diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/Makefile | 2 | ||||
| -rwxr-xr-x | tests/scripts/gen-xfail | 8 | ||||
| -rwxr-xr-x | tests/scripts/gen-xpass | 5 | ||||
| -rw-r--r-- | tests/scripts/makefile | 22 | ||||
| -rwxr-xr-x | tests/scripts/run-xfail | 58 | ||||
| -rwxr-xr-x | tests/scripts/run-xpass | 53 |
6 files changed, 130 insertions, 18 deletions
diff --git a/tests/Makefile b/tests/Makefile index ce009fc..81cc205 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,3 +1,5 @@ +export VALGRIND ?= 1 + check: $(MAKE) -f scripts/makefile check diff --git a/tests/scripts/gen-xfail b/tests/scripts/gen-xfail index c4cf382..a04f8d9 100755 --- a/tests/scripts/gen-xfail +++ b/tests/scripts/gen-xfail @@ -9,13 +9,7 @@ do EMSG=${s#${NAME},} echo ".PHONY: $NAME" >> tests.mk echo "$NAME:" >> tests.mk - echo " ../fwd $NAME/$NAME.fwd \\" >> tests.mk - echo " > reports/$NAME/gen.c 2> reports/$NAME/log \\" >> tests.mk - echo " && echo 'Wrong retval' > reports/$NAME/OK \\" >> tests.mk - echo " || :" >> tests.mk - echo " grep '$EMSG' reports/$NAME/log > /dev/null \\" >> tests.mk - echo " && echo OK > reports/$NAME/OK \\" >> tests.mk - echo " || echo EMSG > reports/$NAME/OK" >> tests.mk + echo " ./scripts/run-xfail $NAME '$EMSG'" >> tests.mk done echo -n "TESTS +=" >> tests.mk diff --git a/tests/scripts/gen-xpass b/tests/scripts/gen-xpass index 762650e..8ec383f 100755 --- a/tests/scripts/gen-xpass +++ b/tests/scripts/gen-xpass @@ -6,10 +6,7 @@ for s in "${@}" do echo ".PHONY: $s" >> tests.mk echo "$s:" >> tests.mk - echo " ../fwd $s/$s.fwd > reports/$s/gen.c 2> reports/$s/log" >> tests.mk - echo " \$(COMPILE_TEST) reports/$s/gen.c -o reports/$s/$s \\" >> tests.mk - echo " \$(TEST_LIBS)" >> tests.mk - echo " ./reports/$s/$s > reports/$s/OK 2>&1" >> tests.mk + echo " ./scripts/run-xpass $s" >> tests.mk done echo "TESTS += " "${@}" >> tests.mk diff --git a/tests/scripts/makefile b/tests/scripts/makefile index 21b1418..e6ea394 100644 --- a/tests/scripts/makefile +++ b/tests/scripts/makefile @@ -4,16 +4,24 @@ all: check COMPILE_TEST := $(CC) -L../mod -I../include -I../lib -Wl,-rpath=../mod -O2 TEST_LIBS := -lfwdio -lfwdmem -lfwdutil +export COMPILE_TEST +export TEST_LIBS + TESTS := include tests.mk .PHONY: check check: $(TESTS) - @cd reports; for d in * ; do \ - if [ ! -f "$$d/OK" ]; then \ - echo "BROKEN: $$d" ; \ - elif [ "$$(tail -n1 $$d/OK)" != "OK" ]; then \ - echo "FAIL: $$d" ; \ - fi \ - done + @cd reports; OK=1; for d in * ; do \ + if [ ! -f "$$d/OK" ]; then \ + echo "BROKEN: $$d" ; \ + elif [ "$$(tail -n1 $$d/OK)" != "OK" ]; then \ + OK=0 ; \ + echo "FAIL: $$d" ; \ + fi \ + done; \ + if [ "$$OK" != 1 ]; then \ + echo "There were failing tests"; \ + exit 1; \ + fi @echo "Done." diff --git a/tests/scripts/run-xfail b/tests/scripts/run-xfail new file mode 100755 index 0000000..35bcc75 --- /dev/null +++ b/tests/scripts/run-xfail @@ -0,0 +1,58 @@ +#!/bin/sh +NAME="$1" +EMSG="$2" + +try_vg() { + if [ "x$VALGRIND" != "x0" ] && which valgrind > /dev/null >&1; then + eval "valgrind -q --leak-check=full --error-exitcode=169 $1" + else + eval "$1" + fi + + RETVAL=$? + return "$RETVAL" +} + +I=0 +while :; do + if try_vg "../fwd $NAME/$NAME.fwd" \ + 1> "reports/$NAME/gen.c" \ + 2> "reports/$NAME/run-$I" + then + # success was not expected + echo "SUCC" > reports/$NAME/OK + exit 0 + fi + + if [ "$RETVAL" = 169 ]; then + echo "VALGRIND" > reports/$NAME/OK + + # don't return on an error so that makefile continues running + # other tests that are maybe failing, and reports the status + # after all tests have been run + exit 0 + fi + + # correctly handled coverage exception, continue to next run + # here we check for the special 'internal error' which should be + # reported if the error was handled gracefully, otherwise since the + # error message is not the one we expected and not a covsrv error, we + # don't know what it is and could end up in an infinite loop if we don't + # stop here. + if grep 'COVSRV: EXIT' "reports/$NAME/gen.c" > /dev/null \ + && grep 'internal error:' "reports/$NAME/run-$I" > /dev/null; then + I=$((I+1)) + continue + fi + + if grep "$EMSG" "reports/$NAME/run-$I" > /dev/null; then + # we exited with the expected error message and without leaking + # memory, presume covsrv is happy as well + echo "OK" > reports/$NAME/OK + exit 0 + fi + + # incorrectly handled coverage exception, stop run + echo "MEMERR" > reports/$NAME/OK + exit 0 +done diff --git a/tests/scripts/run-xpass b/tests/scripts/run-xpass new file mode 100755 index 0000000..93540c9 --- /dev/null +++ b/tests/scripts/run-xpass @@ -0,0 +1,53 @@ +#!/bin/sh +NAME="$1" + +if [ -z "${COMPILE_TEST+x}" -o -z "${TEST_LIBS+x}" ]; then + echo "Test runner script must have COMPILE_TEST and TEST_LIBS defined." + exit 1 +fi + +try_vg() { + if [ "x$VALGRIND" != "x0" ] && which valgrind > /dev/null >&1; then + eval "valgrind -q --leak-check=full --error-exitcode=169 $1" + else + eval "$1" + fi + + RETVAL=$? + return "$RETVAL" +} + +I=0 +while :; do + if try_vg "../fwd $NAME/$NAME.fwd" \ + 1> "reports/$NAME/gen.c" \ + 2> "reports/$NAME/run-$I" + then + # fwd succeeded, compile and run program to check result + # remove possible COVSRV lines in output + sed -i "s/COVSRV:.*$//" "reports/$NAME/gen.c" + eval "$COMPILE_TEST reports/$NAME/gen.c -o reports/$NAME/$NAME $TEST_LIBS" + ./reports/$NAME/$NAME > reports/$NAME/OK 2>&1 + exit 0 + fi + + if [ "$RETVAL" = 169 ]; then + echo "VALGRIND" > reports/$NAME/OK + + # don't return on an error so that makefile continues running + # other tests that are maybe failing, and reports the status + # after all tests have been run + exit 0 + fi + + # correctly handled coverage exception, continue to next run + if grep 'COVSRV: EXIT' "reports/$NAME/gen.c" > /dev/null \ + && grep 'internal error:' "reports/$NAME/run-$I" > /dev/null; then + I=$((I+1)) + continue + fi + + # incorrectly handled coverage exception, stop run + echo "MEMERR" > reports/$NAME/OK + exit 0 +done |