5 files changed, 173 insertions, 0 deletions
diff --git a/tests/scripts/gen-xfail b/tests/scripts/gen-xfail
new file mode 100755
index 0000000..a04f8d9
--- /dev/null
+++ b/tests/scripts/gen-xfail
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+mkdir -p $(for d in "${@}"; do echo "$d"; done \
+		| sed "s|,.*||" | uniq | sed "s|^|reports/|")
+
+for s in "${@}"
+do
+	NAME=${s%%,*}
+	EMSG=${s#${NAME},}
+	echo ".PHONY: $NAME"						>> tests.mk
+	echo "$NAME:"							>> tests.mk
+	echo "	./scripts/run-xfail $NAME '$EMSG'"			>> tests.mk
+done
+
+echo -n "TESTS +=" >> tests.mk
+for s in "${@}"
+do
+	NAME=${s%%,*}
+	echo -n " $NAME" >> tests.mk
+done
+
+# append newline
+echo "" >> tests.mk
diff --git a/tests/scripts/gen-xpass b/tests/scripts/gen-xpass
new file mode 100755
index 0000000..8ec383f
--- /dev/null
+++ b/tests/scripts/gen-xpass
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+mkdir -p $(for d in "${@}"; do echo "$d"; done | uniq | sed "s|^|reports/|")
+
+for s in "${@}"
+do
+	echo ".PHONY: $s"						>> tests.mk
+	echo "$s:"							>> tests.mk
+	echo "	./scripts/run-xpass $s"					>> tests.mk
+done
+
+echo "TESTS += " "${@}" >> tests.mk
diff --git a/tests/scripts/makefile b/tests/scripts/makefile
new file mode 100644
index 0000000..e6ea394
--- /dev/null
+++ b/tests/scripts/makefile
@@ -0,0 +1,27 @@
+.PHONY: all
+all: check
+
+COMPILE_TEST := $(CC) -L../mod -I../include -I../lib -Wl,-rpath=../mod -O2
+TEST_LIBS    := -lfwdio -lfwdmem -lfwdutil
+
+export COMPILE_TEST
+export TEST_LIBS
+
+TESTS :=
+include tests.mk
+
+.PHONY: check
+check: $(TESTS)
+	@cd reports; OK=1; for d in * ; do			\
+		if [ ! -f "$$d/OK" ]; then			\
+			echo "BROKEN: $$d" ;			\
+		elif [ "$$(tail -n1 $$d/OK)" != "OK" ]; then	\
+			OK=0             ;			\
+			echo "FAIL: $$d" ;			\
+		fi						\
+	done;							\
+	if [ "$$OK" != 1 ]; then				\
+		echo "There were failing tests";		\
+		exit 1; \
+	fi
+	@echo "Done."
diff --git a/tests/scripts/run-xfail b/tests/scripts/run-xfail
new file mode 100755
index 0000000..35bcc75
--- /dev/null
+++ b/tests/scripts/run-xfail
@@ -0,0 +1,58 @@
+#!/bin/sh
+NAME="$1"
+EMSG="$2"
+
+try_vg() {
+	if [ "x$VALGRIND" != "x0" ] && which valgrind > /dev/null >&1; then
+		eval "valgrind -q --leak-check=full --error-exitcode=169 $1"
+	else
+		eval "$1"
+	fi
+
+	RETVAL=$?
+	return "$RETVAL"
+}
+
+I=0
+while :; do
+	if try_vg "../fwd $NAME/$NAME.fwd" \
+		1> "reports/$NAME/gen.c"   \
+		2> "reports/$NAME/run-$I"
+	then
+		# success was not expected
+		echo "SUCC" > reports/$NAME/OK
+		exit 0
+	fi
+
+	if [ "$RETVAL" = 169 ]; then
+		echo "VALGRIND" > reports/$NAME/OK
+
+		# don't return on an error so that makefile continues running
+		# other tests that are maybe failing, and reports the status
+		# after all tests have been run
+		exit 0
+	fi
+
+	# correctly handled coverage exception, continue to next run
+	# here we check for the special 'internal error' which should be
+	# reported if the error was handled gracefully, otherwise since the
+	# error message is not the one we expected and not a covsrv error, we
+	# don't know what it is and could end up in an infinite loop if we don't
+	# stop here.
+	if grep 'COVSRV: EXIT' "reports/$NAME/gen.c" > /dev/null \
+	&& grep 'internal error:' "reports/$NAME/run-$I" > /dev/null; then
+		I=$((I+1))
+		continue
+	fi
+
+	if grep "$EMSG" "reports/$NAME/run-$I" > /dev/null; then
+		# we exited with the expected error message and without leaking
+		# memory, presume covsrv is happy as well
+		echo "OK" > reports/$NAME/OK
+		exit 0
+	fi
+
+	# incorrectly handled coverage exception, stop run
+	echo "MEMERR" > reports/$NAME/OK
+	exit 0
+done
diff --git a/tests/scripts/run-xpass b/tests/scripts/run-xpass
new file mode 100755
index 0000000..93540c9
--- /dev/null
+++ b/tests/scripts/run-xpass
@@ -0,0 +1,53 @@
+#!/bin/sh
+NAME="$1"
+
+if [ -z "${COMPILE_TEST+x}" -o -z "${TEST_LIBS+x}" ]; then
+	echo "Test runner script must have COMPILE_TEST and TEST_LIBS defined."
+	exit 1
+fi
+
+try_vg() {
+	if [ "x$VALGRIND" != "x0" ] && which valgrind > /dev/null >&1; then
+		eval "valgrind -q --leak-check=full --error-exitcode=169 $1"
+	else
+		eval "$1"
+	fi
+
+	RETVAL=$?
+	return "$RETVAL"
+}
+
+I=0
+while :; do
+	if try_vg "../fwd $NAME/$NAME.fwd" \
+		1> "reports/$NAME/gen.c"   \
+		2> "reports/$NAME/run-$I"
+	then
+		# fwd succeeded, compile and run program to check result
+		# remove possible COVSRV lines in output
+		sed -i "s/COVSRV:.*$//" "reports/$NAME/gen.c"
+		eval "$COMPILE_TEST reports/$NAME/gen.c -o reports/$NAME/$NAME $TEST_LIBS"
+		./reports/$NAME/$NAME > reports/$NAME/OK 2>&1
+		exit 0
+	fi
+
+	if [ "$RETVAL" = 169 ]; then
+		echo "VALGRIND" > reports/$NAME/OK
+
+		# don't return on an error so that makefile continues running
+		# other tests that are maybe failing, and reports the status
+		# after all tests have been run
+		exit 0
+	fi
+
+	# correctly handled coverage exception, continue to next run
+	if grep 'COVSRV: EXIT' "reports/$NAME/gen.c" > /dev/null \
+	&& grep 'internal error:' "reports/$NAME/run-$I" > /dev/null; then
+		I=$((I+1))
+		continue
+	fi
+
+	# incorrectly handled coverage exception, stop run
+	echo "MEMERR" > reports/$NAME/OK
+	exit 0
+done