make tests a bit more sensible

+ At least by my standards. Some amount of scripting going on, but
  should still be fine.

  Effectively, source.mk contains information to build the test,
  check.mk checks the generated log from running the test. As a results,
  check.mk should output OK into reports/$TEST/OK if everything went
  well, and anything else otherwise. The user can then look at
  reports/$TEST/log to see what went wrong.

  I expect to add some more features, such as starting QEMU with gdb and
  running a single test a bit easier than right now (same limitation as
  in the top level makefile, i.e. you have to specify

	make -f scripts/makefile $TEST

  from within `tests` and you must have ran `make check` at that point
  so that `tests.mk` is fully generated. Not huge issues, but slightly
  annoying, somewhat unsure how to work around them properly but we'll
  see.

11 files changed, 165 insertions, 48 deletions

diff --git a/.gitignore b/.gitignore
index 9cf038c..beadd64 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ deps.mk
 initrd
 init
 build
+reports
 
 # keep gitkeep files wherever they are
 !.gitkeep
diff --git a/scripts/makefile b/scripts/makefile
index da0afa0..45b0aa4 100644
--- a/scripts/makefile
+++ b/scripts/makefile
@@ -18,7 +18,8 @@ ASSERTFLAGS	!= [ "$(ASSERT)" != "0" ] \
 			|| echo
 
 USE_LLVM	!= [ "$(LLVM)" -a "$(LLVM)" != "0" ] \
-			&& echo "1"
+			&& echo "1" \
+			|| echo ""
 
 DEPFLAGS	= -MT $@ -MMD -MP -MF $@.d
 LINTFLAGS	= -fsyntax-only
diff --git a/tests/Makefile b/tests/Makefile
index 00b523e..8a27691 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,40 +1,18 @@
-TESTS		:=
+check:
+	$(MAKE) -f scripts/makefile check
 
-ARCH		?= riscv64
-CROSS_COMPILE	?= $(ARCH)-unknown-elf-
+.DEFAULT:
+	$(MAKE) -f scripts/makefile $<
 
-LLVM		?= 0
-COMPILER	!= [ "$(LLVM)" != "0" ] \
-			&& echo clang --target="$(CROSS_COMPILE)" \
-			|| echo $(CROSS_COMPILE)gcc
-
-OBFLAGS		:= -ffreestanding -nostdlib -std=c17 -g
-INCLUDEFLAGS	:= -I ../include -I.
-WARNFLAGS	:= -Wall -Wextra
-COMPILE		:= $(COMPILER) $(WARNFLAGS) $(INCLUDEFLAGS) $(OBFLAGS)
-
-GEN_INITRD	:= cpio -H newc -o >
-QEMU		:= qemu-system-$(ARCH) -machine virt -kernel ../kmi.bin \
-		   -serial stdio \
-		   -monitor none \
-		   -nographic \
-		   -no-reboot \
-		   -initrd
-
-COMMON		:= build/printf.o ../kmi.bin ../include/kmi/syscalls.h
-
-include noop/source.mk
-include hello-world/source.mk
-
-.PHONY: check
-check: $(TESTS)
+SOURCES		!= echo */source.mk
+DO		!= echo -n > tests.mk
+include $(SOURCES)
 
 build/printf.o: common/printf.c
-	mkdir -p build
-	$(COMPILE) -c common/printf.c -o build/printf.o
+	mkdir -p build && $(COMPILE_TEST) -c common/printf.c -o build/printf.o
 
 RM	?= rm
 
 .PHONY: clean
 clean:
-	$(RM) -rf build
+	$(RM) -rf build reports
diff --git a/tests/hello-world/check.mk b/tests/hello-world/check.mk
new file mode 100644
index 0000000..7fbabc9
--- /dev/null
+++ b/tests/hello-world/check.mk
@@ -0,0 +1,5 @@
+REPORT	:= reports/hello-world
+hello-world: do-hello-world
+	@grep 'Hello, world!' $(REPORT)/log > /dev/null	\
+		&& echo "OK" > $(REPORT)/OK		\
+		|| echo "ERR" > $(REPORT)/ERR
diff --git a/tests/hello-world/source.mk b/tests/hello-world/source.mk
index 52d642d..5fd60ff 100644
--- a/tests/hello-world/source.mk
+++ b/tests/hello-world/source.mk
@@ -1,8 +1,2 @@
-TESTS	:= $(TESTS) hello-world
-
-.PHONY: hello-world
-hello-world: hello-world/init.c $(COMMON)
-	mkdir -p build/hello-world
-	$(COMPILE) hello-world/init.c build/printf.o -o build/hello-world/init
-	echo build/hello-world/init | $(GEN_INITRD) build/hello-world/initrd
-	$(QEMU) build/hello-world/initrd | grep 'Hello, world!'
+DO	!= ./scripts/gen-prog -n hello-world -p init init.c
+DO	!= ./scripts/gen-simple -n hello-world -p init
diff --git a/tests/noop/check.mk b/tests/noop/check.mk
new file mode 100644
index 0000000..f4741cb
--- /dev/null
+++ b/tests/noop/check.mk
@@ -0,0 +1,3 @@
+noop: do-noop
+	@# remove carriage returns before writing status
+	@tail -n1 reports/noop/log | tr -d '\r' > reports/noop/OK
diff --git a/tests/noop/source.mk b/tests/noop/source.mk
index 3745e37..cbb87b7 100644
--- a/tests/noop/source.mk
+++ b/tests/noop/source.mk
@@ -1,8 +1,2 @@
-TESTS	:= $(TESTS) noop
-
-.PHONY: noop
-noop: noop/init.c $(COMMON)
-	mkdir -p build/noop
-	$(COMPILE) noop/init.c build/printf.o -o build/noop/init
-	echo build/noop/init | $(GEN_INITRD) build/noop/initrd
-	$(QEMU) build/noop/initrd | grep 'OK'
+DO	!= ./scripts/gen-prog -n noop -p init init.c
+DO	!= ./scripts/gen-simple -n noop -p init
diff --git a/tests/scripts/gen-prog b/tests/scripts/gen-prog
new file mode 100755
index 0000000..e56ecc3
--- /dev/null
+++ b/tests/scripts/gen-prog
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+NAME=
+PROG=
+while getopts "n:p:" opt; do
+	case "$opt" in
+		n) NAME="$OPTARG";;
+		p) PROG="$OPTARG";;
+		*) echo "unrecognised options -$OPTARG" >&2; exit 1;
+	esac
+done
+
+shift $((OPTIND - 1))
+
+# create all subdirectories
+mkdir -p $(echo "${@}" | tr ' ' '\n' | sed "s|[^/]*$||;s|^|build/$NAME|" | uniq)
+
+for s in ${@}
+do
+	obj="build/${NAME}/${s%.*}.o"
+	dep="${obj}.d"
+
+	echo "${NAME}_OBJS += ${obj}"				>> tests.mk
+	echo "${dep}:"						>> tests.mk
+	echo "-include ${dep}"					>> tests.mk
+	echo "${obj}: $NAME/${s}"				>> tests.mk
+	echo "	\$(COMPILE_TEST) -c $NAME/${s} -o ${obj}"	>> tests.mk
+done
+
+echo "build/${NAME}/$PROG: \$(${NAME}_OBJS) \$(COMMON)"	>> tests.mk
+echo "	\$(COMPILE_TEST) \$(${NAME}_OBJS) \$(COMMON) \
+		-o build/${NAME}/$PROG"	>> tests.mk
+
+echo "include ${NAME}/check.mk" >> tests.mk
diff --git a/tests/scripts/gen-simple b/tests/scripts/gen-simple
new file mode 100755
index 0000000..ddec9a4
--- /dev/null
+++ b/tests/scripts/gen-simple
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+NAME=
+PROGS=
+while getopts "n:p:" opt; do
+	case "$opt" in
+		n) NAME="$OPTARG";;
+		p) PROGS="$PROGS $OPTARG";;
+		*) echo "unrecognised option -$OPTARG" >&2; exit 1;;
+	esac
+done
+
+if [ -z "$NAME" ]; then
+	echo "No name given for tests" >&2; exit 2;
+fi
+
+if [ -z "$PROGS" ]; then
+	echo "No programs to add to initrd" >&2; exit 3;
+fi
+
+echo "build/$NAME/initrd: build/$NAME/init \$(COMMON) \$(KMI)"		>> tests.mk
+echo "	echo build/$NAME/init | \$(GEN_INITRD) build/$NAME/initrd"	>> tests.mk
+echo "reports/$NAME/log: build/$NAME/initrd"				>> tests.mk
+echo "	mkdir -p reports/$NAME"						>> tests.mk
+echo "	rm -f reports/$NAME/*"						>> tests.mk
+echo "	timeout --foreground 30s \$(QEMU) \
+	build/$NAME/initrd > reports/$NAME/log" >> tests.mk
+
+echo "TESTS += $NAME"			>> tests.mk
+echo ".PHONY: do-$NAME"			>> tests.mk
+echo "do-$NAME: reports/$NAME/log"	>> tests.mk
diff --git a/tests/scripts/makefile b/tests/scripts/makefile
new file mode 100644
index 0000000..8ea802f
--- /dev/null
+++ b/tests/scripts/makefile
@@ -0,0 +1,42 @@
+.PHONY: all
+all: check
+
+ARCH		?= riscv64
+CROSS_COMPILE	?= $(ARCH)-unknown-elf-
+
+LLVM		?= 0
+COMPILER	!= [ "$(LLVM)" != "0" ] \
+			&& echo clang --target="$(CROSS_COMPILE)" \
+			|| echo $(CROSS_COMPILE)gcc
+
+OBFLAGS		:= -ffreestanding -nostdlib -std=c17 -g
+INCLUDEFLAGS	:= -I ../include -I.
+WARNFLAGS	:= -Wall -Wextra
+COMPILE_TEST	:= $(COMPILER) $(WARNFLAGS) $(INCLUDEFLAGS) $(OBFLAGS)
+
+GEN_INITRD	:= cpio -H newc -o >
+QEMU		:= qemu-system-$(ARCH) -machine virt -kernel ../kmi.bin \
+		   -serial stdio \
+		   -monitor none \
+		   -nographic \
+		   -no-reboot \
+		   -initrd
+
+KMI		:= ../kmi.bin
+COMMON		:= build/printf.o
+
+build/printf.o: common/printf.c
+	$(COMPILE_TEST) -c common/printf.c -o build/printf.o
+
+include tests.mk
+
+.PHONY: check
+check: $(TESTS)
+	@for d in reports/* ; do				\
+		if [ ! -f "$$d/OK" ]; then			\
+			echo "BROKEN: $$d" ;			\
+		elif [ "$$(tail -n1 $$d/OK)" != "OK" ]; then	\
+			echo "FAIL: $$d" ;			\
+		fi						\
+	done
+	@echo "Done."
diff --git a/tests/tests.mk b/tests/tests.mk
new file mode 100644
index 0000000..78d104b
--- /dev/null
+++ b/tests/tests.mk
@@ -0,0 +1,34 @@
+hello-world_OBJS += build/hello-world/init.o
+build/hello-world/init.o.d:
+-include build/hello-world/init.o.d
+build/hello-world/init.o: hello-world/init.c
+	$(COMPILE_TEST) -c hello-world/init.c -o build/hello-world/init.o
+build/hello-world/init: $(hello-world_OBJS) $(COMMON)
+	$(COMPILE_TEST) $(hello-world_OBJS) $(COMMON) 		-o build/hello-world/init
+include hello-world/check.mk
+build/hello-world/initrd: build/hello-world/init $(COMMON) $(KMI)
+	echo build/hello-world/init | $(GEN_INITRD) build/hello-world/initrd
+reports/hello-world/log: build/hello-world/initrd
+	mkdir -p reports/hello-world
+	rm -f reports/hello-world/*
+	timeout --foreground 30s $(QEMU) 	build/hello-world/initrd > reports/hello-world/log
+TESTS += hello-world
+.PHONY: do-hello-world
+do-hello-world: reports/hello-world/log
+noop_OBJS += build/noop/init.o
+build/noop/init.o.d:
+-include build/noop/init.o.d
+build/noop/init.o: noop/init.c
+	$(COMPILE_TEST) -c noop/init.c -o build/noop/init.o
+build/noop/init: $(noop_OBJS) $(COMMON)
+	$(COMPILE_TEST) $(noop_OBJS) $(COMMON) 		-o build/noop/init
+include noop/check.mk
+build/noop/initrd: build/noop/init $(COMMON) $(KMI)
+	echo build/noop/init | $(GEN_INITRD) build/noop/initrd
+reports/noop/log: build/noop/initrd
+	mkdir -p reports/noop
+	rm -f reports/noop/*
+	timeout --foreground 30s $(QEMU) 	build/noop/initrd > reports/noop/log
+TESTS += noop
+.PHONY: do-noop
+do-noop: reports/noop/log