parser seems to work

1 files changed, 615 insertions, 13 deletions

diff --git a/src/fwdscript.c b/src/fwdscript.c
index c524810..a44fa93 100644
--- a/src/fwdscript.c
+++ b/src/fwdscript.c
@@ -23,6 +23,20 @@ enum fwdtype {
 	FWDSCRIPT_OPQ
 };
 
+static const char *fwdtype_str(enum fwdtype type)
+{
+	switch (type) {
+	case FWDSCRIPT_NUM: return "/num";
+	case FWDSCRIPT_STR: return "/str";
+	case FWDSCRIPT_CNT: return "/cnt";
+	case FWDSCRIPT_TBL: return "/tbl";
+	case FWDSCRIPT_OPQ: return "/opq";
+	default: break;
+	}
+
+	return "/???";
+}
+
 struct fwdval {
 	enum fwdtype kind;
 
@@ -135,6 +149,13 @@ static struct fwdscript fwdscript_create()
 	return inst;
 }
 
+static void *fwdscript_move(void **p)
+{
+	void *c = *p;
+	*p = NULL;
+	return c;
+}
+
 static int fwdscript_register(struct fwdscript *s, const char *name,
 		fwd_callback_t cb, fwd_typecheck_t tp)
 {
@@ -207,7 +228,7 @@ static int fwdscript_register_builtins(struct fwdscript *s)
 
 static const char *skip_ignored(const char *buf)
 {
-	while (*buf && isspace(*buf))
+	while (*buf && isspace(*buf) && *buf != '\n')
 		buf++;
 
 	if (*buf == '#')
@@ -232,17 +253,22 @@ static const char *next_separator(const char *buf)
 	return buf;
 }
 
+enum fwdtokentype {
+	FWDTOKEN_ID,
+	FWDTOKEN_STR,
+	FWDTOKEN_NUM,
+	FWDTOKEN_OPEN,
+	FWDTOKEN_CLOSE,
+	FWDTOKEN_TO,
+	FWDTOKEN_END,
+	FWDTOKEN_EOL,
+
+	/* special token used for peeking */
+	FWDTOKEN_EOF
+};
+
 struct fwdtoken {
-	enum {
-		FWDTOKEN_ID,
-		FWDTOKEN_STR,
-		FWDTOKEN_NUM,
-		FWDTOKEN_OPEN,
-		FWDTOKEN_CLOSE,
-		FWDTOKEN_TO,
-		FWDTOKEN_END,
-		FWDTOKEN_EOL
-	} kind;
+	enum fwdtokentype kind;
 
 	size_t line;
 	union {
@@ -368,7 +394,7 @@ static int fwdscript_tokenize(struct fwdtokens *tokens, const char *buf)
 				return -1;
 			}
 
-			char *s = strndup(tok + 1, end - tok - 2);
+			char *s = strndup(tok + 1, end - tok - 1);
 			if (!s) {
 				fprintf(stderr, "failed duping str token\n");
 				return -1;
@@ -451,6 +477,573 @@ static void free_tokens(struct fwdtokens *tokens)
 	fwdtokens_destroy(tokens);
 }
 
+struct fwdast {
+	enum {
+		FWDAST_ID,
+		FWDAST_VAR,
+		FWDAST_STR,
+		FWDAST_NUM,
+		FWDAST_CNT,
+		FWDAST_CMD,
+		FWDAST_TOP
+	} kind;
+
+	char *s;
+
+	union {
+		double d;
+		enum fwdtype t;
+	};
+
+	size_t argc;
+	struct fwdast *args;
+
+	size_t bodyc;
+	struct fwdast *body;
+};
+
+#define VEC_TYPE struct fwdast
+#define VEC_NAME fwdasts
+#include <conts/vec.h>
+
+static struct fwdasts fwdast_body(struct fwdast *ast)
+{
+	return (struct fwdasts){.n = ast->bodyc, .s = 0, .buf = ast->body};
+}
+
+static struct fwdasts fwdast_args(struct fwdast *ast)
+{
+	return (struct fwdasts){.n = ast->argc, .s = 0, .buf = ast->args};
+}
+
+static void free_asts(struct fwdasts *asts);
+
+static void free_ast(struct fwdast *ast)
+{
+	struct fwdasts args = fwdast_args(ast);
+	struct fwdasts body = fwdast_body(ast);
+
+	switch (ast->kind) {
+	case FWDAST_TOP:
+		free_asts(&body);
+		break;
+
+	case FWDAST_CMD:
+		free_asts(&args);
+		break;
+
+	case FWDAST_STR:
+	case FWDAST_ID:
+	case FWDAST_VAR:
+		free(ast->s);
+		break;
+
+	case FWDAST_CNT:
+		free_asts(&args);
+		free_asts(&body);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void free_asts(struct fwdasts *asts)
+{
+	foreach(fwdasts, n, asts) {
+		free_ast(n);
+	}
+
+	fwdasts_destroy(asts);
+}
+
+static enum fwdtokentype fwdtoken_peek(struct fwdtokens *tokens, ssize_t pos)
+{
+	if ((size_t)pos >= fwdtokens_len(tokens))
+		return FWDTOKEN_EOF;
+
+	return fwdtokens_at(tokens, (size_t)pos)->kind;
+}
+
+static int fwdtoken_to_fwdtype(struct fwdtoken *token, enum fwdtype *type)
+{
+	assert(token->kind == FWDTOKEN_ID);
+	if (strcmp(token->s, "/num") == 0) {
+		*type = FWDSCRIPT_NUM;
+		return 0;
+	}
+
+	if (strcmp(token->s, "/str") == 0) {
+		*type = FWDSCRIPT_STR;
+		return 0;
+	}
+
+	if (strcmp(token->s, "/cnt") == 0) {
+		*type = FWDSCRIPT_CNT;
+		return 0;
+	}
+
+	if (strcmp(token->s, "/tbl") == 0) {
+		*type = FWDSCRIPT_TBL;
+		return 0;
+	}
+
+	if (strcmp(token->s, "/opq") == 0) {
+		*type = FWDSCRIPT_OPQ;
+		return 0;
+	}
+
+	fprintf(stderr, "line %zu, unknown type %s\n", token->line, token->s);
+	return -1;
+}
+
+static const char *fwdtoken_kind_str(enum fwdtokentype kind)
+{
+	switch (kind) {
+	case FWDTOKEN_ID: return "id";
+	case FWDTOKEN_STR: return "str";
+	case FWDTOKEN_NUM: return "num";
+	case FWDTOKEN_OPEN: return "{";
+	case FWDTOKEN_CLOSE: return "}";
+	case FWDTOKEN_TO: return "=>";
+	case FWDTOKEN_END: return ";";
+	case FWDTOKEN_EOL: return "end of line";
+	case FWDTOKEN_EOF: return "end of file";
+	default: break;
+	}
+
+	return "???";
+}
+
+static struct fwdtoken *fwdtokens_consume(struct fwdtokens *tokens, ssize_t *pos, enum fwdtokentype ex)
+{
+	if ((size_t)*pos >= fwdtokens_len(tokens)) {
+		fprintf(stderr, "ran out of tokens\n");
+		return NULL;
+	}
+
+	struct fwdtoken *tok = fwdtokens_at(tokens, *pos);
+	if (tok->kind != ex) {
+		fprintf(stderr, "line %zu, expected %s but found %s\n",
+				tok->line,
+				fwdtoken_kind_str(ex),
+				fwdtoken_kind_str(tok->kind));
+		return NULL;
+	}
+
+	*pos += 1;
+	return tok;
+}
+
+static ssize_t fwdscript_parse_cmds(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *cmds);
+static ssize_t fwdscript_parse_body(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *body);
+
+static ssize_t fwdscript_parse_empty(struct fwdtokens *tokens, ssize_t pos)
+{
+	/* skip until first interesting feature */
+	while (1) {
+		if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END) {
+			pos++;
+			continue;
+		}
+
+		if (fwdtoken_peek(tokens, pos) == FWDTOKEN_EOL) {
+			pos++;
+			continue;
+		}
+
+		break;
+	}
+
+	return pos;
+}
+
+
+static ssize_t fwdscript_parse_var(struct fwdtokens *tokens, ssize_t pos, struct fwdast *var)
+{
+	struct fwdtoken *type = NULL;
+	struct fwdtoken *arg  = NULL;
+
+	if ((type = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL)
+		return -1;
+
+	if ((arg = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL)
+		return -1;
+
+	enum fwdtype tk;
+	if (fwdtoken_to_fwdtype(type, &tk))
+		return -1;
+
+	var->kind = FWDAST_VAR;
+	var->s    = fwdscript_move((void **)&arg->s);
+	var->t    = tk;
+	return pos;
+}
+
+static ssize_t fwdscript_parse_closure(struct fwdtokens *tokens, ssize_t pos, struct fwdast *arg)
+{
+	if (fwdtokens_consume(tokens, &pos, FWDTOKEN_TO) == NULL)
+		return -1;
+
+	struct fwdasts vars = fwdasts_create(0);
+
+	while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) {
+		struct fwdast var = {};
+		if ((pos = fwdscript_parse_var(tokens, pos, &var)) < 0) {
+			free_asts(&vars);
+			return -1;
+		}
+
+		if (fwdasts_append(&vars, var) == NULL) {
+			fprintf(stderr, "failed appending ast var to closure\n");
+			free_asts(&vars);
+			return -1;
+		}
+	}
+
+	struct fwdasts body = fwdasts_create(0);
+	if (fwdtoken_peek(tokens, pos) == FWDTOKEN_OPEN) {
+		if ((pos = fwdscript_parse_body(tokens, pos, &body)) < 0) {
+			free_asts(&vars);
+			free_asts(&body);
+			return -1;
+		}
+
+	} else if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END) {
+		if ((pos = fwdscript_parse_cmds(tokens, pos, &body)) < 0) {
+			free_asts(&vars);
+			free_asts(&body);
+			return -1;
+		}
+	}
+
+	arg->kind = FWDAST_CNT;
+
+	arg->argc = vars.n;
+	arg->args = vars.buf;
+
+	arg->bodyc = body.n;
+	arg->body  = body.buf;
+	return pos;
+}
+
+static ssize_t fwdscript_parse_arg(struct fwdtokens *tokens, ssize_t pos, struct fwdast *arg)
+{
+	struct fwdtoken *tok = fwdtokens_at(tokens, pos);
+	switch (tok->kind) {
+	case FWDTOKEN_ID:
+		arg->kind = FWDAST_ID;
+		arg->s    = fwdscript_move((void **)&tok->s);
+		return pos + 1;
+
+	case FWDTOKEN_STR:
+		arg->kind = FWDAST_STR;
+		arg->s    = fwdscript_move((void **)&tok->s);
+		return pos + 1;
+
+	case FWDTOKEN_NUM:
+		arg->kind = FWDAST_NUM;
+		arg->d    = tok->d;
+		return pos + 1;
+
+	case FWDTOKEN_TO:
+		return fwdscript_parse_closure(tokens, pos, arg);
+
+	default:
+		break;
+	}
+
+	/* let whoever called us handle other cases (?) */
+	return pos;
+}
+
+static ssize_t fwdscript_parse_cmd(struct fwdtokens *tokens, ssize_t pos, struct fwdast *cmd)
+{
+	if ((pos = fwdscript_parse_empty(tokens, pos)) < 0)
+		return -1;
+
+	struct fwdasts parts = fwdasts_create(0);
+
+	struct fwdtoken *name = NULL;
+	if ((name = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL) {
+		free_asts(&parts);
+		return -1;
+	}
+
+	/* parse args */
+	while (1) {
+		if (fwdtoken_peek(tokens, pos) == FWDTOKEN_EOL) {
+			pos++;
+			continue;
+		}
+
+		if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END)
+			break;
+
+
+		if (fwdtoken_peek(tokens, pos) == FWDTOKEN_CLOSE)
+			break;
+
+		struct fwdast part = {};
+		if ((pos = fwdscript_parse_arg(tokens, pos, &part)) < 0) {
+			free_asts(&parts);
+			return -1;
+		}
+
+		if (fwdasts_append(&parts, part) == NULL) {
+			fprintf(stderr, "failed appending ast arg to parts\n");
+			free_asts(&parts);
+			return -1;
+		}
+	}
+
+	cmd->kind = FWDAST_CMD;
+	cmd->s    = fwdscript_move((void **)&name->s);
+	cmd->argc = parts.n;
+	cmd->args = parts.buf;
+	return pos;
+}
+
+static ssize_t fwdscript_parse_cmds(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *cmds)
+{
+	if ((pos = fwdscript_parse_empty(tokens, pos)) < 0) {
+		return -1;
+	}
+
+	/* each command must start with an identifier */
+	while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) {
+		struct fwdast cmd = {};
+		if ((pos = fwdscript_parse_cmd(tokens, pos, &cmd)) < 0)
+			return -1;
+
+		if (fwdasts_append(cmds, cmd) == NULL) {
+			fprintf(stderr, "failed appending at cmd to body\n");
+			return -1;
+		}
+
+	}
+
+	if ((pos = fwdscript_parse_empty(tokens, pos)) < 0)
+		return -1;
+
+	return pos;
+}
+
+static ssize_t fwdscript_parse_body(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *body)
+{
+	if (fwdtokens_consume(tokens, &pos, FWDTOKEN_OPEN) == NULL)
+		return -1;
+
+	if ((pos = fwdscript_parse_cmds(tokens, pos, body)) < 0)
+		return -1;
+
+	if (fwdtokens_consume(tokens, &pos, FWDTOKEN_CLOSE) == NULL)
+		return -1;
+
+	return pos;
+}
+
+static ssize_t fwdscript_parse_def(struct fwdtokens *tokens, ssize_t pos, struct fwdast *def)
+{
+	struct fwdtoken *name = NULL;
+	if ((name = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL)
+		return -1;
+
+	struct fwdasts args = fwdasts_create(0);
+	while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) {
+		struct fwdast var = {};
+		if ((pos = fwdscript_parse_var(tokens, pos, &var)) < 0) {
+			free_asts(&args);
+			return -1;
+		}
+
+		if (fwdasts_append(&args, var) == NULL) {
+			fprintf(stderr, "failed appending ast var to def\n");
+			free_asts(&args);
+			return -1;
+		}
+	}
+
+	struct fwdasts body = fwdasts_create(0);
+	if ((pos = fwdscript_parse_body(tokens, pos, &body)) < 0) {
+		free_asts(&args);
+		return -1;
+	}
+
+	/* move everything into ast node */
+	def->kind = FWDAST_CNT,
+	def->s    = fwdscript_move((void **)&name->s),
+
+	def->argc = args.n,
+	def->args = args.buf,
+
+	def->bodyc = body.n,
+	def->body  = body.buf;
+
+	return pos;
+}
+
+static int fwdscript_parse_top(struct fwdtokens *tokens, struct fwdast *top)
+{
+	ssize_t pos = 0;
+	struct fwdasts defs = fwdasts_create(1);
+
+	while (1) {
+		while (pos < (ssize_t)fwdtokens_len(tokens)
+		    && fwdtokens_at(tokens, pos)->kind == FWDTOKEN_EOL)
+			pos++;
+
+		/* all input succesfully read */
+		if (pos == (ssize_t)fwdtokens_len(tokens))
+			break;
+
+		struct fwdast tmp = {};
+		struct fwdast *def = NULL;
+		if ((def = fwdasts_append(&defs, tmp)) == NULL) {
+			fprintf(stderr, "couldn't append temp ast def\n");
+			free_asts(&defs);
+			return -1;
+		}
+
+		if ((pos = fwdscript_parse_def(tokens, pos, def)) < 0) {
+			free_asts(&defs);
+			return -1;
+		}
+	}
+
+	/* transfer ownership or array to top */
+	top->kind = FWDAST_TOP;
+	top->s = NULL;
+	top->argc = 0;
+	top->args = NULL;
+	top->bodyc = defs.n;
+	top->body = defs.buf;
+	return 0;
+}
+
+static void fwdscript_debug_tokens(struct fwdtokens *tokens)
+{
+	foreach(fwdtokens, tok, tokens) {
+		switch (tok->kind) {
+		case FWDTOKEN_ID:
+			fprintf(stderr, "line %4zu: %s (%s)\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind),
+					tok->s);
+			break;
+
+		case FWDTOKEN_STR:
+			fprintf(stderr, "line %4zu: %s \"%s\"\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind),
+					tok->s);
+			break;
+
+		case FWDTOKEN_NUM:
+			fprintf(stderr, "line %4zu: %s (%f)\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind),
+					tok->d);
+			break;
+
+		case FWDTOKEN_OPEN:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+
+		case FWDTOKEN_CLOSE:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+
+		case FWDTOKEN_TO:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+
+		case FWDTOKEN_END:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+
+		case FWDTOKEN_EOL:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+
+		case FWDTOKEN_EOF:
+			fprintf(stderr, "line %4zu: %s\n",
+					tok->line,
+					fwdtoken_kind_str(tok->kind));
+			break;
+		}
+	}
+
+}
+
+static void fwdscript_debug_ast(struct fwdast *ast, int depth)
+{
+	/** @todo line numbering disappears from ast, not fantastic I guess? */
+	struct fwdasts args = fwdast_args(ast);
+	struct fwdasts body = fwdast_body(ast);
+
+	switch (ast->kind) {
+	case FWDAST_ID:
+		fprintf(stderr, "%*s+ id (%s)\n", depth, "|",
+				ast->s);
+		break;
+
+	case FWDAST_VAR:
+		fprintf(stderr, "%*s+ var (%s, %s)\n", depth, "|",
+				ast->s, fwdtype_str(ast->t));
+		break;
+
+	case FWDAST_STR:
+		fprintf(stderr, "%*s+ str \"%s\"\n", depth, "|",
+				ast->s);
+		break;
+
+	case FWDAST_NUM:
+		fprintf(stderr, "%*s+ num (%f)\n", depth, "|",
+				ast->d);
+		break;
+
+	case FWDAST_CNT:
+		fprintf(stderr, "%*s+ cnt (%s)\n", depth, "|",
+				ast->s ? ast->s : "closure");
+
+		foreach(fwdasts, n, &args) {
+			fwdscript_debug_ast(n, depth + 1);
+		}
+
+		foreach(fwdasts, n, &body) {
+			fwdscript_debug_ast(n, depth + 1);
+		}
+		break;
+
+	case FWDAST_CMD:
+		fprintf(stderr, "%*s+ cmd (%s)\n", depth, "|",
+				ast->s);
+
+		foreach(fwdasts, n, &args) {
+			fwdscript_debug_ast(n, depth + 1);
+		}
+		break;
+
+	case FWDAST_TOP:
+		fprintf(stderr, "TOP\n");
+		foreach(fwdasts, n, &body) {
+			fwdscript_debug_ast(n, depth + 1);
+		}
+		break;
+	}
+}
+
 static int fwdscript_parse(struct fwdscript *s, const char *buf)
 {
 	struct fwdtokens tokens = fwdtokens_create(0);
@@ -460,8 +1053,17 @@ static int fwdscript_parse(struct fwdscript *s, const char *buf)
 		return -1;
 	}
 
+	fwdscript_debug_tokens(&tokens);
+
 	/* convert tokens into some kind of structure */
-	TODO();
+	struct fwdast top = {};
+	if (fwdscript_parse_top(&tokens, &top)) {
+		free_tokens(&tokens);
+		return -1;
+	}
+
+	fwdscript_debug_ast(&top, 0);
+
 	free_tokens(&tokens);
 	return 0;
 }