From 2e688cefa1620a6dd79f917003e1317b277d805e Mon Sep 17 00:00:00 2001 From: Kimplul Date: Sat, 1 Aug 2026 16:35:54 +0300 Subject: parser seems to work --- src/fwdscript.c | 628 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 615 insertions(+), 13 deletions(-) diff --git a/src/fwdscript.c b/src/fwdscript.c index c524810..a44fa93 100644 --- a/src/fwdscript.c +++ b/src/fwdscript.c @@ -23,6 +23,20 @@ enum fwdtype { FWDSCRIPT_OPQ }; +static const char *fwdtype_str(enum fwdtype type) +{ + switch (type) { + case FWDSCRIPT_NUM: return "/num"; + case FWDSCRIPT_STR: return "/str"; + case FWDSCRIPT_CNT: return "/cnt"; + case FWDSCRIPT_TBL: return "/tbl"; + case FWDSCRIPT_OPQ: return "/opq"; + default: break; + } + + return "/???"; +} + struct fwdval { enum fwdtype kind; @@ -135,6 +149,13 @@ static struct fwdscript fwdscript_create() return inst; } +static void *fwdscript_move(void **p) +{ + void *c = *p; + *p = NULL; + return c; +} + static int fwdscript_register(struct fwdscript *s, const char *name, fwd_callback_t cb, fwd_typecheck_t tp) { @@ -207,7 +228,7 @@ static int fwdscript_register_builtins(struct fwdscript *s) static const char *skip_ignored(const char *buf) { - while (*buf && isspace(*buf)) + while (*buf && isspace(*buf) && *buf != '\n') buf++; if (*buf == '#') @@ -232,17 +253,22 @@ static const char *next_separator(const char *buf) return buf; } +enum fwdtokentype { + FWDTOKEN_ID, + FWDTOKEN_STR, + FWDTOKEN_NUM, + FWDTOKEN_OPEN, + FWDTOKEN_CLOSE, + FWDTOKEN_TO, + FWDTOKEN_END, + FWDTOKEN_EOL, + + /* special token used for peeking */ + FWDTOKEN_EOF +}; + struct fwdtoken { - enum { - FWDTOKEN_ID, - FWDTOKEN_STR, - FWDTOKEN_NUM, - FWDTOKEN_OPEN, - FWDTOKEN_CLOSE, - FWDTOKEN_TO, - FWDTOKEN_END, - FWDTOKEN_EOL - } kind; + enum fwdtokentype kind; size_t line; union { @@ -368,7 +394,7 @@ static int fwdscript_tokenize(struct fwdtokens *tokens, const char *buf) return -1; } - char *s = strndup(tok + 1, end - tok - 2); + char *s = strndup(tok + 1, end - tok - 1); if (!s) { fprintf(stderr, "failed duping str token\n"); return -1; @@ -451,6 +477,573 @@ static void free_tokens(struct fwdtokens *tokens) fwdtokens_destroy(tokens); } +struct fwdast { + enum { + FWDAST_ID, + FWDAST_VAR, + FWDAST_STR, + FWDAST_NUM, + FWDAST_CNT, + FWDAST_CMD, + FWDAST_TOP + } kind; + + char *s; + + union { + double d; + enum fwdtype t; + }; + + size_t argc; + struct fwdast *args; + + size_t bodyc; + struct fwdast *body; +}; + +#define VEC_TYPE struct fwdast +#define VEC_NAME fwdasts +#include + +static struct fwdasts fwdast_body(struct fwdast *ast) +{ + return (struct fwdasts){.n = ast->bodyc, .s = 0, .buf = ast->body}; +} + +static struct fwdasts fwdast_args(struct fwdast *ast) +{ + return (struct fwdasts){.n = ast->argc, .s = 0, .buf = ast->args}; +} + +static void free_asts(struct fwdasts *asts); + +static void free_ast(struct fwdast *ast) +{ + struct fwdasts args = fwdast_args(ast); + struct fwdasts body = fwdast_body(ast); + + switch (ast->kind) { + case FWDAST_TOP: + free_asts(&body); + break; + + case FWDAST_CMD: + free_asts(&args); + break; + + case FWDAST_STR: + case FWDAST_ID: + case FWDAST_VAR: + free(ast->s); + break; + + case FWDAST_CNT: + free_asts(&args); + free_asts(&body); + break; + + default: + break; + } +} + +static void free_asts(struct fwdasts *asts) +{ + foreach(fwdasts, n, asts) { + free_ast(n); + } + + fwdasts_destroy(asts); +} + +static enum fwdtokentype fwdtoken_peek(struct fwdtokens *tokens, ssize_t pos) +{ + if ((size_t)pos >= fwdtokens_len(tokens)) + return FWDTOKEN_EOF; + + return fwdtokens_at(tokens, (size_t)pos)->kind; +} + +static int fwdtoken_to_fwdtype(struct fwdtoken *token, enum fwdtype *type) +{ + assert(token->kind == FWDTOKEN_ID); + if (strcmp(token->s, "/num") == 0) { + *type = FWDSCRIPT_NUM; + return 0; + } + + if (strcmp(token->s, "/str") == 0) { + *type = FWDSCRIPT_STR; + return 0; + } + + if (strcmp(token->s, "/cnt") == 0) { + *type = FWDSCRIPT_CNT; + return 0; + } + + if (strcmp(token->s, "/tbl") == 0) { + *type = FWDSCRIPT_TBL; + return 0; + } + + if (strcmp(token->s, "/opq") == 0) { + *type = FWDSCRIPT_OPQ; + return 0; + } + + fprintf(stderr, "line %zu, unknown type %s\n", token->line, token->s); + return -1; +} + +static const char *fwdtoken_kind_str(enum fwdtokentype kind) +{ + switch (kind) { + case FWDTOKEN_ID: return "id"; + case FWDTOKEN_STR: return "str"; + case FWDTOKEN_NUM: return "num"; + case FWDTOKEN_OPEN: return "{"; + case FWDTOKEN_CLOSE: return "}"; + case FWDTOKEN_TO: return "=>"; + case FWDTOKEN_END: return ";"; + case FWDTOKEN_EOL: return "end of line"; + case FWDTOKEN_EOF: return "end of file"; + default: break; + } + + return "???"; +} + +static struct fwdtoken *fwdtokens_consume(struct fwdtokens *tokens, ssize_t *pos, enum fwdtokentype ex) +{ + if ((size_t)*pos >= fwdtokens_len(tokens)) { + fprintf(stderr, "ran out of tokens\n"); + return NULL; + } + + struct fwdtoken *tok = fwdtokens_at(tokens, *pos); + if (tok->kind != ex) { + fprintf(stderr, "line %zu, expected %s but found %s\n", + tok->line, + fwdtoken_kind_str(ex), + fwdtoken_kind_str(tok->kind)); + return NULL; + } + + *pos += 1; + return tok; +} + +static ssize_t fwdscript_parse_cmds(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *cmds); +static ssize_t fwdscript_parse_body(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *body); + +static ssize_t fwdscript_parse_empty(struct fwdtokens *tokens, ssize_t pos) +{ + /* skip until first interesting feature */ + while (1) { + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END) { + pos++; + continue; + } + + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_EOL) { + pos++; + continue; + } + + break; + } + + return pos; +} + + +static ssize_t fwdscript_parse_var(struct fwdtokens *tokens, ssize_t pos, struct fwdast *var) +{ + struct fwdtoken *type = NULL; + struct fwdtoken *arg = NULL; + + if ((type = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL) + return -1; + + if ((arg = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL) + return -1; + + enum fwdtype tk; + if (fwdtoken_to_fwdtype(type, &tk)) + return -1; + + var->kind = FWDAST_VAR; + var->s = fwdscript_move((void **)&arg->s); + var->t = tk; + return pos; +} + +static ssize_t fwdscript_parse_closure(struct fwdtokens *tokens, ssize_t pos, struct fwdast *arg) +{ + if (fwdtokens_consume(tokens, &pos, FWDTOKEN_TO) == NULL) + return -1; + + struct fwdasts vars = fwdasts_create(0); + + while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) { + struct fwdast var = {}; + if ((pos = fwdscript_parse_var(tokens, pos, &var)) < 0) { + free_asts(&vars); + return -1; + } + + if (fwdasts_append(&vars, var) == NULL) { + fprintf(stderr, "failed appending ast var to closure\n"); + free_asts(&vars); + return -1; + } + } + + struct fwdasts body = fwdasts_create(0); + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_OPEN) { + if ((pos = fwdscript_parse_body(tokens, pos, &body)) < 0) { + free_asts(&vars); + free_asts(&body); + return -1; + } + + } else if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END) { + if ((pos = fwdscript_parse_cmds(tokens, pos, &body)) < 0) { + free_asts(&vars); + free_asts(&body); + return -1; + } + } + + arg->kind = FWDAST_CNT; + + arg->argc = vars.n; + arg->args = vars.buf; + + arg->bodyc = body.n; + arg->body = body.buf; + return pos; +} + +static ssize_t fwdscript_parse_arg(struct fwdtokens *tokens, ssize_t pos, struct fwdast *arg) +{ + struct fwdtoken *tok = fwdtokens_at(tokens, pos); + switch (tok->kind) { + case FWDTOKEN_ID: + arg->kind = FWDAST_ID; + arg->s = fwdscript_move((void **)&tok->s); + return pos + 1; + + case FWDTOKEN_STR: + arg->kind = FWDAST_STR; + arg->s = fwdscript_move((void **)&tok->s); + return pos + 1; + + case FWDTOKEN_NUM: + arg->kind = FWDAST_NUM; + arg->d = tok->d; + return pos + 1; + + case FWDTOKEN_TO: + return fwdscript_parse_closure(tokens, pos, arg); + + default: + break; + } + + /* let whoever called us handle other cases (?) */ + return pos; +} + +static ssize_t fwdscript_parse_cmd(struct fwdtokens *tokens, ssize_t pos, struct fwdast *cmd) +{ + if ((pos = fwdscript_parse_empty(tokens, pos)) < 0) + return -1; + + struct fwdasts parts = fwdasts_create(0); + + struct fwdtoken *name = NULL; + if ((name = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL) { + free_asts(&parts); + return -1; + } + + /* parse args */ + while (1) { + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_EOL) { + pos++; + continue; + } + + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_END) + break; + + + if (fwdtoken_peek(tokens, pos) == FWDTOKEN_CLOSE) + break; + + struct fwdast part = {}; + if ((pos = fwdscript_parse_arg(tokens, pos, &part)) < 0) { + free_asts(&parts); + return -1; + } + + if (fwdasts_append(&parts, part) == NULL) { + fprintf(stderr, "failed appending ast arg to parts\n"); + free_asts(&parts); + return -1; + } + } + + cmd->kind = FWDAST_CMD; + cmd->s = fwdscript_move((void **)&name->s); + cmd->argc = parts.n; + cmd->args = parts.buf; + return pos; +} + +static ssize_t fwdscript_parse_cmds(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *cmds) +{ + if ((pos = fwdscript_parse_empty(tokens, pos)) < 0) { + return -1; + } + + /* each command must start with an identifier */ + while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) { + struct fwdast cmd = {}; + if ((pos = fwdscript_parse_cmd(tokens, pos, &cmd)) < 0) + return -1; + + if (fwdasts_append(cmds, cmd) == NULL) { + fprintf(stderr, "failed appending at cmd to body\n"); + return -1; + } + + } + + if ((pos = fwdscript_parse_empty(tokens, pos)) < 0) + return -1; + + return pos; +} + +static ssize_t fwdscript_parse_body(struct fwdtokens *tokens, ssize_t pos, struct fwdasts *body) +{ + if (fwdtokens_consume(tokens, &pos, FWDTOKEN_OPEN) == NULL) + return -1; + + if ((pos = fwdscript_parse_cmds(tokens, pos, body)) < 0) + return -1; + + if (fwdtokens_consume(tokens, &pos, FWDTOKEN_CLOSE) == NULL) + return -1; + + return pos; +} + +static ssize_t fwdscript_parse_def(struct fwdtokens *tokens, ssize_t pos, struct fwdast *def) +{ + struct fwdtoken *name = NULL; + if ((name = fwdtokens_consume(tokens, &pos, FWDTOKEN_ID)) == NULL) + return -1; + + struct fwdasts args = fwdasts_create(0); + while (fwdtoken_peek(tokens, pos) == FWDTOKEN_ID) { + struct fwdast var = {}; + if ((pos = fwdscript_parse_var(tokens, pos, &var)) < 0) { + free_asts(&args); + return -1; + } + + if (fwdasts_append(&args, var) == NULL) { + fprintf(stderr, "failed appending ast var to def\n"); + free_asts(&args); + return -1; + } + } + + struct fwdasts body = fwdasts_create(0); + if ((pos = fwdscript_parse_body(tokens, pos, &body)) < 0) { + free_asts(&args); + return -1; + } + + /* move everything into ast node */ + def->kind = FWDAST_CNT, + def->s = fwdscript_move((void **)&name->s), + + def->argc = args.n, + def->args = args.buf, + + def->bodyc = body.n, + def->body = body.buf; + + return pos; +} + +static int fwdscript_parse_top(struct fwdtokens *tokens, struct fwdast *top) +{ + ssize_t pos = 0; + struct fwdasts defs = fwdasts_create(1); + + while (1) { + while (pos < (ssize_t)fwdtokens_len(tokens) + && fwdtokens_at(tokens, pos)->kind == FWDTOKEN_EOL) + pos++; + + /* all input succesfully read */ + if (pos == (ssize_t)fwdtokens_len(tokens)) + break; + + struct fwdast tmp = {}; + struct fwdast *def = NULL; + if ((def = fwdasts_append(&defs, tmp)) == NULL) { + fprintf(stderr, "couldn't append temp ast def\n"); + free_asts(&defs); + return -1; + } + + if ((pos = fwdscript_parse_def(tokens, pos, def)) < 0) { + free_asts(&defs); + return -1; + } + } + + /* transfer ownership or array to top */ + top->kind = FWDAST_TOP; + top->s = NULL; + top->argc = 0; + top->args = NULL; + top->bodyc = defs.n; + top->body = defs.buf; + return 0; +} + +static void fwdscript_debug_tokens(struct fwdtokens *tokens) +{ + foreach(fwdtokens, tok, tokens) { + switch (tok->kind) { + case FWDTOKEN_ID: + fprintf(stderr, "line %4zu: %s (%s)\n", + tok->line, + fwdtoken_kind_str(tok->kind), + tok->s); + break; + + case FWDTOKEN_STR: + fprintf(stderr, "line %4zu: %s \"%s\"\n", + tok->line, + fwdtoken_kind_str(tok->kind), + tok->s); + break; + + case FWDTOKEN_NUM: + fprintf(stderr, "line %4zu: %s (%f)\n", + tok->line, + fwdtoken_kind_str(tok->kind), + tok->d); + break; + + case FWDTOKEN_OPEN: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + + case FWDTOKEN_CLOSE: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + + case FWDTOKEN_TO: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + + case FWDTOKEN_END: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + + case FWDTOKEN_EOL: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + + case FWDTOKEN_EOF: + fprintf(stderr, "line %4zu: %s\n", + tok->line, + fwdtoken_kind_str(tok->kind)); + break; + } + } + +} + +static void fwdscript_debug_ast(struct fwdast *ast, int depth) +{ + /** @todo line numbering disappears from ast, not fantastic I guess? */ + struct fwdasts args = fwdast_args(ast); + struct fwdasts body = fwdast_body(ast); + + switch (ast->kind) { + case FWDAST_ID: + fprintf(stderr, "%*s+ id (%s)\n", depth, "|", + ast->s); + break; + + case FWDAST_VAR: + fprintf(stderr, "%*s+ var (%s, %s)\n", depth, "|", + ast->s, fwdtype_str(ast->t)); + break; + + case FWDAST_STR: + fprintf(stderr, "%*s+ str \"%s\"\n", depth, "|", + ast->s); + break; + + case FWDAST_NUM: + fprintf(stderr, "%*s+ num (%f)\n", depth, "|", + ast->d); + break; + + case FWDAST_CNT: + fprintf(stderr, "%*s+ cnt (%s)\n", depth, "|", + ast->s ? ast->s : "closure"); + + foreach(fwdasts, n, &args) { + fwdscript_debug_ast(n, depth + 1); + } + + foreach(fwdasts, n, &body) { + fwdscript_debug_ast(n, depth + 1); + } + break; + + case FWDAST_CMD: + fprintf(stderr, "%*s+ cmd (%s)\n", depth, "|", + ast->s); + + foreach(fwdasts, n, &args) { + fwdscript_debug_ast(n, depth + 1); + } + break; + + case FWDAST_TOP: + fprintf(stderr, "TOP\n"); + foreach(fwdasts, n, &body) { + fwdscript_debug_ast(n, depth + 1); + } + break; + } +} + static int fwdscript_parse(struct fwdscript *s, const char *buf) { struct fwdtokens tokens = fwdtokens_create(0); @@ -460,8 +1053,17 @@ static int fwdscript_parse(struct fwdscript *s, const char *buf) return -1; } + fwdscript_debug_tokens(&tokens); + /* convert tokens into some kind of structure */ - TODO(); + struct fwdast top = {}; + if (fwdscript_parse_top(&tokens, &top)) { + free_tokens(&tokens); + return -1; + } + + fwdscript_debug_ast(&top, 0); + free_tokens(&tokens); return 0; } -- cgit v1.3