From ca038a85d6efaf3cfc7be7b44296adafa8281ac9 Mon Sep 17 00:00:00 2001 From: Quinn Stephens Date: Sun, 8 Jun 2025 10:10:04 -0400 Subject: lexer: Add more token kinds Signed-off-by: Quinn Stephens --- src/lexer/lexer.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'src/lexer/lexer.c') diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index 1431789..e7f277e 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -73,6 +73,9 @@ lex_identifier(struct lexer *ctx, struct token *tok) bool lexer_next(struct lexer *ctx, struct token *tok) { + char ch; + uint16_t ch_info; + if (ctx == NULL || tok == NULL) { return false; } @@ -84,12 +87,22 @@ lexer_next(struct lexer *ctx, struct token *tok) tok->line = ctx->line; tok->col = (int)(tok->pos - ctx->line_start) + 1; - if (char_info[(int)*ctx->pos] & CHAR_ALPHA || *ctx->pos == '_') { + ch = *ctx->pos; + ch_info = char_info[(int)ch]; + + if (ch_info & CHAR_ALPHA || ch == '_') { lex_identifier(ctx, tok); return true; } - if (*ctx->pos == '\0') { + if (ch_info & CHAR_SINGLE) { + tok->kind = ch_info >> CHAR_SINGLE_SHIFT; + tok->len = 1; + ctx->pos++; + return true; + } + + if (ch == '\0') { tok->kind = TOK_EOF; return true; } -- cgit v1.2.3