diff options
| author | Quinn Stephens <quinn@osmora.org> | 2025-06-08 10:10:04 -0400 |
|---|---|---|
| committer | Quinn Stephens <quinn@osmora.org> | 2025-06-08 10:10:04 -0400 |
| commit | ca038a85d6efaf3cfc7be7b44296adafa8281ac9 (patch) | |
| tree | cce1a812b94c7465b6543f7f3665b682e83834d1 /src | |
| parent | 7ea671c8417f7abfffd7276348a93a7eb2665aaa (diff) | |
lexer: Add more token kinds
Signed-off-by: Quinn Stephens <quinn@osmora.org>
Diffstat (limited to 'src')
| -rw-r--r-- | src/lexer/char_info.c | 16 | ||||
| -rw-r--r-- | src/lexer/lexer.c | 17 |
2 files changed, 23 insertions, 10 deletions
diff --git a/src/lexer/char_info.c b/src/lexer/char_info.c index d66c651..144bbe7 100644 --- a/src/lexer/char_info.c +++ b/src/lexer/char_info.c @@ -29,7 +29,7 @@ #include "lexer/char_info.h" -uint8_t char_info[CHAR_INFO_COUNT] = { +uint16_t char_info[CHAR_INFO_COUNT] = { /* NUL SOH STX ETX EOT ENQ ACK BEL @@ -60,8 +60,8 @@ uint8_t char_info[CHAR_INFO_COUNT] = { */ CHAR_HORZ_WS, 0 , 0 , 0 , 0 , 0 , 0 , 0 , - 0 , 0 , 0 , 0 , - 0 , 0 , 0 , 0 , + CHAR_LPAREN , CHAR_RPAREN , 0 , 0 , + CHAR_COMMA , 0 , CHAR_DOT , 0 , /* 0 1 2 3 @@ -71,7 +71,7 @@ uint8_t char_info[CHAR_INFO_COUNT] = { */ CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , CHAR_DIGIT , - CHAR_DIGIT , CHAR_DIGIT , 0 , 0 , + CHAR_DIGIT , CHAR_DIGIT , CHAR_COLON , CHAR_SEMI , 0 , 0 , 0 , 0 , /* @@ -93,8 +93,8 @@ uint8_t char_info[CHAR_INFO_COUNT] = { */ CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , - CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , 0 , - 0 , 0 , 0 , 0 , + CHAR_UPPER , CHAR_UPPER , CHAR_UPPER , CHAR_LSQUARE, + 0 , CHAR_RSQUARE, 0 , 0 , /* ` a b c @@ -115,6 +115,6 @@ uint8_t char_info[CHAR_INFO_COUNT] = { */ CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , - CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , 0 , - 0 , 0 , 0 , 0 , + CHAR_LOWER , CHAR_LOWER , CHAR_LOWER , CHAR_LCURLY , + 0 , CHAR_RCURLY , 0 , 0 , }; diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index 1431789..e7f277e 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -73,6 +73,9 @@ lex_identifier(struct lexer *ctx, struct token *tok) bool lexer_next(struct lexer *ctx, struct token *tok) { + char ch; + uint16_t ch_info; + if (ctx == NULL || tok == NULL) { return false; } @@ -84,12 +87,22 @@ lexer_next(struct lexer *ctx, struct token *tok) tok->line = ctx->line; tok->col = (int)(tok->pos - ctx->line_start) + 1; - if (char_info[(int)*ctx->pos] & CHAR_ALPHA || *ctx->pos == '_') { + ch = *ctx->pos; + ch_info = char_info[(int)ch]; + + if (ch_info & CHAR_ALPHA || ch == '_') { lex_identifier(ctx, tok); return true; } - if (*ctx->pos == '\0') { + if (ch_info & CHAR_SINGLE) { + tok->kind = ch_info >> CHAR_SINGLE_SHIFT; + tok->len = 1; + ctx->pos++; + return true; + } + + if (ch == '\0') { tok->kind = TOK_EOF; return true; } |