diff options
| author | Quinn Stephens <quinn@osmora.org> | 2025-06-08 10:10:04 -0400 |
|---|---|---|
| committer | Quinn Stephens <quinn@osmora.org> | 2025-06-08 10:10:04 -0400 |
| commit | ca038a85d6efaf3cfc7be7b44296adafa8281ac9 (patch) | |
| tree | cce1a812b94c7465b6543f7f3665b682e83834d1 /include | |
| parent | 7ea671c8417f7abfffd7276348a93a7eb2665aaa (diff) | |
lexer: Add more token kinds
Signed-off-by: Quinn Stephens <quinn@osmora.org>
Diffstat (limited to 'include')
| -rw-r--r-- | include/lexer/char_info.h | 17 | ||||
| -rw-r--r-- | include/lexer/token.h | 11 |
2 files changed, 27 insertions, 1 deletions
diff --git a/include/lexer/char_info.h b/include/lexer/char_info.h index 59db2be..d9205b0 100644 --- a/include/lexer/char_info.h +++ b/include/lexer/char_info.h @@ -31,6 +31,7 @@ #define _LEXER_CHAR_INFO_H 1 #include <stdint.h> +#include "lexer/token.h" #define CHAR_HORZ_WS (1 << 0) #define CHAR_VERT_WS (1 << 1) @@ -38,6 +39,7 @@ #define CHAR_XDIGIT (1 << 3) #define CHAR_UPPER (1 << 4) #define CHAR_LOWER (1 << 5) +#define CHAR_SINGLE (1 << 6) #define CHAR_HEX (CHAR_DIGIT | CHAR_XDIGIT) #define CHAR_XUPPER (CHAR_XDIGIT | CHAR_UPPER) @@ -46,8 +48,21 @@ #define CHAR_ALPHA (CHAR_UPPER | CHAR_LOWER) #define CHAR_ALNUM (CHAR_ALPHA | CHAR_DIGIT) +#define CHAR_SINGLE_SHIFT 8 +#define MAKE_SINGLE(type) ((type << CHAR_SINGLE_SHIFT) | CHAR_SINGLE) +#define CHAR_COMMA MAKE_SINGLE(TOK_COMMA) +#define CHAR_DOT MAKE_SINGLE(TOK_DOT) +#define CHAR_COLON MAKE_SINGLE(TOK_COLON) +#define CHAR_SEMI MAKE_SINGLE(TOK_SEMICOLON) +#define CHAR_LPAREN MAKE_SINGLE(TOK_LPAREN) +#define CHAR_RPAREN MAKE_SINGLE(TOK_RPAREN) +#define CHAR_LCURLY MAKE_SINGLE(TOK_LCURLY) +#define CHAR_RCURLY MAKE_SINGLE(TOK_RCURLY) +#define CHAR_LSQUARE MAKE_SINGLE(TOK_LSQUARE) +#define CHAR_RSQUARE MAKE_SINGLE(TOK_RSQUARE) + #define CHAR_INFO_COUNT 256 -extern uint8_t char_info[CHAR_INFO_COUNT]; +extern uint16_t char_info[CHAR_INFO_COUNT]; #endif /* !_LEXER_CHAR_INFO_H */ diff --git a/include/lexer/token.h b/include/lexer/token.h index ab8231d..f51b5e9 100644 --- a/include/lexer/token.h +++ b/include/lexer/token.h @@ -39,6 +39,17 @@ enum token_kind { TOK_IDENTIFIER, + TOK_COMMA, + TOK_DOT, + TOK_COLON, + TOK_SEMICOLON, + TOK_LPAREN, + TOK_RPAREN, + TOK_LCURLY, + TOK_RCURLY, + TOK_LSQUARE, + TOK_RSQUARE, + /* * Do not modify this ordering without updating parser/types.c */ |