lexer: Add more token kinds

Signed-off-by: Quinn Stephens <quinn@osmora.org>

1 files changed, 15 insertions, 2 deletions

diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c
index 1431789..e7f277e 100644
--- a/src/lexer/lexer.c
+++ b/src/lexer/lexer.c
@@ -73,6 +73,9 @@ lex_identifier(struct lexer *ctx, struct token *tok)
 bool
 lexer_next(struct lexer *ctx, struct token *tok)
 {
+    char ch;
+    uint16_t ch_info;
+
     if (ctx == NULL || tok == NULL) {
         return false;
     }
@@ -84,12 +87,22 @@ lexer_next(struct lexer *ctx, struct token *tok)
     tok->line = ctx->line;
     tok->col = (int)(tok->pos - ctx->line_start) + 1;
 
-    if (char_info[(int)*ctx->pos] & CHAR_ALPHA || *ctx->pos == '_') {
+    ch = *ctx->pos;
+    ch_info = char_info[(int)ch];
+
+    if (ch_info & CHAR_ALPHA || ch == '_') {
         lex_identifier(ctx, tok);
         return true;
     }
 
-    if (*ctx->pos == '\0') {
+    if (ch_info & CHAR_SINGLE) {
+        tok->kind = ch_info >> CHAR_SINGLE_SHIFT;
+        tok->len = 1;
+        ctx->pos++;
+        return true;
+    }
+
+    if (ch == '\0') {
         tok->kind = TOK_EOF;
         return true;
     }