From ca038a85d6efaf3cfc7be7b44296adafa8281ac9 Mon Sep 17 00:00:00 2001
From: Quinn Stephens <quinn@osmora.org>
Date: Sun, 8 Jun 2025 10:10:04 -0400
Subject: lexer: Add more token kinds

Signed-off-by: Quinn Stephens <quinn@osmora.org>
---
 include/lexer/char_info.h | 17 ++++++++++++++++-
 include/lexer/token.h     | 11 +++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

(limited to 'include/lexer')

diff --git a/include/lexer/char_info.h b/include/lexer/char_info.h
index 59db2be..d9205b0 100644
--- a/include/lexer/char_info.h
+++ b/include/lexer/char_info.h
@@ -31,6 +31,7 @@
 #define _LEXER_CHAR_INFO_H 1
 
 #include <stdint.h>
+#include "lexer/token.h"
 
 #define CHAR_HORZ_WS (1 << 0)
 #define CHAR_VERT_WS (1 << 1)
@@ -38,6 +39,7 @@
 #define CHAR_XDIGIT  (1 << 3)
 #define CHAR_UPPER   (1 << 4)
 #define CHAR_LOWER   (1 << 5)
+#define CHAR_SINGLE  (1 << 6)
 
 #define CHAR_HEX        (CHAR_DIGIT   | CHAR_XDIGIT)
 #define CHAR_XUPPER     (CHAR_XDIGIT  | CHAR_UPPER)
@@ -46,8 +48,21 @@
 #define CHAR_ALPHA      (CHAR_UPPER   | CHAR_LOWER)
 #define CHAR_ALNUM      (CHAR_ALPHA   | CHAR_DIGIT)
 
+#define CHAR_SINGLE_SHIFT 8
+#define MAKE_SINGLE(type) ((type << CHAR_SINGLE_SHIFT) | CHAR_SINGLE)
+#define CHAR_COMMA   MAKE_SINGLE(TOK_COMMA)
+#define CHAR_DOT     MAKE_SINGLE(TOK_DOT)
+#define CHAR_COLON   MAKE_SINGLE(TOK_COLON)
+#define CHAR_SEMI    MAKE_SINGLE(TOK_SEMICOLON)
+#define CHAR_LPAREN  MAKE_SINGLE(TOK_LPAREN)
+#define CHAR_RPAREN  MAKE_SINGLE(TOK_RPAREN)
+#define CHAR_LCURLY  MAKE_SINGLE(TOK_LCURLY)
+#define CHAR_RCURLY  MAKE_SINGLE(TOK_RCURLY)
+#define CHAR_LSQUARE MAKE_SINGLE(TOK_LSQUARE)
+#define CHAR_RSQUARE MAKE_SINGLE(TOK_RSQUARE)
+
 #define CHAR_INFO_COUNT 256
 
-extern uint8_t char_info[CHAR_INFO_COUNT];
+extern uint16_t char_info[CHAR_INFO_COUNT];
 
 #endif /* !_LEXER_CHAR_INFO_H */
diff --git a/include/lexer/token.h b/include/lexer/token.h
index ab8231d..f51b5e9 100644
--- a/include/lexer/token.h
+++ b/include/lexer/token.h
@@ -39,6 +39,17 @@ enum token_kind {
 
     TOK_IDENTIFIER,
 
+    TOK_COMMA,
+    TOK_DOT,
+    TOK_COLON,
+    TOK_SEMICOLON,
+    TOK_LPAREN,
+    TOK_RPAREN,
+    TOK_LCURLY,
+    TOK_RCURLY,
+    TOK_LSQUARE,
+    TOK_RSQUARE,
+
     /*
      * Do not modify this ordering without updating parser/types.c
      */
-- 
cgit v1.2.3