minor implement lexer

include/globals.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <stdint.h>
+#include <stddef.h>
 
 typedef enum {
     LBRACK,
@@ -21,15 +22,23 @@ typedef enum {
     NUM,
     STRING_LITERAL,
 
+    VAL,
     RETURN,
     IF,
     ELSE,
-    
+
+    EOF_TOKEN,
     ERROR
 } TokenType;
 
+typedef struct {
+    size_t len;
+    char* string;
+} TokenString;
+
 typedef struct {
     TokenType type;
-    char *data;
     uint32_t line;
+    TokenString data;
 } Token;
+

include/lex.h

@@ -22,7 +22,7 @@ typedef struct Lexer {
     size_t bytes_in_buffer;
 } Lexer;
 
-Lexer *new_lexer();
+Lexer *lexer_new();
 
 void lexer_set_source(Lexer *lexer, FILE *source);
 
@@ -36,4 +36,4 @@ char lexer_peek(Lexer *lexer);
 
 Token lexer_next_token(Lexer *lexer);
 
-void free_lexer(Lexer *lexer);
+void lexer_free(Lexer *lexer);

include/util.h

@@ -0,0 +1,12 @@
+#pragma once
+#include "globals.h"
+
+int is_whitespace(char c);
+
+int is_alpha(char c);
+
+int is_digit(char c);
+
+int is_alpha_digit(char c);
+
+void print_token(Token tok);