add lex.h and lex.c except next_token

include/globals.h

@@ -1,30 +1,35 @@
 #pragma once
+#include <stdint.h>
 
 typedef enum {
     LBRACK,
     RBRACK,
     LCURLY,
     RCURLY,
+    LPAREN,
+    RPAREN,
 
-    ID,
     SEMI,
     COMMA,
     ARROW,
     STAR,
+    ANDREF,
+    DOLLAR,
+
     COMMENT,
+    ID,
     NUM,
+    STRING_LITERAL,
 
     RETURN,
     IF,
     ELSE,
-
-    STRING_LITERAL,
-
+    
     ERROR
 } TokenType;
 
 typedef struct {
     TokenType type;
     char *data;
-    int line;
+    uint32_t line;
 } Token;

include/lex.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "globals.h"
+
+#define LEX_BUF_SIZE 4096
+
+typedef struct Lexer {
+    uint32_t linepos;
+    uint32_t lineno;
+
+    FILE *source;
+
+    int flag_EOF;
+
+    char *buffer;
+    char *curr;
+
+    size_t bytes_in_buffer;
+} Lexer;
+
+Lexer *new_lexer();
+
+void lexer_set_source(Lexer *lexer, FILE *source);
+
+void lexer_fill_buffer(Lexer *lexer);
+
+void lexer_consume(Lexer *lexer);
+
+char lexer_curr(Lexer *lexer);
+
+char lexer_peek(Lexer *lexer);
+
+Token lexer_next_token(Lexer *lexer);
+
+void free_lexer(Lexer *lexer);