add tokens to state struct

src/lexer/lexer.c

@@ -1,31 +1,26 @@
 #include "lex.yy.h"
-#include "token.h"
 #include "lexer.h"
+#include "../string/string.h"
+#include <stdlib.h>
 
-int lexer() {
+void lexer(LexerState state) {
     yyscan_t scanner;
-    LexerState state = { "file1.src", 1 };
 
-    const char *input = "let x = 10";
+    char *unquoted = unquote(state.content);
+    if (unquoted) {
+        printf("%s\n", unquoted);
+        free(unquoted);
+    }
 
     yylex_init(&scanner);
 
-    // Set the extra data *before* scanning
     yyset_extra(&state, scanner);
 
-    void* buffer = yy_scan_string(input, scanner);
+    void* buffer = yy_scan_string(state.content, scanner);
     yy_switch_to_buffer(buffer, scanner);
 
-    yylex(scanner);  // This fills the token array
+    yylex(scanner);
 
     yy_delete_buffer(buffer, scanner);
     yylex_destroy(scanner);
-
-    // print tokens etc.
-    for (int i = 0; i < token_count; i++) {
-        printf("Token(type=%d, value='%s')\n", tokens[i].type, tokens[i].value);
-    }
-
-    free_tokens();
-    return 0;
 }