add flex lexer

src/lexer/lex.l

@@ -0,0 +1,47 @@
+%{
+#include "token.h"
+int current_line = 1;
+int current_column = 1;
+
+int yywrap() {
+    return 1;
+}
+%}
+
+%%
+
+\"(\\[a-z\"'`]|[^\\"])*\" {
+    add_token(TOKEN_STRING, yytext, current_line, current_column);
+    current_column += yyleng;
+}
+
+[0-9]+ {
+    add_token(TOKEN_NUMBER, yytext, current_line, current_column);
+    current_column += yyleng;
+}
+
+[a-zA-Z_][a-zA-Z0-9_]* {
+    add_token(TOKEN_IDENTIFIER, yytext, current_line, current_column);
+    current_column += yyleng;
+}
+
+"." {
+    add_token(TOKEN_DOT, yytext, current_line, current_column);
+    current_column += yyleng;
+}
+
+\n {
+    add_token(TOKEN_NEW_LINE, yytext, current_line, current_column);
+    current_line++;
+    current_column = 1;
+}
+
+[ \t]+ {
+    current_column += yyleng;  // Advance column for whitespace
+}
+
+.   { 
+    fprintf(stderr, "Error: Unexpected character '%c' at line %d\n", *yytext, yylineno);
+    exit(1);
+}
+%%

src/lexer/lexer.c

@@ -0,0 +1,20 @@
+#include "lex.yy.h"
+#include "token.h"
+
+int lexer() {
+    const char *input = "term.log\n";
+
+
+
+    void* buffer = yy_scan_string(input);
+    yy_switch_to_buffer(buffer);
+    yylex();  // This fills the token array
+    yy_delete_buffer(buffer);
+
+    for (int i = 0; i < token_count; i++) {
+        printf("Token(type=%d, value='%s')\n", tokens[i].type, tokens[i].value);
+    }
+
+    free_tokens();
+    return 0;
+}

src/lexer/lexer.h

@@ -0,0 +1 @@
+int lexer();

src/lexer/token.c

@@ -0,0 +1,35 @@
+#include <stdlib.h>
+#include <string.h>
+#include "token.h"
+
+#define INITIAL_CAPACITY 64
+
+Token* tokens = NULL;
+int token_count = 0;
+static int token_capacity = 0;
+
+void add_token(TokenType type, const char* value, int line, int column) {
+    if (tokens == NULL) {
+        token_capacity = INITIAL_CAPACITY;
+        tokens = malloc(sizeof(Token) * token_capacity);
+    } else if (token_count >= token_capacity) {
+        token_capacity *= 2;
+        tokens = realloc(tokens, sizeof(Token) * token_capacity);
+    }
+
+    tokens[token_count].type = type;
+    tokens[token_count].value = strdup(value);
+    tokens[token_count].line = line;
+    tokens[token_count].column = column;
+    token_count++;
+}
+
+void free_tokens() {
+    for (int i = 0; i < token_count; ++i) {
+        free(tokens[i].value);
+    }
+    free(tokens);
+    tokens = NULL;
+    token_count = 0;
+    token_capacity = 0;
+}

src/lexer/token.h

@@ -0,0 +1,29 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+typedef enum {
+    TOKEN_STRING,
+    TOKEN_NUMBER,
+    TOKEN_IDENTIFIER,
+    TOKEN_KEYWORD,
+    TOKEN_DOT,
+    TOKEN_NEW_LINE,
+} TokenType;
+
+typedef struct {
+    TokenType type;
+    char* value;
+    int line;
+    int column;
+} Token;
+
+extern int token_count;
+
+extern Token* tokens;
+
+
+void add_token(TokenType type, const char* value, int line, int column);
+
+void free_tokens();
+
+#endif