Open-Argon/Chloride
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add state to lexer for parrellel support

Browse Source
This commit is contained in:
William Bell
2025-05-27 04:07:53 +01:00
parent 1540645759
commit 296600ee11
5 changed files with 55 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/lexer/lex.l
View File

@@ -1,9 +1,14 @@
%option reentrant
%{
#include "token.h"
int current_line = 1;
int current_column = 1;
#include "lexer.h"
#define GET_STATE LexerState *state = (LexerState *)yyget_extra(yyscanner);
#define GET_ADD_COLUMN COLUMN_NO += yyleng;
#define LINE_NO yylineno+1
#define COLUMN_NO state->current_column
int yywrap() {
int yywrap(void *) {
return 1;
}
%}
@@ -11,37 +16,43 @@ int yywrap() {
%%
\"(\\[a-z\"'`]|[^\\"])*\" {
add_token(TOKEN_STRING, yytext, current_line, current_column);
current_column += yyleng;
GET_STATE
add_token(TOKEN_STRING, yytext, LINE_NO, COLUMN_NO);
GET_ADD_COLUMN
}
[0-9]+ {
add_token(TOKEN_NUMBER, yytext, current_line, current_column);
current_column += yyleng;
GET_STATE
add_token(TOKEN_NUMBER, yytext, LINE_NO, COLUMN_NO);
GET_ADD_COLUMN
}
[a-zA-Z_][a-zA-Z0-9_]* {
add_token(TOKEN_IDENTIFIER, yytext, current_line, current_column);
current_column += yyleng;
GET_STATE
add_token(TOKEN_IDENTIFIER, yytext, LINE_NO, COLUMN_NO);
GET_ADD_COLUMN
}
"." {
add_token(TOKEN_DOT, yytext, current_line, current_column);
current_column += yyleng;
GET_STATE
add_token(TOKEN_DOT, yytext, LINE_NO, COLUMN_NO);
GET_ADD_COLUMN
}
\n {
add_token(TOKEN_NEW_LINE, yytext, current_line, current_column);
current_line++;
current_column = 1;
GET_STATE
add_token(TOKEN_NEW_LINE, yytext, LINE_NO, COLUMN_NO);
COLUMN_NO = 1;
}
[ \t]+ {
current_column += yyleng; // Advance column for whitespace
GET_STATE
GET_ADD_COLUMN // Advance column for whitespace
}
. {
fprintf(stderr, "Error: Unexpected character '%c' at line %d\n", *yytext, yylineno);
. {
GET_STATE
fprintf(stderr, "Error in file %s on line %d column %d: unexpected character '%s'\n", state->filename, LINE_NO, COLUMN_NO, yytext);
exit(1);
}
%%