Open-Argon/Chloride
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change over to a linked list system and start trying to fix bug which causes the parser to not output anything meaningful due to memory deallocation

Browse Source
This commit is contained in:
William Bell
2025-05-28 22:51:27 +01:00
parent 43bc7663fc
commit b82e351daf
13 changed files with 465 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
src/lexer/lex.l
View File

@@ -18,95 +18,318 @@ int yywrap(void *) {
(\"(\\[a-z\"'`]|[^\\"])*\") {
GET_STATE
add_token(TOKENS,TOKEN_STRING, yytext, LINE_NO, COLUMN_NO);
Token * token = create_token(
TOKEN_STRING,
LINE_NO,
COLUMN_NO,
yytext
);
append(TOKENS, token);
ADD_TO_COLUMN
}
('((\\([a-z'\"`]))|[^'])*') {
GET_STATE
add_token(TOKENS,TOKEN_STRING, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(
TOKEN_STRING,
LINE_NO,
COLUMN_NO,
yytext
));
ADD_TO_COLUMN
}
((([0-9]+(\.[0-9]+)?)|(\.[0-9]+))(e((\-|\+)?([0-9]+(\.[0-9]+)?)))?) {
GET_STATE
add_token(TOKENS,TOKEN_NUMBER, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(
TOKEN_NUMBER,
LINE_NO,
COLUMN_NO,
yytext
));
ADD_TO_COLUMN
}
([0-9]+\/[0-9]+) {
GET_STATE
add_token(TOKENS,TOKEN_FRACTION, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(
TOKEN_FRACTION,
LINE_NO,
COLUMN_NO,
yytext
));
ADD_TO_COLUMN
}
"not"[ \t]+"in" { GET_STATE; add_token(TOKENS,TOKEN_NOT_IN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"&&" { GET_STATE; add_token(TOKENS,TOKEN_AND, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"||" { GET_STATE; add_token(TOKENS,TOKEN_OR, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"<=" { GET_STATE; add_token(TOKENS,TOKEN_LE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
">=" { GET_STATE; add_token(TOKENS,TOKEN_GE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"!=" { GET_STATE; add_token(TOKENS,TOKEN_NE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"==" { GET_STATE; add_token(TOKENS,TOKEN_EQ, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"=" { GET_STATE; add_token(TOKENS,TOKEN_ASSIGN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"//" { GET_STATE; add_token(TOKENS,TOKEN_FLOORDIV, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"<" { GET_STATE; add_token(TOKENS,TOKEN_LT, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
">" { GET_STATE; add_token(TOKENS,TOKEN_GT, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"+" { GET_STATE; add_token(TOKENS,TOKEN_PLUS, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"-" { GET_STATE; add_token(TOKENS,TOKEN_MINUS, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"%" { GET_STATE; add_token(TOKENS,TOKEN_MODULO, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"*" { GET_STATE; add_token(TOKENS,TOKEN_STAR, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"/" { GET_STATE; add_token(TOKENS,TOKEN_SLASH, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"^" { GET_STATE; add_token(TOKENS,TOKEN_CARET, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"not"[ \t]+"in" { GET_STATE; append(TOKENS, create_token(TOKEN_NOT_IN,
LINE_NO,
COLUMN_NO,
yytext
));
append(TOKENS, create_token(
TOKEN_NOT_IN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"&&" { GET_STATE; append(TOKENS, create_token(TOKEN_AND,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"||" { GET_STATE; append(TOKENS, create_token(TOKEN_OR,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"<=" { GET_STATE; append(TOKENS, create_token(TOKEN_LE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
">=" { GET_STATE; append(TOKENS, create_token(TOKEN_GE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"!=" { GET_STATE; append(TOKENS, create_token(TOKEN_NE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"==" { GET_STATE; append(TOKENS, create_token(TOKEN_EQ,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"=" { GET_STATE; append(TOKENS, create_token(TOKEN_ASSIGN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"//" { GET_STATE; append(TOKENS, create_token(TOKEN_FLOORDIV,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"<" { GET_STATE; append(TOKENS, create_token(TOKEN_LT,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
">" { GET_STATE; append(TOKENS, create_token(TOKEN_GT,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"+" { GET_STATE; append(TOKENS, create_token(TOKEN_PLUS,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"-" { GET_STATE; append(TOKENS, create_token(TOKEN_MINUS,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"%" { GET_STATE; append(TOKENS, create_token(TOKEN_MODULO,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"*" { GET_STATE; append(TOKENS, create_token(TOKEN_STAR,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"/" { GET_STATE; append(TOKENS, create_token(TOKEN_SLASH,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"^" { GET_STATE; append(TOKENS, create_token(TOKEN_CARET,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"if" { GET_STATE; add_token(TOKENS,TOKEN_IF, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"else" { GET_STATE; add_token(TOKENS,TOKEN_ELSE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"while" { GET_STATE; add_token(TOKENS,TOKEN_WHILE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"forever" { GET_STATE; add_token(TOKENS,TOKEN_FOREVER, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"for" { GET_STATE; add_token(TOKENS,TOKEN_FOR, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"break" { GET_STATE; add_token(TOKENS,TOKEN_BREAK, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"continue" { GET_STATE; add_token(TOKENS,TOKEN_CONTINUE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"return" { GET_STATE; add_token(TOKENS,TOKEN_RETURN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"let" { GET_STATE; add_token(TOKENS,TOKEN_LET, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"import" { GET_STATE; add_token(TOKENS,TOKEN_IMPORT, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"from" { GET_STATE; add_token(TOKENS,TOKEN_FROM, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"do" { GET_STATE; add_token(TOKENS,TOKEN_DO, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"true" { GET_STATE; add_token(TOKENS,TOKEN_TRUE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"false" { GET_STATE; add_token(TOKENS,TOKEN_FALSE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"null" { GET_STATE; add_token(TOKENS,TOKEN_NULL, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"delete" { GET_STATE; add_token(TOKENS,TOKEN_DELETE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"not" { GET_STATE; add_token(TOKENS,TOKEN_NOT, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"in" { GET_STATE; add_token(TOKENS,TOKEN_IN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"try" { GET_STATE; add_token(TOKENS,TOKEN_TRY, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"catch" { GET_STATE; add_token(TOKENS,TOKEN_CATCH, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"if" { GET_STATE; append(TOKENS, create_token(TOKEN_IF,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"else" { GET_STATE; append(TOKENS, create_token(TOKEN_ELSE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"while" { GET_STATE; append(TOKENS, create_token(TOKEN_WHILE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"forever" { GET_STATE; append(TOKENS, create_token(TOKEN_FOREVER,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"for" { GET_STATE; append(TOKENS, create_token(TOKEN_FOR,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"break" { GET_STATE; append(TOKENS, create_token(TOKEN_BREAK,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"continue" { GET_STATE; append(TOKENS, create_token(TOKEN_CONTINUE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"return" { GET_STATE; append(TOKENS, create_token(TOKEN_RETURN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"let" { GET_STATE; append(TOKENS, create_token(TOKEN_LET,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"import" { GET_STATE; append(TOKENS, create_token(TOKEN_IMPORT,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"from" { GET_STATE; append(TOKENS, create_token(TOKEN_FROM,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"do" { GET_STATE; append(TOKENS, create_token(TOKEN_DO,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"true" { GET_STATE; append(TOKENS, create_token(TOKEN_TRUE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"false" { GET_STATE; append(TOKENS, create_token(TOKEN_FALSE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"null" { GET_STATE; append(TOKENS, create_token(TOKEN_NULL,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"delete" { GET_STATE; append(TOKENS, create_token(TOKEN_DELETE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"not" { GET_STATE; append(TOKENS, create_token(TOKEN_NOT,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"in" { GET_STATE; append(TOKENS, create_token(TOKEN_IN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"try" { GET_STATE; append(TOKENS, create_token(TOKEN_TRY,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"catch" { GET_STATE; append(TOKENS, create_token(TOKEN_CATCH,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"(" { GET_STATE; add_token(TOKENS,TOKEN_LPAREN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
")" { GET_STATE; add_token(TOKENS,TOKEN_RPAREN, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"[" { GET_STATE; add_token(TOKENS,TOKEN_LBRACKET, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"]" { GET_STATE; add_token(TOKENS,TOKEN_RBRACKET, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"{" { GET_STATE; add_token(TOKENS,TOKEN_LBRACE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"}" { GET_STATE; add_token(TOKENS,TOKEN_RBRACE, yytext, LINE_NO, COLUMN_NO); ADD_TO_COLUMN; }
"(" { GET_STATE; append(TOKENS, create_token(TOKEN_LPAREN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
")" { GET_STATE; append(TOKENS, create_token(TOKEN_RPAREN,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"[" { GET_STATE; append(TOKENS, create_token(TOKEN_LBRACKET,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"]" { GET_STATE; append(TOKENS, create_token(TOKEN_RBRACKET,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"{" { GET_STATE; append(TOKENS, create_token(TOKEN_LBRACE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
"}" { GET_STATE; append(TOKENS, create_token(TOKEN_RBRACE,
LINE_NO,
COLUMN_NO,
yytext
)); ADD_TO_COLUMN; }
[a-zA-Z_][a-zA-Z0-9_]* {
GET_STATE
add_token(TOKENS,TOKEN_IDENTIFIER, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(TOKEN_IDENTIFIER,
LINE_NO,
COLUMN_NO,
yytext
));
ADD_TO_COLUMN
}
"." {GET_STATE;add_token(TOKENS,TOKEN_DOT, yytext, LINE_NO, COLUMN_NO);ADD_TO_COLUMN}
"," {GET_STATE;add_token(TOKENS,TOKEN_COMMA, yytext, LINE_NO, COLUMN_NO);ADD_TO_COLUMN}
":" {GET_STATE;add_token(TOKENS,TOKEN_COLON, yytext, LINE_NO, COLUMN_NO);ADD_TO_COLUMN}
"." {GET_STATE;append(TOKENS, create_token(TOKEN_DOT,
LINE_NO,
COLUMN_NO,
yytext
));ADD_TO_COLUMN}
"," {GET_STATE;append(TOKENS, create_token(TOKEN_COMMA,
LINE_NO,
COLUMN_NO,
yytext
));ADD_TO_COLUMN}
":" {GET_STATE;append(TOKENS, create_token(TOKEN_COLON,
LINE_NO,
COLUMN_NO,
yytext
));ADD_TO_COLUMN}
\n {
GET_STATE
add_token(TOKENS,TOKEN_NEW_LINE, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(TOKEN_NEW_LINE,
LINE_NO,
COLUMN_NO,
yytext
));
COLUMN_NO = 1;
}
[ \t]+ {
GET_STATE
if (COLUMN_NO == 1){
add_token(TOKENS,TOKEN_INDENT, yytext, LINE_NO, COLUMN_NO);
append(TOKENS, create_token(TOKEN_INDENT,
LINE_NO,
COLUMN_NO,
yytext
));
}
ADD_TO_COLUMN // Advance column for whitespace
}

3
src/lexer/lexer.h
View File

@@ -1,10 +1,11 @@
#include "token.h"
#include "../list/list.h"
typedef struct {
const char *path;
const char *content;
int current_column;
TokenStruct* tokens;
LinkedList* tokens;
// add more fields as needed
} LexerState;

54
src/lexer/token.c
View File

@@ -1,47 +1,13 @@
#include <stdlib.h>
#include <string.h>
#include "token.h"
#include <stdio.h>
#include <stdlib.h>
#define INITIAL_CAPACITY 64
TokenStruct* init_token() {
TokenStruct *tokenStruct = malloc(sizeof(TokenStruct));
if (tokenStruct == NULL) {
// handle malloc failure
return NULL;
}
tokenStruct->count = 0;
tokenStruct->capacity = INITIAL_CAPACITY;
tokenStruct->tokens = malloc(sizeof(Token) * INITIAL_CAPACITY);
if (tokenStruct->tokens == NULL) {
// handle malloc failure
free(tokenStruct);
return NULL;
}
return tokenStruct;
}
void add_token(TokenStruct* token,TokenType type, const char* value, int line, int column) {
if (token->count >= token->capacity) {
token->capacity *= 2;
token->tokens = realloc(token->tokens, sizeof(Token) * token->capacity);
}
token->tokens[token->count].type = type;
token->tokens[token->count].value = strdup(value);
token->tokens[token->count].line = line;
token->tokens[token->count].column = column;
token->count++;
}
void free_tokens(TokenStruct* token) {
for (int i = 0; i < token->count; ++i) {
free(token->tokens[i].value);
}
free(token->tokens);
token->tokens = NULL;
token->count = 0;
token->capacity = 0;
free(token);
Token *create_token(TokenType type, int line, int column, char *value) {
Token * token = malloc(sizeof(Token));
printf("%s\n", value);
token->type = type;
token->line=line;
token->column=column;
token->value=value;
return token;
}

14
src/lexer/token.h
View File

@@ -71,17 +71,5 @@ typedef struct {
char *value;
} Token;
typedef struct {
int count;
int capacity;
Token *tokens;
} TokenStruct;
TokenStruct *init_token();
void add_token(TokenStruct *token, TokenType type, const char *value, int line,
int column);
void free_tokens(TokenStruct *token);
Token *create_token(TokenType type, int line, int column, char *value);
#endif