change string literals to be length terminated instead of null terminated, so null characters can be embeded

src/lexer/lexer.c

@@ -1,7 +1,27 @@
 #include "lexer.h"
 #include "lex.yy.h"
+#include "../string/string.h"
 
 void lexer(LexerState state) {
+  size_t line = 1;
+  size_t column = 1;
+  int ch;
+  while ((ch = fgetc(state.file)) != EOF) {
+    if (ch == 0 || (ch < 0x20 && ch != '\n' && ch != '\r' && ch != '\t')) {
+      fprintf(stderr, "%s:%zu:%zu error: disallowed character\n", state.path,
+              line, column);
+      exit(1);
+    }
+
+    if (ch == '\n') {
+      line++;
+      column = 1;
+    } else {
+      column++;
+    }
+  }
+  rewind(state.file);
+
   yyscan_t scanner;
 
   yylex_init(&scanner);
@@ -12,11 +32,14 @@ void lexer(LexerState state) {
 
   int token;
   while ((token = yylex(scanner)) != 0) {
-    Token *token_struct =
-        create_token(token, state.current_line+1, state.current_column + 1,
-                     yyget_text(scanner));
-    darray_push(state.tokens, token_struct);
-    free(token_struct);
+    Token token_struct = (Token){
+      token,
+      state.current_line+1,
+      state.current_column+1,
+      yyget_leng(scanner),
+      cloneString(yyget_text(scanner))
+    };
+    darray_push(state.tokens, &token_struct);
     if (token == TOKEN_NEW_LINE) {
       state.current_line += 1;
       state.current_column = 0;