diff --git a/CMakeLists.txt b/CMakeLists.txt index 65f0cb54..58633983 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,19 +61,15 @@ set(BUILTINS_HEADERS # Set lexer source and headers set(LEXER_SOURCE - src/lexer/tokenizer/token.c + src/parser/lexer/token.c - src/lexer/tokenizer/tokenizer.c - - src/lexer/lexer.c + src/parser/lexer/lexer.c ) set(LEXER_HEADERS - src/lexer/tokenizer/token.h - - src/lexer/tokenizer/tokenizer.h + src/parser/lexer/token.h - src/lexer/lexer.h + src/parser/lexer/lexer.h ) # Set AST source and headers @@ -100,7 +96,7 @@ set(PARSER_HEADERS # Set Main executables for lexer, ast, and parser set(LEXER_MAIN - src/lexer/main.c + src/parser/lexer/main.c ) set(AST_MAIN diff --git a/Dockerfile b/Dockerfile index 8614d524..fe211bfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:latest LABEL name="The One Programming Language" # LLVM version -ARG LLVM_VERSION=12.0.1 +ARG LLVM_VERSION=16.0.3 # LLVM dependencies RUN apk --no-cache add \ @@ -58,8 +58,7 @@ RUN cmake --build ./build --config Debug --target all -j 6 -- WORKDIR /One/build # Running example input.one -RUN ./lexer ../src/input.one log -RUN cat log +RUN ./lexer ../src/input.one log && cat log # Running tests RUN ./lexer_test diff --git a/src/ast/ast.h b/src/ast/ast.h index 0259a470..583f1d61 100644 --- a/src/ast/ast.h +++ b/src/ast/ast.h @@ -14,7 +14,7 @@ #include #include "../builtins/array.h" -#include "../lexer/tokenizer/token.h" +#include "../parser/lexer/token.h" typedef enum _token_type TokenType; typedef struct _location Location; diff --git a/src/builtins/error.c b/src/builtins/error.c index 9a3d5a8b..7d118e3a 100644 --- a/src/builtins/error.c +++ b/src/builtins/error.c @@ -8,7 +8,7 @@ **/ -#include +#include "../parser/lexer/lexer.h" #include "error.h" diff --git a/src/parser/lexer/lexer.c b/src/parser/lexer/lexer.c index f64bda15..bf662439 100644 --- a/src/parser/lexer/lexer.c +++ b/src/parser/lexer/lexer.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include "../../builtins/error.h" #ifdef _ONE_TEST_ #define debug_lexer(format, args...) \ @@ -617,3 +617,49 @@ void lexer_free() // free(lexer.start); // free(lexer.current); } + +/* + * @function: lexer_trace + * @description: Log and trace items of tokens + * @arguments: FILE* file_out, char* data, Token** tokens + * @return: nothing, void + */ +void lexer_trace(FILE* file_out, const char* data, Token** tokens) +{ + debug_lexer("lexer_trace"); + + while (tokens != NULL && *tokens != NULL) + if (tokens == NULL) return; + + while (*tokens != NULL) + { + Token* t = *tokens; + printf("sizeof(t) = %zu\n", sizeof t); + printf("-->1\n"); + printf("-->%s\n", t == NULL ? "y" : "n"); + // printf("sizeof(t) = %zu\n", sizeof t); + // printf("-->1\n"); + // printf("-->%s\n", t == NULL ? "y" : "n"); + char* t_name = token_name(t->type); + printf("-->2\n"); + // printf("-->2\n"); + bool has1 = file_convert_index_to_rc(data, t->pos.index, &t->pos.line, &t->pos.column); + printf("-->3\n"); + // printf("-->3\n"); + bool has2 = file_convert_index_to_rc(data, t->pos_end.index, &t->pos_end.line, &t->pos_end.column); + printf("-->4\n"); + // printf("-->4\n"); + + fprintf(file_out, "[%zu:%zu] [%zu:%zu - %zu:%zu] %s", t->pos.tokens, t->length, t->pos.line, t->pos.column, t->pos_end.line, t->pos_end.column, t_name); + printf("-->5\n"); + // printf("-->5\n"); + + if (t->value != NULL) + { + fprintf(file_out, ": \"%s\"", t->value); + } + fprintf(file_out, "\n"); + if (t->type == TOKEN_EOF) break; + tokens++; + } +} diff --git a/src/parser/lexer/main.c b/src/parser/lexer/main.c index b30b8ec6..6a59fec7 100644 --- a/src/parser/lexer/main.c +++ b/src/parser/lexer/main.c @@ -11,7 +11,7 @@ #include #include -#include +#include "../../builtins/file.h" #include "lexer.h" extern Token* current; diff --git a/src/parser/lexer/test.sh b/src/parser/lexer/test.sh index 70ce8571..dce91f55 100755 --- a/src/parser/lexer/test.sh +++ b/src/parser/lexer/test.sh @@ -2,5 +2,5 @@ clear bash build.sh -./lexer ../input.one log +./lexer ../../input.one log cat log diff --git a/src/parser/lexer/token.c b/src/parser/lexer/token.c index d766b7b2..7298445a 100644 --- a/src/parser/lexer/token.c +++ b/src/parser/lexer/token.c @@ -10,7 +10,8 @@ #include "token.h" #include "lexer.h" -#include +#include "../../builtins/array.h" +#include "../../builtins/error.h" #ifdef _ONE_TEST_ #define debug_token(format, args...) \ @@ -587,3 +588,49 @@ bool token_is_skip(TokenType type) type == TOKEN_SKIP_COMMENT_SINGLE || type == TOKEN_SKIP_COMMENT_MULTI); } + +/* + * @function: tokenizer_string + * @description: Create a array of tokens from a one program source-code char* + * @inputs: char* of a One program source-code + * @return: Array of Token + */ +Token** tokenizer_string(char* data) +{ + debug_token("tokenizer_string"); + debug_token("tokenizer_string: %s", data); + + lexer_init(data); + + Array tokens; + array_init(&tokens); + + size_t i; + Token* t; + for (;;) + { + t = lexer_scan(); + // printf("==>%s\n", token_name(t->type)); + array_push(&tokens, t); + // debug_lexer("parser_scan: print_token %s", token_name(t->type)); + if (t->type == TOKEN_ERROR) + { + printf("Error: %s\n", t->value); + break; + } + else if (t->type == TOKEN_EOF) + break; + } + + debug_token("tokenizer_string: count of tokens is %d", tokens.count); + debug_token("tokenizer_string: size of tokens is %d", tokens.size); + + // while (*data != '\0') + // { + // Token* t = token_make_value(TOKEN_VALUE_IDENTIFIER, (char*){data}); + // array_push(&tokens, t); + // data++; + // } + + return (Token**)tokens.data; +} diff --git a/src/parser/lexer/token.h b/src/parser/lexer/token.h index 992e9b72..20de20d1 100644 --- a/src/parser/lexer/token.h +++ b/src/parser/lexer/token.h @@ -407,4 +407,12 @@ size_t token_utf8_string_length(char* s); */ bool token_is_skip(TokenType type); +/* + * @function: tokenizer_string + * @description: Create a array of tokens from a one program source-code char* + * @inputs: char* of a One program source-code + * @return: Array of Token + */ +Token** tokenizer_string(char* data); + #endif // _ONE_TOKENIZER_TOKEN_H_ diff --git a/src/parser/parser.h b/src/parser/parser.h index 1f72f363..6dd891d8 100644 --- a/src/parser/parser.h +++ b/src/parser/parser.h @@ -17,7 +17,7 @@ #include "parser_token.h" -#include +#include "../builtins/array.h" typedef struct { diff --git a/test/build.sh b/test/build.sh index 45c9061a..11b243da 100755 --- a/test/build.sh +++ b/test/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash FLAGS="-o" -FILES="../src/builtins/array.c ../src/builtins/error.c ../src/builtins/file.c ../src/ast/ast.c ../src/parser/parser.c ../src/parser/parser_token.c ../src/lexer/lexer.c ../src/lexer/tokenizer/token.c ../src/lexer/tokenizer/tokenizer.c -lLLVM-12 -D_ONE_TEST_" +FILES="../src/builtins/array.c ../src/builtins/error.c ../src/builtins/file.c ../src/ast/ast.c ../src/parser/parser.c ../src/parser/parser_token.c ../src/parser/lexer/lexer.c ../src/parser/lexer/token.c -lLLVM-12 -D_ONE_TEST_" CC="clang" # compile argument diff --git a/test/test.h b/test/test.h index c5945def..b22a393e 100644 --- a/test/test.h +++ b/test/test.h @@ -23,7 +23,7 @@ #include "../src/builtins/file.h" -#include "../src/lexer/lexer.h" +#include "../src/parser/lexer/lexer.h" #include "../src/parser/parser.h" typedef struct