diff --git a/compile_commands.json b/compile_commands.json index ae1e537..7b9ccfe 100644 --- a/compile_commands.json +++ b/compile_commands.json @@ -118,12 +118,12 @@ "-x", "c", "-o", - "/tmp/main-92ad35.o", + "/tmp/main-4f432a.o", "src/main.c" ], "directory": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json", "file": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json/src/main.c", - "output": "/tmp/main-92ad35.o" + "output": "/tmp/main-4f432a.o" }, { "arguments": [ @@ -187,12 +187,12 @@ "-x", "c", "-o", - "/tmp/dstring-0fd2b9.o", + "/tmp/dstring-841d2e.o", "src/dstring/dstring.c" ], "directory": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json", "file": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json/src/dstring/dstring.c", - "output": "/tmp/dstring-0fd2b9.o" + "output": "/tmp/dstring-841d2e.o" }, { "arguments": [ @@ -256,11 +256,11 @@ "-x", "c", "-o", - "/tmp/lexer-77effd.o", + "/tmp/lexer-5cd579.o", "src/lexer/lexer.c" ], "directory": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json", "file": "/mnt/3A5CDF785CDF2CFF/Users/abdoo/dev/say_it_in_json/src/lexer/lexer.c", - "output": "/tmp/lexer-77effd.o" + "output": "/tmp/lexer-5cd579.o" } ] diff --git a/src/lexer/lexer.c b/src/lexer/lexer.c index 5627520..0b2712f 100644 --- a/src/lexer/lexer.c +++ b/src/lexer/lexer.c @@ -90,6 +90,8 @@ struct lexer { dstr_t *current_string; bool token_ready; token_t token; + bool has_extra_token; + token_t extra_token; }; void stack_push(state_stack_t *stack, lexer_state_t value); @@ -97,11 +99,13 @@ lexer_state_t stack_pop(state_stack_t *stack); void append_to_lex_str(lexer_string_t *str, char input); void clear_lex_str(lexer_string_t *str); -bool strequal(const char *first, const char *second); +bool strequal(const char *const first, const char *const second); bool is_valid_hex_char(const char input); bool ishex(const char input); -void set_token(lexer_t *lexer, token_type_t type, token_value_t value); +token_t dstr_to_numerical_token(const dstr_t *str); +void set_token(token_t *token, u64 line, u64 column, token_type_t type, + token_value_t value); void lexer_state_machine(lexer_t *lexer, char input); lexer_state_t handle_lexer_start(lexer_t *lexer, char input); @@ -151,6 +155,8 @@ void lexer_init(lexer_t **lexer) { (*lexer)->current_string = dstr_with_capacity(STRING_BUF_START_CAPACITY); (*lexer)->token_ready = false; (*lexer)->token = (token_t){0}; + (*lexer)->has_extra_token = false; + (*lexer)->extra_token = (token_t){0}; if (!((*lexer)->current_string)) { lexer_free(lexer); @@ -178,19 +184,23 @@ token_t get_next_token(lexer_t *lexer, const char *text) { char c; while (lexer->cursor < lexer->text_length) { + if (lexer->has_extra_token) { + lexer->has_extra_token = false; + + return lexer->extra_token; + } + c = lexer->text[(lexer->cursor)++]; + lexer_state_machine(lexer, c); + if (c == '\n') { ++(lexer->line); lexer->column = 0; - continue; + } else { + ++(lexer->column); } - lexer_state_machine(lexer, c); - - // Track the position in the text - ++(lexer->column); - if (lexer->current == LEXER_STATE_ERROR) { } else if (lexer->token_ready) { lexer->token_ready = false; @@ -269,9 +279,10 @@ void print_token(token_t token) { printf("%15s, VALUE: %s", "TK_STR_VAL", token.value.string); break; case TK_INTEGER: + printf("%15s, VALUE: %ld", "TK_INTEGER", token.value.num_int); break; case TK_DOUBLE: - printf("%15s, VALUE: %s", "TK_DOUBLE", token.value.string); + printf("%15s, VALUE: %f", "TK_DOUBLE", token.value.num_frac); break; } @@ -345,7 +356,7 @@ void clear_lex_str(lexer_string_t *lex_str) { lex_str->size = 0; } -bool strequal(const char *first, const char *second) { +bool strequal(const char *const first, const char *const second) { return strcmp(first, second) == 0; } @@ -373,12 +384,27 @@ bool ishex(const char input) { return isdigit(input) || is_valid_hex_char(input); } -void set_token(lexer_t *lexer, token_type_t type, token_value_t value) { - lexer->token_ready = true; +token_t dstr_to_numerical_token(const dstr_t *str) { + token_t token = {0}; - lexer->token = (token_t){ - .line = lexer->line, - .column = lexer->column, + bool is_double = dstr_find(str, ".") != -1; + + token.type = is_double ? TK_DOUBLE : TK_INTEGER; + + if (is_double) { + token.value.num_frac = strtod(dstr_to_cstr(str), NULL); + } else { + token.value.num_int = atol(dstr_to_cstr(str)); + } + + return token; +} + +void set_token(token_t *token, u64 line, u64 column, token_type_t type, + token_value_t value) { + *token = (token_t){ + .line = line, + .column = column, .type = type, .value = value, }; @@ -484,11 +510,15 @@ lexer_state_t handle_lexer_start(lexer_t *lexer, char input) { switch (input) { case '{': - set_token(lexer, TK_L_BRACE, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_L_BRACE, + (token_value_t){0}); return LEXER_STATE_OBJECT_START; case '[': - set_token(lexer, TK_L_BRACKET, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_L_BRACKET, + (token_value_t){0}); return LEXER_STATE_ARRAY_START; } @@ -512,7 +542,18 @@ lexer_state_t handle_collection_end(lexer_t *lexer, char input) { bool object_end = lexer->current == LEXER_STATE_OBJECT && input == '}'; if (object_end) { - set_token(lexer, TK_R_BRACE, (token_value_t){0}); + token_t *token; + + if (lexer->token_ready) { + lexer->has_extra_token = true; + token = &(lexer->extra_token); + } else { + lexer->token_ready = true; + token = &(lexer->token); + } + + set_token(token, lexer->line, lexer->column, TK_R_BRACE, + (token_value_t){0}); return LEXER_STATE_OBJECT_END; } @@ -520,7 +561,18 @@ lexer_state_t handle_collection_end(lexer_t *lexer, char input) { bool array_end = lexer->current == LEXER_STATE_ARRAY && input == ']'; if (array_end) { - set_token(lexer, TK_R_BRACKET, (token_value_t){0}); + token_t *token; + + if (lexer->token_ready) { + lexer->has_extra_token = true; + token = &(lexer->extra_token); + } else { + lexer->token_ready = true; + token = &(lexer->token); + } + + set_token(token, lexer->line, lexer->column, TK_R_BRACKET, + (token_value_t){0}); return LEXER_STATE_ARRAY_END; } @@ -531,11 +583,15 @@ lexer_state_t handle_collection_end(lexer_t *lexer, char input) { void handle_input_after_collection_end(lexer_t *lexer, char input) { switch (input) { case '}': - set_token(lexer, TK_R_BRACE, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_R_BRACE, + (token_value_t){0}); break; case ']': - set_token(lexer, TK_R_BRACKET, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_R_BRACKET, + (token_value_t){0}); break; } @@ -590,11 +646,15 @@ lexer_state_t handle_value(lexer_t *lexer, char input) { return LEXER_STATE_DECIMAL; case '{': - set_token(lexer, TK_L_BRACE, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_L_BRACE, + (token_value_t){0}); return LEXER_STATE_OBJECT_START; case '[': - set_token(lexer, TK_L_BRACKET, (token_value_t){0}); + lexer->token_ready = true; + set_token(&(lexer->token), lexer->line, lexer->column, TK_L_BRACKET, + (token_value_t){0}); return LEXER_STATE_ARRAY_START; case 't': @@ -617,12 +677,15 @@ lexer_state_t handle_string(lexer_t *lexer, char input) { case '"': { lexer_state_t string_type = lexer->stack.stack[lexer->stack.size - 1]; + lexer->token_ready = true; + token_t *token = &(lexer->token); + u64 column = lexer->column - dstr_length(lexer->current_string); + token_value_t value = {.string = dstr_to_cstr(lexer->current_string)}; + if (string_type == LEXER_STATE_KEY) { - set_token(lexer, TK_STR_KEY, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + set_token(token, lexer->line, column, TK_STR_KEY, value); } else if (string_type == LEXER_STATE_VALUE) { - set_token(lexer, TK_STR_VAL, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + set_token(token, lexer->line, column, TK_STR_VAL, value); } return LEXER_STATE_STRING_END; @@ -718,22 +781,34 @@ lexer_state_t handle_number(lexer_t *lexer, char input) { } else if (input == '}' || input == ']') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return handle_collection_end(lexer, input); } else if (input == ',') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return lexer->stack.stack[lexer->stack.size - 1]; } else if (isspace(input)) { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return LEXER_STATE_NUMBER_END; } @@ -749,8 +824,12 @@ lexer_state_t handle_fraction(lexer_t *lexer, char input) { } else if (input == '}' || input == ']') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return handle_collection_end(lexer, input); } else if (input == 'e' || input == 'E') { @@ -760,15 +839,23 @@ lexer_state_t handle_fraction(lexer_t *lexer, char input) { } else if (input == ',') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return lexer->stack.stack[lexer->stack.size - 1]; } else if (isspace(input)) { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return LEXER_STATE_NUMBER_END; } @@ -806,22 +893,34 @@ lexer_state_t handle_power(lexer_t *lexer, char input) { } else if (input == '}' || input == ']') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return handle_collection_end(lexer, input); } else if (input == ',') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return lexer->stack.stack[lexer->stack.size - 1]; } else if (isspace(input)) { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return LEXER_STATE_NUMBER_END; } @@ -835,8 +934,12 @@ lexer_state_t handle_number_end(lexer_t *lexer, char input) { } else if (input == ',') { // TODO (Abdelrahman): Set the token type correctly based on whether the // number is an integer or a double - set_token(lexer, TK_DOUBLE, - (token_value_t){.string = dstr_to_cstr(lexer->current_string)}); + lexer->token_ready = true; + u64 column = lexer->column - dstr_length(lexer->current_string); + + token_t token = dstr_to_numerical_token(lexer->current_string); + + set_token(&(lexer->token), lexer->line, column, token.type, token.value); return lexer->stack.stack[lexer->stack.size - 1]; } @@ -924,15 +1027,21 @@ lexer_state_t handle_null(lexer_t *lexer, char input) { lexer_state_t handle_keyword_end(lexer_t *lexer, char input) { const char *keyword = lexer->keyword.keyword.str; - if (strequal(keyword, "null")) { - set_token(lexer, TK_NULL, (token_value_t){0}); - } else if (strequal(keyword, "true")) { - set_token(lexer, TK_TRUE, (token_value_t){0}); - } else if (strequal(keyword, "false")) { - set_token(lexer, TK_FALSE, (token_value_t){0}); - } + if (lexer->keyword.size > 0) { + lexer->token_ready = true; + token_t *token = &(lexer->token); + u64 column = lexer->column - lexer->keyword.size; - clear_lex_str(&(lexer->keyword)); + if (strequal(keyword, "null")) { + set_token(token, lexer->line, column, TK_NULL, (token_value_t){0}); + } else if (strequal(keyword, "true")) { + set_token(token, lexer->line, column, TK_TRUE, (token_value_t){0}); + } else if (strequal(keyword, "false")) { + set_token(token, lexer->line, column, TK_FALSE, (token_value_t){0}); + } + + clear_lex_str(&(lexer->keyword)); + } if (isspace(input)) { return LEXER_STATE_KEYWORD_END;