From 8dabbc426f229d76f855592eaee5d7e04559a990 Mon Sep 17 00:00:00 2001 From: gazhonsepaskwa Date: Wed, 22 Jan 2025 16:05:02 +0100 Subject: [PATCH] tokenizer update --- tests/parse.c | 10 ++++++++-- tests/tokenizer/linked_list.c | 24 +++++++++++++----------- tests/tokenizer/token_and_pres.c | 6 ++++-- tests/tokenizer/tokenizer.c | 5 ++++- tests/tokenizer/tokenizer.h | 5 +++-- tests/tokenizer/tokenizer_utils.c | 14 ++++++++++++-- tests/tokenizer/tokenizer_utils2.c | 0 7 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 tests/tokenizer/tokenizer_utils2.c diff --git a/tests/parse.c b/tests/parse.c index 3986e0d..7249baf 100644 --- a/tests/parse.c +++ b/tests/parse.c @@ -15,14 +15,20 @@ void truncate_comment(char *str) { - int i; + int i; + char inquote; if (!str) return ; i = 0; + inquote = 0; while (str[i]) { - if (str[i] == '#') + if (!inquote && (str[i] == 39 || str[i] == '"')) + inquote = str[i]; + else if (inquote && (str[i] == inquote)) + inquote = 0; + if (str[i] == '#' && !inquote && str[i - 1] && str[i - 1] != '\\') { str[i] = 0; return ; diff --git a/tests/tokenizer/linked_list.c b/tests/tokenizer/linked_list.c index e775c44..6596f89 100644 --- a/tests/tokenizer/linked_list.c +++ b/tests/tokenizer/linked_list.c @@ -106,27 +106,29 @@ void debug_linked_list(t_node *head, char *msg) // set vals for pressision token if (current->pressision == COMMAND) - pres = ft_strdup("COMMAND"); + pres = ft_strdup("COMMAND "); else if (current->pressision == UNDEFINED) - pres = ft_strdup("UNDEF "); + pres = ft_strdup("UNDEF "); else if (current->pressision == AND) - pres = ft_strdup("AND "); + pres = ft_strdup("AND "); else if (current->pressision == OR) - pres = ft_strdup("OR "); + pres = ft_strdup("OR "); else if (current->pressision == PIPE) - pres = ft_strdup("PIPE "); + pres = ft_strdup("PIPE "); else if (current->pressision == SUBSH_S) - pres = ft_strdup("SUBSH_S"); + pres = ft_strdup("SUBSH_S "); else if (current->pressision == SUBSH_E) - pres = ft_strdup("SUBSH_E"); + pres = ft_strdup("SUBSH_E "); else if (current->pressision == RED_L) - pres = ft_strdup("RED_L "); + pres = ft_strdup("RED_L "); else if (current->pressision == RED_R) - pres = ft_strdup("RED_R "); + pres = ft_strdup("RED_R "); else if (current->pressision == HEREDOC) - pres = ft_strdup("HEREDOC"); + pres = ft_strdup("HEREDOC "); else if (current->pressision == D_RED_R) - pres = ft_strdup("D_RED_R"); + pres = ft_strdup("D_RED_R "); + else if (current->pressision == PARAMETER) + pres = ft_strdup("PARAMETER"); else pres = ft_strdup("??? "); diff --git a/tests/tokenizer/token_and_pres.c b/tests/tokenizer/token_and_pres.c index 3b2a492..a09b5aa 100644 --- a/tests/tokenizer/token_and_pres.c +++ b/tests/tokenizer/token_and_pres.c @@ -48,9 +48,11 @@ t_token get_token(char *str) return (token); } -t_pres get_pressision(char *s, t_token token) +t_pres get_pressision(char *s, t_token token, t_token last_token) { if (token == OPERATOR) return (get_operator(s)); - return (COMMAND); + else if (last_token == OPERATOR || last_token == UNSET) + return (COMMAND); + return (PARAMETER); } diff --git a/tests/tokenizer/tokenizer.c b/tests/tokenizer/tokenizer.c index 969a83b..3bc34a8 100644 --- a/tests/tokenizer/tokenizer.c +++ b/tests/tokenizer/tokenizer.c @@ -38,12 +38,15 @@ static t_node *tokenize_base(char *str) static void set_token(t_node *head) { t_node *it; + t_token last_token; it = head; + last_token = UNSET; while (it != NULL) { it->token = get_token(it->val); - it->pressision = get_pressision(it->val, it->token); + it->pressision = get_pressision(it->val, it->token, last_token); + last_token = it->token; it = it->next; } } diff --git a/tests/tokenizer/tokenizer.h b/tests/tokenizer/tokenizer.h index 3cdeecf..e4e708c 100644 --- a/tests/tokenizer/tokenizer.h +++ b/tests/tokenizer/tokenizer.h @@ -34,7 +34,8 @@ typedef enum e_pres RED_L, RED_R, HEREDOC, - D_RED_R + D_RED_R, + PARAMETER } t_pres; typedef struct s_node @@ -51,7 +52,7 @@ int add_node_back(t_node *head, char *val, t_token token); int merge_with_next_node(t_node *node); void free_linked_list(t_node *stack); t_token get_token(char *str); -t_pres get_pressision(char *str, t_token token); +t_pres get_pressision(char *s, t_token token, t_token last_token); int create_node_after(t_node *elem, char *val); char *copy_meta_xor(char *val, int *copied, int rev); int is_meta(char c); diff --git a/tests/tokenizer/tokenizer_utils.c b/tests/tokenizer/tokenizer_utils.c index 62f9f62..c099bc9 100644 --- a/tests/tokenizer/tokenizer_utils.c +++ b/tests/tokenizer/tokenizer_utils.c @@ -1,3 +1,15 @@ +/* ************************************************************************** */ +/* */ +/* ::: :::::::: */ +/* tokenizer_utils.c :+: :+: :+: */ +/* +:+ +:+ +:+ */ +/* By: nalebrun +#+ +:+ +#+ */ +/* +#+#+#+#+#+ +#+ */ +/* Created: 2025/01/22 14:24:05 by nalebrun #+# #+# */ +/* Updated: 2025/01/22 14:24:05 by nalebrun ### ########.fr */ +/* */ +/* ************************************************************************** */ + #include "tokenizer.h" int is_meta(char c) @@ -43,7 +55,6 @@ int ft_str_count(char *s, char c) return (count); } - int trim_nodes(t_node *head) { t_node *it; @@ -88,4 +99,3 @@ int find_quote_node(t_node *head, char q) } return (0); } - diff --git a/tests/tokenizer/tokenizer_utils2.c b/tests/tokenizer/tokenizer_utils2.c new file mode 100644 index 0000000..e69de29