44 * Tokenizer
55 */
66
7- #include < cstring >
7+ #include " vtr_token.h "
88
99#include " vtr_assert.h"
1010#include " vtr_util.h"
1111#include " vtr_memory.h"
12- #include " vtr_token.h"
1312
14- enum e_token_type GetTokenTypeFromChar (const enum e_token_type cur_token_type,
15- const char cur);
16-
17- bool IsWhitespace (char c);
18-
19- // /@brief Returns true if character is whatspace between tokens
20- bool IsWhitespace (char c) {
21- switch (c) {
22- case ' '
23- case ' \t '
24- case ' \r '
25- case ' \n '
26- return true ;
27- default :
28- return false ;
29- }
30- }
13+ #include < cctype>
3114
32- // /@brief Returns a token list of the text for a given string.
33- t_token* GetTokensFromString (const char * inString, int * num_tokens) {
34- const char * cur;
35- t_token* tokens;
36- int i, in_string_index, prev_in_string_index;
37- bool has_null;
38- enum e_token_type cur_token_type, new_token_type;
15+ // / @brief Returns a token type of the given char
16+ static e_token_type get_token_type_from_char (e_token_type cur_token_type, char cur);
3917
40- *num_tokens = i = 0 ;
41- cur_token_type = TOKEN_NULL;
18+ const t_token Tokens::null_token_{e_token_type::NULL_TOKEN, " "
4219
43- if (inString == nullptr ) {
44- return nullptr ;
45- };
46-
47- cur = inString;
48-
49- /* Count number of tokens */
50- while (*cur) {
51- new_token_type = GetTokenTypeFromChar (cur_token_type, *cur);
52- if (new_token_type != cur_token_type) {
53- cur_token_type = new_token_type;
54- if (new_token_type != TOKEN_NULL) {
55- i++;
56- }
57- }
58- ++cur;
20+ Tokens::Tokens (std::string_view inString) {
21+ if (inString.empty ()) {
22+ return ;
5923 }
60- *num_tokens = i;
6124
62- if (*num_tokens > 0 ) {
63- tokens = (t_token*)vtr::calloc (*num_tokens + 1 , sizeof (t_token));
64- } else {
65- return nullptr ;
66- }
25+ e_token_type cur_token_type = e_token_type::NULL_TOKEN;
26+ size_t in_string_index = 0 ;
27+ size_t prev_in_string_index = 0 ;
6728
68- /* populate tokens */
69- i = 0 ;
70- in_string_index = 0 ;
71- has_null = true ;
72- prev_in_string_index = 0 ;
73- cur_token_type = TOKEN_NULL;
74-
75- cur = inString;
76-
77- while (*cur) {
78- new_token_type = GetTokenTypeFromChar (cur_token_type, *cur);
29+ for (char cur : inString) {
30+ e_token_type new_token_type = get_token_type_from_char (cur_token_type, cur);
7931 if (new_token_type != cur_token_type) {
80- if (!has_null) {
81- tokens[i - 1 ].data [in_string_index - prev_in_string_index] = ' \0 ' /* NULL the end of the data string */
82- has_null = true ;
32+ if (cur_token_type != e_token_type::NULL_TOKEN) {
33+ // Finalize the current token
34+ t_token& current_token = tokens_.back ();
35+ current_token.data = inString.substr (prev_in_string_index,
36+ in_string_index - prev_in_string_index);
8337 }
84- if (new_token_type != TOKEN_NULL) {
85- tokens[i].type = new_token_type;
86- tokens[i].data = vtr::strdup (inString + in_string_index);
38+ if (new_token_type != e_token_type::NULL_TOKEN) {
39+ // Start a new token
40+ t_token new_token;
41+ new_token.type = new_token_type;
42+ tokens_.push_back (new_token);
8743 prev_in_string_index = in_string_index;
88- has_null = false ;
89- i++;
9044 }
9145 cur_token_type = new_token_type;
9246 }
93- ++cur;
9447 in_string_index++;
9548 }
9649
97- VTR_ASSERT (i == *num_tokens);
98-
99- tokens[*num_tokens].type = TOKEN_NULL;
100- tokens[*num_tokens].data = nullptr ;
101-
102- /* Return the list */
103- return tokens;
50+ // Finalize the last token if it exists
51+ if (cur_token_type != e_token_type::NULL_TOKEN && !tokens_.empty ()) {
52+ t_token& current_token = tokens_.back ();
53+ current_token.data = inString.substr (prev_in_string_index,
54+ in_string_index - prev_in_string_index);
55+ }
10456}
10557
106- // /@brief Free (tokens)
107- void freeTokens (t_token* tokens, const int num_tokens ) {
108- int i ;
109- for (i = 0 ; i < num_tokens; i++) {
110- free (tokens[i]. data ) ;
58+ const t_token& Tokens:: operator []( size_t idx) const {
59+ if (idx < tokens_. size () ) {
60+ return tokens_[idx] ;
61+ } else {
62+ return null_token_ ;
11163 }
112- free (tokens);
11364}
11465
115- // /@brief Returns a token type of the given char
116- enum e_token_type GetTokenTypeFromChar (const enum e_token_type cur_token_type,
117- const char cur) {
118- if (IsWhitespace (cur)) {
119- return TOKEN_NULL;
66+ static e_token_type get_token_type_from_char (e_token_type cur_token_type, char cur) {
67+ if (std::isspace (cur)) {
68+ return e_token_type::NULL_TOKEN;
12069 } else {
12170 if (cur == ' ['
122- return TOKEN_OPEN_SQUARE_BRACKET ;
71+ return e_token_type::OPEN_SQUARE_BRACKET ;
12372 } else if (cur == ' ]'
124- return TOKEN_CLOSE_SQUARE_BRACKET ;
73+ return e_token_type::CLOSE_SQUARE_BRACKET ;
12574 } else if (cur == ' {'
126- return TOKEN_OPEN_SQUIG_BRACKET ;
75+ return e_token_type::OPEN_SQUIG_BRACKET ;
12776 } else if (cur == ' }'
128- return TOKEN_CLOSE_SQUIG_BRACKET ;
77+ return e_token_type::CLOSE_SQUIG_BRACKET ;
12978 } else if (cur == ' :'
130- return TOKEN_COLON ;
79+ return e_token_type::COLON ;
13180 } else if (cur == ' .'
132- return TOKEN_DOT ;
133- } else if (cur >= ' 0' ' 9' TOKEN_STRING ) {
134- return TOKEN_INT ;
81+ return e_token_type::DOT ;
82+ } else if (cur >= ' 0' ' 9' e_token_type::STRING ) {
83+ return e_token_type::INT ;
13584 } else {
136- return TOKEN_STRING ;
85+ return e_token_type::STRING ;
13786 }
13887 }
13988}
14089
141- // /@brief Returns true if the token's type equals to token_type
142- bool checkTokenType (const t_token token, enum e_token_type token_type) {
143- if (token.type != token_type) {
144- return false ;
145- }
146- return true ;
147- }
148-
149- // /@brief Returns a 2D array representing the atof result of all the input string entries seperated by whitespace
15090void my_atof_2D (float ** matrix, const int max_i, const int max_j, const char * instring) {
15191 int i, j;
15292 char *cur, *cur2, *copy, *final ;
@@ -160,7 +100,7 @@ void my_atof_2D(float** matrix, const int max_i, const int max_j, const char* in
160100 cur = copy;
161101 i = j = 0 ;
162102 while (cur != final ) {
163- while (IsWhitespace (*cur) && cur != final ) {
103+ while (std::isspace (*cur) && cur != final ) {
164104 if (j == max_j) {
165105 i++;
166106 j = 0 ;
@@ -171,7 +111,7 @@ void my_atof_2D(float** matrix, const int max_i, const int max_j, const char* in
171111 break ;
172112 }
173113 cur2 = cur;
174- while (!IsWhitespace (*cur2) && cur2 != final ) {
114+ while (!std::isspace (*cur2) && cur2 != final ) {
175115 cur2++;
176116 }
177117 *cur2 = ' \0 '
@@ -187,13 +127,6 @@ void my_atof_2D(float** matrix, const int max_i, const int max_j, const char* in
187127 free (copy);
188128}
189129
190- /* Date:July 2nd, 2013 *
191- * Author: Daniel Chen */
192- /* *
193- * @brief Checks if the number of entries (separated by whitespace) matches the the expected number (max_i * max_j)
194- *
195- * can be used before calling my_atof_2D
196- */
197130bool check_my_atof_2D (const int max_i, const int max_j, const char * instring, int * num_entries) {
198131 /* Check if max_i * max_j matches number of entries in instring */
199132 const char * cur = instring;
@@ -202,10 +135,10 @@ bool check_my_atof_2D(const int max_i, const int max_j, const char* instring, in
202135
203136 /* First count number of entries in instring */
204137 while (*cur != ' \0 '
205- if (!IsWhitespace (*cur) && !in_str) {
138+ if (!std::isspace (*cur) && !in_str) {
206139 in_str = true ;
207140 entry_count++;
208- } else if (IsWhitespace (*cur)) {
141+ } else if (std::isspace (*cur)) {
209142 in_str = false ;
210143 }
211144 cur++;
0 commit comments