1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/lexer/token.h"
25 #include "data/identifier.h"
26 #include "libpspp/assertion.h"
27 #include "libpspp/cast.h"
28 #include "libpspp/misc.h"
30 #include "gl/ftoastr.h"
31 #include "gl/xalloc.h"
34 token_copy (struct token *dst, const struct token *src)
36 *dst = (struct token) {
38 .number = src->number,
40 ss_alloc_substring (&dst->string, src->string);
43 /* Frees the string that TOKEN contains. */
45 token_uninit (struct token *token)
49 ss_dealloc (&token->string);
50 *token = (struct token) { .type = T_STOP };
55 token_equal (const struct token *a, const struct token *b)
57 if (a->type != b->type)
64 return a->number == b->number;
70 return ss_equals (a->string, b->string);
78 number_token_to_string (const struct token *token)
80 char buffer[DBL_BUFSIZE_BOUND];
82 c_dtoastr (buffer, sizeof buffer, 0, 0, fabs (token->number));
83 return (token->type == T_POS_NUM
85 : xasprintf ("-%s", buffer));
89 quoted_string_representation (struct substring ss, size_t n_quotes)
95 p = rep = xmalloc (1 + ss.length + n_quotes + 1 + 1);
97 for (i = 0; i < ss.length; i++)
99 uint8_t c = ss.string[i];
111 hex_string_representation (struct substring ss)
117 p = rep = xmalloc (2 + 2 * ss.length + 1 + 1);
120 for (i = 0; i < ss.length; i++)
122 static const char hex_digits[] = "0123456789abcdef";
123 uint8_t c = ss.string[i];
124 *p++ = hex_digits[c >> 4];
125 *p++ = hex_digits[c & 15];
134 string_representation (struct substring ss)
141 for (ofs = 0; ofs < ss.length; ofs += mblen)
145 mblen = u8_mbtoucr (&uc,
146 CHAR_CAST (const uint8_t *, ss.string + ofs),
148 if (mblen < 0 || !uc_is_print (uc))
149 return hex_string_representation (ss);
153 return quoted_string_representation (ss, n_quotes);
156 /* Returns a UTF-8 string that would yield TOKEN if it appeared in a syntax
157 file. The caller should free the returned string, with free(), when it is
160 The T_STOP token has no representation, so this function returns NULL. */
162 token_to_string (const struct token *token)
168 return number_token_to_string (token);
173 return ss_xstrdup (token->string);
176 return string_representation (token->string);
179 return xstrdup_if_nonnull (token_type_to_string (token->type));
183 /* Prints TOKEN on STREAM, for debugging. */
185 token_print (const struct token *token, FILE *stream)
187 fputs (token_type_to_name (token->type), stream);
188 if (token->type == T_POS_NUM || token->type == T_NEG_NUM
189 || token->number != 0.0)
191 char s[DBL_BUFSIZE_BOUND];
193 c_dtoastr (s, sizeof s, 0, 0, token->number);
194 fprintf (stream, "\t%s", s);
196 if (token->type == T_ID || token->type == T_STRING || token->string.length)
197 fprintf (stream, "\t\"%.*s\"",
198 (int) token->string.length, token->string.string);
203 tokens_copy (struct tokens *dst, const struct tokens *src)
205 *dst = (struct tokens) {
206 .tokens = xnmalloc (src->n, sizeof *dst->tokens),
211 for (size_t i = 0; i < src->n; i++)
212 token_copy (&dst->tokens[i], &src->tokens[i]);
216 tokens_uninit (struct tokens *tokens)
218 for (size_t i = 0; i < tokens->n; i++)
219 token_destroy (&tokens->tokens[i]);
220 free (tokens->tokens);
224 tokens_add (struct tokens *tokens, const struct token *t)
226 if (tokens->allocated >= tokens->n)
227 tokens->tokens = x2nrealloc (tokens->tokens, &tokens->allocated,
228 sizeof *tokens->tokens);
230 token_copy (&tokens->tokens[tokens->n++], t);
234 tokens_print (const struct tokens *tokens, FILE *stream)
236 for (size_t i = 0; i < tokens->n; i++)
237 token_print (&tokens->tokens[i], stream);