1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2005, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
18 This file is concerned with the definition of the PSPP syntax, NOT the
19 action of scanning/parsing code .
24 #include "data/identifier.h"
29 #include "libpspp/assertion.h"
31 #include "gl/c-ctype.h"
34 #define _(msgid) gettext (msgid)
38 /* Returns TYPE as a string, e.g. "ID" for T_ID. */
40 token_type_to_name (enum token_type type)
44 #define TOKEN_TYPE(TYPE) case T_##TYPE: return #TYPE;
49 return "unknown token type";
53 /* Returns an ASCII string that yields TOKEN if it appeared in a syntax file,
54 as a statically allocated constant string. This function returns NULL for
55 tokens that don't have any fixed string representation, such as identifier
58 token_type_to_string (enum token_type token)
151 /* Recognizing identifiers. */
154 is_ascii_id1 (unsigned char c)
156 return c_isalpha (c) || c == '@' || c == '#' || c == '$';
160 is_ascii_idn (unsigned char c)
162 return is_ascii_id1 (c) || isdigit (c) || c == '.' || c == '_';
165 /* Returns true if C may be the first byte in an identifier in the current
168 (PSPP is transitioning to using Unicode internally for syntax, so please
169 use lex_uc_is_id1() instead, if possible.) */
173 return is_ascii_id1 (c) || (unsigned char) c >= 128;
176 /* Returns true if C may be a byte in an identifier other than the first.
178 (PSPP is transitioning to using Unicode internally for syntax, so please
179 use lex_uc_is_idn() instead, if possible.) */
183 return is_ascii_idn (c) || (unsigned char) c >= 128;
186 /* Returns true if Unicode code point UC may be the first character in an
187 identifier in the current locale. */
189 lex_uc_is_id1 (ucs4_t uc)
193 : (uc_is_general_category_withtable (uc,
197 && uc != 0xfffc && uc != 0xfffd));
200 /* Returns true if Unicode code point UC may be a character in an identifier
201 other than the first. */
203 lex_uc_is_idn (ucs4_t uc)
206 ? is_ascii_id1 (uc) || isdigit (uc) || uc == '.' || uc == '_'
207 : (uc_is_general_category_withtable (uc,
212 && uc != 0xfffc && uc != 0xfffd));
215 /* Returns true if Unicode code point UC is a space that separates tokens. */
217 lex_uc_is_space (ucs4_t uc)
219 /* These are all of the Unicode characters in category Zs, Zl, or Zp. */
220 return (uc == ' ' || (uc <= 0x000d && uc >= 0x0009)
222 && (uc == 0xa0 || uc == 0x85 || uc == 0x1680 || uc == 0x180e
223 || (uc >= 0x2000 && uc <= 0x200a)
224 || uc == 0x2028 || uc == 0x2029 || uc == 0x202f
225 || uc == 0x205f || uc == 0x3000)));
229 /* Returns the length of the longest prefix of STRING that forms
230 a valid identifier. Returns zero if STRING does not begin
231 with a valid identifier. */
233 lex_id_get_length (struct substring string)
236 if (!ss_is_empty (string) && lex_is_id1 (ss_first (string)))
239 while (length < ss_length (string)
240 && lex_is_idn (ss_at (string, length)))
246 /* Comparing identifiers. */
248 /* Returns true if TOKEN is a case-insensitive match for KEYWORD.
250 Keywords match if one of the following is true: KEYWORD and
251 TOKEN are identical, or TOKEN is at least 3 characters long
252 and those characters are identical to KEYWORD. (Letters that
253 differ only in case are considered identical.)
255 KEYWORD must be ASCII, but TOKEN may be ASCII or UTF-8. */
257 lex_id_match (struct substring keyword, struct substring token)
259 return lex_id_match_n (keyword, token, 3);
262 /* Returns true if TOKEN is a case-insensitive match for at least
263 the first N characters of KEYWORD.
265 KEYWORD must be ASCII, but TOKEN may be ASCII or UTF-8. */
267 lex_id_match_n (struct substring keyword, struct substring token, size_t n)
269 size_t token_len = ss_length (token);
270 size_t keyword_len = ss_length (keyword);
272 if (token_len >= n && token_len < keyword_len)
273 return ss_equals_case (ss_head (keyword, token_len), token);
275 return ss_equals_case (keyword, token);
278 /* Table of keywords. */
282 const struct substring identifier;
285 static const struct keyword keywords[] =
287 { T_AND, SS_LITERAL_INITIALIZER ("AND") },
288 { T_OR, SS_LITERAL_INITIALIZER ("OR") },
289 { T_NOT, SS_LITERAL_INITIALIZER ("NOT") },
290 { T_EQ, SS_LITERAL_INITIALIZER ("EQ") },
291 { T_GE, SS_LITERAL_INITIALIZER ("GE") },
292 { T_GT, SS_LITERAL_INITIALIZER ("GT") },
293 { T_LE, SS_LITERAL_INITIALIZER ("LE") },
294 { T_LT, SS_LITERAL_INITIALIZER ("LT") },
295 { T_NE, SS_LITERAL_INITIALIZER ("NE") },
296 { T_ALL, SS_LITERAL_INITIALIZER ("ALL") },
297 { T_BY, SS_LITERAL_INITIALIZER ("BY") },
298 { T_TO, SS_LITERAL_INITIALIZER ("TO") },
299 { T_WITH, SS_LITERAL_INITIALIZER ("WITH") },
301 static const size_t keyword_cnt = sizeof keywords / sizeof *keywords;
303 /* Returns true if TOKEN is representable as a keyword. */
305 lex_is_keyword (enum token_type token)
307 const struct keyword *kw;
308 for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
309 if (kw->token == token)
314 /* Returns the proper token type, either T_ID or a reserved
315 keyword enum, for ID. */
317 lex_id_to_token (struct substring id)
319 if (ss_length (id) >= 2 && ss_length (id) <= 4)
321 const struct keyword *kw;
322 for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
323 if (ss_equals_case (kw->identifier, id))