1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2005, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
18 This file is concerned with the definition of the PSPP syntax, NOT the
19 action of scanning/parsing code .
24 #include "data/identifier.h"
30 #include "libpspp/assertion.h"
31 #include "libpspp/cast.h"
33 #include "gl/c-ctype.h"
36 #define _(msgid) gettext (msgid)
40 /* Returns TYPE as a string, e.g. "ID" for T_ID. */
42 token_type_to_name (enum token_type type)
46 #define TOKEN_TYPE(TYPE) case T_##TYPE: return #TYPE;
51 return "unknown token type";
55 /* Returns an ASCII string that yields TOKEN if it appeared in a syntax file,
56 as a statically allocated constant string. This function returns NULL for
57 tokens that don't have any fixed string representation, such as identifier
60 token_type_to_string (enum token_type token)
155 /* Recognizing identifiers. */
158 is_ascii_id1 (unsigned char c)
160 return c_isalpha (c) || c == '@' || c == '#' || c == '$';
164 is_ascii_idn (unsigned char c)
166 return is_ascii_id1 (c) || isdigit (c) || c == '.' || c == '_';
169 /* Returns true if C may be the first byte in an identifier in the current
172 (PSPP is transitioning to using Unicode internally for syntax, so please
173 use lex_uc_is_id1() instead, if possible.) */
177 return is_ascii_id1 (c) || (unsigned char) c >= 128;
180 /* Returns true if C may be a byte in an identifier other than the first.
182 (PSPP is transitioning to using Unicode internally for syntax, so please
183 use lex_uc_is_idn() instead, if possible.) */
187 return is_ascii_idn (c) || (unsigned char) c >= 128;
190 /* Returns true if Unicode code point UC may be the first character in an
191 identifier in the current locale. */
193 lex_uc_is_id1 (ucs4_t uc)
197 : (uc_is_general_category_withtable (uc,
201 && uc != 0xfffc && uc != 0xfffd));
204 /* Returns true if Unicode code point UC may be a character in an identifier
205 other than the first. */
207 lex_uc_is_idn (ucs4_t uc)
210 ? is_ascii_id1 (uc) || isdigit (uc) || uc == '.' || uc == '_'
211 : (uc_is_general_category_withtable (uc,
216 && uc != 0xfffc && uc != 0xfffd));
219 /* Returns true if Unicode code point UC is a space that separates tokens. */
221 lex_uc_is_space (ucs4_t uc)
223 /* These are all of the Unicode characters in category Zs, Zl, or Zp. */
224 return (uc == ' ' || (uc <= 0x000d && uc >= 0x0009)
226 && (uc == 0xa0 || uc == 0x85 || uc == 0x1680 || uc == 0x180e
227 || (uc >= 0x2000 && uc <= 0x200a)
228 || uc == 0x2028 || uc == 0x2029 || uc == 0x202f
229 || uc == 0x205f || uc == 0x3000)));
233 /* Returns the length of the longest prefix of STRING that forms
234 a valid identifier. Returns zero if STRING does not begin
235 with a valid identifier. */
237 lex_id_get_length (struct substring string)
239 const uint8_t *s = CHAR_CAST (const uint8_t *, string.string);
240 size_t len = string.length;
244 for (ofs = 0; ofs < string.length; ofs += mblen)
248 mblen = u8_mbtouc (&uc, s + ofs, len - ofs);
249 if (!(ofs == 0 ? lex_uc_is_id1 (uc) : lex_uc_is_idn (uc)))
256 /* Comparing identifiers. */
258 /* Returns true if TOKEN is a case-insensitive match for KEYWORD.
260 Keywords match if one of the following is true: KEYWORD and
261 TOKEN are identical, or TOKEN is at least 3 characters long
262 and those characters are identical to KEYWORD. (Letters that
263 differ only in case are considered identical.)
265 KEYWORD must be ASCII, but TOKEN may be ASCII or UTF-8. */
267 lex_id_match (struct substring keyword, struct substring token)
269 return lex_id_match_n (keyword, token, 3);
272 /* Returns true if TOKEN is a case-insensitive match for at least
273 the first N characters of KEYWORD.
275 KEYWORD must be ASCII, but TOKEN may be ASCII or UTF-8. */
277 lex_id_match_n (struct substring keyword, struct substring token, size_t n)
279 size_t token_len = ss_length (token);
280 size_t keyword_len = ss_length (keyword);
282 if (token_len >= n && token_len < keyword_len)
283 return ss_equals_case (ss_head (keyword, token_len), token);
285 return ss_equals_case (keyword, token);
288 /* Table of keywords. */
292 const struct substring identifier;
295 static const struct keyword keywords[] =
297 { T_AND, SS_LITERAL_INITIALIZER ("AND") },
298 { T_OR, SS_LITERAL_INITIALIZER ("OR") },
299 { T_NOT, SS_LITERAL_INITIALIZER ("NOT") },
300 { T_EQ, SS_LITERAL_INITIALIZER ("EQ") },
301 { T_GE, SS_LITERAL_INITIALIZER ("GE") },
302 { T_GT, SS_LITERAL_INITIALIZER ("GT") },
303 { T_LE, SS_LITERAL_INITIALIZER ("LE") },
304 { T_LT, SS_LITERAL_INITIALIZER ("LT") },
305 { T_NE, SS_LITERAL_INITIALIZER ("NE") },
306 { T_ALL, SS_LITERAL_INITIALIZER ("ALL") },
307 { T_BY, SS_LITERAL_INITIALIZER ("BY") },
308 { T_TO, SS_LITERAL_INITIALIZER ("TO") },
309 { T_WITH, SS_LITERAL_INITIALIZER ("WITH") },
311 static const size_t keyword_cnt = sizeof keywords / sizeof *keywords;
313 /* Returns true if TOKEN is representable as a keyword. */
315 lex_is_keyword (enum token_type token)
317 const struct keyword *kw;
318 for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
319 if (kw->token == token)
324 /* Returns the proper token type, either T_ID or a reserved
325 keyword enum, for ID. */
327 lex_id_to_token (struct substring id)
329 if (ss_length (id) >= 2 && ss_length (id) <= 4)
331 const struct keyword *kw;
332 for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
333 if (ss_equals_case (kw->identifier, id))