1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/lexer/lexer.h"
30 #include "data/settings.h"
31 #include "language/command.h"
32 #include "libpspp/assertion.h"
33 #include "libpspp/getl.h"
34 #include "libpspp/message.h"
35 #include "libpspp/str.h"
36 #include "output/journal.h"
37 #include "output/text-item.h"
39 #include "gl/xalloc.h"
42 #define _(msgid) gettext (msgid)
43 #define N_(msgid) msgid
47 struct string line_buffer;
49 struct source_stream *ss;
51 int token; /* Current token. */
52 double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
54 struct string tokstr; /* T_ID, T_STRING: token string value. */
56 char *prog; /* Pointer to next token in line_buffer. */
57 bool dot; /* True only if this line ends with a terminal dot. */
59 int put_token ; /* If nonzero, next token returned by lex_get().
60 Used only in exceptional circumstances. */
62 struct string put_tokstr;
67 static int parse_id (struct lexer *);
69 /* How a string represents its contents. */
72 CHARACTER_STRING, /* Characters. */
73 BINARY_STRING, /* Binary digits. */
74 OCTAL_STRING, /* Octal digits. */
75 HEX_STRING /* Hexadecimal digits. */
78 static int parse_string (struct lexer *, enum string_type);
82 /* Initializes the lexer. */
84 lex_create (struct source_stream *ss)
86 struct lexer *lexer = xzalloc (sizeof (*lexer));
88 ds_init_empty (&lexer->tokstr);
89 ds_init_empty (&lexer->put_tokstr);
90 ds_init_empty (&lexer->line_buffer);
96 struct source_stream *
97 lex_get_source_stream (const struct lexer *lex)
103 lex_current_syntax_mode (const struct lexer *lex)
105 return source_stream_current_syntax_mode (lex->ss);
109 lex_current_error_mode (const struct lexer *lex)
111 return source_stream_current_error_mode (lex->ss);
116 lex_destroy (struct lexer *lexer)
120 ds_destroy (&lexer->put_tokstr);
121 ds_destroy (&lexer->tokstr);
122 ds_destroy (&lexer->line_buffer);
129 /* Common functions. */
131 /* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
132 tokval, respectively, and sets tokid appropriately. */
134 restore_token (struct lexer *lexer)
136 assert (lexer->put_token != 0);
137 lexer->token = lexer->put_token;
138 ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
139 lexer->tokval = lexer->put_tokval;
140 lexer->put_token = 0;
143 /* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
144 put_lexer->tokval respectively. */
146 save_token (struct lexer *lexer)
148 lexer->put_token = lexer->token;
149 ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
150 lexer->put_tokval = lexer->tokval;
153 /* Parses a single token, setting appropriate global variables to
154 indicate the token's attributes. */
156 lex_get (struct lexer *lexer)
161 if (NULL == lexer->prog && ! lex_get_line (lexer) )
163 lexer->token = T_STOP;
167 /* If a token was pushed ahead, return it. */
168 if (lexer->put_token)
170 restore_token (lexer);
176 /* Skip whitespace. */
177 while (c_isspace ((unsigned char) *lexer->prog))
186 lexer->token = T_ENDCMD;
189 else if (!lex_get_line (lexer))
192 lexer->token = T_STOP;
196 if (lexer->put_token)
198 restore_token (lexer);
204 /* Actually parse the token. */
205 ds_clear (&lexer->tokstr);
207 switch (*lexer->prog)
210 case '0': case '1': case '2': case '3': case '4':
211 case '5': case '6': case '7': case '8': case '9':
215 /* `-' can introduce a negative number, or it can be a token by
217 if (*lexer->prog == '-')
219 ds_put_byte (&lexer->tokstr, *lexer->prog++);
220 while (c_isspace ((unsigned char) *lexer->prog))
223 if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
225 lexer->token = T_DASH;
228 lexer->token = T_NEG_NUM;
231 lexer->token = T_POS_NUM;
233 /* Parse the number, copying it into tokstr. */
234 while (c_isdigit ((unsigned char) *lexer->prog))
235 ds_put_byte (&lexer->tokstr, *lexer->prog++);
236 if (*lexer->prog == '.')
238 ds_put_byte (&lexer->tokstr, *lexer->prog++);
239 while (c_isdigit ((unsigned char) *lexer->prog))
240 ds_put_byte (&lexer->tokstr, *lexer->prog++);
242 if (*lexer->prog == 'e' || *lexer->prog == 'E')
244 ds_put_byte (&lexer->tokstr, *lexer->prog++);
245 if (*lexer->prog == '+' || *lexer->prog == '-')
246 ds_put_byte (&lexer->tokstr, *lexer->prog++);
247 while (c_isdigit ((unsigned char) *lexer->prog))
248 ds_put_byte (&lexer->tokstr, *lexer->prog++);
251 /* Parse as floating point. */
252 lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail);
255 msg (SE, _("%s does not form a valid number."),
256 ds_cstr (&lexer->tokstr));
259 ds_clear (&lexer->tokstr);
260 ds_put_byte (&lexer->tokstr, '0');
267 lexer->token = parse_string (lexer, CHARACTER_STRING);
271 lexer->token = T_PLUS;
276 lexer->token = T_SLASH;
281 lexer->token = T_EQUALS;
286 lexer->token = T_LPAREN;
291 lexer->token = T_RPAREN;
296 lexer->token = T_LBRACK;
301 lexer->token = T_RBRACK;
306 lexer->token = T_COMMA;
311 if (*++lexer->prog == '*')
314 lexer->token = T_EXP;
317 lexer->token = T_ASTERISK;
321 if (*++lexer->prog == '=')
326 else if (*lexer->prog == '>')
336 if (*++lexer->prog == '=')
346 if (*++lexer->prog == '=')
352 lexer->token = T_NOT;
357 lexer->token = T_AND;
366 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
367 lexer->token = parse_string (lexer, BINARY_STRING);
369 lexer->token = parse_id (lexer);
373 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
374 lexer->token = parse_string (lexer, OCTAL_STRING);
376 lexer->token = parse_id (lexer);
380 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
381 lexer->token = parse_string (lexer, HEX_STRING);
383 lexer->token = parse_id (lexer);
387 if (lex_is_id1 (*lexer->prog))
389 lexer->token = parse_id (lexer);
394 unsigned char c = *lexer->prog++;
395 char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c);
396 msg (SE, _("Bad character in input: `%s'."), c_name);
405 /* Parses an identifier at the current position into tokstr.
406 Returns the correct token type. */
408 parse_id (struct lexer *lexer)
410 struct substring rest_of_line
411 = ss_substr (ds_ss (&lexer->line_buffer),
412 ds_pointer_to_position (&lexer->line_buffer, lexer->prog),
414 struct substring id = ss_head (rest_of_line,
415 lex_id_get_length (rest_of_line));
416 lexer->prog += ss_length (id);
418 ds_assign_substring (&lexer->tokstr, id);
419 return lex_id_to_token (id);
422 /* Reports an error to the effect that subcommand SBC may only be
425 lex_sbc_only_once (const char *sbc)
427 msg (SE, _("Subcommand %s may only be specified once."), sbc);
430 /* Reports an error to the effect that subcommand SBC is
433 lex_sbc_missing (struct lexer *lexer, const char *sbc)
435 lex_error (lexer, _("missing required subcommand %s"), sbc);
438 /* Prints a syntax error message containing the current token and
439 given message MESSAGE (if non-null). */
441 lex_error (struct lexer *lexer, const char *message, ...)
447 if (lexer->token == T_STOP)
448 ds_put_cstr (&s, _("Syntax error at end of file"));
449 else if (lexer->token == T_ENDCMD)
450 ds_put_cstr (&s, _("Syntax error at end of command"));
453 char *token_rep = lex_token_representation (lexer);
454 ds_put_format (&s, _("Syntax error at `%s'"), token_rep);
462 ds_put_cstr (&s, ": ");
464 va_start (args, message);
465 ds_put_vformat (&s, message, args);
469 msg (SE, "%s.", ds_cstr (&s));
473 /* Checks that we're at end of command.
474 If so, returns a successful command completion code.
475 If not, flags a syntax error and returns an error command
478 lex_end_of_command (struct lexer *lexer)
480 if (lexer->token != T_ENDCMD)
482 lex_error (lexer, _("expecting end of command"));
489 /* Token testing functions. */
491 /* Returns true if the current token is a number. */
493 lex_is_number (struct lexer *lexer)
495 return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
499 /* Returns true if the current token is a string. */
501 lex_is_string (struct lexer *lexer)
503 return lexer->token == T_STRING;
507 /* Returns the value of the current token, which must be a
508 floating point number. */
510 lex_number (struct lexer *lexer)
512 assert (lex_is_number (lexer));
513 return lexer->tokval;
516 /* Returns true iff the current token is an integer. */
518 lex_is_integer (struct lexer *lexer)
520 return (lex_is_number (lexer)
521 && lexer->tokval > LONG_MIN
522 && lexer->tokval <= LONG_MAX
523 && floor (lexer->tokval) == lexer->tokval);
526 /* Returns the value of the current token, which must be an
529 lex_integer (struct lexer *lexer)
531 assert (lex_is_integer (lexer));
532 return lexer->tokval;
535 /* Token matching functions. */
537 /* If TOK is the current token, skips it and returns true
538 Otherwise, returns false. */
540 lex_match (struct lexer *lexer, enum token_type t)
542 if (lexer->token == t)
551 /* If the current token is the identifier S, skips it and returns
552 true. The identifier may be abbreviated to its first three
554 Otherwise, returns false. */
556 lex_match_id (struct lexer *lexer, const char *s)
558 return lex_match_id_n (lexer, s, 3);
561 /* If the current token is the identifier S, skips it and returns
562 true. The identifier may be abbreviated to its first N
564 Otherwise, returns false. */
566 lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
568 if (lexer->token == T_ID
569 && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
578 /* If the current token is integer N, skips it and returns true.
579 Otherwise, returns false. */
581 lex_match_int (struct lexer *lexer, int x)
583 if (lex_is_integer (lexer) && lex_integer (lexer) == x)
592 /* Forced matches. */
594 /* If this token is identifier S, fetches the next token and returns
596 Otherwise, reports an error and returns zero. */
598 lex_force_match_id (struct lexer *lexer, const char *s)
600 if (lex_match_id (lexer, s))
604 lex_error (lexer, _("expecting `%s'"), s);
609 /* If the current token is T, skips the token. Otherwise, reports an
610 error and returns from the current function with return value false. */
612 lex_force_match (struct lexer *lexer, enum token_type t)
614 if (lexer->token == t)
621 lex_error (lexer, _("expecting `%s'"), lex_token_name (t));
626 /* If this token is a string, does nothing and returns true.
627 Otherwise, reports an error and returns false. */
629 lex_force_string (struct lexer *lexer)
631 if (lex_is_string (lexer))
635 lex_error (lexer, _("expecting string"));
640 /* If this token is an integer, does nothing and returns true.
641 Otherwise, reports an error and returns false. */
643 lex_force_int (struct lexer *lexer)
645 if (lex_is_integer (lexer))
649 lex_error (lexer, _("expecting integer"));
654 /* If this token is a number, does nothing and returns true.
655 Otherwise, reports an error and returns false. */
657 lex_force_num (struct lexer *lexer)
659 if (lex_is_number (lexer))
662 lex_error (lexer, _("expecting number"));
666 /* If this token is an identifier, does nothing and returns true.
667 Otherwise, reports an error and returns false. */
669 lex_force_id (struct lexer *lexer)
671 if (lexer->token == T_ID)
674 lex_error (lexer, _("expecting identifier"));
678 /* Weird token functions. */
680 /* Returns the likely type of the next token, or 0 if it's hard to tell. */
682 lex_look_ahead (struct lexer *lexer)
684 if (lexer->put_token)
685 return lexer->put_token;
689 if (NULL == lexer->prog && ! lex_get_line (lexer) )
694 while (c_isspace ((unsigned char) *lexer->prog))
701 else if (!lex_get_line (lexer))
704 if (lexer->put_token)
705 return lexer->put_token;
708 switch (toupper ((unsigned char) *lexer->prog))
710 case 'X': case 'B': case 'O':
711 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
719 case '0': case '1': case '2': case '3': case '4':
720 case '5': case '6': case '7': case '8': case '9':
751 return lexer->prog[1] == '*' ? T_EXP : T_ASTERISK;
754 return (lexer->prog[1] == '=' ? T_LE
755 : lexer->prog[1] == '>' ? T_NE
759 return lexer->prog[1] == '=' ? T_GE : T_GT;
762 return lexer->prog[1] == '=' ? T_NE : T_NOT;
771 if (lex_is_id1 (*lexer->prog))
778 /* Makes the current token become the next token to be read; the
779 current token is set to T. */
781 lex_put_back (struct lexer *lexer, enum token_type t)
787 /* Weird line processing functions. */
789 /* Returns the entire contents of the current line. */
791 lex_entire_line (const struct lexer *lexer)
793 return ds_cstr (&lexer->line_buffer);
796 const struct string *
797 lex_entire_line_ds (const struct lexer *lexer)
799 return &lexer->line_buffer;
802 /* As lex_entire_line(), but only returns the part of the current line
803 that hasn't already been tokenized. */
805 lex_rest_of_line (const struct lexer *lexer)
810 /* Returns true if the current line ends in a terminal dot,
813 lex_end_dot (const struct lexer *lexer)
818 /* Causes the rest of the current input line to be ignored for
819 tokenization purposes. */
821 lex_discard_line (struct lexer *lexer)
823 ds_cstr (&lexer->line_buffer); /* Ensures ds_end points to something valid */
824 lexer->prog = ds_end (&lexer->line_buffer);
826 lexer->put_token = 0;
830 /* Discards the rest of the current command.
831 When we're reading commands from a file, we skip tokens until
832 a terminal dot or EOF.
833 When we're reading commands interactively from the user,
834 that's just discarding the current line, because presumably
835 the user doesn't want to finish typing a command that will be
838 lex_discard_rest_of_command (struct lexer *lexer)
840 if (!getl_is_interactive (lexer->ss))
842 while (lexer->token != T_STOP && lexer->token != T_ENDCMD)
846 lex_discard_line (lexer);
849 /* Weird line reading functions. */
851 /* Remove C-style comments in STRING, begun by slash-star and
852 terminated by star-slash or newline. */
854 strip_comments (struct string *string)
862 for (cp = ds_cstr (string); *cp; )
864 /* If we're not in a comment, check for quote marks. */
869 else if (*cp == '\'' || *cp == '"')
873 /* If we're not inside a quotation, check for comment. */
876 if (cp[0] == '/' && cp[1] == '*')
883 else if (in_comment && cp[0] == '*' && cp[1] == '/')
892 /* Check commenting. */
899 /* Prepares LINE, which is subject to the given SYNTAX rules, for
900 tokenization by stripping comments and determining whether it
901 is the beginning or end of a command and storing into
902 *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */
904 lex_preprocess_line (struct string *line,
905 enum syntax_mode syntax,
906 bool *line_starts_command,
907 bool *line_ends_command)
909 strip_comments (line);
910 ds_rtrim (line, ss_cstr (CC_SPACES));
911 *line_ends_command = ds_chomp_byte (line, '.') || ds_is_empty (line);
912 *line_starts_command = false;
913 if (syntax == GETL_BATCH)
915 int first = ds_first (line);
916 *line_starts_command = !c_isspace (first);
917 if (first == '+' || first == '-')
918 *ds_data (line) = ' ';
922 /* Reads a line, without performing any preprocessing. */
924 lex_get_line_raw (struct lexer *lexer)
926 bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
929 const char *line = ds_cstr (&lexer->line_buffer);
930 text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line));
937 /* Reads a line for use by the tokenizer, and preprocesses it by
938 removing comments, stripping trailing whitespace and the
939 terminal dot, and removing leading indentors. */
941 lex_get_line (struct lexer *lexer)
943 bool line_starts_command;
945 if (!lex_get_line_raw (lexer))
948 lex_preprocess_line (&lexer->line_buffer,
949 lex_current_syntax_mode (lexer),
950 &line_starts_command, &lexer->dot);
952 if (line_starts_command)
953 lexer->put_token = T_ENDCMD;
955 lexer->prog = ds_cstr (&lexer->line_buffer);
961 /* Returns the name of a token. */
963 lex_token_name (enum token_type token)
1056 /* Returns an ASCII representation of the current token as a
1057 malloc()'d string. */
1059 lex_token_representation (struct lexer *lexer)
1063 switch (lexer->token)
1068 return ss_xstrdup (lex_tokss (lexer));
1072 struct substring ss;
1076 ss = lex_tokss (lexer);
1077 for (sp = ss_data (ss); sp < ss_end (ss); sp++)
1078 if (!c_isprint ((unsigned char) *sp))
1084 token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
1091 for (sp = ss_data (ss); sp < ss_end (ss); sp++)
1096 *dp++ = (unsigned char) *sp;
1100 *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
1101 *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
1110 return xstrdup (lex_token_name (lexer->token));
1114 /* Really weird functions. */
1116 /* Skip a COMMENT command. */
1118 lex_skip_comment (struct lexer *lexer)
1122 if (!lex_get_line (lexer))
1124 lexer->put_token = T_STOP;
1129 if (lexer->put_token == T_ENDCMD)
1132 ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
1133 lexer->prog = ds_end (&lexer->line_buffer);
1139 /* Private functions. */
1141 /* When invoked, tokstr contains a string of binary, octal, or
1142 hex digits, according to TYPE. The string is converted to
1143 characters having the specified values. */
1145 convert_numeric_string_to_char_string (struct lexer *lexer,
1146 enum string_type type)
1148 const char *base_name;
1158 base_name = _("binary");
1163 base_name = _("octal");
1168 base_name = _("hex");
1176 byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
1177 if (ds_length (&lexer->tokstr) % chars_per_byte)
1178 msg (SE, _("String of %s digits has %zu characters, which is not a "
1180 base_name, ds_length (&lexer->tokstr), chars_per_byte);
1182 p = ds_cstr (&lexer->tokstr);
1183 for (i = 0; i < byte_cnt; i++)
1189 for (j = 0; j < chars_per_byte; j++, p++)
1193 if (*p >= '0' && *p <= '9')
1197 static const char alpha[] = "abcdef";
1198 const char *q = strchr (alpha, tolower ((unsigned char) *p));
1207 msg (SE, _("`%c' is not a valid %s digit."), *p, base_name);
1209 value = value * base + v;
1212 ds_cstr (&lexer->tokstr)[i] = (unsigned char) value;
1215 ds_truncate (&lexer->tokstr, byte_cnt);
1218 /* Parses a string from the input buffer into tokstr. The input
1219 buffer pointer lexer->prog must point to the initial single or double
1220 quote. TYPE indicates the type of string to be parsed.
1221 Returns token type. */
1223 parse_string (struct lexer *lexer, enum string_type type)
1225 if (type != CHARACTER_STRING)
1228 /* Accumulate the entire string, joining sections indicated by +
1232 /* Single or double quote. */
1233 int c = *lexer->prog++;
1235 /* Accumulate section. */
1238 /* Check end of line. */
1239 if (*lexer->prog == '\0')
1241 msg (SE, _("Unterminated string constant."));
1245 /* Double quote characters to embed them in strings. */
1246 if (*lexer->prog == c)
1248 if (lexer->prog[1] == c)
1254 ds_put_byte (&lexer->tokstr, *lexer->prog++);
1258 /* Skip whitespace after final quote mark. */
1259 if (lexer->prog == NULL)
1263 while (c_isspace ((unsigned char) *lexer->prog))
1271 if (!lex_get_line (lexer))
1275 /* Skip plus sign. */
1276 if (*lexer->prog != '+')
1280 /* Skip whitespace after plus sign. */
1281 if (lexer->prog == NULL)
1285 while (c_isspace ((unsigned char) *lexer->prog))
1293 if (!lex_get_line (lexer))
1295 msg (SE, _("Unexpected end of file in string concatenation."));
1300 /* Ensure that a valid string follows. */
1301 if (*lexer->prog != '\'' && *lexer->prog != '"')
1303 msg (SE, _("String expected following `+'."));
1308 /* We come here when we've finished concatenating all the string sections
1309 into one large string. */
1311 if (type != CHARACTER_STRING)
1312 convert_numeric_string_to_char_string (lexer, type);
1317 /* Token Accessor Functions */
1320 lex_token (const struct lexer *lexer)
1322 return lexer->token;
1326 lex_tokval (const struct lexer *lexer)
1328 return lexer->tokval;
1331 /* Returns the null-terminated string value associated with LEXER's current
1332 token. For a T_ID token, this is the identifier, and for a T_STRING token,
1333 this is the string. For other tokens the value is undefined. */
1335 lex_tokcstr (const struct lexer *lexer)
1337 return ds_cstr (&lexer->tokstr);
1340 /* Returns the string value associated with LEXER's current token. For a T_ID
1341 token, this is the identifier, and for a T_STRING token, this is the string.
1342 For other tokens the value is undefined. */
1344 lex_tokss (const struct lexer *lexer)
1346 return ds_ss (&lexer->tokstr);
1349 /* If the lexer is positioned at the (pseudo)identifier S, which
1350 may contain a hyphen ('-'), skips it and returns true. Each
1351 half of the identifier may be abbreviated to its first three
1353 Otherwise, returns false. */
1355 lex_match_hyphenated_word (struct lexer *lexer, const char *s)
1357 const char *hyphen = strchr (s, '-');
1359 return lex_match_id (lexer, s);
1360 else if (lexer->token != T_ID
1361 || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
1362 || lex_look_ahead (lexer) != T_DASH)
1367 lex_force_match (lexer, T_DASH);
1368 lex_force_match_id (lexer, hyphen + 1);