1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <libpspp/message.h>
28 #include <libpspp/assertion.h>
29 #include <language/command.h>
30 #include <libpspp/message.h>
31 #include <data/settings.h>
32 #include <libpspp/getl.h>
33 #include <libpspp/str.h>
34 #include <output/journal.h>
39 #define _(msgid) gettext (msgid)
40 #define N_(msgid) msgid
49 struct string line_buffer;
51 struct source_stream *ss;
53 int token; /* Current token. */
54 double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
56 char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
58 struct string tokstr; /* T_ID, T_STRING: token string value.
59 For T_ID, this is not truncated as is
62 char *prog; /* Pointer to next token in line_buffer. */
63 bool dot; /* True only if this line ends with a terminal dot. */
65 int put_token ; /* If nonzero, next token returned by lex_get().
66 Used only in exceptional circumstances. */
68 struct string put_tokstr;
73 static int parse_id (struct lexer *);
75 /* How a string represents its contents. */
78 CHARACTER_STRING, /* Characters. */
79 BINARY_STRING, /* Binary digits. */
80 OCTAL_STRING, /* Octal digits. */
81 HEX_STRING /* Hexadecimal digits. */
84 static int parse_string (struct lexer *, enum string_type);
87 static void dump_token (struct lexer *);
92 /* Initializes the lexer. */
94 lex_create (struct source_stream *ss)
96 struct lexer *lexer = xzalloc (sizeof (*lexer));
98 ds_init_empty (&lexer->tokstr);
99 ds_init_empty (&lexer->put_tokstr);
100 ds_init_empty (&lexer->line_buffer);
106 struct source_stream *
107 lex_get_source_stream (const struct lexer *lex)
113 lex_current_syntax_mode (const struct lexer *lex)
115 return source_stream_current_syntax_mode (lex->ss);
119 lex_current_error_mode (const struct lexer *lex)
121 return source_stream_current_error_mode (lex->ss);
126 lex_destroy (struct lexer *lexer)
130 ds_destroy (&lexer->put_tokstr);
131 ds_destroy (&lexer->tokstr);
132 ds_destroy (&lexer->line_buffer);
139 /* Common functions. */
141 /* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
142 tokval, respectively, and sets tokid appropriately. */
144 restore_token (struct lexer *lexer)
146 assert (lexer->put_token != 0);
147 lexer->token = lexer->put_token;
148 ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
149 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
150 lexer->tokval = lexer->put_tokval;
151 lexer->put_token = 0;
154 /* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
155 put_lexer->tokval respectively. */
157 save_token (struct lexer *lexer)
159 lexer->put_token = lexer->token;
160 ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
161 lexer->put_tokval = lexer->tokval;
164 /* Parses a single token, setting appropriate global variables to
165 indicate the token's attributes. */
167 lex_get (struct lexer *lexer)
172 if (NULL == lexer->prog && ! lex_get_line (lexer) )
174 lexer->token = T_STOP;
178 /* If a token was pushed ahead, return it. */
179 if (lexer->put_token)
181 restore_token (lexer);
190 /* Skip whitespace. */
191 while (c_isspace ((unsigned char) *lexer->prog))
206 else if (!lex_get_line (lexer))
209 lexer->token = T_STOP;
216 if (lexer->put_token)
218 restore_token (lexer);
227 /* Actually parse the token. */
228 ds_clear (&lexer->tokstr);
230 switch (*lexer->prog)
233 case '0': case '1': case '2': case '3': case '4':
234 case '5': case '6': case '7': case '8': case '9':
238 /* `-' can introduce a negative number, or it can be a
239 token by itself. If it is not followed by a digit or a
240 decimal point, it is definitely not a number.
241 Otherwise, it might be either, but most of the time we
242 want it as a number. When the syntax calls for a `-'
243 token, lex_negative_to_dash() must be used to break
244 negative numbers into two tokens. */
245 if (*lexer->prog == '-')
247 ds_put_char (&lexer->tokstr, *lexer->prog++);
248 while (c_isspace ((unsigned char) *lexer->prog))
251 if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
256 lexer->token = T_NEG_NUM;
259 lexer->token = T_POS_NUM;
261 /* Parse the number, copying it into tokstr. */
262 while (c_isdigit ((unsigned char) *lexer->prog))
263 ds_put_char (&lexer->tokstr, *lexer->prog++);
264 if (*lexer->prog == '.')
266 ds_put_char (&lexer->tokstr, *lexer->prog++);
267 while (c_isdigit ((unsigned char) *lexer->prog))
268 ds_put_char (&lexer->tokstr, *lexer->prog++);
270 if (*lexer->prog == 'e' || *lexer->prog == 'E')
272 ds_put_char (&lexer->tokstr, *lexer->prog++);
273 if (*lexer->prog == '+' || *lexer->prog == '-')
274 ds_put_char (&lexer->tokstr, *lexer->prog++);
275 while (c_isdigit ((unsigned char) *lexer->prog))
276 ds_put_char (&lexer->tokstr, *lexer->prog++);
279 /* Parse as floating point. */
280 lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail);
283 msg (SE, _("%s does not form a valid number."),
284 ds_cstr (&lexer->tokstr));
287 ds_clear (&lexer->tokstr);
288 ds_put_char (&lexer->tokstr, '0');
295 lexer->token = parse_string (lexer, CHARACTER_STRING);
298 case '(': case ')': case ',': case '=': case '+': case '/':
300 lexer->token = *lexer->prog++;
304 if (*++lexer->prog == '*')
307 lexer->token = T_EXP;
314 if (*++lexer->prog == '=')
319 else if (*lexer->prog == '>')
329 if (*++lexer->prog == '=')
339 if (*++lexer->prog == '=')
345 lexer->token = T_NOT;
350 lexer->token = T_AND;
359 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
360 lexer->token = parse_string (lexer, BINARY_STRING);
362 lexer->token = parse_id (lexer);
366 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
367 lexer->token = parse_string (lexer, OCTAL_STRING);
369 lexer->token = parse_id (lexer);
373 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
374 lexer->token = parse_string (lexer, HEX_STRING);
376 lexer->token = parse_id (lexer);
380 if (lex_is_id1 (*lexer->prog))
382 lexer->token = parse_id (lexer);
387 if (c_isgraph ((unsigned char) *lexer->prog))
388 msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
390 msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
402 /* Parses an identifier at the current position into tokid and
404 Returns the correct token type. */
406 parse_id (struct lexer *lexer)
408 struct substring rest_of_line
409 = ss_substr (ds_ss (&lexer->line_buffer),
410 ds_pointer_to_position (&lexer->line_buffer, lexer->prog),
412 struct substring id = ss_head (rest_of_line,
413 lex_id_get_length (rest_of_line));
414 lexer->prog += ss_length (id);
416 ds_assign_substring (&lexer->tokstr, id);
417 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
418 return lex_id_to_token (id);
421 /* Reports an error to the effect that subcommand SBC may only be
424 lex_sbc_only_once (const char *sbc)
426 msg (SE, _("Subcommand %s may only be specified once."), sbc);
429 /* Reports an error to the effect that subcommand SBC is
432 lex_sbc_missing (struct lexer *lexer, const char *sbc)
434 lex_error (lexer, _("missing required subcommand %s"), sbc);
437 /* Prints a syntax error message containing the current token and
438 given message MESSAGE (if non-null). */
440 lex_error (struct lexer *lexer, const char *message, ...)
445 token_rep = lex_token_representation (lexer);
446 if (lexer->token == T_STOP)
447 strcpy (where, "end of file");
448 else if (lexer->token == '.')
449 strcpy (where, "end of command");
451 snprintf (where, sizeof where, "`%s'", token_rep);
459 va_start (args, message);
460 vsnprintf (buf, 1024, message, args);
463 msg (SE, _("Syntax error %s at %s."), buf, where);
466 msg (SE, _("Syntax error at %s."), where);
469 /* Checks that we're at end of command.
470 If so, returns a successful command completion code.
471 If not, flags a syntax error and returns an error command
474 lex_end_of_command (struct lexer *lexer)
476 if (lexer->token != '.')
478 lex_error (lexer, _("expecting end of command"));
485 /* Token testing functions. */
487 /* Returns true if the current token is a number. */
489 lex_is_number (struct lexer *lexer)
491 return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
495 /* Returns true if the current token is a string. */
497 lex_is_string (struct lexer *lexer)
499 return lexer->token == T_STRING;
503 /* Returns the value of the current token, which must be a
504 floating point number. */
506 lex_number (struct lexer *lexer)
508 assert (lex_is_number (lexer));
509 return lexer->tokval;
512 /* Returns true iff the current token is an integer. */
514 lex_is_integer (struct lexer *lexer)
516 return (lex_is_number (lexer)
517 && lexer->tokval > LONG_MIN
518 && lexer->tokval <= LONG_MAX
519 && floor (lexer->tokval) == lexer->tokval);
522 /* Returns the value of the current token, which must be an
525 lex_integer (struct lexer *lexer)
527 assert (lex_is_integer (lexer));
528 return lexer->tokval;
531 /* Token matching functions. */
533 /* If TOK is the current token, skips it and returns true
534 Otherwise, returns false. */
536 lex_match (struct lexer *lexer, int t)
538 if (lexer->token == t)
547 /* If the current token is the identifier S, skips it and returns
548 true. The identifier may be abbreviated to its first three
550 Otherwise, returns false. */
552 lex_match_id (struct lexer *lexer, const char *s)
554 return lex_match_id_n (lexer, s, 3);
557 /* If the current token is the identifier S, skips it and returns
558 true. The identifier may be abbreviated to its first N
560 Otherwise, returns false. */
562 lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
564 if (lexer->token == T_ID
565 && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
574 /* If the current token is integer N, skips it and returns true.
575 Otherwise, returns false. */
577 lex_match_int (struct lexer *lexer, int x)
579 if (lex_is_integer (lexer) && lex_integer (lexer) == x)
588 /* Forced matches. */
590 /* If this token is identifier S, fetches the next token and returns
592 Otherwise, reports an error and returns zero. */
594 lex_force_match_id (struct lexer *lexer, const char *s)
596 if (lex_match_id (lexer, s))
600 lex_error (lexer, _("expecting `%s'"), s);
605 /* If the current token is T, skips the token. Otherwise, reports an
606 error and returns from the current function with return value false. */
608 lex_force_match (struct lexer *lexer, int t)
610 if (lexer->token == t)
617 lex_error (lexer, _("expecting `%s'"), lex_token_name (t));
622 /* If this token is a string, does nothing and returns true.
623 Otherwise, reports an error and returns false. */
625 lex_force_string (struct lexer *lexer)
627 if (lexer->token == T_STRING)
631 lex_error (lexer, _("expecting string"));
636 /* If this token is an integer, does nothing and returns true.
637 Otherwise, reports an error and returns false. */
639 lex_force_int (struct lexer *lexer)
641 if (lex_is_integer (lexer))
645 lex_error (lexer, _("expecting integer"));
650 /* If this token is a number, does nothing and returns true.
651 Otherwise, reports an error and returns false. */
653 lex_force_num (struct lexer *lexer)
655 if (lex_is_number (lexer))
658 lex_error (lexer, _("expecting number"));
662 /* If this token is an identifier, does nothing and returns true.
663 Otherwise, reports an error and returns false. */
665 lex_force_id (struct lexer *lexer)
667 if (lexer->token == T_ID)
670 lex_error (lexer, _("expecting identifier"));
674 /* Weird token functions. */
676 /* Returns the first character of the next token, except that if the
677 next token is not an identifier, the character returned will not be
678 a character that can begin an identifier. Specifically, the
679 hexstring lead-in X' causes lookahead() to return '. Note that an
680 alphanumeric return value doesn't guarantee an ID token, it could
681 also be a reserved-word token. */
683 lex_look_ahead (struct lexer *lexer)
685 if (lexer->put_token)
686 return lexer->put_token;
690 if (NULL == lexer->prog && ! lex_get_line (lexer) )
695 while (c_isspace ((unsigned char) *lexer->prog))
702 else if (!lex_get_line (lexer))
705 if (lexer->put_token)
706 return lexer->put_token;
709 if ((toupper ((unsigned char) *lexer->prog) == 'X'
710 || toupper ((unsigned char) *lexer->prog) == 'B'
711 || toupper ((unsigned char) *lexer->prog) == 'O')
712 && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
719 /* Makes the current token become the next token to be read; the
720 current token is set to T. */
722 lex_put_back (struct lexer *lexer, int t)
728 /* Makes the current token become the next token to be read; the
729 current token is set to the identifier ID. */
731 lex_put_back_id (struct lexer *lexer, const char *id)
733 assert (lex_id_to_token (ss_cstr (id)) == T_ID);
736 ds_assign_cstr (&lexer->tokstr, id);
737 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
740 /* Weird line processing functions. */
742 /* Returns the entire contents of the current line. */
744 lex_entire_line (const struct lexer *lexer)
746 return ds_cstr (&lexer->line_buffer);
749 const struct string *
750 lex_entire_line_ds (const struct lexer *lexer)
752 return &lexer->line_buffer;
755 /* As lex_entire_line(), but only returns the part of the current line
756 that hasn't already been tokenized. */
758 lex_rest_of_line (const struct lexer *lexer)
763 /* Returns true if the current line ends in a terminal dot,
766 lex_end_dot (const struct lexer *lexer)
771 /* Causes the rest of the current input line to be ignored for
772 tokenization purposes. */
774 lex_discard_line (struct lexer *lexer)
776 ds_cstr (&lexer->line_buffer); /* Ensures ds_end points to something valid */
777 lexer->prog = ds_end (&lexer->line_buffer);
779 lexer->put_token = 0;
783 /* Discards the rest of the current command.
784 When we're reading commands from a file, we skip tokens until
785 a terminal dot or EOF.
786 When we're reading commands interactively from the user,
787 that's just discarding the current line, because presumably
788 the user doesn't want to finish typing a command that will be
791 lex_discard_rest_of_command (struct lexer *lexer)
793 if (!getl_is_interactive (lexer->ss))
795 while (lexer->token != T_STOP && lexer->token != '.')
799 lex_discard_line (lexer);
802 /* Weird line reading functions. */
804 /* Remove C-style comments in STRING, begun by slash-star and
805 terminated by star-slash or newline. */
807 strip_comments (struct string *string)
815 for (cp = ds_cstr (string); *cp; )
817 /* If we're not in a comment, check for quote marks. */
822 else if (*cp == '\'' || *cp == '"')
826 /* If we're not inside a quotation, check for comment. */
829 if (cp[0] == '/' && cp[1] == '*')
836 else if (in_comment && cp[0] == '*' && cp[1] == '/')
845 /* Check commenting. */
852 /* Prepares LINE, which is subject to the given SYNTAX rules, for
853 tokenization by stripping comments and determining whether it
854 is the beginning or end of a command and storing into
855 *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */
857 lex_preprocess_line (struct string *line,
858 enum syntax_mode syntax,
859 bool *line_starts_command,
860 bool *line_ends_command)
862 strip_comments (line);
863 ds_rtrim (line, ss_cstr (CC_SPACES));
864 *line_ends_command = (ds_chomp (line, settings_get_endcmd ())
865 || (ds_is_empty (line) && settings_get_nulline ()));
866 *line_starts_command = false;
867 if (syntax == GETL_BATCH)
869 int first = ds_first (line);
870 *line_starts_command = !c_isspace (first);
871 if (first == '+' || first == '-')
872 *ds_data (line) = ' ';
876 /* Reads a line, without performing any preprocessing.
877 Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
880 lex_get_line_raw (struct lexer *lexer)
882 bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
883 enum syntax_mode mode = lex_current_syntax_mode (lexer);
884 journal_write (mode == GETL_BATCH, ds_cstr (&lexer->line_buffer));
889 /* Reads a line for use by the tokenizer, and preprocesses it by
890 removing comments, stripping trailing whitespace and the
891 terminal dot, and removing leading indentors. */
893 lex_get_line (struct lexer *lexer)
895 bool line_starts_command;
897 if (!lex_get_line_raw (lexer))
903 lex_preprocess_line (&lexer->line_buffer,
904 lex_current_syntax_mode (lexer),
905 &line_starts_command, &lexer->dot);
907 if (line_starts_command)
908 lexer->put_token = '.';
910 lexer->prog = ds_cstr (&lexer->line_buffer);
916 /* Returns the name of a token. */
918 lex_token_name (int token)
920 if (lex_is_keyword (token))
921 return lex_id_name (token);
922 else if (token < 256)
924 static char t[256][2];
934 /* Returns an ASCII representation of the current token as a
935 malloc()'d string. */
937 lex_token_representation (struct lexer *lexer)
941 switch (lexer->token)
946 return ds_xstrdup (&lexer->tokstr);
954 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
955 if (!c_isprint ((unsigned char) *sp))
961 token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
969 for (sp = ds_cstr (&lexer->tokstr); *sp; )
973 *dp++ = (unsigned char) *sp++;
976 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
978 *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
979 *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
989 token_rep = xmalloc (1);
994 return xstrdup ("**");
997 return xstrdup (lex_token_name (lexer->token));
1003 /* Really weird functions. */
1005 /* Most of the time, a `-' is a lead-in to a negative number. But
1006 sometimes it's actually part of the syntax. If a dash can be part
1007 of syntax then this function is called to rip it off of a
1010 lex_negative_to_dash (struct lexer *lexer)
1012 if (lexer->token == T_NEG_NUM)
1014 lexer->token = T_POS_NUM;
1015 lexer->tokval = -lexer->tokval;
1016 ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
1022 /* Skip a COMMENT command. */
1024 lex_skip_comment (struct lexer *lexer)
1028 if (!lex_get_line (lexer))
1030 lexer->put_token = T_STOP;
1035 if (lexer->put_token == '.')
1038 ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
1039 lexer->prog = ds_end (&lexer->line_buffer);
1045 /* Private functions. */
1047 /* When invoked, tokstr contains a string of binary, octal, or
1048 hex digits, according to TYPE. The string is converted to
1049 characters having the specified values. */
1051 convert_numeric_string_to_char_string (struct lexer *lexer,
1052 enum string_type type)
1054 const char *base_name;
1064 base_name = _("binary");
1069 base_name = _("octal");
1074 base_name = _("hex");
1082 byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
1083 if (ds_length (&lexer->tokstr) % chars_per_byte)
1084 msg (SE, _("String of %s digits has %zu characters, which is not a "
1086 base_name, ds_length (&lexer->tokstr), chars_per_byte);
1088 p = ds_cstr (&lexer->tokstr);
1089 for (i = 0; i < byte_cnt; i++)
1095 for (j = 0; j < chars_per_byte; j++, p++)
1099 if (*p >= '0' && *p <= '9')
1103 static const char alpha[] = "abcdef";
1104 const char *q = strchr (alpha, tolower ((unsigned char) *p));
1113 msg (SE, _("`%c' is not a valid %s digit."), *p, base_name);
1115 value = value * base + v;
1118 ds_cstr (&lexer->tokstr)[i] = (unsigned char) value;
1121 ds_truncate (&lexer->tokstr, byte_cnt);
1124 /* Parses a string from the input buffer into tokstr. The input
1125 buffer pointer lexer->prog must point to the initial single or double
1126 quote. TYPE indicates the type of string to be parsed.
1127 Returns token type. */
1129 parse_string (struct lexer *lexer, enum string_type type)
1131 if (type != CHARACTER_STRING)
1134 /* Accumulate the entire string, joining sections indicated by +
1138 /* Single or double quote. */
1139 int c = *lexer->prog++;
1141 /* Accumulate section. */
1144 /* Check end of line. */
1145 if (*lexer->prog == '\0')
1147 msg (SE, _("Unterminated string constant."));
1151 /* Double quote characters to embed them in strings. */
1152 if (*lexer->prog == c)
1154 if (lexer->prog[1] == c)
1160 ds_put_char (&lexer->tokstr, *lexer->prog++);
1164 /* Skip whitespace after final quote mark. */
1165 if (lexer->prog == NULL)
1169 while (c_isspace ((unsigned char) *lexer->prog))
1177 if (!lex_get_line (lexer))
1181 /* Skip plus sign. */
1182 if (*lexer->prog != '+')
1186 /* Skip whitespace after plus sign. */
1187 if (lexer->prog == NULL)
1191 while (c_isspace ((unsigned char) *lexer->prog))
1199 if (!lex_get_line (lexer))
1201 msg (SE, _("Unexpected end of file in string concatenation."));
1206 /* Ensure that a valid string follows. */
1207 if (*lexer->prog != '\'' && *lexer->prog != '"')
1209 msg (SE, _("String expected following `+'."));
1214 /* We come here when we've finished concatenating all the string sections
1215 into one large string. */
1217 if (type != CHARACTER_STRING)
1218 convert_numeric_string_to_char_string (lexer, type);
1220 if (ds_length (&lexer->tokstr) > 255)
1222 msg (SE, _("String exceeds 255 characters in length (%zu characters)."),
1223 ds_length (&lexer->tokstr));
1224 ds_truncate (&lexer->tokstr, 255);
1231 /* Reads one token from the lexer and writes a textual representation
1232 on stdout for debugging purposes. */
1234 dump_token (struct lexer *lexer)
1240 curln = getl_source_location (lexer->ss);
1241 curfn = getl_source_name (lexer->ss);
1243 fprintf (stderr, "%s:%d\t", curfn, curln);
1246 switch (lexer->token)
1249 fprintf (stderr, "ID\t%s\n", lexer->tokid);
1254 fprintf (stderr, "NUM\t%f\n", lexer->tokval);
1258 fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
1262 fprintf (stderr, "STOP\n");
1266 fprintf (stderr, "MISC\tEXP\"");
1270 fprintf (stderr, "MISC\tEOF\n");
1274 if (lex_is_keyword (lexer->token))
1275 fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
1277 fprintf (stderr, "PUNCT\t%c\n", lexer->token);
1281 #endif /* DUMP_TOKENS */
1284 /* Token Accessor Functions */
1287 lex_token (const struct lexer *lexer)
1289 return lexer->token;
1293 lex_tokval (const struct lexer *lexer)
1295 return lexer->tokval;
1299 lex_tokid (const struct lexer *lexer)
1301 return lexer->tokid;
1304 const struct string *
1305 lex_tokstr (const struct lexer *lexer)
1307 return &lexer->tokstr;
1310 /* If the lexer is positioned at the (pseudo)identifier S, which
1311 may contain a hyphen ('-'), skips it and returns true. Each
1312 half of the identifier may be abbreviated to its first three
1314 Otherwise, returns false. */
1316 lex_match_hyphenated_word (struct lexer *lexer, const char *s)
1318 const char *hyphen = strchr (s, '-');
1320 return lex_match_id (lexer, s);
1321 else if (lexer->token != T_ID
1322 || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
1323 || lex_look_ahead (lexer) != '-')
1328 lex_force_match (lexer, '-');
1329 lex_force_match_id (lexer, hyphen + 1);