1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <libpspp/message.h>
26 #include <libpspp/alloc.h>
27 #include <libpspp/assertion.h>
28 #include <language/command.h>
29 #include <libpspp/message.h>
30 #include <libpspp/magic.h>
31 #include <data/settings.h>
32 #include <libpspp/getl.h>
33 #include <libpspp/str.h>
38 #define _(msgid) gettext (msgid)
39 #define N_(msgid) msgid
47 struct string line_buffer;
49 struct source_stream *ss;
51 int token; /* Current token. */
52 double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
54 char tokid [LONG_NAME_LEN + 1]; /* T_ID: the identifier. */
56 struct string tokstr; /* T_ID, T_STRING: token string value.
57 For T_ID, this is not truncated as is
60 char *prog; /* Pointer to next token in line_buffer. */
61 bool dot; /* True only if this line ends with a terminal dot. */
63 int put_token ; /* If nonzero, next token returned by lex_get().
64 Used only in exceptional circumstances. */
66 struct string put_tokstr;
71 static int parse_id (struct lexer *);
73 /* How a string represents its contents. */
76 CHARACTER_STRING, /* Characters. */
77 BINARY_STRING, /* Binary digits. */
78 OCTAL_STRING, /* Octal digits. */
79 HEX_STRING /* Hexadecimal digits. */
82 static int parse_string (struct lexer *, enum string_type);
85 static void dump_token (struct lexer *);
90 /* Initializes the lexer. */
92 lex_create (struct source_stream *ss)
94 struct lexer *lexer = xzalloc (sizeof (*lexer));
96 ds_init_empty (&lexer->tokstr);
97 ds_init_empty (&lexer->put_tokstr);
98 ds_init_empty (&lexer->line_buffer);
104 struct source_stream *
105 lex_get_source_stream (const struct lexer *lex)
112 lex_destroy (struct lexer *lexer)
116 ds_destroy (&lexer->put_tokstr);
117 ds_destroy (&lexer->tokstr);
118 ds_destroy (&lexer->line_buffer);
125 /* Common functions. */
127 /* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
128 tokval, respectively, and sets tokid appropriately. */
130 restore_token (struct lexer *lexer)
132 assert (lexer->put_token != 0);
133 lexer->token = lexer->put_token;
134 ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
135 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
136 lexer->tokval = lexer->put_tokval;
137 lexer->put_token = 0;
140 /* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
141 put_lexer->tokval respectively. */
143 save_token (struct lexer *lexer)
145 lexer->put_token = lexer->token;
146 ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
147 lexer->put_tokval = lexer->tokval;
150 /* Parses a single token, setting appropriate global variables to
151 indicate the token's attributes. */
153 lex_get (struct lexer *lexer)
158 if (NULL == lexer->prog && ! lex_get_line (lexer) )
160 lexer->token = T_STOP;
164 /* If a token was pushed ahead, return it. */
165 if (lexer->put_token)
167 restore_token (lexer);
176 /* Skip whitespace. */
177 while (isspace ((unsigned char) *lexer->prog))
192 else if (!lex_get_line (lexer))
195 lexer->token = T_STOP;
202 if (lexer->put_token)
204 restore_token (lexer);
213 /* Actually parse the token. */
214 ds_clear (&lexer->tokstr);
216 switch (*lexer->prog)
219 case '0': case '1': case '2': case '3': case '4':
220 case '5': case '6': case '7': case '8': case '9':
224 /* `-' can introduce a negative number, or it can be a
225 token by itself. If it is not followed by a digit or a
226 decimal point, it is definitely not a number.
227 Otherwise, it might be either, but most of the time we
228 want it as a number. When the syntax calls for a `-'
229 token, lex_negative_to_dash() must be used to break
230 negative numbers into two tokens. */
231 if (*lexer->prog == '-')
233 ds_put_char (&lexer->tokstr, *lexer->prog++);
234 while (isspace ((unsigned char) *lexer->prog))
237 if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
242 lexer->token = T_NEG_NUM;
245 lexer->token = T_POS_NUM;
247 /* Parse the number, copying it into tokstr. */
248 while (isdigit ((unsigned char) *lexer->prog))
249 ds_put_char (&lexer->tokstr, *lexer->prog++);
250 if (*lexer->prog == '.')
252 ds_put_char (&lexer->tokstr, *lexer->prog++);
253 while (isdigit ((unsigned char) *lexer->prog))
254 ds_put_char (&lexer->tokstr, *lexer->prog++);
256 if (*lexer->prog == 'e' || *lexer->prog == 'E')
258 ds_put_char (&lexer->tokstr, *lexer->prog++);
259 if (*lexer->prog == '+' || *lexer->prog == '-')
260 ds_put_char (&lexer->tokstr, *lexer->prog++);
261 while (isdigit ((unsigned char) *lexer->prog))
262 ds_put_char (&lexer->tokstr, *lexer->prog++);
265 /* Parse as floating point. */
266 lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
269 msg (SE, _("%s does not form a valid number."),
270 ds_cstr (&lexer->tokstr));
273 ds_clear (&lexer->tokstr);
274 ds_put_char (&lexer->tokstr, '0');
281 lexer->token = parse_string (lexer, CHARACTER_STRING);
284 case '(': case ')': case ',': case '=': case '+': case '/':
285 lexer->token = *lexer->prog++;
289 if (*++lexer->prog == '*')
292 lexer->token = T_EXP;
299 if (*++lexer->prog == '=')
304 else if (*lexer->prog == '>')
314 if (*++lexer->prog == '=')
324 if (*++lexer->prog == '=')
330 lexer->token = T_NOT;
335 lexer->token = T_AND;
344 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
345 lexer->token = parse_string (lexer, BINARY_STRING);
347 lexer->token = parse_id (lexer);
351 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
352 lexer->token = parse_string (lexer, OCTAL_STRING);
354 lexer->token = parse_id (lexer);
358 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
359 lexer->token = parse_string (lexer, HEX_STRING);
361 lexer->token = parse_id (lexer);
365 if (lex_is_id1 (*lexer->prog))
367 lexer->token = parse_id (lexer);
372 if (isgraph ((unsigned char) *lexer->prog))
373 msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
375 msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
387 /* Parses an identifier at the current position into tokid and
389 Returns the correct token type. */
391 parse_id (struct lexer *lexer)
393 struct substring rest_of_line
394 = ss_substr (ds_ss (&lexer->line_buffer),
395 ds_pointer_to_position (&lexer->line_buffer, lexer->prog),
397 struct substring id = ss_head (rest_of_line,
398 lex_id_get_length (rest_of_line));
399 lexer->prog += ss_length (id);
401 ds_assign_substring (&lexer->tokstr, id);
402 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
403 return lex_id_to_token (id);
406 /* Reports an error to the effect that subcommand SBC may only be
409 lex_sbc_only_once (const char *sbc)
411 msg (SE, _("Subcommand %s may only be specified once."), sbc);
414 /* Reports an error to the effect that subcommand SBC is
417 lex_sbc_missing (struct lexer *lexer, const char *sbc)
419 lex_error (lexer, _("missing required subcommand %s"), sbc);
422 /* Prints a syntax error message containing the current token and
423 given message MESSAGE (if non-null). */
425 lex_error (struct lexer *lexer, const char *message, ...)
430 token_rep = lex_token_representation (lexer);
431 if (lexer->token == T_STOP)
432 strcpy (where, "end of file");
433 else if (lexer->token == '.')
434 strcpy (where, "end of command");
436 snprintf (where, sizeof where, "`%s'", token_rep);
444 va_start (args, message);
445 vsnprintf (buf, 1024, message, args);
448 msg (SE, _("Syntax error %s at %s."), buf, where);
451 msg (SE, _("Syntax error at %s."), where);
454 /* Checks that we're at end of command.
455 If so, returns a successful command completion code.
456 If not, flags a syntax error and returns an error command
459 lex_end_of_command (struct lexer *lexer)
461 if (lexer->token != '.')
463 lex_error (lexer, _("expecting end of command"));
470 /* Token testing functions. */
472 /* Returns true if the current token is a number. */
474 lex_is_number (struct lexer *lexer)
476 return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
480 /* Returns true if the current token is a string. */
482 lex_is_string (struct lexer *lexer)
484 return lexer->token == T_STRING;
488 /* Returns the value of the current token, which must be a
489 floating point number. */
491 lex_number (struct lexer *lexer)
493 assert (lex_is_number (lexer));
494 return lexer->tokval;
497 /* Returns true iff the current token is an integer. */
499 lex_is_integer (struct lexer *lexer)
501 return (lex_is_number (lexer)
502 && lexer->tokval != NOT_LONG
503 && lexer->tokval >= LONG_MIN
504 && lexer->tokval <= LONG_MAX
505 && floor (lexer->tokval) == lexer->tokval);
508 /* Returns the value of the current token, which must be an
511 lex_integer (struct lexer *lexer)
513 assert (lex_is_integer (lexer));
514 return lexer->tokval;
517 /* Token matching functions. */
519 /* If TOK is the current token, skips it and returns true
520 Otherwise, returns false. */
522 lex_match (struct lexer *lexer, int t)
524 if (lexer->token == t)
533 /* If the current token is the identifier S, skips it and returns
534 true. The identifier may be abbreviated to its first three
536 Otherwise, returns false. */
538 lex_match_id (struct lexer *lexer, const char *s)
540 if (lexer->token == T_ID
541 && lex_id_match (ss_cstr (s), ss_cstr (lexer->tokid)))
550 /* If the current token is integer N, skips it and returns true.
551 Otherwise, returns false. */
553 lex_match_int (struct lexer *lexer, int x)
555 if (lex_is_integer (lexer) && lex_integer (lexer) == x)
564 /* Forced matches. */
566 /* If this token is identifier S, fetches the next token and returns
568 Otherwise, reports an error and returns zero. */
570 lex_force_match_id (struct lexer *lexer, const char *s)
572 if (lex_match_id (lexer, s))
576 lex_error (lexer, _("expecting `%s'"), s);
581 /* If the current token is T, skips the token. Otherwise, reports an
582 error and returns from the current function with return value false. */
584 lex_force_match (struct lexer *lexer, int t)
586 if (lexer->token == t)
593 lex_error (lexer, _("expecting `%s'"), lex_token_name (t));
598 /* If this token is a string, does nothing and returns true.
599 Otherwise, reports an error and returns false. */
601 lex_force_string (struct lexer *lexer)
603 if (lexer->token == T_STRING)
607 lex_error (lexer, _("expecting string"));
612 /* If this token is an integer, does nothing and returns true.
613 Otherwise, reports an error and returns false. */
615 lex_force_int (struct lexer *lexer)
617 if (lex_is_integer (lexer))
621 lex_error (lexer, _("expecting integer"));
626 /* If this token is a number, does nothing and returns true.
627 Otherwise, reports an error and returns false. */
629 lex_force_num (struct lexer *lexer)
631 if (lex_is_number (lexer))
634 lex_error (lexer, _("expecting number"));
638 /* If this token is an identifier, does nothing and returns true.
639 Otherwise, reports an error and returns false. */
641 lex_force_id (struct lexer *lexer)
643 if (lexer->token == T_ID)
646 lex_error (lexer, _("expecting identifier"));
650 /* Weird token functions. */
652 /* Returns the first character of the next token, except that if the
653 next token is not an identifier, the character returned will not be
654 a character that can begin an identifier. Specifically, the
655 hexstring lead-in X' causes lookahead() to return '. Note that an
656 alphanumeric return value doesn't guarantee an ID token, it could
657 also be a reserved-word token. */
659 lex_look_ahead (struct lexer *lexer)
661 if (lexer->put_token)
662 return lexer->put_token;
666 if (NULL == lexer->prog && ! lex_get_line (lexer) )
671 while (isspace ((unsigned char) *lexer->prog))
678 else if (!lex_get_line (lexer))
681 if (lexer->put_token)
682 return lexer->put_token;
685 if ((toupper ((unsigned char) *lexer->prog) == 'X'
686 || toupper ((unsigned char) *lexer->prog) == 'B'
687 || toupper ((unsigned char) *lexer->prog) == 'O')
688 && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
695 /* Makes the current token become the next token to be read; the
696 current token is set to T. */
698 lex_put_back (struct lexer *lexer, int t)
704 /* Makes the current token become the next token to be read; the
705 current token is set to the identifier ID. */
707 lex_put_back_id (struct lexer *lexer, const char *id)
709 assert (lex_id_to_token (ss_cstr (id)) == T_ID);
712 ds_assign_cstr (&lexer->tokstr, id);
713 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
716 /* Weird line processing functions. */
718 /* Returns the entire contents of the current line. */
720 lex_entire_line (const struct lexer *lexer)
722 return ds_cstr (&lexer->line_buffer);
725 const struct string *
726 lex_entire_line_ds (const struct lexer *lexer)
728 return &lexer->line_buffer;
731 /* As lex_entire_line(), but only returns the part of the current line
732 that hasn't already been tokenized. */
734 lex_rest_of_line (const struct lexer *lexer)
739 /* Returns true if the current line ends in a terminal dot,
742 lex_end_dot (const struct lexer *lexer)
747 /* Causes the rest of the current input line to be ignored for
748 tokenization purposes. */
750 lex_discard_line (struct lexer *lexer)
752 ds_cstr (&lexer->line_buffer); /* Ensures ds_end points to something valid */
753 lexer->prog = ds_end (&lexer->line_buffer);
755 lexer->put_token = 0;
759 /* Discards the rest of the current command.
760 When we're reading commands from a file, we skip tokens until
761 a terminal dot or EOF.
762 When we're reading commands interactively from the user,
763 that's just discarding the current line, because presumably
764 the user doesn't want to finish typing a command that will be
767 lex_discard_rest_of_command (struct lexer *lexer)
769 if (!getl_is_interactive (lexer->ss))
771 while (lexer->token != T_STOP && lexer->token != '.')
775 lex_discard_line (lexer);
778 /* Weird line reading functions. */
780 /* Remove C-style comments in STRING, begun by slash-star and
781 terminated by star-slash or newline. */
783 strip_comments (struct string *string)
791 for (cp = ds_cstr (string); *cp; )
793 /* If we're not in a comment, check for quote marks. */
798 else if (*cp == '\'' || *cp == '"')
802 /* If we're not inside a quotation, check for comment. */
805 if (cp[0] == '/' && cp[1] == '*')
812 else if (in_comment && cp[0] == '*' && cp[1] == '/')
821 /* Check commenting. */
828 /* Prepares LINE, which is subject to the given SYNTAX rules, for
829 tokenization by stripping comments and determining whether it
830 is the beginning or end of a command and storing into
831 *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */
833 lex_preprocess_line (struct string *line,
834 enum getl_syntax syntax,
835 bool *line_starts_command,
836 bool *line_ends_command)
838 strip_comments (line);
839 ds_rtrim (line, ss_cstr (CC_SPACES));
840 *line_ends_command = (ds_chomp (line, get_endcmd ())
841 || (ds_is_empty (line) && get_nulline ()));
842 *line_starts_command = false;
843 if (syntax == GETL_BATCH)
845 int first = ds_first (line);
846 *line_starts_command = !isspace (first);
847 if (first == '+' || first == '-')
848 *ds_data (line) = ' ';
852 /* Reads a line, without performing any preprocessing.
853 Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
856 lex_get_line_raw (struct lexer *lexer, enum getl_syntax *syntax)
858 enum getl_syntax dummy;
859 bool ok = getl_read_line (lexer->ss, &lexer->line_buffer,
860 syntax != NULL ? syntax : &dummy);
864 /* Reads a line for use by the tokenizer, and preprocesses it by
865 removing comments, stripping trailing whitespace and the
866 terminal dot, and removing leading indentors. */
868 lex_get_line (struct lexer *lexer)
870 bool line_starts_command;
871 enum getl_syntax syntax = GETL_BATCH;
873 if (!lex_get_line_raw (lexer, &syntax))
879 lex_preprocess_line (&lexer->line_buffer, syntax,
880 &line_starts_command, &lexer->dot);
882 if (line_starts_command)
883 lexer->put_token = '.';
885 lexer->prog = ds_cstr (&lexer->line_buffer);
891 /* Returns the name of a token. */
893 lex_token_name (int token)
895 if (lex_is_keyword (token))
896 return lex_id_name (token);
897 else if (token < 256)
899 static char t[256][2];
909 /* Returns an ASCII representation of the current token as a
910 malloc()'d string. */
912 lex_token_representation (struct lexer *lexer)
916 switch (lexer->token)
921 return ds_xstrdup (&lexer->tokstr);
929 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
930 if (!isprint ((unsigned char) *sp))
936 token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
944 for (sp = ds_cstr (&lexer->tokstr); *sp; )
948 *dp++ = (unsigned char) *sp++;
951 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
953 *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
954 *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
964 token_rep = xmalloc (1);
969 return xstrdup ("**");
972 return xstrdup (lex_token_name (lexer->token));
978 /* Really weird functions. */
980 /* Most of the time, a `-' is a lead-in to a negative number. But
981 sometimes it's actually part of the syntax. If a dash can be part
982 of syntax then this function is called to rip it off of a
985 lex_negative_to_dash (struct lexer *lexer)
987 if (lexer->token == T_NEG_NUM)
989 lexer->token = T_POS_NUM;
990 lexer->tokval = -lexer->tokval;
991 ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
997 /* Skip a COMMENT command. */
999 lex_skip_comment (struct lexer *lexer)
1003 if (!lex_get_line (lexer))
1005 lexer->put_token = T_STOP;
1010 if (lexer->put_token == '.')
1013 ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
1014 lexer->prog = ds_end (&lexer->line_buffer);
1020 /* Private functions. */
1022 /* When invoked, tokstr contains a string of binary, octal, or
1023 hex digits, according to TYPE. The string is converted to
1024 characters having the specified values. */
1026 convert_numeric_string_to_char_string (struct lexer *lexer,
1027 enum string_type type)
1029 const char *base_name;
1039 base_name = _("binary");
1044 base_name = _("octal");
1049 base_name = _("hex");
1057 byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
1058 if (ds_length (&lexer->tokstr) % chars_per_byte)
1059 msg (SE, _("String of %s digits has %d characters, which is not a "
1061 base_name, (int) ds_length (&lexer->tokstr), chars_per_byte);
1063 p = ds_cstr (&lexer->tokstr);
1064 for (i = 0; i < byte_cnt; i++)
1070 for (j = 0; j < chars_per_byte; j++, p++)
1074 if (*p >= '0' && *p <= '9')
1078 static const char alpha[] = "abcdef";
1079 const char *q = strchr (alpha, tolower ((unsigned char) *p));
1088 msg (SE, _("`%c' is not a valid %s digit."), *p, base_name);
1090 value = value * base + v;
1093 ds_cstr (&lexer->tokstr)[i] = (unsigned char) value;
1096 ds_truncate (&lexer->tokstr, byte_cnt);
1099 /* Parses a string from the input buffer into tokstr. The input
1100 buffer pointer lexer->prog must point to the initial single or double
1101 quote. TYPE indicates the type of string to be parsed.
1102 Returns token type. */
1104 parse_string (struct lexer *lexer, enum string_type type)
1106 if (type != CHARACTER_STRING)
1109 /* Accumulate the entire string, joining sections indicated by +
1113 /* Single or double quote. */
1114 int c = *lexer->prog++;
1116 /* Accumulate section. */
1119 /* Check end of line. */
1120 if (*lexer->prog == '\0')
1122 msg (SE, _("Unterminated string constant."));
1126 /* Double quote characters to embed them in strings. */
1127 if (*lexer->prog == c)
1129 if (lexer->prog[1] == c)
1135 ds_put_char (&lexer->tokstr, *lexer->prog++);
1139 /* Skip whitespace after final quote mark. */
1140 if (lexer->prog == NULL)
1144 while (isspace ((unsigned char) *lexer->prog))
1152 if (!lex_get_line (lexer))
1156 /* Skip plus sign. */
1157 if (*lexer->prog != '+')
1161 /* Skip whitespace after plus sign. */
1162 if (lexer->prog == NULL)
1166 while (isspace ((unsigned char) *lexer->prog))
1174 if (!lex_get_line (lexer))
1176 msg (SE, _("Unexpected end of file in string concatenation."));
1181 /* Ensure that a valid string follows. */
1182 if (*lexer->prog != '\'' && *lexer->prog != '"')
1184 msg (SE, _("String expected following `+'."));
1189 /* We come here when we've finished concatenating all the string sections
1190 into one large string. */
1192 if (type != CHARACTER_STRING)
1193 convert_numeric_string_to_char_string (lexer, type);
1195 if (ds_length (&lexer->tokstr) > 255)
1197 msg (SE, _("String exceeds 255 characters in length (%d characters)."),
1198 (int) ds_length (&lexer->tokstr));
1199 ds_truncate (&lexer->tokstr, 255);
1206 /* Reads one token from the lexer and writes a textual representation
1207 on stdout for debugging purposes. */
1209 dump_token (struct lexer *lexer)
1215 curln = getl_source_location (lexer->ss);
1216 curfn = getl_source_name (lexer->ss);
1218 fprintf (stderr, "%s:%d\t", curfn, curln);
1221 switch (lexer->token)
1224 fprintf (stderr, "ID\t%s\n", lexer->tokid);
1229 fprintf (stderr, "NUM\t%f\n", lexer->tokval);
1233 fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
1237 fprintf (stderr, "STOP\n");
1241 fprintf (stderr, "MISC\tEXP\"");
1245 fprintf (stderr, "MISC\tEOF\n");
1249 if (lex_is_keyword (lexer->token))
1250 fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
1252 fprintf (stderr, "PUNCT\t%c\n", lexer->token);
1256 #endif /* DUMP_TOKENS */
1259 /* Token Accessor Functions */
1262 lex_token (const struct lexer *lexer)
1264 return lexer->token;
1268 lex_tokval (const struct lexer *lexer)
1270 return lexer->tokval;
1274 lex_tokid (const struct lexer *lexer)
1276 return lexer->tokid;
1279 const struct string *
1280 lex_tokstr (const struct lexer *lexer)
1282 return &lexer->tokstr;