1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 #include <libpspp/message.h>
29 #include <libpspp/alloc.h>
30 #include <libpspp/assertion.h>
31 #include <language/command.h>
32 #include <libpspp/message.h>
33 #include <libpspp/magic.h>
34 #include <data/settings.h>
35 #include <libpspp/getl.h>
36 #include <libpspp/str.h>
41 #define _(msgid) gettext (msgid)
42 #define N_(msgid) msgid
50 struct string line_buffer;
52 struct source_stream *ss;
54 int token; /* Current token. */
55 double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
57 char tokid [LONG_NAME_LEN + 1]; /* T_ID: the identifier. */
59 struct string tokstr; /* T_ID, T_STRING: token string value.
60 For T_ID, this is not truncated as is
63 char *prog; /* Pointer to next token in line_buffer. */
64 bool dot; /* True only if this line ends with a terminal dot. */
66 int put_token ; /* If nonzero, next token returned by lex_get().
67 Used only in exceptional circumstances. */
69 struct string put_tokstr;
74 static int parse_id (struct lexer *);
76 /* How a string represents its contents. */
79 CHARACTER_STRING, /* Characters. */
80 BINARY_STRING, /* Binary digits. */
81 OCTAL_STRING, /* Octal digits. */
82 HEX_STRING /* Hexadecimal digits. */
85 static int parse_string (struct lexer *, enum string_type);
88 static void dump_token (struct lexer *);
93 /* Initializes the lexer. */
95 lex_create (struct source_stream *ss)
97 struct lexer *lexer = xzalloc (sizeof (*lexer));
99 ds_init_empty (&lexer->tokstr);
100 ds_init_empty (&lexer->put_tokstr);
101 ds_init_empty (&lexer->line_buffer);
107 struct source_stream *
108 lex_get_source_stream (const struct lexer *lex)
115 lex_destroy (struct lexer *lexer)
119 ds_destroy (&lexer->put_tokstr);
120 ds_destroy (&lexer->tokstr);
121 ds_destroy (&lexer->line_buffer);
128 /* Common functions. */
130 /* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
131 tokval, respectively, and sets tokid appropriately. */
133 restore_token (struct lexer *lexer)
135 assert (lexer->put_token != 0);
136 lexer->token = lexer->put_token;
137 ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
138 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
139 lexer->tokval = lexer->put_tokval;
140 lexer->put_token = 0;
143 /* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
144 put_lexer->tokval respectively. */
146 save_token (struct lexer *lexer)
148 lexer->put_token = lexer->token;
149 ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
150 lexer->put_tokval = lexer->tokval;
153 /* Parses a single token, setting appropriate global variables to
154 indicate the token's attributes. */
156 lex_get (struct lexer *lexer)
161 if (NULL == lexer->prog && ! lex_get_line (lexer) )
163 lexer->token = T_STOP;
167 /* If a token was pushed ahead, return it. */
168 if (lexer->put_token)
170 restore_token (lexer);
179 /* Skip whitespace. */
180 while (isspace ((unsigned char) *lexer->prog))
195 else if (!lex_get_line (lexer))
198 lexer->token = T_STOP;
205 if (lexer->put_token)
207 restore_token (lexer);
216 /* Actually parse the token. */
217 ds_clear (&lexer->tokstr);
219 switch (*lexer->prog)
222 case '0': case '1': case '2': case '3': case '4':
223 case '5': case '6': case '7': case '8': case '9':
227 /* `-' can introduce a negative number, or it can be a
228 token by itself. If it is not followed by a digit or a
229 decimal point, it is definitely not a number.
230 Otherwise, it might be either, but most of the time we
231 want it as a number. When the syntax calls for a `-'
232 token, lex_negative_to_dash() must be used to break
233 negative numbers into two tokens. */
234 if (*lexer->prog == '-')
236 ds_put_char (&lexer->tokstr, *lexer->prog++);
237 while (isspace ((unsigned char) *lexer->prog))
240 if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
245 lexer->token = T_NEG_NUM;
248 lexer->token = T_POS_NUM;
250 /* Parse the number, copying it into tokstr. */
251 while (isdigit ((unsigned char) *lexer->prog))
252 ds_put_char (&lexer->tokstr, *lexer->prog++);
253 if (*lexer->prog == '.')
255 ds_put_char (&lexer->tokstr, *lexer->prog++);
256 while (isdigit ((unsigned char) *lexer->prog))
257 ds_put_char (&lexer->tokstr, *lexer->prog++);
259 if (*lexer->prog == 'e' || *lexer->prog == 'E')
261 ds_put_char (&lexer->tokstr, *lexer->prog++);
262 if (*lexer->prog == '+' || *lexer->prog == '-')
263 ds_put_char (&lexer->tokstr, *lexer->prog++);
264 while (isdigit ((unsigned char) *lexer->prog))
265 ds_put_char (&lexer->tokstr, *lexer->prog++);
268 /* Parse as floating point. */
269 lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
272 msg (SE, _("%s does not form a valid number."),
273 ds_cstr (&lexer->tokstr));
276 ds_clear (&lexer->tokstr);
277 ds_put_char (&lexer->tokstr, '0');
284 lexer->token = parse_string (lexer, CHARACTER_STRING);
287 case '(': case ')': case ',': case '=': case '+': case '/':
288 lexer->token = *lexer->prog++;
292 if (*++lexer->prog == '*')
295 lexer->token = T_EXP;
302 if (*++lexer->prog == '=')
307 else if (*lexer->prog == '>')
317 if (*++lexer->prog == '=')
327 if (*++lexer->prog == '=')
333 lexer->token = T_NOT;
338 lexer->token = T_AND;
347 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
348 lexer->token = parse_string (lexer, BINARY_STRING);
350 lexer->token = parse_id (lexer);
354 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
355 lexer->token = parse_string (lexer, OCTAL_STRING);
357 lexer->token = parse_id (lexer);
361 if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
362 lexer->token = parse_string (lexer, HEX_STRING);
364 lexer->token = parse_id (lexer);
368 if (lex_is_id1 (*lexer->prog))
370 lexer->token = parse_id (lexer);
375 if (isgraph ((unsigned char) *lexer->prog))
376 msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
378 msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
390 /* Parses an identifier at the current position into tokid and
392 Returns the correct token type. */
394 parse_id (struct lexer *lexer)
396 struct substring rest_of_line
397 = ss_substr (ds_ss (&lexer->line_buffer),
398 ds_pointer_to_position (&lexer->line_buffer, lexer->prog),
400 struct substring id = ss_head (rest_of_line,
401 lex_id_get_length (rest_of_line));
402 lexer->prog += ss_length (id);
404 ds_assign_substring (&lexer->tokstr, id);
405 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
406 return lex_id_to_token (id);
409 /* Reports an error to the effect that subcommand SBC may only be
412 lex_sbc_only_once (const char *sbc)
414 msg (SE, _("Subcommand %s may only be specified once."), sbc);
417 /* Reports an error to the effect that subcommand SBC is
420 lex_sbc_missing (struct lexer *lexer, const char *sbc)
422 lex_error (lexer, _("missing required subcommand %s"), sbc);
425 /* Prints a syntax error message containing the current token and
426 given message MESSAGE (if non-null). */
428 lex_error (struct lexer *lexer, const char *message, ...)
433 token_rep = lex_token_representation (lexer);
434 if (lexer->token == T_STOP)
435 strcpy (where, "end of file");
436 else if (lexer->token == '.')
437 strcpy (where, "end of command");
439 snprintf (where, sizeof where, "`%s'", token_rep);
447 va_start (args, message);
448 vsnprintf (buf, 1024, message, args);
451 msg (SE, _("Syntax error %s at %s."), buf, where);
454 msg (SE, _("Syntax error at %s."), where);
457 /* Checks that we're at end of command.
458 If so, returns a successful command completion code.
459 If not, flags a syntax error and returns an error command
462 lex_end_of_command (struct lexer *lexer)
464 if (lexer->token != '.')
466 lex_error (lexer, _("expecting end of command"));
473 /* Token testing functions. */
475 /* Returns true if the current token is a number. */
477 lex_is_number (struct lexer *lexer)
479 return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
482 /* Returns the value of the current token, which must be a
483 floating point number. */
485 lex_number (struct lexer *lexer)
487 assert (lex_is_number (lexer));
488 return lexer->tokval;
491 /* Returns true iff the current token is an integer. */
493 lex_is_integer (struct lexer *lexer)
495 return (lex_is_number (lexer)
496 && lexer->tokval != NOT_LONG
497 && lexer->tokval >= LONG_MIN
498 && lexer->tokval <= LONG_MAX
499 && floor (lexer->tokval) == lexer->tokval);
502 /* Returns the value of the current token, which must be an
505 lex_integer (struct lexer *lexer)
507 assert (lex_is_integer (lexer));
508 return lexer->tokval;
511 /* Token matching functions. */
513 /* If TOK is the current token, skips it and returns true
514 Otherwise, returns false. */
516 lex_match (struct lexer *lexer, int t)
518 if (lexer->token == t)
527 /* If the current token is the identifier S, skips it and returns
528 true. The identifier may be abbreviated to its first three
530 Otherwise, returns false. */
532 lex_match_id (struct lexer *lexer, const char *s)
534 if (lexer->token == T_ID
535 && lex_id_match (ss_cstr (s), ss_cstr (lexer->tokid)))
544 /* If the current token is integer N, skips it and returns true.
545 Otherwise, returns false. */
547 lex_match_int (struct lexer *lexer, int x)
549 if (lex_is_integer (lexer) && lex_integer (lexer) == x)
558 /* Forced matches. */
560 /* If this token is identifier S, fetches the next token and returns
562 Otherwise, reports an error and returns zero. */
564 lex_force_match_id (struct lexer *lexer, const char *s)
566 if (lex_match_id (lexer, s))
570 lex_error (lexer, _("expecting `%s'"), s);
575 /* If the current token is T, skips the token. Otherwise, reports an
576 error and returns from the current function with return value false. */
578 lex_force_match (struct lexer *lexer, int t)
580 if (lexer->token == t)
587 lex_error (lexer, _("expecting `%s'"), lex_token_name (t));
592 /* If this token is a string, does nothing and returns true.
593 Otherwise, reports an error and returns false. */
595 lex_force_string (struct lexer *lexer)
597 if (lexer->token == T_STRING)
601 lex_error (lexer, _("expecting string"));
606 /* If this token is an integer, does nothing and returns true.
607 Otherwise, reports an error and returns false. */
609 lex_force_int (struct lexer *lexer)
611 if (lex_is_integer (lexer))
615 lex_error (lexer, _("expecting integer"));
620 /* If this token is a number, does nothing and returns true.
621 Otherwise, reports an error and returns false. */
623 lex_force_num (struct lexer *lexer)
625 if (lex_is_number (lexer))
628 lex_error (lexer, _("expecting number"));
632 /* If this token is an identifier, does nothing and returns true.
633 Otherwise, reports an error and returns false. */
635 lex_force_id (struct lexer *lexer)
637 if (lexer->token == T_ID)
640 lex_error (lexer, _("expecting identifier"));
644 /* Weird token functions. */
646 /* Returns the first character of the next token, except that if the
647 next token is not an identifier, the character returned will not be
648 a character that can begin an identifier. Specifically, the
649 hexstring lead-in X' causes lookahead() to return '. Note that an
650 alphanumeric return value doesn't guarantee an ID token, it could
651 also be a reserved-word token. */
653 lex_look_ahead (struct lexer *lexer)
655 if (lexer->put_token)
656 return lexer->put_token;
660 if (NULL == lexer->prog && ! lex_get_line (lexer) )
665 while (isspace ((unsigned char) *lexer->prog))
672 else if (!lex_get_line (lexer))
675 if (lexer->put_token)
676 return lexer->put_token;
679 if ((toupper ((unsigned char) *lexer->prog) == 'X'
680 || toupper ((unsigned char) *lexer->prog) == 'B'
681 || toupper ((unsigned char) *lexer->prog) == 'O')
682 && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
689 /* Makes the current token become the next token to be read; the
690 current token is set to T. */
692 lex_put_back (struct lexer *lexer, int t)
698 /* Makes the current token become the next token to be read; the
699 current token is set to the identifier ID. */
701 lex_put_back_id (struct lexer *lexer, const char *id)
703 assert (lex_id_to_token (ss_cstr (id)) == T_ID);
706 ds_assign_cstr (&lexer->tokstr, id);
707 str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
710 /* Weird line processing functions. */
712 /* Returns the entire contents of the current line. */
714 lex_entire_line (struct lexer *lexer)
716 return ds_cstr (&lexer->line_buffer);
719 const struct string *
720 lex_entire_line_ds (struct lexer *lexer)
722 return &lexer->line_buffer;
725 /* As lex_entire_line(), but only returns the part of the current line
726 that hasn't already been tokenized.
727 If END_DOT is non-null, stores nonzero into *END_DOT if the line
728 ends with a terminal dot, or zero if it doesn't. */
730 lex_rest_of_line (struct lexer *lexer, int *end_dot)
733 *end_dot = lexer->dot;
737 /* Causes the rest of the current input line to be ignored for
738 tokenization purposes. */
740 lex_discard_line (struct lexer *lexer)
742 ds_cstr (&lexer->line_buffer); /* Ensures ds_end points to something valid */
743 lexer->prog = ds_end (&lexer->line_buffer);
745 lexer->put_token = 0;
749 /* Discards the rest of the current command.
750 When we're reading commands from a file, we skip tokens until
751 a terminal dot or EOF.
752 When we're reading commands interactively from the user,
753 that's just discarding the current line, because presumably
754 the user doesn't want to finish typing a command that will be
757 lex_discard_rest_of_command (struct lexer *lexer)
759 if (!getl_is_interactive (lexer->ss))
761 while (lexer->token != T_STOP && lexer->token != '.')
765 lex_discard_line (lexer);
768 /* Weird line reading functions. */
770 /* Remove C-style comments in STRING, begun by slash-star and
771 terminated by star-slash or newline. */
773 strip_comments (struct string *string)
781 for (cp = ds_cstr (string); *cp; )
783 /* If we're not in a comment, check for quote marks. */
788 else if (*cp == '\'' || *cp == '"')
792 /* If we're not inside a quotation, check for comment. */
795 if (cp[0] == '/' && cp[1] == '*')
802 else if (in_comment && cp[0] == '*' && cp[1] == '/')
811 /* Check commenting. */
818 /* Prepares LINE, which is subject to the given SYNTAX rules, for
819 tokenization by stripping comments and determining whether it
820 is the beginning or end of a command and storing into
821 *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */
823 lex_preprocess_line (struct string *line,
824 enum getl_syntax syntax,
825 bool *line_starts_command,
826 bool *line_ends_command)
828 strip_comments (line);
829 ds_rtrim (line, ss_cstr (CC_SPACES));
830 *line_ends_command = (ds_chomp (line, get_endcmd ())
831 || (ds_is_empty (line) && get_nulline ()));
832 *line_starts_command = false;
833 if (syntax == GETL_BATCH)
835 int first = ds_first (line);
836 *line_starts_command = !isspace (first);
837 if (first == '+' || first == '-')
838 *ds_data (line) = ' ';
842 /* Reads a line, without performing any preprocessing.
843 Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
846 lex_get_line_raw (struct lexer *lexer, enum getl_syntax *syntax)
848 enum getl_syntax dummy;
849 bool ok = getl_read_line (lexer->ss, &lexer->line_buffer,
850 syntax != NULL ? syntax : &dummy);
854 /* Reads a line for use by the tokenizer, and preprocesses it by
855 removing comments, stripping trailing whitespace and the
856 terminal dot, and removing leading indentors. */
858 lex_get_line (struct lexer *lexer)
860 bool line_starts_command;
861 enum getl_syntax syntax;
863 if (!lex_get_line_raw (lexer, &syntax))
869 lex_preprocess_line (&lexer->line_buffer, syntax,
870 &line_starts_command, &lexer->dot);
872 if (line_starts_command)
873 lexer->put_token = '.';
875 lexer->prog = ds_cstr (&lexer->line_buffer);
881 /* Returns the name of a token. */
883 lex_token_name (int token)
885 if (lex_is_keyword (token))
886 return lex_id_name (token);
887 else if (token < 256)
889 static char t[256][2];
899 /* Returns an ASCII representation of the current token as a
900 malloc()'d string. */
902 lex_token_representation (struct lexer *lexer)
906 switch (lexer->token)
911 return ds_xstrdup (&lexer->tokstr);
919 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
920 if (!isprint ((unsigned char) *sp))
926 token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
934 for (sp = ds_cstr (&lexer->tokstr); *sp; )
938 *dp++ = (unsigned char) *sp++;
941 for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
943 *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
944 *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
954 token_rep = xmalloc (1);
959 return xstrdup ("**");
962 return xstrdup (lex_token_name (lexer->token));
968 /* Really weird functions. */
970 /* Most of the time, a `-' is a lead-in to a negative number. But
971 sometimes it's actually part of the syntax. If a dash can be part
972 of syntax then this function is called to rip it off of a
975 lex_negative_to_dash (struct lexer *lexer)
977 if (lexer->token == T_NEG_NUM)
979 lexer->token = T_POS_NUM;
980 lexer->tokval = -lexer->tokval;
981 ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
987 /* Skip a COMMENT command. */
989 lex_skip_comment (struct lexer *lexer)
993 if (!lex_get_line (lexer))
995 lexer->put_token = T_STOP;
1000 if (lexer->put_token == '.')
1003 ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
1004 lexer->prog = ds_end (&lexer->line_buffer);
1010 /* Private functions. */
1012 /* When invoked, tokstr contains a string of binary, octal, or
1013 hex digits, according to TYPE. The string is converted to
1014 characters having the specified values. */
1016 convert_numeric_string_to_char_string (struct lexer *lexer,
1017 enum string_type type)
1019 const char *base_name;
1029 base_name = _("binary");
1034 base_name = _("octal");
1039 base_name = _("hex");
1047 byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
1048 if (ds_length (&lexer->tokstr) % chars_per_byte)
1049 msg (SE, _("String of %s digits has %d characters, which is not a "
1051 base_name, ds_length (&lexer->tokstr), chars_per_byte);
1053 p = ds_cstr (&lexer->tokstr);
1054 for (i = 0; i < byte_cnt; i++)
1060 for (j = 0; j < chars_per_byte; j++, p++)
1064 if (*p >= '0' && *p <= '9')
1068 static const char alpha[] = "abcdef";
1069 const char *q = strchr (alpha, tolower ((unsigned char) *p));
1078 msg (SE, _("`%c' is not a valid %s digit."), *p, base_name);
1080 value = value * base + v;
1083 ds_cstr (&lexer->tokstr)[i] = (unsigned char) value;
1086 ds_truncate (&lexer->tokstr, byte_cnt);
1089 /* Parses a string from the input buffer into tokstr. The input
1090 buffer pointer lexer->prog must point to the initial single or double
1091 quote. TYPE indicates the type of string to be parsed.
1092 Returns token type. */
1094 parse_string (struct lexer *lexer, enum string_type type)
1096 if (type != CHARACTER_STRING)
1099 /* Accumulate the entire string, joining sections indicated by +
1103 /* Single or double quote. */
1104 int c = *lexer->prog++;
1106 /* Accumulate section. */
1109 /* Check end of line. */
1110 if (*lexer->prog == '\0')
1112 msg (SE, _("Unterminated string constant."));
1116 /* Double quote characters to embed them in strings. */
1117 if (*lexer->prog == c)
1119 if (lexer->prog[1] == c)
1125 ds_put_char (&lexer->tokstr, *lexer->prog++);
1129 /* Skip whitespace after final quote mark. */
1130 if (lexer->prog == NULL)
1134 while (isspace ((unsigned char) *lexer->prog))
1142 if (!lex_get_line (lexer))
1146 /* Skip plus sign. */
1147 if (*lexer->prog != '+')
1151 /* Skip whitespace after plus sign. */
1152 if (lexer->prog == NULL)
1156 while (isspace ((unsigned char) *lexer->prog))
1164 if (!lex_get_line (lexer))
1166 msg (SE, _("Unexpected end of file in string concatenation."));
1171 /* Ensure that a valid string follows. */
1172 if (*lexer->prog != '\'' && *lexer->prog != '"')
1174 msg (SE, _("String expected following `+'."));
1179 /* We come here when we've finished concatenating all the string sections
1180 into one large string. */
1182 if (type != CHARACTER_STRING)
1183 convert_numeric_string_to_char_string (lexer, type);
1185 if (ds_length (&lexer->tokstr) > 255)
1187 msg (SE, _("String exceeds 255 characters in length (%d characters)."),
1188 ds_length (&lexer->tokstr));
1189 ds_truncate (&lexer->tokstr, 255);
1196 /* Reads one token from the lexer and writes a textual representation
1197 on stdout for debugging purposes. */
1199 dump_token (struct lexer *lexer)
1205 curln = getl_source_location (lexer->ss);
1206 curfn = getl_source_name (lexer->ss);
1208 fprintf (stderr, "%s:%d\t", curfn, curln);
1211 switch (lexer->token)
1214 fprintf (stderr, "ID\t%s\n", lexer->tokid);
1219 fprintf (stderr, "NUM\t%f\n", lexer->tokval);
1223 fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
1227 fprintf (stderr, "STOP\n");
1231 fprintf (stderr, "MISC\tEXP\"");
1235 fprintf (stderr, "MISC\tEOF\n");
1239 if (lex_is_keyword (lexer->token))
1240 fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
1242 fprintf (stderr, "PUNCT\t%c\n", lexer->token);
1246 #endif /* DUMP_TOKENS */
1249 /* Token Accessor Functions */
1252 lex_token (const struct lexer *lexer)
1254 return lexer->token;
1258 lex_tokval (const struct lexer *lexer)
1260 return lexer->tokval;
1264 lex_tokid (const struct lexer *lexer)
1266 return lexer->tokid;
1269 const struct string *
1270 lex_tokstr (const struct lexer *lexer)
1272 return &lexer->tokstr;