X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=aa681c80306b65f0cfb7f2c72f3edfae479ccead;hb=774441e68b4d2e3a4b5c6975e9614dcd4369955e;hp=39b674c554d6891e037e43a97cc6d41b61a25f39;hpb=3816248a008a4af75aac6319d0c9929cb7ff679e;p=pspp diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c index 39b674c554..aa681c8030 100644 --- a/src/language/lexer/lexer.c +++ b/src/language/lexer/lexer.c @@ -1,72 +1,66 @@ -/* PSPP - computes sample statistics. - Copyright (C) 1997-9, 2000 Free Software Foundation, Inc. - Written by Ben Pfaff . +/* PSPP - a program for statistical analysis. + Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. */ + along with this program. If not, see . */ #include #include "lexer.h" #include -#include +#include +#include #include #include #include #include +#include #include -#include #include #include #include -#include -#include #include +#include #include +#include +#include -#include "size_max.h" +#include "xalloc.h" #include "gettext.h" #define _(msgid) gettext (msgid) #define N_(msgid) msgid -/* -#define DUMP_TOKENS 1 -*/ - -struct lexer +struct lexer { struct string line_buffer; - bool (*read_line) (struct string *, bool *); + struct source_stream *ss; int token; /* Current token. */ double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */ - char tokid [LONG_NAME_LEN + 1]; /* T_ID: the identifier. */ + char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */ struct string tokstr; /* T_ID, T_STRING: token string value. - For T_ID, this is not truncated as is + For T_ID, this is not truncated as is tokid. */ char *prog; /* Pointer to next token in line_buffer. */ bool dot; /* True only if this line ends with a terminal dot. */ - bool eof; /* True only if the last token returned was T_STOP. */ int put_token ; /* If nonzero, next token returned by lex_get(). Used only in exceptional circumstances. */ - + struct string put_tokstr; double put_tokval; }; @@ -75,7 +69,7 @@ struct lexer static int parse_id (struct lexer *); /* How a string represents its contents. */ -enum string_type +enum string_type { CHARACTER_STRING, /* Characters. */ BINARY_STRING, /* Binary digits. */ @@ -84,38 +78,53 @@ enum string_type }; static int parse_string (struct lexer *, enum string_type); - -#if DUMP_TOKENS -static void dump_token (void); -#endif /* Initialization. */ /* Initializes the lexer. */ struct lexer * -lex_create (bool (*read_line_func) (struct string *, bool *)) +lex_create (struct source_stream *ss) { struct lexer *lexer = xzalloc (sizeof (*lexer)); ds_init_empty (&lexer->tokstr); ds_init_empty (&lexer->put_tokstr); ds_init_empty (&lexer->line_buffer); - lexer->read_line = read_line_func; - - if (!lex_get_line (lexer)) - lexer->eof = true; + lexer->ss = ss; return lexer; } +struct source_stream * +lex_get_source_stream (const struct lexer *lex) +{ + return lex->ss; +} + +enum syntax_mode +lex_current_syntax_mode (const struct lexer *lex) +{ + return source_stream_current_syntax_mode (lex->ss); +} + +enum error_mode +lex_current_error_mode (const struct lexer *lex) +{ + return source_stream_current_error_mode (lex->ss); +} + + void lex_destroy (struct lexer *lexer) { - ds_destroy (&lexer->put_tokstr); - ds_destroy (&lexer->tokstr); - ds_destroy (&lexer->line_buffer); + if ( NULL != lexer ) + { + ds_destroy (&lexer->put_tokstr); + ds_destroy (&lexer->tokstr); + ds_destroy (&lexer->line_buffer); - free (lexer); + free (lexer); + } } @@ -124,7 +133,7 @@ lex_destroy (struct lexer *lexer) /* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr, tokval, respectively, and sets tokid appropriately. */ static void -restore_token (struct lexer *lexer) +restore_token (struct lexer *lexer) { assert (lexer->put_token != 0); lexer->token = lexer->put_token; @@ -137,7 +146,7 @@ restore_token (struct lexer *lexer) /* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr, put_lexer->tokval respectively. */ static void -save_token (struct lexer *lexer) +save_token (struct lexer *lexer) { lexer->put_token = lexer->token; ds_assign_string (&lexer->put_tokstr, &lexer->tokstr); @@ -149,30 +158,28 @@ save_token (struct lexer *lexer) void lex_get (struct lexer *lexer) { - /* If a token was pushed ahead, return it. */ - if (lexer->put_token) - { - restore_token (lexer); -#if DUMP_TOKENS - dump_token (); -#endif - return; - } - /* Find a token. */ for (;;) { - /* Skip whitespace. */ - if (lexer->eof) + if (NULL == lexer->prog && ! lex_get_line (lexer) ) + { + lexer->token = T_STOP; + return; + } + + /* If a token was pushed ahead, return it. */ + if (lexer->put_token) { - lexer->token = T_STOP; + restore_token (lexer); return; } for (;;) - { - while (isspace ((unsigned char) *lexer->prog)) + { + /* Skip whitespace. */ + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; + if (*lexer->prog) break; @@ -180,27 +187,18 @@ lex_get (struct lexer *lexer) { lexer->dot = 0; lexer->token = '.'; -#if DUMP_TOKENS - dump_token (); -#endif return; } else if (!lex_get_line (lexer)) { - lexer->eof = true; + lexer->prog = NULL; lexer->token = T_STOP; -#if DUMP_TOKENS - dump_token (); -#endif return; } if (lexer->put_token) { restore_token (lexer); -#if DUMP_TOKENS - dump_token (); -#endif return; } } @@ -208,7 +206,7 @@ lex_get (struct lexer *lexer) /* Actually parse the token. */ ds_clear (&lexer->tokstr); - + switch (*lexer->prog) { case '-': case '.': @@ -227,26 +225,26 @@ lex_get (struct lexer *lexer) if (*lexer->prog == '-') { ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; - if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.') + if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.') { lexer->token = '-'; break; } lexer->token = T_NEG_NUM; } - else + else lexer->token = T_POS_NUM; - + /* Parse the number, copying it into tokstr. */ - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '.') { ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); } if (*lexer->prog == 'e' || *lexer->prog == 'E') @@ -254,12 +252,12 @@ lex_get (struct lexer *lexer) ds_put_char (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '+' || *lexer->prog == '-') ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); } /* Parse as floating point. */ - lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail); + lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail); if (*tail) { msg (SE, _("%s does not form a valid number."), @@ -278,6 +276,7 @@ lex_get (struct lexer *lexer) break; case '(': case ')': case ',': case '=': case '+': case '/': + case '[': case ']': lexer->token = *lexer->prog++; break; @@ -342,62 +341,63 @@ lex_get (struct lexer *lexer) else lexer->token = parse_id (lexer); break; - + case 'o': case 'O': if (lexer->prog[1] == '\'' || lexer->prog[1] == '"') lexer->token = parse_string (lexer, OCTAL_STRING); else lexer->token = parse_id (lexer); break; - + case 'x': case 'X': if (lexer->prog[1] == '\'' || lexer->prog[1] == '"') lexer->token = parse_string (lexer, HEX_STRING); else lexer->token = parse_id (lexer); break; - + default: - if (lex_is_id1 (*lexer->prog)) + if (lex_is_id1 (*lexer->prog)) { lexer->token = parse_id (lexer); - break; + break; } else { - if (isgraph ((unsigned char) *lexer->prog)) - msg (SE, _("Bad character in input: `%c'."), *lexer->prog++); - else - msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++); - continue; + unsigned char c = *lexer->prog++; + char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c); + msg (SE, _("Bad character in input: `%s'."), c_name); + free (c_name); + continue; } } break; } - -#if DUMP_TOKENS - dump_token (); -#endif } /* Parses an identifier at the current position into tokid and tokstr. Returns the correct token type. */ static int -parse_id (struct lexer *lexer) +parse_id (struct lexer *lexer) { - const char *start = lexer->prog; - lexer->prog = lex_skip_identifier (start); - - ds_put_substring (&lexer->tokstr, ss_buffer (start, lexer->prog - start)); + struct substring rest_of_line + = ss_substr (ds_ss (&lexer->line_buffer), + ds_pointer_to_position (&lexer->line_buffer, lexer->prog), + SIZE_MAX); + struct substring id = ss_head (rest_of_line, + lex_id_get_length (rest_of_line)); + lexer->prog += ss_length (id); + + ds_assign_substring (&lexer->tokstr, id); str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr)); - return lex_id_to_token (ds_cstr (&lexer->tokstr), ds_length (&lexer->tokstr)); + return lex_id_to_token (id); } /* Reports an error to the effect that subcommand SBC may only be specified once. */ void -lex_sbc_only_once (const char *sbc) +lex_sbc_only_once (const char *sbc) { msg (SE, _("Subcommand %s may only be specified once."), sbc); } @@ -405,7 +405,7 @@ lex_sbc_only_once (const char *sbc) /* Reports an error to the effect that subcommand SBC is missing. */ void -lex_sbc_missing (struct lexer *lexer, const char *sbc) +lex_sbc_missing (struct lexer *lexer, const char *sbc) { lex_error (lexer, _("missing required subcommand %s"), sbc); } @@ -415,31 +415,34 @@ lex_sbc_missing (struct lexer *lexer, const char *sbc) void lex_error (struct lexer *lexer, const char *message, ...) { - char *token_rep; - char where[128]; + struct string s; + + ds_init_empty (&s); - token_rep = lex_token_representation (lexer); if (lexer->token == T_STOP) - strcpy (where, "end of file"); + ds_put_cstr (&s, _("Syntax error at end of file")); else if (lexer->token == '.') - strcpy (where, "end of command"); + ds_put_cstr (&s, _("Syntax error at end of command")); else - snprintf (where, sizeof where, "`%s'", token_rep); - free (token_rep); + { + char *token_rep = lex_token_representation (lexer); + ds_put_format (&s, _("Syntax error at `%s'"), token_rep); + free (token_rep); + } if (message) { - char buf[1024]; va_list args; - + + ds_put_cstr (&s, ": "); + va_start (args, message); - vsnprintf (buf, 1024, message, args); + ds_put_vformat (&s, message, args); va_end (args); - - msg (SE, _("Syntax error %s at %s."), buf, where); } - else - msg (SE, _("Syntax error at %s."), where); + + msg (SE, "%s.", ds_cstr (&s)); + ds_destroy (&s); } /* Checks that we're at end of command. @@ -462,11 +465,20 @@ lex_end_of_command (struct lexer *lexer) /* Returns true if the current token is a number. */ bool -lex_is_number (struct lexer *lexer) +lex_is_number (struct lexer *lexer) { return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM; } + +/* Returns true if the current token is a string. */ +bool +lex_is_string (struct lexer *lexer) +{ + return lexer->token == T_STRING; +} + + /* Returns the value of the current token, which must be a floating point number. */ double @@ -481,8 +493,7 @@ bool lex_is_integer (struct lexer *lexer) { return (lex_is_number (lexer) - && lexer->tokval != NOT_LONG - && lexer->tokval >= LONG_MIN + && lexer->tokval > LONG_MIN && lexer->tokval <= LONG_MAX && floor (lexer->tokval) == lexer->tokval); } @@ -495,7 +506,7 @@ lex_integer (struct lexer *lexer) assert (lex_is_integer (lexer)); return lexer->tokval; } - + /* Token matching functions. */ /* If TOK is the current token, skips it and returns true @@ -519,7 +530,18 @@ lex_match (struct lexer *lexer, int t) bool lex_match_id (struct lexer *lexer, const char *s) { - if (lexer->token == T_ID && lex_id_match (s, lexer->tokid)) + return lex_match_id_n (lexer, s, 3); +} + +/* If the current token is the identifier S, skips it and returns + true. The identifier may be abbreviated to its first N + letters. + Otherwise, returns false. */ +bool +lex_match_id_n (struct lexer *lexer, const char *s, size_t n) +{ + if (lexer->token == T_ID + && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n)) { lex_get (lexer); return true; @@ -550,11 +572,8 @@ lex_match_int (struct lexer *lexer, int x) bool lex_force_match_id (struct lexer *lexer, const char *s) { - if (lexer->token == T_ID && lex_id_match (s, lexer->tokid)) - { - lex_get (lexer); - return true; - } + if (lex_match_id (lexer, s)) + return true; else { lex_error (lexer, _("expecting `%s'"), s); @@ -584,7 +603,7 @@ lex_force_match (struct lexer *lexer, int t) bool lex_force_string (struct lexer *lexer) { - if (lexer->token == T_STRING) + if (lex_is_string (lexer)) return true; else { @@ -606,7 +625,7 @@ lex_force_int (struct lexer *lexer) return false; } } - + /* If this token is a number, does nothing and returns true. Otherwise, reports an error and returns false. */ bool @@ -618,7 +637,7 @@ lex_force_num (struct lexer *lexer) lex_error (lexer, _("expecting number")); return false; } - + /* If this token is an identifier, does nothing and returns true. Otherwise, reports an error and returns false. */ bool @@ -647,12 +666,12 @@ lex_look_ahead (struct lexer *lexer) for (;;) { - if (lexer->eof) + if (NULL == lexer->prog && ! lex_get_line (lexer) ) return 0; for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -662,7 +681,7 @@ lex_look_ahead (struct lexer *lexer) else if (!lex_get_line (lexer)) return 0; - if (lexer->put_token) + if (lexer->put_token) return lexer->put_token; } @@ -690,7 +709,7 @@ lex_put_back (struct lexer *lexer, int t) void lex_put_back_id (struct lexer *lexer, const char *id) { - assert (lex_id_to_token (id, strlen (id)) == T_ID); + assert (lex_id_to_token (ss_cstr (id)) == T_ID); save_token (lexer); lexer->token = T_ID; ds_assign_cstr (&lexer->tokstr, id); @@ -701,29 +720,33 @@ lex_put_back_id (struct lexer *lexer, const char *id) /* Returns the entire contents of the current line. */ const char * -lex_entire_line (struct lexer *lexer) +lex_entire_line (const struct lexer *lexer) { return ds_cstr (&lexer->line_buffer); } const struct string * -lex_entire_line_ds (struct lexer *lexer) +lex_entire_line_ds (const struct lexer *lexer) { return &lexer->line_buffer; } /* As lex_entire_line(), but only returns the part of the current line - that hasn't already been tokenized. - If END_DOT is non-null, stores nonzero into *END_DOT if the line - ends with a terminal dot, or zero if it doesn't. */ + that hasn't already been tokenized. */ const char * -lex_rest_of_line (struct lexer *lexer, int *end_dot) +lex_rest_of_line (const struct lexer *lexer) { - if (end_dot) - *end_dot = lexer->dot; return lexer->prog; } +/* Returns true if the current line ends in a terminal dot, + false otherwise. */ +bool +lex_end_dot (const struct lexer *lexer) +{ + return lexer->dot; +} + /* Causes the rest of the current input line to be ignored for tokenization purposes. */ void @@ -744,15 +767,15 @@ lex_discard_line (struct lexer *lexer) the user doesn't want to finish typing a command that will be ignored anyway. */ void -lex_discard_rest_of_command (struct lexer *lexer) +lex_discard_rest_of_command (struct lexer *lexer) { - if (!getl_is_interactive ()) + if (!getl_is_interactive (lexer->ss)) { while (lexer->token != T_STOP && lexer->token != '.') lex_get (lexer); } - else - lex_discard_line (lexer); + else + lex_discard_line (lexer); } /* Weird line reading functions. */ @@ -778,7 +801,7 @@ strip_comments (struct string *string) else if (*cp == '\'' || *cp == '"') quote = *cp; } - + /* If we're not inside a quotation, check for comment. */ if (quote == EOF) { @@ -797,7 +820,7 @@ strip_comments (struct string *string) continue; } } - + /* Check commenting. */ if (in_comment) *cp = ' '; @@ -805,12 +828,41 @@ strip_comments (struct string *string) } } -/* Reads a line, without performing any preprocessing */ -bool +/* Prepares LINE, which is subject to the given SYNTAX rules, for + tokenization by stripping comments and determining whether it + is the beginning or end of a command and storing into + *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */ +void +lex_preprocess_line (struct string *line, + enum syntax_mode syntax, + bool *line_starts_command, + bool *line_ends_command) +{ + strip_comments (line); + ds_rtrim (line, ss_cstr (CC_SPACES)); + *line_ends_command = (ds_chomp (line, settings_get_endcmd ()) + || (ds_is_empty (line) && settings_get_nulline ())); + *line_starts_command = false; + if (syntax == GETL_BATCH) + { + int first = ds_first (line); + *line_starts_command = !c_isspace (first); + if (first == '+' || first == '-') + *ds_data (line) = ' '; + } +} + +/* Reads a line, without performing any preprocessing. */ +bool lex_get_line_raw (struct lexer *lexer) { - bool dummy; - return lexer->read_line (&lexer->line_buffer, &dummy); + bool ok = getl_read_line (lexer->ss, &lexer->line_buffer); + if (ok) + { + const char *line = ds_cstr (&lexer->line_buffer); + text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line)); + } + return ok; } /* Reads a line for use by the tokenizer, and preprocesses it by @@ -819,53 +871,43 @@ lex_get_line_raw (struct lexer *lexer) bool lex_get_line (struct lexer *lexer) { - struct string *line = &lexer->line_buffer; - bool interactive; + bool line_starts_command; - if (!lexer->read_line (line, &interactive)) - return false; - - strip_comments (line); - ds_rtrim (line, ss_cstr (CC_SPACES)); - - /* Check for and remove terminal dot. */ - lexer->dot = (ds_chomp (line, get_endcmd ()) - || (ds_is_empty (line) && get_nulline ())); - - /* Strip leading indentors or insert a terminal dot (unless the - line was obtained interactively). */ - if (!interactive) + if (!lex_get_line_raw (lexer)) { - int first = ds_first (line); - - if (first == '+' || first == '-') - *ds_data (line) = ' '; - else if (first != EOF && !isspace (first)) - lexer->put_token = '.'; + lexer->prog = NULL; + return false; } - lexer->prog = ds_cstr (line); + lex_preprocess_line (&lexer->line_buffer, + lex_current_syntax_mode (lexer), + &line_starts_command, &lexer->dot); + if (line_starts_command) + lexer->put_token = '.'; + + lexer->prog = ds_cstr (&lexer->line_buffer); return true; } /* Token names. */ -/* Returns the name of a token in a static buffer. */ +/* Returns the name of a token. */ const char * lex_token_name (int token) { - if (token >= T_FIRST_KEYWORD && token <= T_LAST_KEYWORD) - return keywords[token - T_FIRST_KEYWORD]; - - if (token < 256) + if (lex_is_keyword (token)) + return lex_id_name (token); + else if (token < 256) { - static char t[2]; - t[0] = token; - return t; + static char t[256][2]; + char *s = t[token]; + s[0] = token; + s[1] = '\0'; + return s; } - - NOT_REACHED (); + else + NOT_REACHED (); } /* Returns an ASCII representation of the current token as a @@ -874,7 +916,7 @@ char * lex_token_representation (struct lexer *lexer) { char *token_rep; - + switch (lexer->token) { case T_ID: @@ -889,12 +931,12 @@ lex_token_representation (struct lexer *lexer) char *sp, *dp; for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++) - if (!isprint ((unsigned char) *sp)) + if (!c_isprint ((unsigned char) *sp)) { hexstring = 1; break; } - + token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1); dp = token_rep; @@ -917,7 +959,7 @@ lex_token_representation (struct lexer *lexer) } *dp++ = '\''; *dp = '\0'; - + return token_rep; } break; @@ -931,17 +973,9 @@ lex_token_representation (struct lexer *lexer) return xstrdup ("**"); default: - if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD) - return xstrdup (keywords [lexer->token - T_FIRST_KEYWORD]); - else - { - token_rep = xmalloc (2); - token_rep[0] = lexer->token; - token_rep[1] = '\0'; - return token_rep; - } + return xstrdup (lex_token_name (lexer->token)); } - + NOT_REACHED (); } @@ -963,13 +997,6 @@ lex_negative_to_dash (struct lexer *lexer) lexer->token = '-'; } } - -/* We're not at eof any more. */ -void -lex_reset_eof (struct lexer *lexer) -{ - lexer->eof = false; -} /* Skip a COMMENT command. */ void @@ -977,13 +1004,13 @@ lex_skip_comment (struct lexer *lexer) { for (;;) { - if (!lex_get_line (lexer)) + if (!lex_get_line (lexer)) { lexer->put_token = T_STOP; - lexer->eof = true; + lexer->prog = NULL; return; } - + if (lexer->put_token == '.') break; @@ -1000,7 +1027,7 @@ lex_skip_comment (struct lexer *lexer) hex digits, according to TYPE. The string is converted to characters having the specified values. */ static void -convert_numeric_string_to_char_string (struct lexer *lexer, +convert_numeric_string_to_char_string (struct lexer *lexer, enum string_type type) { const char *base_name; @@ -1010,7 +1037,7 @@ convert_numeric_string_to_char_string (struct lexer *lexer, size_t i; char *p; - switch (type) + switch (type) { case BINARY_STRING: base_name = _("binary"); @@ -1030,10 +1057,10 @@ convert_numeric_string_to_char_string (struct lexer *lexer, default: NOT_REACHED (); } - + byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte; if (ds_length (&lexer->tokstr) % chars_per_byte) - msg (SE, _("String of %s digits has %d characters, which is not a " + msg (SE, _("String of %s digits has %zu characters, which is not a " "multiple of %d."), base_name, ds_length (&lexer->tokstr), chars_per_byte); @@ -1042,7 +1069,7 @@ convert_numeric_string_to_char_string (struct lexer *lexer, { int value; int j; - + value = 0; for (j = 0; j < chars_per_byte; j++, p++) { @@ -1077,7 +1104,7 @@ convert_numeric_string_to_char_string (struct lexer *lexer, buffer pointer lexer->prog must point to the initial single or double quote. TYPE indicates the type of string to be parsed. Returns token type. */ -static int +static int parse_string (struct lexer *lexer, enum string_type type) { if (type != CHARACTER_STRING) @@ -1089,7 +1116,7 @@ parse_string (struct lexer *lexer, enum string_type type) { /* Single or double quote. */ int c = *lexer->prog++; - + /* Accumulate section. */ for (;;) { @@ -1099,7 +1126,7 @@ parse_string (struct lexer *lexer, enum string_type type) msg (SE, _("Unterminated string constant.")); goto finish; } - + /* Double quote characters to embed them in strings. */ if (*lexer->prog == c) { @@ -1114,11 +1141,11 @@ parse_string (struct lexer *lexer, enum string_type type) lexer->prog++; /* Skip whitespace after final quote mark. */ - if (lexer->eof) + if (lexer->prog == NULL) break; for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -1136,11 +1163,11 @@ parse_string (struct lexer *lexer, enum string_type type) lexer->prog++; /* Skip whitespace after plus sign. */ - if (lexer->eof) + if (lexer->prog == NULL) break; for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -1169,78 +1196,18 @@ finish: if (type != CHARACTER_STRING) convert_numeric_string_to_char_string (lexer, type); - if (ds_length (&lexer->tokstr) > 255) - { - msg (SE, _("String exceeds 255 characters in length (%d characters)."), - ds_length (&lexer->tokstr)); - ds_truncate (&lexer->tokstr, 255); - } - return T_STRING; } - -#if DUMP_TOKENS -/* Reads one token from the lexer and writes a textual representation - on stdout for debugging purposes. */ -static void -dump_token (struct lexer *lexer) -{ - { - const char *curfn; - int curln; - - getl_location (&curfn, &curln); - if (curfn) - fprintf (stderr, "%s:%d\t", curfn, curln); - } - - switch (lexer->token) - { - case T_ID: - fprintf (stderr, "ID\t%s\n", lexer->tokid); - break; - - case T_POS_NUM: - case T_NEG_NUM: - fprintf (stderr, "NUM\t%f\n", lexer->tokval); - break; - - case T_STRING: - fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr)); - break; - - case T_STOP: - fprintf (stderr, "STOP\n"); - break; - - case T_EXP: - fprintf (stderr, "MISC\tEXP\""); - break; - - case 0: - fprintf (stderr, "MISC\tEOF\n"); - break; - - default: - if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD) - fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (token)); - else - fprintf (stderr, "PUNCT\t%c\n", lexer->token); - break; - } -} -#endif /* DUMP_TOKENS */ - - + /* Token Accessor Functions */ -int +int lex_token (const struct lexer *lexer) { return lexer->token; } -double +double lex_tokval (const struct lexer *lexer) { return lexer->tokval; @@ -1257,3 +1224,28 @@ lex_tokstr (const struct lexer *lexer) { return &lexer->tokstr; } + +/* If the lexer is positioned at the (pseudo)identifier S, which + may contain a hyphen ('-'), skips it and returns true. Each + half of the identifier may be abbreviated to its first three + letters. + Otherwise, returns false. */ +bool +lex_match_hyphenated_word (struct lexer *lexer, const char *s) +{ + const char *hyphen = strchr (s, '-'); + if (hyphen == NULL) + return lex_match_id (lexer, s); + else if (lexer->token != T_ID + || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid)) + || lex_look_ahead (lexer) != '-') + return false; + else + { + lex_get (lexer); + lex_force_match (lexer, '-'); + lex_force_match_id (lexer, hyphen + 1); + return true; + } +} +