-/* PSPP - computes sample statistics.
- Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
- Written by Ben Pfaff <blp@gnu.org>.
+/* PSPP - a program for statistical analysis.
+ Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc.
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of the
- License, or (at your option) any later version.
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
- This program is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- 02110-1301, USA. */
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
#include <config.h>
#include "lexer.h"
#include <libpspp/message.h>
-#include <ctype.h>
+#include <c-ctype.h>
+#include <c-strtod.h>
#include <errno.h>
#include <limits.h>
#include <math.h>
#include <stdarg.h>
+#include <stdint.h>
#include <stdlib.h>
-#include <libpspp/alloc.h>
#include <libpspp/assertion.h>
#include <language/command.h>
#include <libpspp/message.h>
-#include <libpspp/magic.h>
#include <data/settings.h>
#include <libpspp/getl.h>
#include <libpspp/str.h>
+#include <output/journal.h>
+#include <output/text-item.h>
-#include "size_max.h"
+#include "xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
#define N_(msgid) msgid
-/*
-#define DUMP_TOKENS 1
-*/
-
-struct lexer
+struct lexer
{
struct string line_buffer;
- bool (*read_line) (struct string *, bool *);
+ struct source_stream *ss;
int token; /* Current token. */
double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
- char tokid [LONG_NAME_LEN + 1]; /* T_ID: the identifier. */
+ char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
struct string tokstr; /* T_ID, T_STRING: token string value.
- For T_ID, this is not truncated as is
+ For T_ID, this is not truncated as is
tokid. */
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
- bool eof; /* True only if the last token returned was T_STOP. */
int put_token ; /* If nonzero, next token returned by lex_get().
Used only in exceptional circumstances. */
-
+
struct string put_tokstr;
double put_tokval;
};
static int parse_id (struct lexer *);
/* How a string represents its contents. */
-enum string_type
+enum string_type
{
CHARACTER_STRING, /* Characters. */
BINARY_STRING, /* Binary digits. */
};
static int parse_string (struct lexer *, enum string_type);
-
-#if DUMP_TOKENS
-static void dump_token (void);
-#endif
\f
/* Initialization. */
/* Initializes the lexer. */
struct lexer *
-lex_create (bool (*read_line_func) (struct string *, bool *))
+lex_create (struct source_stream *ss)
{
struct lexer *lexer = xzalloc (sizeof (*lexer));
ds_init_empty (&lexer->tokstr);
ds_init_empty (&lexer->put_tokstr);
ds_init_empty (&lexer->line_buffer);
- lexer->read_line = read_line_func;
-
- if (!lex_get_line (lexer))
- lexer->eof = true;
+ lexer->ss = ss;
return lexer;
}
+struct source_stream *
+lex_get_source_stream (const struct lexer *lex)
+{
+ return lex->ss;
+}
+
+enum syntax_mode
+lex_current_syntax_mode (const struct lexer *lex)
+{
+ return source_stream_current_syntax_mode (lex->ss);
+}
+
+enum error_mode
+lex_current_error_mode (const struct lexer *lex)
+{
+ return source_stream_current_error_mode (lex->ss);
+}
+
+
void
lex_destroy (struct lexer *lexer)
{
- if ( NULL != lexer )
+ if ( NULL != lexer )
{
ds_destroy (&lexer->put_tokstr);
ds_destroy (&lexer->tokstr);
/* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
tokval, respectively, and sets tokid appropriately. */
static void
-restore_token (struct lexer *lexer)
+restore_token (struct lexer *lexer)
{
assert (lexer->put_token != 0);
lexer->token = lexer->put_token;
/* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
put_lexer->tokval respectively. */
static void
-save_token (struct lexer *lexer)
+save_token (struct lexer *lexer)
{
lexer->put_token = lexer->token;
ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
void
lex_get (struct lexer *lexer)
{
- /* If a token was pushed ahead, return it. */
- if (lexer->put_token)
- {
- restore_token (lexer);
-#if DUMP_TOKENS
- dump_token ();
-#endif
- return;
- }
-
/* Find a token. */
for (;;)
{
- /* Skip whitespace. */
- if (lexer->eof)
+ if (NULL == lexer->prog && ! lex_get_line (lexer) )
+ {
+ lexer->token = T_STOP;
+ return;
+ }
+
+ /* If a token was pushed ahead, return it. */
+ if (lexer->put_token)
{
- lexer->token = T_STOP;
+ restore_token (lexer);
return;
}
for (;;)
- {
- while (isspace ((unsigned char) *lexer->prog))
+ {
+ /* Skip whitespace. */
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
+
if (*lexer->prog)
break;
{
lexer->dot = 0;
lexer->token = '.';
-#if DUMP_TOKENS
- dump_token ();
-#endif
return;
}
else if (!lex_get_line (lexer))
{
- lexer->eof = true;
+ lexer->prog = NULL;
lexer->token = T_STOP;
-#if DUMP_TOKENS
- dump_token ();
-#endif
return;
}
if (lexer->put_token)
{
restore_token (lexer);
-#if DUMP_TOKENS
- dump_token ();
-#endif
return;
}
}
/* Actually parse the token. */
ds_clear (&lexer->tokstr);
-
+
switch (*lexer->prog)
{
case '-': case '.':
if (*lexer->prog == '-')
{
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
- if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
+ if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
{
lexer->token = '-';
break;
}
lexer->token = T_NEG_NUM;
}
- else
+ else
lexer->token = T_POS_NUM;
-
+
/* Parse the number, copying it into tokstr. */
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '.')
{
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
}
if (*lexer->prog == 'e' || *lexer->prog == 'E')
ds_put_char (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '+' || *lexer->prog == '-')
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
}
/* Parse as floating point. */
- lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
+ lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail);
if (*tail)
{
msg (SE, _("%s does not form a valid number."),
break;
case '(': case ')': case ',': case '=': case '+': case '/':
+ case '[': case ']':
lexer->token = *lexer->prog++;
break;
else
lexer->token = parse_id (lexer);
break;
-
+
case 'o': case 'O':
if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
lexer->token = parse_string (lexer, OCTAL_STRING);
else
lexer->token = parse_id (lexer);
break;
-
+
case 'x': case 'X':
if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
lexer->token = parse_string (lexer, HEX_STRING);
else
lexer->token = parse_id (lexer);
break;
-
+
default:
- if (lex_is_id1 (*lexer->prog))
+ if (lex_is_id1 (*lexer->prog))
{
lexer->token = parse_id (lexer);
- break;
+ break;
}
else
{
- if (isgraph ((unsigned char) *lexer->prog))
- msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
- else
- msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
- continue;
+ unsigned char c = *lexer->prog++;
+ char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c);
+ msg (SE, _("Bad character in input: `%s'."), c_name);
+ free (c_name);
+ continue;
}
}
break;
}
-
-#if DUMP_TOKENS
- dump_token ();
-#endif
}
/* Parses an identifier at the current position into tokid and
tokstr.
Returns the correct token type. */
static int
-parse_id (struct lexer *lexer)
+parse_id (struct lexer *lexer)
{
- const char *start = lexer->prog;
- lexer->prog = lex_skip_identifier (start);
-
- ds_put_substring (&lexer->tokstr, ss_buffer (start, lexer->prog - start));
+ struct substring rest_of_line
+ = ss_substr (ds_ss (&lexer->line_buffer),
+ ds_pointer_to_position (&lexer->line_buffer, lexer->prog),
+ SIZE_MAX);
+ struct substring id = ss_head (rest_of_line,
+ lex_id_get_length (rest_of_line));
+ lexer->prog += ss_length (id);
+
+ ds_assign_substring (&lexer->tokstr, id);
str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
- return lex_id_to_token (ds_cstr (&lexer->tokstr), ds_length (&lexer->tokstr));
+ return lex_id_to_token (id);
}
/* Reports an error to the effect that subcommand SBC may only be
specified once. */
void
-lex_sbc_only_once (const char *sbc)
+lex_sbc_only_once (const char *sbc)
{
msg (SE, _("Subcommand %s may only be specified once."), sbc);
}
/* Reports an error to the effect that subcommand SBC is
missing. */
void
-lex_sbc_missing (struct lexer *lexer, const char *sbc)
+lex_sbc_missing (struct lexer *lexer, const char *sbc)
{
lex_error (lexer, _("missing required subcommand %s"), sbc);
}
void
lex_error (struct lexer *lexer, const char *message, ...)
{
- char *token_rep;
- char where[128];
+ struct string s;
+
+ ds_init_empty (&s);
- token_rep = lex_token_representation (lexer);
if (lexer->token == T_STOP)
- strcpy (where, "end of file");
+ ds_put_cstr (&s, _("Syntax error at end of file"));
else if (lexer->token == '.')
- strcpy (where, "end of command");
+ ds_put_cstr (&s, _("Syntax error at end of command"));
else
- snprintf (where, sizeof where, "`%s'", token_rep);
- free (token_rep);
+ {
+ char *token_rep = lex_token_representation (lexer);
+ ds_put_format (&s, _("Syntax error at `%s'"), token_rep);
+ free (token_rep);
+ }
if (message)
{
- char buf[1024];
va_list args;
-
+
+ ds_put_cstr (&s, ": ");
+
va_start (args, message);
- vsnprintf (buf, 1024, message, args);
+ ds_put_vformat (&s, message, args);
va_end (args);
-
- msg (SE, _("Syntax error %s at %s."), buf, where);
}
- else
- msg (SE, _("Syntax error at %s."), where);
+
+ msg (SE, "%s.", ds_cstr (&s));
+ ds_destroy (&s);
}
/* Checks that we're at end of command.
/* Returns true if the current token is a number. */
bool
-lex_is_number (struct lexer *lexer)
+lex_is_number (struct lexer *lexer)
{
return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
}
+
+/* Returns true if the current token is a string. */
+bool
+lex_is_string (struct lexer *lexer)
+{
+ return lexer->token == T_STRING;
+}
+
+
/* Returns the value of the current token, which must be a
floating point number. */
double
lex_is_integer (struct lexer *lexer)
{
return (lex_is_number (lexer)
- && lexer->tokval != NOT_LONG
- && lexer->tokval >= LONG_MIN
+ && lexer->tokval > LONG_MIN
&& lexer->tokval <= LONG_MAX
&& floor (lexer->tokval) == lexer->tokval);
}
assert (lex_is_integer (lexer));
return lexer->tokval;
}
-\f
+\f
/* Token matching functions. */
/* If TOK is the current token, skips it and returns true
bool
lex_match_id (struct lexer *lexer, const char *s)
{
- if (lexer->token == T_ID && lex_id_match (s, lexer->tokid))
+ return lex_match_id_n (lexer, s, 3);
+}
+
+/* If the current token is the identifier S, skips it and returns
+ true. The identifier may be abbreviated to its first N
+ letters.
+ Otherwise, returns false. */
+bool
+lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
+{
+ if (lexer->token == T_ID
+ && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
{
lex_get (lexer);
return true;
bool
lex_force_match_id (struct lexer *lexer, const char *s)
{
- if (lexer->token == T_ID && lex_id_match (s, lexer->tokid))
- {
- lex_get (lexer);
- return true;
- }
+ if (lex_match_id (lexer, s))
+ return true;
else
{
lex_error (lexer, _("expecting `%s'"), s);
return false;
}
}
-
+
/* If this token is a number, does nothing and returns true.
Otherwise, reports an error and returns false. */
bool
lex_error (lexer, _("expecting number"));
return false;
}
-
+
/* If this token is an identifier, does nothing and returns true.
Otherwise, reports an error and returns false. */
bool
for (;;)
{
- if (lexer->eof)
+ if (NULL == lexer->prog && ! lex_get_line (lexer) )
return 0;
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
else if (!lex_get_line (lexer))
return 0;
- if (lexer->put_token)
+ if (lexer->put_token)
return lexer->put_token;
}
void
lex_put_back_id (struct lexer *lexer, const char *id)
{
- assert (lex_id_to_token (id, strlen (id)) == T_ID);
+ assert (lex_id_to_token (ss_cstr (id)) == T_ID);
save_token (lexer);
lexer->token = T_ID;
ds_assign_cstr (&lexer->tokstr, id);
/* Returns the entire contents of the current line. */
const char *
-lex_entire_line (struct lexer *lexer)
+lex_entire_line (const struct lexer *lexer)
{
return ds_cstr (&lexer->line_buffer);
}
const struct string *
-lex_entire_line_ds (struct lexer *lexer)
+lex_entire_line_ds (const struct lexer *lexer)
{
return &lexer->line_buffer;
}
/* As lex_entire_line(), but only returns the part of the current line
- that hasn't already been tokenized.
- If END_DOT is non-null, stores nonzero into *END_DOT if the line
- ends with a terminal dot, or zero if it doesn't. */
+ that hasn't already been tokenized. */
const char *
-lex_rest_of_line (struct lexer *lexer, int *end_dot)
+lex_rest_of_line (const struct lexer *lexer)
{
- if (end_dot)
- *end_dot = lexer->dot;
return lexer->prog;
}
+/* Returns true if the current line ends in a terminal dot,
+ false otherwise. */
+bool
+lex_end_dot (const struct lexer *lexer)
+{
+ return lexer->dot;
+}
+
/* Causes the rest of the current input line to be ignored for
tokenization purposes. */
void
the user doesn't want to finish typing a command that will be
ignored anyway. */
void
-lex_discard_rest_of_command (struct lexer *lexer)
+lex_discard_rest_of_command (struct lexer *lexer)
{
- if (!getl_is_interactive ())
+ if (!getl_is_interactive (lexer->ss))
{
while (lexer->token != T_STOP && lexer->token != '.')
lex_get (lexer);
}
- else
- lex_discard_line (lexer);
+ else
+ lex_discard_line (lexer);
}
\f
/* Weird line reading functions. */
else if (*cp == '\'' || *cp == '"')
quote = *cp;
}
-
+
/* If we're not inside a quotation, check for comment. */
if (quote == EOF)
{
continue;
}
}
-
+
/* Check commenting. */
if (in_comment)
*cp = ' ';
}
}
-/* Reads a line, without performing any preprocessing */
-bool
+/* Prepares LINE, which is subject to the given SYNTAX rules, for
+ tokenization by stripping comments and determining whether it
+ is the beginning or end of a command and storing into
+ *LINE_STARTS_COMMAND and *LINE_ENDS_COMMAND appropriately. */
+void
+lex_preprocess_line (struct string *line,
+ enum syntax_mode syntax,
+ bool *line_starts_command,
+ bool *line_ends_command)
+{
+ strip_comments (line);
+ ds_rtrim (line, ss_cstr (CC_SPACES));
+ *line_ends_command = (ds_chomp (line, settings_get_endcmd ())
+ || (ds_is_empty (line) && settings_get_nulline ()));
+ *line_starts_command = false;
+ if (syntax == GETL_BATCH)
+ {
+ int first = ds_first (line);
+ *line_starts_command = !c_isspace (first);
+ if (first == '+' || first == '-')
+ *ds_data (line) = ' ';
+ }
+}
+
+/* Reads a line, without performing any preprocessing. */
+bool
lex_get_line_raw (struct lexer *lexer)
{
- bool dummy;
- return lexer->read_line (&lexer->line_buffer, &dummy);
+ bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
+ if (ok)
+ {
+ const char *line = ds_cstr (&lexer->line_buffer);
+ text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line));
+ }
+ return ok;
}
/* Reads a line for use by the tokenizer, and preprocesses it by
bool
lex_get_line (struct lexer *lexer)
{
- struct string *line = &lexer->line_buffer;
- bool interactive;
-
- if (!lexer->read_line (line, &interactive))
- return false;
+ bool line_starts_command;
- strip_comments (line);
- ds_rtrim (line, ss_cstr (CC_SPACES));
-
- /* Check for and remove terminal dot. */
- lexer->dot = (ds_chomp (line, get_endcmd ())
- || (ds_is_empty (line) && get_nulline ()));
-
- /* Strip leading indentors or insert a terminal dot (unless the
- line was obtained interactively). */
- if (!interactive)
+ if (!lex_get_line_raw (lexer))
{
- int first = ds_first (line);
-
- if (first == '+' || first == '-')
- *ds_data (line) = ' ';
- else if (first != EOF && !isspace (first))
- lexer->put_token = '.';
+ lexer->prog = NULL;
+ return false;
}
- lexer->prog = ds_cstr (line);
+ lex_preprocess_line (&lexer->line_buffer,
+ lex_current_syntax_mode (lexer),
+ &line_starts_command, &lexer->dot);
+
+ if (line_starts_command)
+ lexer->put_token = '.';
+ lexer->prog = ds_cstr (&lexer->line_buffer);
return true;
}
\f
/* Token names. */
-/* Returns the name of a token in a static buffer. */
+/* Returns the name of a token. */
const char *
lex_token_name (int token)
{
- if (token >= T_FIRST_KEYWORD && token <= T_LAST_KEYWORD)
- return keywords[token - T_FIRST_KEYWORD];
-
- if (token < 256)
+ if (lex_is_keyword (token))
+ return lex_id_name (token);
+ else if (token < 256)
{
- static char t[2];
- t[0] = token;
- return t;
+ static char t[256][2];
+ char *s = t[token];
+ s[0] = token;
+ s[1] = '\0';
+ return s;
}
-
- NOT_REACHED ();
+ else
+ NOT_REACHED ();
}
/* Returns an ASCII representation of the current token as a
lex_token_representation (struct lexer *lexer)
{
char *token_rep;
-
+
switch (lexer->token)
{
case T_ID:
char *sp, *dp;
for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
- if (!isprint ((unsigned char) *sp))
+ if (!c_isprint ((unsigned char) *sp))
{
hexstring = 1;
break;
}
-
+
token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
dp = token_rep;
}
*dp++ = '\'';
*dp = '\0';
-
+
return token_rep;
}
break;
return xstrdup ("**");
default:
- if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD)
- return xstrdup (keywords [lexer->token - T_FIRST_KEYWORD]);
- else
- {
- token_rep = xmalloc (2);
- token_rep[0] = lexer->token;
- token_rep[1] = '\0';
- return token_rep;
- }
+ return xstrdup (lex_token_name (lexer->token));
}
-
+
NOT_REACHED ();
}
\f
lexer->token = '-';
}
}
-
-/* We're not at eof any more. */
-void
-lex_reset_eof (struct lexer *lexer)
-{
- lexer->eof = false;
-}
/* Skip a COMMENT command. */
void
{
for (;;)
{
- if (!lex_get_line (lexer))
+ if (!lex_get_line (lexer))
{
lexer->put_token = T_STOP;
- lexer->eof = true;
+ lexer->prog = NULL;
return;
}
-
+
if (lexer->put_token == '.')
break;
hex digits, according to TYPE. The string is converted to
characters having the specified values. */
static void
-convert_numeric_string_to_char_string (struct lexer *lexer,
+convert_numeric_string_to_char_string (struct lexer *lexer,
enum string_type type)
{
const char *base_name;
size_t i;
char *p;
- switch (type)
+ switch (type)
{
case BINARY_STRING:
base_name = _("binary");
default:
NOT_REACHED ();
}
-
+
byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
if (ds_length (&lexer->tokstr) % chars_per_byte)
- msg (SE, _("String of %s digits has %d characters, which is not a "
+ msg (SE, _("String of %s digits has %zu characters, which is not a "
"multiple of %d."),
base_name, ds_length (&lexer->tokstr), chars_per_byte);
{
int value;
int j;
-
+
value = 0;
for (j = 0; j < chars_per_byte; j++, p++)
{
buffer pointer lexer->prog must point to the initial single or double
quote. TYPE indicates the type of string to be parsed.
Returns token type. */
-static int
+static int
parse_string (struct lexer *lexer, enum string_type type)
{
if (type != CHARACTER_STRING)
{
/* Single or double quote. */
int c = *lexer->prog++;
-
+
/* Accumulate section. */
for (;;)
{
msg (SE, _("Unterminated string constant."));
goto finish;
}
-
+
/* Double quote characters to embed them in strings. */
if (*lexer->prog == c)
{
lexer->prog++;
/* Skip whitespace after final quote mark. */
- if (lexer->eof)
+ if (lexer->prog == NULL)
break;
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
lexer->prog++;
/* Skip whitespace after plus sign. */
- if (lexer->eof)
+ if (lexer->prog == NULL)
break;
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
if (type != CHARACTER_STRING)
convert_numeric_string_to_char_string (lexer, type);
- if (ds_length (&lexer->tokstr) > 255)
- {
- msg (SE, _("String exceeds 255 characters in length (%d characters)."),
- ds_length (&lexer->tokstr));
- ds_truncate (&lexer->tokstr, 255);
- }
-
return T_STRING;
}
-\f
-#if DUMP_TOKENS
-/* Reads one token from the lexer and writes a textual representation
- on stdout for debugging purposes. */
-static void
-dump_token (struct lexer *lexer)
-{
- {
- const char *curfn;
- int curln;
-
- getl_location (&curfn, &curln);
- if (curfn)
- fprintf (stderr, "%s:%d\t", curfn, curln);
- }
-
- switch (lexer->token)
- {
- case T_ID:
- fprintf (stderr, "ID\t%s\n", lexer->tokid);
- break;
-
- case T_POS_NUM:
- case T_NEG_NUM:
- fprintf (stderr, "NUM\t%f\n", lexer->tokval);
- break;
-
- case T_STRING:
- fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
- break;
-
- case T_STOP:
- fprintf (stderr, "STOP\n");
- break;
-
- case T_EXP:
- fprintf (stderr, "MISC\tEXP\"");
- break;
-
- case 0:
- fprintf (stderr, "MISC\tEOF\n");
- break;
-
- default:
- if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD)
- fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (token));
- else
- fprintf (stderr, "PUNCT\t%c\n", lexer->token);
- break;
- }
-}
-#endif /* DUMP_TOKENS */
-
-
+\f
/* Token Accessor Functions */
-int
+int
lex_token (const struct lexer *lexer)
{
return lexer->token;
}
-double
+double
lex_tokval (const struct lexer *lexer)
{
return lexer->tokval;
{
return &lexer->tokstr;
}
+
+/* If the lexer is positioned at the (pseudo)identifier S, which
+ may contain a hyphen ('-'), skips it and returns true. Each
+ half of the identifier may be abbreviated to its first three
+ letters.
+ Otherwise, returns false. */
+bool
+lex_match_hyphenated_word (struct lexer *lexer, const char *s)
+{
+ const char *hyphen = strchr (s, '-');
+ if (hyphen == NULL)
+ return lex_match_id (lexer, s);
+ else if (lexer->token != T_ID
+ || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+ || lex_look_ahead (lexer) != '-')
+ return false;
+ else
+ {
+ lex_get (lexer);
+ lex_force_match (lexer, '-');
+ lex_force_match_id (lexer, hyphen + 1);
+ return true;
+ }
+}
+