/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
#include "lexer.h"
#include <libpspp/message.h>
-#include <ctype.h>
+#include <c-ctype.h>
+#include <c-strtod.h>
#include <errno.h>
#include <limits.h>
#include <math.h>
#include <stdarg.h>
+#include <stdint.h>
#include <stdlib.h>
-#include <libpspp/alloc.h>
#include <libpspp/assertion.h>
#include <language/command.h>
#include <libpspp/message.h>
-#include <libpspp/magic.h>
#include <data/settings.h>
#include <libpspp/getl.h>
#include <libpspp/str.h>
#include <output/journal.h>
+#include <output/text-item.h>
-#include "size_max.h"
+#include "xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
#define N_(msgid) msgid
-
-#define DUMP_TOKENS 0
-
-
-
struct lexer
{
struct string line_buffer;
int token; /* Current token. */
double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
- char tokid [LONG_NAME_LEN + 1]; /* T_ID: the identifier. */
+ char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
struct string tokstr; /* T_ID, T_STRING: token string value.
For T_ID, this is not truncated as is
};
static int parse_string (struct lexer *, enum string_type);
-
-#if DUMP_TOKENS
-static void dump_token (struct lexer *);
-#endif
\f
/* Initialization. */
return;
}
- /* If a token was pushed ahead, return it. */
- if (lexer->put_token)
- {
- restore_token (lexer);
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
- return;
- }
+ /* If a token was pushed ahead, return it. */
+ if (lexer->put_token)
+ {
+ restore_token (lexer);
+ return;
+ }
- for (;;)
- {
- /* Skip whitespace. */
- while (isspace ((unsigned char) *lexer->prog))
+ for (;;)
+ {
+ /* Skip whitespace. */
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
{
lexer->dot = 0;
lexer->token = '.';
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
return;
}
else if (!lex_get_line (lexer))
{
lexer->prog = NULL;
lexer->token = T_STOP;
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
return;
}
if (lexer->put_token)
{
restore_token (lexer);
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
return;
}
}
if (*lexer->prog == '-')
{
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
- if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
+ if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
{
lexer->token = '-';
break;
lexer->token = T_POS_NUM;
/* Parse the number, copying it into tokstr. */
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '.')
{
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
}
if (*lexer->prog == 'e' || *lexer->prog == 'E')
ds_put_char (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '+' || *lexer->prog == '-')
ds_put_char (&lexer->tokstr, *lexer->prog++);
- while (isdigit ((unsigned char) *lexer->prog))
+ while (c_isdigit ((unsigned char) *lexer->prog))
ds_put_char (&lexer->tokstr, *lexer->prog++);
}
/* Parse as floating point. */
- lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
+ lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail);
if (*tail)
{
msg (SE, _("%s does not form a valid number."),
break;
case '(': case ')': case ',': case '=': case '+': case '/':
+ case '[': case ']':
lexer->token = *lexer->prog++;
break;
}
else
{
- if (isgraph ((unsigned char) *lexer->prog))
- msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
- else
- msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
+ unsigned char c = *lexer->prog++;
+ char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c);
+ msg (SE, _("Bad character in input: `%s'."), c_name);
+ free (c_name);
continue;
}
}
break;
}
-
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
}
/* Parses an identifier at the current position into tokid and
void
lex_error (struct lexer *lexer, const char *message, ...)
{
- char *token_rep;
- char where[128];
+ struct string s;
+
+ ds_init_empty (&s);
- token_rep = lex_token_representation (lexer);
if (lexer->token == T_STOP)
- strcpy (where, "end of file");
+ ds_put_cstr (&s, _("Syntax error at end of file"));
else if (lexer->token == '.')
- strcpy (where, "end of command");
+ ds_put_cstr (&s, _("Syntax error at end of command"));
else
- snprintf (where, sizeof where, "`%s'", token_rep);
- free (token_rep);
+ {
+ char *token_rep = lex_token_representation (lexer);
+ ds_put_format (&s, _("Syntax error at `%s'"), token_rep);
+ free (token_rep);
+ }
if (message)
{
- char buf[1024];
va_list args;
+ ds_put_cstr (&s, ": ");
+
va_start (args, message);
- vsnprintf (buf, 1024, message, args);
+ ds_put_vformat (&s, message, args);
va_end (args);
-
- msg (SE, _("Syntax error %s at %s."), buf, where);
}
- else
- msg (SE, _("Syntax error at %s."), where);
+
+ msg (SE, "%s.", ds_cstr (&s));
+ ds_destroy (&s);
}
/* Checks that we're at end of command.
lex_is_integer (struct lexer *lexer)
{
return (lex_is_number (lexer)
- && lexer->tokval != NOT_LONG
- && lexer->tokval >= LONG_MIN
+ && lexer->tokval > LONG_MIN
&& lexer->tokval <= LONG_MAX
&& floor (lexer->tokval) == lexer->tokval);
}
Otherwise, returns false. */
bool
lex_match_id (struct lexer *lexer, const char *s)
+{
+ return lex_match_id_n (lexer, s, 3);
+}
+
+/* If the current token is the identifier S, skips it and returns
+ true. The identifier may be abbreviated to its first N
+ letters.
+ Otherwise, returns false. */
+bool
+lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
{
if (lexer->token == T_ID
- && lex_id_match (ss_cstr (s), ss_cstr (lexer->tokid)))
+ && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
{
lex_get (lexer);
return true;
bool
lex_force_string (struct lexer *lexer)
{
- if (lexer->token == T_STRING)
+ if (lex_is_string (lexer))
return true;
else
{
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
{
strip_comments (line);
ds_rtrim (line, ss_cstr (CC_SPACES));
- *line_ends_command = (ds_chomp (line, get_endcmd ())
- || (ds_is_empty (line) && get_nulline ()));
+ *line_ends_command = (ds_chomp (line, settings_get_endcmd ())
+ || (ds_is_empty (line) && settings_get_nulline ()));
*line_starts_command = false;
if (syntax == GETL_BATCH)
{
int first = ds_first (line);
- *line_starts_command = !isspace (first);
+ *line_starts_command = !c_isspace (first);
if (first == '+' || first == '-')
*ds_data (line) = ' ';
}
}
-/* Reads a line, without performing any preprocessing.
- Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
- mode. */
+/* Reads a line, without performing any preprocessing. */
bool
lex_get_line_raw (struct lexer *lexer)
{
bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
- enum syntax_mode mode = lex_current_syntax_mode (lexer);
- journal_write (mode == GETL_BATCH, ds_cstr (&lexer->line_buffer));
-
+ if (ok)
+ {
+ const char *line = ds_cstr (&lexer->line_buffer);
+ text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line));
+ }
return ok;
}
char *sp, *dp;
for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
- if (!isprint ((unsigned char) *sp))
+ if (!c_isprint ((unsigned char) *sp))
{
hexstring = 1;
break;
byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
if (ds_length (&lexer->tokstr) % chars_per_byte)
- msg (SE, _("String of %s digits has %d characters, which is not a "
+ msg (SE, _("String of %s digits has %zu characters, which is not a "
"multiple of %d."),
- base_name, (int) ds_length (&lexer->tokstr), chars_per_byte);
+ base_name, ds_length (&lexer->tokstr), chars_per_byte);
p = ds_cstr (&lexer->tokstr);
for (i = 0; i < byte_cnt; i++)
break;
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
break;
for (;;)
{
- while (isspace ((unsigned char) *lexer->prog))
+ while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (*lexer->prog)
break;
if (type != CHARACTER_STRING)
convert_numeric_string_to_char_string (lexer, type);
- if (ds_length (&lexer->tokstr) > 255)
- {
- msg (SE, _("String exceeds 255 characters in length (%d characters)."),
- (int) ds_length (&lexer->tokstr));
- ds_truncate (&lexer->tokstr, 255);
- }
-
return T_STRING;
}
\f
-#if DUMP_TOKENS
-/* Reads one token from the lexer and writes a textual representation
- on stdout for debugging purposes. */
-static void
-dump_token (struct lexer *lexer)
-{
- {
- const char *curfn;
- int curln;
-
- curln = getl_source_location (lexer->ss);
- curfn = getl_source_name (lexer->ss);
- if (curfn)
- fprintf (stderr, "%s:%d\t", curfn, curln);
- }
-
- switch (lexer->token)
- {
- case T_ID:
- fprintf (stderr, "ID\t%s\n", lexer->tokid);
- break;
-
- case T_POS_NUM:
- case T_NEG_NUM:
- fprintf (stderr, "NUM\t%f\n", lexer->tokval);
- break;
-
- case T_STRING:
- fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
- break;
-
- case T_STOP:
- fprintf (stderr, "STOP\n");
- break;
-
- case T_EXP:
- fprintf (stderr, "MISC\tEXP\"");
- break;
-
- case 0:
- fprintf (stderr, "MISC\tEOF\n");
- break;
-
- default:
- if (lex_is_keyword (lexer->token))
- fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
- else
- fprintf (stderr, "PUNCT\t%c\n", lexer->token);
- break;
- }
-}
-#endif /* DUMP_TOKENS */
-
-
/* Token Accessor Functions */
int
{
return &lexer->tokstr;
}
+
+/* If the lexer is positioned at the (pseudo)identifier S, which
+ may contain a hyphen ('-'), skips it and returns true. Each
+ half of the identifier may be abbreviated to its first three
+ letters.
+ Otherwise, returns false. */
+bool
+lex_match_hyphenated_word (struct lexer *lexer, const char *s)
+{
+ const char *hyphen = strchr (s, '-');
+ if (hyphen == NULL)
+ return lex_match_id (lexer, s);
+ else if (lexer->token != T_ID
+ || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+ || lex_look_ahead (lexer) != '-')
+ return false;
+ else
+ {
+ lex_get (lexer);
+ lex_force_match (lexer, '-');
+ lex_force_match_id (lexer, hyphen + 1);
+ return true;
+ }
+}
+
projects / pspp / blobdiff