/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
along with this program. If not, see <http://www.gnu.org/licenses/>. */
#include <config.h>
+
#include "lexer.h"
-#include <libpspp/message.h>
#include <c-ctype.h>
#include <c-strtod.h>
#include <errno.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
-#include <libpspp/assertion.h>
#include <language/command.h>
-#include <libpspp/message.h>
#include <data/settings.h>
+#include <libpspp/assertion.h>
#include <libpspp/getl.h>
+#include <libpspp/message.h>
#include <libpspp/str.h>
#include <output/journal.h>
+#include <output/text-item.h>
#include "xalloc.h"
#define _(msgid) gettext (msgid)
#define N_(msgid) msgid
-
-#define DUMP_TOKENS 0
-
-
-
struct lexer
{
struct string line_buffer;
int token; /* Current token. */
double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
- char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
-
- struct string tokstr; /* T_ID, T_STRING: token string value.
- For T_ID, this is not truncated as is
- tokid. */
+ struct string tokstr; /* T_ID, T_STRING: token string value. */
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
};
static int parse_string (struct lexer *, enum string_type);
-
-#if DUMP_TOKENS
-static void dump_token (struct lexer *);
-#endif
\f
/* Initialization. */
assert (lexer->put_token != 0);
lexer->token = lexer->put_token;
ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
lexer->tokval = lexer->put_tokval;
lexer->put_token = 0;
}
return;
}
- /* If a token was pushed ahead, return it. */
- if (lexer->put_token)
- {
- restore_token (lexer);
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
- return;
- }
+ /* If a token was pushed ahead, return it. */
+ if (lexer->put_token)
+ {
+ restore_token (lexer);
+ return;
+ }
- for (;;)
- {
- /* Skip whitespace. */
+ for (;;)
+ {
+ /* Skip whitespace. */
while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (lexer->dot)
{
lexer->dot = 0;
- lexer->token = '.';
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
+ lexer->token = T_ENDCMD;
return;
}
else if (!lex_get_line (lexer))
{
lexer->prog = NULL;
lexer->token = T_STOP;
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
return;
}
if (lexer->put_token)
{
restore_token (lexer);
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
return;
}
}
{
char *tail;
- /* `-' can introduce a negative number, or it can be a
- token by itself. If it is not followed by a digit or a
- decimal point, it is definitely not a number.
- Otherwise, it might be either, but most of the time we
- want it as a number. When the syntax calls for a `-'
- token, lex_negative_to_dash() must be used to break
- negative numbers into two tokens. */
+ /* `-' can introduce a negative number, or it can be a token by
+ itself. */
if (*lexer->prog == '-')
{
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
while (c_isspace ((unsigned char) *lexer->prog))
lexer->prog++;
if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
{
- lexer->token = '-';
+ lexer->token = T_DASH;
break;
}
lexer->token = T_NEG_NUM;
/* Parse the number, copying it into tokstr. */
while (c_isdigit ((unsigned char) *lexer->prog))
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '.')
{
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
while (c_isdigit ((unsigned char) *lexer->prog))
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
}
if (*lexer->prog == 'e' || *lexer->prog == 'E')
{
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
if (*lexer->prog == '+' || *lexer->prog == '-')
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
while (c_isdigit ((unsigned char) *lexer->prog))
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
}
/* Parse as floating point. */
lexer->tokval = 0.0;
ds_clear (&lexer->tokstr);
- ds_put_char (&lexer->tokstr, '0');
+ ds_put_byte (&lexer->tokstr, '0');
}
break;
lexer->token = parse_string (lexer, CHARACTER_STRING);
break;
- case '(': case ')': case ',': case '=': case '+': case '/':
- case '[': case ']':
- lexer->token = *lexer->prog++;
- break;
+ case '+':
+ lexer->token = T_PLUS;
+ lexer->prog++;
+ break;
+
+ case '/':
+ lexer->token = T_SLASH;
+ lexer->prog++;
+ break;
+
+ case '=':
+ lexer->token = T_EQUALS;
+ lexer->prog++;
+ break;
+
+ case '(':
+ lexer->token = T_LPAREN;
+ lexer->prog++;
+ break;
+
+ case ')':
+ lexer->token = T_RPAREN;
+ lexer->prog++;
+ break;
+
+ case '[':
+ lexer->token = T_LBRACK;
+ lexer->prog++;
+ break;
+
+ case ']':
+ lexer->token = T_RBRACK;
+ lexer->prog++;
+ break;
+
+ case ',':
+ lexer->token = T_COMMA;
+ lexer->prog++;
+ break;
case '*':
if (*++lexer->prog == '*')
lexer->token = T_EXP;
}
else
- lexer->token = '*';
+ lexer->token = T_ASTERISK;
break;
case '<':
}
else
{
- if (c_isgraph ((unsigned char) *lexer->prog))
- msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
- else
- msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
+ unsigned char c = *lexer->prog++;
+ char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c);
+ msg (SE, _("Bad character in input: `%s'."), c_name);
+ free (c_name);
continue;
}
}
break;
}
-
-#if DUMP_TOKENS
- dump_token (lexer);
-#endif
}
-/* Parses an identifier at the current position into tokid and
- tokstr.
+/* Parses an identifier at the current position into tokstr.
Returns the correct token type. */
static int
parse_id (struct lexer *lexer)
lexer->prog += ss_length (id);
ds_assign_substring (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
return lex_id_to_token (id);
}
void
lex_error (struct lexer *lexer, const char *message, ...)
{
- char *token_rep;
- char where[128];
+ struct string s;
+
+ ds_init_empty (&s);
- token_rep = lex_token_representation (lexer);
if (lexer->token == T_STOP)
- strcpy (where, "end of file");
- else if (lexer->token == '.')
- strcpy (where, "end of command");
+ ds_put_cstr (&s, _("Syntax error at end of file"));
+ else if (lexer->token == T_ENDCMD)
+ ds_put_cstr (&s, _("Syntax error at end of command"));
else
- snprintf (where, sizeof where, "`%s'", token_rep);
- free (token_rep);
+ {
+ char *token_rep = lex_token_representation (lexer);
+ ds_put_format (&s, _("Syntax error at `%s'"), token_rep);
+ free (token_rep);
+ }
if (message)
{
- char buf[1024];
va_list args;
+ ds_put_cstr (&s, ": ");
+
va_start (args, message);
- vsnprintf (buf, 1024, message, args);
+ ds_put_vformat (&s, message, args);
va_end (args);
-
- msg (SE, _("Syntax error %s at %s."), buf, where);
}
- else
- msg (SE, _("Syntax error at %s."), where);
+
+ msg (SE, "%s.", ds_cstr (&s));
+ ds_destroy (&s);
}
/* Checks that we're at end of command.
int
lex_end_of_command (struct lexer *lexer)
{
- if (lexer->token != '.')
+ if (lexer->token != T_ENDCMD)
{
lex_error (lexer, _("expecting end of command"));
return CMD_FAILURE;
/* If TOK is the current token, skips it and returns true
Otherwise, returns false. */
bool
-lex_match (struct lexer *lexer, int t)
+lex_match (struct lexer *lexer, enum token_type t)
{
if (lexer->token == t)
{
lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
{
if (lexer->token == T_ID
- && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+ && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
{
lex_get (lexer);
return true;
/* If the current token is T, skips the token. Otherwise, reports an
error and returns from the current function with return value false. */
bool
-lex_force_match (struct lexer *lexer, int t)
+lex_force_match (struct lexer *lexer, enum token_type t)
{
if (lexer->token == t)
{
bool
lex_force_string (struct lexer *lexer)
{
- if (lexer->token == T_STRING)
+ if (lex_is_string (lexer))
return true;
else
{
/* Weird token functions. */
-/* Returns the first character of the next token, except that if the
- next token is not an identifier, the character returned will not be
- a character that can begin an identifier. Specifically, the
- hexstring lead-in X' causes lookahead() to return '. Note that an
- alphanumeric return value doesn't guarantee an ID token, it could
- also be a reserved-word token. */
-int
+/* Returns the likely type of the next token, or 0 if it's hard to tell. */
+enum token_type
lex_look_ahead (struct lexer *lexer)
{
if (lexer->put_token)
break;
if (lexer->dot)
- return '.';
+ return T_ENDCMD;
else if (!lex_get_line (lexer))
return 0;
return lexer->put_token;
}
- if ((toupper ((unsigned char) *lexer->prog) == 'X'
- || toupper ((unsigned char) *lexer->prog) == 'B'
- || toupper ((unsigned char) *lexer->prog) == 'O')
- && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
- return '\'';
+ switch (toupper ((unsigned char) *lexer->prog))
+ {
+ case 'X': case 'B': case 'O':
+ if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
+ return T_STRING;
+ /* Fall through */
+
+ case '-':
+ return T_DASH;
+
+ case '.':
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ return T_POS_NUM;
+
+ case '\'': case '"':
+ return T_STRING;
+
+ case '+':
+ return T_PLUS;
+
+ case '/':
+ return T_SLASH;
+
+ case '=':
+ return T_EQUALS;
+
+ case '(':
+ return T_LPAREN;
+
+ case ')':
+ return T_RPAREN;
+
+ case '[':
+ return T_LBRACK;
+
+ case ']':
+ return T_RBRACK;
+
+ case ',':
+ return T_COMMA;
+
+ case '*':
+ return lexer->prog[1] == '*' ? T_EXP : T_ASTERISK;
+
+ case '<':
+ return (lexer->prog[1] == '=' ? T_LE
+ : lexer->prog[1] == '>' ? T_NE
+ : T_LT);
+
+ case '>':
+ return lexer->prog[1] == '=' ? T_GE : T_GT;
- return *lexer->prog;
+ case '~':
+ return lexer->prog[1] == '=' ? T_NE : T_NOT;
+
+ case '&':
+ return T_AND;
+
+ case '|':
+ return T_OR;
+
+ default:
+ if (lex_is_id1 (*lexer->prog))
+ return T_ID;
+ return 0;
+ }
}
}
/* Makes the current token become the next token to be read; the
current token is set to T. */
void
-lex_put_back (struct lexer *lexer, int t)
+lex_put_back (struct lexer *lexer, enum token_type t)
{
save_token (lexer);
lexer->token = t;
}
-
-/* Makes the current token become the next token to be read; the
- current token is set to the identifier ID. */
-void
-lex_put_back_id (struct lexer *lexer, const char *id)
-{
- assert (lex_id_to_token (ss_cstr (id)) == T_ID);
- save_token (lexer);
- lexer->token = T_ID;
- ds_assign_cstr (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
-}
\f
/* Weird line processing functions. */
{
if (!getl_is_interactive (lexer->ss))
{
- while (lexer->token != T_STOP && lexer->token != '.')
+ while (lexer->token != T_STOP && lexer->token != T_ENDCMD)
lex_get (lexer);
}
else
{
strip_comments (line);
ds_rtrim (line, ss_cstr (CC_SPACES));
- *line_ends_command = (ds_chomp (line, settings_get_endcmd ())
- || (ds_is_empty (line) && settings_get_nulline ()));
+ *line_ends_command = ds_chomp (line, '.') || ds_is_empty (line);
*line_starts_command = false;
if (syntax == GETL_BATCH)
{
}
}
-/* Reads a line, without performing any preprocessing.
- Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
- mode. */
+/* Reads a line, without performing any preprocessing. */
bool
lex_get_line_raw (struct lexer *lexer)
{
bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
- enum syntax_mode mode = lex_current_syntax_mode (lexer);
- journal_write (mode == GETL_BATCH, ds_cstr (&lexer->line_buffer));
-
+ if (ok)
+ {
+ const char *line = ds_cstr (&lexer->line_buffer);
+ text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line));
+ }
+ else
+ lexer->prog = NULL;
return ok;
}
bool line_starts_command;
if (!lex_get_line_raw (lexer))
- {
- lexer->prog = NULL;
- return false;
- }
+ return false;
lex_preprocess_line (&lexer->line_buffer,
lex_current_syntax_mode (lexer),
&line_starts_command, &lexer->dot);
if (line_starts_command)
- lexer->put_token = '.';
+ lexer->put_token = T_ENDCMD;
lexer->prog = ds_cstr (&lexer->line_buffer);
return true;
/* Returns the name of a token. */
const char *
-lex_token_name (int token)
+lex_token_name (enum token_type token)
{
- if (lex_is_keyword (token))
- return lex_id_name (token);
- else if (token < 256)
+ switch (token)
{
- static char t[256][2];
- char *s = t[token];
- s[0] = token;
- s[1] = '\0';
- return s;
+ case T_ID:
+ case T_POS_NUM:
+ case T_NEG_NUM:
+ case T_STRING:
+ case TOKEN_N_TYPES:
+ NOT_REACHED ();
+
+ case T_STOP:
+ return "";
+
+ case T_ENDCMD:
+ return ".";
+
+ case T_PLUS:
+ return "+";
+
+ case T_DASH:
+ return "-";
+
+ case T_ASTERISK:
+ return "*";
+
+ case T_SLASH:
+ return "/";
+
+ case T_EQUALS:
+ return "=";
+
+ case T_LPAREN:
+ return "(";
+
+ case T_RPAREN:
+ return ")";
+
+ case T_LBRACK:
+ return "[";
+
+ case T_RBRACK:
+ return "]";
+
+ case T_COMMA:
+ return ",";
+
+ case T_AND:
+ return "AND";
+
+ case T_OR:
+ return "OR";
+
+ case T_NOT:
+ return "NOT";
+
+ case T_EQ:
+ return "EQ";
+
+ case T_GE:
+ return ">=";
+
+ case T_GT:
+ return ">";
+
+ case T_LE:
+ return "<=";
+
+ case T_LT:
+ return "<";
+
+ case T_NE:
+ return "~=";
+
+ case T_ALL:
+ return "ALL";
+
+ case T_BY:
+ return "BY";
+
+ case T_TO:
+ return "TO";
+
+ case T_WITH:
+ return "WITH";
+
+ case T_EXP:
+ return "**";
}
- else
- NOT_REACHED ();
+
+ NOT_REACHED ();
}
/* Returns an ASCII representation of the current token as a
case T_ID:
case T_POS_NUM:
case T_NEG_NUM:
- return ds_xstrdup (&lexer->tokstr);
- break;
+ return ss_xstrdup (lex_tokss (lexer));
case T_STRING:
{
+ struct substring ss;
int hexstring = 0;
char *sp, *dp;
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+ ss = lex_tokss (lexer);
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
if (!c_isprint ((unsigned char) *sp))
{
hexstring = 1;
break;
}
- token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+ token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
dp = token_rep;
if (hexstring)
*dp++ = 'X';
*dp++ = '\'';
- if (!hexstring)
- for (sp = ds_cstr (&lexer->tokstr); *sp; )
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+ if (!hexstring)
{
if (*sp == '\'')
*dp++ = '\'';
- *dp++ = (unsigned char) *sp++;
+ *dp++ = (unsigned char) *sp;
}
- else
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+ else
{
*dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
*dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
return token_rep;
}
- break;
-
- case T_STOP:
- token_rep = xmalloc (1);
- *token_rep = '\0';
- return token_rep;
-
- case T_EXP:
- return xstrdup ("**");
default:
return xstrdup (lex_token_name (lexer->token));
}
-
- NOT_REACHED ();
}
\f
/* Really weird functions. */
-/* Most of the time, a `-' is a lead-in to a negative number. But
- sometimes it's actually part of the syntax. If a dash can be part
- of syntax then this function is called to rip it off of a
- number. */
-void
-lex_negative_to_dash (struct lexer *lexer)
-{
- if (lexer->token == T_NEG_NUM)
- {
- lexer->token = T_POS_NUM;
- lexer->tokval = -lexer->tokval;
- ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
- save_token (lexer);
- lexer->token = '-';
- }
-}
-
/* Skip a COMMENT command. */
void
lex_skip_comment (struct lexer *lexer)
return;
}
- if (lexer->put_token == '.')
+ if (lexer->put_token == T_ENDCMD)
break;
ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
break;
}
- ds_put_char (&lexer->tokstr, *lexer->prog++);
+ ds_put_byte (&lexer->tokstr, *lexer->prog++);
}
lexer->prog++;
if (type != CHARACTER_STRING)
convert_numeric_string_to_char_string (lexer, type);
- if (ds_length (&lexer->tokstr) > 255)
- {
- msg (SE, _("String exceeds 255 characters in length (%zu characters)."),
- ds_length (&lexer->tokstr));
- ds_truncate (&lexer->tokstr, 255);
- }
-
return T_STRING;
}
\f
-#if DUMP_TOKENS
-/* Reads one token from the lexer and writes a textual representation
- on stdout for debugging purposes. */
-static void
-dump_token (struct lexer *lexer)
-{
- {
- const char *curfn;
- int curln;
-
- curln = getl_source_location (lexer->ss);
- curfn = getl_source_name (lexer->ss);
- if (curfn)
- fprintf (stderr, "%s:%d\t", curfn, curln);
- }
-
- switch (lexer->token)
- {
- case T_ID:
- fprintf (stderr, "ID\t%s\n", lexer->tokid);
- break;
-
- case T_POS_NUM:
- case T_NEG_NUM:
- fprintf (stderr, "NUM\t%f\n", lexer->tokval);
- break;
-
- case T_STRING:
- fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
- break;
-
- case T_STOP:
- fprintf (stderr, "STOP\n");
- break;
-
- case T_EXP:
- fprintf (stderr, "MISC\tEXP\"");
- break;
-
- case 0:
- fprintf (stderr, "MISC\tEOF\n");
- break;
-
- default:
- if (lex_is_keyword (lexer->token))
- fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
- else
- fprintf (stderr, "PUNCT\t%c\n", lexer->token);
- break;
- }
-}
-#endif /* DUMP_TOKENS */
-
-
/* Token Accessor Functions */
-int
+enum token_type
lex_token (const struct lexer *lexer)
{
return lexer->token;
return lexer->tokval;
}
+/* Returns the null-terminated string value associated with LEXER's current
+ token. For a T_ID token, this is the identifier, and for a T_STRING token,
+ this is the string. For other tokens the value is undefined. */
const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
{
- return lexer->tokid;
+ return ds_cstr (&lexer->tokstr);
}
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token. For a T_ID
+ token, this is the identifier, and for a T_STRING token, this is the string.
+ For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
{
- return &lexer->tokstr;
+ return ds_ss (&lexer->tokstr);
}
/* If the lexer is positioned at the (pseudo)identifier S, which
if (hyphen == NULL)
return lex_match_id (lexer, s);
else if (lexer->token != T_ID
- || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
- || lex_look_ahead (lexer) != '-')
+ || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
+ || lex_look_ahead (lexer) != T_DASH)
return false;
else
{
lex_get (lexer);
- lex_force_match (lexer, '-');
+ lex_force_match (lexer, T_DASH);
lex_force_match_id (lexer, hyphen + 1);
return true;
}
projects / pspp-builds.git / blobdiff