/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
-#include "lexer.h"
+#include "language/lexer/lexer.h"
+
#include <c-ctype.h>
#include <c-strtod.h>
#include <errno.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdlib.h>
-#include <language/command.h>
-#include <data/settings.h>
-#include <libpspp/assertion.h>
-#include <libpspp/getl.h>
-#include <libpspp/message.h>
-#include <libpspp/str.h>
-#include <output/journal.h>
-#include <output/text-item.h>
-#include "xalloc.h"
+#include "data/settings.h"
+#include "language/command.h"
+#include "libpspp/assertion.h"
+#include "libpspp/getl.h"
+#include "libpspp/message.h"
+#include "libpspp/str.h"
+#include "output/journal.h"
+#include "output/text-item.h"
+
+#include "gl/xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
int token; /* Current token. */
double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
- char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
-
- struct string tokstr; /* T_ID, T_STRING: token string value.
- For T_ID, this is not truncated as is
- tokid. */
+ struct string tokstr; /* T_ID, T_STRING: token string value. */
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
assert (lexer->put_token != 0);
lexer->token = lexer->put_token;
ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
lexer->tokval = lexer->put_tokval;
lexer->put_token = 0;
}
{
char *tail;
- /* `-' can introduce a negative number, or it can be a
- token by itself. If it is not followed by a digit or a
- decimal point, it is definitely not a number.
- Otherwise, it might be either, but most of the time we
- want it as a number. When the syntax calls for a `-'
- token, lex_negative_to_dash() must be used to break
- negative numbers into two tokens. */
+ /* `-' can introduce a negative number, or it can be a token by
+ itself. */
if (*lexer->prog == '-')
{
ds_put_byte (&lexer->tokstr, *lexer->prog++);
}
}
-/* Parses an identifier at the current position into tokid and
- tokstr.
+/* Parses an identifier at the current position into tokstr.
Returns the correct token type. */
static int
parse_id (struct lexer *lexer)
lexer->prog += ss_length (id);
ds_assign_substring (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
return lex_id_to_token (id);
}
lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
{
if (lexer->token == T_ID
- && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+ && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
{
lex_get (lexer);
return true;
save_token (lexer);
lexer->token = t;
}
-
-/* Makes the current token become the next token to be read; the
- current token is set to the identifier ID. */
-void
-lex_put_back_id (struct lexer *lexer, const char *id)
-{
- assert (lex_id_to_token (ss_cstr (id)) == T_ID);
- save_token (lexer);
- lexer->token = T_ID;
- ds_assign_cstr (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
-}
\f
/* Weird line processing functions. */
{
strip_comments (line);
ds_rtrim (line, ss_cstr (CC_SPACES));
- *line_ends_command = (ds_chomp (line, settings_get_endcmd ())
- || (ds_is_empty (line) && settings_get_nulline ()));
+ *line_ends_command = ds_chomp_byte (line, '.') || ds_is_empty (line);
*line_starts_command = false;
if (syntax == GETL_BATCH)
{
case T_POS_NUM:
case T_NEG_NUM:
case T_STRING:
+ case TOKEN_N_TYPES:
NOT_REACHED ();
case T_STOP:
case T_ID:
case T_POS_NUM:
case T_NEG_NUM:
- return ds_xstrdup (&lexer->tokstr);
+ return ss_xstrdup (lex_tokss (lexer));
case T_STRING:
{
+ struct substring ss;
int hexstring = 0;
char *sp, *dp;
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+ ss = lex_tokss (lexer);
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
if (!c_isprint ((unsigned char) *sp))
{
hexstring = 1;
break;
}
- token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+ token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
dp = token_rep;
if (hexstring)
*dp++ = 'X';
*dp++ = '\'';
- if (!hexstring)
- for (sp = ds_cstr (&lexer->tokstr); *sp; )
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+ if (!hexstring)
{
if (*sp == '\'')
*dp++ = '\'';
- *dp++ = (unsigned char) *sp++;
+ *dp++ = (unsigned char) *sp;
}
- else
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+ else
{
*dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
*dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
\f
/* Really weird functions. */
-/* Most of the time, a `-' is a lead-in to a negative number. But
- sometimes it's actually part of the syntax. If a dash can be part
- of syntax then this function is called to rip it off of a
- number. */
-void
-lex_negative_to_dash (struct lexer *lexer)
-{
- if (lexer->token == T_NEG_NUM)
- {
- lexer->token = T_POS_NUM;
- lexer->tokval = -lexer->tokval;
- ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
- save_token (lexer);
- lexer->token = T_DASH;
- }
-}
-
/* Skip a COMMENT command. */
void
lex_skip_comment (struct lexer *lexer)
return lexer->tokval;
}
+/* Returns the null-terminated string value associated with LEXER's current
+ token. For a T_ID token, this is the identifier, and for a T_STRING token,
+ this is the string. For other tokens the value is undefined. */
const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
{
- return lexer->tokid;
+ return ds_cstr (&lexer->tokstr);
}
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token. For a T_ID
+ token, this is the identifier, and for a T_STRING token, this is the string.
+ For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
{
- return &lexer->tokstr;
+ return ds_ss (&lexer->tokstr);
}
/* If the lexer is positioned at the (pseudo)identifier S, which
if (hyphen == NULL)
return lex_match_id (lexer, s);
else if (lexer->token != T_ID
- || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+ || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
|| lex_look_ahead (lexer) != T_DASH)
return false;
else
projects / pspp-builds.git / blobdiff