Removed eof state from lexer. A lexer which reaches the end of its input, can
now continue, if new input is appended to the input stream.
/* PSPP - computes sample statistics.
/* PSPP - computes sample statistics.
- Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
Written by Ben Pfaff <blp@gnu.org>.
This program is free software; you can redistribute it and/or
Written by Ben Pfaff <blp@gnu.org>.
This program is free software; you can redistribute it and/or
#define _(msgid) gettext (msgid)
#define N_(msgid) msgid
#define _(msgid) gettext (msgid)
#define N_(msgid) msgid
-/*
-#define DUMP_TOKENS 1
-*/
+
+#define DUMP_TOKENS 0
+
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
- bool eof; /* True only if the last token returned was T_STOP. */
int put_token ; /* If nonzero, next token returned by lex_get().
Used only in exceptional circumstances. */
int put_token ; /* If nonzero, next token returned by lex_get().
Used only in exceptional circumstances. */
static int parse_string (struct lexer *, enum string_type);
#if DUMP_TOKENS
static int parse_string (struct lexer *, enum string_type);
#if DUMP_TOKENS
-static void dump_token (void);
+static void dump_token (struct lexer *);
#endif
\f
/* Initialization. */
#endif
\f
/* Initialization. */
ds_init_empty (&lexer->line_buffer);
lexer->ss = ss;
ds_init_empty (&lexer->line_buffer);
lexer->ss = ss;
- if (!lex_get_line (lexer))
- lexer->eof = true;
-
void
lex_get (struct lexer *lexer)
{
void
lex_get (struct lexer *lexer)
{
+ /* Find a token. */
+ for (;;)
+ {
+ if (NULL == lexer->prog && ! lex_get_line (lexer) )
+ {
+ lexer->token = T_STOP;
+ return;
+ }
+
/* If a token was pushed ahead, return it. */
if (lexer->put_token)
{
restore_token (lexer);
#if DUMP_TOKENS
/* If a token was pushed ahead, return it. */
if (lexer->put_token)
{
restore_token (lexer);
#if DUMP_TOKENS
for (;;)
{
/* Skip whitespace. */
for (;;)
{
/* Skip whitespace. */
- if (lexer->eof)
- {
- lexer->token = T_STOP;
- return;
- }
-
- for (;;)
- {
while (isspace ((unsigned char) *lexer->prog))
lexer->prog++;
while (isspace ((unsigned char) *lexer->prog))
lexer->prog++;
lexer->dot = 0;
lexer->token = '.';
#if DUMP_TOKENS
lexer->dot = 0;
lexer->token = '.';
#if DUMP_TOKENS
#endif
return;
}
else if (!lex_get_line (lexer))
{
#endif
return;
}
else if (!lex_get_line (lexer))
{
lexer->token = T_STOP;
#if DUMP_TOKENS
lexer->token = T_STOP;
#if DUMP_TOKENS
{
restore_token (lexer);
#if DUMP_TOKENS
{
restore_token (lexer);
#if DUMP_TOKENS
+ if (NULL == lexer->prog && ! lex_get_line (lexer) )
enum getl_syntax syntax;
if (!lex_get_line_raw (lexer, &syntax))
enum getl_syntax syntax;
if (!lex_get_line_raw (lexer, &syntax))
+ {
+ lexer->prog = NULL;
lex_preprocess_line (&lexer->line_buffer, syntax,
&line_starts_command, &lexer->dot);
lex_preprocess_line (&lexer->line_buffer, syntax,
&line_starts_command, &lexer->dot);
if (line_starts_command)
lexer->put_token = '.';
if (line_starts_command)
lexer->put_token = '.';
if (!lex_get_line (lexer))
{
lexer->put_token = T_STOP;
if (!lex_get_line (lexer))
{
lexer->put_token = T_STOP;
lexer->prog++;
/* Skip whitespace after final quote mark. */
lexer->prog++;
/* Skip whitespace after final quote mark. */
+ if (lexer->prog == NULL)
lexer->prog++;
/* Skip whitespace after plus sign. */
lexer->prog++;
/* Skip whitespace after plus sign. */
+ if (lexer->prog == NULL)
const char *curfn;
int curln;
const char *curfn;
int curln;
- getl_location (&curfn, &curln);
+ curln = getl_source_location (lexer->ss);
+ curfn = getl_source_name (lexer->ss);
if (curfn)
fprintf (stderr, "%s:%d\t", curfn, curln);
}
if (curfn)
fprintf (stderr, "%s:%d\t", curfn, curln);
}
- if (lex_is_keyword (token))
- fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (token));
+ if (lex_is_keyword (lexer->token))
+ fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
else
fprintf (stderr, "PUNCT\t%c\n", lexer->token);
break;
else
fprintf (stderr, "PUNCT\t%c\n", lexer->token);
break;