#include "libpspp/cast.h"
#include "libpspp/deque.h"
#include "libpspp/i18n.h"
+#include "libpspp/intern.h"
#include "libpspp/ll.h"
#include "libpspp/message.h"
#include "libpspp/misc.h"
call. */
size_t token_pos; /* Offset into src->buffer of token start. */
size_t token_len; /* Length of source for token in bytes. */
- int first_line; /* Line number at token_pos. */
/* For a token obtained through macro expansion, this is just this token.
static size_t lex_stage_count (const struct lex_stage *);
static bool lex_stage_is_empty (const struct lex_stage *);
-static struct lex_token *lex_stage_last (struct lex_stage *);
static struct lex_token *lex_stage_first (struct lex_stage *);
static struct lex_token *lex_stage_nth (struct lex_stage *, size_t ofs);
return deque_count (&stage->deque);
}
-/* Returns the last token in STAGE, which must be nonempty. The last token is
- the one accessed with the greatest lookahead. */
-static struct lex_token *
-lex_stage_last (struct lex_stage *stage)
-{
- return stage->tokens[deque_front (&stage->deque, 0)];
-}
-
/* Returns the first token in STAGE, which must be nonempty.
The first token is the one accessed with the least lookahead. */
static struct lex_token *
stage->tokens[deque_push_front (&stage->deque)] = token;
}
+/* Removes and returns the first token from STAGE. */
+static struct lex_token *
+lex_stage_take_first (struct lex_stage *stage)
+{
+ return stage->tokens[deque_pop_back (&stage->deque)];
+}
+
/* Removes the first token from STAGE and uninitializes it. */
static void
lex_stage_pop_first (struct lex_stage *stage)
{
- lex_token_destroy (stage->tokens[deque_pop_back (&stage->deque)]);
+ lex_token_destroy (lex_stage_take_first (stage));
}
/* Removes the first N tokens from SRC, appending them to DST as the last
lex_stage_shift (struct lex_stage *dst, struct lex_stage *src, size_t n)
{
for (size_t i = 0; i < n; i++)
- {
- lex_stage_push_last (dst, lex_stage_first (src));
- deque_pop_back (&src->deque);
- }
+ lex_stage_push_last (dst, lex_stage_take_first (src));
}
/* A source of tokens, corresponding to a syntax file.
size_t journal_pos; /* First byte not yet output to journal. */
size_t seg_pos; /* First byte not yet scanned as token. */
- int n_newlines; /* Number of new-lines up to seg_pos. */
+ /* Offset into 'buffer' of starts of lines. */
+ size_t *lines;
+ size_t n_lines, allocated_lines;
+
bool suppress_next_newline;
/* Tokens.
in 'merge'.
- merge: Tokens that need to pass through scan_merge() to end up in
- 'lookahead'.
+ 'parse'.
- - lookahead: Tokens available to the client for parsing. */
+ - parse: Tokens available to the client for parsing.
+
+ 'pp' and 'merge' store tokens only temporarily until they pass into
+ 'parse'. Tokens then live in 'parse' until the command is fully
+ consumed, at which time they are freed together. */
struct lex_stage pp;
struct lex_stage merge;
- struct lex_stage lookahead;
+ struct lex_token **parse;
+ size_t n_parse, allocated_parse, parse_ofs;
};
static struct lex_source *lex_source_create (struct lexer *,
int n0, int n1);
static const struct lex_token *lex_next__ (const struct lexer *, int n);
static void lex_source_push_endcmd__ (struct lex_source *);
+static void lex_source_push_parse (struct lex_source *, struct lex_token *);
+static void lex_source_clear_parse (struct lex_source *);
-static bool lex_source_get_lookahead (struct lex_source *);
+static bool lex_source_get_parse (struct lex_source *);
static void lex_source_error_valist (struct lex_source *, int n0, int n1,
const char *format, va_list)
PRINTF_FORMAT (4, 0);
if (src == NULL)
return;
- if (!lex_stage_is_empty (&src->lookahead))
- lex_stage_pop_first (&src->lookahead);
+ if (src->parse_ofs < src->n_parse)
+ {
+ if (src->parse[src->parse_ofs]->token.type == T_ENDCMD)
+ lex_source_clear_parse (src);
+ else
+ src->parse_ofs++;
+ }
- while (lex_stage_is_empty (&src->lookahead))
- if (!lex_source_get_lookahead (src))
+ while (src->parse_ofs == src->n_parse)
+ if (!lex_source_get_parse (src))
{
lex_source_destroy (src);
src = lex_source__ (lexer);
lex_source_next__ (const struct lex_source *src_, int n)
{
struct lex_source *src = CONST_CAST (struct lex_source *, src_);
- while (lex_stage_count (&src->lookahead) <= n)
+
+ if (n < 0)
+ {
+ if (-n <= src->parse_ofs)
+ return src->parse[src->parse_ofs - (-n)];
+ else
+ {
+ static const struct lex_token endcmd_token
+ = { .token = { .type = T_ENDCMD } };
+ return &endcmd_token;
+ }
+ }
+
+ while (src->n_parse - src->parse_ofs <= n)
{
- if (!lex_stage_is_empty (&src->lookahead))
+ if (src->n_parse > 0)
{
- const struct lex_token *t = lex_stage_last (&src->lookahead);
+ const struct lex_token *t = src->parse[src->n_parse - 1];
if (t->token.type == T_STOP || t->token.type == T_ENDCMD)
return t;
}
- lex_source_get_lookahead (src);
+ lex_source_get_parse (src);
}
- return lex_stage_nth (&src->lookahead, n);
+ return src->parse[src->parse_ofs + n];
}
/* Returns the "struct token" of the token N after the current one in LEXER.
return lex_next (lexer, n)->string;
}
+int
+lex_ofs (const struct lexer *lexer)
+{
+ struct lex_source *src = lex_source__ (lexer);
+ return src ? src->parse_ofs : 0;
+}
+
+const struct token *
+lex_ofs_token (const struct lexer *lexer_, int ofs)
+{
+ struct lexer *lexer = CONST_CAST (struct lexer *, lexer_);
+ struct lex_source *src = lex_source__ (lexer);
+
+ if (src != NULL)
+ return &lex_source_next__ (src, ofs - src->parse_ofs)->token;
+ else
+ {
+ static const struct token stop_token = { .type = T_STOP };
+ return &stop_token;
+ }
+}
+
+struct msg_location *
+lex_ofs_location (const struct lexer *lexer, int ofs0, int ofs1)
+{
+ int ofs = lex_ofs (lexer);
+ return lex_get_location (lexer, ofs0 - ofs, ofs1 - ofs);
+}
+
/* Returns the text of the syntax in tokens N0 ahead of the current one,
through N1 ahead of the current one, inclusive. (For example, if N0 and N1
are both zero, this requests the syntax for the current token.) The caller
lex_token_get_last_line_number (const struct lex_source *src,
const struct lex_token *token)
{
+ size_t end = token->token_pos + token->token_len
+ return lex_source_ofs_to_line_number (src,
if (token->first_line == 0)
return 0;
else
const struct lex_token *t0,
const struct lex_token *t1)
{
+ int first_column = lex_token_get_first_column (src, t0);
+ int last_line = lex_token_get_last_line_number (src, t1) - 1;
+ int last_column = lex_token_get_last_column (src, t1) - 1;
return (struct msg_location) {
- .file_name = src->reader->file_name,
- .first_line = t0->first_line,
- .last_line = lex_token_get_last_line_number (src, t1),
- .first_column = lex_token_get_first_column (src, t0),
- .last_column = lex_token_get_last_column (src, t1),
+ .file_name = intern_new_if_nonnull (src->reader->file_name),
+ .p[0] = { .line = t0->first_line, .column = first_column },
+ .p[1] = { .line = last_line, .column = last_column },
};
}
lex_get_location (const struct lexer *lexer, int n0, int n1)
{
struct msg_location *loc = lex_get_lines (lexer, n0, n1);
- loc->first_column = lex_get_first_column (lexer, n0);
- loc->last_column = lex_get_last_column (lexer, n1);
+ loc->p[0].column = lex_get_first_column (lexer, n0);
+ loc->p[1].column = lex_get_last_column (lexer, n1) - 1;
return loc;
}
lex_get_lines (const struct lexer *lexer, int n0, int n1)
{
struct msg_location *loc = xmalloc (sizeof *loc);
+ int first_line = lex_get_first_line_number (lexer, n0);
+ int last_line = lex_get_last_line_number (lexer, n1) - 1;
*loc = (struct msg_location) {
- .file_name = xstrdup_if_nonnull (lex_get_file_name (lexer)),
- .first_line = lex_get_first_line_number (lexer, n0),
- .last_line = lex_get_last_line_number (lexer, n1),
+ .file_name = intern_new_if_nonnull (lex_get_file_name (lexer)),
+ .p[0] = { .line = first_line },
+ .p[1] = { .line = last_line },
};
return loc;
}
+void
+lex_extend_location (const struct lexer *lexer, int n, struct msg_location **loc)
+{
+ struct msg_location *new = lex_get_location (lexer, n, n);
+ msg_location_merge (loc, new);
+ msg_location_destroy (new);
+}
+
const char *
lex_get_encoding (const struct lexer *lexer)
{
false);
lex_stage_clear (&src->pp);
lex_stage_clear (&src->merge);
- lex_stage_clear (&src->lookahead);
+ lex_source_clear_parse (src);
lex_source_push_endcmd__ (src);
}
}
{
lex_stage_clear (&src->pp);
lex_stage_clear (&src->merge);
- lex_stage_clear (&src->lookahead);
+ lex_source_clear_parse (src);
for (; src != NULL && src->reader->error != LEX_ERROR_TERMINAL;
src = lex_source__ (lexer))
Returns true if successful, false on failure. In the latter case, SRC is
exhausted and 'src->eof' is now true. */
static bool
-lex_source_get_lookahead (struct lex_source *src)
+lex_source_get_parse (struct lex_source *src)
{
struct merger m = MERGER_INIT;
struct token out;
&out);
if (!retval)
{
- lex_stage_shift (&src->lookahead, &src->merge, 1);
+ lex_source_push_parse (src, lex_stage_take_first (&src->merge));
return true;
}
else if (retval > 0)
};
if (t->ref_cnt)
++*t->ref_cnt;
- lex_stage_push_last (&src->lookahead, t);
+ lex_source_push_parse (src, t);
for (int i = 0; i < retval; i++)
lex_stage_pop_first (&src->merge);
static void
lex_source_push_endcmd__ (struct lex_source *src)
{
- assert (lex_stage_is_empty (&src->lookahead));
+ assert (src->n_parse == 0);
+
struct lex_token *token = xmalloc (sizeof *token);
*token = (struct lex_token) { .token = { .type = T_ENDCMD } };
- lex_stage_push_last (&src->lookahead, token);
+ lex_source_push_parse (src, token);
+}
+
+static void
+lex_source_push_parse (struct lex_source *src, struct lex_token *token)
+{
+ if (src->n_parse >= src->allocated_parse)
+ src->parse = x2nrealloc (src->parse, &src->allocated_parse,
+ sizeof *src->parse);
+ src->parse[src->n_parse++] = token;
+}
+
+static void
+lex_source_clear_parse (struct lex_source *src)
+{
+ for (size_t i = 0; i < src->n_parse; i++)
+ lex_token_destroy (src->parse[i]);
+ src->n_parse = src->parse_ofs = 0;
}
static struct lex_source *
free (src->buffer);
lex_stage_uninit (&src->pp);
lex_stage_uninit (&src->merge);
- lex_stage_uninit (&src->lookahead);
+ lex_source_clear_parse (src);
+ free (src->parse);
ll_remove (&src->ll);
free (src);
}
projects / pspp / blobdiff