struct string_lexer slex;
string_lexer_init (&slex, s.string, s.length, SEG_MODE_INTERACTIVE, true);
struct token another_token = { .type = T_STOP };
- if (!string_lexer_next (&slex, token)
- || string_lexer_next (&slex, &another_token))
+ if (string_lexer_next (&slex, token) != SLR_TOKEN
+ || string_lexer_next (&slex, &another_token) != SLR_END)
{
token_uninit (token);
token_uninit (&another_token);
struct macro *m = xmalloc (sizeof *m);
*m = (struct macro) {
.name = ss_xstrdup (lex_tokss (lexer)),
+ .location = xmalloc (sizeof *m->location),
+ };
+ *m->location = (struct msg_location) {
.file_name = xstrdup_if_nonnull (lex_get_file_name (lexer)),
.first_line = lex_get_first_line_number (lexer, 0),
};
ds_put_byte (&body, '\n');
lex_get (lexer);
}
- m->last_line = lex_get_last_line_number (lexer, 0);
+ m->location->last_line = lex_get_last_line_number (lexer, 0);
macro_tokens_from_string (&m->body, body.ss, lex_get_syntax_mode (lexer));
ds_destroy (&body);
};
static void
-lex_token_uninit (struct lex_token *t)
+lex_token_destroy (struct lex_token *t)
{
token_uninit (&t->token);
if (t->ref_cnt)
free (t->ref_cnt);
}
}
+ free (t);
+}
+\f
+/* A deque of lex_tokens that comprises one stage in the token pipeline in a
+ lex_source. */
+struct lex_stage
+ {
+ struct deque deque;
+ struct lex_token **tokens;
+ };
+
+static void lex_stage_clear (struct lex_stage *);
+static void lex_stage_uninit (struct lex_stage *);
+
+static size_t lex_stage_count (const struct lex_stage *);
+static bool lex_stage_is_empty (const struct lex_stage *);
+
+static struct lex_token *lex_stage_last (struct lex_stage *);
+static struct lex_token *lex_stage_first (struct lex_stage *);
+static struct lex_token *lex_stage_nth (struct lex_stage *, size_t ofs);
+
+static void lex_stage_push_last (struct lex_stage *, struct lex_token *);
+static void lex_stage_pop_first (struct lex_stage *);
+
+static void lex_stage_shift (struct lex_stage *dst, struct lex_stage *src,
+ size_t n);
+
+/* Deletes all the tokens from STAGE. */
+static void
+lex_stage_clear (struct lex_stage *stage)
+{
+ while (!deque_is_empty (&stage->deque))
+ lex_stage_pop_first (stage);
+}
+
+/* Deletes all the tokens from STAGE and frees storage for the deque. */
+static void
+lex_stage_uninit (struct lex_stage *stage)
+{
+ lex_stage_clear (stage);
+ free (stage->tokens);
+}
+
+/* Returns true if STAGE contains no tokens, otherwise false. */
+static bool
+lex_stage_is_empty (const struct lex_stage *stage)
+{
+ return deque_is_empty (&stage->deque);
+}
+
+/* Returns the number of tokens in STAGE. */
+static size_t
+lex_stage_count (const struct lex_stage *stage)
+{
+ return deque_count (&stage->deque);
+}
+
+/* Returns the last token in STAGE, which must be nonempty. The last token is
+ the one accessed with the greatest lookahead. */
+static struct lex_token *
+lex_stage_last (struct lex_stage *stage)
+{
+ return stage->tokens[deque_front (&stage->deque, 0)];
+}
+
+/* Returns the first token in STAGE, which must be nonempty.
+ The first token is the one accessed with the least lookahead. */
+static struct lex_token *
+lex_stage_first (struct lex_stage *stage)
+{
+ return lex_stage_nth (stage, 0);
+}
+
+/* Returns the token the given INDEX in STAGE. The first token (with the least
+ lookahead) is 0, the second token is 1, and so on. There must be at least
+ INDEX + 1 tokens in STAGE. */
+static struct lex_token *
+lex_stage_nth (struct lex_stage *stage, size_t index)
+{
+ return stage->tokens[deque_back (&stage->deque, index)];
+}
+
+/* Adds TOKEN so that it becomes the last token in STAGE. */
+static void
+lex_stage_push_last (struct lex_stage *stage, struct lex_token *token)
+{
+ if (deque_is_full (&stage->deque))
+ stage->tokens = deque_expand (&stage->deque, stage->tokens,
+ sizeof *stage->tokens);
+ stage->tokens[deque_push_front (&stage->deque)] = token;
+}
+
+/* Removes the first token from STAGE and uninitializes it. */
+static void
+lex_stage_pop_first (struct lex_stage *stage)
+{
+ lex_token_destroy (stage->tokens[deque_pop_back (&stage->deque)]);
+}
+
+/* Removes the first N tokens from SRC, appending them to DST as the last
+ tokens. */
+static void
+lex_stage_shift (struct lex_stage *dst, struct lex_stage *src, size_t n)
+{
+ for (size_t i = 0; i < n; i++)
+ {
+ lex_stage_push_last (dst, lex_stage_first (src));
+ deque_pop_back (&src->deque);
+ }
}
/* A source of tokens, corresponding to a syntax file.
/* Tokens.
- This is mostly like a deque, with the invariant that 'back <= middle <=
- front' (modulo SIZE_MAX+1). The tokens available for parsing are
- between 'back' and 'middle': the token at 'back' is the current token,
- the token at 'back + 1' is the next token, and so on. There are usually
- no tokens between 'middle' and 'front'; if there are, then they need to
- go through macro expansion and are not yet available for parsing.
-
- 'capacity' is the current number of elements in 'tokens'. It is always
- a power of 2. 'front', 'middle', and 'back' refer to indexes in
- 'tokens' modulo 'capacity'. */
- size_t front;
- size_t middle;
- size_t back;
- size_t capacity;
- size_t mask; /* capacity - 1 */
- struct lex_token *tokens;
+ This is a pipeline with the following stages. Each token eventually
+ made available to the parser passes through of these stages. The stages
+ are named after the processing that happens in each one.
+
+ Initially, tokens come from the segmenter and scanner to 'pp':
+
+ - pp: Tokens that need to pass through the macro preprocessor to end up
+ in 'merge'.
+
+ - merge: Tokens that need to pass through scan_merge() to end up in
+ 'lookahead'.
+
+ - lookahead: Tokens available to the client for parsing. */
+ struct lex_stage pp;
+ struct lex_stage merge;
+ struct lex_stage lookahead;
};
static struct lex_source *lex_source_create (struct lexer *,
static const struct lex_token *lex_next__ (const struct lexer *, int n);
static void lex_source_push_endcmd__ (struct lex_source *);
-static void lex_source_pop_back (struct lex_source *);
-static bool lex_source_get (const struct lex_source *);
+static bool lex_source_get_lookahead (struct lex_source *);
static void lex_source_error_valist (struct lex_source *, int n0, int n1,
const char *format, va_list)
PRINTF_FORMAT (4, 0);
\f
/* Advancing. */
-/* Adds a new token at the front of SRC and returns a pointer to it. The
- caller should initialize it. Does not advance the middle pointer, so the
- token isn't immediately available to the parser. */
-static struct lex_token *
-lex_push_token__ (struct lex_source *src)
-{
- if (src->front - src->back >= src->capacity)
- {
- /* Expansion works just like a deque, so we reuse the code. */
- struct deque deque = {
- .capacity = src->capacity,
- .front = src->front,
- .back = src->back,
- };
- src->tokens = deque_expand (&deque, src->tokens, sizeof *src->tokens);
- src->capacity = deque.capacity;
- src->mask = src->capacity - 1;
- }
-
- struct lex_token *token = &src->tokens[src->front++ & src->mask];
- token->token = (struct token) { .type = T_STOP };
- token->macro_rep = NULL;
- token->ref_cnt = NULL;
- return token;
-}
-
-/* Removes the current token from SRC and uninitializes it. */
-static void
-lex_source_pop_back (struct lex_source *src)
-{
- assert (src->middle - src->back > 0);
- lex_token_uninit (&src->tokens[src->back++ & src->mask]);
-}
-
-/* Removes the token at the greatest lookahead from SRC and uninitializes
- it. */
-static void
-lex_source_pop_front (struct lex_source *src)
-{
- assert (src->front - src->middle > 0);
- lex_token_uninit (&src->tokens[--src->front & src->mask]);
-}
-
/* Advances LEXER to the next token, consuming the current token. */
void
lex_get (struct lexer *lexer)
if (src == NULL)
return;
- if (src->middle - src->back > 0)
- lex_source_pop_back (src);
+ if (!lex_stage_is_empty (&src->lookahead))
+ lex_stage_pop_first (&src->lookahead);
- while (src->back == src->middle)
- if (!lex_source_get (src))
+ while (lex_stage_is_empty (&src->lookahead))
+ if (!lex_source_get_lookahead (src))
{
lex_source_destroy (src);
src = lex_source__ (lexer);
}
}
-/* Returns the token in SRC with the greatest lookahead. */
-static const struct lex_token *
-lex_source_middle (const struct lex_source *src)
-{
- assert (src->middle - src->back > 0);
- return &src->tokens[(src->middle - 1) & src->mask];
-}
-
static const struct lex_token *
-lex_source_next__ (const struct lex_source *src, int n)
+lex_source_next__ (const struct lex_source *src_, int n)
{
- while (src->middle - src->back <= n)
+ struct lex_source *src = CONST_CAST (struct lex_source *, src_);
+ while (lex_stage_count (&src->lookahead) <= n)
{
- if (src->middle - src->back > 0)
+ if (!lex_stage_is_empty (&src->lookahead))
{
- const struct lex_token *middle = lex_source_middle (src);
- if (middle->token.type == T_STOP || middle->token.type == T_ENDCMD)
- return middle;
+ const struct lex_token *t = lex_stage_last (&src->lookahead);
+ if (t->token.type == T_STOP || t->token.type == T_ENDCMD)
+ return t;
}
- lex_source_get (src);
+ lex_source_get_lookahead (src);
}
- return &src->tokens[(src->back + n) & src->mask];
+ return lex_stage_nth (&src->lookahead, n);
}
/* Returns the "struct token" of the token N after the current one in LEXER.
i = 0;
string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
while (string_lexer_next (&slex, &token))
- if (token.type != SCAN_SKIP)
- {
- bool match = lex_tokens_match (lex_next (lexer, i++), &token);
- token_uninit (&token);
- if (!match)
- return false;
- }
+ {
+ bool match = lex_tokens_match (lex_next (lexer, i++), &token);
+ token_uninit (&token);
+ if (!match)
+ return false;
+ }
while (i-- > 0)
lex_get (lexer);
return true;
}
-static int
-lex_source_get_first_line_number (const struct lex_source *src, int n)
-{
- return lex_source_next__ (src, n)->first_line;
-}
-
static int
count_newlines (char *s, size_t length)
{
}
static int
-lex_source_get_last_line_number (const struct lex_source *src, int n)
+lex_token_get_last_line_number (const struct lex_source *src,
+ const struct lex_token *token)
{
- const struct lex_token *token = lex_source_next__ (src, n);
-
if (token->first_line == 0)
return 0;
else
}
static int
-lex_source_get_first_column (const struct lex_source *src, int n)
+lex_token_get_first_column (const struct lex_source *src,
+ const struct lex_token *token)
{
- const struct lex_token *token = lex_source_next__ (src, n);
return count_columns (&src->buffer[token->line_pos - src->tail],
token->token_pos - token->line_pos);
}
static int
-lex_source_get_last_column (const struct lex_source *src, int n)
+lex_token_get_last_column (const struct lex_source *src,
+ const struct lex_token *token)
{
- const struct lex_token *token = lex_source_next__ (src, n);
char *start, *end, *newline;
start = &src->buffer[token->line_pos - src->tail];
return count_columns (start, end - start);
}
+static struct msg_location
+lex_token_location (const struct lex_source *src,
+ const struct lex_token *t0,
+ const struct lex_token *t1)
+{
+ return (struct msg_location) {
+ .file_name = src->reader->file_name,
+ .first_line = t0->first_line,
+ .last_line = lex_token_get_last_line_number (src, t1),
+ .first_column = lex_token_get_first_column (src, t0),
+ .last_column = lex_token_get_last_column (src, t1),
+ };
+}
+
+static struct msg_location *
+lex_token_location_rw (const struct lex_source *src,
+ const struct lex_token *t0,
+ const struct lex_token *t1)
+{
+ struct msg_location location = lex_token_location (src, t0, t1);
+ return msg_location_dup (&location);
+}
+
+static struct msg_location *
+lex_source_get_location (const struct lex_source *src, int n0, int n1)
+{
+ return lex_token_location_rw (src,
+ lex_source_next__ (src, n0),
+ lex_source_next__ (src, n1));
+}
+
/* Returns the 1-based line number of the start of the syntax that represents
the token N after the current one in LEXER. Returns 0 for a T_STOP token or
if the token is drawn from a source that does not have line numbers. */
lex_get_first_line_number (const struct lexer *lexer, int n)
{
const struct lex_source *src = lex_source__ (lexer);
- return src != NULL ? lex_source_get_first_line_number (src, n) : 0;
+ return src ? lex_source_next__ (src, n)->first_line : 0;
}
/* Returns the 1-based line number of the end of the syntax that represents the
lex_get_last_line_number (const struct lexer *lexer, int n)
{
const struct lex_source *src = lex_source__ (lexer);
- return src != NULL ? lex_source_get_last_line_number (src, n) : 0;
+ return src ? lex_token_get_last_line_number (src,
+ lex_source_next__ (src, n)) : 0;
}
/* Returns the 1-based column number of the start of the syntax that represents
lex_get_first_column (const struct lexer *lexer, int n)
{
const struct lex_source *src = lex_source__ (lexer);
- return src != NULL ? lex_source_get_first_column (src, n) : 0;
+ return src ? lex_token_get_first_column (src, lex_source_next__ (src, n)) : 0;
}
/* Returns the 1-based column number of the end of the syntax that represents
lex_get_last_column (const struct lexer *lexer, int n)
{
const struct lex_source *src = lex_source__ (lexer);
- return src != NULL ? lex_source_get_last_column (src, n) : 0;
+ return src ? lex_token_get_last_column (src, lex_source_next__ (src, n)) : 0;
}
/* Returns the name of the syntax file from which the current command is drawn.
src->suppress_next_newline = false;
src->segmenter = segmenter_init (segmenter_get_mode (&src->segmenter),
false);
- while (src->middle - src->back > 0)
- lex_source_pop_back (src);
- while (src->front - src->middle > 0)
- lex_source_pop_front (src);
+ lex_stage_clear (&src->pp);
+ lex_stage_clear (&src->merge);
+ lex_stage_clear (&src->lookahead);
lex_source_push_endcmd__ (src);
}
}
if (src != NULL)
{
- while (src->middle - src->back > 0)
- lex_source_pop_back (src);
+ lex_stage_clear (&src->pp);
+ lex_stage_clear (&src->merge);
+ lex_stage_clear (&src->lookahead);
for (; src != NULL && src->reader->error != LEX_ERROR_TERMINAL;
src = lex_source__ (lexer))
}
\f
static size_t
-lex_source_max_tail__ (const struct lex_source *src)
+lex_source_max_tail__ (const struct lex_source *src_)
{
- const struct lex_token *token;
- size_t max_tail;
+ struct lex_source *src = CONST_CAST (struct lex_source *, src_);
assert (src->seg_pos >= src->line_pos);
- max_tail = MIN (src->journal_pos, src->line_pos);
+ size_t max_tail = MIN (src->journal_pos, src->line_pos);
- /* Use the oldest token also. (We know that src->deque cannot be empty
- because we are in the process of adding a new token, which is already
- initialized enough to use here.) */
- token = &src->tokens[src->back & src->mask];
- assert (token->token_pos >= token->line_pos);
- max_tail = MIN (max_tail, token->line_pos);
+ /* Use the oldest token also. */
+ struct lex_stage *stages[] = { &src->lookahead, &src->merge, &src->pp };
+ for (size_t i = 0; i < sizeof stages / sizeof *stages; i++)
+ if (!lex_stage_is_empty (stages[i]))
+ {
+ struct lex_token *first = lex_stage_first (stages[i]);
+ assert (first->token_pos >= first->line_pos);
+ return MIN (max_tail, first->line_pos);
+ }
return max_tail;
}
if (ds_last (&s) != '.')
ds_put_byte (&s, '.');
- struct msg_location *location = xmalloc (sizeof *location);
- *location = (struct msg_location) {
- .file_name = xstrdup_if_nonnull (src->reader->file_name),
- .first_line = lex_source_get_first_line_number (src, n0),
- .last_line = lex_source_get_last_line_number (src, n1),
- .first_column = lex_source_get_first_column (src, n0),
- .last_column = lex_source_get_last_column (src, n1),
- };
struct msg *m = xmalloc (sizeof *m);
*m = (struct msg) {
.category = MSG_C_SYNTAX,
.severity = MSG_S_ERROR,
- .location = location,
+ .location = lex_source_get_location (src, n0, n1),
.text = ds_steal_cstr (&s),
};
msg_emit (m);
}
-static void PRINTF_FORMAT (4, 5)
-lex_source_error (struct lex_source *src, int n0, int n1,
- const char *format, ...)
-{
- va_list args;
- va_start (args, format);
- lex_source_error_valist (src, n0, n1, format, args);
- va_end (args);
-}
-
static void
-lex_get_error (struct lex_source *src, const char *s)
+lex_get_error (struct lex_source *src, const struct lex_token *token)
{
- size_t old_middle = src->middle;
- src->middle = src->front;
- size_t n = src->front - src->back - 1;
- lex_source_error (src, n, n, "%s", s);
- src->middle = old_middle;
+ char syntax[64];
+ str_ellipsize (ss_buffer (&src->buffer[token->token_pos - src->tail],
+ token->token_len),
+ syntax, sizeof syntax);
- lex_source_pop_front (src);
-}
+ struct string s = DS_EMPTY_INITIALIZER;
+ ds_put_format (&s, _("Syntax error at `%s'"), syntax);
+ ds_put_format (&s, ": %s", token->token.string.string);
-/* Attempts to append an additional token at the front of SRC, reading more
- from the underlying lex_reader if necessary. Returns true if a new token
- was added to SRC's deque, false otherwise. The caller should retry failures
- unless SRC's 'eof' marker was set to true indicating that there will be no
- more tokens from this source.
+ struct msg *m = xmalloc (sizeof *m);
+ *m = (struct msg) {
+ .category = MSG_C_SYNTAX,
+ .severity = MSG_S_ERROR,
+ .location = lex_token_location_rw (src, token, token),
+ .text = ds_steal_cstr (&s),
+ };
+ msg_emit (m);
+}
- Does not make the new token available for lookahead yet; the caller must
- adjust SRC's 'middle' pointer to do so. */
+/* Attempts to append an additional token to 'pp' in SRC, reading more from the
+ underlying lex_reader if necessary. Returns true if a new token was added
+ to SRC's deque, false otherwise. The caller should retry failures unless
+ SRC's 'eof' marker was set to true indicating that there will be no more
+ tokens from this source. */
static bool
-lex_source_try_get__ (struct lex_source *src)
+lex_source_try_get_pp (struct lex_source *src)
{
- /* State maintained while scanning tokens. Usually we only need a single
- state, but scanner_push() can return SCAN_SAVE to indicate that the state
- needs to be saved and possibly restored later with SCAN_BACK. */
- struct state
- {
- struct segmenter segmenter;
- enum segment_type last_segment;
- int newlines; /* Number of newlines encountered so far. */
- /* Maintained here so we can update lex_source's similar members when we
- finish. */
- size_t line_pos;
- size_t seg_pos;
- };
-
- /* Initialize state. */
- struct state state =
- {
- .segmenter = src->segmenter,
- .newlines = 0,
- .seg_pos = src->seg_pos,
- .line_pos = src->line_pos,
- };
- struct state saved = state;
-
/* Append a new token to SRC and initialize it. */
- struct lex_token *token = lex_push_token__ (src);
- struct scanner scanner;
- scanner_init (&scanner, &token->token);
+ struct lex_token *token = xmalloc (sizeof *token);
+ token->token = (struct token) { .type = T_STOP };
+ token->macro_rep = NULL;
+ token->ref_cnt = NULL;
token->line_pos = src->line_pos;
token->token_pos = src->seg_pos;
if (src->reader->line_number > 0)
else
token->first_line = 0;
- /* Extract segments and pass them through the scanner until we obtain a
- token. */
+ /* Extract a segment. */
+ const char *segment;
+ enum segment_type seg_type;
+ int seg_len;
for (;;)
{
- /* Extract a segment. */
- const char *segment = &src->buffer[state.seg_pos - src->tail];
- size_t seg_maxlen = src->head - state.seg_pos;
- enum segment_type type;
- int seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen,
- src->reader->eof, &type);
- if (seg_len < 0)
- {
- /* The segmenter needs more input to produce a segment. */
- assert (!src->reader->eof);
- lex_source_read__ (src);
- continue;
- }
+ segment = &src->buffer[src->seg_pos - src->tail];
+ seg_len = segmenter_push (&src->segmenter, segment,
+ src->head - src->seg_pos,
+ src->reader->eof, &seg_type);
+ if (seg_len >= 0)
+ break;
- /* Update state based on the segment. */
- state.last_segment = type;
- state.seg_pos += seg_len;
- if (type == SEG_NEWLINE)
- {
- state.newlines++;
- state.line_pos = state.seg_pos;
- }
+ /* The segmenter needs more input to produce a segment. */
+ assert (!src->reader->eof);
+ lex_source_read__ (src);
+ }
- /* Pass the segment into the scanner and try to get a token out. */
- enum scan_result result = scanner_push (&scanner, type,
- ss_buffer (segment, seg_len),
- &token->token);
- if (result == SCAN_SAVE)
- saved = state;
- else if (result == SCAN_BACK)
- {
- state = saved;
- break;
- }
- else if (result == SCAN_DONE)
- break;
+ /* Update state based on the segment. */
+ token->token_len = seg_len;
+ src->seg_pos += seg_len;
+ if (seg_type == SEG_NEWLINE)
+ {
+ src->line_pos = src->seg_pos;
+ src->n_newlines++;
}
+ /* Get a token from the segment. */
+ enum tokenize_result result = token_from_segment (
+ seg_type, ss_buffer (segment, seg_len), &token->token);
+
/* If we've reached the end of a line, or the end of a command, then pass
the line to the output engine as a syntax text item. */
- int n_lines = state.newlines;
- if (state.last_segment == SEG_END_COMMAND && !src->suppress_next_newline)
+ int n_lines = seg_type == SEG_NEWLINE;
+ if (seg_type == SEG_END_COMMAND && !src->suppress_next_newline)
{
n_lines++;
src->suppress_next_newline = true;
/* Calculate line length, including \n or \r\n end-of-line if present.
We use src->head even though that may be beyond what we've actually
- converted to tokens (which is only through state.line_pos). That's
- because, if we're emitting the line due to SEG_END_COMMAND, we want to
- take the whole line through the newline, not just through the '.'. */
+ converted to tokens (which is only through line_pos). That's because,
+ if we're emitting the line due to SEG_END_COMMAND, we want to take the
+ whole line through the newline, not just through the '.'. */
size_t max_len = src->head - src->journal_pos;
const char *newline = memchr (line, '\n', max_len);
size_t line_len = newline ? newline - line + 1 : max_len;
src->journal_pos += line_len;
}
- token->token_len = state.seg_pos - src->seg_pos;
-
- src->segmenter = state.segmenter;
- src->seg_pos = state.seg_pos;
- src->line_pos = state.line_pos;
- src->n_newlines += state.newlines;
-
- switch (token->token.type)
+ switch (result)
{
- default:
- return true;
-
- case T_STOP:
- token->token.type = T_ENDCMD;
- src->eof = true;
- return true;
-
- case SCAN_BAD_HEX_LENGTH:
- case SCAN_BAD_HEX_DIGIT:
- case SCAN_BAD_UNICODE_DIGIT:
- case SCAN_BAD_UNICODE_LENGTH:
- case SCAN_BAD_UNICODE_CODE_POINT:
- case SCAN_EXPECTED_QUOTE:
- case SCAN_EXPECTED_EXPONENT:
- case SCAN_UNEXPECTED_CHAR:
- {
- char *msg = scan_token_to_error (&token->token);
- lex_get_error (src, msg);
- free (msg);
+ case TOKENIZE_ERROR:
+ lex_get_error (src, token);
+ /* Fall through. */
+ case TOKENIZE_EMPTY:
+ lex_token_destroy (token);
return false;
- }
- case SCAN_SKIP:
- lex_source_pop_front (src);
- return false;
+ case TOKENIZE_TOKEN:
+ if (token->token.type == T_STOP)
+ {
+ token->token.type = T_ENDCMD;
+ src->eof = true;
+ }
+ lex_stage_push_last (&src->pp, token);
+ return true;
}
-
NOT_REACHED ();
}
-/* Attempts to add a new token at the front of SRC. Returns true if
- successful, false on failure. On failure, the end of SRC has been reached
- and no more tokens will be forthcoming from it.
+/* Attempts to append a new token to SRC. Returns true if successful, false on
+ failure. On failure, the end of SRC has been reached and no more tokens
+ will be forthcoming from it.
Does not make the new token available for lookahead yet; the caller must
adjust SRC's 'middle' pointer to do so. */
static bool
-lex_source_get__ (struct lex_source *src)
+lex_source_get_pp (struct lex_source *src)
{
while (!src->eof)
- if (lex_source_try_get__ (src))
+ if (lex_source_try_get_pp (src))
return true;
return false;
}
-/* Attempts to obtain a new token for SRC, in particular expanding the number
- of lookahead tokens (the tokens between 'back' and 'middle').
-
- Returns true if successful, false on failure. In the latter case, SRC is
- exhausted and 'src->eof' is now true. */
static bool
-lex_source_get (const struct lex_source *src_)
+lex_source_try_get_merge (const struct lex_source *src_)
{
struct lex_source *src = CONST_CAST (struct lex_source *, src_);
- /* In the common case, call into the scanner and segmenter to obtain a new
- token between 'middle' and 'front'. In the uncommon case, there can be one
- or a few tokens there already, leftovers from a macro expansion.
-
- If we call into the scanner and it fails, then we've hit EOF and we're
- done. */
- if (src->front - src->middle == 0 && !lex_source_get__ (src))
+ if (lex_stage_is_empty (&src->pp) && !lex_source_get_pp (src))
return false;
- /* We have at least one token available between 'middle' and 'front'.
-
- The remaining complication is all about macro expansion. If macro
- expansion is disabled, we're done. */
if (!settings_get_mexpand ())
{
- src->middle++;
+ lex_stage_shift (&src->merge, &src->pp, lex_stage_count (&src->pp));
return true;
}
In the common case where there is no macro to expand, the loop is not
entered. */
struct macro_call *mc;
- int n_call = macro_call_create (
- src->lexer->macros, &src->tokens[src->middle & src->mask].token,
- &mc);
- for (int middle_ofs = 1; !n_call; middle_ofs++)
+ int n_call = macro_call_create (src->lexer->macros,
+ &lex_stage_first (&src->pp)->token, &mc);
+ for (int ofs = 1; !n_call; ofs++)
{
- if (src->front - src->middle <= middle_ofs && !lex_source_get__ (src))
+ if (lex_stage_count (&src->pp) <= ofs && !lex_source_get_pp (src))
{
/* This should not be reachable because we always get a T_ENDCMD at
the end of an input file (transformed from T_STOP by
- lex_source_try_get__()) and the macro_expander should always
+ lex_source_try_get_pp()) and the macro_expander should always
terminate expansion on T_ENDCMD. */
NOT_REACHED ();
}
- const struct lex_token *t = &src->tokens[(src->middle + middle_ofs)
- & src->mask];
+ const struct lex_token *t = lex_stage_nth (&src->pp, ofs);
size_t start = t->token_pos;
size_t end = t->token_pos + t->token_len;
const struct macro_token mt = {
.token = t->token,
.syntax = ss_buffer (&src->buffer[start - src->tail], end - start),
};
-
- /* We temporarily add the tokens to the source to avoid re-entry if
- macro_expander_add() reports an error and to give better error
- messages. */
- src->middle += middle_ofs + 1;
- n_call = macro_call_add (mc, &mt);
- src->middle -= middle_ofs + 1;
+ const struct msg_location loc = lex_token_location (src, t, t);
+ n_call = macro_call_add (mc, &mt, &loc);
}
if (n_call < 0)
{
lookahead. We'll retry macro expansion from the second token next
time around. */
macro_call_destroy (mc);
- src->middle++;
+ lex_stage_shift (&src->merge, &src->pp, 1);
return true;
}
- /* Now expand the macro.
-
- We temporarily add the macro call's tokens to the source in case the macro
- expansion calls msg() to report an error and error processing tries to get
- the location of the error with, e.g. lex_get_first_line_number(), which
- would re-enter this code. This is a kluge; it might be cleaner to pass
- the line number into macro_expander_get_expansion(). */
- src->middle += n_call;
+ /* The first 'n_call' tokens in 'pp', which we bracket as C0...C1, inclusive,
+ are a macro call. (These are likely to be the only tokens in 'pp'.)
+ Expand them. */
+ const struct lex_token *c0 = lex_stage_first (&src->pp);
+ const struct lex_token *c1 = lex_stage_nth (&src->pp, n_call - 1);
struct macro_tokens expansion = { .n = 0 };
- macro_call_expand (mc, src->reader->syntax, &expansion);
+ struct msg_location loc = lex_token_location (src, c0, c1);
+ macro_call_expand (mc, src->reader->syntax, &loc, &expansion);
macro_call_destroy (mc);
- src->middle -= n_call;
- /* Convert the macro expansion into syntax for possible error messages later. */
+ /* Convert the macro expansion into syntax for possible error messages
+ later. */
size_t *ofs = xnmalloc (expansion.n, sizeof *ofs);
size_t *len = xnmalloc (expansion.n, sizeof *len);
struct string s = DS_EMPTY_INITIALIZER;
output_item_submit (text_item_create (TEXT_ITEM_LOG, ds_cstr (&s),
_("Macro Expansion")));
- /* The first 'n_call' tokens starting at 'middle' will be replaced by the
- macro expansion. There might be more tokens after that, up to 'front'.
-
- Figure out the boundary of the macro call in the syntax, to go into the
- lex_tokens for the expansion so that later error messages can report what
- macro was called. */
- const struct lex_token *call_first = &src->tokens[src->middle & src->mask];
- const struct lex_token *call_last
- = &src->tokens[(src->middle + n_call - 1) & src->mask];
- size_t call_pos = call_first->token_pos;
- size_t call_len = (call_last->token_pos + call_last->token_len) - call_pos;
- size_t line_pos = call_first->line_pos;
- int first_line = call_first->first_line;
-
- /* Destroy the tokens for the call, and save any tokens following the call so
- we can add them back later. */
- for (size_t i = src->middle; i != src->middle + n_call; i++)
- lex_token_uninit (&src->tokens[i & src->mask]);
- size_t n_save = src->front - (src->middle + n_call);
- struct lex_token *save_tokens = xnmalloc (n_save, sizeof *save_tokens);
- for (size_t i = 0; i < n_save; i++)
- save_tokens[i] = src->tokens[(src->middle + n_call + i) & src->mask];
- src->front = src->middle;
-
/* Append the macro expansion tokens to the lookahead. */
char *macro_rep = ds_steal_cstr (&s);
size_t *ref_cnt = xmalloc (sizeof *ref_cnt);
*ref_cnt = expansion.n;
for (size_t i = 0; i < expansion.n; i++)
{
- *lex_push_token__ (src) = (struct lex_token) {
+ struct lex_token *token = xmalloc (sizeof *token);
+ *token = (struct lex_token) {
.token = expansion.mts[i].token,
- .token_pos = call_pos,
- .token_len = call_len,
- .line_pos = line_pos,
- .first_line = first_line,
+ .token_pos = c0->token_pos,
+ .token_len = (c1->token_pos + c1->token_len) - c0->token_pos,
+ .line_pos = c0->line_pos,
+ .first_line = c0->first_line,
.macro_rep = macro_rep,
.ofs = ofs[i],
.len = len[i],
.ref_cnt = ref_cnt,
};
- src->middle++;
+ lex_stage_push_last (&src->merge, token);
ss_dealloc (&expansion.mts[i].syntax);
}
free (ofs);
free (len);
- /* Finally, put the saved tokens back. */
- for (size_t i = 0; i < n_save; i++)
- *lex_push_token__ (src) = save_tokens[i];
- free (save_tokens);
+ /* Destroy the tokens for the call. */
+ for (size_t i = 0; i < n_call; i++)
+ lex_stage_pop_first (&src->pp);
- return true;
+ return expansion.n > 0;
+}
+
+/* Attempts to obtain at least one new token into 'merge' in SRC.
+
+ Returns true if successful, false on failure. In the latter case, SRC is
+ exhausted and 'src->eof' is now true. */
+static bool
+lex_source_get_merge (struct lex_source *src)
+{
+ while (!src->eof)
+ if (lex_source_try_get_merge (src))
+ return true;
+ return false;
+}
+
+/* Attempts to obtain at least one new token into 'lookahead' in SRC.
+
+ Returns true if successful, false on failure. In the latter case, SRC is
+ exhausted and 'src->eof' is now true. */
+static bool
+lex_source_get_lookahead (struct lex_source *src)
+{
+ struct merger m = MERGER_INIT;
+ for (size_t i = 0; ; i++)
+ {
+ while (lex_stage_count (&src->merge) <= i && !lex_source_get_merge (src))
+ {
+ /* We always get a T_ENDCMD at the end of an input file
+ (transformed from T_STOP by lex_source_try_get_pp()) and
+ merger_add() should never return -1 on T_ENDCMD. */
+ assert (lex_stage_is_empty (&src->merge));
+ return false;
+ }
+
+ struct token out;
+ int retval = merger_add (&m, &lex_stage_nth (&src->merge, i)->token,
+ &out);
+ if (!retval)
+ {
+ lex_stage_shift (&src->lookahead, &src->merge, 1);
+ return true;
+ }
+ else if (retval > 0)
+ {
+ /* Add a token that merges all the tokens together. */
+ const struct lex_token *first = lex_stage_first (&src->merge);
+ const struct lex_token *last = lex_stage_nth (&src->merge,
+ retval - 1);
+ bool macro = first->macro_rep && first->macro_rep == last->macro_rep;
+ struct lex_token *t = xmalloc (sizeof *t);
+ *t = (struct lex_token) {
+ .token = out,
+ .token_pos = first->token_pos,
+ .token_len = (last->token_pos - first->token_pos) + last->token_len,
+ .line_pos = first->line_pos,
+ .first_line = first->first_line,
+
+ /* This works well if all the tokens were not expanded from macros,
+ or if they came from the same macro expansion. It just gives up
+ in the other (corner) cases. */
+ .macro_rep = macro ? first->macro_rep : NULL,
+ .ofs = macro ? first->ofs : 0,
+ .len = macro ? (last->ofs - first->ofs) + last->len : 0,
+ .ref_cnt = first->ref_cnt,
+ };
+ if (t->ref_cnt)
+ ++*t->ref_cnt;
+ lex_stage_push_last (&src->lookahead, t);
+
+ for (int i = 0; i < retval; i++)
+ lex_stage_pop_first (&src->merge);
+ return true;
+ }
+ }
}
\f
static void
lex_source_push_endcmd__ (struct lex_source *src)
{
- assert (src->back == src->middle && src->middle == src->front);
- *lex_push_token__ (src) = (struct lex_token) {
- .token = { .type = T_ENDCMD } };
- src->middle++;
+ assert (lex_stage_is_empty (&src->lookahead));
+ struct lex_token *token = xmalloc (sizeof *token);
+ *token = (struct lex_token) { .token = { .type = T_ENDCMD } };
+ lex_stage_push_last (&src->lookahead, token);
}
static struct lex_source *
free (file_name);
free (encoding);
free (src->buffer);
- while (src->middle - src->back > 0)
- lex_source_pop_back (src);
- while (src->front - src->middle > 0)
- lex_source_pop_front (src);
- free (src->tokens);
+ lex_stage_uninit (&src->pp);
+ lex_stage_uninit (&src->merge);
+ lex_stage_uninit (&src->lookahead);
ll_remove (&src->ll);
free (src);
}
/* An entry in the stack of macros and macro directives being expanded. The
stack is maintained as a linked list. Entries are not dynamically allocated
- but on the program stack. */
+ but on the program stack.
+
+ The outermost entry, where 'next' is NULL, represents the source location of
+ the call to the macro. */
struct macro_expansion_stack
{
- /* Points to an outer stack entry, or NULL if this is the outermost. */
- const struct macro_expansion_stack *next;
-
- /* A macro name or !IF, !DO, etc. */
- const char *name;
-
- /* Location of the macro definition, if available. */
- const char *file_name;
- int first_line;
- int last_line;
+ const struct macro_expansion_stack *next; /* Next outer stack entry. */
+ const char *name; /* A macro name or !IF, !DO, etc. */
+ const struct msg_location *location; /* Source location if available. */
};
/* Reports an error during macro expansion. STACK is the stack for reporting
the location of the error, MT is the optional token at which the error was
detected, and FORMAT along with the varargs is the message to report. */
-static void PRINTF_FORMAT (3, 4)
-macro_error (const struct macro_expansion_stack *stack,
- const struct macro_token *mt,
- const char *format, ...)
+static void PRINTF_FORMAT (3, 0)
+macro_error_valist (const struct macro_expansion_stack *stack,
+ const struct macro_token *mt, const char *format,
+ va_list args)
{
struct msg_stack **ms = NULL;
size_t allocated_ms = 0;
size_t n_ms = 0;
- for (const struct macro_expansion_stack *p = stack; p; p = p->next)
+ const struct macro_expansion_stack *p;
+ for (p = stack; p && p->next; p = p->next)
{
if (n_ms >= allocated_ms)
ms = x2nrealloc (ms, &allocated_ms, sizeof *ms);
ms[n_ms] = xmalloc (sizeof *ms[n_ms]);
*ms[n_ms] = (struct msg_stack) {
- .location = {
- .file_name = xstrdup_if_nonnull (p->file_name),
- .first_line = p->first_line,
- .last_line = p->last_line,
- },
+ .location = msg_location_dup (p->location),
.description = description,
};
n_ms++;
}
- va_list args;
- va_start (args, format);
- char *s = xvasprintf (format, args);
- va_end (args);
-
struct msg *m = xmalloc (sizeof *m);
*m = (struct msg) {
.category = MSG_C_SYNTAX,
.severity = MSG_S_ERROR,
.stack = ms,
.n_stack = n_ms,
- .text = s,
+ .location = msg_location_dup (p ? p->location : NULL),
+ .text = xvasprintf (format, args),
};
msg_emit (m);
}
+/* Reports an error during macro expansion. STACK is the stack for reporting
+ the location of the error, MT is the optional token at which the error was
+ detected, and FORMAT along with the varargs is the message to report. */
+static void PRINTF_FORMAT (3, 4)
+macro_error (const struct macro_expansion_stack *stack,
+ const struct macro_token *mt, const char *format, ...)
+{
+ va_list args;
+ va_start (args, format);
+ macro_error_valist (stack, mt, format, args);
+ va_end (args);
+}
+
void
macro_token_copy (struct macro_token *dst, const struct macro_token *src)
{
enum segmenter_mode mode,
const struct macro_expansion_stack *stack)
{
- struct state
- {
- struct segmenter segmenter;
- struct substring body;
- };
-
- struct state state = {
- .segmenter = segmenter_init (mode, true),
- .body = src,
- };
- struct state saved = state;
+ struct segmenter segmenter = segmenter_init (mode, true);
+ struct substring body = src;
- while (state.body.length > 0)
+ while (body.length > 0)
{
struct macro_token mt = {
.token = { .type = T_STOP },
- .syntax = { .string = state.body.string },
+ .syntax = { .string = body.string },
};
struct token *token = &mt.token;
- struct scanner scanner;
- scanner_init (&scanner, token);
+ enum segment_type type;
+ int seg_len = segmenter_push (&segmenter, body.string,
+ body.length, true, &type);
+ assert (seg_len >= 0);
- for (;;)
- {
- enum segment_type type;
- int seg_len = segmenter_push (&state.segmenter, state.body.string,
- state.body.length, true, &type);
- assert (seg_len >= 0);
-
- struct substring segment = ss_head (state.body, seg_len);
- ss_advance (&state.body, seg_len);
-
- enum scan_result result = scanner_push (&scanner, type, segment, token);
- if (result == SCAN_SAVE)
- saved = state;
- else if (result == SCAN_BACK)
- {
- state = saved;
- break;
- }
- else if (result == SCAN_DONE)
- break;
- }
+ struct substring segment = ss_head (body, seg_len);
+ enum tokenize_result result = token_from_segment (type, segment, token);
+ ss_advance (&body, seg_len);
- /* We have a token in 'token'. */
- mt.syntax.length = state.body.string - mt.syntax.string;
- if (is_scan_type (token->type))
+ switch (result)
{
- if (token->type != SCAN_SKIP)
- {
- char *s = scan_token_to_error (token);
- if (stack)
- {
- mt.token.type = T_STRING;
- macro_error (stack, &mt, "%s", s);
- }
- else
- msg (SE, "%s", s);
- free (s);
- }
+ case TOKENIZE_EMPTY:
+ break;
+
+ case TOKENIZE_TOKEN:
+ mt.syntax.length = body.string - mt.syntax.string;
+ macro_tokens_add (mts, &mt);
+ break;
+
+ case TOKENIZE_ERROR:
+ mt.syntax.length = body.string - mt.syntax.string;
+ macro_error (stack, &mt, "%s", token->string.string);
+ break;
}
- else
- macro_tokens_add (mts, &mt);
+
token_uninit (token);
}
}
return;
free (m->name);
- free (m->file_name);
+ msg_location_destroy (m->location);
for (size_t i = 0; i < m->n_params; i++)
{
struct macro_param *p = &m->params[i];
const struct macro_set *macros;
const struct macro *macro;
struct macro_tokens **args;
+ const struct macro_expansion_stack *stack;
enum mc_state state;
size_t n_tokens;
}
}
-static int
-mc_error (struct macro_call *mc)
+static void PRINTF_FORMAT (3, 4)
+mc_error (const struct macro_call *mc, const struct msg_location *loc,
+ const char *format, ...)
{
- mc->state = MC_ERROR;
- return -1;
+ va_list args;
+ va_start (args, format);
+ if (!mc->stack)
+ {
+ const struct macro_expansion_stack stack = { .location = loc };
+ macro_error_valist (&stack, NULL, format, args);
+ }
+ else
+ macro_error_valist (mc->stack, NULL, format, args);
+ va_end (args);
}
static int
-mc_add_arg (struct macro_call *mc, const struct macro_token *mt)
+mc_add_arg (struct macro_call *mc, const struct macro_token *mt,
+ const struct msg_location *loc)
{
const struct macro_param *p = mc->param;
if ((token->type == T_ENDCMD || token->type == T_STOP)
&& p->arg_type != ARG_CMDEND)
{
- msg (SE, _("Unexpected end of command reading argument %s "
- "to macro %s."), mc->param->name, mc->macro->name);
+ mc_error (mc, loc,
+ _("Unexpected end of command reading argument %s "
+ "to macro %s."), mc->param->name, mc->macro->name);
- return mc_error (mc);
+ mc->state = MC_ERROR;
+ return -1;
}
mc->n_tokens++;
static int
mc_expected (struct macro_call *mc, const struct macro_token *actual,
- const struct token *expected)
+ const struct msg_location *loc, const struct token *expected)
{
const struct substring actual_s = (actual->syntax.length ? actual->syntax
: ss_cstr (_("<end of input>")));
char *expected_s = token_to_string (expected);
- msg (SE, _("Found `%.*s' while expecting `%s' reading argument %s "
- "to macro %s."),
- (int) actual_s.length, actual_s.string, expected_s,
- mc->param->name, mc->macro->name);
+ mc_error (mc, loc,
+ _("Found `%.*s' while expecting `%s' reading argument %s "
+ "to macro %s."),
+ (int) actual_s.length, actual_s.string, expected_s,
+ mc->param->name, mc->macro->name);
free (expected_s);
- return mc_error (mc);
+ mc->state = MC_ERROR;
+ return -1;
}
static int
-mc_enclose (struct macro_call *mc, const struct macro_token *mt)
+mc_enclose (struct macro_call *mc, const struct macro_token *mt,
+ const struct msg_location *loc)
{
const struct token *token = &mt->token;
mc->n_tokens++;
return 0;
}
- return mc_expected (mc, mt, &mc->param->enclose[0]);
+ return mc_expected (mc, mt, loc, &mc->param->enclose[0]);
}
static const struct macro_param *
}
static int
-mc_keyword (struct macro_call *mc, const struct macro_token *mt)
+mc_keyword (struct macro_call *mc, const struct macro_token *mt,
+ const struct msg_location *loc)
{
const struct token *token = &mt->token;
if (token->type != T_ID)
mc->param = p;
if (mc->args[arg_index])
{
- msg (SE,
- _("Argument %s multiply specified in call to macro %s."),
- p->name, mc->macro->name);
- return mc_error (mc);
+ mc_error (mc, loc,
+ _("Argument %s multiply specified in call to macro %s."),
+ p->name, mc->macro->name);
+ mc->state = MC_ERROR;
+ return -1;
}
mc->n_tokens++;
}
static int
-mc_equals (struct macro_call *mc, const struct macro_token *mt)
+mc_equals (struct macro_call *mc, const struct macro_token *mt,
+ const struct msg_location *loc)
{
const struct token *token = &mt->token;
mc->n_tokens++;
return 0;
}
- return mc_expected (mc, mt, &(struct token) { .type = T_EQUALS });
+ return mc_expected (mc, mt, loc, &(struct token) { .type = T_EQUALS });
}
-/* If TOKEN is the first token of a call to a macro in MACROS, create a new
- macro expander, initializes *MCP to it. Returns 0 if more tokens are needed
- and should be added via macro_call_add() or 1 if the caller should next call
- macro_call_get_expansion().
-
- If TOKEN is not the first token of a macro call, returns -1 and sets *MCP to
- NULL. */
-int
-macro_call_create (const struct macro_set *macros,
- const struct token *token,
- struct macro_call **mcp)
+static int
+macro_call_create__ (const struct macro_set *macros,
+ const struct macro_expansion_stack *stack,
+ const struct token *token,
+ struct macro_call **mcp)
{
const struct macro *macro = (token->type == T_ID || token->type == T_MACRO_ID
? macro_set_find (macros, token->string.string)
: MC_ARG),
.args = macro->n_params ? xcalloc (macro->n_params, sizeof *mc->args) : NULL,
.param = macro->params,
+ .stack = stack,
};
*mcp = mc;
return mc->state == MC_FINISHED ? 1 : 0;
}
+/* If TOKEN is the first token of a call to a macro in MACROS, create a new
+ macro expander, initializes *MCP to it. Returns 0 if more tokens are needed
+ and should be added via macro_call_add() or 1 if the caller should next call
+ macro_call_get_expansion().
+
+ If TOKEN is not the first token of a macro call, returns -1 and sets *MCP to
+ NULL. */
+int
+macro_call_create (const struct macro_set *macros,
+ const struct token *token,
+ struct macro_call **mcp)
+{
+ return macro_call_create__ (macros, NULL, token, mcp);
+}
+
void
macro_call_destroy (struct macro_call *mc)
{
macro invocation is finished. The caller should call
macro_call_get_expansion() to obtain the expansion. */
int
-macro_call_add (struct macro_call *mc, const struct macro_token *mt)
+macro_call_add (struct macro_call *mc, const struct macro_token *mt,
+ const struct msg_location *loc)
{
switch (mc->state)
{
return -1;
case MC_ARG:
- return mc_add_arg (mc, mt);
+ return mc_add_arg (mc, mt, loc);
case MC_ENCLOSE:
- return mc_enclose (mc, mt);
+ return mc_enclose (mc, mt, loc);
case MC_KEYWORD:
- return mc_keyword (mc, mt);
+ return mc_keyword (mc, mt, loc);
case MC_EQUALS:
- return mc_equals (mc, mt);
+ return mc_equals (mc, mt, loc);
default:
NOT_REACHED ();
int nesting_countdown; /* Remaining nesting levels. */
const struct macro_expansion_stack *stack; /* Stack for error reporting. */
bool *expand; /* May macro calls be expanded? */
- struct stringi_map *vars; /* Variables from !DO and !LET. */
+ struct stringi_map *vars; /* Variables from !do and !let. */
/* Only nonnull if inside a !DO loop. */
bool *break_; /* Set to true to break out of loop. */
string_lexer_init (&slex, s, strlen (s), segmenter_mode, true);
struct token token1;
- if (!string_lexer_next (&slex, &token1))
- return false;
-
- if (token1.type != T_STRING)
+ if (string_lexer_next (&slex, &token1) != SLR_TOKEN
+ || token1.type != T_STRING)
{
token_uninit (&token1);
return false;
}
struct token token2;
- if (string_lexer_next (&slex, &token2))
+ if (string_lexer_next (&slex, &token2) != SLR_END)
{
token_uninit (&token1);
token_uninit (&token2);
if (*me->expand)
{
struct macro_call *submc;
- int n_call = macro_call_create (me->macros, token, &submc);
+ int n_call = macro_call_create__ (me->macros, me->stack, token, &submc);
for (size_t j = 1; !n_call; j++)
{
const struct macro_token endcmd
= { .token = { .type = T_ENDCMD } };
- n_call = macro_call_add (submc, j < n ? &mts[j] : &endcmd);
+ n_call = macro_call_add (submc, j < n ? &mts[j] : &endcmd, NULL);
}
if (n_call > 0)
{
struct stringi_map vars = STRINGI_MAP_INITIALIZER (vars);
struct macro_expansion_stack stack = {
.name = submc->macro->name,
- .file_name = submc->macro->file_name,
- .first_line = submc->macro->first_line,
- .last_line = submc->macro->last_line,
+ .location = submc->macro->location,
.next = me->stack,
};
struct macro_expander subme = {
void
macro_call_expand (struct macro_call *mc, enum segmenter_mode segmenter_mode,
+ const struct msg_location *call_loc,
struct macro_tokens *exp)
{
assert (mc->state == MC_FINISHED);
bool expand = true;
struct stringi_map vars = STRINGI_MAP_INITIALIZER (vars);
- struct macro_expansion_stack stack = {
+ struct macro_expansion_stack stack0 = {
+ .location = call_loc,
+ };
+ struct macro_expansion_stack stack1 = {
+ .next = &stack0,
.name = mc->macro->name,
- .file_name = mc->macro->file_name,
- .first_line = mc->macro->first_line,
- .last_line = mc->macro->last_line,
+ .location = mc->macro->location,
};
struct macro_expander me = {
.macros = mc->macros,
.break_ = NULL,
.vars = &vars,
.nesting_countdown = settings_get_mnest (),
- .stack = &stack,
+ .stack = &stack1,
};
const struct macro_tokens *body = &mc->macro->body;
#include "language/lexer/segment.h"
#include "language/lexer/token.h"
+struct msg_location;
+
/* A token along with the syntax that was tokenized to produce it. The syntax
allows the token to be turned back into syntax accurately. */
struct macro_token
char *name;
/* Source code location of macro definition, for error reporting. */
- char *file_name;
- int first_line;
- int last_line;
+ struct msg_location *location;
/* Parameters. */
struct macro_param *params;
struct macro_call;
int macro_call_create (const struct macro_set *, const struct token *,
- struct macro_call **);
-int macro_call_add (struct macro_call *, const struct macro_token *);
+ struct macro_call **);
+int macro_call_add (struct macro_call *, const struct macro_token *,
+ const struct msg_location *);
void macro_call_expand (struct macro_call *, enum segmenter_mode segmenter_mode,
- struct macro_tokens *);
+ const struct msg_location *call_loc, struct macro_tokens *);
void macro_call_destroy (struct macro_call *);
#include "gettext.h"
#define _(msgid) gettext (msgid)
-enum
- {
- S_START,
- S_DASH,
- S_STRING
- };
-
-#define SS_NL_BEFORE_PLUS (1u << 0)
-#define SS_PLUS (1u << 1)
-#define SS_NL_AFTER_PLUS (1u << 2)
-
/* Returns the integer value of (hex) digit C. */
static int
digit_value (int c)
}
}
-static bool
-scan_quoted_string__ (struct substring s, struct token *token)
+static void
+scan_quoted_string (struct substring in, struct token *token)
{
- int quote;
-
/* Trim ' or " from front and back. */
- quote = s.string[s.length - 1];
- s.string++;
- s.length -= 2;
+ int quote = in.string[0];
+ in.string++;
+ in.length -= 2;
- ss_realloc (&token->string, token->string.length + s.length + 1);
+ struct substring out = { .string = xmalloc (in.length + 1) };
for (;;)
{
- size_t pos = ss_find_byte (s, quote);
+ size_t pos = ss_find_byte (in, quote);
if (pos == SIZE_MAX)
break;
- memcpy (ss_end (token->string), s.string, pos + 1);
- token->string.length += pos + 1;
- ss_advance (&s, pos + 2);
+ memcpy (ss_end (out), in.string, pos + 1);
+ out.length += pos + 1;
+ ss_advance (&in, pos + 2);
}
- memcpy (ss_end (token->string), s.string, ss_length (s));
- token->string.length += ss_length (s);
+ memcpy (ss_end (out), in.string, in.length);
+ out.length += in.length;
+ out.string[out.length] = '\0';
- return true;
+ *token = (struct token) { .type = T_STRING, .string = out };
}
-static bool
-scan_hex_string__ (struct substring s, struct token *token)
+static char *
+scan_hex_string__ (struct substring in, struct substring *out)
{
- uint8_t *dst;
- size_t i;
-
- /* Trim X' from front and ' from back. */
- s.string += 2;
- s.length -= 3;
-
- if (s.length % 2 != 0)
- {
- token->type = SCAN_BAD_HEX_LENGTH;
- token->number = s.length;
- return false;
- }
-
- ss_realloc (&token->string, token->string.length + s.length / 2 + 1);
- dst = CHAR_CAST (uint8_t *, ss_end (token->string));
- token->string.length += s.length / 2;
- for (i = 0; i < s.length; i += 2)
+ if (in.length % 2 != 0)
+ return xasprintf (_("String of hex digits has %zu characters, which "
+ "is not a multiple of 2."), in.length);
+
+ ss_realloc (out, in.length / 2 + 1);
+ uint8_t *dst = CHAR_CAST (uint8_t *, out->string);
+ out->length = in.length / 2;
+ for (size_t i = 0; i < in.length; i += 2)
{
- int hi = digit_value (s.string[i]);
- int lo = digit_value (s.string[i + 1]);
+ int hi = digit_value (in.string[i]);
+ int lo = digit_value (in.string[i + 1]);
if (hi >= 16 || lo >= 16)
- {
- token->type = SCAN_BAD_HEX_DIGIT;
- token->number = s.string[hi >= 16 ? i : i + 1];
- return false;
- }
+ return xasprintf (_("`%c' is not a valid hex digit."),
+ in.string[hi >= 16 ? i : i + 1]);
*dst++ = hi * 16 + lo;
}
- return true;
+ return NULL;
}
-static bool
-scan_unicode_string__ (struct substring s, struct token *token)
+static char *
+scan_unicode_string__ (struct substring in, struct substring *out)
{
- uint8_t *dst;
- ucs4_t uc;
- size_t i;
+ if (in.length < 1 || in.length > 8)
+ return xasprintf (_("Unicode string contains %zu bytes, which is "
+ "not in the valid range of 1 to 8 bytes."),
+ in.length);
- /* Trim U' from front and ' from back. */
- s.string += 2;
- s.length -= 3;
-
- if (s.length < 1 || s.length > 8)
+ ucs4_t uc = 0;
+ for (size_t i = 0; i < in.length; i++)
{
- token->type = SCAN_BAD_UNICODE_LENGTH;
- token->number = s.length;
- return 0;
- }
-
- ss_realloc (&token->string, token->string.length + 4 + 1);
-
- uc = 0;
- for (i = 0; i < s.length; i++)
- {
- int digit = digit_value (s.string[i]);
+ int digit = digit_value (in.string[i]);
if (digit >= 16)
- {
- token->type = SCAN_BAD_UNICODE_DIGIT;
- token->number = s.string[i];
- return 0;
- }
+ return xasprintf (_("`%c' is not a valid hex digit."), in.string[i]);
uc = uc * 16 + digit;
}
if ((uc >= 0xd800 && uc < 0xe000) || uc > 0x10ffff)
- {
- token->type = SCAN_BAD_UNICODE_CODE_POINT;
- token->number = uc;
- return 0;
- }
-
- dst = CHAR_CAST (uint8_t *, ss_end (token->string));
- token->string.length += u8_uctomb (dst, uc, 4);
+ return xasprintf (_("U+%04llX is not a valid Unicode code point."),
+ (long long) uc);
- return true;
-}
-
-static enum scan_result
-scan_string_segment__ (struct scanner *scanner, enum segment_type type,
- struct substring s, struct token *token)
-{
- bool ok;
-
- switch (type)
- {
- case SEG_QUOTED_STRING:
- ok = scan_quoted_string__ (s, token);
- break;
-
- case SEG_HEX_STRING:
- ok = scan_hex_string__ (s, token);
- break;
-
- case SEG_UNICODE_STRING:
- ok = scan_unicode_string__ (s, token);
- break;
-
- default:
- NOT_REACHED ();
- }
+ ss_realloc (out, 4 + 1);
+ out->length = u8_uctomb (CHAR_CAST (uint8_t *, ss_end (*out)), uc, 4);
- if (ok)
- {
- token->type = T_STRING;
- token->string.string[token->string.length] = '\0';
- scanner->state = S_STRING;
- scanner->substate = 0;
- return SCAN_SAVE;
- }
- else
- {
- /* The function we called above should have filled in token->type and
- token->number properly to describe the error. */
- ss_dealloc (&token->string);
- token->string = ss_empty ();
- return SCAN_DONE;
- }
-
-}
-
-static enum scan_result
-add_bit (struct scanner *scanner, unsigned int bit)
-{
- if (!(scanner->substate & bit))
- {
- scanner->substate |= bit;
- return SCAN_MORE;
- }
- else
- return SCAN_BACK;
-}
-
-static enum scan_result
-scan_string__ (struct scanner *scanner, enum segment_type type,
- struct substring s, struct token *token)
-{
- switch (type)
- {
- case SEG_SPACES:
- case SEG_COMMENT:
- return SCAN_MORE;
-
- case SEG_NEWLINE:
- if (scanner->substate & SS_PLUS)
- return add_bit (scanner, SS_NL_AFTER_PLUS);
- else
- return add_bit (scanner, SS_NL_BEFORE_PLUS);
-
- case SEG_PUNCT:
- return (s.length == 1 && s.string[0] == '+'
- ? add_bit (scanner, SS_PLUS)
- : SCAN_BACK);
-
- case SEG_QUOTED_STRING:
- case SEG_HEX_STRING:
- case SEG_UNICODE_STRING:
- return (scanner->substate & SS_PLUS
- ? scan_string_segment__ (scanner, type, s, token)
- : SCAN_BACK);
-
- default:
- return SCAN_BACK;
- }
+ return NULL;
}
static enum token_type
if (p != buf)
free (p);
}
-
-static enum scan_result
-scan_unexpected_char (const struct substring *s, struct token *token)
+\f
+static void
+tokenize_error__ (struct token *token, char *error)
{
- ucs4_t uc;
-
- token->type = SCAN_UNEXPECTED_CHAR;
- u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, s->string), s->length);
- token->number = uc;
-
- return SCAN_DONE;
+ *token = (struct token) { .type = T_STRING, .string = ss_cstr (error) };
}
-const char *
-scan_type_to_string (enum scan_type type)
+static enum tokenize_result
+tokenize_string_segment__ (enum segment_type type,
+ struct substring s, struct token *token)
{
- switch (type)
- {
-#define SCAN_TYPE(NAME) case SCAN_##NAME: return #NAME;
- SCAN_TYPES
-#undef SCAN_TYPE
+ /* Trim X' or U' from front and ' from back. */
+ s.string += 2;
+ s.length -= 3;
- default:
- return token_type_to_name ((enum token_type) type);
+ struct substring out = SS_EMPTY_INITIALIZER;
+ char *error = (type == SEG_HEX_STRING
+ ? scan_hex_string__ (s, &out)
+ : scan_unicode_string__ (s, &out));
+ if (!error)
+ {
+ out.string[out.length] = '\0';
+ *token = (struct token) { .type = T_STRING, .string = out };
+ return TOKENIZE_TOKEN;
+ }
+ else
+ {
+ tokenize_error__ (token, error);
+ return TOKENIZE_ERROR;
}
}
-bool
-is_scan_type (enum scan_type type)
-{
- return type > SCAN_FIRST && type < SCAN_LAST;
-}
-
-/* If TOKEN has the type of a scan error (a subset of those identified by
- is_scan_type()), returns an appropriate error message. Otherwise, returns
- NULL. */
-char *
-scan_token_to_error (const struct token *token)
+static void
+tokenize_unexpected_char (const struct substring *s, struct token *token)
{
- switch (token->type)
- {
- case SCAN_BAD_HEX_LENGTH:
- return xasprintf (_("String of hex digits has %d characters, which "
- "is not a multiple of 2."), (int) token->number);
-
- case SCAN_BAD_HEX_DIGIT:
- case SCAN_BAD_UNICODE_DIGIT:
- return xasprintf (_("`%c' is not a valid hex digit."),
- (int) token->number);
-
- case SCAN_BAD_UNICODE_LENGTH:
- return xasprintf (_("Unicode string contains %d bytes, which is "
- "not in the valid range of 1 to 8 bytes."),
- (int) token->number);
-
- case SCAN_BAD_UNICODE_CODE_POINT:
- return xasprintf (_("U+%04X is not a valid Unicode code point."),
- (int) token->number);
-
- case SCAN_EXPECTED_QUOTE:
- return xasprintf (_("Unterminated string constant."));
-
- case SCAN_EXPECTED_EXPONENT:
- return xasprintf (_("Missing exponent following `%s'."),
- token->string.string);
-
- case SCAN_UNEXPECTED_CHAR:
- {
- char c_name[16];
- return xasprintf (_("Bad character %s in input."),
- uc_name (token->number, c_name));
- }
- }
+ ucs4_t uc;
+ u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, s->string), s->length);
- return NULL;
+ char c_name[16];
+ tokenize_error__ (token, xasprintf (_("Bad character %s in input."),
+ uc_name (uc, c_name)));
}
-static enum scan_result
-scan_start__ (struct scanner *scanner, enum segment_type type,
- struct substring s, struct token *token)
+enum tokenize_result
+token_from_segment (enum segment_type type, struct substring s,
+ struct token *token)
{
switch (type)
{
case SEG_NUMBER:
scan_number__ (s, token);
- return SCAN_DONE;
+ return TOKENIZE_TOKEN;
case SEG_QUOTED_STRING:
+ scan_quoted_string (s, token);
+ return TOKENIZE_TOKEN;
+
case SEG_HEX_STRING:
case SEG_UNICODE_STRING:
- return scan_string_segment__ (scanner, type, s, token);
+ return tokenize_string_segment__ (type, s, token);
case SEG_UNQUOTED_STRING:
case SEG_DO_REPEAT_COMMAND:
case SEG_INLINE_DATA:
case SEG_DOCUMENT:
case SEG_MACRO_BODY:
- token->type = T_STRING;
+ *token = (struct token) { .type = T_STRING };
ss_alloc_substring (&token->string, s);
- return SCAN_DONE;
+ return TOKENIZE_TOKEN;
case SEG_RESERVED_WORD:
- token->type = scan_reserved_word__ (s);
- return SCAN_DONE;
+ *token = (struct token) { .type = scan_reserved_word__ (s) };
+ return TOKENIZE_TOKEN;
case SEG_IDENTIFIER:
- token->type = T_ID;
+ *token = (struct token) { .type = T_ID };
ss_alloc_substring (&token->string, s);
- return SCAN_DONE;
+ return TOKENIZE_TOKEN;
case SEG_MACRO_ID:
- token->type = T_MACRO_ID;
+ *token = (struct token) { .type = T_MACRO_ID };
ss_alloc_substring (&token->string, s);
- return SCAN_DONE;
+ return TOKENIZE_TOKEN;
case SEG_PUNCT:
- if (s.length == 1 && s.string[0] == '-')
- {
- scanner->state = S_DASH;
- return SCAN_SAVE;
- }
- else
- {
- token->type = scan_punct__ (s);
- if (token->type == T_MACRO_PUNCT)
- ss_alloc_substring (&token->string, s);
- return SCAN_DONE;
- }
+ *token = (struct token) { .type = scan_punct__ (s) };
+ if (token->type == T_MACRO_PUNCT)
+ ss_alloc_substring (&token->string, s);
+ return TOKENIZE_TOKEN;
case SEG_SHBANG:
case SEG_SPACES:
case SEG_COMMENT:
case SEG_NEWLINE:
case SEG_COMMENT_COMMAND:
- token->type = SCAN_SKIP;
- return SCAN_DONE;
+ return TOKENIZE_EMPTY;
case SEG_START_DOCUMENT:
- token->type = T_ID;
+ *token = (struct token) { .type = T_ID };
ss_alloc_substring (&token->string, ss_cstr ("DOCUMENT"));
- return SCAN_DONE;
+ return TOKENIZE_TOKEN;
case SEG_START_COMMAND:
case SEG_SEPARATE_COMMANDS:
case SEG_END_COMMAND:
- token->type = T_ENDCMD;
- return SCAN_DONE;
+ *token = (struct token) { .type = T_ENDCMD };
+ return TOKENIZE_TOKEN;
case SEG_END:
- token->type = T_STOP;
- return SCAN_DONE;
+ *token = (struct token) { .type = T_STOP };
+ return TOKENIZE_TOKEN;
case SEG_EXPECTED_QUOTE:
- token->type = SCAN_EXPECTED_QUOTE;
- return SCAN_DONE;
+ tokenize_error__ (token, xasprintf (_("Unterminated string constant.")));
+ return TOKENIZE_ERROR;
case SEG_EXPECTED_EXPONENT:
- token->type = SCAN_EXPECTED_EXPONENT;
- ss_alloc_substring (&token->string, s);
- return SCAN_DONE;
+ tokenize_error__ (token,
+ xasprintf (_("Missing exponent following `%.*s'."),
+ (int) s.length, s.string));
+ return TOKENIZE_ERROR;
case SEG_UNEXPECTED_CHAR:
- return scan_unexpected_char (&s, token);
+ tokenize_unexpected_char (&s, token);
+ return TOKENIZE_ERROR;
}
NOT_REACHED ();
}
-static enum scan_result
-scan_dash__ (enum segment_type type, struct substring s, struct token *token)
-{
- switch (type)
- {
- case SEG_SPACES:
- case SEG_COMMENT:
- return SCAN_MORE;
-
- case SEG_NUMBER:
- scan_number__ (s, token);
- token->type = T_NEG_NUM;
- token->number = -token->number;
- return SCAN_DONE;
-
- default:
- token->type = T_DASH;
- return SCAN_BACK;
- }
-}
-
-/* Initializes SCANNER for scanning a token from a sequence of segments.
- Initializes TOKEN as the output token. (The client retains ownership of
- TOKEN, but it must be preserved across subsequent calls to scanner_push()
- for SCANNER.)
-
- A scanner only produces a single token. To obtain the next token,
- re-initialize it by calling this function again.
-
- A scanner does not contain any external references, so nothing needs to be
- done to destroy one. For the same reason, scanners may be copied with plain
- struct assignment (or memcpy). */
-void
-scanner_init (struct scanner *scanner, struct token *token)
-{
- scanner->state = S_START;
- *token = (struct token) { .type = T_STOP };
-}
-
-/* Adds the segment with type TYPE and UTF-8 text S to SCANNER. TOKEN must be
- the same token passed to scanner_init() for SCANNER, or a copy of it.
- scanner_push() may modify TOKEN. The client retains ownership of TOKEN,
-
- The possible return values are:
-
- - SCAN_DONE: All of the segments that have been passed to scanner_push()
- form the token now stored in TOKEN. SCANNER is now "used up" and must
- be reinitialized with scanner_init() if it is to be used again.
-
- Most tokens only consist of a single segment, so this is the most common
- return value.
-
- - SCAN_MORE: The segments passed to scanner_push() don't yet determine a
- token. The caller should call scanner_push() again with the next token.
- (This won't happen if TYPE is SEG_END indicating the end of input.)
-
- - SCAN_SAVE: This is similar to SCAN_MORE, with one difference: the caller
- needs to "save its place" in the stream of segments for a possible
- future SCAN_BACK return. This value can be returned more than once in a
- sequence of scanner_push() calls for SCANNER, but the caller only needs
- to keep track of the most recent position.
-
- - SCAN_BACK: This is similar to SCAN_DONE, but the token consists of only
- the segments up to and including the segment for which SCAN_SAVE was
- most recently returned. Segments following that one should be passed to
- the next scanner to be initialized.
-*/
-enum scan_result
-scanner_push (struct scanner *scanner, enum segment_type type,
- struct substring s, struct token *token)
-{
- switch (scanner->state)
- {
- case S_START:
- return scan_start__ (scanner, type, s, token);
-
- case S_DASH:
- return scan_dash__ (type, s, token);
-
- case S_STRING:
- return scan_string__ (scanner, type, s, token);
- }
-
- NOT_REACHED ();
-}
\f
/* Initializes SLEX for parsing INPUT, which is LENGTH bytes long, in the
specified MODE.
}
/* */
-bool
+enum string_lexer_result
string_lexer_next (struct string_lexer *slex, struct token *token)
{
- struct segmenter saved_segmenter;
- size_t saved_offset = 0;
-
- struct scanner scanner;
-
- scanner_init (&scanner, token);
for (;;)
{
const char *s = slex->input + slex->offset;
assert (n >= 0);
slex->offset += n;
- switch (scanner_push (&scanner, type, ss_buffer (s, n), token))
+ switch (token_from_segment (type, ss_buffer (s, n), token))
{
- case SCAN_BACK:
- slex->segmenter = saved_segmenter;
- slex->offset = saved_offset;
- /* Fall through. */
- case SCAN_DONE:
- return token->type != T_STOP;
-
- case SCAN_MORE:
- break;
+ case TOKENIZE_TOKEN:
+ return token->type == T_STOP ? SLR_END : SLR_TOKEN;
+
+ case TOKENIZE_ERROR:
+ return SLR_ERROR;
- case SCAN_SAVE:
- saved_segmenter = slex->segmenter;
- saved_offset = slex->offset;
+ case TOKENIZE_EMPTY:
break;
}
}
}
+
+static struct substring
+concat (struct substring a, struct substring b)
+{
+ size_t length = a.length + b.length;
+ struct substring out = { .string = xmalloc (length + 1), .length = length };
+ memcpy (out.string, a.string, a.length);
+ memcpy (out.string + a.length, b.string, b.length);
+ out.string[length] = '\0';
+ return out;
+}
+
+/* Attempts to merge a sequence of tokens together into a single token. The
+ caller feeds tokens in one by one and the merger FSM reports progress. The
+ caller must supply a merger structure M that is set to MERGER_INIT before
+ the first call. The caller must also supply a token OUT for storage, which
+ need not be initialized.
+
+ Returns:
+
+ * -1 if more tokens are needed. Token OUT might be in use for temporary
+ storage; to ensure that it is freed, continue calling merger_add() until
+ it returns something other than -1. (T_STOP or T_ENDCMD will make it do
+ that.)
+
+ * 0 if the first token submitted to the merger is the output. This is the
+ common case for the first call, and it can be returned for subsequent
+ calls as well.
+
+ * A positive number if OUT is initialized to the output token. The return
+ value is the number of tokens being merged to produce this one. */
+int
+merger_add (struct merger *m, const struct token *in, struct token *out)
+{
+ /* We perform two different kinds of token merging:
+
+ - String concatenation, where syntax like "a" + "b" is converted into a
+ single string token. This is definitely needed because the parser
+ relies on it.
+
+ - Negative number merging, where syntax like -5 is converted from a pair
+ of tokens (T_DASH then T_POS_NUM) into a single token (T_NEG_NUM). This
+ might not be needed anymore because the segmenter directly treats a dash
+ followed by a number, with optional intervening white space, as a
+ negative number. It's only needed if we want intervening comments to be
+ allowed or for part of the negative number token to be produced by macro
+ expansion. */
+ switch (++m->state)
+ {
+ case 1:
+ if (in->type == T_DASH || in->type == T_STRING)
+ {
+ *out = *in;
+ return -1;
+ }
+ else
+ return 0;
+
+ case 2:
+ if (out->type == T_DASH)
+ {
+ if (in->type == T_POS_NUM)
+ {
+ *out = (struct token) {
+ .type = T_NEG_NUM,
+ .number = -in->number
+ };
+ return 2;
+ }
+ else
+ return 0;
+ }
+ else
+ return in->type == T_PLUS ? -1 : 0;
+ NOT_REACHED ();
+
+ case 3:
+ if (in->type == T_STRING)
+ {
+ out->string = concat (out->string, in->string);
+ return -1;
+ }
+ else
+ return 0;
+ NOT_REACHED ();
+
+ default:
+ if (!(m->state % 2))
+ return in->type == T_PLUS ? -1 : m->state - 1;
+ else
+ {
+ if (in->type == T_STRING)
+ {
+ struct substring s = concat (out->string, in->string);
+ ss_swap (&s, &out->string);
+ ss_dealloc (&s);
+ return -1;
+ }
+ else
+ return m->state - 2;
+ }
+ NOT_REACHED ();
+ }
+}
types.
*/
-#define SCAN_TYPES \
- SCAN_TYPE(BAD_HEX_LENGTH) \
- SCAN_TYPE(BAD_HEX_DIGIT) \
- \
- SCAN_TYPE(BAD_UNICODE_LENGTH) \
- SCAN_TYPE(BAD_UNICODE_DIGIT) \
- SCAN_TYPE(BAD_UNICODE_CODE_POINT) \
- \
- SCAN_TYPE(EXPECTED_QUOTE) \
- SCAN_TYPE(EXPECTED_EXPONENT) \
- SCAN_TYPE(UNEXPECTED_CHAR) \
- \
- SCAN_TYPE(SKIP)
-
-/* Types of scan tokens.
-
- Scan token types are a superset of enum token_type. Only the additional
- scan token types are defined here, so see the definition of enum token_type
- for the others. */
-enum scan_type
+enum tokenize_result
{
-#define SCAN_TYPE(TYPE) SCAN_##TYPE,
- SCAN_FIRST = 255,
- SCAN_TYPES
- SCAN_LAST
-#undef SCAN_TYPE
+ TOKENIZE_EMPTY,
+ TOKENIZE_TOKEN,
+ TOKENIZE_ERROR
};
-const char *scan_type_to_string (enum scan_type);
-bool is_scan_type (enum scan_type);
+enum tokenize_result token_from_segment (enum segment_type, struct substring,
+ struct token *);
-char *scan_token_to_error (const struct token *);
-
-/* A scanner. Opaque. */
-struct scanner
- {
- unsigned char state;
- unsigned char substate;
- };
-
-/* scanner_push() return type. */
-enum scan_result
+struct merger
{
- /* Complete token. */
- SCAN_DONE, /* Token successfully scanned. */
- SCAN_MORE, /* More segments needed to scan token. */
-
- /* Incomplete token. */
- SCAN_BACK, /* Done, but go back to saved position too. */
- SCAN_SAVE /* Need more segments, and save position. */
+ unsigned int state;
};
+#define MERGER_INIT { 0 }
-void scanner_init (struct scanner *, struct token *);
-enum scan_result scanner_push (struct scanner *, enum segment_type,
- struct substring, struct token *);
+int merger_add (struct merger *m, const struct token *in, struct token *out);
\f
/* A simplified lexer for handling syntax in a string. */
struct segmenter segmenter;
};
+enum string_lexer_result
+ {
+ SLR_END,
+ SLR_TOKEN,
+ SLR_ERROR
+ };
+
void string_lexer_init (struct string_lexer *, const char *input,
size_t length, enum segmenter_mode, bool is_snippet);
-bool string_lexer_next (struct string_lexer *, struct token *);
+enum string_lexer_result string_lexer_next (struct string_lexer *,
+ struct token *);
#endif /* scan.h */
#include "libpspp/str.h"
#include "data/identifier.h"
-/* A PSPP syntax token.
-
- The 'type' member is used by the scanner (see scan.h) for SCAN_* values as
- well, which is why it is not declared as type "enum token_type". */
+/* A PSPP syntax token. */
struct token
{
- int type; /* Usually a "enum token_type" value. */
+ enum token_type type;
double number;
struct substring string;
};
{
if (stack)
{
- msg_location_uninit (&stack->location);
+ msg_location_destroy (stack->location);
free (stack->description);
free (stack);
}
{
struct msg_stack *dst = xmalloc (sizeof *src);
*dst = (struct msg_stack) {
- .location = src->location,
+ .location = msg_location_dup (src->location),
.description = xstrdup_if_nonnull (src->description),
};
- dst->location.file_name = xstrdup_if_nonnull (dst->location.file_name);
return dst;
}
\f
for (size_t i = 0; i < m->n_stack; i++)
{
const struct msg_stack *ms = m->stack[i];
- if (!msg_location_is_empty (&ms->location))
+ if (!msg_location_is_empty (ms->location))
{
- msg_location_format (&ms->location, &s);
+ msg_location_format (ms->location, &s);
ds_put_cstr (&s, ": ");
}
ds_put_format (&s, "%s\n", ms->description);
struct msg_stack
{
- struct msg_location location;
+ struct msg_location *location;
char *description;
};
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
"Too few tokens for !TOKENS."
-define.sps:13: error: DEBUG EXPAND: Unexpected end of command reading
+define.sps:13.7: error: DEBUG EXPAND: Unexpected end of command reading
argument !3 to macro !p.
note: unexpanded token "!p"
note: unexpanded token "b"
-define.sps:14: error: DEBUG EXPAND: Unexpected end of command reading
+define.sps:14.5: error: DEBUG EXPAND: Unexpected end of command reading
argument !2 to macro !p.
note: unexpanded token "!p"
note: unexpanded token "a"
-define.sps:15: error: DEBUG EXPAND: Unexpected end of command reading
+define.sps:15.3: error: DEBUG EXPAND: Unexpected end of command reading
argument !1 to macro !p.
note: unexpanded token "!p"
"Missing charend delimiter."
-define.sps:18: error: DEBUG EXPAND: Unexpected end of command reading
+define.sps:18.10: error: DEBUG EXPAND: Unexpected end of command reading
argument !1 to macro !ce.
note: unexpanded token "!ce"
"Missing start delimiter."
-define.sps:21: error: DEBUG EXPAND: Found `a' while expecting `{' reading
+define.sps:21.7: error: DEBUG EXPAND: Found `a' while expecting `{' reading
argument !1 to macro !enc1.
note: unexpanded token "!enc1"
"Missing end delimiter."
-define.sps:24: error: DEBUG EXPAND: Unexpected end of command reading
+define.sps:24.12: error: DEBUG EXPAND: Unexpected end of command reading
argument !1 to macro !enc1.
note: unexpanded token "!enc1"
[DEFINE !k(arg1 = !TOKENS(1)) k(!arg1) !ENDDEFINE.],
[!k arg1.
!k arg1=.], [dnl
-define.sps:3: error: DEBUG EXPAND: Found `.' while expecting `=' reading
+define.sps:3.8: error: DEBUG EXPAND: Found `.' while expecting `=' reading
argument !arg1 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
-define.sps:4: error: DEBUG EXPAND: Unexpected end of command reading argument !
-arg1 to macro !k.
+define.sps:4.9: error: DEBUG EXPAND: Unexpected end of command reading argument
+!arg1 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
note: unexpanded token "="], [1])
!k arg1=.
!k arg1=x.
!k arg1=x/ arg2=y.],
- [define.sps:6: error: DEBUG EXPAND: Found `.' while expecting `=' reading
+ [define.sps:6.8: error: DEBUG EXPAND: Found `.' while expecting `=' reading
argument !arg1 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
-define.sps:7: error: DEBUG EXPAND: Unexpected end of command reading argument !
-arg1 to macro !k.
+define.sps:7.9: error: DEBUG EXPAND: Unexpected end of command reading argument
+!arg1 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
note: unexpanded token "="
-define.sps:8: error: DEBUG EXPAND: Unexpected end of command reading argument !
-arg1 to macro !k.
+define.sps:8.10: error: DEBUG EXPAND: Unexpected end of command reading
+argument !arg1 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
note: unexpanded token "="
note: unexpanded token "x"
-define.sps:9: error: DEBUG EXPAND: Unexpected end of command reading argument !
-arg2 to macro !k.
+define.sps:9.18: error: DEBUG EXPAND: Unexpected end of command reading
+argument !arg2 to macro !k.
note: unexpanded token "!k"
note: unexpanded token "arg1"
note: unexpanded token "="
[!s.],
[define.sps:1-10: At `"ba' in the expansion of `!s',dnl "
-define.sps:12: error: DEBUG EXPAND: Unterminated string constant.
+define.sps:12.1-12.2: error: DEBUG EXPAND: Unterminated string constant.
nana.
nan.
.
define.sps:1-3: inside the expansion of `!macro',
define.sps:1-3: inside the expansion of `!macro',
define.sps:1-3: inside the expansion of `!macro',
-define.sps:4: error: DEFINE: Maximum nesting level 50 exceeded. (Use SET MNEST to change the limit.)"
+define.sps:4.1-4.6: error: DEFINE: Maximum nesting level 50 exceeded. (Use SET MNEST to change the limit.)"
define.sps:4.1-4.6: error: Syntax error at `!macro' (in expansion of `!macro'): expecting command name.
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1-3: At `!x' in the expansion of `!for',
-define.sps:10: error: DEBUG EXPAND: Cannot use argument name or macro keyword
-as !DO variable.
+define.sps:10.1-10.12: error: DEBUG EXPAND: Cannot use argument name or macro
+keyword as !DO variable.
!DO 1 = 1 !TO 5 !var !DOEND.
define.sps:5-7: At `!noexpand' in the expansion of `!for2',
-define.sps:11: error: DEBUG EXPAND: Cannot use argument name or macro keyword
-as !DO variable.
+define.sps:11.1-11.13: error: DEBUG EXPAND: Cannot use argument name or macro
+keyword as !DO variable.
!DO !noexpand = 1 !TO 5 !var !DOEND.
])
In the expansion of `!DO',
define.sps:3-5: inside the expansion of `!for',
-define.sps:14: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum number
-of iterations 3. (Use SET MITERATE to change the limit.)
+define.sps:14.1-14.8: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum
+number of iterations 3. (Use SET MITERATE to change the limit.)
1 2 3 4.
In the expansion of `!DO',
define.sps:7-9: inside the expansion of `!forby',
-define.sps:15: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum number
-of iterations 3. (Use SET MITERATE to change the limit.)
+define.sps:15.1-15.12: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum
+number of iterations 3. (Use SET MITERATE to change the limit.)
1 2 3 4.
In the expansion of `!DO',
define.sps:7-9: inside the expansion of `!forby',
-define.sps:23: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum number
-of iterations 3. (Use SET MITERATE to change the limit.)
+define.sps:23.1-23.13: error: DEBUG EXPAND: Numerical !DO loop exceeded maximum
+number of iterations 3. (Use SET MITERATE to change the limit.)
5 4 3 2.
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
In the expansion of `!DO',
define.sps:1-3: inside the expansion of `!for',
-define.sps:7: error: DEBUG EXPAND: !DO loop over list exceeded maximum number
-of iterations 2. (Use SET MITERATE to change the limit.)
+define.sps:7.1-7.11: error: DEBUG EXPAND: !DO loop over list exceeded maximum
+number of iterations 2. (Use SET MITERATE to change the limit.)
( (a) (b) ).
In the expansion of `!DO',
define.sps:1-3: inside the expansion of `!for',
-define.sps:8: error: DEBUG EXPAND: !DO loop over list exceeded maximum number
-of iterations 2. (Use SET MITERATE to change the limit.)
+define.sps:8.1-8.24: error: DEBUG EXPAND: !DO loop over list exceeded maximum
+number of iterations 2. (Use SET MITERATE to change the limit.)
( (foo) (bar) ).
])
AT_CHECK([pspp --testing-mode define.sps -O format=csv], [1], [dnl
"define.sps:1-3: At `!x' in the expansion of `!macro',
-define.sps:10: error: DEBUG EXPAND: Cannot use argument name or macro keyword ""!x"" as !LET variable."
+define.sps:10.1-10.10: error: DEBUG EXPAND: Cannot use argument name or macro keyword ""!x"" as !LET variable."
!LET 1 = 1
"define.sps:5-7: At `!do' in the expansion of `!macro2',
-define.sps:11: error: DEBUG EXPAND: Cannot use argument name or macro keyword ""!do"" as !LET variable."
+define.sps:11.1-11.7: error: DEBUG EXPAND: Cannot use argument name or macro keyword ""!do"" as !LET variable."
"define.sps:5-7: At `=' in the expansion of `!macro2',
-define.sps:11: error: DEBUG EXPAND: Expected macro variable name following !DO."
+define.sps:11.1-11.7: error: DEBUG EXPAND: Expected macro variable name following !DO."
!LET !do = x
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1: In the expansion of `!a',
-define.sps:18: error: DEBUG EXPAND: `@{:@' expected following !SUBSTR.
+define.sps:18.1-18.2: error: DEBUG EXPAND: `@{:@' expected following !SUBSTR.
!SUBSTR
define.sps:2: At `x' in the expansion of `!b',
-define.sps:19: error: DEBUG EXPAND: `@{:@' expected following !SUBSTR.
+define.sps:19.1-19.2: error: DEBUG EXPAND: `@{:@' expected following !SUBSTR.
!SUBSTR x
define.sps:3: At `x' in the expansion of `!c',
-define.sps:20: error: DEBUG EXPAND: `,' or `@:}@' expected in call to macro
+define.sps:20.1-20.2: error: DEBUG EXPAND: `,' or `@:}@' expected in call to macro
function !SUBSTR.
!SUBSTR(1 x)
define.sps:4: In the expansion of `!d',
-define.sps:21: error: DEBUG EXPAND: Missing `@:}@' in call to macro function !
-SUBSTR.
+define.sps:21.1-21.2: error: DEBUG EXPAND: Missing `@:}@' in call to macro
+function !SUBSTR.
!SUBSTR@{:@1
define.sps:5: In the expansion of `!narg_blanks',
-define.sps:22: error: DEBUG EXPAND: Macro function !BLANKS takes one argument
-(not 0).
+define.sps:22.1-22.12: error: DEBUG EXPAND: Macro function !BLANKS takes one
+argument (not 0).
!BLANKS( )
define.sps:6: In the expansion of `!narg_concat',
-define.sps:23: error: DEBUG EXPAND: Macro function !CONCAT needs at least one
-argument.
+define.sps:23.1-23.12: error: DEBUG EXPAND: Macro function !CONCAT needs at
+least one argument.
!CONCAT( )
define.sps:7: In the expansion of `!narg_eval',
-define.sps:24: error: DEBUG EXPAND: Macro function !EVAL takes one argument
-(not 0).
+define.sps:24.1-24.10: error: DEBUG EXPAND: Macro function !EVAL takes one
+argument (not 0).
!EVAL( )
define.sps:8: In the expansion of `!narg_head',
-define.sps:25: error: DEBUG EXPAND: Macro function !HEAD takes one argument
-(not 0).
+define.sps:25.1-25.10: error: DEBUG EXPAND: Macro function !HEAD takes one
+argument (not 0).
!HEAD( )
define.sps:9: In the expansion of `!narg_index',
-define.sps:26: error: DEBUG EXPAND: Macro function !INDEX takes two arguments
-(not 0).
+define.sps:26.1-26.11: error: DEBUG EXPAND: Macro function !INDEX takes two
+arguments (not 0).
!INDEX( )
define.sps:10: In the expansion of `!narg_length',
-define.sps:27: error: DEBUG EXPAND: Macro function !LENGTH takes one argument
-(not 0).
+define.sps:27.1-27.12: error: DEBUG EXPAND: Macro function !LENGTH takes one
+argument (not 0).
!LENGTH( )
( )
define.sps:12: In the expansion of `!narg_quote',
-define.sps:29: error: DEBUG EXPAND: Macro function !QUOTE takes one argument
-(not 0).
+define.sps:29.1-29.11: error: DEBUG EXPAND: Macro function !QUOTE takes one
+argument (not 0).
!QUOTE( )
define.sps:13: In the expansion of `!narg_substr',
-define.sps:30: error: DEBUG EXPAND: Macro function !SUBSTR takes two or three
-arguments (not 0).
+define.sps:30.1-30.12: error: DEBUG EXPAND: Macro function !SUBSTR takes two or
+three arguments (not 0).
!SUBSTR( )
define.sps:14: In the expansion of `!narg_tail',
-define.sps:31: error: DEBUG EXPAND: Macro function !TAIL takes one argument
-(not 0).
+define.sps:31.1-31.10: error: DEBUG EXPAND: Macro function !TAIL takes one
+argument (not 0).
!TAIL( )
define.sps:15: In the expansion of `!narg_unquote',
-define.sps:32: error: DEBUG EXPAND: Macro function !UNQUOTE takes one argument
-(not 0).
+define.sps:32.1-32.13: error: DEBUG EXPAND: Macro function !UNQUOTE takes one
+argument (not 0).
!UNQUOTE( )
define.sps:16: In the expansion of `!narg_upcase',
-define.sps:33: error: DEBUG EXPAND: Macro function !UPCASE takes one argument
-(not 0).
+define.sps:33.1-33.12: error: DEBUG EXPAND: Macro function !UPCASE takes one
+argument (not 0).
!UPCASE( )
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1: In the expansion of `!a',
-define.sps:5: error: DEBUG EXPAND: Argument to !BLANKS must be non-negative
-integer (not "x").
+define.sps:5.1-5.2: error: DEBUG EXPAND: Argument to !BLANKS must be non-
+negative integer (not "x").
!BLANKS(x).
define.sps:2: In the expansion of `!b',
-define.sps:6: error: DEBUG EXPAND: Second argument of !SUBSTR must be positive
-integer (not "y").
+define.sps:6.1-6.2: error: DEBUG EXPAND: Second argument of !SUBSTR must be
+positive integer (not "y").
!SUBSTR(x, y).
define.sps:3: In the expansion of `!c',
-define.sps:7: error: DEBUG EXPAND: Third argument of !SUBSTR must be non-
+define.sps:7.1-7.2: error: DEBUG EXPAND: Third argument of !SUBSTR must be non-
negative integer (not "z").
!SUBSTR(x, 1, z).
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1-2: At `.' in the expansion of `!a',
-define.sps:5: error: DEBUG EXPAND: Expecting ')' in macro expression.
+define.sps:5.1-5.2: error: DEBUG EXPAND: Expecting ')' in macro expression.
!LET !x = (1.
At `x' in the expansion of `!DO',
define.sps:2: inside the expansion of `!b',
-define.sps:6: error: DEBUG EXPAND: Macro expression must evaluate to a number
-(not "x").
+define.sps:6.1-6.2: error: DEBUG EXPAND: Macro expression must evaluate to a
+number (not "x").
!DO !x = x.
define.sps:3: At `)' in the expansion of `!c',
-define.sps:7: error: DEBUG EXPAND: Expecting literal or function invocation in
-macro expression.
+define.sps:7.1-7.2: error: DEBUG EXPAND: Expecting literal or function
+invocation in macro expression.
!LET !x = ( ).
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1: In the expansion of `!a',
-define.sps:5: error: DEBUG EXPAND: !THEN expected in macro !IF construct.
+define.sps:5.1-5.2: error: DEBUG EXPAND: !THEN expected in macro !IF construct.
!IF 1
define.sps:2: In the expansion of `!b',
-define.sps:6: error: DEBUG EXPAND: !ELSE or !IFEND expected in macro !IF
+define.sps:6.1-6.2: error: DEBUG EXPAND: !ELSE or !IFEND expected in macro !IF
construct.
!IF 1 !THEN
define.sps:3: In the expansion of `!c',
-define.sps:7: error: DEBUG EXPAND: !IFEND expected in macro !IF construct.
+define.sps:7.1-7.2: error: DEBUG EXPAND: !IFEND expected in macro !IF
+construct.
!IF 1 !THEN !ELSE
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1: In the expansion of `!a',
-define.sps:6: error: DEBUG EXPAND: Expected macro variable name following !LET.
+define.sps:6.1-6.2: error: DEBUG EXPAND: Expected macro variable name following
+!LET.
!LET
define.sps:2: At `0' in the expansion of `!b',
-define.sps:7: error: DEBUG EXPAND: Expected macro variable name following !LET.
+define.sps:7.1-7.2: error: DEBUG EXPAND: Expected macro variable name following
+!LET.
!LET 0
define.sps:3: In the expansion of `!c',
-define.sps:8: error: DEBUG EXPAND: Expected `=' following !LET.
+define.sps:8.1-8.2: error: DEBUG EXPAND: Expected `=' following !LET.
!LET !x
define.sps:4: At `y' in the expansion of `!d',
-define.sps:9: error: DEBUG EXPAND: Expected `=' following !LET.
+define.sps:9.1-9.2: error: DEBUG EXPAND: Expected `=' following !LET.
!LET !x y
])
])
AT_CHECK([pspp --testing-mode define.sps], [1], [dnl
define.sps:1: In the expansion of `!a',
-define.sps:12: error: DEBUG EXPAND: Expected macro variable name following !DO.
+define.sps:12.1-12.2: error: DEBUG EXPAND: Expected macro variable name
+following !DO.
!DO
define.sps:2: At `0' in the expansion of `!b',
-define.sps:13: error: DEBUG EXPAND: Expected macro variable name following !DO.
+define.sps:13.1-13.2: error: DEBUG EXPAND: Expected macro variable name
+following !DO.
!DO 0
define.sps:3: In the expansion of `!c',
-define.sps:14: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
+define.sps:14.1-14.2: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
!DO !x
In the expansion of `!DO',
define.sps:4: inside the expansion of `!d',
-define.sps:15: error: DEBUG EXPAND: Missing !DOEND.
+define.sps:15.1-15.2: error: DEBUG EXPAND: Missing !DOEND.
!DO !x !in(x)
At `x' in the expansion of `!DO',
define.sps:5: inside the expansion of `!e',
-define.sps:16: error: DEBUG EXPAND: Macro expression must evaluate to a number
-(not "x").
+define.sps:16.1-16.2: error: DEBUG EXPAND: Macro expression must evaluate to a
+number (not "x").
!DO !x = x.
At `x' in the expansion of `!DO',
define.sps:6: inside the expansion of `!f',
-define.sps:17: error: DEBUG EXPAND: Expected !TO in numerical !DO loop.
+define.sps:17.1-17.2: error: DEBUG EXPAND: Expected !TO in numerical !DO loop.
!DO !x = 5 x
In the expansion of `!DO',
define.sps:7: inside the expansion of `!g',
-define.sps:18: error: DEBUG EXPAND: !BY value cannot be zero.
+define.sps:18.1-18.2: error: DEBUG EXPAND: !BY value cannot be zero.
!DO !x = 5 !TO 6 !BY 0
define.sps:8: In the expansion of `!h',
-define.sps:19: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
+define.sps:19.1-19.2: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
!DO !x
define.sps:9: At `0' in the expansion of `!i',
-define.sps:20: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
+define.sps:20.1-20.2: error: DEBUG EXPAND: Expected `=' or !IN in !DO loop.
!DO !x 0
define.sps:10: At `!BREAK' in the expansion of `!j',
-define.sps:21: error: DEBUG EXPAND: !BREAK outside !DO.
+define.sps:21.1-21.2: error: DEBUG EXPAND: !BREAK outside !DO.
])
AT_CLEANUP
define.sps:23.1: error: DEFINE: Syntax error at end of command: Expecting macro
body or !ENDDEFINE.
])
+AT_CLEANUP
+
+AT_SETUP([macro expansion with token merging])
+AT_DATA([define.sps], [dnl
+DEFINE !foo() "foo" !ENDDEFINE.
+DEFINE !bar() "bar" !ENDDEFINE.
+DEFINE !plus() + !ENDDEFINE.
+DEFINE !minus() - !ENDDEFINE.
+DEFINE !one() 1 !ENDDEFINE.
+ECHO "foo" + "bar".
+ECHO !foo.
+ECHO !bar.
+ECHO !foo + "quux".
+ECHO "baz" + !bar.
+ECHO !foo + !bar.
+ECHO !foo !plus !bar.
+ECHO "two" "strings".
+N OF CASES -/**/1.
+N OF CASES !minus 1.
+N OF CASES - !one.
+N OF CASES !minus !one.
+])
+AT_CHECK([pspp define.sps], [1], [dnl
+foobar
+
+foo
+
+bar
+
+fooquux
+
+bazbar
+
+foobar
+
+foobar
+
+two
+
+define.sps:13.12-13.20: error: ECHO: Syntax error at `"strings"': expecting end
+of command.
+
+define.sps:14.12-14.17: error: N OF CASES: Syntax error at `-/**/1': Expected
+positive integer for N OF CASES.
+
+define.sps:15.12-15.19: error: N OF CASES: Syntax error at `!minus 1': Expected
+positive integer for N OF CASES.
+
+define.sps:16.12-16.17: error: N OF CASES: Syntax error at `- !one': Expected
+positive integer for N OF CASES.
+
+define.sps:17.12-17.22: error: N OF CASES: Syntax error at `!minus !one':
+Expected positive integer for N OF CASES.
+])
AT_CLEANUP
\ No newline at end of file
lexer.sps:2: error: LIST: LIST is allowed only after the active dataset has been defined.
-lexer.sps:2.6: error: LIST: Syntax error: Bad character U+0000 in input.
+lexer.sps:2.6: error: LIST: Syntax error at `...': Bad character U+0000 in input.
])
AT_CLEANUP
char *input;
struct string_lexer slex;
- bool more;
set_program_name (argv[0]);
file_name = parse_options (argc, argv);
length--;
}
+ struct token *tokens = NULL;
+ size_t n_tokens = 0;
+ size_t allocated_tokens = 0;
string_lexer_init (&slex, input, length, mode, false);
- do
+ for (;;)
{
- struct token token;
+ if (n_tokens >= allocated_tokens)
+ tokens = x2nrealloc (tokens, &allocated_tokens, sizeof *tokens);
+ enum string_lexer_result result
+ = string_lexer_next (&slex, &tokens[n_tokens]);
+
+ if (result == SLR_ERROR)
+ tokens[n_tokens].type = T_STOP;
+ n_tokens++;
- more = string_lexer_next (&slex, &token);
+ if (result == SLR_END)
+ break;
+ }
- printf ("%s", scan_type_to_string (token.type));
- if (token.number != 0.0)
+ for (size_t i = 0; i < n_tokens; )
+ {
+ struct merger m = MERGER_INIT;
+ int retval;
+ struct token out;
+ for (size_t j = i; ; j++)
{
- double x = token.number;
+ assert (j < n_tokens);
+ retval = merger_add (&m, &tokens[j], &out);
+ if (retval != -1)
+ break;
+ }
+
+ const struct token *t = retval ? &out : &tokens[i];
+
+ fputs (token_type_to_name (t->type), stdout);
+ if (t->number != 0.0)
+ {
+ double x = t->number;
if (x > LONG_MIN && x <= LONG_MAX && floor (x) == x)
printf (" %ld", (long int) x);
else
printf (" %.3g", x);
}
- if (token.string.string != NULL || token.string.length > 0)
- printf (" \"%.*s\"", (int) token.string.length, token.string.string);
+ if (t->string.string != NULL || t->string.length > 0)
+ printf (" \"%.*s\"", (int) t->string.length, t->string.string);
printf ("\n");
- token_uninit (&token);
+ if (retval)
+ {
+ i += retval;
+ token_uninit (&out);
+ }
+ else
+ i++;
}
- while (more);
+ for (size_t i = 0; i < n_tokens; i++)
+ token_uninit (&tokens[i]);
+ free (tokens);
free (input);
return 0;
])
AT_DATA([expout-base], [dnl
ID "a"
-SKIP
ID "aB"
-SKIP
ID "i5"
-SKIP
ID "$x"
-SKIP
ID "@efg"
-SKIP
ID "@@."
-SKIP
MACRO_ID "!abcd"
-SKIP
ID "#.#"
-SKIP
MACRO_PUNCT "."
ID "x"
-SKIP
MACRO_PUNCT "_"
ID "z"
ENDCMD
-SKIP
ID "abcd."
-SKIP
ID "abcd"
ENDCMD
-SKIP
ID "QRSTUV"
ENDCMD
-SKIP
-SKIP
ID "QrStUv"
ENDCMD
-SKIP
-SKIP
-SKIP
ID "WXYZ"
ENDCMD
-SKIP
-SKIP
-SKIP
-UNEXPECTED_CHAR 65533
+STOP "Bad character U+FFFD in input."
ENDCMD
-SKIP
-SKIP
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
AND
-SKIP
OR
-SKIP
NOT
-SKIP
EQ
-SKIP
GE
-SKIP
GT
-SKIP
LE
-SKIP
LT
-SKIP
NE
-SKIP
ALL
-SKIP
BY
-SKIP
TO
-SKIP
WITH
-SKIP
AND
-SKIP
OR
-SKIP
NOT
-SKIP
EQ
-SKIP
GE
-SKIP
GT
-SKIP
LE
-SKIP
LT
-SKIP
NE
-SKIP
ALL
-SKIP
BY
-SKIP
TO
-SKIP
WITH
-SKIP
ID "andx"
-SKIP
ID "orx"
-SKIP
ID "notx"
-SKIP
ID "eqx"
-SKIP
ID "gex"
-SKIP
ID "gtx"
-SKIP
ID "lex"
-SKIP
ID "ltx"
-SKIP
ID "nex"
-SKIP
ID "allx"
-SKIP
ID "byx"
-SKIP
ID "tox"
-SKIP
ID "withx"
-SKIP
ID "and."
-SKIP
WITH
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
NOT
-SKIP
AND
-SKIP
OR
-SKIP
EQUALS
-SKIP
GE
-SKIP
GT
-SKIP
LE
-SKIP
LT
-SKIP
NE
-SKIP
NE
-SKIP
LPAREN
-SKIP
RPAREN
-SKIP
COMMA
-SKIP
DASH
-SKIP
PLUS
-SKIP
ASTERISK
-SKIP
SLASH
-SKIP
LBRACK
-SKIP
RBRACK
-SKIP
EXP
-SKIP
NOT
AND
OR
LBRACK
RBRACK
EXP
-SKIP
MACRO_PUNCT "%"
-SKIP
MACRO_PUNCT ":"
-SKIP
MACRO_PUNCT ";"
-SKIP
MACRO_PUNCT "?"
-SKIP
MACRO_PUNCT "_"
-SKIP
MACRO_PUNCT "`"
-SKIP
MACRO_PUNCT "{"
-SKIP
MACRO_PUNCT "}"
-SKIP
NOT
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
POS_NUM
-SKIP
POS_NUM 1
-SKIP
POS_NUM 1
-SKIP
POS_NUM 1
-SKIP
POS_NUM 1
ENDCMD
-SKIP
POS_NUM 123
ENDCMD
-SKIP
-SKIP
-SKIP
-SKIP
-SKIP
ENDCMD
POS_NUM 1
-SKIP
POS_NUM 0.1
-SKIP
POS_NUM 0.1
-SKIP
POS_NUM 0.1
-SKIP
POS_NUM 50
-SKIP
POS_NUM 0.6
-SKIP
POS_NUM 70
-SKIP
POS_NUM 60
-SKIP
POS_NUM 0.006
-SKIP
ENDCMD
POS_NUM 30
-SKIP
POS_NUM 0.04
-SKIP
POS_NUM 5
-SKIP
POS_NUM 6
-SKIP
POS_NUM 0.0007
-SKIP
POS_NUM 12.3
-SKIP
POS_NUM 4.56
-SKIP
POS_NUM 789
-SKIP
POS_NUM 999
-SKIP
POS_NUM 0.0112
-SKIP
ENDCMD
-SKIP
-EXPECTED_EXPONENT "1e"
-SKIP
+STOP "Missing exponent following `1e'."
ID "e1"
-SKIP
-EXPECTED_EXPONENT "1e+"
-SKIP
-EXPECTED_EXPONENT "1e-"
--SKIP
+STOP "Missing exponent following `1e+'."
+STOP "Missing exponent following `1e-'."
STOP
])
PSPP_CHECK_SCAN([-i])
-. -1e -e1 -1e+ -1e- -1.
])
AT_DATA([expout-base0], [dnl
-SKIP
NEG_NUM
-SKIP
NEG_NUM -1
-SKIP
NEG_NUM -1
-SKIP
NEG_NUM -1
-SKIP
NEG_NUM -1
ENDCMD
-SKIP
-SKIP
NEG_NUM -123
ENDCMD
-SKIP
-SKIP
-SKIP
-SKIP
-SKIP
-SKIP
NEG_NUM -0.1
-SKIP
NEG_NUM -0.1
-SKIP
NEG_NUM -0.1
-SKIP
NEG_NUM -0.1
-SKIP
-SKIP
NEG_NUM -50
-SKIP
NEG_NUM -0.6
-SKIP
NEG_NUM -70
-SKIP
NEG_NUM -60
-SKIP
NEG_NUM -0.006
-SKIP
-SKIP
NEG_NUM -3
-SKIP
NEG_NUM -0.04
-SKIP
NEG_NUM -5
-SKIP
NEG_NUM -6
-SKIP
NEG_NUM -0.0007
-SKIP
-SKIP
NEG_NUM -12.3
-SKIP
NEG_NUM -4.56
-SKIP
NEG_NUM -789
-SKIP
NEG_NUM -999
-SKIP
NEG_NUM -0.0112
-SKIP
-SKIP
NEG_NUM -1
-SKIP
-SKIP
DASH
-+SKIP
MACRO_PUNCT "."
-SKIP
-EXPECTED_EXPONENT "-1e"
-SKIP
+STOP "Missing exponent following `-1e'."
DASH
-+SKIP
ID "e1"
-SKIP
-EXPECTED_EXPONENT "-1e+"
-SKIP
-EXPECTED_EXPONENT "-1e-"
-SKIP
+STOP "Missing exponent following `-1e+'."
+STOP "Missing exponent following `-1e-'."
NEG_NUM -1
ENDCMD
--SKIP
STOP
])
-AS_BOX([without extra spaces])
cp input-base input
-sed '/^+/d' < expout-base0 > expout-base
+cp expout-base0 expout-base
PSPP_CHECK_SCAN([-i])
-AS_BOX([with extra spaces])
sed 's/ -/ - /g' < input-base > input
-sed 's/EXPONENT "-/EXPONENT "- /
- s/^+//' < expout-base0 > expout-base
+sed 's/following `-/following `- /' < expout-base0 > expout-base
PSPP_CHECK_SCAN([-i])
AT_CLEANUP
\f
])
AT_DATA([expout-base], [dnl
STRING "x"
-SKIP
STRING "y"
-SKIP
STRING "abc"
-SKIP
STRING "Don't"
-SKIP
STRING "Can't"
-SKIP
STRING "Won't"
-SKIP
STRING ""quoted""
-SKIP
STRING ""quoted""
-SKIP
STRING ""
-SKIP
STRING ""
-SKIP
STRING "'"
-SKIP
STRING """
-SKIP
-EXPECTED_QUOTE
-SKIP
-EXPECTED_QUOTE
-SKIP
+STOP "Unterminated string constant."
+STOP "Unterminated string constant."
STRING "xyzabcde"
-SKIP
STRING "foobar"
-SKIP
STRING "foobar"
-SKIP
STRING "foo"
-SKIP
PLUS
-SKIP
ENDCMD
-SKIP
STRING "bar"
-SKIP
ENDCMD
-SKIP
PLUS
-SKIP
STRING "AB5152"
-SKIP
STRING "4142QR"
-SKIP
STRING "ABお"
-SKIP
STRING "�あいうえお"
-SKIP
STRING "abc�えxyz"
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
#! /usr/bin/pspp
])
AT_DATA([expout-base], [dnl
-SKIP
-SKIP
ID "#"
MACRO_ID "!"
-SKIP
SLASH
ID "usr"
SLASH
ID "bin"
SLASH
ID "pspp"
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
-SKIP
-SKIP
-SKIP
ENDCMD
-SKIP
ENDCMD
-SKIP
-SKIP
ENDCMD
-SKIP
-SKIP
ENDCMD
-SKIP
ENDCMD
-SKIP
-SKIP
ENDCMD
-SKIP
-SKIP
ENDCMD
-SKIP
ID "com"
-SKIP
ID "is"
-SKIP
ID "ambiguous"
-SKIP
WITH
-SKIP
ID "COMPUTE"
ENDCMD
-SKIP
ENDCMD
-SKIP
-SKIP
-SKIP
ENDCMD
-SKIP
ENDCMD
-SKIP
-SKIP
-SKIP
ENDCMD
-SKIP
ID "next"
-SKIP
ID "command"
ENDCMD
-SKIP
-ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
STRING "DOCUMENT one line."
ENDCMD
ENDCMD
-SKIP
ID "DOCUMENT"
STRING "DOC more"
-SKIP
STRING " than"
-SKIP
STRING " one"
-SKIP
STRING " line."
ENDCMD
ENDCMD
-SKIP
ID "DOCUMENT"
STRING "docu"
-SKIP
STRING "first.paragraph"
-SKIP
STRING "isn't parsed as tokens"
-SKIP
STRING ""
-SKIP
STRING "second paragraph."
-ENDCMD
-ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "FIL"
-SKIP
ID "label"
-SKIP
STRING "isn't quoted"
ENDCMD
-SKIP
ID "FILE"
-SKIP
-SKIP
ID "lab"
-SKIP
STRING "is quoted"
ENDCMD
-SKIP
ID "FILE"
-SKIP
-SKIP
-SKIP
-SKIP
-SKIP
ID "lab"
-SKIP
STRING "not quoted here either"
-SKIP
-ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "begin"
-SKIP
ID "data"
ENDCMD
-SKIP
STRING "123"
-SKIP
STRING "xxx"
-SKIP
ID "end"
-SKIP
ID "data"
ENDCMD
-SKIP
ENDCMD
-SKIP
ID "BEG"
-SKIP
-SKIP
-SKIP
ID "DAT"
-SKIP
-SKIP
-SKIP
STRING "5 6 7 /* x"
-SKIP
STRING ""
-SKIP
STRING "end data"
-SKIP
ID "end"
-SKIP
ID "data"
-SKIP
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "do"
-SKIP
ID "repeat"
-SKIP
ID "x"
EQUALS
ID "a"
-SKIP
ID "b"
-SKIP
ID "c"
-SKIP
-SKIP
ID "y"
EQUALS
ID "d"
-SKIP
ID "e"
-SKIP
ID "f"
ENDCMD
-SKIP
STRING " do repeat a=1 thru 5."
-SKIP
STRING "another command."
-SKIP
STRING "second command"
-SKIP
STRING "+ third command."
-SKIP
STRING "end /* x */ /* y */ repeat print."
-SKIP
ID "end"
-SKIP
-SKIP
ID "repeat"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "do"
-SKIP
ID "repeat"
-SKIP
ID "x"
EQUALS
ID "a"
-SKIP
ID "b"
-SKIP
ID "c"
-SKIP
-SKIP
ID "y"
EQUALS
ID "d"
-SKIP
ID "e"
-SKIP
ID "f"
-SKIP
ENDCMD
STRING "do repeat a=1 thru 5"
-SKIP
STRING "another command"
-SKIP
STRING "second command"
-SKIP
STRING "+ third command"
-SKIP
STRING "end /* x */ /* y */ repeat print"
-SKIP
ID "end"
-SKIP
-SKIP
ID "repeat"
-SKIP
ENDCMD
ID "do"
-SKIP
-SKIP
ID "repeat"
-SKIP
ID "#a"
EQUALS
POS_NUM 1
-SKIP
ENDCMD
-SKIP
STRING " inner command"
-SKIP
ID "end"
-SKIP
ID "repeat"
--SKIP
STOP
])
PSPP_CHECK_SCAN([-b])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
-SKIP
-SKIP
STRING "var1 var2 var3"
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
STRING " var1 var2 var3"
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
-SKIP
-SKIP
STRING "var1 var2 var3"
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
STRING "var1 var2 var3"
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
-SKIP
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
-SKIP
-SKIP
STRING ""
-SKIP
STRING ""
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
ID "a"
LPAREN
RPAREN
COMMA
-SKIP
ID "b"
LPAREN
RPAREN
COMMA
-SKIP
ID "c"
LPAREN
RPAREN
RPAREN
-SKIP
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
-SKIP
-SKIP
ID "a"
LPAREN
RPAREN
COMMA
-SKIP
ID "b"
LPAREN
-SKIP
-SKIP
RPAREN
COMMA
-SKIP
-SKIP
ID "c"
LPAREN
RPAREN
-SKIP
RPAREN
-SKIP
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
-SKIP
LPAREN
ID "x"
COMMA
ID "y"
COMMA
ID "z"
-SKIP
RPAREN
-SKIP
-SKIP
STRING "content 1"
-SKIP
STRING "content 2"
-SKIP
MACRO_ID "!enddefine"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
ENDCMD
-SKIP
ID "data"
-SKIP
ID "list"
-SKIP
SLASH
ID "x"
-SKIP
POS_NUM 1
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
-SKIP
ID "x"
ENDCMD
-SKIP
ID "data"
-SKIP
ID "list"
-SKIP
SLASH
ID "x"
-SKIP
POS_NUM 1
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
ENDCMD
-SKIP
ID "x"
ENDCMD
-SKIP
ID "data"
-SKIP
ID "list"
-SKIP
SLASH
ID "x"
-SKIP
POS_NUM 1
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
ENDCMD
-SKIP
ID "data"
-SKIP
ID "list"
-SKIP
SLASH
ID "x"
-SKIP
POS_NUM 1
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "define"
-SKIP
MACRO_ID "!macro1"
LPAREN
RPAREN
-SKIP
-SKIP
STRING "content line 1"
-SKIP
STRING "content line 2"
--SKIP
STOP
])
PSPP_CHECK_SCAN([-i])
])
AT_DATA([expout-base], [dnl
ID "first"
-SKIP
ID "command"
-SKIP
-SKIP
ID "another"
-SKIP
ID "line"
-SKIP
ID "of"
-SKIP
ID "first"
-SKIP
ID "command"
-SKIP
ENDCMD
-SKIP
ID "second"
-SKIP
ID "command"
-SKIP
ENDCMD
ID "third"
-SKIP
ID "command"
-SKIP
ENDCMD
-SKIP
ID "fourth"
-SKIP
ID "command"
ENDCMD
-SKIP
-SKIP
ID "fifth"
-SKIP
ID "command"
ENDCMD
--SKIP
STOP
])
PSPP_CHECK_SCAN([-b])
projects / pspp / commitdiff