#include <uniwidth.h>
#include "language/command.h"
+#include "language/lexer/macro.h"
#include "language/lexer/scan.h"
#include "language/lexer/segment.h"
#include "language/lexer/token.h"
/* The regular token information. */
struct token token;
- /* Location of token in terms of the lex_source's buffer.
+ /* For a token obtained through the lexer in an ordinary way, this is the
+ location of the token in terms of the lex_source's buffer.
+
+ For a token produced through macro expansion, this is the entire macro
+ call.
+
src->tail <= line_pos <= token_pos <= src->head. */
size_t token_pos; /* Start of token. */
size_t token_len; /* Length of source for token in bytes. */
size_t line_pos; /* Start of line containing token_pos. */
int first_line; /* Line number at token_pos. */
+
+ /* For a token obtained through macro expansion, this is just this token. */
+ char *macro_rep; /* The whole macro expansion. */
+ size_t ofs; /* Offset of this token in macro_rep. */
+ size_t len; /* Length of this token in macro_rep. */
+ size_t *ref_cnt; /* Number of lex_tokens that refer to macro_rep. */
};
+static void
+lex_token_uninit (struct lex_token *t)
+{
+ token_uninit (&t->token);
+ if (t->ref_cnt)
+ {
+ assert (*t->ref_cnt > 0);
+ if (!--*t->ref_cnt)
+ {
+ free (t->macro_rep);
+ free (t->ref_cnt);
+ }
+ }
+}
+
/* A source of tokens, corresponding to a syntax file.
This is conceptually a lex_reader wrapped with everything needed to convert
{
struct ll ll; /* In lexer's list of sources. */
struct lex_reader *reader;
+ struct lexer *lexer;
struct segmenter segmenter;
bool eof; /* True if T_STOP was read from 'reader'. */
struct lex_token *tokens; /* Lookahead tokens for parser. */
};
-static struct lex_source *lex_source_create (struct lex_reader *);
+static struct lex_source *lex_source_create (struct lexer *,
+ struct lex_reader *);
static void lex_source_destroy (struct lex_source *);
/* Lexer. */
struct lexer
{
struct ll_list sources; /* Contains "struct lex_source"s. */
+ struct macro_set *macros;
};
static struct lex_source *lex_source__ (const struct lexer *);
-static struct substring lex_source_get_syntax__ (const struct lex_source *,
- int n0, int n1);
+static char *lex_source_get_syntax__ (const struct lex_source *,
+ int n0, int n1);
static const struct lex_token *lex_next__ (const struct lexer *, int n);
static void lex_source_push_endcmd__ (struct lex_source *);
static void lex_source_pop__ (struct lex_source *);
-static bool lex_source_get__ (const struct lex_source *);
+static bool lex_source_get (const struct lex_source *);
static void lex_source_error_valist (struct lex_source *, int n0, int n1,
const char *format, va_list)
PRINTF_FORMAT (4, 0);
struct lexer *
lex_create (void)
{
- struct lexer *lexer = xzalloc (sizeof *lexer);
- ll_init (&lexer->sources);
+ struct lexer *lexer = xmalloc (sizeof *lexer);
+ *lexer = (struct lexer) {
+ .sources = LL_INITIALIZER (lexer->sources),
+ .macros = macro_set_create (),
+ };
return lexer;
}
ll_for_each_safe (source, next, struct lex_source, ll, &lexer->sources)
lex_source_destroy (source);
+ macro_set_destroy (lexer->macros);
free (lexer);
}
}
+/* Adds M to LEXER's set of macros. M replaces any existing macro with the
+ same name. Takes ownership of M. */
+void
+lex_define_macro (struct lexer *lexer, struct macro *m)
+{
+ macro_set_add (lexer->macros, m);
+}
+
/* Inserts READER into LEXER so that the next token read by LEXER comes from
READER. Before the caller, LEXER must either be empty or at a T_ENDCMD
token. */
lex_include (struct lexer *lexer, struct lex_reader *reader)
{
assert (ll_is_empty (&lexer->sources) || lex_token (lexer) == T_ENDCMD);
- ll_push_head (&lexer->sources, &lex_source_create (reader)->ll);
+ ll_push_head (&lexer->sources, &lex_source_create (lexer, reader)->ll);
}
/* Appends READER to LEXER, so that it will be read after all other current
void
lex_append (struct lexer *lexer, struct lex_reader *reader)
{
- ll_push_tail (&lexer->sources, &lex_source_create (reader)->ll);
+ ll_push_tail (&lexer->sources, &lex_source_create (lexer, reader)->ll);
}
\f
/* Advancing. */
src->tokens = deque_expand (&src->deque, src->tokens, sizeof *src->tokens);
token = &src->tokens[deque_push_front (&src->deque)];
- token_init (&token->token);
+ token->token = (struct token) { .type = T_STOP };
+ token->macro_rep = NULL;
+ token->ref_cnt = NULL;
return token;
}
static void
lex_source_pop__ (struct lex_source *src)
{
- token_uninit (&src->tokens[deque_pop_back (&src->deque)].token);
+ lex_token_uninit (&src->tokens[deque_pop_back (&src->deque)]);
}
static void
lex_source_pop_front (struct lex_source *src)
{
- token_uninit (&src->tokens[deque_pop_front (&src->deque)].token);
+ lex_token_uninit (&src->tokens[deque_pop_front (&src->deque)]);
}
/* Advances LEXER to the next token, consuming the current token. */
lex_source_pop__ (src);
while (deque_is_empty (&src->deque))
- if (!lex_source_get__ (src))
+ if (!lex_source_get (src))
{
lex_source_destroy (src);
src = lex_source__ (lexer);
return lex_source_next__ (src, n);
else
{
- static const struct lex_token stop_token =
- { TOKEN_INITIALIZER (T_STOP, 0.0, ""), 0, 0, 0, 0 };
-
+ static const struct lex_token stop_token = { .token = { .type = T_STOP } };
return &stop_token;
}
}
+static const struct lex_token *
+lex_source_front (const struct lex_source *src)
+{
+ return &src->tokens[deque_front (&src->deque, 0)];
+}
+
static const struct lex_token *
lex_source_next__ (const struct lex_source *src, int n)
{
{
if (!deque_is_empty (&src->deque))
{
- struct lex_token *front;
-
- front = &src->tokens[deque_front (&src->deque, 0)];
+ const struct lex_token *front = lex_source_front (src);
if (front->token.type == T_STOP || front->token.type == T_ENDCMD)
return front;
}
- lex_source_get__ (src);
+ lex_source_get (src);
}
return &src->tokens[deque_back (&src->deque, n)];
/* Returns the text of the syntax in tokens N0 ahead of the current one,
through N1 ahead of the current one, inclusive. (For example, if N0 and N1
are both zero, this requests the syntax for the current token.) The caller
- must not modify or free the returned string. The syntax is encoded in UTF-8
- and in the original form supplied to the lexer so that, for example, it may
- include comments, spaces, and new-lines if it spans multiple tokens. */
-struct substring
+ must eventually free the returned string (with free()). The syntax is
+ encoded in UTF-8 and in the original form supplied to the lexer so that, for
+ example, it may include comments, spaces, and new-lines if it spans multiple
+ tokens. Macro expansion, however, has already been performed. */
+char *
lex_next_representation (const struct lexer *lexer, int n0, int n1)
{
return lex_source_get_syntax__ (lex_source__ (lexer), n0, n1);
}
+bool
+lex_next_is_from_macro (const struct lexer *lexer, int n)
+{
+ return lex_next__ (lexer, n)->macro_rep != NULL;
+}
+
static bool
lex_tokens_match (const struct token *actual, const struct token *expected)
{
int i;
i = 0;
- string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+ string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
while (string_lexer_next (&slex, &token))
if (token.type != SCAN_SKIP)
{
return src == NULL ? NULL : src->reader->encoding;
}
-
/* Returns the syntax mode for the syntax file from which the current drawn is
drawn. Returns SEG_MODE_AUTO for a T_STOP token or if the command's source
does not have line numbers.
src->journal_pos = src->seg_pos = src->line_pos = 0;
src->n_newlines = 0;
src->suppress_next_newline = false;
- segmenter_init (&src->segmenter, segmenter_get_mode (&src->segmenter));
+ src->segmenter = segmenter_init (segmenter_get_mode (&src->segmenter),
+ false);
while (!deque_is_empty (&src->deque))
lex_source_pop__ (src);
lex_source_push_endcmd__ (src);
: ll_data (ll_head (&lexer->sources), struct lex_source, ll));
}
-static struct substring
-lex_tokens_get_syntax__ (const struct lex_source *src,
- const struct lex_token *token0,
- const struct lex_token *token1)
+static char *
+lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
{
- size_t start = token0->token_pos;
- size_t end = token1->token_pos + token1->token_len;
+ struct string s = DS_EMPTY_INITIALIZER;
+ for (size_t i = n0; i <= n1; )
+ {
+ /* Find [I,J) as the longest sequence of tokens not produced by macro
+ expansion, or otherwise the longest sequence expanded from a single
+ macro call. */
+ const struct lex_token *first = lex_source_next__ (src, i);
+ size_t j;
+ for (j = i + 1; j <= n1; j++)
+ {
+ const struct lex_token *cur = lex_source_next__ (src, j);
+ if ((first->macro_rep != NULL) != (cur->macro_rep != NULL)
+ || first->macro_rep != cur->macro_rep)
+ break;
+ }
+ const struct lex_token *last = lex_source_next__ (src, j - 1);
- return ss_buffer (&src->buffer[start - src->tail], end - start);
-}
+ if (!ds_is_empty (&s))
+ ds_put_byte (&s, ' ');
+ if (!first->macro_rep)
+ {
+ size_t start = first->token_pos;
+ size_t end = last->token_pos + last->token_len;
+ ds_put_substring (&s, ss_buffer (&src->buffer[start - src->tail],
+ end - start));
+ }
+ else
+ {
+ size_t start = first->ofs;
+ size_t end = last->ofs + last->len;
+ ds_put_substring (&s, ss_buffer (first->macro_rep + start,
+ end - start));
+ }
-static struct substring
-lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
-{
- return lex_tokens_get_syntax__ (src,
- lex_source_next__ (src, n0),
- lex_source_next__ (src, MAX (n0, n1)));
+ i = j;
+ }
+ return ds_steal_cstr (&s);
}
static void
strcpy (&out[out_len], out_len < in.length ? "..." : "");
}
+static bool
+lex_source_contains_macro_call (struct lex_source *src, int n0, int n1)
+{
+ for (size_t i = n0; i <= n1; i++)
+ if (lex_source_next__ (src, i)->macro_rep)
+ return true;
+ return false;
+}
+
+static struct substring
+lex_source_get_macro_call (struct lex_source *src, int n0, int n1)
+{
+ if (!lex_source_contains_macro_call (src, n0, n1))
+ return ss_empty ();
+
+ const struct lex_token *token0 = lex_source_next__ (src, n0);
+ const struct lex_token *token1 = lex_source_next__ (src, MAX (n0, n1));
+ size_t start = token0->token_pos;
+ size_t end = token1->token_pos + token1->token_len;
+
+ return ss_buffer (&src->buffer[start - src->tail], end - start);
+}
+
static void
lex_source_error_valist (struct lex_source *src, int n0, int n1,
const char *format, va_list args)
ds_put_cstr (&s, _("Syntax error at end of command"));
else
{
- struct substring syntax = lex_source_get_syntax__ (src, n0, n1);
- if (!ss_is_empty (syntax))
+ /* Get the syntax that caused the error. */
+ char *syntax = lex_source_get_syntax__ (src, n0, n1);
+ char syntax_cstr[64];
+ lex_ellipsize__ (ss_cstr (syntax), syntax_cstr, sizeof syntax_cstr);
+ free (syntax);
+
+ /* Get the macro call(s) that expanded to the syntax that caused the
+ error. */
+ char call_cstr[64];
+ struct substring call = lex_source_get_macro_call (src, n0, n1);
+ lex_ellipsize__ (call, call_cstr, sizeof call_cstr);
+
+ if (syntax_cstr[0])
{
- char syntax_cstr[64];
-
- lex_ellipsize__ (syntax, syntax_cstr, sizeof syntax_cstr);
- ds_put_format (&s, _("Syntax error at `%s'"), syntax_cstr);
+ if (call_cstr[0])
+ ds_put_format (&s, _("Syntax error at `%s' "
+ "(in expansion of `%s')"),
+ syntax_cstr, call_cstr);
+ else
+ ds_put_format (&s, _("Syntax error at `%s'"), syntax_cstr);
}
+ else if (call_cstr[0])
+ ds_put_format (&s, _("Syntax error in syntax expanded from `%s'"),
+ call_cstr);
else
ds_put_cstr (&s, _("Syntax error"));
}
}
/* Attempts to append an additional token into SRC's deque, reading more from
- the underlying lex_reader if necessary. Returns true if successful, false
- if the deque already represents (a suffix of) the whole lex_reader's
- contents, */
+ the underlying lex_reader if necessary. Returns true if a new token was
+ added to SRC's deque, false otherwise. */
static bool
-lex_source_get__ (const struct lex_source *src_)
+lex_source_try_get (struct lex_source *src)
{
- struct lex_source *src = CONST_CAST (struct lex_source *, src_);
- if (src->eof)
- return false;
-
/* State maintained while scanning tokens. Usually we only need a single
state, but scanner_push() can return SCAN_SAVE to indicate that the state
needs to be saved and possibly restored later with SCAN_BACK. */
switch (token->token.type)
{
default:
- break;
+ return true;
case T_STOP:
token->token.type = T_ENDCMD;
src->eof = true;
- break;
+ return true;
case SCAN_BAD_HEX_LENGTH:
lex_get_error (src, _("String of hex digits has %d characters, which "
"is not a multiple of 2"),
(int) token->token.number);
- break;
+ return false;
case SCAN_BAD_HEX_DIGIT:
case SCAN_BAD_UNICODE_DIGIT:
lex_get_error (src, _("`%c' is not a valid hex digit"),
(int) token->token.number);
- break;
+ return false;
case SCAN_BAD_UNICODE_LENGTH:
lex_get_error (src, _("Unicode string contains %d bytes, which is "
"not in the valid range of 1 to 8 bytes"),
(int) token->token.number);
- break;
+ return false;
case SCAN_BAD_UNICODE_CODE_POINT:
lex_get_error (src, _("U+%04X is not a valid Unicode code point"),
(int) token->token.number);
- break;
+ return false;
case SCAN_EXPECTED_QUOTE:
lex_get_error (src, _("Unterminated string constant"));
- break;
+ return false;
case SCAN_EXPECTED_EXPONENT:
lex_get_error (src, _("Missing exponent following `%s'"),
token->token.string.string);
- break;
+ return false;
case SCAN_UNEXPECTED_CHAR:
{
char c_name[16];
lex_get_error (src, _("Bad character %s in input"),
uc_name (token->token.number, c_name));
+ return false;
}
- break;
case SCAN_SKIP:
lex_source_pop_front (src);
- break;
+ return false;
}
+ NOT_REACHED ();
+}
+
+static bool
+lex_source_get__ (struct lex_source *src)
+{
+ for (;;)
+ {
+ if (src->eof)
+ return false;
+ else if (lex_source_try_get (src))
+ return true;
+ }
+}
+
+static bool
+lex_source_get (const struct lex_source *src_)
+{
+ struct lex_source *src = CONST_CAST (struct lex_source *, src_);
+
+ size_t old_count = deque_count (&src->deque);
+ if (!lex_source_get__ (src))
+ return false;
+
+ if (!settings_get_mexpand ())
+ return true;
+
+ struct macro_expander *me;
+ int retval = macro_expander_create (src->lexer->macros,
+ &lex_source_front (src)->token,
+ &me);
+ while (!retval)
+ {
+ if (!lex_source_get__ (src))
+ {
+ /* This should not be reachable because we always get a T_ENDCMD at
+ the end of an input file (transformed from T_STOP by
+ lex_source_try_get()) and the macro_expander should always
+ terminate expansion on T_ENDCMD. */
+ NOT_REACHED ();
+ }
+
+ const struct lex_token *front = lex_source_front (src);
+ size_t start = front->token_pos;
+ size_t end = front->token_pos + front->token_len;
+ const struct macro_token mt = {
+ .token = front->token,
+ .representation = ss_buffer (&src->buffer[start - src->tail],
+ end - start),
+ };
+ retval = macro_expander_add (me, &mt);
+ }
+ if (retval < 0)
+ {
+ /* XXX handle case where there's a macro invocation starting from some
+ later token we've already obtained */
+ macro_expander_destroy (me);
+ return true;
+ }
+
+ /* XXX handle case where the macro invocation doesn't use all the tokens */
+ const struct lex_token *call_first = lex_source_next__ (src, old_count);
+ const struct lex_token *call_last = lex_source_front (src);
+ size_t call_pos = call_first->token_pos;
+ size_t call_len = (call_last->token_pos + call_last->token_len) - call_pos;
+ size_t line_pos = call_first->line_pos;
+ int first_line = call_first->first_line;
+ while (deque_count (&src->deque) > old_count)
+ lex_source_pop_front (src);
+
+ struct macro_tokens expansion = { .n = 0 };
+ macro_expander_get_expansion (me, &expansion);
+ macro_expander_destroy (me);
+
+ size_t *ofs = xnmalloc (expansion.n, sizeof *ofs);
+ size_t *len = xnmalloc (expansion.n, sizeof *len);
+ struct string s = DS_EMPTY_INITIALIZER;
+ macro_tokens_to_representation (&expansion, &s, ofs, len);
+
+ if (settings_get_mprint ())
+ output_item_submit (text_item_create (TEXT_ITEM_LOG, ds_cstr (&s),
+ _("Macro Expansion")));
+
+ char *macro_rep = ds_steal_cstr (&s);
+ size_t *ref_cnt = xmalloc (sizeof *ref_cnt);
+ *ref_cnt = expansion.n;
+ for (size_t i = 0; i < expansion.n; i++)
+ {
+ *lex_push_token__ (src) = (struct lex_token) {
+ .token = expansion.mts[i].token,
+ .token_pos = call_pos,
+ .token_len = call_len,
+ .line_pos = line_pos,
+ .first_line = first_line,
+ .macro_rep = macro_rep,
+ .ofs = ofs[i],
+ .len = len[i],
+ .ref_cnt = ref_cnt,
+ };
+
+ ss_dealloc (&expansion.mts[i].representation);
+ }
+ free (expansion.mts);
+ free (ofs);
+ free (len);
+
return true;
}
\f
static void
lex_source_push_endcmd__ (struct lex_source *src)
{
- struct lex_token *token = lex_push_token__ (src);
- token->token.type = T_ENDCMD;
- token->token_pos = 0;
- token->token_len = 0;
- token->line_pos = 0;
- token->first_line = 0;
+ *lex_push_token__ (src) = (struct lex_token) { .token = { .type = T_ENDCMD } };
}
static struct lex_source *
-lex_source_create (struct lex_reader *reader)
+lex_source_create (struct lexer *lexer, struct lex_reader *reader)
{
struct lex_source *src;
src = xzalloc (sizeof *src);
src->reader = reader;
- segmenter_init (&src->segmenter, reader->syntax);
+ src->segmenter = segmenter_init (reader->syntax, false);
+ src->lexer = lexer;
src->tokens = deque_init (&src->deque, 4, sizeof *src->tokens);
lex_source_push_endcmd__ (src);
projects / pspp / blobdiff