+ NOT_REACHED ();
+}
+
+/* Attempts to add a new token at the front of SRC. Returns true if
+ successful, false on failure. On failure, the end of SRC has been reached
+ and no more tokens will be forthcoming from it.
+
+ Does not make the new token available for lookahead yet; the caller must
+ adjust SRC's 'middle' pointer to do so. */
+static bool
+lex_source_get__ (struct lex_source *src)
+{
+ while (!src->eof)
+ if (lex_source_try_get__ (src))
+ return true;
+ return false;
+}
+
+static bool
+lex_source_get (const struct lex_source *src_)
+{
+ struct lex_source *src = CONST_CAST (struct lex_source *, src_);
+
+ if (src->front - src->middle == 0)
+ {
+ if (!lex_source_get__ (src))
+ return false;
+ }
+
+ if (!settings_get_mexpand ())
+ {
+ src->middle++;
+ return true;
+ }
+
+ struct macro_expander *me;
+ int n_call = macro_expander_create (
+ src->lexer->macros, &src->tokens[src->middle & (src->capacity - 1)].token,
+ &me);
+ for (int middle_ofs = 1; !n_call; middle_ofs++)
+ {
+ if (src->front - src->middle <= middle_ofs && !lex_source_get__ (src))
+ {
+ /* This should not be reachable because we always get a T_ENDCMD at
+ the end of an input file (transformed from T_STOP by
+ lex_source_try_get__()) and the macro_expander should always
+ terminate expansion on T_ENDCMD. */
+ NOT_REACHED ();
+ }
+
+ const struct lex_token *t = &src->tokens[(src->middle + middle_ofs)
+ & (src->capacity - 1)];
+ size_t start = t->token_pos;
+ size_t end = t->token_pos + t->token_len;
+ const struct macro_token mt = {
+ .token = t->token,
+ .representation = ss_buffer (&src->buffer[start - src->tail],
+ end - start),
+ };
+ src->middle += middle_ofs + 1;
+ n_call = macro_expander_add (me, &mt);
+ src->middle -= middle_ofs + 1;
+ }
+ if (n_call < 0)
+ {
+ /* False alarm: no macro expansion after all. Use first token as
+ lookahead. We'll retry macro expansion from the second token next
+ time around. */
+ macro_expander_destroy (me);
+ src->middle++;
+ return true;
+ }
+
+ /* Now expand the macro.
+
+ We temporarily add the macro call's tokens to the source in case the macro
+ expansion calls msg() to report an error and error processing tries to get
+ the location of the error with, e.g. lex_get_first_line_number(), which
+ would re-enter this code. This is a kluge; it might be cleaner to pass
+ the line number into macro_expander_get_expansion(). */
+ src->middle += n_call;
+ struct macro_tokens expansion = { .n = 0 };
+ macro_expander_get_expansion (me, src->reader->syntax, &expansion);
+ macro_expander_destroy (me);
+ src->middle -= n_call;
+
+ /* Convert the macro expansion into syntax for possible error messages later. */
+ size_t *ofs = xnmalloc (expansion.n, sizeof *ofs);
+ size_t *len = xnmalloc (expansion.n, sizeof *len);
+ struct string s = DS_EMPTY_INITIALIZER;
+ macro_tokens_to_representation (&expansion, &s, ofs, len);
+
+ if (settings_get_mprint ())
+ output_item_submit (text_item_create (TEXT_ITEM_LOG, ds_cstr (&s),
+ _("Macro Expansion")));
+
+ /* The first 'n_call' tokens starting at 'middle' will be replaced by the
+ macro expansion. There might be more tokens after that, up to 'front'.
+
+ Figure out the boundary of the macro call in the syntax, to go into the
+ lex_tokens for the expansion so that later error messages can report what
+ macro was called. */
+ const struct lex_token *call_first
+ = &src->tokens[src->middle & (src->capacity - 1)];
+ const struct lex_token *call_last
+ = &src->tokens[(src->middle + n_call - 1) & (src->capacity - 1)];
+ size_t call_pos = call_first->token_pos;
+ size_t call_len = (call_last->token_pos + call_last->token_len) - call_pos;
+ size_t line_pos = call_first->line_pos;
+ int first_line = call_first->first_line;
+
+ /* Destroy the tokens for the call, and save any tokens following the call so
+ we can add them back later. */
+ for (size_t i = src->middle; i != src->middle + n_call; i++)
+ lex_token_uninit (&src->tokens[i & (src->capacity - 1)]);
+ size_t n_save = src->front - (src->middle + n_call);
+ struct lex_token *save_tokens = xnmalloc (n_save, sizeof *save_tokens);
+ for (size_t i = 0; i < n_save; i++)
+ save_tokens[i] = src->tokens[(src->middle + n_call + i)
+ & (src->capacity - 1)];
+ src->front = src->middle;
+
+ /* Append the macro expansion tokens to the lookahead. */
+ char *macro_rep = ds_steal_cstr (&s);
+ size_t *ref_cnt = xmalloc (sizeof *ref_cnt);
+ *ref_cnt = expansion.n;
+ for (size_t i = 0; i < expansion.n; i++)
+ {
+ *lex_push_token__ (src) = (struct lex_token) {
+ .token = expansion.mts[i].token,
+ .token_pos = call_pos,
+ .token_len = call_len,
+ .line_pos = line_pos,
+ .first_line = first_line,
+ .macro_rep = macro_rep,
+ .ofs = ofs[i],
+ .len = len[i],
+ .ref_cnt = ref_cnt,
+ };
+ src->middle++;
+
+ ss_dealloc (&expansion.mts[i].representation);
+ }
+ free (expansion.mts);
+ free (ofs);
+ free (len);
+
+ /* Finally, put the saved tokens back. */
+ for (size_t i = 0; i < n_save; i++)
+ *lex_push_token__ (src) = save_tokens[i];
+ free (save_tokens);
+