From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 9 May 2021 21:59:16 +0000 (-0700)
Subject: now macros keep track of representation
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8c5a1c4933d05641d01ac5cd74b6f9df5f2ad8b7;p=pspp

now macros keep track of representation
---

diff --git a/src/language/control/define.c b/src/language/control/define.c
index 2f5ffd9d6b..b0699a56b1 100644
--- a/src/language/control/define.c
+++ b/src/language/control/define.c
@@ -63,8 +63,8 @@ parse_quoted_token (struct lexer *lexer, struct token *token)
   if (!string_lexer_next (&slex, token)
       || string_lexer_next (&slex, &another_token))
     {
-      token_destroy (token);
-      token_destroy (&another_token);
+      token_uninit (token);
+      token_uninit (&another_token);
       lex_error (lexer, _("String must contain exactly one token."));
       return false;
     }
@@ -73,7 +73,7 @@ parse_quoted_token (struct lexer *lexer, struct token *token)
 }
 
 static void
-macro_tokenize (struct macro *m, struct lexer *lexer)
+macro_tokenize (struct macro *m, const struct substring body, struct lexer *lexer)
 {
   struct state
     {
@@ -83,16 +83,20 @@ macro_tokenize (struct macro *m, struct lexer *lexer)
 
   struct state state = {
     .segmenter = SEGMENTER_INIT (lex_get_syntax_mode (lexer)),
-    .body = m->body,
+    .body = body,
   };
   struct state saved = state;
 
-  struct token token = { .type = T_STOP };
-
   while (state.body.length > 0)
     {
+      struct macro_token mt = {
+        .token = { .type = T_STOP },
+        .representation = { .string = state.body.string },
+      };
+      struct token *token = &mt.token;
+
       struct scanner scanner;
-      scanner_init (&scanner, &token);
+      scanner_init (&scanner, token);
 
       for (;;)
         {
@@ -104,7 +108,7 @@ macro_tokenize (struct macro *m, struct lexer *lexer)
           struct substring segment = ss_head (state.body, seg_len);
           ss_advance (&state.body, seg_len);
 
-          enum scan_result result = scanner_push (&scanner, type, segment, &token);
+          enum scan_result result = scanner_push (&scanner, type, segment, token);
           if (result == SCAN_SAVE)
             saved = state;
           else if (result == SCAN_BACK)
@@ -117,16 +121,19 @@ macro_tokenize (struct macro *m, struct lexer *lexer)
         }
 
       /* We have a token in 'token'. */
-      if (is_scan_type (token.type))
+      if (is_scan_type (token->type))
         {
-          if (token.type != SCAN_SKIP)
+          if (token->type != SCAN_SKIP)
             {
               /* XXX report error */
             }
         }
       else
-        tokens_add (&m->body_tokens, &token);
-      token_destroy (&token);
+        {
+          mt.representation.length = state.body.string - mt.representation.string;
+          macro_tokens_add (&m->body, &mt);
+        }
+      token_uninit (token);
     }
 }
 
@@ -195,7 +202,11 @@ cmd_define (struct lexer *lexer, struct dataset *ds UNUSED)
                   lex_error_expecting (lexer, ")");
                   goto error;
                 }
-              tokens_add (&p->def, lex_next (lexer, 0));
+              const struct macro_token mt = {
+                .token = *lex_next (lexer, 0),
+                .representation = lex_next_representation (lexer, 0, 0),
+              };
+              macro_tokens_add (&p->def, &mt);
               lex_get (lexer);
             }
         }
@@ -263,9 +274,9 @@ cmd_define (struct lexer *lexer, struct dataset *ds UNUSED)
       ds_put_byte (&body, '\n');
       lex_get (lexer);
     }
-  m->body = ds_ss (&body);
 
-  macro_tokenize (m, lexer);
+  macro_tokenize (m, body.ss, lexer);
+  ds_destroy (&body);
 
   lex_define_macro (lexer, m);
 
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index ff3c43e35e..5ff5099652 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -114,6 +114,8 @@ struct lexer
   };
 
 static struct lex_source *lex_source__ (const struct lexer *);
+static struct substring lex_source_get_syntax__ (const struct lex_source *,
+                                                 int n0, int n1);
 static const struct lex_token *lex_next__ (const struct lexer *, int n);
 static void lex_source_push_endcmd__ (struct lex_source *);
 
@@ -964,6 +966,12 @@ lex_next_tokss (const struct lexer *lexer, int n)
   return lex_next (lexer, n)->string;
 }
 
+struct substring
+lex_next_representation (const struct lexer *lexer, int n0, int n1)
+{
+  return lex_source_get_syntax__ (lex_source__ (lexer), n0, n1);
+}
+
 static bool
 lex_tokens_match (const struct token *actual, const struct token *expected)
 {
@@ -1179,7 +1187,6 @@ lex_get_encoding (const struct lexer *lexer)
   return src == NULL ? NULL : src->reader->encoding;
 }
 
-
 /* Returns the syntax mode for the syntax file from which the current drawn is
    drawn.  Returns SEG_MODE_AUTO for a T_STOP token or if the command's source
    does not have line numbers.
@@ -1339,16 +1346,24 @@ lex_source__ (const struct lexer *lexer)
 }
 
 static struct substring
-lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
+lex_tokens_get_syntax__ (const struct lex_source *src,
+                         const struct lex_token *token0,
+                         const struct lex_token *token1)
 {
-  const struct lex_token *token0 = lex_source_next__ (src, n0);
-  const struct lex_token *token1 = lex_source_next__ (src, MAX (n0, n1));
   size_t start = token0->token_pos;
   size_t end = token1->token_pos + token1->token_len;
 
   return ss_buffer (&src->buffer[start - src->tail], end - start);
 }
 
+static struct substring
+lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
+{
+  return lex_tokens_get_syntax__ (src,
+                                  lex_source_next__ (src, n0),
+                                  lex_source_next__ (src, MAX (n0, n1)));
+}
+
 static void
 lex_ellipsize__ (struct substring in, char *out, size_t out_size)
 {
@@ -1673,7 +1688,8 @@ lex_source_get (const struct lex_source *src_)
 
   struct macro_expander *me;
   int retval = macro_expander_create (src->lexer->macros,
-                                      &lex_source_front (src)->token, &me);
+                                      &lex_source_front (src)->token,
+                                      &me);
   while (!retval)
     {
       if (!lex_source_get__ (src))
@@ -1684,7 +1700,12 @@ lex_source_get (const struct lex_source *src_)
           NOT_REACHED ();
         }
 
-      retval = macro_expander_add (me, &lex_source_front (src)->token);
+      const struct lex_token *front = lex_source_front (src);
+      const struct macro_token mt = {
+        .token = front->token,
+        .representation = lex_tokens_get_syntax__ (src, front, front)
+      };
+      retval = macro_expander_add (me, &mt);
     }
   if (retval < 0)
     {
@@ -1698,19 +1719,21 @@ lex_source_get (const struct lex_source *src_)
   while (deque_count (&src->deque) > old_count)
     lex_source_pop_front (src);
 
-  struct tokens expansion = { .tokens = NULL };
+  struct macro_tokens expansion = { .n = 0 };
   macro_expander_get_expansion (me, &expansion);
   macro_expander_destroy (me);
 
   for (size_t i = 0; i < expansion.n; i++)
     {
       *lex_push_token__ (src) = (struct lex_token) {
-        .token = expansion.tokens[i],
+        .token = expansion.mts[i].token,
         .from_macro = true,
         /* XXX the rest */
       };
+
+      ss_dealloc (&expansion.mts[i].representation); /* XXX should feed into lexer */
     }
-  free (expansion.tokens);
+  free (expansion.mts);
 
   return true;
 }
diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index 5aa82e75fb..86bb4f2b5d 100644
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -146,6 +146,10 @@ const char *lex_next_tokcstr (const struct lexer *, int n);
 double lex_next_tokval (const struct lexer *, int n);
 struct substring lex_next_tokss (const struct lexer *, int n);
 
+/* Token representation. */
+struct substring lex_next_representation (const struct lexer *,
+                                          int n0, int n1);
+
 /* Current position. */
 int lex_get_first_line_number (const struct lexer *, int n);
 int lex_get_last_line_number (const struct lexer *, int n);
diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c
index 7726e4c3ec..e0b964fd69 100644
--- a/src/language/lexer/macro.c
+++ b/src/language/lexer/macro.c
@@ -31,6 +31,55 @@
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
 
+void
+macro_token_copy (struct macro_token *dst, const struct macro_token *src)
+{
+  token_copy (&dst->token, &src->token);
+  ss_alloc_substring (&dst->representation, src->representation);
+}
+
+void
+macro_token_uninit (struct macro_token *mt)
+{
+  token_uninit (&mt->token);
+  ss_dealloc (&mt->representation);
+}
+
+void
+macro_tokens_copy (struct macro_tokens *dst, const struct macro_tokens *src)
+{
+  *dst = (struct macro_tokens) {
+    .mts = xmalloc (src->n * sizeof *dst->mts),
+    .n = src->n,
+    .allocated = src->n,
+  };
+  for (size_t i = 0; i < src->n; i++)
+    macro_token_copy (&dst->mts[i], &src->mts[i]);
+}
+
+void
+macro_tokens_uninit (struct macro_tokens *mts)
+{
+  for (size_t i = 0; i < mts->n; i++)
+    macro_token_uninit (&mts->mts[i]);
+  free (mts->mts);
+}
+
+void
+macro_tokens_add (struct macro_tokens *mts, const struct macro_token *mt)
+{
+  if (mts->n >= mts->allocated)
+    mts->mts = x2nrealloc (mts->mts, &mts->allocated, sizeof *mts->mts);
+  macro_token_copy (&mts->mts[mts->n++], mt);
+}
+
+void
+macro_tokens_print (const struct macro_tokens *mts, FILE *stream)
+{
+  for (size_t i = 0; i < mts->n; i++)
+    token_print (&mts->mts[i].token, stream);
+}
+
 void
 macro_destroy (struct macro *m)
 {
@@ -43,7 +92,7 @@ macro_destroy (struct macro *m)
       struct macro_param *p = &m->params[i];
       free (p->name);
 
-      tokens_uninit (&p->def);
+      macro_tokens_uninit (&p->def);
 
       switch (p->arg_type)
         {
@@ -51,12 +100,12 @@ macro_destroy (struct macro *m)
           break;
 
         case ARG_CHAREND:
-          token_destroy (&p->charend);
+          token_uninit (&p->charend);
           break;
 
         case ARG_ENCLOSE:
-          token_destroy (&p->enclose[0]);
-          token_destroy (&p->enclose[1]);
+          token_uninit (&p->enclose[0]);
+          token_uninit (&p->enclose[1]);
           break;
 
         case ARG_CMDEND:
@@ -64,8 +113,7 @@ macro_destroy (struct macro *m)
         }
     }
   free (m->params);
-  ss_dealloc (&m->body);
-  tokens_uninit (&m->body_tokens);
+  macro_tokens_uninit (&m->body);
   free (m);
 }
 
@@ -162,7 +210,7 @@ struct macro_expander
     size_t n_tokens;
 
     const struct macro *macro;
-    struct tokens **args;
+    struct macro_tokens **args;
     const struct macro_param *param;
   };
 
@@ -173,7 +221,7 @@ me_finished (struct macro_expander *me)
     if (!me->args[i])
       {
         me->args[i] = xmalloc (sizeof *me->args[i]);
-        tokens_copy (me->args[i], &me->macro->params[i].def);
+        macro_tokens_copy (me->args[i], &me->macro->params[i].def);
       }
   return me->n_tokens;
 }
@@ -217,8 +265,9 @@ me_error (struct macro_expander *me)
 }
 
 static int
-me_add_arg (struct macro_expander *me, const struct token *token)
+me_add_arg (struct macro_expander *me, const struct macro_token *mt)
 {
+  const struct token *token = &mt->token;
   if (token->type == T_STOP)
     {
       msg (SE, _("Unexpected end of file reading argument %s "
@@ -230,13 +279,13 @@ me_add_arg (struct macro_expander *me, const struct token *token)
   me->n_tokens++;
 
   const struct macro_param *p = me->param;
-  struct tokens **argp = &me->args[p - me->macro->params];
+  struct macro_tokens **argp = &me->args[p - me->macro->params];
   if (!*argp)
     *argp = xzalloc (sizeof **argp);
-  struct tokens *arg = *argp;
+  struct macro_tokens *arg = *argp;
   if (p->arg_type == ARG_N_TOKENS)
     {
-      tokens_add (arg, token);
+      macro_tokens_add (arg, mt);
       if (arg->n >= p->n_tokens)
         return me_next_arg (me);
       return 0;
@@ -245,7 +294,7 @@ me_add_arg (struct macro_expander *me, const struct token *token)
     {
       if (token->type == T_ENDCMD || token->type == T_STOP)
         return me_next_arg (me);
-      tokens_add (arg, token);
+      macro_tokens_add (arg, mt);
       return 0;
     }
   else
@@ -254,31 +303,32 @@ me_add_arg (struct macro_expander *me, const struct token *token)
         = p->arg_type == ARG_CMDEND ? &p->charend : &p->enclose[1];
       if (token_equal (token, end))
         return me_next_arg (me);
-      tokens_add (arg, token);
+      macro_tokens_add (arg, mt);
       return 0;
     }
 }
 
 static int
-me_expected (struct macro_expander *me, const struct token *token,
-             const struct token *wanted)
+me_expected (struct macro_expander *me, const struct macro_token *actual,
+             const struct token *expected)
 {
-  char *actual = token_to_string (token);
-  if (!actual)
-    actual = xstrdup ("<eof>");
-  char *expected = token_to_string (wanted);
-  msg (SE, _("Found `%s' while expecting `%s' reading argument %s "
+  const struct substring actual_s
+    = (actual->representation.length ? actual->representation
+       : ss_cstr (_("<end of input>")));
+  char *expected_s = token_to_string (expected);
+  msg (SE, _("Found `%.*s' while expecting `%s' reading argument %s "
              "to macro %s."),
-       actual, expected, me->param->name, me->macro->name);
-  free (expected);
-  free (actual);
+       (int) actual_s.length, actual_s.string, expected_s,
+       me->param->name, me->macro->name);
+  free (expected_s);
 
   return me_error (me);
 }
 
 static int
-me_enclose (struct macro_expander *me, const struct token *token)
+me_enclose (struct macro_expander *me, const struct macro_token *mt)
 {
+  const struct token *token = &mt->token;
   me->n_tokens++;
 
   if (token_equal (&me->param->enclose[0], token))
@@ -287,7 +337,7 @@ me_enclose (struct macro_expander *me, const struct token *token)
       return 0;
     }
 
-  return me_expected (me, token, &me->param->enclose[0]);
+  return me_expected (me, mt, &me->param->enclose[0]);
 }
 
 static const struct macro_param *
@@ -305,8 +355,9 @@ macro_find_parameter_by_name (const struct macro *m, struct substring name)
 }
 
 static int
-me_keyword (struct macro_expander *me, const struct token *token)
+me_keyword (struct macro_expander *me, const struct macro_token *mt)
 {
+  const struct token *token = &mt->token;
   if (token->type != T_ID)
     return me_finished (me);
 
@@ -333,8 +384,9 @@ me_keyword (struct macro_expander *me, const struct token *token)
 }
 
 static int
-me_equals (struct macro_expander *me, const struct token *token)
+me_equals (struct macro_expander *me, const struct macro_token *mt)
 {
+  const struct token *token = &mt->token;
   me->n_tokens++;
 
   if (token->type == T_EQUALS)
@@ -343,8 +395,7 @@ me_equals (struct macro_expander *me, const struct token *token)
       return 0;
     }
 
-  const struct token equals = { .type = T_EQUALS };
-  return me_expected (me, token, &equals);
+  return me_expected (me, mt, &(struct token) { .type = T_EQUALS });
 }
 
 int
@@ -390,7 +441,7 @@ macro_expander_destroy (struct macro_expander *me)
   for (size_t i = 0; i < me->macro->n_params; i++)
     if (me->args[i])
       {
-        tokens_uninit (me->args[i]);
+        macro_tokens_uninit (me->args[i]);
         free (me->args[i]);
       }
   free (me->args);
@@ -413,7 +464,7 @@ macro_expander_destroy (struct macro_expander *me)
    macro invocation is finished.  The caller should call
    macro_expander_get_expansion() to obtain the expansion. */
 int
-macro_expander_add (struct macro_expander *me, const struct token *token)
+macro_expander_add (struct macro_expander *me, const struct macro_token *mt)
 {
   switch (me->state)
     {
@@ -421,16 +472,16 @@ macro_expander_add (struct macro_expander *me, const struct token *token)
       return -1;
 
     case ME_ARG:
-      return me_add_arg (me, token);
+      return me_add_arg (me, mt);
 
     case ME_ENCLOSE:
-      return me_enclose (me, token);
+      return me_enclose (me, mt);
 
     case ME_KEYWORD:
-      return me_keyword (me, token);
+      return me_keyword (me, mt);
 
     case ME_EQUALS:
-      return me_equals (me, token);
+      return me_equals (me, mt);
 
     default:
       NOT_REACHED ();
@@ -439,9 +490,9 @@ macro_expander_add (struct macro_expander *me, const struct token *token)
 
 struct parse_macro_function_ctx
   {
-    const struct tokens *tokens;
+    const struct macro_tokens *mts;
     size_t *idx;
-    struct tokens *args;
+    struct macro_tokens *args;
     int nesting_countdown;
     const struct macro_set *macros;
     const struct macro_expander *me;
@@ -449,29 +500,29 @@ struct parse_macro_function_ctx
   };
 
 static void
-macro_expand (const struct tokens *tokens, int nesting_countdown,
-              const struct macro_set *macros, const struct macro_expander *me,
-              bool *expand, struct tokens *exp);
+macro_expand (const struct macro_tokens *,
+              int nesting_countdown, const struct macro_set *,
+              const struct macro_expander *, bool *expand, struct macro_tokens *exp);
 
 static bool
 parse_macro_function (struct parse_macro_function_ctx *ctx,
                       struct substring function,
                       int min_args, int max_args)
 {
-  const struct token *tokens = ctx->tokens->tokens;
-  size_t n_tokens = ctx->tokens->n;
+  struct macro_token *tokens = ctx->mts->mts;
+  size_t n_tokens = ctx->mts->n;
 
-  if (!ss_equals_case (tokens[0].string, function))
+  if (!ss_equals_case (tokens[0].token.string, function))
     return false;
 
   size_t lparen_idx = *ctx->idx + 1;
-  if (lparen_idx >= n_tokens || tokens[lparen_idx].type != T_LPAREN)
+  if (lparen_idx >= n_tokens || tokens[lparen_idx].token.type != T_LPAREN)
     {
       printf ("`(' expected following %s'\n", function.string);
       return false;
     }
 
-  *ctx->args = (struct tokens) { .n = 0 };
+  *ctx->args = (struct macro_tokens) { .n = 0 };
 
   size_t i = lparen_idx + 1;
   for (size_t j = i; ; j++)
@@ -483,7 +534,7 @@ parse_macro_function (struct parse_macro_function_ctx *ctx,
           goto error;
         }
 
-      int type = tokens[j].type;
+      int type = tokens[j].token.type;
       if (type == T_LPAREN)
         {
           int paren_nesting_level = 1;
@@ -496,21 +547,18 @@ parse_macro_function (struct parse_macro_function_ctx *ctx,
                           ctx->args->n + 1, function.string);
                   goto error;
                 }
-              if (tokens[j].type == T_LPAREN)
+              if (tokens[j].token.type == T_LPAREN)
                 paren_nesting_level++;
-              else if (tokens[j].type == T_RPAREN)
+              else if (tokens[j].token.type == T_RPAREN)
                 paren_nesting_level--;
             }
           while (paren_nesting_level != 0);
         }
       else if (type == T_RPAREN || type == T_COMMA)
         {
-          const struct tokens unexpanded_arg = {
-            .tokens = CONST_CAST (struct token *, &tokens[i]),
-            .n = j - i,
-          };
-          struct tokens expanded_arg = { .n = 0 };
-          macro_expand (&unexpanded_arg, ctx->nesting_countdown, ctx->macros,
+          struct macro_tokens expanded_arg = { .n = 0 };
+          macro_expand (&(const struct macro_tokens) { .mts = &tokens[i], .n = j - i },
+                        ctx->nesting_countdown, ctx->macros,
                         ctx->me, ctx->expand, &expanded_arg);
 
           if (expanded_arg.n != 1)
@@ -518,12 +566,12 @@ parse_macro_function (struct parse_macro_function_ctx *ctx,
               printf ("argument %zu to %s must be a single token "
                       "(not %zu tokens)\n", ctx->args->n + 1, function.string,
                       expanded_arg.n);
-              tokens_uninit (&expanded_arg);
+              macro_tokens_uninit (&expanded_arg);
               goto error;
             }
 
-          tokens_add (ctx->args, &expanded_arg.tokens[0]);
-          tokens_uninit (&expanded_arg);
+          macro_tokens_add (ctx->args, &expanded_arg.mts[0]);
+          macro_tokens_uninit (&expanded_arg);
 
           i = j + 1;
           if (type == T_RPAREN)
@@ -540,26 +588,28 @@ parse_macro_function (struct parse_macro_function_ctx *ctx,
   return true;
 
 error:
-  tokens_uninit (ctx->args);
+  macro_tokens_uninit (ctx->args);
   return false;
 }
 
 static void
-macro_expand (const struct tokens *tokens, int nesting_countdown,
-              const struct macro_set *macros, const struct macro_expander *me,
-              bool *expand, struct tokens *exp)
+macro_expand (const struct macro_tokens *mts,
+              int nesting_countdown, const struct macro_set *macros,
+              const struct macro_expander *me, bool *expand,
+              struct macro_tokens *exp)
 {
   if (nesting_countdown <= 0)
     {
       printf ("maximum nesting level exceeded\n");
-      for (size_t i = 0; i < tokens->n; i++)
-        tokens_add (exp, &tokens->tokens[i]);
+      for (size_t i = 0; i < mts->n; i++)
+        macro_tokens_add (exp, &mts->mts[i]);
       return;
     }
 
-  for (size_t i = 0; i < tokens->n; i++)
+  for (size_t i = 0; i < mts->n; i++)
     {
-      const struct token *token = &tokens->tokens[i];
+      const struct macro_token *mt = &mts->mts[i];
+      const struct token *token = &mt->token;
       if (token->type == T_MACRO_ID && me)
         {
           const struct macro_param *param = macro_find_parameter_by_name (
@@ -567,13 +617,13 @@ macro_expand (const struct tokens *tokens, int nesting_countdown,
           if (param)
             {
               printf ("expand %s to:\n", param->name);
-              const struct tokens *arg = me->args[param - me->macro->params];
-              tokens_print (arg, stdout);
+              const struct macro_tokens *arg = me->args[param - me->macro->params];
+              macro_tokens_print (arg, stdout);
               if (*expand && param->expand_arg)
                 macro_expand (arg, nesting_countdown, macros, NULL, expand, exp);
               else
                 for (size_t i = 0; i < arg->n; i++)
-                  tokens_add (exp, &arg->tokens[i]);
+                  macro_tokens_add (exp, &arg->mts[i]);
               continue;
             }
         }
@@ -584,15 +634,15 @@ macro_expand (const struct tokens *tokens, int nesting_countdown,
           int retval = macro_expander_create (macros, token, &subme);
           for (size_t j = 1; !retval; j++)
             {
-              static const struct token stop = { .type = T_STOP };
+              const struct macro_token stop = { .token = { .type = T_STOP } };
               retval = macro_expander_add (
-                subme, i + j < tokens->n ? &tokens->tokens[i + j] : &stop);
+                subme, i + j < mts->n ? &mts->mts[i + j] : &stop);
             }
           if (retval > 0)
             {
               i += retval - 1;
-              macro_expand (&subme->macro->body_tokens, nesting_countdown - 1,
-                            macros, subme, expand, exp);
+              macro_expand (&subme->macro->body, nesting_countdown - 1, macros,
+                            subme, expand, exp);
               macro_expander_destroy (subme);
               continue;
             }
@@ -602,7 +652,7 @@ macro_expand (const struct tokens *tokens, int nesting_countdown,
 
       if (token->type != T_MACRO_ID)
         {
-          tokens_add (exp, token);
+          macro_tokens_add (exp, mt);
           continue;
         }
 
@@ -627,9 +677,11 @@ macro_expand (const struct tokens *tokens, int nesting_countdown,
         { "!eval", 1, 1 },
       };
 #endif
-      struct tokens args;
+      /* Maybe each arg should just be a string, either a quoted string or a
+         non-quoted string containing tokens. */
+      struct macro_tokens args;
       struct parse_macro_function_ctx ctx = {
-        .tokens = tokens,
+        .mts = mts,
         .idx = &i,
         .args = &args,
         .nesting_countdown = nesting_countdown,
@@ -639,51 +691,60 @@ macro_expand (const struct tokens *tokens, int nesting_countdown,
       };
       if (parse_macro_function (&ctx, ss_cstr ("!length"), 1, 1))
         {
-          char *s = token_to_string (&args.tokens[0]);
-          struct token t = { .type = T_POS_NUM, .number = strlen (s) };
-          tokens_add (exp, &t);
-          free (s);
+          size_t length = args.mts[0].representation.length;
+          struct macro_token mt = {
+            .token = { .type = T_POS_NUM, .number = length },
+            .representation = ss_cstr (xasprintf ("%zu", length)),
+          };
+          macro_tokens_add (exp, &mt);
+          macro_token_uninit (&mt);
 
-          tokens_uninit (&args);
+          macro_tokens_uninit (&args);
         }
       else if (parse_macro_function (&ctx, ss_cstr ("!blanks"), 1, 1))
         {
-          if (args.tokens[0].type != T_POS_NUM)
+          /* XXX this isn't right, it might be a character string containing a
+             positive integer, e.g. via !CONCAT. */
+          if (args.mts[0].token.type != T_POS_NUM)
             printf ("argument to !BLANKS must be positive integer\n");
           else
             {
               struct string s = DS_EMPTY_INITIALIZER;
-              ds_put_byte_multiple (&s, ' ', args.tokens[0].number);
-              struct token t = { .type = T_ID, .string = s.ss };
-              tokens_add (exp, &t);
+              ds_put_byte_multiple (&s, ' ', args.mts[0].token.number);
+
+              struct macro_token mt = {
+                .token = { .type = T_ID, .string = s.ss },
+                .representation = s.ss
+              };
+              macro_tokens_add (exp, &mt);
+
               ds_destroy (&s);
             }
-          tokens_uninit (&args);
+          macro_tokens_uninit (&args);
         }
       else if (ss_equals_case (token->string, ss_cstr ("!onexpand")))
         *expand = true;
       else if (ss_equals_case (token->string, ss_cstr ("!offexpand")))
         *expand = false;
       else
-        tokens_add (exp, token);
+        macro_tokens_add (exp, mt);
     }
 }
 
-
 void
-macro_expander_get_expansion (struct macro_expander *me, struct tokens *exp)
+macro_expander_get_expansion (struct macro_expander *me, struct macro_tokens *exp)
 {
   for (size_t i = 0; i < me->macro->n_params; i++)
     {
       printf ("%s:\n", me->macro->params[i].name);
-      tokens_print (me->args[i], stdout);
+      macro_tokens_print (me->args[i], stdout);
     }
 
   bool expand = true;
-  macro_expand (&me->macro->body_tokens, settings_get_mnest (),
+  macro_expand (&me->macro->body, settings_get_mnest (),
                 me->macros, me, &expand, exp);
 
   printf ("expansion:\n");
-  tokens_print (exp, stdout);
+  macro_tokens_print (exp, stdout);
 }
 
diff --git a/src/language/lexer/macro.h b/src/language/lexer/macro.h
index ba0f1fe735..61cfbc0937 100644
--- a/src/language/lexer/macro.h
+++ b/src/language/lexer/macro.h
@@ -26,11 +26,33 @@
 
 struct macro_expander;
 
+struct macro_token
+  {
+    struct token token;
+    struct substring representation;
+  };
+
+void macro_token_copy (struct macro_token *, const struct macro_token *);
+void macro_token_uninit (struct macro_token *);
+
+struct macro_tokens
+  {
+    struct macro_token *mts;
+    size_t n;
+    size_t allocated;
+  };
+
+void macro_tokens_copy (struct macro_tokens *, const struct macro_tokens *);
+void macro_tokens_uninit (struct macro_tokens *);
+void macro_tokens_add (struct macro_tokens *, const struct macro_token *);
+
+void macro_tokens_print (const struct macro_tokens *, FILE *);
+
 struct macro_param
   {
     bool positional;            /* Is this a positional parameter? */
     char *name;                 /* "!1" or "!name". */
-    struct tokens def;          /* Default expansion. */
+    struct macro_tokens def;    /* Default expansion. */
     bool expand_arg;            /* Macro-expand the argument? */
 
     enum
@@ -57,8 +79,7 @@ struct macro
     struct macro_param *params;
     size_t n_params;
 
-    struct substring body;
-    struct tokens body_tokens;
+    struct macro_tokens body;
   };
 
 void macro_destroy (struct macro *);
@@ -87,8 +108,8 @@ int macro_expander_create (const struct macro_set *,
                            struct macro_expander **);
 void macro_expander_destroy (struct macro_expander *);
 
-int macro_expander_add (struct macro_expander *, const struct token *);
+int macro_expander_add (struct macro_expander *, const struct macro_token *);
 
-void macro_expander_get_expansion (struct macro_expander *, struct tokens *);
+void macro_expander_get_expansion (struct macro_expander *, struct macro_tokens *);
 
 #endif /* macro.h */
diff --git a/src/language/lexer/token.c b/src/language/lexer/token.c
index 7becaaffad..9fa5bbb6ba 100644
--- a/src/language/lexer/token.c
+++ b/src/language/lexer/token.c
@@ -216,7 +216,7 @@ void
 tokens_uninit (struct tokens *tokens)
 {
   for (size_t i = 0; i < tokens->n; i++)
-    token_destroy (&tokens->tokens[i]);
+    token_uninit (&tokens->tokens[i]);
   free (tokens->tokens);
 }