From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 4 Jul 2021 02:56:47 +0000 (-0700)
Subject: More error reporting improvemnts.
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=37cbc94c3ed75eb0ac345c7ada138ef4662f1433;p=pspp

More error reporting improvemnts.
---

diff --git a/doc/flow-control.texi b/doc/flow-control.texi
index a3f39b6ccb..7804e8b013 100644
--- a/doc/flow-control.texi
+++ b/doc/flow-control.texi
@@ -622,7 +622,7 @@ extends to the end of @var{arg}.
 @example
 !SUBSTR(banana, 3)           @expansion{} nana
 !SUBSTR(banana, 3, 3)        @expansion{} nan
-!SUBSTR("banana", 3)         @expansion{} @r{error (@code{anana"} is not a valid token)}
+!SUBSTR("banana", 1, 3)         @expansion{} @r{error (@code{"ba} is not a valid token)}
 !SUBSTR(!UNQUOTE("banana"), 3) @expansion{} nana
 !SUBSTR("banana", 3, 3)      @expansion{} ana
 
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index c109077a89..562fcc5827 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1895,7 +1895,7 @@ lex_source_get (const struct lex_source *src_)
 
   /* Now expand the macro. */
   struct macro_tokens expansion = { .n = 0 };
-  macro_expander_get_expansion (me, &expansion);
+  macro_expander_get_expansion (me, src->reader->syntax, &expansion);
   macro_expander_destroy (me);
 
   /* Convert the macro expansion into syntax for possible error messages later. */
diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c
index 42ccf4255f..8678a0bcdb 100644
--- a/src/language/lexer/macro.c
+++ b/src/language/lexer/macro.c
@@ -41,6 +41,84 @@
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
 
+struct macro_expansion_stack
+  {
+    const struct macro_expansion_stack *next;
+    const char *name;
+    const char *file_name;
+    int first_line;
+    int last_line;
+  };
+
+static void PRINTF_FORMAT (3, 4)
+macro_error (const struct macro_expansion_stack *stack,
+             const struct macro_token *mt,
+             const char *format, ...)
+{
+  struct msg_stack **ms = NULL;
+  size_t allocated_ms = 0;
+  size_t n_ms = 0;
+
+  for (const struct macro_expansion_stack *p = stack; p; p = p->next)
+    {
+      if (n_ms >= allocated_ms)
+        ms = x2nrealloc (ms, &allocated_ms, sizeof *ms);
+
+      /* TRANSLATORS: These strings are used for explaining the context of an
+         error.  The "While expanding" message appears first, followed by zero
+         or more of the "inside expansion" messages.  `innermost',
+         `next_inner`, etc., are names of macros, and `foobar' is a piece of
+         PSPP syntax:
+
+         foo.sps:12: At `foobar' in the expansion of 'innermost',
+         foo.sps:23: inside the expansion of 'next_inner',
+         foo.sps:34: inside the expansion of 'next_inner2',
+         foo.sps:45: inside the expansion of 'outermost',
+         foo.sps:76: This is the actual error message. */
+      char *description;
+      if (p == stack)
+        {
+          if (mt && mt->representation.length)
+            {
+              char syntax[64];
+              lex_ellipsize (mt->representation, syntax, sizeof syntax);
+              description = xasprintf (_("At `%s' in the expansion of `%s',"),
+                                       syntax, p->name);
+            }
+          else
+            description = xasprintf (_("In the expansion of `%s',"), p->name);
+        }
+      else
+        description = xasprintf (_("inside the expansion of `%s',"), p->name);
+
+      ms[n_ms] = xmalloc (sizeof *ms[n_ms]);
+      *ms[n_ms] = (struct msg_stack) {
+        .location = {
+          .file_name = xstrdup_if_nonnull (p->file_name),
+          .first_line = p->first_line,
+          .last_line = p->last_line,
+        },
+        .description = description,
+      };
+      n_ms++;
+    }
+
+  va_list args;
+  va_start (args, format);
+  char *s = xvasprintf (format, args);
+  va_end (args);
+
+  struct msg *m = xmalloc (sizeof *m);
+  *m = (struct msg) {
+    .category = MSG_C_SYNTAX,
+    .severity = MSG_S_ERROR,
+    .stack = ms,
+    .n_stack = n_ms,
+    .text = s,
+  };
+  msg_emit (m);
+}
+
 void
 macro_token_copy (struct macro_token *dst, const struct macro_token *src)
 {
@@ -130,9 +208,10 @@ macro_tokens_add (struct macro_tokens *mts, const struct macro_token *mt)
   macro_token_copy (macro_tokens_add_uninit (mts), mt);
 }
 
-void
-macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
-                          enum segmenter_mode mode)
+static void
+macro_tokens_from_string__ (struct macro_tokens *mts, const struct substring src,
+                            enum segmenter_mode mode,
+                            const struct macro_expansion_stack *stack)
 {
   struct state
     {
@@ -180,23 +259,35 @@ macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
         }
 
       /* We have a token in 'token'. */
+      mt.representation.length = state.body.string - mt.representation.string;
       if (is_scan_type (token->type))
         {
           if (token->type != SCAN_SKIP)
             {
-              printf ("error\n");
-              /* XXX report error */
+              char *s = scan_token_to_error (token);
+              if (stack)
+                {
+                  mt.token.type = T_STRING;
+                  macro_error (stack, &mt, "%s", s);
+                }
+              else
+                msg (SE, "%s", s);
+              free (s);
             }
         }
       else
-        {
-          mt.representation.length = state.body.string - mt.representation.string;
-          macro_tokens_add (mts, &mt);
-        }
+        macro_tokens_add (mts, &mt);
       token_uninit (token);
     }
 }
 
+void
+macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
+                          enum segmenter_mode mode)
+{
+  macro_tokens_from_string__ (mts, src, mode, NULL);
+}
+
 void
 macro_tokens_print (const struct macro_tokens *mts, FILE *stream)
 {
@@ -425,84 +516,6 @@ macro_set_add (struct macro_set *set, struct macro *m)
   hmap_insert (&set->macros, &m->hmap_node, hash_macro_name (m->name));
 }
 
-struct macro_expansion_stack
-  {
-    const struct macro_expansion_stack *next;
-    const char *name;
-    const char *file_name;
-    int first_line;
-    int last_line;
-  };
-
-static void PRINTF_FORMAT (3, 4)
-macro_error (const struct macro_expansion_stack *stack,
-             const struct macro_token *mt,
-             const char *format, ...)
-{
-  struct msg_stack **ms = NULL;
-  size_t allocated_ms = 0;
-  size_t n_ms = 0;
-
-  for (const struct macro_expansion_stack *p = stack; p; p = p->next)
-    {
-      if (n_ms >= allocated_ms)
-        ms = x2nrealloc (ms, &allocated_ms, sizeof *ms);
-
-      /* TRANSLATORS: These strings are used for explaining the context of an
-         error.  The "While expanding" message appears first, followed by zero
-         or more of the "inside expansion" messages.  `innermost',
-         `next_inner`, etc., are names of macros, and `foobar' is a piece of
-         PSPP syntax:
-
-         foo.sps:12: At `foobar' in the expansion of 'innermost',
-         foo.sps:23: inside the expansion of 'next_inner',
-         foo.sps:34: inside the expansion of 'next_inner2',
-         foo.sps:45: inside the expansion of 'outermost',
-         foo.sps:76: This is the actual error message. */
-      char *description;
-      if (p == stack)
-        {
-          if (mt && mt->representation.length)
-            {
-              char syntax[64];
-              lex_ellipsize (mt->representation, syntax, sizeof syntax);
-              description = xasprintf (_("At `%s' in the expansion of `%s',"),
-                                       syntax, p->name);
-            }
-          else
-            description = xasprintf (_("In the expansion of `%s',"), p->name);
-        }
-      else
-        description = xasprintf (_("inside the expansion of `%s',"), p->name);
-
-      ms[n_ms] = xmalloc (sizeof *ms[n_ms]);
-      *ms[n_ms] = (struct msg_stack) {
-        .location = {
-          .file_name = xstrdup_if_nonnull (p->file_name),
-          .first_line = p->first_line,
-          .last_line = p->last_line,
-        },
-        .description = description,
-      };
-      n_ms++;
-    }
-
-  va_list args;
-  va_start (args, format);
-  char *s = xvasprintf (format, args);
-  va_end (args);
-
-  struct msg *m = xmalloc (sizeof *m);
-  *m = (struct msg) {
-    .category = MSG_C_SYNTAX,
-    .severity = MSG_S_ERROR,
-    .stack = ms,
-    .n_stack = n_ms,
-    .text = s,
-  };
-  msg_emit (m);
-}
-
 enum me_state
   {
     /* Error state. */
@@ -836,6 +849,7 @@ struct parse_macro_function_ctx
     const struct macro_token *input;
     size_t n_input;
     int nesting_countdown;
+    enum segmenter_mode segmenter_mode;
     const struct macro_set *macros;
     const struct macro_expander *me;
     const struct macro_expansion_stack *stack;
@@ -844,8 +858,8 @@ struct parse_macro_function_ctx
   };
 
 static void
-macro_expand (const struct macro_tokens *,
-              int nesting_countdown, const struct macro_set *,
+macro_expand (const struct macro_tokens *, int nesting_countdown,
+              enum segmenter_mode segmenter_mode, const struct macro_set *,
               const struct macro_expander *, struct string_map *vars,
               const struct macro_expansion_stack *stack,
               bool *expand, bool *break_,
@@ -923,6 +937,7 @@ parse_function_arg (struct parse_macro_function_ctx *ctx,
         .input = &ctx->input[i],
         .n_input = ctx->n_input - i,
         .nesting_countdown = ctx->nesting_countdown,
+        .segmenter_mode = ctx->segmenter_mode,
         .macros = ctx->macros,
         .me = ctx->me,
         .stack = ctx->stack,
@@ -956,7 +971,7 @@ parse_macro_function (struct parse_macro_function_ctx *ctx,
   if (n_tokens < 2 || tokens[1].token.type != T_LPAREN)
     {
       macro_error (ctx->stack, n_tokens > 1 ? &tokens[1] : NULL,
-                   _("`(' expected following %s"), function.string);
+                   _("`(' expected following %s."), function.string);
       return false;
     }
 
@@ -1009,11 +1024,11 @@ error:
 }
 
 static bool
-unquote_string (const char *s, struct string *content)
+unquote_string (const char *s, enum segmenter_mode segmenter_mode,
+                struct string *content)
 {
   struct string_lexer slex;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */,
-                     true);
+  string_lexer_init (&slex, s, strlen (s), segmenter_mode, true);
 
   struct token token1;
   if (!string_lexer_next (&slex, &token1))
@@ -1039,10 +1054,11 @@ unquote_string (const char *s, struct string *content)
 }
 
 static const char *
-unquote_string_in_place (const char *s, struct string *tmp)
+unquote_string_in_place (const char *s, enum segmenter_mode segmenter_mode,
+                         struct string *tmp)
 {
   ds_init_empty (tmp);
-  return unquote_string (s, tmp) ? ds_cstr (tmp) : s;
+  return unquote_string (s, segmenter_mode, tmp) ? ds_cstr (tmp) : s;
 }
 
 static bool
@@ -1075,7 +1091,7 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
         {
           macro_error (ctx->stack, NULL,
                        _("Argument to !BLANKS must be non-negative integer "
-                         "(not \"%s\")"), args.strings[0]);
+                         "(not \"%s\")."), args.strings[0]);
           string_array_destroy (&args);
           return false;
         }
@@ -1086,17 +1102,19 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
                                  input_consumed))
     {
       for (size_t i = 0; i < args.n; i++)
-        if (!unquote_string (args.strings[i], output))
+        if (!unquote_string (args.strings[i], ctx->segmenter_mode, output))
           ds_put_cstr (output, args.strings[i]);
     }
   else if (parse_macro_function (ctx, &args, ss_cstr ("!head"), 1, 1,
                                  input_consumed))
     {
       struct string tmp;
-      const char *s = unquote_string_in_place (args.strings[0], &tmp);
+      const char *s = unquote_string_in_place (args.strings[0],
+                                               ctx->segmenter_mode, &tmp);
 
       struct macro_tokens mts = { .n = 0 };
-      macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
+      macro_tokens_from_string__ (&mts, ss_cstr (s), ctx->segmenter_mode,
+                                  ctx->stack);
       if (mts.n > 0)
         ds_put_substring (output, mts.mts[0].representation);
       macro_tokens_uninit (&mts);
@@ -1112,7 +1130,7 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
   else if (parse_macro_function (ctx, &args, ss_cstr ("!quote"), 1, 1,
                                  input_consumed))
     {
-      if (unquote_string (args.strings[0], NULL))
+      if (unquote_string (args.strings[0], ctx->segmenter_mode, NULL))
         ds_put_cstr (output, args.strings[0]);
       else
         {
@@ -1135,7 +1153,7 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
         {
           macro_error (ctx->stack, NULL,
                        _("Second argument of !SUBSTR must be "
-                         "positive integer (not \"%s\")"),
+                         "positive integer (not \"%s\")."),
                        args.strings[1]);
           string_array_destroy (&args);
           return false;
@@ -1146,7 +1164,7 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
         {
           macro_error (ctx->stack, NULL,
                        _("Third argument of !SUBSTR must be "
-                         "non-negative integer (not \"%s\")"),
+                         "non-negative integer (not \"%s\")."),
                        args.strings[1]);
           string_array_destroy (&args);
           return false;
@@ -1159,10 +1177,12 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
                                  input_consumed))
     {
       struct string tmp;
-      const char *s = unquote_string_in_place (args.strings[0], &tmp);
+      const char *s = unquote_string_in_place (args.strings[0],
+                                               ctx->segmenter_mode, &tmp);
 
       struct macro_tokens mts = { .n = 0 };
-      macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
+      macro_tokens_from_string__ (&mts, ss_cstr (s), ctx->segmenter_mode,
+                                  ctx->stack);
       if (mts.n > 1)
         {
           struct macro_tokens tail = { .mts = mts.mts + 1, .n = mts.n - 1 };
@@ -1174,14 +1194,15 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
   else if (parse_macro_function (ctx, &args, ss_cstr ("!unquote"), 1, 1,
                                  input_consumed))
     {
-      if (!unquote_string (args.strings[0], output))
+      if (!unquote_string (args.strings[0], ctx->segmenter_mode, output))
         ds_put_cstr (output, args.strings[0]);
     }
   else if (parse_macro_function (ctx, &args, ss_cstr ("!upcase"), 1, 1,
                                  input_consumed))
     {
       struct string tmp;
-      const char *s = unquote_string_in_place (args.strings[0], &tmp);
+      const char *s = unquote_string_in_place (args.strings[0],
+                                               ctx->segmenter_mode, &tmp);
       char *upper = utf8_to_upper (s);
       ds_put_cstr (output, upper);
       free (upper);
@@ -1191,11 +1212,11 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
                                  input_consumed))
     {
       struct macro_tokens mts = { .n = 0 };
-      macro_tokens_from_string (&mts, ss_cstr (args.strings[0]),
-                                SEG_MODE_INTERACTIVE /* XXX */);
+      macro_tokens_from_string__ (&mts, ss_cstr (args.strings[0]),
+                                  ctx->segmenter_mode, ctx->stack);
       struct macro_tokens exp = { .n = 0 };
-      macro_expand (&mts, ctx->nesting_countdown - 1, ctx->macros, ctx->me,
-                    ctx->vars,
+      macro_expand (&mts, ctx->nesting_countdown - 1, ctx->segmenter_mode,
+                    ctx->macros, ctx->me, ctx->vars,
                     &(struct macro_expansion_stack) {
                       .name = "!EVAL",
                       .next = ctx->stack,
@@ -1221,6 +1242,7 @@ expand_macro_function (struct parse_macro_function_ctx *ctx,
 struct expr_context
   {
     int nesting_countdown;
+    enum segmenter_mode segmenter_mode;
     const struct macro_set *macros;
     const struct macro_expander *me;
     const struct macro_expansion_stack *stack;
@@ -1262,6 +1284,7 @@ macro_evaluate_literal (const struct expr_context *ctx,
     .input = p,
     .n_input = end - p,
     .nesting_countdown = ctx->nesting_countdown,
+    .segmenter_mode = ctx->segmenter_mode,
     .macros = ctx->macros,
     .me = ctx->me,
     .stack = ctx->stack,
@@ -1271,7 +1294,8 @@ macro_evaluate_literal (const struct expr_context *ctx,
   struct string function_output = DS_EMPTY_INITIALIZER;
   size_t function_consumed = parse_function_arg (&fctx, 0, &function_output);
   struct string unquoted = DS_EMPTY_INITIALIZER;
-  if (unquote_string (ds_cstr (&function_output), &unquoted))
+  if (unquote_string (ds_cstr (&function_output), ctx->segmenter_mode,
+                      &unquoted))
     {
       ds_swap (&function_output, &unquoted);
       ds_destroy (&unquoted);
@@ -1345,8 +1369,10 @@ macro_evaluate_relational (const struct expr_context *ctx,
     }
 
   struct string lhs_tmp, rhs_tmp;
-  int cmp = strcmp/*XXX*/ (unquote_string_in_place (lhs, &lhs_tmp),
-                           unquote_string_in_place (rhs, &rhs_tmp));
+  int cmp = strcmp (unquote_string_in_place (lhs, ctx->segmenter_mode,
+                                             &lhs_tmp),
+                    unquote_string_in_place (rhs, ctx->segmenter_mode,
+                                             &rhs_tmp));
   ds_destroy (&lhs_tmp);
   ds_destroy (&rhs_tmp);
 
@@ -1358,7 +1384,7 @@ macro_evaluate_relational (const struct expr_context *ctx,
             : op == T_LT ? cmp < 0
             : op == T_GT ? cmp > 0
             : op == T_LE ? cmp <= 0
-            :/*op == T_GE*/cmp >= 0);
+            : /* T_GE */ cmp >= 0);
 
   *tokens = p;
   return xstrdup (b ? "1" : "0");
@@ -1457,13 +1483,16 @@ macro_evaluate_or (const struct expr_context *ctx,
 
 static char *
 macro_evaluate_expression (const struct macro_token **tokens, size_t n_tokens,
-                           int nesting_countdown, const struct macro_set *macros,
+                           int nesting_countdown,
+                           enum segmenter_mode segmenter_mode,
+                           const struct macro_set *macros,
                            const struct macro_expander *me,
                            const struct macro_expansion_stack *stack,
                            struct string_map *vars, bool *expand)
 {
   const struct expr_context ctx = {
     .nesting_countdown = nesting_countdown,
+    .segmenter_mode = segmenter_mode,
     .macros = macros,
     .me = me,
     .stack = stack,
@@ -1475,25 +1504,28 @@ macro_evaluate_expression (const struct macro_token **tokens, size_t n_tokens,
 
 static bool
 macro_evaluate_number (const struct macro_token **tokens, size_t n_tokens,
-                       int nesting_countdown, const struct macro_set *macros,
+                       int nesting_countdown,
+                       enum segmenter_mode segmenter_mode,
+                       const struct macro_set *macros,
                        const struct macro_expander *me,
                        const struct macro_expansion_stack *stack,
                        struct string_map *vars,
                        bool *expand, double *number)
 {
   char *s = macro_evaluate_expression (tokens, n_tokens, nesting_countdown,
-                                       macros, me, stack, vars, expand);
+                                       segmenter_mode, macros, me, stack, vars,
+                                       expand);
   if (!s)
     return false;
 
   struct macro_tokens mts = { .n = 0 };
-  macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
+  macro_tokens_from_string__ (&mts, ss_cstr (s), segmenter_mode, stack);
   if (mts.n != 1 || !token_is_number (&mts.mts[0].token))
     {
       macro_tokens_print (&mts, stdout);
       macro_error (stack, mts.n > 0 ? &mts.mts[0] : NULL,
                    _("Macro expression must evaluate to "
-                     "a number (not \"%s\")"), s);
+                     "a number (not \"%s\")."), s);
       free (s);
       macro_tokens_uninit (&mts);
       return false;
@@ -1530,7 +1562,8 @@ find_ifend_clause (const struct macro_token *p, const struct macro_token *end)
 
 static size_t
 macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
-                 int nesting_countdown, const struct macro_set *macros,
+                 int nesting_countdown, enum segmenter_mode segmenter_mode,
+                 const struct macro_set *macros,
                  const struct macro_expander *me,
                  const struct macro_expansion_stack *stack,
                  struct string_map *vars,
@@ -1544,7 +1577,8 @@ macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
 
   p++;
   char *result = macro_evaluate_expression (&p, end - p,
-                                            nesting_countdown, macros, me,
+                                            nesting_countdown, segmenter_mode,
+                                            macros, me,
                                             stack, vars, expand);
   if (!result)
     return 0;
@@ -1612,7 +1646,7 @@ macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
         .mts = CONST_CAST (struct macro_token *, start),
         .n = n,
       };
-      macro_expand (&mts, nesting_countdown, macros, me, vars,
+      macro_expand (&mts, nesting_countdown, segmenter_mode, macros, me, vars,
                     &(struct macro_expansion_stack) {
                       .name = "!IF",
                       .next = stack,
@@ -1624,7 +1658,8 @@ macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
 
 static size_t
 macro_parse_let (const struct macro_token *tokens, size_t n_tokens,
-                 int nesting_countdown, const struct macro_set *macros,
+                 int nesting_countdown, enum segmenter_mode segmenter_mode,
+                 const struct macro_set *macros,
                  const struct macro_expander *me,
                  const struct macro_expansion_stack *stack,
                  struct string_map *vars, bool *expand)
@@ -1648,7 +1683,7 @@ macro_parse_let (const struct macro_token *tokens, size_t n_tokens,
     {
       macro_error (stack, p < end ? p : NULL,
                    _("Cannot use argument name or macro keyword "
-                     "\"%.*s\" as !LET variable"),
+                     "\"%.*s\" as !LET variable."),
                    (int) var_name.length, var_name.string);
       return 0;
     }
@@ -1657,13 +1692,13 @@ macro_parse_let (const struct macro_token *tokens, size_t n_tokens,
   if (p >= end || p->token.type != T_EQUALS)
     {
       macro_error (stack, p < end ? p : NULL,
-                   _("Expected `=' following !LET"));
+                   _("Expected `=' following !LET."));
       return 0;
     }
   p++;
 
-  char *value = macro_evaluate_expression (&p, end - p,
-                                           nesting_countdown, macros, me, stack,
+  char *value = macro_evaluate_expression (&p, end - p, nesting_countdown,
+                                           segmenter_mode, macros, me, stack,
                                            vars, expand);
   if (!value)
     return 0;
@@ -1697,7 +1732,8 @@ find_doend (const struct macro_expansion_stack *stack,
 
 static size_t
 macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
-                 int nesting_countdown, const struct macro_set *macros,
+                 int nesting_countdown, enum segmenter_mode segmenter_mode,
+                 const struct macro_set *macros,
                  const struct macro_expander *me,
                  const struct macro_expansion_stack *stack,
                  struct string_map *vars,
@@ -1713,7 +1749,7 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
   if (p >= end || p->token.type != T_MACRO_ID)
     {
       macro_error (stack, p < end ? p : NULL,
-                   _("Expected macro variable name following !DO"));
+                   _("Expected macro variable name following !DO."));
       return 0;
     }
   const struct substring var_name = p->token.string;
@@ -1721,7 +1757,7 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
       || macro_find_parameter_by_name (me->macro, var_name))
     {
       macro_error (stack, p, _("Cannot use argument name or macro "
-                               "keyword as !DO variable"));
+                               "keyword as !DO variable."));
       return 0;
     }
   p++;
@@ -1734,15 +1770,15 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
       && ss_equals_case (p->token.string, ss_cstr ("!IN")))
     {
       p++;
-      char *list = macro_evaluate_expression (&p, end - p,
-                                              nesting_countdown, macros, me,
+      char *list = macro_evaluate_expression (&p, end - p, nesting_countdown,
+                                              segmenter_mode, macros, me,
                                               &next_stack, vars, expand);
       if (!list)
         return 0;
 
       struct macro_tokens items = { .n = 0 };
-      macro_tokens_from_string (&items, ss_cstr (list),
-                                SEG_MODE_INTERACTIVE /* XXX */);
+      macro_tokens_from_string__ (&items, ss_cstr (list), segmenter_mode,
+                                  stack);
       free (list);
 
       const struct macro_token *do_end = find_doend (stack, p, end);
@@ -1771,7 +1807,7 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
                                      ss_xstrdup (items.mts[i].representation));
 
           bool break_ = false;
-          macro_expand (&inner, nesting_countdown, macros,
+          macro_expand (&inner, nesting_countdown, segmenter_mode, macros,
                         me, vars, &next_stack, expand, &break_, exp);
           if (break_)
             break;
@@ -1782,22 +1818,24 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
     {
       p++;
       double first;
-      if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
-                                  &next_stack, vars, expand, &first))
+      if (!macro_evaluate_number (&p, end - p, nesting_countdown,
+                                  segmenter_mode, macros, me, &next_stack,
+                                  vars, expand, &first))
         return 0;
 
       if (p >= end || p->token.type != T_MACRO_ID
           || !ss_equals_case (p->token.string, ss_cstr ("!TO")))
         {
           macro_error (stack, p < end ? p : NULL,
-                       _("Expected !TO in numerical !DO loop"));
+                       _("Expected !TO in numerical !DO loop."));
           return 0;
         }
       p++;
 
       double last;
-      if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
-                                  &next_stack, vars, expand, &last))
+      if (!macro_evaluate_number (&p, end - p, nesting_countdown,
+                                  segmenter_mode, macros, me, &next_stack,
+                                  vars, expand, &last))
         return 0;
 
       double by = 1.0;
@@ -1805,8 +1843,9 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
           && ss_equals_case (p->token.string, ss_cstr ("!BY")))
         {
           p++;
-          if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
-                                      &next_stack, vars, expand, &by))
+          if (!macro_evaluate_number (&p, end - p, nesting_countdown,
+                                      segmenter_mode, macros, me, &next_stack,
+                                      vars, expand, &by))
             return 0;
 
           if (by == 0.0)
@@ -1847,7 +1886,7 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
                                          xstrdup (index_s));
 
               bool break_ = false;
-              macro_expand (&inner, nesting_countdown, macros,
+              macro_expand (&inner, nesting_countdown, segmenter_mode, macros,
                             me, vars, &next_stack, expand, &break_, exp);
               if (break_)
                 break;
@@ -1865,15 +1904,17 @@ macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
 }
 
 static void
-macro_expand (const struct macro_tokens *mts,
-              int nesting_countdown, const struct macro_set *macros,
+macro_expand (const struct macro_tokens *mts, int nesting_countdown,
+              enum segmenter_mode segmenter_mode,
+              const struct macro_set *macros,
               const struct macro_expander *me, struct string_map *vars,
               const struct macro_expansion_stack *stack,
               bool *expand, bool *break_, struct macro_tokens *exp)
 {
   if (nesting_countdown <= 0)
     {
-      macro_error (stack, NULL, _("Maximum nesting level %d exceeded."),
+      macro_error (stack, NULL, _("Maximum nesting level %d exceeded.  "
+                                  "(Use SET MNEST to change the limit.)"),
                    settings_get_mnest ());
       for (size_t i = 0; i < mts->n; i++)
         macro_tokens_add (exp, &mts->mts[i]);
@@ -1896,7 +1937,8 @@ macro_expand (const struct macro_tokens *mts,
             {
               const struct macro_tokens *arg = me->args[param - me->macro->params];
               if (*expand && param->expand_arg)
-                macro_expand (arg, nesting_countdown, macros, NULL, NULL,
+                macro_expand (arg, nesting_countdown, segmenter_mode,
+                              macros, NULL, NULL,
                               &(struct macro_expansion_stack) {
                                 .name = param->name,
                                 .next = stack,
@@ -1917,7 +1959,8 @@ macro_expand (const struct macro_tokens *mts,
 
                   const struct macro_tokens *arg = me->args[j];
                   if (*expand && param->expand_arg)
-                    macro_expand (arg, nesting_countdown, macros, NULL, NULL,
+                    macro_expand (arg, nesting_countdown, segmenter_mode,
+                                  macros, NULL, NULL,
                                   &(struct macro_expansion_stack) {
                                     .name = "!*",
                                     .next = stack,
@@ -1931,7 +1974,8 @@ macro_expand (const struct macro_tokens *mts,
             }
 
           size_t n = macro_expand_if (&mts->mts[i], mts->n - i,
-                                      nesting_countdown, macros, me, stack,
+                                      nesting_countdown, segmenter_mode,
+                                      macros, me, stack,
                                       vars, expand, break_, exp);
           if (n > 0)
             {
@@ -1946,8 +1990,8 @@ macro_expand (const struct macro_tokens *mts,
                                                  token->string.length);
           if (value)
             {
-              macro_tokens_from_string (exp, ss_cstr (value),
-                                        SEG_MODE_INTERACTIVE /* XXX */);
+              macro_tokens_from_string__ (exp, ss_cstr (value), segmenter_mode,
+                                          stack);
               continue;
             }
         }
@@ -1965,8 +2009,8 @@ macro_expand (const struct macro_tokens *mts,
           if (retval > 0)
             {
               i += retval - 1;
-              macro_expand (&subme->macro->body, nesting_countdown - 1, macros,
-                            subme, NULL,
+              macro_expand (&subme->macro->body, nesting_countdown - 1,
+                            segmenter_mode, macros, subme, NULL,
                             &(struct macro_expansion_stack) {
                               .name = subme->macro->name,
                               .file_name = subme->macro->file_name,
@@ -2002,6 +2046,7 @@ macro_expand (const struct macro_tokens *mts,
         .input = &mts->mts[i],
         .n_input = mts->n - i,
         .nesting_countdown = nesting_countdown,
+        .segmenter_mode = segmenter_mode,
         .macros = macros,
         .me = me,
         .stack = stack,
@@ -2014,16 +2059,16 @@ macro_expand (const struct macro_tokens *mts,
         {
           i += function_consumed - 1;
 
-          macro_tokens_from_string (exp, function_output.ss,
-                                    SEG_MODE_INTERACTIVE /* XXX */);
+          macro_tokens_from_string__ (exp, function_output.ss, segmenter_mode,
+                                      stack);
           ds_destroy (&function_output);
 
           continue;
         }
 
       size_t n = macro_parse_let (&mts->mts[i], mts->n - i,
-                                  nesting_countdown, macros, me, stack,
-                                  vars, expand);
+                                  nesting_countdown, segmenter_mode,
+                                  macros, me, stack, vars, expand);
       if (n > 0)
         {
           i += n - 1;
@@ -2031,8 +2076,8 @@ macro_expand (const struct macro_tokens *mts,
         }
 
       n = macro_expand_do (&mts->mts[i], mts->n - i,
-                           nesting_countdown, macros, me, stack, vars,
-                           expand, exp);
+                           nesting_countdown, segmenter_mode,
+                           macros, me, stack, vars, expand, exp);
       if (n > 0)
         {
           i += n - 1;
@@ -2051,7 +2096,9 @@ macro_expand (const struct macro_tokens *mts,
 }
 
 void
-macro_expander_get_expansion (struct macro_expander *me, struct macro_tokens *exp)
+macro_expander_get_expansion (struct macro_expander *me,
+                              enum segmenter_mode segmenter_mode,
+                              struct macro_tokens *exp)
 {
   bool expand = true;
   struct macro_expansion_stack stack = {
@@ -2060,7 +2107,7 @@ macro_expander_get_expansion (struct macro_expander *me, struct macro_tokens *ex
     .first_line = me->macro->first_line,
     .last_line = me->macro->last_line,
   };
-  macro_expand (&me->macro->body, settings_get_mnest (),
+  macro_expand (&me->macro->body, settings_get_mnest (), segmenter_mode,
                 me->macros, me, NULL, &stack, &expand, NULL, exp);
 }
 
diff --git a/src/language/lexer/macro.h b/src/language/lexer/macro.h
index 529eed05fd..305ab85f94 100644
--- a/src/language/lexer/macro.h
+++ b/src/language/lexer/macro.h
@@ -127,6 +127,8 @@ void macro_expander_destroy (struct macro_expander *);
 
 int macro_expander_add (struct macro_expander *, const struct macro_token *);
 
-void macro_expander_get_expansion (struct macro_expander *, struct macro_tokens *);
+void macro_expander_get_expansion (struct macro_expander *,
+                                   enum segmenter_mode segmenter_mode,
+                                   struct macro_tokens *);
 
 #endif /* macro.h */
diff --git a/tests/language/control/define.at b/tests/language/control/define.at
index 92f255fbcc..f1ea2539fb 100644
--- a/tests/language/control/define.at
+++ b/tests/language/control/define.at
@@ -527,7 +527,7 @@ PSPP_CHECK_MACRO_EXPANSION([!SUBSTR],
   [DEFINE !s()
 !SUBSTR(banana, 3).
 !SUBSTR(banana, 3, 3).
-!SUBSTR("banana", 3).
+!SUBSTR("banana", 1, 3).
 !SUBSTR(!UNQUOTE("banana"), 3).
 !SUBSTR("banana", 3, 3).
 !SUBSTR(banana, 3, 0).
@@ -538,7 +538,7 @@ PSPP_CHECK_MACRO_EXPANSION([!SUBSTR],
   [error
 nana.
 nan.
-anana.
+.
 nana.
 ana.
 .