DEFINE: Properly support redefining a macro.

[pspp] / tests / language / lexer / scan-test.c

diff --git a/tests/language/lexer/scan-test.c b/tests/language/lexer/scan-test.c
index abbf0f9455196331dc617dc47abd0ddee6926bef..95c52a42be48cc204f9bd79ef0ef0a692213d1ce 100644 (file)
--- a/tests/language/lexer/scan-test.c
+++ b/tests/language/lexer/scan-test.c
@@ -54,15 +54,14 @@ main (int argc, char *argv[])
   char *input;
 
   struct string_lexer slex;
-  bool more;
 
   set_program_name (argv[0]);
   file_name = parse_options (argc, argv);
 
   /* Read from stdin into 'input'. */
   input = (!strcmp (file_name, "-")
-           ? fread_file (stdin, &length)
-           : read_file (file_name, &length));
+           ? fread_file (stdin, 0, &length)
+           : read_file (file_name, 0, &length));
   if (input == NULL)
     error (EXIT_FAILURE, errno, "reading %s failed", file_name);
 
@@ -73,31 +72,66 @@ main (int argc, char *argv[])
         length--;
     }
 
-  string_lexer_init (&slex, input, length, mode);
-  do
+  struct token *tokens = NULL;
+  size_t n_tokens = 0;
+  size_t allocated_tokens = 0;
+  string_lexer_init (&slex, input, length, mode, false);
+  for (;;)
     {
-      struct token token;
+      if (n_tokens >= allocated_tokens)
+        tokens = x2nrealloc (tokens, &allocated_tokens, sizeof *tokens);
+      enum string_lexer_result result
+        = string_lexer_next (&slex, &tokens[n_tokens]);
+
+      if (result == SLR_ERROR)
+        tokens[n_tokens].type = T_STOP;
+      n_tokens++;
 
-      more = string_lexer_next (&slex, &token);
+      if (result == SLR_END)
+        break;
+    }
 
-      printf ("%s", scan_type_to_string (token.type));
-      if (token.number != 0.0)
+  for (size_t i = 0; i < n_tokens; )
+    {
+      struct merger m = MERGER_INIT;
+      int retval;
+      struct token out;
+      for (size_t j = i; ; j++)
         {
-          double x = token.number;
+          assert (j < n_tokens);
+          retval = merger_add (&m, &tokens[j], &out);
+          if (retval != -1)
+            break;
+        }
+
+      const struct token *t = retval ? &out : &tokens[i];
+
+      fputs (token_type_to_name (t->type), stdout);
+      if (t->number != 0.0)
+        {
+          double x = t->number;
 
           if (x > LONG_MIN && x <= LONG_MAX && floor (x) == x)
             printf (" %ld", (long int) x);
           else
             printf (" %.3g", x);
         }
-      if (token.string.string != NULL || token.string.length > 0)
-        printf (" \"%.*s\"", (int) token.string.length, token.string.string);
+      if (t->string.string != NULL || t->string.length > 0)
+        printf (" \"%.*s\"", (int) t->string.length, t->string.string);
       printf ("\n");
 
-      token_destroy (&token);
+      if (retval)
+        {
+          i += retval;
+          token_uninit (&out);
+        }
+      else
+        i++;
     }
-  while (more);
 
+  for (size_t i = 0; i < n_tokens; i++)
+    token_uninit (&tokens[i]);
+  free (tokens);
   free (input);
 
   return 0;