token: Rename token_destroy() to token_uninit().

[pspp] / tests / language / lexer / scan-test.c

diff --git a/tests/language/lexer/scan-test.c b/tests/language/lexer/scan-test.c
index 313fa16398590589e0eeced47b584516eb237354..1eb04338e3b39b183b3c957c68288839ce68d4ab 100644 (file)
--- a/tests/language/lexer/scan-test.c
+++ b/tests/language/lexer/scan-test.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 2010, 2011, 2013 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -39,108 +39,66 @@
 /* -a/--auto, -b/--batch, -i/--interactive: syntax mode. */
 static enum segmenter_mode mode = SEG_MODE_AUTO;
 
+/* -s, --strip-trailing-newline: Strip trailing newline from last line of
+    input. */
+static bool strip_trailing_newline;
+
 static const char *parse_options (int argc, char **argv);
 static void usage (void) NO_RETURN;
 
 int
 main (int argc, char *argv[])
 {
-  struct segment
-    {
-      enum segment_type type;
-      struct substring string;
-    };
-
-  size_t offset;
   const char *file_name;
-  char *input;
-  struct segmenter s;
-  struct segment *segs;
-  size_t n_segs, allocated_segs;
   size_t length;
-  size_t i;
-  int n;
+  char *input;
+
+  struct string_lexer slex;
+  bool more;
 
   set_program_name (argv[0]);
   file_name = parse_options (argc, argv);
 
-  /* Read from stdin into 'input'.  Ensure that 'input' ends in a new-line
-     followed by a null byte. */
+  /* Read from stdin into 'input'. */
   input = (!strcmp (file_name, "-")
-           ? fread_file (stdin, &length)
-           : read_file (file_name, &length));
+           ? fread_file (stdin, 0, &length)
+           : read_file (file_name, 0, &length));
   if (input == NULL)
     error (EXIT_FAILURE, errno, "reading %s failed", file_name);
-  input = xrealloc (input, length + 3);
-  if (length == 0 || input[length - 1] != '\n')
-    input[length++] = '\n';
-  input[length++] = '\0';
-
-  segs = NULL;
-  n_segs = allocated_segs = 0;
 
-  segmenter_init (&s, mode);
-  for (offset = 0; offset < length; offset += n)
+  if (strip_trailing_newline && length && input[length - 1] == '\n')
     {
-      enum segment_type type;
-
-      n = segmenter_push (&s, input + offset, length - offset, &type);
-      assert (n >= 0);
-      assert (offset + n <= length);
-
-      if (n_segs >= allocated_segs)
-        segs = x2nrealloc (segs, &allocated_segs, sizeof *segs);
-
-      segs[n_segs].type = type;
-      segs[n_segs].string.string = input + offset;
-      segs[n_segs].string.length = n;
-      n_segs++;
+      length--;
+      if (length && input[length - 1] == '\r')
+        length--;
     }
 
-  for (i = 0; i < n_segs; )
+  string_lexer_init (&slex, input, length, mode);
+  do
     {
-      enum scan_result result;
-      struct scanner scanner;
       struct token token;
-      int saved = -1;
 
-      scanner_init (&scanner, &token);
-      do
-        {
-          struct segment *seg;
-
-          assert (i < n_segs);
-
-          seg = &segs[i++];
-          result = scanner_push (&scanner, seg->type, seg->string, &token);
-          if (result == SCAN_SAVE)
-            saved = i;
-        }
-      while (result == SCAN_MORE || result == SCAN_SAVE);
-
-      if (result == SCAN_BACK)
-        {
-          assert (saved >= 0);
-          i = saved;
-        }
+      more = string_lexer_next (&slex, &token);
 
       printf ("%s", scan_type_to_string (token.type));
       if (token.number != 0.0)
         {
-          char s[DBL_BUFSIZE_BOUND];
+          double x = token.number;
 
-          dtoastr (s, sizeof s, 0, 0, token.number);
-          printf (" %s", s);
+          if (x > LONG_MIN && x <= LONG_MAX && floor (x) == x)
+            printf (" %ld", (long int) x);
+          else
+            printf (" %.3g", x);
         }
       if (token.string.string != NULL || token.string.length > 0)
         printf (" \"%.*s\"", (int) token.string.length, token.string.string);
       printf ("\n");
 
-      token_destroy (&token);
+      token_uninit (&token);
     }
+  while (more);
 
   free (input);
-  free (segs);
 
   return 0;
 }
@@ -155,11 +113,12 @@ parse_options (int argc, char **argv)
           {"auto", no_argument, NULL, 'a'},
           {"batch", no_argument, NULL, 'b'},
           {"interactive", no_argument, NULL, 'i'},
+          {"strip-trailing-newline", no_argument, NULL, 's'},
           {"help", no_argument, NULL, 'h'},
           {NULL, 0, NULL, 0},
         };
 
-      int c = getopt_long (argc, argv, "abih", options, NULL);
+      int c = getopt_long (argc, argv, "sabih", options, NULL);
       if (c == -1)
         break;
 
@@ -177,6 +136,10 @@ parse_options (int argc, char **argv)
           mode = SEG_MODE_INTERACTIVE;
           break;
 
+        case 's':
+          strip_trailing_newline = true;
+          break;
+
         case 'h':
           usage ();
 
@@ -207,10 +170,10 @@ usage (void)
 usage: %s [OPTIONS] INPUT\n\
 \n\
 Options:\n\
-  -1, --one-segment   feed one segment at a time\n\
   -a, --auto          use \"auto\" syntax mode\n\
   -b, --batch         use \"batch\" syntax mode\n\
   -i, --interactive   use \"interactive\" syntax mode (default)\n\
+  -s, --strip-trailing-newline  remove newline from end of input\n\
   -v, --verbose       include rows and column numbers in output\n\
   -h, --help          print this help message\n",
           program_name, program_name);