scan: Get rid of scan token types in favor of new scan result state.

[pspp] / src / language / lexer / lexer.c

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 9753024cc3ef6330f8388538ea4ad1dbc4f1a532..f8b5a840821d6046a6881c1af27e4727a6c0797f 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1081,13 +1081,12 @@ lex_match_phrase (struct lexer *lexer, const char *s)
   i = 0;
   string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
   while (string_lexer_next (&slex, &token))
-    if (token.type != SCAN_SKIP)
-      {
-        bool match = lex_tokens_match (lex_next (lexer, i++), &token);
-        token_uninit (&token);
-        if (!match)
-          return false;
-      }
+    {
+      bool match = lex_tokens_match (lex_next (lexer, i++), &token);
+      token_uninit (&token);
+      if (!match)
+        return false;
+    }
 
   while (i-- > 0)
     lex_get (lexer);
@@ -1667,6 +1666,7 @@ lex_source_try_get__ (struct lex_source *src)
 
   /* Extract segments and pass them through the scanner until we obtain a
      token. */
+  enum scan_result result;
   for (;;)
     {
       /* Extract a segment. */
@@ -1693,9 +1693,8 @@ lex_source_try_get__ (struct lex_source *src)
         }
 
       /* Pass the segment into the scanner and try to get a token out. */
-      enum scan_result result = scanner_push (&scanner, type,
-                                              ss_buffer (segment, seg_len),
-                                              &token->token);
+      result = scanner_push (&scanner, type, ss_buffer (segment, seg_len),
+                             &token->token);
       if (result == SCAN_SAVE)
         saved = state;
       else if (result == SCAN_BACK)
@@ -1703,7 +1702,9 @@ lex_source_try_get__ (struct lex_source *src)
           state = saved;
           break;
         }
-      else if (result == SCAN_DONE)
+      else if (result == SCAN_DONE
+               || result == SCAN_EMPTY
+               || result == SCAN_ERROR)
         break;
     }
 
@@ -1757,37 +1758,24 @@ lex_source_try_get__ (struct lex_source *src)
   src->line_pos = state.line_pos;
   src->n_newlines += state.newlines;
 
-  switch (token->token.type)
+  if (result == SCAN_EMPTY)
+    {
+      lex_source_pop_front (src);
+      return false;
+    }
+  else if (result == SCAN_ERROR)
+    {
+      lex_get_error (src, token->token.string.string);
+      return false;
+    }
+  else if (token->token.type == T_STOP)
     {
-    default:
-      return true;
-
-    case T_STOP:
       token->token.type = T_ENDCMD;
       src->eof = true;
       return true;
-
-    case SCAN_BAD_HEX_LENGTH:
-    case SCAN_BAD_HEX_DIGIT:
-    case SCAN_BAD_UNICODE_DIGIT:
-    case SCAN_BAD_UNICODE_LENGTH:
-    case SCAN_BAD_UNICODE_CODE_POINT:
-    case SCAN_EXPECTED_QUOTE:
-    case SCAN_EXPECTED_EXPONENT:
-    case SCAN_UNEXPECTED_CHAR:
-     {
-      char *msg = scan_token_to_error (&token->token);
-      lex_get_error (src, msg);
-      free (msg);
-      return false;
-     }
-
-    case SCAN_SKIP:
-      lex_source_pop_front (src);
-      return false;
     }
-
-  NOT_REACHED ();
+  else
+    return true;
 }
 
 /* Attempts to add a new token at the front of SRC.  Returns true if