X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Fscan.c;h=7aa01593f68be02dc424e1950800607e70ea0948;hb=347a2a2db59f2e3baef3d6345f63cce75e69c6a6;hp=6e9fc618e1568727ef04c6c41f4bf5ae951b9538;hpb=ff5e81803b409939e921211f1ffd46cb24df33e9;p=pspp

diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 6e9fc618e1..7aa01593f6 100644
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -324,6 +324,7 @@ scan_punct1__ (char c0)
     case '<': return T_LT;
     case '>': return T_GT;
     case '~': return T_NOT;
+    default: return T_MACRO_PUNCT;
     }
 
   NOT_REACHED ();
@@ -410,7 +411,7 @@ scan_type_to_string (enum scan_type type)
 #undef SCAN_TYPE
 
     default:
-      return token_type_to_name (type);
+      return token_type_to_name ((enum token_type) type);
     }
 }
 
@@ -440,6 +441,7 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
     case SEG_DO_REPEAT_COMMAND:
     case SEG_INLINE_DATA:
     case SEG_DOCUMENT:
+    case SEG_MACRO_BODY:
       token->type = T_STRING;
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
@@ -453,6 +455,11 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
 
+    case SEG_MACRO_ID:
+      token->type = T_MACRO_ID;
+      ss_alloc_substring (&token->string, s);
+      return SCAN_DONE;
+
     case SEG_PUNCT:
       if (s.length == 1 && s.string[0] == '-')
         {
@@ -462,6 +469,8 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
       else
         {
           token->type = scan_punct__ (s);
+          if (token->type == T_MACRO_PUNCT)
+            ss_alloc_substring (&token->string, s);
           return SCAN_DONE;
         }
 
@@ -497,10 +506,6 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
 
-    case SEG_UNEXPECTED_DOT:
-      token->type = SCAN_UNEXPECTED_DOT;
-      return SCAN_DONE;
-
     case SEG_UNEXPECTED_CHAR:
       return scan_unexpected_char (&s, token);
     }
@@ -543,7 +548,7 @@ void
 scanner_init (struct scanner *scanner, struct token *token)
 {
   scanner->state = S_START;
-  token_init (token);
+  *token = (struct token) { .type = T_STOP };
 }
 
 /* Adds the segment with type TYPE and UTF-8 text S to SCANNER.  TOKEN must be
@@ -593,18 +598,21 @@ scanner_push (struct scanner *scanner, enum segment_type type,
   NOT_REACHED ();
 }
 
-/* Initializes SLEX for parsing INPUT in the specified MODE.
+/* Initializes SLEX for parsing INPUT, which is LENGTH bytes long, in the
+   specified MODE.
 
    SLEX has no internal state to free, but it retains a reference to INPUT, so
    INPUT must not be modified or freed while SLEX is still in use. */
 void
-string_lexer_init (struct string_lexer *slex, const char *input,
-                   enum segmenter_mode mode)
+string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
+                   enum segmenter_mode mode, bool is_snippet)
 {
-  slex->input = input;
-  slex->length = strlen (input) + 1;
-  slex->offset = 0;
-  segmenter_init (&slex->segmenter, mode);
+  *slex = (struct string_lexer) {
+    .input = input,
+    .length = length,
+    .offset = 0,
+    .segmenter = segmenter_init (mode, is_snippet),
+  };
 }
 
 /*  */
@@ -624,7 +632,7 @@ string_lexer_next (struct string_lexer *slex, struct token *token)
       enum segment_type type;
       int n;
 
-      n = segmenter_push (&slex->segmenter, s, left, &type);
+      n = segmenter_push (&slex->segmenter, s, left, true, &type);
       assert (n >= 0);
 
       slex->offset += n;