segment: Add support for segmenting "snippets".

[pspp] / src / language / lexer / scan.c

diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index cae523cb37efbe83fc71ba8bbba1aa28b735309f..57e3c2d1a871cc13c4970a1b349449db12cea68e 100644 (file)
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -441,6 +441,7 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
     case SEG_DO_REPEAT_COMMAND:
     case SEG_INLINE_DATA:
     case SEG_DOCUMENT:
+    case SEG_MACRO_BODY:
       token->type = T_STRING;
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
@@ -505,10 +506,6 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
 
-    case SEG_UNEXPECTED_DOT:
-      token->type = SCAN_UNEXPECTED_DOT;
-      return SCAN_DONE;
-
     case SEG_UNEXPECTED_CHAR:
       return scan_unexpected_char (&s, token);
     }
@@ -551,7 +548,7 @@ void
 scanner_init (struct scanner *scanner, struct token *token)
 {
   scanner->state = S_START;
-  token_init (token);
+  *token = (struct token) { .type = T_STOP };
 }
 
 /* Adds the segment with type TYPE and UTF-8 text S to SCANNER.  TOKEN must be
@@ -610,10 +607,12 @@ void
 string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
                    enum segmenter_mode mode)
 {
-  slex->input = input;
-  slex->length = length;
-  slex->offset = 0;
-  segmenter_init (&slex->segmenter, mode);
+  *slex = (struct string_lexer) {
+    .input = input,
+    .length = length,
+    .offset = 0,
+    .segmenter = segmenter_init (mode, true),
+  };
 }
 
 /*  */