From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 27 Jun 2021 07:10:23 +0000 (-0700)
Subject: fix string lexer
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pspp;a=commitdiff_plain;h=a3fd68bc627bda62cd08e2041b9910f0ee802f97

fix string lexer
---

diff --git a/src/language/control/define.c b/src/language/control/define.c
index 7e9e5f37d4..c21c66a73c 100644
--- a/src/language/control/define.c
+++ b/src/language/control/define.c
@@ -59,7 +59,7 @@ parse_quoted_token (struct lexer *lexer, struct token *token)
 
   struct substring s = lex_tokss (lexer);
   struct string_lexer slex;
-  string_lexer_init (&slex, s.string, s.length, SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s.string, s.length, SEG_MODE_INTERACTIVE, true);
   struct token another_token = { .type = T_STOP };
   if (!string_lexer_next (&slex, token)
       || string_lexer_next (&slex, &another_token))
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 1148cc29e0..8711470fb0 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1008,7 +1008,7 @@ lex_match_phrase (struct lexer *lexer, const char *s)
   int i;
 
   i = 0;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
   while (string_lexer_next (&slex, &token))
     if (token.type != SCAN_SKIP)
       {
diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c
index ca32800f30..12e27620c1 100644
--- a/src/language/lexer/macro.c
+++ b/src/language/lexer/macro.c
@@ -879,7 +879,8 @@ static bool
 unquote_string (const char *s, struct string *content)
 {
   struct string_lexer slex;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */,
+                     true);
 
   struct token token1;
   if (!string_lexer_next (&slex, &token1))
diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 57e3c2d1a8..7aa01593f6 100644
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -605,13 +605,13 @@ scanner_push (struct scanner *scanner, enum segment_type type,
    INPUT must not be modified or freed while SLEX is still in use. */
 void
 string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
-                   enum segmenter_mode mode)
+                   enum segmenter_mode mode, bool is_snippet)
 {
   *slex = (struct string_lexer) {
     .input = input,
     .length = length,
     .offset = 0,
-    .segmenter = segmenter_init (mode, true),
+    .segmenter = segmenter_init (mode, is_snippet),
   };
 }
 
diff --git a/src/language/lexer/scan.h b/src/language/lexer/scan.h
index 866321b0c8..1c0ff7a1e5 100644
--- a/src/language/lexer/scan.h
+++ b/src/language/lexer/scan.h
@@ -100,7 +100,7 @@ struct string_lexer
   };
 
 void string_lexer_init (struct string_lexer *, const char *input,
-                        size_t length, enum segmenter_mode);
+                        size_t length, enum segmenter_mode, bool is_snippet);
 bool string_lexer_next (struct string_lexer *, struct token *);
 
 #endif /* scan.h */
diff --git a/tests/language/lexer/scan-test.c b/tests/language/lexer/scan-test.c
index 1eb04338e3..2a77e127ac 100644
--- a/tests/language/lexer/scan-test.c
+++ b/tests/language/lexer/scan-test.c
@@ -73,7 +73,7 @@ main (int argc, char *argv[])
         length--;
     }
 
-  string_lexer_init (&slex, input, length, mode);
+  string_lexer_init (&slex, input, length, mode, false);
   do
     {
       struct token token;