fix string lexer

diff --git a/src/language/control/define.c b/src/language/control/define.c
index 7e9e5f37d4273e59506fe549a5ea9867dbd25e28..c21c66a73cab91b9eed75dfd1713159317561857 100644 (file)
--- a/src/language/control/define.c
+++ b/src/language/control/define.c
@@ -59,7 +59,7 @@ parse_quoted_token (struct lexer *lexer, struct token *token)
 
   struct substring s = lex_tokss (lexer);
   struct string_lexer slex;
-  string_lexer_init (&slex, s.string, s.length, SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s.string, s.length, SEG_MODE_INTERACTIVE, true);
   struct token another_token = { .type = T_STOP };
   if (!string_lexer_next (&slex, token)
       || string_lexer_next (&slex, &another_token))

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 1148cc29e0856752d071b784207944c5d95fa536..8711470fb0687314f43f4ce8377cec571678edbc 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1008,7 +1008,7 @@ lex_match_phrase (struct lexer *lexer, const char *s)
   int i;
 
   i = 0;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
   while (string_lexer_next (&slex, &token))
     if (token.type != SCAN_SKIP)
       {

diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c
index ca32800f30a9dc5129d8b6f38f52a597da4398e5..12e27620c1c23c997c285ef4f755ba00348be44a 100644 (file)
--- a/src/language/lexer/macro.c
+++ b/src/language/lexer/macro.c
@@ -879,7 +879,8 @@ static bool
 unquote_string (const char *s, struct string *content)
 {
   struct string_lexer slex;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */,
+                     true);
 
   struct token token1;
   if (!string_lexer_next (&slex, &token1))

diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 57e3c2d1a871cc13c4970a1b349449db12cea68e..7aa01593f68be02dc424e1950800607e70ea0948 100644 (file)
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -605,13 +605,13 @@ scanner_push (struct scanner *scanner, enum segment_type type,
    INPUT must not be modified or freed while SLEX is still in use. */
 void
 string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
-                   enum segmenter_mode mode)
+                   enum segmenter_mode mode, bool is_snippet)
 {
   *slex = (struct string_lexer) {
     .input = input,
     .length = length,
     .offset = 0,
-    .segmenter = segmenter_init (mode, true),
+    .segmenter = segmenter_init (mode, is_snippet),
   };
 }
 

diff --git a/src/language/lexer/scan.h b/src/language/lexer/scan.h
index 866321b0c84c5c61e8dec03b29fc3901554db3b6..1c0ff7a1e5477286ff1d1256a4e6c5823877fba0 100644 (file)
--- a/src/language/lexer/scan.h
+++ b/src/language/lexer/scan.h
@@ -100,7 +100,7 @@ struct string_lexer
   };
 
 void string_lexer_init (struct string_lexer *, const char *input,
-                        size_t length, enum segmenter_mode);
+                        size_t length, enum segmenter_mode, bool is_snippet);
 bool string_lexer_next (struct string_lexer *, struct token *);
 
 #endif /* scan.h */

diff --git a/tests/language/lexer/scan-test.c b/tests/language/lexer/scan-test.c
index 1eb04338e3b39b183b3c957c68288839ce68d4ab..2a77e127ace3405ce6770b31aaf0cd7544d3db9d 100644 (file)
--- a/tests/language/lexer/scan-test.c
+++ b/tests/language/lexer/scan-test.c
@@ -73,7 +73,7 @@ main (int argc, char *argv[])
         length--;
     }
 
-  string_lexer_init (&slex, input, length, mode);
+  string_lexer_init (&slex, input, length, mode, false);
   do
     {
       struct token token;