spv: ALlow an extra 01 at the end of X3.

[pspp] / src / language / lexer / scan.c

diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 3150470caa9e379af16d2072b63d2b5be3717488..e4fe405d47c340963666604f3c43a4059fd2a456 100644 (file)
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -188,6 +188,8 @@ scan_punct1__ (char c0)
     case '-': return T_DASH;
     case '[': return T_LBRACK;
     case ']': return T_RBRACK;
+    case '{': return T_LCURLY;
+    case '}': return T_RCURLY;
     case '&': return T_AND;
     case '|': return T_OR;
     case '+': return T_PLUS;
@@ -196,6 +198,8 @@ scan_punct1__ (char c0)
     case '<': return T_LT;
     case '>': return T_GT;
     case '~': return T_NOT;
+    case ';': return T_SEMICOLON;
+    case ':': return T_COLON;
     default: return T_MACRO_PUNCT;
     }
 
@@ -290,6 +294,7 @@ tokenize_string_segment__ (enum segment_type type,
   else
     {
       tokenize_error__ (token, error);
+      ss_dealloc (&out);
       return TOKENIZE_ERROR;
     }
 }
@@ -328,8 +333,8 @@ token_from_segment (enum segment_type type, struct substring s,
     case SEG_INLINE_DATA:
     case SEG_DOCUMENT:
     case SEG_MACRO_BODY:
-      *token = (struct token) { .type = T_STRING };
-      ss_alloc_substring (&token->string, s);
+    case SEG_MACRO_NAME:
+      *token = (struct token) { .type = T_STRING, .string = ss_clone (s) };
       return TOKENIZE_TOKEN;
 
     case SEG_RESERVED_WORD:
@@ -337,19 +342,17 @@ token_from_segment (enum segment_type type, struct substring s,
       return TOKENIZE_TOKEN;
 
     case SEG_IDENTIFIER:
-      *token = (struct token) { .type = T_ID };
-      ss_alloc_substring (&token->string, s);
+      *token = (struct token) { .type = T_ID, .string = ss_clone (s) };
       return TOKENIZE_TOKEN;
 
     case SEG_MACRO_ID:
-      *token = (struct token) { .type = T_MACRO_ID };
-      ss_alloc_substring (&token->string, s);
+      *token = (struct token) { .type = T_MACRO_ID, .string = ss_clone (s)};
       return TOKENIZE_TOKEN;
 
     case SEG_PUNCT:
       *token = (struct token) { .type = scan_punct__ (s) };
       if (token->type == T_MACRO_PUNCT)
-        ss_alloc_substring (&token->string, s);
+        token->string = ss_clone (s);
       return TOKENIZE_TOKEN;
 
     case SEG_SHBANG:
@@ -360,8 +363,10 @@ token_from_segment (enum segment_type type, struct substring s,
       return TOKENIZE_EMPTY;
 
     case SEG_START_DOCUMENT:
-      *token = (struct token) { .type = T_ID };
-      ss_alloc_substring (&token->string, ss_cstr ("DOCUMENT"));
+      *token = (struct token) {
+        .type = T_ID,
+        .string = ss_clone (ss_cstr ("DOCUMENT"))
+      };
       return TOKENIZE_TOKEN;
 
     case SEG_START_COMMAND: