From: Ben Pfaff <blp@cs.stanford.edu>
Date: Tue, 13 Apr 2021 17:35:02 +0000 (-0700)
Subject: segment: Allow '.' as separate token with a line, for macros.
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pspp;a=commitdiff_plain;h=e546c5d3c1ef7bc06c8d7bb6752a863ba3b041e5

segment: Allow '.' as separate token with a line, for macros.

The description of macros just says that ^ and \ can't be used as macro
delimiters, so . is probably OK.
---

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 5044731348..baebda3f6d 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1607,10 +1607,6 @@ lex_source_get__ (const struct lex_source *src_)
                      token->token.string.string);
       break;
 
-    case SCAN_UNEXPECTED_DOT:
-      lex_get_error (src, _("Unexpected `.' in middle of command"));
-      break;
-
     case SCAN_UNEXPECTED_CHAR:
       {
         char c_name[16];
diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 2ad467d978..86ebb7d006 100644
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -506,10 +506,6 @@ scan_start__ (struct scanner *scanner, enum segment_type type,
       ss_alloc_substring (&token->string, s);
       return SCAN_DONE;
 
-    case SEG_UNEXPECTED_DOT:
-      token->type = SCAN_UNEXPECTED_DOT;
-      return SCAN_DONE;
-
     case SEG_UNEXPECTED_CHAR:
       return scan_unexpected_char (&s, token);
     }
diff --git a/src/language/lexer/scan.h b/src/language/lexer/scan.h
index 4327e9bb0b..866321b0c8 100644
--- a/src/language/lexer/scan.h
+++ b/src/language/lexer/scan.h
@@ -45,7 +45,6 @@ struct token;
                                                 \
     SCAN_TYPE(EXPECTED_QUOTE)                   \
     SCAN_TYPE(EXPECTED_EXPONENT)                \
-    SCAN_TYPE(UNEXPECTED_DOT)                   \
     SCAN_TYPE(UNEXPECTED_CHAR)                  \
                                                 \
     SCAN_TYPE(SKIP)
diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c
index d2aa391748..62e9970bb1 100644
--- a/src/language/lexer/segment.c
+++ b/src/language/lexer/segment.c
@@ -676,7 +676,6 @@ next_id_in_command (const struct segmenter *s, const char *input, size_t n,
         case SEG_END:
         case SEG_EXPECTED_QUOTE:
         case SEG_EXPECTED_EXPONENT:
-        case SEG_UNEXPECTED_DOT:
         case SEG_UNEXPECTED_CHAR:
           id[0] = '\0';
           return ofs + retval;
@@ -981,7 +980,7 @@ segmenter_parse_mid_command__ (struct segmenter *s,
           s->substate = SS_START_OF_COMMAND;
         }
       else
-        *type = SEG_UNEXPECTED_DOT;
+        *type = SEG_PUNCT;
       return 1;
 
     case '0': case '1': case '2': case '3': case '4':
diff --git a/src/language/lexer/segment.h b/src/language/lexer/segment.h
index bb788f4230..02a269bdd2 100644
--- a/src/language/lexer/segment.h
+++ b/src/language/lexer/segment.h
@@ -91,7 +91,6 @@ enum segmenter_mode
                                                 \
     SEG_TYPE(EXPECTED_QUOTE)                    \
     SEG_TYPE(EXPECTED_EXPONENT)                 \
-    SEG_TYPE(UNEXPECTED_DOT)                    \
     SEG_TYPE(UNEXPECTED_CHAR)
 
 /* Types of segments. */
diff --git a/tests/language/lexer/lexer.at b/tests/language/lexer/lexer.at
index 8438bfb26d..c572e5fd86 100644
--- a/tests/language/lexer/lexer.at
+++ b/tests/language/lexer/lexer.at
@@ -68,9 +68,7 @@ lexer.sps:8.1-8.70: error: Syntax error at `'very long unterminated string that
 
 lexer.sps:9.1-9.2: error: Syntax error at `1e': Missing exponent following `1e'.
 
-lexer.sps:9.4: error: Syntax error at `.': Unexpected `.' in middle of command.
-
-lexer.sps:9: error: Unknown command `x'.
+lexer.sps:9.4: error: Syntax error at `.': expecting command name.
 
 lexer.sps:10.1: error: Syntax error at `^': Bad character `^' in input.
 
@@ -89,8 +87,6 @@ lexer.sps:1: error: Unknown command `datA dist'.
 
 lexer.sps:2: error: LIST: LIST is allowed only after the active dataset has been defined.
 
-lexer.sps:2.5: error: LIST: Syntax error at `.': Unexpected `.' in middle of command.
-
 lexer.sps:2.6: error: LIST: Syntax error at `...': Bad character U+0000 in input.
 ])
 AT_CLEANUP
diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at
index 6df0caae9e..d263352d88 100644
--- a/tests/language/lexer/scan.at
+++ b/tests/language/lexer/scan.at
@@ -49,7 +49,7 @@ MACRO_ID "!abcd"
 SKIP
 ID "#.#"
 SKIP
-UNEXPECTED_DOT
+MACRO_PUNCT "."
 ID "x"
 SKIP
 MACRO_PUNCT "_"