From: Ben Pfaff <blp@cs.stanford.edu>
Date: Mon, 6 Dec 2021 05:14:08 +0000 (-0800)
Subject: lexer: Add tokens for '{', '}', ':', ';' for use in the matrix language.
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pspp;a=commitdiff_plain;h=64ba9bcf7a234b51dbaeac545338981bfc2a700c

lexer: Add tokens for '{', '}', ':', ';' for use in the matrix language.
---

diff --git a/src/data/identifier.c b/src/data/identifier.c
index 07c7675bd5..baa0abd1c5 100644
--- a/src/data/identifier.c
+++ b/src/data/identifier.c
@@ -99,9 +99,21 @@ token_type_to_string (enum token_type token)
     case T_RBRACK:
       return "]";
 
+    case T_LCURLY:
+      return "{";
+
+    case T_RCURLY:
+      return "}";
+
     case T_COMMA:
       return ",";
 
+    case T_SEMICOLON:
+      return ";";
+
+    case T_COLON:
+      return ":";
+
     case T_AND:
       return "AND";
 
diff --git a/src/data/identifier.h b/src/data/identifier.h
index d694a5201d..dcbce970cd 100644
--- a/src/data/identifier.h
+++ b/src/data/identifier.h
@@ -41,7 +41,11 @@
     TOKEN_TYPE(RPAREN)              /* ) */                             \
     TOKEN_TYPE(LBRACK)              /* [ */                             \
     TOKEN_TYPE(RBRACK)              /* ] */                             \
+    TOKEN_TYPE(LCURLY)              /* { */                             \
+    TOKEN_TYPE(RCURLY)              /* } */                             \
     TOKEN_TYPE(COMMA)               /* , */                             \
+    TOKEN_TYPE(SEMICOLON)           /* ; */                             \
+    TOKEN_TYPE(COLON)               /* : */                             \
                                                                         \
     TOKEN_TYPE(AND)                 /* AND */                           \
     TOKEN_TYPE(OR)                  /* OR */                            \
diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c
index a089a3a7c3..e5805f0356 100644
--- a/src/language/lexer/macro.c
+++ b/src/language/lexer/macro.c
@@ -334,6 +334,8 @@ classify_token (enum token_type type)
     case T_RPAREN:
     case T_LBRACK:
     case T_RBRACK:
+    case T_LCURLY:
+    case T_RCURLY:
       return TC_PUNCT;
 
     case T_PLUS:
@@ -341,6 +343,7 @@ classify_token (enum token_type type)
     case T_ASTERISK:
     case T_SLASH:
     case T_EQUALS:
+    case T_COLON:
     case T_AND:
     case T_OR:
     case T_NOT:
@@ -359,6 +362,7 @@ classify_token (enum token_type type)
       return TC_BINOP;
 
     case T_COMMA:
+    case T_SEMICOLON:
       return TC_COMMA;
     }
 
diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index eed9f57c22..f2dcebca15 100644
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -188,6 +188,8 @@ scan_punct1__ (char c0)
     case '-': return T_DASH;
     case '[': return T_LBRACK;
     case ']': return T_RBRACK;
+    case '{': return T_LCURLY;
+    case '}': return T_RCURLY;
     case '&': return T_AND;
     case '|': return T_OR;
     case '+': return T_PLUS;
@@ -196,6 +198,8 @@ scan_punct1__ (char c0)
     case '<': return T_LT;
     case '>': return T_GT;
     case '~': return T_NOT;
+    case ';': return T_SEMICOLON;
+    case ':': return T_COLON;
     default: return T_MACRO_PUNCT;
     }
 
diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c
index 05a9f57236..346910898c 100644
--- a/src/language/lexer/segment.c
+++ b/src/language/lexer/segment.c
@@ -956,8 +956,8 @@ segmenter_parse_mid_command__ (struct segmenter *s,
             return segmenter_parse_number__ (s, input, n, eof, type, ofs);
         }
       /* Fall through. */
-    case '(': case ')': case ',': case '=':
-    case '[': case ']': case '&': case '|': case '+':
+    case '(': case ')': case '{': case ',': case '=': case ';': case ':':
+    case '[': case ']': case '}': case '&': case '|': case '+':
       *type = SEG_PUNCT;
       s->substate = 0;
       return 1;
diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at
index 90dea5d346..c877628fdf 100644
--- a/tests/language/lexer/scan.at
+++ b/tests/language/lexer/scan.at
@@ -170,13 +170,13 @@ LBRACK
 RBRACK
 EXP
 MACRO_PUNCT "%"
-MACRO_PUNCT ":"
-MACRO_PUNCT ";"
+COLON
+SEMICOLON
 MACRO_PUNCT "?"
 MACRO_PUNCT "_"
 MACRO_PUNCT "`"
-MACRO_PUNCT "{"
-MACRO_PUNCT "}"
+LCURLY
+RCURLY
 NOT
 STOP
 ])