X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fidentifier.c;h=d9d9b2a6444c25a8f76fec3df66089e1306e6b40;hb=ea411b6c81ab5895051535a4730210a66d4be388;hp=6191f0db9028257e7facbddc566c5ba2e0ef08cb;hpb=b3fcf4b1644bf4af9b5eb7b0b0f8856c51118128;p=pspp

diff --git a/src/data/identifier.c b/src/data/identifier.c
index 6191f0db90..d9d9b2a644 100644
--- a/src/data/identifier.c
+++ b/src/data/identifier.c
@@ -24,9 +24,11 @@
 #include "data/identifier.h"
 
 #include <string.h>
+#include <unistr.h>
 #include <unictype.h>
 
 #include "libpspp/assertion.h"
+#include "libpspp/cast.h"
 
 #include "gl/c-ctype.h"
 
@@ -44,7 +46,6 @@ token_type_to_name (enum token_type type)
 #define TOKEN_TYPE(TYPE) case T_##TYPE: return #TYPE;
       TOKEN_TYPES
 #undef TOKEN_TYPE
-    case TOKEN_N_TYPES:
     default:
       return "unknown token type";
     }
@@ -63,6 +64,8 @@ token_type_to_string (enum token_type token)
     case T_POS_NUM:
     case T_NEG_NUM:
     case T_STRING:
+    case T_MACRO_ID:
+    case T_MACRO_PUNCT:
     case T_STOP:
       return NULL;
 
@@ -96,9 +99,21 @@ token_type_to_string (enum token_type token)
     case T_RBRACK:
       return "]";
 
+    case T_LCURLY:
+      return "{";
+
+    case T_RCURLY:
+      return "}";
+
     case T_COMMA:
       return ",";
 
+    case T_SEMICOLON:
+      return ";";
+
+    case T_COLON:
+      return ":";
+
     case T_AND:
       return "AND";
 
@@ -140,9 +155,6 @@ token_type_to_string (enum token_type token)
 
     case T_EXP:
       return "**";
-
-    case TOKEN_N_TYPES:
-      NOT_REACHED ();
     }
 
   NOT_REACHED ();
@@ -232,15 +244,21 @@ lex_uc_is_space (ucs4_t uc)
 size_t
 lex_id_get_length (struct substring string)
 {
-  size_t length = 0;
-  if (!ss_is_empty (string) && lex_is_id1 (ss_first (string)))
+  const uint8_t *s = CHAR_CAST (const uint8_t *, string.string);
+  size_t len = string.length;
+  size_t ofs;
+  int mblen;
+
+  for (ofs = 0; ofs < string.length; ofs += mblen)
     {
-      length = 1;
-      while (length < ss_length (string)
-             && lex_is_idn (ss_at (string, length)))
-        length++;
+      ucs4_t uc;
+
+      mblen = u8_mbtouc (&uc, s + ofs, len - ofs);
+      if (!(ofs == 0 ? lex_uc_is_id1 (uc) : lex_uc_is_idn (uc)))
+        break;
     }
-  return length;
+
+  return ofs;
 }
 
 /* Comparing identifiers. */
@@ -298,14 +316,14 @@ static const struct keyword keywords[] =
     { T_TO,   SS_LITERAL_INITIALIZER ("TO") },
     { T_WITH, SS_LITERAL_INITIALIZER ("WITH") },
   };
-static const size_t keyword_cnt = sizeof keywords / sizeof *keywords;
+static const size_t n_keywords = sizeof keywords / sizeof *keywords;
 
 /* Returns true if TOKEN is representable as a keyword. */
 bool
 lex_is_keyword (enum token_type token)
 {
   const struct keyword *kw;
-  for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
+  for (kw = keywords; kw < &keywords[n_keywords]; kw++)
     if (kw->token == token)
       return true;
   return false;
@@ -319,7 +337,7 @@ lex_id_to_token (struct substring id)
   if (ss_length (id) >= 2 && ss_length (id) <= 4)
     {
       const struct keyword *kw;
-      for (kw = keywords; kw < &keywords[keyword_cnt]; kw++)
+      for (kw = keywords; kw < &keywords[n_keywords]; kw++)
         if (ss_equals_case (kw->identifier, id))
           return kw->token;
     }