lexer: Add support for macro identifiers (that begin with '!').

[pspp] / src / data / identifier.c

diff --git a/src/data/identifier.c b/src/data/identifier.c
index a757b31e3a03a8a644a7520fccec4ec8e81e6c14..c613734c94dc647ec4d2c1ca93e9e0dfb8232542 100644 (file)
--- a/src/data/identifier.c
+++ b/src/data/identifier.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2005, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2005, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -24,9 +24,11 @@
 #include "data/identifier.h"
 
 #include <string.h>
+#include <unistr.h>
 #include <unictype.h>
 
 #include "libpspp/assertion.h"
+#include "libpspp/cast.h"
 
 #include "gl/c-ctype.h"
 
@@ -60,6 +62,7 @@ token_type_to_string (enum token_type token)
   switch (token)
     {
     case T_ID:
+    case T_MACRO_ID:
     case T_POS_NUM:
     case T_NEG_NUM:
     case T_STRING:
@@ -188,7 +191,13 @@ lex_is_idn (char c)
 bool
 lex_uc_is_id1 (ucs4_t uc)
 {
-  return is_ascii_id1 (uc) || (uc >= 0x80 && uc_is_property_id_start (uc));
+  return (uc < 0x80
+          ? is_ascii_id1 (uc)
+          : (uc_is_general_category_withtable (uc,
+                                               UC_CATEGORY_MASK_L |
+                                               UC_CATEGORY_MASK_M |
+                                               UC_CATEGORY_MASK_S)
+             && uc != 0xfffc && uc != 0xfffd));
 }
 
 /* Returns true if Unicode code point UC may be a character in an identifier
@@ -198,7 +207,12 @@ lex_uc_is_idn (ucs4_t uc)
 {
   return (uc < 0x80
           ? is_ascii_id1 (uc) || isdigit (uc) || uc == '.' || uc == '_'
-          : uc >= 0x80 && uc_is_property_id_continue (uc));
+          : (uc_is_general_category_withtable (uc,
+                                               UC_CATEGORY_MASK_L |
+                                               UC_CATEGORY_MASK_M |
+                                               UC_CATEGORY_MASK_S |
+                                               UC_CATEGORY_MASK_N)
+             && uc != 0xfffc && uc != 0xfffd));
 }
 
 /* Returns true if Unicode code point UC is a space that separates tokens. */
@@ -221,15 +235,21 @@ lex_uc_is_space (ucs4_t uc)
 size_t
 lex_id_get_length (struct substring string)
 {
-  size_t length = 0;
-  if (!ss_is_empty (string) && lex_is_id1 (ss_first (string)))
+  const uint8_t *s = CHAR_CAST (const uint8_t *, string.string);
+  size_t len = string.length;
+  size_t ofs;
+  int mblen;
+
+  for (ofs = 0; ofs < string.length; ofs += mblen)
     {
-      length = 1;
-      while (length < ss_length (string)
-             && lex_is_idn (ss_at (string, length)))
-        length++;
+      ucs4_t uc;
+
+      mblen = u8_mbtouc (&uc, s + ofs, len - ofs);
+      if (!(ofs == 0 ? lex_uc_is_id1 (uc) : lex_uc_is_idn (uc)))
+        break;
     }
-  return length;
+
+  return ofs;
 }
 \f
 /* Comparing identifiers. */