lexer: Add support for macro punctuation.

[pspp] / src / data / identifier.h

diff --git a/src/data/identifier.h b/src/data/identifier.h
index 02fc0db97b28aef3f975ad79466921f4186779b0..1fc63b4808f44e7e07ba3a92696f340f338d6922 100644 (file)
--- a/src/data/identifier.h
+++ b/src/data/identifier.h
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2010 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2010, 2011 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -22,50 +22,66 @@
 #include <unitypes.h>
 #include "libpspp/str.h"
 
+#define TOKEN_TYPES                                                     \
+    TOKEN_TYPE(ID)                  /* Identifier. */                   \
+    TOKEN_TYPE(POS_NUM)             /* Positive number. */              \
+    TOKEN_TYPE(NEG_NUM)             /* Negative number. */              \
+    TOKEN_TYPE(STRING)              /* Quoted string. */                \
+    TOKEN_TYPE(STOP)                /* End of input. */                 \
+                                                                        \
+    TOKEN_TYPE(ENDCMD)              /* . */                             \
+    TOKEN_TYPE(PLUS)                /* + */                             \
+    TOKEN_TYPE(DASH)                /* - */                             \
+    TOKEN_TYPE(ASTERISK)            /* * */                             \
+    TOKEN_TYPE(SLASH)               /* / */                             \
+    TOKEN_TYPE(EQUALS)              /* = */                             \
+    TOKEN_TYPE(LPAREN)              /* (*/                              \
+    TOKEN_TYPE(RPAREN)              /* ) */                             \
+    TOKEN_TYPE(LBRACK)              /* [ */                             \
+    TOKEN_TYPE(RBRACK)              /* ] */                             \
+    TOKEN_TYPE(COMMA)               /* , */                             \
+                                                                        \
+    TOKEN_TYPE(AND)                 /* AND */                           \
+    TOKEN_TYPE(OR)                  /* OR */                            \
+    TOKEN_TYPE(NOT)                 /* NOT */                           \
+                                                                        \
+    TOKEN_TYPE(EQ)                  /* EQ */                            \
+    TOKEN_TYPE(GE)                  /* GE or >= */                      \
+    TOKEN_TYPE(GT)                  /* GT or > */                       \
+    TOKEN_TYPE(LE)                  /* LE or <= */                      \
+    TOKEN_TYPE(LT)                  /* LT or < */                       \
+    TOKEN_TYPE(NE)                  /* NE or ~= */                      \
+                                                                        \
+    TOKEN_TYPE(ALL)                 /* ALL */                           \
+    TOKEN_TYPE(BY)                  /* BY */                            \
+    TOKEN_TYPE(TO)                  /* TO */                            \
+    TOKEN_TYPE(WITH)                /* WITH */                          \
+                                                                        \
+    TOKEN_TYPE(EXP)                 /* ** */                            \
+                                                                        \
+    TOKEN_TYPE(MACRO_ID)            /* Identifier starting with '!'. */ \
+    TOKEN_TYPE(MACRO_PUNCT)         /* Miscellaneous punctuator. */
 /* Token types. */
 enum token_type
   {
-    T_ID = 1,   /* Identifier. */
-    T_POS_NUM, /* Positive number. */
-    T_NEG_NUM, /* Negative number. */
-    T_STRING,  /* Quoted string. */
-    T_STOP,    /* End of input. */
-
-    T_ENDCMD,   /* End of command (e.g. '.'). */
-
-    T_PLUS,     /* + */
-    T_DASH,     /* - */
-    T_ASTERISK, /* * */
-    T_SLASH,    /* / */
-    T_EQUALS,   /* = */
-    T_LPAREN,   /* ( */
-    T_RPAREN,   /* ) */
-    T_LBRACK,   /* [ */
-    T_RBRACK,   /* ] */
-    T_COMMA,    /* , */
-
-    T_AND,     /* AND */
-    T_OR,      /* OR */
-    T_NOT,     /* NOT */
-
-    T_EQ,      /* EQ */
-    T_GE,      /* GE or >= */
-    T_GT,      /* GT or > */
-    T_LE,      /* LE or <= */
-    T_LT,      /* LT or < */
-    T_NE,      /* NE or ~= */
-
-    T_ALL,     /* ALL */
-    T_BY,      /* BY */
-    T_TO,      /* TO */
-    T_WITH,    /* WITH */
-
-    T_EXP,     /* ** */
+#define TOKEN_TYPE(TYPE) T_##TYPE,
+    TOKEN_TYPES
+    TOKEN_N_TYPES
+#undef TOKEN_TYPE
   };
 
+const char *token_type_to_name (enum token_type);
+const char *token_type_to_string (enum token_type);
+
 /* Tokens. */
 bool lex_is_keyword (enum token_type);
 
+/* Validating identifiers. */
+#define ID_MAX_LEN 64          /* Maximum length of identifier, in bytes. */
+
+bool id_is_valid (const char *id, const char *dict_encoding, bool issue_error);
+bool id_is_plausible (const char *id, bool issue_error);
+
 /* Recognizing identifiers. */
 bool lex_is_id1 (char);
 bool lex_is_idn (char);
@@ -80,7 +96,4 @@ bool lex_id_match_n (struct substring keyword, struct substring token,
                      size_t n);
 int lex_id_to_token (struct substring);
 
-/* Identifier names. */
-const char *lex_id_name (enum token_type);
-
 #endif /* !data/identifier.h */