token: Update functional interface and add token_copy(), token_equal().

These will have users in an upcoming commit.

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 718458f8da1f5364767f7ed27f4461279a815398..4467042e1464bc91195eb37a5fd7281ad94f74eb 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -198,7 +198,7 @@ lex_push_token__ (struct lex_source *src)
     src->tokens = deque_expand (&src->deque, src->tokens, sizeof *src->tokens);
 
   token = &src->tokens[deque_push_front (&src->deque)];
-  token_init (&token->token);
+  token->token = (struct token) { .type = T_STOP };
   return token;
 }
 
@@ -852,9 +852,7 @@ lex_next__ (const struct lexer *lexer_, int n)
     return lex_source_next__ (src, n);
   else
     {
-      static const struct lex_token stop_token =
-        { TOKEN_INITIALIZER (T_STOP, 0.0, ""), 0, 0, 0, 0 };
-
+      static const struct lex_token stop_token = { .token = { .type = T_STOP } };
       return &stop_token;
     }
 }

diff --git a/src/language/lexer/scan.c b/src/language/lexer/scan.c
index 0acff46cdd119925f98e5845f67e8bfb975eae94..141e1e0f1522de638c7ed2bae5de6e78341ba146 100644 (file)
--- a/src/language/lexer/scan.c
+++ b/src/language/lexer/scan.c
@@ -594,7 +594,7 @@ void
 scanner_init (struct scanner *scanner, struct token *token)
 {
   scanner->state = S_START;
-  token_init (token);
+  *token = (struct token) { .type = T_STOP };
 }
 
 /* Adds the segment with type TYPE and UTF-8 text S to SCANNER.  TOKEN must be

diff --git a/src/language/lexer/token.c b/src/language/lexer/token.c
index ec64bbfb4a431cfcf0044cf0dd9a05b31755569f..71e98f2d7de44369f4648b498be2569d85978c37 100644 (file)
--- a/src/language/lexer/token.c
+++ b/src/language/lexer/token.c
@@ -27,17 +27,18 @@
 #include "libpspp/cast.h"
 #include "libpspp/misc.h"
 
-
 #include "gl/ftoastr.h"
 #include "gl/xalloc.h"
 
-/* Initializes TOKEN with an arbitrary type, number 0, and a null string. */
+/* Initializes DST as a copy of SRC. */
 void
-token_init (struct token *token)
+token_copy (struct token *dst, const struct token *src)
 {
-  token->type = 0;
-  token->number = 0.0;
-  token->string = ss_empty ();
+  *dst = (struct token) {
+    .type = src->type,
+    .number = src->number,
+  };
+  ss_alloc_substring (&dst->string, src->string);
 }
 
 /* Frees the string that TOKEN contains. */
@@ -45,7 +46,34 @@ void
 token_uninit (struct token *token)
 {
   if (token != NULL)
-    ss_dealloc (&token->string);
+    {
+      ss_dealloc (&token->string);
+      *token = (struct token) { .type = T_STOP };
+    }
+}
+
+/* Returns true if A and B are the same token, false otherwise. */
+bool
+token_equal (const struct token *a, const struct token *b)
+{
+  if (a->type != b->type)
+    return false;
+
+  switch (a->type)
+    {
+    case T_POS_NUM:
+    case T_NEG_NUM:
+      return a->number == b->number;
+
+    case T_ID:
+    case T_MACRO_ID:
+    case T_MACRO_PUNCT:
+    case T_STRING:
+      return ss_equals (a->string, b->string);
+
+    default:
+      return true;
+    }
 }
 
 static char *
@@ -150,7 +178,7 @@ token_to_string (const struct token *token)
       return string_representation (token->string);
 
     default:
-      return xstrdup_if_nonnull (token_type_to_name (token->type));
+      return xstrdup_if_nonnull (token_type_to_string (token->type));
     }
 }
 
@@ -173,6 +201,8 @@ token_print (const struct token *token, FILE *stream)
   putc ('\n', stream);
 }
 
+/* Returns true if T is a numeric token for an integer in the range of "long",
+   except that LONG_MIN is excluded. */
 bool
 token_is_integer (const struct token *t)
 {
@@ -182,6 +212,7 @@ token_is_integer (const struct token *t)
           && floor (t->number) == t->number);
 }
 
+/* Returns the "long int" value of T, which must satisfy token_is_integer(T). */
 long
 token_integer (const struct token *t)
 {

diff --git a/src/language/lexer/token.h b/src/language/lexer/token.h
index 67bc7ddc3b74753d587e4e1a29e4b356caf033d0..dca1452c6a32aed957ce98e8e9c129fc254fff32 100644 (file)
--- a/src/language/lexer/token.h
+++ b/src/language/lexer/token.h
@@ -17,6 +17,7 @@
 #ifndef TOKEN_H
 #define TOKEN_H 1
 
+#include <stdbool.h>
 #include <stdio.h>
 #include "libpspp/assertion.h"
 #include "libpspp/str.h"
@@ -33,12 +34,11 @@ struct token
     struct substring string;
   };
 
-#define TOKEN_INITIALIZER(TYPE, NUMBER, STRING) \
-        { TYPE, NUMBER, SS_LITERAL_INITIALIZER (STRING) }
-
-void token_init (struct token *);
+void token_copy (struct token *, const struct token *);
 void token_uninit (struct token *);
 
+bool token_equal (const struct token *, const struct token *);
+
 char *token_to_string (const struct token *);
 
 void token_print (const struct token *, FILE *);