lexer: Reimplement for better testability and internationalization.

[pspp-builds.git] / src / language / expressions / parse.c

diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 79576b4482176744d3950ca0ebefd2e61ead8da5..ed5a07093c712b3b034b82f71d6af1fbdc4db0fd 100644 (file)
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2006, 2010 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2010, 2011 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -23,22 +23,23 @@
 #include <limits.h>
 #include <stdlib.h>
 
-#include "helpers.h"
-#include <data/case.h>
-#include <data/dictionary.h>
-#include <data/settings.h>
-#include <data/variable.h>
-#include <language/lexer/format-parser.h>
-#include <language/lexer/lexer.h>
-#include <language/lexer/variable-parser.h>
-#include <libpspp/array.h>
-#include <libpspp/assertion.h>
-#include <libpspp/message.h>
-#include <libpspp/misc.h>
-#include <libpspp/pool.h>
-#include <libpspp/str.h>
-
-#include "xalloc.h"
+#include "data/case.h"
+#include "data/dictionary.h"
+#include "data/settings.h"
+#include "data/variable.h"
+#include "language/expressions/helpers.h"
+#include "language/lexer/format-parser.h"
+#include "language/lexer/lexer.h"
+#include "language/lexer/variable-parser.h"
+#include "libpspp/array.h"
+#include "libpspp/assertion.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "libpspp/misc.h"
+#include "libpspp/pool.h"
+#include "libpspp/str.h"
+
+#include "gl/xalloc.h"
 \f
 /* Declarations. */
 
@@ -785,12 +786,14 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
       time_t last_proc_time = time_of_last_procedure (e->ds);
       struct tm *time;
       char temp_buf[10];
+      struct substring s;
 
       time = localtime (&last_proc_time);
       sprintf (temp_buf, "%02d %s %02d", abs (time->tm_mday) % 100,
                months[abs (time->tm_mon) % 12], abs (time->tm_year) % 100);
 
-      return expr_allocate_string_buffer (e, temp_buf, strlen (temp_buf));
+      ss_alloc_substring (&s, ss_cstr (temp_buf));
+      return expr_allocate_string (e, s);
     }
   else if (lex_match_id (lexer, "$TRUE"))
     return expr_allocate_boolean (e, 1.0);
@@ -824,7 +827,7 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
     return expr_allocate_number (e, settings_get_viewwidth ());
   else
     {
-      msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
+      msg (SE, _("Unknown system variable %s."), lex_tokcstr (lexer));
       return NULL;
     }
 }
@@ -836,22 +839,22 @@ parse_primary (struct lexer *lexer, struct expression *e)
   switch (lex_token (lexer))
     {
     case T_ID:
-      if (lex_look_ahead (lexer) == T_LPAREN)
+      if (lex_next_token (lexer, 1) == T_LPAREN)
         {
           /* An identifier followed by a left parenthesis may be
              a vector element reference.  If not, it's a function
              call. */
-          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer)) != NULL)
+          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer)) != NULL)
             return parse_vector_element (lexer, e);
           else
             return parse_function (lexer, e);
         }
-      else if (lex_tokid (lexer)[0] == '$')
+      else if (lex_tokcstr (lexer)[0] == '$')
         {
           /* $ at the beginning indicates a system variable. */
           return parse_sysvar (lexer, e);
         }
-      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokid (lexer)))
+      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokcstr (lexer)))
         {
           /* It looks like a user variable.
              (It could be a format specifier, but we'll assume
@@ -872,7 +875,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
             return expr_allocate_format (e, &fmt);
 
           /* All attempts failed. */
-          msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
+          msg (SE, _("Unknown identifier %s."), lex_tokcstr (lexer));
           return NULL;
         }
       break;
@@ -887,8 +890,17 @@ parse_primary (struct lexer *lexer, struct expression *e)
 
     case T_STRING:
       {
-        union any_node *node = expr_allocate_string_buffer (
-          e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+        const char *dict_encoding;
+        union any_node *node;
+        char *s;
+
+        dict_encoding = (e->ds != NULL
+                         ? dict_get_encoding (dataset_dict (e->ds))
+                         : "UTF-8");
+        s = recode_string (dict_encoding, "UTF-8", lex_tokcstr (lexer),
+                           ss_length (lex_tokss (lexer)));
+        node = expr_allocate_string (e, ss_cstr (s));
+
        lex_get (lexer);
        return node;
       }
@@ -918,7 +930,7 @@ parse_vector_element (struct lexer *lexer, struct expression *e)
   /* Find vector, skip token.
      The caller must already have verified that the current token
      is the name of a vector. */
-  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
+  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer));
   assert (vector != NULL);
   lex_get (lexer);
 
@@ -1033,7 +1045,7 @@ lookup_function (const char *name,
 }
 
 static int
-extract_min_valid (char *s)
+extract_min_valid (const char *s)
 {
   char *p = strrchr (s, '.');
   if (p == NULL
@@ -1209,11 +1221,11 @@ parse_function (struct lexer *lexer, struct expression *e)
 
   union any_node *n;
 
-  ds_init_string (&func_name, lex_tokstr (lexer));
-  min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
-  if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last))
+  ds_init_substring (&func_name, lex_tokss (lexer));
+  min_valid = extract_min_valid (lex_tokcstr (lexer));
+  if (!lookup_function (lex_tokcstr (lexer), &first, &last))
     {
-      msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("No function or vector named %s."), lex_tokcstr (lexer));
       ds_destroy (&func_name);
       return NULL;
     }
@@ -1231,7 +1243,7 @@ parse_function (struct lexer *lexer, struct expression *e)
     for (;;)
       {
         if (lex_token (lexer) == T_ID
-            && toupper (lex_look_ahead (lexer)) == T_ID)
+            && lex_next_token (lexer, 1) == T_TO)
           {
             const struct variable **vars;
             size_t var_cnt;
@@ -1473,18 +1485,6 @@ expr_allocate_vector (struct expression *e, const struct vector *vector)
   return n;
 }
 
-union any_node *
-expr_allocate_string_buffer (struct expression *e,
-                             const char *string, size_t length)
-{
-  union any_node *n = pool_alloc (e->expr_pool, sizeof n->string);
-  n->type = OP_string;
-  if (length > MAX_STRING)
-    length = MAX_STRING;
-  n->string.s = copy_string (e, string, length);
-  return n;
-}
-
 union any_node *
 expr_allocate_string (struct expression *e, struct substring s)
 {