lexer: Change the functions for retrieving token strings.

Until now, lex_tokid() has been for T_ID tokens only and lex_tokstr() has
been for T_ID and T_STRING tokens.  For T_ID tokens, lex_tokid() and
lex_tokstr() had slightly different semantics.

This doesn't entirely make sense, and these particular functions are not
the ones wanted most by clients, so this commit removes these functions
in favor of lex_tokcstr() and lex_tokss(), which are both applicable
to both T_ID and T_STRING tokens, with the same semantics in each case.

These functions are also easier for the upcoming reimplementation of the
lexer.

diff --git a/src/language/command.c b/src/language/command.c
index 3da5da69919a510adbd56f4f39c79df1de31847b..3bf9571fa5a4065537ce1225a01fb42a1ff49559 100644 (file)
--- a/src/language/command.c
+++ b/src/language/command.c
@@ -288,7 +288,7 @@ parse_command_name (struct lexer *lexer)
         {
           if (!ds_is_empty (&s) && ds_last (&s) != '-')
             ds_put_byte (&s, ' ');
-          ds_put_cstr (&s, lex_tokid (lexer));
+          ds_put_cstr (&s, lex_tokcstr (lexer));
         }
       else if (lex_is_integer (lexer) && lex_integer (lexer) >= 0)
         {
@@ -514,10 +514,10 @@ cmd_erase (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  if (remove (ds_cstr (lex_tokstr (lexer))) == -1)
+  if (remove (lex_tokcstr (lexer)) == -1)
     {
       msg (SW, _("Error removing `%s': %s."),
-          ds_cstr (lex_tokstr (lexer)), strerror (errno));
+           lex_tokcstr (lexer), strerror (errno));
       return CMD_FAILURE;
     }
 

diff --git a/src/language/control/loop.c b/src/language/control/loop.c
index f17542cb42febf44918373fb60e5fa52721f70c8..5e99232fc751d4115565973bc40f06aef0156c75 100644 (file)
--- a/src/language/control/loop.c
+++ b/src/language/control/loop.c
@@ -221,13 +221,13 @@ parse_index_clause (struct dataset *ds, struct lexer *lexer,
       return false;
     }
 
-  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
   if (loop->index_var != NULL)
     *created_index_var = false;
   else
     {
       loop->index_var = dict_create_var_assert (dataset_dict (ds),
-                                                lex_tokid (lexer), 0);
+                                                lex_tokcstr (lexer), 0);
       *created_index_var = true;
     }
   lex_get (lexer);

diff --git a/src/language/control/repeat.c b/src/language/control/repeat.c
index ddbba09aa9c04e9d582cbd884807019e235a829d..130dfd68800d5327a3f66d872f199a9ab76d1c9a 100644 (file)
--- a/src/language/control/repeat.c
+++ b/src/language/control/repeat.c
@@ -168,21 +168,19 @@ parse_specification (struct lexer *lexer, struct repeat_block *block)
       /* Get a stand-in variable name and make sure it's unique. */
       if (!lex_force_id (lexer))
        return false;
-      if (dict_lookup_var (dict, lex_tokid (lexer)))
-        msg (SW, _("Dummy variable name `%s' hides dictionary "
-                   "variable `%s'."),
-             lex_tokid (lexer), lex_tokid (lexer));
-      if (find_macro (block, ss_cstr (lex_tokid (lexer))))
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)))
+        msg (SW, _("Dummy variable name `%s' hides dictionary variable `%s'."),
+             lex_tokcstr (lexer), lex_tokcstr (lexer));
+      if (find_macro (block, lex_tokss (lexer)))
          {
            msg (SE, _("Dummy variable name `%s' is given twice."),
-                lex_tokid (lexer));
+                lex_tokcstr (lexer));
            return false;
          }
 
       /* Make a new macro. */
       macro = pool_alloc (block->pool, sizeof *macro);
-      ss_alloc_substring_pool (&macro->name, ss_cstr (lex_tokid (lexer)),
-                               block->pool);
+      ss_alloc_substring_pool (&macro->name, lex_tokss (lexer), block->pool);
       ll_push_tail (&block->macros, &macro->ll);
 
       /* Skip equals sign. */

diff --git a/src/language/data-io/combine-files.c b/src/language/data-io/combine-files.c
index 66163ee0b3d23a4bf1e417249d1a417fd79240cc..aa6d3b98ec319b60f1efd31b3fcf299b7ed87654 100644 (file)
--- a/src/language/data-io/combine-files.c
+++ b/src/language/data-io/combine-files.c
@@ -251,7 +251,7 @@ combine_files (enum comb_command_type command,
                            "TABLE."));
                 goto error;
               }
-            strcpy (file->in_name, lex_tokid (lexer));
+            strcpy (file->in_name, lex_tokcstr (lexer));
             lex_get (lexer);
           }
         else if (lex_match_id (lexer, "SORT"))
@@ -325,7 +325,7 @@ combine_files (enum comb_command_type command,
          lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (first_name, lex_tokid (lexer));
+          strcpy (first_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
@@ -339,7 +339,7 @@ combine_files (enum comb_command_type command,
          lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (last_name, lex_tokid (lexer));
+          strcpy (last_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "MAP"))

diff --git a/src/language/data-io/data-list.c b/src/language/data-io/data-list.c
index aec243a59256cdcb858847acc39795a82a634536..046c7b0e0cc165001f35dc0c04d2b572bdf8f7b6 100644 (file)
--- a/src/language/data-io/data-list.c
+++ b/src/language/data-io/data-list.c
@@ -108,7 +108,7 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
          if (!lex_force_string (lexer))
            goto error;
 
-         ds_init_string (&encoding, lex_tokstr (lexer));
+         ds_init_substring (&encoding, lex_tokss (lexer));
 
          lex_get (lexer);
        }
@@ -147,9 +147,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
          lex_match (lexer, T_EQUALS);
          if (!lex_force_id (lexer))
            goto error;
-         end = dict_lookup_var (dict, lex_tokid (lexer));
+         end = dict_lookup_var (dict, lex_tokcstr (lexer));
          if (!end)
-            end = dict_create_var_assert (dict, lex_tokid (lexer), 0);
+            end = dict_create_var_assert (dict, lex_tokcstr (lexer), 0);
          lex_get (lexer);
        }
       else if (lex_match_id (lexer, "NOTABLE"))
@@ -197,9 +197,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
                       if (lex_match_id (lexer, "TAB"))
                         delim = '\t';
                       else if (lex_is_string (lexer)
-                               && ds_length (lex_tokstr (lexer)) == 1)
+                               && ss_length (lex_tokss (lexer)) == 1)
                         {
-                          delim = ds_first (lex_tokstr (lexer));
+                          delim = ss_first (lex_tokss (lexer));
                           lex_get (lexer);
                         }
                       else

diff --git a/src/language/data-io/file-handle.q b/src/language/data-io/file-handle.q
index ba84a15dc54f2127576227a13e95a22aecd95be8..5f2a44075a78835d36afe933184cb7e4bf6c6ec9 100644 (file)
--- a/src/language/data-io/file-handle.q
+++ b/src/language/data-io/file-handle.q
@@ -57,7 +57,7 @@ cmd_file_handle (struct lexer *lexer, struct dataset *ds)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  str_copy_trunc (handle_name, sizeof handle_name, lex_tokid (lexer));
+  str_copy_trunc (handle_name, sizeof handle_name, lex_tokcstr (lexer));
 
   handle = fh_from_id (handle_name);
   if (handle != NULL)
@@ -159,7 +159,7 @@ cmd_close_file_handle (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  handle = fh_from_id (lex_tokid (lexer));
+  handle = fh_from_id (lex_tokcstr (lexer));
   if (handle == NULL)
     return CMD_CASCADING_FAILURE;
 
@@ -208,14 +208,15 @@ fh_parse (struct lexer *lexer, enum fh_referent referent_mask)
 
       handle = NULL;
       if (lex_token (lexer) == T_ID)
-        handle = fh_from_id (lex_tokid (lexer));
+        handle = fh_from_id (lex_tokcstr (lexer));
       if (handle == NULL)
         {
-          if (lex_token (lexer) != T_ID || lex_tokid (lexer)[0] != '#' || settings_get_syntax () != ENHANCED)
-            handle = fh_create_file (NULL, ds_cstr (lex_tokstr (lexer)),
+          if (lex_token (lexer) != T_ID || lex_tokcstr (lexer)[0] != '#'
+              || settings_get_syntax () != ENHANCED)
+            handle = fh_create_file (NULL, lex_tokcstr (lexer),
                                      fh_default_properties ());
           else
-            handle = fh_create_scratch (lex_tokid (lexer));
+            handle = fh_create_scratch (lex_tokcstr (lexer));
         }
       lex_get (lexer);
     }

diff --git a/src/language/data-io/get-data.c b/src/language/data-io/get-data.c
index eeb3827a58924dfa13eedd6bf2d2c2df43ffd8b5..05e115a9f560019cfaffcf13d5b97e525e028194 100644 (file)
--- a/src/language/data-io/get-data.c
+++ b/src/language/data-io/get-data.c
@@ -61,7 +61,7 @@ cmd_get_data (struct lexer *lexer, struct dataset *ds)
   else if (lex_match_id (lexer, "PSQL"))
     return parse_get_psql (lexer, ds);
 
-  msg (SE, _("Unsupported TYPE %s"), lex_tokid (lexer));
+  msg (SE, _("Unsupported TYPE %s"), lex_tokcstr (lexer));
   return CMD_FAILURE;
 }
 
@@ -85,7 +85,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  psql.conninfo = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  psql.conninfo = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -113,7 +113,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
          if ( ! lex_force_string (lexer) )
            goto error;
 
-         ds_put_substring (&psql.sql,  lex_tokstr (lexer)->ss);
+         ds_put_substring (&psql.sql, lex_tokss (lexer));
          lex_get (lexer);
        }
      }
@@ -153,7 +153,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  gri.file_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  gri.file_name = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -172,7 +172,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
              if ( ! lex_force_string (lexer) )
                goto error;
 
-             gri.sheet_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+             gri.sheet_name = ss_xstrdup (lex_tokss (lexer));
              gri.sheet_index = -1;
            }
          else if (lex_match_id (lexer, "INDEX"))
@@ -196,7 +196,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
              if ( ! lex_force_string (lexer) )
                goto error;
 
-             gri.cell_range = xstrdup (ds_cstr (lex_tokstr (lexer)));
+             gri.cell_range = ss_xstrdup (lex_tokss (lexer));
            }
          else
            goto error;
@@ -219,6 +219,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
        }
       else
        {
+         printf ("Unknown data file type `%s'\n", lex_tokcstr (lexer));
          goto error;
        }
       lex_get (lexer);
@@ -415,7 +416,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
           if (!lex_force_string (lexer))
             goto error;
 
-          s = ds_ss (lex_tokstr (lexer));
+          s = lex_tokss (lexer);
           if (ss_match_string (&s, ss_cstr ("\\t")))
             ds_put_cstr (&hard_seps, "\t");
           if (ss_match_string (&s, ss_cstr ("\\\\")))
@@ -441,14 +442,14 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
             goto error;
 
           if (settings_get_syntax () == COMPATIBLE
-              && ds_length (lex_tokstr (lexer)) != 1)
+              && ss_length (lex_tokss (lexer)) != 1)
             {
               msg (SE, _("In compatible syntax mode, the QUALIFIER string "
                          "must contain exactly one character."));
               goto error;
             }
 
-          data_parser_set_quotes (parser, ds_ss (lex_tokstr (lexer)));
+          data_parser_set_quotes (parser, lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (settings_get_syntax () == ENHANCED
@@ -501,7 +502,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
 
       if (!lex_force_id (lexer))
         goto error;
-      strcpy (name, lex_tokid (lexer));
+      strcpy (name, lex_tokcstr (lexer));
       lex_get (lexer);
 
       if (type == DP_DELIMITED)

diff --git a/src/language/data-io/print.c b/src/language/data-io/print.c
index cfbe04396191e5b1f7098f0b7df40ce6fac29edf..96ac15e15298aae2b3624dc560c148f47327a188 100644 (file)
--- a/src/language/data-io/print.c
+++ b/src/language/data-io/print.c
@@ -280,7 +280,7 @@ parse_string_argument (struct lexer *lexer, struct print_trns *trns, int record,
   spec->type = PRT_LITERAL;
   spec->record = record;
   spec->first_column = *column;
-  ds_init_string (&spec->string, lex_tokstr (lexer));
+  ds_init_substring (&spec->string, lex_tokss (lexer));
   ds_register_pool (&spec->string, trns->pool);
   lex_get (lexer);
 

diff --git a/src/language/data-io/save-translate.c b/src/language/data-io/save-translate.c
index dcfdb83c7c618b0af02c1dc80e679149a6b15dc7..213f0bf735e40fe6b9ead6aac5aa66e3a1d698ad 100644 (file)
--- a/src/language/data-io/save-translate.c
+++ b/src/language/data-io/save-translate.c
@@ -159,13 +159,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "DELIMITER");
                       goto error;
                     }
-                  delimiter = ds_first (lex_tokstr (lexer));
+                  delimiter = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "QUALIFIER"))
@@ -173,13 +173,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "QUALIFIER");
                       goto error;
                     }
-                  qualifier = ds_first (lex_tokstr (lexer));
+                  qualifier = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "DECIMAL"))

diff --git a/src/language/data-io/trim.c b/src/language/data-io/trim.c
index be637c403365be64ff788d3df6ae6fcc4b4ac698..9f76b10516b1b8d88dc2eba668b919020c784d58 100644 (file)
--- a/src/language/data-io/trim.c
+++ b/src/language/data-io/trim.c
@@ -83,18 +83,18 @@ parse_dict_rename (struct lexer *lexer, struct dictionary *dict)
       if (!lex_force_match (lexer, T_EQUALS)
          || !lex_force_id (lexer))
        return 0;
-      if (dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
        {
          msg (SE, _("Cannot rename %s as %s because there already exists "
                     "a variable named %s.  To rename variables with "
                     "overlapping names, use a single RENAME subcommand "
                     "such as `/RENAME (A=B)(B=C)(C=A)', or equivalently, "
                     "`/RENAME (A B C=B C A)'."),
-               var_get_name (v), lex_tokid (lexer), lex_tokid (lexer));
+               var_get_name (v), lex_tokcstr (lexer), lex_tokcstr (lexer));
          return 0;
        }
 
-      dict_rename_var (dict, v, lex_tokid (lexer));
+      dict_rename_var (dict, v, lex_tokcstr (lexer));
       lex_get (lexer);
       return 1;
     }

diff --git a/src/language/dictionary/attributes.c b/src/language/dictionary/attributes.c
index 35e69542950300498484ab8b4506b992c2be0423..c992ff806817156fa20f2fe27ab3fc6e14da382f 100644 (file)
--- a/src/language/dictionary/attributes.c
+++ b/src/language/dictionary/attributes.c
@@ -79,7 +79,7 @@ static bool
 match_subcommand (struct lexer *lexer, const char *keyword) 
 {
   if (lex_token (lexer) == T_ID
-      && lex_id_match (ss_cstr (lex_tokid (lexer)), ss_cstr (keyword))
+      && lex_id_match (lex_tokss (lexer), ss_cstr (keyword))
       && lex_look_ahead (lexer) == T_EQUALS)
     {
       lex_get (lexer);          /* Skip keyword. */
@@ -96,7 +96,7 @@ parse_attribute_name (struct lexer *lexer, char name[VAR_NAME_LEN + 1],
 {
   if (!lex_force_id (lexer))
     return false;
-  strcpy (name, lex_tokid (lexer));
+  strcpy (name, lex_tokcstr (lexer));
   lex_get (lexer);
 
   if (lex_match (lexer, T_LBRACK))
@@ -122,14 +122,14 @@ static bool
 add_attribute (struct lexer *lexer, struct attrset **sets, size_t n) 
 {
   char name[VAR_NAME_LEN + 1];
+  const char *value;
   size_t index, i;
-  char *value;
 
   if (!parse_attribute_name (lexer, name, &index)
       || !lex_force_match (lexer, T_LPAREN)
       || !lex_force_string (lexer))
     return false;
-  value = ds_cstr (lex_tokstr (lexer));
+  value = lex_tokcstr (lexer);
 
   for (i = 0; i < n; i++)
     {

diff --git a/src/language/dictionary/missing-values.c b/src/language/dictionary/missing-values.c
index bbc2c84d4d89e225cb8540e08662a3ed2b044378..4a14ee195b8c91f26cf0e589497f2d607bfb325c 100644 (file)
--- a/src/language/dictionary/missing-values.c
+++ b/src/language/dictionary/missing-values.c
@@ -106,7 +106,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       break;
                     }
 
-                  length = ds_length (lex_tokstr (lexer));
+                  length = ss_length (lex_tokss (lexer));
                   if (length > MV_MAX_STRING)
                     {
                       msg (SE, _("Truncating missing value to maximum "
@@ -115,7 +115,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       length = MV_MAX_STRING;
                     }
                   memset (value, ' ', MV_MAX_STRING);
-                  memcpy (value, ds_data (lex_tokstr (lexer)), length);
+                  memcpy (value, ss_data (lex_tokss (lexer)), length);
 
                   if (!mv_add_str (&mv, value))
                     deferred_errors = true;

diff --git a/src/language/dictionary/modify-variables.c b/src/language/dictionary/modify-variables.c
index 6afd321590a7d63f0721c370561e2139c0693ee4..cc847372d3d20090dc093666c6d58487e60c4b5b 100644 (file)
--- a/src/language/dictionary/modify-variables.c
+++ b/src/language/dictionary/modify-variables.c
@@ -300,7 +300,7 @@ cmd_modify_vars (struct lexer *lexer, struct dataset *ds)
       else
        {
          if (lex_token (lexer) == T_ID)
-           msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokid (lexer));
+           msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokcstr (lexer));
          else
            msg (SE, _("Subcommand name expected."));
          goto done;

diff --git a/src/language/dictionary/mrsets.c b/src/language/dictionary/mrsets.c
index 0708e89ccfa72a9499f598ade8527fe8a99abe49..c775f49784eab659d715cc0bb2408a7c20ad21aa 100644 (file)
--- a/src/language/dictionary/mrsets.c
+++ b/src/language/dictionary/mrsets.c
@@ -93,16 +93,16 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_match (lexer, T_EQUALS) || !lex_force_id (lexer))
             goto error;
-          if (lex_tokid (lexer)[0] != '$')
+          if (lex_tokcstr (lexer)[0] != '$')
             {
               msg (SE, _("%s is not a valid name for a multiple response "
                          "set.  Multiple response set names must begin with "
-                         "`$'."), lex_tokid (lexer));
+                         "`$'."), lex_tokcstr (lexer));
               goto error;
             }
 
           free (mrset->name);
-          mrset->name = xstrdup (lex_tokid (lexer));
+          mrset->name = xstrdup (lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "VARIABLES"))
@@ -129,7 +129,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             goto error;
 
           free (mrset->label);
-          mrset->label = ds_xstrdup (lex_tokstr (lexer));
+          mrset->label = ss_xstrdup (lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (type == MRSET_MD && lex_match_id (lexer, "LABELSOURCE"))
@@ -159,7 +159,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             }
           else if (lex_is_string (lexer))
             {
-              const char *s = ds_cstr (lex_tokstr (lexer));
+              const char *s = lex_tokcstr (lexer);
               int width;
 
               /* Trim off trailing spaces, but don't trim the string until
@@ -480,14 +480,14 @@ parse_mrset_names (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_id (lexer))
             return false;
-          if (dict_lookup_mrset (dict, lex_tokid (lexer)) == NULL)
+          if (dict_lookup_mrset (dict, lex_tokcstr (lexer)) == NULL)
             {
               msg (SE, _("No multiple response set named %s."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
               stringi_set_destroy (mrset_names);
               return false;
             }
-          stringi_set_insert (mrset_names, lex_tokid (lexer));
+          stringi_set_insert (mrset_names, lex_tokcstr (lexer));
           lex_get (lexer);
         }
     }

diff --git a/src/language/dictionary/value-labels.c b/src/language/dictionary/value-labels.c
index 0fdd7a2a362e550d69b90b33a8d42704c2349a90..1471fc86876c77ba89fe54248c055a03b4873dc9 100644 (file)
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -142,7 +142,7 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
           return 0;
         }
 
-      ds_init_string (&label, lex_tokstr (lexer));
+      ds_init_substring (&label, lex_tokss (lexer));
 
       if (ds_length (&label) > 60)
        {

diff --git a/src/language/dictionary/variable-label.c b/src/language/dictionary/variable-label.c
index e6bd62296f8486c30f78af84ec5591ae7a37e18c..804a4213d1c4ac8967c8ec1f18b4e8e6322c6e66 100644 (file)
--- a/src/language/dictionary/variable-label.c
+++ b/src/language/dictionary/variable-label.c
@@ -52,7 +52,7 @@ cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
          return CMD_FAILURE;
        }
 
-      ds_init_string (&label, lex_tokstr (lexer) );
+      ds_init_substring (&label, lex_tokss (lexer));
       if (ds_length (&label) > 255)
        {
          msg (SW, _("Truncating variable label to 255 characters."));

diff --git a/src/language/dictionary/vector.c b/src/language/dictionary/vector.c
index a884331ce30ba33eb3a7a3c01be318f9ea9a32d5..3be0723f705d0da04534011c2a2e884d7728755a 100644 (file)
--- a/src/language/dictionary/vector.c
+++ b/src/language/dictionary/vector.c
@@ -59,25 +59,25 @@ cmd_vector (struct lexer *lexer, struct dataset *ds)
        {
           size_t i;
 
-         if (dict_lookup_vector (dict, lex_tokid (lexer)))
+         if (dict_lookup_vector (dict, lex_tokcstr (lexer)))
            {
              msg (SE, _("A vector named %s already exists."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
              goto fail;
            }
 
           for (i = 0; i < vector_cnt; i++)
-            if (!strcasecmp (vectors[i], lex_tokid (lexer)))
+            if (!strcasecmp (vectors[i], lex_tokcstr (lexer)))
              {
                msg (SE, _("Vector name %s is given twice."),
-                     lex_tokid (lexer));
+                     lex_tokcstr (lexer));
                goto fail;
              }
 
           if (vector_cnt == vector_cap)
             vectors = pool_2nrealloc (pool,
                                        vectors, &vector_cap, sizeof *vectors);
-          vectors[vector_cnt++] = pool_strdup (pool, lex_tokid (lexer));
+          vectors[vector_cnt++] = pool_strdup (pool, lex_tokcstr (lexer));
 
          lex_get (lexer);
          lex_match (lexer, T_COMMA);

diff --git a/src/language/expressions/evaluate.c b/src/language/expressions/evaluate.c
index 9bcfe31e6f11d1f69a36629541c4e7bf3cf16688..79b52159528d020d065034f1ef5c0a3f6dc59ec6 100644 (file)
--- a/src/language/expressions/evaluate.c
+++ b/src/language/expressions/evaluate.c
@@ -134,7 +134,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
 
           if (!lex_force_id (lexer))
             goto done;
-          strcpy (name, lex_tokid (lexer));
+          strcpy (name, lex_tokcstr (lexer));
 
           lex_get (lexer);
           if (!lex_force_match (lexer, T_EQUALS))
@@ -143,7 +143,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
           if (lex_is_number (lexer))
             width = 0;
           else if (lex_is_string (lexer))
-            width = ds_length (lex_tokstr (lexer));
+            width = ss_length (lex_tokss (lexer));
           else
             {
               lex_error (lexer, _("expecting number or string"));

diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 79576b4482176744d3950ca0ebefd2e61ead8da5..ae968d0078b5023f5647c1c600f1afcf0ea25b90 100644 (file)
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -824,7 +824,7 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
     return expr_allocate_number (e, settings_get_viewwidth ());
   else
     {
-      msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
+      msg (SE, _("Unknown system variable %s."), lex_tokcstr (lexer));
       return NULL;
     }
 }
@@ -841,17 +841,17 @@ parse_primary (struct lexer *lexer, struct expression *e)
           /* An identifier followed by a left parenthesis may be
              a vector element reference.  If not, it's a function
              call. */
-          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer)) != NULL)
+          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer)) != NULL)
             return parse_vector_element (lexer, e);
           else
             return parse_function (lexer, e);
         }
-      else if (lex_tokid (lexer)[0] == '$')
+      else if (lex_tokcstr (lexer)[0] == '$')
         {
           /* $ at the beginning indicates a system variable. */
           return parse_sysvar (lexer, e);
         }
-      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokid (lexer)))
+      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokcstr (lexer)))
         {
           /* It looks like a user variable.
              (It could be a format specifier, but we'll assume
@@ -872,7 +872,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
             return expr_allocate_format (e, &fmt);
 
           /* All attempts failed. */
-          msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
+          msg (SE, _("Unknown identifier %s."), lex_tokcstr (lexer));
           return NULL;
         }
       break;
@@ -888,7 +888,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
     case T_STRING:
       {
         union any_node *node = expr_allocate_string_buffer (
-          e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+          e, lex_tokcstr (lexer), ss_length (lex_tokss (lexer)));
        lex_get (lexer);
        return node;
       }
@@ -918,7 +918,7 @@ parse_vector_element (struct lexer *lexer, struct expression *e)
   /* Find vector, skip token.
      The caller must already have verified that the current token
      is the name of a vector. */
-  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
+  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer));
   assert (vector != NULL);
   lex_get (lexer);
 
@@ -1209,11 +1209,11 @@ parse_function (struct lexer *lexer, struct expression *e)
 
   union any_node *n;
 
-  ds_init_string (&func_name, lex_tokstr (lexer));
-  min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
-  if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last))
+  ds_init_substring (&func_name, lex_tokss (lexer));
+  min_valid = extract_min_valid (lex_tokcstr (lexer));
+  if (!lookup_function (lex_tokcstr (lexer), &first, &last))
     {
-      msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("No function or vector named %s."), lex_tokcstr (lexer));
       ds_destroy (&func_name);
       return NULL;
     }

diff --git a/src/language/lexer/format-parser.c b/src/language/lexer/format-parser.c
index 1730b4936c69590d7157655eca72d11671043c86..4b25020ed4afdc6630c8e08ac014e40ae6003a8b 100644 (file)
--- a/src/language/lexer/format-parser.c
+++ b/src/language/lexer/format-parser.c
@@ -46,7 +46,7 @@ parse_abstract_format_specifier__ (struct lexer *lexer,
     goto error;
 
   /* Extract pieces. */
-  s = ds_ss (lex_tokstr (lexer));
+  s = ss_cstr (lex_tokcstr (lexer));
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_LETTERS)), &type_ss);
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_DIGITS)), &width_ss);
   if (ss_match_byte (&s, '.'))
@@ -133,9 +133,9 @@ parse_format_specifier_name (struct lexer *lexer, enum fmt_type *type)
       lex_error (lexer, _("expecting format type"));
       return false;
     }
-  if (!fmt_from_name (ds_cstr (lex_tokstr (lexer)), type))
+  if (!fmt_from_name (lex_tokcstr (lexer), type))
     {
-      msg (SE, _("Unknown format type `%s'."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("Unknown format type `%s'."), lex_tokcstr (lexer));
       return false;
     }
   lex_get (lexer);

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index c08c428c4a4bf696bf2211318166b17e225bb77a..4a899bdff91b7b9b13d8a57d8f1b6c1f521a96ff 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -49,11 +49,7 @@ struct lexer
   int token;      /* Current token. */
   double tokval;  /* T_POS_NUM, T_NEG_NUM: the token's value. */
 
-  char tokid [VAR_NAME_LEN + 1];   /* T_ID: the identifier. */
-
-  struct string tokstr;   /* T_ID, T_STRING: token string value.
-                           For T_ID, this is not truncated as is
-                           tokid. */
+  struct string tokstr;   /* T_ID, T_STRING: token string value. */
 
   char *prog; /* Pointer to next token in line_buffer. */
   bool dot;   /* True only if this line ends with a terminal dot. */
@@ -138,7 +134,6 @@ restore_token (struct lexer *lexer)
   assert (lexer->put_token != 0);
   lexer->token = lexer->put_token;
   ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   lexer->tokval = lexer->put_tokval;
   lexer->put_token = 0;
 }
@@ -405,8 +400,7 @@ lex_get (struct lexer *lexer)
     }
 }
 
-/* Parses an identifier at the current position into tokid and
-   tokstr.
+/* Parses an identifier at the current position into tokstr.
    Returns the correct token type. */
 static int
 parse_id (struct lexer *lexer)
@@ -420,7 +414,6 @@ parse_id (struct lexer *lexer)
   lexer->prog += ss_length (id);
 
   ds_assign_substring (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   return lex_id_to_token (id);
 }
 
@@ -571,7 +564,7 @@ bool
 lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
 {
   if (lexer->token == T_ID
-      && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+      && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
     {
       lex_get (lexer);
       return true;
@@ -798,7 +791,6 @@ lex_put_back_id (struct lexer *lexer, const char *id)
   save_token (lexer);
   lexer->token = T_ID;
   ds_assign_cstr (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
 }
 \f
 /* Weird line processing functions. */
@@ -1082,36 +1074,37 @@ lex_token_representation (struct lexer *lexer)
     case T_ID:
     case T_POS_NUM:
     case T_NEG_NUM:
-      return ds_xstrdup (&lexer->tokstr);
+      return ss_xstrdup (lex_tokss (lexer));
 
     case T_STRING:
       {
+        struct substring ss;
        int hexstring = 0;
        char *sp, *dp;
 
-       for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+        ss = lex_tokss (lexer);
+       for (sp = ss_data (ss); sp < ss_end (ss); sp++)
          if (!c_isprint ((unsigned char) *sp))
            {
              hexstring = 1;
              break;
            }
 
-       token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+       token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
 
        dp = token_rep;
        if (hexstring)
          *dp++ = 'X';
        *dp++ = '\'';
 
-       if (!hexstring)
-         for (sp = ds_cstr (&lexer->tokstr); *sp; )
+        for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+          if (!hexstring)
            {
              if (*sp == '\'')
                *dp++ = '\'';
-             *dp++ = (unsigned char) *sp++;
+             *dp++ = (unsigned char) *sp;
            }
-       else
-         for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+          else
            {
              *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
              *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
@@ -1344,16 +1337,22 @@ lex_tokval (const struct lexer *lexer)
   return lexer->tokval;
 }
 
+/* Returns the null-terminated string value associated with LEXER's current
+   token.  For a T_ID token, this is the identifier, and for a T_STRING token,
+   this is the string.  For other tokens the value is undefined. */
 const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
 {
-  return lexer->tokid;
+  return ds_cstr (&lexer->tokstr);
 }
 
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token.  For a T_ID
+   token, this is the identifier, and for a T_STRING token, this is the string.
+   For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
 {
-  return &lexer->tokstr;
+  return ds_ss (&lexer->tokstr);
 }
 
 /* If the lexer is positioned at the (pseudo)identifier S, which
@@ -1368,7 +1367,7 @@ lex_match_hyphenated_word (struct lexer *lexer, const char *s)
   if (hyphen == NULL)
     return lex_match_id (lexer, s);
   else if (lexer->token != T_ID
-          || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+          || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
           || lex_look_ahead (lexer) != T_DASH)
     return false;
   else

diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index 82f3fe23c10f40c1b941f0b8a0a965ba7b307d72..7af40234cc03d40d157a824f3803db05afac2266 100644 (file)
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -93,8 +93,8 @@ char *lex_token_representation (struct lexer *);
 /* Token accessors */
 enum token_type lex_token (const struct lexer *);
 double lex_tokval (const struct lexer *);
-const char *lex_tokid (const struct lexer *);
-const struct string *lex_tokstr (const struct lexer *);
+const char *lex_tokcstr (const struct lexer *);
+struct substring lex_tokss (const struct lexer *);
 
 /* Really weird functions. */
 void lex_skip_comment (struct lexer *);

diff --git a/src/language/lexer/q2c.c b/src/language/lexer/q2c.c
index c03b6b21da6b4d16018d5a1768492440f05be614..cf5a04ee132c6df3449cccc4da8d86c1239a07e1 100644 (file)
--- a/src/language/lexer/q2c.c
+++ b/src/language/lexer/q2c.c
@@ -1542,7 +1542,7 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
              dump (0, "goto lossage;");
              dump (-1, "}");
               dump (-1, "free (p->%s%s);", sbc->prefix, st_lower (s->valname));
-              dump (0, "p->%s%s = xstrdup (ds_cstr (lex_tokstr (lexer)));",
+              dump (0, "p->%s%s = ss_xstrdup (ss_tokss (lexer));",
                     sbc->prefix, st_lower (s->valname));
             }
           else
@@ -1705,7 +1705,7 @@ dump_subcommand (const subcommand *sbc)
       outdent ();
       if (sbc->restriction)
        {
-         dump (0, "x = ds_length (lex_tokstr (lexer));");
+         dump (0, "x = ss_length (lex_tokss (lexer));");
          dump (1, "if (!(%s))", sbc->restriction);
          dump (1, "{");
          dump (0, "msg (SE, _(\"String for %s must be %s.\"));",
@@ -1715,7 +1715,7 @@ dump_subcommand (const subcommand *sbc)
          outdent ();
        }
       dump (0, "free(p->s_%s);", st_lower(sbc->name) );
-      dump (0, "p->s_%s = ds_xstrdup (lex_tokstr (lexer));",
+      dump (0, "p->s_%s = ss_xstrdup (lex_tokss (lexer));",
            st_lower (sbc->name));
       dump (0, "lex_get (lexer);");
       if (sbc->restriction)
@@ -1833,13 +1833,13 @@ dump_parser (int persistent)
     {
       if (def->type == SBC_VARLIST)
        dump (1, "if (lex_token (lexer) == T_ID "
-              "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL "
-             "&& lex_look_ahead (lexer) != T_EQUALS)");
+              "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL "
+             "&& lex_look_ahead (lexer) != '=')");
       else
        {
          dump (0, "if ((lex_token (lexer) == T_ID "
-                "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) "
-               "&& lex_look_ahead () != T_EQUALS)");
+                "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) "
+               "&& lex_look_ahead () != '=')");
          dump (1, "     || token == T_ALL)");
        }
       dump (1, "{");

diff --git a/src/language/lexer/value-parser.c b/src/language/lexer/value-parser.c
index 269d24bca48d04b13da5c7a054b08fcdb281d4fd..4ef092de3a3e6c884e3c236e2d159b19023eed01 100644 (file)
--- a/src/language/lexer/value-parser.c
+++ b/src/language/lexer/value-parser.c
@@ -106,7 +106,7 @@ parse_number (struct lexer *lexer, double *x, const enum fmt_type *format)
 
       assert (fmt_get_category (*format) != FMT_CAT_STRING);
 
-      if (!data_in_msg (ds_ss (lex_tokstr (lexer)), LEGACY_NATIVE,
+      if (!data_in_msg (lex_tokss (lexer), LEGACY_NATIVE,
                         *format, &v, 0, NULL))
         return false;
 
@@ -143,7 +143,12 @@ parse_value (struct lexer *lexer, union value *v, int width)
     }
   else if (lex_force_string (lexer))
     {
-      const char *s = ds_cstr (lex_tokstr (lexer));
+      const char *s;
+
+      if (!lex_force_string (lexer))
+       return false;
+
+      s = lex_tokcstr (lexer);
       value_copy_str_rpad (v, width, CHAR_CAST_BUG (const uint8_t *, s), ' ');
     }
   else

diff --git a/src/language/lexer/variable-parser.c b/src/language/lexer/variable-parser.c
index b05b7ce7f9009e9771c4fe1f1d469d4e024216cf..5b698bbf4a2c6eefc063bcf431cffe6c559f61af 100644 (file)
--- a/src/language/lexer/variable-parser.c
+++ b/src/language/lexer/variable-parser.c
@@ -65,14 +65,14 @@ parse_vs_variable_idx (struct lexer *lexer, const struct var_set *vs,
       lex_error (lexer, _("expecting variable name"));
       return false;
     }
-  else if (var_set_lookup_var_idx (vs, lex_tokid (lexer), idx))
+  else if (var_set_lookup_var_idx (vs, lex_tokcstr (lexer), idx))
     {
       lex_get (lexer);
       return true;
     }
   else
     {
-      msg (SE, _("%s is not a variable name."), lex_tokid (lexer));
+      msg (SE, _("%s is not a variable name."), lex_tokcstr (lexer));
       return false;
     }
 }
@@ -339,7 +339,7 @@ parse_var_set_vars (struct lexer *lexer, const struct var_set *vs,
       lex_match (lexer, T_COMMA);
     }
   while (lex_token (lexer) == T_ALL
-         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokid (lexer)) != NULL));
+         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokcstr (lexer)) != NULL));
 
   if (*nv == 0)
     goto fail;
@@ -444,13 +444,13 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
          lex_error (lexer, "expecting variable name");
          goto fail;
        }
-      if (dict_class_from_id (lex_tokid (lexer)) == DC_SCRATCH
+      if (dict_class_from_id (lex_tokcstr (lexer)) == DC_SCRATCH
           && (pv_opts & PV_NO_SCRATCH))
        {
          msg (SE, _("Scratch variables not allowed here."));
          goto fail;
        }
-      strcpy (name1, lex_tokid (lexer));
+      strcpy (name1, lex_tokcstr (lexer));
       lex_get (lexer);
       if (lex_token (lexer) == T_TO)
        {
@@ -460,7 +460,7 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
              lex_error (lexer, "expecting variable name");
              goto fail;
            }
-         strcpy (name2, lex_tokid (lexer));
+         strcpy (name2, lex_tokcstr (lexer));
          lex_get (lexer);
 
          if (!extract_num (name1, root1, &n1, &d1)
@@ -588,7 +588,7 @@ parse_mixed_vars (struct lexer *lexer, const struct dictionary *dict,
     }
   while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
     {
-      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
        {
          struct variable **v;
          size_t nv;

diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c
index bbe444d548a6bcc0af9774c9d0d26ac1a6887fe3..cfa9ea66248a2ca0879d5c183f778f72cff2a657 100644 (file)
--- a/src/language/stats/aggregate.c
+++ b/src/language/stats/aggregate.c
@@ -435,7 +435,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
          if (lex_is_string (lexer))
            {
              struct string label;
-             ds_init_string (&label, lex_tokstr (lexer));
+             ds_init_substring (&label, lex_tokss (lexer));
 
              ds_truncate (&label, 255);
              dest_label[n_dest - 1] = ds_xstrdup (&label);
@@ -451,7 +451,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
          goto error;
        }
 
-      ds_assign_string (&function_name, lex_tokstr (lexer));
+      ds_assign_substring (&function_name, lex_tokss (lexer));
       exclude = ds_chomp (&function_name, '.') ? MV_SYSTEM : MV_ANY;
 
       for (function = agr_func_tab; function->name; function++)
@@ -501,7 +501,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
                lex_match (lexer, T_COMMA);
                if (lex_is_string (lexer))
                  {
-                   arg[i].c = ds_xstrdup (lex_tokstr (lexer));
+                   arg[i].c = ss_xstrdup (lex_tokss (lexer));
                    type = VAL_STRING;
                  }
                else if (lex_is_number (lexer))

diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q
index 0e7b3daecb01ce1ddc3c0e70fc45e9f39c79af3c..02cd4deb980e6d2e627c696b1139a82476b1bd95 100644 (file)
--- a/src/language/stats/crosstabs.q
+++ b/src/language/stats/crosstabs.q
@@ -381,7 +381,7 @@ crs_custom_tables (struct lexer *lexer, struct dataset *ds,
   /* Ensure that this is a TABLES subcommand. */
   if (!lex_match_id (lexer, "TABLES")
       && (lex_token (lexer) != T_ID ||
-         dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+         dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
   lex_match (lexer, T_EQUALS);

diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c
index 17eb4450446f7961a40cbaee4c2d0a5776cf55b7..ae472294a8f1e7eca3457156f89619f58f6cf100 100644 (file)
--- a/src/language/stats/descriptives.c
+++ b/src/language/stats/descriptives.c
@@ -334,14 +334,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds)
                       lex_error (lexer, NULL);
                       goto error;
                     }
-                  if (try_name (dict, dsc, lex_tokid (lexer)))
+                  if (try_name (dict, dsc, lex_tokcstr (lexer)))
                     {
-                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer));
+                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokcstr (lexer));
                       z_cnt++;
                     }
                   else
                     msg (SE, _("Z-score variable name %s would be"
-                               " a duplicate variable name."), lex_tokid (lexer));
+                               " a duplicate variable name."), lex_tokcstr (lexer));
                   lex_get (lexer);
                   if (!lex_force_match (lexer, T_RPAREN))
                    goto error;

diff --git a/src/language/stats/examine.q b/src/language/stats/examine.q
index db11ef7f2f01014835ad054afb974052031a7bf0..aa578e1e2d6907be4bbe1044fc61b7ebb76ac4e4 100644 (file)
--- a/src/language/stats/examine.q
+++ b/src/language/stats/examine.q
@@ -675,7 +675,7 @@ xmn_custom_variables (struct lexer *lexer, struct dataset *ds,
   const struct dictionary *dict = dataset_dict (ds);
   lex_match (lexer, T_EQUALS);
 
-  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       return 2;
@@ -719,7 +719,7 @@ examine_parse_independent_vars (struct lexer *lexer,
   ll_init (&sf->result_list);
 
   if ( (lex_token (lexer) != T_ID ||
-       dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+       dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       free ( sf ) ;
@@ -734,7 +734,7 @@ examine_parse_independent_vars (struct lexer *lexer,
       lex_match (lexer, T_BY);
 
       if ( (lex_token (lexer) != T_ID ||
-           dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+           dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
           && lex_token (lexer) != T_ALL)
        {
          free (sf);

diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index e397eb8760361fe70fa1a679f6955cc9585daebf..ecaefdf64e421196160009769ffe2e4438797262 100644 (file)
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -621,7 +621,7 @@ frq_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
   if (lex_token (lexer) != T_ALL
       && (lex_token (lexer) != T_ID
-          || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+          || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
     return 2;
 
   /* Get list of current variables, to avoid duplicates. */
@@ -663,7 +663,8 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
   struct frq_proc *frq = frq_;
 
   lex_match (lexer, T_EQUALS);
-  if ((lex_token (lexer) == T_ID && dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+  if ((lex_token (lexer) == T_ID
+       && dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
       || lex_token (lexer) == T_ID)
     for (;;)
       {
@@ -739,7 +740,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
        free (v);
        if (!lex_match (lexer, T_SLASH))
          break;
-       if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+       if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
             && lex_token (lexer) != T_ALL)
          {
            lex_put_back (lexer, T_SLASH);

diff --git a/src/language/stats/npar.c b/src/language/stats/npar.c
index b3a24dce77eace0e2a5e5f0359c04c3329fabe39..7dd94f7813df4f4f11ed5690f002bb6e6c65f74b 100644 (file)
--- a/src/language/stats/npar.c
+++ b/src/language/stats/npar.c
@@ -277,7 +277,7 @@ parse_npar_tests (struct lexer *lexer, struct dataset *ds, struct cmd_npar_tests
       else if (lex_match_hyphenated_word (lexer, "M-W") ||
               lex_match_hyphenated_word (lexer, "MANN-WHITNEY"))
         {
-          lex_match (lexer, '=');
+          lex_match (lexer, T_EQUALS);
           npt->mann_whitney++;
           switch (npar_mann_whitney (lexer, ds, nps))
             {
@@ -574,7 +574,7 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
   nt->execute = runs_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  if ( lex_force_match (lexer, '(') )
+  if ( lex_force_match (lexer, T_LPAREN) )
     {
       if ( lex_match_id (lexer, "MEAN"))
        {
@@ -600,8 +600,8 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
          return 0;
        }
                  
-      lex_force_match (lexer, ')');
-      lex_force_match (lexer, '=');
+      lex_force_match (lexer, T_RPAREN);
+      lex_force_match (lexer, T_EQUALS);
       if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
                                  &tp->vars, &tp->n_vars,
                                  PV_NO_SCRATCH | PV_NO_DUPLICATE | PV_NUMERIC))
@@ -632,7 +632,7 @@ npar_friedman (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
                                   &ost->vars, &ost->n_vars,
@@ -663,7 +663,7 @@ npar_kendall (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
                                   &ost->vars, &ost->n_vars,
@@ -693,7 +693,7 @@ npar_cochran (struct lexer *lexer, struct dataset *ds,
   nt->execute = cochran_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
                                   &ft->vars, &ft->n_vars,
@@ -772,7 +772,7 @@ npar_chisquare (struct lexer *lexer, struct dataset *ds,
                  n = 1;
                  f = lex_number (lexer);
                  lex_get (lexer);
-                 if ( lex_match (lexer, '*'))
+                 if ( lex_match (lexer, T_ASTERISK))
                    {
                      n = f;
                      f = lex_number (lexer);

diff --git a/src/language/stats/rank.q b/src/language/stats/rank.q
index 386ef03a877c9baf08e1d86c623e2abca8574416..f53ef2d01475a8c76040615898449e51f554f301 100644 (file)
--- a/src/language/stats/rank.q
+++ b/src/language/stats/rank.q
@@ -798,7 +798,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 {
   lex_match (lexer, T_EQUALS);
 
-  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
       return 2;
 
@@ -808,7 +808,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 
   if ( lex_match (lexer, T_BY)  )
     {
-      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
        {
          return 2;
        }
@@ -847,9 +847,9 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
       while( lex_token (lexer) == T_ID )
        {
 
-         if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
+         if ( dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL )
            {
-             msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
+             msg(SE, _("Variable %s already exists."), lex_tokcstr (lexer));
              return 0;
            }
          if ( var_count >= subcase_get_n_fields (&sc) )
@@ -858,7 +858,7 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
              return 0;
            }
 
-         destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokid (lexer));
+         destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokcstr (lexer));
          rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
 
          lex_get (lexer);

diff --git a/src/language/stats/regression.q b/src/language/stats/regression.q
index 83accc9e055a55b51c6f1562c62e017868ad99d1..e5b86ea2bc84a024a9f2503796b16a3654c8a2d1 100644 (file)
--- a/src/language/stats/regression.q
+++ b/src/language/stats/regression.q
@@ -755,7 +755,7 @@ regression_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
 
   if ((lex_token (lexer) != T_ID
-       || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+       || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
 

diff --git a/src/language/stats/reliability.c b/src/language/stats/reliability.c
index c6331970fc5c0b215063d0c4720c38a2e5e8a04a..3633b3dd8937e048e4062c3f1ae20df5d0c229b4 100644 (file)
--- a/src/language/stats/reliability.c
+++ b/src/language/stats/reliability.c
@@ -179,7 +179,7 @@ cmd_reliability (struct lexer *lexer, struct dataset *ds)
          if ( ! lex_force_string (lexer) ) 
            goto error;
 
-         ds_init_string (&reliability.scale_name, lex_tokstr (lexer));
+         ds_init_substring (&reliability.scale_name, lex_tokss (lexer));
 
          lex_get (lexer);
 

diff --git a/src/language/stats/sort-criteria.c b/src/language/stats/sort-criteria.c
index d808d1c203a161129a6de377dc834330d7528bff..649760235432fc4222ed70fed7353d7e815a0027 100644 (file)
--- a/src/language/stats/sort-criteria.c
+++ b/src/language/stats/sort-criteria.c
@@ -94,7 +94,7 @@ parse_sort_criteria (struct lexer *lexer, const struct dictionary *dict,
         }
     }
   while (lex_token (lexer) == T_ID
-         && dict_lookup_var (dict, lex_tokid (lexer)) != NULL);
+         && dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL);
 
   free (local_vars);
   return true;

diff --git a/src/language/tests/float-format.c b/src/language/tests/float-format.c
index 91e4d9ec737544a096f4a7dbd669962b65d85df1..554b2c33c782987255139125172946946922784d 100644 (file)
--- a/src/language/tests/float-format.c
+++ b/src/language/tests/float-format.c
@@ -116,7 +116,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
           || !lex_force_string (lexer))
         return false;
 
-      length = ds_length (lex_tokstr (lexer));
+      length = ss_length (lex_tokss (lexer));
       if (fp->format != FLOAT_HEX)
         {
           if (length != float_get_size (fp->format))
@@ -127,7 +127,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               return false;
             }
           assert (length <= sizeof fp->data);
-          memcpy (fp->data, ds_data (lex_tokstr (lexer)), length);
+          memcpy (fp->data, ss_data (lex_tokss (lexer)), length);
         }
       else
         {
@@ -136,7 +136,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               msg (SE, _("Hexadecimal floating constant too long."));
               return false;
             }
-          strncpy (CHAR_CAST_BUG (char *,fp->data), ds_cstr (lex_tokstr (lexer)), sizeof fp->data);
+          strncpy (CHAR_CAST_BUG (char *,fp->data), lex_tokcstr (lexer), sizeof fp->data);
         }
 
       lex_get (lexer);

diff --git a/src/language/tests/format-guesser-test.c b/src/language/tests/format-guesser-test.c
index 6285fc042c04a07cac583e68a38c814921ab4022..c95ea35f00cf5db60b5c8917a9d35a04f838b03a 100644 (file)
--- a/src/language/tests/format-guesser-test.c
+++ b/src/language/tests/format-guesser-test.c
@@ -35,8 +35,8 @@ cmd_debug_format_guesser (struct lexer *lexer, struct dataset *ds UNUSED)
   g = fmt_guesser_create ();
   while (lex_is_string (lexer))
     {
-      fprintf (stderr, "\"%s\" ", ds_cstr (lex_tokstr (lexer)));
-      fmt_guesser_add (g, ds_ss (lex_tokstr (lexer)));
+      fprintf (stderr, "\"%s\" ", lex_tokcstr (lexer));
+      fmt_guesser_add (g, lex_tokss (lexer));
       lex_get (lexer);
     }
 

diff --git a/src/language/tests/paper-size.c b/src/language/tests/paper-size.c
index 5aea2c334f34f5157a7dbcd77d4fa0f4a478f6b7..4dbd94600a454a437225d7c5aa96eb51bf90ab4c 100644 (file)
--- a/src/language/tests/paper-size.c
+++ b/src/language/tests/paper-size.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_debug_paper_size (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
-  paper_size = ds_cstr (lex_tokstr (lexer));
+  paper_size = lex_tokcstr (lexer);
 
   printf ("\"%s\" => ", paper_size);
   if (measure_paper (paper_size, &h, &v))

diff --git a/src/language/utilities/cd.c b/src/language/utilities/cd.c
index 38211e7661f8e6b0ac1978b56e6808150f794fbb..220164f0b420069480580ff481c4c467f692b610 100644 (file)
--- a/src/language/utilities/cd.c
+++ b/src/language/utilities/cd.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2008 Free Software Foundation, Inc.
+   Copyright (C) 2008, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_cd (struct lexer *lexer, struct dataset *ds UNUSED)
   if ( ! lex_force_string (lexer))
     goto error;
 
-  path = ds_xstrdup (lex_tokstr (lexer));
+  path = ss_xstrdup (lex_tokss (lexer));
 
   if ( -1 == chdir (path) )
     {

diff --git a/src/language/utilities/echo.c b/src/language/utilities/echo.c
index b27b40017a367c397901acfb932bb1e81166ad2e..149cc0eb0c131454fd3b717ab56a6eb2ec1bca61 100644 (file)
--- a/src/language/utilities/echo.c
+++ b/src/language/utilities/echo.c
@@ -34,7 +34,7 @@ cmd_echo (struct lexer *lexer, struct dataset *ds UNUSED)
 
   tab = tab_create(1, 1);
 
-  tab_text(tab, 0, 0, 0, ds_cstr (lex_tokstr (lexer)));
+  tab_text (tab, 0, 0, 0, lex_tokcstr (lexer));
 
   tab_submit(tab);
 

diff --git a/src/language/utilities/host.c b/src/language/utilities/host.c
index 86109574f61c84741735704589fb73049c06635a..f9b10480d2507c95d20dc677767c41c98639565f 100644 (file)
--- a/src/language/utilities/host.c
+++ b/src/language/utilities/host.c
@@ -145,7 +145,7 @@ cmd_host (struct lexer *lexer, struct dataset *ds UNUSED)
         {
           if (!ds_is_empty (&command))
             ds_put_byte (&command, '\n');
-          ds_put_substring (&command, ds_ss (lex_tokstr (lexer)));
+          ds_put_substring (&command, lex_tokss (lexer));
           lex_get (lexer);
         }
       if (!lex_force_match (lexer, T_RBRACK))

diff --git a/src/language/utilities/include.c b/src/language/utilities/include.c
index d259855e6ff707feab6b43679625412ff0987e63..f152684d41f73c6ba2091262ab368f3da7526c50 100644 (file)
--- a/src/language/utilities/include.c
+++ b/src/language/utilities/include.c
@@ -170,7 +170,7 @@ cmd_insert (struct lexer *lexer, struct dataset *ds UNUSED)
 static int
 parse_insert (struct lexer *lexer, char **filename)
 {
-  char *target_fn;
+  const char *target_fn;
   char *relative_filename;
 
   /* Skip optional FILE=. */
@@ -184,7 +184,7 @@ parse_insert (struct lexer *lexer, char **filename)
       return CMD_FAILURE;
     }
 
-  target_fn = ds_cstr (lex_tokstr (lexer));
+  target_fn = lex_tokcstr (lexer);
 
   relative_filename =
     fn_search_path (target_fn,

diff --git a/src/language/utilities/permissions.c b/src/language/utilities/permissions.c
index 34fa6fd83defb8701b97c9d505f3da22100c0ed4..ad9ffe751732037db38dfc2297cdde2bc58a6886 100644 (file)
--- a/src/language/utilities/permissions.c
+++ b/src/language/utilities/permissions.c
@@ -50,7 +50,7 @@ cmd_permissions (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  fn = ds_xstrdup (lex_tokstr (lexer));
+  fn = ss_xstrdup (lex_tokss (lexer));
   lex_force_match (lexer, T_STRING);
 
 

diff --git a/src/language/utilities/set.q b/src/language/utilities/set.q
index 3883f09649e4f18797526b6bce8ffb585d7a0fc4..f109e4505fd9d84fa17674deff4c3cc018f433cf 100644 (file)
--- a/src/language/utilities/set.q
+++ b/src/language/utilities/set.q
@@ -383,27 +383,26 @@ static int
 stc_custom_locale (struct lexer *lexer, struct dataset *ds UNUSED,
                   struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  const struct string *s;
+  const char *s;
 
   lex_match (lexer, T_EQUALS);
 
   if ( !lex_force_string (lexer))
     return 0;
 
-  s = lex_tokstr (lexer);
+  s = lex_tokcstr (lexer);
 
   /* First try this string as an encoding name */
-  if ( valid_encoding (ds_cstr (s)))
-    set_default_encoding (ds_cstr (s));
+  if ( valid_encoding (s))
+    set_default_encoding (s);
 
   /* Now try as a locale name (or alias) */
-  else if (set_encoding_from_locale (ds_cstr (s)))
+  else if (set_encoding_from_locale (s))
     {
     }
   else
     {
-      msg (ME, _("%s is not a recognized encoding or locale name"),
-          ds_cstr (s));
+      msg (ME, _("%s is not a recognized encoding or locale name"), s);
       return 0;
     }
 
@@ -511,7 +510,7 @@ stc_custom_journal (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_s
     journal_disable ();
   else if (lex_is_string (lexer) || lex_token (lexer) == T_ID)
     {
-      journal_set_file_name (ds_cstr (lex_tokstr (lexer)));
+      journal_set_file_name (lex_tokcstr (lexer));
       lex_get (lexer);
     }
   else

diff --git a/src/language/utilities/title.c b/src/language/utilities/title.c
index a3d89b3cf25493c4431d6e753564b4e282bb1e2c..82f7be50f5341b5b8cdddf69d0aba116dbafd302 100644 (file)
--- a/src/language/utilities/title.c
+++ b/src/language/utilities/title.c
@@ -57,7 +57,7 @@ parse_title (struct lexer *lexer, enum text_item_type type)
       lex_get (lexer);
       if (!lex_force_string (lexer))
        return CMD_FAILURE;
-      set_title (ds_cstr (lex_tokstr (lexer)), type);
+      set_title (lex_tokcstr (lexer), type);
       lex_get (lexer);
       return lex_end_of_command (lexer);
     }
@@ -149,7 +149,7 @@ cmd_add_documents (struct lexer *lexer, struct dataset *ds)
 
   while ( lex_is_string (lexer))
     {
-      dict_add_document_line (dict, ds_cstr (lex_tokstr (lexer)));
+      dict_add_document_line (dict, lex_tokcstr (lexer));
       lex_get (lexer);
     }
 

diff --git a/src/language/xforms/compute.c b/src/language/xforms/compute.c
index 8245ffed0ff7cef3cefe09394cfbe9c322387d27..442c15a4964d7118bf5d125bb6a640053c722d6d 100644 (file)
--- a/src/language/xforms/compute.c
+++ b/src/language/xforms/compute.c
@@ -349,10 +349,10 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   if (lex_look_ahead (lexer) == T_LPAREN)
     {
       /* Vector. */
-      lvalue->vector = dict_lookup_vector (dict, lex_tokid (lexer));
+      lvalue->vector = dict_lookup_vector (dict, lex_tokcstr (lexer));
       if (lvalue->vector == NULL)
        {
-         msg (SE, _("There is no vector named %s."), lex_tokid (lexer));
+         msg (SE, _("There is no vector named %s."), lex_tokcstr (lexer));
           goto lossage;
        }
 
@@ -369,7 +369,7 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   else
     {
       /* Variable name. */
-      const char *var_name = lex_tokid (lexer);
+      const char *var_name = lex_tokcstr (lexer);
       lvalue->variable = dict_lookup_var (dict, var_name);
       if (lvalue->variable == NULL)
         {

diff --git a/src/language/xforms/count.c b/src/language/xforms/count.c
index 6d055f836974fcfab1554de48d18bf436c1faa24..550db8a5315e454fde88f775b19002c2cee77a92 100644 (file)
--- a/src/language/xforms/count.c
+++ b/src/language/xforms/count.c
@@ -114,7 +114,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
       /* Get destination variable, or at least its name. */
       if (!lex_force_id (lexer))
        goto fail;
-      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
       if (dv->var != NULL)
         {
           if (var_is_alpha (dv->var))
@@ -124,7 +124,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
             }
         }
       else
-        dv->name = pool_strdup (trns->pool, lex_tokid (lexer));
+        dv->name = pool_strdup (trns->pool, lex_tokcstr (lexer));
 
       lex_get (lexer);
       if (!lex_force_match (lexer, T_EQUALS))
@@ -253,7 +253,7 @@ parse_string_criteria (struct lexer *lexer, struct pool *pool, struct criteria *
        return false;
       cur = &crit->values.str[crit->value_cnt++];
       *cur = pool_alloc (pool, len + 1);
-      str_copy_rpad (*cur, len + 1, ds_cstr (lex_tokstr (lexer)));
+      str_copy_rpad (*cur, len + 1, lex_tokcstr (lexer));
       lex_get (lexer);
 
       lex_match (lexer, T_COMMA);

diff --git a/src/language/xforms/recode.c b/src/language/xforms/recode.c
index 12f1eb435f003c5bfe312efcb42f37ff3379b246..236b49832e07f0be63b0900a1c3e66733f05711e 100644 (file)
--- a/src/language/xforms/recode.c
+++ b/src/language/xforms/recode.c
@@ -116,12 +116,12 @@ static bool parse_map_in (struct lexer *lexer, struct map_in *, struct pool *,
 static void set_map_in_generic (struct map_in *, enum map_in_type);
 static void set_map_in_num (struct map_in *, enum map_in_type, double, double);
 static void set_map_in_str (struct map_in *, struct pool *,
-                            const struct string *, size_t width);
+                            struct substring, size_t width);
 
 static bool parse_map_out (struct lexer *lexer, struct pool *, struct map_out *);
 static void set_map_out_num (struct map_out *, double);
 static void set_map_out_str (struct map_out *, struct pool *,
-                             const struct string *);
+                             struct substring);
 
 static void enlarge_dst_widths (struct recode_trns *);
 static void create_dst_vars (struct recode_trns *, struct dictionary *);
@@ -318,10 +318,10 @@ parse_map_in (struct lexer *lexer, struct map_in *in, struct pool *pool,
         return false;
       else 
        {
-         set_map_in_str (in, pool, lex_tokstr (lexer), max_src_width);
+         set_map_in_str (in, pool, lex_tokss (lexer), max_src_width);
          lex_get (lexer);
          if (lex_token (lexer) == T_ID
-             && lex_id_match (ss_cstr ("THRU"), ss_cstr (lex_tokid (lexer))))
+             && lex_id_match (ss_cstr ("THRU"), lex_tokss (lexer)))
            {
              msg (SE, _("THRU is not allowed with string variables."));
              return false;
@@ -370,13 +370,13 @@ set_map_in_num (struct map_in *in, enum map_in_type type, double x, double y)
    right to WIDTH characters long. */
 static void
 set_map_in_str (struct map_in *in, struct pool *pool,
-                const struct string *string, size_t width)
+                struct substring string, size_t width)
 {
   in->type = MAP_SINGLE;
   value_init_pool (pool, &in->x, width);
   value_copy_buf_rpad (&in->x, width,
-                       CHAR_CAST_BUG (uint8_t *, ds_data (string)),
-                       ds_length (string), ' ');
+                       CHAR_CAST_BUG (uint8_t *, ss_data (string)),
+                       ss_length (string), ' ');
 }
 
 /* Parses a mapping output value into OUT, allocating memory from
@@ -393,7 +393,7 @@ parse_map_out (struct lexer *lexer, struct pool *pool, struct map_out *out)
     set_map_out_num (out, SYSMIS);
   else if (lex_is_string (lexer))
     {
-      set_map_out_str (out, pool, lex_tokstr (lexer));
+      set_map_out_str (out, pool, lex_tokss (lexer));
       lex_get (lexer);
     }
   else if (lex_match_id (lexer, "COPY")) 
@@ -421,10 +421,10 @@ set_map_out_num (struct map_out *out, double value)
 /* Sets OUT as a string mapping output with the given VALUE. */
 static void
 set_map_out_str (struct map_out *out, struct pool *pool,
-                 const struct string *value)
+                 const struct substring value)
 {
-  const char *string = ds_data (value);
-  size_t length = ds_length (value);
+  const char *string = ss_data (value);
+  size_t length = ss_length (value);
 
   if (length == 0)
     {