From 55e6e7ba37a30570f5a31e2d78c22dfa7b61a36f Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Fri, 19 Nov 2010 21:05:04 -0800
Subject: [PATCH] lexer: Change the functions for retrieving token strings.

Until now, lex_tokid() has been for T_ID tokens only and lex_tokstr() has
been for T_ID and T_STRING tokens.  For T_ID tokens, lex_tokid() and
lex_tokstr() had slightly different semantics.

This doesn't entirely make sense, and these particular functions are not
the ones wanted most by clients, so this commit removes these functions
in favor of lex_tokcstr() and lex_tokss(), which are both applicable
to both T_ID and T_STRING tokens, with the same semantics in each case.

These functions are also easier for the upcoming reimplementation of the
lexer.
---
 src/language/command.c                     |  6 +--
 src/language/control/loop.c                |  4 +-
 src/language/control/repeat.c              | 14 +++----
 src/language/data-io/combine-files.c       |  6 +--
 src/language/data-io/data-list.c           | 10 ++---
 src/language/data-io/file-handle.q         | 13 +++---
 src/language/data-io/get-data.c            | 21 +++++-----
 src/language/data-io/print.c               |  2 +-
 src/language/data-io/save-translate.c      |  8 ++--
 src/language/data-io/trim.c                |  6 +--
 src/language/dictionary/attributes.c       |  8 ++--
 src/language/dictionary/missing-values.c   |  4 +-
 src/language/dictionary/modify-variables.c |  2 +-
 src/language/dictionary/mrsets.c           | 16 +++----
 src/language/dictionary/value-labels.c     |  2 +-
 src/language/dictionary/variable-label.c   |  2 +-
 src/language/dictionary/vector.c           | 10 ++---
 src/language/expressions/evaluate.c        |  4 +-
 src/language/expressions/parse.c           | 22 +++++-----
 src/language/lexer/format-parser.c         |  6 +--
 src/language/lexer/lexer.c                 | 49 +++++++++++-----------
 src/language/lexer/lexer.h                 |  4 +-
 src/language/lexer/q2c.c                   | 14 +++----
 src/language/lexer/value-parser.c          |  9 +++-
 src/language/lexer/variable-parser.c       | 14 +++----
 src/language/stats/aggregate.c             |  6 +--
 src/language/stats/crosstabs.q             |  2 +-
 src/language/stats/descriptives.c          |  6 +--
 src/language/stats/examine.q               |  6 +--
 src/language/stats/frequencies.q           |  7 ++--
 src/language/stats/npar.c                  | 16 +++----
 src/language/stats/rank.q                  | 10 ++---
 src/language/stats/regression.q            |  2 +-
 src/language/stats/reliability.c           |  2 +-
 src/language/stats/sort-criteria.c         |  2 +-
 src/language/tests/float-format.c          |  6 +--
 src/language/tests/format-guesser-test.c   |  4 +-
 src/language/tests/paper-size.c            |  4 +-
 src/language/utilities/cd.c                |  4 +-
 src/language/utilities/echo.c              |  2 +-
 src/language/utilities/host.c              |  2 +-
 src/language/utilities/include.c           |  4 +-
 src/language/utilities/permissions.c       |  2 +-
 src/language/utilities/set.q               | 15 ++++---
 src/language/utilities/title.c             |  4 +-
 src/language/xforms/compute.c              |  6 +--
 src/language/xforms/count.c                |  6 +--
 src/language/xforms/recode.c               | 22 +++++-----
 48 files changed, 200 insertions(+), 196 deletions(-)

diff --git a/src/language/command.c b/src/language/command.c
index 3da5da6991..3bf9571fa5 100644
--- a/src/language/command.c
+++ b/src/language/command.c
@@ -288,7 +288,7 @@ parse_command_name (struct lexer *lexer)
         {
           if (!ds_is_empty (&s) && ds_last (&s) != '-')
             ds_put_byte (&s, ' ');
-          ds_put_cstr (&s, lex_tokid (lexer));
+          ds_put_cstr (&s, lex_tokcstr (lexer));
         }
       else if (lex_is_integer (lexer) && lex_integer (lexer) >= 0)
         {
@@ -514,10 +514,10 @@ cmd_erase (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  if (remove (ds_cstr (lex_tokstr (lexer))) == -1)
+  if (remove (lex_tokcstr (lexer)) == -1)
     {
       msg (SW, _("Error removing `%s': %s."),
-	   ds_cstr (lex_tokstr (lexer)), strerror (errno));
+           lex_tokcstr (lexer), strerror (errno));
       return CMD_FAILURE;
     }
 
diff --git a/src/language/control/loop.c b/src/language/control/loop.c
index f17542cb42..5e99232fc7 100644
--- a/src/language/control/loop.c
+++ b/src/language/control/loop.c
@@ -221,13 +221,13 @@ parse_index_clause (struct dataset *ds, struct lexer *lexer,
       return false;
     }
 
-  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
   if (loop->index_var != NULL)
     *created_index_var = false;
   else
     {
       loop->index_var = dict_create_var_assert (dataset_dict (ds),
-                                                lex_tokid (lexer), 0);
+                                                lex_tokcstr (lexer), 0);
       *created_index_var = true;
     }
   lex_get (lexer);
diff --git a/src/language/control/repeat.c b/src/language/control/repeat.c
index ddbba09aa9..130dfd6880 100644
--- a/src/language/control/repeat.c
+++ b/src/language/control/repeat.c
@@ -168,21 +168,19 @@ parse_specification (struct lexer *lexer, struct repeat_block *block)
       /* Get a stand-in variable name and make sure it's unique. */
       if (!lex_force_id (lexer))
 	return false;
-      if (dict_lookup_var (dict, lex_tokid (lexer)))
-        msg (SW, _("Dummy variable name `%s' hides dictionary "
-                   "variable `%s'."),
-             lex_tokid (lexer), lex_tokid (lexer));
-      if (find_macro (block, ss_cstr (lex_tokid (lexer))))
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)))
+        msg (SW, _("Dummy variable name `%s' hides dictionary variable `%s'."),
+             lex_tokcstr (lexer), lex_tokcstr (lexer));
+      if (find_macro (block, lex_tokss (lexer)))
 	  {
 	    msg (SE, _("Dummy variable name `%s' is given twice."),
-		 lex_tokid (lexer));
+		 lex_tokcstr (lexer));
 	    return false;
 	  }
 
       /* Make a new macro. */
       macro = pool_alloc (block->pool, sizeof *macro);
-      ss_alloc_substring_pool (&macro->name, ss_cstr (lex_tokid (lexer)),
-                               block->pool);
+      ss_alloc_substring_pool (&macro->name, lex_tokss (lexer), block->pool);
       ll_push_tail (&block->macros, &macro->ll);
 
       /* Skip equals sign. */
diff --git a/src/language/data-io/combine-files.c b/src/language/data-io/combine-files.c
index 66163ee0b3..aa6d3b98ec 100644
--- a/src/language/data-io/combine-files.c
+++ b/src/language/data-io/combine-files.c
@@ -251,7 +251,7 @@ combine_files (enum comb_command_type command,
                            "TABLE."));
                 goto error;
               }
-            strcpy (file->in_name, lex_tokid (lexer));
+            strcpy (file->in_name, lex_tokcstr (lexer));
             lex_get (lexer);
           }
         else if (lex_match_id (lexer, "SORT"))
@@ -325,7 +325,7 @@ combine_files (enum comb_command_type command,
 	  lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (first_name, lex_tokid (lexer));
+          strcpy (first_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
@@ -339,7 +339,7 @@ combine_files (enum comb_command_type command,
 	  lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (last_name, lex_tokid (lexer));
+          strcpy (last_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "MAP"))
diff --git a/src/language/data-io/data-list.c b/src/language/data-io/data-list.c
index aec243a592..046c7b0e0c 100644
--- a/src/language/data-io/data-list.c
+++ b/src/language/data-io/data-list.c
@@ -108,7 +108,7 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
 	  if (!lex_force_string (lexer))
 	    goto error;
 
-	  ds_init_string (&encoding, lex_tokstr (lexer));
+	  ds_init_substring (&encoding, lex_tokss (lexer));
 
 	  lex_get (lexer);
 	}
@@ -147,9 +147,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
 	  lex_match (lexer, T_EQUALS);
 	  if (!lex_force_id (lexer))
 	    goto error;
-	  end = dict_lookup_var (dict, lex_tokid (lexer));
+	  end = dict_lookup_var (dict, lex_tokcstr (lexer));
 	  if (!end)
-            end = dict_create_var_assert (dict, lex_tokid (lexer), 0);
+            end = dict_create_var_assert (dict, lex_tokcstr (lexer), 0);
 	  lex_get (lexer);
 	}
       else if (lex_match_id (lexer, "NOTABLE"))
@@ -197,9 +197,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
                       if (lex_match_id (lexer, "TAB"))
                         delim = '\t';
                       else if (lex_is_string (lexer)
-                               && ds_length (lex_tokstr (lexer)) == 1)
+                               && ss_length (lex_tokss (lexer)) == 1)
                         {
-                          delim = ds_first (lex_tokstr (lexer));
+                          delim = ss_first (lex_tokss (lexer));
                           lex_get (lexer);
                         }
                       else
diff --git a/src/language/data-io/file-handle.q b/src/language/data-io/file-handle.q
index ba84a15dc5..5f2a44075a 100644
--- a/src/language/data-io/file-handle.q
+++ b/src/language/data-io/file-handle.q
@@ -57,7 +57,7 @@ cmd_file_handle (struct lexer *lexer, struct dataset *ds)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  str_copy_trunc (handle_name, sizeof handle_name, lex_tokid (lexer));
+  str_copy_trunc (handle_name, sizeof handle_name, lex_tokcstr (lexer));
 
   handle = fh_from_id (handle_name);
   if (handle != NULL)
@@ -159,7 +159,7 @@ cmd_close_file_handle (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  handle = fh_from_id (lex_tokid (lexer));
+  handle = fh_from_id (lex_tokcstr (lexer));
   if (handle == NULL)
     return CMD_CASCADING_FAILURE;
 
@@ -208,14 +208,15 @@ fh_parse (struct lexer *lexer, enum fh_referent referent_mask)
 
       handle = NULL;
       if (lex_token (lexer) == T_ID)
-        handle = fh_from_id (lex_tokid (lexer));
+        handle = fh_from_id (lex_tokcstr (lexer));
       if (handle == NULL)
         {
-          if (lex_token (lexer) != T_ID || lex_tokid (lexer)[0] != '#' || settings_get_syntax () != ENHANCED)
-            handle = fh_create_file (NULL, ds_cstr (lex_tokstr (lexer)),
+          if (lex_token (lexer) != T_ID || lex_tokcstr (lexer)[0] != '#'
+              || settings_get_syntax () != ENHANCED)
+            handle = fh_create_file (NULL, lex_tokcstr (lexer),
                                      fh_default_properties ());
           else
-            handle = fh_create_scratch (lex_tokid (lexer));
+            handle = fh_create_scratch (lex_tokcstr (lexer));
         }
       lex_get (lexer);
     }
diff --git a/src/language/data-io/get-data.c b/src/language/data-io/get-data.c
index eeb3827a58..05e115a9f5 100644
--- a/src/language/data-io/get-data.c
+++ b/src/language/data-io/get-data.c
@@ -61,7 +61,7 @@ cmd_get_data (struct lexer *lexer, struct dataset *ds)
   else if (lex_match_id (lexer, "PSQL"))
     return parse_get_psql (lexer, ds);
 
-  msg (SE, _("Unsupported TYPE %s"), lex_tokid (lexer));
+  msg (SE, _("Unsupported TYPE %s"), lex_tokcstr (lexer));
   return CMD_FAILURE;
 }
 
@@ -85,7 +85,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  psql.conninfo = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  psql.conninfo = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -113,7 +113,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
 	  if ( ! lex_force_string (lexer) )
 	    goto error;
 
-	  ds_put_substring (&psql.sql,  lex_tokstr (lexer)->ss);
+	  ds_put_substring (&psql.sql, lex_tokss (lexer));
 	  lex_get (lexer);
 	}
      }
@@ -153,7 +153,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  gri.file_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  gri.file_name = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -172,7 +172,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	      if ( ! lex_force_string (lexer) )
 		goto error;
 
-	      gri.sheet_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+	      gri.sheet_name = ss_xstrdup (lex_tokss (lexer));
 	      gri.sheet_index = -1;
 	    }
 	  else if (lex_match_id (lexer, "INDEX"))
@@ -196,7 +196,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	      if ( ! lex_force_string (lexer) )
 		goto error;
 
-	      gri.cell_range = xstrdup (ds_cstr (lex_tokstr (lexer)));
+	      gri.cell_range = ss_xstrdup (lex_tokss (lexer));
 	    }
 	  else
 	    goto error;
@@ -219,6 +219,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	}
       else
 	{
+	  printf ("Unknown data file type `%s'\n", lex_tokcstr (lexer));
 	  goto error;
 	}
       lex_get (lexer);
@@ -415,7 +416,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
           if (!lex_force_string (lexer))
             goto error;
 
-          s = ds_ss (lex_tokstr (lexer));
+          s = lex_tokss (lexer);
           if (ss_match_string (&s, ss_cstr ("\\t")))
             ds_put_cstr (&hard_seps, "\t");
           if (ss_match_string (&s, ss_cstr ("\\\\")))
@@ -441,14 +442,14 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
             goto error;
 
           if (settings_get_syntax () == COMPATIBLE
-              && ds_length (lex_tokstr (lexer)) != 1)
+              && ss_length (lex_tokss (lexer)) != 1)
             {
               msg (SE, _("In compatible syntax mode, the QUALIFIER string "
                          "must contain exactly one character."));
               goto error;
             }
 
-          data_parser_set_quotes (parser, ds_ss (lex_tokstr (lexer)));
+          data_parser_set_quotes (parser, lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (settings_get_syntax () == ENHANCED
@@ -501,7 +502,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
 
       if (!lex_force_id (lexer))
         goto error;
-      strcpy (name, lex_tokid (lexer));
+      strcpy (name, lex_tokcstr (lexer));
       lex_get (lexer);
 
       if (type == DP_DELIMITED)
diff --git a/src/language/data-io/print.c b/src/language/data-io/print.c
index cfbe043961..96ac15e152 100644
--- a/src/language/data-io/print.c
+++ b/src/language/data-io/print.c
@@ -280,7 +280,7 @@ parse_string_argument (struct lexer *lexer, struct print_trns *trns, int record,
   spec->type = PRT_LITERAL;
   spec->record = record;
   spec->first_column = *column;
-  ds_init_string (&spec->string, lex_tokstr (lexer));
+  ds_init_substring (&spec->string, lex_tokss (lexer));
   ds_register_pool (&spec->string, trns->pool);
   lex_get (lexer);
 
diff --git a/src/language/data-io/save-translate.c b/src/language/data-io/save-translate.c
index dcfdb83c7c..213f0bf735 100644
--- a/src/language/data-io/save-translate.c
+++ b/src/language/data-io/save-translate.c
@@ -159,13 +159,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "DELIMITER");
                       goto error;
                     }
-                  delimiter = ds_first (lex_tokstr (lexer));
+                  delimiter = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "QUALIFIER"))
@@ -173,13 +173,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "QUALIFIER");
                       goto error;
                     }
-                  qualifier = ds_first (lex_tokstr (lexer));
+                  qualifier = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "DECIMAL"))
diff --git a/src/language/data-io/trim.c b/src/language/data-io/trim.c
index be637c4033..9f76b10516 100644
--- a/src/language/data-io/trim.c
+++ b/src/language/data-io/trim.c
@@ -83,18 +83,18 @@ parse_dict_rename (struct lexer *lexer, struct dictionary *dict)
       if (!lex_force_match (lexer, T_EQUALS)
 	  || !lex_force_id (lexer))
 	return 0;
-      if (dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
 	{
 	  msg (SE, _("Cannot rename %s as %s because there already exists "
 		     "a variable named %s.  To rename variables with "
 		     "overlapping names, use a single RENAME subcommand "
 		     "such as `/RENAME (A=B)(B=C)(C=A)', or equivalently, "
 		     "`/RENAME (A B C=B C A)'."),
-               var_get_name (v), lex_tokid (lexer), lex_tokid (lexer));
+               var_get_name (v), lex_tokcstr (lexer), lex_tokcstr (lexer));
 	  return 0;
 	}
 
-      dict_rename_var (dict, v, lex_tokid (lexer));
+      dict_rename_var (dict, v, lex_tokcstr (lexer));
       lex_get (lexer);
       return 1;
     }
diff --git a/src/language/dictionary/attributes.c b/src/language/dictionary/attributes.c
index 35e6954295..c992ff8068 100644
--- a/src/language/dictionary/attributes.c
+++ b/src/language/dictionary/attributes.c
@@ -79,7 +79,7 @@ static bool
 match_subcommand (struct lexer *lexer, const char *keyword) 
 {
   if (lex_token (lexer) == T_ID
-      && lex_id_match (ss_cstr (lex_tokid (lexer)), ss_cstr (keyword))
+      && lex_id_match (lex_tokss (lexer), ss_cstr (keyword))
       && lex_look_ahead (lexer) == T_EQUALS)
     {
       lex_get (lexer);          /* Skip keyword. */
@@ -96,7 +96,7 @@ parse_attribute_name (struct lexer *lexer, char name[VAR_NAME_LEN + 1],
 {
   if (!lex_force_id (lexer))
     return false;
-  strcpy (name, lex_tokid (lexer));
+  strcpy (name, lex_tokcstr (lexer));
   lex_get (lexer);
 
   if (lex_match (lexer, T_LBRACK))
@@ -122,14 +122,14 @@ static bool
 add_attribute (struct lexer *lexer, struct attrset **sets, size_t n) 
 {
   char name[VAR_NAME_LEN + 1];
+  const char *value;
   size_t index, i;
-  char *value;
 
   if (!parse_attribute_name (lexer, name, &index)
       || !lex_force_match (lexer, T_LPAREN)
       || !lex_force_string (lexer))
     return false;
-  value = ds_cstr (lex_tokstr (lexer));
+  value = lex_tokcstr (lexer);
 
   for (i = 0; i < n; i++)
     {
diff --git a/src/language/dictionary/missing-values.c b/src/language/dictionary/missing-values.c
index bbc2c84d4d..4a14ee195b 100644
--- a/src/language/dictionary/missing-values.c
+++ b/src/language/dictionary/missing-values.c
@@ -106,7 +106,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       break;
                     }
 
-                  length = ds_length (lex_tokstr (lexer));
+                  length = ss_length (lex_tokss (lexer));
                   if (length > MV_MAX_STRING)
                     {
                       msg (SE, _("Truncating missing value to maximum "
@@ -115,7 +115,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       length = MV_MAX_STRING;
                     }
                   memset (value, ' ', MV_MAX_STRING);
-                  memcpy (value, ds_data (lex_tokstr (lexer)), length);
+                  memcpy (value, ss_data (lex_tokss (lexer)), length);
 
                   if (!mv_add_str (&mv, value))
                     deferred_errors = true;
diff --git a/src/language/dictionary/modify-variables.c b/src/language/dictionary/modify-variables.c
index 6afd321590..cc847372d3 100644
--- a/src/language/dictionary/modify-variables.c
+++ b/src/language/dictionary/modify-variables.c
@@ -300,7 +300,7 @@ cmd_modify_vars (struct lexer *lexer, struct dataset *ds)
       else
 	{
 	  if (lex_token (lexer) == T_ID)
-	    msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokid (lexer));
+	    msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokcstr (lexer));
 	  else
 	    msg (SE, _("Subcommand name expected."));
 	  goto done;
diff --git a/src/language/dictionary/mrsets.c b/src/language/dictionary/mrsets.c
index 0708e89ccf..c775f49784 100644
--- a/src/language/dictionary/mrsets.c
+++ b/src/language/dictionary/mrsets.c
@@ -93,16 +93,16 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_match (lexer, T_EQUALS) || !lex_force_id (lexer))
             goto error;
-          if (lex_tokid (lexer)[0] != '$')
+          if (lex_tokcstr (lexer)[0] != '$')
             {
               msg (SE, _("%s is not a valid name for a multiple response "
                          "set.  Multiple response set names must begin with "
-                         "`$'."), lex_tokid (lexer));
+                         "`$'."), lex_tokcstr (lexer));
               goto error;
             }
 
           free (mrset->name);
-          mrset->name = xstrdup (lex_tokid (lexer));
+          mrset->name = xstrdup (lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "VARIABLES"))
@@ -129,7 +129,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             goto error;
 
           free (mrset->label);
-          mrset->label = ds_xstrdup (lex_tokstr (lexer));
+          mrset->label = ss_xstrdup (lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (type == MRSET_MD && lex_match_id (lexer, "LABELSOURCE"))
@@ -159,7 +159,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             }
           else if (lex_is_string (lexer))
             {
-              const char *s = ds_cstr (lex_tokstr (lexer));
+              const char *s = lex_tokcstr (lexer);
               int width;
 
               /* Trim off trailing spaces, but don't trim the string until
@@ -480,14 +480,14 @@ parse_mrset_names (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_id (lexer))
             return false;
-          if (dict_lookup_mrset (dict, lex_tokid (lexer)) == NULL)
+          if (dict_lookup_mrset (dict, lex_tokcstr (lexer)) == NULL)
             {
               msg (SE, _("No multiple response set named %s."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
               stringi_set_destroy (mrset_names);
               return false;
             }
-          stringi_set_insert (mrset_names, lex_tokid (lexer));
+          stringi_set_insert (mrset_names, lex_tokcstr (lexer));
           lex_get (lexer);
         }
     }
diff --git a/src/language/dictionary/value-labels.c b/src/language/dictionary/value-labels.c
index 0fdd7a2a36..1471fc8687 100644
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -142,7 +142,7 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
           return 0;
         }
 
-      ds_init_string (&label, lex_tokstr (lexer));
+      ds_init_substring (&label, lex_tokss (lexer));
 
       if (ds_length (&label) > 60)
 	{
diff --git a/src/language/dictionary/variable-label.c b/src/language/dictionary/variable-label.c
index e6bd62296f..804a4213d1 100644
--- a/src/language/dictionary/variable-label.c
+++ b/src/language/dictionary/variable-label.c
@@ -52,7 +52,7 @@ cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
 	  return CMD_FAILURE;
 	}
 
-      ds_init_string (&label, lex_tokstr (lexer) );
+      ds_init_substring (&label, lex_tokss (lexer));
       if (ds_length (&label) > 255)
 	{
 	  msg (SW, _("Truncating variable label to 255 characters."));
diff --git a/src/language/dictionary/vector.c b/src/language/dictionary/vector.c
index a884331ce3..3be0723f70 100644
--- a/src/language/dictionary/vector.c
+++ b/src/language/dictionary/vector.c
@@ -59,25 +59,25 @@ cmd_vector (struct lexer *lexer, struct dataset *ds)
 	{
           size_t i;
 
-	  if (dict_lookup_vector (dict, lex_tokid (lexer)))
+	  if (dict_lookup_vector (dict, lex_tokcstr (lexer)))
 	    {
 	      msg (SE, _("A vector named %s already exists."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
 	      goto fail;
 	    }
 
           for (i = 0; i < vector_cnt; i++)
-            if (!strcasecmp (vectors[i], lex_tokid (lexer)))
+            if (!strcasecmp (vectors[i], lex_tokcstr (lexer)))
 	      {
 		msg (SE, _("Vector name %s is given twice."),
-                     lex_tokid (lexer));
+                     lex_tokcstr (lexer));
 		goto fail;
 	      }
 
           if (vector_cnt == vector_cap)
             vectors = pool_2nrealloc (pool,
                                        vectors, &vector_cap, sizeof *vectors);
-          vectors[vector_cnt++] = pool_strdup (pool, lex_tokid (lexer));
+          vectors[vector_cnt++] = pool_strdup (pool, lex_tokcstr (lexer));
 
 	  lex_get (lexer);
 	  lex_match (lexer, T_COMMA);
diff --git a/src/language/expressions/evaluate.c b/src/language/expressions/evaluate.c
index 9bcfe31e6f..79b5215952 100644
--- a/src/language/expressions/evaluate.c
+++ b/src/language/expressions/evaluate.c
@@ -134,7 +134,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
 
           if (!lex_force_id (lexer))
             goto done;
-          strcpy (name, lex_tokid (lexer));
+          strcpy (name, lex_tokcstr (lexer));
 
           lex_get (lexer);
           if (!lex_force_match (lexer, T_EQUALS))
@@ -143,7 +143,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
           if (lex_is_number (lexer))
             width = 0;
           else if (lex_is_string (lexer))
-            width = ds_length (lex_tokstr (lexer));
+            width = ss_length (lex_tokss (lexer));
           else
             {
               lex_error (lexer, _("expecting number or string"));
diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 79576b4482..ae968d0078 100644
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -824,7 +824,7 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
     return expr_allocate_number (e, settings_get_viewwidth ());
   else
     {
-      msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
+      msg (SE, _("Unknown system variable %s."), lex_tokcstr (lexer));
       return NULL;
     }
 }
@@ -841,17 +841,17 @@ parse_primary (struct lexer *lexer, struct expression *e)
           /* An identifier followed by a left parenthesis may be
              a vector element reference.  If not, it's a function
              call. */
-          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer)) != NULL)
+          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer)) != NULL)
             return parse_vector_element (lexer, e);
           else
             return parse_function (lexer, e);
         }
-      else if (lex_tokid (lexer)[0] == '$')
+      else if (lex_tokcstr (lexer)[0] == '$')
         {
           /* $ at the beginning indicates a system variable. */
           return parse_sysvar (lexer, e);
         }
-      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokid (lexer)))
+      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokcstr (lexer)))
         {
           /* It looks like a user variable.
              (It could be a format specifier, but we'll assume
@@ -872,7 +872,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
             return expr_allocate_format (e, &fmt);
 
           /* All attempts failed. */
-          msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
+          msg (SE, _("Unknown identifier %s."), lex_tokcstr (lexer));
           return NULL;
         }
       break;
@@ -888,7 +888,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
     case T_STRING:
       {
         union any_node *node = expr_allocate_string_buffer (
-          e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+          e, lex_tokcstr (lexer), ss_length (lex_tokss (lexer)));
 	lex_get (lexer);
 	return node;
       }
@@ -918,7 +918,7 @@ parse_vector_element (struct lexer *lexer, struct expression *e)
   /* Find vector, skip token.
      The caller must already have verified that the current token
      is the name of a vector. */
-  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
+  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer));
   assert (vector != NULL);
   lex_get (lexer);
 
@@ -1209,11 +1209,11 @@ parse_function (struct lexer *lexer, struct expression *e)
 
   union any_node *n;
 
-  ds_init_string (&func_name, lex_tokstr (lexer));
-  min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
-  if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last))
+  ds_init_substring (&func_name, lex_tokss (lexer));
+  min_valid = extract_min_valid (lex_tokcstr (lexer));
+  if (!lookup_function (lex_tokcstr (lexer), &first, &last))
     {
-      msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("No function or vector named %s."), lex_tokcstr (lexer));
       ds_destroy (&func_name);
       return NULL;
     }
diff --git a/src/language/lexer/format-parser.c b/src/language/lexer/format-parser.c
index 1730b4936c..4b25020ed4 100644
--- a/src/language/lexer/format-parser.c
+++ b/src/language/lexer/format-parser.c
@@ -46,7 +46,7 @@ parse_abstract_format_specifier__ (struct lexer *lexer,
     goto error;
 
   /* Extract pieces. */
-  s = ds_ss (lex_tokstr (lexer));
+  s = ss_cstr (lex_tokcstr (lexer));
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_LETTERS)), &type_ss);
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_DIGITS)), &width_ss);
   if (ss_match_byte (&s, '.'))
@@ -133,9 +133,9 @@ parse_format_specifier_name (struct lexer *lexer, enum fmt_type *type)
       lex_error (lexer, _("expecting format type"));
       return false;
     }
-  if (!fmt_from_name (ds_cstr (lex_tokstr (lexer)), type))
+  if (!fmt_from_name (lex_tokcstr (lexer), type))
     {
-      msg (SE, _("Unknown format type `%s'."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("Unknown format type `%s'."), lex_tokcstr (lexer));
       return false;
     }
   lex_get (lexer);
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index c08c428c4a..4a899bdff9 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -49,11 +49,7 @@ struct lexer
   int token;      /* Current token. */
   double tokval;  /* T_POS_NUM, T_NEG_NUM: the token's value. */
 
-  char tokid [VAR_NAME_LEN + 1];   /* T_ID: the identifier. */
-
-  struct string tokstr;   /* T_ID, T_STRING: token string value.
-  			    For T_ID, this is not truncated as is
-  			    tokid. */
+  struct string tokstr;   /* T_ID, T_STRING: token string value. */
 
   char *prog; /* Pointer to next token in line_buffer. */
   bool dot;   /* True only if this line ends with a terminal dot. */
@@ -138,7 +134,6 @@ restore_token (struct lexer *lexer)
   assert (lexer->put_token != 0);
   lexer->token = lexer->put_token;
   ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   lexer->tokval = lexer->put_tokval;
   lexer->put_token = 0;
 }
@@ -405,8 +400,7 @@ lex_get (struct lexer *lexer)
     }
 }
 
-/* Parses an identifier at the current position into tokid and
-   tokstr.
+/* Parses an identifier at the current position into tokstr.
    Returns the correct token type. */
 static int
 parse_id (struct lexer *lexer)
@@ -420,7 +414,6 @@ parse_id (struct lexer *lexer)
   lexer->prog += ss_length (id);
 
   ds_assign_substring (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   return lex_id_to_token (id);
 }
 
@@ -571,7 +564,7 @@ bool
 lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
 {
   if (lexer->token == T_ID
-      && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+      && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
     {
       lex_get (lexer);
       return true;
@@ -798,7 +791,6 @@ lex_put_back_id (struct lexer *lexer, const char *id)
   save_token (lexer);
   lexer->token = T_ID;
   ds_assign_cstr (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
 }
 
 /* Weird line processing functions. */
@@ -1082,36 +1074,37 @@ lex_token_representation (struct lexer *lexer)
     case T_ID:
     case T_POS_NUM:
     case T_NEG_NUM:
-      return ds_xstrdup (&lexer->tokstr);
+      return ss_xstrdup (lex_tokss (lexer));
 
     case T_STRING:
       {
+        struct substring ss;
 	int hexstring = 0;
 	char *sp, *dp;
 
-	for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+        ss = lex_tokss (lexer);
+	for (sp = ss_data (ss); sp < ss_end (ss); sp++)
 	  if (!c_isprint ((unsigned char) *sp))
 	    {
 	      hexstring = 1;
 	      break;
 	    }
 
-	token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+	token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
 
 	dp = token_rep;
 	if (hexstring)
 	  *dp++ = 'X';
 	*dp++ = '\'';
 
-	if (!hexstring)
-	  for (sp = ds_cstr (&lexer->tokstr); *sp; )
+        for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+          if (!hexstring)
 	    {
 	      if (*sp == '\'')
 		*dp++ = '\'';
-	      *dp++ = (unsigned char) *sp++;
+	      *dp++ = (unsigned char) *sp;
 	    }
-	else
-	  for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+          else
 	    {
 	      *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
 	      *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
@@ -1344,16 +1337,22 @@ lex_tokval (const struct lexer *lexer)
   return lexer->tokval;
 }
 
+/* Returns the null-terminated string value associated with LEXER's current
+   token.  For a T_ID token, this is the identifier, and for a T_STRING token,
+   this is the string.  For other tokens the value is undefined. */
 const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
 {
-  return lexer->tokid;
+  return ds_cstr (&lexer->tokstr);
 }
 
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token.  For a T_ID
+   token, this is the identifier, and for a T_STRING token, this is the string.
+   For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
 {
-  return &lexer->tokstr;
+  return ds_ss (&lexer->tokstr);
 }
 
 /* If the lexer is positioned at the (pseudo)identifier S, which
@@ -1368,7 +1367,7 @@ lex_match_hyphenated_word (struct lexer *lexer, const char *s)
   if (hyphen == NULL)
     return lex_match_id (lexer, s);
   else if (lexer->token != T_ID
-	   || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+	   || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
 	   || lex_look_ahead (lexer) != T_DASH)
     return false;
   else
diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index 82f3fe23c1..7af40234cc 100644
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -93,8 +93,8 @@ char *lex_token_representation (struct lexer *);
 /* Token accessors */
 enum token_type lex_token (const struct lexer *);
 double lex_tokval (const struct lexer *);
-const char *lex_tokid (const struct lexer *);
-const struct string *lex_tokstr (const struct lexer *);
+const char *lex_tokcstr (const struct lexer *);
+struct substring lex_tokss (const struct lexer *);
 
 /* Really weird functions. */
 void lex_skip_comment (struct lexer *);
diff --git a/src/language/lexer/q2c.c b/src/language/lexer/q2c.c
index c03b6b21da..cf5a04ee13 100644
--- a/src/language/lexer/q2c.c
+++ b/src/language/lexer/q2c.c
@@ -1542,7 +1542,7 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
 	      dump (0, "goto lossage;");
 	      dump (-1, "}");
               dump (-1, "free (p->%s%s);", sbc->prefix, st_lower (s->valname));
-              dump (0, "p->%s%s = xstrdup (ds_cstr (lex_tokstr (lexer)));",
+              dump (0, "p->%s%s = ss_xstrdup (ss_tokss (lexer));",
                     sbc->prefix, st_lower (s->valname));
             }
           else
@@ -1705,7 +1705,7 @@ dump_subcommand (const subcommand *sbc)
       outdent ();
       if (sbc->restriction)
 	{
-	  dump (0, "x = ds_length (lex_tokstr (lexer));");
+	  dump (0, "x = ss_length (lex_tokss (lexer));");
 	  dump (1, "if (!(%s))", sbc->restriction);
 	  dump (1, "{");
 	  dump (0, "msg (SE, _(\"String for %s must be %s.\"));",
@@ -1715,7 +1715,7 @@ dump_subcommand (const subcommand *sbc)
 	  outdent ();
 	}
       dump (0, "free(p->s_%s);", st_lower(sbc->name) );
-      dump (0, "p->s_%s = ds_xstrdup (lex_tokstr (lexer));",
+      dump (0, "p->s_%s = ss_xstrdup (lex_tokss (lexer));",
 	    st_lower (sbc->name));
       dump (0, "lex_get (lexer);");
       if (sbc->restriction)
@@ -1833,13 +1833,13 @@ dump_parser (int persistent)
     {
       if (def->type == SBC_VARLIST)
 	dump (1, "if (lex_token (lexer) == T_ID "
-              "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL "
-	      "&& lex_look_ahead (lexer) != T_EQUALS)");
+              "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL "
+	      "&& lex_look_ahead (lexer) != '=')");
       else
 	{
 	  dump (0, "if ((lex_token (lexer) == T_ID "
-                "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) "
-		"&& lex_look_ahead () != T_EQUALS)");
+                "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) "
+		"&& lex_look_ahead () != '=')");
 	  dump (1, "     || token == T_ALL)");
 	}
       dump (1, "{");
diff --git a/src/language/lexer/value-parser.c b/src/language/lexer/value-parser.c
index 269d24bca4..4ef092de3a 100644
--- a/src/language/lexer/value-parser.c
+++ b/src/language/lexer/value-parser.c
@@ -106,7 +106,7 @@ parse_number (struct lexer *lexer, double *x, const enum fmt_type *format)
 
       assert (fmt_get_category (*format) != FMT_CAT_STRING);
 
-      if (!data_in_msg (ds_ss (lex_tokstr (lexer)), LEGACY_NATIVE,
+      if (!data_in_msg (lex_tokss (lexer), LEGACY_NATIVE,
                         *format, &v, 0, NULL))
         return false;
 
@@ -143,7 +143,12 @@ parse_value (struct lexer *lexer, union value *v, int width)
     }
   else if (lex_force_string (lexer))
     {
-      const char *s = ds_cstr (lex_tokstr (lexer));
+      const char *s;
+
+      if (!lex_force_string (lexer))
+	return false;
+
+      s = lex_tokcstr (lexer);
       value_copy_str_rpad (v, width, CHAR_CAST_BUG (const uint8_t *, s), ' ');
     }
   else
diff --git a/src/language/lexer/variable-parser.c b/src/language/lexer/variable-parser.c
index b05b7ce7f9..5b698bbf4a 100644
--- a/src/language/lexer/variable-parser.c
+++ b/src/language/lexer/variable-parser.c
@@ -65,14 +65,14 @@ parse_vs_variable_idx (struct lexer *lexer, const struct var_set *vs,
       lex_error (lexer, _("expecting variable name"));
       return false;
     }
-  else if (var_set_lookup_var_idx (vs, lex_tokid (lexer), idx))
+  else if (var_set_lookup_var_idx (vs, lex_tokcstr (lexer), idx))
     {
       lex_get (lexer);
       return true;
     }
   else
     {
-      msg (SE, _("%s is not a variable name."), lex_tokid (lexer));
+      msg (SE, _("%s is not a variable name."), lex_tokcstr (lexer));
       return false;
     }
 }
@@ -339,7 +339,7 @@ parse_var_set_vars (struct lexer *lexer, const struct var_set *vs,
       lex_match (lexer, T_COMMA);
     }
   while (lex_token (lexer) == T_ALL
-         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokid (lexer)) != NULL));
+         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokcstr (lexer)) != NULL));
 
   if (*nv == 0)
     goto fail;
@@ -444,13 +444,13 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
 	  lex_error (lexer, "expecting variable name");
 	  goto fail;
 	}
-      if (dict_class_from_id (lex_tokid (lexer)) == DC_SCRATCH
+      if (dict_class_from_id (lex_tokcstr (lexer)) == DC_SCRATCH
           && (pv_opts & PV_NO_SCRATCH))
 	{
 	  msg (SE, _("Scratch variables not allowed here."));
 	  goto fail;
 	}
-      strcpy (name1, lex_tokid (lexer));
+      strcpy (name1, lex_tokcstr (lexer));
       lex_get (lexer);
       if (lex_token (lexer) == T_TO)
 	{
@@ -460,7 +460,7 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
 	      lex_error (lexer, "expecting variable name");
 	      goto fail;
 	    }
-	  strcpy (name2, lex_tokid (lexer));
+	  strcpy (name2, lex_tokcstr (lexer));
 	  lex_get (lexer);
 
 	  if (!extract_num (name1, root1, &n1, &d1)
@@ -588,7 +588,7 @@ parse_mixed_vars (struct lexer *lexer, const struct dictionary *dict,
     }
   while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
     {
-      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
 	{
 	  struct variable **v;
 	  size_t nv;
diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c
index bbe444d548..cfa9ea6624 100644
--- a/src/language/stats/aggregate.c
+++ b/src/language/stats/aggregate.c
@@ -435,7 +435,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 	  if (lex_is_string (lexer))
 	    {
 	      struct string label;
-	      ds_init_string (&label, lex_tokstr (lexer));
+	      ds_init_substring (&label, lex_tokss (lexer));
 
 	      ds_truncate (&label, 255);
 	      dest_label[n_dest - 1] = ds_xstrdup (&label);
@@ -451,7 +451,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 	  goto error;
 	}
 
-      ds_assign_string (&function_name, lex_tokstr (lexer));
+      ds_assign_substring (&function_name, lex_tokss (lexer));
       exclude = ds_chomp (&function_name, '.') ? MV_SYSTEM : MV_ANY;
 
       for (function = agr_func_tab; function->name; function++)
@@ -501,7 +501,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 		lex_match (lexer, T_COMMA);
 		if (lex_is_string (lexer))
 		  {
-		    arg[i].c = ds_xstrdup (lex_tokstr (lexer));
+		    arg[i].c = ss_xstrdup (lex_tokss (lexer));
 		    type = VAL_STRING;
 		  }
 		else if (lex_is_number (lexer))
diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q
index 0e7b3daecb..02cd4deb98 100644
--- a/src/language/stats/crosstabs.q
+++ b/src/language/stats/crosstabs.q
@@ -381,7 +381,7 @@ crs_custom_tables (struct lexer *lexer, struct dataset *ds,
   /* Ensure that this is a TABLES subcommand. */
   if (!lex_match_id (lexer, "TABLES")
       && (lex_token (lexer) != T_ID ||
-	  dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+	  dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
   lex_match (lexer, T_EQUALS);
diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c
index 17eb445044..ae472294a8 100644
--- a/src/language/stats/descriptives.c
+++ b/src/language/stats/descriptives.c
@@ -334,14 +334,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds)
                       lex_error (lexer, NULL);
                       goto error;
                     }
-                  if (try_name (dict, dsc, lex_tokid (lexer)))
+                  if (try_name (dict, dsc, lex_tokcstr (lexer)))
                     {
-                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer));
+                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokcstr (lexer));
                       z_cnt++;
                     }
                   else
                     msg (SE, _("Z-score variable name %s would be"
-                               " a duplicate variable name."), lex_tokid (lexer));
+                               " a duplicate variable name."), lex_tokcstr (lexer));
                   lex_get (lexer);
                   if (!lex_force_match (lexer, T_RPAREN))
 		    goto error;
diff --git a/src/language/stats/examine.q b/src/language/stats/examine.q
index db11ef7f2f..aa578e1e2d 100644
--- a/src/language/stats/examine.q
+++ b/src/language/stats/examine.q
@@ -675,7 +675,7 @@ xmn_custom_variables (struct lexer *lexer, struct dataset *ds,
   const struct dictionary *dict = dataset_dict (ds);
   lex_match (lexer, T_EQUALS);
 
-  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       return 2;
@@ -719,7 +719,7 @@ examine_parse_independent_vars (struct lexer *lexer,
   ll_init (&sf->result_list);
 
   if ( (lex_token (lexer) != T_ID ||
-	dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+	dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       free ( sf ) ;
@@ -734,7 +734,7 @@ examine_parse_independent_vars (struct lexer *lexer,
       lex_match (lexer, T_BY);
 
       if ( (lex_token (lexer) != T_ID ||
-	    dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+	    dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
 	   && lex_token (lexer) != T_ALL)
 	{
 	  free (sf);
diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index e397eb8760..ecaefdf64e 100644
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -621,7 +621,7 @@ frq_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
   if (lex_token (lexer) != T_ALL
       && (lex_token (lexer) != T_ID
-          || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+          || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
     return 2;
 
   /* Get list of current variables, to avoid duplicates. */
@@ -663,7 +663,8 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
   struct frq_proc *frq = frq_;
 
   lex_match (lexer, T_EQUALS);
-  if ((lex_token (lexer) == T_ID && dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+  if ((lex_token (lexer) == T_ID
+       && dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
       || lex_token (lexer) == T_ID)
     for (;;)
       {
@@ -739,7 +740,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
 	free (v);
 	if (!lex_match (lexer, T_SLASH))
 	  break;
-	if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+	if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
             && lex_token (lexer) != T_ALL)
 	  {
 	    lex_put_back (lexer, T_SLASH);
diff --git a/src/language/stats/npar.c b/src/language/stats/npar.c
index b3a24dce77..7dd94f7813 100644
--- a/src/language/stats/npar.c
+++ b/src/language/stats/npar.c
@@ -277,7 +277,7 @@ parse_npar_tests (struct lexer *lexer, struct dataset *ds, struct cmd_npar_tests
       else if (lex_match_hyphenated_word (lexer, "M-W") ||
 	       lex_match_hyphenated_word (lexer, "MANN-WHITNEY"))
         {
-          lex_match (lexer, '=');
+          lex_match (lexer, T_EQUALS);
           npt->mann_whitney++;
           switch (npar_mann_whitney (lexer, ds, nps))
             {
@@ -574,7 +574,7 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
   nt->execute = runs_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  if ( lex_force_match (lexer, '(') )
+  if ( lex_force_match (lexer, T_LPAREN) )
     {
       if ( lex_match_id (lexer, "MEAN"))
 	{
@@ -600,8 +600,8 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
 	  return 0;
 	}
 		  
-      lex_force_match (lexer, ')');
-      lex_force_match (lexer, '=');
+      lex_force_match (lexer, T_RPAREN);
+      lex_force_match (lexer, T_EQUALS);
       if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				  &tp->vars, &tp->n_vars,
 				  PV_NO_SCRATCH | PV_NO_DUPLICATE | PV_NUMERIC))
@@ -632,7 +632,7 @@ npar_friedman (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ost->vars, &ost->n_vars,
@@ -663,7 +663,7 @@ npar_kendall (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ost->vars, &ost->n_vars,
@@ -693,7 +693,7 @@ npar_cochran (struct lexer *lexer, struct dataset *ds,
   nt->execute = cochran_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ft->vars, &ft->n_vars,
@@ -772,7 +772,7 @@ npar_chisquare (struct lexer *lexer, struct dataset *ds,
 		  n = 1;
 		  f = lex_number (lexer);
 		  lex_get (lexer);
-		  if ( lex_match (lexer, '*'))
+		  if ( lex_match (lexer, T_ASTERISK))
 		    {
 		      n = f;
 		      f = lex_number (lexer);
diff --git a/src/language/stats/rank.q b/src/language/stats/rank.q
index 386ef03a87..f53ef2d014 100644
--- a/src/language/stats/rank.q
+++ b/src/language/stats/rank.q
@@ -798,7 +798,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 {
   lex_match (lexer, T_EQUALS);
 
-  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
       return 2;
 
@@ -808,7 +808,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 
   if ( lex_match (lexer, T_BY)  )
     {
-      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
 	{
 	  return 2;
 	}
@@ -847,9 +847,9 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
       while( lex_token (lexer) == T_ID )
 	{
 
-	  if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
+	  if ( dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL )
 	    {
-	      msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
+	      msg(SE, _("Variable %s already exists."), lex_tokcstr (lexer));
 	      return 0;
 	    }
 	  if ( var_count >= subcase_get_n_fields (&sc) )
@@ -858,7 +858,7 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
 	      return 0;
 	    }
 
-	  destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokid (lexer));
+	  destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokcstr (lexer));
 	  rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
 
 	  lex_get (lexer);
diff --git a/src/language/stats/regression.q b/src/language/stats/regression.q
index 83accc9e05..e5b86ea2bc 100644
--- a/src/language/stats/regression.q
+++ b/src/language/stats/regression.q
@@ -755,7 +755,7 @@ regression_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
 
   if ((lex_token (lexer) != T_ID
-       || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+       || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
 
diff --git a/src/language/stats/reliability.c b/src/language/stats/reliability.c
index c6331970fc..3633b3dd89 100644
--- a/src/language/stats/reliability.c
+++ b/src/language/stats/reliability.c
@@ -179,7 +179,7 @@ cmd_reliability (struct lexer *lexer, struct dataset *ds)
 	  if ( ! lex_force_string (lexer) ) 
 	    goto error;
 
-	  ds_init_string (&reliability.scale_name, lex_tokstr (lexer));
+	  ds_init_substring (&reliability.scale_name, lex_tokss (lexer));
 
 	  lex_get (lexer);
 
diff --git a/src/language/stats/sort-criteria.c b/src/language/stats/sort-criteria.c
index d808d1c203..6497602354 100644
--- a/src/language/stats/sort-criteria.c
+++ b/src/language/stats/sort-criteria.c
@@ -94,7 +94,7 @@ parse_sort_criteria (struct lexer *lexer, const struct dictionary *dict,
         }
     }
   while (lex_token (lexer) == T_ID
-         && dict_lookup_var (dict, lex_tokid (lexer)) != NULL);
+         && dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL);
 
   free (local_vars);
   return true;
diff --git a/src/language/tests/float-format.c b/src/language/tests/float-format.c
index 91e4d9ec73..554b2c33c7 100644
--- a/src/language/tests/float-format.c
+++ b/src/language/tests/float-format.c
@@ -116,7 +116,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
           || !lex_force_string (lexer))
         return false;
 
-      length = ds_length (lex_tokstr (lexer));
+      length = ss_length (lex_tokss (lexer));
       if (fp->format != FLOAT_HEX)
         {
           if (length != float_get_size (fp->format))
@@ -127,7 +127,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               return false;
             }
           assert (length <= sizeof fp->data);
-          memcpy (fp->data, ds_data (lex_tokstr (lexer)), length);
+          memcpy (fp->data, ss_data (lex_tokss (lexer)), length);
         }
       else
         {
@@ -136,7 +136,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               msg (SE, _("Hexadecimal floating constant too long."));
               return false;
             }
-          strncpy (CHAR_CAST_BUG (char *,fp->data), ds_cstr (lex_tokstr (lexer)), sizeof fp->data);
+          strncpy (CHAR_CAST_BUG (char *,fp->data), lex_tokcstr (lexer), sizeof fp->data);
         }
 
       lex_get (lexer);
diff --git a/src/language/tests/format-guesser-test.c b/src/language/tests/format-guesser-test.c
index 6285fc042c..c95ea35f00 100644
--- a/src/language/tests/format-guesser-test.c
+++ b/src/language/tests/format-guesser-test.c
@@ -35,8 +35,8 @@ cmd_debug_format_guesser (struct lexer *lexer, struct dataset *ds UNUSED)
   g = fmt_guesser_create ();
   while (lex_is_string (lexer))
     {
-      fprintf (stderr, "\"%s\" ", ds_cstr (lex_tokstr (lexer)));
-      fmt_guesser_add (g, ds_ss (lex_tokstr (lexer)));
+      fprintf (stderr, "\"%s\" ", lex_tokcstr (lexer));
+      fmt_guesser_add (g, lex_tokss (lexer));
       lex_get (lexer);
     }
 
diff --git a/src/language/tests/paper-size.c b/src/language/tests/paper-size.c
index 5aea2c334f..4dbd94600a 100644
--- a/src/language/tests/paper-size.c
+++ b/src/language/tests/paper-size.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_debug_paper_size (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
-  paper_size = ds_cstr (lex_tokstr (lexer));
+  paper_size = lex_tokcstr (lexer);
 
   printf ("\"%s\" => ", paper_size);
   if (measure_paper (paper_size, &h, &v))
diff --git a/src/language/utilities/cd.c b/src/language/utilities/cd.c
index 38211e7661..220164f0b4 100644
--- a/src/language/utilities/cd.c
+++ b/src/language/utilities/cd.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2008 Free Software Foundation, Inc.
+   Copyright (C) 2008, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_cd (struct lexer *lexer, struct dataset *ds UNUSED)
   if ( ! lex_force_string (lexer))
     goto error;
 
-  path = ds_xstrdup (lex_tokstr (lexer));
+  path = ss_xstrdup (lex_tokss (lexer));
 
   if ( -1 == chdir (path) )
     {
diff --git a/src/language/utilities/echo.c b/src/language/utilities/echo.c
index b27b40017a..149cc0eb0c 100644
--- a/src/language/utilities/echo.c
+++ b/src/language/utilities/echo.c
@@ -34,7 +34,7 @@ cmd_echo (struct lexer *lexer, struct dataset *ds UNUSED)
 
   tab = tab_create(1, 1);
 
-  tab_text(tab, 0, 0, 0, ds_cstr (lex_tokstr (lexer)));
+  tab_text (tab, 0, 0, 0, lex_tokcstr (lexer));
 
   tab_submit(tab);
 
diff --git a/src/language/utilities/host.c b/src/language/utilities/host.c
index 86109574f6..f9b10480d2 100644
--- a/src/language/utilities/host.c
+++ b/src/language/utilities/host.c
@@ -145,7 +145,7 @@ cmd_host (struct lexer *lexer, struct dataset *ds UNUSED)
         {
           if (!ds_is_empty (&command))
             ds_put_byte (&command, '\n');
-          ds_put_substring (&command, ds_ss (lex_tokstr (lexer)));
+          ds_put_substring (&command, lex_tokss (lexer));
           lex_get (lexer);
         }
       if (!lex_force_match (lexer, T_RBRACK))
diff --git a/src/language/utilities/include.c b/src/language/utilities/include.c
index d259855e6f..f152684d41 100644
--- a/src/language/utilities/include.c
+++ b/src/language/utilities/include.c
@@ -170,7 +170,7 @@ cmd_insert (struct lexer *lexer, struct dataset *ds UNUSED)
 static int
 parse_insert (struct lexer *lexer, char **filename)
 {
-  char *target_fn;
+  const char *target_fn;
   char *relative_filename;
 
   /* Skip optional FILE=. */
@@ -184,7 +184,7 @@ parse_insert (struct lexer *lexer, char **filename)
       return CMD_FAILURE;
     }
 
-  target_fn = ds_cstr (lex_tokstr (lexer));
+  target_fn = lex_tokcstr (lexer);
 
   relative_filename =
     fn_search_path (target_fn,
diff --git a/src/language/utilities/permissions.c b/src/language/utilities/permissions.c
index 34fa6fd83d..ad9ffe7517 100644
--- a/src/language/utilities/permissions.c
+++ b/src/language/utilities/permissions.c
@@ -50,7 +50,7 @@ cmd_permissions (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  fn = ds_xstrdup (lex_tokstr (lexer));
+  fn = ss_xstrdup (lex_tokss (lexer));
   lex_force_match (lexer, T_STRING);
 
 
diff --git a/src/language/utilities/set.q b/src/language/utilities/set.q
index 3883f09649..f109e4505f 100644
--- a/src/language/utilities/set.q
+++ b/src/language/utilities/set.q
@@ -383,27 +383,26 @@ static int
 stc_custom_locale (struct lexer *lexer, struct dataset *ds UNUSED,
 		   struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  const struct string *s;
+  const char *s;
 
   lex_match (lexer, T_EQUALS);
 
   if ( !lex_force_string (lexer))
     return 0;
 
-  s = lex_tokstr (lexer);
+  s = lex_tokcstr (lexer);
 
   /* First try this string as an encoding name */
-  if ( valid_encoding (ds_cstr (s)))
-    set_default_encoding (ds_cstr (s));
+  if ( valid_encoding (s))
+    set_default_encoding (s);
 
   /* Now try as a locale name (or alias) */
-  else if (set_encoding_from_locale (ds_cstr (s)))
+  else if (set_encoding_from_locale (s))
     {
     }
   else
     {
-      msg (ME, _("%s is not a recognized encoding or locale name"),
-	   ds_cstr (s));
+      msg (ME, _("%s is not a recognized encoding or locale name"), s);
       return 0;
     }
 
@@ -511,7 +510,7 @@ stc_custom_journal (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_s
     journal_disable ();
   else if (lex_is_string (lexer) || lex_token (lexer) == T_ID)
     {
-      journal_set_file_name (ds_cstr (lex_tokstr (lexer)));
+      journal_set_file_name (lex_tokcstr (lexer));
       lex_get (lexer);
     }
   else
diff --git a/src/language/utilities/title.c b/src/language/utilities/title.c
index a3d89b3cf2..82f7be50f5 100644
--- a/src/language/utilities/title.c
+++ b/src/language/utilities/title.c
@@ -57,7 +57,7 @@ parse_title (struct lexer *lexer, enum text_item_type type)
       lex_get (lexer);
       if (!lex_force_string (lexer))
 	return CMD_FAILURE;
-      set_title (ds_cstr (lex_tokstr (lexer)), type);
+      set_title (lex_tokcstr (lexer), type);
       lex_get (lexer);
       return lex_end_of_command (lexer);
     }
@@ -149,7 +149,7 @@ cmd_add_documents (struct lexer *lexer, struct dataset *ds)
 
   while ( lex_is_string (lexer))
     {
-      dict_add_document_line (dict, ds_cstr (lex_tokstr (lexer)));
+      dict_add_document_line (dict, lex_tokcstr (lexer));
       lex_get (lexer);
     }
 
diff --git a/src/language/xforms/compute.c b/src/language/xforms/compute.c
index 8245ffed0f..442c15a496 100644
--- a/src/language/xforms/compute.c
+++ b/src/language/xforms/compute.c
@@ -349,10 +349,10 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   if (lex_look_ahead (lexer) == T_LPAREN)
     {
       /* Vector. */
-      lvalue->vector = dict_lookup_vector (dict, lex_tokid (lexer));
+      lvalue->vector = dict_lookup_vector (dict, lex_tokcstr (lexer));
       if (lvalue->vector == NULL)
 	{
-	  msg (SE, _("There is no vector named %s."), lex_tokid (lexer));
+	  msg (SE, _("There is no vector named %s."), lex_tokcstr (lexer));
           goto lossage;
 	}
 
@@ -369,7 +369,7 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   else
     {
       /* Variable name. */
-      const char *var_name = lex_tokid (lexer);
+      const char *var_name = lex_tokcstr (lexer);
       lvalue->variable = dict_lookup_var (dict, var_name);
       if (lvalue->variable == NULL)
         {
diff --git a/src/language/xforms/count.c b/src/language/xforms/count.c
index 6d055f8369..550db8a531 100644
--- a/src/language/xforms/count.c
+++ b/src/language/xforms/count.c
@@ -114,7 +114,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
       /* Get destination variable, or at least its name. */
       if (!lex_force_id (lexer))
 	goto fail;
-      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
       if (dv->var != NULL)
         {
           if (var_is_alpha (dv->var))
@@ -124,7 +124,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
             }
         }
       else
-        dv->name = pool_strdup (trns->pool, lex_tokid (lexer));
+        dv->name = pool_strdup (trns->pool, lex_tokcstr (lexer));
 
       lex_get (lexer);
       if (!lex_force_match (lexer, T_EQUALS))
@@ -253,7 +253,7 @@ parse_string_criteria (struct lexer *lexer, struct pool *pool, struct criteria *
 	return false;
       cur = &crit->values.str[crit->value_cnt++];
       *cur = pool_alloc (pool, len + 1);
-      str_copy_rpad (*cur, len + 1, ds_cstr (lex_tokstr (lexer)));
+      str_copy_rpad (*cur, len + 1, lex_tokcstr (lexer));
       lex_get (lexer);
 
       lex_match (lexer, T_COMMA);
diff --git a/src/language/xforms/recode.c b/src/language/xforms/recode.c
index 12f1eb435f..236b49832e 100644
--- a/src/language/xforms/recode.c
+++ b/src/language/xforms/recode.c
@@ -116,12 +116,12 @@ static bool parse_map_in (struct lexer *lexer, struct map_in *, struct pool *,
 static void set_map_in_generic (struct map_in *, enum map_in_type);
 static void set_map_in_num (struct map_in *, enum map_in_type, double, double);
 static void set_map_in_str (struct map_in *, struct pool *,
-                            const struct string *, size_t width);
+                            struct substring, size_t width);
 
 static bool parse_map_out (struct lexer *lexer, struct pool *, struct map_out *);
 static void set_map_out_num (struct map_out *, double);
 static void set_map_out_str (struct map_out *, struct pool *,
-                             const struct string *);
+                             struct substring);
 
 static void enlarge_dst_widths (struct recode_trns *);
 static void create_dst_vars (struct recode_trns *, struct dictionary *);
@@ -318,10 +318,10 @@ parse_map_in (struct lexer *lexer, struct map_in *in, struct pool *pool,
         return false;
       else 
 	{
-	  set_map_in_str (in, pool, lex_tokstr (lexer), max_src_width);
+	  set_map_in_str (in, pool, lex_tokss (lexer), max_src_width);
 	  lex_get (lexer);
 	  if (lex_token (lexer) == T_ID
-	      && lex_id_match (ss_cstr ("THRU"), ss_cstr (lex_tokid (lexer))))
+	      && lex_id_match (ss_cstr ("THRU"), lex_tokss (lexer)))
 	    {
 	      msg (SE, _("THRU is not allowed with string variables."));
 	      return false;
@@ -370,13 +370,13 @@ set_map_in_num (struct map_in *in, enum map_in_type type, double x, double y)
    right to WIDTH characters long. */
 static void
 set_map_in_str (struct map_in *in, struct pool *pool,
-                const struct string *string, size_t width)
+                struct substring string, size_t width)
 {
   in->type = MAP_SINGLE;
   value_init_pool (pool, &in->x, width);
   value_copy_buf_rpad (&in->x, width,
-                       CHAR_CAST_BUG (uint8_t *, ds_data (string)),
-                       ds_length (string), ' ');
+                       CHAR_CAST_BUG (uint8_t *, ss_data (string)),
+                       ss_length (string), ' ');
 }
 
 /* Parses a mapping output value into OUT, allocating memory from
@@ -393,7 +393,7 @@ parse_map_out (struct lexer *lexer, struct pool *pool, struct map_out *out)
     set_map_out_num (out, SYSMIS);
   else if (lex_is_string (lexer))
     {
-      set_map_out_str (out, pool, lex_tokstr (lexer));
+      set_map_out_str (out, pool, lex_tokss (lexer));
       lex_get (lexer);
     }
   else if (lex_match_id (lexer, "COPY")) 
@@ -421,10 +421,10 @@ set_map_out_num (struct map_out *out, double value)
 /* Sets OUT as a string mapping output with the given VALUE. */
 static void
 set_map_out_str (struct map_out *out, struct pool *pool,
-                 const struct string *value)
+                 const struct substring value)
 {
-  const char *string = ds_data (value);
-  size_t length = ds_length (value);
+  const char *string = ss_data (value);
+  size_t length = ss_length (value);
 
   if (length == 0)
     {
-- 
2.30.2