From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sat, 20 Nov 2010 05:05:04 +0000 (-0800)
Subject: lexer: Change the functions for retrieving token strings.
X-Git-Tag: v0.7.7~123
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=55e6e7ba37a30570f5a31e2d78c22dfa7b61a36f;p=pspp-builds.git

lexer: Change the functions for retrieving token strings.

Until now, lex_tokid() has been for T_ID tokens only and lex_tokstr() has
been for T_ID and T_STRING tokens.  For T_ID tokens, lex_tokid() and
lex_tokstr() had slightly different semantics.

This doesn't entirely make sense, and these particular functions are not
the ones wanted most by clients, so this commit removes these functions
in favor of lex_tokcstr() and lex_tokss(), which are both applicable
to both T_ID and T_STRING tokens, with the same semantics in each case.

These functions are also easier for the upcoming reimplementation of the
lexer.
---

diff --git a/src/language/command.c b/src/language/command.c
index 3da5da69..3bf9571f 100644
--- a/src/language/command.c
+++ b/src/language/command.c
@@ -288,7 +288,7 @@ parse_command_name (struct lexer *lexer)
         {
           if (!ds_is_empty (&s) && ds_last (&s) != '-')
             ds_put_byte (&s, ' ');
-          ds_put_cstr (&s, lex_tokid (lexer));
+          ds_put_cstr (&s, lex_tokcstr (lexer));
         }
       else if (lex_is_integer (lexer) && lex_integer (lexer) >= 0)
         {
@@ -514,10 +514,10 @@ cmd_erase (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  if (remove (ds_cstr (lex_tokstr (lexer))) == -1)
+  if (remove (lex_tokcstr (lexer)) == -1)
     {
       msg (SW, _("Error removing `%s': %s."),
-	   ds_cstr (lex_tokstr (lexer)), strerror (errno));
+           lex_tokcstr (lexer), strerror (errno));
       return CMD_FAILURE;
     }
 
diff --git a/src/language/control/loop.c b/src/language/control/loop.c
index f17542cb..5e99232f 100644
--- a/src/language/control/loop.c
+++ b/src/language/control/loop.c
@@ -221,13 +221,13 @@ parse_index_clause (struct dataset *ds, struct lexer *lexer,
       return false;
     }
 
-  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+  loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
   if (loop->index_var != NULL)
     *created_index_var = false;
   else
     {
       loop->index_var = dict_create_var_assert (dataset_dict (ds),
-                                                lex_tokid (lexer), 0);
+                                                lex_tokcstr (lexer), 0);
       *created_index_var = true;
     }
   lex_get (lexer);
diff --git a/src/language/control/repeat.c b/src/language/control/repeat.c
index ddbba09a..130dfd68 100644
--- a/src/language/control/repeat.c
+++ b/src/language/control/repeat.c
@@ -168,21 +168,19 @@ parse_specification (struct lexer *lexer, struct repeat_block *block)
       /* Get a stand-in variable name and make sure it's unique. */
       if (!lex_force_id (lexer))
 	return false;
-      if (dict_lookup_var (dict, lex_tokid (lexer)))
-        msg (SW, _("Dummy variable name `%s' hides dictionary "
-                   "variable `%s'."),
-             lex_tokid (lexer), lex_tokid (lexer));
-      if (find_macro (block, ss_cstr (lex_tokid (lexer))))
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)))
+        msg (SW, _("Dummy variable name `%s' hides dictionary variable `%s'."),
+             lex_tokcstr (lexer), lex_tokcstr (lexer));
+      if (find_macro (block, lex_tokss (lexer)))
 	  {
 	    msg (SE, _("Dummy variable name `%s' is given twice."),
-		 lex_tokid (lexer));
+		 lex_tokcstr (lexer));
 	    return false;
 	  }
 
       /* Make a new macro. */
       macro = pool_alloc (block->pool, sizeof *macro);
-      ss_alloc_substring_pool (&macro->name, ss_cstr (lex_tokid (lexer)),
-                               block->pool);
+      ss_alloc_substring_pool (&macro->name, lex_tokss (lexer), block->pool);
       ll_push_tail (&block->macros, &macro->ll);
 
       /* Skip equals sign. */
diff --git a/src/language/data-io/combine-files.c b/src/language/data-io/combine-files.c
index 66163ee0..aa6d3b98 100644
--- a/src/language/data-io/combine-files.c
+++ b/src/language/data-io/combine-files.c
@@ -251,7 +251,7 @@ combine_files (enum comb_command_type command,
                            "TABLE."));
                 goto error;
               }
-            strcpy (file->in_name, lex_tokid (lexer));
+            strcpy (file->in_name, lex_tokcstr (lexer));
             lex_get (lexer);
           }
         else if (lex_match_id (lexer, "SORT"))
@@ -325,7 +325,7 @@ combine_files (enum comb_command_type command,
 	  lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (first_name, lex_tokid (lexer));
+          strcpy (first_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
@@ -339,7 +339,7 @@ combine_files (enum comb_command_type command,
 	  lex_match (lexer, T_EQUALS);
           if (!lex_force_id (lexer))
             goto error;
-          strcpy (last_name, lex_tokid (lexer));
+          strcpy (last_name, lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "MAP"))
diff --git a/src/language/data-io/data-list.c b/src/language/data-io/data-list.c
index aec243a5..046c7b0e 100644
--- a/src/language/data-io/data-list.c
+++ b/src/language/data-io/data-list.c
@@ -108,7 +108,7 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
 	  if (!lex_force_string (lexer))
 	    goto error;
 
-	  ds_init_string (&encoding, lex_tokstr (lexer));
+	  ds_init_substring (&encoding, lex_tokss (lexer));
 
 	  lex_get (lexer);
 	}
@@ -147,9 +147,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
 	  lex_match (lexer, T_EQUALS);
 	  if (!lex_force_id (lexer))
 	    goto error;
-	  end = dict_lookup_var (dict, lex_tokid (lexer));
+	  end = dict_lookup_var (dict, lex_tokcstr (lexer));
 	  if (!end)
-            end = dict_create_var_assert (dict, lex_tokid (lexer), 0);
+            end = dict_create_var_assert (dict, lex_tokcstr (lexer), 0);
 	  lex_get (lexer);
 	}
       else if (lex_match_id (lexer, "NOTABLE"))
@@ -197,9 +197,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
                       if (lex_match_id (lexer, "TAB"))
                         delim = '\t';
                       else if (lex_is_string (lexer)
-                               && ds_length (lex_tokstr (lexer)) == 1)
+                               && ss_length (lex_tokss (lexer)) == 1)
                         {
-                          delim = ds_first (lex_tokstr (lexer));
+                          delim = ss_first (lex_tokss (lexer));
                           lex_get (lexer);
                         }
                       else
diff --git a/src/language/data-io/file-handle.q b/src/language/data-io/file-handle.q
index ba84a15d..5f2a4407 100644
--- a/src/language/data-io/file-handle.q
+++ b/src/language/data-io/file-handle.q
@@ -57,7 +57,7 @@ cmd_file_handle (struct lexer *lexer, struct dataset *ds)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  str_copy_trunc (handle_name, sizeof handle_name, lex_tokid (lexer));
+  str_copy_trunc (handle_name, sizeof handle_name, lex_tokcstr (lexer));
 
   handle = fh_from_id (handle_name);
   if (handle != NULL)
@@ -159,7 +159,7 @@ cmd_close_file_handle (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  handle = fh_from_id (lex_tokid (lexer));
+  handle = fh_from_id (lex_tokcstr (lexer));
   if (handle == NULL)
     return CMD_CASCADING_FAILURE;
 
@@ -208,14 +208,15 @@ fh_parse (struct lexer *lexer, enum fh_referent referent_mask)
 
       handle = NULL;
       if (lex_token (lexer) == T_ID)
-        handle = fh_from_id (lex_tokid (lexer));
+        handle = fh_from_id (lex_tokcstr (lexer));
       if (handle == NULL)
         {
-          if (lex_token (lexer) != T_ID || lex_tokid (lexer)[0] != '#' || settings_get_syntax () != ENHANCED)
-            handle = fh_create_file (NULL, ds_cstr (lex_tokstr (lexer)),
+          if (lex_token (lexer) != T_ID || lex_tokcstr (lexer)[0] != '#'
+              || settings_get_syntax () != ENHANCED)
+            handle = fh_create_file (NULL, lex_tokcstr (lexer),
                                      fh_default_properties ());
           else
-            handle = fh_create_scratch (lex_tokid (lexer));
+            handle = fh_create_scratch (lex_tokcstr (lexer));
         }
       lex_get (lexer);
     }
diff --git a/src/language/data-io/get-data.c b/src/language/data-io/get-data.c
index eeb3827a..05e115a9 100644
--- a/src/language/data-io/get-data.c
+++ b/src/language/data-io/get-data.c
@@ -61,7 +61,7 @@ cmd_get_data (struct lexer *lexer, struct dataset *ds)
   else if (lex_match_id (lexer, "PSQL"))
     return parse_get_psql (lexer, ds);
 
-  msg (SE, _("Unsupported TYPE %s"), lex_tokid (lexer));
+  msg (SE, _("Unsupported TYPE %s"), lex_tokcstr (lexer));
   return CMD_FAILURE;
 }
 
@@ -85,7 +85,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  psql.conninfo = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  psql.conninfo = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -113,7 +113,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
 	  if ( ! lex_force_string (lexer) )
 	    goto error;
 
-	  ds_put_substring (&psql.sql,  lex_tokstr (lexer)->ss);
+	  ds_put_substring (&psql.sql, lex_tokss (lexer));
 	  lex_get (lexer);
 	}
      }
@@ -153,7 +153,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
   if (!lex_force_string (lexer))
     goto error;
 
-  gri.file_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+  gri.file_name = ss_xstrdup (lex_tokss (lexer));
 
   lex_get (lexer);
 
@@ -172,7 +172,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	      if ( ! lex_force_string (lexer) )
 		goto error;
 
-	      gri.sheet_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+	      gri.sheet_name = ss_xstrdup (lex_tokss (lexer));
 	      gri.sheet_index = -1;
 	    }
 	  else if (lex_match_id (lexer, "INDEX"))
@@ -196,7 +196,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	      if ( ! lex_force_string (lexer) )
 		goto error;
 
-	      gri.cell_range = xstrdup (ds_cstr (lex_tokstr (lexer)));
+	      gri.cell_range = ss_xstrdup (lex_tokss (lexer));
 	    }
 	  else
 	    goto error;
@@ -219,6 +219,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
 	}
       else
 	{
+	  printf ("Unknown data file type `%s'\n", lex_tokcstr (lexer));
 	  goto error;
 	}
       lex_get (lexer);
@@ -415,7 +416,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
           if (!lex_force_string (lexer))
             goto error;
 
-          s = ds_ss (lex_tokstr (lexer));
+          s = lex_tokss (lexer);
           if (ss_match_string (&s, ss_cstr ("\\t")))
             ds_put_cstr (&hard_seps, "\t");
           if (ss_match_string (&s, ss_cstr ("\\\\")))
@@ -441,14 +442,14 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
             goto error;
 
           if (settings_get_syntax () == COMPATIBLE
-              && ds_length (lex_tokstr (lexer)) != 1)
+              && ss_length (lex_tokss (lexer)) != 1)
             {
               msg (SE, _("In compatible syntax mode, the QUALIFIER string "
                          "must contain exactly one character."));
               goto error;
             }
 
-          data_parser_set_quotes (parser, ds_ss (lex_tokstr (lexer)));
+          data_parser_set_quotes (parser, lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (settings_get_syntax () == ENHANCED
@@ -501,7 +502,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
 
       if (!lex_force_id (lexer))
         goto error;
-      strcpy (name, lex_tokid (lexer));
+      strcpy (name, lex_tokcstr (lexer));
       lex_get (lexer);
 
       if (type == DP_DELIMITED)
diff --git a/src/language/data-io/print.c b/src/language/data-io/print.c
index cfbe0439..96ac15e1 100644
--- a/src/language/data-io/print.c
+++ b/src/language/data-io/print.c
@@ -280,7 +280,7 @@ parse_string_argument (struct lexer *lexer, struct print_trns *trns, int record,
   spec->type = PRT_LITERAL;
   spec->record = record;
   spec->first_column = *column;
-  ds_init_string (&spec->string, lex_tokstr (lexer));
+  ds_init_substring (&spec->string, lex_tokss (lexer));
   ds_register_pool (&spec->string, trns->pool);
   lex_get (lexer);
 
diff --git a/src/language/data-io/save-translate.c b/src/language/data-io/save-translate.c
index dcfdb83c..213f0bf7 100644
--- a/src/language/data-io/save-translate.c
+++ b/src/language/data-io/save-translate.c
@@ -159,13 +159,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "DELIMITER");
                       goto error;
                     }
-                  delimiter = ds_first (lex_tokstr (lexer));
+                  delimiter = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "QUALIFIER"))
@@ -173,13 +173,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset *ds)
                   lex_match (lexer, T_EQUALS);
                   if (!lex_force_string (lexer))
                     goto error;
-                  if (ds_length (lex_tokstr (lexer)) != 1)
+                  if (ss_length (lex_tokss (lexer)) != 1)
                     {
                       msg (SE, _("The %s string must contain exactly one "
                                  "character."), "QUALIFIER");
                       goto error;
                     }
-                  qualifier = ds_first (lex_tokstr (lexer));
+                  qualifier = ss_first (lex_tokss (lexer));
                   lex_get (lexer);
                 }
               else if (lex_match_id (lexer, "DECIMAL"))
diff --git a/src/language/data-io/trim.c b/src/language/data-io/trim.c
index be637c40..9f76b105 100644
--- a/src/language/data-io/trim.c
+++ b/src/language/data-io/trim.c
@@ -83,18 +83,18 @@ parse_dict_rename (struct lexer *lexer, struct dictionary *dict)
       if (!lex_force_match (lexer, T_EQUALS)
 	  || !lex_force_id (lexer))
 	return 0;
-      if (dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
 	{
 	  msg (SE, _("Cannot rename %s as %s because there already exists "
 		     "a variable named %s.  To rename variables with "
 		     "overlapping names, use a single RENAME subcommand "
 		     "such as `/RENAME (A=B)(B=C)(C=A)', or equivalently, "
 		     "`/RENAME (A B C=B C A)'."),
-               var_get_name (v), lex_tokid (lexer), lex_tokid (lexer));
+               var_get_name (v), lex_tokcstr (lexer), lex_tokcstr (lexer));
 	  return 0;
 	}
 
-      dict_rename_var (dict, v, lex_tokid (lexer));
+      dict_rename_var (dict, v, lex_tokcstr (lexer));
       lex_get (lexer);
       return 1;
     }
diff --git a/src/language/dictionary/attributes.c b/src/language/dictionary/attributes.c
index 35e69542..c992ff80 100644
--- a/src/language/dictionary/attributes.c
+++ b/src/language/dictionary/attributes.c
@@ -79,7 +79,7 @@ static bool
 match_subcommand (struct lexer *lexer, const char *keyword) 
 {
   if (lex_token (lexer) == T_ID
-      && lex_id_match (ss_cstr (lex_tokid (lexer)), ss_cstr (keyword))
+      && lex_id_match (lex_tokss (lexer), ss_cstr (keyword))
       && lex_look_ahead (lexer) == T_EQUALS)
     {
       lex_get (lexer);          /* Skip keyword. */
@@ -96,7 +96,7 @@ parse_attribute_name (struct lexer *lexer, char name[VAR_NAME_LEN + 1],
 {
   if (!lex_force_id (lexer))
     return false;
-  strcpy (name, lex_tokid (lexer));
+  strcpy (name, lex_tokcstr (lexer));
   lex_get (lexer);
 
   if (lex_match (lexer, T_LBRACK))
@@ -122,14 +122,14 @@ static bool
 add_attribute (struct lexer *lexer, struct attrset **sets, size_t n) 
 {
   char name[VAR_NAME_LEN + 1];
+  const char *value;
   size_t index, i;
-  char *value;
 
   if (!parse_attribute_name (lexer, name, &index)
       || !lex_force_match (lexer, T_LPAREN)
       || !lex_force_string (lexer))
     return false;
-  value = ds_cstr (lex_tokstr (lexer));
+  value = lex_tokcstr (lexer);
 
   for (i = 0; i < n; i++)
     {
diff --git a/src/language/dictionary/missing-values.c b/src/language/dictionary/missing-values.c
index bbc2c84d..4a14ee19 100644
--- a/src/language/dictionary/missing-values.c
+++ b/src/language/dictionary/missing-values.c
@@ -106,7 +106,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       break;
                     }
 
-                  length = ds_length (lex_tokstr (lexer));
+                  length = ss_length (lex_tokss (lexer));
                   if (length > MV_MAX_STRING)
                     {
                       msg (SE, _("Truncating missing value to maximum "
@@ -115,7 +115,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
                       length = MV_MAX_STRING;
                     }
                   memset (value, ' ', MV_MAX_STRING);
-                  memcpy (value, ds_data (lex_tokstr (lexer)), length);
+                  memcpy (value, ss_data (lex_tokss (lexer)), length);
 
                   if (!mv_add_str (&mv, value))
                     deferred_errors = true;
diff --git a/src/language/dictionary/modify-variables.c b/src/language/dictionary/modify-variables.c
index 6afd3215..cc847372 100644
--- a/src/language/dictionary/modify-variables.c
+++ b/src/language/dictionary/modify-variables.c
@@ -300,7 +300,7 @@ cmd_modify_vars (struct lexer *lexer, struct dataset *ds)
       else
 	{
 	  if (lex_token (lexer) == T_ID)
-	    msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokid (lexer));
+	    msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokcstr (lexer));
 	  else
 	    msg (SE, _("Subcommand name expected."));
 	  goto done;
diff --git a/src/language/dictionary/mrsets.c b/src/language/dictionary/mrsets.c
index 0708e89c..c775f497 100644
--- a/src/language/dictionary/mrsets.c
+++ b/src/language/dictionary/mrsets.c
@@ -93,16 +93,16 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_match (lexer, T_EQUALS) || !lex_force_id (lexer))
             goto error;
-          if (lex_tokid (lexer)[0] != '$')
+          if (lex_tokcstr (lexer)[0] != '$')
             {
               msg (SE, _("%s is not a valid name for a multiple response "
                          "set.  Multiple response set names must begin with "
-                         "`$'."), lex_tokid (lexer));
+                         "`$'."), lex_tokcstr (lexer));
               goto error;
             }
 
           free (mrset->name);
-          mrset->name = xstrdup (lex_tokid (lexer));
+          mrset->name = xstrdup (lex_tokcstr (lexer));
           lex_get (lexer);
         }
       else if (lex_match_id (lexer, "VARIABLES"))
@@ -129,7 +129,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             goto error;
 
           free (mrset->label);
-          mrset->label = ds_xstrdup (lex_tokstr (lexer));
+          mrset->label = ss_xstrdup (lex_tokss (lexer));
           lex_get (lexer);
         }
       else if (type == MRSET_MD && lex_match_id (lexer, "LABELSOURCE"))
@@ -159,7 +159,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
             }
           else if (lex_is_string (lexer))
             {
-              const char *s = ds_cstr (lex_tokstr (lexer));
+              const char *s = lex_tokcstr (lexer);
               int width;
 
               /* Trim off trailing spaces, but don't trim the string until
@@ -480,14 +480,14 @@ parse_mrset_names (struct lexer *lexer, struct dictionary *dict,
         {
           if (!lex_force_id (lexer))
             return false;
-          if (dict_lookup_mrset (dict, lex_tokid (lexer)) == NULL)
+          if (dict_lookup_mrset (dict, lex_tokcstr (lexer)) == NULL)
             {
               msg (SE, _("No multiple response set named %s."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
               stringi_set_destroy (mrset_names);
               return false;
             }
-          stringi_set_insert (mrset_names, lex_tokid (lexer));
+          stringi_set_insert (mrset_names, lex_tokcstr (lexer));
           lex_get (lexer);
         }
     }
diff --git a/src/language/dictionary/value-labels.c b/src/language/dictionary/value-labels.c
index 0fdd7a2a..1471fc86 100644
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -142,7 +142,7 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
           return 0;
         }
 
-      ds_init_string (&label, lex_tokstr (lexer));
+      ds_init_substring (&label, lex_tokss (lexer));
 
       if (ds_length (&label) > 60)
 	{
diff --git a/src/language/dictionary/variable-label.c b/src/language/dictionary/variable-label.c
index e6bd6229..804a4213 100644
--- a/src/language/dictionary/variable-label.c
+++ b/src/language/dictionary/variable-label.c
@@ -52,7 +52,7 @@ cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
 	  return CMD_FAILURE;
 	}
 
-      ds_init_string (&label, lex_tokstr (lexer) );
+      ds_init_substring (&label, lex_tokss (lexer));
       if (ds_length (&label) > 255)
 	{
 	  msg (SW, _("Truncating variable label to 255 characters."));
diff --git a/src/language/dictionary/vector.c b/src/language/dictionary/vector.c
index a884331c..3be0723f 100644
--- a/src/language/dictionary/vector.c
+++ b/src/language/dictionary/vector.c
@@ -59,25 +59,25 @@ cmd_vector (struct lexer *lexer, struct dataset *ds)
 	{
           size_t i;
 
-	  if (dict_lookup_vector (dict, lex_tokid (lexer)))
+	  if (dict_lookup_vector (dict, lex_tokcstr (lexer)))
 	    {
 	      msg (SE, _("A vector named %s already exists."),
-                   lex_tokid (lexer));
+                   lex_tokcstr (lexer));
 	      goto fail;
 	    }
 
           for (i = 0; i < vector_cnt; i++)
-            if (!strcasecmp (vectors[i], lex_tokid (lexer)))
+            if (!strcasecmp (vectors[i], lex_tokcstr (lexer)))
 	      {
 		msg (SE, _("Vector name %s is given twice."),
-                     lex_tokid (lexer));
+                     lex_tokcstr (lexer));
 		goto fail;
 	      }
 
           if (vector_cnt == vector_cap)
             vectors = pool_2nrealloc (pool,
                                        vectors, &vector_cap, sizeof *vectors);
-          vectors[vector_cnt++] = pool_strdup (pool, lex_tokid (lexer));
+          vectors[vector_cnt++] = pool_strdup (pool, lex_tokcstr (lexer));
 
 	  lex_get (lexer);
 	  lex_match (lexer, T_COMMA);
diff --git a/src/language/expressions/evaluate.c b/src/language/expressions/evaluate.c
index 9bcfe31e..79b52159 100644
--- a/src/language/expressions/evaluate.c
+++ b/src/language/expressions/evaluate.c
@@ -134,7 +134,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
 
           if (!lex_force_id (lexer))
             goto done;
-          strcpy (name, lex_tokid (lexer));
+          strcpy (name, lex_tokcstr (lexer));
 
           lex_get (lexer);
           if (!lex_force_match (lexer, T_EQUALS))
@@ -143,7 +143,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
           if (lex_is_number (lexer))
             width = 0;
           else if (lex_is_string (lexer))
-            width = ds_length (lex_tokstr (lexer));
+            width = ss_length (lex_tokss (lexer));
           else
             {
               lex_error (lexer, _("expecting number or string"));
diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 79576b44..ae968d00 100644
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -824,7 +824,7 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
     return expr_allocate_number (e, settings_get_viewwidth ());
   else
     {
-      msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
+      msg (SE, _("Unknown system variable %s."), lex_tokcstr (lexer));
       return NULL;
     }
 }
@@ -841,17 +841,17 @@ parse_primary (struct lexer *lexer, struct expression *e)
           /* An identifier followed by a left parenthesis may be
              a vector element reference.  If not, it's a function
              call. */
-          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer)) != NULL)
+          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer)) != NULL)
             return parse_vector_element (lexer, e);
           else
             return parse_function (lexer, e);
         }
-      else if (lex_tokid (lexer)[0] == '$')
+      else if (lex_tokcstr (lexer)[0] == '$')
         {
           /* $ at the beginning indicates a system variable. */
           return parse_sysvar (lexer, e);
         }
-      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokid (lexer)))
+      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokcstr (lexer)))
         {
           /* It looks like a user variable.
              (It could be a format specifier, but we'll assume
@@ -872,7 +872,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
             return expr_allocate_format (e, &fmt);
 
           /* All attempts failed. */
-          msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
+          msg (SE, _("Unknown identifier %s."), lex_tokcstr (lexer));
           return NULL;
         }
       break;
@@ -888,7 +888,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
     case T_STRING:
       {
         union any_node *node = expr_allocate_string_buffer (
-          e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+          e, lex_tokcstr (lexer), ss_length (lex_tokss (lexer)));
 	lex_get (lexer);
 	return node;
       }
@@ -918,7 +918,7 @@ parse_vector_element (struct lexer *lexer, struct expression *e)
   /* Find vector, skip token.
      The caller must already have verified that the current token
      is the name of a vector. */
-  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
+  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer));
   assert (vector != NULL);
   lex_get (lexer);
 
@@ -1209,11 +1209,11 @@ parse_function (struct lexer *lexer, struct expression *e)
 
   union any_node *n;
 
-  ds_init_string (&func_name, lex_tokstr (lexer));
-  min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
-  if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last))
+  ds_init_substring (&func_name, lex_tokss (lexer));
+  min_valid = extract_min_valid (lex_tokcstr (lexer));
+  if (!lookup_function (lex_tokcstr (lexer), &first, &last))
     {
-      msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("No function or vector named %s."), lex_tokcstr (lexer));
       ds_destroy (&func_name);
       return NULL;
     }
diff --git a/src/language/lexer/format-parser.c b/src/language/lexer/format-parser.c
index 1730b493..4b25020e 100644
--- a/src/language/lexer/format-parser.c
+++ b/src/language/lexer/format-parser.c
@@ -46,7 +46,7 @@ parse_abstract_format_specifier__ (struct lexer *lexer,
     goto error;
 
   /* Extract pieces. */
-  s = ds_ss (lex_tokstr (lexer));
+  s = ss_cstr (lex_tokcstr (lexer));
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_LETTERS)), &type_ss);
   ss_get_bytes (&s, ss_span (s, ss_cstr (CC_DIGITS)), &width_ss);
   if (ss_match_byte (&s, '.'))
@@ -133,9 +133,9 @@ parse_format_specifier_name (struct lexer *lexer, enum fmt_type *type)
       lex_error (lexer, _("expecting format type"));
       return false;
     }
-  if (!fmt_from_name (ds_cstr (lex_tokstr (lexer)), type))
+  if (!fmt_from_name (lex_tokcstr (lexer), type))
     {
-      msg (SE, _("Unknown format type `%s'."), ds_cstr (lex_tokstr (lexer)));
+      msg (SE, _("Unknown format type `%s'."), lex_tokcstr (lexer));
       return false;
     }
   lex_get (lexer);
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index c08c428c..4a899bdf 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -49,11 +49,7 @@ struct lexer
   int token;      /* Current token. */
   double tokval;  /* T_POS_NUM, T_NEG_NUM: the token's value. */
 
-  char tokid [VAR_NAME_LEN + 1];   /* T_ID: the identifier. */
-
-  struct string tokstr;   /* T_ID, T_STRING: token string value.
-  			    For T_ID, this is not truncated as is
-  			    tokid. */
+  struct string tokstr;   /* T_ID, T_STRING: token string value. */
 
   char *prog; /* Pointer to next token in line_buffer. */
   bool dot;   /* True only if this line ends with a terminal dot. */
@@ -138,7 +134,6 @@ restore_token (struct lexer *lexer)
   assert (lexer->put_token != 0);
   lexer->token = lexer->put_token;
   ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   lexer->tokval = lexer->put_tokval;
   lexer->put_token = 0;
 }
@@ -405,8 +400,7 @@ lex_get (struct lexer *lexer)
     }
 }
 
-/* Parses an identifier at the current position into tokid and
-   tokstr.
+/* Parses an identifier at the current position into tokstr.
    Returns the correct token type. */
 static int
 parse_id (struct lexer *lexer)
@@ -420,7 +414,6 @@ parse_id (struct lexer *lexer)
   lexer->prog += ss_length (id);
 
   ds_assign_substring (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
   return lex_id_to_token (id);
 }
 
@@ -571,7 +564,7 @@ bool
 lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
 {
   if (lexer->token == T_ID
-      && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+      && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
     {
       lex_get (lexer);
       return true;
@@ -798,7 +791,6 @@ lex_put_back_id (struct lexer *lexer, const char *id)
   save_token (lexer);
   lexer->token = T_ID;
   ds_assign_cstr (&lexer->tokstr, id);
-  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
 }
 
 /* Weird line processing functions. */
@@ -1082,36 +1074,37 @@ lex_token_representation (struct lexer *lexer)
     case T_ID:
     case T_POS_NUM:
     case T_NEG_NUM:
-      return ds_xstrdup (&lexer->tokstr);
+      return ss_xstrdup (lex_tokss (lexer));
 
     case T_STRING:
       {
+        struct substring ss;
 	int hexstring = 0;
 	char *sp, *dp;
 
-	for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+        ss = lex_tokss (lexer);
+	for (sp = ss_data (ss); sp < ss_end (ss); sp++)
 	  if (!c_isprint ((unsigned char) *sp))
 	    {
 	      hexstring = 1;
 	      break;
 	    }
 
-	token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+	token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
 
 	dp = token_rep;
 	if (hexstring)
 	  *dp++ = 'X';
 	*dp++ = '\'';
 
-	if (!hexstring)
-	  for (sp = ds_cstr (&lexer->tokstr); *sp; )
+        for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+          if (!hexstring)
 	    {
 	      if (*sp == '\'')
 		*dp++ = '\'';
-	      *dp++ = (unsigned char) *sp++;
+	      *dp++ = (unsigned char) *sp;
 	    }
-	else
-	  for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+          else
 	    {
 	      *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
 	      *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
@@ -1344,16 +1337,22 @@ lex_tokval (const struct lexer *lexer)
   return lexer->tokval;
 }
 
+/* Returns the null-terminated string value associated with LEXER's current
+   token.  For a T_ID token, this is the identifier, and for a T_STRING token,
+   this is the string.  For other tokens the value is undefined. */
 const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
 {
-  return lexer->tokid;
+  return ds_cstr (&lexer->tokstr);
 }
 
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token.  For a T_ID
+   token, this is the identifier, and for a T_STRING token, this is the string.
+   For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
 {
-  return &lexer->tokstr;
+  return ds_ss (&lexer->tokstr);
 }
 
 /* If the lexer is positioned at the (pseudo)identifier S, which
@@ -1368,7 +1367,7 @@ lex_match_hyphenated_word (struct lexer *lexer, const char *s)
   if (hyphen == NULL)
     return lex_match_id (lexer, s);
   else if (lexer->token != T_ID
-	   || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+	   || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
 	   || lex_look_ahead (lexer) != T_DASH)
     return false;
   else
diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index 82f3fe23..7af40234 100644
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -93,8 +93,8 @@ char *lex_token_representation (struct lexer *);
 /* Token accessors */
 enum token_type lex_token (const struct lexer *);
 double lex_tokval (const struct lexer *);
-const char *lex_tokid (const struct lexer *);
-const struct string *lex_tokstr (const struct lexer *);
+const char *lex_tokcstr (const struct lexer *);
+struct substring lex_tokss (const struct lexer *);
 
 /* Really weird functions. */
 void lex_skip_comment (struct lexer *);
diff --git a/src/language/lexer/q2c.c b/src/language/lexer/q2c.c
index c03b6b21..cf5a04ee 100644
--- a/src/language/lexer/q2c.c
+++ b/src/language/lexer/q2c.c
@@ -1542,7 +1542,7 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
 	      dump (0, "goto lossage;");
 	      dump (-1, "}");
               dump (-1, "free (p->%s%s);", sbc->prefix, st_lower (s->valname));
-              dump (0, "p->%s%s = xstrdup (ds_cstr (lex_tokstr (lexer)));",
+              dump (0, "p->%s%s = ss_xstrdup (ss_tokss (lexer));",
                     sbc->prefix, st_lower (s->valname));
             }
           else
@@ -1705,7 +1705,7 @@ dump_subcommand (const subcommand *sbc)
       outdent ();
       if (sbc->restriction)
 	{
-	  dump (0, "x = ds_length (lex_tokstr (lexer));");
+	  dump (0, "x = ss_length (lex_tokss (lexer));");
 	  dump (1, "if (!(%s))", sbc->restriction);
 	  dump (1, "{");
 	  dump (0, "msg (SE, _(\"String for %s must be %s.\"));",
@@ -1715,7 +1715,7 @@ dump_subcommand (const subcommand *sbc)
 	  outdent ();
 	}
       dump (0, "free(p->s_%s);", st_lower(sbc->name) );
-      dump (0, "p->s_%s = ds_xstrdup (lex_tokstr (lexer));",
+      dump (0, "p->s_%s = ss_xstrdup (lex_tokss (lexer));",
 	    st_lower (sbc->name));
       dump (0, "lex_get (lexer);");
       if (sbc->restriction)
@@ -1833,13 +1833,13 @@ dump_parser (int persistent)
     {
       if (def->type == SBC_VARLIST)
 	dump (1, "if (lex_token (lexer) == T_ID "
-              "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL "
-	      "&& lex_look_ahead (lexer) != T_EQUALS)");
+              "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL "
+	      "&& lex_look_ahead (lexer) != '=')");
       else
 	{
 	  dump (0, "if ((lex_token (lexer) == T_ID "
-                "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) "
-		"&& lex_look_ahead () != T_EQUALS)");
+                "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) "
+		"&& lex_look_ahead () != '=')");
 	  dump (1, "     || token == T_ALL)");
 	}
       dump (1, "{");
diff --git a/src/language/lexer/value-parser.c b/src/language/lexer/value-parser.c
index 269d24bc..4ef092de 100644
--- a/src/language/lexer/value-parser.c
+++ b/src/language/lexer/value-parser.c
@@ -106,7 +106,7 @@ parse_number (struct lexer *lexer, double *x, const enum fmt_type *format)
 
       assert (fmt_get_category (*format) != FMT_CAT_STRING);
 
-      if (!data_in_msg (ds_ss (lex_tokstr (lexer)), LEGACY_NATIVE,
+      if (!data_in_msg (lex_tokss (lexer), LEGACY_NATIVE,
                         *format, &v, 0, NULL))
         return false;
 
@@ -143,7 +143,12 @@ parse_value (struct lexer *lexer, union value *v, int width)
     }
   else if (lex_force_string (lexer))
     {
-      const char *s = ds_cstr (lex_tokstr (lexer));
+      const char *s;
+
+      if (!lex_force_string (lexer))
+	return false;
+
+      s = lex_tokcstr (lexer);
       value_copy_str_rpad (v, width, CHAR_CAST_BUG (const uint8_t *, s), ' ');
     }
   else
diff --git a/src/language/lexer/variable-parser.c b/src/language/lexer/variable-parser.c
index b05b7ce7..5b698bbf 100644
--- a/src/language/lexer/variable-parser.c
+++ b/src/language/lexer/variable-parser.c
@@ -65,14 +65,14 @@ parse_vs_variable_idx (struct lexer *lexer, const struct var_set *vs,
       lex_error (lexer, _("expecting variable name"));
       return false;
     }
-  else if (var_set_lookup_var_idx (vs, lex_tokid (lexer), idx))
+  else if (var_set_lookup_var_idx (vs, lex_tokcstr (lexer), idx))
     {
       lex_get (lexer);
       return true;
     }
   else
     {
-      msg (SE, _("%s is not a variable name."), lex_tokid (lexer));
+      msg (SE, _("%s is not a variable name."), lex_tokcstr (lexer));
       return false;
     }
 }
@@ -339,7 +339,7 @@ parse_var_set_vars (struct lexer *lexer, const struct var_set *vs,
       lex_match (lexer, T_COMMA);
     }
   while (lex_token (lexer) == T_ALL
-         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokid (lexer)) != NULL));
+         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokcstr (lexer)) != NULL));
 
   if (*nv == 0)
     goto fail;
@@ -444,13 +444,13 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
 	  lex_error (lexer, "expecting variable name");
 	  goto fail;
 	}
-      if (dict_class_from_id (lex_tokid (lexer)) == DC_SCRATCH
+      if (dict_class_from_id (lex_tokcstr (lexer)) == DC_SCRATCH
           && (pv_opts & PV_NO_SCRATCH))
 	{
 	  msg (SE, _("Scratch variables not allowed here."));
 	  goto fail;
 	}
-      strcpy (name1, lex_tokid (lexer));
+      strcpy (name1, lex_tokcstr (lexer));
       lex_get (lexer);
       if (lex_token (lexer) == T_TO)
 	{
@@ -460,7 +460,7 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
 	      lex_error (lexer, "expecting variable name");
 	      goto fail;
 	    }
-	  strcpy (name2, lex_tokid (lexer));
+	  strcpy (name2, lex_tokcstr (lexer));
 	  lex_get (lexer);
 
 	  if (!extract_num (name1, root1, &n1, &d1)
@@ -588,7 +588,7 @@ parse_mixed_vars (struct lexer *lexer, const struct dictionary *dict,
     }
   while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
     {
-      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
 	{
 	  struct variable **v;
 	  size_t nv;
diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c
index bbe444d5..cfa9ea66 100644
--- a/src/language/stats/aggregate.c
+++ b/src/language/stats/aggregate.c
@@ -435,7 +435,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 	  if (lex_is_string (lexer))
 	    {
 	      struct string label;
-	      ds_init_string (&label, lex_tokstr (lexer));
+	      ds_init_substring (&label, lex_tokss (lexer));
 
 	      ds_truncate (&label, 255);
 	      dest_label[n_dest - 1] = ds_xstrdup (&label);
@@ -451,7 +451,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 	  goto error;
 	}
 
-      ds_assign_string (&function_name, lex_tokstr (lexer));
+      ds_assign_substring (&function_name, lex_tokss (lexer));
       exclude = ds_chomp (&function_name, '.') ? MV_SYSTEM : MV_ANY;
 
       for (function = agr_func_tab; function->name; function++)
@@ -501,7 +501,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict,
 		lex_match (lexer, T_COMMA);
 		if (lex_is_string (lexer))
 		  {
-		    arg[i].c = ds_xstrdup (lex_tokstr (lexer));
+		    arg[i].c = ss_xstrdup (lex_tokss (lexer));
 		    type = VAL_STRING;
 		  }
 		else if (lex_is_number (lexer))
diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q
index 0e7b3dae..02cd4deb 100644
--- a/src/language/stats/crosstabs.q
+++ b/src/language/stats/crosstabs.q
@@ -381,7 +381,7 @@ crs_custom_tables (struct lexer *lexer, struct dataset *ds,
   /* Ensure that this is a TABLES subcommand. */
   if (!lex_match_id (lexer, "TABLES")
       && (lex_token (lexer) != T_ID ||
-	  dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+	  dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
   lex_match (lexer, T_EQUALS);
diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c
index 17eb4450..ae472294 100644
--- a/src/language/stats/descriptives.c
+++ b/src/language/stats/descriptives.c
@@ -334,14 +334,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds)
                       lex_error (lexer, NULL);
                       goto error;
                     }
-                  if (try_name (dict, dsc, lex_tokid (lexer)))
+                  if (try_name (dict, dsc, lex_tokcstr (lexer)))
                     {
-                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer));
+                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokcstr (lexer));
                       z_cnt++;
                     }
                   else
                     msg (SE, _("Z-score variable name %s would be"
-                               " a duplicate variable name."), lex_tokid (lexer));
+                               " a duplicate variable name."), lex_tokcstr (lexer));
                   lex_get (lexer);
                   if (!lex_force_match (lexer, T_RPAREN))
 		    goto error;
diff --git a/src/language/stats/examine.q b/src/language/stats/examine.q
index db11ef7f..aa578e1e 100644
--- a/src/language/stats/examine.q
+++ b/src/language/stats/examine.q
@@ -675,7 +675,7 @@ xmn_custom_variables (struct lexer *lexer, struct dataset *ds,
   const struct dictionary *dict = dataset_dict (ds);
   lex_match (lexer, T_EQUALS);
 
-  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+  if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       return 2;
@@ -719,7 +719,7 @@ examine_parse_independent_vars (struct lexer *lexer,
   ll_init (&sf->result_list);
 
   if ( (lex_token (lexer) != T_ID ||
-	dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+	dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
        && lex_token (lexer) != T_ALL)
     {
       free ( sf ) ;
@@ -734,7 +734,7 @@ examine_parse_independent_vars (struct lexer *lexer,
       lex_match (lexer, T_BY);
 
       if ( (lex_token (lexer) != T_ID ||
-	    dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+	    dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
 	   && lex_token (lexer) != T_ALL)
 	{
 	  free (sf);
diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index e397eb87..ecaefdf6 100644
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -621,7 +621,7 @@ frq_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
   if (lex_token (lexer) != T_ALL
       && (lex_token (lexer) != T_ID
-          || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+          || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
     return 2;
 
   /* Get list of current variables, to avoid duplicates. */
@@ -663,7 +663,8 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
   struct frq_proc *frq = frq_;
 
   lex_match (lexer, T_EQUALS);
-  if ((lex_token (lexer) == T_ID && dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+  if ((lex_token (lexer) == T_ID
+       && dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
       || lex_token (lexer) == T_ID)
     for (;;)
       {
@@ -739,7 +740,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
 	free (v);
 	if (!lex_match (lexer, T_SLASH))
 	  break;
-	if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+	if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
             && lex_token (lexer) != T_ALL)
 	  {
 	    lex_put_back (lexer, T_SLASH);
diff --git a/src/language/stats/npar.c b/src/language/stats/npar.c
index b3a24dce..7dd94f78 100644
--- a/src/language/stats/npar.c
+++ b/src/language/stats/npar.c
@@ -277,7 +277,7 @@ parse_npar_tests (struct lexer *lexer, struct dataset *ds, struct cmd_npar_tests
       else if (lex_match_hyphenated_word (lexer, "M-W") ||
 	       lex_match_hyphenated_word (lexer, "MANN-WHITNEY"))
         {
-          lex_match (lexer, '=');
+          lex_match (lexer, T_EQUALS);
           npt->mann_whitney++;
           switch (npar_mann_whitney (lexer, ds, nps))
             {
@@ -574,7 +574,7 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
   nt->execute = runs_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  if ( lex_force_match (lexer, '(') )
+  if ( lex_force_match (lexer, T_LPAREN) )
     {
       if ( lex_match_id (lexer, "MEAN"))
 	{
@@ -600,8 +600,8 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
 	  return 0;
 	}
 		  
-      lex_force_match (lexer, ')');
-      lex_force_match (lexer, '=');
+      lex_force_match (lexer, T_RPAREN);
+      lex_force_match (lexer, T_EQUALS);
       if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				  &tp->vars, &tp->n_vars,
 				  PV_NO_SCRATCH | PV_NO_DUPLICATE | PV_NUMERIC))
@@ -632,7 +632,7 @@ npar_friedman (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ost->vars, &ost->n_vars,
@@ -663,7 +663,7 @@ npar_kendall (struct lexer *lexer, struct dataset *ds,
   nt->execute = friedman_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ost->vars, &ost->n_vars,
@@ -693,7 +693,7 @@ npar_cochran (struct lexer *lexer, struct dataset *ds,
   nt->execute = cochran_execute;
   nt->insert_variables = one_sample_insert_variables;
 
-  lex_match (lexer, '=');
+  lex_match (lexer, T_EQUALS);
 
   if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
 				   &ft->vars, &ft->n_vars,
@@ -772,7 +772,7 @@ npar_chisquare (struct lexer *lexer, struct dataset *ds,
 		  n = 1;
 		  f = lex_number (lexer);
 		  lex_get (lexer);
-		  if ( lex_match (lexer, '*'))
+		  if ( lex_match (lexer, T_ASTERISK))
 		    {
 		      n = f;
 		      f = lex_number (lexer);
diff --git a/src/language/stats/rank.q b/src/language/stats/rank.q
index 386ef03a..f53ef2d0 100644
--- a/src/language/stats/rank.q
+++ b/src/language/stats/rank.q
@@ -798,7 +798,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 {
   lex_match (lexer, T_EQUALS);
 
-  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
       return 2;
 
@@ -808,7 +808,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank
 
   if ( lex_match (lexer, T_BY)  )
     {
-      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
 	{
 	  return 2;
 	}
@@ -847,9 +847,9 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
       while( lex_token (lexer) == T_ID )
 	{
 
-	  if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
+	  if ( dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL )
 	    {
-	      msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
+	      msg(SE, _("Variable %s already exists."), lex_tokcstr (lexer));
 	      return 0;
 	    }
 	  if ( var_count >= subcase_get_n_fields (&sc) )
@@ -858,7 +858,7 @@ parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_ra
 	      return 0;
 	    }
 
-	  destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokid (lexer));
+	  destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokcstr (lexer));
 	  rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
 
 	  lex_get (lexer);
diff --git a/src/language/stats/regression.q b/src/language/stats/regression.q
index 83accc9e..e5b86ea2 100644
--- a/src/language/stats/regression.q
+++ b/src/language/stats/regression.q
@@ -755,7 +755,7 @@ regression_custom_variables (struct lexer *lexer, struct dataset *ds,
   lex_match (lexer, T_EQUALS);
 
   if ((lex_token (lexer) != T_ID
-       || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+       || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
       && lex_token (lexer) != T_ALL)
     return 2;
 
diff --git a/src/language/stats/reliability.c b/src/language/stats/reliability.c
index c6331970..3633b3dd 100644
--- a/src/language/stats/reliability.c
+++ b/src/language/stats/reliability.c
@@ -179,7 +179,7 @@ cmd_reliability (struct lexer *lexer, struct dataset *ds)
 	  if ( ! lex_force_string (lexer) ) 
 	    goto error;
 
-	  ds_init_string (&reliability.scale_name, lex_tokstr (lexer));
+	  ds_init_substring (&reliability.scale_name, lex_tokss (lexer));
 
 	  lex_get (lexer);
 
diff --git a/src/language/stats/sort-criteria.c b/src/language/stats/sort-criteria.c
index d808d1c2..64976023 100644
--- a/src/language/stats/sort-criteria.c
+++ b/src/language/stats/sort-criteria.c
@@ -94,7 +94,7 @@ parse_sort_criteria (struct lexer *lexer, const struct dictionary *dict,
         }
     }
   while (lex_token (lexer) == T_ID
-         && dict_lookup_var (dict, lex_tokid (lexer)) != NULL);
+         && dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL);
 
   free (local_vars);
   return true;
diff --git a/src/language/tests/float-format.c b/src/language/tests/float-format.c
index 91e4d9ec..554b2c33 100644
--- a/src/language/tests/float-format.c
+++ b/src/language/tests/float-format.c
@@ -116,7 +116,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
           || !lex_force_string (lexer))
         return false;
 
-      length = ds_length (lex_tokstr (lexer));
+      length = ss_length (lex_tokss (lexer));
       if (fp->format != FLOAT_HEX)
         {
           if (length != float_get_size (fp->format))
@@ -127,7 +127,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               return false;
             }
           assert (length <= sizeof fp->data);
-          memcpy (fp->data, ds_data (lex_tokstr (lexer)), length);
+          memcpy (fp->data, ss_data (lex_tokss (lexer)), length);
         }
       else
         {
@@ -136,7 +136,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
               msg (SE, _("Hexadecimal floating constant too long."));
               return false;
             }
-          strncpy (CHAR_CAST_BUG (char *,fp->data), ds_cstr (lex_tokstr (lexer)), sizeof fp->data);
+          strncpy (CHAR_CAST_BUG (char *,fp->data), lex_tokcstr (lexer), sizeof fp->data);
         }
 
       lex_get (lexer);
diff --git a/src/language/tests/format-guesser-test.c b/src/language/tests/format-guesser-test.c
index 6285fc04..c95ea35f 100644
--- a/src/language/tests/format-guesser-test.c
+++ b/src/language/tests/format-guesser-test.c
@@ -35,8 +35,8 @@ cmd_debug_format_guesser (struct lexer *lexer, struct dataset *ds UNUSED)
   g = fmt_guesser_create ();
   while (lex_is_string (lexer))
     {
-      fprintf (stderr, "\"%s\" ", ds_cstr (lex_tokstr (lexer)));
-      fmt_guesser_add (g, ds_ss (lex_tokstr (lexer)));
+      fprintf (stderr, "\"%s\" ", lex_tokcstr (lexer));
+      fmt_guesser_add (g, lex_tokss (lexer));
       lex_get (lexer);
     }
 
diff --git a/src/language/tests/paper-size.c b/src/language/tests/paper-size.c
index 5aea2c33..4dbd9460 100644
--- a/src/language/tests/paper-size.c
+++ b/src/language/tests/paper-size.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2007 Free Software Foundation, Inc.
+   Copyright (C) 2007, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_debug_paper_size (struct lexer *lexer, struct dataset *ds UNUSED)
 
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
-  paper_size = ds_cstr (lex_tokstr (lexer));
+  paper_size = lex_tokcstr (lexer);
 
   printf ("\"%s\" => ", paper_size);
   if (measure_paper (paper_size, &h, &v))
diff --git a/src/language/utilities/cd.c b/src/language/utilities/cd.c
index 38211e76..220164f0 100644
--- a/src/language/utilities/cd.c
+++ b/src/language/utilities/cd.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 2008 Free Software Foundation, Inc.
+   Copyright (C) 2008, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_cd (struct lexer *lexer, struct dataset *ds UNUSED)
   if ( ! lex_force_string (lexer))
     goto error;
 
-  path = ds_xstrdup (lex_tokstr (lexer));
+  path = ss_xstrdup (lex_tokss (lexer));
 
   if ( -1 == chdir (path) )
     {
diff --git a/src/language/utilities/echo.c b/src/language/utilities/echo.c
index b27b4001..149cc0eb 100644
--- a/src/language/utilities/echo.c
+++ b/src/language/utilities/echo.c
@@ -34,7 +34,7 @@ cmd_echo (struct lexer *lexer, struct dataset *ds UNUSED)
 
   tab = tab_create(1, 1);
 
-  tab_text(tab, 0, 0, 0, ds_cstr (lex_tokstr (lexer)));
+  tab_text (tab, 0, 0, 0, lex_tokcstr (lexer));
 
   tab_submit(tab);
 
diff --git a/src/language/utilities/host.c b/src/language/utilities/host.c
index 86109574..f9b10480 100644
--- a/src/language/utilities/host.c
+++ b/src/language/utilities/host.c
@@ -145,7 +145,7 @@ cmd_host (struct lexer *lexer, struct dataset *ds UNUSED)
         {
           if (!ds_is_empty (&command))
             ds_put_byte (&command, '\n');
-          ds_put_substring (&command, ds_ss (lex_tokstr (lexer)));
+          ds_put_substring (&command, lex_tokss (lexer));
           lex_get (lexer);
         }
       if (!lex_force_match (lexer, T_RBRACK))
diff --git a/src/language/utilities/include.c b/src/language/utilities/include.c
index d259855e..f152684d 100644
--- a/src/language/utilities/include.c
+++ b/src/language/utilities/include.c
@@ -170,7 +170,7 @@ cmd_insert (struct lexer *lexer, struct dataset *ds UNUSED)
 static int
 parse_insert (struct lexer *lexer, char **filename)
 {
-  char *target_fn;
+  const char *target_fn;
   char *relative_filename;
 
   /* Skip optional FILE=. */
@@ -184,7 +184,7 @@ parse_insert (struct lexer *lexer, char **filename)
       return CMD_FAILURE;
     }
 
-  target_fn = ds_cstr (lex_tokstr (lexer));
+  target_fn = lex_tokcstr (lexer);
 
   relative_filename =
     fn_search_path (target_fn,
diff --git a/src/language/utilities/permissions.c b/src/language/utilities/permissions.c
index 34fa6fd8..ad9ffe75 100644
--- a/src/language/utilities/permissions.c
+++ b/src/language/utilities/permissions.c
@@ -50,7 +50,7 @@ cmd_permissions (struct lexer *lexer, struct dataset *ds UNUSED)
   if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  fn = ds_xstrdup (lex_tokstr (lexer));
+  fn = ss_xstrdup (lex_tokss (lexer));
   lex_force_match (lexer, T_STRING);
 
 
diff --git a/src/language/utilities/set.q b/src/language/utilities/set.q
index 3883f096..f109e450 100644
--- a/src/language/utilities/set.q
+++ b/src/language/utilities/set.q
@@ -383,27 +383,26 @@ static int
 stc_custom_locale (struct lexer *lexer, struct dataset *ds UNUSED,
 		   struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  const struct string *s;
+  const char *s;
 
   lex_match (lexer, T_EQUALS);
 
   if ( !lex_force_string (lexer))
     return 0;
 
-  s = lex_tokstr (lexer);
+  s = lex_tokcstr (lexer);
 
   /* First try this string as an encoding name */
-  if ( valid_encoding (ds_cstr (s)))
-    set_default_encoding (ds_cstr (s));
+  if ( valid_encoding (s))
+    set_default_encoding (s);
 
   /* Now try as a locale name (or alias) */
-  else if (set_encoding_from_locale (ds_cstr (s)))
+  else if (set_encoding_from_locale (s))
     {
     }
   else
     {
-      msg (ME, _("%s is not a recognized encoding or locale name"),
-	   ds_cstr (s));
+      msg (ME, _("%s is not a recognized encoding or locale name"), s);
       return 0;
     }
 
@@ -511,7 +510,7 @@ stc_custom_journal (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_s
     journal_disable ();
   else if (lex_is_string (lexer) || lex_token (lexer) == T_ID)
     {
-      journal_set_file_name (ds_cstr (lex_tokstr (lexer)));
+      journal_set_file_name (lex_tokcstr (lexer));
       lex_get (lexer);
     }
   else
diff --git a/src/language/utilities/title.c b/src/language/utilities/title.c
index a3d89b3c..82f7be50 100644
--- a/src/language/utilities/title.c
+++ b/src/language/utilities/title.c
@@ -57,7 +57,7 @@ parse_title (struct lexer *lexer, enum text_item_type type)
       lex_get (lexer);
       if (!lex_force_string (lexer))
 	return CMD_FAILURE;
-      set_title (ds_cstr (lex_tokstr (lexer)), type);
+      set_title (lex_tokcstr (lexer), type);
       lex_get (lexer);
       return lex_end_of_command (lexer);
     }
@@ -149,7 +149,7 @@ cmd_add_documents (struct lexer *lexer, struct dataset *ds)
 
   while ( lex_is_string (lexer))
     {
-      dict_add_document_line (dict, ds_cstr (lex_tokstr (lexer)));
+      dict_add_document_line (dict, lex_tokcstr (lexer));
       lex_get (lexer);
     }
 
diff --git a/src/language/xforms/compute.c b/src/language/xforms/compute.c
index 8245ffed..442c15a4 100644
--- a/src/language/xforms/compute.c
+++ b/src/language/xforms/compute.c
@@ -349,10 +349,10 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   if (lex_look_ahead (lexer) == T_LPAREN)
     {
       /* Vector. */
-      lvalue->vector = dict_lookup_vector (dict, lex_tokid (lexer));
+      lvalue->vector = dict_lookup_vector (dict, lex_tokcstr (lexer));
       if (lvalue->vector == NULL)
 	{
-	  msg (SE, _("There is no vector named %s."), lex_tokid (lexer));
+	  msg (SE, _("There is no vector named %s."), lex_tokcstr (lexer));
           goto lossage;
 	}
 
@@ -369,7 +369,7 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
   else
     {
       /* Variable name. */
-      const char *var_name = lex_tokid (lexer);
+      const char *var_name = lex_tokcstr (lexer);
       lvalue->variable = dict_lookup_var (dict, var_name);
       if (lvalue->variable == NULL)
         {
diff --git a/src/language/xforms/count.c b/src/language/xforms/count.c
index 6d055f83..550db8a5 100644
--- a/src/language/xforms/count.c
+++ b/src/language/xforms/count.c
@@ -114,7 +114,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
       /* Get destination variable, or at least its name. */
       if (!lex_force_id (lexer))
 	goto fail;
-      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
       if (dv->var != NULL)
         {
           if (var_is_alpha (dv->var))
@@ -124,7 +124,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
             }
         }
       else
-        dv->name = pool_strdup (trns->pool, lex_tokid (lexer));
+        dv->name = pool_strdup (trns->pool, lex_tokcstr (lexer));
 
       lex_get (lexer);
       if (!lex_force_match (lexer, T_EQUALS))
@@ -253,7 +253,7 @@ parse_string_criteria (struct lexer *lexer, struct pool *pool, struct criteria *
 	return false;
       cur = &crit->values.str[crit->value_cnt++];
       *cur = pool_alloc (pool, len + 1);
-      str_copy_rpad (*cur, len + 1, ds_cstr (lex_tokstr (lexer)));
+      str_copy_rpad (*cur, len + 1, lex_tokcstr (lexer));
       lex_get (lexer);
 
       lex_match (lexer, T_COMMA);
diff --git a/src/language/xforms/recode.c b/src/language/xforms/recode.c
index 12f1eb43..236b4983 100644
--- a/src/language/xforms/recode.c
+++ b/src/language/xforms/recode.c
@@ -116,12 +116,12 @@ static bool parse_map_in (struct lexer *lexer, struct map_in *, struct pool *,
 static void set_map_in_generic (struct map_in *, enum map_in_type);
 static void set_map_in_num (struct map_in *, enum map_in_type, double, double);
 static void set_map_in_str (struct map_in *, struct pool *,
-                            const struct string *, size_t width);
+                            struct substring, size_t width);
 
 static bool parse_map_out (struct lexer *lexer, struct pool *, struct map_out *);
 static void set_map_out_num (struct map_out *, double);
 static void set_map_out_str (struct map_out *, struct pool *,
-                             const struct string *);
+                             struct substring);
 
 static void enlarge_dst_widths (struct recode_trns *);
 static void create_dst_vars (struct recode_trns *, struct dictionary *);
@@ -318,10 +318,10 @@ parse_map_in (struct lexer *lexer, struct map_in *in, struct pool *pool,
         return false;
       else 
 	{
-	  set_map_in_str (in, pool, lex_tokstr (lexer), max_src_width);
+	  set_map_in_str (in, pool, lex_tokss (lexer), max_src_width);
 	  lex_get (lexer);
 	  if (lex_token (lexer) == T_ID
-	      && lex_id_match (ss_cstr ("THRU"), ss_cstr (lex_tokid (lexer))))
+	      && lex_id_match (ss_cstr ("THRU"), lex_tokss (lexer)))
 	    {
 	      msg (SE, _("THRU is not allowed with string variables."));
 	      return false;
@@ -370,13 +370,13 @@ set_map_in_num (struct map_in *in, enum map_in_type type, double x, double y)
    right to WIDTH characters long. */
 static void
 set_map_in_str (struct map_in *in, struct pool *pool,
-                const struct string *string, size_t width)
+                struct substring string, size_t width)
 {
   in->type = MAP_SINGLE;
   value_init_pool (pool, &in->x, width);
   value_copy_buf_rpad (&in->x, width,
-                       CHAR_CAST_BUG (uint8_t *, ds_data (string)),
-                       ds_length (string), ' ');
+                       CHAR_CAST_BUG (uint8_t *, ss_data (string)),
+                       ss_length (string), ' ');
 }
 
 /* Parses a mapping output value into OUT, allocating memory from
@@ -393,7 +393,7 @@ parse_map_out (struct lexer *lexer, struct pool *pool, struct map_out *out)
     set_map_out_num (out, SYSMIS);
   else if (lex_is_string (lexer))
     {
-      set_map_out_str (out, pool, lex_tokstr (lexer));
+      set_map_out_str (out, pool, lex_tokss (lexer));
       lex_get (lexer);
     }
   else if (lex_match_id (lexer, "COPY")) 
@@ -421,10 +421,10 @@ set_map_out_num (struct map_out *out, double value)
 /* Sets OUT as a string mapping output with the given VALUE. */
 static void
 set_map_out_str (struct map_out *out, struct pool *pool,
-                 const struct string *value)
+                 const struct substring value)
 {
-  const char *string = ds_data (value);
-  size_t length = ds_length (value);
+  const char *string = ss_data (value);
+  size_t length = ss_length (value);
 
   if (length == 0)
     {