From: John Darrington <john@darrington.wattle.id.au>
Date: Sat, 11 Nov 2006 23:10:00 +0000 (+0000)
Subject: Encapsulated lexer and updated calling functions accordingly.
X-Git-Tag: v0.6.0~697
X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3816248a008a4af75aac6319d0c9929cb7ff679e;p=pspp-builds.git

Encapsulated lexer and updated calling functions accordingly.
---

diff --git a/src/language/command.c b/src/language/command.c
index f98277aa..cfae2ba0 100644
--- a/src/language/command.c
+++ b/src/language/command.c
@@ -108,7 +108,7 @@ struct command
     enum states states;         /* States in which command is allowed. */
     enum flags flags;           /* Other command requirements. */
     const char *name;		/* Command name. */
-    int (*function) (struct dataset *);	/* Function to call. */
+    int (*function) (struct lexer *, struct dataset *);	/* Function to call. */
   };
 
 /* Define the command array. */
@@ -130,22 +130,22 @@ static void set_completion_state (enum cmd_state);
 
 /* Command parser. */
 
-static const struct command *parse_command_name (void);
-static enum cmd_result do_parse_command (struct dataset *ds, enum cmd_state);
+static const struct command *parse_command_name (struct lexer *lexer);
+static enum cmd_result do_parse_command (struct lexer *, struct dataset *, enum cmd_state);
 
 /* Parses an entire command, from command name to terminating
    dot.  On failure, skips to the terminating dot.
    Returns the command's success or failure result. */
 enum cmd_result
-cmd_parse (struct dataset *ds, enum cmd_state state) 
+cmd_parse (struct lexer *lexer, struct dataset *ds, enum cmd_state state) 
 {
   int result;
   
   som_new_series ();
 
-  result = do_parse_command (ds, state);
+  result = do_parse_command (lexer, ds, state);
   if (cmd_result_is_failure (result)) 
-    lex_discard_rest_of_command ();
+    lex_discard_rest_of_command (lexer);
 
   unset_cmd_algorithm ();
   dict_clear_aux (dataset_dict (ds));
@@ -156,7 +156,7 @@ cmd_parse (struct dataset *ds, enum cmd_state state)
 /* Parses an entire command, from command name to terminating
    dot. */
 static enum cmd_result
-do_parse_command (struct dataset *ds, enum cmd_state state)
+do_parse_command (struct lexer *lexer, struct dataset *ds, enum cmd_state state)
 {
   const struct command *command;
   enum cmd_result result;
@@ -164,10 +164,10 @@ do_parse_command (struct dataset *ds, enum cmd_state state)
   /* Read the command's first token. */
   getl_set_prompt_style (GETL_PROMPT_FIRST);
   set_completion_state (state);
-  lex_get ();
-  if (token == T_STOP)
+  lex_get (lexer);
+  if (lex_token (lexer) == T_STOP)
     return CMD_EOF;
-  else if (token == '.') 
+  else if (lex_token (lexer) == '.') 
     {
       /* Null commands can result from extra empty lines. */
       return CMD_SUCCESS; 
@@ -175,7 +175,7 @@ do_parse_command (struct dataset *ds, enum cmd_state state)
   getl_set_prompt_style (GETL_PROMPT_LATER);
 
   /* Parse the command name. */
-  command = parse_command_name ();
+  command = parse_command_name (lexer);
   if (command == NULL)
     return CMD_FAILURE;
   else if (command->function == NULL) 
@@ -203,7 +203,7 @@ do_parse_command (struct dataset *ds, enum cmd_state state)
   /* Execute command. */
   msg_set_command_name (command->name);
   tab_set_command_name (command->name);
-  result = command->function (ds);
+  result = command->function (lexer, ds);
   tab_set_command_name (NULL);
   msg_set_command_name (NULL);
     
@@ -441,10 +441,10 @@ free_words (char *words[], size_t word_cnt)
 /* Flags an error that the command whose name is given by the
    WORD_CNT words in WORDS is unknown. */
 static void
-unknown_command_error (char *const words[], size_t word_cnt) 
+unknown_command_error (struct lexer *lexer, char *const words[], size_t word_cnt) 
 {
   if (word_cnt == 0) 
-    lex_error (_("expecting command name"));
+    lex_error (lexer, _("expecting command name"));
   else 
     {
       struct string s;
@@ -468,29 +468,30 @@ unknown_command_error (char *const words[], size_t word_cnt)
    struct command if successful.
    If not successful, return a null pointer. */
 static const struct command *
-parse_command_name (void)
+parse_command_name (struct lexer *lexer)
 {
   char *words[16];
   int word_cnt;
   int complete_word_cnt;
   int dash_possible;
 
-  if (token == T_EXP || token == '*' || token == '[') 
+  if (lex_token (lexer) == T_EXP || 
+		  lex_token (lexer) == '*' || lex_token (lexer) == '[') 
     return find_command ("COMMENT");
 
   dash_possible = 0;
   word_cnt = complete_word_cnt = 0;
-  while (token == T_ID || (dash_possible && token == '-')) 
+  while (lex_token (lexer) == T_ID || (dash_possible && lex_token (lexer) == '-')) 
     {
       int cmd_match_cnt;
       
       assert (word_cnt < sizeof words / sizeof *words);
-      if (token == T_ID) 
+      if (lex_token (lexer) == T_ID) 
         {
-          words[word_cnt] = ds_xstrdup (&tokstr);
+          words[word_cnt] = ds_xstrdup (lex_tokstr (lexer));
           str_uppercase (words[word_cnt]); 
         }
-      else if (token == '-')
+      else if (lex_token (lexer) == '-')
         words[word_cnt] = xstrdup ("-");
       word_cnt++;
 
@@ -504,7 +505,7 @@ parse_command_name (void)
           if (command != NULL) 
             {
               if (!(command->flags & F_KEEP_FINAL_TOKEN))
-                lex_get ();
+                lex_get (lexer);
               free_words (words, word_cnt);
               return command;
             }
@@ -515,7 +516,7 @@ parse_command_name (void)
           if (get_complete_match (words, word_cnt) != NULL)
             complete_word_cnt = word_cnt;
         }
-      lex_get ();
+      lex_get (lexer);
     }
 
   /* If we saw a complete command name earlier, drop back to
@@ -542,9 +543,9 @@ parse_command_name (void)
         {
           word_cnt--;
           if (strcmp (words[word_cnt], "-")) 
-            lex_put_back_id (words[word_cnt]);
+            lex_put_back_id (lexer, words[word_cnt]);
           else
-            lex_put_back ('-');
+            lex_put_back (lexer, '-');
           free (words[word_cnt]);
         }
 
@@ -553,7 +554,7 @@ parse_command_name (void)
     }
 
   /* We didn't get a valid command name. */
-  unknown_command_error (words, word_cnt);
+  unknown_command_error (lexer, words, word_cnt);
   free_words (words, word_cnt);
   return NULL;
 }
@@ -649,40 +650,40 @@ cmd_complete (const char *prefix, const struct command **cmd)
 
 /* Parse and execute FINISH command. */
 int
-cmd_finish (struct dataset *ds UNUSED)
+cmd_finish (struct lexer *lexer UNUSED, struct dataset *ds UNUSED)
 {
   return CMD_FINISH;
 }
 
 /* Parses the N command. */
 int
-cmd_n_of_cases (struct dataset *ds)
+cmd_n_of_cases (struct lexer *lexer, struct dataset *ds)
 {
   /* Value for N. */
   int x;
 
-  if (!lex_force_int ())
+  if (!lex_force_int (lexer))
     return CMD_FAILURE;
-  x = lex_integer ();
-  lex_get ();
-  if (!lex_match_id ("ESTIMATED"))
+  x = lex_integer (lexer);
+  lex_get (lexer);
+  if (!lex_match_id (lexer, "ESTIMATED"))
     dict_set_case_limit (dataset_dict (ds), x);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Parses, performs the EXECUTE procedure. */
 int
-cmd_execute (struct dataset *ds)
+cmd_execute (struct lexer *lexer, struct dataset *ds)
 {
   if (!procedure (ds, NULL, NULL))
     return CMD_CASCADING_FAILURE;
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Parses, performs the ERASE command. */
 int
-cmd_erase (struct dataset *ds UNUSED)
+cmd_erase (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   if (get_safer_mode ()) 
     { 
@@ -690,16 +691,16 @@ cmd_erase (struct dataset *ds UNUSED)
       return CMD_FAILURE; 
     } 
   
-  if (!lex_force_match_id ("FILE"))
+  if (!lex_force_match_id (lexer, "FILE"))
     return CMD_FAILURE;
-  lex_match ('=');
-  if (!lex_force_string ())
+  lex_match (lexer, '=');
+  if (!lex_force_string (lexer))
     return CMD_FAILURE;
 
-  if (remove (ds_cstr (&tokstr)) == -1)
+  if (remove (ds_cstr (lex_tokstr (lexer))) == -1)
     {
       msg (SW, _("Error removing `%s': %s."),
-	   ds_cstr (&tokstr), strerror (errno));
+	   ds_cstr (lex_tokstr (lexer)), strerror (errno));
       return CMD_FAILURE;
     }
 
@@ -763,27 +764,27 @@ shell (void)
 /* Parses the HOST command argument and executes the specified
    command.  Returns a suitable command return code. */
 static int
-run_command (void)
+run_command (struct lexer *lexer)
 {
   const char *cmd;
   int string;
 
   /* Handle either a string argument or a full-line argument. */
   {
-    int c = lex_look_ahead ();
+    int c = lex_look_ahead (lexer);
 
     if (c == '\'' || c == '"')
       {
-	lex_get ();
-	if (!lex_force_string ())
+	lex_get (lexer);
+	if (!lex_force_string (lexer))
 	  return CMD_FAILURE;
-	cmd = ds_cstr (&tokstr);
+	cmd = ds_cstr (lex_tokstr (lexer));
 	string = 1;
       }
     else
       {
-	cmd = lex_rest_of_line (NULL);
-        lex_discard_line ();
+	cmd = lex_rest_of_line (lexer, NULL);
+        lex_discard_line (lexer);
 	string = 0;
       }
   }
@@ -795,23 +796,21 @@ run_command (void)
   /* Finish parsing. */
   if (string)
     {
-      lex_get ();
+      lex_get (lexer);
 
-      if (token != '.')
+      if (lex_token (lexer) != '.')
 	{
-	  lex_error (_("expecting end of command"));
+	  lex_error (lexer, _("expecting end of command"));
 	  return CMD_FAILURE;
 	}
     }
-  else
-    token = '.';
 
   return CMD_SUCCESS;
 }
 
 /* Parses, performs the HOST command. */
 int
-cmd_host (struct dataset *ds UNUSED)
+cmd_host (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   int code;
 
@@ -824,18 +823,18 @@ cmd_host (struct dataset *ds UNUSED)
 #ifdef unix
   /* Figure out whether to invoke an interactive shell or to execute a
      single shell command. */
-  if (lex_look_ahead () == '.')
+  if (lex_look_ahead (lexer) == '.')
     {
-      lex_get ();
+      lex_get (lexer);
       code = shell () ? CMD_FAILURE : CMD_SUCCESS;
     }
   else
-    code = run_command ();
+    code = run_command (lexer);
 #else /* !unix */
   /* Make sure that the system has a command interpreter, then run a
      command. */
   if (system (NULL) != 0)
-    code = run_command ();
+    code = run_command (lexer);
   else
     {
       msg (SE, _("No operating system support for this command."));
@@ -848,17 +847,17 @@ cmd_host (struct dataset *ds UNUSED)
 
 /* Parses, performs the NEW FILE command. */
 int
-cmd_new_file (struct dataset *ds)
+cmd_new_file (struct lexer *lexer, struct dataset *ds)
 {
   discard_variables (ds);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Parses a comment. */
 int
-cmd_comment (struct dataset *ds UNUSED)
+cmd_comment (struct lexer *lexer, struct dataset *ds UNUSED)
 {
-  lex_skip_comment ();
+  lex_skip_comment (lexer);
   return CMD_SUCCESS;
 }
diff --git a/src/language/command.h b/src/language/command.h
index cca9f839..e38707dd 100644
--- a/src/language/command.h
+++ b/src/language/command.h
@@ -55,7 +55,9 @@ enum cmd_state
   };
 
 struct dataset;
-enum cmd_result cmd_parse (struct dataset *ds, enum cmd_state);
+struct lexer; 
+
+enum cmd_result cmd_parse (struct lexer *lexer, struct dataset *ds, enum cmd_state);
 
 struct command;
 const char *cmd_complete (const char *, const struct command **);
@@ -63,7 +65,7 @@ const char *cmd_complete (const char *, const struct command **);
 struct dataset;
 
 /* Prototype all the command functions. */
-#define DEF_CMD(STATES, FLAGS, NAME, FUNCTION) int FUNCTION (struct dataset *);
+#define DEF_CMD(STATES, FLAGS, NAME, FUNCTION) int FUNCTION (struct lexer *, struct dataset *);
 #define UNIMPL_CMD(NAME, DESCRIPTION)
 #include "command.def"
 #undef DEF_CMD
diff --git a/src/language/control/do-if.c b/src/language/control/do-if.c
index e53fc898..51c3ec9f 100644
--- a/src/language/control/do-if.c
+++ b/src/language/control/do-if.c
@@ -83,7 +83,7 @@ struct do_if_trns
 
 static const struct ctl_class do_if_class;
 
-static int parse_clause (struct do_if_trns *, struct dataset *ds);
+static int parse_clause (struct lexer *, struct do_if_trns *, struct dataset *ds);
 static void add_clause (struct do_if_trns *,
                         struct expression *condition, int target_index);
 static void add_else (struct do_if_trns *);
@@ -98,7 +98,7 @@ static trns_free_func do_if_trns_free;
 
 /* Parse DO IF. */
 int
-cmd_do_if (struct dataset *ds)
+cmd_do_if (struct lexer *lexer, struct dataset *ds)
 {
   struct do_if_trns *do_if = xmalloc (sizeof *do_if);
   do_if->clauses = NULL;
@@ -109,34 +109,34 @@ cmd_do_if (struct dataset *ds)
   add_transformation_with_finalizer (ds, do_if_finalize_func,
                                      do_if_trns_proc, do_if_trns_free, do_if);
 
-  return parse_clause (do_if, ds);
+  return parse_clause (lexer, do_if, ds);
 }
 
 /* Parse ELSE IF. */
 int
-cmd_else_if (struct dataset *ds)
+cmd_else_if (struct lexer *lexer, struct dataset *ds)
 {
   struct do_if_trns *do_if = ctl_stack_top (&do_if_class);
   if (do_if == NULL || !must_not_have_else (do_if))
     return CMD_CASCADING_FAILURE;
-  return parse_clause (do_if, ds);
+  return parse_clause (lexer, do_if, ds);
 }
 
 /* Parse ELSE. */
 int
-cmd_else (struct dataset *ds)
+cmd_else (struct lexer *lexer, struct dataset *ds)
 {
   struct do_if_trns *do_if = ctl_stack_top (&do_if_class);
   assert (ds == do_if->ds);
   if (do_if == NULL || !must_not_have_else (do_if))
     return CMD_CASCADING_FAILURE;
   add_else (do_if);
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Parse END IF. */
 int
-cmd_end_if (struct dataset *ds)
+cmd_end_if (struct lexer *lexer, struct dataset *ds)
 {
   struct do_if_trns *do_if = ctl_stack_top (&do_if_class);
   assert (ds == do_if->ds);
@@ -146,7 +146,7 @@ cmd_end_if (struct dataset *ds)
 
   ctl_stack_pop (do_if);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Closes out DO_IF, by adding a sentinel ELSE clause if
@@ -197,17 +197,17 @@ has_else (struct do_if_trns *do_if)
    corresponding clause to DO_IF.  Checks for end of command and
    returns a command return code. */
 static int
-parse_clause (struct do_if_trns *do_if, struct dataset *ds)
+parse_clause (struct lexer *lexer, struct do_if_trns *do_if, struct dataset *ds)
 {
   struct expression *condition;
 
-  condition = expr_parse (ds, EXPR_BOOLEAN);
+  condition = expr_parse (lexer, ds, EXPR_BOOLEAN);
   if (condition == NULL)
     return CMD_CASCADING_FAILURE;
 
   add_clause (do_if, condition, next_transformation (ds));
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Adds a clause to DO_IF that tests for the given CONDITION and,
diff --git a/src/language/control/loop.c b/src/language/control/loop.c
index 5c22f3ca..01aad6fd 100644
--- a/src/language/control/loop.c
+++ b/src/language/control/loop.c
@@ -87,27 +87,27 @@ static trns_proc_func loop_trns_proc, end_loop_trns_proc, break_trns_proc;
 static trns_free_func loop_trns_free;
 
 static struct loop_trns *create_loop_trns (struct dataset *);
-static bool parse_if_clause (struct loop_trns *, struct expression **);
-static bool parse_index_clause (struct loop_trns *, char index_var_name[]);
+static bool parse_if_clause (struct lexer *, struct loop_trns *, struct expression **);
+static bool parse_index_clause (struct lexer *, struct loop_trns *, char index_var_name[]);
 static void close_loop (void *);
 
 /* LOOP. */
 
 /* Parses LOOP. */
 int
-cmd_loop (struct dataset *ds)
+cmd_loop (struct lexer *lexer, struct dataset *ds)
 {
   struct loop_trns *loop;
   char index_var_name[LONG_NAME_LEN + 1];
   bool ok = true;
 
   loop = create_loop_trns (ds);
-  while (token != '.' && ok) 
+  while (lex_token (lexer) != '.' && ok) 
     {
-      if (lex_match_id ("IF")) 
-        ok = parse_if_clause (loop, &loop->loop_condition);
+      if (lex_match_id (lexer, "IF")) 
+        ok = parse_if_clause (lexer, loop, &loop->loop_condition);
       else
-        ok = parse_index_clause (loop, index_var_name);
+        ok = parse_index_clause (lexer, loop, index_var_name);
     }
 
   /* Find index variable and create if necessary. */
@@ -126,7 +126,7 @@ cmd_loop (struct dataset *ds)
 
 /* Parses END LOOP. */
 int
-cmd_end_loop (struct dataset *ds)
+cmd_end_loop (struct lexer *lexer, struct dataset *ds)
 {
   struct loop_trns *loop;
   bool ok = true;
@@ -138,10 +138,10 @@ cmd_end_loop (struct dataset *ds)
   assert (loop->ds == ds);
   
   /* Parse syntax. */
-  if (lex_match_id ("IF"))
-    ok = parse_if_clause (loop, &loop->end_loop_condition);
+  if (lex_match_id (lexer, "IF"))
+    ok = parse_if_clause (lexer, loop, &loop->end_loop_condition);
   if (ok)
-    ok = lex_end_of_command () == CMD_SUCCESS;
+    ok = lex_end_of_command (lexer) == CMD_SUCCESS;
 
   if (!ok)
     loop->max_pass_count = 0;
@@ -153,7 +153,7 @@ cmd_end_loop (struct dataset *ds)
 
 /* Parses BREAK. */
 int
-cmd_break (struct dataset *ds)
+cmd_break (struct lexer *lexer, struct dataset *ds)
 {
   struct ctl_stmt *loop = ctl_stack_search (&loop_class);
   if (loop == NULL)
@@ -161,7 +161,7 @@ cmd_break (struct dataset *ds)
 
   add_transformation (ds, break_trns_proc, NULL, loop);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Closes a LOOP construct by emitting the END LOOP
@@ -187,9 +187,10 @@ close_loop (void *loop_)
    resulting expression to *CONDITION.
    Returns true if successful, false on failure. */
 static bool
-parse_if_clause (struct loop_trns *loop, struct expression **condition) 
+parse_if_clause (struct lexer *lexer, 
+		 struct loop_trns *loop, struct expression **condition) 
 {
-  *condition = expr_parse_pool (loop->pool, loop->ds, EXPR_BOOLEAN);
+  *condition = expr_parse_pool (lexer, loop->pool, loop->ds, EXPR_BOOLEAN);
   return *condition != NULL;
 }
 
@@ -197,29 +198,30 @@ parse_if_clause (struct loop_trns *loop, struct expression **condition)
    Stores the index variable's name in INDEX_VAR_NAME[].
    Returns true if successful, false on failure. */
 static bool
-parse_index_clause (struct loop_trns *loop, char index_var_name[]) 
+parse_index_clause (struct lexer *lexer, struct loop_trns *loop, char index_var_name[]) 
 {
-  if (token != T_ID) 
+  if (lex_token (lexer) != T_ID) 
     {
-      lex_error (NULL);
+      lex_error (lexer, NULL);
       return false;
     }
-  strcpy (index_var_name, tokid);
-  lex_get ();
+  strcpy (index_var_name, lex_tokid (lexer));
+  lex_get (lexer);
 
-  if (!lex_force_match ('='))
+  if (!lex_force_match (lexer, '='))
     return false;
 
-  loop->first_expr = expr_parse_pool (loop->pool, loop->ds, EXPR_NUMBER);
+  loop->first_expr = expr_parse_pool (lexer, loop->pool, 
+				      loop->ds, EXPR_NUMBER);
   if (loop->first_expr == NULL)
     return false;
 
   for (;;)
     {
       struct expression **e;
-      if (lex_match (T_TO)) 
+      if (lex_match (lexer, T_TO)) 
         e = &loop->last_expr;
-      else if (lex_match (T_BY)) 
+      else if (lex_match (lexer, T_BY)) 
         e = &loop->by_expr;
       else
         break;
@@ -229,13 +231,13 @@ parse_index_clause (struct loop_trns *loop, char index_var_name[])
           lex_sbc_only_once (e == &loop->last_expr ? "TO" : "BY");
           return false;
         }
-      *e = expr_parse_pool (loop->pool, loop->ds, EXPR_NUMBER);
+      *e = expr_parse_pool (lexer, loop->pool, loop->ds, EXPR_NUMBER);
       if (*e == NULL)
         return false;
     }
   if (loop->last_expr == NULL) 
     {
-      lex_sbc_missing ("TO");
+      lex_sbc_missing (lexer, "TO");
       return false;
     }
   if (loop->by_expr == NULL)
diff --git a/src/language/control/repeat.c b/src/language/control/repeat.c
index d812c12f..384ac775 100644
--- a/src/language/control/repeat.c
+++ b/src/language/control/repeat.c
@@ -83,13 +83,13 @@ struct repeat_block
     bool print;                         /* Print lines as executed? */
   };
 
-static bool parse_specification (struct repeat_block *);
-static bool parse_lines (struct repeat_block *);
+static bool parse_specification (struct lexer *, struct repeat_block *);
+static bool parse_lines (struct lexer *, struct repeat_block *);
 static void create_vars (struct repeat_block *);
 
-static int parse_ids (const struct dictionary *dict, struct repeat_entry *, struct pool *);
-static int parse_numbers (struct repeat_entry *, struct pool *);
-static int parse_strings (struct repeat_entry *, struct pool *);
+static int parse_ids (struct lexer *, const struct dictionary *dict, struct repeat_entry *, struct pool *);
+static int parse_numbers (struct lexer *, struct repeat_entry *, struct pool *);
+static int parse_strings (struct lexer *, struct repeat_entry *, struct pool *);
 
 static void do_repeat_filter (struct string *line, void *block);
 static bool do_repeat_read (struct string *line, char **file_name,
@@ -97,14 +97,14 @@ static bool do_repeat_read (struct string *line, char **file_name,
 static void do_repeat_close (void *block);
 
 int
-cmd_do_repeat (struct dataset *ds)
+cmd_do_repeat (struct lexer *lexer, struct dataset *ds)
 {
   struct repeat_block *block;
 
   block = pool_create_container (struct repeat_block, pool);
   block->ds = ds;
 
-  if (!parse_specification (block) || !parse_lines (block))
+  if (!parse_specification (lexer, block) || !parse_lines (lexer, block))
     goto error;
   
   create_vars (block);
@@ -124,7 +124,7 @@ cmd_do_repeat (struct dataset *ds)
 /* Parses the whole DO REPEAT command specification.
    Returns success. */
 static bool
-parse_specification (struct repeat_block *block) 
+parse_specification (struct lexer *lexer, struct repeat_block *block) 
 {
   char first_name[LONG_NAME_LEN + 1];
 
@@ -138,16 +138,16 @@ parse_specification (struct repeat_block *block)
       int count;
 
       /* Get a stand-in variable name and make sure it's unique. */
-      if (!lex_force_id ())
+      if (!lex_force_id (lexer))
 	return false;
-      if (dict_lookup_var (dict, tokid))
+      if (dict_lookup_var (dict, lex_tokid (lexer)))
         msg (SW, _("Dummy variable name \"%s\" hides dictionary "
                    "variable \"%s\"."),
-             tokid, tokid);
+             lex_tokid (lexer), lex_tokid (lexer));
       for (iter = block->macros; iter != NULL; iter = iter->next)
-	if (!strcasecmp (iter->id, tokid))
+	if (!strcasecmp (iter->id, lex_tokid (lexer)))
 	  {
-	    msg (SE, _("Dummy variable name \"%s\" is given twice."), tokid);
+	    msg (SE, _("Dummy variable name \"%s\" is given twice."), lex_tokid (lexer));
 	    return false;
 	  }
 
@@ -155,31 +155,31 @@ parse_specification (struct repeat_block *block)
          list. */
       e = pool_alloc (block->pool, sizeof *e);
       e->next = block->macros;
-      strcpy (e->id, tokid);
+      strcpy (e->id, lex_tokid (lexer));
       block->macros = e;
 
       /* Skip equals sign. */
-      lex_get ();
-      if (!lex_force_match ('='))
+      lex_get (lexer);
+      if (!lex_force_match (lexer, '='))
 	return false;
 
       /* Get the details of the variable's possible values. */
-      if (token == T_ID)
-	count = parse_ids (dict, e, block->pool);
-      else if (lex_is_number ())
-	count = parse_numbers (e, block->pool);
-      else if (token == T_STRING)
-	count = parse_strings (e, block->pool);
+      if (lex_token (lexer) == T_ID)
+	count = parse_ids (lexer, dict, e, block->pool);
+      else if (lex_is_number (lexer))
+	count = parse_numbers (lexer, e, block->pool);
+      else if (lex_token (lexer) == T_STRING)
+	count = parse_strings (lexer, e, block->pool);
       else
 	{
-	  lex_error (NULL);
+	  lex_error (lexer, NULL);
 	  return false;
 	}
       if (count == 0)
 	return false;
-      if (token != '/' && token != '.') 
+      if (lex_token (lexer) != '/' && lex_token (lexer) != '.') 
         {
-          lex_error (NULL);
+          lex_error (lexer, NULL);
           return false;
         }
 
@@ -200,9 +200,9 @@ parse_specification (struct repeat_block *block)
 	  return false;
 	}
 
-      lex_match ('/');
+      lex_match (lexer, '/');
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
 
   return true;
 }
@@ -265,7 +265,7 @@ recognize_end_repeat (const char *line, bool *print)
 /* Read all the lines we are going to substitute, inside the DO
    REPEAT...END REPEAT block. */
 static bool
-parse_lines (struct repeat_block *block) 
+parse_lines (struct lexer *lexer, struct repeat_block *block) 
 {
   char *previous_file_name;
   struct line_list **last_line;
@@ -284,7 +284,7 @@ parse_lines (struct repeat_block *block)
       struct string cur_line_copy;
       bool dot;
 
-      if (! lex_get_line_raw ())
+      if (! lex_get_line_raw (lexer))
         return false;
 
       /* If the current file has changed then record the fact. */
@@ -293,7 +293,7 @@ parse_lines (struct repeat_block *block)
           || !strcmp (cur_file_name, previous_file_name))
         previous_file_name = pool_strdup (block->pool, cur_file_name);
 
-      ds_init_string (&cur_line_copy, lex_entire_line_ds () );
+      ds_init_string (&cur_line_copy, lex_entire_line_ds (lexer) );
       ds_rtrim (&cur_line_copy, ss_cstr (CC_SPACES));
       dot = ds_chomp (&cur_line_copy, get_endcmd ());
 
@@ -303,7 +303,7 @@ parse_lines (struct repeat_block *block)
         {
         if (nesting_level-- == 0)
           {
-            lex_discard_line ();
+            lex_discard_line (lexer);
 	    ds_destroy (&cur_line_copy);
             return true;
           } 
@@ -321,7 +321,7 @@ parse_lines (struct repeat_block *block)
       ds_destroy (&cur_line_copy);
     }
 
-  lex_discard_line ();
+  lex_discard_line (lexer);
   return true;
 }
 
@@ -347,11 +347,13 @@ create_vars (struct repeat_block *block)
 
 /* Parses a set of ids for DO REPEAT. */
 static int
-parse_ids (const struct dictionary *dict, struct repeat_entry *e, struct pool *pool)
+parse_ids (struct lexer *lexer, const struct dictionary *dict, 
+	   struct repeat_entry *e, struct pool *pool)
 {
   size_t n = 0;
   e->type = VAR_NAMES;
-  return parse_mixed_vars_pool (dict, pool, &e->replacement, &n, PV_NONE) ? n : 0;
+  return parse_mixed_vars_pool (lexer, dict, pool, 
+				&e->replacement, &n, PV_NONE) ? n : 0;
 }
 
 /* Adds STRING to E's list of replacements, which has *USED
@@ -370,7 +372,7 @@ add_replacement (char *string,
 
 /* Parses a list of numbers for DO REPEAT. */
 static int
-parse_numbers (struct repeat_entry *e, struct pool *pool)
+parse_numbers (struct lexer *lexer, struct repeat_entry *e, struct pool *pool)
 {
   size_t used = 0;
   size_t allocated = 0;
@@ -383,23 +385,23 @@ parse_numbers (struct repeat_entry *e, struct pool *pool)
       long a, b, i;
 
       /* Parse A TO B into a, b. */
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return 0;
-      a = lex_integer ();
+      a = lex_integer (lexer);
 
-      lex_get ();
-      if (token == T_TO)
+      lex_get (lexer);
+      if (lex_token (lexer) == T_TO)
 	{
-	  lex_get ();
-	  if (!lex_force_int ())
+	  lex_get (lexer);
+	  if (!lex_force_int (lexer))
 	    return 0;
-	  b = lex_integer ();
+	  b = lex_integer (lexer);
           if (b < a) 
             {
               msg (SE, _("%ld TO %ld is an invalid range."), a, b);
               return 0;
             }
-	  lex_get ();
+	  lex_get (lexer);
 	}
       else
         b = a;
@@ -409,16 +411,16 @@ parse_numbers (struct repeat_entry *e, struct pool *pool)
                          e, pool, &used, &allocated);
 
 
-      lex_match (',');
+      lex_match (lexer, ',');
     }
-  while (token != '/' && token != '.');
+  while (lex_token (lexer) != '/' && lex_token (lexer) != '.');
 
   return used;
 }
 
 /* Parses a list of strings for DO REPEAT. */
 int
-parse_strings (struct repeat_entry *e, struct pool *pool)
+parse_strings (struct lexer *lexer, struct repeat_entry *e, struct pool *pool)
 {
   size_t used = 0;
   size_t allocated = 0;
@@ -430,26 +432,26 @@ parse_strings (struct repeat_entry *e, struct pool *pool)
     {
       char *string;
       
-      if (token != T_STRING)
+      if (lex_token (lexer) != T_STRING)
 	{
 	  msg (SE, _("String expected."));
 	  return 0;
 	}
 
-      string = lex_token_representation ();
+      string = lex_token_representation (lexer);
       pool_register (pool, free, string);
       add_replacement (string, e, pool, &used, &allocated);
 
-      lex_get ();
-      lex_match (',');
+      lex_get (lexer);
+      lex_match (lexer, ',');
     }
-  while (token != '/' && token != '.');
+  while (lex_token (lexer) != '/' && lex_token (lexer) != '.');
 
   return used;
 }
 
 int
-cmd_end_repeat (struct dataset *ds UNUSED)
+cmd_end_repeat (struct lexer *lexer UNUSED, struct dataset *ds UNUSED)
 {
   msg (SE, _("No matching DO REPEAT."));
   return CMD_CASCADING_FAILURE;
diff --git a/src/language/control/temporary.c b/src/language/control/temporary.c
index 45d7342b..e49fc1ac 100644
--- a/src/language/control/temporary.c
+++ b/src/language/control/temporary.c
@@ -41,12 +41,12 @@
 
 /* Parses the TEMPORARY command. */
 int
-cmd_temporary (struct dataset *ds)
+cmd_temporary (struct lexer *lexer, struct dataset *ds)
 {
   if (!proc_in_temporary_transformations (ds))
     proc_start_temporary_transformations (ds);
   else
     msg (SE, _("This command may only appear once between "
                "procedures and procedure-like commands."));
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/data-io/data-list.c b/src/language/data-io/data-list.c
index 136c8e44..42837c15 100644
--- a/src/language/data-io/data-list.c
+++ b/src/language/data-io/data-list.c
@@ -104,9 +104,9 @@ struct data_list_pgm
 
 static const struct case_source_class data_list_source_class;
 
-static bool parse_fixed (struct dictionary *dict, 
+static bool parse_fixed (struct lexer *, struct dictionary *dict, 
 			 struct pool *tmp_pool, struct data_list_pgm *);
-static bool parse_free (struct dictionary *dict, 
+static bool parse_free (struct lexer *, struct dictionary *dict, 
 			struct pool *tmp_pool, struct data_list_pgm *);
 static void dump_fixed_table (const struct ll_list *,
                               const struct file_handle *, int record_cnt);
@@ -117,7 +117,7 @@ static trns_free_func data_list_trns_free;
 static trns_proc_func data_list_trns_proc;
 
 int
-cmd_data_list (struct dataset *ds)
+cmd_data_list (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   struct data_list_pgm *dls;
@@ -141,34 +141,34 @@ cmd_data_list (struct dataset *ds)
 
   tmp_pool = pool_create_subpool (dls->pool);
 
-  while (token != '/')
+  while (lex_token (lexer) != '/')
     {
-      if (lex_match_id ("FILE"))
+      if (lex_match_id (lexer, "FILE"))
 	{
-	  lex_match ('=');
-	  fh = fh_parse (FH_REF_FILE | FH_REF_INLINE);
+	  lex_match (lexer, '=');
+	  fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE);
 	  if (fh == NULL)
 	    goto error;
 	}
-      else if (lex_match_id ("RECORDS"))
+      else if (lex_match_id (lexer, "RECORDS"))
 	{
-	  lex_match ('=');
-	  lex_match ('(');
-	  if (!lex_force_int ())
+	  lex_match (lexer, '=');
+	  lex_match (lexer, '(');
+	  if (!lex_force_int (lexer))
 	    goto error;
-	  dls->record_cnt = lex_integer ();
-	  lex_get ();
-	  lex_match (')');
+	  dls->record_cnt = lex_integer (lexer);
+	  lex_get (lexer);
+	  lex_match (lexer, ')');
 	}
-      else if (lex_match_id ("SKIP"))
+      else if (lex_match_id (lexer, "SKIP"))
 	{
-	  lex_match ('=');
-	  if (!lex_force_int ())
+	  lex_match (lexer, '=');
+	  if (!lex_force_int (lexer))
 	    goto error;
-	  dls->skip_records = lex_integer ();
-	  lex_get ();
+	  dls->skip_records = lex_integer (lexer);
+	  lex_get (lexer);
 	}
-      else if (lex_match_id ("END"))
+      else if (lex_match_id (lexer, "END"))
 	{
 	  if (dls->end)
 	    {
@@ -176,32 +176,32 @@ cmd_data_list (struct dataset *ds)
 	      goto error;
 	    }
 	  
-	  lex_match ('=');
-	  if (!lex_force_id ())
+	  lex_match (lexer, '=');
+	  if (!lex_force_id (lexer))
 	    goto error;
-	  dls->end = dict_lookup_var (dataset_dict (ds), tokid);
+	  dls->end = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
 	  if (!dls->end) 
-            dls->end = dict_create_var_assert (dataset_dict (ds), tokid, 0);
-	  lex_get ();
+            dls->end = dict_create_var_assert (dataset_dict (ds), lex_tokid (lexer), 0);
+	  lex_get (lexer);
 	}
-      else if (token == T_ID)
+      else if (lex_token (lexer) == T_ID)
 	{
-          if (lex_match_id ("NOTABLE"))
+          if (lex_match_id (lexer, "NOTABLE"))
             table = 0;
-          else if (lex_match_id ("TABLE"))
+          else if (lex_match_id (lexer, "TABLE"))
             table = 1;
           else 
             {
               int type;
-              if (lex_match_id ("FIXED"))
+              if (lex_match_id (lexer, "FIXED"))
                 type = DLS_FIXED;
-              else if (lex_match_id ("FREE"))
+              else if (lex_match_id (lexer, "FREE"))
                 type = DLS_FREE;
-              else if (lex_match_id ("LIST"))
+              else if (lex_match_id (lexer, "LIST"))
                 type = DLS_LIST;
               else 
                 {
-                  lex_error (NULL);
+                  lex_error (lexer, NULL);
                   goto error;
                 }
 
@@ -214,35 +214,35 @@ cmd_data_list (struct dataset *ds)
 	      dls->type = type;
 
               if ((dls->type == DLS_FREE || dls->type == DLS_LIST)
-                  && lex_match ('(')) 
+                  && lex_match (lexer, '(')) 
                 {
-                  while (!lex_match (')'))
+                  while (!lex_match (lexer, ')'))
                     {
                       int delim;
 
-                      if (lex_match_id ("TAB"))
+                      if (lex_match_id (lexer, "TAB"))
                         delim = '\t';
-                      else if (token == T_STRING && ds_length (&tokstr) == 1)
+                      else if (lex_token (lexer) == T_STRING && ds_length (lex_tokstr (lexer)) == 1)
 			{
-			  delim = ds_first (&tokstr);
-			  lex_get ();
+			  delim = ds_first (lex_tokstr (lexer));
+			  lex_get (lexer);
 			}
                       else 
                         {
-                          lex_error (NULL);
+                          lex_error (lexer, NULL);
                           goto error;
                         }
 
                       ds_put_char (&dls->delims, delim);
 
-                      lex_match (',');
+                      lex_match (lexer, ',');
                     }
                 }
             }
         }
       else
 	{
-	  lex_error (NULL);
+	  lex_error (lexer, NULL);
 	  goto error;
 	}
     }
@@ -255,11 +255,11 @@ cmd_data_list (struct dataset *ds)
   if (table == -1)
     table = dls->type != DLS_FREE;
 
-  ok = (dls->type == DLS_FIXED ? parse_fixed : parse_free) (dict, tmp_pool, dls);
+  ok = (dls->type == DLS_FIXED ? parse_fixed : parse_free) (lexer, dict, tmp_pool, dls);
   if (!ok)
     goto error;
 
-  if (lex_end_of_command () != CMD_SUCCESS)
+  if (lex_end_of_command (lexer) != CMD_SUCCESS)
     goto error;
 
   if (table)
@@ -270,7 +270,7 @@ cmd_data_list (struct dataset *ds)
 	dump_free_table (dls, fh);
     }
 
-  dls->reader = dfm_open_reader (fh);
+  dls->reader = dfm_open_reader (fh, lexer);
   if (dls->reader == NULL)
     goto error;
 
@@ -295,14 +295,14 @@ cmd_data_list (struct dataset *ds)
    needed once parsing is complete.  Returns true only if
    successful. */
 static bool
-parse_fixed (struct dictionary *dict, 
+parse_fixed (struct lexer *lexer, struct dictionary *dict, 
 	     struct pool *tmp_pool, struct data_list_pgm *dls)
 {
   int last_nonempty_record;
   int record = 0;
   int column = 1;
 
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
       char **names;
       size_t name_cnt, name_idx;
@@ -310,9 +310,10 @@ parse_fixed (struct dictionary *dict,
       size_t format_cnt;
 
       /* Parse everything. */
-      if (!parse_record_placement (&record, &column)
-          || !parse_DATA_LIST_vars_pool (tmp_pool, &names, &name_cnt, PV_NONE)
-          || !parse_var_placements (tmp_pool, name_cnt, true,
+      if (!parse_record_placement (lexer, &record, &column)
+          || !parse_DATA_LIST_vars_pool (lexer, tmp_pool, 
+					 &names, &name_cnt, PV_NONE)
+          || !parse_var_placements (lexer, tmp_pool, name_cnt, true,
                                     &formats, &format_cnt))
         return false;
 
@@ -447,30 +448,32 @@ dump_fixed_table (const struct ll_list *specs,
    them to DLS.  Uses TMP_POOL for data that is not needed once
    parsing is complete.  Returns true only if successful. */
 static bool
-parse_free (struct dictionary *dict, struct pool *tmp_pool, struct data_list_pgm *dls)
+parse_free (struct lexer *lexer, struct dictionary *dict, struct pool *tmp_pool, 
+		struct data_list_pgm *dls)
 {
-  lex_get ();
-  while (token != '.')
+  lex_get (lexer);
+  while (lex_token (lexer) != '.')
     {
       struct fmt_spec input, output;
       char **name;
       size_t name_cnt;
       size_t i;
 
-      if (!parse_DATA_LIST_vars_pool (tmp_pool, &name, &name_cnt, PV_NONE))
+      if (!parse_DATA_LIST_vars_pool (lexer, tmp_pool, 
+				      &name, &name_cnt, PV_NONE))
 	return 0;
 
-      if (lex_match ('('))
+      if (lex_match (lexer, '('))
 	{
-	  if (!parse_format_specifier (&input)
+	  if (!parse_format_specifier (lexer, &input)
               || !fmt_check_input (&input)
-              || !lex_force_match (')')) 
+              || !lex_force_match (lexer, ')')) 
             return NULL;
 	  output = fmt_for_output_from_input (&input);
 	}
       else
 	{
-	  lex_match ('*');
+	  lex_match (lexer, '*');
           input = fmt_for_input (FMT_F, 8, 0);
 	  output = *get_format ();
 	}
diff --git a/src/language/data-io/data-reader.c b/src/language/data-io/data-reader.c
index 6458a821..0bff8696 100644
--- a/src/language/data-io/data-reader.c
+++ b/src/language/data-io/data-reader.c
@@ -64,6 +64,7 @@ struct dfm_reader
     FILE *file;                 /* Associated file. */
     size_t pos;                 /* Offset in line of current character. */
     unsigned eof_cnt;           /* # of attempts to advance past EOF. */
+    struct lexer *lexer;        /* The lexer reading the file */
   };
 
 /* Closes reader R opened by dfm_open_reader(). */
@@ -111,7 +112,7 @@ dfm_close_reader (struct dfm_reader *r)
    file between BEGIN FILE and END FILE.  Returns a reader if
    successful, or a null pointer otherwise. */
 struct dfm_reader *
-dfm_open_reader (struct file_handle *fh)
+dfm_open_reader (struct file_handle *fh, struct lexer *lexer)
 {
   struct dfm_reader *r;
   void **rp;
@@ -124,6 +125,7 @@ dfm_open_reader (struct file_handle *fh)
   
   r = xmalloc (sizeof *r);
   r->fh = fh;
+  r->lexer = lexer ;
   ds_init_empty (&r->line);
   ds_init_empty (&r->scratch);
   r->flags = DFM_ADVANCE;
@@ -163,14 +165,14 @@ read_inline_record (struct dfm_reader *r)
     {
       r->flags |= DFM_SAW_BEGIN_DATA;
 
-      while (token == '.')
-        lex_get ();
-      if (!lex_force_match_id ("BEGIN") || !lex_force_match_id ("DATA"))
+      while (lex_token (r->lexer) == '.')
+        lex_get (r->lexer);
+      if (!lex_force_match_id (r->lexer, "BEGIN") || !lex_force_match_id (r->lexer, "DATA"))
         return false;
       getl_set_prompt_style (GETL_PROMPT_DATA);
     }
       
-  if (!lex_get_line_raw ())
+  if (!lex_get_line_raw (r->lexer))
     {
       msg (SE, _("Unexpected end-of-file while reading data in BEGIN "
                  "DATA.  This probably indicates "
@@ -180,14 +182,14 @@ read_inline_record (struct dfm_reader *r)
       return false;
     }
 
-  if (ds_length (lex_entire_line_ds() ) >= 8
-      && !strncasecmp (lex_entire_line (), "end data", 8))
+  if (ds_length (lex_entire_line_ds (r->lexer) ) >= 8
+      && !strncasecmp (lex_entire_line (r->lexer), "end data", 8))
     {
-      lex_discard_line ();
+      lex_discard_line (r->lexer);
       return false;
     }
 
-  ds_assign_string (&r->line, lex_entire_line_ds () );
+  ds_assign_string (&r->line, lex_entire_line_ds (r->lexer) );
 
   return true;
 }
@@ -260,7 +262,7 @@ dfm_eof (struct dfm_reader *r)
     {
       r->flags &= ~DFM_ADVANCE;
 
-      if (r->eof_cnt == 0 && read_record (r)) 
+      if (r->eof_cnt == 0 && read_record (r) ) 
         {
           r->pos = 0;
           return 0; 
@@ -424,7 +426,7 @@ dfm_pop (struct dfm_reader *r)
 
 /* Perform BEGIN DATA...END DATA as a procedure in itself. */
 int
-cmd_begin_data (struct dataset *ds)
+cmd_begin_data (struct lexer *lexer, struct dataset *ds)
 {
   struct dfm_reader *r;
   bool ok;
@@ -437,7 +439,7 @@ cmd_begin_data (struct dataset *ds)
     }
 
   /* Open inline file. */
-  r = dfm_open_reader (fh_inline_file ());
+  r = dfm_open_reader (fh_inline_file (), lexer);
   r->flags |= DFM_SAW_BEGIN_DATA;
 
   /* Input procedure reads from inline file. */
diff --git a/src/language/data-io/data-reader.h b/src/language/data-io/data-reader.h
index e7f4cd69..9f8ac931 100644
--- a/src/language/data-io/data-reader.h
+++ b/src/language/data-io/data-reader.h
@@ -31,9 +31,10 @@
 
 struct file_handle;
 struct string;
+struct lexer;
 
 /* Input. */
-struct dfm_reader *dfm_open_reader (struct file_handle *);
+struct dfm_reader *dfm_open_reader (struct file_handle *, struct lexer *);
 void dfm_close_reader (struct dfm_reader *);
 bool dfm_reader_error (const struct dfm_reader *);
 unsigned dfm_eof (struct dfm_reader *);
diff --git a/src/language/data-io/file-handle.h b/src/language/data-io/file-handle.h
index 31333e9e..50f0c57b 100644
--- a/src/language/data-io/file-handle.h
+++ b/src/language/data-io/file-handle.h
@@ -26,6 +26,7 @@
 #include <stddef.h>
 #include <data/file-handle-def.h>
 
-struct file_handle *fh_parse (enum fh_referent);
+struct lexer ;
+struct file_handle *fh_parse (struct lexer *, enum fh_referent);
 
 #endif /* !file_handle.h */
diff --git a/src/language/data-io/file-handle.q b/src/language/data-io/file-handle.q
index b3d15619..31039a35 100644
--- a/src/language/data-io/file-handle.q
+++ b/src/language/data-io/file-handle.q
@@ -51,7 +51,7 @@
 /* (functions) */
 
 int
-cmd_file_handle (struct dataset *ds)
+cmd_file_handle (struct lexer *lexer, struct dataset *ds)
 {
   char handle_name[LONG_NAME_LEN + 1];
   struct fh_properties properties = *fh_default_properties ();
@@ -59,9 +59,9 @@ cmd_file_handle (struct dataset *ds)
   struct cmd_file_handle cmd;
   struct file_handle *handle;
 
-  if (!lex_force_id ())
+  if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  str_copy_trunc (handle_name, sizeof handle_name, tokid);
+  str_copy_trunc (handle_name, sizeof handle_name, lex_tokid (lexer));
 
   handle = fh_from_name (handle_name);
   if (handle != NULL)
@@ -72,19 +72,19 @@ cmd_file_handle (struct dataset *ds)
       return CMD_CASCADING_FAILURE;
     }
 
-  lex_get ();
-  if (!lex_force_match ('/'))
+  lex_get (lexer);
+  if (!lex_force_match (lexer, '/'))
     return CMD_CASCADING_FAILURE;
 
-  if (!parse_file_handle (ds, &cmd, NULL))
+  if (!parse_file_handle (lexer, ds, &cmd, NULL))
     return CMD_CASCADING_FAILURE;
 
-  if (lex_end_of_command () != CMD_SUCCESS)
+  if (lex_end_of_command (lexer) != CMD_SUCCESS)
     goto lossage;
 
   if (cmd.s_name == NULL && cmd.mode != FH_SCRATCH)
     {
-      lex_sbc_missing ("NAME");
+      lex_sbc_missing (lexer, "NAME");
       goto lossage;
     }
 
@@ -127,13 +127,13 @@ cmd_file_handle (struct dataset *ds)
 }
 
 int
-cmd_close_file_handle (struct dataset *ds UNUSED) 
+cmd_close_file_handle (struct lexer *lexer, struct dataset *ds UNUSED) 
 {
   struct file_handle *handle;
 
-  if (!lex_force_id ())
+  if (!lex_force_id (lexer))
     return CMD_CASCADING_FAILURE;
-  handle = fh_from_name (tokid);
+  handle = fh_from_name (lex_tokid (lexer));
   if (handle == NULL)
     return CMD_CASCADING_FAILURE;
 
@@ -164,39 +164,39 @@ referent_name (enum fh_referent referent)
    file handle are restricted to those in REFERENT_MASK.  Returns
    the file handle when successful, a null pointer on failure. */
 struct file_handle *
-fh_parse (enum fh_referent referent_mask)
+fh_parse (struct lexer *lexer, enum fh_referent referent_mask)
 {
   struct file_handle *handle;
 
-  if (lex_match_id ("INLINE")) 
+  if (lex_match_id (lexer, "INLINE")) 
     handle = fh_inline_file ();
   else 
     {
-      if (token != T_ID && token != T_STRING)
+      if (lex_token (lexer) != T_ID && lex_token (lexer) != T_STRING)
         {
-          lex_error (_("expecting a file name or handle name"));
+          lex_error (lexer, _("expecting a file name or handle name"));
           return NULL;
         }
 
       handle = NULL;
-      if (token == T_ID) 
-        handle = fh_from_name (tokid);
+      if (lex_token (lexer) == T_ID) 
+        handle = fh_from_name (lex_tokid (lexer));
       if (handle == NULL) 
-        handle = fh_from_file_name (ds_cstr (&tokstr)); 
+        handle = fh_from_file_name (ds_cstr (lex_tokstr (lexer))); 
       if (handle == NULL)
         {
-          if (token != T_ID || tokid[0] != '#' || get_syntax () != ENHANCED) 
+          if (lex_token (lexer) != T_ID || lex_tokid (lexer)[0] != '#' || get_syntax () != ENHANCED) 
             {
-              char *file_name = ds_cstr (&tokstr);
+              char *file_name = ds_cstr (lex_tokstr (lexer));
               char *handle_name = xasprintf ("\"%s\"", file_name);
               handle = fh_create_file (handle_name, file_name,
                                        fh_default_properties ());
               free (handle_name);
             }
           else
-            handle = fh_create_scratch (tokid);
+            handle = fh_create_scratch (lex_tokid (lexer));
         }
-      lex_get ();
+      lex_get (lexer);
     }
 
   if (!(fh_get_referent (handle) & referent_mask)) 
diff --git a/src/language/data-io/get.c b/src/language/data-io/get.c
index d79206d3..be561416 100644
--- a/src/language/data-io/get.c
+++ b/src/language/data-io/get.c
@@ -60,7 +60,7 @@ static void map_case (const struct case_map *,
                       const struct ccase *, struct ccase *);
 static void destroy_case_map (struct case_map *);
 
-static bool parse_dict_trim (struct dictionary *);
+static bool parse_dict_trim (struct lexer *, struct dictionary *);
 
 /* Reading system and portable files. */
 
@@ -85,7 +85,7 @@ static void case_reader_pgm_free (struct case_reader_pgm *);
 
 /* Parses a GET or IMPORT command. */
 static int
-parse_read_command (struct dataset *ds, enum reader_command type)
+parse_read_command (struct lexer *lexer, struct dataset *ds, enum reader_command type)
 {
   struct case_reader_pgm *pgm = NULL;
   struct file_handle *fh = NULL;
@@ -93,27 +93,27 @@ parse_read_command (struct dataset *ds, enum reader_command type)
 
   for (;;)
     {
-      lex_match ('/');
+      lex_match (lexer, '/');
 
-      if (lex_match_id ("FILE") || token == T_STRING)
+      if (lex_match_id (lexer, "FILE") || lex_token (lexer) == T_STRING)
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
-	  fh = fh_parse (FH_REF_FILE | FH_REF_SCRATCH);
+	  fh = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH);
 	  if (fh == NULL)
             goto error;
 	}
-      else if (type == IMPORT_CMD && lex_match_id ("TYPE"))
+      else if (type == IMPORT_CMD && lex_match_id (lexer, "TYPE"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
-	  if (lex_match_id ("COMM"))
+	  if (lex_match_id (lexer, "COMM"))
 	    type = PFM_COMM;
-	  else if (lex_match_id ("TAPE"))
+	  else if (lex_match_id (lexer, "TAPE"))
 	    type = PFM_TAPE;
 	  else
 	    {
-	      lex_error (_("expecting COMM or TAPE"));
+	      lex_error (lexer, _("expecting COMM or TAPE"));
               goto error;
 	    }
 	}
@@ -123,7 +123,7 @@ parse_read_command (struct dataset *ds, enum reader_command type)
   
   if (fh == NULL) 
     {
-      lex_sbc_missing ("FILE");
+      lex_sbc_missing (lexer, "FILE");
       goto error;
     }
               
@@ -140,10 +140,10 @@ parse_read_command (struct dataset *ds, enum reader_command type)
   
   start_case_map (dict);
 
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
-      lex_match ('/');
-      if (!parse_dict_trim (dict))
+      lex_match (lexer, '/');
+      if (!parse_dict_trim (lexer, dict))
         goto error;
     }
 
@@ -227,16 +227,16 @@ static const struct case_source_class case_reader_source_class =
 
 /* GET. */
 int
-cmd_get (struct dataset *ds) 
+cmd_get (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_read_command (ds, GET_CMD);
+  return parse_read_command (lexer, ds, GET_CMD);
 }
 
 /* IMPORT. */
 int
-cmd_import (struct dataset *ds) 
+cmd_import (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_read_command (ds, IMPORT_CMD);
+  return parse_read_command (lexer, ds, IMPORT_CMD);
 }
 
 /* Writing system and portable files. */ 
@@ -290,7 +290,7 @@ case_writer_destroy (struct case_writer *aw)
 
    On failure, returns a null pointer. */
 static struct case_writer *
-parse_write_command (struct dataset *ds, 
+parse_write_command (struct lexer *lexer, struct dataset *ds, 
 		     enum writer_type writer_type,
                      enum command_type command_type,
                      bool *retain_unselected)
@@ -327,10 +327,10 @@ parse_write_command (struct dataset *ds,
   start_case_map (dict);
   dict_delete_scratch_vars (dict);
 
-  lex_match ('/');
+  lex_match (lexer, '/');
   for (;;)
     {
-      if (lex_match_id ("OUTFILE"))
+      if (lex_match_id (lexer, "OUTFILE"))
 	{
           if (handle != NULL) 
             {
@@ -338,88 +338,88 @@ parse_write_command (struct dataset *ds,
               goto error; 
             }
           
-	  lex_match ('=');
+	  lex_match (lexer, '=');
       
-	  handle = fh_parse (FH_REF_FILE | FH_REF_SCRATCH);
+	  handle = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH);
 	  if (handle == NULL)
 	    goto error;
 	}
-      else if (lex_match_id ("NAMES"))
+      else if (lex_match_id (lexer, "NAMES"))
         print_short_names = true;
-      else if (lex_match_id ("PERMISSIONS")) 
+      else if (lex_match_id (lexer, "PERMISSIONS")) 
         {
           bool cw;
           
-          lex_match ('=');
-          if (lex_match_id ("READONLY"))
+          lex_match (lexer, '=');
+          if (lex_match_id (lexer, "READONLY"))
             cw = false;
-          else if (lex_match_id ("WRITEABLE"))
+          else if (lex_match_id (lexer, "WRITEABLE"))
             cw = true;
           else
             {
-              lex_error (_("expecting %s or %s"), "READONLY", "WRITEABLE");
+              lex_error (lexer, _("expecting %s or %s"), "READONLY", "WRITEABLE");
               goto error;
             }
           sysfile_opts.create_writeable = porfile_opts.create_writeable = cw;
         }
-      else if (command_type == PROC_CMD && lex_match_id ("UNSELECTED")) 
+      else if (command_type == PROC_CMD && lex_match_id (lexer, "UNSELECTED")) 
         {
-          lex_match ('=');
-          if (lex_match_id ("RETAIN"))
+          lex_match (lexer, '=');
+          if (lex_match_id (lexer, "RETAIN"))
             *retain_unselected = true;
-          else if (lex_match_id ("DELETE"))
+          else if (lex_match_id (lexer, "DELETE"))
             *retain_unselected = false;
           else
             {
-              lex_error (_("expecting %s or %s"), "RETAIN", "DELETE");
+              lex_error (lexer, _("expecting %s or %s"), "RETAIN", "DELETE");
               goto error;
             }
         }
-      else if (writer_type == SYSFILE_WRITER && lex_match_id ("COMPRESSED"))
+      else if (writer_type == SYSFILE_WRITER && lex_match_id (lexer, "COMPRESSED"))
 	sysfile_opts.compress = true;
-      else if (writer_type == SYSFILE_WRITER && lex_match_id ("UNCOMPRESSED"))
+      else if (writer_type == SYSFILE_WRITER && lex_match_id (lexer, "UNCOMPRESSED"))
 	sysfile_opts.compress = false;
-      else if (writer_type == SYSFILE_WRITER && lex_match_id ("VERSION"))
+      else if (writer_type == SYSFILE_WRITER && lex_match_id (lexer, "VERSION"))
 	{
-	  lex_match ('=');
-	  if (!lex_force_int ())
+	  lex_match (lexer, '=');
+	  if (!lex_force_int (lexer))
             goto error;
-          sysfile_opts.version = lex_integer ();
-          lex_get ();
+          sysfile_opts.version = lex_integer (lexer);
+          lex_get (lexer);
 	}
-      else if (writer_type == PORFILE_WRITER && lex_match_id ("TYPE")) 
+      else if (writer_type == PORFILE_WRITER && lex_match_id (lexer, "TYPE")) 
         {
-          lex_match ('=');
-          if (lex_match_id ("COMMUNICATIONS"))
+          lex_match (lexer, '=');
+          if (lex_match_id (lexer, "COMMUNICATIONS"))
             porfile_opts.type = PFM_COMM;
-          else if (lex_match_id ("TAPE"))
+          else if (lex_match_id (lexer, "TAPE"))
             porfile_opts.type = PFM_TAPE;
           else
             {
-              lex_error (_("expecting %s or %s"), "COMM", "TAPE");
+              lex_error (lexer, _("expecting %s or %s"), "COMM", "TAPE");
               goto error;
             }
         }
-      else if (writer_type == PORFILE_WRITER && lex_match_id ("DIGITS")) 
+      else if (writer_type == PORFILE_WRITER && lex_match_id (lexer, "DIGITS")) 
         {
-          lex_match ('=');
-          if (!lex_force_int ())
+          lex_match (lexer, '=');
+          if (!lex_force_int (lexer))
             goto error;
-          porfile_opts.digits = lex_integer ();
-          lex_get ();
+          porfile_opts.digits = lex_integer (lexer);
+          lex_get (lexer);
         }
-      else if (!parse_dict_trim (dict))
+      else if (!parse_dict_trim (lexer, dict))
         goto error;
       
-      if (!lex_match ('/'))
+      if (!lex_match (lexer, '/'))
 	break;
     }
-  if (lex_end_of_command () != CMD_SUCCESS)
+  if (lex_end_of_command (lexer) != CMD_SUCCESS)
     goto error;
 
   if (handle == NULL) 
     {
-      lex_sbc_missing ("OUTFILE");
+      lex_sbc_missing (lexer, "OUTFILE");
       goto error;
     }
 
@@ -474,14 +474,14 @@ static bool output_proc (const struct ccase *, void *, const struct dataset *);
 
 /* Parses and performs the SAVE or EXPORT procedure. */
 static int
-parse_output_proc (struct dataset *ds, enum writer_type writer_type)
+parse_output_proc (struct lexer *lexer, struct dataset *ds, enum writer_type writer_type)
 {
   bool retain_unselected;
   struct variable *saved_filter_variable;
   struct case_writer *aw;
   bool ok;
 
-  aw = parse_write_command (ds, writer_type, PROC_CMD, &retain_unselected);
+  aw = parse_write_command (lexer, ds, writer_type, PROC_CMD, &retain_unselected);
   if (aw == NULL) 
     return CMD_CASCADING_FAILURE;
 
@@ -504,15 +504,15 @@ output_proc (const struct ccase *c, void *aw_, const struct dataset *ds UNUSED)
 }
 
 int
-cmd_save (struct dataset *ds) 
+cmd_save (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_output_proc (ds, SYSFILE_WRITER);
+  return parse_output_proc (lexer, ds, SYSFILE_WRITER);
 }
 
 int
-cmd_export (struct dataset *ds) 
+cmd_export (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_output_proc (ds, PORFILE_WRITER);
+  return parse_output_proc (lexer, ds, PORFILE_WRITER);
 }
 
 /* XSAVE and XEXPORT. */
@@ -528,10 +528,10 @@ static trns_free_func output_trns_free;
 
 /* Parses the XSAVE or XEXPORT transformation command. */
 static int
-parse_output_trns (struct dataset *ds, enum writer_type writer_type) 
+parse_output_trns (struct lexer *lexer, struct dataset *ds, enum writer_type writer_type) 
 {
   struct output_trns *t = xmalloc (sizeof *t);
-  t->aw = parse_write_command (ds, writer_type, XFORM_CMD, NULL);
+  t->aw = parse_write_command (lexer, ds, writer_type, XFORM_CMD, NULL);
   if (t->aw == NULL) 
     {
       free (t);
@@ -569,50 +569,50 @@ output_trns_free (void *trns_)
 
 /* XSAVE command. */
 int
-cmd_xsave (struct dataset *ds) 
+cmd_xsave (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_output_trns (ds, SYSFILE_WRITER);
+  return parse_output_trns (lexer, ds, SYSFILE_WRITER);
 }
 
 /* XEXPORT command. */
 int
-cmd_xexport (struct dataset *ds) 
+cmd_xexport (struct lexer *lexer, struct dataset *ds) 
 {
-  return parse_output_trns (ds, PORFILE_WRITER);
+  return parse_output_trns (lexer, ds, PORFILE_WRITER);
 }
 
-static bool rename_variables (struct dictionary *dict);
-static bool drop_variables (struct dictionary *dict);
-static bool keep_variables (struct dictionary *dict);
+static bool rename_variables (struct lexer *lexer, struct dictionary *dict);
+static bool drop_variables (struct lexer *, struct dictionary *dict);
+static bool keep_variables (struct lexer *, struct dictionary *dict);
 
 /* Commands that read and write system files share a great deal
    of common syntactic structure for rearranging and dropping
    variables.  This function parses this syntax and modifies DICT
    appropriately.  Returns true on success, false on failure. */
 static bool
-parse_dict_trim (struct dictionary *dict)
+parse_dict_trim (struct lexer *lexer, struct dictionary *dict)
 {
-  if (lex_match_id ("MAP")) 
+  if (lex_match_id (lexer, "MAP")) 
     {
       /* FIXME. */
       return true;
     }
-  else if (lex_match_id ("DROP"))
-    return drop_variables (dict);
-  else if (lex_match_id ("KEEP"))
-    return keep_variables (dict);
-  else if (lex_match_id ("RENAME"))
-    return rename_variables (dict);
+  else if (lex_match_id (lexer, "DROP"))
+    return drop_variables (lexer, dict);
+  else if (lex_match_id (lexer, "KEEP"))
+    return keep_variables (lexer, dict);
+  else if (lex_match_id (lexer, "RENAME"))
+    return rename_variables (lexer, dict);
   else
     {
-      lex_error (_("expecting a valid subcommand"));
+      lex_error (lexer, _("expecting a valid subcommand"));
       return false;
     }
 }
 
 /* Parses and performs the RENAME subcommand of GET and SAVE. */
 static bool
-rename_variables (struct dictionary *dict)
+rename_variables (struct lexer *lexer, struct dictionary *dict)
 {
   size_t i;
 
@@ -625,29 +625,29 @@ rename_variables (struct dictionary *dict)
 
   int group;
 
-  lex_match ('=');
-  if (token != '(')
+  lex_match (lexer, '=');
+  if (lex_token (lexer) != '(')
     {
       struct variable *v;
 
-      v = parse_variable (dict);
+      v = parse_variable (lexer, dict);
       if (v == NULL)
 	return 0;
-      if (!lex_force_match ('=')
-	  || !lex_force_id ())
+      if (!lex_force_match (lexer, '=')
+	  || !lex_force_id (lexer))
 	return 0;
-      if (dict_lookup_var (dict, tokid) != NULL)
+      if (dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
 	{
 	  msg (SE, _("Cannot rename %s as %s because there already exists "
 		     "a variable named %s.  To rename variables with "
 		     "overlapping names, use a single RENAME subcommand "
 		     "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
-		     "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
+		     "\"/RENAME (A B C=B C A)\"."), v->name, lex_tokid (lexer), lex_tokid (lexer));
 	  return 0;
 	}
       
-      dict_rename_var (dict, v, tokid);
-      lex_get ();
+      dict_rename_var (dict, v, lex_tokid (lexer));
+      lex_get (lexer);
       return 1;
     }
 
@@ -655,18 +655,18 @@ rename_variables (struct dictionary *dict)
   v = NULL;
   new_names = 0;
   group = 1;
-  while (lex_match ('('))
+  while (lex_match (lexer, '('))
     {
       size_t old_nv = nv;
 
-      if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
+      if (!parse_variables (lexer, dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
 	goto done;
-      if (!lex_match ('='))
+      if (!lex_match (lexer, '='))
 	{
 	  msg (SE, _("`=' expected after variable list."));
 	  goto done;
 	}
-      if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
+      if (!parse_DATA_LIST_vars (lexer, &new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
 	goto done;
       if (nn != nv)
 	{
@@ -676,7 +676,7 @@ rename_variables (struct dictionary *dict)
 	       (unsigned) (nv - old_nv), (unsigned) (nn - old_nv), group);
 	  goto done;
 	}
-      if (!lex_force_match (')'))
+      if (!lex_force_match (lexer, ')'))
 	goto done;
       group++;
     }
@@ -700,13 +700,13 @@ rename_variables (struct dictionary *dict)
 /* Parses and performs the DROP subcommand of GET and SAVE.
    Returns true if successful, false on failure.*/
 static bool
-drop_variables (struct dictionary *dict)
+drop_variables (struct lexer *lexer, struct dictionary *dict)
 {
   struct variable **v;
   size_t nv;
 
-  lex_match ('=');
-  if (!parse_variables (dict, &v, &nv, PV_NONE))
+  lex_match (lexer, '=');
+  if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
     return false;
   dict_delete_vars (dict, v, nv);
   free (v);
@@ -722,14 +722,14 @@ drop_variables (struct dictionary *dict)
 /* Parses and performs the KEEP subcommand of GET and SAVE.
    Returns true if successful, false on failure.*/
 static bool
-keep_variables (struct dictionary *dict)
+keep_variables (struct lexer *lexer, struct dictionary *dict)
 {
   struct variable **v;
   size_t nv;
   size_t i;
 
-  lex_match ('=');
-  if (!parse_variables (dict, &v, &nv, PV_NONE))
+  lex_match (lexer, '=');
+  if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
     return false;
 
   /* Move the specified variables to the beginning. */
@@ -810,7 +810,7 @@ static struct variable *get_master (struct variable *);
 
 /* Parse and execute the MATCH FILES command. */
 int
-cmd_match_files (struct dataset *ds)
+cmd_match_files (struct lexer *lexer, struct dataset *ds)
 {
   struct mtf_proc mtf;
   struct mtf_file *first_table = NULL;
@@ -833,22 +833,22 @@ cmd_match_files (struct dataset *ds)
   mtf.seq_nums = NULL;
   dict_set_case_limit (mtf.dict, dict_get_case_limit (dataset_dict (ds)));
 
-  lex_match ('/');
-  while (token == T_ID
-         && (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid)))
+  lex_match (lexer, '/');
+  while (lex_token (lexer) == T_ID
+         && (lex_id_match ("FILE", lex_tokid (lexer)) || lex_id_match ("TABLE", lex_tokid (lexer))))
     {
       struct mtf_file *file = xmalloc (sizeof *file);
 
-      if (lex_match_id ("FILE"))
+      if (lex_match_id (lexer, "FILE"))
         file->type = MTF_FILE;
-      else if (lex_match_id ("TABLE"))
+      else if (lex_match_id (lexer, "TABLE"))
         {
           file->type = MTF_TABLE;
           saw_table = true;
         }
       else
         NOT_REACHED ();
-      lex_match ('=');
+      lex_match (lexer, '=');
 
       file->by = NULL;
       file->handle = NULL;
@@ -883,7 +883,7 @@ cmd_match_files (struct dataset *ds)
           first_table->prev = file;
         }
 
-      if (lex_match ('*'))
+      if (lex_match (lexer, '*'))
         {
           file->handle = NULL;
           file->reader = NULL;
@@ -913,7 +913,7 @@ cmd_match_files (struct dataset *ds)
         }
       else
         {
-          file->handle = fh_parse (FH_REF_FILE | FH_REF_SCRATCH);
+          file->handle = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH);
           if (file->handle == NULL)
             goto error;
 
@@ -924,18 +924,18 @@ cmd_match_files (struct dataset *ds)
           case_create (&file->input, dict_get_next_value_idx (file->dict));
         }
 
-      while (lex_match ('/'))
-        if (lex_match_id ("RENAME")) 
+      while (lex_match (lexer, '/'))
+        if (lex_match_id (lexer, "RENAME")) 
           {
-            if (!rename_variables (file->dict))
+            if (!rename_variables (lexer, file->dict))
               goto error; 
           }
-        else if (lex_match_id ("IN"))
+        else if (lex_match_id (lexer, "IN"))
           {
-            lex_match ('=');
-            if (token != T_ID)
+            lex_match (lexer, '=');
+            if (lex_token (lexer) != T_ID)
               {
-                lex_error (NULL);
+                lex_error (lexer, NULL);
                 goto error;
               }
 
@@ -945,17 +945,17 @@ cmd_match_files (struct dataset *ds)
                            "TABLE."));
                 goto error;
               }
-            file->in_name = xstrdup (tokid);
-            lex_get ();
+            file->in_name = xstrdup (lex_tokid (lexer));
+            lex_get (lexer);
             saw_in = true;
           }
 
       mtf_merge_dictionary (mtf.dict, file);
     }
   
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
-      if (lex_match (T_BY))
+      if (lex_match (lexer, T_BY))
 	{
           struct variable **by;
           
@@ -965,8 +965,8 @@ cmd_match_files (struct dataset *ds)
 	      goto error;
 	    }
 	      
-	  lex_match ('=');
-	  if (!parse_variables (mtf.dict, &by, &mtf.by_cnt,
+	  lex_match (lexer, '=');
+	  if (!parse_variables (lexer, mtf.dict, &by, &mtf.by_cnt,
 				PV_NO_DUPLICATE | PV_NO_SCRATCH))
 	    goto error;
 
@@ -991,7 +991,7 @@ cmd_match_files (struct dataset *ds)
             }
           free (by);
 	}
-      else if (lex_match_id ("FIRST")) 
+      else if (lex_match_id (lexer, "FIRST")) 
         {
           if (mtf.first[0] != '\0')
             {
@@ -999,13 +999,13 @@ cmd_match_files (struct dataset *ds)
               goto error;
             }
 	      
-	  lex_match ('=');
-          if (!lex_force_id ())
+	  lex_match (lexer, '=');
+          if (!lex_force_id (lexer))
             goto error;
-          strcpy (mtf.first, tokid);
-          lex_get ();
+          strcpy (mtf.first, lex_tokid (lexer));
+          lex_get (lexer);
         }
-      else if (lex_match_id ("LAST")) 
+      else if (lex_match_id (lexer, "LAST")) 
         {
           if (mtf.last[0] != '\0')
             {
@@ -1013,35 +1013,35 @@ cmd_match_files (struct dataset *ds)
               goto error;
             }
 	      
-	  lex_match ('=');
-          if (!lex_force_id ())
+	  lex_match (lexer, '=');
+          if (!lex_force_id (lexer))
             goto error;
-          strcpy (mtf.last, tokid);
-          lex_get ();
+          strcpy (mtf.last, lex_tokid (lexer));
+          lex_get (lexer);
         }
-      else if (lex_match_id ("MAP"))
+      else if (lex_match_id (lexer, "MAP"))
 	{
 	  /* FIXME. */
 	}
-      else if (lex_match_id ("DROP")) 
+      else if (lex_match_id (lexer, "DROP")) 
         {
-          if (!drop_variables (mtf.dict))
+          if (!drop_variables (lexer, mtf.dict))
             goto error;
         }
-      else if (lex_match_id ("KEEP")) 
+      else if (lex_match_id (lexer, "KEEP")) 
         {
-          if (!keep_variables (mtf.dict))
+          if (!keep_variables (lexer, mtf.dict))
             goto error;
         }
       else
 	{
-	  lex_error (NULL);
+	  lex_error (lexer, NULL);
 	  goto error;
 	}
 
-      if (!lex_match ('/') && token != '.') 
+      if (!lex_match (lexer, '/') && lex_token (lexer) != '.') 
         {
-          lex_end_of_command ();
+          lex_end_of_command (lexer);
           goto error;
         }
     }
diff --git a/src/language/data-io/inpt-pgm.c b/src/language/data-io/inpt-pgm.c
index f07e119a..3459ee0f 100644
--- a/src/language/data-io/inpt-pgm.c
+++ b/src/language/data-io/inpt-pgm.c
@@ -103,15 +103,15 @@ emit_END_CASE (struct dataset *ds, struct input_program_pgm *inp)
 }
 
 int
-cmd_input_program (struct dataset *ds)
+cmd_input_program (struct lexer *lexer, struct dataset *ds)
 {
   struct input_program_pgm *inp;
   size_t i;
   bool saw_END_CASE = false;
 
   discard_variables (ds);
-  if (token != '.')
-    return lex_end_of_command ();
+  if (lex_token (lexer) != '.')
+    return lex_end_of_command (lexer);
 
   inp = xmalloc (sizeof *inp);
   inp->trns_chain = NULL;
@@ -120,7 +120,7 @@ cmd_input_program (struct dataset *ds)
   inside_input_program = true;
   for (;;) 
     {
-      enum cmd_result result = cmd_parse (ds, CMD_STATE_INPUT_PROGRAM);
+      enum cmd_result result = cmd_parse (lexer, ds, CMD_STATE_INPUT_PROGRAM);
       if (result == CMD_END_INPUT_PROGRAM)
         break;
       else if (result == CMD_END_CASE) 
@@ -181,7 +181,7 @@ cmd_input_program (struct dataset *ds)
 }
 
 int
-cmd_end_input_program (struct dataset *ds UNUSED)
+cmd_end_input_program (struct lexer *lexer UNUSED, struct dataset *ds UNUSED)
 {
   assert (in_input_program ());
   return CMD_END_INPUT_PROGRAM; 
@@ -288,12 +288,12 @@ static const struct case_source_class input_program_source_class =
   };
 
 int
-cmd_end_case (struct dataset *ds UNUSED)
+cmd_end_case (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   assert (in_input_program ());
-  if (token == '.')
+  if (lex_token (lexer) == '.')
     return CMD_END_CASE;
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Sends the current case as the source's output. */
@@ -319,7 +319,7 @@ struct reread_trns
 
 /* Parses REREAD command. */
 int
-cmd_reread (struct dataset *ds)
+cmd_reread (struct lexer *lexer, struct dataset *ds)
 {
   struct file_handle *fh;       /* File to be re-read. */
   struct expression *e;         /* Expression for column to set. */
@@ -327,11 +327,11 @@ cmd_reread (struct dataset *ds)
 
   fh = fh_get_default_handle ();
   e = NULL;
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
-      if (lex_match_id ("COLUMN"))
+      if (lex_match_id (lexer, "COLUMN"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  
 	  if (e)
 	    {
@@ -340,14 +340,14 @@ cmd_reread (struct dataset *ds)
 	      return CMD_CASCADING_FAILURE;
 	    }
 	  
-	  e = expr_parse (ds, EXPR_NUMBER);
+	  e = expr_parse (lexer, ds, EXPR_NUMBER);
 	  if (!e)
 	    return CMD_CASCADING_FAILURE;
 	}
-      else if (lex_match_id ("FILE"))
+      else if (lex_match_id (lexer, "FILE"))
 	{
-	  lex_match ('=');
-          fh = fh_parse (FH_REF_FILE | FH_REF_INLINE);
+	  lex_match (lexer, '=');
+          fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE);
 	  if (fh == NULL)
 	    {
 	      expr_free (e);
@@ -356,13 +356,13 @@ cmd_reread (struct dataset *ds)
 	}
       else
 	{
-	  lex_error (NULL);
+	  lex_error (lexer, NULL);
 	  expr_free (e);
 	}
     }
 
   t = xmalloc (sizeof *t);
-  t->reader = dfm_open_reader (fh);
+  t->reader = dfm_open_reader (fh, lexer);
   t->column = e;
   add_transformation (ds, reread_trns_proc, reread_trns_free, t);
 
@@ -405,13 +405,13 @@ reread_trns_free (void *t_)
 
 /* Parses END FILE command. */
 int
-cmd_end_file (struct dataset *ds)
+cmd_end_file (struct lexer *lexer, struct dataset *ds)
 {
   assert (in_input_program ());
 
   add_transformation (ds, end_file_trns_proc, NULL, NULL);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Executes an END FILE transformation. */
diff --git a/src/language/data-io/list.q b/src/language/data-io/list.q
index d8256e32..71238e8e 100644
--- a/src/language/data-io/list.q
+++ b/src/language/data-io/list.q
@@ -130,12 +130,12 @@ write_line (struct outp_driver *d, const char *s)
     
 /* Parses and executes the LIST procedure. */
 int
-cmd_list (struct dataset *ds)
+cmd_list (struct lexer *lexer, struct dataset *ds)
 {
   struct variable casenum_var;
   bool ok;
 
-  if (!parse_list (ds, &cmd, NULL))
+  if (!parse_list (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
   
   /* Fill in defaults. */
diff --git a/src/language/data-io/matrix-data.c b/src/language/data-io/matrix-data.c
index 5398cd69..a1e5d218 100644
--- a/src/language/data-io/matrix-data.c
+++ b/src/language/data-io/matrix-data.c
@@ -170,11 +170,11 @@ static int compare_variables_by_mxd_var_type (const void *pa,
 					     const void *pb);
 static bool read_matrices_without_rowtype (struct dataset *ds, struct matrix_data_pgm *);
 static bool read_matrices_with_rowtype (struct dataset *ds, struct matrix_data_pgm *);
-static int string_to_content_type (char *, int *);
+static int string_to_content_type (const char *, int *);
 static void attach_mxd_aux (struct variable *, int var_type, int sub_type);
 
 int
-cmd_matrix_data (struct dataset *ds)
+cmd_matrix_data (struct lexer *lexer, struct dataset *ds)
 {
   struct pool *pool;
   struct matrix_data_pgm *mx;
@@ -204,11 +204,11 @@ cmd_matrix_data (struct dataset *ds)
   mx->n_contents = 0;
   mx->n_continuous = 0;
   mx->first_continuous = 0;
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
-      lex_match ('/');
+      lex_match (lexer, '/');
 
-      if (lex_match_id ("VARIABLES"))
+      if (lex_match_id (lexer, "VARIABLES"))
 	{
 	  char **v;
 	  size_t nv;
@@ -220,8 +220,8 @@ cmd_matrix_data (struct dataset *ds)
 	    }
 	  seen |= 1;
 	  
-	  lex_match ('=');
-	  if (!parse_DATA_LIST_vars (&v, &nv, PV_NO_DUPLICATE))
+	  lex_match (lexer, '=');
+	  if (!parse_DATA_LIST_vars (lexer, &v, &nv, PV_NO_DUPLICATE))
 	    goto lossage;
 	  
 	  {
@@ -262,43 +262,43 @@ cmd_matrix_data (struct dataset *ds)
                                                  "ROWTYPE_", 8);
           attach_mxd_aux (mx->rowtype_, MXD_ROWTYPE, 0);
 	}
-      else if (lex_match_id ("FILE"))
+      else if (lex_match_id (lexer, "FILE"))
 	{
-	  lex_match ('=');
-	  fh = fh_parse (FH_REF_FILE | FH_REF_INLINE);
+	  lex_match (lexer, '=');
+	  fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE);
 	  if (fh == NULL)
 	    goto lossage;
 	}
-      else if (lex_match_id ("FORMAT"))
+      else if (lex_match_id (lexer, "FORMAT"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
-	  while (token == T_ID)
+	  while (lex_token (lexer) == T_ID)
 	    {
-	      if (lex_match_id ("LIST"))
+	      if (lex_match_id (lexer, "LIST"))
 		mx->fmt = LIST;
-	      else if (lex_match_id ("FREE"))
+	      else if (lex_match_id (lexer, "FREE"))
 		mx->fmt = FREE;
-	      else if (lex_match_id ("LOWER"))
+	      else if (lex_match_id (lexer, "LOWER"))
 		mx->section = LOWER;
-	      else if (lex_match_id ("UPPER"))
+	      else if (lex_match_id (lexer, "UPPER"))
 		mx->section = UPPER;
-	      else if (lex_match_id ("FULL"))
+	      else if (lex_match_id (lexer, "FULL"))
 		mx->section = FULL;
-	      else if (lex_match_id ("DIAGONAL"))
+	      else if (lex_match_id (lexer, "DIAGONAL"))
 		mx->diag = DIAGONAL;
-	      else if (lex_match_id ("NODIAGONAL"))
+	      else if (lex_match_id (lexer, "NODIAGONAL"))
 		mx->diag = NODIAGONAL;
 	      else 
 		{
-		  lex_error (_("in FORMAT subcommand"));
+		  lex_error (lexer, _("in FORMAT subcommand"));
 		  goto lossage;
 		}
 	    }
 	}
-      else if (lex_match_id ("SPLIT"))
+      else if (lex_match_id (lexer, "SPLIT"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
 	  if (seen & 2)
 	    {
@@ -307,17 +307,17 @@ cmd_matrix_data (struct dataset *ds)
 	    }
 	  seen |= 2;
 	  
-	  if (token != T_ID)
+	  if (lex_token (lexer) != T_ID)
 	    {
-	      lex_error (_("in SPLIT subcommand"));
+	      lex_error (lexer, _("in SPLIT subcommand"));
 	      goto lossage;
 	    }
 	  
-	  if (dict_lookup_var (dataset_dict (ds), tokid) == NULL
-	      && (lex_look_ahead () == '.' || lex_look_ahead () == '/'))
+	  if (dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL
+	      && (lex_look_ahead (lexer) == '.' || lex_look_ahead (lexer) == '/'))
 	    {
-	      if (!strcasecmp (tokid, "ROWTYPE_")
-                  || !strcasecmp (tokid, "VARNAME_"))
+	      if (!strcasecmp (lex_tokid (lexer), "ROWTYPE_")
+                  || !strcasecmp (lex_tokid (lexer), "VARNAME_"))
 		{
 		  msg (SE, _("Split variable may not be named ROWTYPE_ "
 			     "or VARNAME_."));
@@ -325,9 +325,9 @@ cmd_matrix_data (struct dataset *ds)
 		}
 
 	      mx->single_split = dict_create_var_assert (dataset_dict (ds),
-                                                         tokid, 0);
+                                                         lex_tokid (lexer), 0);
               attach_mxd_aux (mx->single_split, MXD_CONTINUOUS, 0);
-	      lex_get ();
+	      lex_get (lexer);
 
               dict_set_split_vars (dataset_dict (ds), &mx->single_split, 1);
 	    }
@@ -336,7 +336,8 @@ cmd_matrix_data (struct dataset *ds)
 	      struct variable **split;
 	      size_t n;
 
-	      if (!parse_variables (dataset_dict (ds), &split, &n, PV_NO_DUPLICATE))
+	      if (!parse_variables (lexer, dataset_dict (ds), 
+				    &split, &n, PV_NO_DUPLICATE))
 		goto lossage;
 
               dict_set_split_vars (dataset_dict (ds), split, n);
@@ -354,7 +355,7 @@ cmd_matrix_data (struct dataset *ds)
 		if (mv->var_type != MXD_CONTINUOUS)
 		  {
 		    msg (SE, _("Split variable %s is already another type."),
-			 tokid);
+			 lex_tokid (lexer));
 		    goto lossage;
 		  }
                 var_clear_aux (split[i]);
@@ -362,9 +363,9 @@ cmd_matrix_data (struct dataset *ds)
               }
 	  }
 	}
-      else if (lex_match_id ("FACTORS"))
+      else if (lex_match_id (lexer, "FACTORS"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  
 	  if (seen & 4)
 	    {
@@ -373,7 +374,7 @@ cmd_matrix_data (struct dataset *ds)
 	    }
 	  seen |= 4;
 
-	  if (!parse_variables (dataset_dict (ds), &mx->factors, &mx->n_factors,
+	  if (!parse_variables (lexer, dataset_dict (ds), &mx->factors, &mx->n_factors,
                                 PV_NONE))
 	    goto lossage;
 	  
@@ -388,7 +389,7 @@ cmd_matrix_data (struct dataset *ds)
 		if (mv->var_type != MXD_CONTINUOUS)
 		  {
 		    msg (SE, _("Factor variable %s is already another type."),
-			 tokid);
+			 lex_tokid (lexer));
 		    goto lossage;
 		  }
                 var_clear_aux (v);
@@ -396,9 +397,9 @@ cmd_matrix_data (struct dataset *ds)
 	      }
 	  }
 	}
-      else if (lex_match_id ("CELLS"))
+      else if (lex_match_id (lexer, "CELLS"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  
 	  if (mx->cells != -1)
 	    {
@@ -406,18 +407,18 @@ cmd_matrix_data (struct dataset *ds)
 	      goto lossage;
 	    }
 
-	  if (!lex_is_integer () || lex_integer () < 1)
+	  if (!lex_is_integer (lexer) || lex_integer (lexer) < 1)
 	    {
-	      lex_error (_("expecting positive integer"));
+	      lex_error (lexer, _("expecting positive integer"));
 	      goto lossage;
 	    }
 
-	  mx->cells = lex_integer ();
-	  lex_get ();
+	  mx->cells = lex_integer (lexer);
+	  lex_get (lexer);
 	}
-      else if (lex_match_id ("N"))
+      else if (lex_match_id (lexer, "N"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
 	  if (mx->pop_n != -1)
 	    {
@@ -425,16 +426,16 @@ cmd_matrix_data (struct dataset *ds)
 	      goto lossage;
 	    }
 
-	  if (!lex_is_integer () || lex_integer () < 1)
+	  if (!lex_is_integer (lexer) || lex_integer (lexer) < 1)
 	    {
-	      lex_error (_("expecting positive integer"));
+	      lex_error (lexer, _("expecting positive integer"));
 	      goto lossage;
 	    }
 
-	  mx->pop_n = lex_integer ();
-	  lex_get ();
+	  mx->pop_n = lex_integer (lexer);
+	  lex_get (lexer);
 	}
-      else if (lex_match_id ("CONTENTS"))
+      else if (lex_match_id (lexer, "CONTENTS"))
 	{
 	  int inside_parens = 0;
 	  unsigned collide = 0;
@@ -447,7 +448,7 @@ cmd_matrix_data (struct dataset *ds)
 	    }
 	  seen |= 8;
 
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  
 	  {
 	    int i;
@@ -458,7 +459,7 @@ cmd_matrix_data (struct dataset *ds)
 
 	  for (;;)
 	    {
-	      if (lex_match ('('))
+	      if (lex_match (lexer, '('))
 		{
 		  if (inside_parens)
 		    {
@@ -468,7 +469,7 @@ cmd_matrix_data (struct dataset *ds)
 		  inside_parens = 1;
 		  item = LPAREN;
 		}
-	      else if (lex_match (')'))
+	      else if (lex_match (lexer, ')'))
 		{
 		  if (!inside_parens)
 		    {
@@ -488,20 +489,20 @@ cmd_matrix_data (struct dataset *ds)
 		  int content_type;
 		  int collide_index;
 		  
-		  if (token != T_ID)
+		  if (lex_token (lexer) != T_ID)
 		    {
-		      lex_error (_("in CONTENTS subcommand"));
+		      lex_error (lexer, _("in CONTENTS subcommand"));
 		      goto lossage;
 		    }
 
-		  content_type = string_to_content_type (tokid,
+		  content_type = string_to_content_type (lex_tokid (lexer),
 							 &collide_index);
 		  if (content_type == -1)
 		    {
-		      lex_error (_("in CONTENTS subcommand"));
+		      lex_error (lexer, _("in CONTENTS subcommand"));
 		      goto lossage;
 		    }
-		  lex_get ();
+		  lex_get (lexer);
 
 		  if (collide & (1 << collide_index))
 		    {
@@ -516,7 +517,7 @@ cmd_matrix_data (struct dataset *ds)
 		}
 	      mx->contents[mx->n_contents++] = item;
 
-	      if (token == '/' || token == '.')
+	      if (lex_token (lexer) == '/' || lex_token (lexer) == '.')
 		break;
 	    }
 
@@ -529,14 +530,14 @@ cmd_matrix_data (struct dataset *ds)
 	}
       else 
 	{
-	  lex_error (NULL);
+	  lex_error (lexer, NULL);
 	  goto lossage;
 	}
     }
   
-  if (token != '.')
+  if (lex_token (lexer) != '.')
     {
-      lex_error (_("expecting end of command"));
+      lex_error (lexer, _("expecting end of command"));
       goto lossage;
     }
   
@@ -623,7 +624,7 @@ cmd_matrix_data (struct dataset *ds)
       goto lossage;
     }
 
-  mx->reader = dfm_open_reader (fh);
+  mx->reader = dfm_open_reader (fh, lexer);
   if (mx->reader == NULL)
     goto lossage;
 
@@ -651,7 +652,7 @@ lossage:
    as a bit-index) which can be used for determining whether a related
    statistic has already been used. */
 static int
-string_to_content_type (char *s, int *collide)
+string_to_content_type (const char *s, int *collide)
 {
   static const struct
     {
diff --git a/src/language/data-io/placement-parser.c b/src/language/data-io/placement-parser.c
index 82f9cd6c..074c5693 100644
--- a/src/language/data-io/placement-parser.c
+++ b/src/language/data-io/placement-parser.c
@@ -44,9 +44,9 @@ enum
     PRS_TYPE_NEW_REC            /* Next record. */
   };
 
-static bool fixed_parse_columns (struct pool *, size_t var_cnt, bool for_input,
+static bool fixed_parse_columns (struct lexer *, struct pool *, size_t var_cnt, bool for_input,
                                  struct fmt_spec **, size_t *);
-static bool fixed_parse_fortran (struct pool *, bool for_input,
+static bool fixed_parse_fortran (struct lexer *l, struct pool *, bool for_input,
                                  struct fmt_spec **, size_t *);
 
 /* Parses Fortran-like or column-based specifications for placing
@@ -65,18 +65,18 @@ static bool fixed_parse_fortran (struct pool *, bool for_input,
    Uses POOL for allocation.  When the caller is finished
    interpreting *FORMATS, POOL may be destroyed. */
 bool
-parse_var_placements (struct pool *pool, size_t var_cnt, bool for_input,
+parse_var_placements (struct lexer *lexer, struct pool *pool, size_t var_cnt, bool for_input,
                       struct fmt_spec **formats, size_t *format_cnt) 
 {
   assert (var_cnt > 0);
-  if (lex_is_number ())
-    return fixed_parse_columns (pool, var_cnt, for_input, formats, format_cnt);
-  else if (lex_match ('(')) 
+  if (lex_is_number (lexer))
+    return fixed_parse_columns (lexer, pool, var_cnt, for_input, formats, format_cnt);
+  else if (lex_match (lexer, '(')) 
     {
       size_t assignment_cnt;
       size_t i;
 
-      if (!fixed_parse_fortran (pool, for_input, formats, format_cnt))
+      if (!fixed_parse_fortran (lexer, pool, for_input, formats, format_cnt))
         return false; 
 
       assignment_cnt = 0;
@@ -103,14 +103,14 @@ parse_var_placements (struct pool *pool, size_t var_cnt, bool for_input,
 
 /* Implements parse_var_placements for column-based formats. */
 static bool
-fixed_parse_columns (struct pool *pool, size_t var_cnt, bool for_input,
+fixed_parse_columns (struct lexer *lexer, struct pool *pool, size_t var_cnt, bool for_input,
                      struct fmt_spec **formats, size_t *format_cnt)
 {
   struct fmt_spec format;
   int fc, lc;
   size_t i;
 
-  if (!parse_column_range (&fc, &lc, NULL))
+  if ( !parse_column_range (lexer, &fc, &lc, NULL) )
     return false;
 
   /* Divide columns evenly. */    
@@ -124,28 +124,28 @@ fixed_parse_columns (struct pool *pool, size_t var_cnt, bool for_input,
     }
 
   /* Format specifier. */
-  if (lex_match ('('))
+  if (lex_match (lexer, '('))
     {
       /* Get format type. */
-      if (token == T_ID)
+      if (lex_token (lexer) == T_ID)
 	{
-	  if (!parse_format_specifier_name (&format.type))
+	  if (!parse_format_specifier_name (lexer, &format.type))
             return false;
-	  lex_match (',');
+	  lex_match (lexer, ',');
 	}
       else
 	format.type = FMT_F;
 
       /* Get decimal places. */
-      if (lex_is_integer ())
+      if (lex_is_integer (lexer))
 	{
-	  format.d = lex_integer ();
-	  lex_get ();
+	  format.d = lex_integer (lexer);
+	  lex_get (lexer);
 	}
       else
 	format.d = 0;
 
-      if (!lex_force_match (')'))
+      if (!lex_force_match (lexer, ')'))
 	return false;
     }
   else
@@ -167,14 +167,14 @@ fixed_parse_columns (struct pool *pool, size_t var_cnt, bool for_input,
 
 /* Implements parse_var_placements for Fortran-like formats. */
 static bool
-fixed_parse_fortran (struct pool *pool, bool for_input,
+fixed_parse_fortran (struct lexer *lexer, struct pool *pool, bool for_input,
                      struct fmt_spec **formats, size_t *format_cnt)
 {
   size_t formats_allocated = 0;
   size_t formats_used = 0;
 
   *formats = NULL;
-  while (!lex_match (')'))
+  while (!lex_match (lexer, ')'))
     {
       struct fmt_spec f;
       struct fmt_spec *new_formats;
@@ -183,19 +183,19 @@ fixed_parse_fortran (struct pool *pool, bool for_input,
       size_t formats_needed;
       
       /* Parse count. */
-      if (lex_is_integer ())
+      if (lex_is_integer (lexer))
 	{
-	  count = lex_integer ();
-	  lex_get ();
+	  count = lex_integer (lexer);
+	  lex_get (lexer);
 	}
       else
 	count = 1;
 
       /* Parse format specifier. */
-      if (lex_match ('('))
+      if (lex_match (lexer, '('))
         {
           /* Call ourselves recursively to handle parentheses. */
-          if (!fixed_parse_fortran (pool, for_input,
+          if (!fixed_parse_fortran (lexer, pool, for_input,
                                     &new_formats, &new_format_cnt))
             return false;
         }
@@ -203,13 +203,13 @@ fixed_parse_fortran (struct pool *pool, bool for_input,
         {
           new_formats = &f;
           new_format_cnt = 1;
-          if (lex_match ('/'))
+          if (lex_match (lexer, '/'))
             f.type = PRS_TYPE_NEW_REC;
           else
             {
               char type[FMT_TYPE_LEN_MAX + 1];
               
-              if (!parse_abstract_format_specifier (type, &f.w, &f.d))
+              if (!parse_abstract_format_specifier (lexer, type, &f.w, &f.d))
                 return false;
 
               if (!strcasecmp (type, "T")) 
@@ -254,7 +254,7 @@ fixed_parse_fortran (struct pool *pool, bool for_input,
           formats_used += new_format_cnt;
         }
 
-      lex_match (',');
+      lex_match (lexer, ',');
     }
 
   *format_cnt = formats_used;
@@ -299,27 +299,27 @@ execute_placement_format (const struct fmt_spec *format,
    successful, false if the syntax was invalid or the values
    specified did not make sense. */
 bool
-parse_column_range (int *first_column, int *last_column,
+parse_column_range (struct lexer *lexer, int *first_column, int *last_column,
                     bool *range_specified) 
 {
   /* First column. */
-  if (!lex_force_int ())
+  if (!lex_force_int (lexer))
     return false;
-  *first_column = lex_integer ();
+  *first_column = lex_integer (lexer);
   if (*first_column < 1)
     {
       msg (SE, _("Column positions for fields must be positive."));
       return false;
     }
-  lex_get ();
+  lex_get (lexer);
 
   /* Last column. */
-  lex_negative_to_dash ();
-  if (lex_match ('-'))
+  lex_negative_to_dash (lexer);
+  if (lex_match (lexer, '-'))
     {
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return false;
-      *last_column = lex_integer ();
+      *last_column = lex_integer (lexer);
       if (*last_column < 1)
 	{
 	  msg (SE, _("Column positions for fields must be positive."));
@@ -334,7 +334,7 @@ parse_column_range (int *first_column, int *last_column,
 
       if (range_specified)
         *range_specified = true;
-      lex_get ();
+      lex_get (lexer);
     }
   else 
     {
@@ -354,23 +354,23 @@ parse_column_range (int *first_column, int *last_column,
 
    Returns true if successful, false on syntax error. */
 bool
-parse_record_placement (int *record, int *column) 
+parse_record_placement (struct lexer *lexer, int *record, int *column) 
 {
-  while (lex_match ('/'))
+  while (lex_match (lexer, '/'))
     {
-      if (lex_is_integer ())
+      if (lex_is_integer (lexer))
         {
-          if (lex_integer () <= *record)
+          if (lex_integer (lexer) <= *record)
             {
               msg (SE, _("The record number specified, %ld, is at or "
                          "before the previous record, %d.  Data "
                          "fields must be listed in order of "
                          "increasing record number."),
-                   lex_integer (), *record);
+                   lex_integer (lexer), *record);
               return false;
             }
-          *record = lex_integer ();
-          lex_get ();
+          *record = lex_integer (lexer);
+          lex_get (lexer);
         }
       else
         (*record)++;
diff --git a/src/language/data-io/placement-parser.h b/src/language/data-io/placement-parser.h
index 1204ceb8..166b9f99 100644
--- a/src/language/data-io/placement-parser.h
+++ b/src/language/data-io/placement-parser.h
@@ -25,13 +25,14 @@
 
 struct fmt_spec;
 struct pool;
+struct lexer;
 
-bool parse_record_placement (int *record, int *column);
-bool parse_var_placements (struct pool *, size_t var_cnt, bool for_input,
+bool parse_record_placement (struct lexer *, int *record, int *column);
+bool parse_var_placements (struct lexer *, struct pool *, size_t var_cnt, bool for_input,
                            struct fmt_spec **, size_t *format_cnt);
 bool execute_placement_format (const struct fmt_spec *,
                                int *record, int *column);
-bool parse_column_range (int *first_column, int *last_column,
+bool parse_column_range (struct lexer *, int *first_column, int *last_column,
                          bool *range_specified);
 
 #endif /* language/data-io/placement-parser.h */
diff --git a/src/language/data-io/print-space.c b/src/language/data-io/print-space.c
index 61a046b5..8ddd0be5 100644
--- a/src/language/data-io/print-space.c
+++ b/src/language/data-io/print-space.c
@@ -47,32 +47,32 @@ static trns_proc_func print_space_trns_proc;
 static trns_free_func print_space_trns_free;
 
 int
-cmd_print_space (struct dataset *ds)
+cmd_print_space (struct lexer *lexer, struct dataset *ds)
 {
   struct print_space_trns *trns;
   struct file_handle *handle;
   struct expression *expr;
   struct dfm_writer *writer;
 
-  if (lex_match_id ("OUTFILE"))
+  if (lex_match_id (lexer, "OUTFILE"))
     {
-      lex_match ('=');
+      lex_match (lexer, '=');
 
-      handle = fh_parse (FH_REF_FILE);
+      handle = fh_parse (lexer, FH_REF_FILE);
       if (handle == NULL)
 	return CMD_FAILURE;
-      lex_get ();
+      lex_get (lexer);
     }
   else
     handle = NULL;
 
-  if (token != '.')
+  if (lex_token (lexer) != '.')
     {
-      expr = expr_parse (ds, EXPR_NUMBER);
-      if (token != '.')
+      expr = expr_parse (lexer, ds, EXPR_NUMBER);
+      if (lex_token (lexer) != '.')
 	{
 	  expr_free (expr);
-	  lex_error (_("expecting end of command"));
+	  lex_error (lexer, _("expecting end of command"));
 	  return CMD_FAILURE;
 	}
     }
diff --git a/src/language/data-io/print.c b/src/language/data-io/print.c
index 262d5397..f15bcdcb 100644
--- a/src/language/data-io/print.c
+++ b/src/language/data-io/print.c
@@ -96,11 +96,11 @@ enum which_formats
     WRITE
   };
 
-static int internal_cmd_print (struct dataset *ds, 
+static int internal_cmd_print (struct lexer *, struct dataset *ds, 
 			       enum which_formats, bool eject);
 static trns_proc_func print_trns_proc;
 static trns_free_func print_trns_free;
-static bool parse_specs (struct pool *tmp_pool, struct print_trns *,
+static bool parse_specs (struct lexer *, struct pool *tmp_pool, struct print_trns *,
 			 struct dictionary *dict, enum which_formats);
 static void dump_table (struct print_trns *, const struct file_handle *);
 
@@ -108,28 +108,28 @@ static void dump_table (struct print_trns *, const struct file_handle *);
 
 /* Parses PRINT command. */
 int
-cmd_print (struct dataset *ds)
+cmd_print (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_print (ds, PRINT, false);
+  return internal_cmd_print (lexer, ds, PRINT, false);
 }
 
 /* Parses PRINT EJECT command. */
 int
-cmd_print_eject (struct dataset *ds)
+cmd_print_eject (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_print (ds, PRINT, true);
+  return internal_cmd_print (lexer, ds, PRINT, true);
 }
 
 /* Parses WRITE command. */
 int
-cmd_write (struct dataset *ds)
+cmd_write (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_print (ds, WRITE, false);
+  return internal_cmd_print (lexer, ds, WRITE, false);
 }
 
 /* Parses the output commands. */
 static int
-internal_cmd_print (struct dataset *ds, 
+internal_cmd_print (struct lexer *lexer, struct dataset *ds, 
 		    enum which_formats which_formats, bool eject)
 {
   bool print_table = 0;
@@ -149,33 +149,33 @@ internal_cmd_print (struct dataset *ds,
   tmp_pool = pool_create_subpool (trns->pool);
 
   /* Parse the command options. */
-  while (token != '/' && token != '.')
+  while (lex_token (lexer) != '/' && lex_token (lexer) != '.')
     {
-      if (lex_match_id ("OUTFILE"))
+      if (lex_match_id (lexer, "OUTFILE"))
 	{
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 
-	  fh = fh_parse (FH_REF_FILE);
+	  fh = fh_parse (lexer, FH_REF_FILE);
 	  if (fh == NULL)
 	    goto error;
 	}
-      else if (lex_match_id ("RECORDS"))
+      else if (lex_match_id (lexer, "RECORDS"))
 	{
-	  lex_match ('=');
-	  lex_match ('(');
-	  if (!lex_force_int ())
+	  lex_match (lexer, '=');
+	  lex_match (lexer, '(');
+	  if (!lex_force_int (lexer))
 	    goto error;
-	  trns->record_cnt = lex_integer ();
-	  lex_get ();
-	  lex_match (')');
+	  trns->record_cnt = lex_integer (lexer);
+	  lex_get (lexer);
+	  lex_match (lexer, ')');
 	}
-      else if (lex_match_id ("TABLE"))
+      else if (lex_match_id (lexer, "TABLE"))
 	print_table = true;
-      else if (lex_match_id ("NOTABLE"))
+      else if (lex_match_id (lexer, "NOTABLE"))
 	print_table = false;
       else
 	{
-	  lex_error (_("expecting a valid subcommand"));
+	  lex_error (lexer, _("expecting a valid subcommand"));
 	  goto error;
 	}
     }
@@ -185,10 +185,10 @@ internal_cmd_print (struct dataset *ds,
   trns->include_prefix = which_formats == PRINT && fh != NULL;
 
   /* Parse variables and strings. */
-  if (!parse_specs (tmp_pool, trns, dataset_dict (ds), which_formats))
+  if (!parse_specs (lexer, tmp_pool, trns, dataset_dict (ds), which_formats))
     goto error;
 
-  if (lex_end_of_command () != CMD_SUCCESS)
+  if (lex_end_of_command (lexer) != CMD_SUCCESS)
     goto error;
 
   if (fh != NULL)
@@ -214,9 +214,9 @@ internal_cmd_print (struct dataset *ds,
   return CMD_FAILURE;
 }
 
-static bool parse_string_argument (struct print_trns *,
+static bool parse_string_argument (struct lexer *, struct print_trns *,
                                    int record, int *column);
-static bool parse_variable_argument (const struct dictionary *, 
+static bool parse_variable_argument (struct lexer *, const struct dictionary *, 
 				     struct print_trns *,
                                      struct pool *tmp_pool,
                                      int *record, int *column,
@@ -226,35 +226,35 @@ static bool parse_variable_argument (const struct dictionary *,
    PRINT, PRINT EJECT, or WRITE command into the prt structure.
    Returns success. */
 static bool
-parse_specs (struct pool *tmp_pool, struct print_trns *trns,
+parse_specs (struct lexer *lexer, struct pool *tmp_pool, struct print_trns *trns,
 	     struct dictionary *dict, 
              enum which_formats which_formats)
 {
   int record = 0;
   int column = 1;
 
-  if (token == '.') 
+  if (lex_token (lexer) == '.') 
     {
       trns->record_cnt = 1;
       return true;
     }
 
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
       bool ok;
 
-      if (!parse_record_placement (&record, &column))
+      if (!parse_record_placement (lexer, &record, &column))
         return false;
 
-      if (token == T_STRING)
-	ok = parse_string_argument (trns, record, &column);
+      if (lex_token (lexer) == T_STRING)
+	ok = parse_string_argument (lexer, trns, record, &column);
       else
-	ok = parse_variable_argument (dict, trns, tmp_pool, &record, &column,
+	ok = parse_variable_argument (lexer, dict, trns, tmp_pool, &record, &column,
                                       which_formats);
       if (!ok)
 	return 0;
 
-      lex_match (',');
+      lex_match (lexer, ',');
     }
 
   if (trns->record_cnt != 0 && trns->record_cnt != record)
@@ -268,23 +268,23 @@ parse_specs (struct pool *tmp_pool, struct print_trns *trns,
 
 /* Parses a string argument to the PRINT commands.  Returns success. */
 static bool
-parse_string_argument (struct print_trns *trns, int record, int *column)
+parse_string_argument (struct lexer *lexer, struct print_trns *trns, int record, int *column)
 {
   struct prt_out_spec *spec = pool_alloc (trns->pool, sizeof *spec);
   spec->type = PRT_LITERAL;
   spec->record = record;
   spec->first_column = *column;
-  ds_init_string (&spec->string, &tokstr);
+  ds_init_string (&spec->string, lex_tokstr (lexer));
   ds_register_pool (&spec->string, trns->pool);
-  lex_get ();
+  lex_get (lexer);
 
   /* Parse the included column range. */
-  if (lex_is_number ())
+  if (lex_is_number (lexer))
     {
       int first_column, last_column;
       bool range_specified;
 
-      if (!parse_column_range (&first_column, &last_column, &range_specified)) 
+      if (!parse_column_range (lexer, &first_column, &last_column, &range_specified)) 
         return false; 
 
       spec->first_column = first_column;
@@ -301,7 +301,7 @@ parse_string_argument (struct print_trns *trns, int record, int *column)
    to fixed_parse_compatible() or fixed_parse_fortran() as appropriate.
    Returns success. */
 static bool
-parse_variable_argument (const struct dictionary *dict,
+parse_variable_argument (struct lexer *lexer, const struct dictionary *dict,
 			 struct print_trns *trns, struct pool *tmp_pool,
                          int *record, int *column,
                          enum which_formats which_formats)
@@ -312,12 +312,13 @@ parse_variable_argument (const struct dictionary *dict,
   size_t format_cnt;
   bool add_space;
   
-  if (!parse_variables_pool (tmp_pool, dict, &vars, &var_cnt, PV_DUPLICATE))
+  if (!parse_variables_pool (lexer, tmp_pool, dict, 
+			     &vars, &var_cnt, PV_DUPLICATE))
     return false;
 
-  if (lex_is_number () || token == '(')
+  if (lex_is_number (lexer) || lex_token (lexer) == '(')
     {
-      if (!parse_var_placements (tmp_pool, var_cnt, false,
+      if (!parse_var_placements (lexer, tmp_pool, var_cnt, false,
                                  &formats, &format_cnt))
         return false;
       add_space = false;
@@ -326,7 +327,7 @@ parse_variable_argument (const struct dictionary *dict,
     {
       size_t i;
 
-      lex_match ('*');
+      lex_match (lexer, '*');
       
       formats = pool_nmalloc (tmp_pool, var_cnt, sizeof *formats);
       format_cnt = var_cnt;
diff --git a/src/language/dictionary/apply-dictionary.c b/src/language/dictionary/apply-dictionary.c
index e0434161..6756b58b 100644
--- a/src/language/dictionary/apply-dictionary.c
+++ b/src/language/dictionary/apply-dictionary.c
@@ -39,7 +39,7 @@
 
 /* Parses and executes APPLY DICTIONARY. */
 int
-cmd_apply_dictionary (struct dataset *ds)
+cmd_apply_dictionary (struct lexer *lexer, struct dataset *ds)
 {
   struct file_handle *handle;
   struct any_reader *reader;
@@ -49,9 +49,9 @@ cmd_apply_dictionary (struct dataset *ds)
 
   int i;
   
-  lex_match_id ("FROM");
-  lex_match ('=');
-  handle = fh_parse (FH_REF_FILE | FH_REF_SCRATCH);
+  lex_match_id (lexer, "FROM");
+  lex_match (lexer, '=');
+  handle = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH);
   if (!handle)
     return CMD_FAILURE;
 
@@ -136,5 +136,5 @@ cmd_apply_dictionary (struct dataset *ds)
   
   any_reader_close (reader);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/dictionary/formats.c b/src/language/dictionary/formats.c
index 384e45a9..71e4d95f 100644
--- a/src/language/dictionary/formats.c
+++ b/src/language/dictionary/formats.c
@@ -42,28 +42,28 @@ enum
     FORMATS_WRITE = 002
   };
 
-static int internal_cmd_formats (struct dataset *ds, int);
+static int internal_cmd_formats (struct lexer *, struct dataset *ds, int);
 
 int
-cmd_print_formats (struct dataset *ds)
+cmd_print_formats (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_formats (ds, FORMATS_PRINT);
+  return internal_cmd_formats (lexer, ds, FORMATS_PRINT);
 }
 
 int
-cmd_write_formats (struct dataset *ds)
+cmd_write_formats (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_formats (ds, FORMATS_WRITE);
+  return internal_cmd_formats (lexer, ds, FORMATS_WRITE);
 }
 
 int
-cmd_formats (struct dataset *ds)
+cmd_formats (struct lexer *lexer, struct dataset *ds)
 {
-  return internal_cmd_formats (ds, FORMATS_PRINT | FORMATS_WRITE);
+  return internal_cmd_formats (lexer, ds, FORMATS_PRINT | FORMATS_WRITE);
 }
 
 static int
-internal_cmd_formats (struct dataset *ds, int which)
+internal_cmd_formats (struct lexer *lexer, struct dataset *ds, int which)
 {
   /* Variables. */
   struct variable **v;
@@ -80,24 +80,24 @@ internal_cmd_formats (struct dataset *ds, int which)
 
   for (;;)
     {
-      if (token == '.')
+      if (lex_token (lexer) == '.')
 	break;
 
-      if (!parse_variables (dataset_dict (ds), &v, &cv, PV_NUMERIC))
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &cv, PV_NUMERIC))
 	return CMD_FAILURE;
       type = v[0]->type;
 
-      if (!lex_match ('('))
+      if (!lex_match (lexer, '('))
 	{
 	  msg (SE, _("`(' expected after variable list."));
 	  goto fail;
 	}
-      if (!parse_format_specifier (&f)
+      if (!parse_format_specifier (lexer, &f)
           || !fmt_check_output (&f)
           || !fmt_check_type_compat (&f, NUMERIC))
 	goto fail;
 
-      if (!lex_match (')'))
+      if (!lex_match (lexer, ')'))
 	{
 	  msg (SE, _("`)' expected after output format."));
 	  goto fail;
diff --git a/src/language/dictionary/missing-values.c b/src/language/dictionary/missing-values.c
index a330c46e..f6836996 100644
--- a/src/language/dictionary/missing-values.c
+++ b/src/language/dictionary/missing-values.c
@@ -37,7 +37,7 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_missing_values (struct dataset *ds)
+cmd_missing_values (struct lexer *lexer, struct dataset *ds)
 {
   struct variable **v;
   size_t nv;
@@ -45,23 +45,23 @@ cmd_missing_values (struct dataset *ds)
   int retval = CMD_FAILURE;
   bool deferred_errors = false;
 
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
       size_t i;
 
-      if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE)) 
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE)) 
         goto done;
 
-      if (!lex_match ('('))
+      if (!lex_match (lexer, '('))
         {
-          lex_error (_("expecting `('"));
+          lex_error (lexer, _("expecting `('"));
           goto done;
         }
 
       for (i = 0; i < nv; i++)
         mv_init (&v[i]->miss, v[i]->width);
 
-      if (!lex_match (')')) 
+      if (!lex_match (lexer, ')')) 
         {
           struct missing_values mv;
 
@@ -79,12 +79,12 @@ cmd_missing_values (struct dataset *ds)
           if (v[0]->type == NUMERIC) 
             {
               mv_init (&mv, 0);
-              while (!lex_match (')'))
+              while (!lex_match (lexer, ')'))
                 {
                   double x, y;
                   bool ok;
 
-                  if (!parse_num_range (&x, &y, &v[0]->print))
+                  if (!parse_num_range (lexer, &x, &y, &v[0]->print))
                     goto done;
                   
                   ok = (x == y
@@ -93,35 +93,40 @@ cmd_missing_values (struct dataset *ds)
                   if (!ok)
                     deferred_errors = true;
 
-                  lex_match (',');
+                  lex_match (lexer, ',');
                 }
             }
           else 
             {
+	      struct string value;
+
               mv_init (&mv, MAX_SHORT_STRING);
-              while (!lex_match (')')) 
+              while (!lex_match (lexer, ')')) 
                 {
-                  if (!lex_force_string ())
+                  if (!lex_force_string (lexer))
                     {
                       deferred_errors = true;
                       break;
                     }
+		  
+		  ds_init_string (&value, lex_tokstr (lexer));
 
-                  if (ds_length (&tokstr) > MAX_SHORT_STRING) 
+                  if (ds_length (&value) > MAX_SHORT_STRING) 
                     {
-                      ds_truncate (&tokstr, MAX_SHORT_STRING);
+                      ds_truncate (&value, MAX_SHORT_STRING);
                       msg (SE, _("Truncating missing value to short string "
                                  "length (%d characters)."),
                            MAX_SHORT_STRING);
                     }
                   else
-                    ds_rpad (&tokstr, MAX_SHORT_STRING, ' ');
+                    ds_rpad (&value, MAX_SHORT_STRING, ' ');
 
-                  if (!mv_add_str (&mv, ds_data (&tokstr)))
+                  if (!mv_add_str (&mv, ds_data (&value)))
                     deferred_errors = true;
+		  ds_destroy (&value);
 
-                  lex_get ();
-                  lex_match (',');
+                  lex_get (lexer);
+                  lex_match (lexer, ',');
                 }
             }
           
@@ -142,11 +147,11 @@ cmd_missing_values (struct dataset *ds)
             }
         }
 
-      lex_match ('/');
+      lex_match (lexer, '/');
       free (v);
       v = NULL;
     }
-  retval = lex_end_of_command ();
+  retval = lex_end_of_command (lexer);
   
  done:
   free (v);
diff --git a/src/language/dictionary/modify-variables.c b/src/language/dictionary/modify-variables.c
index f46dca7d..7c9a4b24 100644
--- a/src/language/dictionary/modify-variables.c
+++ b/src/language/dictionary/modify-variables.c
@@ -78,7 +78,7 @@ static bool rearrange_dict (struct dictionary *d,
 
 /* Performs MODIFY VARS command. */
 int
-cmd_modify_vars (struct dataset *ds)
+cmd_modify_vars (struct lexer *lexer, struct dataset *ds)
 {
   /* Bits indicated whether we've already encountered a subcommand of
      this type. */
@@ -105,10 +105,10 @@ cmd_modify_vars (struct dataset *ds)
   vm.drop_cnt = 0;
 
   /* Parse each subcommand. */
-  lex_match ('/');
+  lex_match (lexer, '/');
   for (;;)
     {
-      if (lex_match_id ("REORDER"))
+      if (lex_match_id (lexer, "REORDER"))
 	{
 	  struct variable **v = NULL;
 	  size_t nv = 0;
@@ -120,21 +120,21 @@ cmd_modify_vars (struct dataset *ds)
 	    }
 	  already_encountered |= 1;
 
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  do
 	    {
               struct ordering ordering;
 	      size_t prev_nv = nv;
 
 	      ordering.forward = ordering.positional = 1;
-	      if (lex_match_id ("FORWARD"));
-	      else if (lex_match_id ("BACKWARD"))
+	      if (lex_match_id (lexer, "FORWARD"));
+	      else if (lex_match_id (lexer, "BACKWARD"))
 		ordering.forward = 0;
-	      if (lex_match_id ("POSITIONAL"));
-	      else if (lex_match_id ("ALPHA"))
+	      if (lex_match_id (lexer, "POSITIONAL"));
+	      else if (lex_match_id (lexer, "ALPHA"))
 		ordering.positional = 0;
 
-	      if (lex_match (T_ALL) || token == '/' || token == '.')
+	      if (lex_match (lexer, T_ALL) || lex_token (lexer) == '/' || lex_token (lexer) == '.')
 		{
 		  if (prev_nv != 0)
 		    {
@@ -146,19 +146,19 @@ cmd_modify_vars (struct dataset *ds)
 		}
 	      else
 		{
-		  if (!lex_match ('('))
+		  if (!lex_match (lexer, '('))
 		    {
 		      msg (SE, _("`(' expected on REORDER subcommand."));
 		      free (v);
 		      goto done;
 		    }
-		  if (!parse_variables (dataset_dict (ds), &v, &nv,
+		  if (!parse_variables (lexer, dataset_dict (ds), &v, &nv,
 					PV_APPEND | PV_NO_DUPLICATE))
 		    {
 		      free (v);
 		      goto done;
 		    }
-		  if (!lex_match (')'))
+		  if (!lex_match (lexer, ')'))
 		    {
 		      msg (SE, _("`)' expected following variable names on "
 			   "REORDER subcommand."));
@@ -169,12 +169,12 @@ cmd_modify_vars (struct dataset *ds)
 	      sort (&v[prev_nv], nv - prev_nv, sizeof *v,
                     compare_variables_given_ordering, &ordering);
 	    }
-	  while (token != '/' && token != '.');
+	  while (lex_token (lexer) != '/' && lex_token (lexer) != '.');
 
 	  vm.reorder_vars = v;
           vm.reorder_cnt = nv;
 	}
-      else if (lex_match_id ("RENAME"))
+      else if (lex_match_id (lexer, "RENAME"))
 	{
 	  if (already_encountered & 2)
 	    {
@@ -183,27 +183,29 @@ cmd_modify_vars (struct dataset *ds)
 	    }
 	  already_encountered |= 2;
 
-	  lex_match ('=');
+	  lex_match (lexer, '=');
 	  do
 	    {
 	      size_t prev_nv_1 = vm.rename_cnt;
 	      size_t prev_nv_2 = vm.rename_cnt;
 
-	      if (!lex_match ('('))
+	      if (!lex_match (lexer, '('))
 		{
 		  msg (SE, _("`(' expected on RENAME subcommand."));
 		  goto done;
 		}
-	      if (!parse_variables (dataset_dict (ds), &vm.rename_vars, &vm.rename_cnt,
+	      if (!parse_variables (lexer, dataset_dict (ds), 
+				    &vm.rename_vars, &vm.rename_cnt,
 				    PV_APPEND | PV_NO_DUPLICATE))
 		goto done;
-	      if (!lex_match ('='))
+	      if (!lex_match (lexer, '='))
 		{
 		  msg (SE, _("`=' expected between lists of new and old variable "
 		       "names on RENAME subcommand."));
 		  goto done;
 		}
-	      if (!parse_DATA_LIST_vars (&vm.new_names, &prev_nv_1, PV_APPEND))
+	      if (!parse_DATA_LIST_vars (lexer, &vm.new_names, 
+					 &prev_nv_1, PV_APPEND))
 		goto done;
 	      if (prev_nv_1 != vm.rename_cnt)
 		{
@@ -216,16 +218,16 @@ cmd_modify_vars (struct dataset *ds)
 		  vm.new_names = NULL;
 		  goto done;
 		}
-	      if (!lex_match (')'))
+	      if (!lex_match (lexer, ')'))
 		{
 		  msg (SE, _("`)' expected after variable lists on RENAME "
 		       "subcommand."));
 		  goto done;
 		}
 	    }
-	  while (token != '.' && token != '/');
+	  while (lex_token (lexer) != '.' && lex_token (lexer) != '/');
 	}
-      else if (lex_match_id ("KEEP"))
+      else if (lex_match_id (lexer, "KEEP"))
 	{
 	  struct variable **keep_vars, **all_vars, **drop_vars;
 	  size_t keep_cnt, all_cnt, drop_cnt;
@@ -238,8 +240,8 @@ cmd_modify_vars (struct dataset *ds)
 	    }
 	  already_encountered |= 4;
 
-	  lex_match ('=');
-	  if (!parse_variables (dataset_dict (ds), &keep_vars, &keep_cnt, PV_NONE))
+	  lex_match (lexer, '=');
+	  if (!parse_variables (lexer, dataset_dict (ds), &keep_vars, &keep_cnt, PV_NONE))
 	    goto done;
 
 	  /* Transform the list of variables to keep into a list of
@@ -268,7 +270,7 @@ cmd_modify_vars (struct dataset *ds)
           vm.drop_vars = drop_vars;
           vm.drop_cnt = drop_cnt;
 	}
-      else if (lex_match_id ("DROP"))
+      else if (lex_match_id (lexer, "DROP"))
 	{
 	  struct variable **drop_vars;
 	  size_t drop_cnt;
@@ -282,13 +284,13 @@ cmd_modify_vars (struct dataset *ds)
 	    }
 	  already_encountered |= 4;
 
-	  lex_match ('=');
-	  if (!parse_variables (dataset_dict (ds), &drop_vars, &drop_cnt, PV_NONE))
+	  lex_match (lexer, '=');
+	  if (!parse_variables (lexer, dataset_dict (ds), &drop_vars, &drop_cnt, PV_NONE))
 	    goto done;
           vm.drop_vars = drop_vars;
           vm.drop_cnt = drop_cnt;
 	}
-      else if (lex_match_id ("MAP"))
+      else if (lex_match_id (lexer, "MAP"))
 	{
           struct dictionary *temp = dict_clone (dataset_dict (ds));
           int success = rearrange_dict (temp, &vm);
@@ -300,21 +302,21 @@ cmd_modify_vars (struct dataset *ds)
 	}
       else
 	{
-	  if (token == T_ID)
-	    msg (SE, _("Unrecognized subcommand name `%s'."), tokid);
+	  if (lex_token (lexer) == T_ID)
+	    msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokid (lexer));
 	  else
 	    msg (SE, _("Subcommand name expected."));
 	  goto done;
 	}
 
-      if (token == '.')
+      if (lex_token (lexer) == '.')
 	break;
-      if (token != '/')
+      if (lex_token (lexer) != '/')
 	{
 	  msg (SE, _("`/' or `.' expected."));
 	  goto done;
 	}
-      lex_get ();
+      lex_get (lexer);
     }
 
   if (already_encountered & (1 | 4))
diff --git a/src/language/dictionary/numeric.c b/src/language/dictionary/numeric.c
index ae961963..b7ad96eb 100644
--- a/src/language/dictionary/numeric.c
+++ b/src/language/dictionary/numeric.c
@@ -37,7 +37,7 @@
 
 /* Parses the NUMERIC command. */
 int
-cmd_numeric (struct dataset *ds)
+cmd_numeric (struct lexer *lexer, struct dataset *ds)
 {
   size_t i;
 
@@ -51,13 +51,13 @@ cmd_numeric (struct dataset *ds)
 
   do
     {
-      if (!parse_DATA_LIST_vars (&v, &nv, PV_NONE))
+      if (!parse_DATA_LIST_vars (lexer, &v, &nv, PV_NONE))
 	return CMD_FAILURE;
 
       /* Get the optional format specification. */
-      if (lex_match ('('))
+      if (lex_match (lexer, '('))
 	{
-	  if (!parse_format_specifier (&f))
+	  if (!parse_format_specifier (lexer, &f))
 	    goto fail;
 	  if (fmt_is_string (f.type))
 	    {
@@ -67,7 +67,7 @@ cmd_numeric (struct dataset *ds)
 	      goto fail;
 	    }
 
-	  if (!lex_match (')'))
+	  if (!lex_match (lexer, ')'))
 	    {
 	      msg (SE, _("`)' expected after output format."));
 	      goto fail;
@@ -94,9 +94,9 @@ cmd_numeric (struct dataset *ds)
 	free (v[i]);
       free (v);
     }
-  while (lex_match ('/'));
+  while (lex_match (lexer, '/'));
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 
   /* If we have an error at a point where cleanup is required,
      flow-of-control comes here. */
@@ -109,7 +109,7 @@ fail:
 
 /* Parses the STRING command. */
 int
-cmd_string (struct dataset *ds)
+cmd_string (struct lexer *lexer, struct dataset *ds)
 {
   size_t i;
 
@@ -125,12 +125,12 @@ cmd_string (struct dataset *ds)
 
   do
     {
-      if (!parse_DATA_LIST_vars (&v, &nv, PV_NONE))
+      if (!parse_DATA_LIST_vars (lexer, &v, &nv, PV_NONE))
 	return CMD_FAILURE;
 
-      if (!lex_force_match ('(')
-          || !parse_format_specifier (&f)
-          || !lex_force_match (')'))
+      if (!lex_force_match (lexer, '(')
+          || !parse_format_specifier (lexer, &f)
+          || !lex_force_match (lexer, ')'))
 	goto fail;
       if (!fmt_is_string (f.type))
 	{
@@ -160,9 +160,9 @@ cmd_string (struct dataset *ds)
 	free (v[i]);
       free (v);
     }
-  while (lex_match ('/'));
+  while (lex_match (lexer, '/'));
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 
   /* If we have an error at a point where cleanup is required,
      flow-of-control comes here. */
@@ -175,18 +175,18 @@ fail:
 
 /* Parses the LEAVE command. */
 int
-cmd_leave (struct dataset *ds)
+cmd_leave (struct lexer *lexer, struct dataset *ds)
 {
   struct variable **v;
   size_t nv;
 
   size_t i;
 
-  if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE))
+  if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE))
     return CMD_CASCADING_FAILURE;
   for (i = 0; i < nv; i++)
     v[i]->leave = true;
   free (v);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/dictionary/rename-variables.c b/src/language/dictionary/rename-variables.c
index 0e0df1ec..0dff0b02 100644
--- a/src/language/dictionary/rename-variables.c
+++ b/src/language/dictionary/rename-variables.c
@@ -38,7 +38,7 @@
 /* The code for this function is very similar to the code for the
    RENAME subcommand of MODIFY VARS. */
 int
-cmd_rename_variables (struct dataset *ds)
+cmd_rename_variables (struct lexer *lexer, struct dataset *ds)
 {
   struct variable **rename_vars = NULL;
   char **rename_new_names = NULL;
@@ -56,20 +56,20 @@ cmd_rename_variables (struct dataset *ds)
       size_t prev_nv_1 = rename_cnt;
       size_t prev_nv_2 = rename_cnt;
 
-      if (!lex_match ('('))
+      if (!lex_match (lexer, '('))
 	{
 	  msg (SE, _("`(' expected."));
 	  goto lossage;
 	}
-      if (!parse_variables (dataset_dict (ds), &rename_vars, &rename_cnt,
+      if (!parse_variables (lexer, dataset_dict (ds), &rename_vars, &rename_cnt,
 			    PV_APPEND | PV_NO_DUPLICATE))
 	goto lossage;
-      if (!lex_match ('='))
+      if (!lex_match (lexer, '='))
 	{
 	  msg (SE, _("`=' expected between lists of new and old variable names."));
 	  goto lossage;
 	}
-      if (!parse_DATA_LIST_vars (&rename_new_names, &prev_nv_1, PV_APPEND))
+      if (!parse_DATA_LIST_vars (lexer, &rename_new_names, &prev_nv_1, PV_APPEND))
 	goto lossage;
       if (prev_nv_1 != rename_cnt)
 	{
@@ -85,13 +85,13 @@ cmd_rename_variables (struct dataset *ds)
 	  rename_new_names = NULL;
 	  goto lossage;
 	}
-      if (!lex_match (')'))
+      if (!lex_match (lexer, ')'))
 	{
 	  msg (SE, _("`)' expected after variable names."));
 	  goto lossage;
 	}
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
 
   if (!dict_rename_vars (dataset_dict (ds),
                          rename_vars, rename_new_names, rename_cnt,
diff --git a/src/language/dictionary/split-file.c b/src/language/dictionary/split-file.c
index 8c5a9555..0f648e53 100644
--- a/src/language/dictionary/split-file.c
+++ b/src/language/dictionary/split-file.c
@@ -42,9 +42,9 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_split_file (struct dataset *ds)
+cmd_split_file (struct lexer *lexer, struct dataset *ds)
 {
-  if (lex_match_id ("OFF"))
+  if (lex_match_id (lexer, "OFF"))
     dict_set_split_vars (dataset_dict (ds), NULL, 0);
   else
     {
@@ -52,17 +52,17 @@ cmd_split_file (struct dataset *ds)
       size_t n;
 
       /* For now, ignore SEPARATE and LAYERED. */
-      (void) ( lex_match_id ("SEPARATE") || lex_match_id ("LAYERED") );
+      (void) ( lex_match_id (lexer, "SEPARATE") || lex_match_id (lexer, "LAYERED") );
       
-      lex_match (T_BY);
-      if (!parse_variables (dataset_dict (ds), &v, &n, PV_NO_DUPLICATE))
+      lex_match (lexer, T_BY);
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &n, PV_NO_DUPLICATE))
 	return CMD_CASCADING_FAILURE;
 
       dict_set_split_vars (dataset_dict (ds), v, n);
       free (v);
     }
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Dumps out the values of all the split variables for the case C. */
diff --git a/src/language/dictionary/sys-file-info.c b/src/language/dictionary/sys-file-info.c
index 2a0439c2..0aa32fa2 100644
--- a/src/language/dictionary/sys-file-info.c
+++ b/src/language/dictionary/sys-file-info.c
@@ -78,7 +78,7 @@ sysfile_info_dim (struct tab_table *t, struct outp_driver *d)
 
 /* SYSFILE INFO utility. */
 int
-cmd_sysfile_info (struct dataset *ds UNUSED)
+cmd_sysfile_info (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   struct file_handle *h;
   struct dictionary *d;
@@ -88,10 +88,10 @@ cmd_sysfile_info (struct dataset *ds UNUSED)
   int r, nr;
   int i;
 
-  lex_match_id ("FILE");
-  lex_match ('=');
+  lex_match_id (lexer, "FILE");
+  lex_match (lexer, '=');
 
-  h = fh_parse (FH_REF_FILE);
+  h = fh_parse (lexer, FH_REF_FILE);
   if (!h)
     return CMD_FAILURE;
 
@@ -170,7 +170,7 @@ cmd_sysfile_info (struct dataset *ds UNUSED)
 
   dict_destroy (d);
   
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* DISPLAY utility. */
@@ -181,7 +181,7 @@ static void display_variables (struct variable **, size_t, int);
 static void display_vectors (const struct dictionary *dict, int sorted);
 
 int
-cmd_display (struct dataset *ds)
+cmd_display (struct lexer *lexer, struct dataset *ds)
 {
   /* Whether to sort the list of variables alphabetically. */
   int sorted;
@@ -190,14 +190,14 @@ cmd_display (struct dataset *ds)
   size_t n;
   struct variable **vl;
 
-  if (lex_match_id ("MACROS"))
+  if (lex_match_id (lexer, "MACROS"))
     display_macros ();
-  else if (lex_match_id ("DOCUMENTS"))
+  else if (lex_match_id (lexer, "DOCUMENTS"))
     display_documents (dataset_dict (ds));
-  else if (lex_match_id ("FILE"))
+  else if (lex_match_id (lexer, "FILE"))
     {
       som_blank_line ();
-      if (!lex_force_match_id ("LABEL"))
+      if (!lex_force_match_id (lexer, "LABEL"))
 	return CMD_FAILURE;
       if (dict_get_label (dataset_dict (ds)) == NULL)
 	tab_output_text (TAB_LEFT,
@@ -216,12 +216,12 @@ cmd_display (struct dataset *ds)
       const char **cp;
       int as;
 
-      sorted = lex_match_id ("SORTED");
+      sorted = lex_match_id (lexer, "SORTED");
 
       for (cp = sbc; *cp; cp++)
-	if (token == T_ID && lex_id_match (*cp, tokid))
+	if (lex_token (lexer) == T_ID && lex_id_match (*cp, lex_tokid (lexer)))
 	  {
-	    lex_get ();
+	    lex_get (lexer);
 	    break;
 	  }
       as = cp - sbc;
@@ -235,13 +235,13 @@ cmd_display (struct dataset *ds)
 	  return CMD_SUCCESS;
 	}
 
-      lex_match ('/');
-      lex_match_id ("VARIABLES");
-      lex_match ('=');
+      lex_match (lexer, '/');
+      lex_match_id (lexer, "VARIABLES");
+      lex_match (lexer, '=');
 
-      if (token != '.')
+      if (lex_token (lexer) != '.')
 	{
-	  if (!parse_variables (dataset_dict (ds), &vl, &n, PV_NONE))
+	  if (!parse_variables (lexer, dataset_dict (ds), &vl, &n, PV_NONE))
 	    {
 	      free (vl);
 	      return CMD_FAILURE;
@@ -278,7 +278,7 @@ cmd_display (struct dataset *ds)
       free (vl);
     }
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 static void
diff --git a/src/language/dictionary/value-labels.c b/src/language/dictionary/value-labels.c
index c39bebd4..65b8762e 100644
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -38,39 +38,40 @@
 
 /* Declarations. */
 
-static int do_value_labels (const struct dictionary *dict, int);
+static int do_value_labels (struct lexer *, 
+			    const struct dictionary *dict, int);
 static int verify_val_labs (struct variable **vars, size_t var_cnt);
 static void erase_labels (struct variable **vars, size_t var_cnt);
-static int get_label (struct variable **vars, size_t var_cnt);
+static int get_label (struct lexer *, struct variable **vars, size_t var_cnt);
 
 /* Stubs. */
 
 int
-cmd_value_labels (struct dataset *ds)
+cmd_value_labels (struct lexer *lexer, struct dataset *ds)
 {
-  return do_value_labels (dataset_dict (ds), 1);
+  return do_value_labels (lexer, dataset_dict (ds), 1);
 }
 
 int
-cmd_add_value_labels (struct dataset *ds)
+cmd_add_value_labels (struct lexer *lexer, struct dataset *ds)
 {
-  return do_value_labels (dataset_dict (ds), 0);
+  return do_value_labels (lexer, dataset_dict (ds), 0);
 }
 
 /* Do it. */
 
 static int
-do_value_labels (const struct dictionary *dict, int erase)
+do_value_labels (struct lexer *lexer, const struct dictionary *dict, int erase)
 {
   struct variable **vars; /* Variable list. */
   size_t var_cnt;         /* Number of variables. */
   int parse_err=0;        /* true if error parsing variables */
 
-  lex_match ('/');
+  lex_match (lexer, '/');
   
-  while (token != '.')
+  while (lex_token (lexer) != '.')
     {
-      parse_err = !parse_variables (dict, &vars, &var_cnt, 
+      parse_err = !parse_variables (lexer, dict, &vars, &var_cnt, 
 				    PV_SAME_TYPE) ;
       if (var_cnt < 1)
 	{
@@ -81,17 +82,17 @@ do_value_labels (const struct dictionary *dict, int erase)
         goto lossage;
       if (erase)
         erase_labels (vars, var_cnt);
-      while (token != '/' && token != '.')
-	if (!get_label (vars, var_cnt))
+      while (lex_token (lexer) != '/' && lex_token (lexer) != '.')
+	if (!get_label (lexer, vars, var_cnt))
           goto lossage;
 
-      if (token != '/')
+      if (lex_token (lexer) != '/')
 	{
 	free (vars);
 	break;
 	}
 
-      lex_get ();
+      lex_get (lexer);
 
       free (vars);
     }
@@ -99,7 +100,7 @@ do_value_labels (const struct dictionary *dict, int erase)
   if (parse_err)
     return CMD_FAILURE;
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 
  lossage:
   free (vars);
@@ -141,54 +142,58 @@ erase_labels (struct variable **vars, size_t var_cnt)
 /* Parse all the labels for the VAR_CNT variables in VARS and add
    the specified labels to those variables.  */
 static int
-get_label (struct variable **vars, size_t var_cnt)
+get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
 {
   /* Parse all the labels and add them to the variables. */
   do
     {
       union value value;
-      char *label;
+      struct string label;
       size_t i;
 
       /* Set value. */
       if (vars[0]->type == ALPHA)
 	{
-	  if (token != T_STRING)
+	  if (lex_token (lexer) != T_STRING)
 	    {
-              lex_error (_("expecting string"));
+              lex_error (lexer, _("expecting string"));
 	      return 0;
 	    }
-	  buf_copy_str_rpad (value.s, MAX_SHORT_STRING, ds_cstr (&tokstr));
+	  buf_copy_str_rpad (value.s, MAX_SHORT_STRING, ds_cstr (lex_tokstr (lexer)));
 	}
       else
 	{
-	  if (!lex_is_number ())
+	  if (!lex_is_number (lexer))
 	    {
-	      lex_error (_("expecting integer"));
+	      lex_error (lexer, _("expecting integer"));
 	      return 0;
 	    }
-	  if (!lex_is_integer ())
-	    msg (SW, _("Value label `%g' is not integer."), tokval);
-	  value.f = tokval;
+	  if (!lex_is_integer (lexer))
+	    msg (SW, _("Value label `%g' is not integer."), lex_tokval (lexer));
+	  value.f = lex_tokval (lexer);
 	}
-      lex_get ();
+      lex_get (lexer);
 
       /* Set label. */
-      if (!lex_force_string ())
+      if (!lex_force_string (lexer))
 	return 0;
-      if (ds_length (&tokstr) > 60)
+      
+      ds_init_string (&label, lex_tokstr (lexer));
+
+      if (ds_length (&label) > 60)
 	{
 	  msg (SW, _("Truncating value label to 60 characters."));
-	  ds_truncate (&tokstr, 60);
+	  ds_truncate (&label, 60);
 	}
-      label = ds_cstr (&tokstr);
 
       for (i = 0; i < var_cnt; i++)
-        val_labs_replace (vars[i]->val_labs, value, label);
+        val_labs_replace (vars[i]->val_labs, value, ds_cstr (&label));
+
+      ds_destroy (&label);
 
-      lex_get ();
+      lex_get (lexer);
     }
-  while (token != '/' && token != '.');
+  while (lex_token (lexer) != '/' && lex_token (lexer) != '.');
 
   return 1;
 }
diff --git a/src/language/dictionary/variable-display.c b/src/language/dictionary/variable-display.c
index fa6eae46..e6afcb98 100644
--- a/src/language/dictionary/variable-display.c
+++ b/src/language/dictionary/variable-display.c
@@ -36,7 +36,7 @@
    It affects nothing but GUIs
 */
 int
-cmd_variable_alignment (struct dataset *ds)
+cmd_variable_alignment (struct lexer *lexer, struct dataset *ds)
 {
   do
     {
@@ -46,16 +46,16 @@ cmd_variable_alignment (struct dataset *ds)
       size_t i;
       enum alignment align;
 
-      if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE))
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE))
         return CMD_FAILURE;
 
-      if ( lex_force_match('(') ) 
+      if ( lex_force_match (lexer, '(') ) 
 	{
-	  if ( lex_match_id("LEFT"))
+	  if ( lex_match_id (lexer, "LEFT"))
 	    align = ALIGN_LEFT;
-	  else if ( lex_match_id("RIGHT"))
+	  else if ( lex_match_id (lexer, "RIGHT"))
 	    align = ALIGN_RIGHT;
-	  else if ( lex_match_id("CENTER"))
+	  else if ( lex_match_id (lexer, "CENTER"))
 	    align = ALIGN_CENTRE;
 	  else 
             {
@@ -63,7 +63,7 @@ cmd_variable_alignment (struct dataset *ds)
               return CMD_FAILURE; 
             }
 
-	  lex_force_match(')');
+	  lex_force_match (lexer, ')');
 	}
       else 
         {
@@ -75,12 +75,12 @@ cmd_variable_alignment (struct dataset *ds)
 	v[i]->alignment = align;
 
 
-      while (token == '/')
-	lex_get ();
+      while (lex_token (lexer) == '/')
+	lex_get (lexer);
       free (v);
 
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
   return CMD_SUCCESS;
 }
 
@@ -89,7 +89,7 @@ cmd_variable_alignment (struct dataset *ds)
    It affects nothing but GUIs
 */
 int
-cmd_variable_width (struct dataset *ds)
+cmd_variable_width (struct lexer *lexer, struct dataset *ds)
 {
   do
     {
@@ -97,33 +97,33 @@ cmd_variable_width (struct dataset *ds)
       size_t nv;
       size_t i;
 
-      if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE))
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE))
         return CMD_FAILURE;
 
-      if ( lex_force_match('(') ) 
+      if ( lex_force_match (lexer, '(') ) 
 	{
-	  if ( lex_force_int()) 
-	    lex_get();
+	  if ( lex_force_int (lexer)) 
+	    lex_get (lexer);
 	  else
 	    return CMD_FAILURE;
-	  lex_force_match(')');
+	  lex_force_match (lexer, ')');
 	}
 
       for( i = 0 ; i < nv ; ++i ) 
-	  v[i]->display_width = tokval;
+	  v[i]->display_width = lex_tokval (lexer);
 
-      while (token == '/')
-	lex_get ();
+      while (lex_token (lexer) == '/')
+	lex_get (lexer);
       free (v);
 
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
   return CMD_SUCCESS;
 }
 
 /* Set variables' measurement level */
 int
-cmd_variable_level (struct dataset *ds)
+cmd_variable_level (struct lexer *lexer, struct dataset *ds)
 {
   do
     {
@@ -132,16 +132,16 @@ cmd_variable_level (struct dataset *ds)
       enum measure level;
       size_t i;
 
-      if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE))
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE))
         return CMD_FAILURE;
 
-      if ( lex_force_match('(') ) 
+      if ( lex_force_match (lexer, '(') ) 
 	{
-	  if ( lex_match_id("SCALE"))
+	  if ( lex_match_id (lexer, "SCALE"))
 	    level = MEASURE_SCALE;
-	  else if ( lex_match_id("ORDINAL"))
+	  else if ( lex_match_id (lexer, "ORDINAL"))
 	    level = MEASURE_ORDINAL;
-	  else if ( lex_match_id("NOMINAL"))
+	  else if ( lex_match_id (lexer, "NOMINAL"))
 	    level = MEASURE_NOMINAL;
 	  else 
             {
@@ -149,7 +149,7 @@ cmd_variable_level (struct dataset *ds)
               return CMD_FAILURE; 
             }
 
-	  lex_force_match(')');
+	  lex_force_match (lexer, ')');
 	}
       else
         {
@@ -161,11 +161,11 @@ cmd_variable_level (struct dataset *ds)
 	v[i]->measure = level ;
 
 
-      while (token == '/')
-	lex_get ();
+      while (lex_token (lexer) == '/')
+	lex_get (lexer);
       free (v);
 
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
   return CMD_SUCCESS;
 }
diff --git a/src/language/dictionary/variable-label.c b/src/language/dictionary/variable-label.c
index c3bb4c22..1ae3bc5f 100644
--- a/src/language/dictionary/variable-label.c
+++ b/src/language/dictionary/variable-label.c
@@ -35,42 +35,46 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_variable_labels (struct dataset *ds)
+cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
 {
   do
     {
       struct variable **v;
+      struct string label;
       size_t nv;
 
       size_t i;
 
-      if (!parse_variables (dataset_dict (ds), &v, &nv, PV_NONE))
+      if (!parse_variables (lexer, dataset_dict (ds), &v, &nv, PV_NONE))
         return CMD_FAILURE;
 
-      if (token != T_STRING)
+      if (lex_token (lexer) != T_STRING)
 	{
 	  msg (SE, _("String expected for variable label."));
 	  free (v);
 	  return CMD_FAILURE;
 	}
-      if (ds_length (&tokstr) > 255)
+
+      ds_init_string (&label, lex_tokstr (lexer) );
+      if (ds_length (&label) > 255)
 	{
 	  msg (SW, _("Truncating variable label to 255 characters."));
-	  ds_truncate (&tokstr, 255);
+	  ds_truncate (&label, 255);
 	}
       for (i = 0; i < nv; i++)
 	{
 	  if (v[i]->label)
 	    free (v[i]->label);
-	  v[i]->label = ds_xstrdup (&tokstr);
+	  v[i]->label = ds_xstrdup (&label);
 	}
+      ds_destroy (&label);
 
-      lex_get ();
-      while (token == '/')
-	lex_get ();
+      lex_get (lexer);
+      while (lex_token (lexer) == '/')
+	lex_get (lexer);
       free (v);
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
   return CMD_SUCCESS;
 }
 
diff --git a/src/language/dictionary/vector.c b/src/language/dictionary/vector.c
index 1216d9dd..7492015c 100644
--- a/src/language/dictionary/vector.c
+++ b/src/language/dictionary/vector.c
@@ -37,7 +37,7 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_vector (struct dataset *ds)
+cmd_vector (struct lexer *lexer, struct dataset *ds)
 {
   /* Just to be different, points to a set of null terminated strings
      containing the names of the vectors to be created.  The list
@@ -58,9 +58,9 @@ cmd_vector (struct dataset *ds)
   do
     {
       /* Get the name(s) of the new vector(s). */
-      if (!lex_force_id ())
+      if (!lex_force_id (lexer))
 	return CMD_CASCADING_FAILURE;
-      while (token == T_ID)
+      while (lex_token (lexer) == T_ID)
 	{
 	  if (cp + 16 > endp)
 	    {
@@ -71,27 +71,27 @@ cmd_vector (struct dataset *ds)
 	    }
 
 	  for (cp2 = cp; cp2 < cp; cp2 += strlen (cp))
-	    if (!strcasecmp (cp2, tokid))
+	    if (!strcasecmp (cp2, lex_tokid (lexer)))
 	      {
-		msg (SE, _("Vector name %s is given twice."), tokid);
+		msg (SE, _("Vector name %s is given twice."), lex_tokid (lexer));
 		goto fail;
 	      }
 
-	  if (dict_lookup_vector (dict, tokid))
+	  if (dict_lookup_vector (dict, lex_tokid (lexer)))
 	    {
-	      msg (SE, _("There is already a vector with name %s."), tokid);
+	      msg (SE, _("There is already a vector with name %s."), lex_tokid (lexer));
 	      goto fail;
 	    }
 
-	  cp = stpcpy (cp, tokid) + 1;
-	  lex_get ();
-	  lex_match (',');
+	  cp = stpcpy (cp, lex_tokid (lexer)) + 1;
+	  lex_get (lexer);
+	  lex_match (lexer, ',');
 	}
       *cp++ = 0;
 
       /* Now that we have the names it's time to check for the short
          or long forms. */
-      if (lex_match ('='))
+      if (lex_match (lexer, '='))
 	{
 	  /* Long form. */
           struct variable **v;
@@ -106,14 +106,14 @@ cmd_vector (struct dataset *ds)
 	      goto fail;
 	    }
 
-	  if (!parse_variables (dict, &v, &nv,
+	  if (!parse_variables (lexer, dict, &v, &nv,
                                 PV_SAME_TYPE | PV_DUPLICATE))
 	    goto fail;
 
           dict_create_vector (dict, vecnames, v, nv);
           free (v);
 	}
-      else if (lex_match ('('))
+      else if (lex_match (lexer, '('))
 	{
 	  int i;
 
@@ -128,16 +128,16 @@ cmd_vector (struct dataset *ds)
           struct variable **v;
           int nv;
 
-	  if (!lex_force_int ())
+	  if (!lex_force_int (lexer))
 	    return CMD_CASCADING_FAILURE;
-	  nv = lex_integer ();
-	  lex_get ();
+	  nv = lex_integer (lexer);
+	  lex_get (lexer);
 	  if (nv <= 0)
 	    {
 	      msg (SE, _("Vectors must have at least one element."));
 	      goto fail;
 	    }
-	  if (!lex_force_match (')'))
+	  if (!lex_force_match (lexer, ')'))
 	    goto fail;
 
 	  /* First check that all the generated variable names
@@ -196,11 +196,11 @@ cmd_vector (struct dataset *ds)
       free (vecnames);
       vecnames = NULL;
     }
-  while (lex_match ('/'));
+  while (lex_match (lexer, '/'));
 
-  if (token != '.')
+  if (lex_token (lexer) != '.')
     {
-      lex_error (_("expecting end of command"));
+      lex_error (lexer, _("expecting end of command"));
       goto fail;
     }
   return CMD_SUCCESS;
diff --git a/src/language/dictionary/weight.c b/src/language/dictionary/weight.c
index ec6d712b..33f0cbda 100644
--- a/src/language/dictionary/weight.c
+++ b/src/language/dictionary/weight.c
@@ -34,17 +34,17 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_weight (struct dataset *ds)
+cmd_weight (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
-  if (lex_match_id ("OFF"))
+  if (lex_match_id (lexer, "OFF"))
     dict_set_weight (dataset_dict (ds), NULL);
   else
     {
       struct variable *v;
 
-      lex_match (T_BY);
-      v = parse_variable (dict);
+      lex_match (lexer, T_BY);
+      v = parse_variable (lexer, dict);
       if (!v)
 	return CMD_CASCADING_FAILURE;
       if (v->type == ALPHA)
@@ -61,5 +61,5 @@ cmd_weight (struct dataset *ds)
       dict_set_weight (dict, v);
     }
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/expressions/evaluate.c b/src/language/expressions/evaluate.c
index c3807541..b597fd4c 100644
--- a/src/language/expressions/evaluate.c
+++ b/src/language/expressions/evaluate.c
@@ -109,7 +109,7 @@ expr_evaluate_str (struct expression *e, const struct ccase *c, int case_idx,
 #include <language/command.h>
 
 int
-cmd_debug_evaluate (struct dataset *dsother UNUSED)
+cmd_debug_evaluate (struct lexer *lexer, struct dataset *dsother UNUSED)
 {
   bool optimize = true;
   int retval = CMD_FAILURE;
@@ -124,38 +124,38 @@ cmd_debug_evaluate (struct dataset *dsother UNUSED)
   for (;;) 
     {
       struct dictionary *d = NULL;
-      if (lex_match_id ("NOOPTIMIZE"))
+      if (lex_match_id (lexer, "NOOPTIMIZE"))
         optimize = 0;
-      else if (lex_match_id ("POSTFIX"))
+      else if (lex_match_id (lexer, "POSTFIX"))
         dump_postfix = 1;
-      else if (lex_match ('('))
+      else if (lex_match (lexer, '('))
         {
           char name[LONG_NAME_LEN + 1];
           struct variable *v;
           size_t old_value_cnt;
           int width;
 
-          if (!lex_force_id ())
+          if (!lex_force_id (lexer))
             goto done;
-          strcpy (name, tokid);
+          strcpy (name, lex_tokid (lexer));
 
-          lex_get ();
-          if (!lex_force_match ('='))
+          lex_get (lexer);
+          if (!lex_force_match (lexer, '='))
             goto done;
 
-          if (lex_is_number ())
+          if (lex_is_number (lexer))
             {
               width = 0;
-              fprintf (stderr, "(%s = %.2f)", name, tokval); 
+              fprintf (stderr, "(%s = %.2f)", name, lex_tokval (lexer)); 
             }
-          else if (token == T_STRING) 
+          else if (lex_token (lexer) == T_STRING) 
             {
-              width = ds_length (&tokstr);
-              fprintf (stderr, "(%s = \"%.2s\")", name, ds_cstr (&tokstr)); 
+              width = ds_length (lex_tokstr (lexer));
+              fprintf (stderr, "(%s = \"%.2s\")", name, ds_cstr (lex_tokstr (lexer))); 
             }
           else
             {
-              lex_error (_("expecting number or string"));
+              lex_error (lexer, _("expecting number or string"));
               goto done;
             }
 	  
@@ -181,32 +181,32 @@ cmd_debug_evaluate (struct dataset *dsother UNUSED)
           else
             case_resize (c, old_value_cnt, dict_get_next_value_idx (d));
 
-          if (lex_is_number ())
-            case_data_rw (c, v->fv)->f = tokval;
+          if (lex_is_number (lexer))
+            case_data_rw (c, v->fv)->f = lex_tokval (lexer);
           else
-            memcpy (case_data_rw (c, v->fv)->s, ds_data (&tokstr),
+            memcpy (case_data_rw (c, v->fv)->s, ds_data (lex_tokstr (lexer)),
                     v->width);
-          lex_get ();
+          lex_get (lexer);
 
-          if (!lex_force_match (')'))
+          if (!lex_force_match (lexer, ')'))
             goto done;
         }
       else 
         break;
     }
-  if (token != '/') 
+  if (lex_token (lexer) != '/') 
     {
-      lex_force_match ('/');
+      lex_force_match (lexer, '/');
       goto done;
     }
 
   if ( ds != NULL ) 
     fprintf(stderr, "; ");
-  fprintf (stderr, "%s => ", lex_rest_of_line (NULL));
-  lex_get ();
+  fprintf (stderr, "%s => ", lex_rest_of_line (lexer, NULL));
+  lex_get (lexer);
 
-  expr = expr_parse_any (ds, optimize);
-  if (!expr || lex_end_of_command () != CMD_SUCCESS)
+  expr = expr_parse_any (lexer, ds, optimize);
+  if (!expr || lex_end_of_command (lexer) != CMD_SUCCESS)
     {
       if (expr != NULL)
         expr_free (expr);
diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 43aa553d..0c89be63 100644
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -45,7 +45,7 @@
 /* Declarations. */
 
 /* Recursive descent parser in order of increasing precedence. */
-typedef union any_node *parse_recursively_func (struct expression *);
+typedef union any_node *parse_recursively_func (struct lexer *, struct expression *);
 static parse_recursively_func parse_or, parse_and, parse_not;
 static parse_recursively_func parse_rel, parse_add, parse_mul;
 static parse_recursively_func parse_neg, parse_exp;
@@ -73,7 +73,7 @@ static union any_node *allocate_unary_variable (struct expression *,
    Returns the new expression if successful or a null pointer
    otherwise. */
 struct expression *
-expr_parse (struct dataset *ds, enum expr_type type) 
+expr_parse (struct lexer *lexer, struct dataset *ds, enum expr_type type) 
 {
   union any_node *n;
   struct expression *e;
@@ -81,7 +81,7 @@ expr_parse (struct dataset *ds, enum expr_type type)
   assert (type == EXPR_NUMBER || type == EXPR_STRING || type == EXPR_BOOLEAN);
 
   e = expr_create (ds);
-  n = parse_or (e);
+  n = parse_or (lexer, e);
   if (n != NULL && type_check (e, &n, type))
     return finish_expression (expr_optimize (n, e), e);
   else
@@ -95,11 +95,12 @@ expr_parse (struct dataset *ds, enum expr_type type)
    expr_parse(), and sets up so that destroying POOL will free
    the expression as well. */
 struct expression *
-expr_parse_pool (struct pool *pool,
+expr_parse_pool (struct lexer *lexer, 
+		 struct pool *pool,
 		 struct dataset *ds, 
                  enum expr_type type) 
 {
-  struct expression *e = expr_parse (ds, type);
+  struct expression *e = expr_parse (lexer, ds, type);
   if (e != NULL)
     pool_add_subpool (pool, e->expr_pool);
   return e;
@@ -114,13 +115,13 @@ expr_free (struct expression *e)
 }
 
 struct expression *
-expr_parse_any (struct dataset *ds, bool optimize)
+expr_parse_any (struct lexer *lexer, struct dataset *ds, bool optimize)
 {
   union any_node *n;
   struct expression *e;
 
   e = expr_create (ds);
-  n = parse_or (e);
+  n = parse_or (lexer, e);
   if (n == NULL)
     {
       expr_free (e);
@@ -469,7 +470,7 @@ struct operator
    On failure, returns false and, if OPERATOR is non-null, sets
    *OPERATOR to a null pointer. */
 static bool
-match_operator (const struct operator ops[], size_t op_cnt,
+match_operator (struct lexer *lexer, const struct operator ops[], size_t op_cnt,
                 const struct operator **operator) 
 {
   const struct operator *op;
@@ -477,8 +478,8 @@ match_operator (const struct operator ops[], size_t op_cnt,
   for (op = ops; op < ops + op_cnt; op++)
     {
       if (op->token == '-')
-        lex_negative_to_dash ();
-      if (lex_match (op->token)) 
+        lex_negative_to_dash (lexer);
+      if (lex_match (lexer, op->token)) 
         {
           if (operator != NULL)
             *operator = op;
@@ -529,7 +530,7 @@ get_operand_type (const struct operator *op)
    is non-null, then it will be issued as a warning if more than
    one operator/operand pair is parsed. */
 static union any_node *
-parse_binary_operators (struct expression *e, union any_node *node,
+parse_binary_operators (struct lexer *lexer, struct expression *e, union any_node *node,
                         const struct operator ops[], size_t op_cnt,
                         parse_recursively_func *parse_next_level,
                         const char *chain_warning)
@@ -542,7 +543,7 @@ parse_binary_operators (struct expression *e, union any_node *node,
   if (node == NULL)
     return node;
 
-  for (op_count = 0; match_operator (ops, op_cnt, &operator); op_count++)
+  for (op_count = 0; match_operator (lexer, ops, op_cnt, &operator); op_count++)
     {
       union any_node *rhs;
 
@@ -552,7 +553,7 @@ parse_binary_operators (struct expression *e, union any_node *node,
 
       /* Parse the right-hand side and coerce to type
          OPERAND_TYPE. */
-      rhs = parse_next_level (e);
+      rhs = parse_next_level (lexer, e);
       if (!type_coercion (e, operand_type, &rhs, operator->name))
         return NULL;
       node = expr_allocate_binary (e, operator->type, node, rhs);
@@ -565,7 +566,7 @@ parse_binary_operators (struct expression *e, union any_node *node,
 }
 
 static union any_node *
-parse_inverting_unary_operator (struct expression *e,
+parse_inverting_unary_operator (struct lexer *lexer, struct expression *e,
                                 const struct operator *op,
                                 parse_recursively_func *parse_next_level) 
 {
@@ -575,10 +576,10 @@ parse_inverting_unary_operator (struct expression *e,
   check_operator (op, 1, get_operand_type (op));
 
   op_count = 0;
-  while (match_operator (op, 1, NULL))
+  while (match_operator (lexer, op, 1, NULL))
     op_count++;
 
-  node = parse_next_level (e);
+  node = parse_next_level (lexer, e);
   if (op_count > 0
       && type_coercion (e, get_operand_type (op), &node, op->name)
       && op_count % 2 != 0)
@@ -589,36 +590,37 @@ parse_inverting_unary_operator (struct expression *e,
 
 /* Parses the OR level. */
 static union any_node *
-parse_or (struct expression *e)
+parse_or (struct lexer *lexer, struct expression *e)
 {
   static const struct operator op = 
     { T_OR, OP_OR, "logical disjunction (\"OR\")" };
   
-  return parse_binary_operators (e, parse_and (e), &op, 1, parse_and, NULL);
+  return parse_binary_operators (lexer, e, parse_and (lexer, e), &op, 1, parse_and, NULL);
 }
 
 /* Parses the AND level. */
 static union any_node *
-parse_and (struct expression *e)
+parse_and (struct lexer *lexer, struct expression *e)
 {
   static const struct operator op = 
     { T_AND, OP_AND, "logical conjunction (\"AND\")" };
   
-  return parse_binary_operators (e, parse_not (e), &op, 1, parse_not, NULL);
+  return parse_binary_operators (lexer, e, parse_not (lexer, e), 
+				 &op, 1, parse_not, NULL);
 }
 
 /* Parses the NOT level. */
 static union any_node *
-parse_not (struct expression *e)
+parse_not (struct lexer *lexer, struct expression *e)
 {
   static const struct operator op
     = { T_NOT, OP_NOT, "logical negation (\"NOT\")" };
-  return parse_inverting_unary_operator (e, &op, parse_rel);
+  return parse_inverting_unary_operator (lexer, e, &op, parse_rel);
 }
 
 /* Parse relational operators. */
 static union any_node *
-parse_rel (struct expression *e)
+parse_rel (struct lexer *lexer, struct expression *e)
 {
   const char *chain_warning = 
     _("Chaining relational operators (e.g. \"a < b < c\") will "
@@ -628,7 +630,7 @@ parse_rel (struct expression *e)
       "If chaining is really intended, parentheses will disable "
       "this warning (e.g. \"(a < b) < c\".)");
 
-  union any_node *node = parse_add (e);
+  union any_node *node = parse_add (lexer, e);
 
   if (node == NULL)
     return NULL;
@@ -649,7 +651,8 @@ parse_rel (struct expression *e)
             { T_NE, OP_NE, "numeric inequality (\"<>\")" },
           };
 
-        return parse_binary_operators (e, node, ops, sizeof ops / sizeof *ops,
+        return parse_binary_operators (lexer, e, node, ops, 
+				       sizeof ops / sizeof *ops,
                                        parse_add, chain_warning);
       }
       
@@ -666,7 +669,8 @@ parse_rel (struct expression *e)
             { T_NE, OP_NE_STRING, "string inequality (\"<>\")" },
           };
 
-        return parse_binary_operators (e, node, ops, sizeof ops / sizeof *ops,
+        return parse_binary_operators (lexer, e, node, ops, 
+				       sizeof ops / sizeof *ops,
                                        parse_add, chain_warning);
       }
       
@@ -677,7 +681,7 @@ parse_rel (struct expression *e)
 
 /* Parses the addition and subtraction level. */
 static union any_node *
-parse_add (struct expression *e)
+parse_add (struct lexer *lexer, struct expression *e)
 {
   static const struct operator ops[] = 
     {
@@ -685,14 +689,14 @@ parse_add (struct expression *e)
       { '-', OP_SUB, "subtraction (\"-\")" },
     };
   
-  return parse_binary_operators (e, parse_mul (e),
+  return parse_binary_operators (lexer, e, parse_mul (lexer, e),
                                  ops, sizeof ops / sizeof *ops,
                                  parse_mul, NULL);
 }
 
 /* Parses the multiplication and division level. */
 static union any_node *
-parse_mul (struct expression *e)
+parse_mul (struct lexer *lexer, struct expression *e)
 {
   static const struct operator ops[] = 
     {
@@ -700,21 +704,21 @@ parse_mul (struct expression *e)
       { '/', OP_DIV, "division (\"/\")" },
     };
   
-  return parse_binary_operators (e, parse_neg (e),
+  return parse_binary_operators (lexer, e, parse_neg (lexer, e),
                                  ops, sizeof ops / sizeof *ops,
                                  parse_neg, NULL);
 }
 
 /* Parses the unary minus level. */
 static union any_node *
-parse_neg (struct expression *e)
+parse_neg (struct lexer *lexer, struct expression *e)
 {
   static const struct operator op = { '-', OP_NEG, "negation (\"-\")" };
-  return parse_inverting_unary_operator (e, &op, parse_exp);
+  return parse_inverting_unary_operator (lexer, e, &op, parse_exp);
 }
 
 static union any_node *
-parse_exp (struct expression *e)
+parse_exp (struct lexer *lexer, struct expression *e)
 {
   static const struct operator op = 
     { T_EXP, OP_POW, "exponentiation (\"**\")" };
@@ -725,17 +729,17 @@ parse_exp (struct expression *e)
       "That is, \"a**b**c\" equals \"(a**b)**c\", not as \"a**(b**c)\".  "
       "To disable this warning, insert parentheses.");
 
-  return parse_binary_operators (e, parse_primary (e), &op, 1,
+  return parse_binary_operators (lexer, e, parse_primary (lexer, e), &op, 1,
                                  parse_primary, chain_warning);
 }
 
 /* Parses system variables. */
 static union any_node *
-parse_sysvar (struct expression *e)
+parse_sysvar (struct lexer *lexer, struct expression *e)
 {
-  if (lex_match_id ("$CASENUM"))
+  if (lex_match_id (lexer, "$CASENUM"))
     return expr_allocate_nullary (e, OP_CASENUM);
-  else if (lex_match_id ("$DATE"))
+  else if (lex_match_id (lexer, "$DATE"))
     {
       static const char *months[12] =
         {
@@ -753,13 +757,13 @@ parse_sysvar (struct expression *e)
 
       return expr_allocate_string_buffer (e, temp_buf, strlen (temp_buf));
     }
-  else if (lex_match_id ("$TRUE"))
+  else if (lex_match_id (lexer, "$TRUE"))
     return expr_allocate_boolean (e, 1.0);
-  else if (lex_match_id ("$FALSE"))
+  else if (lex_match_id (lexer, "$FALSE"))
     return expr_allocate_boolean (e, 0.0);
-  else if (lex_match_id ("$SYSMIS"))
+  else if (lex_match_id (lexer, "$SYSMIS"))
     return expr_allocate_number (e, SYSMIS);
-  else if (lex_match_id ("$JDATE"))
+  else if (lex_match_id (lexer, "$JDATE"))
     {
       time_t time = time_of_last_procedure (e->ds);
       struct tm *tm = localtime (&time);
@@ -767,7 +771,7 @@ parse_sysvar (struct expression *e)
                                                        tm->tm_mon + 1,
                                                        tm->tm_mday));
     }
-  else if (lex_match_id ("$TIME"))
+  else if (lex_match_id (lexer, "$TIME"))
     {
       time_t time = time_of_last_procedure (e->ds);
       struct tm *tm = localtime (&time);
@@ -779,45 +783,45 @@ parse_sysvar (struct expression *e)
                                    + tm->tm_min * 60.
                                    + tm->tm_sec);
     }
-  else if (lex_match_id ("$LENGTH"))
+  else if (lex_match_id (lexer, "$LENGTH"))
     return expr_allocate_number (e, get_viewlength ());
-  else if (lex_match_id ("$WIDTH"))
+  else if (lex_match_id (lexer, "$WIDTH"))
     return expr_allocate_number (e, get_viewwidth ());
   else
     {
-      msg (SE, _("Unknown system variable %s."), tokid);
+      msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
       return NULL;
     }
 }
 
 /* Parses numbers, varnames, etc. */
 static union any_node *
-parse_primary (struct expression *e)
+parse_primary (struct lexer *lexer, struct expression *e)
 {
-  switch (token)
+  switch (lex_token (lexer))
     {
     case T_ID:
-      if (lex_look_ahead () == '(') 
+      if (lex_look_ahead (lexer) == '(') 
         {
           /* An identifier followed by a left parenthesis may be
              a vector element reference.  If not, it's a function
              call. */
-          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), tokid) != NULL) 
-            return parse_vector_element (e);
+          if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer)) != NULL) 
+            return parse_vector_element (lexer, e);
           else
-            return parse_function (e);
+            return parse_function (lexer, e);
         }
-      else if (tokid[0] == '$')
+      else if (lex_tokid (lexer)[0] == '$')
         {
           /* $ at the beginning indicates a system variable. */
-          return parse_sysvar (e);
+          return parse_sysvar (lexer, e);
         }
-      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), tokid))
+      else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds), lex_tokid (lexer)))
         {
           /* It looks like a user variable.
              (It could be a format specifier, but we'll assume
              it's a variable unless proven otherwise. */
-          return allocate_unary_variable (e, parse_variable (dataset_dict (e->ds)));
+          return allocate_unary_variable (e, parse_variable (lexer, dataset_dict (e->ds)));
         }
       else 
         {
@@ -826,14 +830,14 @@ parse_primary (struct expression *e)
           bool ok;
           
           msg_disable ();
-          ok = parse_format_specifier (&fmt);
+          ok = parse_format_specifier (lexer, &fmt);
           msg_enable ();
 
           if (ok)
             return expr_allocate_format (e, &fmt);
 
           /* All attempts failed. */
-          msg (SE, _("Unknown identifier %s."), tokid);
+          msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
           return NULL;
         }
       break;
@@ -841,40 +845,40 @@ parse_primary (struct expression *e)
     case T_POS_NUM: 
     case T_NEG_NUM: 
       {
-        union any_node *node = expr_allocate_number (e, tokval);
-        lex_get ();
+        union any_node *node = expr_allocate_number (e, lex_tokval (lexer) );
+        lex_get (lexer);
         return node; 
       }
 
     case T_STRING:
       {
         union any_node *node = expr_allocate_string_buffer (
-          e, ds_cstr (&tokstr), ds_length (&tokstr));
-	lex_get ();
+          e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+	lex_get (lexer);
 	return node;
       }
 
     case '(':
       {
         union any_node *node;
-	lex_get ();
-	node = parse_or (e);
-	if (node != NULL && !lex_match (')'))
+	lex_get (lexer);
+	node = parse_or (lexer, e);
+	if (node != NULL && !lex_match (lexer, ')'))
 	  {
-	    lex_error (_("expecting `)'"));
+	    lex_error (lexer, _("expecting `)'"));
             return NULL;
 	  }
         return node;
       }
 
     default:
-      lex_error (_("in expression"));
+      lex_error (lexer, _("in expression"));
       return NULL;
     }
 }
 
 static union any_node *
-parse_vector_element (struct expression *e)
+parse_vector_element (struct lexer *lexer, struct expression *e)
 {
   const struct vector *vector;
   union any_node *element;
@@ -882,19 +886,19 @@ parse_vector_element (struct expression *e)
   /* Find vector, skip token.
      The caller must already have verified that the current token
      is the name of a vector. */
-  vector = dict_lookup_vector (dataset_dict (e->ds), tokid);
+  vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
   assert (vector != NULL);
-  lex_get ();
+  lex_get (lexer);
 
   /* Skip left parenthesis token.
      The caller must have verified that the lookahead is a left
      parenthesis. */
-  assert (token == '(');
-  lex_get ();
+  assert (lex_token (lexer) == '(');
+  lex_get (lexer);
 
-  element = parse_or (e);
+  element = parse_or (lexer, e);
   if (!type_coercion (e, OP_number, &element, "vector indexing")
-      || !lex_match (')'))
+      || !lex_match (lexer, ')'))
     return NULL;
 
   return expr_allocate_binary (e, (vector->var[0]->type == NUMERIC
@@ -1155,7 +1159,7 @@ no_match (const char *func_name,
 }
 
 static union any_node *
-parse_function (struct expression *e)
+parse_function (struct lexer *lexer, struct expression *e)
 {
   int min_valid;
   const struct operation *f, *first, *last;
@@ -1168,17 +1172,17 @@ parse_function (struct expression *e)
 
   union any_node *n;
 
-  ds_init_string (&func_name, &tokstr);
-  min_valid = extract_min_valid (ds_cstr (&tokstr));
-  if (!lookup_function (ds_cstr (&tokstr), &first, &last)) 
+  ds_init_string (&func_name, lex_tokstr (lexer));
+  min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
+  if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last)) 
     {
-      msg (SE, _("No function or vector named %s."), ds_cstr (&tokstr));
+      msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr (lexer)));
       ds_destroy (&func_name);
       return NULL;
     }
 
-  lex_get ();
-  if (!lex_force_match ('(')) 
+  lex_get (lexer);
+  if (!lex_force_match (lexer, '(')) 
     {
       ds_destroy (&func_name);
       return NULL; 
@@ -1186,16 +1190,16 @@ parse_function (struct expression *e)
   
   args = NULL;
   arg_cnt = arg_cap = 0;
-  if (token != ')')
+  if (lex_token (lexer) != ')')
     for (;;)
       {
-        if (token == T_ID && lex_look_ahead () == 'T')
+        if (lex_token (lexer) == T_ID && lex_look_ahead (lexer) == 'T')
           {
             struct variable **vars;
             size_t var_cnt;
             size_t i;
 
-            if (!parse_variables (dataset_dict (e->ds), &vars, &var_cnt, PV_SINGLE))
+            if (!parse_variables (lexer, dataset_dict (e->ds), &vars, &var_cnt, PV_SINGLE))
               goto fail;
             for (i = 0; i < var_cnt; i++)
               add_arg (&args, &arg_cnt, &arg_cap,
@@ -1204,17 +1208,17 @@ parse_function (struct expression *e)
           }
         else
           {
-            union any_node *arg = parse_or (e);
+            union any_node *arg = parse_or (lexer, e);
             if (arg == NULL)
               goto fail;
 
             add_arg (&args, &arg_cnt, &arg_cap, arg);
           }
-        if (lex_match (')'))
+        if (lex_match (lexer, ')'))
           break;
-        else if (!lex_match (','))
+        else if (!lex_match (lexer, ','))
           {
-            lex_error (_("expecting `,' or `)' invoking %s function"),
+            lex_error (lexer, _("expecting `,' or `)' invoking %s function"),
                        first->name);
             goto fail;
           }
diff --git a/src/language/expressions/private.h b/src/language/expressions/private.h
index 48ac8897..baf69c11 100644
--- a/src/language/expressions/private.h
+++ b/src/language/expressions/private.h
@@ -168,7 +168,7 @@ struct expression
     struct pool *eval_pool;     /* Pool for evaluation temporaries. */
   };
 
-struct expression *expr_parse_any (struct dataset *,  bool optimize);
+struct expression *expr_parse_any (struct lexer *lexer, struct dataset *,  bool optimize);
 void expr_debug_print_postfix (const struct expression *);
 
 union any_node *expr_optimize (union any_node *, struct expression *);
diff --git a/src/language/expressions/public.h b/src/language/expressions/public.h
index 864a258d..dc8aadc9 100644
--- a/src/language/expressions/public.h
+++ b/src/language/expressions/public.h
@@ -36,9 +36,11 @@ struct ccase;
 struct pool;
 union value;
 struct dataset ;
+struct lexer ;
 
-struct expression *expr_parse (struct dataset *, enum expr_type);
-struct expression *expr_parse_pool (struct pool *,
+struct expression *expr_parse (struct lexer *lexer, struct dataset *, enum expr_type);
+struct expression *expr_parse_pool (struct lexer *, 
+				    struct pool *,
 				    struct dataset *,
                                     enum expr_type);
 void expr_free (struct expression *);
diff --git a/src/language/lexer/ChangeLog b/src/language/lexer/ChangeLog
index 29678623..fd98ea9c 100644
--- a/src/language/lexer/ChangeLog
+++ b/src/language/lexer/ChangeLog
@@ -1,3 +1,12 @@
+Sun Nov 12 06:34:06 WST 2006 John Darrrington <john@darrington.wattle.id.au>
+
+	* format-parser.c format-parser.h lexer.c lexer.h q2c.c range-parser.c
+	  range-parser.h subcommand-list.c variable-parser.c
+	  variable-parser.h:
+
+	Encapsulated the lexer into an object, and updated everything
+	accordingly.
+
 Tue Oct 31 18:09:32 2006  Ben Pfaff  <blp@gnu.org>
 
 	* range-parser.c (parse_number): Fix error message.
diff --git a/src/language/lexer/format-parser.c b/src/language/lexer/format-parser.c
index 939559cb..b05a5926 100644
--- a/src/language/lexer/format-parser.c
+++ b/src/language/lexer/format-parser.c
@@ -47,18 +47,18 @@
    format.  Both width and decimals are considered optional.  If
    missing, *WIDTH or *DECIMALS or both will be set to 0. */
 bool
-parse_abstract_format_specifier (char type[FMT_TYPE_LEN_MAX + 1],
+parse_abstract_format_specifier (struct lexer *lexer, char type[FMT_TYPE_LEN_MAX + 1],
                                  int *width, int *decimals) 
 {
   struct substring s;
   struct substring type_ss, width_ss, decimals_ss;
   bool has_decimals;
 
-  if (token != T_ID)
+  if (lex_token (lexer) != T_ID)
     goto error;
 
   /* Extract pieces. */
-  s = ds_ss (&tokstr);
+  s = ds_ss (lex_tokstr (lexer));
   ss_get_chars (&s, ss_span (s, ss_cstr (CC_LETTERS)), &type_ss);
   ss_get_chars (&s, ss_span (s, ss_cstr (CC_DIGITS)), &width_ss);
   if (ss_match_char (&s, '.')) 
@@ -85,11 +85,11 @@ parse_abstract_format_specifier (char type[FMT_TYPE_LEN_MAX + 1],
   *width = strtol (ss_data (width_ss), NULL, 10);
   *decimals = has_decimals ? strtol (ss_data (decimals_ss), NULL, 10) : 0;
 
-  lex_get ();
+  lex_get (lexer);
   return true;
 
  error:
-  lex_error (_("expecting valid format specifier"));
+  lex_error (lexer, _("expecting valid format specifier"));
   return false;
 }
 
@@ -99,11 +99,11 @@ parse_abstract_format_specifier (char type[FMT_TYPE_LEN_MAX + 1],
    check_output_specifier() on the parsed format as
    necessary.  */
 bool
-parse_format_specifier (struct fmt_spec *format)
+parse_format_specifier (struct lexer *lexer, struct fmt_spec *format)
 {
   char type[FMT_TYPE_LEN_MAX + 1];
 
-  if (!parse_abstract_format_specifier (type, &format->w, &format->d))
+  if (!parse_abstract_format_specifier (lexer, type, &format->w, &format->d))
     return false;
 
   if (!fmt_from_name (type, &format->type))
@@ -118,18 +118,18 @@ parse_format_specifier (struct fmt_spec *format)
 /* Parses a token containing just the name of a format type and
    returns true if successful. */
 bool
-parse_format_specifier_name (enum fmt_type *type) 
+parse_format_specifier_name (struct lexer *lexer, enum fmt_type *type) 
 {
-  if (token != T_ID) 
+  if (lex_token (lexer) != T_ID) 
     {
-      lex_error (_("expecting format type"));
+      lex_error (lexer, _("expecting format type"));
       return false;
     }
-  if (!fmt_from_name (ds_cstr (&tokstr), type))
+  if (!fmt_from_name (ds_cstr (lex_tokstr (lexer)), type))
     {
-      msg (SE, _("Unknown format type \"%s\"."), ds_cstr (&tokstr));
+      msg (SE, _("Unknown format type \"%s\"."), ds_cstr (lex_tokstr (lexer)));
       return false;
     }
-  lex_get ();
+  lex_get (lexer);
   return true;
 }
diff --git a/src/language/lexer/format-parser.h b/src/language/lexer/format-parser.h
index d16b2f5b..11890645 100644
--- a/src/language/lexer/format-parser.h
+++ b/src/language/lexer/format-parser.h
@@ -24,9 +24,12 @@
 
 #include <data/format.h>
 
-bool parse_abstract_format_specifier (char type[FMT_TYPE_LEN_MAX + 1],
+
+struct lexer;
+
+bool parse_abstract_format_specifier (struct lexer *, char type[FMT_TYPE_LEN_MAX + 1],
                                       int *width, int *decimals);
-bool parse_format_specifier (struct fmt_spec *);
-bool parse_format_specifier_name (enum fmt_type *type);
+bool parse_format_specifier (struct lexer *, struct fmt_spec *);
+bool parse_format_specifier_name (struct lexer *, enum fmt_type *type);
 
 #endif /* language/lexer/format-parser.h. */
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index a1300418..39b674c5 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -45,38 +45,34 @@
 #define DUMP_TOKENS 1
 */
 
-
-/* Current token. */
-int token;
+struct lexer 
+{
+  struct string line_buffer;
 
-/* T_POS_NUM, T_NEG_NUM: the token's value. */
-double tokval;
+  bool (*read_line) (struct string *, bool *);
 
-/* T_ID: the identifier. */
-char tokid[LONG_NAME_LEN + 1];
+  int token;      /* Current token. */
+  double tokval;  /* T_POS_NUM, T_NEG_NUM: the token's value. */
 
-/* T_ID, T_STRING: token string value.
-   For T_ID, this is not truncated as is tokid. */
-struct string tokstr;
-
-/* Static variables. */
+  char tokid [LONG_NAME_LEN + 1];   /* T_ID: the identifier. */
 
-/* Pointer to next token in line_buffer. */
-static char *prog;
+  struct string tokstr;   /* T_ID, T_STRING: token string value.
+  			    For T_ID, this is not truncated as is 
+  			    tokid. */
 
-/* True only if this line ends with a terminal dot. */
-static bool dot;
+  char *prog; /* Pointer to next token in line_buffer. */
+  bool dot;   /* True only if this line ends with a terminal dot. */
+  bool eof;   /* True only if the last token returned was T_STOP. */
 
-/* True only if the last token returned was T_STOP. */
-static bool eof;
+  int put_token ; /* If nonzero, next token returned by lex_get().
+  		    Used only in exceptional circumstances. */
+  
+  struct string put_tokstr;
+  double put_tokval;
+};
 
-/* If nonzero, next token returned by lex_get().
-   Used only in exceptional circumstances. */
-static int put_token;
-static struct string put_tokstr;
-static double put_tokval;
 
-static int parse_id (void);
+static int parse_id (struct lexer *);
 
 /* How a string represents its contents. */
 enum string_type 
@@ -87,7 +83,7 @@ enum string_type
     HEX_STRING          /* Hexadecimal digits. */
   };
 
-static int parse_string (enum string_type);
+static int parse_string (struct lexer *, enum string_type);
 
 #if DUMP_TOKENS
 static void dump_token (void);
@@ -95,66 +91,68 @@ static void dump_token (void);
 
 /* Initialization. */
 
-static struct string line_buffer;
-
-static bool (*lex_read_line) (struct string *, bool *);
-
 /* Initializes the lexer. */
-void
-lex_init (bool (*read_line_func) (struct string *, bool *))
+struct lexer *
+lex_create (bool (*read_line_func) (struct string *, bool *))
 {
-  ds_init_empty (&tokstr);
-  ds_init_empty (&put_tokstr);
-  ds_init_empty (&line_buffer);
-  lex_read_line = read_line_func;
+  struct lexer *lexer = xzalloc (sizeof (*lexer));
+
+  ds_init_empty (&lexer->tokstr);
+  ds_init_empty (&lexer->put_tokstr);
+  ds_init_empty (&lexer->line_buffer);
+  lexer->read_line = read_line_func;
 
-  if (!lex_get_line ())
-    eof = true;
+  if (!lex_get_line (lexer))
+    lexer->eof = true;
+
+  return lexer;
 }
 
 void
-lex_done (void)
+lex_destroy (struct lexer *lexer)
 {
-  ds_destroy (&put_tokstr);
-  ds_destroy (&tokstr);
-  ds_destroy (&line_buffer);
+  ds_destroy (&lexer->put_tokstr);
+  ds_destroy (&lexer->tokstr);
+  ds_destroy (&lexer->line_buffer);
+
+  free (lexer);
 }
 
 
 /* Common functions. */
 
-/* Copies put_token, put_tokstr, put_tokval into token, tokstr,
+/* Copies put_token, lexer->put_tokstr, put_tokval into token, tokstr,
    tokval, respectively, and sets tokid appropriately. */
 static void
-restore_token (void) 
+restore_token (struct lexer *lexer) 
 {
-  assert (put_token != 0);
-  token = put_token;
-  ds_assign_string (&tokstr, &put_tokstr);
-  str_copy_trunc (tokid, sizeof tokid, ds_cstr (&tokstr));
-  tokval = put_tokval;
-  put_token = 0;
+  assert (lexer->put_token != 0);
+  lexer->token = lexer->put_token;
+  ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
+  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
+  lexer->tokval = lexer->put_tokval;
+  lexer->put_token = 0;
 }
 
-/* Copies token, tokstr, tokval into put_token, put_tokstr,
-   put_tokval respectively. */
+/* Copies token, tokstr, lexer->tokval into lexer->put_token, put_tokstr,
+   put_lexer->tokval respectively. */
 static void
-save_token (void) 
+save_token (struct lexer *lexer) 
 {
-  put_token = token;
-  ds_assign_string (&put_tokstr, &tokstr);
-  put_tokval = tokval;
+  lexer->put_token = lexer->token;
+  ds_assign_string (&lexer->put_tokstr, &lexer->tokstr);
+  lexer->put_tokval = lexer->tokval;
 }
 
 /* Parses a single token, setting appropriate global variables to
    indicate the token's attributes. */
 void
-lex_get (void)
+lex_get (struct lexer *lexer)
 {
   /* If a token was pushed ahead, return it. */
-  if (put_token)
+  if (lexer->put_token)
     {
-      restore_token ();
+      restore_token (lexer);
 #if DUMP_TOKENS
       dump_token ();
 #endif
@@ -165,41 +163,41 @@ lex_get (void)
   for (;;)
     {
       /* Skip whitespace. */
-      if (eof) 
+      if (lexer->eof) 
         {
-          token = T_STOP;
+          lexer->token = T_STOP;
           return;
         }
 
       for (;;)
 	{
-	  while (isspace ((unsigned char) *prog))
-	    prog++;
-	  if (*prog)
+	  while (isspace ((unsigned char) *lexer->prog))
+	    lexer->prog++;
+	  if (*lexer->prog)
 	    break;
 
-	  if (dot)
+	  if (lexer->dot)
 	    {
-	      dot = 0;
-	      token = '.';
+	      lexer->dot = 0;
+	      lexer->token = '.';
 #if DUMP_TOKENS
 	      dump_token ();
 #endif
 	      return;
 	    }
-	  else if (!lex_get_line ())
+	  else if (!lex_get_line (lexer))
 	    {
-	      eof = true;
-	      token = T_STOP;
+	      lexer->eof = true;
+	      lexer->token = T_STOP;
 #if DUMP_TOKENS
 	      dump_token ();
 #endif
 	      return;
 	    }
 
-	  if (put_token)
+	  if (lexer->put_token)
 	    {
-              restore_token ();
+              restore_token (lexer);
 #if DUMP_TOKENS
 	      dump_token ();
 #endif
@@ -209,9 +207,9 @@ lex_get (void)
 
 
       /* Actually parse the token. */
-      ds_clear (&tokstr);
+      ds_clear (&lexer->tokstr);
       
-      switch (*prog)
+      switch (*lexer->prog)
 	{
 	case '-': case '.':
 	case '0': case '1': case '2': case '3': case '4':
@@ -226,151 +224,151 @@ lex_get (void)
 	       want it as a number.  When the syntax calls for a `-'
 	       token, lex_negative_to_dash() must be used to break
 	       negative numbers into two tokens. */
-	    if (*prog == '-')
+	    if (*lexer->prog == '-')
 	      {
-		ds_put_char (&tokstr, *prog++);
-		while (isspace ((unsigned char) *prog))
-		  prog++;
+		ds_put_char (&lexer->tokstr, *lexer->prog++);
+		while (isspace ((unsigned char) *lexer->prog))
+		  lexer->prog++;
 
-		if (!isdigit ((unsigned char) *prog) && *prog != '.')
+		if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
 		  {
-		    token = '-';
+		    lexer->token = '-';
 		    break;
 		  }
-                token = T_NEG_NUM;
+                lexer->token = T_NEG_NUM;
 	      }
             else 
-              token = T_POS_NUM;
+              lexer->token = T_POS_NUM;
                 
 	    /* Parse the number, copying it into tokstr. */
-	    while (isdigit ((unsigned char) *prog))
-	      ds_put_char (&tokstr, *prog++);
-	    if (*prog == '.')
+	    while (isdigit ((unsigned char) *lexer->prog))
+	      ds_put_char (&lexer->tokstr, *lexer->prog++);
+	    if (*lexer->prog == '.')
 	      {
-		ds_put_char (&tokstr, *prog++);
-		while (isdigit ((unsigned char) *prog))
-		  ds_put_char (&tokstr, *prog++);
+		ds_put_char (&lexer->tokstr, *lexer->prog++);
+		while (isdigit ((unsigned char) *lexer->prog))
+		  ds_put_char (&lexer->tokstr, *lexer->prog++);
 	      }
-	    if (*prog == 'e' || *prog == 'E')
+	    if (*lexer->prog == 'e' || *lexer->prog == 'E')
 	      {
-		ds_put_char (&tokstr, *prog++);
-		if (*prog == '+' || *prog == '-')
-		  ds_put_char (&tokstr, *prog++);
-		while (isdigit ((unsigned char) *prog))
-		  ds_put_char (&tokstr, *prog++);
+		ds_put_char (&lexer->tokstr, *lexer->prog++);
+		if (*lexer->prog == '+' || *lexer->prog == '-')
+		  ds_put_char (&lexer->tokstr, *lexer->prog++);
+		while (isdigit ((unsigned char) *lexer->prog))
+		  ds_put_char (&lexer->tokstr, *lexer->prog++);
 	      }
 
 	    /* Parse as floating point. */
-	    tokval = strtod (ds_cstr (&tokstr), &tail);
+	    lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
 	    if (*tail)
 	      {
 		msg (SE, _("%s does not form a valid number."),
-		     ds_cstr (&tokstr));
-		tokval = 0.0;
+		     ds_cstr (&lexer->tokstr));
+		lexer->tokval = 0.0;
 
-		ds_clear (&tokstr);
-		ds_put_char (&tokstr, '0');
+		ds_clear (&lexer->tokstr);
+		ds_put_char (&lexer->tokstr, '0');
 	      }
 
 	    break;
 	  }
 
 	case '\'': case '"':
-	  token = parse_string (CHARACTER_STRING);
+	  lexer->token = parse_string (lexer, CHARACTER_STRING);
 	  break;
 
 	case '(': case ')': case ',': case '=': case '+': case '/':
-	  token = *prog++;
+	  lexer->token = *lexer->prog++;
 	  break;
 
 	case '*':
-	  if (*++prog == '*')
+	  if (*++lexer->prog == '*')
 	    {
-	      prog++;
-	      token = T_EXP;
+	      lexer->prog++;
+	      lexer->token = T_EXP;
 	    }
 	  else
-	    token = '*';
+	    lexer->token = '*';
 	  break;
 
 	case '<':
-	  if (*++prog == '=')
+	  if (*++lexer->prog == '=')
 	    {
-	      prog++;
-	      token = T_LE;
+	      lexer->prog++;
+	      lexer->token = T_LE;
 	    }
-	  else if (*prog == '>')
+	  else if (*lexer->prog == '>')
 	    {
-	      prog++;
-	      token = T_NE;
+	      lexer->prog++;
+	      lexer->token = T_NE;
 	    }
 	  else
-	    token = T_LT;
+	    lexer->token = T_LT;
 	  break;
 
 	case '>':
-	  if (*++prog == '=')
+	  if (*++lexer->prog == '=')
 	    {
-	      prog++;
-	      token = T_GE;
+	      lexer->prog++;
+	      lexer->token = T_GE;
 	    }
 	  else
-	    token = T_GT;
+	    lexer->token = T_GT;
 	  break;
 
 	case '~':
-	  if (*++prog == '=')
+	  if (*++lexer->prog == '=')
 	    {
-	      prog++;
-	      token = T_NE;
+	      lexer->prog++;
+	      lexer->token = T_NE;
 	    }
 	  else
-	    token = T_NOT;
+	    lexer->token = T_NOT;
 	  break;
 
 	case '&':
-	  prog++;
-	  token = T_AND;
+	  lexer->prog++;
+	  lexer->token = T_AND;
 	  break;
 
 	case '|':
-	  prog++;
-	  token = T_OR;
+	  lexer->prog++;
+	  lexer->token = T_OR;
 	  break;
 
         case 'b': case 'B':
-          if (prog[1] == '\'' || prog[1] == '"')
-            token = parse_string (BINARY_STRING);
+          if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
+            lexer->token = parse_string (lexer, BINARY_STRING);
           else
-            token = parse_id ();
+            lexer->token = parse_id (lexer);
           break;
           
         case 'o': case 'O':
-          if (prog[1] == '\'' || prog[1] == '"')
-            token = parse_string (OCTAL_STRING);
+          if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
+            lexer->token = parse_string (lexer, OCTAL_STRING);
           else
-            token = parse_id ();
+            lexer->token = parse_id (lexer);
           break;
           
         case 'x': case 'X':
-          if (prog[1] == '\'' || prog[1] == '"')
-            token = parse_string (HEX_STRING);
+          if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
+            lexer->token = parse_string (lexer, HEX_STRING);
           else
-            token = parse_id ();
+            lexer->token = parse_id (lexer);
           break;
           
 	default:
-          if (lex_is_id1 (*prog)) 
+          if (lex_is_id1 (*lexer->prog)) 
             {
-              token = parse_id ();
+              lexer->token = parse_id (lexer);
               break; 
             }
           else
             {
-              if (isgraph ((unsigned char) *prog))
-                msg (SE, _("Bad character in input: `%c'."), *prog++);
+              if (isgraph ((unsigned char) *lexer->prog))
+                msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
               else
-                msg (SE, _("Bad character in input: `\\%o'."), *prog++);
+                msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
               continue; 
             }
         }
@@ -386,14 +384,14 @@ lex_get (void)
    tokstr.
    Returns the correct token type. */
 static int
-parse_id (void) 
+parse_id (struct lexer *lexer) 
 {
-  const char *start = prog;
-  prog = lex_skip_identifier (start);
+  const char *start = lexer->prog;
+  lexer->prog = lex_skip_identifier (start);
 
-  ds_put_substring (&tokstr, ss_buffer (start, prog - start));
-  str_copy_trunc (tokid, sizeof tokid, ds_cstr (&tokstr));
-  return lex_id_to_token (ds_cstr (&tokstr), ds_length (&tokstr));
+  ds_put_substring (&lexer->tokstr, ss_buffer (start, lexer->prog - start));
+  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
+  return lex_id_to_token (ds_cstr (&lexer->tokstr), ds_length (&lexer->tokstr));
 }
 
 /* Reports an error to the effect that subcommand SBC may only be
@@ -407,23 +405,23 @@ lex_sbc_only_once (const char *sbc)
 /* Reports an error to the effect that subcommand SBC is
    missing. */
 void
-lex_sbc_missing (const char *sbc) 
+lex_sbc_missing (struct lexer *lexer, const char *sbc) 
 {
-  lex_error (_("missing required subcommand %s"), sbc);
+  lex_error (lexer, _("missing required subcommand %s"), sbc);
 }
 
 /* Prints a syntax error message containing the current token and
    given message MESSAGE (if non-null). */
 void
-lex_error (const char *message, ...)
+lex_error (struct lexer *lexer, const char *message, ...)
 {
   char *token_rep;
   char where[128];
 
-  token_rep = lex_token_representation ();
-  if (token == T_STOP)
+  token_rep = lex_token_representation (lexer);
+  if (lexer->token == T_STOP)
     strcpy (where, "end of file");
-  else if (token == '.')
+  else if (lexer->token == '.')
     strcpy (where, "end of command");
   else
     snprintf (where, sizeof where, "`%s'", token_rep);
@@ -449,11 +447,11 @@ lex_error (const char *message, ...)
    If not, flags a syntax error and returns an error command
    completion code. */
 int
-lex_end_of_command (void)
+lex_end_of_command (struct lexer *lexer)
 {
-  if (token != '.')
+  if (lexer->token != '.')
     {
-      lex_error (_("expecting end of command"));
+      lex_error (lexer, _("expecting end of command"));
       return CMD_FAILURE;
     }
   else
@@ -464,38 +462,38 @@ lex_end_of_command (void)
 
 /* Returns true if the current token is a number. */
 bool
-lex_is_number (void) 
+lex_is_number (struct lexer *lexer) 
 {
-  return token == T_POS_NUM || token == T_NEG_NUM;
+  return lexer->token == T_POS_NUM || lexer->token == T_NEG_NUM;
 }
 
 /* Returns the value of the current token, which must be a
    floating point number. */
 double
-lex_number (void)
+lex_number (struct lexer *lexer)
 {
-  assert (lex_is_number ());
-  return tokval;
+  assert (lex_is_number (lexer));
+  return lexer->tokval;
 }
 
 /* Returns true iff the current token is an integer. */
 bool
-lex_is_integer (void)
+lex_is_integer (struct lexer *lexer)
 {
-  return (lex_is_number ()
-	  && tokval != NOT_LONG
-	  && tokval >= LONG_MIN
-	  && tokval <= LONG_MAX
-	  && floor (tokval) == tokval);
+  return (lex_is_number (lexer)
+	  && lexer->tokval != NOT_LONG
+	  && lexer->tokval >= LONG_MIN
+	  && lexer->tokval <= LONG_MAX
+	  && floor (lexer->tokval) == lexer->tokval);
 }
 
 /* Returns the value of the current token, which must be an
    integer. */
 long
-lex_integer (void)
+lex_integer (struct lexer *lexer)
 {
-  assert (lex_is_integer ());
-  return tokval;
+  assert (lex_is_integer (lexer));
+  return lexer->tokval;
 }
   
 /* Token matching functions. */
@@ -503,11 +501,11 @@ lex_integer (void)
 /* If TOK is the current token, skips it and returns true
    Otherwise, returns false. */
 bool
-lex_match (int t)
+lex_match (struct lexer *lexer, int t)
 {
-  if (token == t)
+  if (lexer->token == t)
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else
@@ -519,11 +517,11 @@ lex_match (int t)
    letters.
    Otherwise, returns false. */
 bool
-lex_match_id (const char *s)
+lex_match_id (struct lexer *lexer, const char *s)
 {
-  if (token == T_ID && lex_id_match (s, tokid))
+  if (lexer->token == T_ID && lex_id_match (s, lexer->tokid))
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else
@@ -533,11 +531,11 @@ lex_match_id (const char *s)
 /* If the current token is integer N, skips it and returns true.
    Otherwise, returns false. */
 bool
-lex_match_int (int x)
+lex_match_int (struct lexer *lexer, int x)
 {
-  if (lex_is_integer () && lex_integer () == x)
+  if (lex_is_integer (lexer) && lex_integer (lexer) == x)
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else
@@ -550,16 +548,16 @@ lex_match_int (int x)
    nonzero.
    Otherwise, reports an error and returns zero. */
 bool
-lex_force_match_id (const char *s)
+lex_force_match_id (struct lexer *lexer, const char *s)
 {
-  if (token == T_ID && lex_id_match (s, tokid))
+  if (lexer->token == T_ID && lex_id_match (s, lexer->tokid))
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else
     {
-      lex_error (_("expecting `%s'"), s);
+      lex_error (lexer, _("expecting `%s'"), s);
       return false;
     }
 }
@@ -567,16 +565,16 @@ lex_force_match_id (const char *s)
 /* If the current token is T, skips the token.  Otherwise, reports an
    error and returns from the current function with return value false. */
 bool
-lex_force_match (int t)
+lex_force_match (struct lexer *lexer, int t)
 {
-  if (token == t)
+  if (lexer->token == t)
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else
     {
-      lex_error (_("expecting `%s'"), lex_token_name (t));
+      lex_error (lexer, _("expecting `%s'"), lex_token_name (t));
       return false;
     }
 }
@@ -584,13 +582,13 @@ lex_force_match (int t)
 /* If this token is a string, does nothing and returns true.
    Otherwise, reports an error and returns false. */
 bool
-lex_force_string (void)
+lex_force_string (struct lexer *lexer)
 {
-  if (token == T_STRING)
+  if (lexer->token == T_STRING)
     return true;
   else
     {
-      lex_error (_("expecting string"));
+      lex_error (lexer, _("expecting string"));
       return false;
     }
 }
@@ -598,13 +596,13 @@ lex_force_string (void)
 /* If this token is an integer, does nothing and returns true.
    Otherwise, reports an error and returns false. */
 bool
-lex_force_int (void)
+lex_force_int (struct lexer *lexer)
 {
-  if (lex_is_integer ())
+  if (lex_is_integer (lexer))
     return true;
   else
     {
-      lex_error (_("expecting integer"));
+      lex_error (lexer, _("expecting integer"));
       return false;
     }
 }
@@ -612,30 +610,27 @@ lex_force_int (void)
 /* If this token is a number, does nothing and returns true.
    Otherwise, reports an error and returns false. */
 bool
-lex_force_num (void)
+lex_force_num (struct lexer *lexer)
 {
-  if (lex_is_number ())
+  if (lex_is_number (lexer))
     return true;
-  else
-    {
-      lex_error (_("expecting number"));
-      return false;
-    }
+
+  lex_error (lexer, _("expecting number"));
+  return false;
 }
 	
 /* If this token is an identifier, does nothing and returns true.
    Otherwise, reports an error and returns false. */
 bool
-lex_force_id (void)
+lex_force_id (struct lexer *lexer)
 {
-  if (token == T_ID)
+  if (lexer->token == T_ID)
     return true;
-  else
-    {
-      lex_error (_("expecting identifier"));
-      return false;
-    }
+
+  lex_error (lexer, _("expecting identifier"));
+  return false;
 }
+
 /* Weird token functions. */
 
 /* Returns the first character of the next token, except that if the
@@ -645,76 +640,76 @@ lex_force_id (void)
    alphanumeric return value doesn't guarantee an ID token, it could
    also be a reserved-word token. */
 int
-lex_look_ahead (void)
+lex_look_ahead (struct lexer *lexer)
 {
-  if (put_token)
-    return put_token;
+  if (lexer->put_token)
+    return lexer->put_token;
 
   for (;;)
     {
-      if (eof)
+      if (lexer->eof)
         return 0;
 
       for (;;)
 	{
-	  while (isspace ((unsigned char) *prog))
-	    prog++;
-	  if (*prog)
+	  while (isspace ((unsigned char) *lexer->prog))
+	    lexer->prog++;
+	  if (*lexer->prog)
 	    break;
 
-	  if (dot)
+	  if (lexer->dot)
 	    return '.';
-	  else if (!lex_get_line ())
+	  else if (!lex_get_line (lexer))
             return 0;
 
-	  if (put_token) 
-	    return put_token;
+	  if (lexer->put_token) 
+	    return lexer->put_token;
 	}
 
-      if ((toupper ((unsigned char) *prog) == 'X'
-	   || toupper ((unsigned char) *prog) == 'B'
-           || toupper ((unsigned char) *prog) == 'O')
-	  && (prog[1] == '\'' || prog[1] == '"'))
+      if ((toupper ((unsigned char) *lexer->prog) == 'X'
+	   || toupper ((unsigned char) *lexer->prog) == 'B'
+           || toupper ((unsigned char) *lexer->prog) == 'O')
+	  && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
 	return '\'';
 
-      return *prog;
+      return *lexer->prog;
     }
 }
 
 /* Makes the current token become the next token to be read; the
    current token is set to T. */
 void
-lex_put_back (int t)
+lex_put_back (struct lexer *lexer, int t)
 {
-  save_token ();
-  token = t;
+  save_token (lexer);
+  lexer->token = t;
 }
 
 /* Makes the current token become the next token to be read; the
    current token is set to the identifier ID. */
 void
-lex_put_back_id (const char *id)
+lex_put_back_id (struct lexer *lexer, const char *id)
 {
   assert (lex_id_to_token (id, strlen (id)) == T_ID);
-  save_token ();
-  token = T_ID;
-  ds_assign_cstr (&tokstr, id);
-  str_copy_trunc (tokid, sizeof tokid, ds_cstr (&tokstr));
+  save_token (lexer);
+  lexer->token = T_ID;
+  ds_assign_cstr (&lexer->tokstr, id);
+  str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
 }
 
 /* Weird line processing functions. */
 
 /* Returns the entire contents of the current line. */
 const char *
-lex_entire_line (void)
+lex_entire_line (struct lexer *lexer)
 {
-  return ds_cstr (&line_buffer);
+  return ds_cstr (&lexer->line_buffer);
 }
 
 const struct string *
-lex_entire_line_ds (void)
+lex_entire_line_ds (struct lexer *lexer)
 {
-  return &line_buffer;
+  return &lexer->line_buffer;
 }
 
 /* As lex_entire_line(), but only returns the part of the current line
@@ -722,22 +717,22 @@ lex_entire_line_ds (void)
    If END_DOT is non-null, stores nonzero into *END_DOT if the line
    ends with a terminal dot, or zero if it doesn't. */
 const char *
-lex_rest_of_line (int *end_dot)
+lex_rest_of_line (struct lexer *lexer, int *end_dot)
 {
   if (end_dot)
-    *end_dot = dot;
-  return prog;
+    *end_dot = lexer->dot;
+  return lexer->prog;
 }
 
 /* Causes the rest of the current input line to be ignored for
    tokenization purposes. */
 void
-lex_discard_line (void)
+lex_discard_line (struct lexer *lexer)
 {
-  ds_cstr (&line_buffer);  /* Ensures ds_end points to something valid */
-  prog = ds_end (&line_buffer);
-  dot = false;
-  put_token = 0;
+  ds_cstr (&lexer->line_buffer);  /* Ensures ds_end points to something valid */
+  lexer->prog = ds_end (&lexer->line_buffer);
+  lexer->dot = false;
+  lexer->put_token = 0;
 }
 
 
@@ -749,15 +744,15 @@ lex_discard_line (void)
    the user doesn't want to finish typing a command that will be
    ignored anyway. */
 void
-lex_discard_rest_of_command (void) 
+lex_discard_rest_of_command (struct lexer *lexer) 
 {
   if (!getl_is_interactive ())
     {
-      while (token != T_STOP && token != '.')
-	lex_get ();
+      while (lexer->token != T_STOP && lexer->token != '.')
+	lex_get (lexer);
     }
   else 
-    lex_discard_line (); 
+    lex_discard_line (lexer); 
 }
 
 /* Weird line reading functions. */
@@ -812,29 +807,29 @@ strip_comments (struct string *string)
 
 /* Reads a line, without performing any preprocessing */
 bool 
-lex_get_line_raw (void)
+lex_get_line_raw (struct lexer *lexer)
 {
   bool dummy;
-  return lex_read_line (&line_buffer, &dummy);
+  return lexer->read_line (&lexer->line_buffer, &dummy);
 }
 
 /* Reads a line for use by the tokenizer, and preprocesses it by
    removing comments, stripping trailing whitespace and the
    terminal dot, and removing leading indentors. */
 bool
-lex_get_line (void)
+lex_get_line (struct lexer *lexer)
 {
-  struct string *line = &line_buffer;
+  struct string *line = &lexer->line_buffer;
   bool interactive;
 
-  if (!lex_read_line (line, &interactive))
+  if (!lexer->read_line (line, &interactive))
     return false;
 
   strip_comments (line);
   ds_rtrim (line, ss_cstr (CC_SPACES));
   
   /* Check for and remove terminal dot. */
-  dot = (ds_chomp (line, get_endcmd ())
+  lexer->dot = (ds_chomp (line, get_endcmd ())
          || (ds_is_empty (line) && get_nulline ()));
   
   /* Strip leading indentors or insert a terminal dot (unless the
@@ -846,10 +841,10 @@ lex_get_line (void)
       if (first == '+' || first == '-')
 	*ds_data (line) = ' ';
       else if (first != EOF && !isspace (first))
-	put_token = '.';
+	lexer->put_token = '.';
     }
 
-  prog = ds_cstr (line);
+  lexer->prog = ds_cstr (line);
 
   return true;
 }
@@ -876,16 +871,16 @@ lex_token_name (int token)
 /* Returns an ASCII representation of the current token as a
    malloc()'d string. */
 char *
-lex_token_representation (void)
+lex_token_representation (struct lexer *lexer)
 {
   char *token_rep;
   
-  switch (token)
+  switch (lexer->token)
     {
     case T_ID:
     case T_POS_NUM:
     case T_NEG_NUM:
-      return ds_xstrdup (&tokstr);
+      return ds_xstrdup (&lexer->tokstr);
       break;
 
     case T_STRING:
@@ -893,14 +888,14 @@ lex_token_representation (void)
 	int hexstring = 0;
 	char *sp, *dp;
 
-	for (sp = ds_cstr (&tokstr); sp < ds_end (&tokstr); sp++)
+	for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
 	  if (!isprint ((unsigned char) *sp))
 	    {
 	      hexstring = 1;
 	      break;
 	    }
 	      
-	token_rep = xmalloc (2 + ds_length (&tokstr) * 2 + 1 + 1);
+	token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
 
 	dp = token_rep;
 	if (hexstring)
@@ -908,14 +903,14 @@ lex_token_representation (void)
 	*dp++ = '\'';
 
 	if (!hexstring)
-	  for (sp = ds_cstr (&tokstr); *sp; )
+	  for (sp = ds_cstr (&lexer->tokstr); *sp; )
 	    {
 	      if (*sp == '\'')
 		*dp++ = '\'';
 	      *dp++ = (unsigned char) *sp++;
 	    }
 	else
-	  for (sp = ds_cstr (&tokstr); sp < ds_end (&tokstr); sp++)
+	  for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
 	    {
 	      *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
 	      *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
@@ -936,12 +931,12 @@ lex_token_representation (void)
       return xstrdup ("**");
 
     default:
-      if (token >= T_FIRST_KEYWORD && token <= T_LAST_KEYWORD)
-	return xstrdup (keywords [token - T_FIRST_KEYWORD]);
+      if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD)
+	return xstrdup (keywords [lexer->token - T_FIRST_KEYWORD]);
       else
 	{
 	  token_rep = xmalloc (2);
-	  token_rep[0] = token;
+	  token_rep[0] = lexer->token;
 	  token_rep[1] = '\0';
 	  return token_rep;
 	}
@@ -957,44 +952,44 @@ lex_token_representation (void)
    of syntax then this function is called to rip it off of a
    number. */
 void
-lex_negative_to_dash (void)
+lex_negative_to_dash (struct lexer *lexer)
 {
-  if (token == T_NEG_NUM)
+  if (lexer->token == T_NEG_NUM)
     {
-      token = T_POS_NUM;
-      tokval = -tokval;
-      ds_assign_substring (&tokstr, ds_substr (&tokstr, 1, SIZE_MAX));
-      save_token ();
-      token = '-';
+      lexer->token = T_POS_NUM;
+      lexer->tokval = -lexer->tokval;
+      ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
+      save_token (lexer);
+      lexer->token = '-';
     }
 }
    
 /* We're not at eof any more. */
 void
-lex_reset_eof (void)
+lex_reset_eof (struct lexer *lexer)
 {
-  eof = false;
+  lexer->eof = false;
 }
 
 /* Skip a COMMENT command. */
 void
-lex_skip_comment (void)
+lex_skip_comment (struct lexer *lexer)
 {
   for (;;)
     {
-      if (!lex_get_line ()) 
+      if (!lex_get_line (lexer)) 
         {
-          put_token = T_STOP;
-          eof = true;
+          lexer->put_token = T_STOP;
+          lexer->eof = true;
           return;
         }
       
-      if (put_token == '.')
+      if (lexer->put_token == '.')
 	break;
 
-      ds_cstr (&line_buffer); /* Ensures ds_end will point to a valid char */
-      prog = ds_end (&line_buffer);
-      if (dot)
+      ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
+      lexer->prog = ds_end (&lexer->line_buffer);
+      if (lexer->dot)
 	break;
     }
 }
@@ -1005,7 +1000,8 @@ lex_skip_comment (void)
    hex digits, according to TYPE.  The string is converted to
    characters having the specified values. */
 static void
-convert_numeric_string_to_char_string (enum string_type type)
+convert_numeric_string_to_char_string (struct lexer *lexer, 
+				       enum string_type type)
 {
   const char *base_name;
   int base;
@@ -1035,13 +1031,13 @@ convert_numeric_string_to_char_string (enum string_type type)
       NOT_REACHED ();
     }
   
-  byte_cnt = ds_length (&tokstr) / chars_per_byte;
-  if (ds_length (&tokstr) % chars_per_byte)
+  byte_cnt = ds_length (&lexer->tokstr) / chars_per_byte;
+  if (ds_length (&lexer->tokstr) % chars_per_byte)
     msg (SE, _("String of %s digits has %d characters, which is not a "
 	       "multiple of %d."),
-	 base_name, ds_length (&tokstr), chars_per_byte);
+	 base_name, ds_length (&lexer->tokstr), chars_per_byte);
 
-  p = ds_cstr (&tokstr);
+  p = ds_cstr (&lexer->tokstr);
   for (i = 0; i < byte_cnt; i++)
     {
       int value;
@@ -1071,88 +1067,88 @@ convert_numeric_string_to_char_string (enum string_type type)
 	  value = value * base + v;
 	}
 
-      ds_cstr (&tokstr)[i] = (unsigned char) value;
+      ds_cstr (&lexer->tokstr)[i] = (unsigned char) value;
     }
 
-  ds_truncate (&tokstr, byte_cnt);
+  ds_truncate (&lexer->tokstr, byte_cnt);
 }
 
 /* Parses a string from the input buffer into tokstr.  The input
-   buffer pointer prog must point to the initial single or double
+   buffer pointer lexer->prog must point to the initial single or double
    quote.  TYPE indicates the type of string to be parsed.
    Returns token type. */
 static int 
-parse_string (enum string_type type)
+parse_string (struct lexer *lexer, enum string_type type)
 {
   if (type != CHARACTER_STRING)
-    prog++;
+    lexer->prog++;
 
   /* Accumulate the entire string, joining sections indicated by +
      signs. */
   for (;;)
     {
       /* Single or double quote. */
-      int c = *prog++;
+      int c = *lexer->prog++;
       
       /* Accumulate section. */
       for (;;)
 	{
 	  /* Check end of line. */
-	  if (*prog == '\0')
+	  if (*lexer->prog == '\0')
 	    {
 	      msg (SE, _("Unterminated string constant."));
 	      goto finish;
 	    }
 	  
 	  /* Double quote characters to embed them in strings. */
-	  if (*prog == c)
+	  if (*lexer->prog == c)
 	    {
-	      if (prog[1] == c)
-		prog++;
+	      if (lexer->prog[1] == c)
+		lexer->prog++;
 	      else
 		break;
 	    }
 
-	  ds_put_char (&tokstr, *prog++);
+	  ds_put_char (&lexer->tokstr, *lexer->prog++);
 	}
-      prog++;
+      lexer->prog++;
 
       /* Skip whitespace after final quote mark. */
-      if (eof)
+      if (lexer->eof)
 	break;
       for (;;)
 	{
-	  while (isspace ((unsigned char) *prog))
-	    prog++;
-	  if (*prog)
+	  while (isspace ((unsigned char) *lexer->prog))
+	    lexer->prog++;
+	  if (*lexer->prog)
 	    break;
 
-	  if (dot)
+	  if (lexer->dot)
 	    goto finish;
 
-	  if (!lex_get_line ())
+	  if (!lex_get_line (lexer))
             goto finish;
 	}
 
       /* Skip plus sign. */
-      if (*prog != '+')
+      if (*lexer->prog != '+')
 	break;
-      prog++;
+      lexer->prog++;
 
       /* Skip whitespace after plus sign. */
-      if (eof)
+      if (lexer->eof)
 	break;
       for (;;)
 	{
-	  while (isspace ((unsigned char) *prog))
-	    prog++;
-	  if (*prog)
+	  while (isspace ((unsigned char) *lexer->prog))
+	    lexer->prog++;
+	  if (*lexer->prog)
 	    break;
 
-	  if (dot)
+	  if (lexer->dot)
 	    goto finish;
 
-	  if (!lex_get_line ())
+	  if (!lex_get_line (lexer))
             {
               msg (SE, _("Unexpected end of file in string concatenation."));
               goto finish;
@@ -1160,7 +1156,7 @@ parse_string (enum string_type type)
 	}
 
       /* Ensure that a valid string follows. */
-      if (*prog != '\'' && *prog != '"')
+      if (*lexer->prog != '\'' && *lexer->prog != '"')
 	{
 	  msg (SE, _("String expected following `+'."));
 	  goto finish;
@@ -1171,13 +1167,13 @@ parse_string (enum string_type type)
      into one large string. */
 finish:
   if (type != CHARACTER_STRING)
-    convert_numeric_string_to_char_string (type);
+    convert_numeric_string_to_char_string (lexer, type);
 
-  if (ds_length (&tokstr) > 255)
+  if (ds_length (&lexer->tokstr) > 255)
     {
       msg (SE, _("String exceeds 255 characters in length (%d characters)."),
-	   ds_length (&tokstr));
-      ds_truncate (&tokstr, 255);
+	   ds_length (&lexer->tokstr));
+      ds_truncate (&lexer->tokstr, 255);
     }
       
   return T_STRING;
@@ -1187,7 +1183,7 @@ finish:
 /* Reads one token from the lexer and writes a textual representation
    on stdout for debugging purposes. */
 static void
-dump_token (void)
+dump_token (struct lexer *lexer)
 {
   {
     const char *curfn;
@@ -1198,19 +1194,19 @@ dump_token (void)
       fprintf (stderr, "%s:%d\t", curfn, curln);
   }
   
-  switch (token)
+  switch (lexer->token)
     {
     case T_ID:
-      fprintf (stderr, "ID\t%s\n", tokid);
+      fprintf (stderr, "ID\t%s\n", lexer->tokid);
       break;
 
     case T_POS_NUM:
     case T_NEG_NUM:
-      fprintf (stderr, "NUM\t%f\n", tokval);
+      fprintf (stderr, "NUM\t%f\n", lexer->tokval);
       break;
 
     case T_STRING:
-      fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&tokstr));
+      fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
       break;
 
     case T_STOP:
@@ -1226,11 +1222,38 @@ dump_token (void)
       break;
 
     default:
-      if (token >= T_FIRST_KEYWORD && token <= T_LAST_KEYWORD)
+      if (lexer->token >= T_FIRST_KEYWORD && lexer->token <= T_LAST_KEYWORD)
 	fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (token));
       else
-	fprintf (stderr, "PUNCT\t%c\n", token);
+	fprintf (stderr, "PUNCT\t%c\n", lexer->token);
       break;
     }
 }
 #endif /* DUMP_TOKENS */
+
+
+/* Token Accessor Functions */
+
+int 
+lex_token (const struct lexer *lexer)
+{
+  return lexer->token;
+}
+
+double 
+lex_tokval (const struct lexer *lexer)
+{
+  return lexer->tokval;
+}
+
+const char *
+lex_tokid (const struct lexer *lexer)
+{
+  return lexer->tokid;
+}
+
+const struct string *
+lex_tokstr (const struct lexer *lexer)
+{
+  return &lexer->tokstr;
+}
diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index c367bbb1..792788c5 100644
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -23,70 +23,74 @@
 #include <data/variable.h>
 #include <ctype.h>
 #include <stdbool.h>
-
 #include <data/identifier.h>
 
-
-extern int token;
-extern double tokval;
-extern char tokid[LONG_NAME_LEN + 1];
-extern struct string tokstr;
-
 #include <stddef.h>
 
+struct lexer ;
+
 /* Initialization. */
-void lex_init (bool (*)(struct string *, bool*));
-void lex_done (void);
+struct lexer * lex_create (bool (*)(struct string *, bool*));
+void lex_destroy (struct lexer *);
+
+
+
 
 /* Common functions. */
-void lex_get (void);
-void lex_error (const char *, ...);
+void lex_get (struct lexer *);
+void lex_error (struct lexer *, const char *, ...);
 void lex_sbc_only_once (const char *);
-void lex_sbc_missing (const char *);
-int lex_end_of_command (void);
+void lex_sbc_missing (struct lexer *, const char *);
+int lex_end_of_command (struct lexer *);
 
 /* Token testing functions. */
-bool lex_is_number (void);
-double lex_number (void);
-bool lex_is_integer (void);
-long lex_integer (void);
+bool lex_is_number (struct lexer *);
+double lex_number (struct lexer *);
+bool lex_is_integer (struct lexer *);
+long lex_integer (struct lexer *);
 
 /* Token matching functions. */
-bool lex_match (int);
-bool lex_match_id (const char *);
-bool lex_match_int (int);
+bool lex_match (struct lexer *, int);
+bool lex_match_id (struct lexer *, const char *);
+bool lex_match_int (struct lexer *, int);
 
 /* Forcible matching functions. */
-bool lex_force_match (int);
-bool lex_force_match_id (const char *);
-bool lex_force_int (void);
-bool lex_force_num (void);
-bool lex_force_id (void);
-bool lex_force_string (void);
+bool lex_force_match (struct lexer *, int);
+bool lex_force_match_id (struct lexer *, const char *);
+bool lex_force_int (struct lexer *);
+bool lex_force_num (struct lexer *);
+bool lex_force_id (struct lexer *);
+bool lex_force_string (struct lexer *);
 	
 /* Weird token functions. */
-int lex_look_ahead (void);
-void lex_put_back (int);
-void lex_put_back_id (const char *tokid);
+int lex_look_ahead (struct lexer *);
+void lex_put_back (struct lexer *, int);
+void lex_put_back_id (struct lexer *, const char *tokid);
 
 /* Weird line processing functions. */
-const char *lex_entire_line (void);
-const struct string *lex_entire_line_ds (void);
-const char *lex_rest_of_line (int *end_dot);
-void lex_discard_line (void);
-void lex_discard_rest_of_command (void);
+const char *lex_entire_line (struct lexer *);
+const struct string *lex_entire_line_ds (struct lexer *);
+const char *lex_rest_of_line (struct lexer *, int *end_dot);
+void lex_discard_line (struct lexer *);
+void lex_discard_rest_of_command (struct lexer *);
 
 /* Weird line reading functions. */
-bool lex_get_line (void);
-bool lex_get_line_raw (void);
+bool lex_get_line (struct lexer *);
+bool lex_get_line_raw (struct lexer *);
 
 /* Token names. */
 const char *lex_token_name (int);
-char *lex_token_representation (void);
+char *lex_token_representation (struct lexer *);
+
+/* Token accessors */
+int lex_token (const struct lexer *);
+double lex_tokval (const struct lexer *);
+const char *lex_tokid (const struct lexer *);
+const struct string *lex_tokstr (const struct lexer *);
 
 /* Really weird functions. */
-void lex_negative_to_dash (void);
-void lex_reset_eof (void);
-void lex_skip_comment (void);
+void lex_negative_to_dash (struct lexer *);
+void lex_reset_eof (struct lexer *);
+void lex_skip_comment (struct lexer *);
 
 #endif /* !lexer_h */
diff --git a/src/language/lexer/q2c.c b/src/language/lexer/q2c.c
index 70a4d780..4d60abbe 100644
--- a/src/language/lexer/q2c.c
+++ b/src/language/lexer/q2c.c
@@ -40,22 +40,22 @@
 #define MAX_TOK_LEN 1024
 
 /* argv[0]. */
-char *program_name;
+static char *program_name;
 
 /* Have the input and output files been opened yet? */
-bool is_open;
+static bool is_open;
 
 /* Input, output files. */
-FILE *in, *out;
+static FILE *in, *out;
 
 /* Input, output file names. */
-char *ifn, *ofn;
+static char *ifn, *ofn;
 
 /* Input, output file line number. */
-int ln, oln = 1;
+static int ln, oln = 1;
 
 /* Input line buffer, current position. */
-char *buf, *cp;
+static char *buf, *cp;
 
 /* Token types. */
 enum
@@ -65,14 +65,14 @@ enum
   };
 
 /* Current token: either one of the above, or a single character. */
-int token;
+static int token;
 
 /* Token string value. */
-char *tokstr;
+static char *tokstr;
 
 /* Utility functions. */
 
-char nullstr[] = "";
+static char nullstr[] = "";
 
 /* Close all open files and delete the output file, on failure. */
 static void
@@ -1177,7 +1177,7 @@ dump_declarations (void)
 	      dump (0, "/* Prototype for custom subcommands of %s. */",
 		    cmdname);
 	    }
-	  dump (0, "static int %scustom_%s (struct dataset *, struct cmd_%s *, void *);",
+	  dump (0, "static int %scustom_%s (struct lexer *, struct dataset *, struct cmd_%s *, void *);",
 		st_lower (prefix), st_lower (sbc->name),
 		make_identifier (cmdname));
 	}
@@ -1189,7 +1189,7 @@ dump_declarations (void)
   /* Prototypes for parsing and freeing functions. */
   {
     dump (0, "/* Command parsing functions. */");
-    dump (0, "static int parse_%s (struct dataset *, struct cmd_%s *, void *);",
+    dump (0, "static int parse_%s (struct lexer *, struct dataset *, struct cmd_%s *, void *);",
 	  make_identifier (cmdname), make_identifier (cmdname));
     dump (0, "static void free_%s (struct cmd_%s *);",
 	  make_identifier (cmdname), make_identifier (cmdname));
@@ -1342,17 +1342,17 @@ make_match (const char *t)
     t++;
       
   if (is_keyword (t))
-    sprintf (s, "lex_match (T_%s)", t);
+    sprintf (s, "lex_match (lexer, T_%s)", t);
   else if (!strcmp (t, "ON") || !strcmp (t, "YES"))
-    strcpy (s, "(lex_match_id (\"ON\") || lex_match_id (\"YES\") "
-	    "|| lex_match_id (\"TRUE\"))");
+    strcpy (s, "(lex_match_id (lexer, \"ON\") || lex_match_id (lexer, \"YES\") "
+	    "|| lex_match_id (lexer, \"TRUE\"))");
   else if (!strcmp (t, "OFF") || !strcmp (t, "NO"))
-    strcpy (s, "(lex_match_id (\"OFF\") || lex_match_id (\"NO\") "
-	    "|| lex_match_id (\"FALSE\"))");
+    strcpy (s, "(lex_match_id (lexer, \"OFF\") || lex_match_id (lexer, \"NO\") "
+	    "|| lex_match_id (lexer, \"FALSE\"))");
   else if (isdigit ((unsigned char) t[0]))
-    sprintf (s, "lex_match_int (%s)", t);
+    sprintf (s, "lex_match_int (lexer, %s)", t);
   else
-    sprintf (s, "lex_match_id (\"%s\")", t);
+    sprintf (s, "lex_match_id (lexer, \"%s\")", t);
   
   return s;
 }
@@ -1416,12 +1416,12 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
 	    {
 	      if (s->optvalue)
 		{
-		  dump (1, "if (lex_match ('('))");
+		  dump (1, "if (lex_match (lexer, '('))");
 		  dump (1, "{");
 		}
 	      else
 		{
-		  dump (1, "if (!lex_match ('('))");
+		  dump (1, "if (!lex_match (lexer, '('))");
 		  dump (1, "{");
 		  dump (0, "msg (SE, _(\"`(' expected after %s "
 			"specifier of %s subcommand.\"));",
@@ -1434,26 +1434,26 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
 
 	  if (s->value == VAL_INT)
 	    {
-	      dump (1, "if (!lex_is_integer ())");
+	      dump (1, "if (!lex_is_integer (lexer))");
 	      dump (1, "{");
 	      dump (0, "msg (SE, _(\"%s specifier of %s subcommand "
 		    "requires an integer argument.\"));",
 		    s->specname, sbc->name);
 	      dump (0, "goto lossage;");
 	      dump (-1, "}");
-	      dump (-1, "p->%s%s = lex_integer ();",
+	      dump (-1, "p->%s%s = lex_integer (lexer);",
 		    sbc->prefix, st_lower (s->valname));
 	    }
 	  else
 	    {
-	      dump (1, "if (!lex_is_number ())");
+	      dump (1, "if (!lex_is_number (lexer))");
 	      dump (1, "{");
 	      dump (0, "msg (SE, _(\"Number expected after %s "
 		    "specifier of %s subcommand.\"));",
 		    s->specname, sbc->name);
 	      dump (0, "goto lossage;");
 	      dump (-1, "}");
-	      dump (-1, "p->%s%s = tokval;", sbc->prefix,
+	      dump (-1, "p->%s%s = lex_tokval (lexer);", sbc->prefix,
 		    st_lower (s->valname));
 	    }
 	  
@@ -1480,11 +1480,11 @@ dump_specifier_parse (const specifier *spec, const subcommand *sbc)
 	      outdent ();
 	    }
 	  
-	  dump (0, "lex_get ();");
+	  dump (0, "lex_get (lexer);");
 	  
 	  if (s->valtype == VT_PAREN)
 	    {
-	      dump (1, "if (!lex_match (')'))");
+	      dump (1, "if (!lex_match (lexer, ')'))");
 	      dump (1, "{");
 	      dump (0, "msg (SE, _(\"`)' expected after argument for "
 		    "%s specifier of %s.\"));",
@@ -1520,7 +1520,7 @@ dump_subcommand (const subcommand *sbc)
     {
       int count;
 
-      dump (1, "while (token != '/' && token != '.')");
+      dump (1, "while (lex_token (lexer) != '/' && lex_token (lexer) != '.')");
       dump (1, "{");
       
       {
@@ -1571,20 +1571,20 @@ dump_subcommand (const subcommand *sbc)
 	  {
 	    dump (1, "else");
 	    dump (1, "{");
-	    dump (0, "lex_error (NULL);");
+	    dump (0, "lex_error (lexer, NULL);");
 	    dump (0, "goto lossage;");
 	    dump (-1, "}");
 	    outdent ();
 	  }
       }
 
-      dump (0, "lex_match (',');");
+      dump (0, "lex_match (lexer, ',');");
       dump (-1, "}");
       outdent ();
     }
   else if (sbc->type == SBC_VARLIST)
     {
-      dump (1, "if (!parse_variables (dataset_dict (ds), &p->%sv_%s, &p->%sn_%s, "
+      dump (1, "if (!parse_variables (lexer, dataset_dict (ds), &p->%sv_%s, &p->%sn_%s, "
 	    "PV_APPEND%s%s))",
 	    st_lower (sbc->prefix), st_lower (sbc->name),
 	    st_lower (sbc->prefix), st_lower (sbc->name),
@@ -1595,7 +1595,7 @@ dump_subcommand (const subcommand *sbc)
     }
   else if (sbc->type == SBC_VAR)
     {
-      dump (0, "p->%sv_%s = parse_variable (dataset_dict (ds));",
+      dump (0, "p->%sv_%s = parse_variable (lexer, dataset_dict (ds));",
 	    st_lower (sbc->prefix), st_lower (sbc->name));
       dump (1, "if (!p->%sv_%s)",
 	    st_lower (sbc->prefix), st_lower (sbc->name));
@@ -1609,12 +1609,12 @@ dump_subcommand (const subcommand *sbc)
 	  dump (1, "{");
 	  dump (0, "int x;");
 	}
-      dump (1, "if (!lex_force_string ())");
+      dump (1, "if (!lex_force_string (lexer))");
       dump (0, "return false;");
       outdent ();
       if (sbc->restriction)
 	{
-	  dump (0, "x = ds_length (&tokstr);");
+	  dump (0, "x = ds_length (lex_tokstr (lexer));");
 	  dump (1, "if (!(%s))", sbc->restriction);
 	  dump (1, "{");
 	  dump (0, "msg (SE, _(\"String for %s must be %s.\"));",
@@ -1624,28 +1624,28 @@ dump_subcommand (const subcommand *sbc)
 	  outdent ();
 	}
       dump (0, "free(p->s_%s);", st_lower(sbc->name) );
-      dump (0, "p->s_%s = ds_xstrdup (&tokstr);",
+      dump (0, "p->s_%s = ds_xstrdup (lex_tokstr (lexer));",
 	    st_lower (sbc->name));
-      dump (0, "lex_get ();");
+      dump (0, "lex_get (lexer);");
       if (sbc->restriction)
 	dump (-1, "}");
     }
   else if (sbc->type == SBC_DBL)
     {
-      dump (1, "if (!lex_force_num ())");
+      dump (1, "if (!lex_force_num (lexer))");
       dump (0, "goto lossage;");
-      dump (-1, "p->n_%s[p->sbc_%s - 1] = lex_number ();", 
+      dump (-1, "p->n_%s[p->sbc_%s - 1] = lex_number (lexer);", 
 	    st_lower (sbc->name), st_lower (sbc->name) );
-      dump (0, "lex_get();");
+      dump (0, "lex_get(lexer);");
     }
   else if (sbc->type == SBC_INT)
     {
       dump(1, "{");
       dump(0, "int x;");
-      dump (1, "if (!lex_force_int ())");
+      dump (1, "if (!lex_force_int (lexer))");
       dump (0, "goto lossage;");
-      dump (-1, "x = lex_integer ();");
-      dump (0, "lex_get();");
+      dump (-1, "x = lex_integer (lexer);");
+      dump (0, "lex_get(lexer);");
       if (sbc->restriction)
        {
 	  char buf[1024];
@@ -1664,11 +1664,11 @@ dump_subcommand (const subcommand *sbc)
     }
   else if (sbc->type == SBC_PINT)
     {
-      dump (0, "lex_match ('(');");
-      dump (1, "if (!lex_force_int ())");
+      dump (0, "lex_match (lexer, '(');");
+      dump (1, "if (!lex_force_int (lexer))");
       dump (0, "goto lossage;");
-      dump (-1, "p->n_%s = lex_integer ();", st_lower (sbc->name));
-      dump (0, "lex_match (')');");
+      dump (-1, "p->n_%s = lex_integer (lexer);", st_lower (sbc->name));
+      dump (0, "lex_match (lexer, ')');");
     }
   else if (sbc->type == SBC_DBL_LIST)
     {
@@ -1678,25 +1678,25 @@ dump_subcommand (const subcommand *sbc)
       dump (0, "goto lossage;");
       dump (-1,"}");
 
-      dump (1, "while (token != '/' && token != '.')");
+      dump (1, "while (lex_token (lexer) != '/' && lex_token (lexer) != '.')");
       dump (1, "{");
-      dump (0, "lex_match(',');");
-      dump (0, "if (!lex_force_num ())");
+      dump (0, "lex_match (lexer, ',');");
+      dump (0, "if (!lex_force_num (lexer))");
       dump (1, "{");
       dump (0, "goto lossage;");
       dump (-1,"}");
 
-      dump (0, "subc_list_double_push(&p->dl_%s[p->sbc_%s-1],lex_number ());", 
-	    st_lower (sbc->name),st_lower (sbc->name)
+      dump (0, "subc_list_double_push (&p->dl_%s[p->sbc_%s-1], lex_number (lexer));", 
+	    st_lower (sbc->name), st_lower (sbc->name)
 	    );
 
-      dump (0, "lex_get();");
+      dump (0, "lex_get (lexer);");
       dump (-1,"}");
 
     }
   else if (sbc->type == SBC_CUSTOM)
     {
-      dump (1, "switch (%scustom_%s (ds, p, aux))",
+      dump (1, "switch (%scustom_%s (lexer, ds, p, aux))",
 	    st_lower (prefix), st_lower (sbc->name));
       dump (0, "{");
       dump (1, "case 0:");
@@ -1706,7 +1706,7 @@ dump_subcommand (const subcommand *sbc)
       dump (0, "break;");
       dump (-1, "case 2:");
       indent ();
-      dump (0, "lex_error (NULL);");
+      dump (0, "lex_error (lexer, NULL);");
       dump (0, "goto lossage;");
       dump (-1, "default:");
       indent ();
@@ -1725,7 +1725,7 @@ dump_parser (int persistent)
   indent = 0;
 
   dump (0, "static int");
-  dump (0, "parse_%s (struct dataset *ds%s, struct cmd_%s *p, void *aux UNUSED)",
+  dump (0, "parse_%s (struct lexer *lexer, struct dataset *ds%s, struct cmd_%s *p, void *aux UNUSED)",
         make_identifier (cmdname),
 	(def && ( def->type == SBC_VARLIST && def->type == SBC_CUSTOM))?"":" UNUSED",
 	make_identifier (cmdname));
@@ -1740,19 +1740,19 @@ dump_parser (int persistent)
   if (def && (def->type == SBC_VARLIST))
     {
       if (def->type == SBC_VARLIST)
-	dump (1, "if (token == T_ID "
-              "&& dict_lookup_var (dataset_dict (ds), tokid) != NULL "
-	      "&& lex_look_ahead () != '=')");
+	dump (1, "if (lex_token (lexer) == T_ID "
+              "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL "
+	      "&& lex_look_ahead (lexer) != '=')");
       else
 	{
-	  dump (0, "if ((token == T_ID "
-                "&& dict_lookup_var (dataset_dict (ds), tokid) "
+	  dump (0, "if ((lex_token (lexer) == T_ID "
+                "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) "
 		"&& lex_look_ahead () != '=')");
 	  dump (1, "     || token == T_ALL)");
 	}
       dump (1, "{");
       dump (0, "p->sbc_%s++;", st_lower (def->name));
-      dump (1, "if (!parse_variables (dataset_dict (ds), &p->%sv_%s, &p->%sn_%s, "
+      dump (1, "if (!parse_variables (lexer, dataset_dict (ds), &p->%sv_%s, &p->%sn_%s, "
 	    "PV_APPEND))",
 	    st_lower (def->prefix), st_lower (def->name),
 	    st_lower (def->prefix), st_lower (def->name));
@@ -1763,7 +1763,7 @@ dump_parser (int persistent)
     }
   else if (def && def->type == SBC_CUSTOM)
     {
-      dump (1, "switch (%scustom_%s (ds, p, aux))",
+      dump (1, "switch (%scustom_%s (lexer, ds, p, aux))",
 	    st_lower (prefix), st_lower (def->name));
       dump (0, "{");
       dump (1, "case 0:");
@@ -1791,7 +1791,7 @@ dump_parser (int persistent)
 	f = 1;
 	dump (1, "{");
 
-	dump (0, "lex_match ('=');");
+	dump (0, "lex_match (lexer, '=');");
 	dump (0, "p->sbc_%s++;", st_lower (sbc->name));
 	if (sbc->arity != ARITY_MANY)
 	  {
@@ -1811,15 +1811,15 @@ dump_parser (int persistent)
 
 
   /* Now deal with the /ALGORITHM subcommand implicit to all commands */
-  dump(1,"else if ( get_syntax() != COMPATIBLE && lex_match_id(\"ALGORITHM\"))");
+  dump(1,"else if ( get_syntax() != COMPATIBLE && lex_match_id(lexer, \"ALGORITHM\"))");
   dump(1,"{");
 
-  dump (0, "lex_match ('=');");
+  dump (0, "lex_match (lexer, '=');");
 
-  dump(1,"if (lex_match_id(\"COMPATIBLE\"))");
+  dump(1,"if (lex_match_id(lexer, \"COMPATIBLE\"))");
   dump(0,"set_cmd_algorithm(COMPATIBLE);");
   outdent();
-  dump(1,"else if (lex_match_id(\"ENHANCED\"))");
+  dump(1,"else if (lex_match_id(lexer, \"ENHANCED\"))");
   dump(0,"set_cmd_algorithm(ENHANCED);");
 
   dump (-1, "}");
@@ -1827,14 +1827,14 @@ dump_parser (int persistent)
 
 
   
-  dump (1, "if (!lex_match ('/'))");
+  dump (1, "if (!lex_match (lexer, '/'))");
   dump (0, "break;");
   dump (-2, "}");
   outdent ();
   dump (0, nullstr);
-  dump (1, "if (token != '.')");
+  dump (1, "if (lex_token (lexer) != '.')");
   dump (1, "{");
-  dump (0, "lex_error (_(\"expecting end of command\"));");
+  dump (0, "lex_error (lexer, _(\"expecting end of command\"));");
   dump (0, "goto lossage;");
   dump (-1, "}");
   dump (0, nullstr);
diff --git a/src/language/lexer/range-parser.c b/src/language/lexer/range-parser.c
index 1682935c..ae25b631 100644
--- a/src/language/lexer/range-parser.c
+++ b/src/language/lexer/range-parser.c
@@ -31,7 +31,7 @@
 #define _(msgid) gettext (msgid)
 #define N_(msgid) msgid
 
-static bool parse_number (double *, const struct fmt_spec *);
+static bool parse_number (struct lexer *, double *, const struct fmt_spec *);
 
 /* Parses and stores a numeric value, or a range of the form "x
    THRU y".  Open-ended ranges may be specified as "LO(WEST) THRU
@@ -42,18 +42,18 @@ static bool parse_number (double *, const struct fmt_spec *);
    string values are also accepted, and converted to numeric
    values using the specified format. */
 bool
-parse_num_range (double *x, double *y, const struct fmt_spec *f) 
+parse_num_range (struct lexer *lexer, double *x, double *y, const struct fmt_spec *f) 
 {
-  if (lex_match_id ("LO") || lex_match_id ("LOWEST"))
+  if (lex_match_id (lexer, "LO") || lex_match_id (lexer, "LOWEST"))
     *x = LOWEST;
-  else if (!parse_number (x, f))
+  else if (!parse_number (lexer, x, f))
     return false;
 
-  if (lex_match_id ("THRU")) 
+  if (lex_match_id (lexer, "THRU")) 
     {
-      if (lex_match_id ("HI") || lex_match_id ("HIGHEST"))
+      if (lex_match_id (lexer, "HI") || lex_match_id (lexer, "HIGHEST"))
         *y = HIGHEST;
-      else if (!parse_number (y, f))
+      else if (!parse_number (lexer, y, f))
         return false;
 
       if (*y < *x) 
@@ -90,27 +90,27 @@ parse_num_range (double *x, double *y, const struct fmt_spec *f)
    string values are also accepted, and converted to numeric
    values using the specified format. */
 static bool
-parse_number (double *x, const struct fmt_spec *f)
+parse_number (struct lexer *lexer, double *x, const struct fmt_spec *f)
 {
-  if (lex_is_number ()) 
+  if (lex_is_number (lexer)) 
     {
-      *x = lex_number ();
-      lex_get ();
+      *x = lex_number (lexer);
+      lex_get (lexer);
       return true;
     }
-  else if (token == T_STRING && f != NULL) 
+  else if (lex_token (lexer) == T_STRING && f != NULL) 
     {
       struct data_in di;
       union value v;
-      di.s = ds_data (&tokstr);
-      di.e = ds_end (&tokstr);
+      di.s = ds_data (lex_tokstr (lexer));
+      di.e = ds_end (lex_tokstr (lexer));
       di.v = &v;
       di.flags = 0;
       di.f1 = 1;
-      di.f2 = ds_length (&tokstr);
+      di.f2 = ds_length (lex_tokstr (lexer));
       di.format = *f;
       data_in (&di);
-      lex_get ();
+      lex_get (lexer);
       *x = v.f;
       if (*x == SYSMIS)
         {
@@ -122,9 +122,9 @@ parse_number (double *x, const struct fmt_spec *f)
   else 
     {
       if (f != NULL)
-        lex_error (_("expecting number or data string"));
+        lex_error (lexer, _("expecting number or data string"));
       else
-        lex_force_num ();
+        lex_force_num (lexer);
       return false; 
     }
 }
diff --git a/src/language/lexer/range-parser.h b/src/language/lexer/range-parser.h
index f03a7e88..030a1927 100644
--- a/src/language/lexer/range-parser.h
+++ b/src/language/lexer/range-parser.h
@@ -23,6 +23,7 @@
 #include <stdbool.h>
 
 struct fmt_spec;
-bool parse_num_range (double *x, double *y, const struct fmt_spec *fmt);
+struct lexer;
+bool parse_num_range (struct lexer *, double *, double *, const struct fmt_spec *fmt);
 
 #endif /* range-prs.h */
diff --git a/src/language/lexer/subcommand-list.c b/src/language/lexer/subcommand-list.c
index 95469aa2..e309cf1b 100644
--- a/src/language/lexer/subcommand-list.c
+++ b/src/language/lexer/subcommand-list.c
@@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 02110-1301, USA. */
 
 
+#include <config.h>
 #include "subcommand-list.h"
 #include <stdlib.h>
 #include "xalloc.h"
diff --git a/src/language/lexer/variable-parser.c b/src/language/lexer/variable-parser.c
index 2de98829..44be48b9 100644
--- a/src/language/lexer/variable-parser.c
+++ b/src/language/lexer/variable-parser.c
@@ -44,23 +44,24 @@
    variable's index and returns true if successful.  On failure
    emits an error message and returns false. */
 static bool
-parse_vs_variable_idx (const struct var_set *vs, size_t *idx)
+parse_vs_variable_idx (struct lexer *lexer, const struct var_set *vs, 
+		size_t *idx)
 {
   assert (idx != NULL);
   
-  if (token != T_ID)
+  if (lex_token (lexer) != T_ID)
     {
-      lex_error (_("expecting variable name"));
+      lex_error (lexer, _("expecting variable name"));
       return false;
     }
-  else if (var_set_lookup_var_idx (vs, tokid, idx)) 
+  else if (var_set_lookup_var_idx (vs, lex_tokid (lexer), idx)) 
     {
-      lex_get ();
+      lex_get (lexer);
       return true;
     }
   else 
     {
-      msg (SE, _("%s is not a variable name."), tokid);
+      msg (SE, _("%s is not a variable name."), lex_tokid (lexer));
       return false;
     }
 }
@@ -69,20 +70,20 @@ parse_vs_variable_idx (const struct var_set *vs, size_t *idx)
    if successful.  On failure emits an error message and returns
    a null pointer. */
 static struct variable *
-parse_vs_variable (const struct var_set *vs)
+parse_vs_variable (struct lexer *lexer, const struct var_set *vs)
 {
   size_t idx;
-  return parse_vs_variable_idx (vs, &idx) ? var_set_get_var (vs, idx) : NULL;
+  return parse_vs_variable_idx (lexer, vs, &idx) ? var_set_get_var (vs, idx) : NULL;
 }
 
 /* Parses a variable name in dictionary D and returns the
    variable if successful.  On failure emits an error message and
    returns a null pointer. */
 struct variable *
-parse_variable (const struct dictionary *d) 
+parse_variable (struct lexer *lexer, const struct dictionary *d) 
 {
   struct var_set *vs = var_set_create_from_dict (d);
-  struct variable *var = parse_vs_variable (vs);
+  struct variable *var = parse_vs_variable (lexer, vs);
   var_set_destroy (vs);
   return var;
 }
@@ -92,8 +93,9 @@ parse_variable (const struct dictionary *d)
    number of variables into *CNT.  Returns true only if
    successful. */
 bool
-parse_variables (const struct dictionary *d, struct variable ***var,
-                 size_t *cnt, int opts) 
+parse_variables (struct lexer *lexer, const struct dictionary *d, 
+			struct variable ***var,
+			size_t *cnt, int opts) 
 {
   struct var_set *vs;
   int success;
@@ -103,7 +105,7 @@ parse_variables (const struct dictionary *d, struct variable ***var,
   assert (cnt != NULL);
 
   vs = var_set_create_from_dict (d);
-  success = parse_var_set_vars (vs, var, cnt, opts);
+  success = parse_var_set_vars (lexer, vs, var, cnt, opts);
   if ( success == 0 )
      free ( *var ) ;
   var_set_destroy (vs);
@@ -116,8 +118,9 @@ parse_variables (const struct dictionary *d, struct variable ***var,
    successful.  Same behavior as parse_variables, except that all
    allocations are taken from the given POOL. */
 bool
-parse_variables_pool (struct pool *pool, const struct dictionary *dict,
-                      struct variable ***vars, size_t *var_cnt, int opts) 
+parse_variables_pool (struct lexer *lexer, struct pool *pool, 
+		const struct dictionary *dict,
+		struct variable ***vars, size_t *var_cnt, int opts) 
 {
   int retval;
 
@@ -127,7 +130,7 @@ parse_variables_pool (struct pool *pool, const struct dictionary *dict,
      later. */
   assert (!(opts & PV_APPEND));
   
-  retval = parse_variables (dict, vars, var_cnt, opts);
+  retval = parse_variables (lexer, dict, vars, var_cnt, opts);
   if (retval)
     pool_register (pool, free, *vars);
   return retval;
@@ -138,10 +141,11 @@ parse_variables_pool (struct pool *pool, const struct dictionary *dict,
    dictionary class, and returns true.  Returns false on
    failure. */
 static bool
-parse_var_idx_class (const struct var_set *vs, size_t *idx,
-                     enum dict_class *class)
+parse_var_idx_class (struct lexer *lexer, const struct var_set *vs, 
+			size_t *idx,
+			enum dict_class *class)
 {
-  if (!parse_vs_variable_idx (vs, idx))
+  if (!parse_vs_variable_idx (lexer, vs, idx))
     return false;
 
   *class = dict_class_from_id (var_set_get_var (vs, *idx)->name);
@@ -212,7 +216,7 @@ add_variables (struct variable ***v, size_t *nv, size_t *mv, char *included,
    Conversely, if parse_variables() returns true, then *nv is
    nonzero and *v is non-NULL. */
 bool
-parse_var_set_vars (const struct var_set *vs, 
+parse_var_set_vars (struct lexer *lexer, const struct var_set *vs, 
                     struct variable ***v, size_t *nv,
                     int pv_opts)
 {
@@ -254,7 +258,7 @@ parse_var_set_vars (const struct var_set *vs,
 
   do
     {
-      if (lex_match (T_ALL))
+      if (lex_match (lexer, T_ALL))
         add_variables (v, nv, &mv, included, pv_opts,
                        vs, 0, var_set_get_cnt (vs) - 1, DC_ORDINARY);
       else 
@@ -262,10 +266,10 @@ parse_var_set_vars (const struct var_set *vs,
           enum dict_class class;
           size_t first_idx;
 
-          if (!parse_var_idx_class (vs, &first_idx, &class))
+          if (!parse_var_idx_class (lexer, vs, &first_idx, &class))
             goto fail;
 
-          if (!lex_match (T_TO))
+          if (!lex_match (lexer, T_TO))
             add_variable (v, nv, &mv, included, pv_opts, vs, first_idx);
           else 
             {
@@ -273,7 +277,7 @@ parse_var_set_vars (const struct var_set *vs,
               enum dict_class last_class;
               struct variable *first_var, *last_var;
 
-              if (!parse_var_idx_class (vs, &last_idx, &last_class))
+              if (!parse_var_idx_class (lexer, vs, &last_idx, &last_class))
                 goto fail;
 
               first_var = var_set_get_var (vs, first_idx);
@@ -307,10 +311,10 @@ parse_var_set_vars (const struct var_set *vs,
 
       if (pv_opts & PV_SINGLE)
         break;
-      lex_match (',');
+      lex_match (lexer, ',');
     }
-  while (token == T_ALL
-         || (token == T_ID && var_set_lookup_var (vs, tokid) != NULL));
+  while (lex_token (lexer) == T_ALL
+         || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokid (lexer)) != NULL));
   
   if (*nv == 0)
     goto fail;
@@ -372,7 +376,7 @@ extract_num (char *s, char *r, int *n, int *d)
 /* Parses a list of variable names according to the DATA LIST version
    of the TO convention.  */
 bool
-parse_DATA_LIST_vars (char ***names, size_t *nnames, int pv_opts)
+parse_DATA_LIST_vars (struct lexer *lexer, char ***names, size_t *nnames, int pv_opts)
 {
   int n1, n2;
   int d1, d2;
@@ -398,29 +402,29 @@ parse_DATA_LIST_vars (char ***names, size_t *nnames, int pv_opts)
 
   do
     {
-      if (token != T_ID)
+      if (lex_token (lexer) != T_ID)
 	{
-	  lex_error ("expecting variable name");
+	  lex_error (lexer, "expecting variable name");
 	  goto fail;
 	}
-      if (dict_class_from_id (tokid) == DC_SCRATCH
+      if (dict_class_from_id (lex_tokid (lexer)) == DC_SCRATCH
           && (pv_opts & PV_NO_SCRATCH))
 	{
 	  msg (SE, _("Scratch variables not allowed here."));
 	  goto fail;
 	}
-      strcpy (name1, tokid);
-      lex_get ();
-      if (token == T_TO)
+      strcpy (name1, lex_tokid (lexer));
+      lex_get (lexer);
+      if (lex_token (lexer) == T_TO)
 	{
-	  lex_get ();
-	  if (token != T_ID)
+	  lex_get (lexer);
+	  if (lex_token (lexer) != T_ID)
 	    {
-	      lex_error ("expecting variable name");
+	      lex_error (lexer, "expecting variable name");
 	      goto fail;
 	    }
-	  strcpy (name2, tokid);
-	  lex_get ();
+	  strcpy (name2, lex_tokid (lexer));
+	  lex_get (lexer);
 
 	  if (!extract_num (name1, root1, &n1, &d1)
 	      || !extract_num (name2, root2, &n2, &d2))
@@ -463,12 +467,12 @@ parse_DATA_LIST_vars (char ***names, size_t *nnames, int pv_opts)
 	  (*names)[nvar++] = xstrdup (name1);
 	}
 
-      lex_match (',');
+      lex_match (lexer, ',');
 
       if (pv_opts & PV_SINGLE)
 	break;
     }
-  while (token == T_ID);
+  while (lex_token (lexer) == T_ID);
   success = 1;
 
 fail:
@@ -502,7 +506,7 @@ register_vars_pool (struct pool *pool, char **names, size_t nnames)
    parse_DATA_LIST_vars(), except that all allocations are taken
    from the given POOL. */
 bool
-parse_DATA_LIST_vars_pool (struct pool *pool,
+parse_DATA_LIST_vars_pool (struct lexer *lexer, struct pool *pool,
                            char ***names, size_t *nnames, int pv_opts)
 {
   int retval;
@@ -513,7 +517,7 @@ parse_DATA_LIST_vars_pool (struct pool *pool,
      re-free it later. */
   assert (!(pv_opts & PV_APPEND));
   
-  retval = parse_DATA_LIST_vars (names, nnames, pv_opts);
+  retval = parse_DATA_LIST_vars (lexer, names, nnames, pv_opts);
   if (retval)
     register_vars_pool (pool, *names, *nnames);
   return retval;
@@ -523,7 +527,7 @@ parse_DATA_LIST_vars_pool (struct pool *pool,
    existing and the rest are to be created.  Same args as
    parse_DATA_LIST_vars(). */
 bool
-parse_mixed_vars (const struct dictionary *dict, 
+parse_mixed_vars (struct lexer *lexer, const struct dictionary *dict, 
 		  char ***names, size_t *nnames, int pv_opts)
 {
   size_t i;
@@ -537,14 +541,14 @@ parse_mixed_vars (const struct dictionary *dict,
       *names = NULL;
       *nnames = 0;
     }
-  while (token == T_ID || token == T_ALL)
+  while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
     {
-      if (token == T_ALL || dict_lookup_var (dict, tokid) != NULL)
+      if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
 	{
 	  struct variable **v;
 	  size_t nv;
 
-	  if (!parse_variables (dict, &v, &nv, PV_NONE))
+	  if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
 	    goto fail;
 	  *names = xnrealloc (*names, *nnames + nv, sizeof **names);
 	  for (i = 0; i < nv; i++)
@@ -552,7 +556,7 @@ parse_mixed_vars (const struct dictionary *dict,
 	  free (v);
 	  *nnames += nv;
 	}
-      else if (!parse_DATA_LIST_vars (names, nnames, PV_APPEND))
+      else if (!parse_DATA_LIST_vars (lexer, names, nnames, PV_APPEND))
 	goto fail;
     }
   return 1;
@@ -571,7 +575,7 @@ fail:
    parse_mixed_vars(), except that all allocations are taken
    from the given POOL. */
 bool
-parse_mixed_vars_pool (const struct dictionary *dict, struct pool *pool,
+parse_mixed_vars_pool (struct lexer *lexer, const struct dictionary *dict, struct pool *pool,
                        char ***names, size_t *nnames, int pv_opts)
 {
   int retval;
@@ -582,7 +586,7 @@ parse_mixed_vars_pool (const struct dictionary *dict, struct pool *pool,
      re-free it later. */
   assert (!(pv_opts & PV_APPEND));
 
-  retval = parse_mixed_vars (dict, names, nnames, pv_opts);
+  retval = parse_mixed_vars (lexer, dict, names, nnames, pv_opts);
   if (retval)
     register_vars_pool (pool, *names, *nnames);
   return retval;
diff --git a/src/language/lexer/variable-parser.h b/src/language/lexer/variable-parser.h
index 9760d857..0a204844 100644
--- a/src/language/lexer/variable-parser.h
+++ b/src/language/lexer/variable-parser.h
@@ -27,6 +27,7 @@ struct pool;
 struct dictionary;
 struct var_set;
 struct variable;
+struct lexer ;
 
 struct var_set *var_set_create_from_dict (const struct dictionary *d);
 struct var_set *var_set_create_from_array (struct variable *const *var,
@@ -55,19 +56,19 @@ enum
     PV_NO_SCRATCH = 00200 	/* Disallow scratch variables. */
   };
 
-struct variable *parse_variable (const struct dictionary *);
-bool parse_variables (const struct dictionary *, struct variable ***, size_t *,
+struct variable *parse_variable (struct lexer *, const struct dictionary *);
+bool parse_variables (struct lexer *, const struct dictionary *, struct variable ***, size_t *,
                      int opts);
-bool parse_variables_pool (struct pool *, const struct dictionary *,
+bool parse_variables_pool (struct lexer *, struct pool *, const struct dictionary *,
                           struct variable ***, size_t *, int opts);
-bool parse_var_set_vars (const struct var_set *, struct variable ***, size_t *,
+bool parse_var_set_vars (struct lexer *, const struct var_set *, struct variable ***, size_t *,
                         int opts);
-bool parse_DATA_LIST_vars (char ***names, size_t *cnt, int opts);
-bool parse_DATA_LIST_vars_pool (struct pool *,
+bool parse_DATA_LIST_vars (struct lexer *, char ***names, size_t *cnt, int opts);
+bool parse_DATA_LIST_vars_pool (struct lexer *, struct pool *,
                                char ***names, size_t *cnt, int opts);
-bool parse_mixed_vars (const struct dictionary *dict, 
+bool parse_mixed_vars (struct lexer *, const struct dictionary *dict, 
 		       char ***names, size_t *cnt, int opts);
-bool parse_mixed_vars_pool (const struct dictionary *dict, 
+bool parse_mixed_vars_pool (struct lexer *, const struct dictionary *dict, 
 			    struct pool *,
                            char ***names, size_t *cnt, int opts);
 
diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c
index 006936d7..031a84f8 100644
--- a/src/language/stats/aggregate.c
+++ b/src/language/stats/aggregate.c
@@ -155,7 +155,7 @@ static void initialize_aggregate_info (struct agr_proc *,
                                        const struct ccase *);
 
 /* Prototypes. */
-static bool parse_aggregate_functions (const struct dictionary *,
+static bool parse_aggregate_functions (struct lexer *, const struct dictionary *,
 				       struct agr_proc *);
 static void agr_destroy (struct agr_proc *);
 static bool aggregate_single_case (struct agr_proc *agr,
@@ -173,7 +173,7 @@ static bool presorted_agr_to_sysfile (const struct ccase *, void *aux, const str
 
 /* Parses and executes the AGGREGATE procedure. */
 int
-cmd_aggregate (struct dataset *ds)
+cmd_aggregate (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   struct agr_proc agr;
@@ -193,12 +193,12 @@ cmd_aggregate (struct dataset *ds)
   dict_set_documents (agr.dict, dict_get_documents (dict));
 
   /* OUTFILE subcommand must be first. */
-  if (!lex_force_match_id ("OUTFILE"))
+  if (!lex_force_match_id (lexer, "OUTFILE"))
     goto error;
-  lex_match ('=');
-  if (!lex_match ('*'))
+  lex_match (lexer, '=');
+  if (!lex_match (lexer, '*'))
     {
-      out_file = fh_parse (FH_REF_FILE | FH_REF_SCRATCH);
+      out_file = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH);
       if (out_file == NULL)
         goto error;
     }
@@ -206,28 +206,28 @@ cmd_aggregate (struct dataset *ds)
   /* Read most of the subcommands. */
   for (;;)
     {
-      lex_match ('/');
+      lex_match (lexer, '/');
       
-      if (lex_match_id ("MISSING"))
+      if (lex_match_id (lexer, "MISSING"))
 	{
-	  lex_match ('=');
-	  if (!lex_match_id ("COLUMNWISE"))
+	  lex_match (lexer, '=');
+	  if (!lex_match_id (lexer, "COLUMNWISE"))
 	    {
-	      lex_error (_("while expecting COLUMNWISE"));
+	      lex_error (lexer, _("while expecting COLUMNWISE"));
               goto error;
 	    }
 	  agr.missing = COLUMNWISE;
 	}
-      else if (lex_match_id ("DOCUMENT"))
+      else if (lex_match_id (lexer, "DOCUMENT"))
         copy_documents = true;
-      else if (lex_match_id ("PRESORTED"))
+      else if (lex_match_id (lexer, "PRESORTED"))
         presorted = true;
-      else if (lex_match_id ("BREAK"))
+      else if (lex_match_id (lexer, "BREAK"))
 	{
           int i;
 
-	  lex_match ('=');
-          agr.sort = sort_parse_criteria (dict,
+	  lex_match (lexer, '=');
+          agr.sort = sort_parse_criteria (lexer, dict,
                                           &agr.break_vars, &agr.break_var_cnt,
                                           &saw_direction, NULL);
           if (agr.sort == NULL)
@@ -242,7 +242,7 @@ cmd_aggregate (struct dataset *ds)
 	}
       else
         {
-          lex_error (_("expecting BREAK"));
+          lex_error (lexer, _("expecting BREAK"));
           goto error;
         }
     }
@@ -252,8 +252,8 @@ cmd_aggregate (struct dataset *ds)
                "the same way as the input data."));
       
   /* Read in the aggregate functions. */
-  lex_match ('/');
-  if (!parse_aggregate_functions (dict, &agr))
+  lex_match (lexer, '/');
+  if (!parse_aggregate_functions (lexer, dict, &agr))
     goto error;
 
   /* Delete documents. */
@@ -359,7 +359,7 @@ error:
 
 /* Parse all the aggregate functions. */
 static bool
-parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
+parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, struct agr_proc *agr)
 {
   struct agr_var *tail; /* Tail of linked list starting at agr->vars. */
 
@@ -370,6 +370,7 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
       char **dest;
       char **dest_label;
       size_t n_dest;
+      struct string function_name;
 
       int include_missing;
       const struct agr_func *function;
@@ -392,11 +393,11 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
       arg[1].c = NULL;
 
       /* Parse the list of target variables. */
-      while (!lex_match ('='))
+      while (!lex_match (lexer, '='))
 	{
 	  size_t n_dest_prev = n_dest;
 	  
-	  if (!parse_DATA_LIST_vars (&dest, &n_dest,
+	  if (!parse_DATA_LIST_vars (lexer, &dest, &n_dest,
                                      PV_APPEND | PV_SINGLE | PV_NO_SCRATCH))
 	    goto error;
 
@@ -408,42 +409,52 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 	    for (j = n_dest_prev; j < n_dest; j++)
 	      dest_label[j] = NULL;
 	  }
+
+
 	  
-	  if (token == T_STRING)
+	  if (lex_token (lexer) == T_STRING)
 	    {
-	      ds_truncate (&tokstr, 255);
-	      dest_label[n_dest - 1] = ds_xstrdup (&tokstr);
-	      lex_get ();
+	      struct string label;
+	      ds_init_string (&label, lex_tokstr (lexer));
+
+	      ds_truncate (&label, 255);
+	      dest_label[n_dest - 1] = ds_xstrdup (&label);
+	      lex_get (lexer);
+	      ds_destroy (&label);
 	    }
 	}
 
       /* Get the name of the aggregation function. */
-      if (token != T_ID)
+      if (lex_token (lexer) != T_ID)
 	{
-	  lex_error (_("expecting aggregation function"));
+	  lex_error (lexer, _("expecting aggregation function"));
 	  goto error;
 	}
 
       include_missing = 0;
-      if (tokid[strlen (tokid) - 1] == '.')
-	{
+
+      ds_init_string (&function_name, lex_tokstr (lexer));
+
+      ds_chomp (&function_name, '.');
+
+      if (lex_tokid(lexer)[strlen (lex_tokid (lexer)) - 1] == '.')
 	  include_missing = 1;
-	  tokid[strlen (tokid) - 1] = 0;
-	}
-      
+
       for (function = agr_func_tab; function->name; function++)
-	if (!strcasecmp (function->name, tokid))
+	if (!strcasecmp (function->name, ds_cstr (&function_name)))
 	  break;
       if (NULL == function->name)
 	{
-	  msg (SE, _("Unknown aggregation function %s."), tokid);
+	  msg (SE, _("Unknown aggregation function %s."), 
+	       ds_cstr (&function_name));
 	  goto error;
 	}
+      ds_destroy (&function_name);
       func_index = function - agr_func_tab;
-      lex_get ();
+      lex_get (lexer);
 
       /* Check for leading lparen. */
-      if (!lex_match ('('))
+      if (!lex_match (lexer, '('))
 	{
 	  if (func_index == N)
 	    func_index = N_NO_VARS;
@@ -451,7 +462,7 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 	    func_index = NU_NO_VARS;
 	  else
 	    {
-	      lex_error (_("expecting `('"));
+	      lex_error (lexer, _("expecting `('"));
 	      goto error;
 	    }
 	}
@@ -466,7 +477,7 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 	    else if (function->n_args)
 	      pv_opts |= PV_SAME_TYPE;
 
-	    if (!parse_variables (dict, &src, &n_src, pv_opts))
+	    if (!parse_variables (lexer, dict, &src, &n_src, pv_opts))
 	      goto error;
 	  }
 
@@ -477,15 +488,15 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 	      {
 		int type;
 	    
-		lex_match (',');
-		if (token == T_STRING)
+		lex_match (lexer, ',');
+		if (lex_token (lexer) == T_STRING)
 		  {
-		    arg[i].c = ds_xstrdup (&tokstr);
+		    arg[i].c = ds_xstrdup (lex_tokstr (lexer));
 		    type = ALPHA;
 		  }
-		else if (lex_is_number ())
+		else if (lex_is_number (lexer))
 		  {
-		    arg[i].f = tokval;
+		    arg[i].f = lex_tokval (lexer);
 		    type = NUMERIC;
 		  } else {
 		    msg (SE, _("Missing argument %d to %s."), i + 1,
@@ -493,7 +504,7 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 		    goto error;
 		  }
 	    
-		lex_get ();
+		lex_get (lexer);
 
 		if (type != src[0]->type)
 		  {
@@ -505,9 +516,9 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
 	      }
 
 	  /* Trailing rparen. */
-	  if (!lex_match(')'))
+	  if (!lex_match (lexer, ')'))
 	    {
-	      lex_error (_("expecting `)'"));
+	      lex_error (lexer, _("expecting `)'"));
 	      goto error;
 	    }
 	  
@@ -649,17 +660,18 @@ parse_aggregate_functions (const struct dictionary *dict, struct agr_proc *agr)
       free (dest);
       free (dest_label);
 
-      if (!lex_match ('/'))
+      if (!lex_match (lexer, '/'))
 	{
-	  if (token == '.')
+	  if (lex_token (lexer) == '.')
 	    return true;
 
-	  lex_error ("expecting end of command");
+	  lex_error (lexer, "expecting end of command");
 	  return false;
 	}
       continue;
       
     error:
+      ds_destroy (&function_name);
       for (i = 0; i < n_dest; i++)
 	{
 	  free (dest[i]);
@@ -1106,7 +1118,8 @@ agr_to_active_file (const struct ccase *c, void *agr_, const struct dataset *ds
 /* Aggregate the current case and output it if we passed a
    breakpoint. */
 static bool
-presorted_agr_to_sysfile (const struct ccase *c, void *agr_, const struct dataset *ds UNUSED) 
+presorted_agr_to_sysfile (const struct ccase *c, void *agr_, 
+			  const struct dataset *ds UNUSED) 
 {
   struct agr_proc *agr = agr_;
 
diff --git a/src/language/stats/autorecode.c b/src/language/stats/autorecode.c
index 41ac1058..d8b72ca6 100644
--- a/src/language/stats/autorecode.c
+++ b/src/language/stats/autorecode.c
@@ -102,7 +102,7 @@ static void arc_free (struct autorecode_pgm *);
 
 /* Performs the AUTORECODE procedure. */
 int
-cmd_autorecode (struct dataset *ds)
+cmd_autorecode (struct lexer *lexer, struct dataset *ds)
 {
   struct autorecode_pgm arc;
   size_t dst_cnt;
@@ -119,15 +119,15 @@ cmd_autorecode (struct dataset *ds)
   arc.print = 0;
   dst_cnt = 0;
 
-  lex_match_id ("VARIABLES");
-  lex_match ('=');
-  if (!parse_variables (dataset_dict (ds), &arc.src_vars, &arc.var_cnt,
+  lex_match_id (lexer, "VARIABLES");
+  lex_match (lexer, '=');
+  if (!parse_variables (lexer, dataset_dict (ds), &arc.src_vars, &arc.var_cnt,
                         PV_NO_DUPLICATE))
     goto lossage;
-  if (!lex_force_match_id ("INTO"))
+  if (!lex_force_match_id (lexer, "INTO"))
     goto lossage;
-  lex_match ('=');
-  if (!parse_DATA_LIST_vars (&arc.dst_names, &dst_cnt, PV_NONE))
+  lex_match (lexer, '=');
+  if (!parse_DATA_LIST_vars (lexer, &arc.dst_names, &dst_cnt, PV_NONE))
     goto lossage;
   if (dst_cnt != arc.var_cnt)
     {
@@ -144,14 +144,14 @@ cmd_autorecode (struct dataset *ds)
 
       goto lossage;
     }
-  while (lex_match ('/'))
-    if (lex_match_id ("DESCENDING"))
+  while (lex_match (lexer, '/'))
+    if (lex_match_id (lexer, "DESCENDING"))
       arc.direction = DESCENDING;
-    else if (lex_match_id ("PRINT"))
+    else if (lex_match_id (lexer, "PRINT"))
       arc.print = 1;
-  if (token != '.')
+  if (lex_token (lexer) != '.')
     {
-      lex_error (_("expecting end of command"));
+      lex_error (lexer, _("expecting end of command"));
       goto lossage;
     }
 
diff --git a/src/language/stats/correlations.q b/src/language/stats/correlations.q
index 4564ec2c..15d21ece 100644
--- a/src/language/stats/correlations.q
+++ b/src/language/stats/correlations.q
@@ -46,12 +46,12 @@ static struct cor_set *cor_list, *cor_last;
 static struct file_handle *matrix_file;
 
 static void free_correlations_state (void);
-static int internal_cmd_correlations (struct dataset *ds);
+static int internal_cmd_correlations (struct lexer *lexer, struct dataset *ds);
 
 int
-cmd_correlations (struct dataset *ds)
+cmd_correlations (struct lexer *lexer, struct dataset *ds)
 {
-  int result = internal_cmd_correlations (ds);
+  int result = internal_cmd_correlations (lexer, ds);
   free_correlations_state ();
   return result;
 }
@@ -71,14 +71,14 @@ cmd_correlations (struct dataset *ds)
 /* (functions) */
 
 int
-internal_cmd_correlations (struct dataset *ds)
+internal_cmd_correlations (struct lexer *lexer, struct dataset *ds)
 {
   struct cmd_correlations cmd;
 
   cor_list = cor_last = NULL;
   matrix_file = NULL;
 
-  if (!parse_correlations (ds, &cmd, NULL))
+  if (!parse_correlations (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
 
   free_correlations (&cmd);
@@ -87,26 +87,26 @@ internal_cmd_correlations (struct dataset *ds)
 }
 
 static int
-cor_custom_variables (struct dataset *ds, struct cmd_correlations *cmd UNUSED, void *aux UNUSED)
+cor_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_correlations *cmd UNUSED, void *aux UNUSED)
 {
   struct variable **v1, **v2;
   size_t nv1, nv2;
   struct cor_set *cor;
 
   /* Ensure that this is a VARIABLES subcommand. */
-  if (!lex_match_id ("VARIABLES")
-      && (token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) != NULL)
-      && token != T_ALL)
+  if (!lex_match_id (lexer, "VARIABLES")
+      && (lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+      && lex_token (lexer) != T_ALL)
     return 2;
-  lex_match ('=');
+  lex_match (lexer, '=');
 
-  if (!parse_variables (dataset_dict (ds), &v1, &nv1,
+  if (!parse_variables (lexer, dataset_dict (ds), &v1, &nv1,
 			PV_NO_DUPLICATE | PV_NUMERIC))
     return 0;
   
-  if (lex_match (T_WITH))
+  if (lex_match (lexer, T_WITH))
     {
-      if (!parse_variables (dataset_dict (ds), &v2, &nv2,
+      if (!parse_variables (lexer, dataset_dict (ds), &v2, &nv2,
 			    PV_NO_DUPLICATE | PV_NUMERIC))
 	{
 	  free (v1);
@@ -134,21 +134,21 @@ cor_custom_variables (struct dataset *ds, struct cmd_correlations *cmd UNUSED, v
 }
 
 static int
-cor_custom_matrix (struct dataset *ds UNUSED, struct cmd_correlations *cmd UNUSED, void *aux UNUSED)
+cor_custom_matrix (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_correlations *cmd UNUSED, void *aux UNUSED)
 {
-  if (!lex_force_match ('('))
+  if (!lex_force_match (lexer, '('))
     return 0;
   
-  if (lex_match ('*'))
+  if (lex_match (lexer, '*'))
     matrix_file = NULL;
   else 
     {
-      matrix_file = fh_parse (FH_REF_FILE);
+      matrix_file = fh_parse (lexer, FH_REF_FILE);
       if (matrix_file == NULL)
         return 0; 
     }
 
-  if (!lex_force_match (')'))
+  if (!lex_force_match (lexer, ')'))
     return 0;
 
   return 1;
diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q
index 67ee5aab..87386d6c 100644
--- a/src/language/stats/crosstabs.q
+++ b/src/language/stats/crosstabs.q
@@ -176,7 +176,7 @@ static struct cmd_crosstabs cmd;
 static struct pool *pl_tc;	/* For table cells. */
 static struct pool *pl_col;	/* For column data. */
 
-static int internal_cmd_crosstabs (struct dataset *ds);
+static int internal_cmd_crosstabs (struct lexer *lexer, struct dataset *ds);
 static void precalc (const struct ccase *, void *, const struct dataset *);
 static bool calc_general (const struct ccase *, void *, const struct dataset *);
 static bool calc_integer (const struct ccase *, void *, const struct dataset *);
@@ -188,9 +188,9 @@ static void format_short (char *s, const struct fmt_spec *fp,
 
 /* Parse and execute CROSSTABS, then clean up. */
 int
-cmd_crosstabs (struct dataset *ds)
+cmd_crosstabs (struct lexer *lexer, struct dataset *ds)
 {
-  int result = internal_cmd_crosstabs (ds);
+  int result = internal_cmd_crosstabs (lexer, ds);
 
   free (variables);
   pool_destroy (pl_tc);
@@ -201,7 +201,7 @@ cmd_crosstabs (struct dataset *ds)
 
 /* Parses and executes the CROSSTABS procedure. */
 static int
-internal_cmd_crosstabs (struct dataset *ds)
+internal_cmd_crosstabs (struct lexer *lexer, struct dataset *ds)
 {
   int i;
   bool ok;
@@ -213,7 +213,7 @@ internal_cmd_crosstabs (struct dataset *ds)
   pl_tc = pool_create ();
   pl_col = pool_create ();
 
-  if (!parse_crosstabs (ds, &cmd, NULL))
+  if (!parse_crosstabs (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
 
   mode = variables ? INTEGER : GENERAL;
@@ -303,7 +303,7 @@ internal_cmd_crosstabs (struct dataset *ds)
 
 /* Parses the TABLES subcommand. */
 static int
-crs_custom_tables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *aux UNUSED)
+crs_custom_tables (struct lexer *lexer, struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *aux UNUSED)
 {
   struct var_set *var_set;
   int n_by;
@@ -313,11 +313,11 @@ crs_custom_tables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *a
   int success = 0;
 
   /* Ensure that this is a TABLES subcommand. */
-  if (!lex_match_id ("TABLES")
-      && (token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) == NULL)
-      && token != T_ALL)
+  if (!lex_match_id (lexer, "TABLES")
+      && (lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     return 2;
-  lex_match ('=');
+  lex_match (lexer, '=');
 
   if (variables != NULL)
     var_set = var_set_create_from_array (variables, variables_cnt);
@@ -329,7 +329,7 @@ crs_custom_tables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *a
     {
       by = xnrealloc (by, n_by + 1, sizeof *by);
       by_nvar = xnrealloc (by_nvar, n_by + 1, sizeof *by_nvar);
-      if (!parse_var_set_vars (var_set, &by[n_by], &by_nvar[n_by],
+      if (!parse_var_set_vars (lexer, var_set, &by[n_by], &by_nvar[n_by],
                                PV_NO_DUPLICATE | PV_NO_SCRATCH))
 	goto done;
       if (xalloc_oversized (nx, by_nvar[n_by])) 
@@ -340,11 +340,11 @@ crs_custom_tables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *a
       nx *= by_nvar[n_by];
       n_by++;
 
-      if (!lex_match (T_BY))
+      if (!lex_match (lexer, T_BY))
 	{
 	  if (n_by < 2)
 	    {
-	      lex_error (_("expecting BY"));
+	      lex_error (lexer, _("expecting BY"));
 	      goto done;
 	    }
 	  else 
@@ -407,7 +407,7 @@ crs_custom_tables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *a
 
 /* Parses the VARIABLES subcommand. */
 static int
-crs_custom_variables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *aux UNUSED)
+crs_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void *aux UNUSED)
 {
   if (nxtab)
     {
@@ -415,7 +415,7 @@ crs_custom_variables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void
       return 0;
     }
 
-  lex_match ('=');
+  lex_match (lexer, '=');
   
   for (;;)
     {
@@ -424,42 +424,43 @@ crs_custom_variables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void
 
       long min, max;
       
-      if (!parse_variables (dataset_dict (ds), &variables, &variables_cnt,
+      if (!parse_variables (lexer, dataset_dict (ds), 
+			    &variables, &variables_cnt,
 			    (PV_APPEND | PV_NUMERIC
 			     | PV_NO_DUPLICATE | PV_NO_SCRATCH)))
 	return 0;
 
-      if (token != '(')
+      if (lex_token (lexer) != '(')
 	{
-	  lex_error ("expecting `('");
+	  lex_error (lexer, "expecting `('");
 	  goto lossage;
 	}
-      lex_get ();
+      lex_get (lexer);
 
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	goto lossage;
-      min = lex_integer ();
-      lex_get ();
+      min = lex_integer (lexer);
+      lex_get (lexer);
 
-      lex_match (',');
+      lex_match (lexer, ',');
 
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	goto lossage;
-      max = lex_integer ();
+      max = lex_integer (lexer);
       if (max < min)
 	{
 	  msg (SE, _("Maximum value (%ld) less than minimum value (%ld)."),
 	       max, min);
 	  goto lossage;
 	}
-      lex_get ();
+      lex_get (lexer);
 
-      if (token != ')')
+      if (lex_token (lexer) != ')')
 	{
-	  lex_error ("expecting `)'");
+	  lex_error (lexer, "expecting `)'");
 	  goto lossage;
 	}
-      lex_get ();
+      lex_get (lexer);
       
       for (i = orig_nv; i < variables_cnt; i++) 
         {
@@ -470,7 +471,7 @@ crs_custom_variables (struct dataset *ds, struct cmd_crosstabs *cmd UNUSED, void
           var_attach_aux (variables[i], vr, var_dtor_free);
 	}
       
-      if (token == '/')
+      if (lex_token (lexer) == '/')
 	break;
     }
   
diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c
index de0e417c..c1653dc7 100644
--- a/src/language/stats/descriptives.c
+++ b/src/language/stats/descriptives.c
@@ -169,12 +169,12 @@ struct dsc_proc
   };
 
 /* Parsing. */
-static enum dsc_statistic match_statistic (void);
+static enum dsc_statistic match_statistic (struct lexer *);
 static void free_dsc_proc (struct dsc_proc *);
 
 /* Z-score functions. */
 static bool try_name (const struct dictionary *dict, 
-		      struct dsc_proc *dsc, char *name);
+		      struct dsc_proc *dsc, const char *name);
 static bool generate_z_varname (const struct dictionary *dict, 
 				struct dsc_proc *dsc, char *z_name,
 				const char *name, size_t *z_cnt);
@@ -191,7 +191,7 @@ static void display (struct dsc_proc *dsc);
 
 /* Handles DESCRIPTIVES. */
 int
-cmd_descriptives (struct dataset *ds)
+cmd_descriptives (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   struct dsc_proc *dsc;
@@ -219,106 +219,106 @@ cmd_descriptives (struct dataset *ds)
   dsc->show_stats = dsc->calc_stats = DEFAULT_STATS;
 
   /* Parse DESCRIPTIVES. */
-  while (token != '.') 
+  while (lex_token (lexer) != '.') 
     {
-      if (lex_match_id ("MISSING"))
+      if (lex_match_id (lexer, "MISSING"))
         {
-          lex_match ('=');
-          while (token != '.' && token != '/') 
+          lex_match (lexer, '=');
+          while (lex_token (lexer) != '.' && lex_token (lexer) != '/') 
             {
-              if (lex_match_id ("VARIABLE"))
+              if (lex_match_id (lexer, "VARIABLE"))
                 dsc->missing_type = DSC_VARIABLE;
-              else if (lex_match_id ("LISTWISE"))
+              else if (lex_match_id (lexer, "LISTWISE"))
                 dsc->missing_type = DSC_LISTWISE;
-              else if (lex_match_id ("INCLUDE"))
+              else if (lex_match_id (lexer, "INCLUDE"))
                 dsc->include_user_missing = 1;
               else
                 {
-                  lex_error (NULL);
+                  lex_error (lexer, NULL);
                   goto error;
                 }
-              lex_match (',');
+              lex_match (lexer, ',');
             }
         }
-      else if (lex_match_id ("SAVE"))
+      else if (lex_match_id (lexer, "SAVE"))
         save_z_scores = 1;
-      else if (lex_match_id ("FORMAT")) 
+      else if (lex_match_id (lexer, "FORMAT")) 
         {
-          lex_match ('=');
-          while (token != '.' && token != '/') 
+          lex_match (lexer, '=');
+          while (lex_token (lexer) != '.' && lex_token (lexer) != '/') 
             {
-              if (lex_match_id ("LABELS"))
+              if (lex_match_id (lexer, "LABELS"))
                 dsc->show_var_labels = 1;
-              else if (lex_match_id ("NOLABELS"))
+              else if (lex_match_id (lexer, "NOLABELS"))
                 dsc->show_var_labels = 0;
-              else if (lex_match_id ("INDEX"))
+              else if (lex_match_id (lexer, "INDEX"))
                 dsc->show_index = 1;
-              else if (lex_match_id ("NOINDEX"))
+              else if (lex_match_id (lexer, "NOINDEX"))
                 dsc->show_index = 0;
-              else if (lex_match_id ("LINE"))
+              else if (lex_match_id (lexer, "LINE"))
                 dsc->format = DSC_LINE;
-              else if (lex_match_id ("SERIAL"))
+              else if (lex_match_id (lexer, "SERIAL"))
                 dsc->format = DSC_SERIAL;
               else
                 {
-                  lex_error (NULL);
+                  lex_error (lexer, NULL);
                   goto error;
                 }
-              lex_match (',');
+              lex_match (lexer, ',');
             }
         }
-      else if (lex_match_id ("STATISTICS")) 
+      else if (lex_match_id (lexer, "STATISTICS")) 
         {
-          lex_match ('=');
+          lex_match (lexer, '=');
           dsc->show_stats = 0;
-          while (token != '.' && token != '/') 
+          while (lex_token (lexer) != '.' && lex_token (lexer) != '/') 
             {
-              if (lex_match (T_ALL)) 
+              if (lex_match (lexer, T_ALL)) 
                 dsc->show_stats |= (1ul << DSC_N_STATS) - 1;
-              else if (lex_match_id ("DEFAULT"))
+              else if (lex_match_id (lexer, "DEFAULT"))
                 dsc->show_stats |= DEFAULT_STATS;
               else
-		dsc->show_stats |= 1ul << (match_statistic ());
-              lex_match (',');
+		dsc->show_stats |= 1ul << (match_statistic (lexer));
+              lex_match (lexer, ',');
             }
           if (dsc->show_stats == 0)
             dsc->show_stats = DEFAULT_STATS;
         }
-      else if (lex_match_id ("SORT")) 
+      else if (lex_match_id (lexer, "SORT")) 
         {
-          lex_match ('=');
-          if (lex_match_id ("NAME"))
+          lex_match (lexer, '=');
+          if (lex_match_id (lexer, "NAME"))
             dsc->sort_by_stat = DSC_NAME;
           else 
 	    {
-	      dsc->sort_by_stat = match_statistic ();
+	      dsc->sort_by_stat = match_statistic (lexer);
 	      if (dsc->sort_by_stat == DSC_NONE )
 		dsc->sort_by_stat = DSC_MEAN;
 	    }
-          if (lex_match ('(')) 
+          if (lex_match (lexer, '(')) 
             {
-              if (lex_match_id ("A"))
+              if (lex_match_id (lexer, "A"))
                 dsc->sort_ascending = 1;
-              else if (lex_match_id ("D"))
+              else if (lex_match_id (lexer, "D"))
                 dsc->sort_ascending = 0;
               else
-                lex_error (NULL);
-              lex_force_match (')');
+                lex_error (lexer, NULL);
+              lex_force_match (lexer, ')');
             }
         }
       else if (var_cnt == 0)
         {
-          if (lex_look_ahead () == '=') 
+          if (lex_look_ahead (lexer) == '=') 
             {
-              lex_match_id ("VARIABLES");
-              lex_match ('=');
+              lex_match_id (lexer, "VARIABLES");
+              lex_match (lexer, '=');
             }
 
-          while (token != '.' && token != '/') 
+          while (lex_token (lexer) != '.' && lex_token (lexer) != '/') 
             {
               int i;
               
-              if (!parse_variables (dataset_dict (ds), &vars, &var_cnt,
+              if (!parse_variables (lexer, dataset_dict (ds), &vars, &var_cnt,
                                     PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC))
 		goto error;
 
@@ -332,34 +332,34 @@ cmd_descriptives (struct dataset *ds)
                 }
               dsc->var_cnt = var_cnt;
 
-              if (lex_match ('(')) 
+              if (lex_match (lexer, '(')) 
                 {
-                  if (token != T_ID) 
+                  if (lex_token (lexer) != T_ID) 
                     {
-                      lex_error (NULL);
+                      lex_error (lexer, NULL);
                       goto error;
                     }
-                  if (try_name (dict, dsc, tokid)) 
+                  if (try_name (dict, dsc, lex_tokid (lexer))) 
                     {
-                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, tokid);
+                      strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer));
                       z_cnt++;
                     }
                   else
                     msg (SE, _("Z-score variable name %s would be"
-                               " a duplicate variable name."), tokid);
-                  lex_get ();
-                  if (!lex_force_match (')'))
+                               " a duplicate variable name."), lex_tokid (lexer));
+                  lex_get (lexer);
+                  if (!lex_force_match (lexer, ')'))
 		    goto error;
                 }
             }
         }
       else 
         {
-          lex_error (NULL);
+          lex_error (lexer, NULL);
           goto error; 
         }
 
-      lex_match ('/');
+      lex_match (lexer, '/');
     }
   if (var_cnt == 0)
     {
@@ -436,18 +436,18 @@ cmd_descriptives (struct dataset *ds)
    specifiers). Emits an error if the current token ID does not name a
    statistic. */
 static enum dsc_statistic
-match_statistic (void) 
+match_statistic (struct lexer *lexer) 
 {
-  if (token == T_ID) 
+  if (lex_token (lexer) == T_ID) 
     {
       enum dsc_statistic stat;
 
       for (stat = 0; stat < DSC_N_STATS; stat++)
-        if (lex_match_id (dsc_info[stat].identifier)) 
+        if (lex_match_id (lexer, dsc_info[stat].identifier)) 
 	  return stat;
 
-      lex_get();
-      lex_error (_("expecting statistic name: reverting to default"));
+      lex_get (lexer);
+      lex_error (lexer, _("expecting statistic name: reverting to default"));
     }
 
   return DSC_NONE;
@@ -473,7 +473,8 @@ free_dsc_proc (struct dsc_proc *dsc)
 /* Returns false if NAME is a duplicate of any existing variable name or
    of any previously-declared z-var name; otherwise returns true. */
 static bool
-try_name (const struct dictionary *dict, struct dsc_proc *dsc, char *name)
+try_name (const struct dictionary *dict, struct dsc_proc *dsc, 
+	  const char *name)
 {
   size_t i;
 
diff --git a/src/language/stats/examine.q b/src/language/stats/examine.q
index c88fc90b..3aa9e4f0 100644
--- a/src/language/stats/examine.q
+++ b/src/language/stats/examine.q
@@ -110,7 +110,7 @@ static struct factor *factors=0;
 static struct metrics *totals=0;
 
 /* Parse the clause specifying the factors */
-static int examine_parse_independent_vars (const struct dictionary *dict, struct cmd_examine *cmd);
+static int examine_parse_independent_vars (struct lexer *lexer, const struct dictionary *dict, struct cmd_examine *cmd);
 
 
 
@@ -191,14 +191,14 @@ static short sbc_percentile;
 
 
 int
-cmd_examine (struct dataset *ds)
+cmd_examine (struct lexer *lexer, struct dataset *ds)
 {
   bool ok;
 
   subc_list_double_create(&percentile_list);
   percentile_algorithm = PC_HAVERAGE;
 
-  if ( !parse_examine (ds, &cmd, NULL) )
+  if ( !parse_examine (lexer, ds, &cmd, NULL) )
     return CMD_FAILURE;
 
   /* If /MISSING=INCLUDE is set, then user missing values are ignored */
@@ -429,43 +429,43 @@ list_to_ptile_hash(const subc_list_double *l)
 
 /* Parse the PERCENTILES subcommand */
 static int
-xmn_custom_percentiles(struct dataset *ds UNUSED, 
+xmn_custom_percentiles(struct lexer *lexer, struct dataset *ds UNUSED, 
 		       struct cmd_examine *p UNUSED, void *aux UNUSED)
 {
   sbc_percentile = 1;
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  lex_match('(');
+  lex_match (lexer, '(');
 
-  while ( lex_is_number() ) 
+  while ( lex_is_number (lexer) ) 
     {
-      subc_list_double_push (&percentile_list, lex_number());
+      subc_list_double_push (&percentile_list, lex_number (lexer));
 
-      lex_get();
+      lex_get (lexer);
 
-      lex_match(',') ;
+      lex_match (lexer, ',') ;
     }
-  lex_match(')');
+  lex_match (lexer, ')');
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  if ( lex_match_id("HAVERAGE"))
+  if ( lex_match_id (lexer, "HAVERAGE"))
     percentile_algorithm = PC_HAVERAGE; 
 
-  else if ( lex_match_id("WAVERAGE"))
+  else if ( lex_match_id (lexer, "WAVERAGE"))
     percentile_algorithm = PC_WAVERAGE; 
 
-  else if ( lex_match_id("ROUND"))
+  else if ( lex_match_id (lexer, "ROUND"))
     percentile_algorithm = PC_ROUND;
 
-  else if ( lex_match_id("EMPIRICAL"))
+  else if ( lex_match_id (lexer, "EMPIRICAL"))
     percentile_algorithm = PC_EMPIRICAL;
 
-  else if ( lex_match_id("AEMPIRICAL"))
+  else if ( lex_match_id (lexer, "AEMPIRICAL"))
     percentile_algorithm = PC_AEMPIRICAL; 
 
-  else if ( lex_match_id("NONE"))
+  else if ( lex_match_id (lexer, "NONE"))
     percentile_algorithm = PC_NONE; 
 
 
@@ -485,7 +485,7 @@ xmn_custom_percentiles(struct dataset *ds UNUSED,
 
 /* TOTAL and NOTOTAL are simple, mutually exclusive flags */
 static int
-xmn_custom_total (struct dataset *ds UNUSED, struct cmd_examine *p, void *aux UNUSED)
+xmn_custom_total (struct lexer *lexer UNUSED, struct dataset *ds UNUSED, struct cmd_examine *p, void *aux UNUSED)
 {
   if ( p->sbc_nototal ) 
     {
@@ -497,7 +497,7 @@ xmn_custom_total (struct dataset *ds UNUSED, struct cmd_examine *p, void *aux UN
 }
 
 static int
-xmn_custom_nototal (struct dataset *ds UNUSED, 
+xmn_custom_nototal (struct lexer *lexer UNUSED, struct dataset *ds UNUSED, 
 		    struct cmd_examine *p, void *aux UNUSED)
 {
   if ( p->sbc_total ) 
@@ -514,18 +514,18 @@ xmn_custom_nototal (struct dataset *ds UNUSED,
 /* Parser for the variables sub command  
    Returns 1 on success */
 static int
-xmn_custom_variables(struct dataset *ds, struct cmd_examine *cmd, void *aux UNUSED)
+xmn_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_examine *cmd, void *aux UNUSED)
 {
   const struct dictionary *dict = dataset_dict (ds);
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  if ((token != T_ID || dict_lookup_var (dict, tokid) == NULL)
-      && token != T_ALL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     {
       return 2;
     }
   
-  if (!parse_variables (dict, &dependent_vars, &n_dependent_vars,
+  if (!parse_variables (lexer, dict, &dependent_vars, &n_dependent_vars,
 			PV_NO_DUPLICATE | PV_NUMERIC | PV_NO_SCRATCH) )
     {
       free (dependent_vars);
@@ -536,10 +536,10 @@ xmn_custom_variables(struct dataset *ds, struct cmd_examine *cmd, void *aux UNUS
 
   totals = xnmalloc (n_dependent_vars, sizeof *totals);
 
-  if ( lex_match(T_BY))
+  if ( lex_match (lexer, T_BY))
     {
       int success ; 
-      success =  examine_parse_independent_vars (dict, cmd);
+      success =  examine_parse_independent_vars (lexer, dict, cmd);
       if ( success != 1 ) {
         free (dependent_vars);
       	free (totals) ; 
@@ -554,35 +554,35 @@ xmn_custom_variables(struct dataset *ds, struct cmd_examine *cmd, void *aux UNUS
 
 /* Parse the clause specifying the factors */
 static int
-examine_parse_independent_vars (const struct dictionary *dict, struct cmd_examine *cmd)
+examine_parse_independent_vars (struct lexer *lexer, const struct dictionary *dict, struct cmd_examine *cmd)
 {
   int success;
   struct factor *sf = xmalloc (sizeof *sf);
 
-  if ((token != T_ID || dict_lookup_var (dict, tokid) == NULL)
-      && token != T_ALL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     {
       free ( sf ) ;
       return 2;
     }
 
 
-  sf->indep_var[0] = parse_variable (dict);
+  sf->indep_var[0] = parse_variable (lexer, dict);
   sf->indep_var[1] = 0;
 
-  if ( token == T_BY ) 
+  if ( lex_token (lexer) == T_BY ) 
     {
 
-      lex_match(T_BY);
+      lex_match (lexer, T_BY);
 
-      if ((token != T_ID || dict_lookup_var (dict, tokid) == NULL)
-	  && token != T_ALL)
+      if ((lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+	  && lex_token (lexer) != T_ALL)
 	{
 	  free ( sf ) ;
 	  return 2;
 	}
 
-      sf->indep_var[1] = parse_variable (dict);
+      sf->indep_var[1] = parse_variable (lexer, dict);
 
     }
 
@@ -596,12 +596,12 @@ examine_parse_independent_vars (const struct dictionary *dict, struct cmd_examin
   sf->next = factors;
   factors = sf;
   
-  lex_match(',');
+  lex_match (lexer, ',');
 
-  if ( token == '.' || token == '/' ) 
+  if ( lex_token (lexer) == '.' || lex_token (lexer) == '/' ) 
     return 1;
 
-  success =  examine_parse_independent_vars (dict, cmd);
+  success =  examine_parse_independent_vars (lexer, dict, cmd);
   
   if ( success != 1 ) 
     free ( sf ) ; 
diff --git a/src/language/stats/flip.c b/src/language/stats/flip.c
index 8a6128bc..465ea79c 100644
--- a/src/language/stats/flip.c
+++ b/src/language/stats/flip.c
@@ -90,7 +90,7 @@ static const struct case_sink_class flip_sink_class;
 
 /* Parses and executes FLIP. */
 int
-cmd_flip (struct dataset *ds)
+cmd_flip (struct lexer *lexer, struct dataset *ds)
 {
   struct flip_pgm *flip;
   struct case_sink *sink;
@@ -112,24 +112,24 @@ cmd_flip (struct dataset *ds)
   flip->new_names_tail = NULL;
   flip->file = NULL;
 
-  lex_match ('/');
-  if (lex_match_id ("VARIABLES"))
+  lex_match (lexer, '/');
+  if (lex_match_id (lexer, "VARIABLES"))
     {
-      lex_match ('=');
-      if (!parse_variables (dict, &flip->var, &flip->var_cnt,
+      lex_match (lexer, '=');
+      if (!parse_variables (lexer, dict, &flip->var, &flip->var_cnt,
                             PV_NO_DUPLICATE))
 	goto error;
-      lex_match ('/');
+      lex_match (lexer, '/');
     }
   else
     dict_get_vars (dict, &flip->var, &flip->var_cnt, 1u << DC_SYSTEM);
   pool_register (flip->pool, free, flip->var);
 
-  lex_match ('/');
-  if (lex_match_id ("NEWNAMES"))
+  lex_match (lexer, '/');
+  if (lex_match_id (lexer, "NEWNAMES"))
     {
-      lex_match ('=');
-      flip->new_names = parse_variable (dict);
+      lex_match (lexer, '=');
+      flip->new_names = parse_variable (lexer, dict);
       if (!flip->new_names)
         goto error;
     }
@@ -178,7 +178,7 @@ cmd_flip (struct dataset *ds)
   /* Set up flipped data for reading. */
   proc_set_source (ds, flip_source_create (flip));
 
-  return ok ? lex_end_of_command () : CMD_CASCADING_FAILURE;
+  return ok ? lex_end_of_command (lexer) : CMD_CASCADING_FAILURE;
 
  error:
   destroy_flip_pgm (flip);
diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index 0c0dc63b..929f987a 100644
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -305,15 +305,15 @@ freq_tab_to_hist(const struct freq_tab *ft, const struct variable *var);
 
 /* Parser and outline. */
 
-static int internal_cmd_frequencies (struct dataset *ds);
+static int internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds);
 
 int
-cmd_frequencies (struct dataset *ds)
+cmd_frequencies (struct lexer *lexer, struct dataset *ds)
 {
   int result;
 
   int_pool = pool_create ();
-  result = internal_cmd_frequencies (ds);
+  result = internal_cmd_frequencies (lexer, ds);
   pool_destroy (int_pool);
   int_pool=0;
   pool_destroy (gen_pool);
@@ -324,7 +324,7 @@ cmd_frequencies (struct dataset *ds)
 }
 
 static int
-internal_cmd_frequencies (struct dataset *ds)
+internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
 {
   int i;
   bool ok;
@@ -335,7 +335,7 @@ internal_cmd_frequencies (struct dataset *ds)
   n_variables = 0;
   v_variables = NULL;
 
-  if (!parse_frequencies (ds, &cmd, NULL))
+  if (!parse_frequencies (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
 
   if (cmd.onepage_limit == NOT_LONG)
@@ -376,7 +376,7 @@ internal_cmd_frequencies (struct dataset *ds)
 	  int pl;
 	  subc_list_double *ptl_list = &cmd.dl_percentiles[i];
 	  for ( pl = 0 ; pl < subc_list_double_count(ptl_list); ++pl)
-	      add_percentile(subc_list_double_at(ptl_list,pl) / 100.0 );
+	      add_percentile (subc_list_double_at(ptl_list, pl) / 100.0 );
 	}
     }
   if ( cmd.sbc_ntiles ) 
@@ -385,7 +385,7 @@ internal_cmd_frequencies (struct dataset *ds)
 	{
 	  int j;
 	  for (j = 0; j <= cmd.n_ntiles[i]; ++j ) 
-	      add_percentile(j / (double) cmd.n_ntiles[i]);
+	      add_percentile (j / (double) cmd.n_ntiles[i]);
 	}
     }
   
@@ -665,7 +665,7 @@ postcalc (void *aux UNUSED, const struct dataset *ds  UNUSED)
 
 	  norm.N = vf->tab.valid_cases;
 
-	  calc_stats(v,d);
+	  calc_stats (v, d);
 	  norm.mean = d[frq_mean];
 	  norm.stddev = d[frq_stddev];
 
@@ -793,7 +793,7 @@ cleanup_freq_tab (struct variable *v)
 /* Parses the VARIABLES subcommand, adding to
    {n_variables,v_variables}. */
 static int
-frq_custom_variables (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *aux UNUSED)
+frq_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *aux UNUSED)
 {
   int mode;
   int min = 0, max = 0;
@@ -801,31 +801,31 @@ frq_custom_variables (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, vo
   size_t old_n_variables = n_variables;
   size_t i;
 
-  lex_match ('=');
-  if (token != T_ALL && (token != T_ID
-                         || dict_lookup_var (dataset_dict (ds), tokid) == NULL))
+  lex_match (lexer, '=');
+  if (lex_token (lexer) != T_ALL && (lex_token (lexer) != T_ID
+                         || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
     return 2;
 
-  if (!parse_variables (dataset_dict (ds), &v_variables, &n_variables,
+  if (!parse_variables (lexer, dataset_dict (ds), &v_variables, &n_variables,
 			PV_APPEND | PV_NO_SCRATCH))
     return 0;
 
-  if (!lex_match ('('))
+  if (!lex_match (lexer, '('))
     mode = FRQM_GENERAL;
   else
     {
       mode = FRQM_INTEGER;
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return 0;
-      min = lex_integer ();
-      lex_get ();
-      if (!lex_force_match (','))
+      min = lex_integer (lexer);
+      lex_get (lexer);
+      if (!lex_force_match (lexer, ','))
 	return 0;
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return 0;
-      max = lex_integer ();
-      lex_get ();
-      if (!lex_force_match (')'))
+      max = lex_integer (lexer);
+      lex_get (lexer);
+      if (!lex_force_match (lexer, ')'))
 	return 0;
       if (max < min)
 	{
@@ -881,11 +881,11 @@ frq_custom_variables (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, vo
 /* Parses the GROUPED subcommand, setting the n_grouped, grouped
    fields of specified variables. */
 static int
-frq_custom_grouped (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *aux UNUSED)
+frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *aux UNUSED)
 {
-  lex_match ('=');
-  if ((token == T_ID && dict_lookup_var (dataset_dict (ds), tokid) != NULL)
-      || token == T_ID)
+  lex_match (lexer, '=');
+  if ((lex_token (lexer) == T_ID && dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+      || lex_token (lexer) == T_ID)
     for (;;)
       {
 	size_t i;
@@ -898,27 +898,27 @@ frq_custom_grouped (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void
 	size_t n;
 	struct variable **v;
 
-	if (!parse_variables (dataset_dict (ds), &v, &n,
+	if (!parse_variables (lexer, dataset_dict (ds), &v, &n,
                               PV_NO_DUPLICATE | PV_NUMERIC))
 	  return 0;
-	if (lex_match ('('))
+	if (lex_match (lexer, '('))
 	  {
 	    nl = ml = 0;
 	    dl = NULL;
-	    while (lex_integer ())
+	    while (lex_integer (lexer))
 	      {
 		if (nl >= ml)
 		  {
 		    ml += 16;
 		    dl = pool_nrealloc (int_pool, dl, ml, sizeof *dl);
 		  }
-		dl[nl++] = tokval;
-		lex_get ();
-		lex_match (',');
+		dl[nl++] = lex_tokval (lexer);
+		lex_get (lexer);
+		lex_match (lexer, ',');
 	      }
 	    /* Note that nl might still be 0 and dl might still be
 	       NULL.  That's okay. */
-	    if (!lex_match (')'))
+	    if (!lex_match (lexer, ')'))
 	      {
 		free (v);
 		msg (SE, _("`)' expected after GROUPED interval list."));
@@ -949,12 +949,12 @@ frq_custom_grouped (struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void
                 }
             }
 	free (v);
-	if (!lex_match ('/'))
+	if (!lex_match (lexer, '/'))
 	  break;
-	if ((token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) != NULL)
-            && token != T_ALL)
+	if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) != NULL)
+            && lex_token (lexer) != T_ALL)
 	  {
-	    lex_put_back ('/');
+	    lex_put_back (lexer, '/');
 	    break;
 	  }
       }
@@ -979,7 +979,7 @@ add_percentile (double x)
 	break;
     }
 
-  if (i >= n_percentiles || tokval != percentiles[i].p)
+  if (i >= n_percentiles || x != percentiles[i].p)
     {
       percentiles = pool_nrealloc (int_pool, percentiles,
                                    n_percentiles + 1, sizeof *percentiles);
diff --git a/src/language/stats/means.q b/src/language/stats/means.q
index fc01c07a..b7a2da20 100644
--- a/src/language/stats/means.q
+++ b/src/language/stats/means.q
@@ -61,7 +61,7 @@ static struct variable **v_var;
 
 /* Parses and executes the T-TEST procedure. */
 int
-cmd_means (struct dataset *ds)
+cmd_means (struct lexer *lexer, struct dataset *ds)
 {
   struct cmd_means cmd;
   int success = CMD_FAILURE;
@@ -71,7 +71,7 @@ cmd_means (struct dataset *ds)
   v_dim = NULL;
   v_var = NULL;
 
-  if (!parse_means (ds, &cmd, NULL))
+  if (!parse_means (lexer, ds, &cmd, NULL))
     goto free;
 
   if (cmd.sbc_cells)
@@ -122,15 +122,15 @@ free:
 
 /* Parses the TABLES subcommand. */
 static int
-mns_custom_tables (struct dataset *ds, struct cmd_means *cmd, void *aux UNUSED)
+mns_custom_tables (struct lexer *lexer, struct dataset *ds, struct cmd_means *cmd, void *aux UNUSED)
 {
   struct var_set *var_set;
   
-  if (!lex_match_id ("TABLES")
-      && (token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) == NULL)
-      && token != T_ALL)
+  if (!lex_match_id (lexer, "TABLES")
+      && (lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     return 2;
-  lex_match ('=');
+  lex_match (lexer, '=');
 
   if (cmd->sbc_tables)
     {
@@ -147,7 +147,7 @@ mns_custom_tables (struct dataset *ds, struct cmd_means *cmd, void *aux UNUSED)
       size_t nvl;
       struct variable **vl;
 
-      if (!parse_var_set_vars (var_set, &vl, &nvl,
+      if (!parse_var_set_vars (lexer, var_set, &vl, &nvl,
                                PV_NO_DUPLICATE | PV_NO_SCRATCH)) 
         goto lossage;
       
@@ -158,7 +158,7 @@ mns_custom_tables (struct dataset *ds, struct cmd_means *cmd, void *aux UNUSED)
       nv_dim[n_dim - 1] = nvl;
       v_dim[n_dim - 1] = vl;
     }
-  while (lex_match (T_BY));
+  while (lex_match (lexer, T_BY));
 
   var_set_destroy (var_set);
   return 1;
diff --git a/src/language/stats/oneway.q b/src/language/stats/oneway.q
index afce1efd..a792a2fb 100644
--- a/src/language/stats/oneway.q
+++ b/src/language/stats/oneway.q
@@ -112,12 +112,12 @@ void output_oneway(void);
 
 
 int
-cmd_oneway (struct dataset *ds)
+cmd_oneway (struct lexer *lexer, struct dataset *ds)
 {
   int i;
   bool ok;
 
-  if ( !parse_oneway (ds, &cmd, NULL) )
+  if ( !parse_oneway (lexer, ds, &cmd, NULL) )
     return CMD_FAILURE;
 
   /* What statistics were requested */
@@ -213,18 +213,19 @@ output_oneway(void)
 
 /* Parser for the variables sub command */
 static int
-oneway_custom_variables(struct dataset *ds, struct cmd_oneway *cmd UNUSED, 
+oneway_custom_variables (struct lexer *lexer, 
+			struct dataset *ds, struct cmd_oneway *cmd UNUSED, 
 			void *aux UNUSED)
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  if ((token != T_ID || dict_lookup_var (dict, tokid) == NULL)
-      && token != T_ALL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     return 2;
 
-  if (!parse_variables (dict, &vars, &n_vars,
+  if (!parse_variables (lexer, dict, &vars, &n_vars,
 			PV_DUPLICATE 
 			| PV_NUMERIC | PV_NO_SCRATCH) )
     {
@@ -234,14 +235,14 @@ oneway_custom_variables(struct dataset *ds, struct cmd_oneway *cmd UNUSED,
 
   assert(n_vars);
 
-  if ( ! lex_match(T_BY))
+  if ( ! lex_match (lexer, T_BY))
     return 2;
 
-  indep_var = parse_variable (dict);
+  indep_var = parse_variable (lexer, dict);
 
   if ( !indep_var ) 
     {
-      msg(SE,_("`%s' is not a variable name"),tokid);
+      msg(SE,_("`%s' is not a variable name"),lex_tokid (lexer));
       return 0;
     }
 
diff --git a/src/language/stats/rank.q b/src/language/stats/rank.q
index 7e882ab0..2ab8307d 100644
--- a/src/language/stats/rank.q
+++ b/src/language/stats/rank.q
@@ -729,7 +729,6 @@ create_rank_variable (struct dictionary *dict, enum RANK_FUNC f,
   return var;
 }
 
-int cmd_rank(struct dataset *ds);
 
 static void
 rank_cleanup(void)
@@ -758,14 +757,14 @@ rank_cleanup(void)
 }
 
 int
-cmd_rank (struct dataset *ds)
+cmd_rank (struct lexer *lexer, struct dataset *ds)
 {
   bool result;
   struct variable *order;
   size_t i;
   n_rank_specs = 0;
 
-  if ( !parse_rank (ds, &cmd, NULL) )
+  if ( !parse_rank (lexer, ds, &cmd, NULL) )
     {
       rank_cleanup ();
     return CMD_FAILURE;
@@ -929,27 +928,28 @@ cmd_rank (struct dataset *ds)
 /* Parser for the variables sub command  
    Returns 1 on success */
 static int
-rank_custom_variables (struct dataset *ds, struct cmd_rank *cmd UNUSED, void *aux UNUSED)
+rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd UNUSED, void *aux UNUSED)
 {
   static const int terminators[2] = {T_BY, 0};
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  if ((token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) == NULL)
-      && token != T_ALL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
       return 2;
 
-  sc = sort_parse_criteria (dataset_dict (ds), 
+  sc = sort_parse_criteria (lexer, dataset_dict (ds), 
 			    &src_vars, &n_src_vars, 0, terminators);
 
-  if ( lex_match(T_BY)  )
+  if ( lex_match (lexer, T_BY)  )
     {
-      if ((token != T_ID || dict_lookup_var (dataset_dict (ds), tokid) == NULL))
+      if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
 	{
 	  return 2;
 	}
 
-      if (!parse_variables (dataset_dict (ds), &group_vars, &n_group_vars,
+      if (!parse_variables (lexer, dataset_dict (ds), 
+			    &group_vars, &n_group_vars,
 			    PV_NO_DUPLICATE | PV_NUMERIC | PV_NO_SCRATCH) )
 	{
 	  free (group_vars);
@@ -963,7 +963,7 @@ rank_custom_variables (struct dataset *ds, struct cmd_rank *cmd UNUSED, void *au
 
 /* Parse the [/rank INTO var1 var2 ... varN ] clause */
 static int
-parse_rank_function(struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum RANK_FUNC f)
+parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum RANK_FUNC f)
 {
   int var_count = 0;
   
@@ -975,16 +975,16 @@ parse_rank_function(struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum R
   rank_specs[n_rank_specs - 1].destvars = 
 	    xcalloc (sc->crit_cnt, sizeof (struct variable *));
 	  
-  if (lex_match_id("INTO"))
+  if (lex_match_id (lexer, "INTO"))
     {
       struct variable *destvar;
 
-      while( token == T_ID ) 
+      while( lex_token (lexer) == T_ID ) 
 	{
 
-	  if ( dict_lookup_var (dict, tokid) != NULL )
+	  if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
 	    {
-	      msg(SE, _("Variable %s already exists."), tokid);
+	      msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
 	      return 0;
 	    }
 	  if ( var_count >= sc->crit_cnt ) 
@@ -993,10 +993,10 @@ parse_rank_function(struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum R
 	      return 0;
 	    }
 
-	  destvar = create_rank_variable (dict, f, src_vars[var_count], tokid);
+	  destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokid (lexer));
 	  rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
 
-	  lex_get();
+	  lex_get (lexer);
 	  ++var_count;
 	}
     }
@@ -1006,74 +1006,74 @@ parse_rank_function(struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum R
 
 
 static int
-rank_custom_rank(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_rank (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, RANK);
+  return parse_rank_function (lexer, dict, cmd, RANK);
 }
 
 static int
-rank_custom_normal(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_normal (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct  dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, NORMAL);
+  return parse_rank_function (lexer, dict, cmd, NORMAL);
 }
 
 static int
-rank_custom_percent(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_percent (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, PERCENT);
+  return parse_rank_function (lexer, dict, cmd, PERCENT);
 }
 
 static int
-rank_custom_rfraction(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_rfraction (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, RFRACTION);
+  return parse_rank_function (lexer, dict, cmd, RFRACTION);
 }
 
 static int
-rank_custom_proportion(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_proportion (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, PROPORTION);
+  return parse_rank_function (lexer, dict, cmd, PROPORTION);
 }
 
 static int
-rank_custom_n (struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_n (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, N);
+  return parse_rank_function (lexer, dict, cmd, N);
 }
 
 static int
-rank_custom_savage(struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_savage (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  return parse_rank_function (dict, cmd, SAVAGE);
+  return parse_rank_function (lexer, dict, cmd, SAVAGE);
 }
 
 
 static int
-rank_custom_ntiles (struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
+rank_custom_ntiles (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
 {
   struct dictionary *dict = dataset_dict (ds);
 
-  if ( lex_force_match('(') ) 
+  if ( lex_force_match (lexer, '(') ) 
     {
-      if ( lex_force_int() ) 
+      if ( lex_force_int (lexer) ) 
 	{
-	  k_ntiles = lex_integer ();
-	  lex_get();
-	  lex_force_match(')');
+	  k_ntiles = lex_integer (lexer);
+	  lex_get (lexer);
+	  lex_force_match (lexer, ')');
 	}
       else
 	return 0;
@@ -1081,5 +1081,5 @@ rank_custom_ntiles (struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
   else
     return 0;
 
-  return parse_rank_function(dict, cmd, NTILES);
+  return parse_rank_function (lexer, dict, cmd, NTILES);
 }
diff --git a/src/language/stats/regression.q b/src/language/stats/regression.q
index afe4e77a..1b359703 100644
--- a/src/language/stats/regression.q
+++ b/src/language/stats/regression.q
@@ -916,31 +916,31 @@ subcommand_export (int export, pspp_linreg_cache * c)
 }
 
 static int
-regression_custom_export (struct dataset *ds UNUSED, struct cmd_regression *cmd UNUSED, void *aux UNUSED)
+regression_custom_export (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_regression *cmd UNUSED, void *aux UNUSED)
 {
   /* 0 on failure, 1 on success, 2 on failure that should result in syntax error */
-  if (!lex_force_match ('('))
+  if (!lex_force_match (lexer, '('))
     return 0;
 
-  if (lex_match ('*'))
+  if (lex_match (lexer, '*'))
     model_file = NULL;
   else
     {
-      model_file = fh_parse (FH_REF_FILE);
+      model_file = fh_parse (lexer, FH_REF_FILE);
       if (model_file == NULL)
 	return 0;
     }
 
-  if (!lex_force_match (')'))
+  if (!lex_force_match (lexer, ')'))
     return 0;
 
   return 1;
 }
 
 int
-cmd_regression (struct dataset *ds)
+cmd_regression (struct lexer *lexer, struct dataset *ds)
 {
-  if (!parse_regression (ds, &cmd, NULL))
+  if (!parse_regression (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
 
   models = xnmalloc (cmd.n_dependent, sizeof *models);
@@ -1004,20 +1004,20 @@ mark_missing_cases (const struct casefile *cf, struct variable *v,
 
 /* Parser for the variables sub command */
 static int
-regression_custom_variables (struct dataset *ds, 
+regression_custom_variables (struct lexer *lexer, struct dataset *ds, 
 			     struct cmd_regression *cmd UNUSED,
 			     void *aux UNUSED)
 {
   const struct dictionary *dict = dataset_dict (ds);
 
-  lex_match ('=');
+  lex_match (lexer, '=');
 
-  if ((token != T_ID || dict_lookup_var (dict, tokid) == NULL)
-      && token != T_ALL)
+  if ((lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+      && lex_token (lexer) != T_ALL)
     return 2;
 
 
-  if (!parse_variables (dict, &v_variables, &n_variables, PV_NONE))
+  if (!parse_variables (lexer, dict, &v_variables, &n_variables, PV_NONE))
     {
       free (v_variables);
       return 0;
diff --git a/src/language/stats/sort-cases.c b/src/language/stats/sort-cases.c
index be8e3235..d73a79c4 100644
--- a/src/language/stats/sort-cases.c
+++ b/src/language/stats/sort-cases.c
@@ -40,24 +40,24 @@
 
 /* Performs the SORT CASES procedures. */
 int
-cmd_sort_cases (struct dataset *ds)
+cmd_sort_cases (struct lexer *lexer, struct dataset *ds)
 {
   struct sort_criteria *criteria;
   bool success = false;
 
-  lex_match (T_BY);
+  lex_match (lexer, T_BY);
 
-  criteria = sort_parse_criteria (dataset_dict (ds), NULL, NULL, NULL, NULL);
+  criteria = sort_parse_criteria (lexer, dataset_dict (ds), NULL, NULL, NULL, NULL);
   if (criteria == NULL)
     return CMD_CASCADING_FAILURE;
 
-  if (get_testing_mode () && lex_match ('/')) 
+  if (get_testing_mode () && lex_match (lexer, '/')) 
     {
-      if (!lex_force_match_id ("BUFFERS") || !lex_match ('=')
-          || !lex_force_int ())
+      if (!lex_force_match_id (lexer, "BUFFERS") || !lex_match (lexer, '=')
+          || !lex_force_int (lexer))
         goto done;
 
-      min_buffers = max_buffers = lex_integer ();
+      min_buffers = max_buffers = lex_integer (lexer);
       allow_internal_sort = false;
       if (max_buffers < 2) 
         {
@@ -65,7 +65,7 @@ cmd_sort_cases (struct dataset *ds)
           goto done;
         }
 
-      lex_get ();
+      lex_get (lexer);
     }
 
   success = sort_active_file_in_place (ds, criteria);
@@ -76,6 +76,6 @@ cmd_sort_cases (struct dataset *ds)
   allow_internal_sort = true;
   
   sort_destroy_criteria (criteria);
-  return success ? lex_end_of_command () : CMD_CASCADING_FAILURE;
+  return success ? lex_end_of_command (lexer) : CMD_CASCADING_FAILURE;
 }
 
diff --git a/src/language/stats/sort-criteria.c b/src/language/stats/sort-criteria.c
index b09dee39..3142eb53 100644
--- a/src/language/stats/sort-criteria.c
+++ b/src/language/stats/sort-criteria.c
@@ -50,7 +50,7 @@ static bool  is_terminator(int tok, const int *terminators);
    
 */
 struct sort_criteria *
-sort_parse_criteria (const struct dictionary *dict,
+sort_parse_criteria (struct lexer *lexer, const struct dictionary *dict,
                      struct variable ***vars, size_t *var_cnt,
                      bool *saw_direction,
 		     const int *terminators
@@ -82,23 +82,23 @@ sort_parse_criteria (const struct dictionary *dict,
       enum sort_direction direction;
 
       /* Variables. */
-      if (!parse_variables (dict, vars, var_cnt,
+      if (!parse_variables (lexer, dict, vars, var_cnt,
 			    PV_NO_DUPLICATE | PV_APPEND | PV_NO_SCRATCH))
         goto error;
 
       /* Sort direction. */
-      if (lex_match ('('))
+      if (lex_match (lexer, '('))
 	{
-	  if (lex_match_id ("D") || lex_match_id ("DOWN"))
+	  if (lex_match_id (lexer, "D") || lex_match_id (lexer, "DOWN"))
 	    direction = SRT_DESCEND;
-	  else if (lex_match_id ("A") || lex_match_id ("UP"))
+	  else if (lex_match_id (lexer, "A") || lex_match_id (lexer, "UP"))
             direction = SRT_ASCEND;
           else
 	    {
 	      msg (SE, _("`A' or `D' expected inside parentheses."));
               goto error;
 	    }
-	  if (!lex_match (')'))
+	  if (!lex_match (lexer, ')'))
 	    {
 	      msg (SE, _("`)' expected."));
               goto error;
@@ -120,7 +120,7 @@ sort_parse_criteria (const struct dictionary *dict,
           c->dir = direction;
         }
     }
-  while (token != '.' && token != '/' && !is_terminator(token, terminators));
+  while (lex_token (lexer) != '.' && lex_token (lexer) != '/' && !is_terminator(lex_token (lexer), terminators));
 
   free (local_vars);
   return criteria;
diff --git a/src/language/stats/sort-criteria.h b/src/language/stats/sort-criteria.h
index 0c804a8c..61d4db7c 100644
--- a/src/language/stats/sort-criteria.h
+++ b/src/language/stats/sort-criteria.h
@@ -26,8 +26,9 @@
 
 struct variable;
 struct dictionary;
+struct lexer ;
 
-struct sort_criteria *sort_parse_criteria (const struct dictionary *,
+struct sort_criteria *sort_parse_criteria (struct lexer *, const struct dictionary *,
                                            struct variable ***, size_t *,
                                            bool *saw_direction,
 					   const int *terminators
diff --git a/src/language/stats/t-test.q b/src/language/stats/t-test.q
index 570be931..a75b7356 100644
--- a/src/language/stats/t-test.q
+++ b/src/language/stats/t-test.q
@@ -154,7 +154,7 @@ struct pair
 
 static struct pair *pairs=0;
 
-static int parse_value (union value * v, int type) ;
+static int parse_value (struct lexer *lexer, union value * v, int type) ;
 
 /* Structures and Functions for the Statistics Summary Box */
 struct ssbox;
@@ -258,11 +258,11 @@ static unsigned  hash_group_binary(const struct group_statistics *g,
 
 
 int
-cmd_t_test (struct dataset *ds)
+cmd_t_test (struct lexer *lexer, struct dataset *ds)
 {
   bool ok;
   
-  if ( !parse_t_test (ds, &cmd, NULL) )
+  if ( !parse_t_test (lexer, ds, &cmd, NULL) )
     return CMD_FAILURE;
 
   if (! cmd.sbc_criteria)
@@ -362,16 +362,16 @@ cmd_t_test (struct dataset *ds)
 }
 
 static int
-tts_custom_groups (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux UNUSED)
+tts_custom_groups (struct lexer *lexer, struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux UNUSED)
 {
   int n_group_values=0;
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  indep_var = parse_variable (dataset_dict (ds));
+  indep_var = parse_variable (lexer, dataset_dict (ds));
   if (!indep_var)
     {
-      lex_error ("expecting variable name in GROUPS subcommand");
+      lex_error (lexer, "expecting variable name in GROUPS subcommand");
       return 0;
     }
 
@@ -382,7 +382,7 @@ tts_custom_groups (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux
       return 0;
     }
 
-  if (!lex_match ('('))
+  if (!lex_match (lexer, '('))
     {
       if (indep_var->type == NUMERIC)
 	{
@@ -403,11 +403,11 @@ tts_custom_groups (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux
 	}
     }
 
-  if (!parse_value (&gp.v.g_value[0], indep_var->type))
+  if (!parse_value (lexer, &gp.v.g_value[0], indep_var->type))
       return 0;
 
-  lex_match (',');
-  if (lex_match (')'))
+  lex_match (lexer, ',');
+  if (lex_match (lexer, ')'))
     {
       if (indep_var->type != NUMERIC)
 	{
@@ -423,11 +423,11 @@ tts_custom_groups (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux
       return 1;
     }
 
-  if (!parse_value (&gp.v.g_value[1], indep_var->type))
+  if (!parse_value (lexer, &gp.v.g_value[1], indep_var->type))
     return 0;
 
   n_group_values = 2;
-  if (!lex_force_match (')'))
+  if (!lex_force_match (lexer, ')'))
     return 0;
 
   if ( n_group_values == 2 ) 
@@ -441,7 +441,7 @@ tts_custom_groups (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux
 
 
 static int
-tts_custom_pairs (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux UNUSED)
+tts_custom_pairs (struct lexer *lexer, struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux UNUSED)
 {
   struct variable **vars;
   size_t n_vars;
@@ -451,10 +451,10 @@ tts_custom_pairs (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux U
   size_t n_after_WITH = SIZE_MAX;
   int paired ; /* Was the PAIRED keyword given ? */
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
   n_vars=0;
-  if (!parse_variables (dataset_dict (ds), &vars, &n_vars,
+  if (!parse_variables (lexer, dataset_dict (ds), &vars, &n_vars,
 			PV_DUPLICATE | PV_NUMERIC | PV_NO_SCRATCH))
     {
       free (vars);
@@ -463,10 +463,10 @@ tts_custom_pairs (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux U
   assert (n_vars);
 
   n_before_WITH = 0;
-  if (lex_match (T_WITH))
+  if (lex_match (lexer, T_WITH))
     {
       n_before_WITH = n_vars;
-      if (!parse_variables (dataset_dict (ds), &vars, &n_vars,
+      if (!parse_variables (lexer, dataset_dict (ds), &vars, &n_vars,
 			    PV_DUPLICATE | PV_APPEND
 			    | PV_NUMERIC | PV_NO_SCRATCH))
 	{
@@ -476,7 +476,7 @@ tts_custom_pairs (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux U
       n_after_WITH = n_vars - n_before_WITH;
     }
 
-  paired = (lex_match ('(') && lex_match_id ("PAIRED") && lex_match (')'));
+  paired = (lex_match (lexer, '(') && lex_match_id (lexer, "PAIRED") && lex_match (lexer, ')'));
 
   /* Determine the number of pairs needed */
   if (paired)
@@ -566,22 +566,22 @@ tts_custom_pairs (struct dataset *ds, struct cmd_t_test *cmd UNUSED, void *aux U
 /* Parses the current token (numeric or string, depending on type)
     value v and returns success. */
 static int
-parse_value (union value * v, int type )
+parse_value (struct lexer *lexer, union value * v, int type )
 {
   if (type == NUMERIC)
     {
-      if (!lex_force_num ())
+      if (!lex_force_num (lexer))
 	return 0;
-      v->f = tokval;
+      v->f = lex_tokval (lexer);
     }
   else
     {
-      if (!lex_force_string ())
+      if (!lex_force_string (lexer))
 	return 0;
-      strncpy (v->s, ds_cstr (&tokstr), ds_length (&tokstr));
+      strncpy (v->s, ds_cstr (lex_tokstr (lexer)), ds_length (lex_tokstr (lexer)));
     }
 
-  lex_get ();
+  lex_get (lexer);
 
   return 1;
 }
diff --git a/src/language/tests/casefile-test.c b/src/language/tests/casefile-test.c
index c6885c61..0bafe5ef 100644
--- a/src/language/tests/casefile-test.c
+++ b/src/language/tests/casefile-test.c
@@ -45,7 +45,7 @@ static void test_casereader_clone (struct casereader *reader1, size_t case_cnt);
 static void fail_test (const char *message, ...);
 
 int
-cmd_debug_casefile (struct dataset *ds UNUSED) 
+cmd_debug_casefile (struct lexer *lexer, struct dataset *ds UNUSED) 
 {
   static const size_t sizes[] =
     {
@@ -57,15 +57,15 @@ cmd_debug_casefile (struct dataset *ds UNUSED)
   int pattern;
 
   size_max = sizeof sizes / sizeof *sizes;
-  if (lex_match_id ("SMALL")) 
+  if (lex_match_id (lexer, "SMALL")) 
     {
       size_max -= 4;
       case_max = 511; 
     }
   else
     case_max = 4095;
-  if (token != '.')
-    return lex_end_of_command ();
+  if (lex_token (lexer) != '.')
+    return lex_end_of_command (lexer);
     
   for (pattern = 0; pattern < 7; pattern++) 
     {
diff --git a/src/language/tests/float-format.c b/src/language/tests/float-format.c
index c2ca32f0..52a39253 100644
--- a/src/language/tests/float-format.c
+++ b/src/language/tests/float-format.c
@@ -69,17 +69,17 @@ static const size_t format_cnt = sizeof fp_formats / sizeof *fp_formats;
 /* Parses a floating-point format name into *FORMAT,
    and returns success. */
 static bool
-parse_float_format (enum float_format *format) 
+parse_float_format (struct lexer *lexer, enum float_format *format) 
 {
   size_t i;
 
   for (i = 0; i < format_cnt; i++)
-    if (lex_match_id (fp_formats[i].name)) 
+    if (lex_match_id (lexer, fp_formats[i].name)) 
       {
         *format = fp_formats[i].format;
         return true;
       }
-  lex_error ("expecting floating-point format identifier");
+  lex_error (lexer, "expecting floating-point format identifier");
   return false;
 }
 
@@ -101,25 +101,25 @@ get_float_format_name (enum float_format format)
    representation.  Also supports ordinary floating-point numbers
    written in decimal notation.  Returns success. */
 static bool
-parse_fp (struct fp *fp) 
+parse_fp (struct lexer *lexer, struct fp *fp) 
 {
-  if (lex_is_number ()) 
+  if (lex_is_number (lexer)) 
     {
-      double number = lex_number ();
+      double number = lex_number (lexer);
       fp->format = FLOAT_NATIVE_DOUBLE;
       memcpy (fp->data, &number, sizeof number);
-      lex_get ();
+      lex_get (lexer);
     }
-  else if (token == T_ID)
+  else if (lex_token (lexer) == T_ID)
     {
       size_t length;
       
-      if (!parse_float_format (&fp->format)
-          || !lex_force_match ('(')
-          || !lex_force_string ())
+      if (!parse_float_format (lexer, &fp->format)
+          || !lex_force_match (lexer, '(')
+          || !lex_force_string (lexer))
         return false;
 
-      length = ds_length (&tokstr);
+      length = ds_length (lex_tokstr (lexer));
       if (fp->format != FLOAT_HEX) 
         {
           if (length != float_get_size (fp->format)) 
@@ -129,7 +129,7 @@ parse_fp (struct fp *fp)
               return false;
             }
           assert (length <= sizeof fp->data);
-          memcpy (fp->data, ds_data (&tokstr), length); 
+          memcpy (fp->data, ds_data (lex_tokstr (lexer)), length); 
         }
       else 
         {
@@ -138,16 +138,16 @@ parse_fp (struct fp *fp)
               msg (SE, _("Hexadecimal floating constant too long."));
               return false;
             }
-          strncpy ((char *) fp->data, ds_cstr (&tokstr), sizeof fp->data);
+          strncpy ((char *) fp->data, ds_cstr (lex_tokstr (lexer)), sizeof fp->data);
         }
 
-      lex_get ();
-      if (!lex_force_match (')'))
+      lex_get (lexer);
+      if (!lex_force_match (lexer, ')'))
         return false;
     }
   else
     {
-      lex_error (NULL);
+      lex_error (lexer, NULL);
       return false;
     }
   return true;
@@ -235,7 +235,7 @@ verify_conversion (const struct fp *from, const struct fp *to)
 
 /* Executes the DEBUG FLOAT FORMAT command. */
 int
-cmd_debug_float_format (struct dataset *ds UNUSED) 
+cmd_debug_float_format (struct lexer *lexer, struct dataset *ds UNUSED) 
 {
   struct fp fp[16];
   size_t fp_cnt = 0;
@@ -249,30 +249,30 @@ cmd_debug_float_format (struct dataset *ds UNUSED)
           msg (SE, _("Too many values in single command."));
           return CMD_FAILURE;
         }
-      if (!parse_fp (&fp[fp_cnt++]))
+      if (!parse_fp (lexer, &fp[fp_cnt++]))
         return CMD_FAILURE;
 
-      if (token == '.' && fp_cnt > 1)
+      if (lex_token (lexer) == '.' && fp_cnt > 1)
         break;
-      else if (!lex_force_match ('='))
+      else if (!lex_force_match (lexer, '='))
         return CMD_FAILURE;
       
       if (fp_cnt == 1) 
         {
-          if (lex_match ('='))
+          if (lex_match (lexer, '='))
             bijective = true;
-          else if (lex_match (T_GT))
+          else if (lex_match (lexer, T_GT))
             bijective = false;
           else 
             {
-              lex_error (NULL);
+              lex_error (lexer, NULL);
               return CMD_FAILURE;
             }
         }
       else 
         {
-          if ((bijective && !lex_force_match ('='))
-              || (!bijective && !lex_force_match (T_GT)))
+          if ((bijective && !lex_force_match (lexer, '='))
+              || (!bijective && !lex_force_match (lexer, T_GT)))
             return CMD_FAILURE;
         }
     }
diff --git a/src/language/tests/moments-test.c b/src/language/tests/moments-test.c
index 533b558f..893d2300 100644
--- a/src/language/tests/moments-test.c
+++ b/src/language/tests/moments-test.c
@@ -31,27 +31,27 @@
 #define _(msgid) gettext (msgid)
 
 static bool
-read_values (double **values, double **weights, size_t *cnt) 
+read_values (struct lexer *lexer, double **values, double **weights, size_t *cnt) 
 {
   size_t cap = 0;
 
   *values = NULL;
   *weights = NULL;
   *cnt = 0;
-  while (lex_is_number ())
+  while (lex_is_number (lexer))
     {
-      double value = tokval;
+      double value = lex_tokval (lexer);
       double weight = 1.;
-      lex_get ();
-      if (lex_match ('*'))
+      lex_get (lexer);
+      if (lex_match (lexer, '*'))
         {
-          if (!lex_is_number ())
+          if (!lex_is_number (lexer))
             {
-              lex_error (_("expecting weight value"));
+              lex_error (lexer, _("expecting weight value"));
               return false;
             }
-          weight = tokval;
-          lex_get ();
+          weight = lex_tokval (lexer);
+          lex_get (lexer);
         }
 
       if (*cnt >= cap) 
@@ -70,7 +70,7 @@ read_values (double **values, double **weights, size_t *cnt)
 }
 
 int
-cmd_debug_moments (struct dataset *ds UNUSED) 
+cmd_debug_moments (struct lexer *lexer, struct dataset *ds UNUSED) 
 {
   int retval = CMD_FAILURE;
   double *values = NULL;
@@ -80,22 +80,22 @@ cmd_debug_moments (struct dataset *ds UNUSED)
   size_t cnt;
   size_t i;
 
-  if (lex_match_id ("ONEPASS"))
+  if (lex_match_id (lexer, "ONEPASS"))
     two_pass = 0;
-  if (token != '/') 
+  if (lex_token (lexer) != '/') 
     {
-      lex_force_match ('/');
+      lex_force_match (lexer, '/');
       goto done;
     }
-  fprintf (stderr, "%s => ", lex_rest_of_line (NULL));
-  lex_get ();
+  fprintf (stderr, "%s => ", lex_rest_of_line (lexer, NULL));
+  lex_get (lexer);
 
   if (two_pass) 
     {
       struct moments *m = NULL;
   
       m = moments_create (MOMENT_KURTOSIS);
-      if (!read_values (&values, &weights, &cnt)) 
+      if (!read_values (lexer, &values, &weights, &cnt)) 
         {
           moments_destroy (m);
           goto done; 
@@ -112,7 +112,7 @@ cmd_debug_moments (struct dataset *ds UNUSED)
       struct moments1 *m = NULL;
   
       m = moments1_create (MOMENT_KURTOSIS);
-      if (!read_values (&values, &weights, &cnt)) 
+      if (!read_values (lexer, &values, &weights, &cnt)) 
         {
           moments1_destroy (m);
           goto done; 
@@ -136,7 +136,7 @@ cmd_debug_moments (struct dataset *ds UNUSED)
     }
   fprintf (stderr, "\n");
 
-  retval = lex_end_of_command ();
+  retval = lex_end_of_command (lexer);
   
  done:
   free (values);
diff --git a/src/language/tests/pool-test.c b/src/language/tests/pool-test.c
index 8f1484ff..b888c896 100644
--- a/src/language/tests/pool-test.c
+++ b/src/language/tests/pool-test.c
@@ -32,7 +32,7 @@
 /* Self-test routine.
    This is not exhaustive, but it can be useful. */
 int
-cmd_debug_pool (struct dataset *ds UNUSED)
+cmd_debug_pool (struct lexer *lexer UNUSED, struct dataset *ds UNUSED)
 {
   int seed = time (0) * 257 % 32768;
 
diff --git a/src/language/utilities/date.c b/src/language/utilities/date.c
index 7529f977..25442135 100644
--- a/src/language/utilities/date.c
+++ b/src/language/utilities/date.c
@@ -27,10 +27,10 @@
 
 /* Stub for USE command. */
 int
-cmd_use (struct dataset *ds UNUSED) 
+cmd_use (struct lexer *lexer, struct dataset *ds UNUSED) 
 {
-  if (lex_match (T_ALL))
-    return lex_end_of_command ();
+  if (lex_match (lexer, T_ALL))
+    return lex_end_of_command (lexer);
 
   msg (SW, _("Only USE ALL is currently implemented."));
   return CMD_FAILURE;
diff --git a/src/language/utilities/echo.c b/src/language/utilities/echo.c
index 5405d060..7bb581bd 100644
--- a/src/language/utilities/echo.c
+++ b/src/language/utilities/echo.c
@@ -29,11 +29,11 @@
 
 /* Echos a string to the output stream */
 int
-cmd_echo (struct dataset *ds UNUSED)
+cmd_echo (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   struct tab_table *tab;
 
-  if (token != T_STRING) 
+  if (lex_token (lexer) != T_STRING) 
     return CMD_FAILURE;
   
   tab = tab_create(1, 1, 0);
@@ -41,7 +41,7 @@ cmd_echo (struct dataset *ds UNUSED)
   tab_dim (tab, tab_natural_dimensions);
   tab_flags (tab, SOMF_NO_TITLE );
 
-  tab_text(tab, 0, 0, 0, ds_cstr (&tokstr));
+  tab_text(tab, 0, 0, 0, ds_cstr (lex_tokstr (lexer)));
 
   tab_submit(tab);
 
diff --git a/src/language/utilities/include.c b/src/language/utilities/include.c
index ef58a20e..3e9a5b7b 100644
--- a/src/language/utilities/include.c
+++ b/src/language/utilities/include.c
@@ -31,20 +31,20 @@
 #define _(msgid) gettext (msgid)
 
 int
-cmd_include (struct dataset *ds UNUSED)
+cmd_include (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   /* Skip optional FILE=. */
-  if (lex_match_id ("FILE"))
-    lex_match ('=');
+  if (lex_match_id (lexer, "FILE"))
+    lex_match (lexer, '=');
 
   /* File name can be identifier or string. */
-  if (token != T_ID && token != T_STRING) 
+  if (lex_token (lexer) != T_ID && lex_token (lexer) != T_STRING) 
     {
-      lex_error (_("expecting file name")); 
+      lex_error (lexer, _("expecting file name")); 
       return CMD_CASCADING_FAILURE;
     }
-  getl_include_syntax_file (ds_cstr (&tokstr));
+  getl_include_syntax_file (ds_cstr (lex_tokstr (lexer)));
 
-  lex_get ();
-  return lex_end_of_command ();
+  lex_get (lexer);
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/utilities/permissions.c b/src/language/utilities/permissions.c
index 7ac9143c..bce2194a 100644
--- a/src/language/utilities/permissions.c
+++ b/src/language/utilities/permissions.c
@@ -42,32 +42,32 @@ int change_permissions(const char *file_name, enum PER per);
 
 /* Parses the PERMISSIONS command. */
 int
-cmd_permissions (struct dataset *ds UNUSED)
+cmd_permissions (struct lexer *lexer, struct dataset *ds UNUSED)
 {
   char  *fn = 0;
 
-  lex_match ('/');
+  lex_match (lexer, '/');
 
-  if (lex_match_id ("FILE"))
-    lex_match ('=');
+  if (lex_match_id (lexer, "FILE"))
+    lex_match (lexer, '=');
 
-  fn = ds_xstrdup (&tokstr);
-  lex_force_match(T_STRING);
+  fn = ds_xstrdup (lex_tokstr (lexer));
+  lex_force_match (lexer, T_STRING);
 
 
-  lex_match ('/');
+  lex_match (lexer, '/');
   
-  if ( ! lex_match_id ("PERMISSIONS"))
+  if ( ! lex_match_id (lexer, "PERMISSIONS"))
     goto error;
 
-  lex_match('=');
+  lex_match (lexer, '=');
 
-  if ( lex_match_id("READONLY"))
+  if ( lex_match_id (lexer, "READONLY"))
     {
       if ( ! change_permissions(fn, PER_RO ) ) 
 	goto error;
     }
-  else if ( lex_match_id("WRITEABLE"))
+  else if ( lex_match_id (lexer, "WRITEABLE"))
     {
       if ( ! change_permissions(fn, PER_RW ) ) 
 	goto error;
diff --git a/src/language/utilities/set.q b/src/language/utilities/set.q
index a05878a3..ed7bbbf7 100644
--- a/src/language/utilities/set.q
+++ b/src/language/utilities/set.q
@@ -127,12 +127,15 @@ static enum integer_format stc_to_integer_format (int stc);
 static enum float_format stc_to_float_format (int stc);
 
 int
-cmd_set (struct dataset *ds)
+cmd_set (struct lexer *lexer, struct dataset *ds)
 {
   struct cmd_set cmd;
 
-  if (!parse_set (ds, &cmd, NULL))
-    return CMD_FAILURE;
+  if (!parse_set (lexer, ds, &cmd, NULL))
+    {
+      free_set (&cmd);
+      return CMD_FAILURE;
+    }
 
   if (cmd.sbc_cca)
     do_cc (cmd.s_cca, FMT_CCA);
@@ -212,6 +215,8 @@ cmd_set (struct dataset *ds)
   if (cmd.sbc_compression)
     msg (SW, _("Active file compression is not implemented."));
 
+  free_set (&cmd);
+
   return CMD_SUCCESS;
 }
 
@@ -352,20 +357,22 @@ do_cc (const char *cc_string, enum fmt_type type)
    completely blank fields in numeric data imply.  X, Wnd: Syntax is
    SYSMIS or a numeric value. */
 static int
-stc_custom_blanks (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_blanks (struct lexer *lexer, 
+		   struct dataset *ds UNUSED, 
+		   struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  lex_match ('=');
-  if ((token == T_ID && lex_id_match ("SYSMIS", tokid)))
+  lex_match (lexer, '=');
+  if ((lex_token (lexer) == T_ID && lex_id_match ("SYSMIS", lex_tokid (lexer))))
     {
-      lex_get ();
+      lex_get (lexer);
       set_blanks (SYSMIS);
     }
   else
     {
-      if (!lex_force_num ())
+      if (!lex_force_num (lexer))
 	return 0;
-      set_blanks (lex_number ());
-      lex_get ();
+      set_blanks (lex_number (lexer));
+      lex_get (lexer);
     }
   return 1;
 }
@@ -373,15 +380,17 @@ stc_custom_blanks (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *
 /* Parses the EPOCH subcommand, which controls the epoch used for
    parsing 2-digit years. */
 static int
-stc_custom_epoch (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED) 
+stc_custom_epoch (struct lexer *lexer, 
+		  struct dataset *ds UNUSED, 
+		  struct cmd_set *cmd UNUSED, void *aux UNUSED) 
 {
-  lex_match ('=');
-  if (lex_match_id ("AUTOMATIC"))
+  lex_match (lexer, '=');
+  if (lex_match_id (lexer, "AUTOMATIC"))
     set_epoch (-1);
-  else if (lex_is_integer ()) 
+  else if (lex_is_integer (lexer)) 
     {
-      int new_epoch = lex_integer ();
-      lex_get ();
+      int new_epoch = lex_integer (lexer);
+      lex_get (lexer);
       if (new_epoch < 1500) 
         {
           msg (SE, _("EPOCH must be 1500 or later."));
@@ -391,7 +400,7 @@ stc_custom_epoch (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *a
     }
   else 
     {
-      lex_error (_("expecting AUTOMATIC or year"));
+      lex_error (lexer, _("expecting AUTOMATIC or year"));
       return 0;
     }
 
@@ -399,24 +408,24 @@ stc_custom_epoch (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *a
 }
 
 static int
-stc_custom_length (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_length (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
   int page_length;
 
-  lex_match ('=');
-  if (lex_match_id ("NONE"))
+  lex_match (lexer, '=');
+  if (lex_match_id (lexer, "NONE"))
     page_length = -1;
   else
     {
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return 0;
-      if (lex_integer () < 1)
+      if (lex_integer (lexer) < 1)
 	{
 	  msg (SE, _("LENGTH must be at least 1."));
 	  return 0;
 	}
-      page_length = lex_integer ();
-      lex_get ();
+      page_length = lex_integer (lexer);
+      lex_get (lexer);
     }
 
   if (page_length != -1) 
@@ -426,41 +435,41 @@ stc_custom_length (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *
 }
 
 static int
-stc_custom_seed (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_seed (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  lex_match ('=');
-  if (lex_match_id ("RANDOM"))
+  lex_match (lexer, '=');
+  if (lex_match_id (lexer, "RANDOM"))
     set_rng (time (0));
   else
     {
-      if (!lex_force_num ())
+      if (!lex_force_num (lexer))
 	return 0;
-      set_rng (lex_number ());
-      lex_get ();
+      set_rng (lex_number (lexer));
+      lex_get (lexer);
     }
 
   return 1;
 }
 
 static int
-stc_custom_width (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_width (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  lex_match ('=');
-  if (lex_match_id ("NARROW"))
+  lex_match (lexer, '=');
+  if (lex_match_id (lexer, "NARROW"))
     set_viewwidth (79);
-  else if (lex_match_id ("WIDE"))
+  else if (lex_match_id (lexer, "WIDE"))
     set_viewwidth (131);
   else
     {
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return 0;
-      if (lex_integer () < 40)
+      if (lex_integer (lexer) < 40)
 	{
 	  msg (SE, _("WIDTH must be at least 40."));
 	  return 0;
 	}
-      set_viewwidth (lex_integer ());
-      lex_get ();
+      set_viewwidth (lex_integer (lexer));
+      lex_get (lexer);
     }
 
   return 1;
@@ -469,12 +478,12 @@ stc_custom_width (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *a
 /* Parses FORMAT subcommand, which consists of a numeric format
    specifier. */
 static int
-stc_custom_format (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_format (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
   struct fmt_spec fmt;
 
-  lex_match ('=');
-  if (!parse_format_specifier (&fmt))
+  lex_match (lexer, '=');
+  if (!parse_format_specifier (lexer, &fmt))
     return 0;
   if (fmt_is_string (fmt.type))
     {
@@ -490,16 +499,16 @@ stc_custom_format (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *
 }
 
 static int
-stc_custom_journal (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_journal (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
-  lex_match ('=');
-  if (!lex_match_id ("ON") && !lex_match_id ("OFF")) 
+  lex_match (lexer, '=');
+  if (!lex_match_id (lexer, "ON") && !lex_match_id (lexer, "OFF")) 
     {
-      if (token == T_STRING)
-        lex_get ();
+      if (lex_token (lexer) == T_STRING)
+        lex_get (lexer);
       else
         {
-          lex_error (NULL);
+          lex_error (lexer, NULL);
           return 0;
         }
     }
@@ -507,14 +516,14 @@ stc_custom_journal (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void
 }
 
 static int
-stc_custom_listing (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
+stc_custom_listing (struct lexer *lexer, struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void *aux UNUSED)
 {
   bool listing;
 
-  lex_match ('=');
-  if (lex_match_id ("ON") || lex_match_id ("YES"))
+  lex_match (lexer, '=');
+  if (lex_match_id (lexer, "ON") || lex_match_id (lexer, "YES"))
     listing = true;
-  else if (lex_match_id ("OFF") || lex_match_id ("NO"))
+  else if (lex_match_id (lexer, "OFF") || lex_match_id (lexer, "NO"))
     listing = false;
   else
     {
@@ -527,9 +536,9 @@ stc_custom_listing (struct dataset *ds UNUSED, struct cmd_set *cmd UNUSED, void
 }
 
 static int
-stc_custom_disk (struct dataset *ds, struct cmd_set *cmd UNUSED, void *aux)
+stc_custom_disk (struct lexer *lexer, struct dataset *ds, struct cmd_set *cmd UNUSED, void *aux)
 {
-  return stc_custom_listing (ds, cmd, aux);
+  return stc_custom_listing (lexer, ds, cmd, aux);
 }
 
 static void
@@ -828,9 +837,9 @@ show_copying (const struct dataset *ds UNUSED)
 }
 
 int
-cmd_show (struct dataset *ds) 
+cmd_show (struct lexer *lexer, struct dataset *ds) 
 {
-  if (token == '.') 
+  if (lex_token (lexer) == '.') 
     {
       show_all (ds);
       return CMD_SUCCESS;
@@ -838,38 +847,38 @@ cmd_show (struct dataset *ds)
 
   do 
     {
-      if (lex_match (T_ALL))
+      if (lex_match (lexer, T_ALL))
         show_all (ds);
-      else if (lex_match_id ("CC")) 
+      else if (lex_match_id (lexer, "CC")) 
         show_all_cc ();
-      else if (lex_match_id ("WARRANTY"))
+      else if (lex_match_id (lexer, "WARRANTY"))
         show_warranty (ds);
-      else if (lex_match_id ("COPYING"))
+      else if (lex_match_id (lexer, "COPYING"))
         show_copying (ds);
-      else if (token == T_ID)
+      else if (lex_token (lexer) == T_ID)
         {
           int i;
 
           for (i = 0; i < sizeof show_table / sizeof *show_table; i++)
-            if (lex_match_id (show_table[i].name)) 
+            if (lex_match_id (lexer, show_table[i].name)) 
               {
                 show_table[i].function (ds);
                 goto found;
               }
-          lex_error (NULL);
+          lex_error (lexer, NULL);
           return CMD_FAILURE;
 
         found: ;
         }
       else 
         {
-          lex_error (NULL);
+          lex_error (lexer, NULL);
           return CMD_FAILURE;
         }
 
-      lex_match ('/');
+      lex_match (lexer, '/');
     }
-  while (token != '.');
+  while (lex_token (lexer) != '.');
 
   return CMD_SUCCESS;
 }
diff --git a/src/language/utilities/title.c b/src/language/utilities/title.c
index 9e8bfaf7..d7943126 100644
--- a/src/language/utilities/title.c
+++ b/src/language/utilities/title.c
@@ -36,36 +36,36 @@
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
 
-static int get_title (const char *cmd, char **title);
+static int get_title (struct lexer *, const char *cmd, char **title);
 
 int
-cmd_title (struct dataset *ds UNUSED)
+cmd_title (struct lexer *lexer, struct dataset *ds UNUSED)
 {
-  return get_title ("TITLE", &outp_title);
+  return get_title (lexer, "TITLE", &outp_title);
 }
 
 int
-cmd_subtitle (struct dataset *ds UNUSED)
+cmd_subtitle (struct lexer *lexer, struct dataset *ds UNUSED)
 {
-  return get_title ("SUBTITLE", &outp_subtitle);
+  return get_title (lexer, "SUBTITLE", &outp_subtitle);
 }
 
 static int
-get_title (const char *cmd, char **title)
+get_title (struct lexer *lexer, const char *cmd, char **title)
 {
   int c;
 
-  c = lex_look_ahead ();
+  c = lex_look_ahead (lexer);
   if (c == '"' || c == '\'')
     {
-      lex_get ();
-      if (!lex_force_string ())
+      lex_get (lexer);
+      if (!lex_force_string (lexer))
 	return CMD_FAILURE;
       if (*title)
 	free (*title);
-      *title = ds_xstrdup (&tokstr);
-      lex_get ();
-      if (token != '.')
+      *title = ds_xstrdup (lex_tokstr (lexer));
+      lex_get (lexer);
+      if (lex_token (lexer) != '.')
 	{
 	  msg (SE, _("%s: `.' expected after string."), cmd);
 	  return CMD_FAILURE;
@@ -77,28 +77,26 @@ get_title (const char *cmd, char **title)
 
       if (*title)
 	free (*title);
-      *title = xstrdup (lex_rest_of_line (NULL));
-      lex_discard_line ();
+      *title = xstrdup (lex_rest_of_line (lexer, NULL));
+      lex_discard_line (lexer);
       for (cp = *title; *cp; cp++)
 	*cp = toupper ((unsigned char) (*cp));
-      token = '.';
     }
   return CMD_SUCCESS;
 }
 
 /* Performs the FILE LABEL command. */
 int
-cmd_file_label (struct dataset *ds)
+cmd_file_label (struct lexer *lexer, struct dataset *ds)
 {
   const char *label;
 
-  label = lex_rest_of_line (NULL);
-  lex_discard_line ();
+  label = lex_rest_of_line (lexer, NULL);
+  lex_discard_line (lexer);
   while (isspace ((unsigned char) *label))
     label++;
 
   dict_set_label (dataset_dict (ds), label);
-  token = '.';
 
   return CMD_SUCCESS;
 }
@@ -128,7 +126,7 @@ add_document_line (struct dictionary *dict, const char *line, int indent)
 
 /* Performs the DOCUMENT command. */
 int
-cmd_document (struct dataset *ds)
+cmd_document (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   /* Add a few header lines for reference. */
@@ -148,8 +146,8 @@ cmd_document (struct dataset *ds)
       const char *orig_line;
       char *copy_line;
 
-      orig_line = lex_rest_of_line (&had_dot);
-      lex_discard_line ();
+      orig_line = lex_rest_of_line (lexer, &had_dot);
+      lex_discard_line (lexer);
       while (isspace ((unsigned char) *orig_line))
 	orig_line++;
 
@@ -161,20 +159,19 @@ cmd_document (struct dataset *ds)
       add_document_line (dict, copy_line, 3);
       free (copy_line);
 
-      lex_get_line ();
+      lex_get_line (lexer);
       if (had_dot)
 	break;
     }
 
-  token = '.';
   return CMD_SUCCESS;
 }
 
 /* Performs the DROP DOCUMENTS command. */
 int
-cmd_drop_documents (struct dataset *ds)
+cmd_drop_documents (struct lexer *lexer, struct dataset *ds)
 {
   dict_set_documents (dataset_dict (ds), NULL);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/xforms/compute.c b/src/language/xforms/compute.c
index 46c693bb..a29bb699 100644
--- a/src/language/xforms/compute.c
+++ b/src/language/xforms/compute.c
@@ -43,7 +43,7 @@ struct lvalue;
 
 /* Target of a COMPUTE or IF assignment, either a variable or a
    vector element. */
-static struct lvalue *lvalue_parse (struct dataset *);
+static struct lvalue *lvalue_parse (struct lexer *lexer, struct dataset *);
 static int lvalue_get_type (const struct lvalue *, const struct dictionary *);
 static bool lvalue_is_vector (const struct lvalue *);
 static void lvalue_finalize (struct lvalue *,
@@ -69,7 +69,10 @@ struct compute_trns
     struct expression *rvalue;	 /* Rvalue expression. */
   };
 
-static struct expression *parse_rvalue (const struct lvalue *, struct dataset *);
+static struct expression *parse_rvalue (struct lexer *lexer, 
+					const struct lvalue *, 
+					struct dataset *);
+
 static struct compute_trns *compute_trns_create (void);
 static trns_proc_func *get_proc_func (const struct lvalue *, const struct dictionary *);
 static trns_free_func compute_trns_free;
@@ -77,7 +80,7 @@ static trns_free_func compute_trns_free;
 /* COMPUTE. */
 
 int
-cmd_compute (struct dataset *ds)
+cmd_compute (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   struct lvalue *lvalue = NULL;
@@ -85,13 +88,13 @@ cmd_compute (struct dataset *ds)
 
   compute = compute_trns_create ();
 
-  lvalue = lvalue_parse (ds);
+  lvalue = lvalue_parse (lexer, ds);
   if (lvalue == NULL)
     goto fail;
 
-  if (!lex_force_match ('='))
+  if (!lex_force_match (lexer, '='))
     goto fail;
-  compute->rvalue = parse_rvalue (lvalue, ds);
+  compute->rvalue = parse_rvalue (lexer, lvalue, ds);
   if (compute->rvalue == NULL)
     goto fail;
 
@@ -100,7 +103,7 @@ cmd_compute (struct dataset *ds)
 
   lvalue_finalize (lvalue, compute, dict);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 
  fail:
   lvalue_destroy (lvalue);
@@ -213,7 +216,7 @@ compute_str_vec (void *compute_, struct ccase *c, casenumber case_num)
 /* IF. */
 
 int
-cmd_if (struct dataset *ds)
+cmd_if (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
   struct compute_trns *compute = NULL;
@@ -222,19 +225,19 @@ cmd_if (struct dataset *ds)
   compute = compute_trns_create ();
 
   /* Test expression. */
-  compute->test = expr_parse (ds, EXPR_BOOLEAN);
+  compute->test = expr_parse (lexer, ds, EXPR_BOOLEAN);
   if (compute->test == NULL)
     goto fail;
 
   /* Lvalue variable. */
-  lvalue = lvalue_parse (ds);
+  lvalue = lvalue_parse (lexer, ds);
   if (lvalue == NULL)
     goto fail;
 
   /* Rvalue expression. */
-  if (!lex_force_match ('='))
+  if (!lex_force_match (lexer, '='))
     goto fail;
-  compute->rvalue = parse_rvalue (lvalue, ds);
+  compute->rvalue = parse_rvalue (lexer, lvalue, ds);
   if (compute->rvalue == NULL)
     goto fail;
 
@@ -243,7 +246,7 @@ cmd_if (struct dataset *ds)
 
   lvalue_finalize (lvalue, compute, dict);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 
  fail:
   lvalue_destroy (lvalue);
@@ -267,12 +270,13 @@ get_proc_func (const struct lvalue *lvalue, const struct dictionary *dict)
 /* Parses and returns an rvalue expression of the same type as
    LVALUE, or a null pointer on failure. */
 static struct expression *
-parse_rvalue (const struct lvalue *lvalue, struct dataset *ds)
+parse_rvalue (struct lexer *lexer, 
+	      const struct lvalue *lvalue, struct dataset *ds)
 {
   const struct dictionary *dict = dataset_dict (ds);
   bool is_numeric = lvalue_get_type (lvalue, dict) == NUMERIC;
 
-  return expr_parse (ds, is_numeric ? EXPR_NUMBER : EXPR_STRING);
+  return expr_parse (lexer, ds, is_numeric ? EXPR_NUMBER : EXPR_STRING);
 }
 
 /* Returns a new struct compute_trns after initializing its fields. */
@@ -315,7 +319,7 @@ struct lvalue
 /* Parses the target variable or vector element into a new
    `struct lvalue', which is returned. */
 static struct lvalue *
-lvalue_parse (struct dataset *ds) 
+lvalue_parse (struct lexer *lexer, struct dataset *ds) 
 {
   struct lvalue *lvalue;
 
@@ -324,34 +328,34 @@ lvalue_parse (struct dataset *ds)
   lvalue->vector = NULL;
   lvalue->element = NULL;
 
-  if (!lex_force_id ())
+  if (!lex_force_id (lexer))
     goto lossage;
   
-  if (lex_look_ahead () == '(')
+  if (lex_look_ahead (lexer) == '(')
     {
       /* Vector. */
-      lvalue->vector = dict_lookup_vector (dataset_dict (ds), tokid);
+      lvalue->vector = dict_lookup_vector (dataset_dict (ds), lex_tokid (lexer));
       if (lvalue->vector == NULL)
 	{
-	  msg (SE, _("There is no vector named %s."), tokid);
+	  msg (SE, _("There is no vector named %s."), lex_tokid (lexer));
           goto lossage;
 	}
 
       /* Vector element. */
-      lex_get ();
-      if (!lex_force_match ('('))
+      lex_get (lexer);
+      if (!lex_force_match (lexer, '('))
 	goto lossage;
-      lvalue->element = expr_parse (ds, EXPR_NUMBER);
+      lvalue->element = expr_parse (lexer, ds, EXPR_NUMBER);
       if (lvalue->element == NULL)
         goto lossage;
-      if (!lex_force_match (')'))
+      if (!lex_force_match (lexer, ')'))
         goto lossage;
     }
   else
     {
       /* Variable name. */
-      str_copy_trunc (lvalue->var_name, sizeof lvalue->var_name, tokid);
-      lex_get ();
+      str_copy_trunc (lvalue->var_name, sizeof lvalue->var_name, lex_tokid (lexer));
+      lex_get (lexer);
     }
   return lvalue;
 
diff --git a/src/language/xforms/count.c b/src/language/xforms/count.c
index c392382a..371f2ecb 100644
--- a/src/language/xforms/count.c
+++ b/src/language/xforms/count.c
@@ -93,11 +93,11 @@ struct count_trns
 static trns_proc_func count_trns_proc;
 static trns_free_func count_trns_free;
 
-static bool parse_numeric_criteria (struct pool *, struct criteria *);
-static bool parse_string_criteria (struct pool *, struct criteria *);
+static bool parse_numeric_criteria (struct lexer *, struct pool *, struct criteria *);
+static bool parse_string_criteria (struct lexer *, struct pool *, struct criteria *);
 
 int
-cmd_count (struct dataset *ds)
+cmd_count (struct lexer *lexer, struct dataset *ds)
 {
   struct dst_var *dv;           /* Destination var being parsed. */
   struct count_trns *trns;      /* Transformation. */
@@ -115,9 +115,9 @@ cmd_count (struct dataset *ds)
       dv->crit = NULL;
 
       /* Get destination variable, or at least its name. */
-      if (!lex_force_id ())
+      if (!lex_force_id (lexer))
 	goto fail;
-      dv->var = dict_lookup_var (dataset_dict (ds), tokid);
+      dv->var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
       if (dv->var != NULL)
         {
           if (dv->var->type == ALPHA)
@@ -127,10 +127,10 @@ cmd_count (struct dataset *ds)
             }
         }
       else
-        dv->name = pool_strdup (trns->pool, tokid);
+        dv->name = pool_strdup (trns->pool, lex_tokid (lexer));
 
-      lex_get ();
-      if (!lex_force_match ('='))
+      lex_get (lexer);
+      if (!lex_force_match (lexer, '='))
 	goto fail;
 
       crit = dv->crit = pool_alloc (trns->pool, sizeof *crit);
@@ -140,32 +140,32 @@ cmd_count (struct dataset *ds)
           
 	  crit->next = NULL;
 	  crit->vars = NULL;
-	  if (!parse_variables (dataset_dict (ds), &crit->vars, &crit->var_cnt,
+	  if (!parse_variables (lexer, dataset_dict (ds), &crit->vars, &crit->var_cnt,
                                 PV_DUPLICATE | PV_SAME_TYPE))
 	    goto fail;
           pool_register (trns->pool, free, crit->vars);
 
-	  if (!lex_force_match ('('))
+	  if (!lex_force_match (lexer, '('))
 	    goto fail;
 
           crit->value_cnt = 0;
           if (crit->vars[0]->type == NUMERIC)
-            ok = parse_numeric_criteria (trns->pool, crit);
+            ok = parse_numeric_criteria (lexer, trns->pool, crit);
           else
-            ok = parse_string_criteria (trns->pool, crit);
+            ok = parse_string_criteria (lexer, trns->pool, crit);
 	  if (!ok)
 	    goto fail;
 
-	  if (token == '/' || token == '.')
+	  if (lex_token (lexer) == '/' || lex_token (lexer) == '.')
 	    break;
 
 	  crit = crit->next = pool_alloc (trns->pool, sizeof *crit);
 	}
 
-      if (token == '.')
+      if (lex_token (lexer) == '.')
 	break;
 
-      if (!lex_force_match ('/'))
+      if (!lex_force_match (lexer, '/'))
 	goto fail;
       dv = dv->next = pool_alloc (trns->pool, sizeof *dv);
     }
@@ -192,7 +192,7 @@ fail:
 
 /* Parses a set of numeric criterion values.  Returns success. */
 static bool
-parse_numeric_criteria (struct pool *pool, struct criteria *crit)
+parse_numeric_criteria (struct lexer *lexer, struct pool *pool, struct criteria *crit)
 {
   size_t allocated = 0;
 
@@ -203,11 +203,11 @@ parse_numeric_criteria (struct pool *pool, struct criteria *crit)
     {
       double low, high;
       
-      if (lex_match_id ("SYSMIS"))
+      if (lex_match_id (lexer, "SYSMIS"))
         crit->count_system_missing = true;
-      else if (lex_match_id ("MISSING"))
+      else if (lex_match_id (lexer, "MISSING"))
 	crit->count_user_missing = true;
-      else if (parse_num_range (&low, &high, NULL)) 
+      else if (parse_num_range (lexer, &low, &high, NULL)) 
         {
           struct num_value *cur;
 
@@ -223,8 +223,8 @@ parse_numeric_criteria (struct pool *pool, struct criteria *crit)
       else
         return false;
 
-      lex_match (',');
-      if (lex_match (')'))
+      lex_match (lexer, ',');
+      if (lex_match (lexer, ')'))
 	break;
     }
   return true;
@@ -232,7 +232,7 @@ parse_numeric_criteria (struct pool *pool, struct criteria *crit)
 
 /* Parses a set of string criteria values.  Returns success. */
 static bool
-parse_string_criteria (struct pool *pool, struct criteria *crit)
+parse_string_criteria (struct lexer *lexer, struct pool *pool, struct criteria *crit)
 {
   int len = 0;
   size_t allocated = 0;
@@ -251,15 +251,15 @@ parse_string_criteria (struct pool *pool, struct criteria *crit)
                                            &allocated,
                                            sizeof *crit->values.str);
 
-      if (!lex_force_string ())
+      if (!lex_force_string (lexer))
 	return false;
       cur = &crit->values.str[crit->value_cnt++];
       *cur = pool_alloc (pool, len + 1);
-      str_copy_rpad (*cur, len + 1, ds_cstr (&tokstr));
-      lex_get ();
+      str_copy_rpad (*cur, len + 1, ds_cstr (lex_tokstr (lexer)));
+      lex_get (lexer);
 
-      lex_match (',');
-      if (lex_match (')'))
+      lex_match (lexer, ',');
+      if (lex_match (lexer, ')'))
 	break;
     }
 
diff --git a/src/language/xforms/fail.c b/src/language/xforms/fail.c
index 2b1e7876..fdb2eb19 100644
--- a/src/language/xforms/fail.c
+++ b/src/language/xforms/fail.c
@@ -42,10 +42,10 @@ trns_fail (void *x UNUSED, struct ccase *c UNUSED,
 
 
 int
-cmd_debug_xform_fail (struct dataset *ds)
+cmd_debug_xform_fail (struct lexer *lexer, struct dataset *ds)
 {
 
   add_transformation (ds, trns_fail, NULL, NULL);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/language/xforms/recode.c b/src/language/xforms/recode.c
index 6af3e5d1..7fc187a7 100644
--- a/src/language/xforms/recode.c
+++ b/src/language/xforms/recode.c
@@ -108,21 +108,21 @@ struct recode_trns
     size_t map_cnt;             /* Number of mappings. */
   };
 
-static bool parse_src_vars (struct recode_trns *, const struct dictionary *dict);
-static bool parse_mappings (struct recode_trns *);
-static bool parse_dst_vars (struct recode_trns *, const struct dictionary *dict);
+static bool parse_src_vars (struct lexer *, struct recode_trns *, const struct dictionary *dict);
+static bool parse_mappings (struct lexer *, struct recode_trns *);
+static bool parse_dst_vars (struct lexer *, struct recode_trns *, const struct dictionary *dict);
 
 static void add_mapping (struct recode_trns *,
                          size_t *map_allocated, const struct map_in *);
 
-static bool parse_map_in (struct map_in *, struct pool *,
+static bool parse_map_in (struct lexer *lexer, struct map_in *, struct pool *,
                           enum var_type src_type, size_t max_src_width);
 static void set_map_in_generic (struct map_in *, enum map_in_type);
 static void set_map_in_num (struct map_in *, enum map_in_type, double, double);
 static void set_map_in_str (struct map_in *, struct pool *,
                             const struct string *, size_t width);
 
-static bool parse_map_out (struct pool *, struct map_out *);
+static bool parse_map_out (struct lexer *lexer, struct pool *, struct map_out *);
 static void set_map_out_num (struct map_out *, double);
 static void set_map_out_str (struct map_out *, struct pool *,
                              const struct string *);
@@ -137,7 +137,7 @@ static trns_free_func recode_trns_free;
 
 /* Parses the RECODE transformation. */
 int
-cmd_recode (struct dataset *ds)
+cmd_recode (struct lexer *lexer, struct dataset *ds)
 {
   do
     {
@@ -147,9 +147,9 @@ cmd_recode (struct dataset *ds)
       /* Parse source variable names,
          then input to output mappings,
          then destintation variable names. */
-      if (!parse_src_vars (trns, dataset_dict (ds) )
-          || !parse_mappings (trns)
-          || !parse_dst_vars (trns, dataset_dict (ds)))
+      if (!parse_src_vars (lexer, trns, dataset_dict (ds) )
+          || !parse_mappings (lexer, trns)
+          || !parse_dst_vars (lexer, trns, dataset_dict (ds)))
         {
           recode_trns_free (trns);
           return CMD_FAILURE;
@@ -170,18 +170,19 @@ cmd_recode (struct dataset *ds)
       add_transformation (ds, 
 			  recode_trns_proc, recode_trns_free, trns);
     }
-  while (lex_match ('/'));
+  while (lex_match (lexer, '/'));
   
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Parses a set of variables to recode into TRNS->src_vars and
    TRNS->var_cnt.  Sets TRNS->src_type.  Returns true if
    successful, false on parse error. */
 static bool
-parse_src_vars (struct recode_trns *trns, const struct dictionary *dict) 
+parse_src_vars (struct lexer *lexer, 
+		struct recode_trns *trns, const struct dictionary *dict) 
 {
-  if (!parse_variables (dict, &trns->src_vars, &trns->var_cnt,
+  if (!parse_variables (lexer, dict, &trns->src_vars, &trns->var_cnt,
                         PV_SAME_TYPE))
     return false;
   pool_register (trns->pool, free, trns->src_vars);
@@ -193,7 +194,7 @@ parse_src_vars (struct recode_trns *trns, const struct dictionary *dict)
    into TRNS->mappings and TRNS->map_cnt.  Sets TRNS->dst_type.
    Returns true if successful, false on parse error. */
 static bool
-parse_mappings (struct recode_trns *trns) 
+parse_mappings (struct lexer *lexer, struct recode_trns *trns) 
 {
   size_t max_src_width;
   size_t map_allocated;
@@ -214,13 +215,13 @@ parse_mappings (struct recode_trns *trns)
   trns->map_cnt = 0;
   map_allocated = 0;
   have_dst_type = false;
-  if (!lex_force_match ('('))
+  if (!lex_force_match (lexer, '('))
     return false;
   do
     {
       enum var_type dst_type;
 
-      if (!lex_match_id ("CONVERT")) 
+      if (!lex_match_id (lexer, "CONVERT")) 
         {
           struct map_out out;
           size_t first_map_idx;
@@ -232,15 +233,15 @@ parse_mappings (struct recode_trns *trns)
           do
             {
               struct map_in in;
-              if (!parse_map_in (&in, trns->pool,
+              if (!parse_map_in (lexer, &in, trns->pool,
                                  trns->src_type, max_src_width))
                 return false;
               add_mapping (trns, &map_allocated, &in);
-              lex_match (',');
+              lex_match (lexer, ',');
             }
-          while (!lex_match ('='));
+          while (!lex_match (lexer, '='));
 
-          if (!parse_map_out (trns->pool, &out))
+          if (!parse_map_out (lexer, trns->pool, &out))
             return false;
           dst_type = out.width == 0 ? NUMERIC : ALPHA;
           if (have_dst_type && dst_type != trns->dst_type)
@@ -273,10 +274,10 @@ parse_mappings (struct recode_trns *trns)
       trns->dst_type = dst_type;
       have_dst_type = true;
 
-      if (!lex_force_match (')'))
+      if (!lex_force_match (lexer, ')'))
         return false; 
     }
-  while (lex_match ('('));
+  while (lex_match (lexer, '('));
 
   return true;
 }
@@ -287,31 +288,31 @@ parse_mappings (struct recode_trns *trns)
    be provided in MAX_SRC_WIDTH.  Returns true if successful,
    false on parse error. */
 static bool
-parse_map_in (struct map_in *in, struct pool *pool,
+parse_map_in (struct lexer *lexer, struct map_in *in, struct pool *pool,
               enum var_type src_type, size_t max_src_width)
 {
-  if (lex_match_id ("ELSE"))
+  if (lex_match_id (lexer, "ELSE"))
     set_map_in_generic (in, MAP_ELSE);
   else if (src_type == NUMERIC)
     {
-      if (lex_match_id ("MISSING"))
+      if (lex_match_id (lexer, "MISSING"))
         set_map_in_generic (in, MAP_MISSING);
-      else if (lex_match_id ("SYSMIS"))
+      else if (lex_match_id (lexer, "SYSMIS"))
         set_map_in_generic (in, MAP_SYSMIS);
       else 
         {
           double x, y;
-          if (!parse_num_range (&x, &y, NULL))
+          if (!parse_num_range (lexer, &x, &y, NULL))
             return false;
           set_map_in_num (in, x == y ? MAP_SINGLE : MAP_RANGE, x, y);
         }
     }
   else
     {
-      if (!lex_force_string ())
+      if (!lex_force_string (lexer))
         return false;
-      set_map_in_str (in, pool, &tokstr, max_src_width);
-      lex_get ();
+      set_map_in_str (in, pool, lex_tokstr (lexer), max_src_width);
+      lex_get (lexer);
     }
 
   return true;
@@ -365,25 +366,25 @@ set_map_in_str (struct map_in *in, struct pool *pool,
 /* Parses a mapping output value into OUT, allocating memory from
    POOL.  Returns true if successful, false on parse error. */
 static bool
-parse_map_out (struct pool *pool, struct map_out *out)
+parse_map_out (struct lexer *lexer, struct pool *pool, struct map_out *out)
 {
-  if (lex_is_number ())
+  if (lex_is_number (lexer))
     {
-      set_map_out_num (out, lex_number ());
-      lex_get ();
+      set_map_out_num (out, lex_number (lexer));
+      lex_get (lexer);
     }
-  else if (lex_match_id ("SYSMIS"))
+  else if (lex_match_id (lexer, "SYSMIS"))
     set_map_out_num (out, SYSMIS);
-  else if (token == T_STRING)
+  else if (lex_token (lexer) == T_STRING)
     {
-      set_map_out_str (out, pool, &tokstr);
-      lex_get ();
+      set_map_out_str (out, pool, lex_tokstr (lexer));
+      lex_get (lexer);
     }
-  else if (lex_match_id ("COPY"))
+  else if (lex_match_id (lexer, "COPY"))
     out->copy_input = true;
   else 
     {
-      lex_error (_("expecting output value"));
+      lex_error (lexer, _("expecting output value"));
       return false;
     }
   return true; 
@@ -415,16 +416,17 @@ set_map_out_str (struct map_out *out, struct pool *pool,
 /* Parses a set of target variables into TRNS->dst_vars and
    TRNS->dst_names. */
 static bool
-parse_dst_vars (struct recode_trns *trns, const struct dictionary *dict) 
+parse_dst_vars (struct lexer *lexer, struct recode_trns *trns, 
+		const struct dictionary *dict) 
 {
   size_t i;
   
-  if (lex_match_id ("INTO"))
+  if (lex_match_id (lexer, "INTO"))
     {
       size_t name_cnt;
       size_t i;
 
-      if (!parse_mixed_vars_pool (dict, trns->pool, 
+      if (!parse_mixed_vars_pool (lexer, dict, trns->pool, 
 				  &trns->dst_names, &name_cnt,
                                   PV_NONE))
         return false;
diff --git a/src/language/xforms/sample.c b/src/language/xforms/sample.c
index 83fae887..4cf66c20 100644
--- a/src/language/xforms/sample.c
+++ b/src/language/xforms/sample.c
@@ -57,7 +57,7 @@ static trns_proc_func sample_trns_proc;
 static trns_free_func sample_trns_free;
 
 int
-cmd_sample (struct dataset *ds)
+cmd_sample (struct lexer *lexer, struct dataset *ds)
 {
   struct sample_trns *trns;
 
@@ -65,34 +65,34 @@ cmd_sample (struct dataset *ds)
   int a, b;
   unsigned frac;
 
-  if (!lex_force_num ())
+  if (!lex_force_num (lexer))
     return CMD_FAILURE;
-  if (!lex_is_integer ())
+  if (!lex_is_integer (lexer))
     {
       unsigned long min = gsl_rng_min (get_rng ());
       unsigned long max = gsl_rng_max (get_rng ());
 
       type = TYPE_FRACTION;
-      if (tokval <= 0 || tokval >= 1)
+      if (lex_tokval (lexer) <= 0 || lex_tokval (lexer) >= 1)
 	{
 	  msg (SE, _("The sampling factor must be between 0 and 1 "
 		     "exclusive."));
 	  return CMD_FAILURE;
 	}
 	  
-      frac = tokval * (max - min) + min;
+      frac = lex_tokval (lexer) * (max - min) + min;
       a = b = 0;
     }
   else
     {
       type = TYPE_A_FROM_B;
-      a = lex_integer ();
-      lex_get ();
-      if (!lex_force_match_id ("FROM"))
+      a = lex_integer (lexer);
+      lex_get (lexer);
+      if (!lex_force_match_id (lexer, "FROM"))
 	return CMD_FAILURE;
-      if (!lex_force_int ())
+      if (!lex_force_int (lexer))
 	return CMD_FAILURE;
-      b = lex_integer ();
+      b = lex_integer (lexer);
       if (a >= b)
 	{
 	  msg (SE, _("Cannot sample %d observations from a population of "
@@ -103,7 +103,7 @@ cmd_sample (struct dataset *ds)
       
       frac = 0;
     }
-  lex_get ();
+  lex_get (lexer);
 
   trns = xmalloc (sizeof *trns);
   trns->type = type;
@@ -113,7 +113,7 @@ cmd_sample (struct dataset *ds)
   trns->frac = frac;
   add_transformation (ds, sample_trns_proc, sample_trns_free, trns);
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
 
 /* Executes a SAMPLE transformation. */
diff --git a/src/language/xforms/select-if.c b/src/language/xforms/select-if.c
index 66c8f9e4..2543760b 100644
--- a/src/language/xforms/select-if.c
+++ b/src/language/xforms/select-if.c
@@ -47,19 +47,19 @@ static trns_free_func select_if_free;
 
 /* Parses the SELECT IF transformation. */
 int
-cmd_select_if (struct dataset *ds)
+cmd_select_if (struct lexer *lexer, struct dataset *ds)
 {
   struct expression *e;
   struct select_if_trns *t;
 
-  e = expr_parse (ds, EXPR_BOOLEAN);
+  e = expr_parse (lexer, ds, EXPR_BOOLEAN);
   if (!e)
     return CMD_CASCADING_FAILURE;
 
-  if (token != '.')
+  if (lex_token (lexer) != '.')
     {
       expr_free (e);
-      lex_error (_("expecting end of command"));
+      lex_error (lexer, _("expecting end of command"));
       return CMD_CASCADING_FAILURE;
     }
 
@@ -92,12 +92,12 @@ select_if_free (void *t_)
 
 /* Parses the FILTER command. */
 int
-cmd_filter (struct dataset *ds)
+cmd_filter (struct lexer *lexer, struct dataset *ds)
 {
   struct dictionary *dict = dataset_dict (ds);
-  if (lex_match_id ("OFF"))
+  if (lex_match_id (lexer, "OFF"))
     dict_set_filter (dataset_dict (ds), NULL);
-  else if (token == '.') 
+  else if (lex_token (lexer) == '.') 
     {
       msg (SW, _("Syntax error expecting OFF or BY.  "
                  "Turning off case filtering."));
@@ -107,8 +107,8 @@ cmd_filter (struct dataset *ds)
     {
       struct variable *v;
 
-      lex_match (T_BY);
-      v = parse_variable (dict);
+      lex_match (lexer, T_BY);
+      v = parse_variable (lexer, dict);
       if (!v)
 	return CMD_FAILURE;
 
@@ -127,5 +127,5 @@ cmd_filter (struct dataset *ds)
       dict_set_filter (dict, v);
     }
 
-  return lex_end_of_command ();
+  return lex_end_of_command (lexer);
 }
diff --git a/src/ui/terminal/main.c b/src/ui/terminal/main.c
index b52fd941..0613476b 100644
--- a/src/ui/terminal/main.c
+++ b/src/ui/terminal/main.c
@@ -72,7 +72,7 @@ void bug_handler(int sig);
 void interrupt_handler(int sig);
 static struct dataset * the_dataset = NULL;
 
-
+static struct lexer *the_lexer;
 
 /* Program entry point. */
 int
@@ -102,11 +102,11 @@ main (int argc, char **argv)
     {
       msg_ui_init ();
       outp_read_devices ();
-      lex_init (do_read_line);
+      the_lexer = lex_create (do_read_line);
 
       for (;;)
         {
-          int result = cmd_parse (the_dataset, 
+          int result = cmd_parse (the_lexer, the_dataset, 
 				  proc_has_source (the_dataset)
                                   ? CMD_STATE_DATA : CMD_STATE_INITIAL);
           if (result == CMD_EOF || result == CMD_FINISH)
@@ -188,7 +188,7 @@ terminate (bool success)
       random_done ();
       settings_done ();
       fh_done ();
-      lex_done ();
+      lex_destroy (the_lexer);
       getl_uninitialize ();
       readln_uninitialize ();