X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=c08c428c4a4bf696bf2211318166b17e225bb77a;hb=740218f508466f16a79939cd97027b99f589d682;hp=5b24522f5c9097172b2852c567d21a120dfe5517;hpb=ff46735d4c81b49fa17c8bd26f3f5dd7bde9a15d;p=pspp-builds.git

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 5b24522f..c08c428c 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -15,22 +15,24 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
 
 #include <config.h>
+
 #include "lexer.h"
-#include <libpspp/message.h>
-#include <ctype.h>
+#include <c-ctype.h>
+#include <c-strtod.h>
 #include <errno.h>
 #include <limits.h>
 #include <math.h>
 #include <stdarg.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include <libpspp/assertion.h>
 #include <language/command.h>
-#include <libpspp/message.h>
 #include <data/settings.h>
+#include <libpspp/assertion.h>
 #include <libpspp/getl.h>
+#include <libpspp/message.h>
 #include <libpspp/str.h>
 #include <output/journal.h>
+#include <output/text-item.h>
 
 #include "xalloc.h"
 
@@ -38,11 +40,6 @@
 #define _(msgid) gettext (msgid)
 #define N_(msgid) msgid
 
-
-#define DUMP_TOKENS 0
-
-
-
 struct lexer
 {
   struct string line_buffer;
@@ -81,10 +78,6 @@ enum string_type
   };
 
 static int parse_string (struct lexer *, enum string_type);
-
-#if DUMP_TOKENS
-static void dump_token (struct lexer *);
-#endif
 
 /* Initialization. */
 
@@ -174,20 +167,17 @@ lex_get (struct lexer *lexer)
 	  return;
 	}
 
-  /* If a token was pushed ahead, return it. */
-  if (lexer->put_token)
-    {
-      restore_token (lexer);
-#if DUMP_TOKENS
-	  dump_token (lexer);
-#endif
-      return;
-    }
+      /* If a token was pushed ahead, return it. */
+      if (lexer->put_token)
+        {
+          restore_token (lexer);
+          return;
+        }
 
-  for (;;)
-    {
-      /* Skip whitespace. */
-	  while (isspace ((unsigned char) *lexer->prog))
+      for (;;)
+        {
+          /* Skip whitespace. */
+	  while (c_isspace ((unsigned char) *lexer->prog))
 	    lexer->prog++;
 
 	  if (*lexer->prog)
@@ -196,28 +186,19 @@ lex_get (struct lexer *lexer)
 	  if (lexer->dot)
 	    {
 	      lexer->dot = 0;
-	      lexer->token = '.';
-#if DUMP_TOKENS
-	      dump_token (lexer);
-#endif
+	      lexer->token = T_ENDCMD;
 	      return;
 	    }
 	  else if (!lex_get_line (lexer))
 	    {
 	      lexer->prog = NULL;
 	      lexer->token = T_STOP;
-#if DUMP_TOKENS
-	      dump_token (lexer);
-#endif
 	      return;
 	    }
 
 	  if (lexer->put_token)
 	    {
               restore_token (lexer);
-#if DUMP_TOKENS
-	      dump_token (lexer);
-#endif
 	      return;
 	    }
 	}
@@ -234,22 +215,17 @@ lex_get (struct lexer *lexer)
 	  {
 	    char *tail;
 
-	    /* `-' can introduce a negative number, or it can be a
-	       token by itself.  If it is not followed by a digit or a
-	       decimal point, it is definitely not a number.
-	       Otherwise, it might be either, but most of the time we
-	       want it as a number.  When the syntax calls for a `-'
-	       token, lex_negative_to_dash() must be used to break
-	       negative numbers into two tokens. */
+	    /* `-' can introduce a negative number, or it can be a token by
+	       itself. */
 	    if (*lexer->prog == '-')
 	      {
-		ds_put_char (&lexer->tokstr, *lexer->prog++);
-		while (isspace ((unsigned char) *lexer->prog))
+		ds_put_byte (&lexer->tokstr, *lexer->prog++);
+		while (c_isspace ((unsigned char) *lexer->prog))
 		  lexer->prog++;
 
-		if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
+		if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.')
 		  {
-		    lexer->token = '-';
+		    lexer->token = T_DASH;
 		    break;
 		  }
                 lexer->token = T_NEG_NUM;
@@ -258,25 +234,25 @@ lex_get (struct lexer *lexer)
               lexer->token = T_POS_NUM;
 
 	    /* Parse the number, copying it into tokstr. */
-	    while (isdigit ((unsigned char) *lexer->prog))
-	      ds_put_char (&lexer->tokstr, *lexer->prog++);
+	    while (c_isdigit ((unsigned char) *lexer->prog))
+	      ds_put_byte (&lexer->tokstr, *lexer->prog++);
 	    if (*lexer->prog == '.')
 	      {
-		ds_put_char (&lexer->tokstr, *lexer->prog++);
-		while (isdigit ((unsigned char) *lexer->prog))
-		  ds_put_char (&lexer->tokstr, *lexer->prog++);
+		ds_put_byte (&lexer->tokstr, *lexer->prog++);
+		while (c_isdigit ((unsigned char) *lexer->prog))
+		  ds_put_byte (&lexer->tokstr, *lexer->prog++);
 	      }
 	    if (*lexer->prog == 'e' || *lexer->prog == 'E')
 	      {
-		ds_put_char (&lexer->tokstr, *lexer->prog++);
+		ds_put_byte (&lexer->tokstr, *lexer->prog++);
 		if (*lexer->prog == '+' || *lexer->prog == '-')
-		  ds_put_char (&lexer->tokstr, *lexer->prog++);
-		while (isdigit ((unsigned char) *lexer->prog))
-		  ds_put_char (&lexer->tokstr, *lexer->prog++);
+		  ds_put_byte (&lexer->tokstr, *lexer->prog++);
+		while (c_isdigit ((unsigned char) *lexer->prog))
+		  ds_put_byte (&lexer->tokstr, *lexer->prog++);
 	      }
 
 	    /* Parse as floating point. */
-	    lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail);
+	    lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail);
 	    if (*tail)
 	      {
 		msg (SE, _("%s does not form a valid number."),
@@ -284,7 +260,7 @@ lex_get (struct lexer *lexer)
 		lexer->tokval = 0.0;
 
 		ds_clear (&lexer->tokstr);
-		ds_put_char (&lexer->tokstr, '0');
+		ds_put_byte (&lexer->tokstr, '0');
 	      }
 
 	    break;
@@ -294,9 +270,45 @@ lex_get (struct lexer *lexer)
 	  lexer->token = parse_string (lexer, CHARACTER_STRING);
 	  break;
 
-	case '(': case ')': case ',': case '=': case '+': case '/':
-	  lexer->token = *lexer->prog++;
-	  break;
+        case '+':
+          lexer->token = T_PLUS;
+          lexer->prog++;
+          break;
+
+        case '/':
+          lexer->token = T_SLASH;
+          lexer->prog++;
+          break;
+
+        case '=':
+          lexer->token = T_EQUALS;
+          lexer->prog++;
+          break;
+
+	case '(':
+          lexer->token = T_LPAREN;
+          lexer->prog++;
+          break;
+
+	case ')':
+          lexer->token = T_RPAREN;
+          lexer->prog++;
+          break;
+
+	case '[':
+          lexer->token = T_LBRACK;
+          lexer->prog++;
+          break;
+
+	case ']':
+          lexer->token = T_RBRACK;
+          lexer->prog++;
+          break;
+
+        case ',':
+          lexer->token = T_COMMA;
+          lexer->prog++;
+          break;
 
 	case '*':
 	  if (*++lexer->prog == '*')
@@ -305,7 +317,7 @@ lex_get (struct lexer *lexer)
 	      lexer->token = T_EXP;
 	    }
 	  else
-	    lexer->token = '*';
+	    lexer->token = T_ASTERISK;
 	  break;
 
 	case '<':
@@ -382,19 +394,15 @@ lex_get (struct lexer *lexer)
             }
           else
             {
-              if (isgraph ((unsigned char) *lexer->prog))
-                msg (SE, _("Bad character in input: `%c'."), *lexer->prog++);
-              else
-                msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++);
+              unsigned char c = *lexer->prog++;
+              char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c);
+              msg (SE, _("Bad character in input: `%s'."), c_name);
+              free (c_name);
               continue;
             }
         }
       break;
     }
-
-#if DUMP_TOKENS
-  dump_token (lexer);
-#endif
 }
 
 /* Parses an identifier at the current position into tokid and
@@ -437,31 +445,34 @@ lex_sbc_missing (struct lexer *lexer, const char *sbc)
 void
 lex_error (struct lexer *lexer, const char *message, ...)
 {
-  char *token_rep;
-  char where[128];
+  struct string s;
+
+  ds_init_empty (&s);
 
-  token_rep = lex_token_representation (lexer);
   if (lexer->token == T_STOP)
-    strcpy (where, "end of file");
-  else if (lexer->token == '.')
-    strcpy (where, "end of command");
+    ds_put_cstr (&s, _("Syntax error at end of file"));
+  else if (lexer->token == T_ENDCMD)
+    ds_put_cstr (&s, _("Syntax error at end of command"));
   else
-    snprintf (where, sizeof where, "`%s'", token_rep);
-  free (token_rep);
+    {
+      char *token_rep = lex_token_representation (lexer);
+      ds_put_format (&s, _("Syntax error at `%s'"), token_rep);
+      free (token_rep);
+    }
 
   if (message)
     {
-      char buf[1024];
       va_list args;
 
+      ds_put_cstr (&s, ": ");
+
       va_start (args, message);
-      vsnprintf (buf, 1024, message, args);
+      ds_put_vformat (&s, message, args);
       va_end (args);
-
-      msg (SE, _("Syntax error %s at %s."), buf, where);
     }
-  else
-    msg (SE, _("Syntax error at %s."), where);
+
+  msg (SE, "%s.", ds_cstr (&s));
+  ds_destroy (&s);
 }
 
 /* Checks that we're at end of command.
@@ -471,7 +482,7 @@ lex_error (struct lexer *lexer, const char *message, ...)
 int
 lex_end_of_command (struct lexer *lexer)
 {
-  if (lexer->token != '.')
+  if (lexer->token != T_ENDCMD)
     {
       lex_error (lexer, _("expecting end of command"));
       return CMD_FAILURE;
@@ -531,7 +542,7 @@ lex_integer (struct lexer *lexer)
 /* If TOK is the current token, skips it and returns true
    Otherwise, returns false. */
 bool
-lex_match (struct lexer *lexer, int t)
+lex_match (struct lexer *lexer, enum token_type t)
 {
   if (lexer->token == t)
     {
@@ -603,7 +614,7 @@ lex_force_match_id (struct lexer *lexer, const char *s)
 /* If the current token is T, skips the token.  Otherwise, reports an
    error and returns from the current function with return value false. */
 bool
-lex_force_match (struct lexer *lexer, int t)
+lex_force_match (struct lexer *lexer, enum token_type t)
 {
   if (lexer->token == t)
     {
@@ -622,7 +633,7 @@ lex_force_match (struct lexer *lexer, int t)
 bool
 lex_force_string (struct lexer *lexer)
 {
-  if (lexer->token == T_STRING)
+  if (lex_is_string (lexer))
     return true;
   else
     {
@@ -671,13 +682,8 @@ lex_force_id (struct lexer *lexer)
 
 /* Weird token functions. */
 
-/* Returns the first character of the next token, except that if the
-   next token is not an identifier, the character returned will not be
-   a character that can begin an identifier.  Specifically, the
-   hexstring lead-in X' causes lookahead() to return '.  Note that an
-   alphanumeric return value doesn't guarantee an ID token, it could
-   also be a reserved-word token. */
-int
+/* Returns the likely type of the next token, or 0 if it's hard to tell. */
+enum token_type
 lex_look_ahead (struct lexer *lexer)
 {
   if (lexer->put_token)
@@ -690,13 +696,13 @@ lex_look_ahead (struct lexer *lexer)
 
       for (;;)
 	{
-	  while (isspace ((unsigned char) *lexer->prog))
+	  while (c_isspace ((unsigned char) *lexer->prog))
 	    lexer->prog++;
 	  if (*lexer->prog)
 	    break;
 
 	  if (lexer->dot)
-	    return '.';
+	    return T_ENDCMD;
 	  else if (!lex_get_line (lexer))
             return 0;
 
@@ -704,20 +710,80 @@ lex_look_ahead (struct lexer *lexer)
 	    return lexer->put_token;
 	}
 
-      if ((toupper ((unsigned char) *lexer->prog) == 'X'
-	   || toupper ((unsigned char) *lexer->prog) == 'B'
-           || toupper ((unsigned char) *lexer->prog) == 'O')
-	  && (lexer->prog[1] == '\'' || lexer->prog[1] == '"'))
-	return '\'';
+      switch (toupper ((unsigned char) *lexer->prog))
+        {
+        case 'X': case 'B': case 'O':
+          if (lexer->prog[1] == '\'' || lexer->prog[1] == '"')
+            return T_STRING;
+          /* Fall through */
+
+	case '-':
+          return T_DASH;
+
+        case '.':
+	case '0': case '1': case '2': case '3': case '4':
+	case '5': case '6': case '7': case '8': case '9':
+          return T_POS_NUM;
+
+	case '\'': case '"':
+          return T_STRING;
+
+        case '+':
+          return T_PLUS;
+
+        case '/':
+          return T_SLASH;
+
+        case '=':
+          return T_EQUALS;
+
+	case '(':
+          return T_LPAREN;
+
+	case ')':
+          return T_RPAREN;
+
+	case '[':
+          return T_LBRACK;
+
+	case ']':
+          return T_RBRACK;
+
+        case ',':
+          return T_COMMA;
+
+	case '*':
+	  return lexer->prog[1] == '*' ? T_EXP : T_ASTERISK;
+
+	case '<':
+          return (lexer->prog[1] == '=' ? T_LE
+                  : lexer->prog[1] == '>' ? T_NE
+                  : T_LT);
+
+	case '>':
+          return lexer->prog[1] == '=' ? T_GE : T_GT;
+
+	case '~':
+          return lexer->prog[1] == '=' ? T_NE : T_NOT;
+
+	case '&':
+	  return T_AND;
+
+	case '|':
+	  return T_OR;
 
-      return *lexer->prog;
+        default:
+          if (lex_is_id1 (*lexer->prog))
+            return T_ID;
+          return 0;
+        }
     }
 }
 
 /* Makes the current token become the next token to be read; the
    current token is set to T. */
 void
-lex_put_back (struct lexer *lexer, int t)
+lex_put_back (struct lexer *lexer, enum token_type t)
 {
   save_token (lexer);
   lexer->token = t;
@@ -790,7 +856,7 @@ lex_discard_rest_of_command (struct lexer *lexer)
 {
   if (!getl_is_interactive (lexer->ss))
     {
-      while (lexer->token != T_STOP && lexer->token != '.')
+      while (lexer->token != T_STOP && lexer->token != T_ENDCMD)
 	lex_get (lexer);
     }
   else
@@ -865,22 +931,24 @@ lex_preprocess_line (struct string *line,
   if (syntax == GETL_BATCH)
     {
       int first = ds_first (line);
-      *line_starts_command = !isspace (first);
+      *line_starts_command = !c_isspace (first);
       if (first == '+' || first == '-')
         *ds_data (line) = ' ';
     }
 }
 
-/* Reads a line, without performing any preprocessing.
-   Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax
-   mode. */
+/* Reads a line, without performing any preprocessing. */
 bool
 lex_get_line_raw (struct lexer *lexer)
 {
   bool ok = getl_read_line (lexer->ss, &lexer->line_buffer);
-  enum syntax_mode mode = lex_current_syntax_mode (lexer);
-  journal_write (mode == GETL_BATCH, ds_cstr (&lexer->line_buffer));
-
+  if (ok)
+    {
+      const char *line = ds_cstr (&lexer->line_buffer);
+      text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line));
+    }
+  else
+    lexer->prog = NULL;
   return ok;
 }
 
@@ -893,17 +961,14 @@ lex_get_line (struct lexer *lexer)
   bool line_starts_command;
 
   if (!lex_get_line_raw (lexer))
-    {
-      lexer->prog = NULL;
-      return false;
-    }
+    return false;
 
   lex_preprocess_line (&lexer->line_buffer,
 		       lex_current_syntax_mode (lexer),
                        &line_starts_command, &lexer->dot);
 
   if (line_starts_command)
-    lexer->put_token = '.';
+    lexer->put_token = T_ENDCMD;
 
   lexer->prog = ds_cstr (&lexer->line_buffer);
   return true;
@@ -913,20 +978,96 @@ lex_get_line (struct lexer *lexer)
 
 /* Returns the name of a token. */
 const char *
-lex_token_name (int token)
+lex_token_name (enum token_type token)
 {
-  if (lex_is_keyword (token))
-    return lex_id_name (token);
-  else if (token < 256)
+  switch (token)
     {
-      static char t[256][2];
-      char *s = t[token];
-      s[0] = token;
-      s[1] = '\0';
-      return s;
+    case T_ID:
+    case T_POS_NUM:
+    case T_NEG_NUM:
+    case T_STRING:
+      NOT_REACHED ();
+
+    case T_STOP:
+      return "";
+
+    case T_ENDCMD:
+      return ".";
+
+    case T_PLUS:
+      return "+";
+
+    case T_DASH:
+      return "-";
+
+    case T_ASTERISK:
+      return "*";
+
+    case T_SLASH:
+      return "/";
+
+    case T_EQUALS:
+      return "=";
+
+    case T_LPAREN:
+      return "(";
+
+    case T_RPAREN:
+      return ")";
+
+    case T_LBRACK:
+      return "[";
+
+    case T_RBRACK:
+      return "]";
+
+    case T_COMMA:
+      return ",";
+
+    case T_AND:
+      return "AND";
+
+    case T_OR:
+      return "OR";
+
+    case T_NOT:
+      return "NOT";
+
+    case T_EQ:
+      return "EQ";
+
+    case T_GE:
+      return ">=";
+
+    case T_GT:
+      return ">";
+
+    case T_LE:
+      return "<=";
+
+    case T_LT:
+      return "<";
+
+    case T_NE:
+      return "~=";
+
+    case T_ALL:
+      return "ALL";
+
+    case T_BY:
+      return "BY";
+
+    case T_TO:
+      return "TO";
+
+    case T_WITH:
+      return "WITH";
+
+    case T_EXP:
+      return "**";
     }
-  else
-    NOT_REACHED ();
+
+  NOT_REACHED ();
 }
 
 /* Returns an ASCII representation of the current token as a
@@ -942,7 +1083,6 @@ lex_token_representation (struct lexer *lexer)
     case T_POS_NUM:
     case T_NEG_NUM:
       return ds_xstrdup (&lexer->tokstr);
-      break;
 
     case T_STRING:
       {
@@ -950,7 +1090,7 @@ lex_token_representation (struct lexer *lexer)
 	char *sp, *dp;
 
 	for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
-	  if (!isprint ((unsigned char) *sp))
+	  if (!c_isprint ((unsigned char) *sp))
 	    {
 	      hexstring = 1;
 	      break;
@@ -981,42 +1121,14 @@ lex_token_representation (struct lexer *lexer)
 
 	return token_rep;
       }
-    break;
-
-    case T_STOP:
-      token_rep = xmalloc (1);
-      *token_rep = '\0';
-      return token_rep;
-
-    case T_EXP:
-      return xstrdup ("**");
 
     default:
       return xstrdup (lex_token_name (lexer->token));
     }
-
-  NOT_REACHED ();
 }
 
 /* Really weird functions. */
 
-/* Most of the time, a `-' is a lead-in to a negative number.  But
-   sometimes it's actually part of the syntax.  If a dash can be part
-   of syntax then this function is called to rip it off of a
-   number. */
-void
-lex_negative_to_dash (struct lexer *lexer)
-{
-  if (lexer->token == T_NEG_NUM)
-    {
-      lexer->token = T_POS_NUM;
-      lexer->tokval = -lexer->tokval;
-      ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX));
-      save_token (lexer);
-      lexer->token = '-';
-    }
-}
-
 /* Skip a COMMENT command. */
 void
 lex_skip_comment (struct lexer *lexer)
@@ -1030,7 +1142,7 @@ lex_skip_comment (struct lexer *lexer)
           return;
         }
 
-      if (lexer->put_token == '.')
+      if (lexer->put_token == T_ENDCMD)
 	break;
 
       ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */
@@ -1155,7 +1267,7 @@ parse_string (struct lexer *lexer, enum string_type type)
 		break;
 	    }
 
-	  ds_put_char (&lexer->tokstr, *lexer->prog++);
+	  ds_put_byte (&lexer->tokstr, *lexer->prog++);
 	}
       lexer->prog++;
 
@@ -1164,7 +1276,7 @@ parse_string (struct lexer *lexer, enum string_type type)
 	break;
       for (;;)
 	{
-	  while (isspace ((unsigned char) *lexer->prog))
+	  while (c_isspace ((unsigned char) *lexer->prog))
 	    lexer->prog++;
 	  if (*lexer->prog)
 	    break;
@@ -1186,7 +1298,7 @@ parse_string (struct lexer *lexer, enum string_type type)
 	break;
       for (;;)
 	{
-	  while (isspace ((unsigned char) *lexer->prog))
+	  while (c_isspace ((unsigned char) *lexer->prog))
 	    lexer->prog++;
 	  if (*lexer->prog)
 	    break;
@@ -1215,73 +1327,12 @@ finish:
   if (type != CHARACTER_STRING)
     convert_numeric_string_to_char_string (lexer, type);
 
-  if (ds_length (&lexer->tokstr) > 255)
-    {
-      msg (SE, _("String exceeds 255 characters in length (%zu characters)."),
-	   ds_length (&lexer->tokstr));
-      ds_truncate (&lexer->tokstr, 255);
-    }
-
   return T_STRING;
 }
 
-#if DUMP_TOKENS
-/* Reads one token from the lexer and writes a textual representation
-   on stdout for debugging purposes. */
-static void
-dump_token (struct lexer *lexer)
-{
-  {
-    const char *curfn;
-    int curln;
-
-    curln = getl_source_location (lexer->ss);
-    curfn = getl_source_name (lexer->ss);
-    if (curfn)
-      fprintf (stderr, "%s:%d\t", curfn, curln);
-  }
-
-  switch (lexer->token)
-    {
-    case T_ID:
-      fprintf (stderr, "ID\t%s\n", lexer->tokid);
-      break;
-
-    case T_POS_NUM:
-    case T_NEG_NUM:
-      fprintf (stderr, "NUM\t%f\n", lexer->tokval);
-      break;
-
-    case T_STRING:
-      fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr));
-      break;
-
-    case T_STOP:
-      fprintf (stderr, "STOP\n");
-      break;
-
-    case T_EXP:
-      fprintf (stderr, "MISC\tEXP\"");
-      break;
-
-    case 0:
-      fprintf (stderr, "MISC\tEOF\n");
-      break;
-
-    default:
-      if (lex_is_keyword (lexer->token))
-	fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token));
-      else
-	fprintf (stderr, "PUNCT\t%c\n", lexer->token);
-      break;
-    }
-}
-#endif /* DUMP_TOKENS */
-
-
 /* Token Accessor Functions */
 
-int
+enum token_type
 lex_token (const struct lexer *lexer)
 {
   return lexer->token;
@@ -1318,12 +1369,12 @@ lex_match_hyphenated_word (struct lexer *lexer, const char *s)
     return lex_match_id (lexer, s);
   else if (lexer->token != T_ID
 	   || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
-	   || lex_look_ahead (lexer) != '-')
+	   || lex_look_ahead (lexer) != T_DASH)
     return false;
   else
     {
       lex_get (lexer);
-      lex_force_match (lexer, '-');
+      lex_force_match (lexer, T_DASH);
       lex_force_match_id (lexer, hyphen + 1);
       return true;
     }