lexer: Reimplement for better testability and internationalization.

[pspp-builds.git] / src / language / dictionary / value-labels.c

diff --git a/src/language/dictionary/value-labels.c b/src/language/dictionary/value-labels.c
index 5e3ea11f24654d0af73ff4cc1113123134bca00b..18e890ddb87f87a1b206facc4f771fcc69ae361b 100644 (file)
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2009 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2009, 2010, 2011 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,18 +19,19 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include <data/procedure.h>
-#include <data/value-labels.h>
-#include <data/variable.h>
-#include <language/command.h>
-#include <language/lexer/lexer.h>
-#include <language/lexer/value-parser.h>
-#include <language/lexer/variable-parser.h>
-#include <libpspp/hash.h>
-#include <libpspp/message.h>
-#include <libpspp/str.h>
+#include "data/dictionary.h"
+#include "data/procedure.h"
+#include "data/value-labels.h"
+#include "data/variable.h"
+#include "language/command.h"
+#include "language/lexer/lexer.h"
+#include "language/lexer/value-parser.h"
+#include "language/lexer/variable-parser.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "libpspp/str.h"
 
-#include "xalloc.h"
+#include "gl/xalloc.h"
 
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
@@ -40,7 +41,8 @@
 static int do_value_labels (struct lexer *,
                            const struct dictionary *dict, bool);
 static void erase_labels (struct variable **vars, size_t var_cnt);
-static int get_label (struct lexer *, struct variable **vars, size_t var_cnt);
+static int get_label (struct lexer *, struct variable **vars, size_t var_cnt,
+                      const char *dict_encoding);
 \f
 /* Stubs. */
 
@@ -65,9 +67,9 @@ do_value_labels (struct lexer *lexer, const struct dictionary *dict, bool erase)
   size_t var_cnt;         /* Number of variables. */
   int parse_err=0;        /* true if error parsing variables */
 
-  lex_match (lexer, '/');
+  lex_match (lexer, T_SLASH);
 
-  while (lex_token (lexer) != '.')
+  while (lex_token (lexer) != T_ENDCMD)
     {
       parse_err = !parse_variables (lexer, dict, &vars, &var_cnt,
                                    PV_SAME_WIDTH);
@@ -78,11 +80,11 @@ do_value_labels (struct lexer *lexer, const struct dictionary *dict, bool erase)
        }
       if (erase)
         erase_labels (vars, var_cnt);
-      while (lex_token (lexer) != '/' && lex_token (lexer) != '.')
-       if (!get_label (lexer, vars, var_cnt))
+      while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
+       if (!get_label (lexer, vars, var_cnt, dict_get_encoding (dict)))
           goto lossage;
 
-      if (lex_token (lexer) != '/')
+      if (lex_token (lexer) != T_SLASH)
        {
           free (vars);
           break;
@@ -93,10 +95,7 @@ do_value_labels (struct lexer *lexer, const struct dictionary *dict, bool erase)
       free (vars);
     }
 
-  if (parse_err)
-    return CMD_FAILURE;
-
-  return lex_end_of_command (lexer);
+  return parse_err ? CMD_FAILURE : CMD_SUCCESS;
 
  lossage:
   free (vars);
@@ -117,14 +116,17 @@ erase_labels (struct variable **vars, size_t var_cnt)
 /* Parse all the labels for the VAR_CNT variables in VARS and add
    the specified labels to those variables.  */
 static int
-get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
+get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt,
+           const char *dict_encoding)
 {
   /* Parse all the labels and add them to the variables. */
   do
     {
+      enum { MAX_LABEL_LEN = 255 };
       int width = var_get_width (vars[0]);
       union value value;
       struct string label;
+      size_t trunc_len;
       size_t i;
 
       /* Set value. */
@@ -134,7 +136,7 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
           value_destroy (&value, width);
           return 0;
         }
-      lex_match (lexer, ',');
+      lex_match (lexer, T_COMMA);
 
       /* Set label. */
       if (lex_token (lexer) != T_ID && !lex_force_string (lexer))
@@ -143,12 +145,14 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
           return 0;
         }
 
-      ds_init_string (&label, lex_tokstr (lexer));
+      ds_init_substring (&label, lex_tokss (lexer));
 
-      if (ds_length (&label) > 60)
+      trunc_len = utf8_encoding_trunc_len (ds_cstr (&label), dict_encoding,
+                                           MAX_LABEL_LEN);
+      if (ds_length (&label) > trunc_len)
        {
-         msg (SW, _("Truncating value label to 60 characters."));
-         ds_truncate (&label, 60);
+         msg (SW, _("Truncating value label to %d bytes."), MAX_LABEL_LEN);
+         ds_truncate (&label, trunc_len);
        }
 
       for (i = 0; i < var_cnt; i++)
@@ -158,9 +162,9 @@ get_label (struct lexer *lexer, struct variable **vars, size_t var_cnt)
       value_destroy (&value, width);
 
       lex_get (lexer);
-      lex_match (lexer, ',');
+      lex_match (lexer, T_COMMA);
     }
-  while (lex_token (lexer) != '/' && lex_token (lexer) != '.');
+  while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD);
 
   return 1;
 }