Use default encoding when reading system files if no encoding is given in file.

[pspp-builds.git] / src / data / sys-file-reader.c

diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c
index 5a8402791464b663f596596b0675098d02202370..141fd881a1ea2323d2a999089e6594265bd88015 100644 (file)
--- a/src/data/sys-file-reader.c
+++ b/src/data/sys-file-reader.c
@@ -25,6 +25,7 @@
 #include <setjmp.h>
 #include <stdlib.h>
 
+#include <libpspp/i18n.h>
 #include <libpspp/assertion.h>
 #include <libpspp/message.h>
 #include <libpspp/compiler.h>
@@ -186,6 +187,53 @@ static void read_long_string_value_labels (struct sfm_reader *,
                                           size_t size, size_t count,
                                           struct dictionary *);
 
+/* Convert all the strings in DICT from the dict encoding to UTF8 */
+static void
+recode_strings (struct dictionary *dict)
+{
+  int i;
+
+  const char *enc = dict_get_encoding (dict);
+
+  if ( NULL == enc)
+    enc = get_default_encoding ();
+
+  for (i = 0 ; i < dict_get_var_cnt (dict); ++i)
+    {
+      /* Convert the long variable name */
+      struct variable *var = dict_get_var (dict, i);
+      char *utf8_name = recode_string (UTF8, enc, var_get_name (var), -1);
+      dict_rename_var (dict, var, utf8_name);
+      free (utf8_name);
+
+      /* Convert the variable label */
+      if (var_has_label (var))
+       {
+         char *utf8_label = recode_string (UTF8, enc, var_get_label (var), -1);
+         var_set_label (var, utf8_label);
+         free (utf8_label);
+       }
+
+      if (var_has_value_labels (var))
+       {
+         const struct val_lab *vl = NULL;
+         const struct val_labs *vlabs = var_get_value_labels (var);
+
+         for (vl = val_labs_first (vlabs); vl != NULL; vl = val_labs_next (vlabs, vl))
+           {
+             const union value *val = val_lab_get_value (vl);
+             const char *label = val_lab_get_label (vl);
+             char *new_label = NULL;
+
+             new_label = recode_string (UTF8, enc, label, -1);
+
+             var_replace_value_label (var, val, new_label);
+             free (new_label);
+           }
+       }
+    }
+}
+
 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the
@@ -303,6 +351,8 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
       r->has_long_var_names = true;
     }
 
+  recode_strings (*dict);
+
   /* Read record 999 data, which is just filler. */
   read_int (r);
 
@@ -552,7 +602,7 @@ read_variable_record (struct sfm_reader *r, struct dictionary *dict,
       struct missing_values mv;
       int i;
 
-      mv_init (&mv, var_get_width (var));
+      mv_init_pool (r->pool, &mv, var_get_width (var));
       if (var_is_numeric (var))
         {
           if (missing_value_code < -3 || missing_value_code > 3
@@ -571,21 +621,24 @@ read_variable_record (struct sfm_reader *r, struct dictionary *dict,
         }
       else
         {
+          int mv_width = MAX (width, 8);
+          union value value;
+
           if (missing_value_code < 1 || missing_value_code > 3)
             sys_error (r, _("String missing value indicator field is not "
                             "0, 1, 2, or 3."));
-          if (var_is_long_string (var))
-            sys_warn (r, _("Ignoring missing values on long string variable "
-                           "%s, which PSPP does not yet support."), name);
+
+          value_init (&value, mv_width);
+          value_set_missing (&value, mv_width);
           for (i = 0; i < missing_value_code; i++)
             {
-              char string[9];
-              read_string (r, string, sizeof string);
-              mv_add_str (&mv, string);
+              char *s = value_str_rw (&value, mv_width);
+              read_bytes (r, s, 8);
+              mv_add_str (&mv, s);
             }
+          value_destroy (&value, mv_width);
         }
-      if (!var_is_long_string (var))
-        var_set_missing_values (var, &mv);
+      var_set_missing_values (var, &mv);
     }
 
   /* Set formats. */
@@ -1203,7 +1256,7 @@ read_value_labels (struct sfm_reader *r,
   for (i = 0; i < var_cnt; i++)
     {
       var[i] = lookup_var_by_value_idx (r, var_by_value_idx, read_int (r));
-      if (var_is_long_string (var[i]))
+      if (var_get_width (var[i]) > 8)
         sys_error (r, _("Value labels may not be added to long string "
                         "variables (e.g. %s) using records types 3 and 4."),
                    var_get_name (var[i]));