X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?p=pspp-builds.git;a=blobdiff_plain;f=src%2Fdata%2Fsys-file-reader.c;h=b0a41a83573b0e986fb35bfb3bce46379282662f;hp=fe7b53342128087687d39e74d46721b1cb5917ed;hb=b5c82cc9aabe7e641011130240ae1b2e84348e23;hpb=5ee84736663824fe12474b78dace867e42893a14

diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c
index fe7b5334..b0a41a83 100644
--- a/src/data/sys-file-reader.c
+++ b/src/data/sys-file-reader.c
@@ -25,6 +25,7 @@
 #include <setjmp.h>
 #include <stdlib.h>
 
+#include <libpspp/i18n.h>
 #include <libpspp/assertion.h>
 #include <libpspp/message.h>
 #include <libpspp/compiler.h>
@@ -34,6 +35,7 @@
 #include <libpspp/hash.h>
 #include <libpspp/array.h>
 
+#include <data/attributes.h>
 #include <data/case.h>
 #include <data/casereader-provider.h>
 #include <data/casereader.h>
@@ -70,7 +72,7 @@ struct sfm_reader
     struct fh_lock *lock;       /* Mutual exclusion for file handle. */
     FILE *file;                 /* File stream. */
     bool error;                 /* I/O or corruption error? */
-    size_t value_cnt;           /* Number of "union value"s in struct case. */
+    struct caseproto *proto;    /* Format of output cases. */
 
     /* File format. */
     enum integer_format integer_format; /* On-disk integer format. */
@@ -98,9 +100,11 @@ static struct variable *lookup_var_by_value_idx (struct sfm_reader *,
                                                  struct variable **,
                                                  int value_idx);
 
+static void sys_msg (struct sfm_reader *r, int class,
+                     const char *format, va_list args)
+     PRINTF_FORMAT (3, 0);
 static void sys_warn (struct sfm_reader *, const char *, ...)
      PRINTF_FORMAT (2, 3);
-
 static void sys_error (struct sfm_reader *, const char *, ...)
      PRINTF_FORMAT (2, 3)
      NO_RETURN;
@@ -112,15 +116,23 @@ static double read_float (struct sfm_reader *);
 static void read_string (struct sfm_reader *, char *, size_t);
 static void skip_bytes (struct sfm_reader *, size_t);
 
-static struct variable_to_value_map *open_variable_to_value_map (
-  struct sfm_reader *, size_t size);
-static void close_variable_to_value_map (struct sfm_reader *r,
-                                         struct variable_to_value_map *);
-static bool read_variable_to_value_map (struct sfm_reader *,
-                                        struct dictionary *,
-                                        struct variable_to_value_map *,
-                                        struct variable **var, char **value,
-                                        int *warning_cnt);
+static struct text_record *open_text_record (struct sfm_reader *, size_t size);
+static void close_text_record (struct sfm_reader *r,
+                               struct text_record *);
+static bool read_variable_to_value_pair (struct sfm_reader *,
+                                         struct dictionary *,
+                                         struct text_record *,
+                                         struct variable **var, char **value);
+static void text_warn (struct sfm_reader *r, struct text_record *text,
+                       const char *format, ...)
+  PRINTF_FORMAT (3, 4);
+static char *text_get_token (struct text_record *,
+                             struct substring delimiters);
+static bool text_match (struct text_record *, char c);
+static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
+                                  struct text_record *,
+                                  struct substring delimiters,
+                                  struct variable **);
 
 static bool close_reader (struct sfm_reader *r);
 
@@ -151,7 +163,9 @@ static void read_extension_record (struct sfm_reader *, struct dictionary *,
                                    struct sfm_read_info *);
 static void read_machine_integer_info (struct sfm_reader *,
                                        size_t size, size_t count,
-                                       struct sfm_read_info *);
+                                       struct sfm_read_info *,
+				       struct dictionary *
+				       );
 static void read_machine_float_info (struct sfm_reader *,
                                      size_t size, size_t count);
 static void read_display_parameters (struct sfm_reader *,
@@ -163,7 +177,71 @@ static void read_long_var_name_map (struct sfm_reader *,
 static void read_long_string_map (struct sfm_reader *,
                                   size_t size, size_t count,
                                   struct dictionary *);
+static void read_data_file_attributes (struct sfm_reader *,
+                                       size_t size, size_t count,
+                                       struct dictionary *);
+static void read_variable_attributes (struct sfm_reader *,
+                                      size_t size, size_t count,
+                                      struct dictionary *);
+static void read_long_string_value_labels (struct sfm_reader *,
+					   size_t size, size_t count,
+					   struct dictionary *);
+
+/* Convert all the strings in DICT from the dict encoding to UTF8 */
+static void
+recode_strings (struct dictionary *dict)
+{
+  int i;
+
+  const char *enc = dict_get_encoding (dict);
+
+  if ( NULL == enc)
+    enc = get_default_encoding ();
+
+  for (i = 0 ; i < dict_get_var_cnt (dict); ++i)
+    {
+      /* Convert the long variable name */
+      struct variable *var = dict_get_var (dict, i);
+      const char *native_name = var_get_name (var);
+      char *utf8_name = recode_string (UTF8, enc, native_name, -1);
+      if ( 0 != strcmp (utf8_name, native_name))
+	{
+	  if ( NULL == dict_lookup_var (dict, utf8_name))
+	    dict_rename_var (dict, var, utf8_name);
+	  else
+	    msg (MW,
+	     _("Recoded variable name duplicates an existing `%s' within system file."), utf8_name);
+    }
+
+      free (utf8_name);
+
+      /* Convert the variable label */
+      if (var_has_label (var))
+	{
+	  char *utf8_label = recode_string (UTF8, enc, var_get_label (var), -1);
+	  var_set_label (var, utf8_label);
+	  free (utf8_label);
+	}
+
+      if (var_has_value_labels (var))
+	{
+	  const struct val_lab *vl = NULL;
+	  const struct val_labs *vlabs = var_get_value_labels (var);
+
+	  for (vl = val_labs_first (vlabs); vl != NULL; vl = val_labs_next (vlabs, vl))
+	    {
+	      const union value *val = val_lab_get_value (vl);
+	      const char *label = val_lab_get_label (vl);
+	      char *new_label = NULL;
 
+	      new_label = recode_string (UTF8, enc, label, -1);
+
+	      var_replace_value_label (var, val, new_label);
+	      free (new_label);
+	    }
+	}
+    }
+}
 
 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
@@ -282,6 +360,8 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
       r->has_long_var_names = true;
     }
 
+  recode_strings (*dict);
+
   /* Read record 999 data, which is just filler. */
   read_int (r);
 
@@ -301,11 +381,11 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
      dictionary and may destroy or modify its variables. */
   sfm_dictionary_to_sfm_vars (*dict, &r->sfm_vars, &r->sfm_var_cnt);
   pool_register (r->pool, free, r->sfm_vars);
+  r->proto = caseproto_ref_pool (dict_get_proto (*dict), r->pool);
 
   pool_free (r->pool, var_by_value_idx);
-  r->value_cnt = dict_get_next_value_idx (*dict);
   return casereader_create_sequential
-    (NULL, r->value_cnt,
+    (NULL, r->proto,
      r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
                                        &sys_file_casereader_class, r);
 
@@ -509,7 +589,7 @@ read_variable_record (struct sfm_reader *r, struct dictionary *dict,
 
   /* Create variable. */
   if (width < 0 || width > 255)
-    sys_error (r, _("Bad variable width %d."), width);
+    sys_error (r, _("Bad width %d for variable %s."), width, name);
   var = dict_create_var (dict, name, width);
   if (var == NULL)
     sys_error (r,
@@ -543,7 +623,7 @@ read_variable_record (struct sfm_reader *r, struct dictionary *dict,
       struct missing_values mv;
       int i;
 
-      mv_init (&mv, var_get_width (var));
+      mv_init_pool (r->pool, &mv, var_get_width (var));
       if (var_is_numeric (var))
         {
           if (missing_value_code < -3 || missing_value_code > 3
@@ -562,21 +642,24 @@ read_variable_record (struct sfm_reader *r, struct dictionary *dict,
         }
       else
         {
+          int mv_width = MAX (width, 8);
+          union value value;
+
           if (missing_value_code < 1 || missing_value_code > 3)
             sys_error (r, _("String missing value indicator field is not "
                             "0, 1, 2, or 3."));
-          if (var_is_long_string (var))
-            sys_warn (r, _("Ignoring missing values on long string variable "
-                           "%s, which PSPP does not yet support."), name);
+
+          value_init (&value, mv_width);
+          value_set_missing (&value, mv_width);
           for (i = 0; i < missing_value_code; i++)
             {
-              char string[9];
-              read_string (r, string, sizeof string);
-              mv_add_str (&mv, string);
+              uint8_t *s = value_str_rw (&value, mv_width);
+              read_bytes (r, s, 8);
+              mv_add_str (&mv, s);
             }
+          value_destroy (&value, mv_width);
         }
-      if (!var_is_long_string (var))
-        var_set_missing_values (var, &mv);
+      var_set_missing_values (var, &mv);
     }
 
   /* Set formats. */
@@ -721,7 +804,7 @@ read_extension_record (struct sfm_reader *r, struct dictionary *dict,
   switch (subtype)
     {
     case 3:
-      read_machine_integer_info (r, size, count, info);
+      read_machine_integer_info (r, size, count, info, dict);
       return;
 
     case 4:
@@ -764,21 +847,28 @@ read_extension_record (struct sfm_reader *r, struct dictionary *dict,
       break;
 
     case 17:
-      /* Text field that defines variable attributes.  New in
-         SPSS 14. */
-      break;
+      read_data_file_attributes (r, size, count, dict);
+      return;
+
+    case 18:
+      read_variable_attributes (r, size, count, dict);
+      return;
 
     case 20:
       /* New in SPSS 16.  Contains a single string that describes
          the character encoding, e.g. "windows-1252". */
-      break;
+      {
+	char *encoding = pool_calloc (r->pool, size, count + 1);
+	read_string (r, encoding, count + 1);
+	dict_set_encoding (dict, encoding);
+	return;
+      }
 
     case 21:
       /* New in SPSS 16.  Encodes value labels for long string
          variables. */
-      sys_warn (r, _("Ignoring value labels for long string variables, "
-                     "which PSPP does not yet support."));
-      break;
+      read_long_string_value_labels (r, size, count, dict);
+      return;
 
     default:
       sys_warn (r, _("Unrecognized record type 7, subtype %d.  Please send a copy of this file, and the syntax which created it to %s"),
@@ -792,7 +882,8 @@ read_extension_record (struct sfm_reader *r, struct dictionary *dict,
 /* Read record type 7, subtype 3. */
 static void
 read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count,
-                           struct sfm_read_info *info)
+                           struct sfm_read_info *info,
+			   struct dictionary *dict)
 {
   int version_major = read_int (r);
   int version_minor = read_int (r);
@@ -801,7 +892,7 @@ read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count,
   int float_representation = read_int (r);
   int compression_code UNUSED = read_int (r);
   int integer_representation = read_int (r);
-  int character_code UNUSED = read_int (r);
+  int character_code = read_int (r);
 
   int expected_float_format;
   int expected_integer_format;
@@ -846,6 +937,47 @@ read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count,
                 gettext (endian[integer_representation == 1]),
                 gettext (endian[expected_integer_format == 1]));
     }
+
+
+  /*
+    Record 7 (20) provides a much more reliable way of
+    setting the encoding.
+    The character_code is used as a fallback only.
+  */
+  if ( NULL == dict_get_encoding (dict))
+    {
+      switch (character_code)
+	{
+	case 1:
+	  dict_set_encoding (dict, "EBCDIC-US");
+	  break;
+	case 2:
+	case 3:
+	  /* These ostensibly mean "7-bit ASCII" and "8-bit ASCII"[sic]
+	     respectively.   However, there are known to be many files
+	     in the wild with character code 2, yet have data which are
+	     clearly not ascii.
+	     Therefore we ignore these values.
+	  */
+	  return;
+	case 4:
+	  dict_set_encoding (dict, "MS_KANJI");
+	  break;
+	case 65000:
+	  dict_set_encoding (dict, "UTF-7");
+	  break;
+	case 65001:
+	  dict_set_encoding (dict, "UTF-8");
+	  break;
+	default:
+	  {
+	    char enc[100];
+	    snprintf (enc, 100, "CP%d", character_code);
+	    dict_set_encoding (dict, enc);
+	  }
+	  break;
+	};
+    }
 }
 
 /* Read record type 7, subtype 4. */
@@ -861,11 +993,16 @@ read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
                size, count);
 
   if (sysmis != SYSMIS)
-    sys_warn (r, _("File specifies unexpected value %g as SYSMIS."), sysmis);
+    sys_warn (r, _("File specifies unexpected value %g as %s."),
+              sysmis, "SYSMIS");
+
   if (highest != HIGHEST)
-    sys_warn (r, _("File specifies unexpected value %g as HIGHEST."), highest);
+    sys_warn (r, _("File specifies unexpected value %g as %s."),
+              highest, "HIGHEST");
+
   if (lowest != LOWEST)
-    sys_warn (r, _("File specifies unexpected value %g as LOWEST."), lowest);
+    sys_warn (r, _("File specifies unexpected value %g as %s."),
+              lowest, "LOWEST");
 }
 
 /* Read record type 7, subtype 11, which specifies how variables
@@ -944,14 +1081,12 @@ static void
 read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count,
                         struct dictionary *dict)
 {
-  struct variable_to_value_map *map;
+  struct text_record *text;
   struct variable *var;
   char *long_name;
-  int warning_cnt = 0;
 
-  map = open_variable_to_value_map (r, size * count);
-  while (read_variable_to_value_map (r, dict, map, &var, &long_name,
-                                     &warning_cnt))
+  text = open_text_record (r, size * count);
+  while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
     {
       char **short_names;
       size_t short_name_cnt;
@@ -997,7 +1132,7 @@ read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count,
         }
       free (short_names);
     }
-  close_variable_to_value_map (r, map);
+  close_text_record (r, text);
   r->has_long_var_names = true;
 }
 
@@ -1007,14 +1142,12 @@ static void
 read_long_string_map (struct sfm_reader *r, size_t size, size_t count,
                       struct dictionary *dict)
 {
-  struct variable_to_value_map *map;
+  struct text_record *text;
   struct variable *var;
   char *length_s;
-  int warning_cnt = 0;
 
-  map = open_variable_to_value_map (r, size * count);
-  while (read_variable_to_value_map (r, dict, map, &var, &length_s,
-                                     &warning_cnt))
+  text = open_text_record (r, size * count);
+  while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
     {
       size_t idx = var_get_dict_index (var);
       long int length;
@@ -1062,7 +1195,7 @@ read_long_string_map (struct sfm_reader *r, size_t size, size_t count,
       dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
       var_set_width (var, length);
     }
-  close_variable_to_value_map (r, map);
+  close_text_record (r, text);
   dict_compact_values (dict);
 }
 
@@ -1076,7 +1209,7 @@ read_value_labels (struct sfm_reader *r,
 
   struct label
     {
-      char raw_value[8];        /* Value as uninterpreted bytes. */
+      uint8_t raw_value[8];        /* Value as uninterpreted bytes. */
       union value value;        /* Value. */
       char *label;              /* Null-terminated label string. */
     };
@@ -1086,6 +1219,7 @@ read_value_labels (struct sfm_reader *r,
 
   struct variable **var = NULL;	/* Associated variables. */
   int var_cnt;			/* Number of associated variables. */
+  int max_width;                /* Maximum width of string variables. */
 
   int i;
 
@@ -1144,12 +1278,15 @@ read_value_labels (struct sfm_reader *r,
 
   /* Read the list of variables. */
   var = pool_nalloc (subpool, var_cnt, sizeof *var);
+  max_width = 0;
   for (i = 0; i < var_cnt; i++)
     {
       var[i] = lookup_var_by_value_idx (r, var_by_value_idx, read_int (r));
-      if (var_is_long_string (var[i]))
-        sys_error (r, _("Value labels are not allowed on long string "
-                        "variables (%s)."), var_get_name (var[i]));
+      if (var_get_width (var[i]) > 8)
+        sys_error (r, _("Value labels may not be added to long string "
+                        "variables (e.g. %s) using records types 3 and 4."),
+                   var_get_name (var[i]));
+      max_width = MAX (max_width, var_get_width (var[i]));
     }
 
   /* Type check the variables. */
@@ -1168,9 +1305,10 @@ read_value_labels (struct sfm_reader *r,
     {
       struct label *label = labels + i;
 
+      value_init_pool (subpool, &label->value, max_width);
       if (var_is_alpha (var[0]))
-        buf_copy_rpad (label->value.s, sizeof label->value.s,
-                       label->raw_value, sizeof label->raw_value);
+        u8_buf_copy_rpad (value_str_rw (&label->value, max_width), max_width,
+                       label->raw_value, sizeof label->raw_value, ' ');
       else
         label->value.f = float_get_double (r->float_format, label->raw_value);
     }
@@ -1192,7 +1330,7 @@ read_value_labels (struct sfm_reader *r,
                           label->value.f, var_get_name (v));
               else
                 sys_warn (r, _("Duplicate value label for \"%.*s\" on %s."),
-                          var_get_width (v), label->value.s,
+                          max_width, value_str (&label->value, max_width),
                           var_get_name (v));
             }
 	}
@@ -1200,6 +1338,203 @@ read_value_labels (struct sfm_reader *r,
 
   pool_destroy (subpool);
 }
+
+/* Reads a set of custom attributes from TEXT into ATTRS.
+   ATTRS may be a null pointer, in which case the attributes are
+   read but discarded. */
+static void
+read_attributes (struct sfm_reader *r, struct text_record *text,
+                 struct attrset *attrs)
+{
+  do
+    {
+      struct attribute *attr;
+      char *key;
+      int index;
+
+      /* Parse the key. */
+      key = text_get_token (text, ss_cstr ("("));
+      if (key == NULL)
+        return;
+
+      attr = attribute_create (key);
+      for (index = 1; ; index++)
+        {
+          /* Parse the value. */
+          char *value;
+          size_t length;
+
+          value = text_get_token (text, ss_cstr ("\n"));
+          if (value == NULL)
+            {
+              text_warn (r, text, _("Error parsing attribute value %s[%d]"),
+                         key, index);
+              break;
+            }              
+
+          length = strlen (value);
+          if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'') 
+            {
+              value[length - 1] = '\0';
+              attribute_add_value (attr, value + 1); 
+            }
+          else 
+            {
+              text_warn (r, text,
+                         _("Attribute value %s[%d] is not quoted: %s"),
+                         key, index, value);
+              attribute_add_value (attr, value); 
+            }
+
+          /* Was this the last value for this attribute? */
+          if (text_match (text, ')'))
+            break;
+        }
+      if (attrs != NULL)
+        attrset_add (attrs, attr);
+      else
+        attribute_destroy (attr);
+    }
+  while (!text_match (text, '/'));
+}
+
+/* Reads record type 7, subtype 17, which lists custom
+   attributes on the data file.  */
+static void
+read_data_file_attributes (struct sfm_reader *r,
+                           size_t size, size_t count,
+                           struct dictionary *dict)
+{
+  struct text_record *text = open_text_record (r, size * count);
+  read_attributes (r, text, dict_get_attributes (dict));
+  close_text_record (r, text);
+}
+
+static void
+skip_long_string_value_labels (struct sfm_reader *r, size_t n_labels)
+{
+  size_t i;
+
+  for (i = 0; i < n_labels; i++)
+    {
+      size_t value_length, label_length;
+
+      value_length = read_int (r);
+      skip_bytes (r, value_length);
+      label_length = read_int (r);
+      skip_bytes (r, label_length);
+    }
+}
+
+static void
+read_long_string_value_labels (struct sfm_reader *r,
+			       size_t size, size_t count,
+			       struct dictionary *d)
+{
+  const off_t start = ftello (r->file);
+  while (ftello (r->file) - start < size * count)
+    {
+      char var_name[VAR_NAME_LEN + 1];
+      size_t n_labels, i;
+      struct variable *v;
+      union value value;
+      int var_name_len;
+      int width;
+
+      /* Read header. */
+      var_name_len = read_int (r);
+      if (var_name_len > VAR_NAME_LEN)
+        sys_error (r, _("Variable name length in long string value label "
+                        "record (%d) exceeds %d-byte limit."),
+                   var_name_len, VAR_NAME_LEN);
+      read_string (r, var_name, var_name_len + 1);
+      width = read_int (r);
+      n_labels = read_int (r);
+
+      v = dict_lookup_var (d, var_name);
+      if (v == NULL)
+        {
+          sys_warn (r, _("Ignoring long string value record for "
+                         "unknown variable %s."), var_name);
+          skip_long_string_value_labels (r, n_labels);
+          continue;
+        }
+      if (var_is_numeric (v))
+        {
+          sys_warn (r, _("Ignoring long string value record for "
+                         "numeric variable %s."), var_name);
+          skip_long_string_value_labels (r, n_labels);
+          continue;
+        }
+      if (width != var_get_width (v))
+        {
+          sys_warn (r, _("Ignoring long string value record for variable %s "
+                         "because the record's width (%d) does not match the "
+                         "variable's width (%d)"),
+                    var_name, width, var_get_width (v));
+          skip_long_string_value_labels (r, n_labels);
+          continue;
+        }
+
+      /* Read values. */
+      value_init_pool (r->pool, &value, width);
+      for (i = 0; i < n_labels; i++)
+	{
+          size_t value_length, label_length;
+          char label[256];
+          bool skip = false;
+
+          /* Read value. */
+          value_length = read_int (r);
+          if (value_length == width)
+            read_bytes (r, value_str_rw (&value, width), width);
+          else
+            {
+              sys_warn (r, _("Ignoring long string value %zu for variable %s, "
+                             "with width %d, that has bad value width %zu."),
+                        i, var_get_name (v), width, value_length);
+              skip_bytes (r, value_length);
+              skip = true;
+            }
+
+          /* Read label. */
+          label_length = read_int (r);
+          read_string (r, label, MIN (sizeof label, label_length + 1));
+          if (label_length >= sizeof label)
+            {
+              /* Skip and silently ignore label text after the
+                 first 255 bytes.  The maximum documented length
+                 of a label is 120 bytes so this is more than
+                 generous. */
+              skip_bytes (r, sizeof label - (label_length + 1));
+            }
+
+          if (!skip && !var_add_value_label (v, &value, label))
+            sys_warn (r, _("Duplicate value label for \"%.*s\" on %s."),
+                      width, value_str (&value, width), var_get_name (v));
+        }
+    }
+}
+
+
+/* Reads record type 7, subtype 18, which lists custom
+   attributes on individual variables.  */
+static void
+read_variable_attributes (struct sfm_reader *r,
+                          size_t size, size_t count,
+                          struct dictionary *dict)
+{
+  struct text_record *text = open_text_record (r, size * count);
+  for (;;) 
+    {
+      struct variable *var;
+      if (!text_read_short_name (r, dict, text, ss_cstr (":"), &var))
+        break;
+      read_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
+    }
+  close_text_record (r, text);
+}
+
 
 /* Case reader. */
 
@@ -1209,31 +1544,31 @@ static void partial_record (struct sfm_reader *r)
 static void read_error (struct casereader *, const struct sfm_reader *);
 
 static bool read_case_number (struct sfm_reader *, double *);
-static bool read_case_string (struct sfm_reader *, char *, size_t);
+static bool read_case_string (struct sfm_reader *, uint8_t *, size_t);
 static int read_opcode (struct sfm_reader *);
 static bool read_compressed_number (struct sfm_reader *, double *);
-static bool read_compressed_string (struct sfm_reader *, char *);
-static bool read_whole_strings (struct sfm_reader *, char *, size_t);
+static bool read_compressed_string (struct sfm_reader *, uint8_t *);
+static bool read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
 static bool skip_whole_strings (struct sfm_reader *, size_t);
 
-/* Reads one case from READER's file into C.  Returns true only
-   if successful. */
-static bool
-sys_file_casereader_read (struct casereader *reader, void *r_,
-                          struct ccase *c)
+/* Reads and returns one case from READER's file.  Returns a null
+   pointer if not successful. */
+static struct ccase *
+sys_file_casereader_read (struct casereader *reader, void *r_)
 {
   struct sfm_reader *r = r_;
+  struct ccase *volatile c;
   int i;
 
   if (r->error)
-    return false;
+    return NULL;
 
-  case_create (c, r->value_cnt);
+  c = case_create (r->proto);
   if (setjmp (r->bail_out))
     {
       casereader_force_error (reader);
-      case_destroy (c);
-      return false;
+      case_unref (c);
+      return NULL;
     }
 
   for (i = 0; i < r->sfm_var_cnt; i++)
@@ -1241,28 +1576,29 @@ sys_file_casereader_read (struct casereader *reader, void *r_,
       struct sfm_var *sv = &r->sfm_vars[i];
       union value *v = case_data_rw_idx (c, sv->case_index);
 
-      if (sv->width == 0)
+      if (sv->var_width == 0)
         {
           if (!read_case_number (r, &v->f))
             goto eof;
         }
       else
         {
-          if (!read_case_string (r, v->s + sv->offset, sv->width))
+          uint8_t *s = value_str_rw (v, sv->var_width);
+          if (!read_case_string (r, s + sv->offset, sv->segment_width))
             goto eof;
           if (!skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)))
             partial_record (r);
         }
     }
-  return true;
+  return c;
 
 eof:
-  case_destroy (c);
+  case_unref (c);
   if (i != 0)
     partial_record (r);
   if (r->case_cnt != -1)
     read_error (reader, r);
-  return false;
+  return NULL;
 }
 
 /* Issues an error that R ends in a partial record. */
@@ -1309,7 +1645,7 @@ read_case_number (struct sfm_reader *r, double *d)
    Returns true if successful, false if end of file is
    reached immediately. */
 static bool
-read_case_string (struct sfm_reader *r, char *s, size_t length)
+read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
 {
   size_t whole = ROUND_DOWN (length, 8);
   size_t partial = length % 8;
@@ -1322,7 +1658,7 @@ read_case_string (struct sfm_reader *r, char *s, size_t length)
 
   if (partial)
     {
-      char bounce[8];
+      uint8_t bounce[8];
       if (!read_whole_strings (r, bounce, sizeof bounce))
         {
           if (whole)
@@ -1393,7 +1729,7 @@ read_compressed_number (struct sfm_reader *r, double *d)
    Returns true if successful, false if end of file is
    reached immediately. */
 static bool
-read_compressed_string (struct sfm_reader *r, char *dst)
+read_compressed_string (struct sfm_reader *r, uint8_t *dst)
 {
   switch (read_opcode (r))
     {
@@ -1422,7 +1758,7 @@ read_compressed_string (struct sfm_reader *r, char *dst)
    Returns true if successful, false if end of file is
    reached immediately. */
 static bool
-read_whole_strings (struct sfm_reader *r, char *s, size_t length)
+read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
 {
   assert (length % 8 == 0);
   if (!r->compressed)
@@ -1450,7 +1786,7 @@ read_whole_strings (struct sfm_reader *r, char *s, size_t length)
 static bool
 skip_whole_strings (struct sfm_reader *r, size_t length)
 {
-  char buffer[1024];
+  uint8_t buffer[1024];
   assert (length < sizeof buffer);
   return read_whole_strings (r, buffer, length);
 }
@@ -1535,82 +1871,124 @@ lookup_var_by_short_name (struct dictionary *d, const char *short_name)
   return NULL;
 }
 
-/* Helpers for reading records that contain "variable=value"
-   pairs. */
+/* Helpers for reading records that contain structured text
+   strings. */
+
+/* Maximum number of warnings to issue for a single text
+   record. */
+#define MAX_TEXT_WARNINGS 5
 
 /* State. */
-struct variable_to_value_map
+struct text_record
   {
     struct substring buffer;    /* Record contents. */
     size_t pos;                 /* Current position in buffer. */
+    int n_warnings;             /* Number of warnings issued or suppressed. */
   };
 
-/* Reads SIZE bytes into a "variable=value" map for R,
-   and returns the map. */
-static struct variable_to_value_map *
-open_variable_to_value_map (struct sfm_reader *r, size_t size)
+/* Reads SIZE bytes into a text record for R,
+   and returns the new text record. */
+static struct text_record *
+open_text_record (struct sfm_reader *r, size_t size)
 {
-  struct variable_to_value_map *map = pool_alloc (r->pool, sizeof *map);
+  struct text_record *text = pool_alloc (r->pool, sizeof *text);
   char *buffer = pool_malloc (r->pool, size + 1);
   read_bytes (r, buffer, size);
-  map->buffer = ss_buffer (buffer, size);
-  map->pos = 0;
-  return map;
+  text->buffer = ss_buffer (buffer, size);
+  text->pos = 0;
+  text->n_warnings = 0;
+  return text;
 }
 
-/* Closes MAP and frees its storage.
-   Not really needed, because the pool will free the map anyway,
-   but can be used to free it earlier. */
+/* Closes TEXT, frees its storage, and issues a final warning
+   about suppressed warnings if necesary. */
 static void
-close_variable_to_value_map (struct sfm_reader *r,
-                             struct variable_to_value_map *map)
+close_text_record (struct sfm_reader *r, struct text_record *text)
 {
-  pool_free (r->pool, ss_data (map->buffer));
+  if (text->n_warnings > MAX_TEXT_WARNINGS)
+    sys_warn (r, _("Suppressed %d additional related warnings."),
+              text->n_warnings - MAX_TEXT_WARNINGS);
+  pool_free (r->pool, ss_data (text->buffer));
 }
 
-/* Reads the next variable=value pair from MAP.
+/* Reads a variable=value pair from TEXT.
    Looks up the variable in DICT and stores it into *VAR.
    Stores a null-terminated value into *VALUE. */
 static bool
-read_variable_to_value_map (struct sfm_reader *r, struct dictionary *dict,
-                            struct variable_to_value_map *map,
-                            struct variable **var, char **value,
-                            int *warning_cnt)
+read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
+                             struct text_record *text,
+                             struct variable **var, char **value)
 {
-  int max_warnings = 5;
-
   for (;;)
     {
-      struct substring short_name_ss, value_ss;
+      if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
+        return false;
+      
+      *value = text_get_token (text, ss_buffer ("\t\0", 2));
+      if (*value == NULL)
+        return false;
 
-      if (!ss_tokenize (map->buffer, ss_cstr ("="), &map->pos, &short_name_ss)
-          || !ss_tokenize (map->buffer, ss_buffer ("\t\0", 2), &map->pos,
-                           &value_ss))
-        {
-          if (*warning_cnt > max_warnings)
-            sys_warn (r, _("Suppressed %d additional variable map warnings."),
-                      *warning_cnt - max_warnings);
-          return false;
-        }
+      text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
+                            ss_buffer ("\t\0", 2));
 
-      map->pos += ss_span (ss_substr (map->buffer, map->pos, SIZE_MAX),
-                           ss_buffer ("\t\0", 2));
+      if (*var != NULL)
+        return true;
+    }
+}
 
-      ss_data (short_name_ss)[ss_length (short_name_ss)] = '\0';
-      *var = lookup_var_by_short_name (dict, ss_data (short_name_ss));
-      if (*var == NULL)
-        {
-          if (++*warning_cnt <= max_warnings)
-            sys_warn (r, _("Variable map refers to unknown variable %s."),
-                      ss_data (short_name_ss));
-          continue;
-        }
+static bool
+text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
+                      struct text_record *text, struct substring delimiters,
+                      struct variable **var)
+{
+  char *short_name = text_get_token (text, delimiters);
+  if (short_name == NULL)
+    return false;
 
-      ss_data (value_ss)[ss_length (value_ss)] = '\0';
-      *value = ss_data (value_ss);
+  *var = lookup_var_by_short_name (dict, short_name);
+  if (*var == NULL)
+    text_warn (r, text, _("Variable map refers to unknown variable %s."),
+               short_name);
+  return true;
+}
+
+/* Displays a warning for the current file position, limiting the
+   number to MAX_TEXT_WARNINGS for TEXT. */
+static void
+text_warn (struct sfm_reader *r, struct text_record *text,
+           const char *format, ...)
+{
+  if (text->n_warnings++ < MAX_TEXT_WARNINGS) 
+    {
+      va_list args;
 
+      va_start (args, format);
+      sys_msg (r, MW, format, args);
+      va_end (args);
+    }
+}
+
+static char *
+text_get_token (struct text_record *text, struct substring delimiters)
+{
+  struct substring token;
+
+  if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
+    return NULL;
+  ss_data (token)[ss_length (token)] = '\0';
+  return ss_data (token);
+}
+
+static bool
+text_match (struct text_record *text, char c)
+{
+  if (text->buffer.string[text->pos] == c) 
+    {
+      text->pos++;
       return true;
     }
+  else
+    return false;
 }
 
 /* Messages. */