Made array.h const correct, and dealt with the consequences.

[pspp] / src / data / sys-file-reader.c

diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c
index c8c70342c957f421f6503bbce331665dad4aee1b..246448a1f2a4bd7a733c4b4e856ee95c14859d6f 100644 (file)
--- a/src/data/sys-file-reader.c
+++ b/src/data/sys-file-reader.c
@@ -18,71 +18,77 @@
    02110-1301, USA. */
 
 #include <config.h>
    02110-1301, USA. */
 
 #include <config.h>

-#include "sys-file-reader.h"
-#include "sfm-private.h"
-#include "message.h"
+

 #include <stdlib.h>
 #include <stdlib.h>

-#include <ctype.h>

 #include <errno.h>
 #include <float.h>
 #include <errno.h>
 #include <float.h>

-#include <setjmp.h>
-#include "alloc.h"
+#include <c-ctype.h>
+#include <minmax.h>
+
+#include <libpspp/alloc.h>
+#include <libpspp/assertion.h>
+#include <libpspp/message.h>
+#include <libpspp/compiler.h>
+#include <libpspp/magic.h>
+#include <libpspp/misc.h>
+#include <libpspp/str.h>
+#include <libpspp/hash.h>
+#include <libpspp/array.h>
+
+#include "sys-file-reader.h"
+#include "sfm-private.h"

 #include "case.h"
 #include "dictionary.h"
 #include "case.h"
 #include "dictionary.h"

-#include "message.h"

 #include "file-handle-def.h"
 #include "file-handle-def.h"

-#include "filename.h"
+#include "file-name.h"

 #include "format.h"
 #include "format.h"

-#include "magic.h"
-#include "misc.h"

 #include "value-labels.h"
 #include "value-labels.h"

-#include "str.h"

 #include "variable.h"
 #include "variable.h"

+#include "value.h"

 
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
 
 
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
 

-#include "debug-print.h"
-

 /* System file reader. */
 struct sfm_reader
 /* System file reader. */
 struct sfm_reader

-  {
-    struct file_handle *fh;     /* File handle. */
-    FILE *file;                        /* File stream. */
-
-    int reverse_endian;                /* 1=file has endianness opposite us. */
-    int fix_specials;           /* 1=SYSMIS/HIGHEST/LOWEST differs from us. */
-    int value_cnt;             /* Number of `union values's per case. */
-    long case_cnt;             /* Number of cases, -1 if unknown. */
-    int compressed;            /* 1=compressed, 0=not compressed. */
-    double bias;               /* Compression bias, usually 100.0. */
-    int weight_idx;            /* 0-based index of weighting variable, or -1. */
-    bool ok;                    /* False after an I/O error or corrupt data. */
-
-    /* Variables. */
-    struct sfm_var *vars;       /* Variables. */
-
-    /* File's special constants. */
-    flt64 sysmis;
-    flt64 highest;
-    flt64 lowest;
-
-    /* Decompression buffer. */
-    flt64 *buf;                        /* Buffer data. */
-    flt64 *ptr;                        /* Current location in buffer. */
-    flt64 *end;                        /* End of buffer data. */
-
-    /* Compression instruction octet. */
-    unsigned char x[8];         /* Current instruction octet. */
-    unsigned char *y;          /* Location in current instruction octet. */
-  };
+{
+  struct file_handle *fh;     /* File handle. */
+  FILE *file;                  /* File stream. */
+
+  int reverse_endian;          /* 1=file has endianness opposite us. */
+  int value_cnt;               /* Number of `union values's per case. */
+  long case_cnt;               /* Number of cases, -1 if unknown. */
+  int compressed;              /* 1=compressed, 0=not compressed. */
+  double bias;                 /* Compression bias, usually 100.0. */
+  int weight_idx;              /* 0-based index of weighting variable, or -1. */
+  bool ok;                    /* False after an I/O error or corrupt data. */
+  bool has_vls;         /* True if the file has one or more Very Long Strings*/
+
+  /* Variables. */
+  struct sfm_var *vars;
+  size_t var_cnt;
+
+  /* File's special constants. */
+  flt64 sysmis;
+  flt64 highest;
+  flt64 lowest;
+
+  /* Decompression buffer. */
+  flt64 *buf;                  /* Buffer data. */
+  flt64 *ptr;                  /* Current location in buffer. */
+  flt64 *end;                  /* End of buffer data. */
+
+  /* Compression instruction octet. */
+  unsigned char x[8];         /* Current instruction octet. */
+  unsigned char *y;            /* Location in current instruction octet. */
+};

 
 /* A variable in a system file. */
 struct sfm_var 
 
 /* A variable in a system file. */
 struct sfm_var 

-  {
-    int width;                  /* 0=numeric, otherwise string width. */
-    int fv;                     /* Index into case. */
-  };
+{
+  int width;                  /* 0=numeric, otherwise string width. */
+  int fv;                     /* Index into case. */
+};

 \f
 /* Utilities. */
 
 \f
 /* Utilities. */
 


@@ -97,7 +103,7 @@ bswap (char *a, char *b)
 
 /* Reverse the byte order of 32-bit integer *X. */
 static inline void
 
 /* Reverse the byte order of 32-bit integer *X. */
 static inline void

-bswap_int32 (int32 *x_)
+bswap_int32 (int32_t *x_)

 {
   char *x = (char *) x_;
   bswap (x + 0, x + 3);
 {
   char *x = (char *) x_;
   bswap (x + 0, x + 3);


@@ -119,21 +125,26 @@ static void
 corrupt_msg (int class, const char *format,...)
      PRINTF_FORMAT (2, 3);
 
 corrupt_msg (int class, const char *format,...)
      PRINTF_FORMAT (2, 3);
 

-/* Displays a corrupt sysfile error. */
-static void
-corrupt_msg (int class, const char *format,...)
+     /* Displays a corrupt sysfile error. */
+     static void
+     corrupt_msg (int class, const char *format,...)

 {
 {

-  struct error e;
+  struct msg m;

   va_list args;
   va_list args;

+  struct string text;

 
 

-  e.class = class;
-  e.where.filename = NULL;
-  e.where.line_number = 0;
-  e.title = _("corrupt system file: ");
-
+  ds_init_cstr (&text, _("corrupt system file: "));

   va_start (args, format);
   va_start (args, format);

-  err_vmsg (&e, format, args);
+  ds_put_vformat (&text, format, args);

   va_end (args);
   va_end (args);

+
+  m.category = msg_class_to_category (class);
+  m.severity = msg_class_to_severity (class);
+  m.where.file_name = NULL;
+  m.where.line_number = 0;
+  m.text = ds_cstr (&text);
+
+  msg_emit (&m);

 }
 
 /* Closes a system file after we're done with it. */
 }
 
 /* Closes a system file after we're done with it. */


@@ -145,15 +156,15 @@ sfm_close_reader (struct sfm_reader *r)
 
   if (r->file)
     {
 
   if (r->file)
     {

-      if (fn_close (fh_get_filename (r->fh), r->file) == EOF)
+      if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)

         msg (ME, _("%s: Closing system file: %s."),
         msg (ME, _("%s: Closing system file: %s."),

-             fh_get_filename (r->fh), strerror (errno));
+             fh_get_file_name (r->fh), strerror (errno));

       r->file = NULL;
     }
 
   if (r->fh != NULL)
     fh_close (r->fh, "system file", "rs");
       r->file = NULL;
     }
 
   if (r->fh != NULL)
     fh_close (r->fh, "system file", "rs");

-  
+

   free (r->vars);
   free (r->buf);
   free (r);
   free (r->vars);
   free (r->buf);
   free (r);


@@ -168,7 +179,7 @@ static void *buf_read (struct sfm_reader *, void *buf, size_t byte_cnt,
 
 static int read_header (struct sfm_reader *,
                         struct dictionary *, struct sfm_read_info *);
 
 static int read_header (struct sfm_reader *,
                         struct dictionary *, struct sfm_read_info *);

-static int parse_format_spec (struct sfm_reader *, int32,
+static int parse_format_spec (struct sfm_reader *, int32_t,

                              struct fmt_spec *, const struct variable *);
 static int read_value_labels (struct sfm_reader *, struct dictionary *,
                               struct variable **var_by_idx);
                              struct fmt_spec *, const struct variable *);
 static int read_value_labels (struct sfm_reader *, struct dictionary *,
                               struct variable **var_by_idx);


@@ -196,6 +207,70 @@ static int fread_ok (struct sfm_reader *, void *, size_t);
              goto error;                       \
        } while (0)
 
              goto error;                       \
        } while (0)
 

+
+struct name_pair
+{
+  char *shortname;
+  char *longname;
+};
+
+static int
+pair_sn_compare(const void *_p1, const void *_p2, const void *aux UNUSED)
+{
+  int i;
+
+  const struct name_pair *p1 = _p1;
+  const struct name_pair *p2 = _p2;
+
+  char buf1[SHORT_NAME_LEN + 1];
+  char buf2[SHORT_NAME_LEN + 1];
+
+  memset(buf1, 0, SHORT_NAME_LEN + 1);
+  memset(buf2, 0, SHORT_NAME_LEN + 1);
+
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf1[i] = p1->shortname[i];
+      if ( '\0' == buf1[i]) 
+       break;
+    }
+
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf2[i] = p2->shortname[i];
+      if ( '\0' == buf2[i]) 
+       break;
+    }
+
+  return strncmp(buf1, buf2, SHORT_NAME_LEN);
+}
+
+static unsigned int
+pair_sn_hash(const void *_p, const void *aux UNUSED)
+{
+  int i;
+  const struct name_pair *p = _p;
+  char buf[SHORT_NAME_LEN + 1];
+
+  memset(buf, 0, SHORT_NAME_LEN + 1); 
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf[i] = p->shortname[i];
+      if ( '\0' == buf[i]) 
+       break;
+    }
+
+  return hsh_hash_bytes(buf, strlen(buf));
+}
+
+static void
+pair_sn_free(void *p, const void *aux UNUSED)
+{
+  free(p);
+}
+
+
+

 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the
 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the


@@ -207,6 +282,12 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   struct sfm_reader *r = NULL;
   struct variable **var_by_idx = NULL;
 
   struct sfm_reader *r = NULL;
   struct variable **var_by_idx = NULL;
 

+  /* The data in record 7(14) */
+  char *subrec14data = 0;
+
+  /* A hash table of long variable names indexed by short name */
+  struct hsh_table *short_to_long = NULL;
+

   *dict = dict_create ();
   if (!fh_open (fh, FH_REF_FILE, "system file", "rs"))
     goto error;
   *dict = dict_create ();
   if (!fh_open (fh, FH_REF_FILE, "system file", "rs"))
     goto error;


@@ -214,16 +295,16 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   /* Create and initialize reader. */
   r = xmalloc (sizeof *r);
   r->fh = fh;
   /* Create and initialize reader. */
   r = xmalloc (sizeof *r);
   r->fh = fh;

-  r->file = fn_open (fh_get_filename (fh), "rb");
+  r->file = fn_open (fh_get_file_name (fh), "rb");

 
   r->reverse_endian = 0;
 
   r->reverse_endian = 0;

-  r->fix_specials = 0;

   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;
   r->bias = 100.0;
   r->weight_idx = -1;
   r->ok = true;
   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;
   r->bias = 100.0;
   r->weight_idx = -1;
   r->ok = true;

+  r->has_vls = false;

 
   r->vars = NULL;
 
 
   r->vars = NULL;
 


@@ -239,7 +320,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
     {
       msg (ME, _("An error occurred while opening \"%s\" for reading "
                  "as a system file: %s."),
     {
       msg (ME, _("An error occurred while opening \"%s\" for reading "
                  "as a system file: %s."),

-           fh_get_filename (r->fh), strerror (errno));
+           fh_get_file_name (r->fh), strerror (errno));

       goto error;
     }
 
       goto error;
     }
 


@@ -256,7 +337,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
       if (r->weight_idx < 0 || r->weight_idx >= r->value_cnt)
        lose ((ME, _("%s: Index of weighting variable (%d) is not between 0 "
                     "and number of elements per case (%d)."),
       if (r->weight_idx < 0 || r->weight_idx >= r->value_cnt)
        lose ((ME, _("%s: Index of weighting variable (%d) is not between 0 "
                     "and number of elements per case (%d)."),

-              fh_get_filename (r->fh), r->weight_idx, r->value_cnt));
+              fh_get_file_name (r->fh), r->weight_idx, r->value_cnt));

 
 
       weight_var = var_by_idx[r->weight_idx];
 
 
       weight_var = var_by_idx[r->weight_idx];


@@ -264,10 +345,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
       if (weight_var == NULL)
        lose ((ME,
                _("%s: Weighting variable may not be a continuation of "
       if (weight_var == NULL)
        lose ((ME,
                _("%s: Weighting variable may not be a continuation of "

-              "a long string variable."), fh_get_filename (fh)));
+                "a long string variable."), fh_get_file_name (fh)));

       else if (weight_var->type == ALPHA)
        lose ((ME, _("%s: Weighting variable may not be a string variable."),
       else if (weight_var->type == ALPHA)
        lose ((ME, _("%s: Weighting variable may not be a string variable."),

-              fh_get_filename (fh)));
+              fh_get_file_name (fh)));

 
       dict_set_weight (*dict, weight_var);
     }
 
       dict_set_weight (*dict, weight_var);
     }


@@ -277,12 +358,13 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   /* Read records of types 3, 4, 6, and 7. */
   for (;;)
     {
   /* Read records of types 3, 4, 6, and 7. */
   for (;;)
     {

-      int32 rec_type;
+      int32_t rec_type;

 
       assertive_buf_read (r, &rec_type, sizeof rec_type, 0);
       if (r->reverse_endian)
        bswap_int32 (&rec_type);
 
 
       assertive_buf_read (r, &rec_type, sizeof rec_type, 0);
       if (r->reverse_endian)
        bswap_int32 (&rec_type);
 

+

       switch (rec_type)
        {
        case 3:
       switch (rec_type)
        {
        case 3:


@@ -294,7 +376,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
          lose ((ME, _("%s: Orphaned variable index record (type 4).  Type 4 "
                        "records must always immediately follow type 3 "
                        "records."),
          lose ((ME, _("%s: Orphaned variable index record (type 4).  Type 4 "
                        "records must always immediately follow type 3 "
                        "records."),

-                fh_get_filename (r->fh)));
+                fh_get_file_name (r->fh)));

 
        case 6:
          if (!read_documents (r, *dict))
 
        case 6:
          if (!read_documents (r, *dict))


@@ -304,11 +386,11 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
        case 7:
          {
            struct
        case 7:
          {
            struct

-             {
-               int32 subtype P;
-               int32 size P;
-               int32 count P;
-             }
+           {
+             int32_t subtype ;
+             int32_t size ;
+             int32_t count ;
+           } ATTRIBUTE((packed)) 

            data;
             unsigned long bytes;
 
            data;
             unsigned long bytes;
 


@@ -322,9 +404,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                bswap_int32 (&data.count);
              }
             bytes = data.size * data.count;
                bswap_int32 (&data.count);
              }
             bytes = data.size * data.count;

+

             if (bytes < data.size || bytes < data.count)
               lose ((ME, "%s: Record type %d subtype %d too large.",
             if (bytes < data.size || bytes < data.count)
               lose ((ME, "%s: Record type %d subtype %d too large.",

-                     fh_get_filename (r->fh), rec_type, data.subtype));
+                     fh_get_file_name (r->fh), rec_type, data.subtype));

 
            switch (data.subtype)
              {
 
            switch (data.subtype)
              {


@@ -347,28 +430,39 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                {
                  const int  n_vars = data.count / 3 ;
                  int i;
                {
                  const int  n_vars = data.count / 3 ;
                  int i;

-                 if ( data.count % 3 || n_vars > dict_get_var_cnt(*dict) ) 
+                 if ( data.count % 3 || n_vars != dict_get_var_cnt(*dict) ) 

                    {
                      msg (MW, _("%s: Invalid subrecord length. "
                                 "Record: 7; Subrecord: 11"), 
                    {
                      msg (MW, _("%s: Invalid subrecord length. "
                                 "Record: 7; Subrecord: 11"), 

-                          fh_get_filename (r->fh));
+                          fh_get_file_name (r->fh));

                      skip = 1;
                      skip = 1;

+                     break;

                    }
 
                  for ( i = 0 ; i < min(n_vars, dict_get_var_cnt(*dict)) ; ++i ) 
                    {
                      struct
                      {
                    }
 
                  for ( i = 0 ; i < min(n_vars, dict_get_var_cnt(*dict)) ; ++i ) 
                    {
                      struct
                      {

-                       int32 measure P;
-                       int32 width P;
-                       int32 align P;
-                     }
+                       int32_t measure ;
+                       int32_t width ;
+                       int32_t align ;
+                     } ATTRIBUTE((packed))

                      params;
 
                      struct variable *v;
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 
                      params;
 
                      struct variable *v;
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 

+                     if ( ! measure_is_valid(params.measure) 
+                          || 
+                          ! alignment_is_valid(params.align))
+                       {
+                         msg(MW, 
+                             _("%s: Invalid variable display parameters.  Default parameters substituted."), 
+                             fh_get_file_name(r->fh));
+                         continue;
+                       }
+

                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;
                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;


@@ -380,23 +474,30 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
              case 13: /* SPSS 12.0 Long variable name map */
                {
 
              case 13: /* SPSS 12.0 Long variable name map */
                {

-                 char *buf, *short_name, *save_ptr;
+                 char *short_name; 
+                 char *save_ptr = NULL;

                   int idx;
 
                   /* Read data. */
                   int idx;
 
                   /* Read data. */

-                  buf = xmalloc (bytes + 1);
-                 if (!buf_read (r, buf, bytes, 0)) 
+                  subrec14data = xmalloc (bytes + 1);
+                 if (!buf_read (r, subrec14data, bytes, 0)) 

                     {
                     {

-                      free (buf);

                       goto error;
                     }
                       goto error;
                     }

-                 buf[bytes] = '\0';
+                 subrec14data[bytes] = '\0';
+
+                 short_to_long = hsh_create(4, 
+                                            pair_sn_compare,
+                                            pair_sn_hash,
+                                            pair_sn_free, 
+                                            0);

 
                   /* Parse data. */
 
                   /* Parse data. */

-                 for (short_name = strtok_r (buf, "=", &save_ptr), idx = 0;
+                 for (short_name = strtok_r (subrec14data, "=", &save_ptr), idx = 0;

                        short_name != NULL;
                        short_name = strtok_r (NULL, "=", &save_ptr), idx++)
                    {
                        short_name != NULL;
                        short_name = strtok_r (NULL, "=", &save_ptr), idx++)
                    {

+                     struct name_pair *pair ;

                       char *long_name = strtok_r (NULL, "\t", &save_ptr);
                       struct variable *v;
 
                       char *long_name = strtok_r (NULL, "\t", &save_ptr);
                       struct variable *v;
 


@@ -405,14 +506,14 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                         {
                           msg (MW, _("%s: Trailing garbage in long variable "
                                      "name map."),
                         {
                           msg (MW, _("%s: Trailing garbage in long variable "
                                      "name map."),

-                               fh_get_filename (r->fh));
+                               fh_get_file_name (r->fh));

                           break;
                         }
                       if (!var_is_valid_name (long_name, false))
                         {
                           msg (MW, _("%s: Long variable mapping to invalid "
                                      "variable name `%s'."),
                           break;
                         }
                       if (!var_is_valid_name (long_name, false))
                         {
                           msg (MW, _("%s: Long variable mapping to invalid "
                                      "variable name `%s'."),

-                               fh_get_filename (r->fh), long_name);
+                               fh_get_file_name (r->fh), long_name);

                           break;
                         }
                       
                           break;
                         }
                       


@@ -422,16 +523,16 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                         {
                           msg (MW, _("%s: Long variable mapping for "
                                      "nonexistent variable %s."),
                         {
                           msg (MW, _("%s: Long variable mapping for "
                                      "nonexistent variable %s."),

-                               fh_get_filename (r->fh), short_name);
+                               fh_get_file_name (r->fh), short_name);

                           break;
                         }
 
                       /* Identify any duplicates. */
                      if ( compare_var_names(short_name, long_name, 0) &&
                           break;
                         }
 
                       /* Identify any duplicates. */
                      if ( compare_var_names(short_name, long_name, 0) &&

-                         NULL != dict_lookup_var (*dict, long_name))
+                          NULL != dict_lookup_var (*dict, long_name))

                         lose ((ME, _("%s: Duplicate long variable name `%s' "
                                      "within system file."),
                         lose ((ME, _("%s: Duplicate long variable name `%s' "
                                      "within system file."),

-                               fh_get_filename (r->fh), long_name));
+                               fh_get_file_name (r->fh), long_name));

 
 
                       /* Set long name.
 
 
                       /* Set long name.


@@ -441,24 +542,145 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                       dict_rename_var (*dict, v, long_name);
                       var_set_short_name (v, short_name);
 
                       dict_rename_var (*dict, v, long_name);
                       var_set_short_name (v, short_name);
 

-                      /* For compatability, make sure dictionary
+                     pair = xmalloc(sizeof *pair);
+                     pair->shortname = short_name;
+                     pair->longname = long_name;
+                     hsh_insert(short_to_long, pair);
+#if 0 
+      /* This messes up the processing of subtype 14 (below).
+        I'm not sure if it is needed anyway, so I'm removing it for
+        now.  If it's needed, then it will need to be done after all the
+        records have been processed. --- JMD 27 April 2006
+      */
+                     
+                      /* For compatibility, make sure dictionary

                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long
                          variable name map are already in the
                          same order. */
                       dict_reorder_var (*dict, v, idx);
                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long
                          variable name map are already in the
                          same order. */
                       dict_reorder_var (*dict, v, idx);

+#endif

                    }
                    }

+                 
+               }
+               break;
+
+             case 14:
+               {
+                 int j = 0;
+                 bool eq_seen = false;
+                 int i;
+
+                  /* Read data. */
+                  char *buffer = xmalloc (bytes + 1);
+                 if (!buf_read (r, buffer, bytes, 0)) 
+                    {
+                      free (buffer);
+                      goto error;
+                    }
+                 buffer[bytes] = '\0';

 
 

-                 /* Free data. */
-                 free (buf);
+                 r->has_vls = true;
+
+                 /* Note:  SPSS v13 terminates this record with 00,
+                    whereas SPSS v14 terminates it with 00 09. We must
+                    accept either */ 
+                 for(i = 0; i < bytes ; ++i)
+                   {
+                     long int length;
+                     static char name[SHORT_NAME_LEN + 1]  = {0};
+                     static char len_str[6]  ={0};
+
+                     switch( buffer[i] )
+                       {
+                       case '=':
+                         eq_seen = true;
+                         j = 0;
+                         break;
+                       case '\0':
+                         length = strtol(len_str, 0, 10);
+                         if ( length != LONG_MAX && length != LONG_MIN) 
+                           {
+                             char *lookup_name = name;
+                             int l;
+                             int idx;
+                             struct variable *v;
+
+                             if ( short_to_long ) 
+                               {
+                                 struct name_pair pair;
+                                 struct name_pair *p;
+
+                                 pair.shortname = name;
+                                 p = hsh_find(short_to_long, &pair);
+                                 if ( p ) 
+                                   lookup_name = p->longname;
+                               }
+                               
+                             v = dict_lookup_var(*dict, lookup_name);
+                             if ( !v ) 
+                               {
+                                 corrupt_msg(MW, 
+                                             _("%s: No variable called %s but it is listed in length table."),
+                                             fh_get_file_name (r->fh), lookup_name);
+
+                                 goto error;
+
+                               }
+
+                             l = length;
+                             if ( v->width > EFFECTIVE_LONG_STRING_LENGTH ) 
+                               l -= EFFECTIVE_LONG_STRING_LENGTH;
+                             else
+                               l -= v->width;
+
+                             idx = v->index;
+                             while ( l > 0 ) 
+                               {
+                                 struct variable *v_next;
+                                 v_next = dict_get_var(*dict, idx + 1);
+
+                                 if ( v_next->width > EFFECTIVE_LONG_STRING_LENGTH ) 
+                                   l -= EFFECTIVE_LONG_STRING_LENGTH;
+                                 else
+                                   l -= v_next->width;
+
+                                 dict_delete_var(*dict, v_next);
+                               }
+
+                             assert ( length > MAX_LONG_STRING );
+
+                             v->width = length;
+                             v->print.w = v->width;
+                             v->write.w = v->width;
+                             v->nv = DIV_RND_UP (length, MAX_SHORT_STRING);
+                           }
+                         eq_seen = false;
+                         memset(name, 0, SHORT_NAME_LEN+1); 
+                         memset(len_str, 0, 6); 
+                         j = 0;
+                         break;
+                       case '\t':
+                         break;
+                       default:
+                         if ( eq_seen ) 
+                           len_str[j] = buffer[i];
+                         else
+                           name[j] = buffer[i];
+                         j++;
+                         break;
+                       }
+                   }
+                 free(buffer);
+                 dict_compact_values(*dict);

                }
                break;
 
              default:
                msg (MW, _("%s: Unrecognized record type 7, subtype %d "
                            "encountered in system file."),
                }
                break;
 
              default:
                msg (MW, _("%s: Unrecognized record type 7, subtype %d "
                            "encountered in system file."),

-                     fh_get_filename (r->fh), data.subtype);
+                     fh_get_file_name (r->fh), data.subtype);

                skip = 1;
              }
 
                skip = 1;
              }
 


@@ -474,27 +696,51 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
        case 999:
          {
 
        case 999:
          {

-           int32 filler;
+           int32_t filler;

 
            assertive_buf_read (r, &filler, sizeof filler, 0);
 
            assertive_buf_read (r, &filler, sizeof filler, 0);

+

            goto success;
          }
 
        default:
          corrupt_msg(MW, _("%s: Unrecognized record type %d."),
            goto success;
          }
 
        default:
          corrupt_msg(MW, _("%s: Unrecognized record type %d."),

-                 fh_get_filename (r->fh), rec_type);
+                     fh_get_file_name (r->fh), rec_type);

        }
     }
 
        }
     }
 

-success:
+ success:

   /* Come here on successful completion. */
   /* Come here on successful completion. */

+
+  /* Create an index of dictionary variable widths for
+     sfm_read_case to use.  We cannot use the `struct variables'
+     from the dictionary we created, because the caller owns the
+     dictionary and may destroy or modify its variables. */
+  {
+    size_t i;
+
+    r->var_cnt = dict_get_var_cnt (*dict);
+    r->vars = xnmalloc (r->var_cnt, sizeof *r->vars);
+    for (i = 0; i < r->var_cnt; i++) 
+      {
+        struct variable *v = dict_get_var (*dict, i);
+        struct sfm_var *sv = &r->vars[i];
+        sv->width = v->width;
+        sv->fv = v->fv; 
+      }
+  }
+

   free (var_by_idx);
   free (var_by_idx);

+  hsh_destroy(short_to_long);
+  free (subrec14data);

   return r;
 
   return r;
 

-error:
+ error:

   /* Come here on unsuccessful completion. */
   sfm_close_reader (r);
   free (var_by_idx);
   /* Come here on unsuccessful completion. */
   sfm_close_reader (r);
   free (var_by_idx);

+  hsh_destroy(short_to_long);
+  free (subrec14data);

   if (*dict != NULL) 
     {
       dict_destroy (*dict);
   if (*dict != NULL) 
     {
       dict_destroy (*dict);


@@ -507,15 +753,15 @@ error:
 static int
 read_machine_int32_info (struct sfm_reader *r, int size, int count)
 {
 static int
 read_machine_int32_info (struct sfm_reader *r, int size, int count)
 {

-  int32 data[8];
+  int32_t data[8];

   int file_bigendian;
 
   int i;
 
   int file_bigendian;
 
   int i;
 

-  if (size != sizeof (int32) || count != 8)
+  if (size != sizeof (int32_t) || count != 8)

     lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
                  "subtype 3.   Expected size %d, count 8."),
     lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
                  "subtype 3.   Expected size %d, count 8."),

-          fh_get_filename (r->fh), size, count, sizeof (int32)));
+          fh_get_file_name (r->fh), size, count, sizeof (int32_t)));

 
   assertive_buf_read (r, data, sizeof data, 0);
   if (r->reverse_endian)
 
   assertive_buf_read (r, data, sizeof data, 0);
   if (r->reverse_endian)


@@ -527,7 +773,7 @@ read_machine_int32_info (struct sfm_reader *r, int size, int count)
     lose ((ME, _("%s: Floating-point representation in system file is not "
                  "IEEE-754.  PSPP cannot convert between floating-point "
                  "formats."),
     lose ((ME, _("%s: Floating-point representation in system file is not "
                  "IEEE-754.  PSPP cannot convert between floating-point "
                  "formats."),

-           fh_get_filename (r->fh)));
+           fh_get_file_name (r->fh)));

 #else
 #error Add support for your floating-point format.
 #endif
 #else
 #error Add support for your floating-point format.
 #endif


@@ -542,22 +788,22 @@ read_machine_int32_info (struct sfm_reader *r, int size, int count)
   if (file_bigendian ^ (data[6] == 1))
     lose ((ME, _("%s: File-indicated endianness (%s) does not match "
                  "endianness intuited from file header (%s)."),
   if (file_bigendian ^ (data[6] == 1))
     lose ((ME, _("%s: File-indicated endianness (%s) does not match "
                  "endianness intuited from file header (%s)."),

-          fh_get_filename (r->fh),
+          fh_get_file_name (r->fh),

            file_bigendian ? _("big-endian") : _("little-endian"),
           data[6] == 1 ? _("big-endian") : (data[6] == 2 ? _("little-endian")
            file_bigendian ? _("big-endian") : _("little-endian"),
           data[6] == 1 ? _("big-endian") : (data[6] == 2 ? _("little-endian")

-                                         : _("unknown"))));
+                                            : _("unknown"))));

 
   /* PORTME: Character representation code. */
   if (data[7] != 2 && data[7] != 3) 
     lose ((ME, _("%s: File-indicated character representation code (%s) is "
                  "not ASCII."),
 
   /* PORTME: Character representation code. */
   if (data[7] != 2 && data[7] != 3) 
     lose ((ME, _("%s: File-indicated character representation code (%s) is "
                  "not ASCII."),

-           fh_get_filename (r->fh),
+           fh_get_file_name (r->fh),

            (data[7] == 1 ? "EBCDIC"
             : (data[7] == 4 ? _("DEC Kanji") : _("Unknown")))));
 
   return 1;
 
            (data[7] == 1 ? "EBCDIC"
             : (data[7] == 4 ? _("DEC Kanji") : _("Unknown")))));
 
   return 1;
 

-error:
+ error:

   return 0;
 }
 
   return 0;
 }
 


@@ -571,7 +817,7 @@ read_machine_flt64_info (struct sfm_reader *r, int size, int count)
   if (size != sizeof (flt64) || count != 3)
     lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
                  "subtype 4.   Expected size %d, count 8."),
   if (size != sizeof (flt64) || count != 3)
     lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
                  "subtype 4.   Expected size %d, count 8."),

-          fh_get_filename (r->fh), size, count, sizeof (flt64)));
+          fh_get_file_name (r->fh), size, count, sizeof (flt64)));

 
   assertive_buf_read (r, data, sizeof data, 0);
   if (r->reverse_endian)
 
   assertive_buf_read (r, data, sizeof data, 0);
   if (r->reverse_endian)


@@ -588,14 +834,14 @@ read_machine_flt64_info (struct sfm_reader *r, int size, int count)
                 "for at least one of the three system values.  SYSMIS: "
                 "indicated %g, expected %g; HIGHEST: %g, %g; LOWEST: "
                 "%g, %g."),
                 "for at least one of the three system values.  SYSMIS: "
                 "indicated %g, expected %g; HIGHEST: %g, %g; LOWEST: "
                 "%g, %g."),

-          fh_get_filename (r->fh), (double) data[0], (double) SYSMIS,
+          fh_get_file_name (r->fh), (double) data[0], (double) SYSMIS,

           (double) data[1], (double) FLT64_MAX,
           (double) data[2], (double) second_lowest_flt64);
     }
   
   return 1;
 
           (double) data[1], (double) FLT64_MAX,
           (double) data[2], (double) second_lowest_flt64);
     }
   
   return 1;
 

-error:
+ error:

   return 0;
 }
 
   return 0;
 }
 


@@ -613,15 +859,15 @@ read_header (struct sfm_reader *r,
   if (strncmp ("$FL2", hdr.rec_type, 4) != 0)
     lose ((ME, _("%s: Bad magic.  Proper system files begin with "
                 "the four characters `$FL2'. This file will not be read."),
   if (strncmp ("$FL2", hdr.rec_type, 4) != 0)
     lose ((ME, _("%s: Bad magic.  Proper system files begin with "
                 "the four characters `$FL2'. This file will not be read."),

-          fh_get_filename (r->fh)));
+          fh_get_file_name (r->fh)));

 
   /* Check eye-category.her string. */
   memcpy (prod_name, hdr.prod_name, sizeof hdr.prod_name);
   for (i = 0; i < 60; i++)
 
   /* Check eye-category.her string. */
   memcpy (prod_name, hdr.prod_name, sizeof hdr.prod_name);
   for (i = 0; i < 60; i++)

-    if (!isprint ((unsigned char) prod_name[i]))
+    if (!c_isprint ((unsigned char) prod_name[i]))

       prod_name[i] = ' ';
   for (i = 59; i >= 0; i--)
       prod_name[i] = ' ';
   for (i = 59; i >= 0; i--)

-    if (!isgraph ((unsigned char) prod_name[i]))
+    if (!c_isgraph ((unsigned char) prod_name[i]))

       {
        prod_name[i] = '\0';
        break;
       {
        prod_name[i] = '\0';
        break;


@@ -655,10 +901,10 @@ read_header (struct sfm_reader *r,
       if (hdr.layout_code != 2)
        lose ((ME, _("%s: File layout code has unexpected value %d.  Value "
                      "should be 2, in big-endian or little-endian format."),
       if (hdr.layout_code != 2)
        lose ((ME, _("%s: File layout code has unexpected value %d.  Value "
                      "should be 2, in big-endian or little-endian format."),

-              fh_get_filename (r->fh), hdr.layout_code));
+              fh_get_file_name (r->fh), hdr.layout_code));

 
       r->reverse_endian = 1;
 
       r->reverse_endian = 1;

-      bswap_int32 (&hdr.case_size);
+      bswap_int32 (&hdr.nominal_case_size);

       bswap_int32 (&hdr.compress);
       bswap_int32 (&hdr.weight_idx);
       bswap_int32 (&hdr.case_cnt);
       bswap_int32 (&hdr.compress);
       bswap_int32 (&hdr.weight_idx);
       bswap_int32 (&hdr.case_cnt);


@@ -667,9 +913,10 @@ read_header (struct sfm_reader *r,
 
 
   /* Copy basic info and verify correctness. */
 
 
   /* Copy basic info and verify correctness. */

-  r->value_cnt = hdr.case_size;
+  r->value_cnt = hdr.nominal_case_size;

 
 

-  /* If value count is rediculous, then force it to -1 (a sentinel value) */
+  /* If value count is ridiculous, then force it to -1 (a
+     sentinel value). */

   if ( r->value_cnt < 0 || 
        r->value_cnt > (INT_MAX / (int) sizeof (union value) / 2))
     r->value_cnt = -1;
   if ( r->value_cnt < 0 || 
        r->value_cnt > (INT_MAX / (int) sizeof (union value) / 2))
     r->value_cnt = -1;


@@ -682,13 +929,13 @@ read_header (struct sfm_reader *r,
   if (r->case_cnt < -1 || r->case_cnt > INT_MAX / 2)
     lose ((ME,
            _("%s: Number of cases in file (%ld) is not between -1 and %d."),
   if (r->case_cnt < -1 || r->case_cnt > INT_MAX / 2)
     lose ((ME,
            _("%s: Number of cases in file (%ld) is not between -1 and %d."),

-           fh_get_filename (r->fh), (long) r->case_cnt, INT_MAX / 2));
+           fh_get_file_name (r->fh), (long) r->case_cnt, INT_MAX / 2));

 
   r->bias = hdr.bias;
   if (r->bias != 100.0)
     corrupt_msg (MW, _("%s: Compression bias (%g) is not the usual "
                        "value of 100."),
 
   r->bias = hdr.bias;
   if (r->bias != 100.0)
     corrupt_msg (MW, _("%s: Compression bias (%g) is not the usual "
                        "value of 100."),

-                 fh_get_filename (r->fh), r->bias);
+                 fh_get_file_name (r->fh), r->bias);

 
   /* Make a file label only on the condition that the given label is
      not all spaces or nulls. */
 
   /* Make a file label only on the condition that the given label is
      not all spaces or nulls. */


@@ -696,16 +943,18 @@ read_header (struct sfm_reader *r,
     int i;
 
     for (i = sizeof hdr.file_label - 1; i >= 0; i--)
     int i;
 
     for (i = sizeof hdr.file_label - 1; i >= 0; i--)

-      if (!isspace ((unsigned char) hdr.file_label[i])
-         && hdr.file_label[i] != 0)
-       {
-          char *label = xmalloc (i + 2);
-         memcpy (label, hdr.file_label, i + 1);
-         label[i + 1] = 0;
-          dict_set_label (dict, label);
-          free (label);
-         break;
-       }
+      {
+       if (!c_isspace ((unsigned char) hdr.file_label[i])
+           && hdr.file_label[i] != 0)
+         {
+           char *label = xmalloc (i + 2);
+           memcpy (label, hdr.file_label, i + 1);
+           label[i + 1] = 0;
+           dict_set_label (dict, label);
+           free (label);
+           break;
+         }
+      }

   }
 
   if (info)
   }
 
   if (info)


@@ -729,14 +978,14 @@ read_header (struct sfm_reader *r,
       info->case_cnt = hdr.case_cnt;
 
       for (cp = &prod_name[skip_amt]; cp < &prod_name[60]; cp++)
       info->case_cnt = hdr.case_cnt;
 
       for (cp = &prod_name[skip_amt]; cp < &prod_name[60]; cp++)

-       if (isgraph ((unsigned char) *cp))
+       if (c_isgraph ((unsigned char) *cp))

          break;
       strcpy (info->product, cp);
     }
 
   return 1;
 
          break;
       strcpy (info->product, cp);
     }
 
   return 1;
 

-error:
+ error:

   return 0;
 }
 
   return 0;
 }
 


@@ -757,13 +1006,6 @@ read_variables (struct sfm_reader *r,
 
   *var_by_idx = 0;
 
 
   *var_by_idx = 0;
 

-  /* Pre-allocate variables. */
-  if (r->value_cnt != -1) 
-    {
-      *var_by_idx = xnmalloc (r->value_cnt, sizeof **var_by_idx);
-      r->vars = xnmalloc (r->value_cnt, sizeof *r->vars);
-    }
-

 
   /* Read in the entry for each variable and use the info to
      initialize the dictionary. */
 
   /* Read in the entry for each variable and use the info to
      initialize the dictionary. */


@@ -774,9 +1016,6 @@ read_variables (struct sfm_reader *r,
       int nv;
       int j;
 
       int nv;
       int j;
 

-      if ( r->value_cnt != -1  && i >= r->value_cnt ) 
-       break;
-

       assertive_buf_read (r, &sv, sizeof sv, 0);
 
       if (r->reverse_endian)
       assertive_buf_read (r, &sv, sizeof sv, 0);
 
       if (r->reverse_endian)


@@ -797,11 +1036,7 @@ read_variables (struct sfm_reader *r,
          break;
        }
 
          break;
        }
 

-      if ( -1 == r->value_cnt ) 
-       {
-         *var_by_idx = xnrealloc (*var_by_idx, i + 1, sizeof **var_by_idx);
-         r->vars = xnrealloc (r->vars, i + 1, sizeof *r->vars);
-       }
+      *var_by_idx = xnrealloc (*var_by_idx, i + 1, sizeof **var_by_idx);

 
       /* If there was a long string previously, make sure that the
         continuations are present; otherwise make sure there aren't
 
       /* If there was a long string previously, make sure that the
         continuations are present; otherwise make sure there aren't


@@ -811,10 +1046,9 @@ read_variables (struct sfm_reader *r,
          if (sv.type != -1)
            lose ((ME, _("%s: position %d: String variable does not have "
                         "proper number of continuation records."),
          if (sv.type != -1)
            lose ((ME, _("%s: position %d: String variable does not have "
                         "proper number of continuation records."),

-                   fh_get_filename (r->fh), i));
+                   fh_get_file_name (r->fh), i));

 
 
 
 

-         r->vars[i].width = -1;

          (*var_by_idx)[i] = NULL;
          long_string_count--;
          continue;
          (*var_by_idx)[i] = NULL;
          long_string_count--;
          continue;


@@ -822,72 +1056,51 @@ read_variables (struct sfm_reader *r,
       else if (sv.type == -1)
        lose ((ME, _("%s: position %d: Superfluous long string continuation "
                      "record."),
       else if (sv.type == -1)
        lose ((ME, _("%s: position %d: Superfluous long string continuation "
                      "record."),

-               fh_get_filename (r->fh), i));
+               fh_get_file_name (r->fh), i));

 
       /* Check fields for validity. */
       if (sv.type < 0 || sv.type > 255)
        lose ((ME, _("%s: position %d: Bad variable type code %d."),
 
       /* Check fields for validity. */
       if (sv.type < 0 || sv.type > 255)
        lose ((ME, _("%s: position %d: Bad variable type code %d."),

-              fh_get_filename (r->fh), i, sv.type));
+              fh_get_file_name (r->fh), i, sv.type));

       if (sv.has_var_label != 0 && sv.has_var_label != 1)
        lose ((ME, _("%s: position %d: Variable label indicator field is not "
       if (sv.has_var_label != 0 && sv.has_var_label != 1)
        lose ((ME, _("%s: position %d: Variable label indicator field is not "

-              "0 or 1."), fh_get_filename (r->fh), i));
+                    "0 or 1."), fh_get_file_name (r->fh), i));

       if (sv.n_missing_values < -3 || sv.n_missing_values > 3
          || sv.n_missing_values == -1)
        lose ((ME, _("%s: position %d: Missing value indicator field is not "
       if (sv.n_missing_values < -3 || sv.n_missing_values > 3
          || sv.n_missing_values == -1)
        lose ((ME, _("%s: position %d: Missing value indicator field is not "

-                    "-3, -2, 0, 1, 2, or 3."), fh_get_filename (r->fh), i));
+                    "-3, -2, 0, 1, 2, or 3."), fh_get_file_name (r->fh), i));

 
       /* Copy first character of variable name. */
 
       /* Copy first character of variable name. */

-      if (!isalpha ((unsigned char) sv.name[0])
-         && sv.name[0] != '@' && sv.name[0] != '#')
+      if (sv.name[0] == '@' || sv.name[0] == '#')

        lose ((ME, _("%s: position %d: Variable name begins with invalid "
                      "character."),
        lose ((ME, _("%s: position %d: Variable name begins with invalid "
                      "character."),

-               fh_get_filename (r->fh), i));
-      if (islower ((unsigned char) sv.name[0]))
-       msg (MW, _("%s: position %d: Variable name begins with lowercase letter "
-                   "%c."),
-             fh_get_filename (r->fh), i, sv.name[0]);
-      if (sv.name[0] == '#')
-       msg (MW, _("%s: position %d: Variable name begins with octothorpe "
-                  "(`#').  Scratch variables should not appear in system "
-                  "files."),
-             fh_get_filename (r->fh), i);
-      name[0] = toupper ((unsigned char) (sv.name[0]));
+               fh_get_file_name (r->fh), i));
+
+      name[0] = sv.name[0];

 
       /* Copy remaining characters of variable name. */
       for (j = 1; j < SHORT_NAME_LEN; j++)
        {
          int c = (unsigned char) sv.name[j];
 
 
       /* Copy remaining characters of variable name. */
       for (j = 1; j < SHORT_NAME_LEN; j++)
        {
          int c = (unsigned char) sv.name[j];
 

-         if (isspace (c))
+         if (c == ' ') 

            break;
            break;

-         else if (islower (c))
-           {
-             msg (MW, _("%s: position %d: Variable name character %d is "
-                         "lowercase letter %c."),
-                   fh_get_filename (r->fh), i, j + 1, sv.name[j]);
-             name[j] = toupper ((unsigned char) (c));
-           }
-         else if (isalnum (c) || c == '.' || c == '@'
-                  || c == '#' || c == '$' || c == '_')
+         else 

            name[j] = c;
            name[j] = c;

-         else
-           lose ((ME, _("%s: position %d: character `\\%03o' (%c) is not valid in a "
-                         "variable name."),
-                   fh_get_filename (r->fh), i, c, c));

        }
       name[j] = 0;
 
        }
       name[j] = 0;
 

-      if ( ! var_is_valid_name(name, false) ) 
+      if ( ! var_is_plausible_name(name, false) ) 

         lose ((ME, _("%s: Invalid variable name `%s' within system file."),
         lose ((ME, _("%s: Invalid variable name `%s' within system file."),

-               fh_get_filename (r->fh), name));
+               fh_get_file_name (r->fh), name));

 
       /* Create variable. */
 
       /* Create variable. */

-

       vv = (*var_by_idx)[i] = dict_create_var (dict, name, sv.type);
       if (vv == NULL) 
         lose ((ME, _("%s: Duplicate variable name `%s' within system file."),
       vv = (*var_by_idx)[i] = dict_create_var (dict, name, sv.type);
       if (vv == NULL) 
         lose ((ME, _("%s: Duplicate variable name `%s' within system file."),

-               fh_get_filename (r->fh), name));
+               fh_get_file_name (r->fh), name));

 
 

+      /* Set the short name the same as the long name */

       var_set_short_name (vv, vv->name);
 
       /* Case reading data. */
       var_set_short_name (vv, vv->name);
 
       /* Case reading data. */


@@ -899,7 +1112,7 @@ read_variables (struct sfm_reader *r,
       if (sv.has_var_label == 1)
        {
          /* Disk buffer. */
       if (sv.has_var_label == 1)
        {
          /* Disk buffer. */

-         int32 len;
+         int32_t len;

 
          /* Read length of label. */
          assertive_buf_read (r, &len, sizeof len, 0);
 
          /* Read length of label. */
          assertive_buf_read (r, &len, sizeof len, 0);


@@ -910,12 +1123,12 @@ read_variables (struct sfm_reader *r,
          if (len < 0 || len > 255)
            lose ((ME, _("%s: Variable %s indicates variable label of invalid "
                          "length %d."),
          if (len < 0 || len > 255)
            lose ((ME, _("%s: Variable %s indicates variable label of invalid "
                          "length %d."),

-                   fh_get_filename (r->fh), vv->name, len));
+                   fh_get_file_name (r->fh), vv->name, len));

 
          if ( len != 0 ) 
            {
              /* Read label into variable structure. */
 
          if ( len != 0 ) 
            {
              /* Read label into variable structure. */

-             vv->label = buf_read (r, NULL, ROUND_UP (len, sizeof (int32)), len + 1);
+             vv->label = buf_read (r, NULL, ROUND_UP (len, sizeof (int32_t)), len + 1);

              if (vv->label == NULL)
                goto error;
              vv->label[len] = '\0';
              if (vv->label == NULL)
                goto error;
              vv->label[len] = '\0';


@@ -931,7 +1144,7 @@ read_variables (struct sfm_reader *r,
          if (vv->width > MAX_SHORT_STRING)
            lose ((ME, _("%s: Long string variable %s may not have missing "
                          "values."),
          if (vv->width > MAX_SHORT_STRING)
            lose ((ME, _("%s: Long string variable %s may not have missing "
                          "values."),

-                   fh_get_filename (r->fh), vv->name));
+                   fh_get_file_name (r->fh), vv->name));

 
          assertive_buf_read (r, mv, sizeof *mv * mv_cnt, 0);
 
 
          assertive_buf_read (r, mv, sizeof *mv * mv_cnt, 0);
 


@@ -952,7 +1165,7 @@ read_variables (struct sfm_reader *r,
              if (vv->type == ALPHA)
                lose ((ME, _("%s: String variable %s may not have missing "
                              "values specified as a range."),
              if (vv->type == ALPHA)
                lose ((ME, _("%s: String variable %s may not have missing "
                              "values specified as a range."),

-                       fh_get_filename (r->fh), vv->name));
+                       fh_get_file_name (r->fh), vv->name));

 
              if (mv[0] == r->lowest)
                 mv_add_num_range (&vv->miss, LOWEST, mv[1]);
 
              if (mv[0] == r->lowest)
                 mv_add_num_range (&vv->miss, LOWEST, mv[1]);


@@ -969,46 +1182,42 @@ read_variables (struct sfm_reader *r,
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;

-
-      r->vars[i].width = vv->width;
-      r->vars[i].fv = vv->fv;
-

     }
 
   /* Some consistency checks. */
   if (long_string_count != 0)
     lose ((ME, _("%s: Long string continuation records omitted at end of "
                  "dictionary."),
     }
 
   /* Some consistency checks. */
   if (long_string_count != 0)
     lose ((ME, _("%s: Long string continuation records omitted at end of "
                  "dictionary."),

-           fh_get_filename (r->fh)));
+           fh_get_file_name (r->fh)));

 
   if (next_value != r->value_cnt)
     corrupt_msg(MW, _("%s: System file header indicates %d variable positions but "
 
   if (next_value != r->value_cnt)
     corrupt_msg(MW, _("%s: System file header indicates %d variable positions but "

-                 "%d were read from file."),
-           fh_get_filename (r->fh), r->value_cnt, next_value);
+                     "%d were read from file."),
+               fh_get_file_name (r->fh), r->value_cnt, next_value);

 
 
   return 1;
 
 
 
   return 1;
 

-error:
+ error:

   return 0;
 }
 
 /* Translates the format spec from sysfile format to internal
    format. */
 static int
   return 0;
 }
 
 /* Translates the format spec from sysfile format to internal
    format. */
 static int

-parse_format_spec (struct sfm_reader *r, int32 s,
+parse_format_spec (struct sfm_reader *r, int32_t s,

                    struct fmt_spec *f, const struct variable *v)
 {
   f->type = translate_fmt ((s >> 16) & 0xff);
   if (f->type == -1)
     lose ((ME, _("%s: Bad format specifier byte (%d)."),
                    struct fmt_spec *f, const struct variable *v)
 {
   f->type = translate_fmt ((s >> 16) & 0xff);
   if (f->type == -1)
     lose ((ME, _("%s: Bad format specifier byte (%d)."),

-          fh_get_filename (r->fh), (s >> 16) & 0xff));
+          fh_get_file_name (r->fh), (s >> 16) & 0xff));

   f->w = (s >> 8) & 0xff;
   f->d = s & 0xff;
 
   if ((v->type == ALPHA) ^ ((formats[f->type].cat & FCAT_STRING) != 0))
     lose ((ME, _("%s: %s variable %s has %s format specifier %s."),
   f->w = (s >> 8) & 0xff;
   f->d = s & 0xff;
 
   if ((v->type == ALPHA) ^ ((formats[f->type].cat & FCAT_STRING) != 0))
     lose ((ME, _("%s: %s variable %s has %s format specifier %s."),

-          fh_get_filename (r->fh),
+          fh_get_file_name (r->fh),

            v->type == ALPHA ? _("String") : _("Numeric"),
           v->name,
           formats[f->type].cat & FCAT_STRING ? _("string") : _("numeric"),
            v->type == ALPHA ? _("String") : _("Numeric"),
           v->name,
           formats[f->type].cat & FCAT_STRING ? _("string") : _("numeric"),


@@ -1024,7 +1233,7 @@ parse_format_spec (struct sfm_reader *r, int32 s,
     }
   return 1;
 
     }
   return 1;
 

-error:
+ error:

   return 0;
 }
 
   return 0;
 }
 


@@ -1035,17 +1244,17 @@ read_value_labels (struct sfm_reader *r,
                    struct dictionary *dict, struct variable **var_by_idx)
 {
   struct label 
                    struct dictionary *dict, struct variable **var_by_idx)
 {
   struct label 

-    {
-      char raw_value[8];        /* Value as uninterpreted bytes. */
-      union value value;        /* Value. */
-      char *label;              /* Null-terminated label string. */
-    };
+  {
+    char raw_value[8];        /* Value as uninterpreted bytes. */
+    union value value;        /* Value. */
+    char *label;              /* Null-terminated label string. */
+  };

 
   struct label *labels = NULL;
 
   struct label *labels = NULL;

-  int32 n_labels;              /* Number of labels. */
+  int32_t n_labels;            /* Number of labels. */

 
   struct variable **var = NULL;        /* Associated variables. */
 
   struct variable **var = NULL;        /* Associated variables. */

-  int32 n_vars;                        /* Number of associated variables. */
+  int32_t n_vars;                      /* Number of associated variables. */

 
   int i;
 
 
   int i;
 


@@ -1058,10 +1267,10 @@ read_value_labels (struct sfm_reader *r,
   if (r->reverse_endian)
     bswap_int32 (&n_labels);
 
   if (r->reverse_endian)
     bswap_int32 (&n_labels);
 

-  if ( n_labels >= ((int32) ~0) / sizeof *labels)
+  if ( n_labels >= ((int32_t) ~0) / sizeof *labels)

     {    
       corrupt_msg(MW, _("%s: Invalid number of labels: %d.  Ignoring labels."),
     {    
       corrupt_msg(MW, _("%s: Invalid number of labels: %d.  Ignoring labels."),

-                 fh_get_filename (r->fh), n_labels);
+                 fh_get_file_name (r->fh), n_labels);

       n_labels = 0;
     }
 
       n_labels = 0;
     }
 


@@ -1095,7 +1304,7 @@ read_value_labels (struct sfm_reader *r,
 
   /* Read record type of type 4 record. */
   {
 
   /* Read record type of type 4 record. */
   {

-    int32 rec_type;
+    int32_t rec_type;

     
     assertive_buf_read (r, &rec_type, sizeof rec_type, 0);
     if (r->reverse_endian)
     
     assertive_buf_read (r, &rec_type, sizeof rec_type, 0);
     if (r->reverse_endian)


@@ -1104,7 +1313,7 @@ read_value_labels (struct sfm_reader *r,
     if (rec_type != 4)
       lose ((ME, _("%s: Variable index record (type 4) does not immediately "
                    "follow value label record (type 3) as it should."),
     if (rec_type != 4)
       lose ((ME, _("%s: Variable index record (type 4) does not immediately "
                    "follow value label record (type 3) as it should."),

-             fh_get_filename (r->fh)));
+             fh_get_file_name (r->fh)));

   }
 
   /* Read number of variables associated with value label from type 4
   }
 
   /* Read number of variables associated with value label from type 4


@@ -1115,13 +1324,13 @@ read_value_labels (struct sfm_reader *r,
   if (n_vars < 1 || n_vars > dict_get_var_cnt (dict))
     lose ((ME, _("%s: Number of variables associated with a value label (%d) "
                  "is not between 1 and the number of variables (%d)."),
   if (n_vars < 1 || n_vars > dict_get_var_cnt (dict))
     lose ((ME, _("%s: Number of variables associated with a value label (%d) "
                  "is not between 1 and the number of variables (%d)."),

-          fh_get_filename (r->fh), n_vars, dict_get_var_cnt (dict)));
+          fh_get_file_name (r->fh), n_vars, dict_get_var_cnt (dict)));

 
   /* Read the list of variables. */
   var = xnmalloc (n_vars, sizeof *var);
   for (i = 0; i < n_vars; i++)
     {
 
   /* Read the list of variables. */
   var = xnmalloc (n_vars, sizeof *var);
   for (i = 0; i < n_vars; i++)
     {

-      int32 var_idx;
+      int32_t var_idx;

       struct variable *v;
 
       /* Read variable index, check range. */
       struct variable *v;
 
       /* Read variable index, check range. */


@@ -1131,7 +1340,7 @@ read_value_labels (struct sfm_reader *r,
       if (var_idx < 1 || var_idx > r->value_cnt)
        lose ((ME, _("%s: Variable index associated with value label (%d) is "
                      "not between 1 and the number of values (%d)."),
       if (var_idx < 1 || var_idx > r->value_cnt)
        lose ((ME, _("%s: Variable index associated with value label (%d) is "
                      "not between 1 and the number of values (%d)."),

-              fh_get_filename (r->fh), var_idx, r->value_cnt));
+              fh_get_file_name (r->fh), var_idx, r->value_cnt));

 
       /* Make sure it's a real variable. */
       v = var_by_idx[var_idx - 1];
 
       /* Make sure it's a real variable. */
       v = var_by_idx[var_idx - 1];


@@ -1139,11 +1348,11 @@ read_value_labels (struct sfm_reader *r,
        lose ((ME, _("%s: Variable index associated with value label (%d) "
                      "refers to a continuation of a string variable, not to "
                      "an actual variable."),
        lose ((ME, _("%s: Variable index associated with value label (%d) "
                      "refers to a continuation of a string variable, not to "
                      "an actual variable."),

-               fh_get_filename (r->fh), var_idx));
+               fh_get_file_name (r->fh), var_idx));

       if (v->type == ALPHA && v->width > MAX_SHORT_STRING)
        lose ((ME, _("%s: Value labels are not allowed on long string "
                      "variables (%s)."),
       if (v->type == ALPHA && v->width > MAX_SHORT_STRING)
        lose ((ME, _("%s: Value labels are not allowed on long string "
                      "variables (%s)."),

-               fh_get_filename (r->fh), v->name));
+               fh_get_file_name (r->fh), v->name));

 
       /* Add it to the list of variables. */
       var[i] = v;
 
       /* Add it to the list of variables. */
       var[i] = v;


@@ -1155,7 +1364,7 @@ read_value_labels (struct sfm_reader *r,
       lose ((ME, _("%s: Variables associated with value label are not all of "
                    "identical type.  Variable %s has %s type, but variable "
                    "%s has %s type."),
       lose ((ME, _("%s: Variables associated with value label are not all of "
                    "identical type.  Variable %s has %s type, but variable "
                    "%s has %s type."),

-             fh_get_filename (r->fh),
+             fh_get_file_name (r->fh),

             var[0]->name, var[0]->type == ALPHA ? _("string") : _("numeric"),
             var[i]->name, var[i]->type == ALPHA ? _("string") : _("numeric")));
 
             var[0]->name, var[0]->type == ALPHA ? _("string") : _("numeric"),
             var[i]->name, var[i]->type == ALPHA ? _("string") : _("numeric")));
 


@@ -1195,11 +1404,11 @@ read_value_labels (struct sfm_reader *r,
          if (var[0]->type == NUMERIC)
            msg (MW, _("%s: File contains duplicate label for value %g for "
                        "variable %s."),
          if (var[0]->type == NUMERIC)
            msg (MW, _("%s: File contains duplicate label for value %g for "
                        "variable %s."),

-                 fh_get_filename (r->fh), label->value.f, v->name);
+                 fh_get_file_name (r->fh), label->value.f, v->name);

          else
            msg (MW, _("%s: File contains duplicate label for value `%.*s' "
                        "for variable %s."),
          else
            msg (MW, _("%s: File contains duplicate label for value `%.*s' "
                        "for variable %s."),

-                 fh_get_filename (r->fh), v->width, label->value.s, v->name);
+                 fh_get_file_name (r->fh), v->width, label->value.s, v->name);

        }
     }
 
        }
     }
 


@@ -1209,7 +1418,7 @@ read_value_labels (struct sfm_reader *r,
   free (var);
   return 1;
 
   free (var);
   return 1;
 

-error:
+ error:

   if (labels) 
     {
       for (i = 0; i < n_labels; i++)
   if (labels) 
     {
       for (i = 0; i < n_labels; i++)


@@ -1240,13 +1449,14 @@ buf_read (struct sfm_reader *r, void *buf, size_t byte_cnt, size_t min_alloc)
     {
       if (ferror (r->file))
        msg (ME, _("%s: Reading system file: %s."),
     {
       if (ferror (r->file))
        msg (ME, _("%s: Reading system file: %s."),

-             fh_get_filename (r->fh), strerror (errno));
+             fh_get_file_name (r->fh), strerror (errno));

       else
        corrupt_msg (ME, _("%s: Unexpected end of file."),
       else
        corrupt_msg (ME, _("%s: Unexpected end of file."),

-                     fh_get_filename (r->fh));
+                     fh_get_file_name (r->fh));

       r->ok = false;
       return NULL;
     }
       r->ok = false;
       return NULL;
     }

+

   return buf;
 }
 
   return buf;
 }
 


@@ -1259,7 +1469,7 @@ buf_unread(struct sfm_reader *r, size_t byte_cnt)
   if ( 0 != fseek(r->file, -byte_cnt, SEEK_CUR))
     {
       msg (ME, _("%s: Seeking system file: %s."),
   if ( 0 != fseek(r->file, -byte_cnt, SEEK_CUR))
     {
       msg (ME, _("%s: Seeking system file: %s."),

-          fh_get_filename (r->fh), strerror (errno));
+          fh_get_file_name (r->fh), strerror (errno));

     }
 }
 
     }
 }
 


@@ -1269,19 +1479,19 @@ buf_unread(struct sfm_reader *r, size_t byte_cnt)
 static int
 read_documents (struct sfm_reader *r, struct dictionary *dict)
 {
 static int
 read_documents (struct sfm_reader *r, struct dictionary *dict)
 {

-  int32 line_cnt;
+  int32_t line_cnt;

   char *documents;
 
   if (dict_get_documents (dict) != NULL)
     lose ((ME, _("%s: System file contains multiple "
                  "type 6 (document) records."),
   char *documents;
 
   if (dict_get_documents (dict) != NULL)
     lose ((ME, _("%s: System file contains multiple "
                  "type 6 (document) records."),

-          fh_get_filename (r->fh)));
+          fh_get_file_name (r->fh)));

 
   assertive_buf_read (r, &line_cnt, sizeof line_cnt, 0);
   if (line_cnt <= 0)
     lose ((ME, _("%s: Number of document lines (%ld) "
                  "must be greater than 0."),
 
   assertive_buf_read (r, &line_cnt, sizeof line_cnt, 0);
   if (line_cnt <= 0)
     lose ((ME, _("%s: Number of document lines (%ld) "
                  "must be greater than 0."),

-          fh_get_filename (r->fh), (long) line_cnt));
+          fh_get_file_name (r->fh), (long) line_cnt));

 
   documents = buf_read (r, NULL, 80 * line_cnt, line_cnt * 80 + 1);
   /* FIXME?  Run through asciify. */
 
   documents = buf_read (r, NULL, 80 * line_cnt, line_cnt * 80 + 1);
   /* FIXME?  Run through asciify. */


@@ -1292,7 +1502,7 @@ read_documents (struct sfm_reader *r, struct dictionary *dict)
   free (documents);
   return 1;
 
   free (documents);
   return 1;
 

-error:
+ error:

   return 0;
 }
 \f
   return 0;
 }
 \f


@@ -1314,7 +1524,7 @@ buffer_input (struct sfm_reader *r)
   if (ferror (r->file))
     {
       msg (ME, _("%s: Error reading file: %s."),
   if (ferror (r->file))
     {
       msg (ME, _("%s: Error reading file: %s."),

-           fh_get_filename (r->fh), strerror (errno));
+           fh_get_file_name (r->fh), strerror (errno));

       r->ok = false;
       return 0;
     }
       r->ok = false;
       return 0;
     }


@@ -1354,14 +1564,14 @@ read_compressed_data (struct sfm_reader *r, flt64 *buf)
               return 0;
             lose ((ME, _("%s: Compressed data is corrupted.  Data ends "
                          "in partial case."),
               return 0;
             lose ((ME, _("%s: Compressed data is corrupted.  Data ends "
                          "in partial case."),

-                   fh_get_filename (r->fh)));
+                   fh_get_file_name (r->fh)));

          case 253:
            /* Code 253 indicates that the value is stored explicitly
               following the instruction bytes. */
            if (r->ptr == NULL || r->ptr >= r->end)
              if (!buffer_input (r))
                 lose ((ME, _("%s: Unexpected end of file."),
          case 253:
            /* Code 253 indicates that the value is stored explicitly
               following the instruction bytes. */
            if (r->ptr == NULL || r->ptr >= r->end)
              if (!buffer_input (r))
                 lose ((ME, _("%s: Unexpected end of file."),

-                       fh_get_filename (r->fh)));
+                       fh_get_file_name (r->fh)));

            memcpy (buf++, r->ptr++, sizeof *buf);
            if (buf >= buf_end)
              goto success;
            memcpy (buf++, r->ptr++, sizeof *buf);
            if (buf >= buf_end)
              goto success;


@@ -1399,23 +1609,27 @@ read_compressed_data (struct sfm_reader *r, flt64 *buf)
       if (r->ptr == NULL || r->ptr >= r->end) 
         {
           if (!buffer_input (r))
       if (r->ptr == NULL || r->ptr >= r->end) 
         {
           if (!buffer_input (r))

-            if (buf_beg != buf)
-              lose ((ME, _("%s: Unexpected end of file."),
-                     fh_get_filename (r->fh))); 
+           {
+             if (buf_beg != buf)
+               lose ((ME, _("%s: Unexpected end of file."),
+                      fh_get_file_name (r->fh))); 
+             else
+               return 0;
+           }

         }
       memcpy (r->x, r->ptr++, sizeof *buf);
       p = r->x;
     }
 
         }
       memcpy (r->x, r->ptr++, sizeof *buf);
       p = r->x;
     }
 

-  abort ();
+  NOT_REACHED ();

 
 

-success:
+ success:

   /* We have filled up an entire record.  Update state and return
      successfully. */
   r->y = ++p;
   return 1;
 
   /* We have filled up an entire record.  Update state and return
      successfully. */
   r->y = ++p;
   return 1;
 

-error:
+ error:

   /* I/O error. */
   r->ok = false;
   return 0;
   /* I/O error. */
   r->ok = false;
   return 0;


@@ -1428,8 +1642,8 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
 {
   if (!r->ok)
     return 0;
 {
   if (!r->ok)
     return 0;

-  
-  if (!r->compressed && sizeof (flt64) == sizeof (double)) 
+
+  if (!r->compressed && sizeof (flt64) == sizeof (double) && ! r->has_vls) 

     {
       /* Fast path: external and internal representations are the
          same, except possibly for endianness or SYSMIS.  Read
     {
       /* Fast path: external and internal representations are the
          same, except possibly for endianness or SYSMIS.  Read


@@ -1444,7 +1658,7 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
         {
           int i;
           
         {
           int i;
           

-          for (i = 0; i < r->value_cnt; i++) 
+          for (i = 0; i < r->var_cnt; i++) 

             if (r->vars[i].width == 0)
               bswap_flt64 (&case_data_rw (c, r->vars[i].fv)->f);
         }
             if (r->vars[i].width == 0)
               bswap_flt64 (&case_data_rw (c, r->vars[i].fv)->f);
         }


@@ -1456,7 +1670,7 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
         {
           int i;
           
         {
           int i;
           

-          for (i = 0; i < r->value_cnt; i++) 
+          for (i = 0; i < r->var_cnt; i++) 

             if (r->vars[i].width == 0 && case_num (c, i) == r->sysmis)
               case_data_rw (c, r->vars[i].fv)->f = SYSMIS;
         }
             if (r->vars[i].width == 0 && case_num (c, i) == r->sysmis)
               case_data_rw (c, r->vars[i].fv)->f = SYSMIS;
         }


@@ -1474,6 +1688,8 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
       bounce_size = sizeof *bounce * r->value_cnt;
       bounce = bounce_cur = local_alloc (bounce_size);
 
       bounce_size = sizeof *bounce * r->value_cnt;
       bounce = bounce_cur = local_alloc (bounce_size);
 

+      memset(bounce, 0, bounce_size);
+

       if (!r->compressed)
         read_ok = fread_ok (r, bounce, bounce_size);
       else
       if (!r->compressed)
         read_ok = fread_ok (r, bounce, bounce_size);
       else


@@ -1484,21 +1700,31 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
           return 0;
         }
 
           return 0;
         }
 

-      for (i = 0; i < r->value_cnt; i++)
+      for (i = 0; i < r->var_cnt; i++)

         {
         {

-          struct sfm_var *v = &r->vars[i];
+         struct sfm_var *sv = &r->vars[i];

 
 

-          if (v->width == 0)
+          if (sv->width == 0)

             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);
             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);

-              case_data_rw (c, v->fv)->f = f == r->sysmis ? SYSMIS : f;
+              case_data_rw (c, sv->fv)->f = f == r->sysmis ? SYSMIS : f;

             }
             }

-          else if (v->width != -1)
+          else

             {
             {

-              memcpy (case_data_rw (c, v->fv)->s, bounce_cur, v->width);
-              bounce_cur += DIV_RND_UP (v->width, sizeof (flt64));
+             flt64 *bc_start = bounce_cur;
+             int ofs = 0;
+              while (ofs < sv->width )
+                {
+                  const int chunk = MIN (MAX_LONG_STRING, sv->width - ofs);
+                  memcpy (case_data_rw (c, sv->fv)->s + ofs, bounce_cur, chunk);
+
+                  bounce_cur += DIV_RND_UP (chunk, sizeof (flt64));
+
+                  ofs += chunk;
+                }
+             bounce_cur = bc_start + width_to_bytes(sv->width) / sizeof(flt64);

             }
         }
 
             }
         }
 


@@ -1519,13 +1745,13 @@ fread_ok (struct sfm_reader *r, void *buffer, size_t byte_cnt)
       if (ferror (r->file)) 
         {
           msg (ME, _("%s: Reading system file: %s."),
       if (ferror (r->file)) 
         {
           msg (ME, _("%s: Reading system file: %s."),

-               fh_get_filename (r->fh), strerror (errno));
+               fh_get_file_name (r->fh), strerror (errno));

           r->ok = false; 
         }
       else if (read_bytes != 0) 
         {
           msg (ME, _("%s: Partial record at end of system file."),
           r->ok = false; 
         }
       else if (read_bytes != 0) 
         {
           msg (ME, _("%s: Partial record at end of system file."),

-               fh_get_filename (r->fh));
+               fh_get_file_name (r->fh));

           r->ok = false; 
         }
       return 0;
           r->ok = false; 
         }
       return 0;


@@ -1553,3 +1779,4 @@ sfm_detect (FILE *file)
     return false;
   return true; 
 }
     return false;
   return true; 
 }

+