We've had a mix of min, max from libpspp/misc.h and MIN, MAX from

[pspp-builds.git] / src / data / sys-file-reader.c

diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c
index 89a236d5393561994125fc37953275b573c1e72c..1b66a4d27757a57e9c162738f12fe8b5e3f3856f 100644 (file)
--- a/src/data/sys-file-reader.c
+++ b/src/data/sys-file-reader.c
@@ -26,6 +26,7 @@
 #include <minmax.h>
 
 #include <libpspp/alloc.h>
 #include <minmax.h>
 
 #include <libpspp/alloc.h>

+#include <libpspp/assertion.h>

 #include <libpspp/message.h>
 #include <libpspp/compiler.h>
 #include <libpspp/magic.h>
 #include <libpspp/message.h>
 #include <libpspp/compiler.h>
 #include <libpspp/magic.h>


@@ -55,7 +56,6 @@ struct sfm_reader
   FILE *file;                  /* File stream. */
 
   int reverse_endian;          /* 1=file has endianness opposite us. */
   FILE *file;                  /* File stream. */
 
   int reverse_endian;          /* 1=file has endianness opposite us. */

-  int fix_specials;             /* 1=SYSMIS/HIGHEST/LOWEST differs from us. */

   int value_cnt;               /* Number of `union values's per case. */
   long case_cnt;               /* Number of cases, -1 if unknown. */
   int compressed;              /* 1=compressed, 0=not compressed. */
   int value_cnt;               /* Number of `union values's per case. */
   long case_cnt;               /* Number of cases, -1 if unknown. */
   int compressed;              /* 1=compressed, 0=not compressed. */


@@ -65,8 +65,8 @@ struct sfm_reader
   bool has_vls;         /* True if the file has one or more Very Long Strings*/
 
   /* Variables. */
   bool has_vls;         /* True if the file has one or more Very Long Strings*/
 
   /* Variables. */

-  struct hsh_table *var_hash;
-  struct variable **svars;
+  struct sfm_var *vars;
+  size_t var_cnt;

 
   /* File's special constants. */
   flt64 sysmis;
 
   /* File's special constants. */
   flt64 sysmis;


@@ -86,7 +86,6 @@ struct sfm_reader
 /* A variable in a system file. */
 struct sfm_var 
 {
 /* A variable in a system file. */
 struct sfm_var 
 {

-  char name[SHORT_NAME_LEN + 1];  /* name */

   int width;                  /* 0=numeric, otherwise string width. */
   int fv;                     /* Index into case. */
 };
   int width;                  /* 0=numeric, otherwise string width. */
   int fv;                     /* Index into case. */
 };


@@ -134,16 +133,16 @@ corrupt_msg (int class, const char *format,...)
   va_list args;
   struct string text;
 
   va_list args;
   struct string text;
 

-  ds_create (&text, _("corrupt system file: "));
+  ds_init_cstr (&text, _("corrupt system file: "));

   va_start (args, format);
   va_start (args, format);

-  ds_vprintf (&text, format, args);
+  ds_put_vformat (&text, format, args);

   va_end (args);
 
   m.category = msg_class_to_category (class);
   m.severity = msg_class_to_severity (class);
   m.where.file_name = NULL;
   m.where.line_number = 0;
   va_end (args);
 
   m.category = msg_class_to_category (class);
   m.severity = msg_class_to_severity (class);
   m.where.file_name = NULL;
   m.where.line_number = 0;

-  m.text = ds_c_str (&text);
+  m.text = ds_cstr (&text);

 
   msg_emit (&m);
 }
 
   msg_emit (&m);
 }


@@ -166,7 +165,7 @@ sfm_close_reader (struct sfm_reader *r)
   if (r->fh != NULL)
     fh_close (r->fh, "system file", "rs");
 
   if (r->fh != NULL)
     fh_close (r->fh, "system file", "rs");
 

-  hsh_destroy(r->var_hash);
+  free (r->vars);

   free (r->buf);
   free (r);
 }
   free (r->buf);
   free (r);
 }


@@ -216,7 +215,7 @@ struct name_pair
 };
 
 static int
 };
 
 static int

-pair_sn_compare(const void *_p1, const void *_p2, void *aux UNUSED)
+pair_sn_compare(const void *_p1, const void *_p2, const void *aux UNUSED)

 {
   int i;
 
 {
   int i;
 


@@ -247,7 +246,7 @@ pair_sn_compare(const void *_p1, const void *_p2, void *aux UNUSED)
 }
 
 static unsigned int
 }
 
 static unsigned int

-pair_sn_hash(const void *_p, void *aux UNUSED)
+pair_sn_hash(const void *_p, const void *aux UNUSED)

 {
   int i;
   const struct name_pair *p = _p;
 {
   int i;
   const struct name_pair *p = _p;


@@ -265,66 +264,13 @@ pair_sn_hash(const void *_p, void *aux UNUSED)
 }
 
 static void
 }
 
 static void

-pair_sn_free(void *p, void *aux UNUSED)
+pair_sn_free(void *p, const void *aux UNUSED)

 {
   free(p);
 }
 
 
 
 {
   free(p);
 }
 
 
 

-/* A hsh_compare_func that orders variables A and B by their
-   names. */
-static int
-compare_var_shortnames (const void *a_, const void *b_, void *foo UNUSED) 
-{
-  int i;
-  const struct variable *a = a_;
-  const struct variable *b = b_;
-
-  char buf1[SHORT_NAME_LEN + 1];
-  char buf2[SHORT_NAME_LEN + 1];
-
-  memset(buf1, 0, SHORT_NAME_LEN + 1);
-  memset(buf2, 0, SHORT_NAME_LEN + 1);
-
-  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
-    {
-      buf1[i] = a->short_name[i];
-      if ( '\0' == buf1[i]) 
-       break;
-    }
-
-  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
-    {
-      buf2[i] = b->short_name[i];
-      if ( '\0' == buf2[i]) 
-       break;
-    }
-
-  return strncmp(buf1, buf2, SHORT_NAME_LEN);
-}
-
-/* A hsh_hash_func that hashes variable V based on its name. */
-static unsigned
-hash_var_shortname (const void *v_, void *foo UNUSED) 
-{
-  int i;
-  const struct variable *v = v_;
-  char buf[SHORT_NAME_LEN + 1];
-
-  memset(buf, 0, SHORT_NAME_LEN + 1); 
-  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
-    {
-      buf[i] = v->short_name[i];
-      if ( '\0' == buf[i]) 
-       break;
-    }
-
-  return hsh_hash_bytes(buf, strlen(buf));
-}
-
-
-

 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the
 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the


@@ -342,7 +288,6 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   /* A hash table of long variable names indexed by short name */
   struct hsh_table *short_to_long = NULL;
 
   /* A hash table of long variable names indexed by short name */
   struct hsh_table *short_to_long = NULL;
 

-

   *dict = dict_create ();
   if (!fh_open (fh, FH_REF_FILE, "system file", "rs"))
     goto error;
   *dict = dict_create ();
   if (!fh_open (fh, FH_REF_FILE, "system file", "rs"))
     goto error;


@@ -353,7 +298,6 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   r->file = fn_open (fh_get_file_name (fh), "rb");
 
   r->reverse_endian = 0;
   r->file = fn_open (fh_get_file_name (fh), "rb");
 
   r->reverse_endian = 0;

-  r->fix_specials = 0;

   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;
   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;


@@ -361,9 +305,8 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   r->weight_idx = -1;
   r->ok = true;
   r->has_vls = false;
   r->weight_idx = -1;
   r->ok = true;
   r->has_vls = false;

-  r->svars = 0;

 
 

-  r->var_hash = hsh_create(4, compare_var_shortnames, hash_var_shortname, 0, 0);
+  r->vars = NULL;

 
   r->sysmis = -FLT64_MAX;
   r->highest = FLT64_MAX;
 
   r->sysmis = -FLT64_MAX;
   r->highest = FLT64_MAX;


@@ -444,10 +387,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
          {
            struct
            {
          {
            struct
            {

-             int32_t subtype P;
-             int32_t size P;
-             int32_t count P;
-           }
+             int32_t subtype ;
+             int32_t size ;
+             int32_t count ;
+           } ATTRIBUTE((packed)) 

            data;
             unsigned long bytes;
 
            data;
             unsigned long bytes;
 


@@ -496,20 +439,30 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                      break;
                    }
 
                      break;
                    }
 

-                 for ( i = 0 ; i < min(n_vars, dict_get_var_cnt(*dict)) ; ++i ) 
+                 for ( i = 0 ; i < MIN(n_vars, dict_get_var_cnt(*dict)) ; ++i ) 

                    {
                      struct
                      {
                    {
                      struct
                      {

-                       int32_t measure P;
-                       int32_t width P;
-                       int32_t align P;
-                     }
+                       int32_t measure ;
+                       int32_t width ;
+                       int32_t align ;
+                     } ATTRIBUTE((packed))

                      params;
 
                      struct variable *v;
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 
                      params;
 
                      struct variable *v;
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 

+                     if ( ! measure_is_valid(params.measure) 
+                          || 
+                          ! alignment_is_valid(params.align))
+                       {
+                         msg(MW, 
+                             _("%s: Invalid variable display parameters.  Default parameters substituted."), 
+                             fh_get_file_name(r->fh));
+                         continue;
+                       }
+

                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;
                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;


@@ -521,11 +474,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
              case 13: /* SPSS 12.0 Long variable name map */
                {
 
              case 13: /* SPSS 12.0 Long variable name map */
                {

-                 char *short_name, *save_ptr;
+                 char *short_name; 
+                 char *save_ptr = NULL;

                   int idx;
 
                   int idx;
 

-                 r->has_vls = true;
-

                   /* Read data. */
                   subrec14data = xmalloc (bytes + 1);
                  if (!buf_read (r, subrec14data, bytes, 0)) 
                   /* Read data. */
                   subrec14data = xmalloc (bytes + 1);
                  if (!buf_read (r, subrec14data, bytes, 0)) 


@@ -601,7 +553,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
         records have been processed. --- JMD 27 April 2006
       */
                      
         records have been processed. --- JMD 27 April 2006
       */
                      

-                      /* For compatability, make sure dictionary
+                      /* For compatibility, make sure dictionary

                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long
                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long


@@ -629,6 +581,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                     }
                  buffer[bytes] = '\0';
 
                     }
                  buffer[bytes] = '\0';
 

+                 r->has_vls = true;

 
                  /* Note:  SPSS v13 terminates this record with 00,
                     whereas SPSS v14 terminates it with 00 09. We must
 
                  /* Note:  SPSS v13 terminates this record with 00,
                     whereas SPSS v14 terminates it with 00 09. We must


@@ -693,8 +646,6 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                                  else
                                    l -= v_next->width;
 
                                  else
                                    l -= v_next->width;
 

-                                 hsh_delete(r->var_hash, v_next);
-

                                  dict_delete_var(*dict, v_next);
                                }
 
                                  dict_delete_var(*dict, v_next);
                                }
 


@@ -761,6 +712,24 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
  success:
   /* Come here on successful completion. */
 
  success:
   /* Come here on successful completion. */
 

+  /* Create an index of dictionary variable widths for
+     sfm_read_case to use.  We cannot use the `struct variables'
+     from the dictionary we created, because the caller owns the
+     dictionary and may destroy or modify its variables. */
+  {
+    size_t i;
+
+    r->var_cnt = dict_get_var_cnt (*dict);
+    r->vars = xnmalloc (r->var_cnt, sizeof *r->vars);
+    for (i = 0; i < r->var_cnt; i++) 
+      {
+        struct variable *v = dict_get_var (*dict, i);
+        struct sfm_var *sv = &r->vars[i];
+        sv->width = v->width;
+        sv->fv = v->fv; 
+      }
+  }
+

   free (var_by_idx);
   hsh_destroy(short_to_long);
   free (subrec14data);
   free (var_by_idx);
   hsh_destroy(short_to_long);
   free (subrec14data);


@@ -1213,9 +1182,6 @@ read_variables (struct sfm_reader *r,
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;

-
-      if ( vv->width != -1)
-       hsh_insert(r->var_hash, vv);

     }
 
   /* Some consistency checks. */
     }
 
   /* Some consistency checks. */


@@ -1242,28 +1208,35 @@ static int
 parse_format_spec (struct sfm_reader *r, int32_t s,
                    struct fmt_spec *f, const struct variable *v)
 {
 parse_format_spec (struct sfm_reader *r, int32_t s,
                    struct fmt_spec *f, const struct variable *v)
 {

-  f->type = translate_fmt ((s >> 16) & 0xff);
-  if (f->type == -1)
+  bool ok;
+  
+  if (!fmt_from_io ((s >> 16) & 0xff, &f->type))

     lose ((ME, _("%s: Bad format specifier byte (%d)."),
           fh_get_file_name (r->fh), (s >> 16) & 0xff));
   f->w = (s >> 8) & 0xff;
   f->d = s & 0xff;
 
     lose ((ME, _("%s: Bad format specifier byte (%d)."),
           fh_get_file_name (r->fh), (s >> 16) & 0xff));
   f->w = (s >> 8) & 0xff;
   f->d = s & 0xff;
 

-  if ((v->type == ALPHA) ^ ((formats[f->type].cat & FCAT_STRING) != 0))
+  if ((v->type == ALPHA) ^ (fmt_is_string (f->type) != 0))

     lose ((ME, _("%s: %s variable %s has %s format specifier %s."),
           fh_get_file_name (r->fh),
            v->type == ALPHA ? _("String") : _("Numeric"),
           v->name,
     lose ((ME, _("%s: %s variable %s has %s format specifier %s."),
           fh_get_file_name (r->fh),
            v->type == ALPHA ? _("String") : _("Numeric"),
           v->name,

-          formats[f->type].cat & FCAT_STRING ? _("string") : _("numeric"),
-          formats[f->type].name));
+          fmt_is_string (f->type) ? _("string") : _("numeric"),
+          fmt_name (f->type)));

 
 

-  if (!check_output_specifier (f, false)
-      || !check_specifier_width (f, v->width, false)) 
+  msg_disable ();
+  ok = fmt_check_output (f) && fmt_check_width_compat (f, v->width);
+  msg_enable ();
+  
+  if (!ok) 

     {
     {

+      char fmt_string[FMT_STRING_LEN_MAX + 1];

       msg (ME, _("%s variable %s has invalid format specifier %s."),
            v->type == NUMERIC ? _("Numeric") : _("String"),
       msg (ME, _("%s variable %s has invalid format specifier %s."),
            v->type == NUMERIC ? _("Numeric") : _("String"),

-           v->name, fmt_to_string (f));
-      *f = v->type == NUMERIC ? f8_2 : make_output_format (FMT_A, v->width, 0);
+           v->name, fmt_to_string (f, fmt_string));
+      *f = (v->type == NUMERIC
+            ? fmt_for_output (FMT_F, 8, 2) 
+            : fmt_for_output (FMT_A, v->width, 0));

     }
   return 1;
 
     }
   return 1;
 


@@ -1409,7 +1382,7 @@ read_value_labels (struct sfm_reader *r,
       
       if (var[0]->type == ALPHA)
         {
       
       if (var[0]->type == ALPHA)
         {

-          const int copy_len = min (sizeof label->raw_value,
+          const int copy_len = MIN (sizeof label->raw_value,

                                     sizeof label->label);
           memcpy (label->value.s, label->raw_value, copy_len);
         } else {
                                     sizeof label->label);
           memcpy (label->value.s, label->raw_value, copy_len);
         } else {


@@ -1473,7 +1446,7 @@ buf_read (struct sfm_reader *r, void *buf, size_t byte_cnt, size_t min_alloc)
   assert (r);
 
   if (buf == NULL && byte_cnt > 0 )
   assert (r);
 
   if (buf == NULL && byte_cnt > 0 )

-    buf = xmalloc (max (byte_cnt, min_alloc));
+    buf = xmalloc (MAX (byte_cnt, min_alloc));

 
   if ( byte_cnt == 0 )
     return buf;
 
   if ( byte_cnt == 0 )
     return buf;


@@ -1655,7 +1628,7 @@ read_compressed_data (struct sfm_reader *r, flt64 *buf)
       p = r->x;
     }
 
       p = r->x;
     }
 

-  abort ();
+  NOT_REACHED ();

 
  success:
   /* We have filled up an entire record.  Update state and return
 
  success:
   /* We have filled up an entire record.  Update state and return


@@ -1669,20 +1642,6 @@ read_compressed_data (struct sfm_reader *r, flt64 *buf)
   return 0;
 }
 
   return 0;
 }
 

-
-static int
-compare_var_index(const void *_v1, const void *_v2, void *aux UNUSED)
-{
-  const struct variable *const *v1 = _v1;
-  const struct variable *const *v2 = _v2;
-
-  if ( (*v1)->index < (*v2)->index) 
-    return -1;
-
-  return ( (*v1)->index > (*v2)->index) ;
-}
-
-

 /* Reads one case from READER's file into C.  Returns nonzero
    only if successful. */
 int
 /* Reads one case from READER's file into C.  Returns nonzero
    only if successful. */
 int


@@ -1691,13 +1650,6 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
   if (!r->ok)
     return 0;
 
   if (!r->ok)
     return 0;
 

-  if ( ! r->svars ) 
-    {
-      r->svars = (struct variable **) hsh_data(r->var_hash);
-      sort(r->svars, hsh_count(r->var_hash), 
-          sizeof(*r->svars), compare_var_index, 0);
-    }
-

   if (!r->compressed && sizeof (flt64) == sizeof (double) && ! r->has_vls) 
     {
       /* Fast path: external and internal representations are the
   if (!r->compressed && sizeof (flt64) == sizeof (double) && ! r->has_vls) 
     {
       /* Fast path: external and internal representations are the


@@ -1713,12 +1665,9 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
         {
           int i;
           
         {
           int i;
           

-          for (i = 0; i < hsh_count(r->var_hash); i++) 
-           {
-             struct variable *v = r->svars[i];
-             if (v->width == 0)
-               bswap_flt64 (&case_data_rw (c, v->fv)->f);
-           }
+          for (i = 0; i < r->var_cnt; i++) 
+            if (r->vars[i].width == 0)
+              bswap_flt64 (&case_data_rw (c, r->vars[i].fv)->f);

         }
 
       /* Fix up SYSMIS values if needed.
         }
 
       /* Fix up SYSMIS values if needed.


@@ -1727,12 +1676,10 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
       if (r->sysmis != SYSMIS) 
         {
           int i;
       if (r->sysmis != SYSMIS) 
         {
           int i;

-          for (i = 0; i < hsh_count(r->var_hash); i++) 
-           {
-             struct variable *v = r->svars[i];
-             if (v->width == 0 && case_num (c, i) == r->sysmis)
-               case_data_rw (c, v->fv)->f = SYSMIS;
-           }
+          
+          for (i = 0; i < r->var_cnt; i++) 
+            if (r->vars[i].width == 0 && case_num (c, i) == r->sysmis)
+              case_data_rw (c, r->vars[i].fv)->f = SYSMIS;

         }
     }
   else 
         }
     }
   else 


@@ -1760,31 +1707,31 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
           return 0;
         }
 
           return 0;
         }
 

-      for (i = 0; i < hsh_count(r->var_hash); i++)
+      for (i = 0; i < r->var_cnt; i++)

         {
         {

-         struct variable *tv = r->svars[i];
+         struct sfm_var *sv = &r->vars[i];

 
 

-          if (tv->width == 0)
+          if (sv->width == 0)

             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);
             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);

-              case_data_rw (c, tv->fv)->f = f == r->sysmis ? SYSMIS : f;
+              case_data_rw (c, sv->fv)->f = f == r->sysmis ? SYSMIS : f;

             }
             }

-          else if (tv->width != -1)
+          else

             {
              flt64 *bc_start = bounce_cur;
              int ofs = 0;
             {
              flt64 *bc_start = bounce_cur;
              int ofs = 0;

-              while (ofs < tv->width )
+              while (ofs < sv->width )

                 {
                 {

-                  const int chunk = MIN (MAX_LONG_STRING, tv->width - ofs);
-                  memcpy (case_data_rw (c, tv->fv)->s + ofs, bounce_cur, chunk);
+                  const int chunk = MIN (MAX_LONG_STRING, sv->width - ofs);
+                  memcpy (case_data_rw (c, sv->fv)->s + ofs, bounce_cur, chunk);

 
                   bounce_cur += DIV_RND_UP (chunk, sizeof (flt64));
 
                   ofs += chunk;
                 }
 
                   bounce_cur += DIV_RND_UP (chunk, sizeof (flt64));
 
                   ofs += chunk;
                 }

-             bounce_cur = bc_start + width_to_bytes(tv->width) / sizeof(flt64);
+             bounce_cur = bc_start + width_to_bytes(sv->width) / sizeof(flt64);

             }
         }
 
             }
         }