added new initialization function

[pspp-builds.git] / src / data / sys-file-reader.c

diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c
index 2c53dbb74d793dd5e3f1fbd9652d12c075f88c0a..52a14a45d2e26021d05f9ba32bb20cda5b2f7445 100644 (file)
--- a/src/data/sys-file-reader.c
+++ b/src/data/sys-file-reader.c
@@ -23,6 +23,7 @@
 #include <errno.h>
 #include <float.h>
 #include <c-ctype.h>
 #include <errno.h>
 #include <float.h>
 #include <c-ctype.h>

+#include <minmax.h>

 
 #include <libpspp/alloc.h>
 #include <libpspp/message.h>
 
 #include <libpspp/alloc.h>
 #include <libpspp/message.h>


@@ -31,6 +32,7 @@
 #include <libpspp/misc.h>
 #include <libpspp/str.h>
 #include <libpspp/hash.h>
 #include <libpspp/misc.h>
 #include <libpspp/str.h>
 #include <libpspp/hash.h>

+#include <libpspp/array.h>

 
 #include "sys-file-reader.h"
 #include "sfm-private.h"
 
 #include "sys-file-reader.h"
 #include "sfm-private.h"


@@ -53,16 +55,17 @@ struct sfm_reader
   FILE *file;                  /* File stream. */
 
   int reverse_endian;          /* 1=file has endianness opposite us. */
   FILE *file;                  /* File stream. */
 
   int reverse_endian;          /* 1=file has endianness opposite us. */

-  int fix_specials;           /* 1=SYSMIS/HIGHEST/LOWEST differs from us. */

   int value_cnt;               /* Number of `union values's per case. */
   long case_cnt;               /* Number of cases, -1 if unknown. */
   int compressed;              /* 1=compressed, 0=not compressed. */
   int value_cnt;               /* Number of `union values's per case. */
   long case_cnt;               /* Number of cases, -1 if unknown. */
   int compressed;              /* 1=compressed, 0=not compressed. */

-  double bias;         /* Compression bias, usually 100.0. */
+  double bias;                 /* Compression bias, usually 100.0. */

   int weight_idx;              /* 0-based index of weighting variable, or -1. */
   bool ok;                    /* False after an I/O error or corrupt data. */
   int weight_idx;              /* 0-based index of weighting variable, or -1. */
   bool ok;                    /* False after an I/O error or corrupt data. */

+  bool has_vls;         /* True if the file has one or more Very Long Strings*/

 
   /* Variables. */
 
   /* Variables. */

-  struct sfm_var *vars;       /* Variables. */
+  struct sfm_var *vars;
+  size_t var_cnt;

 
   /* File's special constants. */
   flt64 sysmis;
 
   /* File's special constants. */
   flt64 sysmis;


@@ -129,16 +132,16 @@ corrupt_msg (int class, const char *format,...)
   va_list args;
   struct string text;
 
   va_list args;
   struct string text;
 

-  ds_create (&text, _("corrupt system file: "));
+  ds_init_cstr (&text, _("corrupt system file: "));

   va_start (args, format);
   va_start (args, format);

-  ds_vprintf (&text, format, args);
+  ds_put_vformat (&text, format, args);

   va_end (args);
 
   m.category = msg_class_to_category (class);
   m.severity = msg_class_to_severity (class);
   m.where.file_name = NULL;
   m.where.line_number = 0;
   va_end (args);
 
   m.category = msg_class_to_category (class);
   m.severity = msg_class_to_severity (class);
   m.where.file_name = NULL;
   m.where.line_number = 0;

-  m.text = ds_c_str (&text);
+  m.text = ds_cstr (&text);

 
   msg_emit (&m);
 }
 
   msg_emit (&m);
 }


@@ -213,17 +216,50 @@ struct name_pair
 static int
 pair_sn_compare(const void *_p1, const void *_p2, void *aux UNUSED)
 {
 static int
 pair_sn_compare(const void *_p1, const void *_p2, void *aux UNUSED)
 {

+  int i;
+

   const struct name_pair *p1 = _p1;
   const struct name_pair *p2 = _p2;
   const struct name_pair *p1 = _p1;
   const struct name_pair *p2 = _p2;

-  
-  return strcmp(p1->shortname, p2->shortname);
+
+  char buf1[SHORT_NAME_LEN + 1];
+  char buf2[SHORT_NAME_LEN + 1];
+
+  memset(buf1, 0, SHORT_NAME_LEN + 1);
+  memset(buf2, 0, SHORT_NAME_LEN + 1);
+
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf1[i] = p1->shortname[i];
+      if ( '\0' == buf1[i]) 
+       break;
+    }
+
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf2[i] = p2->shortname[i];
+      if ( '\0' == buf2[i]) 
+       break;
+    }
+
+  return strncmp(buf1, buf2, SHORT_NAME_LEN);

 }
 
 }
 

-static unsigned
+static unsigned int

 pair_sn_hash(const void *_p, void *aux UNUSED)
 {
 pair_sn_hash(const void *_p, void *aux UNUSED)
 {

+  int i;

   const struct name_pair *p = _p;
   const struct name_pair *p = _p;

-  return hsh_hash_bytes(p->shortname, strlen(p->shortname));
+  char buf[SHORT_NAME_LEN + 1];
+
+  memset(buf, 0, SHORT_NAME_LEN + 1); 
+  for (i = 0 ; i <= SHORT_NAME_LEN ; ++i ) 
+    {
+      buf[i] = p->shortname[i];
+      if ( '\0' == buf[i]) 
+       break;
+    }
+
+  return hsh_hash_bytes(buf, strlen(buf));

 }
 
 static void
 }
 
 static void


@@ -233,6 +269,7 @@ pair_sn_free(void *p, void *aux UNUSED)
 }
 
 
 }
 
 

+

 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the
 /* Opens the system file designated by file handle FH for
    reading.  Reads the system file's dictionary into *DICT.
    If INFO is non-null, then it receives additional info about the


@@ -244,6 +281,9 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   struct sfm_reader *r = NULL;
   struct variable **var_by_idx = NULL;
 
   struct sfm_reader *r = NULL;
   struct variable **var_by_idx = NULL;
 

+  /* The data in record 7(14) */
+  char *subrec14data = 0;
+

   /* A hash table of long variable names indexed by short name */
   struct hsh_table *short_to_long = NULL;
 
   /* A hash table of long variable names indexed by short name */
   struct hsh_table *short_to_long = NULL;
 


@@ -257,13 +297,13 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   r->file = fn_open (fh_get_file_name (fh), "rb");
 
   r->reverse_endian = 0;
   r->file = fn_open (fh_get_file_name (fh), "rb");
 
   r->reverse_endian = 0;

-  r->fix_specials = 0;

   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;
   r->bias = 100.0;
   r->weight_idx = -1;
   r->ok = true;
   r->value_cnt = 0;
   r->case_cnt = 0;
   r->compressed = 0;
   r->bias = 100.0;
   r->weight_idx = -1;
   r->ok = true;

+  r->has_vls = false;

 
   r->vars = NULL;
 
 
   r->vars = NULL;
 


@@ -412,6 +452,16 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 
 
                      assertive_buf_read (r, &params, sizeof(params), 0);
 

+                     if ( ! measure_is_valid(params.measure) 
+                          || 
+                          ! alignment_is_valid(params.align))
+                       {
+                         msg(MW, 
+                             _("%s: Invalid variable display parameters.  Default parameters substituted."), 
+                             fh_get_file_name(r->fh));
+                         continue;
+                       }
+

                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;
                      v = dict_get_var(*dict, i);
 
                      v->measure = params.measure;


@@ -423,17 +473,16 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
              case 13: /* SPSS 12.0 Long variable name map */
                {
 
              case 13: /* SPSS 12.0 Long variable name map */
                {

-                 char *buf, *short_name, *save_ptr;
+                 char *short_name, *save_ptr;

                   int idx;
 
                   /* Read data. */
                   int idx;
 
                   /* Read data. */

-                  buf = xmalloc (bytes + 1);
-                 if (!buf_read (r, buf, bytes, 0)) 
+                  subrec14data = xmalloc (bytes + 1);
+                 if (!buf_read (r, subrec14data, bytes, 0)) 

                     {
                     {

-                      free (buf);

                       goto error;
                     }
                       goto error;
                     }

-                 buf[bytes] = '\0';
+                 subrec14data[bytes] = '\0';

 
                  short_to_long = hsh_create(4, 
                                             pair_sn_compare,
 
                  short_to_long = hsh_create(4, 
                                             pair_sn_compare,


@@ -442,7 +491,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                                             0);
 
                   /* Parse data. */
                                             0);
 
                   /* Parse data. */

-                 for (short_name = strtok_r (buf, "=", &save_ptr), idx = 0;
+                 for (short_name = strtok_r (subrec14data, "=", &save_ptr), idx = 0;

                        short_name != NULL;
                        short_name = strtok_r (NULL, "=", &save_ptr), idx++)
                    {
                        short_name != NULL;
                        short_name = strtok_r (NULL, "=", &save_ptr), idx++)
                    {


@@ -502,7 +551,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
         records have been processed. --- JMD 27 April 2006
       */
                      
         records have been processed. --- JMD 27 April 2006
       */
                      

-                      /* For compatability, make sure dictionary
+                      /* For compatibility, make sure dictionary

                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long
                          is in long variable name map order.  In
                          the common case, this has no effect,
                          because the dictionary and the long


@@ -512,9 +561,6 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 #endif
                    }
                  
 #endif
                    }
                  

-
-                 /* Free data. */
-                 free (buf);

                }
                break;
 
                }
                break;
 


@@ -525,14 +571,15 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                  int i;
 
                   /* Read data. */
                  int i;
 
                   /* Read data. */

-                  char *buf = xmalloc (bytes + 1);
-                 if (!buf_read (r, buf, bytes, 0)) 
+                  char *buffer = xmalloc (bytes + 1);
+                 if (!buf_read (r, buffer, bytes, 0)) 

                     {
                     {

-                      free (buf);
+                      free (buffer);

                       goto error;
                     }
                       goto error;
                     }

-                 buf[bytes] = '\0';
+                 buffer[bytes] = '\0';

 
 

+                 r->has_vls = true;

 
                  /* Note:  SPSS v13 terminates this record with 00,
                     whereas SPSS v14 terminates it with 00 09. We must
 
                  /* Note:  SPSS v13 terminates this record with 00,
                     whereas SPSS v14 terminates it with 00 09. We must


@@ -540,10 +587,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                  for(i = 0; i < bytes ; ++i)
                    {
                      long int length;
                  for(i = 0; i < bytes ; ++i)
                    {
                      long int length;

-                     static char name[SHORT_NAME_LEN + 1];
-                     static char len_str[6];
+                     static char name[SHORT_NAME_LEN + 1]  = {0};
+                     static char len_str[6]  ={0};

 
 

-                     switch( buf[i] )
+                     switch( buffer[i] )

                        {
                        case '=':
                          eq_seen = true;
                        {
                        case '=':
                          eq_seen = true;


@@ -569,7 +616,6 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                                    lookup_name = p->longname;
                                }
                                
                                    lookup_name = p->longname;
                                }
                                

-                             

                              v = dict_lookup_var(*dict, lookup_name);
                              if ( !v ) 
                                {
                              v = dict_lookup_var(*dict, lookup_name);
                              if ( !v ) 
                                {


@@ -580,7 +626,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                                  goto error;
 
                                }
                                  goto error;
 
                                }

-                             
+

                              l = length;
                              if ( v->width > EFFECTIVE_LONG_STRING_LENGTH ) 
                                l -= EFFECTIVE_LONG_STRING_LENGTH;
                              l = length;
                              if ( v->width > EFFECTIVE_LONG_STRING_LENGTH ) 
                                l -= EFFECTIVE_LONG_STRING_LENGTH;


@@ -600,10 +646,13 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
 
                                  dict_delete_var(*dict, v_next);
                                }
 
                                  dict_delete_var(*dict, v_next);
                                }

-                             
+
+                             assert ( length > MAX_LONG_STRING );
+

                              v->width = length;
                              v->print.w = v->width;
                              v->write.w = v->width;
                              v->width = length;
                              v->print.w = v->width;
                              v->write.w = v->width;

+                             v->nv = DIV_RND_UP (length, MAX_SHORT_STRING);

                            }
                          eq_seen = false;
                          memset(name, 0, SHORT_NAME_LEN+1); 
                            }
                          eq_seen = false;
                          memset(name, 0, SHORT_NAME_LEN+1); 


@@ -614,14 +663,15 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
                          break;
                        default:
                          if ( eq_seen ) 
                          break;
                        default:
                          if ( eq_seen ) 

-                           len_str[j] = buf[i];
+                           len_str[j] = buffer[i];

                          else
                          else

-                           name[j] = buf[i];
+                           name[j] = buffer[i];

                          j++;
                          break;
                        }
                    }
                          j++;
                          break;
                        }
                    }

-                 free(buf);
+                 free(buffer);
+                 dict_compact_values(*dict);

                }
                break;
 
                }
                break;
 


@@ -660,9 +710,27 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
  success:
   /* Come here on successful completion. */
 
  success:
   /* Come here on successful completion. */
 

+  /* Create an index of dictionary variable widths for
+     sfm_read_case to use.  We cannot use the `struct variables'
+     from the dictionary we created, because the caller owns the
+     dictionary and may destroy or modify its variables. */
+  {
+    size_t i;
+
+    r->var_cnt = dict_get_var_cnt (*dict);
+    r->vars = xnmalloc (r->var_cnt, sizeof *r->vars);
+    for (i = 0; i < r->var_cnt; i++) 
+      {
+        struct variable *v = dict_get_var (*dict, i);
+        struct sfm_var *sv = &r->vars[i];
+        sv->width = v->width;
+        sv->fv = v->fv; 
+      }
+  }

 
   free (var_by_idx);
   hsh_destroy(short_to_long);
 
   free (var_by_idx);
   hsh_destroy(short_to_long);

+  free (subrec14data);

   return r;
 
  error:
   return r;
 
  error:


@@ -670,6 +738,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
   sfm_close_reader (r);
   free (var_by_idx);
   hsh_destroy(short_to_long);
   sfm_close_reader (r);
   free (var_by_idx);
   hsh_destroy(short_to_long);

+  free (subrec14data);

   if (*dict != NULL) 
     {
       dict_destroy (*dict);
   if (*dict != NULL) 
     {
       dict_destroy (*dict);


@@ -935,13 +1004,6 @@ read_variables (struct sfm_reader *r,
 
   *var_by_idx = 0;
 
 
   *var_by_idx = 0;
 

-  /* Pre-allocate variables. */
-  if (r->value_cnt != -1) 
-    {
-      *var_by_idx = xnmalloc (r->value_cnt, sizeof **var_by_idx);
-      r->vars = xnmalloc (r->value_cnt, sizeof *r->vars);
-    }
-

 
   /* Read in the entry for each variable and use the info to
      initialize the dictionary. */
 
   /* Read in the entry for each variable and use the info to
      initialize the dictionary. */


@@ -973,7 +1035,6 @@ read_variables (struct sfm_reader *r,
        }
 
       *var_by_idx = xnrealloc (*var_by_idx, i + 1, sizeof **var_by_idx);
        }
 
       *var_by_idx = xnrealloc (*var_by_idx, i + 1, sizeof **var_by_idx);

-      r->vars = xnrealloc (r->vars, i + 1, sizeof *r->vars);

 
       /* If there was a long string previously, make sure that the
         continuations are present; otherwise make sure there aren't
 
       /* If there was a long string previously, make sure that the
         continuations are present; otherwise make sure there aren't


@@ -986,7 +1047,6 @@ read_variables (struct sfm_reader *r,
                    fh_get_file_name (r->fh), i));
 
 
                    fh_get_file_name (r->fh), i));
 
 

-         r->vars[i].width = -1;

          (*var_by_idx)[i] = NULL;
          long_string_count--;
          continue;
          (*var_by_idx)[i] = NULL;
          long_string_count--;
          continue;


@@ -1120,10 +1180,6 @@ read_variables (struct sfm_reader *r,
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;
       if (!parse_format_spec (r, sv.print, &vv->print, vv)
          || !parse_format_spec (r, sv.write, &vv->write, vv))
        goto error;

-
-      r->vars[i].width = vv->width;
-      r->vars[i].fv = vv->fv;
-

     }
 
   /* Some consistency checks. */
     }
 
   /* Some consistency checks. */


@@ -1584,8 +1640,8 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
 {
   if (!r->ok)
     return 0;
 {
   if (!r->ok)
     return 0;

-  
-  if (!r->compressed && sizeof (flt64) == sizeof (double)) 
+
+  if (!r->compressed && sizeof (flt64) == sizeof (double) && ! r->has_vls) 

     {
       /* Fast path: external and internal representations are the
          same, except possibly for endianness or SYSMIS.  Read
     {
       /* Fast path: external and internal representations are the
          same, except possibly for endianness or SYSMIS.  Read


@@ -1600,7 +1656,7 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
         {
           int i;
           
         {
           int i;
           

-          for (i = 0; i < r->value_cnt; i++) 
+          for (i = 0; i < r->var_cnt; i++) 

             if (r->vars[i].width == 0)
               bswap_flt64 (&case_data_rw (c, r->vars[i].fv)->f);
         }
             if (r->vars[i].width == 0)
               bswap_flt64 (&case_data_rw (c, r->vars[i].fv)->f);
         }


@@ -1612,7 +1668,7 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
         {
           int i;
           
         {
           int i;
           

-          for (i = 0; i < r->value_cnt; i++) 
+          for (i = 0; i < r->var_cnt; i++) 

             if (r->vars[i].width == 0 && case_num (c, i) == r->sysmis)
               case_data_rw (c, r->vars[i].fv)->f = SYSMIS;
         }
             if (r->vars[i].width == 0 && case_num (c, i) == r->sysmis)
               case_data_rw (c, r->vars[i].fv)->f = SYSMIS;
         }


@@ -1630,6 +1686,8 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
       bounce_size = sizeof *bounce * r->value_cnt;
       bounce = bounce_cur = local_alloc (bounce_size);
 
       bounce_size = sizeof *bounce * r->value_cnt;
       bounce = bounce_cur = local_alloc (bounce_size);
 

+      memset(bounce, 0, bounce_size);
+

       if (!r->compressed)
         read_ok = fread_ok (r, bounce, bounce_size);
       else
       if (!r->compressed)
         read_ok = fread_ok (r, bounce, bounce_size);
       else


@@ -1640,21 +1698,31 @@ sfm_read_case (struct sfm_reader *r, struct ccase *c)
           return 0;
         }
 
           return 0;
         }
 

-      for (i = 0; i < r->value_cnt; i++)
+      for (i = 0; i < r->var_cnt; i++)

         {
         {

-          struct sfm_var *v = &r->vars[i];
+         struct sfm_var *sv = &r->vars[i];

 
 

-          if (v->width == 0)
+          if (sv->width == 0)

             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);
             {
               flt64 f = *bounce_cur++;
               if (r->reverse_endian)
                 bswap_flt64 (&f);

-              case_data_rw (c, v->fv)->f = f == r->sysmis ? SYSMIS : f;
+              case_data_rw (c, sv->fv)->f = f == r->sysmis ? SYSMIS : f;

             }
             }

-          else if (v->width != -1)
+          else

             {
             {

-              memcpy (case_data_rw (c, v->fv)->s, bounce_cur, v->width);
-              bounce_cur += DIV_RND_UP (v->width, sizeof (flt64));
+             flt64 *bc_start = bounce_cur;
+             int ofs = 0;
+              while (ofs < sv->width )
+                {
+                  const int chunk = MIN (MAX_LONG_STRING, sv->width - ofs);
+                  memcpy (case_data_rw (c, sv->fv)->s + ofs, bounce_cur, chunk);
+
+                  bounce_cur += DIV_RND_UP (chunk, sizeof (flt64));
+
+                  ofs += chunk;
+                }
+             bounce_cur = bc_start + width_to_bytes(sv->width) / sizeof(flt64);

             }
         }
 
             }
         }