1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2000, 2006-2007, 2009-2016 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
25 #include "data/any-reader.h"
26 #include "data/case.h"
27 #include "data/casereader-provider.h"
28 #include "data/casereader.h"
29 #include "data/dictionary.h"
30 #include "data/file-handle-def.h"
31 #include "data/file-name.h"
32 #include "data/format.h"
33 #include "data/identifier.h"
34 #include "data/missing-values.h"
35 #include "data/value-labels.h"
36 #include "data/value.h"
37 #include "data/variable.h"
38 #include "libpspp/float-format.h"
39 #include "libpspp/i18n.h"
40 #include "libpspp/integer-format.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
46 #include "gl/localcharset.h"
47 #include "gl/minmax.h"
48 #include "gl/xalloc.h"
52 #define _(msgid) gettext (msgid)
53 #define N_(msgid) (msgid)
63 struct pcp_dir_entry main;
64 struct pcp_dir_entry variables;
65 struct pcp_dir_entry labels;
66 struct pcp_dir_entry data;
69 struct pcp_main_header
71 char product[63]; /* "PCSPSS SYSTEM FILE..." */
72 unsigned int nominal_case_size; /* Number of var positions. */
73 char creation_date[9]; /* "[m]m/dd/yy". */
74 char creation_time[9]; /* "[H]H:MM:SS". */
75 char file_label[65]; /* File label. */
76 unsigned int weight_index; /* Index of weighting variable, 0 if none. */
85 struct fmt_spec format;
91 struct pcp_value_label *val_labs;
97 struct pcp_value_label
103 /* System file reader. */
106 struct any_reader any_reader;
108 /* Resource tracking. */
109 struct pool *pool; /* All system file state. */
112 unsigned int file_size;
113 struct any_read_info info;
114 struct pcp_directory directory;
115 struct pcp_main_header header;
116 struct pcp_var_record *vars;
120 struct file_handle *fh; /* File handle. */
121 struct fh_lock *lock; /* Mutual exclusion for file handle. */
122 FILE *file; /* File stream. */
123 unsigned int pos; /* Position in file. */
124 bool error; /* I/O or corruption error? */
125 struct caseproto *proto; /* Format of output cases. */
128 unsigned int n_cases; /* Number of cases */
129 const char *encoding; /* String encoding. */
133 uint8_t opcodes[8]; /* Current block of opcodes. */
134 size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
135 bool corruption_warning; /* Warned about possible corruption? */
138 static struct pcp_reader *
139 pcp_reader_cast (const struct any_reader *r_)
141 assert (r_->klass == &pcp_file_reader_class);
142 return UP_CAST (r_, struct pcp_reader, any_reader);
145 static const struct casereader_class pcp_file_casereader_class;
147 static bool pcp_close (struct any_reader *);
149 static bool read_variables_record (struct pcp_reader *);
151 static void pcp_msg (struct pcp_reader *r, off_t, int class,
152 const char *format, va_list args)
153 PRINTF_FORMAT (4, 0);
154 static void pcp_warn (struct pcp_reader *, off_t, const char *, ...)
155 PRINTF_FORMAT (3, 4);
156 static void pcp_error (struct pcp_reader *, off_t, const char *, ...)
157 PRINTF_FORMAT (3, 4);
159 static bool read_bytes (struct pcp_reader *, void *, size_t)
161 static int try_read_bytes (struct pcp_reader *, void *, size_t)
163 static bool read_uint16 (struct pcp_reader *, unsigned int *)
165 static bool read_uint32 (struct pcp_reader *, unsigned int *)
167 static bool read_float (struct pcp_reader *, double *)
169 static double parse_float (const uint8_t number[8]);
170 static bool read_string (struct pcp_reader *, char *, size_t)
172 static bool skip_bytes (struct pcp_reader *, size_t) WARN_UNUSED_RESULT;
174 static bool pcp_seek (struct pcp_reader *, off_t);
176 static bool pcp_is_sysmis(const uint8_t *);
178 /* Dictionary reader. */
180 static bool read_dictionary (struct pcp_reader *);
181 static bool read_main_header (struct pcp_reader *, struct pcp_main_header *);
182 static void parse_header (struct pcp_reader *,
183 const struct pcp_main_header *,
184 struct any_read_info *, struct dictionary *);
185 static bool parse_variable_records (struct pcp_reader *, struct dictionary *,
186 struct pcp_var_record *, size_t n);
188 /* Tries to open FH for reading as an SPSS/PC+ system file. Returns a
189 pcp_reader if successful, otherwise NULL. */
190 static struct any_reader *
191 pcp_open (struct file_handle *fh)
193 struct pcp_reader *r;
196 /* Create and initialize reader. */
197 r = xzalloc (sizeof *r);
198 r->any_reader.klass = &pcp_file_reader_class;
199 r->pool = pool_create ();
200 pool_register (r->pool, free, r);
202 r->opcode_idx = sizeof r->opcodes;
204 /* TRANSLATORS: this fragment will be interpolated into
205 messages in fh_lock() that identify types of files. */
206 r->lock = fh_lock (fh, FH_REF_FILE, N_("SPSS/PC+ system file"),
212 r->file = fn_open (fh, "rb");
215 msg (ME, _("Error opening `%s' for reading as an SPSS/PC+ "
217 fh_get_file_name (r->fh), strerror (errno));
221 /* Fetch file size. */
222 if (fstat (fileno (r->file), &s))
224 pcp_error (ME, 0, _("%s: stat failed (%s)."),
225 fh_get_file_name (r->fh), strerror (errno));
228 if (s.st_size > UINT_MAX)
230 pcp_error (ME, 0, _("%s: file too large."), fh_get_file_name (r->fh));
233 r->file_size = s.st_size;
235 /* Read dictionary. */
236 if (!read_dictionary (r))
239 if (!pcp_seek (r, r->directory.data.ofs))
242 return &r->any_reader;
245 pcp_close (&r->any_reader);
250 pcp_read_dir_entry (struct pcp_reader *r, struct pcp_dir_entry *de)
252 if (!read_uint32 (r, &de->ofs) || !read_uint32 (r, &de->len))
255 if (de->len > r->file_size || de->ofs > r->file_size - de->len)
257 pcp_error (r, r->pos - 8, _("Directory entry is for a %u-byte record "
258 "starting at offset %u but file is only "
260 de->ofs, de->len, r->file_size);
268 read_dictionary (struct pcp_reader *r)
270 unsigned int two, zero;
272 if (!read_uint32 (r, &two) || !read_uint32 (r, &zero))
274 if (two != 2 || zero != 0)
275 pcp_warn (r, 0, _("Directory fields have unexpected values "
276 "(%u,%u)."), two, zero);
278 if (!pcp_read_dir_entry (r, &r->directory.main)
279 || !pcp_read_dir_entry (r, &r->directory.variables)
280 || !pcp_read_dir_entry (r, &r->directory.labels)
281 || !pcp_read_dir_entry (r, &r->directory.data))
284 if (!read_main_header (r, &r->header))
287 read_variables_record (r);
292 struct get_strings_aux
303 add_string__ (struct get_strings_aux *aux,
304 const char *string, bool id, char *title)
306 if (aux->n >= aux->allocated)
308 aux->allocated = 2 * (aux->allocated + 1);
309 aux->titles = pool_realloc (aux->pool, aux->titles,
310 aux->allocated * sizeof *aux->titles);
311 aux->strings = pool_realloc (aux->pool, aux->strings,
312 aux->allocated * sizeof *aux->strings);
313 aux->ids = pool_realloc (aux->pool, aux->ids,
314 aux->allocated * sizeof *aux->ids);
317 aux->titles[aux->n] = title;
318 aux->strings[aux->n] = pool_strdup (aux->pool, string);
319 aux->ids[aux->n] = id;
323 static void PRINTF_FORMAT (3, 4)
324 add_string (struct get_strings_aux *aux,
325 const char *string, const char *title, ...)
329 va_start (args, title);
330 add_string__ (aux, string, false, pool_vasprintf (aux->pool, title, args));
334 static void PRINTF_FORMAT (3, 4)
335 add_id (struct get_strings_aux *aux, const char *id, const char *title, ...)
339 va_start (args, title);
340 add_string__ (aux, id, true, pool_vasprintf (aux->pool, title, args));
344 /* Retrieves significant string data from R in its raw format, to allow the
345 caller to try to detect the encoding in use.
347 Returns the number of strings retrieved N. Sets each of *TITLESP, *IDSP,
348 and *STRINGSP to an array of N elements allocated from POOL. For each I in
349 0...N-1, UTF-8 string *TITLESP[I] describes *STRINGSP[I], which is in
350 whatever encoding system file R uses. *IDS[I] is true if *STRINGSP[I] must
351 be a valid PSPP language identifier, false if *STRINGSP[I] is free-form
354 pcp_get_strings (const struct any_reader *r_, struct pool *pool,
355 char ***titlesp, bool **idsp, char ***stringsp)
357 struct pcp_reader *r = pcp_reader_cast (r_);
358 struct get_strings_aux aux;
370 for (i = 0; i < r->n_vars; i++)
371 if (r->vars[i].width != -1)
372 add_id (&aux, r->vars[i].name, _("Variable %zu"), ++var_idx);
375 for (i = 0; i < r->n_vars; i++)
376 if (r->vars[i].width != -1)
379 if (r->vars[i].label)
380 add_string (&aux, r->vars[i].label, _("Variable %zu Label"),
383 for (j = 0; j < r->vars[i].n_val_labs; j++)
384 add_string (&aux, r->vars[i].label,
385 _("Variable %zu Value Label %zu"), var_idx, j);
388 add_string (&aux, r->header.creation_date, _("Creation Date"));
389 add_string (&aux, r->header.creation_time, _("Creation Time"));
390 add_string (&aux, r->header.product, _("Product"));
391 add_string (&aux, r->header.file_label, _("File Label"));
393 *titlesp = aux.titles;
395 *stringsp = aux.strings;
400 find_and_delete_var (struct dictionary *dict, const char *name)
402 struct variable *var = dict_lookup_var (dict, name);
404 dict_delete_var (dict, var);
407 /* Decodes the dictionary read from R, saving it into into *DICT. Character
408 strings in R are decoded using ENCODING, or an encoding obtained from R if
409 ENCODING is null, or the locale encoding if R specifies no encoding.
411 If INFOP is non-null, then it receives additional info about the system
412 file, which the caller must eventually free with any_read_info_destroy()
413 when it is no longer needed.
415 This function consumes R. The caller must use it again later, even to
416 destroy it with pcp_close(). */
417 static struct casereader *
418 pcp_decode (struct any_reader *r_, const char *encoding,
419 struct dictionary **dictp, struct any_read_info *infop)
421 struct pcp_reader *r = pcp_reader_cast (r_);
422 struct dictionary *dict;
424 if (encoding == NULL)
426 encoding = locale_charset ();
427 pcp_warn (r, -1, _("Using default encoding %s to read this SPSS/PC+ "
428 "system file. For best results, specify an "
429 "encoding explicitly. Use SYSFILE INFO with "
430 "ENCODING=\"DETECT\" to analyze the possible "
435 dict = dict_create (encoding);
436 r->encoding = dict_get_encoding (dict);
438 parse_header (r, &r->header, &r->info, dict);
439 if (!parse_variable_records (r, dict, r->vars, r->n_vars))
442 /* Create an index of dictionary variable widths for
443 pcp_read_case to use. We cannot use the `struct variable's
444 from the dictionary we created, because the caller owns the
445 dictionary and may destroy or modify its variables. */
446 r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
448 find_and_delete_var (dict, "CASENUM_");
449 find_and_delete_var (dict, "DATE_");
450 find_and_delete_var (dict, "WEIGHT_");
456 memset (&r->info, 0, sizeof r->info);
459 return casereader_create_sequential
460 (NULL, r->proto, r->n_cases, &pcp_file_casereader_class, r);
463 pcp_close (&r->any_reader);
469 /* Closes R, which should have been returned by pcp_open() but not already
470 closed with pcp_decode() or this function.
471 Returns true if an I/O error has occurred on READER, false
474 pcp_close (struct any_reader *r_)
476 struct pcp_reader *r = pcp_reader_cast (r_);
481 if (fn_close (r->fh, r->file) == EOF)
483 msg (ME, _("Error closing system file `%s': %s."),
484 fh_get_file_name (r->fh), strerror (errno));
490 any_read_info_destroy (&r->info);
495 pool_destroy (r->pool);
500 /* Destroys READER. */
502 pcp_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
504 struct pcp_reader *r = r_;
505 pcp_close (&r->any_reader);
508 /* Detects whether FILE is an SPSS/PC+ system file. Returns 1 if so, 0 if
509 not, and a negative errno value if there is an error reading FILE. */
511 pcp_detect (FILE *file)
513 static const char signature[4] = "SPSS";
514 char buf[sizeof signature];
516 if (fseek (file, 0x104, SEEK_SET))
519 if (fread (buf, sizeof buf, 1, file) != 1)
520 return ferror (file) ? -errno : 0;
522 return !memcmp (buf, signature, sizeof buf);
525 /* Reads the main header of the SPSS/PC+ system file. Initializes *HEADER and
526 *INFO, except for the string fields in *INFO, which parse_header() will
527 initialize later once the file's encoding is known. */
529 read_main_header (struct pcp_reader *r, struct pcp_main_header *header)
531 unsigned int base_ofs = r->directory.main.ofs;
532 unsigned int zero0, zero1, zero2, zero3;
533 size_t min_values, min_data_size;
534 unsigned int one0, one1;
535 unsigned int compressed;
536 unsigned int n_cases1;
539 if (!pcp_seek (r, base_ofs))
542 if (r->directory.main.len < 0xb0)
544 pcp_error (r, r->pos, _("This is not an SPSS/PC+ system file."));
547 else if (r->directory.main.len > 0xb0)
548 pcp_warn (r, r->pos, _("Record 0 has unexpected length %u."),
549 r->directory.main.len);
551 if (!read_uint16 (r, &one0)
552 || !read_string (r, header->product, sizeof header->product)
553 || !read_bytes (r, sysmis, sizeof sysmis)
554 || !read_uint32 (r, &zero0)
555 || !read_uint32 (r, &zero1)
556 || !read_uint16 (r, &one1)
557 || !read_uint16 (r, &compressed)
558 || !read_uint16 (r, &header->nominal_case_size)
559 || !read_uint16 (r, &r->n_cases)
560 || !read_uint16 (r, &header->weight_index)
561 || !read_uint16 (r, &zero2)
562 || !read_uint16 (r, &n_cases1)
563 || !read_uint16 (r, &zero3)
564 || !read_string (r, header->creation_date, sizeof header->creation_date)
565 || !read_string (r, header->creation_time, sizeof header->creation_time)
566 || !read_string (r, header->file_label, sizeof header->file_label))
569 if (!pcp_is_sysmis (sysmis))
571 double d = parse_float (sysmis);
572 pcp_warn (r, base_ofs, _("Record 0 specifies unexpected system missing "
573 "value %g (%a)."), d, d);
575 if (one0 != 1 || one1 != 1
576 || zero0 != 0 || zero1 != 0 || zero2 != 0 || zero3 != 0)
577 pcp_warn (r, base_ofs, _("Record 0 reserved fields have unexpected values "
578 "(%u,%u,%u,%u,%u,%u)."),
579 one0, one1, zero0, zero1, zero2, zero3);
580 if (n_cases1 != r->n_cases)
581 pcp_warn (r, base_ofs, _("Record 0 case counts differ (%u versus %u)."),
582 r->n_cases, n_cases1);
583 if (compressed != 0 && compressed != 1)
585 pcp_error (r, base_ofs, _("Invalid compression type %u."), compressed);
589 r->compressed = compressed != 0;
591 min_values = xtimes (header->nominal_case_size, r->n_cases);
592 min_data_size = xtimes (compressed ? 1 : 8, min_values);
593 if (r->directory.data.len < min_data_size
594 || size_overflow_p (min_data_size))
596 pcp_warn (r, base_ofs, _("Record 0 claims %u cases with %u values per "
597 "case (requiring at least %zu bytes) but data "
598 "record is only %u bytes long."),
599 r->n_cases, header->nominal_case_size, min_data_size,
600 r->directory.data.len);
608 read_value_labels (struct pcp_reader *r, struct pcp_var_record *var,
609 unsigned int start, unsigned int end)
611 size_t allocated_val_labs = 0;
615 if (end > r->directory.labels.len)
617 pcp_warn (r, r->pos - 32,
618 _("Value labels claimed to end at offset %u in labels record "
619 "but labels record is only %u bytes."),
620 end, r->directory.labels.len);
624 start += r->directory.labels.ofs;
625 end += r->directory.labels.ofs;
626 if (start > end || end > r->file_size)
628 pcp_warn (r, r->pos - 32,
629 _("Value labels claimed to be at offset %u with length %u "
630 "but file size is only %u bytes."),
631 start, end - start, r->file_size);
635 if (!pcp_seek (r, start))
638 while (r->pos < end && end - r->pos > 8)
640 struct pcp_value_label *vl;
643 if (var->n_val_labs >= allocated_val_labs)
644 var->val_labs = pool_2nrealloc (r->pool, var->val_labs,
646 sizeof *var->val_labs);
647 vl = &var->val_labs[var->n_val_labs];
649 if (!read_bytes (r, vl->value, sizeof vl->value)
650 || !read_bytes (r, &len, 1))
653 if (end - r->pos < len)
656 _("Value labels end with partial label (%u bytes left in "
657 "record, label length %"PRIu8")."),
661 vl->label = pool_malloc (r->pool, len + 1);
662 if (!read_bytes (r, vl->label, len))
665 vl->label[len] = '\0';
669 pcp_warn (r, r->pos, _("%u leftover bytes following value labels."),
676 read_var_label (struct pcp_reader *r, struct pcp_var_record *var,
682 if (ofs >= r->directory.labels.len)
684 pcp_warn (r, r->pos - 32,
685 _("Variable label claimed to start at offset %u in labels "
686 "record but labels record is only %u bytes."),
687 ofs, r->directory.labels.len);
691 if (!pcp_seek (r, ofs + r->directory.labels.ofs) || !read_bytes (r, &len, 1))
694 if (len >= r->directory.labels.len - ofs)
696 pcp_warn (r, r->pos - 1,
697 _("Variable label with length %u starting at offset %u in "
698 "labels record overruns end of %u-byte labels record."),
699 len, ofs + 1, r->directory.labels.len);
703 var->label = pool_malloc (r->pool, len + 1);
704 var->label[len] = '\0';
705 return read_bytes (r, var->label, len);
708 /* Reads the variables record (record 1) into R. */
710 read_variables_record (struct pcp_reader *r)
715 if (!pcp_seek (r, r->directory.variables.ofs))
717 if (r->directory.variables.len != r->header.nominal_case_size * 32)
719 pcp_error (r, r->pos, _("Record 1 has length %u (expected %u)."),
720 r->directory.variables.len, r->header.nominal_case_size * 32);
724 r->vars = pool_calloc (r->pool,
725 r->header.nominal_case_size, sizeof *r->vars);
727 for (i = 0; i < r->header.nominal_case_size; i++)
729 struct pcp_var_record *var = &r->vars[r->n_vars++];
730 unsigned int value_label_start, value_label_end;
731 unsigned int var_label_ofs;
736 if (!read_uint32 (r, &value_label_start)
737 || !read_uint32 (r, &value_label_end)
738 || !read_uint32 (r, &var_label_ofs)
739 || !read_uint32 (r, &format)
740 || !read_string (r, var->name, sizeof var->name)
741 || !read_bytes (r, var->missing, sizeof var->missing))
744 var->weight = r->header.weight_index && i == r->header.weight_index - 1;
748 raw_type = format >> 16;
749 if (!fmt_from_io (raw_type, &var->format.type))
751 pcp_error (r, var->pos, _("Variable %u has invalid type %"PRIu8"."),
756 var->format.w = (format >> 8) & 0xff;
757 var->format.d = format & 0xff;
758 fmt_fix_output (&var->format);
759 var->width = fmt_var_width (&var->format);
763 unsigned int save_pos = r->pos;
764 if (!read_var_label (r, var, var_label_ofs)
765 || !pcp_seek (r, save_pos))
769 if (value_label_end > value_label_start && var->width <= 8)
771 unsigned int save_pos = r->pos;
772 if (!read_value_labels (r, var, value_label_start, value_label_end)
773 || !pcp_seek (r, save_pos))
779 int extra = DIV_RND_UP (var->width - 8, 8);
781 if (!skip_bytes (r, 32 * extra))
786 if (r->header.weight_index && !weighted)
787 pcp_warn (r, -1, _("Invalid weight index %u."), r->header.weight_index);
793 recode_and_trim_string (struct pool *pool, const char *from, const char *in)
795 struct substring out;
797 out = recode_substring_pool ("UTF-8", from, ss_cstr (in), pool);
798 ss_trim (&out, ss_cstr (" "));
799 return ss_xstrdup (out);
803 parse_header (struct pcp_reader *r, const struct pcp_main_header *header,
804 struct any_read_info *info, struct dictionary *dict)
806 const char *dict_encoding = dict_get_encoding (dict);
809 memset (info, 0, sizeof *info);
811 info->integer_format = INTEGER_LSB_FIRST;
812 info->float_format = FLOAT_IEEE_DOUBLE_LE;
813 info->compression = r->compressed ? ANY_COMP_SIMPLE : ANY_COMP_NONE;
814 info->case_cnt = r->n_cases;
816 /* Convert file label to UTF-8 and put it into DICT. */
817 label = recode_and_trim_string (r->pool, dict_encoding, header->file_label);
818 dict_set_label (dict, label);
821 /* Put creation date, time, and product in UTF-8 into INFO. */
822 info->creation_date = recode_and_trim_string (r->pool, dict_encoding,
823 header->creation_date);
824 info->creation_time = recode_and_trim_string (r->pool, dict_encoding,
825 header->creation_time);
826 info->product = recode_and_trim_string (r->pool, dict_encoding,
830 /* Reads a variable (type 2) record from R and adds the
831 corresponding variable to DICT.
832 Also skips past additional variable records for long string
835 parse_variable_records (struct pcp_reader *r, struct dictionary *dict,
836 struct pcp_var_record *var_recs, size_t n_var_recs)
838 const char *dict_encoding = dict_get_encoding (dict);
839 struct pcp_var_record *rec;
841 for (rec = var_recs; rec < &var_recs[n_var_recs]; rec++)
843 struct variable *var;
847 name = recode_string_pool ("UTF-8", dict_encoding,
848 rec->name, -1, r->pool);
849 name[strcspn (name, " ")] = '\0';
851 /* Transform $DATE => DATE_, $WEIGHT => WEIGHT_, $CASENUM => CASENUM_. */
853 name = pool_asprintf (r->pool, "%s_", name + 1);
855 if (!dict_id_is_valid (dict, name, false) || name[0] == '#')
857 pcp_error (r, rec->pos, _("Invalid variable name `%s'."), name);
861 var = rec->var = dict_create_var (dict, name, rec->width);
864 char *new_name = dict_make_unique_var_name (dict, NULL, NULL);
865 pcp_warn (r, rec->pos, _("Renaming variable with duplicate name "
868 var = rec->var = dict_create_var_assert (dict, new_name, rec->width);
874 dict_set_weight (dict, var);
876 pcp_warn (r, rec->pos,
877 _("Cannot weight by string variable `%s'."), name);
880 /* Set the short name the same as the long name. */
881 var_set_short_name (var, 0, name);
883 /* Get variable label, if any. */
888 utf8_label = recode_string ("UTF-8", dict_encoding, rec->label, -1);
889 var_set_label (var, utf8_label);
893 /* Add value labels. */
894 for (i = 0; i < rec->n_val_labs; i++)
899 value_init (&value, rec->width);
900 if (var_is_numeric (var))
901 value.f = parse_float (rec->val_labs[i].value);
903 memcpy (value_str_rw (&value, rec->width),
904 rec->val_labs[i].value, rec->width);
906 utf8_label = recode_string ("UTF-8", dict_encoding,
907 rec->val_labs[i].label, -1);
908 var_add_value_label (var, &value, utf8_label);
911 value_destroy (&value, rec->width);
914 /* Set missing values. */
915 if (rec->width <= 8 && !pcp_is_sysmis (rec->missing))
917 int width = var_get_width (var);
918 struct missing_values mv;
920 mv_init_pool (r->pool, &mv, width);
921 if (var_is_numeric (var))
922 mv_add_num (&mv, parse_float (rec->missing));
924 mv_add_str (&mv, rec->missing, MIN (width, 8));
925 var_set_missing_values (var, &mv);
929 var_set_both_formats (var, &rec->format);
937 static void read_error (struct casereader *, const struct pcp_reader *);
939 static bool read_case_number (struct pcp_reader *, double *);
940 static int read_case_string (struct pcp_reader *, uint8_t *, size_t);
941 static int read_opcode (struct pcp_reader *);
942 static bool read_compressed_number (struct pcp_reader *, double *);
943 static int read_compressed_string (struct pcp_reader *, uint8_t *);
944 static int read_whole_strings (struct pcp_reader *, uint8_t *, size_t);
946 /* Reads and returns one case from READER's file. Returns a null
947 pointer if not successful. */
948 static struct ccase *
949 pcp_file_casereader_read (struct casereader *reader, void *r_)
951 struct pcp_reader *r = r_;
952 unsigned int start_pos = r->pos;
957 if (r->error || !r->n_cases)
961 c = case_create (r->proto);
962 for (i = 0; i < r->n_vars; i++)
964 struct pcp_var_record *var = &r->vars[i];
965 union value *v = case_data_rw_idx (c, i);
968 retval = read_case_number (r, &v->f);
970 retval = read_case_string (r, value_str_rw (v, var->width),
975 pcp_error (r, r->pos, _("File ends in partial case."));
979 if (r->pos > r->directory.data.ofs + r->directory.data.len)
981 pcp_error (r, r->pos, _("Case beginning at offset 0x%08x extends past "
982 "end of data record at offset 0x%08x."),
983 start_pos, r->directory.data.ofs + r->directory.data.len);
990 read_error (reader, r);
995 /* Issues an error that an unspecified error occurred PCP, and
998 read_error (struct casereader *r, const struct pcp_reader *pcp)
1000 msg (ME, _("Error reading case from file %s."), fh_get_name (pcp->fh));
1001 casereader_force_error (r);
1004 /* Reads a number from R and stores its value in *D.
1005 If R is compressed, reads a compressed number;
1006 otherwise, reads a number in the regular way.
1007 Returns true if successful, false if end of file is
1008 reached immediately. */
1010 read_case_number (struct pcp_reader *r, double *d)
1015 if (!try_read_bytes (r, number, sizeof number))
1017 *d = parse_float (number);
1021 return read_compressed_number (r, d);
1024 /* Reads LENGTH string bytes from R into S. Always reads a multiple of 8
1025 bytes; if LENGTH is not a multiple of 8, then extra bytes are read and
1026 discarded without being written to S. Reads compressed strings if S is
1027 compressed. Returns 1 if successful, 0 if end of file is reached
1028 immediately, or -1 for some kind of error. */
1030 read_case_string (struct pcp_reader *r, uint8_t *s, size_t length)
1032 size_t whole = ROUND_DOWN (length, 8);
1033 size_t partial = length % 8;
1037 int retval = read_whole_strings (r, s, whole);
1045 int retval = read_whole_strings (r, bounce, sizeof bounce);
1048 memcpy (s + whole, bounce, partial);
1054 /* Reads and returns the next compression opcode from R. */
1056 read_opcode (struct pcp_reader *r)
1058 assert (r->compressed);
1059 if (r->opcode_idx >= sizeof r->opcodes)
1061 int retval = try_read_bytes (r, r->opcodes, sizeof r->opcodes);
1066 return r->opcodes[r->opcode_idx++];
1069 /* Reads a compressed number from R and stores its value in D.
1070 Returns true if successful, false if end of file is
1071 reached immediately. */
1073 read_compressed_number (struct pcp_reader *r, double *d)
1075 int opcode = read_opcode (r);
1086 return read_float (r, d);
1089 *d = opcode - 105.0;
1094 /* Reads a compressed 8-byte string segment from R and stores it in DST. */
1096 read_compressed_string (struct pcp_reader *r, uint8_t *dst)
1101 opcode = read_opcode (r);
1108 retval = read_bytes (r, dst, 8);
1109 return retval == 1 ? 1 : -1;
1112 if (!r->corruption_warning)
1114 r->corruption_warning = true;
1115 pcp_warn (r, r->pos,
1116 _("Possible compressed data corruption: "
1117 "string contains compressed integer (opcode %d)."),
1120 memset (dst, ' ', 8);
1125 /* Reads LENGTH string bytes from R into S. LENGTH must be a multiple of 8.
1126 Reads compressed strings if S is compressed. Returns 1 if successful, 0 if
1127 end of file is reached immediately, or -1 for some kind of error. */
1129 read_whole_strings (struct pcp_reader *r, uint8_t *s, size_t length)
1131 assert (length % 8 == 0);
1133 return try_read_bytes (r, s, length);
1138 for (ofs = 0; ofs < length; ofs += 8)
1140 int retval = read_compressed_string (r, s + ofs);
1150 /* Displays a corruption message. */
1152 pcp_msg (struct pcp_reader *r, off_t offset,
1153 int class, const char *format, va_list args)
1158 ds_init_empty (&text);
1160 ds_put_format (&text, _("`%s' near offset 0x%llx: "),
1161 fh_get_file_name (r->fh), (long long int) offset);
1163 ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
1164 ds_put_vformat (&text, format, args);
1166 m.category = msg_class_to_category (class);
1167 m.severity = msg_class_to_severity (class);
1173 m.text = ds_cstr (&text);
1178 /* Displays a warning for offset OFFSET in the file. */
1180 pcp_warn (struct pcp_reader *r, off_t offset, const char *format, ...)
1184 va_start (args, format);
1185 pcp_msg (r, offset, MW, format, args);
1189 /* Displays an error for the current file position,
1190 marks it as in an error state,
1191 and aborts reading it using longjmp. */
1193 pcp_error (struct pcp_reader *r, off_t offset, const char *format, ...)
1197 va_start (args, format);
1198 pcp_msg (r, offset, ME, format, args);
1204 /* Reads BYTE_CNT bytes into BUF.
1205 Returns 1 if exactly BYTE_CNT bytes are successfully read.
1206 Returns -1 if an I/O error or a partial read occurs.
1207 Returns 0 for an immediate end-of-file and, if EOF_IS_OK is false, reports
1210 read_bytes_internal (struct pcp_reader *r, bool eof_is_ok,
1211 void *buf, size_t byte_cnt)
1213 size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
1214 r->pos += bytes_read;
1215 if (bytes_read == byte_cnt)
1217 else if (ferror (r->file))
1219 pcp_error (r, r->pos, _("System error: %s."), strerror (errno));
1222 else if (!eof_is_ok || bytes_read != 0)
1224 pcp_error (r, r->pos, _("Unexpected end of file."));
1231 /* Reads BYTE_CNT into BUF.
1232 Returns true if successful.
1233 Returns false upon I/O error or if end-of-file is encountered. */
1235 read_bytes (struct pcp_reader *r, void *buf, size_t byte_cnt)
1237 return read_bytes_internal (r, false, buf, byte_cnt) == 1;
1240 /* Reads BYTE_CNT bytes into BUF.
1241 Returns 1 if exactly BYTE_CNT bytes are successfully read.
1242 Returns 0 if an immediate end-of-file is encountered.
1243 Returns -1 if an I/O error or a partial read occurs. */
1245 try_read_bytes (struct pcp_reader *r, void *buf, size_t byte_cnt)
1247 return read_bytes_internal (r, true, buf, byte_cnt);
1250 /* Reads a 16-bit signed integer from R and stores its value in host format in
1251 *X. Returns true if successful, otherwise false. */
1253 read_uint16 (struct pcp_reader *r, unsigned int *x)
1256 if (read_bytes (r, integer, sizeof integer) != 1)
1258 *x = integer_get (INTEGER_LSB_FIRST, integer, sizeof integer);
1262 /* Reads a 32-bit signed integer from R and stores its value in host format in
1263 *X. Returns true if successful, otherwise false. */
1265 read_uint32 (struct pcp_reader *r, unsigned int *x)
1268 if (read_bytes (r, integer, sizeof integer) != 1)
1270 *x = integer_get (INTEGER_LSB_FIRST, integer, sizeof integer);
1274 /* Reads exactly SIZE - 1 bytes into BUFFER
1275 and stores a null byte into BUFFER[SIZE - 1]. */
1277 read_string (struct pcp_reader *r, char *buffer, size_t size)
1282 ok = read_bytes (r, buffer, size - 1);
1284 buffer[size - 1] = '\0';
1288 /* Skips BYTES bytes forward in R. */
1290 skip_bytes (struct pcp_reader *r, size_t bytes)
1295 size_t chunk = MIN (sizeof buffer, bytes);
1296 if (!read_bytes (r, buffer, chunk))
1305 pcp_seek (struct pcp_reader *r, off_t offset)
1307 if (fseeko (r->file, offset, SEEK_SET))
1309 pcp_error (r, 0, _("%s: seek failed (%s)."),
1310 fh_get_file_name (r->fh), strerror (errno));
1317 /* Reads a 64-bit floating-point number from R and returns its
1318 value in host format. */
1320 read_float (struct pcp_reader *r, double *d)
1324 if (!read_bytes (r, number, sizeof number))
1328 *d = parse_float (number);
1334 parse_float (const uint8_t number[8])
1336 return (pcp_is_sysmis (number)
1338 : float_get_double (FLOAT_IEEE_DOUBLE_LE, number));
1342 pcp_is_sysmis(const uint8_t *p)
1344 static const uint8_t sysmis[8]
1345 = { 0xf5, 0x1e, 0x26, 0x02, 0x8a, 0x8c, 0xed, 0xff };
1346 return !memcmp (p, sysmis, 8);
1349 static const struct casereader_class pcp_file_casereader_class =
1351 pcp_file_casereader_read,
1352 pcp_file_casereader_destroy,
1357 const struct any_reader_class pcp_file_reader_class =
1359 N_("SPSS/PC+ System File"),