1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2000, 2006-2007, 2009-2016, 2021 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/sys-file-private.h"
28 #include "data/any-reader.h"
29 #include "data/attributes.h"
30 #include "data/case.h"
31 #include "data/casereader-provider.h"
32 #include "data/casereader.h"
33 #include "data/dictionary.h"
34 #include "data/file-handle-def.h"
35 #include "data/file-name.h"
36 #include "data/format.h"
37 #include "data/identifier.h"
38 #include "data/missing-values.h"
39 #include "data/mrset.h"
40 #include "data/short-names.h"
41 #include "data/value-labels.h"
42 #include "data/value.h"
43 #include "data/variable.h"
44 #include "libpspp/array.h"
45 #include "libpspp/assertion.h"
46 #include "libpspp/compiler.h"
47 #include "libpspp/i18n.h"
48 #include "libpspp/ll.h"
49 #include "libpspp/message.h"
50 #include "libpspp/misc.h"
51 #include "libpspp/pool.h"
52 #include "libpspp/str.h"
53 #include "libpspp/stringi-set.h"
55 #include "gl/c-strtod.h"
56 #include "gl/c-ctype.h"
57 #include "gl/inttostr.h"
58 #include "gl/localcharset.h"
59 #include "gl/minmax.h"
60 #include "gl/unlocked-io.h"
61 #include "gl/xalloc.h"
62 #include "gl/xalloc-oversized.h"
66 #define _(msgid) gettext (msgid)
67 #define N_(msgid) (msgid)
71 /* subtypes 0-2 unknown */
72 EXT_INTEGER = 3, /* Machine integer info. */
73 EXT_FLOAT = 4, /* Machine floating-point info. */
74 EXT_VAR_SETS = 5, /* Variable sets. */
75 EXT_DATE = 6, /* DATE. */
76 EXT_MRSETS = 7, /* Multiple response sets. */
77 EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
78 /* subtype 9 unknown */
79 EXT_PRODUCT_INFO = 10, /* Extra product info text. */
80 EXT_DISPLAY = 11, /* Variable display parameters. */
81 /* subtype 12 unknown */
82 EXT_LONG_NAMES = 13, /* Long variable names. */
83 EXT_LONG_STRINGS = 14, /* Long strings. */
84 /* subtype 15 unknown */
85 EXT_NCASES = 16, /* Extended number of cases. */
86 EXT_FILE_ATTRS = 17, /* Data file attributes. */
87 EXT_VAR_ATTRS = 18, /* Variable attributes. */
88 EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
89 EXT_ENCODING = 20, /* Character encoding. */
90 EXT_LONG_LABELS = 21, /* Value labels for long strings. */
91 EXT_LONG_MISSING = 22, /* Missing values for long strings. */
92 EXT_DATAVIEW = 24 /* "Format properties in dataview table". */
95 /* Fields from the top-level header record. */
96 struct sfm_header_record
98 char magic[5]; /* First 4 bytes of file, then null. */
99 int weight_idx; /* 0 if unweighted, otherwise a var index. */
100 int nominal_case_size; /* Number of var positions. */
102 /* These correspond to the members of struct any_file_info or a dictionary
103 but in the system file's encoding rather than ASCII. */
104 char creation_date[10]; /* "dd mmm yy". */
105 char creation_time[9]; /* "hh:mm:ss". */
106 char eye_catcher[61]; /* Eye-catcher string, then product name. */
107 char file_label[65]; /* File label. */
110 struct sfm_var_record
117 int missing_value_code;
120 struct variable *var;
123 struct sfm_value_label
129 struct sfm_value_label_record
132 struct sfm_value_label *labels;
133 unsigned int n_labels;
139 struct sfm_document_record
148 const char *name; /* Name. */
149 const char *label; /* Human-readable label for group. */
150 enum mrset_type type; /* Group type. */
151 const char **vars; /* Constituent variables' names. */
152 size_t n_vars; /* Number of constituent variables. */
155 enum mrset_md_cat_source cat_source; /* Source of category labels. */
156 bool label_from_var_label; /* 'label' taken from variable label? */
157 const char *counted; /* Counted value, as string. */
160 struct sfm_extension_record
162 struct ll ll; /* In struct sfm_reader 'var_attrs' list. */
163 int subtype; /* Record subtype. */
164 off_t pos; /* Starting offset in file. */
165 unsigned int size; /* Size of data elements. */
166 unsigned int count; /* Number of data elements. */
167 void *data; /* Contents. */
170 /* System file reader. */
173 struct any_reader any_reader;
175 /* Resource tracking. */
176 struct pool *pool; /* All system file state. */
179 struct any_read_info info;
180 struct sfm_header_record header;
181 struct sfm_var_record *vars;
183 struct sfm_value_label_record *labels;
185 struct sfm_document_record *document;
186 struct sfm_mrset *mrsets;
188 struct sfm_extension_record *extensions[32];
189 struct ll_list var_attrs; /* Contains "struct sfm_extension_record"s. */
192 struct file_handle *fh; /* File handle. */
193 struct fh_lock *lock; /* Mutual exclusion for file handle. */
194 FILE *file; /* File stream. */
195 off_t pos; /* Position in file. */
196 bool error; /* I/O or corruption error? */
197 struct caseproto *proto; /* Format of output cases. */
200 enum integer_format integer_format; /* On-disk integer format. */
201 enum float_format float_format; /* On-disk floating point format. */
202 struct sfm_var *sfm_vars; /* Variables. */
203 size_t sfm_var_cnt; /* Number of variables. */
204 int case_cnt; /* Number of cases */
205 const char *encoding; /* String encoding. */
206 bool written_by_readstat; /* From https://github.com/WizardMac/ReadStat? */
209 enum any_compression compression;
210 double bias; /* Compression bias, usually 100.0. */
211 uint8_t opcodes[8]; /* Current block of opcodes. */
212 size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
213 bool corruption_warning; /* Warned about possible corruption? */
215 /* ZLIB decompression. */
216 long long int ztrailer_ofs; /* Offset of ZLIB trailer at end of file. */
217 #define ZIN_BUF_SIZE 4096
218 uint8_t *zin_buf; /* Inflation input buffer. */
219 #define ZOUT_BUF_SIZE 16384
220 uint8_t *zout_buf; /* Inflation output buffer. */
221 unsigned int zout_end; /* Number of bytes of data in zout_buf. */
222 unsigned int zout_pos; /* First unconsumed byte in zout_buf. */
223 z_stream zstream; /* ZLIB inflater. */
226 static const struct casereader_class sys_file_casereader_class;
228 static struct sfm_reader *
229 sfm_reader_cast (const struct any_reader *r_)
231 assert (r_->klass == &sys_file_reader_class);
232 return UP_CAST (r_, struct sfm_reader, any_reader);
235 static bool sfm_close (struct any_reader *);
237 static void sys_msg (struct sfm_reader *r, off_t, int class,
238 const char *format, va_list args)
239 PRINTF_FORMAT (4, 0);
240 static void sys_warn (struct sfm_reader *, off_t, const char *, ...)
241 PRINTF_FORMAT (3, 4);
242 static void sys_error (struct sfm_reader *, off_t, const char *, ...)
243 PRINTF_FORMAT (3, 4);
245 static bool read_bytes (struct sfm_reader *, void *, size_t)
247 static int try_read_bytes (struct sfm_reader *, void *, size_t)
249 static bool read_int (struct sfm_reader *, int *) WARN_UNUSED_RESULT;
250 static bool read_uint (struct sfm_reader *, unsigned int *) WARN_UNUSED_RESULT;
251 static bool read_int64 (struct sfm_reader *, long long int *)
253 static bool read_uint64 (struct sfm_reader *, unsigned long long int *)
255 static bool read_string (struct sfm_reader *, char *, size_t)
257 static bool skip_bytes (struct sfm_reader *, size_t) WARN_UNUSED_RESULT;
259 /* ZLIB compressed data handling. */
260 static bool read_zheader (struct sfm_reader *) WARN_UNUSED_RESULT;
261 static bool open_zstream (struct sfm_reader *) WARN_UNUSED_RESULT;
262 static bool close_zstream (struct sfm_reader *) WARN_UNUSED_RESULT;
263 static int read_bytes_zlib (struct sfm_reader *, void *, size_t)
265 static int read_compressed_bytes (struct sfm_reader *, void *, size_t)
267 static int try_read_compressed_bytes (struct sfm_reader *, void *, size_t)
269 static bool read_compressed_float (struct sfm_reader *, double *)
272 static char *fix_line_ends (const char *);
274 static int parse_int (const struct sfm_reader *, const void *data, size_t ofs);
275 static double parse_float (const struct sfm_reader *,
276 const void *data, size_t ofs);
278 static bool read_variable_record (struct sfm_reader *,
279 struct sfm_var_record *);
280 static bool read_value_label_record (struct sfm_reader *,
281 struct sfm_value_label_record *);
282 static bool read_document_record (struct sfm_reader *);
283 static bool read_extension_record (struct sfm_reader *, int subtype,
284 struct sfm_extension_record **);
285 static bool skip_extension_record (struct sfm_reader *, int subtype);
287 static struct text_record *open_text_record (
288 struct sfm_reader *, const struct sfm_extension_record *,
289 bool recode_to_utf8);
290 static void close_text_record (struct sfm_reader *,
291 struct text_record *);
292 static bool read_variable_to_value_pair (struct sfm_reader *,
294 struct text_record *,
295 struct variable **var, char **value);
296 static void text_warn (struct sfm_reader *r, struct text_record *text,
297 const char *format, ...) PRINTF_FORMAT (3, 4);
298 static char *text_get_token (struct text_record *,
299 struct substring delimiters, char *delimiter);
300 static bool text_match (struct text_record *, char c);
301 static bool text_read_variable_name (struct sfm_reader *, struct dictionary *,
302 struct text_record *,
303 struct substring delimiters,
305 static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
306 struct text_record *,
307 struct substring delimiters,
309 static const char *text_parse_counted_string (struct sfm_reader *,
310 struct text_record *);
311 static size_t text_pos (const struct text_record *);
312 static const char *text_get_all (const struct text_record *);
314 /* Dictionary reader. */
322 static bool read_dictionary (struct sfm_reader *);
323 static bool read_record (struct sfm_reader *, int type,
324 size_t *allocated_vars, size_t *allocated_labels);
325 static bool read_header (struct sfm_reader *, struct any_read_info *,
326 struct sfm_header_record *);
327 static void parse_header (struct sfm_reader *,
328 const struct sfm_header_record *,
329 struct any_read_info *, struct dictionary *);
330 static bool parse_variable_records (struct sfm_reader *, struct dictionary *,
331 struct sfm_var_record *, size_t n);
332 static void parse_format_spec (struct sfm_reader *, off_t pos,
333 unsigned int format, enum which_format,
334 struct variable *, int *format_warning_cnt);
335 static void parse_document (struct dictionary *, struct sfm_document_record *);
336 static void parse_display_parameters (struct sfm_reader *,
337 const struct sfm_extension_record *,
338 struct dictionary *);
339 static bool parse_machine_integer_info (struct sfm_reader *,
340 const struct sfm_extension_record *,
341 struct any_read_info *);
342 static void parse_machine_float_info (struct sfm_reader *,
343 const struct sfm_extension_record *);
344 static void parse_extra_product_info (struct sfm_reader *,
345 const struct sfm_extension_record *,
346 struct any_read_info *);
347 static void parse_mrsets (struct sfm_reader *,
348 const struct sfm_extension_record *,
349 size_t *allocated_mrsets);
350 static void decode_mrsets (struct sfm_reader *, struct dictionary *);
351 static void parse_long_var_name_map (struct sfm_reader *,
352 const struct sfm_extension_record *,
353 struct dictionary *);
354 static bool parse_long_string_map (struct sfm_reader *,
355 const struct sfm_extension_record *,
356 struct dictionary *);
357 static void parse_value_labels (struct sfm_reader *, struct dictionary *);
358 static struct variable *parse_weight_var (struct sfm_reader *,
359 const struct sfm_var_record *, size_t n_var_recs,
361 static void parse_data_file_attributes (struct sfm_reader *,
362 const struct sfm_extension_record *,
363 struct dictionary *);
364 static void parse_variable_attributes (struct sfm_reader *,
365 const struct sfm_extension_record *,
366 struct dictionary *);
367 static void assign_variable_roles (struct sfm_reader *, struct dictionary *);
368 static void parse_long_string_value_labels (struct sfm_reader *,
369 const struct sfm_extension_record *,
370 struct dictionary *);
371 static void parse_long_string_missing_values (
372 struct sfm_reader *, const struct sfm_extension_record *,
373 struct dictionary *);
375 /* Frees the strings inside INFO. */
377 any_read_info_destroy (struct any_read_info *info)
381 free (info->creation_date);
382 free (info->creation_time);
383 free (info->product);
384 free (info->product_ext);
388 /* Tries to open FH for reading as a system file. Returns an sfm_reader if
389 successful, otherwise NULL. */
390 static struct any_reader *
391 sfm_open (struct file_handle *fh)
393 size_t allocated_mrsets = 0;
395 /* Create and initialize reader. */
396 struct sfm_reader *r = XZALLOC (struct sfm_reader);
397 r->any_reader.klass = &sys_file_reader_class;
398 r->pool = pool_create ();
399 pool_register (r->pool, free, r);
401 r->opcode_idx = sizeof r->opcodes;
402 ll_init (&r->var_attrs);
404 /* TRANSLATORS: this fragment will be interpolated into
405 messages in fh_lock() that identify types of files. */
406 r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false);
410 r->file = fn_open (fh, "rb");
413 msg (ME, _("Error opening `%s' for reading as a system file: %s."),
414 fh_get_file_name (r->fh), strerror (errno));
418 if (!read_dictionary (r))
421 if (r->extensions[EXT_MRSETS] != NULL)
422 parse_mrsets (r, r->extensions[EXT_MRSETS], &allocated_mrsets);
424 if (r->extensions[EXT_MRSETS2] != NULL)
425 parse_mrsets (r, r->extensions[EXT_MRSETS2], &allocated_mrsets);
427 return &r->any_reader;
431 sfm_close (&r->any_reader);
436 read_dictionary (struct sfm_reader *r)
438 size_t allocated_vars;
439 size_t allocated_labels;
441 if (!read_header (r, &r->info, &r->header))
445 allocated_labels = 0;
450 if (!read_int (r, &type))
454 if (!read_record (r, type, &allocated_vars, &allocated_labels))
458 if (!skip_bytes (r, 4))
461 if (r->compression == ANY_COMP_ZLIB && !read_zheader (r))
468 read_record (struct sfm_reader *r, int type,
469 size_t *allocated_vars, size_t *allocated_labels)
476 if (r->n_vars >= *allocated_vars)
477 r->vars = pool_2nrealloc (r->pool, r->vars, allocated_vars,
479 return read_variable_record (r, &r->vars[r->n_vars++]);
482 if (r->n_labels >= *allocated_labels)
483 r->labels = pool_2nrealloc (r->pool, r->labels, allocated_labels,
485 return read_value_label_record (r, &r->labels[r->n_labels++]);
488 /* A Type 4 record is always immediately after a type 3 record,
489 so the code for type 3 records reads the type 4 record too. */
490 sys_error (r, r->pos, _("Misplaced type 4 record."));
494 if (r->document != NULL)
495 sys_warn (r, r->pos, _("Duplicate type 6 (document) record."));
496 return read_document_record (r);
499 if (!read_int (r, &subtype))
502 || subtype >= sizeof r->extensions / sizeof *r->extensions)
505 _("Unrecognized record type 7, subtype %d. For help, "
506 "please send this file to %s and mention that you were "
508 subtype, PACKAGE_BUGREPORT, PACKAGE_STRING);
509 return skip_extension_record (r, subtype);
511 else if (subtype == 18)
513 /* System files written by "Stata 14.1/-savespss- 1.77 by S.Radyakin"
514 put each variable attribute into a separate record with subtype
515 18. I'm surprised that SPSS puts up with this. */
516 struct sfm_extension_record *ext;
517 bool ok = read_extension_record (r, subtype, &ext);
519 ll_push_tail (&r->var_attrs, &ext->ll);
522 else if (r->extensions[subtype] != NULL)
525 _("Record type 7, subtype %d found here has the same "
526 "type as the record found near offset 0x%llx. For "
527 "help, please send this file to %s and mention that "
528 "you were using %s."),
529 subtype, (long long int) r->extensions[subtype]->pos,
530 PACKAGE_BUGREPORT, PACKAGE_STRING);
531 return skip_extension_record (r, subtype);
534 return read_extension_record (r, subtype, &r->extensions[subtype]);
537 sys_error (r, r->pos, _("Unrecognized record type %d."), type);
544 /* Returns the character encoding obtained from R, or a null pointer if R
545 doesn't have an indication of its character encoding. */
547 sfm_get_encoding (const struct sfm_reader *r)
549 /* The EXT_ENCODING record is the best way to determine dictionary
551 if (r->extensions[EXT_ENCODING])
552 return r->extensions[EXT_ENCODING]->data;
554 /* But EXT_INTEGER is better than nothing as a fallback. */
555 if (r->extensions[EXT_INTEGER])
557 int codepage = parse_int (r, r->extensions[EXT_INTEGER]->data, 7 * 4);
558 const char *encoding;
567 /* These ostensibly mean "7-bit ASCII" and "8-bit ASCII"[sic]
568 respectively. However, many files have character code 2 but data
569 which are clearly not ASCII. Therefore, ignore these values. */
576 encoding = sys_get_encoding_from_codepage (codepage);
577 if (encoding != NULL)
583 /* If the file magic number is EBCDIC then its character data is too. */
584 if (!strcmp (r->header.magic, EBCDIC_MAGIC))
590 struct get_strings_aux
601 add_string__ (struct get_strings_aux *aux,
602 const char *string, bool id, char *title)
604 if (aux->n >= aux->allocated)
606 aux->allocated = 2 * (aux->allocated + 1);
607 aux->titles = pool_realloc (aux->pool, aux->titles,
608 aux->allocated * sizeof *aux->titles);
609 aux->strings = pool_realloc (aux->pool, aux->strings,
610 aux->allocated * sizeof *aux->strings);
611 aux->ids = pool_realloc (aux->pool, aux->ids,
612 aux->allocated * sizeof *aux->ids);
615 aux->titles[aux->n] = title;
616 aux->strings[aux->n] = pool_strdup (aux->pool, string);
617 aux->ids[aux->n] = id;
621 static void PRINTF_FORMAT (3, 4)
622 add_string (struct get_strings_aux *aux,
623 const char *string, const char *title, ...)
627 va_start (args, title);
628 add_string__ (aux, string, false, pool_vasprintf (aux->pool, title, args));
632 static void PRINTF_FORMAT (3, 4)
633 add_id (struct get_strings_aux *aux, const char *id, const char *title, ...)
637 va_start (args, title);
638 add_string__ (aux, id, true, pool_vasprintf (aux->pool, title, args));
643 skip_prefix (const char *s, const char *prefix)
645 size_t prefix_len = strlen (prefix);
646 return !strncmp (s, prefix, prefix_len) ? s + prefix_len : s;
649 /* Retrieves significant string data from R in its raw format, to allow the
650 caller to try to detect the encoding in use.
652 Returns the number of strings retrieved N. Sets each of *TITLESP, *IDSP,
653 and *STRINGSP to an array of N elements allocated from POOL. For each I in
654 0...N-1, UTF-8 string *TITLESP[I] describes *STRINGSP[I], which is in
655 whatever encoding system file R uses. *IDS[I] is true if *STRINGSP[I] must
656 be a valid PSPP language identifier, false if *STRINGSP[I] is free-form
659 sfm_get_strings (const struct any_reader *r_, struct pool *pool,
660 char ***titlesp, bool **idsp, char ***stringsp)
662 struct sfm_reader *r = sfm_reader_cast (r_);
663 const struct sfm_mrset *mrset;
664 struct get_strings_aux aux;
676 for (i = 0; i < r->n_vars; i++)
677 if (r->vars[i].width != -1)
678 add_id (&aux, r->vars[i].name, _("Variable %zu"), ++var_idx);
681 for (i = 0; i < r->n_vars; i++)
682 if (r->vars[i].width != -1)
685 if (r->vars[i].label)
686 add_string (&aux, r->vars[i].label, _("Variable %zu Label"),
691 for (i = 0; i < r->n_labels; i++)
692 for (j = 0; j < r->labels[i].n_labels; j++)
693 add_string (&aux, r->labels[i].labels[j].label,
694 _("Value Label %zu"), k++);
696 add_string (&aux, r->header.creation_date, _("Creation Date"));
697 add_string (&aux, r->header.creation_time, _("Creation Time"));
698 add_string (&aux, skip_prefix (r->header.eye_catcher, "@(#) "), _("Product"));
699 add_string (&aux, r->header.file_label, _("File Label"));
701 if (r->extensions[EXT_PRODUCT_INFO])
702 add_string (&aux, r->extensions[EXT_PRODUCT_INFO]->data,
703 _("Extra Product Info"));
709 for (i = 0; i < r->document->n_lines; i++)
713 memcpy (line, r->document->documents + i * 80, 80);
716 add_string (&aux, line, _("Document Line %zu"), i + 1);
720 for (mrset = r->mrsets; mrset < &r->mrsets[r->n_mrsets]; mrset++)
722 size_t mrset_idx = mrset - r->mrsets + 1;
724 add_id (&aux, mrset->name, _("MRSET %zu"), mrset_idx);
726 add_string (&aux, mrset->label, _("MRSET %zu Label"), mrset_idx);
728 /* Skip the variables because they ought to be duplicates. */
731 add_string (&aux, mrset->counted, _("MRSET %zu Counted Value"),
735 /* data file attributes */
736 /* variable attributes */
738 /* long string value labels */
739 /* long string missing values */
741 *titlesp = aux.titles;
743 *stringsp = aux.strings;
747 /* Decodes the dictionary read from R, saving it into into *DICT. Character
748 strings in R are decoded using ENCODING, or an encoding obtained from R if
749 ENCODING is null, or the locale encoding if R specifies no encoding.
751 If INFOP is non-null, then it receives additional info about the system
752 file, which the caller must eventually free with any_read_info_destroy()
753 when it is no longer needed.
755 This function consumes R. The caller must use it again later, even to
756 destroy it with sfm_close(). */
757 static struct casereader *
758 sfm_decode (struct any_reader *r_, const char *encoding,
759 struct dictionary **dictp, struct any_read_info *infop)
761 struct sfm_reader *r = sfm_reader_cast (r_);
762 struct dictionary *dict;
764 if (encoding == NULL)
766 encoding = sfm_get_encoding (r);
767 if (encoding == NULL)
769 sys_warn (r, -1, _("This system file does not indicate its own "
770 "character encoding. Using default encoding "
771 "%s. For best results, specify an encoding "
772 "explicitly. Use SYSFILE INFO with "
773 "ENCODING=\"DETECT\" to analyze the possible "
776 encoding = locale_charset ();
780 dict = dict_create (encoding);
781 r->encoding = dict_get_encoding (dict);
783 /* These records don't use variables at all. */
784 if (r->document != NULL)
785 parse_document (dict, r->document);
787 if (r->extensions[EXT_INTEGER] != NULL
788 && !parse_machine_integer_info (r, r->extensions[EXT_INTEGER], &r->info))
791 if (r->extensions[EXT_FLOAT] != NULL)
792 parse_machine_float_info (r, r->extensions[EXT_FLOAT]);
794 if (r->extensions[EXT_PRODUCT_INFO] != NULL)
795 parse_extra_product_info (r, r->extensions[EXT_PRODUCT_INFO], &r->info);
797 if (r->extensions[EXT_FILE_ATTRS] != NULL)
798 parse_data_file_attributes (r, r->extensions[EXT_FILE_ATTRS], dict);
800 parse_header (r, &r->header, &r->info, dict);
802 /* Parse the variable records, the basis of almost everything else. */
803 if (!parse_variable_records (r, dict, r->vars, r->n_vars))
806 /* Parse value labels and the weight variable immediately after the variable
807 records. These records use indexes into var_recs[], so we must parse them
808 before those indexes become invalidated by very long string variables. */
809 parse_value_labels (r, dict);
810 if (r->header.weight_idx != 0)
811 dict_set_weight (dict, parse_weight_var (r, r->vars, r->n_vars,
812 r->header.weight_idx));
814 if (r->extensions[EXT_DISPLAY] != NULL)
815 parse_display_parameters (r, r->extensions[EXT_DISPLAY], dict);
817 /* The following records use short names, so they need to be parsed before
818 parse_long_var_name_map() changes short names to long names. */
819 decode_mrsets (r, dict);
821 if (r->extensions[EXT_LONG_STRINGS] != NULL
822 && !parse_long_string_map (r, r->extensions[EXT_LONG_STRINGS], dict))
825 /* Now rename variables to their long names. */
826 parse_long_var_name_map (r, r->extensions[EXT_LONG_NAMES], dict);
828 /* The following records use long names, so they need to follow renaming. */
829 if (!ll_is_empty (&r->var_attrs))
831 struct sfm_extension_record *ext;
832 ll_for_each (ext, struct sfm_extension_record, ll, &r->var_attrs)
833 parse_variable_attributes (r, ext, dict);
835 /* Roles use the $@Role attribute. */
836 assign_variable_roles (r, dict);
838 if (r->extensions[EXT_LONG_LABELS] != NULL)
839 parse_long_string_value_labels (r, r->extensions[EXT_LONG_LABELS], dict);
840 if (r->extensions[EXT_LONG_MISSING] != NULL)
841 parse_long_string_missing_values (r, r->extensions[EXT_LONG_MISSING],
844 /* Warn if the actual amount of data per case differs from the
845 amount that the header claims. SPSS version 13 gets this
846 wrong when very long strings are involved, so don't warn in
848 if (r->header.nominal_case_size > 0
849 && r->header.nominal_case_size != r->n_vars
850 && r->info.version_major != 13)
851 sys_warn (r, -1, _("File header claims %d variable positions but "
852 "%zu were read from file."),
853 r->header.nominal_case_size, r->n_vars);
855 /* Create an index of dictionary variable widths for
856 sfm_read_case to use. We cannot use the `struct variable's
857 from the dictionary we created, because the caller owns the
858 dictionary and may destroy or modify its variables. */
859 sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_var_cnt);
860 pool_register (r->pool, free, r->sfm_vars);
861 r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
867 memset (&r->info, 0, sizeof r->info);
870 return casereader_create_sequential
872 r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
873 &sys_file_casereader_class, r);
882 /* Closes R, which should have been returned by sfm_open() but not already
883 closed with sfm_decode() or this function.
884 Returns true if an I/O error has occurred on READER, false
887 sfm_close (struct any_reader *r_)
889 struct sfm_reader *r = sfm_reader_cast (r_);
894 if (fn_close (r->fh, r->file) == EOF)
896 msg (ME, _("Error closing system file `%s': %s."),
897 fh_get_file_name (r->fh), strerror (errno));
903 any_read_info_destroy (&r->info);
908 pool_destroy (r->pool);
913 /* Destroys READER. */
915 sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
917 struct sfm_reader *r = r_;
918 sfm_close (&r->any_reader);
921 /* Detects whether FILE is an SPSS system file. Returns 1 if so, 0 if not, and
922 a negative errno value if there is an error reading FILE. */
924 sfm_detect (FILE *file)
928 if (fseek (file, 0, SEEK_SET) != 0)
930 if (fread (magic, 4, 1, file) != 1)
931 return ferror (file) ? -errno : 0;
934 return (!strcmp (ASCII_MAGIC, magic)
935 || !strcmp (ASCII_ZMAGIC, magic)
936 || !strcmp (EBCDIC_MAGIC, magic));
939 /* Reads the global header of the system file. Initializes *HEADER and *INFO,
940 except for the string fields in *INFO, which parse_header() will initialize
941 later once the file's encoding is known. */
943 read_header (struct sfm_reader *r, struct any_read_info *info,
944 struct sfm_header_record *header)
946 uint8_t raw_layout_code[4];
951 if (!read_string (r, header->magic, sizeof header->magic)
952 || !read_string (r, header->eye_catcher, sizeof header->eye_catcher))
954 r->written_by_readstat = strstr (header->eye_catcher,
955 "https://github.com/WizardMac/ReadStat");
957 if (!strcmp (ASCII_MAGIC, header->magic)
958 || !strcmp (EBCDIC_MAGIC, header->magic))
960 else if (!strcmp (ASCII_ZMAGIC, header->magic))
964 sys_error (r, 0, _("This is not an SPSS system file."));
968 /* Identify integer format. */
969 if (!read_bytes (r, raw_layout_code, sizeof raw_layout_code))
971 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
973 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
975 || (r->integer_format != INTEGER_MSB_FIRST
976 && r->integer_format != INTEGER_LSB_FIRST))
978 sys_error (r, 64, _("This is not an SPSS system file."));
982 if (!read_int (r, &header->nominal_case_size))
985 if (header->nominal_case_size < 0
986 || header->nominal_case_size > INT_MAX / 16)
987 header->nominal_case_size = -1;
989 if (!read_int (r, &compressed))
994 r->compression = ANY_COMP_NONE;
995 else if (compressed == 1)
996 r->compression = ANY_COMP_SIMPLE;
999 sys_error (r, 0, "System file header has invalid compression "
1000 "value %d.", compressed);
1006 if (compressed == 2)
1007 r->compression = ANY_COMP_ZLIB;
1010 sys_error (r, 0, "ZLIB-compressed system file header has invalid "
1011 "compression value %d.", compressed);
1016 if (!read_int (r, &header->weight_idx))
1019 if (!read_int (r, &r->case_cnt))
1021 if (r->case_cnt > INT_MAX / 2)
1024 /* Identify floating-point format and obtain compression bias. */
1025 if (!read_bytes (r, raw_bias, sizeof raw_bias))
1027 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
1029 uint8_t zero_bias[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
1031 if (memcmp (raw_bias, zero_bias, 8))
1032 sys_warn (r, r->pos - 8,
1033 _("Compression bias is not the usual "
1034 "value of 100, or system file uses unrecognized "
1035 "floating-point format."));
1038 /* Some software is known to write all-zeros to this
1039 field. Such software also writes floating-point
1040 numbers in the format that we expect by default
1041 (it seems that all software most likely does, in
1042 reality), so don't warn in this case. */
1045 if (r->integer_format == INTEGER_MSB_FIRST)
1046 r->float_format = FLOAT_IEEE_DOUBLE_BE;
1048 r->float_format = FLOAT_IEEE_DOUBLE_LE;
1050 float_convert (r->float_format, raw_bias, FLOAT_NATIVE_DOUBLE, &r->bias);
1052 if (!read_string (r, header->creation_date, sizeof header->creation_date)
1053 || !read_string (r, header->creation_time, sizeof header->creation_time)
1054 || !read_string (r, header->file_label, sizeof header->file_label)
1055 || !skip_bytes (r, 3))
1058 info->integer_format = r->integer_format;
1059 info->float_format = r->float_format;
1060 info->compression = r->compression;
1061 info->case_cnt = r->case_cnt;
1066 /* Reads a variable (type 2) record from R into RECORD. */
1068 read_variable_record (struct sfm_reader *r, struct sfm_var_record *record)
1070 int has_variable_label;
1072 memset (record, 0, sizeof *record);
1074 record->pos = r->pos;
1075 if (!read_int (r, &record->width)
1076 || !read_int (r, &has_variable_label)
1077 || !read_int (r, &record->missing_value_code)
1078 || !read_int (r, &record->print_format)
1079 || !read_int (r, &record->write_format)
1080 || !read_string (r, record->name, sizeof record->name))
1083 if (has_variable_label == 1)
1085 enum { MAX_LABEL_LEN = 65536 };
1086 unsigned int len, read_len;
1088 if (!read_uint (r, &len))
1091 /* Read up to MAX_LABEL_LEN bytes of label. */
1092 read_len = MIN (MAX_LABEL_LEN, len);
1093 record->label = pool_malloc (r->pool, read_len + 1);
1094 if (!read_string (r, record->label, read_len + 1))
1097 /* Skip unread label bytes. */
1098 if (!skip_bytes (r, len - read_len))
1101 /* Skip label padding up to multiple of 4 bytes. */
1102 if (!skip_bytes (r, ROUND_UP (len, 4) - len))
1105 else if (has_variable_label != 0)
1107 sys_error (r, record->pos,
1108 _("Variable label indicator field is not 0 or 1."));
1112 /* Set missing values. */
1113 if (record->missing_value_code != 0)
1115 int code = record->missing_value_code;
1116 if (record->width == 0)
1118 if (code < -3 || code > 3 || code == -1)
1120 sys_error (r, record->pos,
1121 _("Numeric missing value indicator field is not "
1122 "-3, -2, 0, 1, 2, or 3."));
1128 if (code < 1 || code > 3)
1130 sys_error (r, record->pos,
1131 _("String missing value indicator field is not "
1137 if (!read_bytes (r, record->missing, 8 * abs (code)))
1144 /* Reads value labels from R into RECORD. */
1146 read_value_label_record (struct sfm_reader *r,
1147 struct sfm_value_label_record *record)
1152 /* Read type 3 record. */
1153 record->pos = r->pos;
1154 if (!read_uint (r, &record->n_labels))
1156 if (record->n_labels > UINT_MAX / sizeof *record->labels)
1158 sys_error (r, r->pos - 4, _("Invalid number of labels %u."),
1162 record->labels = pool_nmalloc (r->pool, record->n_labels,
1163 sizeof *record->labels);
1164 for (i = 0; i < record->n_labels; i++)
1166 struct sfm_value_label *label = &record->labels[i];
1167 unsigned char label_len;
1170 if (!read_bytes (r, label->value, sizeof label->value))
1173 /* Read label length. */
1174 if (!read_bytes (r, &label_len, sizeof label_len))
1176 padded_len = ROUND_UP (label_len + 1, 8);
1178 /* Read label, padding. */
1179 label->label = pool_malloc (r->pool, padded_len + 1);
1180 if (!read_bytes (r, label->label, padded_len - 1))
1182 label->label[label_len] = '\0';
1185 /* Read record type of type 4 record. */
1186 if (!read_int (r, &type))
1190 sys_error (r, r->pos - 4,
1191 _("Variable index record (type 4) does not immediately "
1192 "follow value label record (type 3) as it should."));
1196 /* Read number of variables associated with value label from type 4
1198 if (!read_uint (r, &record->n_vars))
1200 if (record->n_vars < 1 || record->n_vars > r->n_vars)
1202 sys_error (r, r->pos - 4,
1203 _("Number of variables associated with a value label (%u) "
1204 "is not between 1 and the number of variables (%zu)."),
1205 record->n_vars, r->n_vars);
1209 record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
1210 for (i = 0; i < record->n_vars; i++)
1211 if (!read_int (r, &record->vars[i]))
1217 /* Reads a document record from R. Returns true if successful, false on
1220 read_document_record (struct sfm_reader *r)
1223 if (!read_int (r, &n_lines))
1225 else if (n_lines == 0)
1227 else if (n_lines < 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
1229 sys_error (r, r->pos,
1230 _("Number of document lines (%d) "
1231 "must be greater than 0 and less than %d."),
1232 n_lines, INT_MAX / DOC_LINE_LENGTH);
1236 struct sfm_document_record *record;
1237 record = pool_malloc (r->pool, sizeof *record);
1238 record->pos = r->pos;
1239 record->n_lines = n_lines;
1240 record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
1241 if (!read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines))
1244 r->document = record;
1249 read_extension_record_header (struct sfm_reader *r, int subtype,
1250 struct sfm_extension_record *record)
1252 record->subtype = subtype;
1253 record->pos = r->pos;
1254 if (!read_uint (r, &record->size) || !read_uint (r, &record->count))
1257 /* Check that SIZE * COUNT + 1 doesn't overflow. Adding 1
1258 allows an extra byte for a null terminator, used by some
1259 extension processing routines. */
1260 if (record->size != 0
1261 && xsum (1, xtimes (record->count, record->size)) >= UINT_MAX)
1263 sys_error (r, record->pos, "Record type 7 subtype %d too large.",
1271 /* Reads an extension record from R into RECORD. */
1273 read_extension_record (struct sfm_reader *r, int subtype,
1274 struct sfm_extension_record **recordp)
1276 struct extension_record_type
1283 static const struct extension_record_type types[] =
1285 /* Implemented record types. */
1286 { EXT_INTEGER, 4, 8 },
1287 { EXT_FLOAT, 8, 3 },
1288 { EXT_MRSETS, 1, 0 },
1289 { EXT_PRODUCT_INFO, 1, 0 },
1290 { EXT_DISPLAY, 4, 0 },
1291 { EXT_LONG_NAMES, 1, 0 },
1292 { EXT_LONG_STRINGS, 1, 0 },
1293 { EXT_NCASES, 8, 2 },
1294 { EXT_FILE_ATTRS, 1, 0 },
1295 { EXT_VAR_ATTRS, 1, 0 },
1296 { EXT_MRSETS2, 1, 0 },
1297 { EXT_ENCODING, 1, 0 },
1298 { EXT_LONG_LABELS, 1, 0 },
1299 { EXT_LONG_MISSING, 1, 0 },
1301 /* Ignored record types. */
1302 { EXT_VAR_SETS, 0, 0 },
1304 { EXT_DATA_ENTRY, 0, 0 },
1305 { EXT_DATAVIEW, 0, 0 },
1308 const struct extension_record_type *type;
1309 struct sfm_extension_record *record;
1313 record = pool_malloc (r->pool, sizeof *record);
1314 if (!read_extension_record_header (r, subtype, record))
1316 n_bytes = record->count * record->size;
1318 for (type = types; type < &types[sizeof types / sizeof *types]; type++)
1319 if (subtype == type->subtype)
1321 if (type->size > 0 && record->size != type->size)
1322 sys_warn (r, record->pos,
1323 _("Record type 7, subtype %d has bad size %u "
1324 "(expected %d)."), subtype, record->size, type->size);
1325 else if (type->count > 0 && record->count != type->count)
1326 sys_warn (r, record->pos,
1327 _("Record type 7, subtype %d has bad count %u "
1328 "(expected %d)."), subtype, record->count, type->count);
1329 else if (type->count == 0 && type->size == 0)
1331 /* Ignore this record. */
1335 char *data = pool_malloc (r->pool, n_bytes + 1);
1336 data[n_bytes] = '\0';
1338 record->data = data;
1339 if (!read_bytes (r, record->data, n_bytes))
1348 sys_warn (r, record->pos,
1349 _("Unrecognized record type 7, subtype %d. For help, please "
1350 "send this file to %s and mention that you were using %s."),
1351 subtype, PACKAGE_BUGREPORT, PACKAGE_STRING);
1354 return skip_bytes (r, n_bytes);
1358 skip_extension_record (struct sfm_reader *r, int subtype)
1360 struct sfm_extension_record record;
1362 return (read_extension_record_header (r, subtype, &record)
1363 && skip_bytes (r, record.count * record.size));
1367 parse_header (struct sfm_reader *r, const struct sfm_header_record *header,
1368 struct any_read_info *info, struct dictionary *dict)
1370 const char *dict_encoding = dict_get_encoding (dict);
1371 struct substring product;
1372 struct substring label;
1375 /* Convert file label to UTF-8 and put it into DICT. */
1376 label = recode_substring_pool ("UTF-8", dict_encoding,
1377 ss_cstr (header->file_label), r->pool);
1378 ss_trim (&label, ss_cstr (" "));
1379 label.string[label.length] = '\0';
1380 fixed_label = fix_line_ends (label.string);
1381 dict_set_label (dict, fixed_label);
1384 /* Put creation date and time in UTF-8 into INFO. */
1385 info->creation_date = recode_string ("UTF-8", dict_encoding,
1386 header->creation_date, -1);
1387 info->creation_time = recode_string ("UTF-8", dict_encoding,
1388 header->creation_time, -1);
1390 /* Put product name into INFO, dropping eye-catcher string if present. */
1391 product = recode_substring_pool ("UTF-8", dict_encoding,
1392 ss_cstr (header->eye_catcher), r->pool);
1393 ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE"));
1394 ss_trim (&product, ss_cstr (" "));
1395 info->product = ss_xstrdup (product);
1398 static struct variable *
1399 add_var_with_generated_name (struct dictionary *dict, int width)
1401 char *name = dict_make_unique_var_name (dict, NULL, NULL);
1402 struct variable *var = dict_create_var_assert (dict, name, width);
1407 /* Reads a variable (type 2) record from R and adds the
1408 corresponding variable to DICT.
1409 Also skips past additional variable records for long string
1412 parse_variable_records (struct sfm_reader *r, struct dictionary *dict,
1413 struct sfm_var_record *var_recs, size_t n_var_recs)
1415 const char *dict_encoding = dict_get_encoding (dict);
1416 struct sfm_var_record *rec;
1419 for (rec = var_recs; rec < &var_recs[n_var_recs];)
1425 name = recode_string_pool ("UTF-8", dict_encoding,
1426 rec->name, -1, r->pool);
1427 name[strcspn (name, " ")] = '\0';
1429 if (rec->width < 0 || rec->width > 255)
1431 sys_error (r, rec->pos,
1432 _("Bad width %d for variable %s."), rec->width, name);
1436 struct variable *var;
1437 if (!dict_id_is_valid (dict, name, false)
1438 || name[0] == '$' || name[0] == '#')
1440 var = add_var_with_generated_name (dict, rec->width);
1441 sys_warn (r, rec->pos, _("Renaming variable with invalid name "
1442 "`%s' to `%s'."), name, var_get_name (var));
1446 var = dict_create_var (dict, name, rec->width);
1449 var = add_var_with_generated_name (dict, rec->width);
1450 sys_warn (r, rec->pos, _("Renaming variable with duplicate name "
1452 name, var_get_name (var));
1457 /* Set the short name the same as the long name (even if we renamed
1459 var_set_short_name (var, 0, var_get_name (var));
1461 /* Get variable label, if any. */
1466 utf8_label = recode_string_pool ("UTF-8", dict_encoding,
1467 rec->label, -1, r->pool);
1468 var_set_label (var, utf8_label);
1471 /* Set missing values. */
1472 if (rec->missing_value_code != 0)
1474 int width = var_get_width (var);
1475 struct missing_values mv;
1477 mv_init_pool (r->pool, &mv, width);
1478 if (var_is_numeric (var))
1480 bool has_range = rec->missing_value_code < 0;
1481 int n_discrete = (has_range
1482 ? rec->missing_value_code == -3
1483 : rec->missing_value_code);
1488 double low = parse_float (r, rec->missing, 0);
1489 double high = parse_float (r, rec->missing, 8);
1491 /* Deal with SPSS 21 change in representation. */
1495 mv_add_range (&mv, low, high);
1499 for (i = 0; i < n_discrete; i++)
1501 mv_add_num (&mv, parse_float (r, rec->missing, ofs));
1506 for (i = 0; i < rec->missing_value_code; i++)
1507 mv_add_str (&mv, rec->missing + 8 * i, MIN (width, 8));
1508 var_set_missing_values (var, &mv);
1512 parse_format_spec (r, rec->pos + 12, rec->print_format,
1513 PRINT_FORMAT, var, &n_warnings);
1514 parse_format_spec (r, rec->pos + 16, rec->write_format,
1515 WRITE_FORMAT, var, &n_warnings);
1517 /* Account for values.
1518 Skip long string continuation records, if any. */
1519 n_values = rec->width == 0 ? 1 : DIV_RND_UP (rec->width, 8);
1520 for (i = 1; i < n_values; i++)
1521 if (i + (rec - var_recs) >= n_var_recs || rec[i].width != -1)
1523 sys_error (r, rec->pos, _("Missing string continuation record."));
1532 /* Translates the format spec from sysfile format to internal
1535 parse_format_spec (struct sfm_reader *r, off_t pos, unsigned int format,
1536 enum which_format which, struct variable *v,
1539 const int max_warnings = 8;
1542 if (fmt_from_u32 (format, var_get_width (v), false, &f))
1544 if (which == PRINT_FORMAT)
1545 var_set_print_format (v, &f);
1547 var_set_write_format (v, &f);
1549 else if (format == 0)
1551 /* Actually observed in the wild. No point in warning about it. */
1553 else if (++*n_warnings <= max_warnings)
1555 if (which == PRINT_FORMAT)
1556 sys_warn (r, pos, _("Variable %s with width %d has invalid print "
1558 var_get_name (v), var_get_width (v), format);
1560 sys_warn (r, pos, _("Variable %s with width %d has invalid write "
1562 var_get_name (v), var_get_width (v), format);
1564 if (*n_warnings == max_warnings)
1565 sys_warn (r, -1, _("Suppressing further invalid format warnings."));
1570 parse_document (struct dictionary *dict, struct sfm_document_record *record)
1574 for (p = record->documents;
1575 p < record->documents + DOC_LINE_LENGTH * record->n_lines;
1576 p += DOC_LINE_LENGTH)
1578 struct substring line;
1580 line = recode_substring_pool ("UTF-8", dict_get_encoding (dict),
1581 ss_buffer (p, DOC_LINE_LENGTH), NULL);
1582 ss_rtrim (&line, ss_cstr (" "));
1583 line.string[line.length] = '\0';
1585 dict_add_document_line (dict, line.string, false);
1591 /* Parses record type 7, subtype 3. */
1593 parse_machine_integer_info (struct sfm_reader *r,
1594 const struct sfm_extension_record *record,
1595 struct any_read_info *info)
1597 int float_representation, expected_float_format;
1598 int integer_representation, expected_integer_format;
1600 /* Save version info. */
1601 info->version_major = parse_int (r, record->data, 0);
1602 info->version_minor = parse_int (r, record->data, 4);
1603 info->version_revision = parse_int (r, record->data, 8);
1605 /* Check floating point format. */
1606 float_representation = parse_int (r, record->data, 16);
1607 if (r->float_format == FLOAT_IEEE_DOUBLE_BE
1608 || r->float_format == FLOAT_IEEE_DOUBLE_LE)
1609 expected_float_format = 1;
1610 else if (r->float_format == FLOAT_Z_LONG)
1611 expected_float_format = 2;
1612 else if (r->float_format == FLOAT_VAX_G || r->float_format == FLOAT_VAX_D)
1613 expected_float_format = 3;
1616 if (float_representation != expected_float_format)
1618 sys_error (r, record->pos,
1619 _("Floating-point representation indicated by "
1620 "system file (%d) differs from expected (%d)."),
1621 float_representation, expected_float_format);
1625 /* Check integer format. */
1626 integer_representation = parse_int (r, record->data, 24);
1627 if (r->integer_format == INTEGER_MSB_FIRST)
1628 expected_integer_format = 1;
1629 else if (r->integer_format == INTEGER_LSB_FIRST)
1630 expected_integer_format = 2;
1633 if (integer_representation != expected_integer_format)
1634 sys_warn (r, record->pos,
1635 _("Integer format indicated by system file (%d) "
1636 "differs from expected (%d)."),
1637 integer_representation, expected_integer_format);
1642 /* Parses record type 7, subtype 4. */
1644 parse_machine_float_info (struct sfm_reader *r,
1645 const struct sfm_extension_record *record)
1647 double sysmis = parse_float (r, record->data, 0);
1648 double highest = parse_float (r, record->data, 8);
1649 double lowest = parse_float (r, record->data, 16);
1651 if (sysmis != SYSMIS)
1652 sys_warn (r, record->pos,
1653 _("File specifies unexpected value %g (%a) as %s, "
1654 "instead of %g (%a)."),
1655 sysmis, sysmis, "SYSMIS", SYSMIS, SYSMIS);
1657 if (highest != HIGHEST)
1658 sys_warn (r, record->pos,
1659 _("File specifies unexpected value %g (%a) as %s, "
1660 "instead of %g (%a)."),
1661 highest, highest, "HIGHEST", HIGHEST, HIGHEST);
1663 /* SPSS before version 21 used a unique value just bigger than SYSMIS as
1664 LOWEST. SPSS 21 uses SYSMIS for LOWEST, which is OK because LOWEST only
1665 appears in a context (missing values) where SYSMIS cannot. */
1666 if (lowest != LOWEST && lowest != SYSMIS)
1667 sys_warn (r, record->pos,
1668 _("File specifies unexpected value %g (%a) as %s, "
1669 "instead of %g (%a) or %g (%a)."),
1670 lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS);
1673 /* Parses record type 7, subtype 10. */
1675 parse_extra_product_info (struct sfm_reader *r,
1676 const struct sfm_extension_record *record,
1677 struct any_read_info *info)
1679 struct text_record *text;
1681 text = open_text_record (r, record, true);
1682 info->product_ext = fix_line_ends (text_get_all (text));
1683 close_text_record (r, text);
1686 /* Parses record type 7, subtype 7 or 19. */
1688 parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record,
1689 size_t *allocated_mrsets)
1691 struct text_record *text;
1693 text = open_text_record (r, record, false);
1696 struct sfm_mrset *mrset = NULL;
1697 size_t allocated_vars = 0;
1698 char delimiter = '4';
1700 /* Skip extra line feeds if present. */
1701 while (text_match (text, '\n'))
1704 if (r->n_mrsets >= *allocated_mrsets)
1705 r->mrsets = pool_2nrealloc (r->pool, r->mrsets, allocated_mrsets,
1707 mrset = &r->mrsets[r->n_mrsets];
1708 memset(mrset, 0, sizeof *mrset);
1710 mrset->name = text_get_token (text, ss_cstr ("="), NULL);
1711 if (mrset->name == NULL)
1714 if (text_match (text, 'C'))
1716 mrset->type = MRSET_MC;
1717 if (!text_match (text, ' '))
1719 sys_warn (r, record->pos,
1720 _("Missing space following `%c' at offset %zu "
1721 "in MRSETS record."), 'C', text_pos (text));
1725 else if (text_match (text, 'D'))
1727 mrset->type = MRSET_MD;
1728 mrset->cat_source = MRSET_VARLABELS;
1730 else if (text_match (text, 'E'))
1734 mrset->type = MRSET_MD;
1735 mrset->cat_source = MRSET_COUNTEDVALUES;
1736 if (!text_match (text, ' '))
1738 sys_warn (r, record->pos,
1739 _("Missing space following `%c' at offset %zu "
1740 "in MRSETS record."), 'E', text_pos (text));
1744 number = text_get_token (text, ss_cstr (" "), NULL);
1746 sys_warn (r, record->pos,
1747 _("Missing label source value "
1748 "following `E' at offset %zu in MRSETS record."),
1750 else if (!strcmp (number, "11"))
1751 mrset->label_from_var_label = true;
1752 else if (strcmp (number, "1"))
1753 sys_warn (r, record->pos,
1754 _("Unexpected label source value following `E' "
1755 "at offset %zu in MRSETS record."),
1760 sys_warn (r, record->pos,
1761 _("Missing `C', `D', or `E' at offset %zu "
1762 "in MRSETS record."),
1767 if (mrset->type == MRSET_MD)
1769 mrset->counted = text_parse_counted_string (r, text);
1770 if (mrset->counted == NULL)
1774 mrset->label = text_parse_counted_string (r, text);
1775 if (mrset->label == NULL)
1783 var = text_get_token (text, ss_cstr (" \n"), &delimiter);
1786 if (delimiter != '\n')
1787 sys_warn (r, record->pos,
1788 _("Missing new-line parsing variable names "
1789 "at offset %zu in MRSETS record."),
1794 if (mrset->n_vars >= allocated_vars)
1795 mrset->vars = pool_2nrealloc (r->pool, mrset->vars,
1797 sizeof *mrset->vars);
1798 mrset->vars[mrset->n_vars++] = var;
1800 while (delimiter != '\n');
1804 close_text_record (r, text);
1808 decode_mrsets (struct sfm_reader *r, struct dictionary *dict)
1810 const struct sfm_mrset *s;
1812 for (s = r->mrsets; s < &r->mrsets[r->n_mrsets]; s++)
1814 struct stringi_set var_names;
1815 struct mrset *mrset;
1820 name = recode_string ("UTF-8", r->encoding, s->name, -1);
1821 if (!mrset_is_valid_name (name, dict_get_encoding (dict), false))
1823 sys_warn (r, -1, _("Invalid multiple response set name `%s'."),
1829 mrset = xzalloc (sizeof *mrset);
1831 mrset->type = s->type;
1832 mrset->cat_source = s->cat_source;
1833 mrset->label_from_var_label = s->label_from_var_label;
1834 if (s->label[0] != '\0')
1835 mrset->label = recode_string ("UTF-8", r->encoding, s->label, -1);
1837 stringi_set_init (&var_names);
1838 mrset->vars = xmalloc (s->n_vars * sizeof *mrset->vars);
1840 for (i = 0; i < s->n_vars; i++)
1842 struct variable *var;
1845 var_name = recode_string ("UTF-8", r->encoding, s->vars[i], -1);
1847 var = dict_lookup_var (dict, var_name);
1853 if (!stringi_set_insert (&var_names, var_name))
1856 _("MRSET %s contains duplicate variable name %s."),
1857 mrset->name, var_name);
1863 if (mrset->label == NULL && mrset->label_from_var_label
1864 && var_has_label (var))
1865 mrset->label = xstrdup (var_get_label (var));
1868 && var_get_type (var) != var_get_type (mrset->vars[0]))
1871 _("MRSET %s contains both string and "
1872 "numeric variables."), mrset->name);
1875 width = MIN (width, var_get_width (var));
1877 mrset->vars[mrset->n_vars++] = var;
1880 if (mrset->n_vars < 2)
1882 if (mrset->n_vars == 0)
1883 sys_warn (r, -1, _("MRSET %s has no variables."), mrset->name);
1885 sys_warn (r, -1, _("MRSET %s has only one variable."),
1887 mrset_destroy (mrset);
1888 stringi_set_destroy (&var_names);
1892 if (mrset->type == MRSET_MD)
1894 mrset->width = width;
1895 value_init (&mrset->counted, width);
1897 mrset->counted.f = c_strtod (s->counted, NULL);
1899 value_copy_str_rpad (&mrset->counted, width,
1900 (const uint8_t *) s->counted, ' ');
1903 dict_add_mrset (dict, mrset);
1904 stringi_set_destroy (&var_names);
1908 /* Read record type 7, subtype 11, which specifies how variables
1909 should be displayed in GUI environments. */
1911 parse_display_parameters (struct sfm_reader *r,
1912 const struct sfm_extension_record *record,
1913 struct dictionary *dict)
1915 bool includes_width;
1916 bool warned = false;
1921 n_vars = dict_get_var_cnt (dict);
1922 if (record->count == 3 * n_vars)
1923 includes_width = true;
1924 else if (record->count == 2 * n_vars)
1925 includes_width = false;
1928 sys_warn (r, record->pos,
1929 _("Extension 11 has bad count %u (for %zu variables)."),
1930 record->count, n_vars);
1935 for (i = 0; i < n_vars; ++i)
1937 struct variable *v = dict_get_var (dict, i);
1938 int measure, width, align;
1940 measure = parse_int (r, record->data, ofs);
1945 width = parse_int (r, record->data, ofs);
1951 align = parse_int (r, record->data, ofs);
1954 /* SPSS sometimes seems to set variables' measure to zero. */
1958 if (measure < 1 || measure > 3 || align < 0 || align > 2)
1961 sys_warn (r, record->pos,
1962 _("Invalid variable display parameters for variable "
1963 "%zu (%s). Default parameters substituted."),
1964 i, var_get_name (v));
1969 var_set_measure (v, (measure == 1 ? MEASURE_NOMINAL
1970 : measure == 2 ? MEASURE_ORDINAL
1972 var_set_alignment (v, (align == 0 ? ALIGN_LEFT
1973 : align == 1 ? ALIGN_RIGHT
1976 /* Older versions (SPSS 9.0) sometimes set the display
1977 width to zero. This causes confusion in the GUI, so
1978 only set the width if it is nonzero. */
1980 var_set_display_width (v, width);
1985 rename_var_and_save_short_names (struct sfm_reader *r, off_t pos,
1986 struct dictionary *dict,
1987 struct variable *var, const char *new_name)
1989 size_t n_short_names;
1993 /* Renaming a variable may clear its short names, but we
1994 want to retain them, so we save them and re-set them
1996 n_short_names = var_get_short_name_cnt (var);
1997 short_names = xnmalloc (n_short_names, sizeof *short_names);
1998 for (i = 0; i < n_short_names; i++)
2000 const char *s = var_get_short_name (var, i);
2001 short_names[i] = xstrdup_if_nonnull (s);
2004 /* Set long name. */
2005 if (!dict_try_rename_var (dict, var, new_name))
2006 sys_warn (r, pos, _("Duplicate long variable name `%s'."), new_name);
2008 /* Restore short names. */
2009 for (i = 0; i < n_short_names; i++)
2011 var_set_short_name (var, i, short_names[i]);
2012 free (short_names[i]);
2017 /* Parses record type 7, subtype 13, which gives the long name that corresponds
2018 to each short name. Modifies variable names in DICT accordingly. */
2020 parse_long_var_name_map (struct sfm_reader *r,
2021 const struct sfm_extension_record *record,
2022 struct dictionary *dict)
2024 struct text_record *text;
2025 struct variable *var;
2030 /* There are no long variable names. Use the short variable names,
2031 converted to lowercase, as the long variable names. */
2034 for (i = 0; i < dict_get_var_cnt (dict); i++)
2036 struct variable *var = dict_get_var (dict, i);
2039 new_name = utf8_to_lower (var_get_name (var));
2040 rename_var_and_save_short_names (r, -1, dict, var, new_name);
2047 /* Rename each of the variables, one by one. (In a correctly constructed
2048 system file, this cannot create any intermediate duplicate variable names,
2049 because all of the new variable names are longer than any of the old
2050 variable names and thus there cannot be any overlaps.) */
2051 text = open_text_record (r, record, true);
2052 while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
2054 /* Validate long name. */
2055 if (!dict_id_is_valid (dict, long_name, false)
2056 || long_name[0] == '$' || long_name[0] == '#')
2058 sys_warn (r, record->pos,
2059 _("Long variable mapping from %s to invalid "
2060 "variable name `%s'."),
2061 var_get_name (var), long_name);
2065 rename_var_and_save_short_names (r, record->pos, dict, var, long_name);
2067 close_text_record (r, text);
2070 /* Reads record type 7, subtype 14, which gives the real length
2071 of each very long string. Rearranges DICT accordingly. */
2073 parse_long_string_map (struct sfm_reader *r,
2074 const struct sfm_extension_record *record,
2075 struct dictionary *dict)
2077 struct text_record *text;
2078 struct variable *var;
2081 text = open_text_record (r, record, true);
2082 while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
2084 size_t idx = var_get_dict_index (var);
2090 length = strtol (length_s, NULL, 10);
2091 if (length < 1 || length > MAX_STRING)
2093 sys_warn (r, record->pos,
2094 _("%s listed as string of invalid length %s "
2095 "in very long string record."),
2096 var_get_name (var), length_s);
2100 /* Check segments. */
2101 segment_cnt = sfm_width_to_segments (length);
2102 if (segment_cnt == 1)
2104 sys_warn (r, record->pos,
2105 _("%s listed in very long string record with width %s, "
2106 "which requires only one segment."),
2107 var_get_name (var), length_s);
2110 if (idx + segment_cnt > dict_get_var_cnt (dict))
2112 sys_error (r, record->pos,
2113 _("Very long string %s overflows dictionary."),
2114 var_get_name (var));
2118 /* Get the short names from the segments and check their
2120 for (i = 0; i < segment_cnt; i++)
2122 struct variable *seg = dict_get_var (dict, idx + i);
2123 int alloc_width = sfm_segment_alloc_width (length, i);
2124 int width = var_get_width (seg);
2127 var_set_short_name (var, i, var_get_short_name (seg, 0));
2128 if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
2130 sys_error (r, record->pos,
2131 _("Very long string with width %ld has segment %d "
2132 "of width %d (expected %d)."),
2133 length, i, width, alloc_width);
2137 dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
2138 var_set_width (var, length);
2140 close_text_record (r, text);
2141 dict_compact_values (dict);
2146 #define MAX_LABEL_WARNINGS 5
2148 /* Displays a warning for offset OFFSET in the file. */
2150 value_label_warning (struct sfm_reader *r, off_t offset, int *n_label_warnings,
2151 const char *format, ...)
2153 if (++*n_label_warnings > MAX_LABEL_WARNINGS)
2158 va_start (args, format);
2159 sys_msg (r, offset, MW, format, args);
2163 #define MAX_LABEL_WARNINGS 5
2166 parse_one_value_label_set (struct sfm_reader *r, struct dictionary *dict,
2167 const struct sfm_var_record *var_recs,
2169 const struct sfm_value_label_record *record,
2170 int *n_label_warnings)
2173 = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
2174 for (size_t i = 0; i < record->n_labels; i++)
2175 utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
2176 record->labels[i].label, -1,
2179 struct variable **vars = pool_nmalloc (r->pool,
2180 record->n_vars, sizeof *vars);
2181 unsigned int n_vars = 0;
2182 for (size_t i = 0; i < record->n_vars; i++)
2184 int idx = record->vars[i];
2185 if (idx < 1 || idx > n_var_recs)
2187 value_label_warning (
2188 r, record->pos, n_label_warnings,
2189 _("Value label variable index %d not in valid range 1...%zu."),
2194 const struct sfm_var_record *rec = &var_recs[idx - 1];
2195 if (rec->var == NULL)
2197 value_label_warning (
2198 r, record->pos, n_label_warnings,
2199 _("Value label variable index %d "
2200 "refers to long string continuation."), idx);
2204 vars[n_vars++] = rec->var;
2209 for (size_t i = 1; i < n_vars; i++)
2210 if (var_get_type (vars[i]) != var_get_type (vars[0]))
2212 value_label_warning (
2213 r, record->pos, n_label_warnings,
2214 _("Variables associated with value label are not all of "
2215 "identical type. Variable %s is %s, but variable "
2217 var_get_name (vars[0]),
2218 var_is_numeric (vars[0]) ? _("numeric") : _("string"),
2219 var_get_name (vars[i]),
2220 var_is_numeric (vars[i]) ? _("numeric") : _("string"));
2224 for (size_t i = 0; i < n_vars; i++)
2226 struct variable *var = vars[i];
2227 int width = var_get_width (var);
2230 value_label_warning (
2231 r, record->pos, n_label_warnings,
2232 _("Value labels may not be added to long string "
2233 "variables (e.g. %s) using records types 3 and 4."),
2234 var_get_name (var));
2238 for (size_t j = 0; j < record->n_labels; j++)
2240 struct sfm_value_label *label = &record->labels[j];
2243 value_init (&value, width);
2245 value.f = parse_float (r, label->value, 0);
2247 memcpy (value.s, label->value, width);
2249 if (!var_add_value_label (var, &value, utf8_labels[j]))
2251 if (r->written_by_readstat)
2253 /* Ignore the problem. ReadStat is buggy and emits value
2254 labels whose values are longer than string variables'
2255 widths, that are identical in the actual width of the
2256 variable, e.g. both values "ABC123" and "ABC456" for a
2257 string variable with width 3. */
2259 else if (var_is_numeric (var))
2260 value_label_warning (r, record->pos, n_label_warnings,
2261 _("Duplicate value label for %g on %s."),
2262 value.f, var_get_name (var));
2264 value_label_warning (
2265 r, record->pos, n_label_warnings,
2266 _("Duplicate value label for `%.*s' on %s."),
2267 width, value.s, var_get_name (var));
2270 value_destroy (&value, width);
2274 pool_free (r->pool, vars);
2275 for (size_t i = 0; i < record->n_labels; i++)
2276 pool_free (r->pool, utf8_labels[i]);
2277 pool_free (r->pool, utf8_labels);
2281 parse_value_labels (struct sfm_reader *r, struct dictionary *dict)
2283 int n_label_warnings = 0;
2284 for (size_t i = 0; i < r->n_labels; i++)
2285 parse_one_value_label_set (r, dict, r->vars, r->n_vars, &r->labels[i],
2287 if (n_label_warnings > MAX_LABEL_WARNINGS)
2289 _("Suppressed %d additional warnings for value labels."),
2290 n_label_warnings - MAX_LABEL_WARNINGS);
2293 static struct variable *
2294 parse_weight_var (struct sfm_reader *r,
2295 const struct sfm_var_record *var_recs, size_t n_var_recs,
2298 off_t offset = 76; /* Offset to variable index in header. */
2300 if (idx < 1 || idx > n_var_recs)
2302 sys_warn (r, offset,
2303 _("Weight variable index %d not in valid range 1...%zu. "
2304 "Treating file as unweighted."),
2309 const struct sfm_var_record *rec = &var_recs[idx - 1];
2310 if (rec->var == NULL)
2312 sys_warn (r, offset,
2313 _("Weight variable index %d refers to long string "
2314 "continuation. Treating file as unweighted."), idx);
2318 struct variable *weight_var = rec->var;
2319 if (!var_is_numeric (weight_var))
2321 sys_warn (r, offset, _("Ignoring string variable `%s' set "
2322 "as weighting variable."),
2323 var_get_name (weight_var));
2330 /* Parses a set of custom attributes from TEXT into ATTRS.
2331 ATTRS may be a null pointer, in which case the attributes are
2332 read but discarded. */
2334 parse_attributes (struct sfm_reader *r, struct text_record *text,
2335 struct attrset *attrs)
2339 struct attribute *attr;
2343 /* Parse the key. */
2344 key = text_get_token (text, ss_cstr ("("), NULL);
2348 attr = attribute_create (key);
2349 for (index = 1; ; index++)
2351 /* Parse the value. */
2355 value = text_get_token (text, ss_cstr ("\n"), NULL);
2358 text_warn (r, text, _("Error parsing attribute value %s[%d]."),
2363 length = strlen (value);
2364 if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
2366 value[length - 1] = '\0';
2367 attribute_add_value (attr, value + 1);
2372 _("Attribute value %s[%d] is not quoted: %s."),
2374 attribute_add_value (attr, value);
2377 /* Was this the last value for this attribute? */
2378 if (text_match (text, ')'))
2381 if (attrs != NULL && attribute_get_n_values (attr) > 0)
2383 if (!attrset_try_add (attrs, attr))
2385 text_warn (r, text, _("Duplicate attribute %s."),
2386 attribute_get_name (attr));
2387 attribute_destroy (attr);
2391 attribute_destroy (attr);
2393 while (!text_match (text, '/'));
2396 /* Reads record type 7, subtype 17, which lists custom
2397 attributes on the data file. */
2399 parse_data_file_attributes (struct sfm_reader *r,
2400 const struct sfm_extension_record *record,
2401 struct dictionary *dict)
2403 struct text_record *text = open_text_record (r, record, true);
2404 parse_attributes (r, text, dict_get_attributes (dict));
2405 close_text_record (r, text);
2408 /* Parses record type 7, subtype 18, which lists custom
2409 attributes on individual variables. */
2411 parse_variable_attributes (struct sfm_reader *r,
2412 const struct sfm_extension_record *record,
2413 struct dictionary *dict)
2415 struct text_record *text;
2416 struct variable *var;
2418 text = open_text_record (r, record, true);
2419 while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
2420 parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
2421 close_text_record (r, text);
2425 assign_variable_roles (struct sfm_reader *r, struct dictionary *dict)
2427 size_t n_warnings = 0;
2430 for (i = 0; i < dict_get_var_cnt (dict); i++)
2432 struct variable *var = dict_get_var (dict, i);
2433 struct attrset *attrs = var_get_attributes (var);
2434 const struct attribute *attr = attrset_lookup (attrs, "$@Role");
2435 if (attr != NULL && attribute_get_n_values (attr) > 0)
2437 int value = atoi (attribute_get_value (attr, 0));
2459 role = ROLE_PARTITION;
2468 if (n_warnings++ == 0)
2469 sys_warn (r, -1, _("Invalid role for variable %s."),
2470 var_get_name (var));
2473 var_set_role (var, role);
2478 sys_warn (r, -1, _("%zu other variables had invalid roles."),
2483 check_overflow (struct sfm_reader *r,
2484 const struct sfm_extension_record *record,
2485 size_t ofs, size_t length)
2487 size_t end = record->size * record->count;
2488 if (length >= end || ofs + length > end)
2490 sys_warn (r, record->pos + end,
2491 _("Extension record subtype %d ends unexpectedly."),
2499 parse_long_string_value_labels (struct sfm_reader *r,
2500 const struct sfm_extension_record *record,
2501 struct dictionary *dict)
2503 const char *dict_encoding = dict_get_encoding (dict);
2504 size_t end = record->size * record->count;
2511 struct variable *var;
2516 /* Parse variable name length. */
2517 if (!check_overflow (r, record, ofs, 4))
2519 var_name_len = parse_int (r, record->data, ofs);
2522 /* Parse variable name, width, and number of labels. */
2523 if (!check_overflow (r, record, ofs, var_name_len)
2524 || !check_overflow (r, record, ofs, var_name_len + 8))
2526 var_name = recode_string_pool ("UTF-8", dict_encoding,
2527 (const char *) record->data + ofs,
2528 var_name_len, r->pool);
2529 width = parse_int (r, record->data, ofs + var_name_len);
2530 n_labels = parse_int (r, record->data, ofs + var_name_len + 4);
2531 ofs += var_name_len + 8;
2533 /* Look up 'var' and validate. */
2534 var = dict_lookup_var (dict, var_name);
2536 sys_warn (r, record->pos + ofs,
2537 _("Ignoring long string value label record for "
2538 "unknown variable %s."), var_name);
2539 else if (var_is_numeric (var))
2541 sys_warn (r, record->pos + ofs,
2542 _("Ignoring long string value label record for "
2543 "numeric variable %s."), var_name);
2546 else if (width != var_get_width (var))
2548 sys_warn (r, record->pos + ofs,
2549 _("Ignoring long string value label record for variable "
2550 "%s because the record's width (%d) does not match the "
2551 "variable's width (%d)."),
2552 var_name, width, var_get_width (var));
2557 value_init_pool (r->pool, &value, width);
2558 for (i = 0; i < n_labels; i++)
2560 size_t value_length, label_length;
2561 bool skip = var == NULL;
2563 /* Parse value length. */
2564 if (!check_overflow (r, record, ofs, 4))
2566 value_length = parse_int (r, record->data, ofs);
2570 if (!check_overflow (r, record, ofs, value_length))
2574 if (value_length == width)
2575 memcpy (value.s, (const uint8_t *) record->data + ofs, width);
2578 sys_warn (r, record->pos + ofs,
2579 _("Ignoring long string value label %zu for "
2580 "variable %s, with width %d, that has bad value "
2582 i, var_get_name (var), width, value_length);
2586 ofs += value_length;
2588 /* Parse label length. */
2589 if (!check_overflow (r, record, ofs, 4))
2591 label_length = parse_int (r, record->data, ofs);
2595 if (!check_overflow (r, record, ofs, label_length))
2601 label = recode_string_pool ("UTF-8", dict_encoding,
2602 (const char *) record->data + ofs,
2603 label_length, r->pool);
2604 if (!var_add_value_label (var, &value, label))
2605 sys_warn (r, record->pos + ofs,
2606 _("Duplicate value label for `%.*s' on %s."),
2607 width, value.s, var_get_name (var));
2608 pool_free (r->pool, label);
2610 ofs += label_length;
2616 parse_long_string_missing_values (struct sfm_reader *r,
2617 const struct sfm_extension_record *record,
2618 struct dictionary *dict)
2620 const char *dict_encoding = dict_get_encoding (dict);
2621 size_t end = record->size * record->count;
2626 struct missing_values mv;
2628 struct variable *var;
2629 int n_missing_values;
2633 /* Parse variable name length. */
2634 if (!check_overflow (r, record, ofs, 4))
2636 var_name_len = parse_int (r, record->data, ofs);
2639 /* Parse variable name. */
2640 if (!check_overflow (r, record, ofs, var_name_len)
2641 || !check_overflow (r, record, ofs, var_name_len + 1))
2643 var_name = recode_string_pool ("UTF-8", dict_encoding,
2644 (const char *) record->data + ofs,
2645 var_name_len, r->pool);
2646 ofs += var_name_len;
2648 /* Parse number of missing values. */
2649 n_missing_values = ((const uint8_t *) record->data)[ofs];
2650 if (n_missing_values < 1 || n_missing_values > 3)
2651 sys_warn (r, record->pos + ofs,
2652 _("Long string missing values record says variable %s "
2653 "has %d missing values, but only 1 to 3 missing values "
2655 var_name, n_missing_values);
2658 /* Look up 'var' and validate. */
2659 var = dict_lookup_var (dict, var_name);
2661 sys_warn (r, record->pos + ofs,
2662 _("Ignoring long string missing value record for "
2663 "unknown variable %s."), var_name);
2664 else if (var_is_numeric (var))
2666 sys_warn (r, record->pos + ofs,
2667 _("Ignoring long string missing value record for "
2668 "numeric variable %s."), var_name);
2673 mv_init_pool (r->pool, &mv, var ? var_get_width (var) : 8);
2674 for (i = 0; i < n_missing_values; i++)
2676 size_t value_length;
2678 /* Parse value length. */
2679 if (!check_overflow (r, record, ofs, 4))
2681 value_length = parse_int (r, record->data, ofs);
2685 if (!check_overflow (r, record, ofs, value_length))
2689 && !mv_add_str (&mv, (const uint8_t *) record->data + ofs,
2691 sys_warn (r, record->pos + ofs,
2692 _("Ignoring long string missing value %zu for variable "
2693 "%s, with width %d, that has bad value width %zu."),
2694 i, var_get_name (var), var_get_width (var),
2696 ofs += value_length;
2699 var_set_missing_values (var, &mv);
2705 static void partial_record (struct sfm_reader *);
2707 static void read_error (struct casereader *, const struct sfm_reader *);
2709 static bool read_case_number (struct sfm_reader *, double *);
2710 static int read_case_string (struct sfm_reader *, uint8_t *, size_t);
2711 static int read_opcode (struct sfm_reader *);
2712 static bool read_compressed_number (struct sfm_reader *, double *);
2713 static int read_compressed_string (struct sfm_reader *, uint8_t *);
2714 static int read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
2715 static bool skip_whole_strings (struct sfm_reader *, size_t);
2717 /* Reads and returns one case from READER's file. Returns a null
2718 pointer if not successful. */
2719 static struct ccase *
2720 sys_file_casereader_read (struct casereader *reader, void *r_)
2722 struct sfm_reader *r = r_;
2727 if (r->error || !r->sfm_var_cnt)
2730 c = case_create (r->proto);
2732 for (i = 0; i < r->sfm_var_cnt; i++)
2734 struct sfm_var *sv = &r->sfm_vars[i];
2735 union value *v = case_data_rw_idx (c, sv->case_index);
2737 if (sv->var_width == 0)
2738 retval = read_case_number (r, &v->f);
2741 retval = read_case_string (r, v->s + sv->offset, sv->segment_width);
2744 retval = skip_whole_strings (r, ROUND_DOWN (sv->padding, 8));
2746 sys_error (r, r->pos, _("File ends in partial string value."));
2758 if (r->case_cnt != -1)
2759 read_error (reader, r);
2764 /* Issues an error that R ends in a partial record. */
2766 partial_record (struct sfm_reader *r)
2768 sys_error (r, r->pos, _("File ends in partial case."));
2771 /* Issues an error that an unspecified error occurred SFM, and
2774 read_error (struct casereader *r, const struct sfm_reader *sfm)
2776 msg (ME, _("Error reading case from file %s."), fh_get_name (sfm->fh));
2777 casereader_force_error (r);
2780 /* Reads a number from R and stores its value in *D.
2781 If R is compressed, reads a compressed number;
2782 otherwise, reads a number in the regular way.
2783 Returns true if successful, false if end of file is
2784 reached immediately. */
2786 read_case_number (struct sfm_reader *r, double *d)
2788 if (r->compression == ANY_COMP_NONE)
2791 if (!try_read_bytes (r, number, sizeof number))
2793 float_convert (r->float_format, number, FLOAT_NATIVE_DOUBLE, d);
2797 return read_compressed_number (r, d);
2800 /* Reads LENGTH string bytes from R into S. Always reads a multiple of 8
2801 bytes; if LENGTH is not a multiple of 8, then extra bytes are read and
2802 discarded without being written to S. Reads compressed strings if S is
2803 compressed. Returns 1 if successful, 0 if end of file is reached
2804 immediately, or -1 for some kind of error. */
2806 read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
2808 size_t whole = ROUND_DOWN (length, 8);
2809 size_t partial = length % 8;
2813 int retval = read_whole_strings (r, s, whole);
2821 int retval = read_whole_strings (r, bounce, sizeof bounce);
2833 memcpy (s + whole, bounce, partial);
2839 /* Reads and returns the next compression opcode from R. */
2841 read_opcode (struct sfm_reader *r)
2843 assert (r->compression != ANY_COMP_NONE);
2847 if (r->opcode_idx >= sizeof r->opcodes)
2850 int retval = try_read_compressed_bytes (r, r->opcodes,
2856 opcode = r->opcodes[r->opcode_idx++];
2863 /* Reads a compressed number from R and stores its value in D.
2864 Returns true if successful, false if end of file is
2865 reached immediately. */
2867 read_compressed_number (struct sfm_reader *r, double *d)
2869 int opcode = read_opcode (r);
2877 return read_compressed_float (r, d);
2880 float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
2881 if (!r->corruption_warning)
2883 r->corruption_warning = true;
2884 sys_warn (r, r->pos,
2885 _("Possible compressed data corruption: "
2886 "compressed spaces appear in numeric field."));
2895 *d = opcode - r->bias;
2902 /* Reads a compressed 8-byte string segment from R and stores it in DST. */
2904 read_compressed_string (struct sfm_reader *r, uint8_t *dst)
2909 opcode = read_opcode (r);
2917 retval = read_compressed_bytes (r, dst, 8);
2918 return retval == 1 ? 1 : -1;
2921 memset (dst, ' ', 8);
2926 double value = opcode - r->bias;
2927 float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
2930 /* This has actually been seen "in the wild". The submitter of the
2931 file that showed that the contents decoded as spaces, but they
2932 were at the end of the field so it's possible that the null
2933 bytes just acted as null terminators. */
2935 else if (!r->corruption_warning)
2937 r->corruption_warning = true;
2938 sys_warn (r, r->pos,
2939 _("Possible compressed data corruption: "
2940 "string contains compressed integer (opcode %d)."),
2948 /* Reads LENGTH string bytes from R into S. LENGTH must be a multiple of 8.
2949 Reads compressed strings if S is compressed. Returns 1 if successful, 0 if
2950 end of file is reached immediately, or -1 for some kind of error. */
2952 read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
2954 assert (length % 8 == 0);
2955 if (r->compression == ANY_COMP_NONE)
2956 return try_read_bytes (r, s, length);
2961 for (ofs = 0; ofs < length; ofs += 8)
2963 int retval = read_compressed_string (r, s + ofs);
2978 /* Skips LENGTH string bytes from R.
2979 LENGTH must be a multiple of 8.
2980 (LENGTH is also limited to 1024, but that's only because the
2981 current caller never needs more than that many bytes.)
2982 Returns true if successful, false if end of file is
2983 reached immediately. */
2985 skip_whole_strings (struct sfm_reader *r, size_t length)
2987 uint8_t buffer[1024];
2988 assert (length < sizeof buffer);
2989 return read_whole_strings (r, buffer, length);
2992 /* Helpers for reading records that contain structured text
2995 /* Maximum number of warnings to issue for a single text
2997 #define MAX_TEXT_WARNINGS 5
3002 struct substring buffer; /* Record contents. */
3003 off_t start; /* Starting offset in file. */
3004 size_t pos; /* Current position in buffer. */
3005 int n_warnings; /* Number of warnings issued or suppressed. */
3006 bool recoded; /* Recoded into UTF-8? */
3009 static struct text_record *
3010 open_text_record (struct sfm_reader *r,
3011 const struct sfm_extension_record *record,
3012 bool recode_to_utf8)
3014 struct text_record *text;
3015 struct substring raw;
3017 text = pool_alloc (r->pool, sizeof *text);
3018 raw = ss_buffer (record->data, record->size * record->count);
3019 text->start = record->pos;
3020 text->buffer = (recode_to_utf8
3021 ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
3024 text->n_warnings = 0;
3025 text->recoded = recode_to_utf8;
3030 /* Closes TEXT, frees its storage, and issues a final warning
3031 about suppressed warnings if necessary. */
3033 close_text_record (struct sfm_reader *r, struct text_record *text)
3035 if (text->n_warnings > MAX_TEXT_WARNINGS)
3036 sys_warn (r, -1, _("Suppressed %d additional related warnings."),
3037 text->n_warnings - MAX_TEXT_WARNINGS);
3039 pool_free (r->pool, ss_data (text->buffer));
3042 /* Reads a variable=value pair from TEXT.
3043 Looks up the variable in DICT and stores it into *VAR.
3044 Stores a null-terminated value into *VALUE. */
3046 read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
3047 struct text_record *text,
3048 struct variable **var, char **value)
3052 if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
3055 *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
3059 text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
3060 ss_buffer ("\t\0", 2));
3068 text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
3069 struct text_record *text, struct substring delimiters,
3070 struct variable **var)
3074 name = text_get_token (text, delimiters, NULL);
3078 *var = dict_lookup_var (dict, name);
3082 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
3089 text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
3090 struct text_record *text, struct substring delimiters,
3091 struct variable **var)
3093 char *short_name = text_get_token (text, delimiters, NULL);
3094 if (short_name == NULL)
3097 *var = dict_lookup_var (dict, short_name);
3099 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
3104 /* Displays a warning for the current file position, limiting the
3105 number to MAX_TEXT_WARNINGS for TEXT. */
3107 text_warn (struct sfm_reader *r, struct text_record *text,
3108 const char *format, ...)
3110 if (text->n_warnings++ < MAX_TEXT_WARNINGS)
3114 va_start (args, format);
3115 sys_msg (r, text->start + text->pos, MW, format, args);
3121 text_get_token (struct text_record *text, struct substring delimiters,
3124 struct substring token;
3127 if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
3129 if (delimiter != NULL)
3130 *delimiter = ss_data (text->buffer)[text->pos-1];
3134 end = &ss_data (token)[ss_length (token)];
3135 if (delimiter != NULL)
3138 return ss_data (token);
3141 /* Reads a integer value expressed in decimal, then a space, then a string that
3142 consists of exactly as many bytes as specified by the integer, then a space,
3143 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
3144 buffer (so the caller should not free the string). */
3146 text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
3154 while (text->pos < text->buffer.length)
3156 int c = text->buffer.string[text->pos];
3157 if (c < '0' || c > '9')
3159 n = (n * 10) + (c - '0');
3162 if (text->pos >= text->buffer.length || start == text->pos)
3164 sys_warn (r, text->start,
3165 _("Expecting digit at offset %zu in MRSETS record."),
3170 if (!text_match (text, ' '))
3172 sys_warn (r, text->start,
3173 _("Expecting space at offset %zu in MRSETS record."),
3178 if (text->pos + n > text->buffer.length)
3180 sys_warn (r, text->start,
3181 _("%zu-byte string starting at offset %zu "
3182 "exceeds record length %zu."),
3183 n, text->pos, text->buffer.length);
3187 s = &text->buffer.string[text->pos];
3190 sys_warn (r, text->start,
3191 _("Expecting space at offset %zu following %zu-byte string."),
3201 text_match (struct text_record *text, char c)
3203 if (text->pos >= text->buffer.length)
3206 if (text->buffer.string[text->pos] == c)
3215 /* Returns the current byte offset (as converted to UTF-8, if it was converted)
3216 inside the TEXT's string. */
3218 text_pos (const struct text_record *text)
3224 text_get_all (const struct text_record *text)
3226 return text->buffer.string;
3231 /* Displays a corruption message. */
3233 sys_msg (struct sfm_reader *r, off_t offset,
3234 int class, const char *format, va_list args)
3238 ds_init_empty (&text);
3240 ds_put_format (&text, _("`%s' near offset 0x%llx: "),
3241 fh_get_file_name (r->fh), (long long int) offset);
3243 ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
3244 ds_put_vformat (&text, format, args);
3246 struct msg *m = xmalloc (sizeof *m);
3248 .category = msg_class_to_category (class),
3249 .severity = msg_class_to_severity (class),
3250 .text = ds_steal_cstr (&text),
3255 /* Displays a warning for offset OFFSET in the file. */
3257 sys_warn (struct sfm_reader *r, off_t offset, const char *format, ...)
3261 va_start (args, format);
3262 sys_msg (r, offset, MW, format, args);
3266 /* Displays an error for the current file position and marks it as in an error
3269 sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
3273 va_start (args, format);
3274 sys_msg (r, offset, ME, format, args);
3280 /* Reads BYTE_CNT bytes into BUF.
3281 Returns 1 if exactly BYTE_CNT bytes are successfully read.
3282 Returns -1 if an I/O error or a partial read occurs.
3283 Returns 0 for an immediate end-of-file and, if EOF_IS_OK is false, reports
3286 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
3287 void *buf, size_t byte_cnt)
3289 size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
3290 r->pos += bytes_read;
3291 if (bytes_read == byte_cnt)
3293 else if (ferror (r->file))
3295 sys_error (r, r->pos, _("System error: %s."), strerror (errno));
3298 else if (!eof_is_ok || bytes_read != 0)
3300 sys_error (r, r->pos, _("Unexpected end of file."));
3307 /* Reads BYTE_CNT into BUF.
3308 Returns true if successful.
3309 Returns false upon I/O error or if end-of-file is encountered. */
3311 read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
3313 return read_bytes_internal (r, false, buf, byte_cnt) == 1;
3316 /* Reads BYTE_CNT bytes into BUF.
3317 Returns 1 if exactly BYTE_CNT bytes are successfully read.
3318 Returns 0 if an immediate end-of-file is encountered.
3319 Returns -1 if an I/O error or a partial read occurs. */
3321 try_read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
3323 return read_bytes_internal (r, true, buf, byte_cnt);
3326 /* Reads a 32-bit signed integer from R and stores its value in host format in
3327 *X. Returns true if successful, otherwise false. */
3329 read_int (struct sfm_reader *r, int *x)
3332 if (read_bytes (r, integer, sizeof integer) != 1)
3334 *x = integer_get (r->integer_format, integer, sizeof integer);
3339 read_uint (struct sfm_reader *r, unsigned int *x)
3344 ok = read_int (r, &y);
3349 /* Reads a 64-bit signed integer from R and returns its value in
3352 read_int64 (struct sfm_reader *r, long long int *x)
3355 if (read_bytes (r, integer, sizeof integer) != 1)
3357 *x = integer_get (r->integer_format, integer, sizeof integer);
3361 /* Reads a 64-bit signed integer from R and returns its value in
3364 read_uint64 (struct sfm_reader *r, unsigned long long int *x)
3369 ok = read_int64 (r, &y);
3375 parse_int (const struct sfm_reader *r, const void *data, size_t ofs)
3377 return integer_get (r->integer_format, (const uint8_t *) data + ofs, 4);
3381 parse_float (const struct sfm_reader *r, const void *data, size_t ofs)
3383 return float_get_double (r->float_format, (const uint8_t *) data + ofs);
3386 /* Reads exactly SIZE - 1 bytes into BUFFER
3387 and stores a null byte into BUFFER[SIZE - 1]. */
3389 read_string (struct sfm_reader *r, char *buffer, size_t size)
3394 ok = read_bytes (r, buffer, size - 1);
3396 buffer[size - 1] = '\0';
3400 /* Skips BYTES bytes forward in R. */
3402 skip_bytes (struct sfm_reader *r, size_t bytes)
3407 size_t chunk = MIN (sizeof buffer, bytes);
3408 if (!read_bytes (r, buffer, chunk))
3416 /* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have
3417 been replaced by LFs.
3419 (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system
3420 files that use CR-only line ends in the file label and extra product
3423 fix_line_ends (const char *s)
3427 d = dst = xmalloc (strlen (s) + 1);
3446 read_ztrailer (struct sfm_reader *r,
3447 long long int zheader_ofs,
3448 long long int ztrailer_len);
3451 zalloc (voidpf pool_, uInt items, uInt size)
3453 struct pool *pool = pool_;
3455 return (!size || xalloc_oversized (items, size)
3457 : pool_malloc (pool, items * size));
3461 zfree (voidpf pool_, voidpf address)
3463 struct pool *pool = pool_;
3465 pool_free (pool, address);
3469 read_zheader (struct sfm_reader *r)
3472 long long int zheader_ofs;
3473 long long int ztrailer_ofs;
3474 long long int ztrailer_len;
3476 if (!read_int64 (r, &zheader_ofs)
3477 || !read_int64 (r, &ztrailer_ofs)
3478 || !read_int64 (r, &ztrailer_len))
3481 if (zheader_ofs != pos)
3483 sys_error (r, pos, _("Wrong ZLIB data header offset %#llx "
3484 "(expected %#llx)."),
3485 zheader_ofs, (long long int) pos);
3489 if (ztrailer_ofs < r->pos)
3491 sys_error (r, pos, _("Impossible ZLIB trailer offset 0x%llx."),
3496 if (ztrailer_len < 24 || ztrailer_len % 24)
3498 sys_error (r, pos, _("Invalid ZLIB trailer length %lld."), ztrailer_len);
3502 r->ztrailer_ofs = ztrailer_ofs;
3503 if (!read_ztrailer (r, zheader_ofs, ztrailer_len))
3506 if (r->zin_buf == NULL)
3508 r->zin_buf = pool_malloc (r->pool, ZIN_BUF_SIZE);
3509 r->zout_buf = pool_malloc (r->pool, ZOUT_BUF_SIZE);
3510 r->zstream.next_in = NULL;
3511 r->zstream.avail_in = 0;
3514 r->zstream.zalloc = zalloc;
3515 r->zstream.zfree = zfree;
3516 r->zstream.opaque = r->pool;
3518 return open_zstream (r);
3522 seek (struct sfm_reader *r, off_t offset)
3524 if (fseeko (r->file, offset, SEEK_SET))
3525 sys_error (r, 0, _("%s: seek failed (%s)."),
3526 fh_get_file_name (r->fh), strerror (errno));
3530 /* Performs some additional consistency checks on the ZLIB compressed data
3533 read_ztrailer (struct sfm_reader *r,
3534 long long int zheader_ofs,
3535 long long int ztrailer_len)
3537 long long int expected_uncmp_ofs;
3538 long long int expected_cmp_ofs;
3541 unsigned int block_size;
3542 unsigned int n_blocks;
3546 if (fstat (fileno (r->file), &s))
3548 sys_error (r, 0, _("%s: stat failed (%s)."),
3549 fh_get_file_name (r->fh), strerror (errno));
3553 if (!S_ISREG (s.st_mode))
3555 /* We can't seek to the trailer and then back to the data in this file,
3556 so skip doing extra checks. */
3560 if (r->ztrailer_ofs + ztrailer_len != s.st_size)
3561 sys_warn (r, r->pos,
3562 _("End of ZLIB trailer (0x%llx) is not file size (0x%llx)."),
3563 r->ztrailer_ofs + ztrailer_len, (long long int) s.st_size);
3565 seek (r, r->ztrailer_ofs);
3567 /* Read fixed header from ZLIB data trailer. */
3568 if (!read_int64 (r, &bias))
3570 if (-bias != r->bias)
3572 sys_error (r, r->pos, _("ZLIB trailer bias (%lld) differs from "
3573 "file header bias (%.2f)."),
3578 if (!read_int64 (r, &zero))
3581 sys_warn (r, r->pos,
3582 _("ZLIB trailer \"zero\" field has nonzero value %lld."), zero);
3584 if (!read_uint (r, &block_size))
3586 if (block_size != ZBLOCK_SIZE)
3587 sys_warn (r, r->pos,
3588 _("ZLIB trailer specifies unexpected %u-byte block size."),
3591 if (!read_uint (r, &n_blocks))
3593 if (n_blocks != (ztrailer_len - 24) / 24)
3595 sys_error (r, r->pos,
3596 _("%lld-byte ZLIB trailer specifies %u data blocks (expected "
3598 ztrailer_len, n_blocks, (ztrailer_len - 24) / 24);
3602 expected_uncmp_ofs = zheader_ofs;
3603 expected_cmp_ofs = zheader_ofs + 24;
3604 for (i = 0; i < n_blocks; i++)
3606 off_t desc_ofs = r->pos;
3607 unsigned long long int uncompressed_ofs;
3608 unsigned long long int compressed_ofs;
3609 unsigned int uncompressed_size;
3610 unsigned int compressed_size;
3612 if (!read_uint64 (r, &uncompressed_ofs)
3613 || !read_uint64 (r, &compressed_ofs)
3614 || !read_uint (r, &uncompressed_size)
3615 || !read_uint (r, &compressed_size))
3618 if (uncompressed_ofs != expected_uncmp_ofs)
3620 sys_error (r, desc_ofs,
3621 _("ZLIB block descriptor %u reported uncompressed data "
3622 "offset %#llx, when %#llx was expected."),
3623 i, uncompressed_ofs, expected_uncmp_ofs);
3627 if (compressed_ofs != expected_cmp_ofs)
3629 sys_error (r, desc_ofs,
3630 _("ZLIB block descriptor %u reported compressed data "
3631 "offset %#llx, when %#llx was expected."),
3632 i, compressed_ofs, expected_cmp_ofs);
3636 if (i < n_blocks - 1)
3638 if (uncompressed_size != block_size)
3639 sys_warn (r, desc_ofs,
3640 _("ZLIB block descriptor %u reported block size %#x, "
3641 "when %#x was expected."),
3642 i, uncompressed_size, block_size);
3646 if (uncompressed_size > block_size)
3647 sys_warn (r, desc_ofs,
3648 _("ZLIB block descriptor %u reported block size %#x, "
3649 "when at most %#x was expected."),
3650 i, uncompressed_size, block_size);
3653 /* http://www.zlib.net/zlib_tech.html says that the maximum expansion
3654 from compression, with worst-case parameters, is 13.5% plus 11 bytes.
3655 This code checks for an expansion of more than 14.3% plus 11
3657 if (compressed_size > uncompressed_size + uncompressed_size / 7 + 11)
3659 sys_error (r, desc_ofs,
3660 _("ZLIB block descriptor %u reports compressed size %u "
3661 "and uncompressed size %u."),
3662 i, compressed_size, uncompressed_size);
3666 expected_uncmp_ofs += uncompressed_size;
3667 expected_cmp_ofs += compressed_size;
3670 if (expected_cmp_ofs != r->ztrailer_ofs)
3672 sys_error (r, r->pos, _("ZLIB trailer is at offset %#llx but %#llx "
3673 "would be expected from block descriptors."),
3674 r->ztrailer_ofs, expected_cmp_ofs);
3678 seek (r, zheader_ofs + 24);
3683 open_zstream (struct sfm_reader *r)
3687 r->zout_pos = r->zout_end = 0;
3688 error = inflateInit (&r->zstream);
3691 sys_error (r, r->pos, _("ZLIB initialization failed (%s)."),
3699 close_zstream (struct sfm_reader *r)
3703 error = inflateEnd (&r->zstream);
3706 sys_error (r, r->pos, _("Inconsistency at end of ZLIB stream (%s)."),
3714 read_bytes_zlib (struct sfm_reader *r, void *buf_, size_t byte_cnt)
3716 uint8_t *buf = buf_;
3725 /* Use already inflated data if there is any. */
3726 if (r->zout_pos < r->zout_end)
3728 unsigned int n = MIN (byte_cnt, r->zout_end - r->zout_pos);
3729 memcpy (buf, &r->zout_buf[r->zout_pos], n);
3738 /* We need to inflate some more data.
3739 Get some more input data if we don't have any. */
3740 if (r->zstream.avail_in == 0)
3742 unsigned int n = MIN (ZIN_BUF_SIZE, r->ztrailer_ofs - r->pos);
3747 int retval = try_read_bytes (r, r->zin_buf, n);
3750 r->zstream.avail_in = n;
3751 r->zstream.next_in = r->zin_buf;
3755 /* Inflate the (remaining) input data. */
3756 r->zstream.avail_out = ZOUT_BUF_SIZE;
3757 r->zstream.next_out = r->zout_buf;
3758 error = inflate (&r->zstream, Z_SYNC_FLUSH);
3760 r->zout_end = r->zstream.next_out - r->zout_buf;
3761 if (r->zout_end == 0)
3763 if (error != Z_STREAM_END)
3765 sys_error (r, r->pos, _("ZLIB stream inconsistency (%s)."),
3769 else if (!close_zstream (r) || !open_zstream (r))
3774 /* Process the output data and ignore 'error' for now. ZLIB will
3775 present it to us again on the next inflate() call. */
3781 read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
3783 if (r->compression == ANY_COMP_SIMPLE)
3784 return read_bytes (r, buf, byte_cnt);
3787 int retval = read_bytes_zlib (r, buf, byte_cnt);
3789 sys_error (r, r->pos, _("Unexpected end of ZLIB compressed data."));
3795 try_read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
3797 if (r->compression == ANY_COMP_SIMPLE)
3798 return try_read_bytes (r, buf, byte_cnt);
3800 return read_bytes_zlib (r, buf, byte_cnt);
3803 /* Reads a 64-bit floating-point number from R and returns its
3804 value in host format. */
3806 read_compressed_float (struct sfm_reader *r, double *d)
3810 if (!read_compressed_bytes (r, number, sizeof number))
3813 *d = float_get_double (r->float_format, number);
3817 static const struct casereader_class sys_file_casereader_class =
3819 sys_file_casereader_read,
3820 sys_file_casereader_destroy,
3825 const struct any_reader_class sys_file_reader_class =
3827 N_("SPSS System File"),