1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2000, 2006-2007, 2009-2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/sys-file-reader.h"
20 #include "data/sys-file-private.h"
28 #include "data/attributes.h"
29 #include "data/case.h"
30 #include "data/casereader-provider.h"
31 #include "data/casereader.h"
32 #include "data/dictionary.h"
33 #include "data/file-handle-def.h"
34 #include "data/file-name.h"
35 #include "data/format.h"
36 #include "data/identifier.h"
37 #include "data/missing-values.h"
38 #include "data/mrset.h"
39 #include "data/short-names.h"
40 #include "data/value-labels.h"
41 #include "data/value.h"
42 #include "data/variable.h"
43 #include "libpspp/array.h"
44 #include "libpspp/assertion.h"
45 #include "libpspp/compiler.h"
46 #include "libpspp/i18n.h"
47 #include "libpspp/message.h"
48 #include "libpspp/misc.h"
49 #include "libpspp/pool.h"
50 #include "libpspp/str.h"
51 #include "libpspp/stringi-set.h"
53 #include "gl/c-ctype.h"
54 #include "gl/inttostr.h"
55 #include "gl/localcharset.h"
56 #include "gl/minmax.h"
57 #include "gl/unlocked-io.h"
58 #include "gl/xalloc.h"
62 #define _(msgid) gettext (msgid)
63 #define N_(msgid) (msgid)
67 /* subtypes 0-2 unknown */
68 EXT_INTEGER = 3, /* Machine integer info. */
69 EXT_FLOAT = 4, /* Machine floating-point info. */
70 EXT_VAR_SETS = 5, /* Variable sets. */
71 EXT_DATE = 6, /* DATE. */
72 EXT_MRSETS = 7, /* Multiple response sets. */
73 EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
74 /* subtypes 9-10 unknown */
75 EXT_DISPLAY = 11, /* Variable display parameters. */
76 /* subtype 12 unknown */
77 EXT_LONG_NAMES = 13, /* Long variable names. */
78 EXT_LONG_STRINGS = 14, /* Long strings. */
79 /* subtype 15 unknown */
80 EXT_NCASES = 16, /* Extended number of cases. */
81 EXT_FILE_ATTRS = 17, /* Data file attributes. */
82 EXT_VAR_ATTRS = 18, /* Variable attributes. */
83 EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
84 EXT_ENCODING = 20, /* Character encoding. */
85 EXT_LONG_LABELS = 21 /* Value labels for long strings. */
95 int missing_value_code;
101 struct sfm_value_label
107 struct sfm_value_label_record
110 struct sfm_value_label *labels;
117 struct sfm_document_record
124 struct sfm_extension_record
126 off_t pos; /* Starting offset in file. */
127 size_t size; /* Size of data elements. */
128 size_t count; /* Number of data elements. */
129 void *data; /* Contents. */
132 /* System file reader. */
135 /* Resource tracking. */
136 struct pool *pool; /* All system file state. */
137 jmp_buf bail_out; /* longjmp() target for error handling. */
140 struct file_handle *fh; /* File handle. */
141 struct fh_lock *lock; /* Mutual exclusion for file handle. */
142 FILE *file; /* File stream. */
143 off_t pos; /* Position in file. */
144 bool error; /* I/O or corruption error? */
145 struct caseproto *proto; /* Format of output cases. */
148 enum integer_format integer_format; /* On-disk integer format. */
149 enum float_format float_format; /* On-disk floating point format. */
150 struct sfm_var *sfm_vars; /* Variables. */
151 size_t sfm_var_cnt; /* Number of variables. */
152 casenumber case_cnt; /* Number of cases */
153 const char *encoding; /* String encoding. */
156 bool compressed; /* File is compressed? */
157 double bias; /* Compression bias, usually 100.0. */
158 uint8_t opcodes[8]; /* Current block of opcodes. */
159 size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
160 bool corruption_warning; /* Warned about possible corruption? */
163 static const struct casereader_class sys_file_casereader_class;
165 static bool close_reader (struct sfm_reader *);
167 static struct variable *lookup_var_by_index (struct sfm_reader *, off_t,
168 const struct sfm_var_record *,
171 static void sys_msg (struct sfm_reader *r, off_t, int class,
172 const char *format, va_list args)
173 PRINTF_FORMAT (4, 0);
174 static void sys_warn (struct sfm_reader *, off_t, const char *, ...)
175 PRINTF_FORMAT (3, 4);
176 static void sys_error (struct sfm_reader *, off_t, const char *, ...)
180 static void read_bytes (struct sfm_reader *, void *, size_t);
181 static bool try_read_bytes (struct sfm_reader *, void *, size_t);
182 static int read_int (struct sfm_reader *);
183 static double read_float (struct sfm_reader *);
184 static void read_string (struct sfm_reader *, char *, size_t);
185 static void skip_bytes (struct sfm_reader *, size_t);
187 static int parse_int (struct sfm_reader *, const void *data, size_t ofs);
188 static double parse_float (struct sfm_reader *, const void *data, size_t ofs);
190 static void read_variable_record (struct sfm_reader *,
191 struct sfm_var_record *);
192 static void read_value_label_record (struct sfm_reader *,
193 struct sfm_value_label_record *,
195 static struct sfm_document_record *read_document_record (struct sfm_reader *);
196 static struct sfm_extension_record *read_extension_record (
197 struct sfm_reader *, int subtype);
198 static void skip_extension_record (struct sfm_reader *, int subtype);
200 static const char *choose_encoding (
202 const struct sfm_extension_record *ext_integer,
203 const struct sfm_extension_record *ext_encoding);
205 static struct text_record *open_text_record (
206 struct sfm_reader *, const struct sfm_extension_record *,
207 bool recode_to_utf8);
208 static void close_text_record (struct sfm_reader *,
209 struct text_record *);
210 static bool read_variable_to_value_pair (struct sfm_reader *,
212 struct text_record *,
213 struct variable **var, char **value);
214 static void text_warn (struct sfm_reader *r, struct text_record *text,
215 const char *format, ...)
216 PRINTF_FORMAT (3, 4);
217 static char *text_get_token (struct text_record *,
218 struct substring delimiters, char *delimiter);
219 static bool text_match (struct text_record *, char c);
220 static bool text_read_variable_name (struct sfm_reader *, struct dictionary *,
221 struct text_record *,
222 struct substring delimiters,
224 static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
225 struct text_record *,
226 struct substring delimiters,
228 static const char *text_parse_counted_string (struct sfm_reader *,
229 struct text_record *);
230 static size_t text_pos (const struct text_record *);
232 static bool close_reader (struct sfm_reader *r);
234 /* Dictionary reader. */
242 static void read_header (struct sfm_reader *, int *weight_idx,
243 int *claimed_oct_cnt, struct sfm_read_info *,
245 static void parse_file_label (struct sfm_reader *, const char *file_label,
246 struct dictionary *);
247 static void parse_variable_records (struct sfm_reader *, struct dictionary *,
248 struct sfm_var_record *, size_t n);
249 static void parse_format_spec (struct sfm_reader *, off_t pos,
250 unsigned int format, enum which_format,
251 struct variable *, int *format_warning_cnt);
252 static void parse_document (struct dictionary *, struct sfm_document_record *);
253 static void parse_display_parameters (struct sfm_reader *,
254 const struct sfm_extension_record *,
255 struct dictionary *);
256 static void parse_machine_integer_info (struct sfm_reader *,
257 const struct sfm_extension_record *,
258 struct sfm_read_info *);
259 static void parse_machine_float_info (struct sfm_reader *,
260 const struct sfm_extension_record *);
261 static void parse_mrsets (struct sfm_reader *,
262 const struct sfm_extension_record *,
263 struct dictionary *);
264 static void parse_long_var_name_map (struct sfm_reader *,
265 const struct sfm_extension_record *,
266 struct dictionary *);
267 static void parse_long_string_map (struct sfm_reader *,
268 const struct sfm_extension_record *,
269 struct dictionary *);
270 static void parse_value_labels (struct sfm_reader *, struct dictionary *,
271 const struct sfm_var_record *,
273 const struct sfm_value_label_record *);
274 static void parse_data_file_attributes (struct sfm_reader *,
275 const struct sfm_extension_record *,
276 struct dictionary *);
277 static void parse_variable_attributes (struct sfm_reader *,
278 const struct sfm_extension_record *,
279 struct dictionary *);
280 static void parse_long_string_value_labels (struct sfm_reader *,
281 const struct sfm_extension_record *,
282 struct dictionary *);
284 /* Opens the system file designated by file handle FH for
285 reading. Reads the system file's dictionary into *DICT.
286 If INFO is non-null, then it receives additional info about the
289 sfm_open_reader (struct file_handle *fh, struct dictionary **dictp,
290 struct sfm_read_info *volatile info)
292 struct sfm_reader *volatile r = NULL;
293 struct sfm_read_info local_info;
295 struct sfm_var_record *vars;
296 size_t n_vars, allocated_vars;
298 struct sfm_value_label_record *labels;
299 size_t n_labels, allocated_labels;
301 struct sfm_document_record *document;
303 struct sfm_extension_record *extensions[32];
309 struct dictionary *dict = NULL;
312 /* Create and initialize reader. */
313 r = pool_create_container (struct sfm_reader, pool);
319 r->opcode_idx = sizeof r->opcodes;
320 r->corruption_warning = false;
322 /* TRANSLATORS: this fragment will be interpolated into
323 messages in fh_lock() that identify types of files. */
324 r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false);
328 r->file = fn_open (fh_get_file_name (fh), "rb");
331 msg (ME, _("Error opening `%s' for reading as a system file: %s."),
332 fh_get_file_name (r->fh), strerror (errno));
336 /* Initialize info. */
339 memset (info, 0, sizeof *info);
341 if (setjmp (r->bail_out))
345 read_header (r, &weight_idx, &claimed_oct_cnt, info, &file_label);
348 n_vars = allocated_vars = 0;
351 n_labels = allocated_labels = 0;
355 memset (extensions, 0, sizeof extensions);
365 read_int (r); /* Skip filler. */
372 if (n_vars >= allocated_vars)
373 vars = pool_2nrealloc (r->pool, vars, &allocated_vars,
375 read_variable_record (r, &vars[n_vars++]);
379 if (n_labels >= allocated_labels)
380 labels = pool_2nrealloc (r->pool, labels, &allocated_labels,
382 read_value_label_record (r, &labels[n_labels++], n_vars);
386 /* A Type 4 record is always immediately after a type 3 record,
387 so the code for type 3 records reads the type 4 record too. */
388 sys_error (r, r->pos, _("Misplaced type 4 record."));
391 if (document != NULL)
392 sys_error (r, r->pos, _("Duplicate type 6 (document) record."));
393 document = read_document_record (r);
397 subtype = read_int (r);
398 if (subtype < 0 || subtype >= sizeof extensions / sizeof *extensions)
401 _("Unrecognized record type 7, subtype %d. Please "
402 "send a copy of this file, and the syntax which "
403 "created it to %s."),
404 subtype, PACKAGE_BUGREPORT);
405 skip_extension_record (r, subtype);
407 else if (extensions[subtype] != NULL)
410 _("Record type 7, subtype %d found here has the same "
411 "type as the record found near offset 0x%llx. "
412 "Please send a copy of this file, and the syntax "
413 "which created it to %s."),
414 subtype, (long long int) extensions[subtype]->pos,
416 skip_extension_record (r, subtype);
419 extensions[subtype] = read_extension_record (r, subtype);
423 sys_error (r, r->pos, _("Unrecognized record type %d."), type);
428 /* Now actually parse what we read.
430 First, figure out the correct character encoding, because this determines
431 how the rest of the header data is to be interpreted. */
432 dict = dict_create (choose_encoding (r, extensions[EXT_INTEGER],
433 extensions[EXT_ENCODING]));
434 r->encoding = dict_get_encoding (dict);
436 /* These records don't use variables at all. */
437 if (document != NULL)
438 parse_document (dict, document);
440 if (extensions[EXT_INTEGER] != NULL)
441 parse_machine_integer_info (r, extensions[EXT_INTEGER], info);
443 if (extensions[EXT_FLOAT] != NULL)
444 parse_machine_float_info (r, extensions[EXT_FLOAT]);
446 if (extensions[EXT_FILE_ATTRS] != NULL)
447 parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict);
449 parse_file_label (r, file_label, dict);
451 /* Parse the variable records, the basis of almost everything else. */
452 parse_variable_records (r, dict, vars, n_vars);
454 /* Parse value labels and the weight variable immediately after the variable
455 records. These records use indexes into var_recs[], so we must parse them
456 before those indexes become invalidated by very long string variables. */
457 for (i = 0; i < n_labels; i++)
458 parse_value_labels (r, dict, vars, n_vars, &labels[i]);
461 struct variable *weight_var;
463 weight_var = lookup_var_by_index (r, 76, vars, n_vars, weight_idx);
464 if (var_is_numeric (weight_var))
465 dict_set_weight (dict, weight_var);
467 sys_error (r, -1, _("Weighting variable must be numeric "
468 "(not string variable `%s')."),
469 var_get_name (weight_var));
472 if (extensions[EXT_DISPLAY] != NULL)
473 parse_display_parameters (r, extensions[EXT_DISPLAY], dict);
475 /* The following records use short names, so they need to be parsed before
476 parse_long_var_name_map() changes short names to long names. */
477 if (extensions[EXT_MRSETS] != NULL)
478 parse_mrsets (r, extensions[EXT_MRSETS], dict);
480 if (extensions[EXT_MRSETS2] != NULL)
481 parse_mrsets (r, extensions[EXT_MRSETS2], dict);
483 if (extensions[EXT_LONG_STRINGS] != NULL)
484 parse_long_string_map (r, extensions[EXT_LONG_STRINGS], dict);
486 /* Now rename variables to their long names. */
487 parse_long_var_name_map (r, extensions[EXT_LONG_NAMES], dict);
489 /* The following records use long names, so they need to follow renaming. */
490 if (extensions[EXT_VAR_ATTRS] != NULL)
491 parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict);
493 if (extensions[EXT_LONG_LABELS] != NULL)
494 parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict);
496 /* Warn if the actual amount of data per case differs from the
497 amount that the header claims. SPSS version 13 gets this
498 wrong when very long strings are involved, so don't warn in
500 if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars
501 && info->version_major != 13)
502 sys_warn (r, -1, _("File header claims %d variable positions but "
503 "%zu were read from file."),
504 claimed_oct_cnt, n_vars);
506 /* Create an index of dictionary variable widths for
507 sfm_read_case to use. We cannot use the `struct variable's
508 from the dictionary we created, because the caller owns the
509 dictionary and may destroy or modify its variables. */
510 sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_var_cnt);
511 pool_register (r->pool, free, r->sfm_vars);
512 r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
515 return casereader_create_sequential
517 r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
518 &sys_file_casereader_class, r);
527 /* Closes a system file after we're done with it.
528 Returns true if an I/O error has occurred on READER, false
531 close_reader (struct sfm_reader *r)
540 if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)
542 msg (ME, _("Error closing system file `%s': %s."),
543 fh_get_file_name (r->fh), strerror (errno));
553 pool_destroy (r->pool);
558 /* Destroys READER. */
560 sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
562 struct sfm_reader *r = r_;
566 /* Returns true if FILE is an SPSS system file,
569 sfm_detect (FILE *file)
573 if (fread (rec_type, 4, 1, file) != 1)
577 return !strcmp ("$FL2", rec_type);
580 /* Reads the global header of the system file. Sets *WEIGHT_IDX to 0 if the
581 system file is unweighted, or to the value index of the weight variable
582 otherwise. Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units) per
583 case that the file claims to have (although it is not always correct).
584 Initializes INFO with header information. Stores the file label as a string
585 in dictionary encoding into *FILE_LABELP. */
587 read_header (struct sfm_reader *r, int *weight_idx,
588 int *claimed_oct_cnt, struct sfm_read_info *info,
592 char eye_catcher[61];
593 uint8_t raw_layout_code[4];
595 char creation_date[10];
596 char creation_time[9];
598 struct substring product;
600 read_string (r, rec_type, sizeof rec_type);
601 read_string (r, eye_catcher, sizeof eye_catcher);
603 if (strcmp ("$FL2", rec_type) != 0)
604 sys_error (r, 0, _("This is not an SPSS system file."));
606 /* Identify integer format. */
607 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
608 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
610 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
612 || (r->integer_format != INTEGER_MSB_FIRST
613 && r->integer_format != INTEGER_LSB_FIRST))
614 sys_error (r, 64, _("This is not an SPSS system file."));
616 *claimed_oct_cnt = read_int (r);
617 if (*claimed_oct_cnt < 0 || *claimed_oct_cnt > INT_MAX / 16)
618 *claimed_oct_cnt = -1;
620 r->compressed = read_int (r) != 0;
622 *weight_idx = read_int (r);
624 r->case_cnt = read_int (r);
625 if ( r->case_cnt > INT_MAX / 2)
628 /* Identify floating-point format and obtain compression bias. */
629 read_bytes (r, raw_bias, sizeof raw_bias);
630 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
632 uint8_t zero_bias[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
634 if (memcmp (raw_bias, zero_bias, 8))
635 sys_warn (r, r->pos - 8,
636 _("Compression bias is not the usual "
637 "value of 100, or system file uses unrecognized "
638 "floating-point format."));
641 /* Some software is known to write all-zeros to this
642 field. Such software also writes floating-point
643 numbers in the format that we expect by default
644 (it seems that all software most likely does, in
645 reality), so don't warn in this case. */
648 if (r->integer_format == INTEGER_MSB_FIRST)
649 r->float_format = FLOAT_IEEE_DOUBLE_BE;
651 r->float_format = FLOAT_IEEE_DOUBLE_LE;
653 float_convert (r->float_format, raw_bias, FLOAT_NATIVE_DOUBLE, &r->bias);
655 read_string (r, creation_date, sizeof creation_date);
656 read_string (r, creation_time, sizeof creation_time);
657 read_string (r, file_label, sizeof file_label);
660 strcpy (info->creation_date, creation_date);
661 strcpy (info->creation_time, creation_time);
662 info->integer_format = r->integer_format;
663 info->float_format = r->float_format;
664 info->compressed = r->compressed;
665 info->case_cnt = r->case_cnt;
667 product = ss_cstr (eye_catcher);
668 ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE"));
669 ss_trim (&product, ss_cstr (" "));
670 str_copy_buf_trunc (info->product, sizeof info->product,
671 ss_data (product), ss_length (product));
673 *file_labelp = pool_strdup0 (r->pool, file_label, sizeof file_label - 1);
676 /* Reads a variable (type 2) record from R into RECORD. */
678 read_variable_record (struct sfm_reader *r, struct sfm_var_record *record)
680 int has_variable_label;
682 memset (record, 0, sizeof *record);
684 record->pos = r->pos;
685 record->width = read_int (r);
686 has_variable_label = read_int (r);
687 record->missing_value_code = read_int (r);
688 record->print_format = read_int (r);
689 record->write_format = read_int (r);
690 read_bytes (r, record->name, sizeof record->name);
692 if (has_variable_label == 1)
694 enum { MAX_LABEL_LEN = 255 };
695 size_t len, read_len;
699 /* Read up to MAX_LABEL_LEN bytes of label. */
700 read_len = MIN (MAX_LABEL_LEN, len);
701 record->label = xmalloc (read_len + 1);
702 read_string (r, record->label, read_len + 1);
704 /* Skip unread label bytes. */
705 skip_bytes (r, len - read_len);
707 /* Skip label padding up to multiple of 4 bytes. */
708 skip_bytes (r, ROUND_UP (len, 4) - len);
710 else if (has_variable_label != 0)
711 sys_error (r, record->pos,
712 _("Variable label indicator field is not 0 or 1."));
714 /* Set missing values. */
715 if (record->missing_value_code != 0)
717 int code = record->missing_value_code;
718 if (record->width == 0)
720 if (code < -3 || code > 3 || code == -1)
721 sys_error (r, record->pos,
722 _("Numeric missing value indicator field is not "
723 "-3, -2, 0, 1, 2, or 3."));
727 if (code < 1 || code > 3)
728 sys_error (r, record->pos,
729 _("String missing value indicator field is not "
733 read_bytes (r, record->missing, 8 * abs (code));
737 /* Reads value labels from R into RECORD. */
739 read_value_label_record (struct sfm_reader *r,
740 struct sfm_value_label_record *record,
745 /* Read type 3 record. */
746 record->pos = r->pos;
747 record->n_labels = read_int (r);
748 if (record->n_labels > SIZE_MAX / sizeof *record->labels)
749 sys_error (r, r->pos - 4, _("Invalid number of labels %zu."),
751 record->labels = pool_nmalloc (r->pool, record->n_labels,
752 sizeof *record->labels);
753 for (i = 0; i < record->n_labels; i++)
755 struct sfm_value_label *label = &record->labels[i];
756 unsigned char label_len;
759 read_bytes (r, label->value, sizeof label->value);
761 /* Read label length. */
762 read_bytes (r, &label_len, sizeof label_len);
763 padded_len = ROUND_UP (label_len + 1, 8);
765 /* Read label, padding. */
766 label->label = pool_malloc (r->pool, padded_len + 1);
767 read_bytes (r, label->label, padded_len - 1);
768 label->label[label_len] = '\0';
771 /* Read record type of type 4 record. */
772 if (read_int (r) != 4)
773 sys_error (r, r->pos - 4,
774 _("Variable index record (type 4) does not immediately "
775 "follow value label record (type 3) as it should."));
777 /* Read number of variables associated with value label from type 4
779 record->n_vars = read_int (r);
780 if (record->n_vars < 1 || record->n_vars > n_vars)
781 sys_error (r, r->pos - 4,
782 _("Number of variables associated with a value label (%zu) "
783 "is not between 1 and the number of variables (%zu)."),
784 record->n_vars, n_vars);
785 record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
786 for (i = 0; i < record->n_vars; i++)
787 record->vars[i] = read_int (r);
790 /* Reads a document record from R and returns it. */
791 static struct sfm_document_record *
792 read_document_record (struct sfm_reader *r)
794 struct sfm_document_record *record;
797 record = pool_malloc (r->pool, sizeof *record);
798 record->pos = r->pos;
800 n_lines = read_int (r);
801 if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
802 sys_error (r, record->pos,
803 _("Number of document lines (%d) "
804 "must be greater than 0 and less than %d."),
805 n_lines, INT_MAX / DOC_LINE_LENGTH);
807 record->n_lines = n_lines;
808 record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
809 read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines);
815 read_extension_record_header (struct sfm_reader *r, int subtype,
816 struct sfm_extension_record *record)
818 record->pos = r->pos;
819 record->size = read_int (r);
820 record->count = read_int (r);
822 /* Check that SIZE * COUNT + 1 doesn't overflow. Adding 1
823 allows an extra byte for a null terminator, used by some
824 extension processing routines. */
825 if (record->size != 0
826 && size_overflow_p (xsum (1, xtimes (record->count, record->size))))
827 sys_error (r, record->pos, "Record type 7 subtype %d too large.", subtype);
830 /* Reads an extension record from R into RECORD. */
831 static struct sfm_extension_record *
832 read_extension_record (struct sfm_reader *r, int subtype)
834 struct extension_record_type
841 static const struct extension_record_type types[] =
843 /* Implemented record types. */
844 { EXT_INTEGER, 4, 8 },
846 { EXT_MRSETS, 1, 0 },
847 { EXT_DISPLAY, 4, 0 },
848 { EXT_LONG_NAMES, 1, 0 },
849 { EXT_LONG_STRINGS, 1, 0 },
850 { EXT_NCASES, 8, 2 },
851 { EXT_FILE_ATTRS, 1, 0 },
852 { EXT_VAR_ATTRS, 1, 0 },
853 { EXT_MRSETS2, 1, 0 },
854 { EXT_ENCODING, 1, 0 },
855 { EXT_LONG_LABELS, 1, 0 },
857 /* Ignored record types. */
858 { EXT_VAR_SETS, 0, 0 },
860 { EXT_DATA_ENTRY, 0, 0 },
863 const struct extension_record_type *type;
864 struct sfm_extension_record *record;
867 record = pool_malloc (r->pool, sizeof *record);
868 read_extension_record_header (r, subtype, record);
869 n_bytes = record->count * record->size;
871 for (type = types; type < &types[sizeof types / sizeof *types]; type++)
872 if (subtype == type->subtype)
874 if (type->size > 0 && record->size != type->size)
875 sys_warn (r, record->pos,
876 _("Record type 7, subtype %d has bad size %zu "
877 "(expected %d)."), subtype, record->size, type->size);
878 else if (type->count > 0 && record->count != type->count)
879 sys_warn (r, record->pos,
880 _("Record type 7, subtype %d has bad count %zu "
881 "(expected %d)."), subtype, record->count, type->count);
882 else if (type->count == 0 && type->size == 0)
884 /* Ignore this record. */
888 char *data = pool_malloc (r->pool, n_bytes + 1);
889 data[n_bytes] = '\0';
892 read_bytes (r, record->data, n_bytes);
899 sys_warn (r, record->pos,
900 _("Unrecognized record type 7, subtype %d. Please send a "
901 "copy of this file, and the syntax which created it to %s."),
902 subtype, PACKAGE_BUGREPORT);
905 skip_bytes (r, n_bytes);
910 skip_extension_record (struct sfm_reader *r, int subtype)
912 struct sfm_extension_record record;
914 read_extension_record_header (r, subtype, &record);
915 skip_bytes (r, record.count * record.size);
919 parse_file_label (struct sfm_reader *r, const char *file_label,
920 struct dictionary *dict)
922 char *utf8_file_label;
923 size_t file_label_len;
925 utf8_file_label = recode_string_pool ("UTF-8", dict_get_encoding (dict),
926 file_label, -1, r->pool);
927 file_label_len = strlen (utf8_file_label);
928 while (file_label_len > 0 && utf8_file_label[file_label_len - 1] == ' ')
930 utf8_file_label[file_label_len] = '\0';
931 dict_set_label (dict, utf8_file_label);
934 /* Reads a variable (type 2) record from R and adds the
935 corresponding variable to DICT.
936 Also skips past additional variable records for long string
939 parse_variable_records (struct sfm_reader *r, struct dictionary *dict,
940 struct sfm_var_record *var_recs, size_t n_var_recs)
942 const char *dict_encoding = dict_get_encoding (dict);
943 struct sfm_var_record *rec;
946 for (rec = var_recs; rec < &var_recs[n_var_recs]; )
948 struct variable *var;
953 name = recode_string_pool ("UTF-8", dict_encoding,
954 rec->name, 8, r->pool);
955 name[strcspn (name, " ")] = '\0';
957 if (!dict_id_is_valid (dict, name, false)
958 || name[0] == '$' || name[0] == '#')
959 sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
961 if (rec->width < 0 || rec->width > 255)
962 sys_error (r, rec->pos,
963 _("Bad width %d for variable %s."), rec->width, name);
965 var = rec->var = dict_create_var (dict, name, rec->width);
967 sys_error (r, rec->pos, _("Duplicate variable name `%s'."), name);
969 /* Set the short name the same as the long name. */
970 var_set_short_name (var, 0, name);
972 /* Get variable label, if any. */
977 utf8_label = recode_string_pool ("UTF-8", dict_encoding,
978 rec->label, -1, r->pool);
979 var_set_label (var, utf8_label, false);
982 /* Set missing values. */
983 if (rec->missing_value_code != 0)
985 int width = var_get_width (var);
986 struct missing_values mv;
988 mv_init_pool (r->pool, &mv, width);
989 if (var_is_numeric (var))
991 bool has_range = rec->missing_value_code < 0;
992 int n_discrete = (has_range
993 ? rec->missing_value_code == -3
994 : rec->missing_value_code);
999 double low = parse_float (r, rec->missing, 0);
1000 double high = parse_float (r, rec->missing, 8);
1001 mv_add_range (&mv, low, high);
1005 for (i = 0; i < n_discrete; i++)
1007 mv_add_num (&mv, parse_float (r, rec->missing, ofs));
1015 value_init_pool (r->pool, &value, width);
1016 value_set_missing (&value, width);
1017 for (i = 0; i < rec->missing_value_code; i++)
1019 uint8_t *s = value_str_rw (&value, width);
1020 memcpy (s, rec->missing + 8 * i, MIN (width, 8));
1021 mv_add_str (&mv, s);
1024 var_set_missing_values (var, &mv);
1028 parse_format_spec (r, rec->pos + 12, rec->print_format,
1029 PRINT_FORMAT, var, &n_warnings);
1030 parse_format_spec (r, rec->pos + 16, rec->write_format,
1031 WRITE_FORMAT, var, &n_warnings);
1033 /* Account for values.
1034 Skip long string continuation records, if any. */
1035 n_values = rec->width == 0 ? 1 : DIV_RND_UP (rec->width, 8);
1036 for (i = 1; i < n_values; i++)
1037 if (i + (rec - var_recs) >= n_var_recs || rec[i].width != -1)
1038 sys_error (r, rec->pos, _("Missing string continuation record."));
1043 /* Translates the format spec from sysfile format to internal
1046 parse_format_spec (struct sfm_reader *r, off_t pos, unsigned int format,
1047 enum which_format which, struct variable *v,
1050 const int max_warnings = 8;
1051 uint8_t raw_type = format >> 16;
1052 uint8_t w = format >> 8;
1058 if (!fmt_from_io (raw_type, &f.type))
1059 sys_error (r, pos, _("Unknown variable format %"PRIu8"."), raw_type);
1064 ok = fmt_check_output (&f) && fmt_check_width_compat (&f, var_get_width (v));
1069 if (which == PRINT_FORMAT)
1070 var_set_print_format (v, &f);
1072 var_set_write_format (v, &f);
1074 else if (++*n_warnings <= max_warnings)
1076 char fmt_string[FMT_STRING_LEN_MAX + 1];
1077 sys_warn (r, pos, _("%s variable %s has invalid %s format %s."),
1078 var_is_numeric (v) ? _("Numeric") : _("String"),
1080 which == PRINT_FORMAT ? _("print") : _("write"),
1081 fmt_to_string (&f, fmt_string));
1083 if (*n_warnings == max_warnings)
1084 sys_warn (r, -1, _("Suppressing further invalid format warnings."));
1089 parse_document (struct dictionary *dict, struct sfm_document_record *record)
1093 for (p = record->documents;
1094 p < record->documents + DOC_LINE_LENGTH * record->n_lines;
1095 p += DOC_LINE_LENGTH)
1097 struct substring line;
1099 line = recode_substring_pool ("UTF-8", dict_get_encoding (dict),
1100 ss_buffer (p, DOC_LINE_LENGTH), NULL);
1101 ss_rtrim (&line, ss_cstr (" "));
1102 line.string[line.length] = '\0';
1104 dict_add_document_line (dict, line.string, false);
1110 /* Parses record type 7, subtype 3. */
1112 parse_machine_integer_info (struct sfm_reader *r,
1113 const struct sfm_extension_record *record,
1114 struct sfm_read_info *info)
1116 int float_representation, expected_float_format;
1117 int integer_representation, expected_integer_format;
1119 /* Save version info. */
1120 info->version_major = parse_int (r, record->data, 0);
1121 info->version_minor = parse_int (r, record->data, 4);
1122 info->version_revision = parse_int (r, record->data, 8);
1124 /* Check floating point format. */
1125 float_representation = parse_int (r, record->data, 16);
1126 if (r->float_format == FLOAT_IEEE_DOUBLE_BE
1127 || r->float_format == FLOAT_IEEE_DOUBLE_LE)
1128 expected_float_format = 1;
1129 else if (r->float_format == FLOAT_Z_LONG)
1130 expected_float_format = 2;
1131 else if (r->float_format == FLOAT_VAX_G || r->float_format == FLOAT_VAX_D)
1132 expected_float_format = 3;
1135 if (float_representation != expected_float_format)
1136 sys_error (r, record->pos, _("Floating-point representation indicated by "
1137 "system file (%d) differs from expected (%d)."),
1138 float_representation, expected_float_format);
1140 /* Check integer format. */
1141 integer_representation = parse_int (r, record->data, 24);
1142 if (r->integer_format == INTEGER_MSB_FIRST)
1143 expected_integer_format = 1;
1144 else if (r->integer_format == INTEGER_LSB_FIRST)
1145 expected_integer_format = 2;
1148 if (integer_representation != expected_integer_format)
1149 sys_warn (r, record->pos,
1150 _("Integer format indicated by system file (%d) "
1151 "differs from expected (%d)."),
1152 integer_representation, expected_integer_format);
1157 choose_encoding (struct sfm_reader *r,
1158 const struct sfm_extension_record *ext_integer,
1159 const struct sfm_extension_record *ext_encoding)
1161 /* The EXT_ENCODING record is a more reliable way to determine dictionary
1164 return ext_encoding->data;
1166 /* But EXT_INTEGER is better than nothing as a fallback. */
1169 int codepage = parse_int (r, ext_integer->data, 7 * 4);
1170 const char *encoding;
1179 /* These ostensibly mean "7-bit ASCII" and "8-bit ASCII"[sic]
1180 respectively. However, there are known to be many files in the wild
1181 with character code 2, yet have data which are clearly not ASCII.
1182 Therefore we ignore these values. */
1189 encoding = sys_get_encoding_from_codepage (codepage);
1190 if (encoding != NULL)
1196 return locale_charset ();
1199 /* Parses record type 7, subtype 4. */
1201 parse_machine_float_info (struct sfm_reader *r,
1202 const struct sfm_extension_record *record)
1204 double sysmis = parse_float (r, record->data, 0);
1205 double highest = parse_float (r, record->data, 8);
1206 double lowest = parse_float (r, record->data, 16);
1208 if (sysmis != SYSMIS)
1209 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1212 if (highest != HIGHEST)
1213 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1214 highest, "HIGHEST");
1216 if (lowest != LOWEST)
1217 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1221 /* Parses record type 7, subtype 7 or 19. */
1223 parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record,
1224 struct dictionary *dict)
1226 struct text_record *text;
1227 struct mrset *mrset;
1229 text = open_text_record (r, record, false);
1232 const char *counted = NULL;
1235 struct stringi_set var_names;
1236 size_t allocated_vars;
1240 mrset = xzalloc (sizeof *mrset);
1242 name = text_get_token (text, ss_cstr ("="), NULL);
1245 mrset->name = recode_string ("UTF-8", r->encoding, name, -1);
1247 if (mrset->name[0] != '$')
1249 sys_warn (r, record->pos,
1250 _("`%s' does not begin with `$' at offset %zu "
1251 "in MRSETS record."), mrset->name, text_pos (text));
1255 if (text_match (text, 'C'))
1257 mrset->type = MRSET_MC;
1258 if (!text_match (text, ' '))
1260 sys_warn (r, record->pos,
1261 _("Missing space following `%c' at offset %zu "
1262 "in MRSETS record."), 'C', text_pos (text));
1266 else if (text_match (text, 'D'))
1268 mrset->type = MRSET_MD;
1269 mrset->cat_source = MRSET_VARLABELS;
1271 else if (text_match (text, 'E'))
1275 mrset->type = MRSET_MD;
1276 mrset->cat_source = MRSET_COUNTEDVALUES;
1277 if (!text_match (text, ' '))
1279 sys_warn (r, record->pos,
1280 _("Missing space following `%c' at offset %zu "
1281 "in MRSETS record."), 'E', text_pos (text));
1285 number = text_get_token (text, ss_cstr (" "), NULL);
1286 if (!strcmp (number, "11"))
1287 mrset->label_from_var_label = true;
1288 else if (strcmp (number, "1"))
1289 sys_warn (r, record->pos,
1290 _("Unexpected label source value `%s' following `E' "
1291 "at offset %zu in MRSETS record."),
1292 number, text_pos (text));
1296 sys_warn (r, record->pos,
1297 _("Missing `C', `D', or `E' at offset %zu "
1298 "in MRSETS record."),
1303 if (mrset->type == MRSET_MD)
1305 counted = text_parse_counted_string (r, text);
1306 if (counted == NULL)
1310 label = text_parse_counted_string (r, text);
1313 if (label[0] != '\0')
1314 mrset->label = recode_string ("UTF-8", r->encoding, label, -1);
1316 stringi_set_init (&var_names);
1321 const char *raw_var_name;
1322 struct variable *var;
1325 raw_var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
1326 if (raw_var_name == NULL)
1328 sys_warn (r, record->pos,
1329 _("Missing new-line parsing variable names "
1330 "at offset %zu in MRSETS record."),
1334 var_name = recode_string ("UTF-8", r->encoding, raw_var_name, -1);
1336 var = dict_lookup_var (dict, var_name);
1342 if (!stringi_set_insert (&var_names, var_name))
1344 sys_warn (r, record->pos,
1345 _("Duplicate variable name %s "
1346 "at offset %zu in MRSETS record."),
1347 var_name, text_pos (text));
1353 if (mrset->label == NULL && mrset->label_from_var_label
1354 && var_has_label (var))
1355 mrset->label = xstrdup (var_get_label (var));
1358 && var_get_type (var) != var_get_type (mrset->vars[0]))
1360 sys_warn (r, record->pos,
1361 _("MRSET %s contains both string and "
1362 "numeric variables."), name);
1365 width = MIN (width, var_get_width (var));
1367 if (mrset->n_vars >= allocated_vars)
1368 mrset->vars = x2nrealloc (mrset->vars, &allocated_vars,
1369 sizeof *mrset->vars);
1370 mrset->vars[mrset->n_vars++] = var;
1372 while (delimiter != '\n');
1374 if (mrset->n_vars < 2)
1376 sys_warn (r, record->pos,
1377 _("MRSET %s has only %zu variables."), mrset->name,
1379 mrset_destroy (mrset);
1383 if (mrset->type == MRSET_MD)
1385 mrset->width = width;
1386 value_init (&mrset->counted, width);
1388 mrset->counted.f = strtod (counted, NULL);
1390 value_copy_str_rpad (&mrset->counted, width,
1391 (const uint8_t *) counted, ' ');
1394 dict_add_mrset (dict, mrset);
1396 stringi_set_destroy (&var_names);
1398 mrset_destroy (mrset);
1399 close_text_record (r, text);
1402 /* Read record type 7, subtype 11, which specifies how variables
1403 should be displayed in GUI environments. */
1405 parse_display_parameters (struct sfm_reader *r,
1406 const struct sfm_extension_record *record,
1407 struct dictionary *dict)
1409 bool includes_width;
1410 bool warned = false;
1415 n_vars = dict_get_var_cnt (dict);
1416 if (record->count == 3 * n_vars)
1417 includes_width = true;
1418 else if (record->count == 2 * n_vars)
1419 includes_width = false;
1422 sys_warn (r, record->pos,
1423 _("Extension 11 has bad count %zu (for %zu variables)."),
1424 record->count, n_vars);
1429 for (i = 0; i < n_vars; ++i)
1431 struct variable *v = dict_get_var (dict, i);
1432 int measure, width, align;
1434 measure = parse_int (r, record->data, ofs);
1439 width = parse_int (r, record->data, ofs);
1445 align = parse_int (r, record->data, ofs);
1448 /* SPSS 14 sometimes seems to set string variables' measure
1450 if (0 == measure && var_is_alpha (v))
1453 if (measure < 1 || measure > 3 || align < 0 || align > 2)
1456 sys_warn (r, record->pos,
1457 _("Invalid variable display parameters for variable "
1458 "%zu (%s). Default parameters substituted."),
1459 i, var_get_name (v));
1464 var_set_measure (v, (measure == 1 ? MEASURE_NOMINAL
1465 : measure == 2 ? MEASURE_ORDINAL
1467 var_set_alignment (v, (align == 0 ? ALIGN_LEFT
1468 : align == 1 ? ALIGN_RIGHT
1471 /* Older versions (SPSS 9.0) sometimes set the display
1472 width to zero. This causes confusion in the GUI, so
1473 only set the width if it is nonzero. */
1475 var_set_display_width (v, width);
1480 rename_var_and_save_short_names (struct dictionary *dict, struct variable *var,
1481 const char *new_name)
1483 size_t n_short_names;
1487 /* Renaming a variable may clear its short names, but we
1488 want to retain them, so we save them and re-set them
1490 n_short_names = var_get_short_name_cnt (var);
1491 short_names = xnmalloc (n_short_names, sizeof *short_names);
1492 for (i = 0; i < n_short_names; i++)
1494 const char *s = var_get_short_name (var, i);
1495 short_names[i] = s != NULL ? xstrdup (s) : NULL;
1498 /* Set long name. */
1499 dict_rename_var (dict, var, new_name);
1501 /* Restore short names. */
1502 for (i = 0; i < n_short_names; i++)
1504 var_set_short_name (var, i, short_names[i]);
1505 free (short_names[i]);
1510 /* Parses record type 7, subtype 13, which gives the long name that corresponds
1511 to each short name. Modifies variable names in DICT accordingly. */
1513 parse_long_var_name_map (struct sfm_reader *r,
1514 const struct sfm_extension_record *record,
1515 struct dictionary *dict)
1517 struct text_record *text;
1518 struct variable *var;
1523 /* Convert variable names to lowercase. */
1526 for (i = 0; i < dict_get_var_cnt (dict); i++)
1528 struct variable *var = dict_get_var (dict, i);
1531 new_name = xstrdup (var_get_name (var));
1532 str_lowercase (new_name);
1534 rename_var_and_save_short_names (dict, var, new_name);
1542 /* Rename each of the variables, one by one. (In a correctly constructed
1543 system file, this cannot create any intermediate duplicate variable names,
1544 because all of the new variable names are longer than any of the old
1545 variable names and thus there cannot be any overlaps.) */
1546 text = open_text_record (r, record, true);
1547 while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
1549 /* Validate long name. */
1550 /* XXX need to reencode name to UTF-8 */
1551 if (!dict_id_is_valid (dict, long_name, false))
1553 sys_warn (r, record->pos,
1554 _("Long variable mapping from %s to invalid "
1555 "variable name `%s'."),
1556 var_get_name (var), long_name);
1560 /* Identify any duplicates. */
1561 if (strcasecmp (var_get_short_name (var, 0), long_name)
1562 && dict_lookup_var (dict, long_name) != NULL)
1564 sys_warn (r, record->pos,
1565 _("Duplicate long variable name `%s'."), long_name);
1569 rename_var_and_save_short_names (dict, var, long_name);
1571 close_text_record (r, text);
1574 /* Reads record type 7, subtype 14, which gives the real length
1575 of each very long string. Rearranges DICT accordingly. */
1577 parse_long_string_map (struct sfm_reader *r,
1578 const struct sfm_extension_record *record,
1579 struct dictionary *dict)
1581 struct text_record *text;
1582 struct variable *var;
1585 text = open_text_record (r, record, true);
1586 while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
1588 size_t idx = var_get_dict_index (var);
1594 length = strtol (length_s, NULL, 10);
1595 if (length < 1 || length > MAX_STRING)
1597 sys_warn (r, record->pos,
1598 _("%s listed as string of invalid length %s "
1599 "in very long string record."),
1600 var_get_name (var), length_s);
1604 /* Check segments. */
1605 segment_cnt = sfm_width_to_segments (length);
1606 if (segment_cnt == 1)
1608 sys_warn (r, record->pos,
1609 _("%s listed in very long string record with width %s, "
1610 "which requires only one segment."),
1611 var_get_name (var), length_s);
1614 if (idx + segment_cnt > dict_get_var_cnt (dict))
1615 sys_error (r, record->pos,
1616 _("Very long string %s overflows dictionary."),
1617 var_get_name (var));
1619 /* Get the short names from the segments and check their
1621 for (i = 0; i < segment_cnt; i++)
1623 struct variable *seg = dict_get_var (dict, idx + i);
1624 int alloc_width = sfm_segment_alloc_width (length, i);
1625 int width = var_get_width (seg);
1628 var_set_short_name (var, i, var_get_short_name (seg, 0));
1629 if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
1630 sys_error (r, record->pos,
1631 _("Very long string with width %ld has segment %d "
1632 "of width %d (expected %d)."),
1633 length, i, width, alloc_width);
1635 dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
1636 var_set_width (var, length);
1638 close_text_record (r, text);
1639 dict_compact_values (dict);
1643 parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
1644 const struct sfm_var_record *var_recs, size_t n_var_recs,
1645 const struct sfm_value_label_record *record)
1647 struct variable **vars;
1651 utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels);
1652 for (i = 0; i < record->n_labels; i++)
1653 utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
1654 record->labels[i].label, -1,
1657 vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
1658 for (i = 0; i < record->n_vars; i++)
1659 vars[i] = lookup_var_by_index (r, record->pos,
1660 var_recs, n_var_recs, record->vars[i]);
1662 for (i = 1; i < record->n_vars; i++)
1663 if (var_get_type (vars[i]) != var_get_type (vars[0]))
1664 sys_error (r, record->pos,
1665 _("Variables associated with value label are not all of "
1666 "identical type. Variable %s is %s, but variable "
1668 var_get_name (vars[0]),
1669 var_is_numeric (vars[0]) ? _("numeric") : _("string"),
1670 var_get_name (vars[i]),
1671 var_is_numeric (vars[i]) ? _("numeric") : _("string"));
1673 for (i = 0; i < record->n_vars; i++)
1675 struct variable *var = vars[i];
1679 width = var_get_width (var);
1681 sys_error (r, record->pos,
1682 _("Value labels may not be added to long string "
1683 "variables (e.g. %s) using records types 3 and 4."),
1684 var_get_name (var));
1686 for (j = 0; j < record->n_labels; j++)
1688 struct sfm_value_label *label = &record->labels[j];
1691 value_init (&value, width);
1693 value.f = parse_float (r, label->value, 0);
1695 memcpy (value_str_rw (&value, width), label->value, width);
1697 if (!var_add_value_label (var, &value, utf8_labels[j]))
1699 if (var_is_numeric (var))
1700 sys_warn (r, record->pos,
1701 _("Duplicate value label for %g on %s."),
1702 value.f, var_get_name (var));
1704 sys_warn (r, record->pos,
1705 _("Duplicate value label for `%.*s' on %s."),
1706 width, value_str (&value, width),
1707 var_get_name (var));
1710 value_destroy (&value, width);
1714 pool_free (r->pool, vars);
1715 for (i = 0; i < record->n_labels; i++)
1716 pool_free (r->pool, utf8_labels[i]);
1717 pool_free (r->pool, utf8_labels);
1720 static struct variable *
1721 lookup_var_by_index (struct sfm_reader *r, off_t offset,
1722 const struct sfm_var_record *var_recs, size_t n_var_recs,
1725 const struct sfm_var_record *rec;
1727 if (idx < 1 || idx > n_var_recs)
1729 sys_error (r, offset,
1730 _("Variable index %d not in valid range 1...%zu."),
1735 rec = &var_recs[idx - 1];
1736 if (rec->var == NULL)
1738 sys_error (r, offset,
1739 _("Variable index %d refers to long string continuation."),
1747 /* Parses a set of custom attributes from TEXT into ATTRS.
1748 ATTRS may be a null pointer, in which case the attributes are
1749 read but discarded. */
1751 parse_attributes (struct sfm_reader *r, struct text_record *text,
1752 struct attrset *attrs)
1756 struct attribute *attr;
1760 /* Parse the key. */
1761 key = text_get_token (text, ss_cstr ("("), NULL);
1765 attr = attribute_create (key);
1766 for (index = 1; ; index++)
1768 /* Parse the value. */
1772 value = text_get_token (text, ss_cstr ("\n"), NULL);
1775 text_warn (r, text, _("Error parsing attribute value %s[%d]."),
1780 length = strlen (value);
1781 if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
1783 value[length - 1] = '\0';
1784 attribute_add_value (attr, value + 1);
1789 _("Attribute value %s[%d] is not quoted: %s."),
1791 attribute_add_value (attr, value);
1794 /* Was this the last value for this attribute? */
1795 if (text_match (text, ')'))
1799 attrset_add (attrs, attr);
1801 attribute_destroy (attr);
1803 while (!text_match (text, '/'));
1806 /* Reads record type 7, subtype 17, which lists custom
1807 attributes on the data file. */
1809 parse_data_file_attributes (struct sfm_reader *r,
1810 const struct sfm_extension_record *record,
1811 struct dictionary *dict)
1813 struct text_record *text = open_text_record (r, record, true);
1814 parse_attributes (r, text, dict_get_attributes (dict));
1815 close_text_record (r, text);
1818 /* Parses record type 7, subtype 18, which lists custom
1819 attributes on individual variables. */
1821 parse_variable_attributes (struct sfm_reader *r,
1822 const struct sfm_extension_record *record,
1823 struct dictionary *dict)
1825 struct text_record *text;
1826 struct variable *var;
1828 text = open_text_record (r, record, true);
1829 while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
1830 parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
1831 close_text_record (r, text);
1835 check_overflow (struct sfm_reader *r,
1836 const struct sfm_extension_record *record,
1837 size_t ofs, size_t length)
1839 size_t end = record->size * record->count;
1840 if (length >= end || ofs + length > end)
1841 sys_error (r, record->pos + end,
1842 _("Long string value label record ends unexpectedly."));
1846 parse_long_string_value_labels (struct sfm_reader *r,
1847 const struct sfm_extension_record *record,
1848 struct dictionary *dict)
1850 const char *dict_encoding = dict_get_encoding (dict);
1851 size_t end = record->size * record->count;
1858 struct variable *var;
1863 /* Parse variable name length. */
1864 check_overflow (r, record, ofs, 4);
1865 var_name_len = parse_int (r, record->data, ofs);
1868 /* Parse variable name, width, and number of labels. */
1869 check_overflow (r, record, ofs, var_name_len + 8);
1870 var_name = recode_string_pool ("UTF-8", dict_encoding,
1871 (const char *) record->data + ofs,
1872 var_name_len, r->pool);
1873 width = parse_int (r, record->data, ofs + var_name_len);
1874 n_labels = parse_int (r, record->data, ofs + var_name_len + 4);
1875 ofs += var_name_len + 8;
1877 /* Look up 'var' and validate. */
1878 var = dict_lookup_var (dict, var_name);
1880 sys_warn (r, record->pos + ofs,
1881 _("Ignoring long string value record for "
1882 "unknown variable %s."), var_name);
1883 else if (var_is_numeric (var))
1885 sys_warn (r, record->pos + ofs,
1886 _("Ignoring long string value record for "
1887 "numeric variable %s."), var_name);
1890 else if (width != var_get_width (var))
1892 sys_warn (r, record->pos + ofs,
1893 _("Ignoring long string value record for variable %s "
1894 "because the record's width (%d) does not match the "
1895 "variable's width (%d)."),
1896 var_name, width, var_get_width (var));
1901 value_init_pool (r->pool, &value, width);
1902 for (i = 0; i < n_labels; i++)
1904 size_t value_length, label_length;
1905 bool skip = var == NULL;
1907 /* Parse value length. */
1908 check_overflow (r, record, ofs, 4);
1909 value_length = parse_int (r, record->data, ofs);
1913 check_overflow (r, record, ofs, value_length);
1916 if (value_length == width)
1917 memcpy (value_str_rw (&value, width),
1918 (const uint8_t *) record->data + ofs, width);
1921 sys_warn (r, record->pos + ofs,
1922 _("Ignoring long string value %zu for variable "
1923 "%s, with width %d, that has bad value "
1925 i, var_get_name (var), width, value_length);
1929 ofs += value_length;
1931 /* Parse label length. */
1932 check_overflow (r, record, ofs, 4);
1933 label_length = parse_int (r, record->data, ofs);
1937 check_overflow (r, record, ofs, label_length);
1942 label = recode_string_pool ("UTF-8", dict_encoding,
1943 (const char *) record->data + ofs,
1944 label_length, r->pool);
1945 if (!var_add_value_label (var, &value, label))
1946 sys_warn (r, record->pos + ofs,
1947 _("Duplicate value label for `%.*s' on %s."),
1948 width, value_str (&value, width),
1949 var_get_name (var));
1950 pool_free (r->pool, label);
1952 ofs += label_length;
1959 static void partial_record (struct sfm_reader *r)
1962 static void read_error (struct casereader *, const struct sfm_reader *);
1964 static bool read_case_number (struct sfm_reader *, double *);
1965 static bool read_case_string (struct sfm_reader *, uint8_t *, size_t);
1966 static int read_opcode (struct sfm_reader *);
1967 static bool read_compressed_number (struct sfm_reader *, double *);
1968 static bool read_compressed_string (struct sfm_reader *, uint8_t *);
1969 static bool read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
1970 static bool skip_whole_strings (struct sfm_reader *, size_t);
1972 /* Reads and returns one case from READER's file. Returns a null
1973 pointer if not successful. */
1974 static struct ccase *
1975 sys_file_casereader_read (struct casereader *reader, void *r_)
1977 struct sfm_reader *r = r_;
1978 struct ccase *volatile c;
1984 c = case_create (r->proto);
1985 if (setjmp (r->bail_out))
1987 casereader_force_error (reader);
1992 for (i = 0; i < r->sfm_var_cnt; i++)
1994 struct sfm_var *sv = &r->sfm_vars[i];
1995 union value *v = case_data_rw_idx (c, sv->case_index);
1997 if (sv->var_width == 0)
1999 if (!read_case_number (r, &v->f))
2004 uint8_t *s = value_str_rw (v, sv->var_width);
2005 if (!read_case_string (r, s + sv->offset, sv->segment_width))
2007 if (!skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)))
2016 if (r->case_cnt != -1)
2017 read_error (reader, r);
2022 /* Issues an error that R ends in a partial record. */
2024 partial_record (struct sfm_reader *r)
2026 sys_error (r, r->pos, _("File ends in partial case."));
2029 /* Issues an error that an unspecified error occurred SFM, and
2032 read_error (struct casereader *r, const struct sfm_reader *sfm)
2034 msg (ME, _("Error reading case from file %s."), fh_get_name (sfm->fh));
2035 casereader_force_error (r);
2038 /* Reads a number from R and stores its value in *D.
2039 If R is compressed, reads a compressed number;
2040 otherwise, reads a number in the regular way.
2041 Returns true if successful, false if end of file is
2042 reached immediately. */
2044 read_case_number (struct sfm_reader *r, double *d)
2049 if (!try_read_bytes (r, number, sizeof number))
2051 float_convert (r->float_format, number, FLOAT_NATIVE_DOUBLE, d);
2055 return read_compressed_number (r, d);
2058 /* Reads LENGTH string bytes from R into S.
2059 Always reads a multiple of 8 bytes; if LENGTH is not a
2060 multiple of 8, then extra bytes are read and discarded without
2062 Reads compressed strings if S is compressed.
2063 Returns true if successful, false if end of file is
2064 reached immediately. */
2066 read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
2068 size_t whole = ROUND_DOWN (length, 8);
2069 size_t partial = length % 8;
2073 if (!read_whole_strings (r, s, whole))
2080 if (!read_whole_strings (r, bounce, sizeof bounce))
2086 memcpy (s + whole, bounce, partial);
2092 /* Reads and returns the next compression opcode from R. */
2094 read_opcode (struct sfm_reader *r)
2096 assert (r->compressed);
2100 if (r->opcode_idx >= sizeof r->opcodes)
2102 if (!try_read_bytes (r, r->opcodes, sizeof r->opcodes))
2106 opcode = r->opcodes[r->opcode_idx++];
2113 /* Reads a compressed number from R and stores its value in D.
2114 Returns true if successful, false if end of file is
2115 reached immediately. */
2117 read_compressed_number (struct sfm_reader *r, double *d)
2119 int opcode = read_opcode (r);
2127 *d = read_float (r);
2131 float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
2132 if (!r->corruption_warning)
2134 r->corruption_warning = true;
2135 sys_warn (r, r->pos,
2136 _("Possible compressed data corruption: "
2137 "compressed spaces appear in numeric field."));
2146 *d = opcode - r->bias;
2153 /* Reads a compressed 8-byte string segment from R and stores it
2155 Returns true if successful, false if end of file is
2156 reached immediately. */
2158 read_compressed_string (struct sfm_reader *r, uint8_t *dst)
2160 int opcode = read_opcode (r);
2168 read_bytes (r, dst, 8);
2172 memset (dst, ' ', 8);
2177 double value = opcode - r->bias;
2178 float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
2181 /* This has actually been seen "in the wild". The submitter of the
2182 file that showed that the contents decoded as spaces, but they
2183 were at the end of the field so it's possible that the null
2184 bytes just acted as null terminators. */
2186 else if (!r->corruption_warning)
2188 r->corruption_warning = true;
2189 sys_warn (r, r->pos,
2190 _("Possible compressed data corruption: "
2191 "string contains compressed integer (opcode %d)."),
2201 /* Reads LENGTH string bytes from R into S.
2202 LENGTH must be a multiple of 8.
2203 Reads compressed strings if S is compressed.
2204 Returns true if successful, false if end of file is
2205 reached immediately. */
2207 read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
2209 assert (length % 8 == 0);
2211 return try_read_bytes (r, s, length);
2215 for (ofs = 0; ofs < length; ofs += 8)
2216 if (!read_compressed_string (r, s + ofs))
2226 /* Skips LENGTH string bytes from R.
2227 LENGTH must be a multiple of 8.
2228 (LENGTH is also limited to 1024, but that's only because the
2229 current caller never needs more than that many bytes.)
2230 Returns true if successful, false if end of file is
2231 reached immediately. */
2233 skip_whole_strings (struct sfm_reader *r, size_t length)
2235 uint8_t buffer[1024];
2236 assert (length < sizeof buffer);
2237 return read_whole_strings (r, buffer, length);
2240 /* Helpers for reading records that contain structured text
2243 /* Maximum number of warnings to issue for a single text
2245 #define MAX_TEXT_WARNINGS 5
2250 struct substring buffer; /* Record contents. */
2251 off_t start; /* Starting offset in file. */
2252 size_t pos; /* Current position in buffer. */
2253 int n_warnings; /* Number of warnings issued or suppressed. */
2254 bool recoded; /* Recoded into UTF-8? */
2257 static struct text_record *
2258 open_text_record (struct sfm_reader *r,
2259 const struct sfm_extension_record *record,
2260 bool recode_to_utf8)
2262 struct text_record *text;
2263 struct substring raw;
2265 text = pool_alloc (r->pool, sizeof *text);
2266 raw = ss_buffer (record->data, record->size * record->count);
2267 text->start = record->pos;
2268 text->buffer = (recode_to_utf8
2269 ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
2272 text->n_warnings = 0;
2273 text->recoded = recode_to_utf8;
2278 /* Closes TEXT, frees its storage, and issues a final warning
2279 about suppressed warnings if necesary. */
2281 close_text_record (struct sfm_reader *r, struct text_record *text)
2283 if (text->n_warnings > MAX_TEXT_WARNINGS)
2284 sys_warn (r, -1, _("Suppressed %d additional related warnings."),
2285 text->n_warnings - MAX_TEXT_WARNINGS);
2287 pool_free (r->pool, ss_data (text->buffer));
2290 /* Reads a variable=value pair from TEXT.
2291 Looks up the variable in DICT and stores it into *VAR.
2292 Stores a null-terminated value into *VALUE. */
2294 read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
2295 struct text_record *text,
2296 struct variable **var, char **value)
2300 if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
2303 *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
2307 text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
2308 ss_buffer ("\t\0", 2));
2316 text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
2317 struct text_record *text, struct substring delimiters,
2318 struct variable **var)
2322 name = text_get_token (text, delimiters, NULL);
2326 *var = dict_lookup_var (dict, name);
2330 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
2337 text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
2338 struct text_record *text, struct substring delimiters,
2339 struct variable **var)
2341 char *short_name = text_get_token (text, delimiters, NULL);
2342 if (short_name == NULL)
2345 *var = dict_lookup_var (dict, short_name);
2347 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
2352 /* Displays a warning for the current file position, limiting the
2353 number to MAX_TEXT_WARNINGS for TEXT. */
2355 text_warn (struct sfm_reader *r, struct text_record *text,
2356 const char *format, ...)
2358 if (text->n_warnings++ < MAX_TEXT_WARNINGS)
2362 va_start (args, format);
2363 sys_msg (r, text->start + text->pos, MW, format, args);
2369 text_get_token (struct text_record *text, struct substring delimiters,
2372 struct substring token;
2375 if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
2378 end = &ss_data (token)[ss_length (token)];
2379 if (delimiter != NULL)
2382 return ss_data (token);
2385 /* Reads a integer value expressed in decimal, then a space, then a string that
2386 consists of exactly as many bytes as specified by the integer, then a space,
2387 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
2388 buffer (so the caller should not free the string). */
2390 text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
2400 int c = text->buffer.string[text->pos];
2401 if (c < '0' || c > '9')
2403 n = (n * 10) + (c - '0');
2406 if (start == text->pos)
2408 sys_warn (r, text->start,
2409 _("Expecting digit at offset %zu in MRSETS record."),
2414 if (!text_match (text, ' '))
2416 sys_warn (r, text->start,
2417 _("Expecting space at offset %zu in MRSETS record."),
2422 if (text->pos + n > text->buffer.length)
2424 sys_warn (r, text->start,
2425 _("%zu-byte string starting at offset %zu "
2426 "exceeds record length %zu."),
2427 n, text->pos, text->buffer.length);
2431 s = &text->buffer.string[text->pos];
2434 sys_warn (r, text->start,
2435 _("Expecting space at offset %zu following %zu-byte string."),
2445 text_match (struct text_record *text, char c)
2447 if (text->buffer.string[text->pos] == c)
2456 /* Returns the current byte offset (as converted to UTF-8, if it was converted)
2457 inside the TEXT's string. */
2459 text_pos (const struct text_record *text)
2466 /* Displays a corruption message. */
2468 sys_msg (struct sfm_reader *r, off_t offset,
2469 int class, const char *format, va_list args)
2474 ds_init_empty (&text);
2476 ds_put_format (&text, _("`%s' near offset 0x%llx: "),
2477 fh_get_file_name (r->fh), (long long int) offset);
2479 ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
2480 ds_put_vformat (&text, format, args);
2482 m.category = msg_class_to_category (class);
2483 m.severity = msg_class_to_severity (class);
2489 m.text = ds_cstr (&text);
2494 /* Displays a warning for offset OFFSET in the file. */
2496 sys_warn (struct sfm_reader *r, off_t offset, const char *format, ...)
2500 va_start (args, format);
2501 sys_msg (r, offset, MW, format, args);
2505 /* Displays an error for the current file position,
2506 marks it as in an error state,
2507 and aborts reading it using longjmp. */
2509 sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
2513 va_start (args, format);
2514 sys_msg (r, offset, ME, format, args);
2518 longjmp (r->bail_out, 1);
2521 /* Reads BYTE_CNT bytes into BUF.
2522 Returns true if exactly BYTE_CNT bytes are successfully read.
2523 Aborts if an I/O error or a partial read occurs.
2524 If EOF_IS_OK, then an immediate end-of-file causes false to be
2525 returned; otherwise, immediate end-of-file causes an abort
2528 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
2529 void *buf, size_t byte_cnt)
2531 size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
2532 r->pos += bytes_read;
2533 if (bytes_read == byte_cnt)
2535 else if (ferror (r->file))
2536 sys_error (r, r->pos, _("System error: %s."), strerror (errno));
2537 else if (!eof_is_ok || bytes_read != 0)
2538 sys_error (r, r->pos, _("Unexpected end of file."));
2543 /* Reads BYTE_CNT into BUF.
2544 Aborts upon I/O error or if end-of-file is encountered. */
2546 read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
2548 read_bytes_internal (r, false, buf, byte_cnt);
2551 /* Reads BYTE_CNT bytes into BUF.
2552 Returns true if exactly BYTE_CNT bytes are successfully read.
2553 Returns false if an immediate end-of-file is encountered.
2554 Aborts if an I/O error or a partial read occurs. */
2556 try_read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
2558 return read_bytes_internal (r, true, buf, byte_cnt);
2561 /* Reads a 32-bit signed integer from R and returns its value in
2564 read_int (struct sfm_reader *r)
2567 read_bytes (r, integer, sizeof integer);
2568 return integer_get (r->integer_format, integer, sizeof integer);
2571 /* Reads a 64-bit floating-point number from R and returns its
2572 value in host format. */
2574 read_float (struct sfm_reader *r)
2577 read_bytes (r, number, sizeof number);
2578 return float_get_double (r->float_format, number);
2582 parse_int (struct sfm_reader *r, const void *data, size_t ofs)
2584 return integer_get (r->integer_format, (const uint8_t *) data + ofs, 4);
2588 parse_float (struct sfm_reader *r, const void *data, size_t ofs)
2590 return float_get_double (r->float_format, (const uint8_t *) data + ofs);
2593 /* Reads exactly SIZE - 1 bytes into BUFFER
2594 and stores a null byte into BUFFER[SIZE - 1]. */
2596 read_string (struct sfm_reader *r, char *buffer, size_t size)
2599 read_bytes (r, buffer, size - 1);
2600 buffer[size - 1] = '\0';
2603 /* Skips BYTES bytes forward in R. */
2605 skip_bytes (struct sfm_reader *r, size_t bytes)
2610 size_t chunk = MIN (sizeof buffer, bytes);
2611 read_bytes (r, buffer, chunk);
2616 static const struct casereader_class sys_file_casereader_class =
2618 sys_file_casereader_read,
2619 sys_file_casereader_destroy,