1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2000, 2006-2007, 2009-2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/sys-file-reader.h"
20 #include "data/sys-file-private.h"
28 #include "data/attributes.h"
29 #include "data/case.h"
30 #include "data/casereader-provider.h"
31 #include "data/casereader.h"
32 #include "data/dictionary.h"
33 #include "data/file-handle-def.h"
34 #include "data/file-name.h"
35 #include "data/format.h"
36 #include "data/identifier.h"
37 #include "data/missing-values.h"
38 #include "data/mrset.h"
39 #include "data/short-names.h"
40 #include "data/value-labels.h"
41 #include "data/value.h"
42 #include "data/variable.h"
43 #include "libpspp/array.h"
44 #include "libpspp/assertion.h"
45 #include "libpspp/compiler.h"
46 #include "libpspp/i18n.h"
47 #include "libpspp/message.h"
48 #include "libpspp/misc.h"
49 #include "libpspp/pool.h"
50 #include "libpspp/str.h"
51 #include "libpspp/stringi-set.h"
53 #include "gl/c-ctype.h"
54 #include "gl/inttostr.h"
55 #include "gl/localcharset.h"
56 #include "gl/minmax.h"
57 #include "gl/unlocked-io.h"
58 #include "gl/xalloc.h"
62 #define _(msgid) gettext (msgid)
63 #define N_(msgid) (msgid)
67 /* subtypes 0-2 unknown */
68 EXT_INTEGER = 3, /* Machine integer info. */
69 EXT_FLOAT = 4, /* Machine floating-point info. */
70 EXT_VAR_SETS = 5, /* Variable sets. */
71 EXT_DATE = 6, /* DATE. */
72 EXT_MRSETS = 7, /* Multiple response sets. */
73 EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
74 /* subtypes 9-10 unknown */
75 EXT_DISPLAY = 11, /* Variable display parameters. */
76 /* subtype 12 unknown */
77 EXT_LONG_NAMES = 13, /* Long variable names. */
78 EXT_LONG_STRINGS = 14, /* Long strings. */
79 /* subtype 15 unknown */
80 EXT_NCASES = 16, /* Extended number of cases. */
81 EXT_FILE_ATTRS = 17, /* Data file attributes. */
82 EXT_VAR_ATTRS = 18, /* Variable attributes. */
83 EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
84 EXT_ENCODING = 20, /* Character encoding. */
85 EXT_LONG_LABELS = 21 /* Value labels for long strings. */
95 int missing_value_code;
101 struct sfm_value_label
107 struct sfm_value_label_record
110 struct sfm_value_label *labels;
117 struct sfm_document_record
124 struct sfm_extension_record
126 off_t pos; /* Starting offset in file. */
127 size_t size; /* Size of data elements. */
128 size_t count; /* Number of data elements. */
129 void *data; /* Contents. */
132 /* System file reader. */
135 /* Resource tracking. */
136 struct pool *pool; /* All system file state. */
137 jmp_buf bail_out; /* longjmp() target for error handling. */
140 struct file_handle *fh; /* File handle. */
141 struct fh_lock *lock; /* Mutual exclusion for file handle. */
142 FILE *file; /* File stream. */
143 off_t pos; /* Position in file. */
144 bool error; /* I/O or corruption error? */
145 struct caseproto *proto; /* Format of output cases. */
148 enum integer_format integer_format; /* On-disk integer format. */
149 enum float_format float_format; /* On-disk floating point format. */
150 struct sfm_var *sfm_vars; /* Variables. */
151 size_t sfm_var_cnt; /* Number of variables. */
152 casenumber case_cnt; /* Number of cases */
153 const char *encoding; /* String encoding. */
156 bool compressed; /* File is compressed? */
157 double bias; /* Compression bias, usually 100.0. */
158 uint8_t opcodes[8]; /* Current block of opcodes. */
159 size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
160 bool corruption_warning; /* Warned about possible corruption? */
163 static const struct casereader_class sys_file_casereader_class;
165 static bool close_reader (struct sfm_reader *);
167 static struct variable *lookup_var_by_index (struct sfm_reader *, off_t,
168 const struct sfm_var_record *,
171 static void sys_msg (struct sfm_reader *r, off_t, int class,
172 const char *format, va_list args)
173 PRINTF_FORMAT (4, 0);
174 static void sys_warn (struct sfm_reader *, off_t, const char *, ...)
175 PRINTF_FORMAT (3, 4);
176 static void sys_error (struct sfm_reader *, off_t, const char *, ...)
180 static void read_bytes (struct sfm_reader *, void *, size_t);
181 static bool try_read_bytes (struct sfm_reader *, void *, size_t);
182 static int read_int (struct sfm_reader *);
183 static double read_float (struct sfm_reader *);
184 static void read_string (struct sfm_reader *, char *, size_t);
185 static void skip_bytes (struct sfm_reader *, size_t);
187 static int parse_int (struct sfm_reader *, const void *data, size_t ofs);
188 static double parse_float (struct sfm_reader *, const void *data, size_t ofs);
190 static void read_variable_record (struct sfm_reader *,
191 struct sfm_var_record *);
192 static void read_value_label_record (struct sfm_reader *,
193 struct sfm_value_label_record *,
195 static struct sfm_document_record *read_document_record (struct sfm_reader *);
196 static struct sfm_extension_record *read_extension_record (
197 struct sfm_reader *, int subtype);
198 static void skip_extension_record (struct sfm_reader *, int subtype);
200 static const char *choose_encoding (
202 const struct sfm_extension_record *ext_integer,
203 const struct sfm_extension_record *ext_encoding);
205 static struct text_record *open_text_record (
206 struct sfm_reader *, const struct sfm_extension_record *);
207 static void close_text_record (struct sfm_reader *,
208 struct text_record *);
209 static bool read_variable_to_value_pair (struct sfm_reader *,
211 struct text_record *,
212 struct variable **var, char **value);
213 static void text_warn (struct sfm_reader *r, struct text_record *text,
214 const char *format, ...)
215 PRINTF_FORMAT (3, 4);
216 static char *text_get_token (struct text_record *,
217 struct substring delimiters, char *delimiter);
218 static bool text_match (struct text_record *, char c);
219 static bool text_read_variable_name (struct sfm_reader *, struct dictionary *,
220 struct text_record *,
221 struct substring delimiters,
223 static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
224 struct text_record *,
225 struct substring delimiters,
227 static const char *text_parse_counted_string (struct sfm_reader *,
228 struct text_record *);
229 static size_t text_pos (const struct text_record *);
231 static bool close_reader (struct sfm_reader *r);
233 /* Dictionary reader. */
241 static void read_header (struct sfm_reader *, int *weight_idx,
242 int *claimed_oct_cnt, struct sfm_read_info *,
244 static void parse_file_label (struct sfm_reader *, const char *file_label,
245 struct dictionary *);
246 static void parse_variable_records (struct sfm_reader *, struct dictionary *,
247 struct sfm_var_record *, size_t n);
248 static void parse_format_spec (struct sfm_reader *, off_t pos,
249 unsigned int format, enum which_format,
250 struct variable *, int *format_warning_cnt);
251 static void parse_document (struct dictionary *, struct sfm_document_record *);
252 static void parse_display_parameters (struct sfm_reader *,
253 const struct sfm_extension_record *,
254 struct dictionary *);
255 static void parse_machine_integer_info (struct sfm_reader *,
256 const struct sfm_extension_record *,
257 struct sfm_read_info *);
258 static void parse_machine_float_info (struct sfm_reader *,
259 const struct sfm_extension_record *);
260 static void parse_mrsets (struct sfm_reader *,
261 const struct sfm_extension_record *,
262 struct dictionary *);
263 static void parse_long_var_name_map (struct sfm_reader *,
264 const struct sfm_extension_record *,
265 struct dictionary *);
266 static void parse_long_string_map (struct sfm_reader *,
267 const struct sfm_extension_record *,
268 struct dictionary *);
269 static void parse_value_labels (struct sfm_reader *, struct dictionary *,
270 const struct sfm_var_record *,
272 const struct sfm_value_label_record *);
273 static void parse_data_file_attributes (struct sfm_reader *,
274 const struct sfm_extension_record *,
275 struct dictionary *);
276 static void parse_variable_attributes (struct sfm_reader *,
277 const struct sfm_extension_record *,
278 struct dictionary *);
279 static void parse_long_string_value_labels (struct sfm_reader *,
280 const struct sfm_extension_record *,
281 struct dictionary *);
283 /* Opens the system file designated by file handle FH for
284 reading. Reads the system file's dictionary into *DICT.
285 If INFO is non-null, then it receives additional info about the
288 sfm_open_reader (struct file_handle *fh, struct dictionary **dictp,
289 struct sfm_read_info *volatile info)
291 struct sfm_reader *volatile r = NULL;
292 struct sfm_read_info local_info;
294 struct sfm_var_record *vars;
295 size_t n_vars, allocated_vars;
297 struct sfm_value_label_record *labels;
298 size_t n_labels, allocated_labels;
300 struct sfm_document_record *document;
302 struct sfm_extension_record *extensions[32];
308 struct dictionary *dict = NULL;
311 /* Create and initialize reader. */
312 r = pool_create_container (struct sfm_reader, pool);
318 r->opcode_idx = sizeof r->opcodes;
319 r->corruption_warning = false;
321 /* TRANSLATORS: this fragment will be interpolated into
322 messages in fh_lock() that identify types of files. */
323 r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false);
327 r->file = fn_open (fh_get_file_name (fh), "rb");
330 msg (ME, _("Error opening `%s' for reading as a system file: %s."),
331 fh_get_file_name (r->fh), strerror (errno));
335 /* Initialize info. */
338 memset (info, 0, sizeof *info);
340 if (setjmp (r->bail_out))
344 read_header (r, &weight_idx, &claimed_oct_cnt, info, &file_label);
347 n_vars = allocated_vars = 0;
350 n_labels = allocated_labels = 0;
354 memset (extensions, 0, sizeof extensions);
364 read_int (r); /* Skip filler. */
371 if (n_vars >= allocated_vars)
372 vars = pool_2nrealloc (r->pool, vars, &allocated_vars,
374 read_variable_record (r, &vars[n_vars++]);
378 if (n_labels >= allocated_labels)
379 labels = pool_2nrealloc (r->pool, labels, &allocated_labels,
381 read_value_label_record (r, &labels[n_labels++], n_vars);
385 /* A Type 4 record is always immediately after a type 3 record,
386 so the code for type 3 records reads the type 4 record too. */
387 sys_error (r, r->pos, _("Misplaced type 4 record."));
390 if (document != NULL)
391 sys_error (r, r->pos, _("Duplicate type 6 (document) record."));
392 document = read_document_record (r);
396 subtype = read_int (r);
397 if (subtype < 0 || subtype >= sizeof extensions / sizeof *extensions)
400 _("Unrecognized record type 7, subtype %d. Please "
401 "send a copy of this file, and the syntax which "
402 "created it to %s."),
403 subtype, PACKAGE_BUGREPORT);
404 skip_extension_record (r, subtype);
406 else if (extensions[subtype] != NULL)
409 _("Record type 7, subtype %d found here has the same "
410 "type as the record found near offset 0x%llx. "
411 "Please send a copy of this file, and the syntax "
412 "which created it to %s."),
413 subtype, (long long int) extensions[subtype]->pos,
415 skip_extension_record (r, subtype);
418 extensions[subtype] = read_extension_record (r, subtype);
422 sys_error (r, r->pos, _("Unrecognized record type %d."), type);
427 /* Now actually parse what we read.
429 First, figure out the correct character encoding, because this determines
430 how the rest of the header data is to be interpreted. */
431 dict = dict_create ();
432 r->encoding = choose_encoding (r, extensions[EXT_INTEGER],
433 extensions[EXT_ENCODING]);
434 dict_set_encoding (dict, r->encoding);
436 /* These records don't use variables at all. */
437 if (document != NULL)
438 parse_document (dict, document);
440 if (extensions[EXT_INTEGER] != NULL)
441 parse_machine_integer_info (r, extensions[EXT_INTEGER], info);
443 if (extensions[EXT_FLOAT] != NULL)
444 parse_machine_float_info (r, extensions[EXT_FLOAT]);
446 if (extensions[EXT_FILE_ATTRS] != NULL)
447 parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict);
449 parse_file_label (r, file_label, dict);
451 /* Parse the variable records, the basis of almost everything else. */
452 parse_variable_records (r, dict, vars, n_vars);
454 /* Parse value labels and the weight variable immediately after the variable
455 records. These records use indexes into var_recs[], so we must parse them
456 before those indexes become invalidated by very long string variables. */
457 for (i = 0; i < n_labels; i++)
458 parse_value_labels (r, dict, vars, n_vars, &labels[i]);
461 struct variable *weight_var;
463 weight_var = lookup_var_by_index (r, 76, vars, n_vars, weight_idx);
464 if (var_is_numeric (weight_var))
465 dict_set_weight (dict, weight_var);
467 sys_error (r, -1, _("Weighting variable must be numeric "
468 "(not string variable `%s')."),
469 var_get_name (weight_var));
472 if (extensions[EXT_DISPLAY] != NULL)
473 parse_display_parameters (r, extensions[EXT_DISPLAY], dict);
475 /* The following records use short names, so they need to be parsed before
476 parse_long_var_name_map() changes short names to long names. */
477 if (extensions[EXT_MRSETS] != NULL)
478 parse_mrsets (r, extensions[EXT_MRSETS], dict);
480 if (extensions[EXT_MRSETS2] != NULL)
481 parse_mrsets (r, extensions[EXT_MRSETS2], dict);
483 if (extensions[EXT_LONG_STRINGS] != NULL)
484 parse_long_string_map (r, extensions[EXT_LONG_STRINGS], dict);
486 /* Now rename variables to their long names. */
487 parse_long_var_name_map (r, extensions[EXT_LONG_NAMES], dict);
489 /* The following records use long names, so they need to follow renaming. */
490 if (extensions[EXT_VAR_ATTRS] != NULL)
491 parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict);
493 if (extensions[EXT_LONG_LABELS] != NULL)
494 parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict);
496 /* Warn if the actual amount of data per case differs from the
497 amount that the header claims. SPSS version 13 gets this
498 wrong when very long strings are involved, so don't warn in
500 if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars
501 && info->version_major != 13)
502 sys_warn (r, -1, _("File header claims %d variable positions but "
503 "%d were read from file."),
504 claimed_oct_cnt, n_vars);
506 /* Create an index of dictionary variable widths for
507 sfm_read_case to use. We cannot use the `struct variable's
508 from the dictionary we created, because the caller owns the
509 dictionary and may destroy or modify its variables. */
510 sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_var_cnt);
511 pool_register (r->pool, free, r->sfm_vars);
512 r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
515 return casereader_create_sequential
517 r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
518 &sys_file_casereader_class, r);
527 /* Closes a system file after we're done with it.
528 Returns true if an I/O error has occurred on READER, false
531 close_reader (struct sfm_reader *r)
540 if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)
542 msg (ME, _("Error closing system file `%s': %s."),
543 fh_get_file_name (r->fh), strerror (errno));
553 pool_destroy (r->pool);
558 /* Destroys READER. */
560 sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
562 struct sfm_reader *r = r_;
566 /* Returns true if FILE is an SPSS system file,
569 sfm_detect (FILE *file)
573 if (fread (rec_type, 4, 1, file) != 1)
577 return !strcmp ("$FL2", rec_type);
580 /* Reads the global header of the system file. Sets *WEIGHT_IDX to 0 if the
581 system file is unweighted, or to the value index of the weight variable
582 otherwise. Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units) per
583 case that the file claims to have (although it is not always correct).
584 Initializes INFO with header information. Stores the file label as a string
585 in dictionary encoding into *FILE_LABELP. */
587 read_header (struct sfm_reader *r, int *weight_idx,
588 int *claimed_oct_cnt, struct sfm_read_info *info,
592 char eye_catcher[61];
593 uint8_t raw_layout_code[4];
595 char creation_date[10];
596 char creation_time[9];
598 struct substring product;
600 read_string (r, rec_type, sizeof rec_type);
601 read_string (r, eye_catcher, sizeof eye_catcher);
603 if (strcmp ("$FL2", rec_type) != 0)
604 sys_error (r, 0, _("This is not an SPSS system file."));
606 /* Identify integer format. */
607 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
608 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
610 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
612 || (r->integer_format != INTEGER_MSB_FIRST
613 && r->integer_format != INTEGER_LSB_FIRST))
614 sys_error (r, 64, _("This is not an SPSS system file."));
616 *claimed_oct_cnt = read_int (r);
617 if (*claimed_oct_cnt < 0 || *claimed_oct_cnt > INT_MAX / 16)
618 *claimed_oct_cnt = -1;
620 r->compressed = read_int (r) != 0;
622 *weight_idx = read_int (r);
624 r->case_cnt = read_int (r);
625 if ( r->case_cnt > INT_MAX / 2)
628 /* Identify floating-point format and obtain compression bias. */
629 read_bytes (r, raw_bias, sizeof raw_bias);
630 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
632 uint8_t zero_bias[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
634 if (memcmp (raw_bias, zero_bias, 8))
635 sys_warn (r, r->pos - 8,
636 _("Compression bias is not the usual "
637 "value of 100, or system file uses unrecognized "
638 "floating-point format."));
641 /* Some software is known to write all-zeros to this
642 field. Such software also writes floating-point
643 numbers in the format that we expect by default
644 (it seems that all software most likely does, in
645 reality), so don't warn in this case. */
648 if (r->integer_format == INTEGER_MSB_FIRST)
649 r->float_format = FLOAT_IEEE_DOUBLE_BE;
651 r->float_format = FLOAT_IEEE_DOUBLE_LE;
653 float_convert (r->float_format, raw_bias, FLOAT_NATIVE_DOUBLE, &r->bias);
655 read_string (r, creation_date, sizeof creation_date);
656 read_string (r, creation_time, sizeof creation_time);
657 read_string (r, file_label, sizeof file_label);
660 strcpy (info->creation_date, creation_date);
661 strcpy (info->creation_time, creation_time);
662 info->integer_format = r->integer_format;
663 info->float_format = r->float_format;
664 info->compressed = r->compressed;
665 info->case_cnt = r->case_cnt;
667 product = ss_cstr (eye_catcher);
668 ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE"));
669 ss_trim (&product, ss_cstr (" "));
670 str_copy_buf_trunc (info->product, sizeof info->product,
671 ss_data (product), ss_length (product));
673 *file_labelp = pool_strdup0 (r->pool, file_label, sizeof file_label - 1);
676 /* Reads a variable (type 2) record from R into RECORD. */
678 read_variable_record (struct sfm_reader *r, struct sfm_var_record *record)
680 int has_variable_label;
682 memset (record, 0, sizeof *record);
684 record->pos = r->pos;
685 record->width = read_int (r);
686 has_variable_label = read_int (r);
687 record->missing_value_code = read_int (r);
688 record->print_format = read_int (r);
689 record->write_format = read_int (r);
690 read_bytes (r, record->name, sizeof record->name);
692 if (has_variable_label == 1)
694 enum { MAX_LABEL_LEN = 255 };
695 size_t len, read_len;
699 /* Read up to MAX_LABEL_LEN bytes of label. */
700 read_len = MIN (MAX_LABEL_LEN, len);
701 record->label = xmalloc (read_len + 1);
702 read_string (r, record->label, read_len + 1);
704 /* Skip unread label bytes. */
705 skip_bytes (r, len - read_len);
707 /* Skip label padding up to multiple of 4 bytes. */
708 skip_bytes (r, ROUND_UP (len, 4) - len);
710 else if (has_variable_label != 0)
711 sys_error (r, record->pos,
712 _("Variable label indicator field is not 0 or 1."));
714 /* Set missing values. */
715 if (record->missing_value_code != 0)
717 int code = record->missing_value_code;
718 if (record->width == 0)
720 if (code < -3 || code > 3 || code == -1)
721 sys_error (r, record->pos,
722 _("Numeric missing value indicator field is not "
723 "-3, -2, 0, 1, 2, or 3."));
727 if (code < 1 || code > 3)
728 sys_error (r, record->pos,
729 _("String missing value indicator field is not "
733 read_bytes (r, record->missing, 8 * abs (code));
737 /* Reads value labels from R into RECORD. */
739 read_value_label_record (struct sfm_reader *r,
740 struct sfm_value_label_record *record,
745 /* Read type 3 record. */
746 record->pos = r->pos;
747 record->n_labels = read_int (r);
748 if (record->n_labels > SIZE_MAX / sizeof *record->labels)
749 sys_error (r, r->pos - 4, _("Invalid number of labels %zu."),
751 record->labels = pool_nmalloc (r->pool, record->n_labels,
752 sizeof *record->labels);
753 for (i = 0; i < record->n_labels; i++)
755 struct sfm_value_label *label = &record->labels[i];
756 unsigned char label_len;
759 read_bytes (r, label->value, sizeof label->value);
761 /* Read label length. */
762 read_bytes (r, &label_len, sizeof label_len);
763 padded_len = ROUND_UP (label_len + 1, 8);
765 /* Read label, padding. */
766 label->label = pool_malloc (r->pool, padded_len + 1);
767 read_bytes (r, label->label, padded_len - 1);
768 label->label[label_len] = '\0';
771 /* Read record type of type 4 record. */
772 if (read_int (r) != 4)
773 sys_error (r, r->pos - 4,
774 _("Variable index record (type 4) does not immediately "
775 "follow value label record (type 3) as it should."));
777 /* Read number of variables associated with value label from type 4
779 record->n_vars = read_int (r);
780 if (record->n_vars < 1 || record->n_vars > n_vars)
781 sys_error (r, r->pos - 4,
782 _("Number of variables associated with a value label (%d) "
783 "is not between 1 and the number of variables (%zu)."),
784 record->n_vars, n_vars);
785 record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
786 for (i = 0; i < record->n_vars; i++)
787 record->vars[i] = read_int (r);
790 /* Reads a document record from R and returns it. */
791 static struct sfm_document_record *
792 read_document_record (struct sfm_reader *r)
794 struct sfm_document_record *record;
797 record = pool_malloc (r->pool, sizeof *record);
798 record->pos = r->pos;
800 n_lines = read_int (r);
801 if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
802 sys_error (r, record->pos,
803 _("Number of document lines (%d) "
804 "must be greater than 0 and less than %d."),
805 n_lines, INT_MAX / DOC_LINE_LENGTH);
807 record->n_lines = n_lines;
808 record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
809 read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines);
815 read_extension_record_header (struct sfm_reader *r, int subtype,
816 struct sfm_extension_record *record)
818 record->pos = r->pos;
819 record->size = read_int (r);
820 record->count = read_int (r);
822 /* Check that SIZE * COUNT + 1 doesn't overflow. Adding 1
823 allows an extra byte for a null terminator, used by some
824 extension processing routines. */
825 if (record->size != 0
826 && size_overflow_p (xsum (1, xtimes (record->count, record->size))))
827 sys_error (r, record->pos, "Record type 7 subtype %d too large.", subtype);
830 /* Reads an extension record from R into RECORD. */
831 static struct sfm_extension_record *
832 read_extension_record (struct sfm_reader *r, int subtype)
834 struct extension_record_type
841 static const struct extension_record_type types[] =
843 /* Implemented record types. */
844 { EXT_INTEGER, 4, 8 },
846 { EXT_MRSETS, 1, 0 },
847 { EXT_DISPLAY, 4, 0 },
848 { EXT_LONG_NAMES, 1, 0 },
849 { EXT_LONG_STRINGS, 1, 0 },
850 { EXT_NCASES, 8, 2 },
851 { EXT_FILE_ATTRS, 1, 0 },
852 { EXT_VAR_ATTRS, 1, 0 },
853 { EXT_MRSETS2, 1, 0 },
854 { EXT_ENCODING, 1, 0 },
855 { EXT_LONG_LABELS, 1, 0 },
857 /* Ignored record types. */
858 { EXT_VAR_SETS, 0, 0 },
860 { EXT_DATA_ENTRY, 0, 0 },
863 const struct extension_record_type *type;
864 struct sfm_extension_record *record;
867 record = pool_malloc (r->pool, sizeof *record);
868 read_extension_record_header (r, subtype, record);
869 n_bytes = record->count * record->size;
871 for (type = types; type < &types[sizeof types / sizeof *types]; type++)
872 if (subtype == type->subtype)
874 if (type->size > 0 && record->size != type->size)
875 sys_warn (r, record->pos,
876 _("Record type 7, subtype %d has bad size %zu "
877 "(expected %d)."), subtype, record->size, type->size);
878 else if (type->count > 0 && record->count != type->count)
879 sys_warn (r, record->pos,
880 _("Record type 7, subtype %d has bad count %zu "
881 "(expected %d)."), subtype, record->count, type->count);
882 else if (type->count == 0 && type->size == 0)
884 /* Ignore this record. */
888 char *data = pool_malloc (r->pool, n_bytes + 1);
889 data[n_bytes] = '\0';
892 read_bytes (r, record->data, n_bytes);
899 sys_warn (r, record->pos,
900 _("Unrecognized record type 7, subtype %d. Please send a "
901 "copy of this file, and the syntax which created it to %s."),
902 subtype, PACKAGE_BUGREPORT);
905 skip_bytes (r, n_bytes);
910 skip_extension_record (struct sfm_reader *r, int subtype)
912 struct sfm_extension_record record;
914 read_extension_record_header (r, subtype, &record);
915 skip_bytes (r, record.count * record.size);
919 parse_file_label (struct sfm_reader *r, const char *file_label,
920 struct dictionary *dict)
922 char *utf8_file_label;
923 size_t file_label_len;
925 utf8_file_label = recode_string_pool ("UTF-8", dict_get_encoding (dict),
926 file_label, -1, r->pool);
927 file_label_len = strlen (utf8_file_label);
928 while (file_label_len > 0 && utf8_file_label[file_label_len - 1] == ' ')
930 utf8_file_label[file_label_len] = '\0';
931 dict_set_label (dict, utf8_file_label);
934 /* Reads a variable (type 2) record from R and adds the
935 corresponding variable to DICT.
936 Also skips past additional variable records for long string
939 parse_variable_records (struct sfm_reader *r, struct dictionary *dict,
940 struct sfm_var_record *var_recs, size_t n_var_recs)
942 const char *dict_encoding = dict_get_encoding (dict);
943 struct sfm_var_record *rec;
946 for (rec = var_recs; rec < &var_recs[n_var_recs]; )
948 struct variable *var;
953 name = recode_string_pool ("UTF-8", dict_encoding,
954 rec->name, 8, r->pool);
955 name[strcspn (name, " ")] = '\0';
957 if (!dict_id_is_valid (dict, name, false)
958 || name[0] == '$' || name[0] == '#')
959 sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
961 if (rec->width < 0 || rec->width > 255)
962 sys_error (r, rec->pos,
963 _("Bad width %d for variable %s."), rec->width, name);
965 var = rec->var = dict_create_var (dict, name, rec->width);
967 sys_error (r, rec->pos, _("Duplicate variable name `%s'."), name);
969 /* Set the short name the same as the long name. */
970 var_set_short_name (var, 0, name);
972 /* Get variable label, if any. */
977 utf8_label = recode_string_pool ("UTF-8", dict_encoding,
978 rec->label, -1, r->pool);
979 var_set_label (var, utf8_label, NULL, false);
982 /* Set missing values. */
983 if (rec->missing_value_code != 0)
985 int width = var_get_width (var);
986 struct missing_values mv;
988 mv_init_pool (r->pool, &mv, width);
989 if (var_is_numeric (var))
991 bool has_range = rec->missing_value_code < 0;
992 int n_discrete = (has_range
993 ? rec->missing_value_code == -3
994 : rec->missing_value_code);
999 double low = parse_float (r, rec->missing, 0);
1000 double high = parse_float (r, rec->missing, 8);
1001 mv_add_range (&mv, low, high);
1005 for (i = 0; i < n_discrete; i++)
1007 mv_add_num (&mv, parse_float (r, rec->missing, ofs));
1015 value_init_pool (r->pool, &value, width);
1016 value_set_missing (&value, width);
1017 for (i = 0; i < rec->missing_value_code; i++)
1019 uint8_t *s = value_str_rw (&value, width);
1020 memcpy (s, rec->missing + 8 * i, MIN (width, 8));
1021 mv_add_str (&mv, s);
1024 var_set_missing_values (var, &mv);
1028 parse_format_spec (r, rec->pos + 12, rec->print_format,
1029 PRINT_FORMAT, var, &n_warnings);
1030 parse_format_spec (r, rec->pos + 16, rec->write_format,
1031 WRITE_FORMAT, var, &n_warnings);
1033 /* Account for values.
1034 Skip long string continuation records, if any. */
1035 n_values = rec->width == 0 ? 1 : DIV_RND_UP (rec->width, 8);
1036 for (i = 1; i < n_values; i++)
1037 if (i + (rec - var_recs) >= n_var_recs || rec[i].width != -1)
1038 sys_error (r, rec->pos, _("Missing string continuation record."));
1043 /* Translates the format spec from sysfile format to internal
1046 parse_format_spec (struct sfm_reader *r, off_t pos, unsigned int format,
1047 enum which_format which, struct variable *v,
1050 const int max_warnings = 8;
1051 uint8_t raw_type = format >> 16;
1052 uint8_t w = format >> 8;
1058 if (!fmt_from_io (raw_type, &f.type))
1059 sys_error (r, pos, _("Unknown variable format %"PRIu8"."), raw_type);
1064 ok = fmt_check_output (&f) && fmt_check_width_compat (&f, var_get_width (v));
1069 if (which == PRINT_FORMAT)
1070 var_set_print_format (v, &f);
1072 var_set_write_format (v, &f);
1074 else if (++*n_warnings <= max_warnings)
1076 char fmt_string[FMT_STRING_LEN_MAX + 1];
1077 sys_warn (r, pos, _("%s variable %s has invalid %s format %s."),
1078 var_is_numeric (v) ? _("Numeric") : _("String"),
1080 which == PRINT_FORMAT ? _("print") : _("write"),
1081 fmt_to_string (&f, fmt_string));
1083 if (*n_warnings == max_warnings)
1084 sys_warn (r, -1, _("Suppressing further invalid format warnings."));
1089 parse_document (struct dictionary *dict, struct sfm_document_record *record)
1093 for (p = record->documents;
1094 p < record->documents + DOC_LINE_LENGTH * record->n_lines;
1095 p += DOC_LINE_LENGTH)
1097 struct substring line;
1099 line = recode_substring_pool ("UTF-8", dict_get_encoding (dict),
1100 ss_buffer (p, DOC_LINE_LENGTH), NULL);
1101 ss_rtrim (&line, ss_cstr (" "));
1102 line.string[line.length] = '\0';
1104 dict_add_document_line (dict, line.string, false);
1110 /* Parses record type 7, subtype 3. */
1112 parse_machine_integer_info (struct sfm_reader *r,
1113 const struct sfm_extension_record *record,
1114 struct sfm_read_info *info)
1116 int float_representation, expected_float_format;
1117 int integer_representation, expected_integer_format;
1119 /* Save version info. */
1120 info->version_major = parse_int (r, record->data, 0);
1121 info->version_minor = parse_int (r, record->data, 4);
1122 info->version_revision = parse_int (r, record->data, 8);
1124 /* Check floating point format. */
1125 float_representation = parse_int (r, record->data, 16);
1126 if (r->float_format == FLOAT_IEEE_DOUBLE_BE
1127 || r->float_format == FLOAT_IEEE_DOUBLE_LE)
1128 expected_float_format = 1;
1129 else if (r->float_format == FLOAT_Z_LONG)
1130 expected_float_format = 2;
1131 else if (r->float_format == FLOAT_VAX_G || r->float_format == FLOAT_VAX_D)
1132 expected_float_format = 3;
1135 if (float_representation != expected_float_format)
1136 sys_error (r, record->pos, _("Floating-point representation indicated by "
1137 "system file (%d) differs from expected (%d)."),
1138 float_representation, expected_float_format);
1140 /* Check integer format. */
1141 integer_representation = parse_int (r, record->data, 24);
1142 if (r->integer_format == INTEGER_MSB_FIRST)
1143 expected_integer_format = 1;
1144 else if (r->integer_format == INTEGER_LSB_FIRST)
1145 expected_integer_format = 2;
1148 if (integer_representation != expected_integer_format)
1149 sys_warn (r, record->pos,
1150 _("Integer format indicated by system file (%d) "
1151 "differs from expected (%d)."),
1152 integer_representation, expected_integer_format);
1157 choose_encoding (struct sfm_reader *r,
1158 const struct sfm_extension_record *ext_integer,
1159 const struct sfm_extension_record *ext_encoding)
1161 /* The EXT_ENCODING record is a more reliable way to determine dictionary
1164 return ext_encoding->data;
1166 /* But EXT_INTEGER is better than nothing as a fallback. */
1169 int codepage = parse_int (r, ext_integer->data, 7 * 4);
1178 /* These ostensibly mean "7-bit ASCII" and "8-bit ASCII"[sic]
1179 respectively. However, there are known to be many files in the wild
1180 with character code 2, yet have data which are clearly not ASCII.
1181 Therefore we ignore these values. */
1194 return pool_asprintf (r->pool, "CP%d", codepage);
1198 return locale_charset ();
1201 /* Parses record type 7, subtype 4. */
1203 parse_machine_float_info (struct sfm_reader *r,
1204 const struct sfm_extension_record *record)
1206 double sysmis = parse_float (r, record->data, 0);
1207 double highest = parse_float (r, record->data, 8);
1208 double lowest = parse_float (r, record->data, 16);
1210 if (sysmis != SYSMIS)
1211 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1214 if (highest != HIGHEST)
1215 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1216 highest, "HIGHEST");
1218 if (lowest != LOWEST)
1219 sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
1223 /* Parses record type 7, subtype 7 or 19. */
1225 parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record,
1226 struct dictionary *dict)
1228 struct text_record *text;
1229 struct mrset *mrset;
1231 text = open_text_record (r, record);
1234 const char *counted = NULL;
1237 struct stringi_set var_names;
1238 size_t allocated_vars;
1242 mrset = xzalloc (sizeof *mrset);
1244 name = text_get_token (text, ss_cstr ("="), NULL);
1247 mrset->name = xstrdup (name);
1249 if (mrset->name[0] != '$')
1251 sys_warn (r, record->pos,
1252 _("`%s' does not begin with `$' at UTF-8 offset %zu "
1253 "in MRSETS record."), mrset->name, text_pos (text));
1257 if (text_match (text, 'C'))
1259 mrset->type = MRSET_MC;
1260 if (!text_match (text, ' '))
1262 sys_warn (r, record->pos,
1263 _("Missing space following `%c' at UTF-8 offset %zu "
1264 "in MRSETS record."), 'C', text_pos (text));
1268 else if (text_match (text, 'D'))
1270 mrset->type = MRSET_MD;
1271 mrset->cat_source = MRSET_VARLABELS;
1273 else if (text_match (text, 'E'))
1277 mrset->type = MRSET_MD;
1278 mrset->cat_source = MRSET_COUNTEDVALUES;
1279 if (!text_match (text, ' '))
1281 sys_warn (r, record->pos,
1282 _("Missing space following `%c' at UTF-8 offset %zu "
1283 "in MRSETS record."), 'E', text_pos (text));
1287 number = text_get_token (text, ss_cstr (" "), NULL);
1288 if (!strcmp (number, "11"))
1289 mrset->label_from_var_label = true;
1290 else if (strcmp (number, "1"))
1291 sys_warn (r, record->pos,
1292 _("Unexpected label source value `%s' following `E' "
1293 "at UTF-8 offset %zu in MRSETS record."),
1294 number, text_pos (text));
1298 sys_warn (r, record->pos,
1299 _("Missing `C', `D', or `E' at UTF-8 offset %zu "
1300 "in MRSETS record."),
1305 if (mrset->type == MRSET_MD)
1307 counted = text_parse_counted_string (r, text);
1308 if (counted == NULL)
1312 label = text_parse_counted_string (r, text);
1315 mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
1317 stringi_set_init (&var_names);
1322 struct variable *var;
1323 const char *var_name;
1325 var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
1326 if (var_name == NULL)
1328 sys_warn (r, record->pos,
1329 _("Missing new-line parsing variable names "
1330 "at UTF-8 offset %zu in MRSETS record."),
1335 var = dict_lookup_var (dict, var_name);
1338 if (!stringi_set_insert (&var_names, var_name))
1340 sys_warn (r, record->pos,
1341 _("Duplicate variable name %s "
1342 "at UTF-8 offset %zu in MRSETS record."),
1343 var_name, text_pos (text));
1347 if (mrset->label == NULL && mrset->label_from_var_label
1348 && var_has_label (var))
1349 mrset->label = xstrdup (var_get_label (var));
1352 && var_get_type (var) != var_get_type (mrset->vars[0]))
1354 sys_warn (r, record->pos,
1355 _("MRSET %s contains both string and "
1356 "numeric variables."), name);
1359 width = MIN (width, var_get_width (var));
1361 if (mrset->n_vars >= allocated_vars)
1362 mrset->vars = x2nrealloc (mrset->vars, &allocated_vars,
1363 sizeof *mrset->vars);
1364 mrset->vars[mrset->n_vars++] = var;
1366 while (delimiter != '\n');
1368 if (mrset->n_vars < 2)
1370 sys_warn (r, record->pos,
1371 _("MRSET %s has only %zu variables."), mrset->name,
1373 mrset_destroy (mrset);
1377 if (mrset->type == MRSET_MD)
1379 mrset->width = width;
1380 value_init (&mrset->counted, width);
1382 mrset->counted.f = strtod (counted, NULL);
1384 value_copy_str_rpad (&mrset->counted, width,
1385 (const uint8_t *) counted, ' ');
1388 dict_add_mrset (dict, mrset);
1390 stringi_set_destroy (&var_names);
1392 mrset_destroy (mrset);
1393 close_text_record (r, text);
1396 /* Read record type 7, subtype 11, which specifies how variables
1397 should be displayed in GUI environments. */
1399 parse_display_parameters (struct sfm_reader *r,
1400 const struct sfm_extension_record *record,
1401 struct dictionary *dict)
1403 bool includes_width;
1404 bool warned = false;
1409 n_vars = dict_get_var_cnt (dict);
1410 if (record->count == 3 * n_vars)
1411 includes_width = true;
1412 else if (record->count == 2 * n_vars)
1413 includes_width = false;
1416 sys_warn (r, record->pos,
1417 _("Extension 11 has bad count %zu (for %zu variables)."),
1418 record->count, n_vars);
1423 for (i = 0; i < n_vars; ++i)
1425 struct variable *v = dict_get_var (dict, i);
1426 int measure, width, align;
1428 measure = parse_int (r, record->data, ofs);
1433 width = parse_int (r, record->data, ofs);
1439 align = parse_int (r, record->data, ofs);
1442 /* SPSS 14 sometimes seems to set string variables' measure
1444 if (0 == measure && var_is_alpha (v))
1447 if (measure < 1 || measure > 3 || align < 0 || align > 2)
1450 sys_warn (r, record->pos,
1451 _("Invalid variable display parameters for variable "
1452 "%zu (%s). Default parameters substituted."),
1453 i, var_get_name (v));
1458 var_set_measure (v, (measure == 1 ? MEASURE_NOMINAL
1459 : measure == 2 ? MEASURE_ORDINAL
1461 var_set_alignment (v, (align == 0 ? ALIGN_LEFT
1462 : align == 1 ? ALIGN_RIGHT
1465 /* Older versions (SPSS 9.0) sometimes set the display
1466 width to zero. This causes confusion in the GUI, so
1467 only set the width if it is nonzero. */
1469 var_set_display_width (v, width);
1474 rename_var_and_save_short_names (struct dictionary *dict, struct variable *var,
1475 const char *new_name)
1477 size_t n_short_names;
1481 /* Renaming a variable may clear its short names, but we
1482 want to retain them, so we save them and re-set them
1484 n_short_names = var_get_short_name_cnt (var);
1485 short_names = xnmalloc (n_short_names, sizeof *short_names);
1486 for (i = 0; i < n_short_names; i++)
1488 const char *s = var_get_short_name (var, i);
1489 short_names[i] = s != NULL ? xstrdup (s) : NULL;
1492 /* Set long name. */
1493 dict_rename_var (dict, var, new_name);
1495 /* Restore short names. */
1496 for (i = 0; i < n_short_names; i++)
1498 var_set_short_name (var, i, short_names[i]);
1499 free (short_names[i]);
1504 /* Parses record type 7, subtype 13, which gives the long name that corresponds
1505 to each short name. Modifies variable names in DICT accordingly. */
1507 parse_long_var_name_map (struct sfm_reader *r,
1508 const struct sfm_extension_record *record,
1509 struct dictionary *dict)
1511 struct text_record *text;
1512 struct variable *var;
1517 /* Convert variable names to lowercase. */
1520 for (i = 0; i < dict_get_var_cnt (dict); i++)
1522 struct variable *var = dict_get_var (dict, i);
1525 new_name = xstrdup (var_get_name (var));
1526 str_lowercase (new_name);
1528 rename_var_and_save_short_names (dict, var, new_name);
1536 /* Rename each of the variables, one by one. (In a correctly constructed
1537 system file, this cannot create any intermediate duplicate variable names,
1538 because all of the new variable names are longer than any of the old
1539 variable names and thus there cannot be any overlaps.) */
1540 text = open_text_record (r, record);
1541 while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
1543 /* Validate long name. */
1544 /* XXX need to reencode name to UTF-8 */
1545 if (!dict_id_is_valid (dict, long_name, false))
1547 sys_warn (r, record->pos,
1548 _("Long variable mapping from %s to invalid "
1549 "variable name `%s'."),
1550 var_get_name (var), long_name);
1554 /* Identify any duplicates. */
1555 if (strcasecmp (var_get_short_name (var, 0), long_name)
1556 && dict_lookup_var (dict, long_name) != NULL)
1558 sys_warn (r, record->pos,
1559 _("Duplicate long variable name `%s'."), long_name);
1563 rename_var_and_save_short_names (dict, var, long_name);
1565 close_text_record (r, text);
1568 /* Reads record type 7, subtype 14, which gives the real length
1569 of each very long string. Rearranges DICT accordingly. */
1571 parse_long_string_map (struct sfm_reader *r,
1572 const struct sfm_extension_record *record,
1573 struct dictionary *dict)
1575 struct text_record *text;
1576 struct variable *var;
1579 text = open_text_record (r, record);
1580 while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
1582 size_t idx = var_get_dict_index (var);
1588 length = strtol (length_s, NULL, 10);
1589 if (length < 1 || length > MAX_STRING)
1591 sys_warn (r, record->pos,
1592 _("%s listed as string of invalid length %s "
1593 "in very long string record."),
1594 var_get_name (var), length_s);
1598 /* Check segments. */
1599 segment_cnt = sfm_width_to_segments (length);
1600 if (segment_cnt == 1)
1602 sys_warn (r, record->pos,
1603 _("%s listed in very long string record with width %s, "
1604 "which requires only one segment."),
1605 var_get_name (var), length_s);
1608 if (idx + segment_cnt > dict_get_var_cnt (dict))
1609 sys_error (r, record->pos,
1610 _("Very long string %s overflows dictionary."),
1611 var_get_name (var));
1613 /* Get the short names from the segments and check their
1615 for (i = 0; i < segment_cnt; i++)
1617 struct variable *seg = dict_get_var (dict, idx + i);
1618 int alloc_width = sfm_segment_alloc_width (length, i);
1619 int width = var_get_width (seg);
1622 var_set_short_name (var, i, var_get_short_name (seg, 0));
1623 if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
1624 sys_error (r, record->pos,
1625 _("Very long string with width %ld has segment %d "
1626 "of width %d (expected %d)."),
1627 length, i, width, alloc_width);
1629 dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
1630 var_set_width (var, length);
1632 close_text_record (r, text);
1633 dict_compact_values (dict);
1637 parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
1638 const struct sfm_var_record *var_recs, size_t n_var_recs,
1639 const struct sfm_value_label_record *record)
1641 struct variable **vars;
1645 utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels);
1646 for (i = 0; i < record->n_labels; i++)
1647 utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
1648 record->labels[i].label, -1,
1651 vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
1652 for (i = 0; i < record->n_vars; i++)
1653 vars[i] = lookup_var_by_index (r, record->pos,
1654 var_recs, n_var_recs, record->vars[i]);
1656 for (i = 1; i < record->n_vars; i++)
1657 if (var_get_type (vars[i]) != var_get_type (vars[0]))
1658 sys_error (r, record->pos,
1659 _("Variables associated with value label are not all of "
1660 "identical type. Variable %s is %s, but variable "
1662 var_get_name (vars[0]),
1663 var_is_numeric (vars[0]) ? _("numeric") : _("string"),
1664 var_get_name (vars[i]),
1665 var_is_numeric (vars[i]) ? _("numeric") : _("string"));
1667 for (i = 0; i < record->n_vars; i++)
1669 struct variable *var = vars[i];
1673 width = var_get_width (var);
1675 sys_error (r, record->pos,
1676 _("Value labels may not be added to long string "
1677 "variables (e.g. %s) using records types 3 and 4."),
1678 var_get_name (var));
1680 for (j = 0; j < record->n_labels; j++)
1682 struct sfm_value_label *label = &record->labels[j];
1685 value_init (&value, width);
1687 value.f = parse_float (r, label->value, 0);
1689 memcpy (value_str_rw (&value, width), label->value, width);
1691 if (!var_add_value_label (var, &value, utf8_labels[j]))
1693 if (var_is_numeric (var))
1694 sys_warn (r, record->pos,
1695 _("Duplicate value label for %g on %s."),
1696 value.f, var_get_name (var));
1698 sys_warn (r, record->pos,
1699 _("Duplicate value label for `%.*s' on %s."),
1700 width, value_str (&value, width),
1701 var_get_name (var));
1704 value_destroy (&value, width);
1708 pool_free (r->pool, vars);
1709 for (i = 0; i < record->n_labels; i++)
1710 pool_free (r->pool, utf8_labels[i]);
1711 pool_free (r->pool, utf8_labels);
1714 static struct variable *
1715 lookup_var_by_index (struct sfm_reader *r, off_t offset,
1716 const struct sfm_var_record *var_recs, size_t n_var_recs,
1719 const struct sfm_var_record *rec;
1721 if (idx < 1 || idx > n_var_recs)
1723 sys_error (r, offset,
1724 _("Variable index %d not in valid range 1...%d."),
1729 rec = &var_recs[idx - 1];
1730 if (rec->var == NULL)
1732 sys_error (r, offset,
1733 _("Variable index %d refers to long string continuation."),
1741 /* Parses a set of custom attributes from TEXT into ATTRS.
1742 ATTRS may be a null pointer, in which case the attributes are
1743 read but discarded. */
1745 parse_attributes (struct sfm_reader *r, struct text_record *text,
1746 struct attrset *attrs)
1750 struct attribute *attr;
1754 /* Parse the key. */
1755 key = text_get_token (text, ss_cstr ("("), NULL);
1759 attr = attribute_create (key);
1760 for (index = 1; ; index++)
1762 /* Parse the value. */
1766 value = text_get_token (text, ss_cstr ("\n"), NULL);
1769 text_warn (r, text, _("Error parsing attribute value %s[%d]."),
1774 length = strlen (value);
1775 if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
1777 value[length - 1] = '\0';
1778 attribute_add_value (attr, value + 1);
1783 _("Attribute value %s[%d] is not quoted: %s."),
1785 attribute_add_value (attr, value);
1788 /* Was this the last value for this attribute? */
1789 if (text_match (text, ')'))
1793 attrset_add (attrs, attr);
1795 attribute_destroy (attr);
1797 while (!text_match (text, '/'));
1800 /* Reads record type 7, subtype 17, which lists custom
1801 attributes on the data file. */
1803 parse_data_file_attributes (struct sfm_reader *r,
1804 const struct sfm_extension_record *record,
1805 struct dictionary *dict)
1807 struct text_record *text = open_text_record (r, record);
1808 parse_attributes (r, text, dict_get_attributes (dict));
1809 close_text_record (r, text);
1812 /* Parses record type 7, subtype 18, which lists custom
1813 attributes on individual variables. */
1815 parse_variable_attributes (struct sfm_reader *r,
1816 const struct sfm_extension_record *record,
1817 struct dictionary *dict)
1819 struct text_record *text;
1820 struct variable *var;
1822 text = open_text_record (r, record);
1823 while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
1824 parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
1825 close_text_record (r, text);
1829 check_overflow (struct sfm_reader *r,
1830 const struct sfm_extension_record *record,
1831 size_t ofs, size_t length)
1833 size_t end = record->size * record->count;
1834 if (length >= end || ofs + length > end)
1835 sys_error (r, record->pos + end,
1836 _("Long string value label record ends unexpectedly."));
1840 parse_long_string_value_labels (struct sfm_reader *r,
1841 const struct sfm_extension_record *record,
1842 struct dictionary *dict)
1844 const char *dict_encoding = dict_get_encoding (dict);
1845 size_t end = record->size * record->count;
1852 struct variable *var;
1857 /* Parse variable name length. */
1858 check_overflow (r, record, ofs, 4);
1859 var_name_len = parse_int (r, record->data, ofs);
1862 /* Parse variable name, width, and number of labels. */
1863 check_overflow (r, record, ofs, var_name_len + 8);
1864 var_name = recode_string_pool ("UTF-8", dict_encoding,
1865 (const char *) record->data + ofs,
1866 var_name_len, r->pool);
1867 width = parse_int (r, record->data, ofs + var_name_len);
1868 n_labels = parse_int (r, record->data, ofs + var_name_len + 4);
1869 ofs += var_name_len + 8;
1871 /* Look up 'var' and validate. */
1872 var = dict_lookup_var (dict, var_name);
1874 sys_warn (r, record->pos + ofs,
1875 _("Ignoring long string value record for "
1876 "unknown variable %s."), var_name);
1877 else if (var_is_numeric (var))
1879 sys_warn (r, record->pos + ofs,
1880 _("Ignoring long string value record for "
1881 "numeric variable %s."), var_name);
1884 else if (width != var_get_width (var))
1886 sys_warn (r, record->pos + ofs,
1887 _("Ignoring long string value record for variable %s "
1888 "because the record's width (%d) does not match the "
1889 "variable's width (%d)."),
1890 var_name, width, var_get_width (var));
1895 value_init_pool (r->pool, &value, width);
1896 for (i = 0; i < n_labels; i++)
1898 size_t value_length, label_length;
1899 bool skip = var == NULL;
1901 /* Parse value length. */
1902 check_overflow (r, record, ofs, 4);
1903 value_length = parse_int (r, record->data, ofs);
1907 check_overflow (r, record, ofs, value_length);
1910 if (value_length == width)
1911 memcpy (value_str_rw (&value, width),
1912 (const uint8_t *) record->data + ofs, width);
1915 sys_warn (r, record->pos + ofs,
1916 _("Ignoring long string value %zu for variable "
1917 "%s, with width %d, that has bad value "
1919 i, var_get_name (var), width, value_length);
1923 ofs += value_length;
1925 /* Parse label length. */
1926 check_overflow (r, record, ofs, 4);
1927 label_length = parse_int (r, record->data, ofs);
1931 check_overflow (r, record, ofs, label_length);
1936 label = recode_string_pool ("UTF-8", dict_encoding,
1937 (const char *) record->data + ofs,
1938 label_length, r->pool);
1939 if (!var_add_value_label (var, &value, label))
1940 sys_warn (r, record->pos + ofs,
1941 _("Duplicate value label for `%.*s' on %s."),
1942 width, value_str (&value, width),
1943 var_get_name (var));
1944 pool_free (r->pool, label);
1946 ofs += label_length;
1953 static void partial_record (struct sfm_reader *r)
1956 static void read_error (struct casereader *, const struct sfm_reader *);
1958 static bool read_case_number (struct sfm_reader *, double *);
1959 static bool read_case_string (struct sfm_reader *, uint8_t *, size_t);
1960 static int read_opcode (struct sfm_reader *);
1961 static bool read_compressed_number (struct sfm_reader *, double *);
1962 static bool read_compressed_string (struct sfm_reader *, uint8_t *);
1963 static bool read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
1964 static bool skip_whole_strings (struct sfm_reader *, size_t);
1966 /* Reads and returns one case from READER's file. Returns a null
1967 pointer if not successful. */
1968 static struct ccase *
1969 sys_file_casereader_read (struct casereader *reader, void *r_)
1971 struct sfm_reader *r = r_;
1972 struct ccase *volatile c;
1978 c = case_create (r->proto);
1979 if (setjmp (r->bail_out))
1981 casereader_force_error (reader);
1986 for (i = 0; i < r->sfm_var_cnt; i++)
1988 struct sfm_var *sv = &r->sfm_vars[i];
1989 union value *v = case_data_rw_idx (c, sv->case_index);
1991 if (sv->var_width == 0)
1993 if (!read_case_number (r, &v->f))
1998 uint8_t *s = value_str_rw (v, sv->var_width);
1999 if (!read_case_string (r, s + sv->offset, sv->segment_width))
2001 if (!skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)))
2010 if (r->case_cnt != -1)
2011 read_error (reader, r);
2016 /* Issues an error that R ends in a partial record. */
2018 partial_record (struct sfm_reader *r)
2020 sys_error (r, r->pos, _("File ends in partial case."));
2023 /* Issues an error that an unspecified error occurred SFM, and
2026 read_error (struct casereader *r, const struct sfm_reader *sfm)
2028 msg (ME, _("Error reading case from file %s."), fh_get_name (sfm->fh));
2029 casereader_force_error (r);
2032 /* Reads a number from R and stores its value in *D.
2033 If R is compressed, reads a compressed number;
2034 otherwise, reads a number in the regular way.
2035 Returns true if successful, false if end of file is
2036 reached immediately. */
2038 read_case_number (struct sfm_reader *r, double *d)
2043 if (!try_read_bytes (r, number, sizeof number))
2045 float_convert (r->float_format, number, FLOAT_NATIVE_DOUBLE, d);
2049 return read_compressed_number (r, d);
2052 /* Reads LENGTH string bytes from R into S.
2053 Always reads a multiple of 8 bytes; if LENGTH is not a
2054 multiple of 8, then extra bytes are read and discarded without
2056 Reads compressed strings if S is compressed.
2057 Returns true if successful, false if end of file is
2058 reached immediately. */
2060 read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
2062 size_t whole = ROUND_DOWN (length, 8);
2063 size_t partial = length % 8;
2067 if (!read_whole_strings (r, s, whole))
2074 if (!read_whole_strings (r, bounce, sizeof bounce))
2080 memcpy (s + whole, bounce, partial);
2086 /* Reads and returns the next compression opcode from R. */
2088 read_opcode (struct sfm_reader *r)
2090 assert (r->compressed);
2094 if (r->opcode_idx >= sizeof r->opcodes)
2096 if (!try_read_bytes (r, r->opcodes, sizeof r->opcodes))
2100 opcode = r->opcodes[r->opcode_idx++];
2107 /* Reads a compressed number from R and stores its value in D.
2108 Returns true if successful, false if end of file is
2109 reached immediately. */
2111 read_compressed_number (struct sfm_reader *r, double *d)
2113 int opcode = read_opcode (r);
2121 *d = read_float (r);
2125 float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
2126 if (!r->corruption_warning)
2128 r->corruption_warning = true;
2129 sys_warn (r, r->pos,
2130 _("Possible compressed data corruption: "
2131 "compressed spaces appear in numeric field."));
2140 *d = opcode - r->bias;
2147 /* Reads a compressed 8-byte string segment from R and stores it
2149 Returns true if successful, false if end of file is
2150 reached immediately. */
2152 read_compressed_string (struct sfm_reader *r, uint8_t *dst)
2154 int opcode = read_opcode (r);
2162 read_bytes (r, dst, 8);
2166 memset (dst, ' ', 8);
2171 double value = opcode - r->bias;
2172 float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
2175 /* This has actually been seen "in the wild". The submitter of the
2176 file that showed that the contents decoded as spaces, but they
2177 were at the end of the field so it's possible that the null
2178 bytes just acted as null terminators. */
2180 else if (!r->corruption_warning)
2182 r->corruption_warning = true;
2183 sys_warn (r, r->pos,
2184 _("Possible compressed data corruption: "
2185 "string contains compressed integer (opcode %d)."),
2195 /* Reads LENGTH string bytes from R into S.
2196 LENGTH must be a multiple of 8.
2197 Reads compressed strings if S is compressed.
2198 Returns true if successful, false if end of file is
2199 reached immediately. */
2201 read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
2203 assert (length % 8 == 0);
2205 return try_read_bytes (r, s, length);
2209 for (ofs = 0; ofs < length; ofs += 8)
2210 if (!read_compressed_string (r, s + ofs))
2220 /* Skips LENGTH string bytes from R.
2221 LENGTH must be a multiple of 8.
2222 (LENGTH is also limited to 1024, but that's only because the
2223 current caller never needs more than that many bytes.)
2224 Returns true if successful, false if end of file is
2225 reached immediately. */
2227 skip_whole_strings (struct sfm_reader *r, size_t length)
2229 uint8_t buffer[1024];
2230 assert (length < sizeof buffer);
2231 return read_whole_strings (r, buffer, length);
2234 /* Helpers for reading records that contain structured text
2237 /* Maximum number of warnings to issue for a single text
2239 #define MAX_TEXT_WARNINGS 5
2244 struct substring buffer; /* Record contents, in UTF-8. */
2245 off_t start; /* Starting offset in file. */
2246 size_t pos; /* Current position in buffer. */
2247 int n_warnings; /* Number of warnings issued or suppressed. */
2250 static struct text_record *
2251 open_text_record (struct sfm_reader *r,
2252 const struct sfm_extension_record *record)
2254 struct text_record *text;
2255 struct substring raw;
2257 text = pool_alloc (r->pool, sizeof *text);
2258 raw = ss_buffer (record->data, record->size * record->count);
2259 text->start = record->pos;
2260 text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool);
2262 text->n_warnings = 0;
2267 /* Closes TEXT, frees its storage, and issues a final warning
2268 about suppressed warnings if necesary. */
2270 close_text_record (struct sfm_reader *r, struct text_record *text)
2272 if (text->n_warnings > MAX_TEXT_WARNINGS)
2273 sys_warn (r, -1, _("Suppressed %d additional related warnings."),
2274 text->n_warnings - MAX_TEXT_WARNINGS);
2275 pool_free (r->pool, ss_data (text->buffer));
2278 /* Reads a variable=value pair from TEXT.
2279 Looks up the variable in DICT and stores it into *VAR.
2280 Stores a null-terminated value into *VALUE. */
2282 read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
2283 struct text_record *text,
2284 struct variable **var, char **value)
2288 if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
2291 *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
2295 text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
2296 ss_buffer ("\t\0", 2));
2304 text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
2305 struct text_record *text, struct substring delimiters,
2306 struct variable **var)
2310 name = text_get_token (text, delimiters, NULL);
2314 *var = dict_lookup_var (dict, name);
2318 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
2325 text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
2326 struct text_record *text, struct substring delimiters,
2327 struct variable **var)
2329 char *short_name = text_get_token (text, delimiters, NULL);
2330 if (short_name == NULL)
2333 *var = dict_lookup_var (dict, short_name);
2335 text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
2340 /* Displays a warning for the current file position, limiting the
2341 number to MAX_TEXT_WARNINGS for TEXT. */
2343 text_warn (struct sfm_reader *r, struct text_record *text,
2344 const char *format, ...)
2346 if (text->n_warnings++ < MAX_TEXT_WARNINGS)
2350 va_start (args, format);
2351 sys_msg (r, text->start + text->pos, MW, format, args);
2357 text_get_token (struct text_record *text, struct substring delimiters,
2360 struct substring token;
2363 if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
2366 end = &ss_data (token)[ss_length (token)];
2367 if (delimiter != NULL)
2370 return ss_data (token);
2373 /* Reads a integer value expressed in decimal, then a space, then a string that
2374 consists of exactly as many bytes as specified by the integer, then a space,
2375 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
2376 buffer (so the caller should not free the string). */
2378 text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
2388 int c = text->buffer.string[text->pos];
2389 if (c < '0' || c > '9')
2391 n = (n * 10) + (c - '0');
2394 if (start == text->pos)
2396 sys_warn (r, text->start,
2397 _("Expecting digit at UTF-8 offset %zu in MRSETS record."),
2402 if (!text_match (text, ' '))
2404 sys_warn (r, text->start,
2405 _("Expecting space at UTF-8 offset %zu in MRSETS record."),
2410 if (text->pos + n > text->buffer.length)
2412 sys_warn (r, text->start,
2413 _("%zu-byte string starting at UTF-8 offset %zu "
2414 "exceeds record length %zu."),
2415 n, text->pos, text->buffer.length);
2419 s = &text->buffer.string[text->pos];
2422 sys_warn (r, text->start,
2423 _("Expecting space at UTF-8 offset %zu following %zu-byte "
2434 text_match (struct text_record *text, char c)
2436 if (text->buffer.string[text->pos] == c)
2445 /* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's
2448 text_pos (const struct text_record *text)
2455 /* Displays a corruption message. */
2457 sys_msg (struct sfm_reader *r, off_t offset,
2458 int class, const char *format, va_list args)
2463 ds_init_empty (&text);
2465 ds_put_format (&text, _("`%s' near offset 0x%llx: "),
2466 fh_get_file_name (r->fh), (long long int) offset);
2468 ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
2469 ds_put_vformat (&text, format, args);
2471 m.category = msg_class_to_category (class);
2472 m.severity = msg_class_to_severity (class);
2478 m.text = ds_cstr (&text);
2483 /* Displays a warning for offset OFFSET in the file. */
2485 sys_warn (struct sfm_reader *r, off_t offset, const char *format, ...)
2489 va_start (args, format);
2490 sys_msg (r, offset, MW, format, args);
2494 /* Displays an error for the current file position,
2495 marks it as in an error state,
2496 and aborts reading it using longjmp. */
2498 sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
2502 va_start (args, format);
2503 sys_msg (r, offset, ME, format, args);
2507 longjmp (r->bail_out, 1);
2510 /* Reads BYTE_CNT bytes into BUF.
2511 Returns true if exactly BYTE_CNT bytes are successfully read.
2512 Aborts if an I/O error or a partial read occurs.
2513 If EOF_IS_OK, then an immediate end-of-file causes false to be
2514 returned; otherwise, immediate end-of-file causes an abort
2517 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
2518 void *buf, size_t byte_cnt)
2520 size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
2521 r->pos += bytes_read;
2522 if (bytes_read == byte_cnt)
2524 else if (ferror (r->file))
2525 sys_error (r, r->pos, _("System error: %s."), strerror (errno));
2526 else if (!eof_is_ok || bytes_read != 0)
2527 sys_error (r, r->pos, _("Unexpected end of file."));
2532 /* Reads BYTE_CNT into BUF.
2533 Aborts upon I/O error or if end-of-file is encountered. */
2535 read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
2537 read_bytes_internal (r, false, buf, byte_cnt);
2540 /* Reads BYTE_CNT bytes into BUF.
2541 Returns true if exactly BYTE_CNT bytes are successfully read.
2542 Returns false if an immediate end-of-file is encountered.
2543 Aborts if an I/O error or a partial read occurs. */
2545 try_read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
2547 return read_bytes_internal (r, true, buf, byte_cnt);
2550 /* Reads a 32-bit signed integer from R and returns its value in
2553 read_int (struct sfm_reader *r)
2556 read_bytes (r, integer, sizeof integer);
2557 return integer_get (r->integer_format, integer, sizeof integer);
2560 /* Reads a 64-bit floating-point number from R and returns its
2561 value in host format. */
2563 read_float (struct sfm_reader *r)
2566 read_bytes (r, number, sizeof number);
2567 return float_get_double (r->float_format, number);
2571 parse_int (struct sfm_reader *r, const void *data, size_t ofs)
2573 return integer_get (r->integer_format, (const uint8_t *) data + ofs, 4);
2577 parse_float (struct sfm_reader *r, const void *data, size_t ofs)
2579 return float_get_double (r->float_format, (const uint8_t *) data + ofs);
2582 /* Reads exactly SIZE - 1 bytes into BUFFER
2583 and stores a null byte into BUFFER[SIZE - 1]. */
2585 read_string (struct sfm_reader *r, char *buffer, size_t size)
2588 read_bytes (r, buffer, size - 1);
2589 buffer[size - 1] = '\0';
2592 /* Skips BYTES bytes forward in R. */
2594 skip_bytes (struct sfm_reader *r, size_t bytes)
2599 size_t chunk = MIN (sizeof buffer, bytes);
2600 read_bytes (r, buffer, chunk);
2605 static const struct casereader_class sys_file_casereader_class =
2607 sys_file_casereader_read,
2608 sys_file_casereader_destroy,