1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
28 #include "data/val-type.h"
29 #include "libpspp/cast.h"
30 #include "libpspp/compiler.h"
31 #include "libpspp/float-format.h"
32 #include "libpspp/integer-format.h"
33 #include "libpspp/misc.h"
36 #include "gl/minmax.h"
37 #include "gl/progname.h"
38 #include "gl/version-etc.h"
39 #include "gl/xalloc.h"
53 const char *file_name;
56 int n_variable_records, n_variables;
59 size_t n_var_widths, allocated_var_widths;
61 enum integer_format integer_format;
62 enum float_format float_format;
64 enum compression compression;
68 static void read_header (struct sfm_reader *);
69 static void read_variable_record (struct sfm_reader *);
70 static void read_value_label_record (struct sfm_reader *);
71 static void read_document_record (struct sfm_reader *);
72 static void read_extension_record (struct sfm_reader *);
73 static void read_machine_integer_info (struct sfm_reader *,
74 size_t size, size_t count);
75 static void read_machine_float_info (struct sfm_reader *,
76 size_t size, size_t count);
77 static void read_extra_product_info (struct sfm_reader *,
78 size_t size, size_t count);
79 static void read_mrsets (struct sfm_reader *, size_t size, size_t count);
80 static void read_display_parameters (struct sfm_reader *,
81 size_t size, size_t count);
82 static void read_long_var_name_map (struct sfm_reader *r,
83 size_t size, size_t count);
84 static void read_long_string_map (struct sfm_reader *r,
85 size_t size, size_t count);
86 static void read_datafile_attributes (struct sfm_reader *r,
87 size_t size, size_t count);
88 static void read_variable_attributes (struct sfm_reader *r,
89 size_t size, size_t count);
90 static void read_ncases64 (struct sfm_reader *, size_t size, size_t count);
91 static void read_character_encoding (struct sfm_reader *r,
92 size_t size, size_t count);
93 static void read_long_string_value_labels (struct sfm_reader *r,
94 size_t size, size_t count);
95 static void read_long_string_missing_values (struct sfm_reader *r,
96 size_t size, size_t count);
97 static void read_unknown_extension (struct sfm_reader *,
98 size_t size, size_t count);
99 static void read_simple_compressed_data (struct sfm_reader *, int max_cases);
100 static void read_zlib_compressed_data (struct sfm_reader *);
102 static struct text_record *open_text_record (
103 struct sfm_reader *, size_t size, size_t count);
104 static void close_text_record (struct text_record *);
105 static bool read_variable_to_value_pair (struct text_record *,
106 char **key, char **value);
107 static char *text_tokenize (struct text_record *, int delimiter);
108 static bool text_match (struct text_record *text, int c);
109 static const char *text_parse_counted_string (struct text_record *);
110 static size_t text_pos (const struct text_record *);
111 static const char *text_get_all (const struct text_record *);
113 static void usage (void);
114 static void sys_warn (struct sfm_reader *, const char *, ...)
115 PRINTF_FORMAT (2, 3);
116 static void sys_error (struct sfm_reader *, const char *, ...)
120 static void read_bytes (struct sfm_reader *, void *, size_t);
121 static bool try_read_bytes (struct sfm_reader *, void *, size_t);
122 static int read_int (struct sfm_reader *);
123 static int64_t read_int64 (struct sfm_reader *);
124 static double read_float (struct sfm_reader *);
125 static void read_string (struct sfm_reader *, char *, size_t);
126 static void skip_bytes (struct sfm_reader *, size_t);
127 static void trim_spaces (char *);
129 static void print_string (const char *s, size_t len);
132 main (int argc, char *argv[])
138 set_program_name (argv[0]);
142 static const struct option long_options[] =
144 { "data", optional_argument, NULL, 'd' },
145 { "help", no_argument, NULL, 'h' },
146 { "version", no_argument, NULL, 'v' },
147 { NULL, 0, NULL, 0 },
152 c = getopt_long (argc, argv, "d::hv", long_options, NULL);
159 max_cases = optarg ? atoi (optarg) : INT_MAX;
163 version_etc (stdout, "pspp-dump-sav", PACKAGE_NAME, PACKAGE_VERSION,
164 "Ben Pfaff", "John Darrington", NULL_SENTINEL);
177 error (1, 0, "at least one non-option argument is required; "
178 "use --help for help");
180 for (i = optind; i < argc; i++)
184 r.file_name = argv[i];
185 r.file = fopen (r.file_name, "rb");
187 error (EXIT_FAILURE, errno, "error opening `%s'", r.file_name);
188 r.n_variable_records = 0;
191 r.allocated_var_widths = 0;
193 r.compression = COMP_NONE;
195 if (argc - optind > 1)
196 printf ("Reading \"%s\":\n", r.file_name);
199 while ((rec_type = read_int (&r)) != 999)
204 read_variable_record (&r);
208 read_value_label_record (&r);
212 sys_error (&r, "Misplaced type 4 record.");
215 read_document_record (&r);
219 read_extension_record (&r);
223 sys_error (&r, "Unrecognized record type %d.", rec_type);
226 printf ("%08llx: end-of-dictionary record "
227 "(first byte of data at %08llx)\n",
228 (long long int) ftello (r.file),
229 (long long int) ftello (r.file) + 4);
231 if (r.compression == COMP_SIMPLE)
234 read_simple_compressed_data (&r, max_cases);
236 else if (r.compression == COMP_ZLIB)
237 read_zlib_compressed_data (&r);
248 read_header (struct sfm_reader *r)
251 char eye_catcher[61];
252 uint8_t raw_layout_code[4];
255 int32_t weight_index;
258 char creation_date[10];
259 char creation_time[9];
263 read_string (r, rec_type, sizeof rec_type);
264 read_string (r, eye_catcher, sizeof eye_catcher);
266 if (!strcmp ("$FL2", rec_type))
268 else if (!strcmp ("$FL3", rec_type))
271 sys_error (r, "This is not an SPSS system file.");
273 /* Identify integer format. */
274 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
275 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
277 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
279 || (r->integer_format != INTEGER_MSB_FIRST
280 && r->integer_format != INTEGER_LSB_FIRST))
281 sys_error (r, "This is not an SPSS system file.");
282 layout_code = integer_get (r->integer_format,
283 raw_layout_code, sizeof raw_layout_code);
285 read_int (r); /* Nominal case size (not actually useful). */
286 compressed = read_int (r);
287 weight_index = read_int (r);
288 ncases = read_int (r);
293 r->compression = COMP_NONE;
294 else if (compressed == 1)
295 r->compression = COMP_SIMPLE;
296 else if (compressed != 0)
297 sys_error (r, "SAV file header has invalid compression value "
298 "%"PRId32".", compressed);
303 r->compression = COMP_ZLIB;
305 sys_error (r, "ZSAV file header has invalid compression value "
306 "%"PRId32".", compressed);
309 /* Identify floating-point format and obtain compression bias. */
310 read_bytes (r, raw_bias, sizeof raw_bias);
311 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
313 sys_warn (r, "Compression bias is not the usual value of 100, or system "
314 "file uses unrecognized floating-point format.");
315 if (r->integer_format == INTEGER_MSB_FIRST)
316 r->float_format = FLOAT_IEEE_DOUBLE_BE;
318 r->float_format = FLOAT_IEEE_DOUBLE_LE;
320 r->bias = float_get_double (r->float_format, raw_bias);
322 read_string (r, creation_date, sizeof creation_date);
323 read_string (r, creation_time, sizeof creation_time);
324 read_string (r, file_label, sizeof file_label);
325 trim_spaces (file_label);
328 printf ("File header record:\n");
329 printf ("\t%17s: %s\n", "Product name", eye_catcher);
330 printf ("\t%17s: %"PRId32"\n", "Layout code", layout_code);
331 printf ("\t%17s: %"PRId32" (%s)\n", "Compressed",
333 r->compression == COMP_NONE ? "no compression"
334 : r->compression == COMP_SIMPLE ? "simple compression"
335 : r->compression == COMP_ZLIB ? "ZLIB compression"
337 printf ("\t%17s: %"PRId32"\n", "Weight index", weight_index);
338 printf ("\t%17s: %"PRId32"\n", "Number of cases", ncases);
339 printf ("\t%17s: %.*g\n", "Compression bias", DBL_DIG + 1, r->bias);
340 printf ("\t%17s: %s\n", "Creation date", creation_date);
341 printf ("\t%17s: %s\n", "Creation time", creation_time);
342 printf ("\t%17s: \"%s\"\n", "File label", file_label);
346 format_name (int format)
348 switch ((format >> 16) & 0xff)
351 case 2: return "AHEX";
352 case 3: return "COMMA";
353 case 4: return "DOLLAR";
356 case 7: return "PIBHEX";
358 case 9: return "PIB";
359 case 10: return "PK";
360 case 11: return "RB";
361 case 12: return "RBHEX";
365 case 20: return "DATE";
366 case 21: return "TIME";
367 case 22: return "DATETIME";
368 case 23: return "ADATE";
369 case 24: return "JDATE";
370 case 25: return "DTIME";
371 case 26: return "WKDAY";
372 case 27: return "MONTH";
373 case 28: return "MOYR";
374 case 29: return "QYR";
375 case 30: return "WKYR";
376 case 31: return "PCT";
377 case 32: return "DOT";
378 case 33: return "CCA";
379 case 34: return "CCB";
380 case 35: return "CCC";
381 case 36: return "CCD";
382 case 37: return "CCE";
383 case 38: return "EDATE";
384 case 39: return "SDATE";
385 case 40: return "MTIME";
386 case 41: return "YMDHMS";
387 default: return "invalid";
391 /* Reads a variable (type 2) record from R and adds the
392 corresponding variable to DICT.
393 Also skips past additional variable records for long string
396 read_variable_record (struct sfm_reader *r)
399 int has_variable_label;
400 int missing_value_code;
405 printf ("%08llx: variable record #%d\n",
406 (long long int) ftello (r->file), ++r->n_variable_records);
408 width = read_int (r);
409 has_variable_label = read_int (r);
410 missing_value_code = read_int (r);
411 print_format = read_int (r);
412 write_format = read_int (r);
413 read_string (r, name, sizeof name);
414 name[strcspn (name, " ")] = '\0';
419 if (r->n_var_widths >= r->allocated_var_widths)
420 r->var_widths = x2nrealloc (r->var_widths, &r->allocated_var_widths,
421 sizeof *r->var_widths);
422 r->var_widths[r->n_var_widths++] = width;
424 printf ("\tWidth: %d (%s)\n",
427 : width == 0 ? "numeric"
428 : "long string continuation record");
429 printf ("\tVariable label: %d\n", has_variable_label);
430 printf ("\tMissing values code: %d (%s)\n", missing_value_code,
431 (missing_value_code == 0 ? "no missing values"
432 : missing_value_code == 1 ? "one missing value"
433 : missing_value_code == 2 ? "two missing values"
434 : missing_value_code == 3 ? "three missing values"
435 : missing_value_code == -2 ? "one missing value range"
436 : missing_value_code == -3 ? "one missing value, one range"
438 printf ("\tPrint format: %06x (%s%d.%d)\n",
439 print_format, format_name (print_format),
440 (print_format >> 8) & 0xff, print_format & 0xff);
441 printf ("\tWrite format: %06x (%s%d.%d)\n",
442 write_format, format_name (write_format),
443 (write_format >> 8) & 0xff, write_format & 0xff);
444 printf ("\tName: %s\n", name);
446 /* Get variable label, if any. */
447 if (has_variable_label != 0 && has_variable_label != 1)
448 sys_error (r, "Variable label indicator field is not 0 or 1.");
449 if (has_variable_label == 1)
451 long long int offset = ftello (r->file);
452 enum { MAX_LABEL_LEN = 65536 };
454 size_t len = read_int (r);
455 size_t read_len = MIN (MAX_LABEL_LEN, len);
456 char *label = xmalloc (read_len + 1);
457 read_string (r, label, read_len + 1);
458 printf("\t%08llx Variable label: \"%s\"\n", offset, label);
461 /* Skip label padding up to multiple of 4 bytes. */
462 skip_bytes (r, ROUND_UP (len, 4) - len);
465 /* Set missing values. */
466 if (missing_value_code != 0)
470 printf ("\t%08llx Missing values:", (long long int) ftello (r->file));
473 if (missing_value_code < -3 || missing_value_code > 3
474 || missing_value_code == -1)
475 sys_error (r, "Numeric missing value indicator field is not "
476 "-3, -2, 0, 1, 2, or 3.");
477 if (missing_value_code < 0)
479 double low = read_float (r);
480 double high = read_float (r);
481 printf (" %.*g...%.*g", DBL_DIG + 1, low, DBL_DIG + 1, high);
482 missing_value_code = -missing_value_code - 2;
484 for (i = 0; i < missing_value_code; i++)
485 printf (" %.*g", DBL_DIG + 1, read_float (r));
489 if (missing_value_code < 1 || missing_value_code > 3)
490 sys_error (r, "String missing value indicator field is not "
492 for (i = 0; i < missing_value_code; i++)
495 read_string (r, string, sizeof string);
496 printf (" \"%s\"", string);
504 print_untyped_value (struct sfm_reader *r, char raw_value[8])
509 value = float_get_double (r->float_format, raw_value);
510 for (n_printable = 0; n_printable < 8; n_printable++)
511 if (!isprint (raw_value[n_printable]))
514 printf ("%.*g/\"%.*s\"", DBL_DIG + 1, value, n_printable, raw_value);
517 /* Reads value labels from sysfile R and inserts them into the
518 associated dictionary. */
520 read_value_label_record (struct sfm_reader *r)
522 int n_labels, n_vars;
525 printf ("%08llx: value labels record\n", (long long int) ftello (r->file));
527 /* Read number of labels. */
528 n_labels = read_int (r);
529 for (i = 0; i < n_labels; i++)
532 unsigned char label_len;
536 read_bytes (r, raw_value, sizeof raw_value);
538 /* Read label length. */
539 read_bytes (r, &label_len, sizeof label_len);
540 padded_len = ROUND_UP (label_len + 1, 8);
542 /* Read label, padding. */
543 read_bytes (r, label, padded_len - 1);
544 label[label_len] = 0;
547 print_untyped_value (r, raw_value);
548 printf (": \"%s\"\n", label);
551 /* Now, read the type 4 record that has the list of variables
552 to which the value labels are to be applied. */
554 /* Read record type of type 4 record. */
555 if (read_int (r) != 4)
556 sys_error (r, "Variable index record (type 4) does not immediately "
557 "follow value label record (type 3) as it should.");
559 /* Read number of variables associated with value label from type 4
561 printf ("\t%08llx: apply to variables", (long long int) ftello (r->file));
562 n_vars = read_int (r);
563 for (i = 0; i < n_vars; i++)
564 printf (" #%d", read_int (r));
569 read_document_record (struct sfm_reader *r)
574 printf ("%08llx: document record\n", (long long int) ftello (r->file));
575 n_lines = read_int (r);
576 printf ("\t%d lines of documents\n", n_lines);
578 for (i = 0; i < n_lines; i++)
581 printf ("\t%08llx: ", (long long int) ftello (r->file));
582 read_string (r, line, sizeof line);
584 printf ("line %d: \"%s\"\n", i, line);
589 read_extension_record (struct sfm_reader *r)
591 long long int offset = ftello (r->file);
592 int subtype = read_int (r);
593 size_t size = read_int (r);
594 size_t count = read_int (r);
595 size_t bytes = size * count;
597 printf ("%08llx: Record 7, subtype %d, size=%zu, count=%zu\n",
598 offset, subtype, size, count);
603 read_machine_integer_info (r, size, count);
607 read_machine_float_info (r, size, count);
611 /* DATE variable information. We don't use it yet, but we
617 read_mrsets (r, size, count);
621 read_extra_product_info (r, size, count);
625 read_display_parameters (r, size, count);
629 read_long_var_name_map (r, size, count);
633 read_long_string_map (r, size, count);
637 read_ncases64 (r, size, count);
641 read_datafile_attributes (r, size, count);
645 read_variable_attributes (r, size, count);
649 read_character_encoding (r, size, count);
653 read_long_string_value_labels (r, size, count);
657 read_long_string_missing_values (r, size, count);
661 sys_warn (r, "Unrecognized record type 7, subtype %d.", subtype);
662 read_unknown_extension (r, size, count);
666 skip_bytes (r, bytes);
670 read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count)
672 long long int offset = ftello (r->file);
673 int version_major = read_int (r);
674 int version_minor = read_int (r);
675 int version_revision = read_int (r);
676 int machine_code = read_int (r);
677 int float_representation = read_int (r);
678 int compression_code = read_int (r);
679 int integer_representation = read_int (r);
680 int character_code = read_int (r);
682 printf ("%08llx: machine integer info\n", offset);
683 if (size != 4 || count != 8)
684 sys_error (r, "Bad size (%zu) or count (%zu) field on record type 7, "
685 "subtype 3.", size, count);
687 printf ("\tVersion: %d.%d.%d\n",
688 version_major, version_minor, version_revision);
689 printf ("\tMachine code: %d\n", machine_code);
690 printf ("\tFloating point representation: %d (%s)\n",
691 float_representation,
692 float_representation == 1 ? "IEEE 754"
693 : float_representation == 2 ? "IBM 370"
694 : float_representation == 3 ? "DEC VAX"
696 printf ("\tCompression code: %d\n", compression_code);
697 printf ("\tEndianness: %d (%s)\n", integer_representation,
698 integer_representation == 1 ? "big"
699 : integer_representation == 2 ? "little" : "unknown");
700 printf ("\tCharacter code: %d\n", character_code);
703 /* Read record type 7, subtype 4. */
705 read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
707 long long int offset = ftello (r->file);
708 double sysmis = read_float (r);
709 double highest = read_float (r);
710 double lowest = read_float (r);
712 printf ("%08llx: machine float info\n", offset);
713 if (size != 8 || count != 3)
714 sys_error (r, "Bad size (%zu) or count (%zu) on extension 4.",
717 printf ("\tsysmis: %.*g (%a)\n", DBL_DIG + 1, sysmis, sysmis);
718 if (sysmis != SYSMIS)
719 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
720 DBL_DIG + 1, sysmis, sysmis, "SYSMIS");
722 printf ("\thighest: %.*g (%a)\n", DBL_DIG + 1, highest, highest);
723 if (highest != HIGHEST)
724 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
725 DBL_DIG + 1, highest, highest, "HIGHEST");
727 printf ("\tlowest: %.*g (%a)\n", DBL_DIG + 1, lowest, lowest);
728 if (lowest != LOWEST && lowest != SYSMIS)
729 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
730 DBL_DIG + 1, lowest, lowest, "LOWEST");
734 read_extra_product_info (struct sfm_reader *r,
735 size_t size, size_t count)
737 struct text_record *text;
740 printf ("%08llx: extra product info\n", (long long int) ftello (r->file));
741 text = open_text_record (r, size, count);
742 s = text_get_all (text);
743 print_string (s, strlen (s));
744 close_text_record (text);
747 /* Read record type 7, subtype 7. */
749 read_mrsets (struct sfm_reader *r, size_t size, size_t count)
751 struct text_record *text;
753 printf ("%08llx: multiple response sets\n",
754 (long long int) ftello (r->file));
755 text = open_text_record (r, size, count);
759 enum { MRSET_MC, MRSET_MD } type;
760 bool cat_label_from_counted_values = false;
761 bool label_from_var_label = false;
764 const char *variables;
766 while (text_match (text, '\n'))
769 name = text_tokenize (text, '=');
773 if (text_match (text, 'C'))
777 if (!text_match (text, ' '))
779 sys_warn (r, "missing space following 'C' at offset %zu "
780 "in mrsets record", text_pos (text));
784 else if (text_match (text, 'D'))
788 else if (text_match (text, 'E'))
793 cat_label_from_counted_values = true;
795 if (!text_match (text, ' '))
797 sys_warn (r, "Missing space following `%c' at offset %zu "
798 "in MRSETS record", 'E', text_pos (text));
802 number = text_tokenize (text, ' ');
804 sys_warn (r, "Missing label source value "
805 "following `E' at offset %zu in MRSETS record",
807 else if (!strcmp (number, "11"))
808 label_from_var_label = true;
809 else if (strcmp (number, "1"))
810 sys_warn (r, "Unexpected label source value `%s' "
811 "following `E' at offset %zu in MRSETS record",
812 number, text_pos (text));
817 sys_warn (r, "missing `C', `D', or `E' at offset %zu "
818 "in mrsets record", text_pos (text));
822 if (type == MRSET_MD)
824 counted = text_parse_counted_string (text);
829 label = text_parse_counted_string (text);
833 variables = text_tokenize (text, '\n');
835 printf ("\t\"%s\": multiple %s set",
836 name, type == MRSET_MC ? "category" : "dichotomy");
838 printf (", counted value \"%s\"", counted);
839 if (cat_label_from_counted_values)
840 printf (", category labels from counted values");
841 if (label[0] != '\0')
842 printf (", label \"%s\"", label);
843 if (label_from_var_label)
844 printf (", label from variable label");
845 if (variables != NULL)
846 printf(", variables \"%s\"\n", variables);
848 printf(", no variables\n");
850 close_text_record (text);
853 /* Read record type 7, subtype 11. */
855 read_display_parameters (struct sfm_reader *r, size_t size, size_t count)
861 printf ("%08llx: variable display parameters\n",
862 (long long int) ftello (r->file));
865 sys_warn (r, "Bad size %zu on extension 11.", size);
866 skip_bytes (r, size * count);
870 n_vars = r->n_variables;
871 if (count == 3 * n_vars)
872 includes_width = true;
873 else if (count == 2 * n_vars)
874 includes_width = false;
877 sys_warn (r, "Extension 11 has bad count %zu (for %zu variables.",
879 skip_bytes (r, size * count);
883 for (i = 0; i < n_vars; ++i)
885 int measure = read_int (r);
886 int width = includes_width ? read_int (r) : 0;
887 int align = read_int (r);
889 printf ("\tVar #%zu: measure=%d (%s)", i, measure,
890 (measure == 1 ? "nominal"
891 : measure == 2 ? "ordinal"
892 : measure == 3 ? "scale"
895 printf (", width=%d", width);
896 printf (", align=%d (%s)\n", align,
898 : align == 1 ? "right"
899 : align == 2 ? "centre"
904 /* Reads record type 7, subtype 13, which gives the long name
905 that corresponds to each short name. */
907 read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count)
909 struct text_record *text;
913 printf ("%08llx: long variable names (short => long)\n",
914 (long long int) ftello (r->file));
915 text = open_text_record (r, size, count);
916 while (read_variable_to_value_pair (text, &var, &long_name))
917 printf ("\t%s => %s\n", var, long_name);
918 close_text_record (text);
921 /* Reads record type 7, subtype 14, which gives the real length
922 of each very long string. Rearranges DICT accordingly. */
924 read_long_string_map (struct sfm_reader *r, size_t size, size_t count)
926 struct text_record *text;
930 printf ("%08llx: very long strings (variable => length)\n",
931 (long long int) ftello (r->file));
932 text = open_text_record (r, size, count);
933 while (read_variable_to_value_pair (text, &var, &length_s))
934 printf ("\t%s => %d\n", var, atoi (length_s));
935 close_text_record (text);
939 read_attributes (struct sfm_reader *r, struct text_record *text,
940 const char *variable)
947 key = text_tokenize (text, '(');
951 for (index = 1; ; index++)
953 /* Parse the value. */
954 const char *value = text_tokenize (text, '\n');
957 sys_warn (r, "%s: Error parsing attribute value %s[%d]",
958 variable, key, index);
961 if (strlen (value) < 2
962 || value[0] != '\'' || value[strlen (value) - 1] != '\'')
963 sys_warn (r, "%s: Attribute value %s[%d] is not quoted: %s",
964 variable, key, index, value);
966 printf ("\t%s: %s[%d] = \"%.*s\"\n",
967 variable, key, index, (int) strlen (value) - 2, value + 1);
969 /* Was this the last value for this attribute? */
970 if (text_match (text, ')'))
974 if (text_match (text, '/'))
979 /* Read extended number of cases record. */
981 read_ncases64 (struct sfm_reader *r, size_t size, size_t count)
983 int64_t unknown, ncases64;
987 sys_warn (r, "Bad size %zu for extended number of cases.", size);
988 skip_bytes (r, size * count);
993 sys_warn (r, "Bad count %zu for extended number of cases.", size);
994 skip_bytes (r, size * count);
997 unknown = read_int64 (r);
998 ncases64 = read_int64 (r);
999 printf ("%08llx: extended number of cases: "
1000 "unknown=%"PRId64", ncases64=%"PRId64"\n",
1001 (long long int) ftello (r->file), unknown, ncases64);
1005 read_datafile_attributes (struct sfm_reader *r, size_t size, size_t count)
1007 struct text_record *text;
1009 printf ("%08llx: datafile attributes\n", (long long int) ftello (r->file));
1010 text = open_text_record (r, size, count);
1011 read_attributes (r, text, "datafile");
1012 close_text_record (text);
1016 read_character_encoding (struct sfm_reader *r, size_t size, size_t count)
1018 long long int posn = ftello (r->file);
1019 char *encoding = xcalloc (size, count + 1);
1020 read_string (r, encoding, count + 1);
1022 printf ("%08llx: Character Encoding: %s\n", posn, encoding);
1028 read_long_string_value_labels (struct sfm_reader *r, size_t size, size_t count)
1030 long long int start = ftello (r->file);
1032 printf ("%08llx: long string value labels\n", start);
1033 while (ftello (r->file) - start < size * count)
1035 long long posn = ftello (r->file);
1036 char var_name[ID_MAX_LEN + 1];
1042 /* Read variable name. */
1043 var_name_len = read_int (r);
1044 if (var_name_len > ID_MAX_LEN)
1045 sys_error (r, "Variable name length in long string value label "
1046 "record (%d) exceeds %d-byte limit.",
1047 var_name_len, ID_MAX_LEN);
1048 read_string (r, var_name, var_name_len + 1);
1050 /* Read width, number of values. */
1051 width = read_int (r);
1052 n_values = read_int (r);
1054 printf ("\t%08llx: %s, width %d, %d values\n",
1055 posn, var_name, width, n_values);
1058 for (i = 0; i < n_values; i++)
1066 posn = ftello (r->file);
1069 value_length = read_int (r);
1070 value = xmalloc (value_length + 1);
1071 read_string (r, value, value_length + 1);
1074 label_length = read_int (r);
1075 label = xmalloc (label_length + 1);
1076 read_string (r, label, label_length + 1);
1078 printf ("\t\t%08llx: \"%s\" (%d bytes) => \"%s\" (%d bytes)\n",
1079 posn, value, value_length, label, label_length);
1088 read_long_string_missing_values (struct sfm_reader *r,
1089 size_t size, size_t count)
1091 long long int start = ftello (r->file);
1093 printf ("%08llx: long string missing values\n", start);
1094 while (ftello (r->file) - start < size * count)
1096 long long posn = ftello (r->file);
1097 char var_name[ID_MAX_LEN + 1];
1098 uint8_t n_missing_values;
1102 /* Read variable name. */
1103 var_name_len = read_int (r);
1104 if (var_name_len > ID_MAX_LEN)
1105 sys_error (r, "Variable name length in long string value label "
1106 "record (%d) exceeds %d-byte limit.",
1107 var_name_len, ID_MAX_LEN);
1108 read_string (r, var_name, var_name_len + 1);
1110 /* Read number of values. */
1111 read_bytes (r, &n_missing_values, 1);
1113 printf ("\t%08llx: %s, %d missing values:",
1114 posn, var_name, n_missing_values);
1117 for (i = 0; i < n_missing_values; i++)
1122 posn = ftello (r->file);
1125 value_length = read_int (r);
1126 value = xmalloc (value_length + 1);
1127 read_string (r, value, value_length + 1);
1129 printf (" \"%s\"", value);
1138 hex_dump (size_t offset, const void *buffer_, size_t buffer_size)
1140 const uint8_t *buffer = buffer_;
1142 while (buffer_size > 0)
1144 size_t n = MIN (buffer_size, 16);
1147 printf ("%04zx", offset);
1148 for (i = 0; i < 16; i++)
1151 printf ("%c%02x", i == 8 ? '-' : ' ', buffer[i]);
1157 for (i = 0; i < 16; i++)
1159 unsigned char c = i < n ? buffer[i] : ' ';
1160 putchar (isprint (c) ? c : '.');
1170 /* Reads and prints any type 7 record that we don't understand. */
1172 read_unknown_extension (struct sfm_reader *r, size_t size, size_t count)
1174 unsigned char *buffer;
1177 if (size == 0 || count > 65536 / size)
1178 skip_bytes (r, size * count);
1181 buffer = xmalloc (size);
1182 for (i = 0; i < count; i++)
1184 read_bytes (r, buffer, size);
1185 hex_dump (i * size, buffer, size);
1191 buffer = xmalloc (count);
1192 read_bytes (r, buffer, count);
1193 print_string (CHAR_CAST (char *, buffer), count);
1199 read_variable_attributes (struct sfm_reader *r, size_t size, size_t count)
1201 struct text_record *text;
1203 printf ("%08llx: variable attributes\n", (long long int) ftello (r->file));
1204 text = open_text_record (r, size, count);
1207 const char *variable = text_tokenize (text, ':');
1208 if (variable == NULL || !read_attributes (r, text, variable))
1211 close_text_record (text);
1215 read_simple_compressed_data (struct sfm_reader *r, int max_cases)
1217 enum { N_OPCODES = 8 };
1218 uint8_t opcodes[N_OPCODES];
1219 long long int opcode_ofs;
1225 printf ("\n%08llx: compressed data:\n", (long long int) ftello (r->file));
1227 opcode_idx = N_OPCODES;
1230 for (case_num = 0; case_num < max_cases; case_num++)
1232 printf ("%08llx: case %d's uncompressible data begins\n",
1233 (long long int) ftello (r->file), case_num);
1234 for (i = 0; i < r->n_var_widths;)
1236 int width = r->var_widths[i];
1240 if (opcode_idx >= N_OPCODES)
1242 opcode_ofs = ftello (r->file);
1245 if (!try_read_bytes (r, opcodes, 8))
1249 read_bytes (r, opcodes, 8);
1252 opcode = opcodes[opcode_idx];
1253 printf ("%08llx: variable %d: opcode %d: ",
1254 opcode_ofs + opcode_idx, i, opcode);
1259 printf ("%.*g", DBL_DIG + 1, opcode - r->bias);
1261 printf (", but this is a string variable (width=%d)", width);
1267 printf ("ignored padding\n");
1271 printf ("end of data\n");
1275 read_bytes (r, raw_value, 8);
1276 printf ("uncompressible data: ");
1277 print_untyped_value (r, raw_value);
1285 printf (", but this is a numeric variable");
1293 printf (", but this is a string variable (width=%d)", width);
1305 read_zlib_compressed_data (struct sfm_reader *r)
1308 long long int this_ofs, next_ofs, next_len;
1309 long long int bias, zero;
1310 long long int expected_uncmp_ofs, expected_cmp_ofs;
1311 unsigned int block_size, n_blocks;
1315 ofs = ftello (r->file);
1316 printf ("\n%08llx: ZLIB compressed data header:\n", ofs);
1318 this_ofs = read_int64 (r);
1319 next_ofs = read_int64 (r);
1320 next_len = read_int64 (r);
1322 printf ("\tzheader_ofs: 0x%llx\n", this_ofs);
1323 if (this_ofs != ofs)
1324 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1325 printf ("\tztrailer_ofs: 0x%llx\n", next_ofs);
1326 printf ("\tztrailer_len: %lld\n", next_len);
1327 if (next_len < 24 || next_len % 24)
1328 printf ("\t\t(Trailer length is not a positive multiple of 24.)\n");
1330 printf ("\n%08llx: 0x%llx bytes of ZLIB compressed data\n",
1331 ofs + 8 * 3, next_ofs - (ofs + 8 * 3));
1333 skip_bytes (r, next_ofs - (ofs + 8 * 3));
1335 printf ("\n%08llx: ZLIB trailer fixed header:\n", next_ofs);
1336 bias = read_int64 (r);
1337 zero = read_int64 (r);
1338 block_size = read_int (r);
1339 n_blocks = read_int (r);
1340 printf ("\tbias: %lld\n", bias);
1341 printf ("\tzero: 0x%llx\n", zero);
1343 printf ("\t\t(Expected 0.)\n");
1344 printf ("\tblock_size: 0x%x\n", block_size);
1345 if (block_size != 0x3ff000)
1346 printf ("\t\t(Expected 0x3ff000.)\n");
1347 printf ("\tn_blocks: %u\n", n_blocks);
1348 if (n_blocks != next_len / 24 - 1)
1349 printf ("\t\t(Expected %llu.)\n", next_len / 24 - 1);
1351 expected_uncmp_ofs = ofs;
1352 expected_cmp_ofs = ofs + 24;
1353 for (i = 0; i < n_blocks; i++)
1355 long long int blockinfo_ofs = ftello (r->file);
1356 unsigned long long int uncompressed_ofs = read_int64 (r);
1357 unsigned long long int compressed_ofs = read_int64 (r);
1358 unsigned int uncompressed_size = read_int (r);
1359 unsigned int compressed_size = read_int (r);
1361 printf ("\n%08llx: ZLIB block descriptor %d\n", blockinfo_ofs, i + 1);
1363 printf ("\tuncompressed_ofs: 0x%llx\n", uncompressed_ofs);
1364 if (uncompressed_ofs != expected_uncmp_ofs)
1365 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1367 printf ("\tcompressed_ofs: 0x%llx\n", compressed_ofs);
1368 if (compressed_ofs != expected_cmp_ofs)
1369 printf ("\t\t(Expected 0x%llx.)\n", ofs + 24);
1371 printf ("\tuncompressed_size: 0x%x\n", uncompressed_size);
1372 if (i < n_blocks - 1 && uncompressed_size != block_size)
1373 printf ("\t\t(Expected 0x%x.)\n", block_size);
1375 printf ("\tcompressed_size: 0x%x\n", compressed_size);
1376 if (i == n_blocks - 1 && compressed_ofs + compressed_size != next_ofs)
1377 printf ("\t\t(This was expected to be 0x%llx.)\n",
1378 next_ofs - compressed_size);
1380 expected_uncmp_ofs += uncompressed_size;
1381 expected_cmp_ofs += compressed_size;
1385 /* Helpers for reading records that consist of structured text
1391 struct sfm_reader *reader; /* Reader. */
1392 char *buffer; /* Record contents. */
1393 size_t size; /* Size of buffer. */
1394 size_t pos; /* Current position in buffer. */
1397 /* Reads SIZE * COUNT bytes into a text record for R,
1398 and returns the new text record. */
1399 static struct text_record *
1400 open_text_record (struct sfm_reader *r, size_t size, size_t count)
1402 struct text_record *text = xmalloc (sizeof *text);
1404 if (size_overflow_p (xsum (1, xtimes (size, count))))
1405 sys_error (r, "Extension record too large.");
1407 size_t n_bytes = size * count;
1408 char *buffer = xmalloc (n_bytes + 1);
1409 read_bytes (r, buffer, n_bytes);
1410 buffer[n_bytes] = '\0';
1412 text->buffer = buffer;
1413 text->size = n_bytes;
1418 /* Closes TEXT and frees its storage.
1419 Not really needed, because the pool will free the text record anyway,
1420 but can be used to free it earlier. */
1422 close_text_record (struct text_record *text)
1424 free (text->buffer);
1429 text_tokenize (struct text_record *text, int delimiter)
1431 size_t start = text->pos;
1432 while (text->pos < text->size
1433 && text->buffer[text->pos] != delimiter
1434 && text->buffer[text->pos] != '\0')
1436 if (start == text->pos)
1438 text->buffer[text->pos++] = '\0';
1439 return &text->buffer[start];
1443 text_match (struct text_record *text, int c)
1445 if (text->pos < text->size && text->buffer[text->pos] == c)
1454 /* Reads a integer value expressed in decimal, then a space, then a string that
1455 consists of exactly as many bytes as specified by the integer, then a space,
1456 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
1457 buffer (so the caller should not free the string). */
1459 text_parse_counted_string (struct text_record *text)
1467 while (isdigit ((unsigned char) text->buffer[text->pos]))
1468 n = (n * 10) + (text->buffer[text->pos++] - '0');
1469 if (start == text->pos)
1471 sys_error (text->reader, "expecting digit at offset %zu in record",
1476 if (!text_match (text, ' '))
1478 sys_error (text->reader, "expecting space at offset %zu in record",
1483 if (text->pos + n > text->size)
1485 sys_error (text->reader, "%zu-byte string starting at offset %zu "
1486 "exceeds record length %zu", n, text->pos, text->size);
1490 s = &text->buffer[text->pos];
1493 sys_error (text->reader, "expecting space at offset %zu following "
1494 "%zu-byte string", text->pos + n, n);
1502 /* Reads a variable=value pair from TEXT.
1503 Looks up the variable in DICT and stores it into *VAR.
1504 Stores a null-terminated value into *VALUE. */
1506 read_variable_to_value_pair (struct text_record *text,
1507 char **key, char **value)
1509 *key = text_tokenize (text, '=');
1510 *value = text_tokenize (text, '\t');
1511 if (!*key || !*value)
1514 while (text->pos < text->size
1515 && (text->buffer[text->pos] == '\t'
1516 || text->buffer[text->pos] == '\0'))
1521 /* Returns the current byte offset inside the TEXT's string. */
1523 text_pos (const struct text_record *text)
1529 text_get_all (const struct text_record *text)
1531 return text->buffer;
1538 %s, a utility for dissecting system files.\n\
1539 Usage: %s [OPTION]... SYSFILE...\n\
1540 where each SYSFILE is the name of a system file.\n\
1543 --data[=MAXCASES] print (up to MAXCASES cases of) compressed data\n\
1544 --help display this help and exit\n\
1545 --version output version information and exit\n",
1546 program_name, program_name);
1549 /* Displays a corruption message. */
1551 sys_msg (struct sfm_reader *r, const char *format, va_list args)
1553 printf ("\"%s\" near offset 0x%llx: ",
1554 r->file_name, (long long int) ftello (r->file));
1555 vprintf (format, args);
1559 /* Displays a warning for the current file position. */
1561 sys_warn (struct sfm_reader *r, const char *format, ...)
1565 va_start (args, format);
1566 sys_msg (r, format, args);
1570 /* Displays an error for the current file position,
1571 marks it as in an error state,
1572 and aborts reading it using longjmp. */
1574 sys_error (struct sfm_reader *r, const char *format, ...)
1578 va_start (args, format);
1579 sys_msg (r, format, args);
1582 exit (EXIT_FAILURE);
1585 /* Reads BYTE_CNT bytes into BUF.
1586 Returns true if exactly BYTE_CNT bytes are successfully read.
1587 Aborts if an I/O error or a partial read occurs.
1588 If EOF_IS_OK, then an immediate end-of-file causes false to be
1589 returned; otherwise, immediate end-of-file causes an abort
1592 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
1593 void *buf, size_t n_bytes)
1595 size_t bytes_read = fread (buf, 1, n_bytes, r->file);
1596 if (bytes_read == n_bytes)
1598 else if (ferror (r->file))
1599 sys_error (r, "System error: %s.", strerror (errno));
1600 else if (!eof_is_ok || bytes_read != 0)
1601 sys_error (r, "Unexpected end of file.");
1606 /* Reads BYTE_CNT into BUF.
1607 Aborts upon I/O error or if end-of-file is encountered. */
1609 read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1611 read_bytes_internal (r, false, buf, n_bytes);
1614 /* Reads BYTE_CNT bytes into BUF.
1615 Returns true if exactly BYTE_CNT bytes are successfully read.
1616 Returns false if an immediate end-of-file is encountered.
1617 Aborts if an I/O error or a partial read occurs. */
1619 try_read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1621 return read_bytes_internal (r, true, buf, n_bytes);
1624 /* Reads a 32-bit signed integer from R and returns its value in
1627 read_int (struct sfm_reader *r)
1630 read_bytes (r, integer, sizeof integer);
1631 return integer_get (r->integer_format, integer, sizeof integer);
1634 /* Reads a 64-bit signed integer from R and returns its value in
1637 read_int64 (struct sfm_reader *r)
1640 read_bytes (r, integer, sizeof integer);
1641 return integer_get (r->integer_format, integer, sizeof integer);
1644 /* Reads a 64-bit floating-point number from R and returns its
1645 value in host format. */
1647 read_float (struct sfm_reader *r)
1650 read_bytes (r, number, sizeof number);
1651 return float_get_double (r->float_format, number);
1654 /* Reads exactly SIZE - 1 bytes into BUFFER
1655 and stores a null byte into BUFFER[SIZE - 1]. */
1657 read_string (struct sfm_reader *r, char *buffer, size_t size)
1660 read_bytes (r, buffer, size - 1);
1661 buffer[size - 1] = '\0';
1664 /* Skips BYTES bytes forward in R. */
1666 skip_bytes (struct sfm_reader *r, size_t bytes)
1671 size_t chunk = MIN (sizeof buffer, bytes);
1672 read_bytes (r, buffer, chunk);
1678 trim_spaces (char *s)
1680 char *end = strchr (s, '\0');
1681 while (end > s && end[-1] == ' ')
1687 print_string (const char *s, size_t len)
1689 if (memchr (s, 0, len) == 0)
1693 for (i = 0; i < len; i++)
1695 unsigned char c = s[i];
1699 else if (c == '\n' || isprint (c))
1702 printf ("\\%02x", c);
1707 hex_dump (0, s, len);