1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
28 #include "data/val-type.h"
29 #include "libpspp/cast.h"
30 #include "libpspp/compiler.h"
31 #include "libpspp/float-format.h"
32 #include "libpspp/integer-format.h"
33 #include "libpspp/misc.h"
36 #include "gl/minmax.h"
37 #include "gl/progname.h"
38 #include "gl/version-etc.h"
39 #include "gl/xalloc.h"
53 const char *file_name;
56 int n_variable_records, n_variables;
59 size_t n_var_widths, allocated_var_widths;
61 enum integer_format integer_format;
62 enum float_format float_format;
64 enum compression compression;
68 static int character_code;
69 static char *encoding;
71 static void read_header (struct sfm_reader *);
72 static void read_variable_record (struct sfm_reader *);
73 static void read_value_label_record (struct sfm_reader *);
74 static void read_document_record (struct sfm_reader *);
75 static void read_extension_record (struct sfm_reader *);
76 static void read_machine_integer_info (struct sfm_reader *,
77 size_t size, size_t count);
78 static void read_machine_float_info (struct sfm_reader *,
79 size_t size, size_t count);
80 static void read_extra_product_info (struct sfm_reader *,
81 size_t size, size_t count);
82 static void read_variable_sets (struct sfm_reader *, size_t size, size_t count);
83 static void read_mrsets (struct sfm_reader *, size_t size, size_t count);
84 static void read_display_parameters (struct sfm_reader *,
85 size_t size, size_t count);
86 static void read_long_var_name_map (struct sfm_reader *r,
87 size_t size, size_t count);
88 static void read_long_string_map (struct sfm_reader *r,
89 size_t size, size_t count);
90 static void read_datafile_attributes (struct sfm_reader *r,
91 size_t size, size_t count);
92 static void read_variable_attributes (struct sfm_reader *r,
93 size_t size, size_t count);
94 static void read_ncases64 (struct sfm_reader *, size_t size, size_t count);
95 static void read_character_encoding (struct sfm_reader *r,
96 size_t size, size_t count);
97 static void read_long_string_value_labels (struct sfm_reader *r,
98 size_t size, size_t count);
99 static void read_long_string_missing_values (struct sfm_reader *r,
100 size_t size, size_t count);
101 static void read_unknown_extension (struct sfm_reader *,
102 size_t size, size_t count);
103 static void read_simple_compressed_data (struct sfm_reader *, int max_cases);
104 static void read_zlib_compressed_data (struct sfm_reader *);
106 static struct text_record *open_text_record (
107 struct sfm_reader *, size_t size, size_t count);
108 static void close_text_record (struct text_record *);
109 static bool read_variable_to_value_pair (struct text_record *,
110 char **key, char **value);
111 static char *text_tokenize (struct text_record *, int delimiter);
112 static bool text_match (struct text_record *text, int c);
113 static const char *text_parse_counted_string (struct text_record *);
114 static size_t text_pos (const struct text_record *);
115 static const char *text_get_all (const struct text_record *);
117 static void usage (void);
118 static void sys_warn (struct sfm_reader *, const char *, ...)
119 PRINTF_FORMAT (2, 3);
120 static void sys_error (struct sfm_reader *, const char *, ...)
124 static void read_bytes (struct sfm_reader *, void *, size_t);
125 static bool try_read_bytes (struct sfm_reader *, void *, size_t);
126 static int read_int (struct sfm_reader *);
127 static int64_t read_int64 (struct sfm_reader *);
128 static double read_float (struct sfm_reader *);
129 static void read_string (struct sfm_reader *, char *, size_t);
130 static void skip_bytes (struct sfm_reader *, size_t);
131 static void trim_spaces (char *);
133 static void print_string (const char *s, size_t len);
136 main (int argc, char *argv[])
142 set_program_name (argv[0]);
146 static const struct option long_options[] =
148 { "data", optional_argument, NULL, 'd' },
149 { "help", no_argument, NULL, 'h' },
150 { "version", no_argument, NULL, 'v' },
151 { NULL, 0, NULL, 0 },
156 c = getopt_long (argc, argv, "d::hv", long_options, NULL);
163 max_cases = optarg ? atoi (optarg) : INT_MAX;
167 version_etc (stdout, "pspp-dump-sav", PACKAGE_NAME, PACKAGE_VERSION,
168 "Ben Pfaff", "John Darrington", NULL_SENTINEL);
181 error (1, 0, "at least one non-option argument is required; "
182 "use --help for help");
184 for (i = optind; i < argc; i++)
188 r.file_name = argv[i];
189 r.file = fopen (r.file_name, "rb");
191 error (EXIT_FAILURE, errno, "error opening `%s'", r.file_name);
192 r.n_variable_records = 0;
195 r.allocated_var_widths = 0;
197 r.compression = COMP_NONE;
199 if (argc - optind > 1)
200 printf ("Reading \"%s\":\n", r.file_name);
203 while ((rec_type = read_int (&r)) != 999)
208 read_variable_record (&r);
212 read_value_label_record (&r);
216 sys_error (&r, "Misplaced type 4 record.");
219 read_document_record (&r);
223 read_extension_record (&r);
227 sys_error (&r, "Unrecognized record type %d.", rec_type);
230 printf ("%08llx: end-of-dictionary record "
231 "(first byte of data at %08llx)\n",
232 (long long int) ftello (r.file),
233 (long long int) ftello (r.file) + 4);
235 printf ("Character Encoding: %s (%d)\n", encoding ? encoding : "none", character_code);
236 if (r.compression == COMP_SIMPLE)
239 read_simple_compressed_data (&r, max_cases);
241 else if (r.compression == COMP_ZLIB)
242 read_zlib_compressed_data (&r);
253 read_header (struct sfm_reader *r)
256 char eye_catcher[61];
257 uint8_t raw_layout_code[4];
260 int32_t weight_index;
263 char creation_date[10];
264 char creation_time[9];
268 read_string (r, rec_type, sizeof rec_type);
269 read_string (r, eye_catcher, sizeof eye_catcher);
271 if (!strcmp ("$FL2", rec_type))
273 else if (!strcmp ("$FL3", rec_type))
276 sys_error (r, "This is not an SPSS system file.");
278 /* Identify integer format. */
279 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
280 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
282 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
284 || (r->integer_format != INTEGER_MSB_FIRST
285 && r->integer_format != INTEGER_LSB_FIRST))
286 sys_error (r, "This is not an SPSS system file.");
287 layout_code = integer_get (r->integer_format,
288 raw_layout_code, sizeof raw_layout_code);
290 read_int (r); /* Nominal case size (not actually useful). */
291 compressed = read_int (r);
292 weight_index = read_int (r);
293 ncases = read_int (r);
298 r->compression = COMP_NONE;
299 else if (compressed == 1)
300 r->compression = COMP_SIMPLE;
301 else if (compressed != 0)
302 sys_error (r, "SAV file header has invalid compression value "
303 "%"PRId32".", compressed);
308 r->compression = COMP_ZLIB;
310 sys_error (r, "ZSAV file header has invalid compression value "
311 "%"PRId32".", compressed);
314 /* Identify floating-point format and obtain compression bias. */
315 read_bytes (r, raw_bias, sizeof raw_bias);
316 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
318 sys_warn (r, "Compression bias is not the usual value of 100, or system "
319 "file uses unrecognized floating-point format.");
320 if (r->integer_format == INTEGER_MSB_FIRST)
321 r->float_format = FLOAT_IEEE_DOUBLE_BE;
323 r->float_format = FLOAT_IEEE_DOUBLE_LE;
325 if ((r->integer_format == INTEGER_MSB_FIRST && r->float_format != FLOAT_IEEE_DOUBLE_BE) ||
326 (r->integer_format == INTEGER_LSB_FIRST && r->float_format != FLOAT_IEEE_DOUBLE_LE))
328 printf ("unexpected floating-point format\n");
331 if (r->float_format != FLOAT_IEEE_DOUBLE_LE && r->float_format != FLOAT_IEEE_DOUBLE_BE)
333 printf ("non-IEEE format\n");
336 printf ("IEEE format\n");
338 r->bias = float_get_double (r->float_format, raw_bias);
340 read_string (r, creation_date, sizeof creation_date);
341 read_string (r, creation_time, sizeof creation_time);
342 read_string (r, file_label, sizeof file_label);
343 trim_spaces (file_label);
346 printf ("File header record:\n");
347 printf ("\t%17s: %s\n", "Product name", eye_catcher);
348 printf ("\t%17s: %"PRId32"\n", "Layout code", layout_code);
349 printf ("\t%17s: %"PRId32" (%s)\n", "Compressed",
351 r->compression == COMP_NONE ? "no compression"
352 : r->compression == COMP_SIMPLE ? "simple compression"
353 : r->compression == COMP_ZLIB ? "ZLIB compression"
355 printf ("\t%17s: %"PRId32"\n", "Weight index", weight_index);
356 printf ("\t%17s: %"PRId32"\n", "Number of cases", ncases);
357 printf ("\t%17s: %.*g\n", "Compression bias", DBL_DIG + 1, r->bias);
358 printf ("\t%17s: %s\n", "Creation date", creation_date);
359 printf ("\t%17s: %s\n", "Creation time", creation_time);
360 printf ("\t%17s: \"%s\"\n", "File label", file_label);
364 format_name (int format)
366 switch ((format >> 16) & 0xff)
369 case 2: return "AHEX";
370 case 3: return "COMMA";
371 case 4: return "DOLLAR";
374 case 7: return "PIBHEX";
376 case 9: return "PIB";
377 case 10: return "PK";
378 case 11: return "RB";
379 case 12: return "RBHEX";
383 case 20: return "DATE";
384 case 21: return "TIME";
385 case 22: return "DATETIME";
386 case 23: return "ADATE";
387 case 24: return "JDATE";
388 case 25: return "DTIME";
389 case 26: return "WKDAY";
390 case 27: return "MONTH";
391 case 28: return "MOYR";
392 case 29: return "QYR";
393 case 30: return "WKYR";
394 case 31: return "PCT";
395 case 32: return "DOT";
396 case 33: return "CCA";
397 case 34: return "CCB";
398 case 35: return "CCC";
399 case 36: return "CCD";
400 case 37: return "CCE";
401 case 38: return "EDATE";
402 case 39: return "SDATE";
403 case 40: return "MTIME";
404 case 41: return "YMDHMS";
405 default: return "invalid";
409 /* Reads a variable (type 2) record from R and adds the
410 corresponding variable to DICT.
411 Also skips past additional variable records for long string
414 read_variable_record (struct sfm_reader *r)
417 int has_variable_label;
418 int missing_value_code;
423 printf ("%08llx: variable record #%d\n",
424 (long long int) ftello (r->file), ++r->n_variable_records);
426 width = read_int (r);
427 has_variable_label = read_int (r);
428 missing_value_code = read_int (r);
429 print_format = read_int (r);
430 write_format = read_int (r);
431 read_string (r, name, sizeof name);
432 name[strcspn (name, " ")] = '\0';
437 if (r->n_var_widths >= r->allocated_var_widths)
438 r->var_widths = x2nrealloc (r->var_widths, &r->allocated_var_widths,
439 sizeof *r->var_widths);
440 r->var_widths[r->n_var_widths++] = width;
442 printf ("\tWidth: %d (%s)\n",
445 : width == 0 ? "numeric"
446 : "long string continuation record");
447 printf ("\tVariable label: %d\n", has_variable_label);
448 printf ("\tMissing values code: %d (%s)\n", missing_value_code,
449 (missing_value_code == 0 ? "no missing values"
450 : missing_value_code == 1 ? "one missing value"
451 : missing_value_code == 2 ? "two missing values"
452 : missing_value_code == 3 ? "three missing values"
453 : missing_value_code == -2 ? "one missing value range"
454 : missing_value_code == -3 ? "one missing value, one range"
456 printf ("\tPrint format: %06x (%s%d.%d)\n",
457 print_format, format_name (print_format),
458 (print_format >> 8) & 0xff, print_format & 0xff);
459 printf ("\tWrite format: %06x (%s%d.%d)\n",
460 write_format, format_name (write_format),
461 (write_format >> 8) & 0xff, write_format & 0xff);
462 printf ("\tName: %s\n", name);
464 /* Get variable label, if any. */
465 if (has_variable_label != 0 && has_variable_label != 1)
466 sys_error (r, "Variable label indicator field is not 0 or 1.");
467 if (has_variable_label == 1)
469 long long int offset = ftello (r->file);
470 enum { MAX_LABEL_LEN = 65536 };
472 size_t len = read_int (r);
473 size_t read_len = MIN (MAX_LABEL_LEN, len);
474 char *label = xmalloc (read_len + 1);
475 read_string (r, label, read_len + 1);
476 printf("\t%08llx Variable label: \"%s\"\n", offset, label);
479 /* Skip label padding up to multiple of 4 bytes. */
480 skip_bytes (r, ROUND_UP (len, 4) - len);
483 /* Set missing values. */
484 if (missing_value_code != 0)
488 printf ("\t%08llx Missing values:", (long long int) ftello (r->file));
491 if (missing_value_code < -3 || missing_value_code > 3
492 || missing_value_code == -1)
493 sys_error (r, "Numeric missing value indicator field is not "
494 "-3, -2, 0, 1, 2, or 3.");
495 if (missing_value_code < 0)
497 double low = read_float (r);
498 double high = read_float (r);
499 printf (" %.*g...%.*g", DBL_DIG + 1, low, DBL_DIG + 1, high);
500 missing_value_code = -missing_value_code - 2;
502 for (i = 0; i < missing_value_code; i++)
503 printf (" %.*g", DBL_DIG + 1, read_float (r));
507 if (missing_value_code < 1 || missing_value_code > 3)
508 sys_error (r, "String missing value indicator field is not "
510 for (i = 0; i < missing_value_code; i++)
513 read_string (r, string, sizeof string);
514 printf (" \"%s\"", string);
522 print_untyped_value (struct sfm_reader *r, char raw_value[8])
527 value = float_get_double (r->float_format, raw_value);
528 for (n_printable = 0; n_printable < 8; n_printable++)
529 if (!isprint (raw_value[n_printable]))
532 printf ("%.*g/\"%.*s\"", DBL_DIG + 1, value, n_printable, raw_value);
535 /* Reads value labels from sysfile R and inserts them into the
536 associated dictionary. */
538 read_value_label_record (struct sfm_reader *r)
540 int n_labels, n_vars;
543 printf ("%08llx: value labels record\n", (long long int) ftello (r->file));
545 /* Read number of labels. */
546 n_labels = read_int (r);
547 for (i = 0; i < n_labels; i++)
550 unsigned char label_len;
554 read_bytes (r, raw_value, sizeof raw_value);
556 /* Read label length. */
557 read_bytes (r, &label_len, sizeof label_len);
558 padded_len = ROUND_UP (label_len + 1, 8);
560 /* Read label, padding. */
561 read_bytes (r, label, padded_len - 1);
562 label[label_len] = 0;
565 print_untyped_value (r, raw_value);
566 printf (": \"%s\"\n", label);
569 /* Now, read the type 4 record that has the list of variables
570 to which the value labels are to be applied. */
572 /* Read record type of type 4 record. */
573 if (read_int (r) != 4)
574 sys_error (r, "Variable index record (type 4) does not immediately "
575 "follow value label record (type 3) as it should.");
577 /* Read number of variables associated with value label from type 4
579 printf ("\t%08llx: apply to variables", (long long int) ftello (r->file));
580 n_vars = read_int (r);
581 for (i = 0; i < n_vars; i++)
582 printf (" #%d", read_int (r));
587 read_document_record (struct sfm_reader *r)
592 printf ("%08llx: document record\n", (long long int) ftello (r->file));
593 n_lines = read_int (r);
594 printf ("\t%d lines of documents\n", n_lines);
596 for (i = 0; i < n_lines; i++)
599 printf ("\t%08llx: ", (long long int) ftello (r->file));
600 read_string (r, line, sizeof line);
602 printf ("line %d: \"%s\"\n", i, line);
607 read_extension_record (struct sfm_reader *r)
609 long long int offset = ftello (r->file);
610 int subtype = read_int (r);
611 size_t size = read_int (r);
612 size_t count = read_int (r);
613 size_t bytes = size * count;
615 printf ("%08llx: Record 7, subtype %d, size=%zu, count=%zu\n",
616 offset, subtype, size, count);
621 read_machine_integer_info (r, size, count);
625 read_machine_float_info (r, size, count);
629 read_variable_sets (r, size, count);
633 /* DATE variable information. We don't use it yet, but we
639 read_mrsets (r, size, count);
643 read_extra_product_info (r, size, count);
647 read_display_parameters (r, size, count);
651 read_long_var_name_map (r, size, count);
655 read_long_string_map (r, size, count);
659 read_ncases64 (r, size, count);
663 read_datafile_attributes (r, size, count);
667 read_variable_attributes (r, size, count);
671 read_character_encoding (r, size, count);
675 read_long_string_value_labels (r, size, count);
679 read_long_string_missing_values (r, size, count);
683 sys_warn (r, "Unrecognized record type 7, subtype %d.", subtype);
684 read_unknown_extension (r, size, count);
688 skip_bytes (r, bytes);
692 read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count)
694 long long int offset = ftello (r->file);
695 int version_major = read_int (r);
696 int version_minor = read_int (r);
697 int version_revision = read_int (r);
698 int machine_code = read_int (r);
699 int float_representation = read_int (r);
700 int compression_code = read_int (r);
701 int integer_representation = read_int (r);
702 character_code = read_int (r);
704 printf ("%08llx: machine integer info\n", offset);
705 if (size != 4 || count != 8)
706 sys_error (r, "Bad size (%zu) or count (%zu) field on record type 7, "
707 "subtype 3.", size, count);
709 printf ("\tVersion: %d.%d.%d\n",
710 version_major, version_minor, version_revision);
711 printf ("\tMachine code: %d\n", machine_code);
712 printf ("\tFloating point representation: %d (%s)\n",
713 float_representation,
714 float_representation == 1 ? "IEEE 754"
715 : float_representation == 2 ? "IBM 370"
716 : float_representation == 3 ? "DEC VAX"
718 printf ("\tCompression code: %d\n", compression_code);
719 printf ("\tEndianness: %d (%s)\n", integer_representation,
720 integer_representation == 1 ? "big"
721 : integer_representation == 2 ? "little" : "unknown");
722 printf ("\tCharacter code: %d\n", character_code);
725 /* Read record type 7, subtype 4. */
727 read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
729 long long int offset = ftello (r->file);
730 double sysmis = read_float (r);
731 double highest = read_float (r);
732 double lowest = read_float (r);
734 printf ("%08llx: machine float info\n", offset);
735 if (size != 8 || count != 3)
736 sys_error (r, "Bad size (%zu) or count (%zu) on extension 4.",
739 printf ("\tsysmis: %.*g (%a)\n", DBL_DIG + 1, sysmis, sysmis);
740 if (sysmis != SYSMIS)
741 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
742 DBL_DIG + 1, sysmis, sysmis, "SYSMIS");
744 printf ("\thighest: %.*g (%a)\n", DBL_DIG + 1, highest, highest);
745 if (highest != HIGHEST)
746 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
747 DBL_DIG + 1, highest, highest, "HIGHEST");
749 printf ("\tlowest: %.*g (%a)\n", DBL_DIG + 1, lowest, lowest);
750 if (lowest != LOWEST && lowest != SYSMIS)
751 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
752 DBL_DIG + 1, lowest, lowest, "LOWEST");
755 /* Read record type 7, subtype 5. */
757 read_variable_sets (struct sfm_reader *r, size_t size, size_t count)
759 printf ("%08llx: variable sets\n", (long long int) ftello (r->file));
760 struct text_record *text = open_text_record (r, size, count);
763 while (text_match (text, '\n'))
766 const char *set = text_tokenize (text, '=');
770 /* Always present even for an empty set. */
771 text_match (text, ' ');
773 char *variables = text_tokenize (text, '\n');
775 printf ("\tset \"%s\" is empty\n", set);
778 size_t length = strlen (variables);
779 if (variables[length - 1] == '\r')
780 variables[length - 1] = '\0';
781 printf ("\tset \"%s\" contains \"%s\"\n", set, variables);
784 close_text_record (text);
788 read_extra_product_info (struct sfm_reader *r,
789 size_t size, size_t count)
791 struct text_record *text;
794 printf ("%08llx: extra product info\n", (long long int) ftello (r->file));
795 text = open_text_record (r, size, count);
796 s = text_get_all (text);
797 print_string (s, strlen (s));
798 close_text_record (text);
801 /* Read record type 7, subtype 7. */
803 read_mrsets (struct sfm_reader *r, size_t size, size_t count)
805 struct text_record *text;
807 printf ("%08llx: multiple response sets\n",
808 (long long int) ftello (r->file));
809 text = open_text_record (r, size, count);
813 enum { MRSET_MC, MRSET_MD } type;
814 bool cat_label_from_counted_values = false;
815 bool label_from_var_label = false;
818 const char *variables;
820 while (text_match (text, '\n'))
823 name = text_tokenize (text, '=');
827 if (text_match (text, 'C'))
831 if (!text_match (text, ' '))
833 sys_warn (r, "missing space following 'C' at offset %zu "
834 "in mrsets record", text_pos (text));
838 else if (text_match (text, 'D'))
842 else if (text_match (text, 'E'))
847 cat_label_from_counted_values = true;
849 if (!text_match (text, ' '))
851 sys_warn (r, "Missing space following `%c' at offset %zu "
852 "in MRSETS record", 'E', text_pos (text));
856 number = text_tokenize (text, ' ');
858 sys_warn (r, "Missing label source value "
859 "following `E' at offset %zu in MRSETS record",
861 else if (!strcmp (number, "11"))
862 label_from_var_label = true;
863 else if (strcmp (number, "1"))
864 sys_warn (r, "Unexpected label source value `%s' "
865 "following `E' at offset %zu in MRSETS record",
866 number, text_pos (text));
871 sys_warn (r, "missing `C', `D', or `E' at offset %zu "
872 "in mrsets record", text_pos (text));
876 if (type == MRSET_MD)
878 counted = text_parse_counted_string (text);
883 label = text_parse_counted_string (text);
887 variables = text_tokenize (text, '\n');
889 printf ("\t\"%s\": multiple %s set",
890 name, type == MRSET_MC ? "category" : "dichotomy");
892 printf (", counted value \"%s\"", counted);
893 if (cat_label_from_counted_values)
894 printf (", category labels from counted values");
895 if (label[0] != '\0')
896 printf (", label \"%s\"", label);
897 if (label_from_var_label)
898 printf (", label from variable label");
899 if (variables != NULL)
900 printf(", variables \"%s\"\n", variables);
902 printf(", no variables\n");
904 close_text_record (text);
907 /* Read record type 7, subtype 11. */
909 read_display_parameters (struct sfm_reader *r, size_t size, size_t count)
915 printf ("%08llx: variable display parameters\n",
916 (long long int) ftello (r->file));
919 sys_warn (r, "Bad size %zu on extension 11.", size);
920 skip_bytes (r, size * count);
924 n_vars = r->n_variables;
925 if (count == 3 * n_vars)
926 includes_width = true;
927 else if (count == 2 * n_vars)
928 includes_width = false;
931 sys_warn (r, "Extension 11 has bad count %zu (for %zu variables).",
933 skip_bytes (r, size * count);
937 for (i = 0; i < n_vars; ++i)
939 int measure = read_int (r);
940 int width = includes_width ? read_int (r) : 0;
941 int align = read_int (r);
943 printf ("\tVar #%zu: measure=%d (%s)", i, measure,
944 (measure == 1 ? "nominal"
945 : measure == 2 ? "ordinal"
946 : measure == 3 ? "scale"
949 printf (", width=%d", width);
950 printf (", align=%d (%s)\n", align,
952 : align == 1 ? "right"
953 : align == 2 ? "centre"
958 /* Reads record type 7, subtype 13, which gives the long name
959 that corresponds to each short name. */
961 read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count)
963 struct text_record *text;
967 printf ("%08llx: long variable names (short => long)\n",
968 (long long int) ftello (r->file));
969 text = open_text_record (r, size, count);
970 while (read_variable_to_value_pair (text, &var, &long_name))
971 printf ("\t%s => %s\n", var, long_name);
972 close_text_record (text);
975 /* Reads record type 7, subtype 14, which gives the real length
976 of each very long string. Rearranges DICT accordingly. */
978 read_long_string_map (struct sfm_reader *r, size_t size, size_t count)
980 struct text_record *text;
984 printf ("%08llx: very long strings (variable => length)\n",
985 (long long int) ftello (r->file));
986 text = open_text_record (r, size, count);
987 while (read_variable_to_value_pair (text, &var, &length_s))
988 printf ("\t%s => %d\n", var, atoi (length_s));
989 close_text_record (text);
993 read_attributes (struct sfm_reader *r, struct text_record *text,
994 const char *variable)
1001 key = text_tokenize (text, '(');
1005 for (index = 1; ; index++)
1007 /* Parse the value. */
1008 const char *value = text_tokenize (text, '\n');
1011 sys_warn (r, "%s: Error parsing attribute value %s[%d]",
1012 variable, key, index);
1015 if (strlen (value) < 2
1016 || value[0] != '\'' || value[strlen (value) - 1] != '\'')
1017 sys_warn (r, "%s: Attribute value %s[%d] is not quoted: %s",
1018 variable, key, index, value);
1020 printf ("\t%s: %s[%d] = \"%.*s\"\n",
1021 variable, key, index, (int) strlen (value) - 2, value + 1);
1023 /* Was this the last value for this attribute? */
1024 if (text_match (text, ')'))
1028 if (text_match (text, '/'))
1033 /* Read extended number of cases record. */
1035 read_ncases64 (struct sfm_reader *r, size_t size, size_t count)
1037 int64_t unknown, ncases64;
1041 sys_warn (r, "Bad size %zu for extended number of cases.", size);
1042 skip_bytes (r, size * count);
1047 sys_warn (r, "Bad count %zu for extended number of cases.", size);
1048 skip_bytes (r, size * count);
1051 unknown = read_int64 (r);
1052 ncases64 = read_int64 (r);
1053 printf ("%08llx: extended number of cases: "
1054 "unknown=%"PRId64", ncases64=%"PRId64"\n",
1055 (long long int) ftello (r->file), unknown, ncases64);
1059 read_datafile_attributes (struct sfm_reader *r, size_t size, size_t count)
1061 struct text_record *text;
1063 printf ("%08llx: datafile attributes\n", (long long int) ftello (r->file));
1064 text = open_text_record (r, size, count);
1065 read_attributes (r, text, "datafile");
1066 close_text_record (text);
1070 read_character_encoding (struct sfm_reader *r, size_t size, size_t count)
1072 encoding = xcalloc (size, count + 1);
1073 read_string (r, encoding, count + 1);
1078 read_long_string_value_labels (struct sfm_reader *r, size_t size, size_t count)
1080 long long int start = ftello (r->file);
1082 printf ("%08llx: long string value labels\n", start);
1083 while (ftello (r->file) - start < size * count)
1085 long long posn = ftello (r->file);
1086 char var_name[ID_MAX_LEN + 1];
1092 /* Read variable name. */
1093 var_name_len = read_int (r);
1094 if (var_name_len > ID_MAX_LEN)
1095 sys_error (r, "Variable name length in long string value label "
1096 "record (%d) exceeds %d-byte limit.",
1097 var_name_len, ID_MAX_LEN);
1098 read_string (r, var_name, var_name_len + 1);
1100 /* Read width, number of values. */
1101 width = read_int (r);
1102 n_values = read_int (r);
1104 printf ("\t%08llx: %s, width %d, %d values\n",
1105 posn, var_name, width, n_values);
1108 for (i = 0; i < n_values; i++)
1116 posn = ftello (r->file);
1119 value_length = read_int (r);
1120 value = xmalloc (value_length + 1);
1121 read_string (r, value, value_length + 1);
1124 label_length = read_int (r);
1125 label = xmalloc (label_length + 1);
1126 read_string (r, label, label_length + 1);
1128 printf ("\t\t%08llx: \"%s\" (%d bytes) => \"%s\" (%d bytes)\n",
1129 posn, value, value_length, label, label_length);
1138 read_long_string_missing_values (struct sfm_reader *r,
1139 size_t size, size_t count)
1141 long long int start = ftello (r->file);
1143 printf ("%08llx: long string missing values\n", start);
1144 while (ftello (r->file) - start < size * count)
1146 long long posn = ftello (r->file);
1148 /* Read variable name. */
1149 int var_name_len = read_int (r);
1150 if (var_name_len > ID_MAX_LEN)
1151 sys_error (r, "Variable name length in long string missing value "
1152 "record (%d) exceeds %d-byte limit.",
1153 var_name_len, ID_MAX_LEN);
1154 char var_name[ID_MAX_LEN + 1];
1155 read_string (r, var_name, var_name_len + 1);
1157 /* Read number of values. */
1158 uint8_t n_missing_values;
1159 read_bytes (r, &n_missing_values, 1);
1161 /* Read value length. */
1162 int value_length = read_int (r);
1164 printf ("\t%08llx: %s, %d missing values, each %d bytes:",
1165 posn, var_name, n_missing_values, value_length);
1168 for (int i = 0; i < n_missing_values; i++)
1171 char *value = xmalloc (value_length + 1);
1172 read_string (r, value, value_length + 1);
1174 printf (" \"%s\"", value);
1183 hex_dump (size_t offset, const void *buffer_, size_t buffer_size)
1185 const uint8_t *buffer = buffer_;
1187 while (buffer_size > 0)
1189 size_t n = MIN (buffer_size, 16);
1192 printf ("%04zx", offset);
1193 for (i = 0; i < 16; i++)
1196 printf ("%c%02x", i == 8 ? '-' : ' ', buffer[i]);
1202 for (i = 0; i < 16; i++)
1204 unsigned char c = i < n ? buffer[i] : ' ';
1205 putchar (isprint (c) ? c : '.');
1215 /* Reads and prints any type 7 record that we don't understand. */
1217 read_unknown_extension (struct sfm_reader *r, size_t size, size_t count)
1219 unsigned char *buffer;
1222 if (size == 0 || count > 65536 / size)
1223 skip_bytes (r, size * count);
1226 buffer = xmalloc (size);
1227 for (i = 0; i < count; i++)
1229 read_bytes (r, buffer, size);
1230 hex_dump (i * size, buffer, size);
1236 buffer = xmalloc (count);
1237 read_bytes (r, buffer, count);
1238 print_string (CHAR_CAST (char *, buffer), count);
1244 read_variable_attributes (struct sfm_reader *r, size_t size, size_t count)
1246 struct text_record *text;
1248 printf ("%08llx: variable attributes\n", (long long int) ftello (r->file));
1249 text = open_text_record (r, size, count);
1252 const char *variable = text_tokenize (text, ':');
1253 if (variable == NULL || !read_attributes (r, text, variable))
1256 close_text_record (text);
1260 read_simple_compressed_data (struct sfm_reader *r, int max_cases)
1262 enum { N_OPCODES = 8 };
1263 uint8_t opcodes[N_OPCODES];
1264 long long int opcode_ofs;
1270 printf ("\n%08llx: compressed data:\n", (long long int) ftello (r->file));
1272 opcode_idx = N_OPCODES;
1275 for (case_num = 0; case_num < max_cases; case_num++)
1277 printf ("%08llx: case %d's uncompressible data begins\n",
1278 (long long int) ftello (r->file), case_num);
1279 for (i = 0; i < r->n_var_widths;)
1281 int width = r->var_widths[i];
1285 if (opcode_idx >= N_OPCODES)
1287 opcode_ofs = ftello (r->file);
1290 if (!try_read_bytes (r, opcodes, 8))
1294 read_bytes (r, opcodes, 8);
1297 opcode = opcodes[opcode_idx];
1298 printf ("%08llx: variable %d: opcode %d: ",
1299 opcode_ofs + opcode_idx, i, opcode);
1304 printf ("%.*g", DBL_DIG + 1, opcode - r->bias);
1306 printf (", but this is a string variable (width=%d)", width);
1312 printf ("ignored padding\n");
1316 printf ("end of data\n");
1320 read_bytes (r, raw_value, 8);
1321 printf ("uncompressible data: ");
1322 print_untyped_value (r, raw_value);
1330 printf (", but this is a numeric variable");
1338 printf (", but this is a string variable (width=%d)", width);
1350 read_zlib_compressed_data (struct sfm_reader *r)
1353 long long int this_ofs, next_ofs, next_len;
1354 long long int bias, zero;
1355 long long int expected_uncmp_ofs, expected_cmp_ofs;
1356 unsigned int block_size, n_blocks;
1360 ofs = ftello (r->file);
1361 printf ("\n%08llx: ZLIB compressed data header:\n", ofs);
1363 this_ofs = read_int64 (r);
1364 next_ofs = read_int64 (r);
1365 next_len = read_int64 (r);
1367 printf ("\tzheader_ofs: 0x%llx\n", this_ofs);
1368 if (this_ofs != ofs)
1369 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1370 printf ("\tztrailer_ofs: 0x%llx\n", next_ofs);
1371 printf ("\tztrailer_len: %lld\n", next_len);
1372 if (next_len < 24 || next_len % 24)
1373 printf ("\t\t(Trailer length is not a positive multiple of 24.)\n");
1375 printf ("\n%08llx: 0x%llx bytes of ZLIB compressed data\n",
1376 ofs + 8 * 3, next_ofs - (ofs + 8 * 3));
1378 skip_bytes (r, next_ofs - (ofs + 8 * 3));
1380 printf ("\n%08llx: ZLIB trailer fixed header:\n", next_ofs);
1381 bias = read_int64 (r);
1382 zero = read_int64 (r);
1383 block_size = read_int (r);
1384 n_blocks = read_int (r);
1385 printf ("\tbias: %lld\n", bias);
1386 printf ("\tzero: 0x%llx\n", zero);
1388 printf ("\t\t(Expected 0.)\n");
1389 printf ("\tblock_size: 0x%x\n", block_size);
1390 if (block_size != 0x3ff000)
1391 printf ("\t\t(Expected 0x3ff000.)\n");
1392 printf ("\tn_blocks: %u\n", n_blocks);
1393 if (n_blocks != next_len / 24 - 1)
1394 printf ("\t\t(Expected %llu.)\n", next_len / 24 - 1);
1396 expected_uncmp_ofs = ofs;
1397 expected_cmp_ofs = ofs + 24;
1398 for (i = 0; i < n_blocks; i++)
1400 long long int blockinfo_ofs = ftello (r->file);
1401 unsigned long long int uncompressed_ofs = read_int64 (r);
1402 unsigned long long int compressed_ofs = read_int64 (r);
1403 unsigned int uncompressed_size = read_int (r);
1404 unsigned int compressed_size = read_int (r);
1406 printf ("\n%08llx: ZLIB block descriptor %d\n", blockinfo_ofs, i + 1);
1408 printf ("\tuncompressed_ofs: 0x%llx\n", uncompressed_ofs);
1409 if (uncompressed_ofs != expected_uncmp_ofs)
1410 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1412 printf ("\tcompressed_ofs: 0x%llx\n", compressed_ofs);
1413 if (compressed_ofs != expected_cmp_ofs)
1414 printf ("\t\t(Expected 0x%llx.)\n", ofs + 24);
1416 printf ("\tuncompressed_size: 0x%x\n", uncompressed_size);
1417 if (i < n_blocks - 1 && uncompressed_size != block_size)
1418 printf ("\t\t(Expected 0x%x.)\n", block_size);
1420 printf ("\tcompressed_size: 0x%x\n", compressed_size);
1421 if (i == n_blocks - 1 && compressed_ofs + compressed_size != next_ofs)
1422 printf ("\t\t(This was expected to be 0x%llx.)\n",
1423 next_ofs - compressed_size);
1425 expected_uncmp_ofs += uncompressed_size;
1426 expected_cmp_ofs += compressed_size;
1430 /* Helpers for reading records that consist of structured text
1436 struct sfm_reader *reader; /* Reader. */
1437 char *buffer; /* Record contents. */
1438 size_t size; /* Size of buffer. */
1439 size_t pos; /* Current position in buffer. */
1442 /* Reads SIZE * COUNT bytes into a text record for R,
1443 and returns the new text record. */
1444 static struct text_record *
1445 open_text_record (struct sfm_reader *r, size_t size, size_t count)
1447 struct text_record *text = xmalloc (sizeof *text);
1449 if (size_overflow_p (xsum (1, xtimes (size, count))))
1450 sys_error (r, "Extension record too large.");
1452 size_t n_bytes = size * count;
1453 char *buffer = xmalloc (n_bytes + 1);
1454 read_bytes (r, buffer, n_bytes);
1455 buffer[n_bytes] = '\0';
1457 text->buffer = buffer;
1458 text->size = n_bytes;
1463 /* Closes TEXT and frees its storage.
1464 Not really needed, because the pool will free the text record anyway,
1465 but can be used to free it earlier. */
1467 close_text_record (struct text_record *text)
1469 free (text->buffer);
1474 text_tokenize (struct text_record *text, int delimiter)
1476 size_t start = text->pos;
1477 while (text->pos < text->size
1478 && text->buffer[text->pos] != delimiter
1479 && text->buffer[text->pos] != '\0')
1481 if (start == text->pos)
1483 text->buffer[text->pos++] = '\0';
1484 return &text->buffer[start];
1488 text_match (struct text_record *text, int c)
1490 if (text->pos < text->size && text->buffer[text->pos] == c)
1499 /* Reads a integer value expressed in decimal, then a space, then a string that
1500 consists of exactly as many bytes as specified by the integer, then a space,
1501 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
1502 buffer (so the caller should not free the string). */
1504 text_parse_counted_string (struct text_record *text)
1512 while (isdigit ((unsigned char) text->buffer[text->pos]))
1513 n = (n * 10) + (text->buffer[text->pos++] - '0');
1514 if (start == text->pos)
1516 sys_error (text->reader, "expecting digit at offset %zu in record",
1521 if (!text_match (text, ' '))
1523 sys_error (text->reader, "expecting space at offset %zu in record",
1528 if (text->pos + n > text->size)
1530 sys_error (text->reader, "%zu-byte string starting at offset %zu "
1531 "exceeds record length %zu", n, text->pos, text->size);
1535 s = &text->buffer[text->pos];
1538 sys_error (text->reader, "expecting space at offset %zu following "
1539 "%zu-byte string", text->pos + n, n);
1547 /* Reads a variable=value pair from TEXT.
1548 Looks up the variable in DICT and stores it into *VAR.
1549 Stores a null-terminated value into *VALUE. */
1551 read_variable_to_value_pair (struct text_record *text,
1552 char **key, char **value)
1554 *key = text_tokenize (text, '=');
1555 *value = text_tokenize (text, '\t');
1556 if (!*key || !*value)
1559 while (text->pos < text->size
1560 && (text->buffer[text->pos] == '\t'
1561 || text->buffer[text->pos] == '\0'))
1566 /* Returns the current byte offset inside the TEXT's string. */
1568 text_pos (const struct text_record *text)
1574 text_get_all (const struct text_record *text)
1576 return text->buffer;
1583 %s, a utility for dissecting system files.\n\
1584 Usage: %s [OPTION]... SYSFILE...\n\
1585 where each SYSFILE is the name of a system file.\n\
1588 --data[=MAXCASES] print (up to MAXCASES cases of) compressed data\n\
1589 --help display this help and exit\n\
1590 --version output version information and exit\n",
1591 program_name, program_name);
1594 /* Displays a corruption message. */
1596 sys_msg (struct sfm_reader *r, const char *format, va_list args)
1598 printf ("\"%s\" near offset 0x%llx: ",
1599 r->file_name, (long long int) ftello (r->file));
1600 vprintf (format, args);
1604 /* Displays a warning for the current file position. */
1606 sys_warn (struct sfm_reader *r, const char *format, ...)
1610 va_start (args, format);
1611 sys_msg (r, format, args);
1615 /* Displays an error for the current file position,
1616 marks it as in an error state,
1617 and aborts reading it using longjmp. */
1619 sys_error (struct sfm_reader *r, const char *format, ...)
1623 va_start (args, format);
1624 sys_msg (r, format, args);
1627 exit (EXIT_FAILURE);
1630 /* Reads BYTE_CNT bytes into BUF.
1631 Returns true if exactly BYTE_CNT bytes are successfully read.
1632 Aborts if an I/O error or a partial read occurs.
1633 If EOF_IS_OK, then an immediate end-of-file causes false to be
1634 returned; otherwise, immediate end-of-file causes an abort
1637 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
1638 void *buf, size_t n_bytes)
1640 size_t bytes_read = fread (buf, 1, n_bytes, r->file);
1641 if (bytes_read == n_bytes)
1643 else if (ferror (r->file))
1644 sys_error (r, "System error: %s.", strerror (errno));
1645 else if (!eof_is_ok || bytes_read != 0)
1646 sys_error (r, "Unexpected end of file.");
1651 /* Reads BYTE_CNT into BUF.
1652 Aborts upon I/O error or if end-of-file is encountered. */
1654 read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1656 read_bytes_internal (r, false, buf, n_bytes);
1659 /* Reads BYTE_CNT bytes into BUF.
1660 Returns true if exactly BYTE_CNT bytes are successfully read.
1661 Returns false if an immediate end-of-file is encountered.
1662 Aborts if an I/O error or a partial read occurs. */
1664 try_read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1666 return read_bytes_internal (r, true, buf, n_bytes);
1669 /* Reads a 32-bit signed integer from R and returns its value in
1672 read_int (struct sfm_reader *r)
1675 read_bytes (r, integer, sizeof integer);
1676 return integer_get (r->integer_format, integer, sizeof integer);
1679 /* Reads a 64-bit signed integer from R and returns its value in
1682 read_int64 (struct sfm_reader *r)
1685 read_bytes (r, integer, sizeof integer);
1686 return integer_get (r->integer_format, integer, sizeof integer);
1689 /* Reads a 64-bit floating-point number from R and returns its
1690 value in host format. */
1692 read_float (struct sfm_reader *r)
1695 read_bytes (r, number, sizeof number);
1696 return float_get_double (r->float_format, number);
1699 /* Reads exactly SIZE - 1 bytes into BUFFER
1700 and stores a null byte into BUFFER[SIZE - 1]. */
1702 read_string (struct sfm_reader *r, char *buffer, size_t size)
1705 read_bytes (r, buffer, size - 1);
1706 buffer[size - 1] = '\0';
1709 /* Skips BYTES bytes forward in R. */
1711 skip_bytes (struct sfm_reader *r, size_t bytes)
1716 size_t chunk = MIN (sizeof buffer, bytes);
1717 read_bytes (r, buffer, chunk);
1723 trim_spaces (char *s)
1725 char *end = strchr (s, '\0');
1726 while (end > s && end[-1] == ' ')
1732 print_string (const char *s, size_t len)
1734 if (memchr (s, 0, len) == 0)
1738 for (i = 0; i < len; i++)
1740 unsigned char c = s[i];
1744 else if (c == '\n' || isprint (c))
1747 printf ("\\%02x", c);
1752 hex_dump (0, s, len);