1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
28 #include "data/val-type.h"
29 #include "libpspp/cast.h"
30 #include "libpspp/compiler.h"
31 #include "libpspp/float-format.h"
32 #include "libpspp/integer-format.h"
33 #include "libpspp/misc.h"
36 #include "gl/minmax.h"
37 #include "gl/progname.h"
38 #include "gl/version-etc.h"
39 #include "gl/xalloc.h"
53 const char *file_name;
56 int n_variable_records, n_variables;
59 size_t n_var_widths, allocated_var_widths;
61 enum integer_format integer_format;
62 enum float_format float_format;
64 enum compression compression;
68 static void read_header (struct sfm_reader *);
69 static void read_variable_record (struct sfm_reader *);
70 static void read_value_label_record (struct sfm_reader *);
71 static void read_document_record (struct sfm_reader *);
72 static void read_extension_record (struct sfm_reader *);
73 static void read_machine_integer_info (struct sfm_reader *,
74 size_t size, size_t count);
75 static void read_machine_float_info (struct sfm_reader *,
76 size_t size, size_t count);
77 static void read_extra_product_info (struct sfm_reader *,
78 size_t size, size_t count);
79 static void read_variable_sets (struct sfm_reader *, size_t size, size_t count);
80 static void read_mrsets (struct sfm_reader *, size_t size, size_t count);
81 static void read_display_parameters (struct sfm_reader *,
82 size_t size, size_t count);
83 static void read_long_var_name_map (struct sfm_reader *r,
84 size_t size, size_t count);
85 static void read_long_string_map (struct sfm_reader *r,
86 size_t size, size_t count);
87 static void read_datafile_attributes (struct sfm_reader *r,
88 size_t size, size_t count);
89 static void read_variable_attributes (struct sfm_reader *r,
90 size_t size, size_t count);
91 static void read_ncases64 (struct sfm_reader *, size_t size, size_t count);
92 static void read_character_encoding (struct sfm_reader *r,
93 size_t size, size_t count);
94 static void read_long_string_value_labels (struct sfm_reader *r,
95 size_t size, size_t count);
96 static void read_long_string_missing_values (struct sfm_reader *r,
97 size_t size, size_t count);
98 static void read_unknown_extension (struct sfm_reader *,
99 size_t size, size_t count);
100 static void read_simple_compressed_data (struct sfm_reader *, int max_cases);
101 static void read_zlib_compressed_data (struct sfm_reader *);
103 static struct text_record *open_text_record (
104 struct sfm_reader *, size_t size, size_t count);
105 static void close_text_record (struct text_record *);
106 static bool read_variable_to_value_pair (struct text_record *,
107 char **key, char **value);
108 static char *text_tokenize (struct text_record *, int delimiter);
109 static bool text_match (struct text_record *text, int c);
110 static const char *text_parse_counted_string (struct text_record *);
111 static size_t text_pos (const struct text_record *);
112 static const char *text_get_all (const struct text_record *);
114 static void usage (void);
115 static void sys_warn (struct sfm_reader *, const char *, ...)
116 PRINTF_FORMAT (2, 3);
117 static void sys_error (struct sfm_reader *, const char *, ...)
121 static void read_bytes (struct sfm_reader *, void *, size_t);
122 static bool try_read_bytes (struct sfm_reader *, void *, size_t);
123 static int read_int (struct sfm_reader *);
124 static int64_t read_int64 (struct sfm_reader *);
125 static double read_float (struct sfm_reader *);
126 static void read_string (struct sfm_reader *, char *, size_t);
127 static void skip_bytes (struct sfm_reader *, size_t);
128 static void trim_spaces (char *);
130 static void print_string (const char *s, size_t len);
133 main (int argc, char *argv[])
139 set_program_name (argv[0]);
143 static const struct option long_options[] =
145 { "data", optional_argument, NULL, 'd' },
146 { "help", no_argument, NULL, 'h' },
147 { "version", no_argument, NULL, 'v' },
148 { NULL, 0, NULL, 0 },
153 c = getopt_long (argc, argv, "d::hv", long_options, NULL);
160 max_cases = optarg ? atoi (optarg) : INT_MAX;
164 version_etc (stdout, "pspp-dump-sav", PACKAGE_NAME, PACKAGE_VERSION,
165 "Ben Pfaff", "John Darrington", NULL_SENTINEL);
178 error (1, 0, "at least one non-option argument is required; "
179 "use --help for help");
181 for (i = optind; i < argc; i++)
185 r.file_name = argv[i];
186 r.file = fopen (r.file_name, "rb");
188 error (EXIT_FAILURE, errno, "error opening `%s'", r.file_name);
189 r.n_variable_records = 0;
192 r.allocated_var_widths = 0;
194 r.compression = COMP_NONE;
196 if (argc - optind > 1)
197 printf ("Reading \"%s\":\n", r.file_name);
200 while ((rec_type = read_int (&r)) != 999)
205 read_variable_record (&r);
209 read_value_label_record (&r);
213 sys_error (&r, "Misplaced type 4 record.");
216 read_document_record (&r);
220 read_extension_record (&r);
224 sys_error (&r, "Unrecognized record type %d.", rec_type);
227 printf ("%08llx: end-of-dictionary record "
228 "(first byte of data at %08llx)\n",
229 (long long int) ftello (r.file),
230 (long long int) ftello (r.file) + 4);
232 if (r.compression == COMP_SIMPLE)
235 read_simple_compressed_data (&r, max_cases);
237 else if (r.compression == COMP_ZLIB)
238 read_zlib_compressed_data (&r);
249 read_header (struct sfm_reader *r)
252 char eye_catcher[61];
253 uint8_t raw_layout_code[4];
256 int32_t weight_index;
259 char creation_date[10];
260 char creation_time[9];
264 read_string (r, rec_type, sizeof rec_type);
265 read_string (r, eye_catcher, sizeof eye_catcher);
267 if (!strcmp ("$FL2", rec_type))
269 else if (!strcmp ("$FL3", rec_type))
272 sys_error (r, "This is not an SPSS system file.");
274 /* Identify integer format. */
275 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
276 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
278 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
280 || (r->integer_format != INTEGER_MSB_FIRST
281 && r->integer_format != INTEGER_LSB_FIRST))
282 sys_error (r, "This is not an SPSS system file.");
283 layout_code = integer_get (r->integer_format,
284 raw_layout_code, sizeof raw_layout_code);
286 read_int (r); /* Nominal case size (not actually useful). */
287 compressed = read_int (r);
288 weight_index = read_int (r);
289 ncases = read_int (r);
294 r->compression = COMP_NONE;
295 else if (compressed == 1)
296 r->compression = COMP_SIMPLE;
297 else if (compressed != 0)
298 sys_error (r, "SAV file header has invalid compression value "
299 "%"PRId32".", compressed);
304 r->compression = COMP_ZLIB;
306 sys_error (r, "ZSAV file header has invalid compression value "
307 "%"PRId32".", compressed);
310 /* Identify floating-point format and obtain compression bias. */
311 read_bytes (r, raw_bias, sizeof raw_bias);
312 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
314 sys_warn (r, "Compression bias is not the usual value of 100, or system "
315 "file uses unrecognized floating-point format.");
316 if (r->integer_format == INTEGER_MSB_FIRST)
317 r->float_format = FLOAT_IEEE_DOUBLE_BE;
319 r->float_format = FLOAT_IEEE_DOUBLE_LE;
321 if ((r->integer_format == INTEGER_MSB_FIRST && r->float_format != FLOAT_IEEE_DOUBLE_BE) ||
322 (r->integer_format == INTEGER_LSB_FIRST && r->float_format != FLOAT_IEEE_DOUBLE_LE))
324 printf ("unexpected floating-point format\n");
327 if (r->float_format != FLOAT_IEEE_DOUBLE_LE && r->float_format != FLOAT_IEEE_DOUBLE_BE)
329 printf ("non-IEEE format\n");
332 printf ("IEEE format\n");
334 r->bias = float_get_double (r->float_format, raw_bias);
336 read_string (r, creation_date, sizeof creation_date);
337 read_string (r, creation_time, sizeof creation_time);
338 read_string (r, file_label, sizeof file_label);
339 trim_spaces (file_label);
342 printf ("File header record:\n");
343 printf ("\t%17s: %s\n", "Product name", eye_catcher);
344 printf ("\t%17s: %"PRId32"\n", "Layout code", layout_code);
345 printf ("\t%17s: %"PRId32" (%s)\n", "Compressed",
347 r->compression == COMP_NONE ? "no compression"
348 : r->compression == COMP_SIMPLE ? "simple compression"
349 : r->compression == COMP_ZLIB ? "ZLIB compression"
351 printf ("\t%17s: %"PRId32"\n", "Weight index", weight_index);
352 printf ("\t%17s: %"PRId32"\n", "Number of cases", ncases);
353 printf ("\t%17s: %.*g\n", "Compression bias", DBL_DIG + 1, r->bias);
354 printf ("\t%17s: %s\n", "Creation date", creation_date);
355 printf ("\t%17s: %s\n", "Creation time", creation_time);
356 printf ("\t%17s: \"%s\"\n", "File label", file_label);
360 format_name (int format)
362 switch ((format >> 16) & 0xff)
365 case 2: return "AHEX";
366 case 3: return "COMMA";
367 case 4: return "DOLLAR";
370 case 7: return "PIBHEX";
372 case 9: return "PIB";
373 case 10: return "PK";
374 case 11: return "RB";
375 case 12: return "RBHEX";
379 case 20: return "DATE";
380 case 21: return "TIME";
381 case 22: return "DATETIME";
382 case 23: return "ADATE";
383 case 24: return "JDATE";
384 case 25: return "DTIME";
385 case 26: return "WKDAY";
386 case 27: return "MONTH";
387 case 28: return "MOYR";
388 case 29: return "QYR";
389 case 30: return "WKYR";
390 case 31: return "PCT";
391 case 32: return "DOT";
392 case 33: return "CCA";
393 case 34: return "CCB";
394 case 35: return "CCC";
395 case 36: return "CCD";
396 case 37: return "CCE";
397 case 38: return "EDATE";
398 case 39: return "SDATE";
399 case 40: return "MTIME";
400 case 41: return "YMDHMS";
401 default: return "invalid";
405 /* Reads a variable (type 2) record from R and adds the
406 corresponding variable to DICT.
407 Also skips past additional variable records for long string
410 read_variable_record (struct sfm_reader *r)
413 int has_variable_label;
414 int missing_value_code;
419 printf ("%08llx: variable record #%d\n",
420 (long long int) ftello (r->file), ++r->n_variable_records);
422 width = read_int (r);
423 has_variable_label = read_int (r);
424 missing_value_code = read_int (r);
425 print_format = read_int (r);
426 write_format = read_int (r);
427 read_string (r, name, sizeof name);
428 name[strcspn (name, " ")] = '\0';
433 if (r->n_var_widths >= r->allocated_var_widths)
434 r->var_widths = x2nrealloc (r->var_widths, &r->allocated_var_widths,
435 sizeof *r->var_widths);
436 r->var_widths[r->n_var_widths++] = width;
438 printf ("\tWidth: %d (%s)\n",
441 : width == 0 ? "numeric"
442 : "long string continuation record");
443 printf ("\tVariable label: %d\n", has_variable_label);
444 printf ("\tMissing values code: %d (%s)\n", missing_value_code,
445 (missing_value_code == 0 ? "no missing values"
446 : missing_value_code == 1 ? "one missing value"
447 : missing_value_code == 2 ? "two missing values"
448 : missing_value_code == 3 ? "three missing values"
449 : missing_value_code == -2 ? "one missing value range"
450 : missing_value_code == -3 ? "one missing value, one range"
452 printf ("\tPrint format: %06x (%s%d.%d)\n",
453 print_format, format_name (print_format),
454 (print_format >> 8) & 0xff, print_format & 0xff);
455 printf ("\tWrite format: %06x (%s%d.%d)\n",
456 write_format, format_name (write_format),
457 (write_format >> 8) & 0xff, write_format & 0xff);
458 printf ("\tName: %s\n", name);
460 /* Get variable label, if any. */
461 if (has_variable_label != 0 && has_variable_label != 1)
462 sys_error (r, "Variable label indicator field is not 0 or 1.");
463 if (has_variable_label == 1)
465 long long int offset = ftello (r->file);
466 enum { MAX_LABEL_LEN = 65536 };
468 size_t len = read_int (r);
469 size_t read_len = MIN (MAX_LABEL_LEN, len);
470 char *label = xmalloc (read_len + 1);
471 read_string (r, label, read_len + 1);
472 printf("\t%08llx Variable label: \"%s\"\n", offset, label);
475 /* Skip label padding up to multiple of 4 bytes. */
476 skip_bytes (r, ROUND_UP (len, 4) - len);
479 /* Set missing values. */
480 if (missing_value_code != 0)
484 printf ("\t%08llx Missing values:", (long long int) ftello (r->file));
487 if (missing_value_code < -3 || missing_value_code > 3
488 || missing_value_code == -1)
489 sys_error (r, "Numeric missing value indicator field is not "
490 "-3, -2, 0, 1, 2, or 3.");
491 if (missing_value_code < 0)
493 double low = read_float (r);
494 double high = read_float (r);
495 printf (" %.*g...%.*g", DBL_DIG + 1, low, DBL_DIG + 1, high);
496 missing_value_code = -missing_value_code - 2;
498 for (i = 0; i < missing_value_code; i++)
499 printf (" %.*g", DBL_DIG + 1, read_float (r));
503 if (missing_value_code < 1 || missing_value_code > 3)
504 sys_error (r, "String missing value indicator field is not "
506 for (i = 0; i < missing_value_code; i++)
509 read_string (r, string, sizeof string);
510 printf (" \"%s\"", string);
518 print_untyped_value (struct sfm_reader *r, char raw_value[8])
523 value = float_get_double (r->float_format, raw_value);
524 for (n_printable = 0; n_printable < 8; n_printable++)
525 if (!isprint (raw_value[n_printable]))
528 printf ("%.*g/\"%.*s\"", DBL_DIG + 1, value, n_printable, raw_value);
531 /* Reads value labels from sysfile R and inserts them into the
532 associated dictionary. */
534 read_value_label_record (struct sfm_reader *r)
536 int n_labels, n_vars;
539 printf ("%08llx: value labels record\n", (long long int) ftello (r->file));
541 /* Read number of labels. */
542 n_labels = read_int (r);
543 for (i = 0; i < n_labels; i++)
546 unsigned char label_len;
550 read_bytes (r, raw_value, sizeof raw_value);
552 /* Read label length. */
553 read_bytes (r, &label_len, sizeof label_len);
554 padded_len = ROUND_UP (label_len + 1, 8);
556 /* Read label, padding. */
557 read_bytes (r, label, padded_len - 1);
558 label[label_len] = 0;
561 print_untyped_value (r, raw_value);
562 printf (": \"%s\"\n", label);
565 /* Now, read the type 4 record that has the list of variables
566 to which the value labels are to be applied. */
568 /* Read record type of type 4 record. */
569 if (read_int (r) != 4)
570 sys_error (r, "Variable index record (type 4) does not immediately "
571 "follow value label record (type 3) as it should.");
573 /* Read number of variables associated with value label from type 4
575 printf ("\t%08llx: apply to variables", (long long int) ftello (r->file));
576 n_vars = read_int (r);
577 for (i = 0; i < n_vars; i++)
578 printf (" #%d", read_int (r));
583 read_document_record (struct sfm_reader *r)
588 printf ("%08llx: document record\n", (long long int) ftello (r->file));
589 n_lines = read_int (r);
590 printf ("\t%d lines of documents\n", n_lines);
592 for (i = 0; i < n_lines; i++)
595 printf ("\t%08llx: ", (long long int) ftello (r->file));
596 read_string (r, line, sizeof line);
598 printf ("line %d: \"%s\"\n", i, line);
603 read_extension_record (struct sfm_reader *r)
605 long long int offset = ftello (r->file);
606 int subtype = read_int (r);
607 size_t size = read_int (r);
608 size_t count = read_int (r);
609 size_t bytes = size * count;
611 printf ("%08llx: Record 7, subtype %d, size=%zu, count=%zu\n",
612 offset, subtype, size, count);
617 read_machine_integer_info (r, size, count);
621 read_machine_float_info (r, size, count);
625 read_variable_sets (r, size, count);
629 /* DATE variable information. We don't use it yet, but we
635 read_mrsets (r, size, count);
639 read_extra_product_info (r, size, count);
643 read_display_parameters (r, size, count);
647 read_long_var_name_map (r, size, count);
651 read_long_string_map (r, size, count);
655 read_ncases64 (r, size, count);
659 read_datafile_attributes (r, size, count);
663 read_variable_attributes (r, size, count);
667 read_character_encoding (r, size, count);
671 read_long_string_value_labels (r, size, count);
675 read_long_string_missing_values (r, size, count);
679 sys_warn (r, "Unrecognized record type 7, subtype %d.", subtype);
680 read_unknown_extension (r, size, count);
684 skip_bytes (r, bytes);
688 read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count)
690 long long int offset = ftello (r->file);
691 int version_major = read_int (r);
692 int version_minor = read_int (r);
693 int version_revision = read_int (r);
694 int machine_code = read_int (r);
695 int float_representation = read_int (r);
696 int compression_code = read_int (r);
697 int integer_representation = read_int (r);
698 int character_code = read_int (r);
700 printf ("%08llx: machine integer info\n", offset);
701 if (size != 4 || count != 8)
702 sys_error (r, "Bad size (%zu) or count (%zu) field on record type 7, "
703 "subtype 3.", size, count);
705 printf ("\tVersion: %d.%d.%d\n",
706 version_major, version_minor, version_revision);
707 printf ("\tMachine code: %d\n", machine_code);
708 printf ("\tFloating point representation: %d (%s)\n",
709 float_representation,
710 float_representation == 1 ? "IEEE 754"
711 : float_representation == 2 ? "IBM 370"
712 : float_representation == 3 ? "DEC VAX"
714 printf ("\tCompression code: %d\n", compression_code);
715 printf ("\tEndianness: %d (%s)\n", integer_representation,
716 integer_representation == 1 ? "big"
717 : integer_representation == 2 ? "little" : "unknown");
718 printf ("\tCharacter code: %d\n", character_code);
721 /* Read record type 7, subtype 4. */
723 read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
725 long long int offset = ftello (r->file);
726 double sysmis = read_float (r);
727 double highest = read_float (r);
728 double lowest = read_float (r);
730 printf ("%08llx: machine float info\n", offset);
731 if (size != 8 || count != 3)
732 sys_error (r, "Bad size (%zu) or count (%zu) on extension 4.",
735 printf ("\tsysmis: %.*g (%a)\n", DBL_DIG + 1, sysmis, sysmis);
736 if (sysmis != SYSMIS)
737 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
738 DBL_DIG + 1, sysmis, sysmis, "SYSMIS");
740 printf ("\thighest: %.*g (%a)\n", DBL_DIG + 1, highest, highest);
741 if (highest != HIGHEST)
742 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
743 DBL_DIG + 1, highest, highest, "HIGHEST");
745 printf ("\tlowest: %.*g (%a)\n", DBL_DIG + 1, lowest, lowest);
746 if (lowest != LOWEST && lowest != SYSMIS)
747 sys_warn (r, "File specifies unexpected value %.*g (%a) as %s.",
748 DBL_DIG + 1, lowest, lowest, "LOWEST");
751 /* Read record type 7, subtype 5. */
753 read_variable_sets (struct sfm_reader *r, size_t size, size_t count)
755 printf ("%08llx: variable sets\n", (long long int) ftello (r->file));
756 struct text_record *text = open_text_record (r, size, count);
759 while (text_match (text, '\n'))
762 const char *set = text_tokenize (text, '=');
766 /* Always present even for an empty set. */
767 text_match (text, ' ');
769 char *variables = text_tokenize (text, '\n');
771 printf ("\tset \"%s\" is empty\n", set);
774 size_t length = strlen (variables);
775 if (variables[length - 1] == '\r')
776 variables[length - 1] = '\0';
777 printf ("\tset \"%s\" contains \"%s\"\n", set, variables);
780 close_text_record (text);
784 read_extra_product_info (struct sfm_reader *r,
785 size_t size, size_t count)
787 struct text_record *text;
790 printf ("%08llx: extra product info\n", (long long int) ftello (r->file));
791 text = open_text_record (r, size, count);
792 s = text_get_all (text);
793 print_string (s, strlen (s));
794 close_text_record (text);
797 /* Read record type 7, subtype 7. */
799 read_mrsets (struct sfm_reader *r, size_t size, size_t count)
801 struct text_record *text;
803 printf ("%08llx: multiple response sets\n",
804 (long long int) ftello (r->file));
805 text = open_text_record (r, size, count);
809 enum { MRSET_MC, MRSET_MD } type;
810 bool cat_label_from_counted_values = false;
811 bool label_from_var_label = false;
814 const char *variables;
816 while (text_match (text, '\n'))
819 name = text_tokenize (text, '=');
823 if (text_match (text, 'C'))
827 if (!text_match (text, ' '))
829 sys_warn (r, "missing space following 'C' at offset %zu "
830 "in mrsets record", text_pos (text));
834 else if (text_match (text, 'D'))
838 else if (text_match (text, 'E'))
843 cat_label_from_counted_values = true;
845 if (!text_match (text, ' '))
847 sys_warn (r, "Missing space following `%c' at offset %zu "
848 "in MRSETS record", 'E', text_pos (text));
852 number = text_tokenize (text, ' ');
854 sys_warn (r, "Missing label source value "
855 "following `E' at offset %zu in MRSETS record",
857 else if (!strcmp (number, "11"))
858 label_from_var_label = true;
859 else if (strcmp (number, "1"))
860 sys_warn (r, "Unexpected label source value `%s' "
861 "following `E' at offset %zu in MRSETS record",
862 number, text_pos (text));
867 sys_warn (r, "missing `C', `D', or `E' at offset %zu "
868 "in mrsets record", text_pos (text));
872 if (type == MRSET_MD)
874 counted = text_parse_counted_string (text);
879 label = text_parse_counted_string (text);
883 variables = text_tokenize (text, '\n');
885 printf ("\t\"%s\": multiple %s set",
886 name, type == MRSET_MC ? "category" : "dichotomy");
888 printf (", counted value \"%s\"", counted);
889 if (cat_label_from_counted_values)
890 printf (", category labels from counted values");
891 if (label[0] != '\0')
892 printf (", label \"%s\"", label);
893 if (label_from_var_label)
894 printf (", label from variable label");
895 if (variables != NULL)
896 printf(", variables \"%s\"\n", variables);
898 printf(", no variables\n");
900 close_text_record (text);
903 /* Read record type 7, subtype 11. */
905 read_display_parameters (struct sfm_reader *r, size_t size, size_t count)
911 printf ("%08llx: variable display parameters\n",
912 (long long int) ftello (r->file));
915 sys_warn (r, "Bad size %zu on extension 11.", size);
916 skip_bytes (r, size * count);
920 n_vars = r->n_variables;
921 if (count == 3 * n_vars)
922 includes_width = true;
923 else if (count == 2 * n_vars)
924 includes_width = false;
927 sys_warn (r, "Extension 11 has bad count %zu (for %zu variables).",
929 skip_bytes (r, size * count);
933 for (i = 0; i < n_vars; ++i)
935 int measure = read_int (r);
936 int width = includes_width ? read_int (r) : 0;
937 int align = read_int (r);
939 printf ("\tVar #%zu: measure=%d (%s)", i, measure,
940 (measure == 1 ? "nominal"
941 : measure == 2 ? "ordinal"
942 : measure == 3 ? "scale"
945 printf (", width=%d", width);
946 printf (", align=%d (%s)\n", align,
948 : align == 1 ? "right"
949 : align == 2 ? "centre"
954 /* Reads record type 7, subtype 13, which gives the long name
955 that corresponds to each short name. */
957 read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count)
959 struct text_record *text;
963 printf ("%08llx: long variable names (short => long)\n",
964 (long long int) ftello (r->file));
965 text = open_text_record (r, size, count);
966 while (read_variable_to_value_pair (text, &var, &long_name))
967 printf ("\t%s => %s\n", var, long_name);
968 close_text_record (text);
971 /* Reads record type 7, subtype 14, which gives the real length
972 of each very long string. Rearranges DICT accordingly. */
974 read_long_string_map (struct sfm_reader *r, size_t size, size_t count)
976 struct text_record *text;
980 printf ("%08llx: very long strings (variable => length)\n",
981 (long long int) ftello (r->file));
982 text = open_text_record (r, size, count);
983 while (read_variable_to_value_pair (text, &var, &length_s))
984 printf ("\t%s => %d\n", var, atoi (length_s));
985 close_text_record (text);
989 read_attributes (struct sfm_reader *r, struct text_record *text,
990 const char *variable)
997 key = text_tokenize (text, '(');
1001 for (index = 1; ; index++)
1003 /* Parse the value. */
1004 const char *value = text_tokenize (text, '\n');
1007 sys_warn (r, "%s: Error parsing attribute value %s[%d]",
1008 variable, key, index);
1011 if (strlen (value) < 2
1012 || value[0] != '\'' || value[strlen (value) - 1] != '\'')
1013 sys_warn (r, "%s: Attribute value %s[%d] is not quoted: %s",
1014 variable, key, index, value);
1016 printf ("\t%s: %s[%d] = \"%.*s\"\n",
1017 variable, key, index, (int) strlen (value) - 2, value + 1);
1019 /* Was this the last value for this attribute? */
1020 if (text_match (text, ')'))
1024 if (text_match (text, '/'))
1029 /* Read extended number of cases record. */
1031 read_ncases64 (struct sfm_reader *r, size_t size, size_t count)
1033 int64_t unknown, ncases64;
1037 sys_warn (r, "Bad size %zu for extended number of cases.", size);
1038 skip_bytes (r, size * count);
1043 sys_warn (r, "Bad count %zu for extended number of cases.", size);
1044 skip_bytes (r, size * count);
1047 unknown = read_int64 (r);
1048 ncases64 = read_int64 (r);
1049 printf ("%08llx: extended number of cases: "
1050 "unknown=%"PRId64", ncases64=%"PRId64"\n",
1051 (long long int) ftello (r->file), unknown, ncases64);
1055 read_datafile_attributes (struct sfm_reader *r, size_t size, size_t count)
1057 struct text_record *text;
1059 printf ("%08llx: datafile attributes\n", (long long int) ftello (r->file));
1060 text = open_text_record (r, size, count);
1061 read_attributes (r, text, "datafile");
1062 close_text_record (text);
1066 read_character_encoding (struct sfm_reader *r, size_t size, size_t count)
1068 long long int posn = ftello (r->file);
1069 char *encoding = xcalloc (size, count + 1);
1070 read_string (r, encoding, count + 1);
1072 printf ("%08llx: Character Encoding: %s\n", posn, encoding);
1078 read_long_string_value_labels (struct sfm_reader *r, size_t size, size_t count)
1080 long long int start = ftello (r->file);
1082 printf ("%08llx: long string value labels\n", start);
1083 while (ftello (r->file) - start < size * count)
1085 long long posn = ftello (r->file);
1086 char var_name[ID_MAX_LEN + 1];
1092 /* Read variable name. */
1093 var_name_len = read_int (r);
1094 if (var_name_len > ID_MAX_LEN)
1095 sys_error (r, "Variable name length in long string value label "
1096 "record (%d) exceeds %d-byte limit.",
1097 var_name_len, ID_MAX_LEN);
1098 read_string (r, var_name, var_name_len + 1);
1100 /* Read width, number of values. */
1101 width = read_int (r);
1102 n_values = read_int (r);
1104 printf ("\t%08llx: %s, width %d, %d values\n",
1105 posn, var_name, width, n_values);
1108 for (i = 0; i < n_values; i++)
1116 posn = ftello (r->file);
1119 value_length = read_int (r);
1120 value = xmalloc (value_length + 1);
1121 read_string (r, value, value_length + 1);
1124 label_length = read_int (r);
1125 label = xmalloc (label_length + 1);
1126 read_string (r, label, label_length + 1);
1128 printf ("\t\t%08llx: \"%s\" (%d bytes) => \"%s\" (%d bytes)\n",
1129 posn, value, value_length, label, label_length);
1138 read_long_string_missing_values (struct sfm_reader *r,
1139 size_t size, size_t count)
1141 long long int start = ftello (r->file);
1143 printf ("%08llx: long string missing values\n", start);
1144 while (ftello (r->file) - start < size * count)
1146 long long posn = ftello (r->file);
1148 /* Read variable name. */
1149 int var_name_len = read_int (r);
1150 if (var_name_len > ID_MAX_LEN)
1151 sys_error (r, "Variable name length in long string missing value "
1152 "record (%d) exceeds %d-byte limit.",
1153 var_name_len, ID_MAX_LEN);
1154 char var_name[ID_MAX_LEN + 1];
1155 read_string (r, var_name, var_name_len + 1);
1157 /* Read number of values. */
1158 uint8_t n_missing_values;
1159 read_bytes (r, &n_missing_values, 1);
1161 /* Read value length. */
1162 int value_length = read_int (r);
1164 printf ("\t%08llx: %s, %d missing values, each %d bytes:",
1165 posn, var_name, n_missing_values, value_length);
1168 for (int i = 0; i < n_missing_values; i++)
1171 char *value = xmalloc (value_length + 1);
1172 read_string (r, value, value_length + 1);
1174 printf (" \"%s\"", value);
1183 hex_dump (size_t offset, const void *buffer_, size_t buffer_size)
1185 const uint8_t *buffer = buffer_;
1187 while (buffer_size > 0)
1189 size_t n = MIN (buffer_size, 16);
1192 printf ("%04zx", offset);
1193 for (i = 0; i < 16; i++)
1196 printf ("%c%02x", i == 8 ? '-' : ' ', buffer[i]);
1202 for (i = 0; i < 16; i++)
1204 unsigned char c = i < n ? buffer[i] : ' ';
1205 putchar (isprint (c) ? c : '.');
1215 /* Reads and prints any type 7 record that we don't understand. */
1217 read_unknown_extension (struct sfm_reader *r, size_t size, size_t count)
1219 unsigned char *buffer;
1222 if (size == 0 || count > 65536 / size)
1223 skip_bytes (r, size * count);
1226 buffer = xmalloc (size);
1227 for (i = 0; i < count; i++)
1229 read_bytes (r, buffer, size);
1230 hex_dump (i * size, buffer, size);
1236 buffer = xmalloc (count);
1237 read_bytes (r, buffer, count);
1238 print_string (CHAR_CAST (char *, buffer), count);
1244 read_variable_attributes (struct sfm_reader *r, size_t size, size_t count)
1246 struct text_record *text;
1248 printf ("%08llx: variable attributes\n", (long long int) ftello (r->file));
1249 text = open_text_record (r, size, count);
1252 const char *variable = text_tokenize (text, ':');
1253 if (variable == NULL || !read_attributes (r, text, variable))
1256 close_text_record (text);
1260 read_simple_compressed_data (struct sfm_reader *r, int max_cases)
1262 enum { N_OPCODES = 8 };
1263 uint8_t opcodes[N_OPCODES];
1264 long long int opcode_ofs;
1270 printf ("\n%08llx: compressed data:\n", (long long int) ftello (r->file));
1272 opcode_idx = N_OPCODES;
1275 for (case_num = 0; case_num < max_cases; case_num++)
1277 printf ("%08llx: case %d's uncompressible data begins\n",
1278 (long long int) ftello (r->file), case_num);
1279 for (i = 0; i < r->n_var_widths;)
1281 int width = r->var_widths[i];
1285 if (opcode_idx >= N_OPCODES)
1287 opcode_ofs = ftello (r->file);
1290 if (!try_read_bytes (r, opcodes, 8))
1294 read_bytes (r, opcodes, 8);
1297 opcode = opcodes[opcode_idx];
1298 printf ("%08llx: variable %d: opcode %d: ",
1299 opcode_ofs + opcode_idx, i, opcode);
1304 printf ("%.*g", DBL_DIG + 1, opcode - r->bias);
1306 printf (", but this is a string variable (width=%d)", width);
1312 printf ("ignored padding\n");
1316 printf ("end of data\n");
1320 read_bytes (r, raw_value, 8);
1321 printf ("uncompressible data: ");
1322 print_untyped_value (r, raw_value);
1330 printf (", but this is a numeric variable");
1338 printf (", but this is a string variable (width=%d)", width);
1350 read_zlib_compressed_data (struct sfm_reader *r)
1353 long long int this_ofs, next_ofs, next_len;
1354 long long int bias, zero;
1355 long long int expected_uncmp_ofs, expected_cmp_ofs;
1356 unsigned int block_size, n_blocks;
1360 ofs = ftello (r->file);
1361 printf ("\n%08llx: ZLIB compressed data header:\n", ofs);
1363 this_ofs = read_int64 (r);
1364 next_ofs = read_int64 (r);
1365 next_len = read_int64 (r);
1367 printf ("\tzheader_ofs: 0x%llx\n", this_ofs);
1368 if (this_ofs != ofs)
1369 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1370 printf ("\tztrailer_ofs: 0x%llx\n", next_ofs);
1371 printf ("\tztrailer_len: %lld\n", next_len);
1372 if (next_len < 24 || next_len % 24)
1373 printf ("\t\t(Trailer length is not a positive multiple of 24.)\n");
1375 printf ("\n%08llx: 0x%llx bytes of ZLIB compressed data\n",
1376 ofs + 8 * 3, next_ofs - (ofs + 8 * 3));
1378 skip_bytes (r, next_ofs - (ofs + 8 * 3));
1380 printf ("\n%08llx: ZLIB trailer fixed header:\n", next_ofs);
1381 bias = read_int64 (r);
1382 zero = read_int64 (r);
1383 block_size = read_int (r);
1384 n_blocks = read_int (r);
1385 printf ("\tbias: %lld\n", bias);
1386 printf ("\tzero: 0x%llx\n", zero);
1388 printf ("\t\t(Expected 0.)\n");
1389 printf ("\tblock_size: 0x%x\n", block_size);
1390 if (block_size != 0x3ff000)
1391 printf ("\t\t(Expected 0x3ff000.)\n");
1392 printf ("\tn_blocks: %u\n", n_blocks);
1393 if (n_blocks != next_len / 24 - 1)
1394 printf ("\t\t(Expected %llu.)\n", next_len / 24 - 1);
1396 expected_uncmp_ofs = ofs;
1397 expected_cmp_ofs = ofs + 24;
1398 for (i = 0; i < n_blocks; i++)
1400 long long int blockinfo_ofs = ftello (r->file);
1401 unsigned long long int uncompressed_ofs = read_int64 (r);
1402 unsigned long long int compressed_ofs = read_int64 (r);
1403 unsigned int uncompressed_size = read_int (r);
1404 unsigned int compressed_size = read_int (r);
1406 printf ("\n%08llx: ZLIB block descriptor %d\n", blockinfo_ofs, i + 1);
1408 printf ("\tuncompressed_ofs: 0x%llx\n", uncompressed_ofs);
1409 if (uncompressed_ofs != expected_uncmp_ofs)
1410 printf ("\t\t(Expected 0x%llx.)\n", ofs);
1412 printf ("\tcompressed_ofs: 0x%llx\n", compressed_ofs);
1413 if (compressed_ofs != expected_cmp_ofs)
1414 printf ("\t\t(Expected 0x%llx.)\n", ofs + 24);
1416 printf ("\tuncompressed_size: 0x%x\n", uncompressed_size);
1417 if (i < n_blocks - 1 && uncompressed_size != block_size)
1418 printf ("\t\t(Expected 0x%x.)\n", block_size);
1420 printf ("\tcompressed_size: 0x%x\n", compressed_size);
1421 if (i == n_blocks - 1 && compressed_ofs + compressed_size != next_ofs)
1422 printf ("\t\t(This was expected to be 0x%llx.)\n",
1423 next_ofs - compressed_size);
1425 expected_uncmp_ofs += uncompressed_size;
1426 expected_cmp_ofs += compressed_size;
1430 /* Helpers for reading records that consist of structured text
1436 struct sfm_reader *reader; /* Reader. */
1437 char *buffer; /* Record contents. */
1438 size_t size; /* Size of buffer. */
1439 size_t pos; /* Current position in buffer. */
1442 /* Reads SIZE * COUNT bytes into a text record for R,
1443 and returns the new text record. */
1444 static struct text_record *
1445 open_text_record (struct sfm_reader *r, size_t size, size_t count)
1447 struct text_record *text = xmalloc (sizeof *text);
1449 if (size_overflow_p (xsum (1, xtimes (size, count))))
1450 sys_error (r, "Extension record too large.");
1452 size_t n_bytes = size * count;
1453 char *buffer = xmalloc (n_bytes + 1);
1454 read_bytes (r, buffer, n_bytes);
1455 buffer[n_bytes] = '\0';
1457 text->buffer = buffer;
1458 text->size = n_bytes;
1463 /* Closes TEXT and frees its storage.
1464 Not really needed, because the pool will free the text record anyway,
1465 but can be used to free it earlier. */
1467 close_text_record (struct text_record *text)
1469 free (text->buffer);
1474 text_tokenize (struct text_record *text, int delimiter)
1476 size_t start = text->pos;
1477 while (text->pos < text->size
1478 && text->buffer[text->pos] != delimiter
1479 && text->buffer[text->pos] != '\0')
1481 if (start == text->pos)
1483 text->buffer[text->pos++] = '\0';
1484 return &text->buffer[start];
1488 text_match (struct text_record *text, int c)
1490 if (text->pos < text->size && text->buffer[text->pos] == c)
1499 /* Reads a integer value expressed in decimal, then a space, then a string that
1500 consists of exactly as many bytes as specified by the integer, then a space,
1501 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
1502 buffer (so the caller should not free the string). */
1504 text_parse_counted_string (struct text_record *text)
1512 while (isdigit ((unsigned char) text->buffer[text->pos]))
1513 n = (n * 10) + (text->buffer[text->pos++] - '0');
1514 if (start == text->pos)
1516 sys_error (text->reader, "expecting digit at offset %zu in record",
1521 if (!text_match (text, ' '))
1523 sys_error (text->reader, "expecting space at offset %zu in record",
1528 if (text->pos + n > text->size)
1530 sys_error (text->reader, "%zu-byte string starting at offset %zu "
1531 "exceeds record length %zu", n, text->pos, text->size);
1535 s = &text->buffer[text->pos];
1538 sys_error (text->reader, "expecting space at offset %zu following "
1539 "%zu-byte string", text->pos + n, n);
1547 /* Reads a variable=value pair from TEXT.
1548 Looks up the variable in DICT and stores it into *VAR.
1549 Stores a null-terminated value into *VALUE. */
1551 read_variable_to_value_pair (struct text_record *text,
1552 char **key, char **value)
1554 *key = text_tokenize (text, '=');
1555 *value = text_tokenize (text, '\t');
1556 if (!*key || !*value)
1559 while (text->pos < text->size
1560 && (text->buffer[text->pos] == '\t'
1561 || text->buffer[text->pos] == '\0'))
1566 /* Returns the current byte offset inside the TEXT's string. */
1568 text_pos (const struct text_record *text)
1574 text_get_all (const struct text_record *text)
1576 return text->buffer;
1583 %s, a utility for dissecting system files.\n\
1584 Usage: %s [OPTION]... SYSFILE...\n\
1585 where each SYSFILE is the name of a system file.\n\
1588 --data[=MAXCASES] print (up to MAXCASES cases of) compressed data\n\
1589 --help display this help and exit\n\
1590 --version output version information and exit\n",
1591 program_name, program_name);
1594 /* Displays a corruption message. */
1596 sys_msg (struct sfm_reader *r, const char *format, va_list args)
1598 printf ("\"%s\" near offset 0x%llx: ",
1599 r->file_name, (long long int) ftello (r->file));
1600 vprintf (format, args);
1604 /* Displays a warning for the current file position. */
1606 sys_warn (struct sfm_reader *r, const char *format, ...)
1610 va_start (args, format);
1611 sys_msg (r, format, args);
1615 /* Displays an error for the current file position,
1616 marks it as in an error state,
1617 and aborts reading it using longjmp. */
1619 sys_error (struct sfm_reader *r, const char *format, ...)
1623 va_start (args, format);
1624 sys_msg (r, format, args);
1627 exit (EXIT_FAILURE);
1630 /* Reads BYTE_CNT bytes into BUF.
1631 Returns true if exactly BYTE_CNT bytes are successfully read.
1632 Aborts if an I/O error or a partial read occurs.
1633 If EOF_IS_OK, then an immediate end-of-file causes false to be
1634 returned; otherwise, immediate end-of-file causes an abort
1637 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
1638 void *buf, size_t n_bytes)
1640 size_t bytes_read = fread (buf, 1, n_bytes, r->file);
1641 if (bytes_read == n_bytes)
1643 else if (ferror (r->file))
1644 sys_error (r, "System error: %s.", strerror (errno));
1645 else if (!eof_is_ok || bytes_read != 0)
1646 sys_error (r, "Unexpected end of file.");
1651 /* Reads BYTE_CNT into BUF.
1652 Aborts upon I/O error or if end-of-file is encountered. */
1654 read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1656 read_bytes_internal (r, false, buf, n_bytes);
1659 /* Reads BYTE_CNT bytes into BUF.
1660 Returns true if exactly BYTE_CNT bytes are successfully read.
1661 Returns false if an immediate end-of-file is encountered.
1662 Aborts if an I/O error or a partial read occurs. */
1664 try_read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
1666 return read_bytes_internal (r, true, buf, n_bytes);
1669 /* Reads a 32-bit signed integer from R and returns its value in
1672 read_int (struct sfm_reader *r)
1675 read_bytes (r, integer, sizeof integer);
1676 return integer_get (r->integer_format, integer, sizeof integer);
1679 /* Reads a 64-bit signed integer from R and returns its value in
1682 read_int64 (struct sfm_reader *r)
1685 read_bytes (r, integer, sizeof integer);
1686 return integer_get (r->integer_format, integer, sizeof integer);
1689 /* Reads a 64-bit floating-point number from R and returns its
1690 value in host format. */
1692 read_float (struct sfm_reader *r)
1695 read_bytes (r, number, sizeof number);
1696 return float_get_double (r->float_format, number);
1699 /* Reads exactly SIZE - 1 bytes into BUFFER
1700 and stores a null byte into BUFFER[SIZE - 1]. */
1702 read_string (struct sfm_reader *r, char *buffer, size_t size)
1705 read_bytes (r, buffer, size - 1);
1706 buffer[size - 1] = '\0';
1709 /* Skips BYTES bytes forward in R. */
1711 skip_bytes (struct sfm_reader *r, size_t bytes)
1716 size_t chunk = MIN (sizeof buffer, bytes);
1717 read_bytes (r, buffer, chunk);
1723 trim_spaces (char *s)
1725 char *end = strchr (s, '\0');
1726 while (end > s && end[-1] == ' ')
1732 print_string (const char *s, size_t len)
1734 if (memchr (s, 0, len) == 0)
1738 for (i = 0; i < len; i++)
1740 unsigned char c = s[i];
1744 else if (c == '\n' || isprint (c))
1747 printf ("\\%02x", c);
1752 hex_dump (0, s, len);