1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
25 #include <data/val-type.h>
26 #include <libpspp/compiler.h>
27 #include <libpspp/float-format.h>
28 #include <libpspp/integer-format.h>
29 #include <libpspp/misc.h>
37 #define _(msgid) gettext (msgid)
41 const char *file_name;
44 int n_variable_records, n_variables;
46 enum integer_format integer_format;
47 enum float_format float_format;
50 static void read_header (struct sfm_reader *);
51 static void read_variable_record (struct sfm_reader *);
52 static void read_value_label_record (struct sfm_reader *);
53 static void read_document_record (struct sfm_reader *);
54 static void read_extension_record (struct sfm_reader *);
55 static void read_machine_integer_info (struct sfm_reader *,
56 size_t size, size_t count);
57 static void read_machine_float_info (struct sfm_reader *,
58 size_t size, size_t count);
59 static void read_display_parameters (struct sfm_reader *,
60 size_t size, size_t count);
61 static void read_long_var_name_map (struct sfm_reader *r,
62 size_t size, size_t count);
63 static void read_long_string_map (struct sfm_reader *r,
64 size_t size, size_t count);
66 static struct variable_to_value_map *open_variable_to_value_map (
67 struct sfm_reader *, size_t size);
68 static void close_variable_to_value_map (struct variable_to_value_map *);
69 static bool read_variable_to_value_map (struct variable_to_value_map *,
70 char **key, char **value);
72 static void usage (int exit_code);
73 static void sys_warn (struct sfm_reader *, const char *, ...)
75 static void sys_error (struct sfm_reader *, const char *, ...)
79 static void read_bytes (struct sfm_reader *, void *, size_t);
80 static int read_int (struct sfm_reader *);
81 static double read_float (struct sfm_reader *);
82 static void read_string (struct sfm_reader *, char *, size_t);
83 static void skip_bytes (struct sfm_reader *, size_t);
84 static void trim_spaces (char *);
87 main (int argc, char *argv[])
92 set_program_name (argv[0]);
96 r.file_name = argv[1];
97 r.file = fopen (r.file_name, "rb");
99 error (EXIT_FAILURE, errno, "error opening \"%s\"", r.file_name);
100 r.n_variable_records = 0;
104 while ((rec_type = read_int (&r)) != 999)
109 read_variable_record (&r);
113 read_value_label_record (&r);
117 sys_error (&r, _("Misplaced type 4 record."));
120 read_document_record (&r);
124 read_extension_record (&r);
128 sys_error (&r, _("Unrecognized record type %d."), rec_type);
131 printf ("%08lx: end-of-dictionary record (first byte of data at %08lx)\n",
132 ftell (r.file), ftell (r.file) + 4);
138 read_header (struct sfm_reader *r)
141 char eye_catcher[61];
142 uint8_t raw_layout_code[4];
144 int32_t nominal_case_size;
146 int32_t weight_index;
150 char creation_date[10];
151 char creation_time[9];
154 read_string (r, rec_type, sizeof rec_type);
155 read_string (r, eye_catcher, sizeof eye_catcher);
157 if (strcmp ("$FL2", rec_type) != 0)
158 sys_error (r, _("This is not an SPSS system file."));
160 /* Identify integer format. */
161 read_bytes (r, raw_layout_code, sizeof raw_layout_code);
162 if ((!integer_identify (2, raw_layout_code, sizeof raw_layout_code,
164 && !integer_identify (3, raw_layout_code, sizeof raw_layout_code,
166 || (r->integer_format != INTEGER_MSB_FIRST
167 && r->integer_format != INTEGER_LSB_FIRST))
168 sys_error (r, _("This is not an SPSS system file."));
169 layout_code = integer_get (r->integer_format,
170 raw_layout_code, sizeof raw_layout_code);
172 nominal_case_size = read_int (r);
173 compressed = read_int (r) != 0;
174 weight_index = read_int (r);
175 ncases = read_int (r);
177 /* Identify floating-point format and obtain compression bias. */
178 read_bytes (r, raw_bias, sizeof raw_bias);
179 if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
181 sys_warn (r, _("Compression bias is not the usual "
182 "value of 100, or system file uses unrecognized "
183 "floating-point format."));
184 if (r->integer_format == INTEGER_MSB_FIRST)
185 r->float_format = FLOAT_IEEE_DOUBLE_BE;
187 r->float_format = FLOAT_IEEE_DOUBLE_LE;
189 bias = float_get_double (r->float_format, raw_bias);
191 read_string (r, creation_date, sizeof creation_date);
192 read_string (r, creation_time, sizeof creation_time);
193 read_string (r, file_label, sizeof file_label);
194 trim_spaces (file_label);
197 printf ("File header record:\n");
198 printf ("\t%17s: %s\n", "Product name", eye_catcher);
199 printf ("\t%17s: %"PRId32"\n", "Layout code", layout_code);
200 printf ("\t%17s: %"PRId32"\n", "Compressed", compressed);
201 printf ("\t%17s: %"PRId32"\n", "Weight index", weight_index);
202 printf ("\t%17s: %"PRId32"\n", "Number of cases", ncases);
203 printf ("\t%17s: %g\n", "Compression bias", bias);
204 printf ("\t%17s: %s\n", "Creation date", creation_date);
205 printf ("\t%17s: %s\n", "Creation time", creation_time);
206 printf ("\t%17s: \"%s\"\n", "File label", file_label);
210 format_name (int format)
212 switch ((format >> 16) & 0xff)
215 case 2: return "AHEX";
216 case 3: return "COMMA";
217 case 4: return "DOLLAR";
220 case 7: return "PIBHEX";
222 case 9: return "PIB";
223 case 10: return "PK";
224 case 11: return "RB";
225 case 12: return "RBHEX";
229 case 20: return "DATE";
230 case 21: return "TIME";
231 case 22: return "DATETIME";
232 case 23: return "ADATE";
233 case 24: return "JDATE";
234 case 25: return "DTIME";
235 case 26: return "WKDAY";
236 case 27: return "MONTH";
237 case 28: return "MOYR";
238 case 29: return "QYR";
239 case 30: return "WKYR";
240 case 31: return "PCT";
241 case 32: return "DOT";
242 case 33: return "CCA";
243 case 34: return "CCB";
244 case 35: return "CCC";
245 case 36: return "CCD";
246 case 37: return "CCE";
247 case 38: return "EDATE";
248 case 39: return "SDATE";
249 default: return "invalid";
253 /* Reads a variable (type 2) record from R and adds the
254 corresponding variable to DICT.
255 Also skips past additional variable records for long string
258 read_variable_record (struct sfm_reader *r)
261 int has_variable_label;
262 int missing_value_code;
267 printf ("%08lx: variable record #%d\n",
268 ftell (r->file), r->n_variable_records++);
270 width = read_int (r);
271 has_variable_label = read_int (r);
272 missing_value_code = read_int (r);
273 print_format = read_int (r);
274 write_format = read_int (r);
275 read_string (r, name, sizeof name);
276 name[strcspn (name, " ")] = '\0';
281 printf ("\tWidth: %d (%s)\n",
284 : width == 0 ? "numeric"
285 : "long string continuation record");
286 printf ("\tVariable label: %d\n", has_variable_label);
287 printf ("\tMissing values code: %d (%s)\n", missing_value_code,
288 (missing_value_code == 0 ? "no missing values"
289 : missing_value_code == 1 ? "one missing value"
290 : missing_value_code == 2 ? "two missing values"
291 : missing_value_code == 3 ? "three missing values"
292 : missing_value_code == -2 ? "one missing value range"
293 : missing_value_code == -3 ? "one missing value, one range"
295 printf ("\tPrint format: %06x (%s%d.%d)\n",
296 print_format, format_name (print_format),
297 (print_format >> 8) & 0xff, print_format & 0xff);
298 printf ("\tWrite format: %06x (%s%d.%d)\n",
299 write_format, format_name (write_format),
300 (write_format >> 8) & 0xff, write_format & 0xff);
301 printf ("\tName: %s\n", name);
303 /* Get variable label, if any. */
304 if (has_variable_label != 0 && has_variable_label != 1)
305 sys_error (r, _("Variable label indicator field is not 0 or 1."));
306 if (has_variable_label == 1)
308 long int offset = ftell (r->file);
313 if (len >= sizeof label)
314 sys_error (r, _("Variable %s has label of invalid length %zu."),
316 read_string (r, label, len + 1);
317 printf("\t%08lx Variable label: \"%s\"\n", offset, label);
319 skip_bytes (r, ROUND_UP (len, 4) - len);
322 /* Set missing values. */
323 if (missing_value_code != 0)
327 printf ("\t%08lx Missing values:", ftell (r->file));
330 if (missing_value_code < -3 || missing_value_code > 3
331 || missing_value_code == -1)
332 sys_error (r, _("Numeric missing value indicator field is not "
333 "-3, -2, 0, 1, 2, or 3."));
334 if (missing_value_code < 0)
336 double low = read_float (r);
337 double high = read_float (r);
338 printf (" %g...%g", low, high);
339 missing_value_code = -missing_value_code - 2;
341 for (i = 0; i < missing_value_code; i++)
342 printf (" %g", read_float (r));
346 if (missing_value_code < 1 || missing_value_code > 3)
347 sys_error (r, _("String missing value indicator field is not "
349 for (i = 0; i < missing_value_code; i++)
352 read_string (r, string, sizeof string);
353 printf (" \"%s\"", string);
360 /* Reads value labels from sysfile R and inserts them into the
361 associated dictionary. */
363 read_value_label_record (struct sfm_reader *r)
365 int label_cnt, var_cnt;
368 printf ("%08lx: value labels record\n", ftell (r->file));
370 /* Read number of labels. */
371 label_cnt = read_int (r);
372 for (i = 0; i < label_cnt; i++)
377 unsigned char label_len;
381 read_bytes (r, raw_value, sizeof raw_value);
382 value = float_get_double (r->float_format, raw_value);
383 for (n_printable = 0; n_printable < sizeof raw_value; n_printable++)
384 if (!isprint (raw_value[n_printable]))
387 /* Read label length. */
388 read_bytes (r, &label_len, sizeof label_len);
389 padded_len = ROUND_UP (label_len + 1, 8);
391 /* Read label, padding. */
392 read_bytes (r, label, padded_len - 1);
393 label[label_len] = 0;
395 printf ("\t%g/\"%.*s\": \"%s\"\n", value, n_printable, raw_value, label);
398 /* Now, read the type 4 record that has the list of variables
399 to which the value labels are to be applied. */
401 /* Read record type of type 4 record. */
402 if (read_int (r) != 4)
403 sys_error (r, _("Variable index record (type 4) does not immediately "
404 "follow value label record (type 3) as it should."));
406 /* Read number of variables associated with value label from type 4
408 printf ("\t%08lx: apply to variables", ftell (r->file));
409 var_cnt = read_int (r);
410 for (i = 0; i < var_cnt; i++)
411 printf (" #%d", read_int (r));
416 read_document_record (struct sfm_reader *r)
421 printf ("%08lx: document record\n", ftell (r->file));
422 n_lines = read_int (r);
423 printf ("\t%d lines of documents\n", n_lines);
425 for (i = 0; i < n_lines; i++)
428 printf ("\t%08lx: ", ftell (r->file));
429 read_string (r, line, sizeof line);
431 printf ("line %d: \"%s\"\n", i, line);
436 read_extension_record (struct sfm_reader *r)
438 long int offset = ftell (r->file);
439 int subtype = read_int (r);
440 size_t size = read_int (r);
441 size_t count = read_int (r);
442 size_t bytes = size * count;
444 printf ("%08lx: Record 7, subtype %d, size=%zu, count=%zu\n",
445 offset, subtype, size, count);
450 read_machine_integer_info (r, size, count);
454 read_machine_float_info (r, size, count);
458 /* Variable sets information. We don't use these yet.
459 They only apply to GUIs; see VARSETS on the APPLY
460 DICTIONARY command in SPSS documentation. */
464 /* DATE variable information. We don't use it yet, but we
469 /* Unknown purpose. */
473 read_display_parameters (r, size, count);
477 read_long_var_name_map (r, size, count);
481 read_long_string_map (r, size, count);
485 /* New in SPSS v14? Unknown purpose. */
489 /* Text field that defines variable attributes. New in
494 sys_warn (r, _("Unrecognized record type 7, subtype %d."), subtype);
498 skip_bytes (r, bytes);
502 read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count)
504 long int offset = ftell (r->file);
505 int version_major = read_int (r);
506 int version_minor = read_int (r);
507 int version_revision = read_int (r);
508 int machine_code = read_int (r);
509 int float_representation = read_int (r);
510 int compression_code = read_int (r);
511 int integer_representation = read_int (r);
512 int character_code = read_int (r);
514 printf ("%08lx: machine integer info\n", offset);
515 if (size != 4 || count != 8)
516 sys_error (r, _("Bad size (%zu) or count (%zu) field on record type 7, "
520 printf ("\tVersion: %d.%d.%d\n",
521 version_major, version_minor, version_revision);
522 printf ("\tMachine code: %d\n", machine_code);
523 printf ("\tFloating point representation: %d (%s)\n",
524 float_representation,
525 float_representation == 1 ? "IEEE 754"
526 : float_representation == 2 ? "IBM 370"
527 : float_representation == 3 ? "DEC VAX"
529 printf ("\tCompression code: %d\n", compression_code);
530 printf ("\tEndianness: %d (%s)\n", integer_representation,
531 integer_representation == 1 ? "big"
532 : integer_representation == 2 ? "little" : "unknown");
533 printf ("\tCharacter code: %d\n", character_code);
536 /* Read record type 7, subtype 4. */
538 read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
540 long int offset = ftell (r->file);
541 double sysmis = read_float (r);
542 double highest = read_float (r);
543 double lowest = read_float (r);
545 printf ("%08lx: machine float info\n", offset);
546 if (size != 8 || count != 3)
547 sys_error (r, _("Bad size (%zu) or count (%zu) on extension 4."),
550 printf ("\tsysmis: %g\n", sysmis);
551 if (sysmis != SYSMIS)
552 sys_warn (r, _("File specifies unexpected value %g as SYSMIS."), sysmis);
553 printf ("\thighest: %g\n", highest);
554 if (highest != HIGHEST)
555 sys_warn (r, _("File specifies unexpected value %g as HIGHEST."), highest);
556 printf ("\tlowest: %g\n", lowest);
557 if (lowest != LOWEST)
558 sys_warn (r, _("File specifies unexpected value %g as LOWEST."), lowest);
561 /* Read record type 7, subtype 11. */
563 read_display_parameters (struct sfm_reader *r, size_t size, size_t count)
569 printf ("%08lx: variable display parameters\n", ftell (r->file));
572 sys_warn (r, _("Bad size %zu on extension 11."), size);
573 skip_bytes (r, size * count);
577 n_vars = r->n_variables;
578 if (count == 3 * n_vars)
579 includes_width = true;
580 else if (count == 2 * n_vars)
581 includes_width = false;
584 sys_warn (r, _("Extension 11 has bad count %zu (for %zu variables)."),
586 skip_bytes (r, size * count);
590 for (i = 0; i < n_vars; ++i)
592 int measure = read_int (r);
593 int width = includes_width ? read_int (r) : 0;
594 int align = read_int (r);
596 printf ("\tVar #%zu: measure=%d (%s)", i, measure,
597 (measure == 1 ? "nominal"
598 : measure == 2 ? "ordinal"
599 : measure == 3 ? "scale"
602 printf (", width=%d", width);
603 printf (", align=%d (%s)\n", align,
605 : align == 1 ? "right"
606 : align == 2 ? "centre"
611 /* Reads record type 7, subtype 13, which gives the long name
612 that corresponds to each short name. */
614 read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count)
616 struct variable_to_value_map *map;
620 printf ("%08lx: long variable names (short => long)\n", ftell (r->file));
621 map = open_variable_to_value_map (r, size * count);
622 while (read_variable_to_value_map (map, &var, &long_name))
623 printf ("\t%s => %s\n", var, long_name);
624 close_variable_to_value_map (map);
627 /* Reads record type 7, subtype 14, which gives the real length
628 of each very long string. Rearranges DICT accordingly. */
630 read_long_string_map (struct sfm_reader *r, size_t size, size_t count)
632 struct variable_to_value_map *map;
636 printf ("%08lx: very long strings (variable => length)\n", ftell (r->file));
637 map = open_variable_to_value_map (r, size * count);
638 while (read_variable_to_value_map (map, &var, &length_s))
639 printf ("\t%s => %d\n", var, atoi (length_s));
640 close_variable_to_value_map (map);
643 /* Helpers for reading records that contain "variable=value"
647 struct variable_to_value_map
649 char *buffer; /* Record contents. */
650 size_t size; /* Size of buffer. */
651 size_t pos; /* Current position in buffer. */
654 /* Reads SIZE bytes into a "variable=value" map for R,
655 and returns the map. */
656 static struct variable_to_value_map *
657 open_variable_to_value_map (struct sfm_reader *r, size_t size)
659 struct variable_to_value_map *map = xmalloc (sizeof *map);
660 char *buffer = xmalloc (size + 1);
661 read_bytes (r, buffer, size);
662 map->buffer = buffer;
668 /* Closes MAP and frees its storage.
669 Not really needed, because the pool will free the map anyway,
670 but can be used to free it earlier. */
672 close_variable_to_value_map (struct variable_to_value_map *map)
679 tokenize (struct variable_to_value_map *map, int delimiter)
681 size_t start = map->pos;
682 while (map->pos < map->size
683 && map->buffer[map->pos] != delimiter
684 && map->buffer[map->pos] != '\0')
686 if (map->pos == map->size)
688 map->buffer[map->pos++] = '\0';
689 return &map->buffer[start];
692 /* Reads the next variable=value pair from MAP.
693 Looks up the variable in DICT and stores it into *VAR.
694 Stores a null-terminated value into *VALUE. */
696 read_variable_to_value_map (struct variable_to_value_map *map,
697 char **key, char **value)
699 *key = tokenize (map, '=');
700 *value = tokenize (map, '\t');
701 if (!*key || !*value)
704 while (map->pos < map->size
705 && (map->buffer[map->pos] == '\t'
706 || map->buffer[map->pos] == '\0'))
712 usage (int exit_code)
714 printf ("usage: %s SYSFILE, where SYSFILE is the name of a system file\n",
719 /* Displays a corruption message. */
721 sys_msg (struct sfm_reader *r, const char *format, va_list args)
723 printf ("\"%s\" near offset 0x%lx: ",
724 r->file_name, (unsigned long) ftell (r->file));
725 vprintf (format, args);
729 /* Displays a warning for the current file position. */
731 sys_warn (struct sfm_reader *r, const char *format, ...)
735 va_start (args, format);
736 sys_msg (r, format, args);
740 /* Displays an error for the current file position,
741 marks it as in an error state,
742 and aborts reading it using longjmp. */
744 sys_error (struct sfm_reader *r, const char *format, ...)
748 va_start (args, format);
749 sys_msg (r, format, args);
755 /* Reads BYTE_CNT bytes into BUF.
756 Returns true if exactly BYTE_CNT bytes are successfully read.
757 Aborts if an I/O error or a partial read occurs.
758 If EOF_IS_OK, then an immediate end-of-file causes false to be
759 returned; otherwise, immediate end-of-file causes an abort
762 read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
763 void *buf, size_t byte_cnt)
765 size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
766 if (bytes_read == byte_cnt)
768 else if (ferror (r->file))
769 sys_error (r, _("System error: %s."), strerror (errno));
770 else if (!eof_is_ok || bytes_read != 0)
771 sys_error (r, _("Unexpected end of file."));
776 /* Reads BYTE_CNT into BUF.
777 Aborts upon I/O error or if end-of-file is encountered. */
779 read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
781 read_bytes_internal (r, false, buf, byte_cnt);
784 /* Reads a 32-bit signed integer from R and returns its value in
787 read_int (struct sfm_reader *r)
790 read_bytes (r, integer, sizeof integer);
791 return integer_get (r->integer_format, integer, sizeof integer);
794 /* Reads a 64-bit floating-point number from R and returns its
795 value in host format. */
797 read_float (struct sfm_reader *r)
800 read_bytes (r, number, sizeof number);
801 return float_get_double (r->float_format, number);
804 /* Reads exactly SIZE - 1 bytes into BUFFER
805 and stores a null byte into BUFFER[SIZE - 1]. */
807 read_string (struct sfm_reader *r, char *buffer, size_t size)
810 read_bytes (r, buffer, size - 1);
811 buffer[size - 1] = '\0';
814 /* Skips BYTES bytes forward in R. */
816 skip_bytes (struct sfm_reader *r, size_t bytes)
821 size_t chunk = MIN (sizeof buffer, bytes);
822 read_bytes (r, buffer, chunk);
828 trim_spaces (char *s)
830 char *end = strchr (s, '\0');
831 while (end > s && end[-1] == ' ')