1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
20 /* AIX requires this to be the first thing in the file. */
23 #define alloca __builtin_alloca
31 #ifndef alloca /* predefined by HP cc +Olibcalls */
46 #include "file-handle.h"
57 /*#define DEBUGGING 1*/
58 #include "debug-print.h"
60 /* PORTME: This file may require substantial revision for those
61 systems that don't meet the typical 32-bit integer/64-bit double
62 model. It's kinda hard to tell without having one of them on my
65 /* sfm's file_handle extension. */
68 FILE *file; /* Actual file. */
69 int opened; /* Reference count. */
71 struct dictionary *dict; /* File's dictionary. */
73 int reverse_endian; /* 1=file has endianness opposite us. */
74 int case_size; /* Number of `values's per case. */
75 long ncases; /* Number of cases, -1 if unknown. */
76 int compressed; /* 1=compressed, 0=not compressed. */
77 double bias; /* Compression bias, usually 100.0. */
78 int weight_index; /* 0-based index of weighting variable, or -1. */
80 /* File's special constants. */
85 /* Uncompression buffer. */
86 flt64 *buf; /* Buffer data. */
87 flt64 *ptr; /* Current location in buffer. */
88 flt64 *end; /* End of buffer data. */
90 /* Compression instruction octet. */
91 unsigned char x[sizeof (flt64)];
92 /* Current instruction octet. */
93 unsigned char *y; /* Location in current instruction octet. */
96 static struct fh_ext_class sfm_r_class;
99 void dump_dictionary (struct dictionary * dict);
104 /* bswap_int32(): Reverse the byte order of 32-bit integer *X. */
106 #include <asm/byteorder.h>
108 bswap_int32 (int32 * x)
112 #else /* not Linux */
114 bswap_int32 (int32 * x)
116 unsigned char *y = (char *) x;
125 #endif /* not Linux */
127 /* Reverse the byte order of 64-bit floating point *X. */
129 bswap_flt64 (flt64 * x)
131 /* Note that under compilers of any quality, half of this function
132 should optimize out as dead code. */
133 unsigned char *y = (char *) x;
135 if (sizeof (flt64) == 8)
156 for (x = 0; x < sizeof (flt64) / 2; x++)
159 y[x] = y[sizeof (flt64) - x];
160 y[sizeof (flt64) - x] = t;
166 corrupt_msg (int class, const char *format,...)
167 __attribute__ ((format (printf, 2, 3)));
169 /* Displays a corrupt sysfile error. */
171 corrupt_msg (int class, const char *format,...)
178 va_start (args, format);
179 vsnprintf (buf, 1024, format, args);
187 getl_location (&e.where.filename, &e.where.line_number);
188 e.title = _("corrupt system file: ");
195 /* Closes a system file after we're done with it. */
197 sfm_close (struct file_handle * h)
199 struct sfm_fhuser_ext *ext = h->ext;
202 assert (ext->opened == 0);
203 if (EOF == fclose (ext->file))
204 msg (ME, _("%s: Closing system file: %s."), h->fn, strerror (errno));
209 /* Closes a system file if we're done with it. */
211 sfm_maybe_close (struct file_handle *h)
213 struct sfm_fhuser_ext *ext = h->ext;
215 if (ext->opened == 1)
221 /* Dictionary reader. */
223 static void *bufread (struct file_handle * handle, void *buf, size_t nbytes,
226 static int read_header (struct file_handle * h, struct sfm_read_info * inf);
227 static int parse_format_spec (struct file_handle * h, int32 s,
228 struct fmt_spec * v, struct variable *vv);
229 static int read_value_labels (struct file_handle * h, struct variable ** var_by_index);
230 static int read_variables (struct file_handle * h, struct variable *** var_by_index);
231 static int read_machine_int32_info (struct file_handle * h, int size, int count);
232 static int read_machine_flt64_info (struct file_handle * h, int size, int count);
233 static int read_documents (struct file_handle * h);
235 /* Displays the message X with corrupt_msg, then jumps to the lossage
245 /* Calls bufread with the specified arguments, and jumps to lossage if
247 #define assertive_bufread(a,b,c,d) \
250 if (!bufread (a,b,c,d)) \
255 /* Reads the dictionary from file with handle H, and returns it in a
256 dictionary structure. This dictionary may be modified in order to
257 rename, reorder, and delete variables, etc. */
259 sfm_read_dictionary (struct file_handle * h, struct sfm_read_info * inf)
261 /* The file handle extension record. */
262 struct sfm_fhuser_ext *ext;
264 /* Allows for quick reference to variables according to indexes
265 relative to position within a case. */
266 struct variable **var_by_index = NULL;
268 /* Check whether the file is already open. */
269 if (h->class == &sfm_r_class)
275 else if (h->class != NULL)
277 msg (ME, _("Cannot read file %s as system file: already opened for %s."),
278 fh_handle_name (h), h->class->name);
282 msg (VM (1), _("%s: Opening system-file handle %s for reading."),
283 fh_handle_filename (h), fh_handle_name (h));
285 /* Open the physical disk file. */
286 ext = xmalloc (sizeof (struct sfm_fhuser_ext));
287 ext->file = fopen (h->norm_fn, "rb");
288 if (ext->file == NULL)
290 msg (ME, _("An error occurred while opening \"%s\" for reading "
291 "as a system file: %s."), h->fn, strerror (errno));
297 /* Initialize the sfm_fhuser_ext structure. */
298 h->class = &sfm_r_class;
301 ext->buf = ext->ptr = ext->end = NULL;
302 ext->y = ext->x + sizeof ext->x;
305 /* Default special constants. */
306 ext->sysmis = -FLT64_MAX;
307 ext->highest = FLT64_MAX;
308 ext->lowest = second_lowest_flt64;
310 /* Read the header. */
311 if (!read_header (h, inf))
314 /* Read about the variables. */
315 if (!read_variables (h, &var_by_index))
318 /* Handle weighting. */
319 if (ext->weight_index != -1)
321 struct variable *wv = var_by_index[ext->weight_index];
324 lose ((ME, _("%s: Weighting variable may not be a continuation of "
325 "a long string variable."), h->fn));
326 else if (wv->type == ALPHA)
327 lose ((ME, _("%s: Weighting variable may not be a string variable."),
330 strcpy (ext->dict->weight_var, wv->name);
333 ext->dict->weight_var[0] = 0;
335 /* Read records of types 3, 4, 6, and 7. */
340 assertive_bufread (h, &rec_type, sizeof rec_type, 0);
341 if (ext->reverse_endian)
342 bswap_int32 (&rec_type);
347 if (!read_value_labels (h, var_by_index))
352 lose ((ME, _("%s: Orphaned variable index record (type 4). Type 4 "
353 "records must always immediately follow type 3 records."),
357 if (!read_documents (h))
373 assertive_bufread (h, &data, sizeof data, 0);
374 if (ext->reverse_endian)
376 bswap_int32 (&data.subtype);
377 bswap_int32 (&data.size);
378 bswap_int32 (&data.count);
381 /*if(data.size != sizeof(int32) && data.size != sizeof(flt64))
382 lose((ME, "%s: Element size in record type 7, subtype %d, is "
383 "not either the size of IN (%d) or OBS (%d); actual value "
385 h->fn, data.subtype, sizeof(int32), sizeof(flt64),
388 switch (data.subtype)
391 if (!read_machine_int32_info (h, data.size, data.count))
396 if (!read_machine_flt64_info (h, data.size, data.count))
402 case 11: /* ?? Used by SPSS 8.0. */
407 msg (MW, _("%s: Unrecognized record type 7, subtype %d "
408 "encountered in system file."), h->fn, data.subtype);
414 void *x = bufread (h, NULL, data.size * data.count, 0);
426 assertive_bufread (h, &filler, sizeof filler, 0);
427 goto break_out_of_loop;
431 lose ((ME, _("%s: Unrecognized record type %d."), h->fn, rec_type));
436 /* Come here on successful completion. */
437 msg (VM (2), _("Read system-file dictionary successfully."));
440 dump_dictionary (ext->dict);
446 /* Come here on unsuccessful completion. */
447 msg (VM (1), _("Error reading system-file header."));
451 if (ext && ext->dict)
452 free_dictionary (ext->dict);
459 /* Read record type 7, subtype 3. */
461 read_machine_int32_info (struct file_handle * h, int size, int count)
463 struct sfm_fhuser_ext *ext = h->ext;
470 if (size != sizeof (int32) || count != 8)
471 lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
472 "subtype 3. Expected size %d, count 8."),
473 h->fn, size, count, sizeof (int32)));
475 assertive_bufread (h, data, sizeof data, 0);
476 if (ext->reverse_endian)
477 for (i = 0; i < 8; i++)
478 bswap_int32 (&data[i]);
480 /* PORTME: Check floating-point representation. */
485 lose ((ME, _("%s: Floating-point representation in system file is not "
486 "IEEE-754. PSPP cannot convert between floating-point "
487 "formats."), h->fn));
493 /* PORTME: Check recorded file endianness against intuited file
495 file_endian = endian;
496 if (ext->reverse_endian)
498 if (file_endian == BIG)
499 file_endian = LITTLE;
500 else if (file_endian == LITTLE)
505 if ((file_endian == BIG) ^ (data[6] == 1))
506 lose ((ME, _("%s: File-indicated endianness (%s) does not match endianness "
507 "intuited from file header (%s)."),
508 h->fn, file_endian == BIG ? _("big-endian") : _("little-endian"),
509 data[6] == 1 ? _("big-endian") : (data[6] == 2 ? _("little-endian")
512 /* PORTME: Character representation code. */
513 if (data[7] != 2 && data[7] != 3)
514 lose ((ME, _("%s: File-indicated character representation code (%s) is not "
516 data[7] == 1 ? "EBCDIC" : (data[7] == 4 ? _("DEC Kanji") : _("Unknown"))));
524 /* Read record type 7, subtype 4. */
526 read_machine_flt64_info (struct file_handle * h, int size, int count)
528 struct sfm_fhuser_ext *ext = h->ext;
534 if (size != sizeof (flt64) || count != 3)
535 lose ((ME, _("%s: Bad size (%d) or count (%d) field on record type 7, "
536 "subtype 4. Expected size %d, count 8."),
537 h->fn, size, count, sizeof (flt64)));
539 assertive_bufread (h, data, sizeof data, 0);
540 if (ext->reverse_endian)
541 for (i = 0; i < 3; i++)
542 bswap_flt64 (&data[i]);
544 if (data[0] != SYSMIS || data[1] != FLT64_MAX
545 || data[2] != second_lowest_flt64)
547 ext->sysmis = data[0];
548 ext->highest = data[1];
549 ext->lowest = data[2];
550 msg (MW, _("%s: File-indicated value is different from internal value "
551 "for at least one of the three system values. SYSMIS: "
552 "indicated %g, expected %g; HIGHEST: %g, %g; LOWEST: "
554 h->fn, (double) data[0], (double) SYSMIS,
555 (double) data[1], (double) FLT64_MAX,
556 (double) data[2], (double) second_lowest_flt64);
566 read_header (struct file_handle * h, struct sfm_read_info * inf)
568 struct sfm_fhuser_ext *ext = h->ext; /* File extension strcut. */
569 struct sysfile_header hdr; /* Disk buffer. */
570 struct dictionary *dict; /* File dictionary. */
571 char prod_name[sizeof hdr.prod_name + 1]; /* Buffer for product name. */
572 int skip_amt; /* Amount of product name to omit. */
575 /* Create the dictionary. */
576 dict = ext->dict = xmalloc (sizeof *dict);
578 dict->var_by_name = NULL;
581 dict->nval = -1; /* Unknown. */
584 dict->weight_var[0] = 0;
585 dict->weight_index = -1;
586 dict->filter_var[0] = 0;
588 dict->n_documents = 0;
589 dict->documents = NULL;
591 /* Read header, check magic. */
592 assertive_bufread (h, &hdr, sizeof hdr, 0);
593 if (0 != strncmp ("$FL2", hdr.rec_type, 4))
594 lose ((ME, _("%s: Bad magic. Proper system files begin with "
595 "the four characters `$FL2'. This file will not be read."),
598 /* Check eye-catcher string. */
599 memcpy (prod_name, hdr.prod_name, sizeof hdr.prod_name);
600 for (i = 0; i < 60; i++)
601 if (!isprint ((unsigned char) prod_name[i]))
603 for (i = 59; i >= 0; i--)
604 if (!isgraph ((unsigned char) prod_name[i]))
609 prod_name[60] = '\0';
613 static const char *prefix[N_PREFIXES] =
615 "@(#) SPSS DATA FILE",
621 for (i = 0; i < N_PREFIXES; i++)
622 if (!strncmp (prefix[i], hdr.prod_name, strlen (prefix[i])))
624 skip_amt = strlen (prefix[i]);
629 /* Check endianness. */
630 /* PORTME: endianness. */
631 if (hdr.layout_code == 2)
632 ext->reverse_endian = 0;
635 bswap_int32 (&hdr.layout_code);
636 if (hdr.layout_code != 2)
637 lose ((ME, _("%s: File layout code has unexpected value %d. Value "
638 "should be 2, in big-endian or little-endian format."),
639 h->fn, hdr.layout_code));
641 ext->reverse_endian = 1;
642 bswap_int32 (&hdr.case_size);
643 bswap_int32 (&hdr.compressed);
644 bswap_int32 (&hdr.weight_index);
645 bswap_int32 (&hdr.ncases);
646 bswap_flt64 (&hdr.bias);
649 /* Copy basic info and verify correctness. */
650 ext->case_size = hdr.case_size;
651 if (hdr.case_size <= 0 || ext->case_size > (INT_MAX
652 / (int) sizeof (union value) / 2))
653 lose ((ME, _("%s: Number of elements per case (%d) is not between 1 "
654 "and %d."), h->fn, hdr.case_size, INT_MAX / sizeof (union value) / 2));
656 ext->compressed = hdr.compressed;
658 ext->weight_index = hdr.weight_index - 1;
659 if (hdr.weight_index < 0 || hdr.weight_index > hdr.case_size)
660 lose ((ME, _("%s: Index of weighting variable (%d) is not between 0 "
661 "and number of elements per case (%d)."),
662 h->fn, hdr.weight_index, ext->case_size));
664 ext->ncases = hdr.ncases;
665 if (ext->ncases < -1 || ext->ncases > INT_MAX / 2)
666 lose ((ME, _("%s: Number of cases in file (%ld) is not between -1 and "
667 "%d."), h->fn, (long) ext->ncases, INT_MAX / 2));
669 ext->bias = hdr.bias;
670 if (ext->bias != 100.0)
671 corrupt_msg (MW, _("%s: Compression bias (%g) is not the usual "
672 "value of 100."), h->fn, ext->bias);
674 /* Make a file label only on the condition that the given label is
675 not all spaces or nulls. */
680 for (i = sizeof hdr.file_label - 1; i >= 0; i--)
681 if (!isspace ((unsigned char) hdr.file_label[i])
682 && hdr.file_label[i] != 0)
684 dict->label = xmalloc (i + 2);
685 memcpy (dict->label, hdr.file_label, i + 1);
686 dict->label[i + 1] = 0;
695 memcpy (inf->creation_date, hdr.creation_date, 9);
696 inf->creation_date[9] = 0;
698 memcpy (inf->creation_time, hdr.creation_time, 8);
699 inf->creation_time[8] = 0;
701 if (!ext->reverse_endian)
702 inf->endianness = endian;
704 inf->endianness = endian == BIG ? LITTLE : BIG;
706 inf->compressed = hdr.compressed;
708 inf->ncases = hdr.ncases;
710 for (cp = &prod_name[skip_amt]; cp < &prod_name[60]; cp++)
711 if (isgraph ((unsigned char) *cp))
713 strcpy (inf->product, cp);
722 /* Reads most of the dictionary from file H; also fills in the
723 associated VAR_BY_INDEX array.
725 Note: the dictionary returned by this function has an invalid NVAL
726 element, also the VAR[] array does not have the FV and LV elements
727 set, however the NV elements *are* set. This is because the caller
728 will probably modify the dictionary before reading it in from the
729 file. Also, the get.* elements are set to appropriate values to
730 allow the file to be read. */
732 read_variables (struct file_handle * h, struct variable *** var_by_index)
736 struct sfm_fhuser_ext *ext = h->ext; /* File extension record. */
737 struct dictionary *dict = ext->dict; /* Dictionary being constructed. */
738 struct sysfile_variable sv; /* Disk buffer. */
739 int long_string_count = 0; /* # of long string continuation
740 records still expected. */
741 int next_value = 0; /* Index to next `value' structure. */
743 /* Allocate variables. */
744 dict->var = xmalloc (sizeof *dict->var * ext->case_size);
745 *var_by_index = xmalloc (sizeof **var_by_index * ext->case_size);
747 /* Read in the entry for each variable and use the info to
748 initialize the dictionary. */
749 for (i = 0; i < ext->case_size; i++)
754 assertive_bufread (h, &sv, sizeof sv, 0);
756 if (ext->reverse_endian)
758 bswap_int32 (&sv.rec_type);
759 bswap_int32 (&sv.type);
760 bswap_int32 (&sv.has_var_label);
761 bswap_int32 (&sv.n_missing_values);
762 bswap_int32 (&sv.print);
763 bswap_int32 (&sv.write);
766 if (sv.rec_type != 2)
767 lose ((ME, _("%s: position %d: Bad record type (%d); "
768 "the expected value was 2."), h->fn, i, sv.rec_type));
770 /* If there was a long string previously, make sure that the
771 continuations are present; otherwise make sure there aren't
773 if (long_string_count)
776 lose ((ME, _("%s: position %d: String variable does not have "
777 "proper number of continuation records."), h->fn, i));
779 (*var_by_index)[i] = NULL;
783 else if (sv.type == -1)
784 lose ((ME, _("%s: position %d: Superfluous long string continuation "
785 "record."), h->fn, i));
787 /* Check fields for validity. */
788 if (sv.type < 0 || sv.type > 255)
789 lose ((ME, _("%s: position %d: Bad variable type code %d."),
791 if (sv.has_var_label != 0 && sv.has_var_label != 1)
792 lose ((ME, _("%s: position %d: Variable label indicator field is not "
793 "0 or 1."), h->fn, i));
794 if (sv.n_missing_values < -3 || sv.n_missing_values > 3
795 || sv.n_missing_values == -1)
796 lose ((ME, _("%s: position %d: Missing value indicator field is not "
797 "-3, -2, 0, 1, 2, or 3."), h->fn, i));
799 /* Construct internal variable structure, initialize critical bits. */
800 vv = (*var_by_index)[i] = dict->var[dict->nvar++] = xmalloc (sizeof *vv);
801 vv->index = dict->nvar - 1;
806 /* Copy first character of variable name. */
807 if (!isalpha ((unsigned char) sv.name[0])
808 && sv.name[0] != '@' && sv.name[0] != '#')
809 lose ((ME, _("%s: position %d: Variable name begins with invalid "
810 "character."), h->fn, i));
811 if (islower ((unsigned char) sv.name[0]))
812 msg (MW, _("%s: position %d: Variable name begins with lowercase letter "
813 "%c."), h->fn, i, sv.name[0]);
814 if (sv.name[0] == '#')
815 msg (MW, _("%s: position %d: Variable name begins with octothorpe "
816 "(`#'). Scratch variables should not appear in system "
817 "files."), h->fn, i);
818 vv->name[0] = toupper ((unsigned char) (sv.name[0]));
820 /* Copy remaining characters of variable name. */
821 for (j = 1; j < 8; j++)
823 int c = (unsigned char) sv.name[j];
827 else if (islower (c))
829 msg (MW, _("%s: position %d: Variable name character %d is "
830 "lowercase letter %c."), h->fn, i, j + 1, sv.name[j]);
831 vv->name[j] = toupper ((unsigned char) (c));
833 else if (isalnum (c) || c == '.' || c == '@'
834 || c == '#' || c == '$' || c == '_')
837 lose ((ME, _("%s: position %d: character `\\%03o' (%c) is not valid in a "
838 "variable name."), h->fn, i, c, c));
842 /* Set type, width, and `left' fields and allocate `value'
849 vv->get.fv = next_value++;
856 vv->nv = DIV_RND_UP (vv->width, MAX_SHORT_STRING);
857 vv->get.nv = DIV_RND_UP (vv->width, sizeof (flt64));
858 vv->get.fv = next_value;
859 next_value += vv->get.nv;
860 long_string_count = vv->get.nv - 1;
862 vv->left = (vv->name[0] == '#');
864 /* Get variable label, if any. */
865 if (sv.has_var_label == 1)
870 /* Read length of label. */
871 assertive_bufread (h, &len, sizeof len, 0);
872 if (ext->reverse_endian)
876 if (len < 0 || len > 255)
877 lose ((ME, _("%s: Variable %s indicates variable label of invalid "
878 "length %d."), h->fn, vv->name, len));
880 /* Read label into variable structure. */
881 vv->label = bufread (h, NULL, ROUND_UP (len, sizeof (int32)), len + 1);
882 if (vv->label == NULL)
884 vv->label[len] = '\0';
887 /* Set missing values. */
888 if (sv.n_missing_values != 0)
892 if (vv->width > MAX_SHORT_STRING)
893 lose ((ME, _("%s: Long string variable %s may not have missing "
894 "values."), h->fn, vv->name));
896 assertive_bufread (h, mv, sizeof *mv * abs (sv.n_missing_values), 0);
898 if (ext->reverse_endian && vv->type == NUMERIC)
899 for (j = 0; j < abs (sv.n_missing_values); j++)
900 bswap_flt64 (&mv[j]);
902 if (sv.n_missing_values > 0)
904 vv->miss_type = sv.n_missing_values;
905 if (vv->type == NUMERIC)
906 for (j = 0; j < sv.n_missing_values; j++)
907 vv->missing[j].f = mv[j];
909 for (j = 0; j < sv.n_missing_values; j++)
910 memcpy (vv->missing[j].s, &mv[j], vv->width);
916 if (vv->type == ALPHA)
917 lose ((ME, _("%s: String variable %s may not have missing "
918 "values specified as a range."), h->fn, vv->name));
920 if (mv[0] == ext->lowest)
922 vv->miss_type = MISSING_LOW;
923 vv->missing[x++].f = mv[1];
925 else if (mv[1] == ext->highest)
927 vv->miss_type = MISSING_HIGH;
928 vv->missing[x++].f = mv[0];
932 vv->miss_type = MISSING_RANGE;
933 vv->missing[x++].f = mv[0];
934 vv->missing[x++].f = mv[1];
937 if (sv.n_missing_values == -3)
940 vv->missing[x++].f = mv[2];
945 vv->miss_type = MISSING_NONE;
947 if (!parse_format_spec (h, sv.print, &vv->print, vv)
948 || !parse_format_spec (h, sv.write, &vv->write, vv))
952 /* Some consistency checks. */
953 if (long_string_count != 0)
954 lose ((ME, _("%s: Long string continuation records omitted at end of "
955 "dictionary."), h->fn));
956 if (next_value != ext->case_size)
957 lose ((ME, _("%s: System file header indicates %d variable positions but "
958 "%d were read from file."), h->fn, ext->case_size, next_value));
959 dict->var = xrealloc (dict->var, sizeof *dict->var * dict->nvar);
961 /* Construct AVL tree of dictionary in order to speed up later
962 processing and to check for duplicate varnames. */
963 dict->var_by_name = avl_create (NULL, cmp_variable, NULL);
964 for (i = 0; i < dict->nvar; i++)
965 if (NULL != avl_insert (dict->var_by_name, dict->var[i]))
966 lose ((ME, _("%s: Duplicate variable name `%s' within system file."),
967 h->fn, dict->var[i]->name));
972 for (i = 0; i < dict->nvar; i++)
974 free (dict->var[i]->label);
978 if (dict->var_by_name)
979 avl_destroy (dict->var_by_name, NULL);
986 /* Translates the format spec from sysfile format to internal
989 parse_format_spec (struct file_handle *h, int32 s, struct fmt_spec *v, struct variable *vv)
991 if ((size_t) ((s >> 16) & 0xff)
992 >= sizeof translate_fmt / sizeof *translate_fmt)
993 lose ((ME, _("%s: Bad format specifier byte (%d)."),
994 h->fn, (s >> 16) & 0xff));
996 v->type = translate_fmt[(s >> 16) & 0xff];
997 v->w = (s >> 8) & 0xff;
1000 /* FIXME? Should verify the resulting specifier more thoroughly. */
1003 lose ((ME, _("%s: Bad format specifier byte (%d)."),
1004 h->fn, (s >> 16) & 0xff));
1005 if ((vv->type == ALPHA) ^ ((formats[v->type].cat & FCAT_STRING) != 0))
1006 lose ((ME, _("%s: %s variable %s has %s format specifier %s."),
1007 h->fn, vv->type == ALPHA ? _("String") : _("Numeric"),
1009 formats[v->type].cat & FCAT_STRING ? _("string") : _("numeric"),
1010 formats[v->type].name));
1017 /* Reads value labels from sysfile H and inserts them into the
1018 associated dictionary. */
1020 read_value_labels (struct file_handle * h, struct variable ** var_by_index)
1022 struct sfm_fhuser_ext *ext = h->ext; /* File extension record. */
1024 flt64 *raw_label = NULL; /* Array of raw label values. */
1025 struct value_label **cooked_label = NULL; /* Array of cooked labels. */
1026 int32 n_labels; /* Number of labels. */
1028 struct variable **var = NULL; /* Associated variables. */
1029 int32 n_vars; /* Number of associated variables. */
1033 /* First step: read the contents of the type 3 record and record its
1034 contents. Note that we can't do much with the data since we
1035 don't know yet whether it is of numeric or string type. */
1037 /* Read number of labels. */
1038 assertive_bufread (h, &n_labels, sizeof n_labels, 0);
1039 if (ext->reverse_endian)
1040 bswap_int32 (&n_labels);
1042 /* Allocate memory. */
1043 raw_label = xmalloc (sizeof *raw_label * n_labels);
1044 cooked_label = xmalloc (sizeof *cooked_label * n_labels);
1045 for (i = 0; i < n_labels; i++)
1046 cooked_label[i] = NULL;
1048 /* Read each value/label tuple. */
1049 for (i = 0; i < n_labels; i++)
1052 unsigned char label_len;
1056 /* Read value, label length. */
1057 assertive_bufread (h, &value, sizeof value, 0);
1058 assertive_bufread (h, &label_len, 1, 0);
1059 memcpy (&raw_label[i], &value, sizeof value);
1062 cooked_label[i] = xmalloc (sizeof **cooked_label);
1063 cooked_label[i]->s = xmalloc (label_len + 1);
1064 assertive_bufread (h, cooked_label[i]->s, label_len, 0);
1065 cooked_label[i]->s[label_len] = 0;
1068 rem = REM_RND_UP (label_len + 1, sizeof (flt64));
1070 assertive_bufread (h, &value, rem, 0);
1073 /* Second step: Read the type 4 record that has the list of
1074 variables to which the value labels are to be applied. */
1076 /* Read record type of type 4 record. */
1080 assertive_bufread (h, &rec_type, sizeof rec_type, 0);
1081 if (ext->reverse_endian)
1082 bswap_int32 (&rec_type);
1085 lose ((ME, _("%s: Variable index record (type 4) does not immediately "
1086 "follow value label record (type 3) as it ought."), h->fn));
1089 /* Read number of variables associated with value label from type 4
1091 assertive_bufread (h, &n_vars, sizeof n_vars, 0);
1092 if (ext->reverse_endian)
1093 bswap_int32 (&n_vars);
1094 if (n_vars < 1 || n_vars > ext->dict->nvar)
1095 lose ((ME, _("%s: Number of variables associated with a value label (%d) "
1096 "is not between 1 and the number of variables (%d)."),
1097 h->fn, n_vars, ext->dict->nvar));
1099 /* Allocate storage. */
1100 var = xmalloc (sizeof *var * n_vars);
1102 /* Read the list of variables. */
1103 for (i = 0; i < n_vars; i++)
1108 /* Read variable index, check range. */
1109 assertive_bufread (h, &var_index, sizeof var_index, 0);
1110 if (ext->reverse_endian)
1111 bswap_int32 (&var_index);
1112 if (var_index < 1 || var_index > ext->case_size)
1113 lose ((ME, _("%s: Variable index associated with value label (%d) is "
1114 "not between 1 and the number of values (%d)."),
1115 h->fn, var_index, ext->case_size));
1117 /* Make sure it's a real variable. */
1118 v = var_by_index[var_index - 1];
1120 lose ((ME, _("%s: Variable index associated with value label (%d) refers "
1121 "to a continuation of a string variable, not to an actual "
1122 "variable."), h->fn, var_index));
1123 if (v->type == ALPHA && v->width > MAX_SHORT_STRING)
1124 lose ((ME, _("%s: Value labels are not allowed on long string variables "
1125 "(%s)."), h->fn, v->name));
1127 /* Add it to the list of variables. */
1131 /* Type check the variables. */
1132 for (i = 1; i < n_vars; i++)
1133 if (var[i]->type != var[0]->type)
1134 lose ((ME, _("%s: Variables associated with value label are not all of "
1135 "identical type. Variable %s has %s type, but variable %s has "
1137 var[0]->name, var[0]->type == ALPHA ? _("string") : _("numeric"),
1138 var[i]->name, var[i]->type == ALPHA ? _("string") : _("numeric")));
1140 /* Create a value_label for each value/label tuple, now that we know
1141 the desired type. */
1142 for (i = 0; i < n_labels; i++)
1144 if (var[0]->type == ALPHA)
1146 const int copy_len = min (sizeof (flt64), MAX_SHORT_STRING);
1147 memcpy (cooked_label[i]->v.s, (char *) &raw_label[i], copy_len);
1148 if (MAX_SHORT_STRING > copy_len)
1149 memset (&cooked_label[i]->v.s[copy_len], ' ',
1150 MAX_SHORT_STRING - copy_len);
1152 cooked_label[i]->v.f = raw_label[i];
1153 if (ext->reverse_endian)
1154 bswap_flt64 (&cooked_label[i]->v.f);
1156 cooked_label[i]->ref_count = n_vars;
1159 /* Assign the value_label's to each variable. */
1160 for (i = 0; i < n_vars; i++)
1162 struct variable *v = var[i];
1165 /* Create AVL tree if necessary. */
1167 v->val_lab = avl_create (NULL, val_lab_cmp, (void *) (v->width));
1169 /* Add each label to the variable. */
1170 for (j = 0; j < n_labels; j++)
1172 struct value_label *old = avl_replace (v->val_lab, cooked_label[j]);
1176 if (var[0]->type == NUMERIC)
1177 msg (MW, _("%s: File contains duplicate label for value %g for "
1178 "variable %s."), h->fn, cooked_label[j]->v.f, v->name);
1180 msg (MW, _("%s: File contains duplicate label for value `%.*s' "
1181 "for variable %s."), h->fn, v->width,
1182 cooked_label[j]->v.s, v->name);
1184 free_value_label (old);
1188 free (cooked_label);
1195 for (i = 0; i < n_labels; i++)
1196 if (cooked_label[i])
1198 free (cooked_label[i]->s);
1199 free (cooked_label[i]);
1206 /* Reads NBYTES bytes from the file represented by H. If BUF is
1207 non-NULL, uses that as the buffer; otherwise allocates at least
1208 MINALLOC bytes. Returns a pointer to the buffer on success, NULL
1211 bufread (struct file_handle * h, void *buf, size_t nbytes, size_t minalloc)
1213 struct sfm_fhuser_ext *ext = h->ext;
1216 buf = xmalloc (max (nbytes, minalloc));
1217 if (1 != fread (buf, nbytes, 1, ext->file))
1219 if (ferror (ext->file))
1220 msg (ME, _("%s: Reading system file: %s."), h->fn, strerror (errno));
1222 corrupt_msg (ME, _("%s: Unexpected end of file."), h->fn);
1228 /* Reads a document record, type 6, from system file H, and sets up
1229 the documents and n_documents fields in the associated
1232 read_documents (struct file_handle * h)
1234 struct sfm_fhuser_ext *ext = h->ext;
1235 struct dictionary *dict = ext->dict;
1238 if (dict->documents != NULL)
1239 lose ((ME, _("%s: System file contains multiple type 6 (document) records."),
1242 assertive_bufread (h, &n_lines, sizeof n_lines, 0);
1243 dict->n_documents = n_lines;
1244 if (dict->n_documents <= 0)
1245 lose ((ME, _("%s: Number of document lines (%ld) must be greater than 0."),
1246 h->fn, (long) dict->n_documents));
1248 dict->documents = bufread (h, NULL, 80 * n_lines, 0);
1249 if (dict->documents == NULL)
1257 #if GLOBAL_DEBUGGING
1259 #include "debug-print.h"
1260 /* Displays dictionary DICT on stdout. */
1262 dump_dictionary (struct dictionary * dict)
1266 debug_printf ((_("dictionary:\n")));
1267 for (i = 0; i < dict->nvar; i++)
1270 struct variable *v = dict->var[i];
1273 debug_printf ((" var %s", v->name));
1274 /*debug_printf (("(indices:%d,%d)", v->index, v->foo));*/
1275 debug_printf (("(type:%s,%d)", (v->type == NUMERIC ? _("num")
1276 : (v->type == ALPHA ? _("str") : "!!!")),
1278 debug_printf (("(fv:%d,%d)", v->fv, v->nv));
1279 /*debug_printf (("(get.fv:%d,%d)", v->get.fv, v->get.nv));*/
1280 debug_printf (("(left:%s)(miss:", v->left ? _("left") : _("right")));
1282 switch (v->miss_type)
1286 debug_printf ((_("none")));
1290 debug_printf ((_("one")));
1294 debug_printf ((_("two")));
1298 debug_printf ((_("three")));
1302 debug_printf ((_("range")));
1306 debug_printf ((_("low")));
1310 debug_printf ((_("high")));
1312 case MISSING_RANGE_1:
1314 debug_printf ((_("range+1")));
1318 debug_printf ((_("low+1")));
1320 case MISSING_HIGH_1:
1322 debug_printf ((_("high+1")));
1327 for (j = 0; j < n; j++)
1328 if (v->type == NUMERIC)
1329 debug_printf ((",%g", v->missing[j].f));
1331 debug_printf ((",\"%.*s\"", v->width, v->missing[j].s));
1332 strcpy (print, fmt_to_string (&v->print));
1333 debug_printf ((")(fmt:%s,%s)(lbl:%s)\n",
1334 print, fmt_to_string (&v->write),
1335 v->label ? v->label : "nolabel"));
1342 /* Reads compressed data into H->BUF and sets other pointers
1343 appropriately. Returns nonzero only if both no errors occur and
1346 buffer_input (struct file_handle * h)
1348 struct sfm_fhuser_ext *ext = h->ext;
1351 if (ext->buf == NULL)
1352 ext->buf = xmalloc (sizeof *ext->buf * 128);
1353 amt = fread (ext->buf, sizeof *ext->buf, 128, ext->file);
1354 if (ferror (ext->file))
1356 msg (ME, _("%s: Error reading file: %s."), h->fn, strerror (errno));
1359 ext->ptr = ext->buf;
1360 ext->end = &ext->buf[amt];
1364 /* Reads a single case consisting of compressed data from system file
1365 H into the array TEMP[] according to dictionary DICT, and returns
1366 nonzero only if successful. */
1367 /* Data in system files is compressed in the following manner:
1368 data values are grouped into sets of eight; each of the eight has
1369 one instruction byte, which are output together in an octet; each
1370 byte gives a value for that byte or indicates that the value can be
1371 found following the instructions. */
1373 read_compressed_data (struct file_handle * h, flt64 * temp)
1375 struct sfm_fhuser_ext *ext = h->ext;
1377 const unsigned char *p_end = ext->x + sizeof (flt64);
1378 unsigned char *p = ext->y;
1380 const flt64 *temp_beg = temp;
1381 const flt64 *temp_end = &temp[ext->case_size];
1385 for (; p < p_end; p++)
1389 /* Code 0 is ignored. */
1392 /* Code 252 is end of file. */
1393 if (temp_beg != temp)
1394 lose ((ME, _("%s: Compressed data is corrupted. Data ends "
1395 "partway through a case."), h->fn));
1398 /* Code 253 indicates that the value is stored explicitly
1399 following the instruction bytes. */
1400 if (ext->ptr == NULL || ext->ptr >= ext->end)
1401 if (!buffer_input (h))
1403 lose ((ME, _("%s: Unexpected end of file."), h->fn));
1406 memcpy (temp++, ext->ptr++, sizeof *temp);
1407 if (temp >= temp_end)
1411 /* Code 254 indicates a string that is all blanks. */
1412 memset (temp++, ' ', sizeof *temp);
1413 if (temp >= temp_end)
1417 /* Code 255 indicates the system-missing value. */
1418 *temp = ext->sysmis;
1419 if (ext->reverse_endian)
1422 if (temp >= temp_end)
1426 /* Codes 1 through 251 inclusive are taken to indicate a
1427 value of (BYTE - BIAS), where BYTE is the byte's value
1428 and BIAS is the compression bias (generally 100.0). */
1429 *temp = *p - ext->bias;
1430 if (ext->reverse_endian)
1433 if (temp >= temp_end)
1438 /* We have reached the end of this instruction octet. Read
1440 if (ext->ptr == NULL || ext->ptr >= ext->end)
1441 if (!buffer_input (h))
1443 if (temp_beg != temp)
1444 lose ((ME, _("%s: Unexpected end of file."), h->fn));
1447 memcpy (ext->x, ext->ptr++, sizeof *temp);
1455 /* We have filled up an entire record. Update state and return
1461 /* We have been unsuccessful at filling a record, either through i/o
1462 error or through an end-of-file indication. Update state and
1463 return unsuccessfully. */
1467 /* Reads one case from system file H into the value array PERM
1468 according to the instructions given in associated dictionary DICT,
1469 which must have the get.* elements appropriately set. Returns
1470 nonzero only if successful. */
1472 sfm_read_case (struct file_handle * h, union value * perm, struct dictionary * dict)
1474 struct sfm_fhuser_ext *ext = h->ext;
1481 /* Make sure the caller remembered to finish polishing the
1482 dictionary returned by sfm_read_dictionary(). */
1483 assert (dict->nval > 0);
1485 /* The first concern is to obtain a full case relative to the data
1486 file. (Cases in the data file have no particular relationship to
1487 cases in the active file.) */
1488 nbytes = sizeof *temp * ext->case_size;
1489 temp = local_alloc (nbytes);
1491 if (ext->compressed == 0)
1493 size_t amt = fread (temp, 1, nbytes, ext->file);
1497 if (ferror (ext->file))
1498 msg (ME, _("%s: Reading system file: %s."), h->fn, strerror (errno));
1500 msg (ME, _("%s: Partial record at end of system file."), h->fn);
1504 else if (!read_compressed_data (h, temp))
1507 /* Translate a case in data file format to a case in active file
1509 for (i = 0; i < dict->nvar; i++)
1511 struct variable *v = dict->var[i];
1513 if (v->get.fv == -1)
1516 if (v->type == NUMERIC)
1518 flt64 src = temp[v->get.fv];
1519 if (ext->reverse_endian)
1521 perm[v->fv].f = src == ext->sysmis ? SYSMIS : src;
1524 memcpy (&perm[v->fv].s, &temp[v->get.fv], v->width);
1535 static struct fh_ext_class sfm_r_class =
1538 N_("reading as a system file"),