1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #include "sfm-write.h"
29 #include <unistd.h> /* Required by SunOS4. */
33 #include "dictionary.h"
35 #include "file-handle.h"
41 #include "value-labels.h"
46 #define _(msgid) gettext (msgid)
48 #include "debug-print.h"
50 /* Compression bias used by PSPP. Values between (1 -
51 COMPRESSION_BIAS) and (251 - COMPRESSION_BIAS) inclusive can be
53 #define COMPRESSION_BIAS 100
55 /* System file writer. */
58 struct file_handle *fh; /* File handle. */
59 FILE *file; /* File stream. */
61 int needs_translation; /* 0=use fast path, 1=translation needed. */
62 int compress; /* 1=compressed, 0=not compressed. */
63 int case_cnt; /* Number of cases written so far. */
64 size_t flt64_cnt; /* Number of flt64 elements in case. */
66 /* Compression buffering. */
67 flt64 *buf; /* Buffered data. */
68 flt64 *end; /* Buffer end. */
69 flt64 *ptr; /* Current location in buffer. */
70 unsigned char *x; /* Location in current instruction octet. */
71 unsigned char *y; /* End of instruction octet. */
74 struct sfm_var *vars; /* Variables. */
75 size_t var_cnt; /* Number of variables. */
78 /* A variable in a system file. */
81 int width; /* 0=numeric, otherwise string width. */
82 int fv; /* Index into case. */
83 size_t flt64_cnt; /* Number of flt64 elements. */
86 static char *append_string_max (char *, const char *, const char *);
87 static int write_header (struct sfm_writer *, const struct dictionary *);
88 static int buf_write (struct sfm_writer *, const void *, size_t);
89 static int write_variable (struct sfm_writer *, struct variable *);
90 static int write_value_labels (struct sfm_writer *,
91 struct variable *, int idx);
92 static int write_rec_7_34 (struct sfm_writer *);
94 static int write_longvar_table (struct sfm_writer *w,
95 const struct dictionary *dict);
97 static int write_variable_display_parameters (struct sfm_writer *w,
98 const struct dictionary *dict);
101 static int write_documents (struct sfm_writer *, const struct dictionary *);
102 static int does_dict_need_translation (const struct dictionary *);
105 var_flt64_cnt (const struct variable *v)
107 return v->type == NUMERIC ? 1 : DIV_RND_UP (v->width, sizeof (flt64));
110 /* Opens the system file designated by file handle FH for writing
111 cases from dictionary D. If COMPRESS is nonzero, the
112 system file will be compressed. If OMIT_LONG_NAMES is nonzero, the
113 long name table will be omitted.
115 No reference to D is retained, so it may be modified or
116 destroyed at will after this function returns. D is not
117 modified by this function, except to assign short names. */
119 sfm_open_writer (struct file_handle *fh,
120 struct dictionary *d, int compress,
121 short omit_long_names)
123 struct sfm_writer *w = NULL;
127 if (!fh_open (fh, "system file", "we"))
130 /* Create and initialize writer. */
131 w = xmalloc (sizeof *w);
133 w->file = fopen (handle_get_filename (fh), "wb");
135 w->needs_translation = does_dict_need_translation (d);
136 w->compress = compress;
140 w->buf = w->end = w->ptr = NULL;
143 w->var_cnt = dict_get_var_cnt (d);
144 w->vars = xmalloc (sizeof *w->vars * w->var_cnt);
145 for (i = 0; i < w->var_cnt; i++)
147 const struct variable *dv = dict_get_var (d, i);
148 struct sfm_var *sv = &w->vars[i];
149 sv->width = dv->width;
151 sv->flt64_cnt = var_flt64_cnt (dv);
154 /* Check that file create succeeded. */
157 msg (ME, _("Error opening \"%s\" for writing "
158 "as a system file: %s."),
159 handle_get_filename (w->fh), strerror (errno));
164 /* Write the file header. */
165 if (!write_header (w, d))
168 /* Write basic variable info. */
169 dict_assign_short_names (d);
170 for (i = 0; i < dict_get_var_cnt (d); i++)
171 write_variable (w, dict_get_var (d, i));
173 /* Write out value labels. */
174 for (idx = i = 0; i < dict_get_var_cnt (d); i++)
176 struct variable *v = dict_get_var (d, i);
178 if (!write_value_labels (w, v, idx))
180 idx += var_flt64_cnt (v);
183 if (dict_get_documents (d) != NULL && !write_documents (w, d))
186 if (!write_rec_7_34 (w))
189 if (!write_variable_display_parameters (w, d))
192 if (!omit_long_names)
194 if (!write_longvar_table (w, d))
198 /* Write end-of-headers record. */
207 rec_999.rec_type = 999;
210 if (!buf_write (w, &rec_999, sizeof rec_999))
216 w->buf = xmalloc (sizeof *w->buf * 128);
218 w->end = &w->buf[128];
219 w->x = (unsigned char *) w->ptr++;
220 w->y = (unsigned char *) w->ptr;
226 sfm_close_writer (w);
231 does_dict_need_translation (const struct dictionary *d)
237 for (i = 0; i < dict_get_var_cnt (d); i++)
239 struct variable *v = dict_get_var (d, i);
240 if (v->fv != case_idx)
247 /* Returns value of X truncated to two least-significant digits. */
258 /* Write the sysfile_header header to system file W. */
260 write_header (struct sfm_writer *w, const struct dictionary *d)
262 struct sysfile_header hdr;
268 memcpy (hdr.rec_type, "$FL2", 4);
270 p = stpcpy (hdr.prod_name, "@(#) SPSS DATA FILE ");
271 p = append_string_max (p, version, &hdr.prod_name[60]);
272 p = append_string_max (p, " - ", &hdr.prod_name[60]);
273 p = append_string_max (p, host_system, &hdr.prod_name[60]);
274 memset (p, ' ', &hdr.prod_name[60] - p);
279 for (i = 0; i < dict_get_var_cnt (d); i++)
280 w->flt64_cnt += var_flt64_cnt (dict_get_var (d, i));
281 hdr.case_size = w->flt64_cnt;
283 hdr.compress = w->compress;
285 if (dict_get_weight (d) != NULL)
287 struct variable *weight_var;
288 int recalc_weight_idx = 1;
291 weight_var = dict_get_weight (d);
294 struct variable *v = dict_get_var (d, i);
297 recalc_weight_idx += var_flt64_cnt (v);
299 hdr.weight_idx = recalc_weight_idx;
305 hdr.bias = COMPRESSION_BIAS;
307 if (time (&t) == (time_t) -1)
309 memcpy (hdr.creation_date, "01 Jan 70", 9);
310 memcpy (hdr.creation_time, "00:00:00", 8);
314 static const char *month_name[12] =
316 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
317 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
319 struct tm *tmp = localtime (&t);
320 int day = rerange (tmp->tm_mday);
321 int mon = rerange (tmp->tm_mon + 1);
322 int year = rerange (tmp->tm_year);
323 int hour = rerange (tmp->tm_hour + 1);
324 int min = rerange (tmp->tm_min + 1);
325 int sec = rerange (tmp->tm_sec + 1);
328 sprintf (buf, "%02d %s %02d", day, month_name[mon - 1], year);
329 memcpy (hdr.creation_date, buf, sizeof hdr.creation_date);
330 sprintf (buf, "%02d:%02d:%02d", hour - 1, min - 1, sec - 1);
331 memcpy (hdr.creation_time, buf, sizeof hdr.creation_time);
335 const char *label = dict_get_label (d);
339 buf_copy_str_rpad (hdr.file_label, sizeof hdr.file_label, label);
342 memset (hdr.padding, 0, sizeof hdr.padding);
344 if (!buf_write (w, &hdr, sizeof hdr))
349 /* Translates format spec from internal form in SRC to system file
352 write_format_spec (struct fmt_spec *src, int32 *dest)
354 *dest = (formats[src->type].spss << 16) | (src->w << 8) | src->d;
357 /* Write the variable record(s) for primary variable P and secondary
358 variable S to system file W. */
360 write_variable (struct sfm_writer *w, struct variable *v)
362 struct sysfile_variable sv;
364 /* Missing values. */
365 flt64 m[3]; /* Missing value values. */
366 int nm; /* Number of missing values, possibly negative. */
370 sv.has_var_label = (v->label != NULL);
372 switch (v->miss_type)
380 for (nm = 0; nm < v->miss_type; nm++)
381 m[nm] = v->missing[nm].f;
384 m[0] = v->missing[0].f;
385 m[1] = v->missing[1].f;
389 m[0] = second_lowest_flt64;
390 m[1] = v->missing[0].f;
394 m[0] = v->missing[0].f;
398 case MISSING_RANGE_1:
399 m[0] = v->missing[0].f;
400 m[1] = v->missing[1].f;
401 m[2] = v->missing[2].f;
405 m[0] = second_lowest_flt64;
406 m[1] = v->missing[0].f;
407 m[2] = v->missing[1].f;
411 m[0] = v->missing[0].f;
412 m[1] = second_lowest_flt64;
413 m[2] = v->missing[1].f;
421 sv.n_missing_values = nm;
422 write_format_spec (&v->print, &sv.print);
423 write_format_spec (&v->write, &sv.write);
424 buf_copy_str_rpad (sv.name, sizeof sv.name, v->short_name);
425 if (!buf_write (w, &sv, sizeof sv))
439 l.label_len = min (strlen (v->label), 255);
440 ext_len = ROUND_UP (l.label_len, sizeof l.label_len);
441 memcpy (l.label, v->label, l.label_len);
442 memset (&l.label[l.label_len], ' ', ext_len - l.label_len);
444 if (!buf_write (w, &l, offsetof (struct label, label) + ext_len))
448 if (nm && !buf_write (w, m, sizeof *m * nm))
451 if (v->type == ALPHA && v->width > (int) sizeof (flt64))
457 sv.has_var_label = 0;
458 sv.n_missing_values = 0;
459 memset (&sv.print, 0, sizeof sv.print);
460 memset (&sv.write, 0, sizeof sv.write);
461 memset (&sv.name, 0, sizeof sv.name);
463 pad_count = DIV_RND_UP (v->width, (int) sizeof (flt64)) - 1;
464 for (i = 0; i < pad_count; i++)
465 if (!buf_write (w, &sv, sizeof sv))
472 /* Writes the value labels for variable V having system file variable
473 index IDX to system file W. Returns
474 nonzero only if successful. */
476 write_value_labels (struct sfm_writer *w, struct variable *v, int idx)
478 struct value_label_rec
492 struct val_labs_iterator *i;
493 struct value_label_rec *vlr;
494 struct var_idx_rec vir;
499 if (!val_labs_count (v->val_labs))
502 /* Pass 1: Count bytes. */
503 vlr_size = (sizeof (struct value_label_rec)
504 + sizeof (flt64) * (val_labs_count (v->val_labs) - 1));
505 for (vl = val_labs_first (v->val_labs, &i); vl != NULL;
506 vl = val_labs_next (v->val_labs, &i))
507 vlr_size += ROUND_UP (strlen (vl->label) + 1, sizeof (flt64));
509 /* Pass 2: Copy bytes. */
510 vlr = xmalloc (vlr_size);
512 vlr->n_labels = val_labs_count (v->val_labs);
514 for (vl = val_labs_first_sorted (v->val_labs, &i); vl != NULL;
515 vl = val_labs_next (v->val_labs, &i))
517 size_t len = strlen (vl->label);
519 *loc++ = vl->value.f;
520 *(unsigned char *) loc = len;
521 memcpy (&((unsigned char *) loc)[1], vl->label, len);
522 memset (&((unsigned char *) loc)[1 + len], ' ',
523 REM_RND_UP (len + 1, sizeof (flt64)));
524 loc += DIV_RND_UP (len + 1, sizeof (flt64));
527 if (!buf_write (w, vlr, vlr_size))
536 vir.vars[0] = idx + 1;
537 if (!buf_write (w, &vir, sizeof vir))
543 /* Writes record type 6, document record. */
545 write_documents (struct sfm_writer *w, const struct dictionary *d)
549 int32 rec_type P; /* Always 6. */
550 int32 n_lines P; /* Number of lines of documents. */
554 const char *documents;
557 documents = dict_get_documents (d);
558 n_lines = strlen (documents) / 80;
561 rec_6.n_lines = n_lines;
562 if (!buf_write (w, &rec_6, sizeof rec_6))
564 if (!buf_write (w, documents, 80 * n_lines))
570 /* Write the alignment, width and scale values */
572 write_variable_display_parameters (struct sfm_writer *w,
573 const struct dictionary *dict)
585 vdp_hdr.rec_type = 7;
586 vdp_hdr.subtype = 11;
587 vdp_hdr.elem_size = 4;
588 vdp_hdr.n_elem = w->var_cnt * 3;
590 if (!buf_write (w, &vdp_hdr, sizeof vdp_hdr))
593 for ( i = 0 ; i < w->var_cnt ; ++i )
604 v = dict_get_var(dict, i);
606 params.measure = v->measure;
607 params.width = v->display_width;
608 params.align = v->alignment;
610 if (!buf_write (w, ¶ms, sizeof(params)))
617 /* Writes the long variable name table */
619 write_longvar_table (struct sfm_writer *w, const struct dictionary *dict)
630 struct string long_name_map;
633 ds_init (&long_name_map, 10 * dict_get_var_cnt (dict));
634 for (i = 0; i < dict_get_var_cnt (dict); i++)
636 struct variable *v = dict_get_var (dict, i);
639 ds_putc (&long_name_map, '\t');
640 ds_printf (&long_name_map, "%s=%s", v->short_name, v->name);
645 lv_hdr.elem_size = 1;
646 lv_hdr.n_elem = ds_length (&long_name_map);
648 if (!buf_write (w, &lv_hdr, sizeof lv_hdr)
649 || !buf_write (w, ds_data (&long_name_map), ds_length (&long_name_map)))
652 ds_destroy (&long_name_map);
656 ds_destroy (&long_name_map);
660 /* Writes record type 7, subtypes 3 and 4. */
662 write_rec_7_34 (struct sfm_writer *w)
679 /* Components of the version number, from major to minor. */
680 int version_component[3];
682 /* Used to step through the version string. */
685 /* Parses the version string, which is assumed to be of the form
686 #.#x, where each # is a string of digits, and x is a single
688 version_component[0] = strtol (bare_version, &p, 10);
691 version_component[1] = strtol (bare_version, &p, 10);
692 version_component[2] = (isalpha ((unsigned char) *p)
693 ? tolower ((unsigned char) *p) - 'a' : 0);
695 rec_7.rec_type_3 = 7;
697 rec_7.data_type_3 = sizeof (int32);
699 rec_7.elem_3[0] = version_component[0];
700 rec_7.elem_3[1] = version_component[1];
701 rec_7.elem_3[2] = version_component[2];
702 rec_7.elem_3[3] = -1;
704 /* PORTME: 1=IEEE754, 2=IBM 370, 3=DEC VAX E. */
711 /* PORTME: 1=big-endian, 2=little-endian. */
718 /* PORTME: 1=EBCDIC, 2=7-bit ASCII, 3=8-bit ASCII, 4=DEC Kanji. */
721 rec_7.rec_type_4 = 7;
723 rec_7.data_type_4 = sizeof (flt64);
725 rec_7.elem_4[0] = -FLT64_MAX;
726 rec_7.elem_4[1] = FLT64_MAX;
727 rec_7.elem_4[2] = second_lowest_flt64;
729 if (!buf_write (w, &rec_7, sizeof rec_7))
734 /* Write NBYTES starting at BUF to the system file represented by
737 buf_write (struct sfm_writer *w, const void *buf, size_t nbytes)
739 assert (buf != NULL);
740 if (fwrite (buf, nbytes, 1, w->file) != 1)
742 msg (ME, _("%s: Writing system file: %s."),
743 handle_get_filename (w->fh), strerror (errno));
749 /* Copies string DEST to SRC with the proviso that DEST does not reach
750 byte END; no null terminator is copied. Returns a pointer to the
751 byte after the last byte copied. */
753 append_string_max (char *dest, const char *src, const char *end)
755 int nbytes = min (end - dest, (int) strlen (src));
756 memcpy (dest, src, nbytes);
757 return dest + nbytes;
760 /* Makes certain that the compression buffer of H has room for another
761 element. If there's not room, pads out the current instruction
762 octet with zero and dumps out the buffer. */
764 ensure_buf_space (struct sfm_writer *w)
766 if (w->ptr >= w->end)
768 memset (w->x, 0, w->y - w->x);
771 if (!buf_write (w, w->buf, sizeof *w->buf * 128))
777 static void write_compressed_data (struct sfm_writer *w, const flt64 *elem);
779 /* Writes case C to system file W.
780 Returns nonzero if successful. */
782 sfm_write_case (struct sfm_writer *w, struct ccase *c)
786 if (!w->needs_translation && !w->compress
787 && sizeof (flt64) == sizeof (union value))
789 /* Fast path: external and internal representations are the
790 same and the dictionary is properly ordered. Write
792 buf_write (w, case_data_all (c), sizeof (union value) * w->flt64_cnt);
796 /* Slow path: internal and external representations differ.
797 Write into a bounce buffer, then write to W. */
803 bounce_size = sizeof *bounce * w->flt64_cnt;
804 bounce = bounce_cur = local_alloc (bounce_size);
806 for (i = 0; i < w->var_cnt; i++)
808 struct sfm_var *v = &w->vars[i];
811 *bounce_cur = case_num (c, v->fv);
813 memcpy (bounce_cur, case_data (c, v->fv)->s, v->width);
814 bounce_cur += v->flt64_cnt;
818 buf_write (w, bounce, bounce_size);
820 write_compressed_data (w, bounce);
829 put_instruction (struct sfm_writer *w, unsigned char instruction)
833 if (!ensure_buf_space (w))
835 w->x = (unsigned char *) w->ptr++;
836 w->y = (unsigned char *) w->ptr;
838 *w->x++ = instruction;
842 put_element (struct sfm_writer *w, const flt64 *elem)
844 if (!ensure_buf_space (w))
846 memcpy (w->ptr++, elem, sizeof *elem);
850 write_compressed_data (struct sfm_writer *w, const flt64 *elem)
854 for (i = 0; i < w->var_cnt; i++)
856 struct sfm_var *v = &w->vars[i];
860 if (*elem == -FLT64_MAX)
861 put_instruction (w, 255);
862 else if (*elem >= 1 - COMPRESSION_BIAS
863 && *elem <= 251 - COMPRESSION_BIAS
864 && *elem == (int) *elem)
865 put_instruction (w, (int) *elem + COMPRESSION_BIAS);
868 put_instruction (w, 253);
869 put_element (w, elem);
877 for (j = 0; j < v->flt64_cnt; j++, elem++)
879 if (!memcmp (elem, " ", sizeof (flt64)))
880 put_instruction (w, 254);
883 put_instruction (w, 253);
884 put_element (w, elem);
891 /* Closes a system file after we're done with it. */
893 sfm_close_writer (struct sfm_writer *w)
898 fh_close (w->fh, "system file", "we");
903 if (w->buf != NULL && w->ptr > w->buf)
905 memset (w->x, 0, w->y - w->x);
906 buf_write (w, w->buf, (w->ptr - w->buf) * sizeof *w->buf);
909 /* Seek back to the beginning and update the number of cases.
910 This is just a courtesy to later readers, so there's no need
911 to check return values or report errors. */
912 if (!fseek (w->file, offsetof (struct sysfile_header, case_cnt), SEEK_SET))
914 int32 case_cnt = w->case_cnt;
916 /* I don't really care about the return value: it doesn't
917 matter whether this data is written. */
918 fwrite (&case_cnt, sizeof case_cnt, 1, w->file);
921 if (fclose (w->file) == EOF)
922 msg (ME, _("%s: Closing system file: %s."),
923 handle_get_filename (w->fh), strerror (errno));