-/* PSPP - computes sample statistics.
- Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
+/* PSPP - a program for statistical analysis.
+ Copyright (C) 1997-2000, 2006-2014 Free Software Foundation, Inc.
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of the
- License, or (at your option) any later version.
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
- This program is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- 02110-1301, USA. */
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
#include <config.h>
-#include "sys-file-writer.h"
-#include "sys-file-private.h"
+#include "data/sys-file-writer.h"
+#include "data/sys-file-private.h"
#include <ctype.h>
#include <errno.h>
-#include <fcntl.h>
+#include <stdint.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <time.h>
-#include <unistd.h>
-
-#include <libpspp/alloc.h>
-#include <libpspp/hash.h>
-#include <libpspp/magic.h>
-#include <libpspp/message.h>
-#include <libpspp/misc.h>
-#include <libpspp/str.h>
-#include <libpspp/version.h>
-
-#include "case.h"
-#include "dictionary.h"
-#include "file-handle-def.h"
-#include "format.h"
-#include "missing-values.h"
-#include "settings.h"
-#include "value-labels.h"
-#include "variable.h"
-
-#include "minmax.h"
+#include <zlib.h>
+
+#include "data/attributes.h"
+#include "data/case.h"
+#include "data/casewriter-provider.h"
+#include "data/casewriter.h"
+#include "data/dictionary.h"
+#include "data/file-handle-def.h"
+#include "data/format.h"
+#include "data/make-file.h"
+#include "data/missing-values.h"
+#include "data/mrset.h"
+#include "data/settings.h"
+#include "data/short-names.h"
+#include "data/value-labels.h"
+#include "data/variable.h"
+#include "libpspp/float-format.h"
+#include "libpspp/i18n.h"
+#include "libpspp/integer-format.h"
+#include "libpspp/message.h"
+#include "libpspp/misc.h"
+#include "libpspp/str.h"
+#include "libpspp/string-array.h"
+#include "libpspp/version.h"
+
+#include "gl/xmemdup0.h"
+#include "gl/minmax.h"
+#include "gl/unlocked-io.h"
+#include "gl/xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
-
-/* Find 64-bit floating-point type. */
-#if SIZEOF_FLOAT == 8
- #define flt64 float
- #define FLT64_MAX FLT_MAX
-#elif SIZEOF_DOUBLE == 8
- #define flt64 double
- #define FLT64_MAX DBL_MAX
-#elif SIZEOF_LONG_DOUBLE == 8
- #define flt64 long double
- #define FLT64_MAX LDBL_MAX
-#else
- #error Which one of your basic types is 64-bit floating point?
-#endif
-
-/* Figure out SYSMIS value for flt64. */
-#include <libpspp/magic.h>
-#if SIZEOF_DOUBLE == 8
-#define second_lowest_flt64 second_lowest_value
-#else
-#error Must define second_lowest_flt64 for your architecture.
-#endif
-
-/* Record Type 1: General Information. */
-struct sysfile_header
- {
- char rec_type[4] ; /* 00: Record-type code, "$FL2". */
- char prod_name[60] ; /* 04: Product identification. */
- int32_t layout_code ; /* 40: 2. */
- int32_t nominal_case_size ; /* 44: Number of `value's per case.
- Note: some systems set this to -1 */
- int32_t compress ; /* 48: 1=compressed, 0=not compressed. */
- int32_t weight_idx ; /* 4c: 1-based index of weighting var, or 0. */
- int32_t case_cnt ; /* 50: Number of cases, -1 if unknown. */
- flt64 bias ; /* 54: Compression bias (100.0). */
- char creation_date[9] ; /* 5c: `dd mmm yy' creation date of file. */
- char creation_time[8] ; /* 65: `hh:mm:ss' 24-hour creation time. */
- char file_label[64] ; /* 6d: File label. */
- char padding[3] ; /* ad: Ignored padding. */
- } ATTRIBUTE((packed)) ;
-
-/* Record Type 2: Variable. */
-struct sysfile_variable
- {
- int32_t rec_type ; /* 2. */
- int32_t type ; /* 0=numeric, 1-255=string width,
- -1=continued string. */
- int32_t has_var_label ; /* 1=has a variable label, 0=doesn't. */
- int32_t n_missing_values ; /* Missing value code of -3,-2,0,1,2, or 3. */
- int32_t print ; /* Print format. */
- int32_t write ; /* Write format. */
- char name[SHORT_NAME_LEN] ; /* Variable name. */
- /* The rest of the structure varies. */
- } ATTRIBUTE((packed)) ;
+#define N_(msgid) (msgid)
/* Compression bias used by PSPP. Values between (1 -
COMPRESSION_BIAS) and (251 - COMPRESSION_BIAS) inclusive can be
struct sfm_writer
{
struct file_handle *fh; /* File handle. */
+ struct fh_lock *lock; /* Mutual exclusion for file. */
FILE *file; /* File stream. */
+ struct replace_file *rf; /* Ticket for replacing output file. */
+
+ enum any_compression compression;
+ casenumber n_cases; /* Number of cases written so far. */
+ uint8_t space; /* ' ' in the file's character encoding. */
- int needs_translation; /* 0=use fast path, 1=translation needed. */
- int compress; /* 1=compressed, 0=not compressed. */
- int case_cnt; /* Number of cases written so far. */
- size_t flt64_cnt; /* Number of flt64 elements in case. */
- bool has_vls; /* Does the dict have very long strings? */
+ /* Simple compression buffering.
- /* Compression buffering. */
- flt64 *buf; /* Buffered data. */
- flt64 *end; /* Buffer end. */
- flt64 *ptr; /* Current location in buffer. */
- unsigned char *x; /* Location in current instruction octet. */
- unsigned char *y; /* End of instruction octet. */
+ Compressed data is output as a series of 8-byte elements, with 1 to 9
+ such elements clustered together. The first element in a cluster is 8
+ 1-byte opcodes. Some opcodes call for an additional element in the
+ cluster (hence, if there are eight such opcodes, then the cluster
+ contains a full 9 elements).
+
+ cbuf[] holds a cluster at a time. */
+ uint8_t cbuf[9][8];
+ int n_opcodes; /* Number of opcodes in cbuf[0] so far. */
+ int n_elements; /* Number of elements in cbuf[] so far. */
+
+ /* ZLIB compression. */
+ z_stream zstream; /* ZLIB deflater. */
+ off_t zstart;
+ struct zblock *blocks;
+ size_t n_blocks, allocated_blocks;
/* Variables. */
- struct sfm_var *vars; /* Variables. */
- size_t var_cnt; /* Number of variables. */
- size_t var_cnt_vls; /* Number of variables including
- very long string components. */
+ struct sfm_var *sfm_vars; /* Variables. */
+ size_t sfm_n_vars; /* Number of variables. */
+ size_t n_segments; /* Number of variables including extra segments
+ for long string variables. */
};
-/* A variable in a system file. */
-struct sfm_var
+struct zblock
{
- int width; /* 0=numeric, otherwise string width. */
- int fv; /* Index into case. */
- size_t flt64_cnt; /* Number of flt64 elements. */
+ unsigned int uncompressed_size;
+ unsigned int compressed_size;
};
-static char *append_string_max (char *, const char *, const char *);
+static const struct casewriter_class sys_file_casewriter_class;
+
static void write_header (struct sfm_writer *, const struct dictionary *);
-static void buf_write (struct sfm_writer *, const void *, size_t);
static void write_variable (struct sfm_writer *, const struct variable *);
static void write_value_labels (struct sfm_writer *,
- struct variable *, int idx);
-static void write_rec_7_34 (struct sfm_writer *);
+ const struct dictionary *);
+static void write_integer_info_record (struct sfm_writer *,
+ const struct dictionary *);
+static void write_float_info_record (struct sfm_writer *);
-static void write_longvar_table (struct sfm_writer *w,
+static void write_longvar_table (struct sfm_writer *w,
const struct dictionary *dict);
-static void write_vls_length_table (struct sfm_writer *w,
+static void write_encoding_record (struct sfm_writer *w,
+ const struct dictionary *);
+
+static void write_vls_length_table (struct sfm_writer *w,
const struct dictionary *dict);
+static void write_long_string_value_labels (struct sfm_writer *,
+ const struct dictionary *);
+static void write_long_string_missing_values (struct sfm_writer *,
+ const struct dictionary *);
+
+static void write_mrsets (struct sfm_writer *, const struct dictionary *,
+ bool pre_v14);
-static void write_variable_display_parameters (struct sfm_writer *w,
+static void write_variable_display_parameters (struct sfm_writer *w,
const struct dictionary *dict);
static void write_documents (struct sfm_writer *, const struct dictionary *);
-static inline int
-var_flt64_cnt (const struct variable *v)
-{
- assert(sizeof(flt64) == MAX_SHORT_STRING);
- return sfm_width_to_bytes(var_get_width (v)) / MAX_SHORT_STRING ;
-}
-
-static inline int
-var_flt64_cnt_nom (const struct variable *v)
-{
- return (var_is_numeric (v)
- ? 1 : DIV_RND_UP (var_get_width (v), sizeof (flt64)));
-}
-
+static void write_data_file_attributes (struct sfm_writer *,
+ const struct dictionary *);
+static void write_variable_attributes (struct sfm_writer *,
+ const struct dictionary *);
+
+static void write_int (struct sfm_writer *, int32_t);
+static void write_int64 (struct sfm_writer *, int64_t);
+static inline void convert_double_to_output_format (double, uint8_t[8]);
+static void write_float (struct sfm_writer *, double);
+static void write_string (struct sfm_writer *, const char *, size_t);
+static void write_utf8_string (struct sfm_writer *, const char *encoding,
+ const char *string, size_t width);
+static void write_utf8_record (struct sfm_writer *, const char *encoding,
+ const struct string *content, int subtype);
+static void write_string_record (struct sfm_writer *,
+ const struct substring content, int subtype);
+static void write_bytes (struct sfm_writer *, const void *, size_t);
+static void write_zeros (struct sfm_writer *, size_t);
+static void write_spaces (struct sfm_writer *, size_t);
+static void write_value (struct sfm_writer *, const union value *, int width);
+
+static void write_case_uncompressed (struct sfm_writer *,
+ const struct ccase *);
+static void write_case_compressed (struct sfm_writer *, const struct ccase *);
+static void flush_compressed (struct sfm_writer *);
+static void put_cmp_opcode (struct sfm_writer *, uint8_t);
+static void put_cmp_number (struct sfm_writer *, double);
+static void put_cmp_string (struct sfm_writer *, const void *, size_t);
+
+static bool start_zstream (struct sfm_writer *);
+static void finish_zstream (struct sfm_writer *);
+static void write_ztrailer (struct sfm_writer *);
+
+static bool write_error (const struct sfm_writer *);
+static bool close_writer (struct sfm_writer *);
/* Returns default options for writing a system file. */
struct sfm_write_options
-sfm_writer_default_options (void)
+sfm_writer_default_options (void)
{
struct sfm_write_options opts;
+ opts.compression = (settings_get_scompression ()
+ ? ANY_COMP_SIMPLE
+ : ANY_COMP_NONE);
opts.create_writeable = true;
- opts.compress = get_scompression ();
opts.version = 3;
return opts;
}
-
-/* Return a short variable name to be used as the continuation of the
- variable with the short name SN.
-
- FIXME: Need to resolve clashes somehow.
-
- */
-static const char *
-cont_var_name(const char *sn, int idx)
-{
- static char s[SHORT_NAME_LEN + 1];
-
- char abb[SHORT_NAME_LEN + 1 - 3]= {0};
-
- strncpy(abb, sn, SHORT_NAME_LEN - 3);
-
- snprintf(s, SHORT_NAME_LEN + 1, "%s%03d", abb, idx);
-
- return s;
-}
-
-
/* Opens the system file designated by file handle FH for writing
- cases from dictionary D according to the given OPTS. If
- COMPRESS is nonzero, the system file will be compressed.
+ cases from dictionary D according to the given OPTS.
No reference to D is retained, so it may be modified or
destroyed at will after this function returns. D is not
modified by this function, except to assign short names. */
-struct sfm_writer *
+struct casewriter *
sfm_open_writer (struct file_handle *fh, struct dictionary *d,
struct sfm_write_options opts)
{
- struct sfm_writer *w = NULL;
+ struct encoding_info encoding_info;
mode_t mode;
- int fd;
- int idx;
int i;
/* Check version. */
- if (opts.version != 2 && opts.version != 3)
+ if (opts.version != 2 && opts.version != 3)
{
msg (ME, _("Unknown system file version %d. Treating as version %d."),
opts.version, 3);
opts.version = 3;
}
- /* Create file. */
- mode = S_IRUSR | S_IRGRP | S_IROTH;
- if (opts.create_writeable)
- mode |= S_IWUSR | S_IWGRP | S_IWOTH;
- fd = open (fh_get_file_name (fh), O_WRONLY | O_CREAT | O_TRUNC, mode);
- if (fd < 0)
- goto open_error;
-
- /* Open file handle. */
- if (!fh_open (fh, FH_REF_FILE, "system file", "we"))
+ /* Create and initialize writer. */
+ struct sfm_writer *w = XZALLOC (struct sfm_writer);
+ w->fh = fh_ref (fh);
+ w->lock = NULL;
+ w->file = NULL;
+ w->rf = NULL;
+
+ /* Use the requested compression, except that no EBCDIC-based ZLIB compressed
+ files have been observed, so drop back to simple compression for those
+ files. */
+ w->compression = opts.compression;
+ if (w->compression == ANY_COMP_ZLIB
+ && is_encoding_ebcdic_compatible (dict_get_encoding (d)))
+ w->compression = ANY_COMP_SIMPLE;
+
+ w->n_cases = 0;
+
+ w->n_opcodes = w->n_elements = 0;
+ memset (w->cbuf[0], 0, 8);
+
+ /* Figure out how to map in-memory case data to on-disk case
+ data. Also count the number of segments. Very long strings
+ occupy multiple segments, otherwise each variable only takes
+ one segment. */
+ w->n_segments = sfm_dictionary_to_sfm_vars (d, &w->sfm_vars, &w->sfm_n_vars);
+
+ /* Open file handle as an exclusive writer. */
+ /* TRANSLATORS: this fragment will be interpolated into
+ messages in fh_lock() that identify types of files. */
+ w->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_WRITE, true);
+ if (w->lock == NULL)
goto error;
- /* Create and initialize writer. */
- w = xmalloc (sizeof *w);
- w->fh = fh;
- w->file = fdopen (fd, "w");
-
- w->needs_translation = dict_compacting_would_change (d);
- w->compress = opts.compress;
- w->case_cnt = 0;
- w->flt64_cnt = 0;
- w->has_vls = false;
-
- w->buf = w->end = w->ptr = NULL;
- w->x = w->y = NULL;
-
- w->var_cnt = dict_get_var_cnt (d);
- w->var_cnt_vls = w->var_cnt;
- w->vars = xnmalloc (w->var_cnt, sizeof *w->vars);
- for (i = 0; i < w->var_cnt; i++)
+ /* Create the file on disk. */
+ mode = 0444;
+ if (opts.create_writeable)
+ mode |= 0222;
+ w->rf = replace_file_start (fh, "wb", mode, &w->file);
+ if (w->rf == NULL)
{
- const struct variable *dv = dict_get_var (d, i);
- struct sfm_var *sv = &w->vars[i];
- sv->width = var_get_width (dv);
- /* spss compatibility nonsense */
- if ( var_get_width (dv) >= MIN_VERY_LONG_STRING )
- w->has_vls = true;
-
- sv->fv = var_get_case_index (dv);
- sv->flt64_cnt = var_flt64_cnt (dv);
+ msg (ME, _("Error opening `%s' for writing as a system file: %s."),
+ fh_get_file_name (fh), strerror (errno));
+ goto error;
}
- /* Check that file create succeeded. */
- if (w->file == NULL)
- {
- close (fd);
- goto open_error;
- }
+ get_encoding_info (&encoding_info, dict_get_encoding (d));
+ w->space = encoding_info.space[0];
/* Write the file header. */
write_header (w, d);
/* Write basic variable info. */
- dict_assign_short_names (d);
- for (i = 0; i < dict_get_var_cnt (d); i++)
- {
- int count = 0;
- const struct variable *v = dict_get_var(d, i);
- int wcount = var_get_width (v);
-
- do {
- struct variable *var_cont = var_clone (v);
- var_set_short_name (var_cont, var_get_short_name (v));
- if ( var_is_alpha (v))
- {
- if ( 0 != count )
- {
- var_clear_missing_values (var_cont);
- var_set_short_name (var_cont,
- cont_var_name (var_get_short_name (v),
- count));
- var_clear_label (var_cont);
- w->var_cnt_vls++;
- }
- count++;
- if ( wcount >= MIN_VERY_LONG_STRING )
- {
- var_set_width (var_cont, MIN_VERY_LONG_STRING - 1);
- wcount -= EFFECTIVE_LONG_STRING_LENGTH;
- }
- else
- {
- var_set_width (var_cont, wcount);
- wcount -= var_get_width (var_cont);
- }
- }
-
- write_variable (w, var_cont);
- var_destroy (var_cont);
- } while(wcount > 0);
- }
+ short_names_assign (d);
+ for (i = 0; i < dict_get_n_vars (d); i++)
+ write_variable (w, dict_get_var (d, i));
- /* Write out value labels. */
- for (idx = i = 0; i < dict_get_var_cnt (d); i++)
- {
- struct variable *v = dict_get_var (d, i);
+ write_value_labels (w, d);
- write_value_labels (w, v, idx);
- idx += var_flt64_cnt (v);
- }
-
- if (dict_get_documents (d) != NULL)
+ if (dict_get_document_n_lines (d) > 0)
write_documents (w, d);
- write_rec_7_34 (w);
+ write_integer_info_record (w, d);
+ write_float_info_record (w);
+
+ write_mrsets (w, d, true);
write_variable_display_parameters (w, d);
- if (opts.version >= 3)
+ if (opts.version >= 3)
write_longvar_table (w, d);
- write_vls_length_table(w, d);
+ write_vls_length_table (w, d);
- /* Write end-of-headers record. */
- {
- struct
- {
- int32_t rec_type ;
- int32_t filler ;
- } ATTRIBUTE((packed))
- rec_999;
+ write_long_string_value_labels (w, d);
+ write_long_string_missing_values (w, d);
+
+ if (opts.version >= 3)
+ {
+ if (attrset_count (dict_get_attributes (d)))
+ write_data_file_attributes (w, d);
+ write_variable_attributes (w, d);
+ }
+
+ write_mrsets (w, d, false);
- rec_999.rec_type = 999;
- rec_999.filler = 0;
+ write_encoding_record (w, d);
- buf_write (w, &rec_999, sizeof rec_999);
- }
+ /* Write end-of-headers record. */
+ write_int (w, 999);
+ write_int (w, 0);
- if (w->compress)
+ if (w->compression == ANY_COMP_ZLIB)
{
- w->buf = xnmalloc (128, sizeof *w->buf);
- w->ptr = w->buf;
- w->end = &w->buf[128];
- w->x = (unsigned char *) w->ptr++;
- w->y = (unsigned char *) w->ptr;
+ w->zstream.zalloc = Z_NULL;
+ w->zstream.zfree = Z_NULL;
+ w->zstream.opaque = Z_NULL;
+ w->zstart = ftello (w->file);
+
+ write_int64 (w, w->zstart);
+ write_int64 (w, 0);
+ write_int64 (w, 0);
+
+ start_zstream (w);
}
- if (sfm_write_error (w))
+ if (write_error (w))
goto error;
-
- return w;
- error:
- sfm_close_writer (w);
- return NULL;
+ return casewriter_create (dict_get_proto (d), &sys_file_casewriter_class, w);
- open_error:
- msg (ME, _("Error opening \"%s\" for writing as a system file: %s."),
- fh_get_file_name (fh), strerror (errno));
- goto error;
+error:
+ close_writer (w);
+ return NULL;
}
/* Returns value of X truncated to two least-significant digits. */
return x;
}
+/* Calculates the offset of data for TARGET_VAR from the
+ beginning of each case's data for dictionary D. The return
+ value is in "octs" (8-byte units). */
+static int
+calc_oct_idx (const struct dictionary *d, struct variable *target_var)
+{
+ int oct_idx;
+ int i;
+
+ oct_idx = 0;
+ for (i = 0; i < dict_get_n_vars (d); i++)
+ {
+ struct variable *var = dict_get_var (d, i);
+ if (var == target_var)
+ break;
+ oct_idx += sfm_width_to_octs (var_get_width (var));
+ }
+ return oct_idx;
+}
+
/* Write the sysfile_header header to system file W. */
static void
write_header (struct sfm_writer *w, const struct dictionary *d)
{
- struct sysfile_header hdr;
- char *p;
- int i;
+ const char *dict_encoding = dict_get_encoding (d);
+ char prod_name[61];
+ const char *file_label;
+ struct variable *weight;
time_t t;
- memcpy (hdr.rec_type, "$FL2", 4);
+ /* Record-type code. */
+ if (is_encoding_ebcdic_compatible (dict_encoding))
+ write_string (w, EBCDIC_MAGIC, 4);
+ else if (w->compression == ANY_COMP_ZLIB)
+ write_string (w, ASCII_ZMAGIC, 4);
+ else
+ write_string (w, ASCII_MAGIC, 4);
- p = stpcpy (hdr.prod_name, "@(#) SPSS DATA FILE ");
- p = append_string_max (p, version, &hdr.prod_name[60]);
- p = append_string_max (p, " - ", &hdr.prod_name[60]);
- p = append_string_max (p, host_system, &hdr.prod_name[60]);
- memset (p, ' ', &hdr.prod_name[60] - p);
+ /* Product identification. */
+ snprintf (prod_name, sizeof prod_name, "@(#) SPSS DATA FILE %s - %s",
+ version, host_system);
+ write_utf8_string (w, dict_encoding, prod_name, 60);
- hdr.layout_code = 2;
+ /* Layout code. */
+ write_int (w, 2);
- w->flt64_cnt = 0;
- for (i = 0; i < dict_get_var_cnt (d); i++)
- {
- w->flt64_cnt += var_flt64_cnt (dict_get_var (d, i));
- }
- hdr.nominal_case_size = w->flt64_cnt;
+ /* Number of `union value's per case. */
+ write_int (w, calc_oct_idx (d, NULL));
- hdr.compress = w->compress;
+ /* Compressed? */
+ write_int (w, (w->compression == ANY_COMP_NONE ? 0
+ : w->compression == ANY_COMP_SIMPLE ? 1
+ : 2));
- if (dict_get_weight (d) != NULL)
- {
- const struct variable *weight_var;
- int recalc_weight_idx = 1;
- int i;
+ /* Weight variable. */
+ weight = dict_get_weight (d);
+ write_int (w, weight != NULL ? calc_oct_idx (d, weight) + 1 : 0);
- weight_var = dict_get_weight (d);
- for (i = 0; ; i++)
- {
- struct variable *v = dict_get_var (d, i);
- if (v == weight_var)
- break;
- recalc_weight_idx += var_flt64_cnt (v);
- }
- hdr.weight_idx = recalc_weight_idx;
- }
- else
- hdr.weight_idx = 0;
+ /* Number of cases. We don't know this in advance, so we write
+ -1 to indicate an unknown number of cases. Later we can
+ come back and overwrite it with the true value. */
+ write_int (w, -1);
- hdr.case_cnt = -1;
- hdr.bias = COMPRESSION_BIAS;
+ /* Compression bias. */
+ write_float (w, COMPRESSION_BIAS);
+ /* Creation date and time. */
+ char *creation_date, *creation_time;
if (time (&t) == (time_t) -1)
{
- memcpy (hdr.creation_date, "01 Jan 70", 9);
- memcpy (hdr.creation_time, "00:00:00", 8);
+ creation_date = xstrdup ("01 Jan 70");
+ creation_time = xstrdup ("00:00:00");
}
else
{
- static const char *month_name[12] =
+ static const char *const month_name[12] =
{
"Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
int hour = rerange (tmp->tm_hour + 1);
int min = rerange (tmp->tm_min + 1);
int sec = rerange (tmp->tm_sec + 1);
- char buf[10];
- sprintf (buf, "%02d %s %02d", day, month_name[mon - 1], year);
- memcpy (hdr.creation_date, buf, sizeof hdr.creation_date);
- sprintf (buf, "%02d:%02d:%02d", hour - 1, min - 1, sec - 1);
- memcpy (hdr.creation_time, buf, sizeof hdr.creation_time);
+ creation_date = xasprintf ("%02d %s %02d",
+ day, month_name[mon - 1], year);
+ creation_time = xasprintf ("%02d:%02d:%02d", hour - 1, min - 1, sec - 1);
}
-
- {
- const char *label = dict_get_label (d);
- if (label == NULL)
- label = "";
+ write_utf8_string (w, dict_encoding, creation_date, 9);
+ write_utf8_string (w, dict_encoding, creation_time, 8);
+ free (creation_time);
+ free (creation_date);
+
+ /* File label. */
+ file_label = dict_get_label (d);
+ if (file_label == NULL)
+ file_label = "";
+ write_utf8_string (w, dict_encoding, file_label, 64);
+
+ /* Padding. */
+ write_zeros (w, 3);
+}
- buf_copy_str_rpad (hdr.file_label, sizeof hdr.file_label, label);
- }
-
- memset (hdr.padding, 0, sizeof hdr.padding);
+/* Write format spec FMT to W, after adjusting it to be
+ compatible with the given WIDTH. */
+static void
+write_format (struct sfm_writer *w, struct fmt_spec fmt, int width)
+{
+ assert (fmt_check_output (&fmt));
+ assert (sfm_width_to_segments (width) == 1);
- buf_write (w, &hdr, sizeof hdr);
+ if (width > 0)
+ fmt_resize (&fmt, width);
+ write_int (w, (fmt_to_io (fmt.type) << 16) | (fmt.w << 8) | fmt.d);
}
-/* Translates format spec from internal form in SRC to system file
- format in DEST. */
-static inline void
-write_format_spec (const struct fmt_spec *src, int32_t *dest)
+/* Write a string continuation variable record for each 8-byte
+ section beyond the initial 8 bytes, for a variable of the
+ given WIDTH. */
+static void
+write_variable_continuation_records (struct sfm_writer *w, int width)
{
- assert (fmt_check_output (src));
- *dest = (fmt_to_io (src->type) << 16) | (src->w << 8) | src->d;
+ int position;
+
+ assert (sfm_width_to_segments (width) == 1);
+ for (position = 8; position < width; position += 8)
+ {
+ write_int (w, 2); /* Record type. */
+ write_int (w, -1); /* Width. */
+ write_int (w, 0); /* No variable label. */
+ write_int (w, 0); /* No missing values. */
+ write_int (w, 0); /* Print format. */
+ write_int (w, 0); /* Write format. */
+ write_zeros (w, 8); /* Name. */
+ }
}
-/* Write the variable record(s) for primary variable P and secondary
- variable S to system file W. */
+/* Write the variable record(s) for variable V to system file
+ W. */
static void
write_variable (struct sfm_writer *w, const struct variable *v)
{
- struct sysfile_variable sv;
+ int width = var_get_width (v);
+ int n_segments = sfm_width_to_segments (width);
+ int seg0_width = sfm_segment_alloc_width (width, 0);
+ const char *encoding = var_get_encoding (v);
+ int i;
- /* Missing values. */
- struct missing_values mv;
- flt64 m[3]; /* Missing value values. */
- int nm; /* Number of missing values, possibly negative. */
- const char *label = var_get_label (v);
+ /* Record type. */
+ write_int (w, 2);
- sv.rec_type = 2;
- sv.type = MIN (var_get_width (v), MIN_VERY_LONG_STRING - 1);
- sv.has_var_label = label != NULL;
+ /* Width. */
+ write_int (w, seg0_width);
- mv_copy (&mv, var_get_missing_values (v));
- nm = 0;
- if (mv_has_range (&mv))
- {
- double x, y;
- mv_pop_range (&mv, &x, &y);
- m[nm++] = x == LOWEST ? second_lowest_flt64 : x;
- m[nm++] = y == HIGHEST ? FLT64_MAX : y;
- }
- while (mv_has_value (&mv))
+ /* Variable has a variable label? */
+ write_int (w, var_has_label (v));
+
+ /* Number of missing values. If there is a range, then the
+ range counts as 2 missing values and causes the number to be
+ negated.
+
+ Missing values for long string variables are written in a separate
+ record. */
+ enum { MAX_SHORT_STRING = 8 };
+ if (width <= MAX_SHORT_STRING)
{
- union value value;
- mv_pop_value (&mv, &value);
- if (var_is_numeric (v))
- m[nm] = value.f;
+ const struct missing_values *mv = var_get_missing_values (v);
+ if (mv_has_range (mv))
+ write_int (w, -2 - mv_n_values (mv));
else
- buf_copy_rpad ((char *) &m[nm], sizeof m[nm], value.s,
- var_get_width (v));
- nm++;
+ write_int (w, mv_n_values (mv));
}
- if (mv_has_range (var_get_missing_values (v)))
- nm = -nm;
-
- sv.n_missing_values = nm;
- write_format_spec (var_get_print_format (v), &sv.print);
- write_format_spec (var_get_write_format (v), &sv.write);
- buf_copy_str_rpad (sv.name, sizeof sv.name, var_get_short_name (v));
- buf_write (w, &sv, sizeof sv);
+ else
+ write_int (w, 0);
- if (label != NULL)
- {
- struct label
- {
- int32_t label_len ;
- char label[255] ;
- } ATTRIBUTE((packed))
- l;
+ /* Print and write formats. */
+ write_format (w, *var_get_print_format (v), seg0_width);
+ write_format (w, *var_get_write_format (v), seg0_width);
- int ext_len;
+ /* Short name.
+ The full name is in a translation table written
+ separately. */
+ write_utf8_string (w, encoding, var_get_short_name (v, 0), 8);
- l.label_len = MIN (strlen (label), 255);
- ext_len = ROUND_UP (l.label_len, sizeof l.label_len);
- memcpy (l.label, label, l.label_len);
- memset (&l.label[l.label_len], ' ', ext_len - l.label_len);
+ /* Value label. */
+ if (var_has_label (v))
+ {
+ char *label = recode_string (encoding, UTF8, var_get_label (v), -1);
+ size_t label_len = MIN (strlen (label), 255);
+ size_t padded_len = ROUND_UP (label_len, 4);
+ write_int (w, label_len);
+ write_string (w, label, padded_len);
+ free (label);
+ }
- buf_write (w, &l, offsetof (struct label, label) + ext_len);
+ /* Write the missing values, if any, range first. */
+ if (width <= MAX_SHORT_STRING)
+ {
+ const struct missing_values *mv = var_get_missing_values (v);
+ if (mv_has_range (mv))
+ {
+ double x, y;
+ mv_get_range (mv, &x, &y);
+ write_float (w, x);
+ write_float (w, y);
+ }
+ for (i = 0; i < mv_n_values (mv); i++)
+ write_value (w, mv_get_value (mv, i), width);
}
- if (nm)
- buf_write (w, m, sizeof *m * abs (nm));
+ write_variable_continuation_records (w, seg0_width);
- if (var_is_alpha (v) && var_get_width (v) > (int) sizeof (flt64))
+ /* Write additional segments for very long string variables. */
+ for (i = 1; i < n_segments; i++)
{
- int i;
- int pad_count;
-
- sv.type = -1;
- sv.has_var_label = 0;
- sv.n_missing_values = 0;
- memset (&sv.print, 0, sizeof sv.print);
- memset (&sv.write, 0, sizeof sv.write);
- memset (&sv.name, 0, sizeof sv.name);
-
- pad_count = DIV_RND_UP (MIN(var_get_width (v), MIN_VERY_LONG_STRING - 1),
- (int) sizeof (flt64)) - 1;
- for (i = 0; i < pad_count; i++)
- buf_write (w, &sv, sizeof sv);
+ int seg_width = sfm_segment_alloc_width (width, i);
+ struct fmt_spec fmt = fmt_for_output (FMT_A, MAX (seg_width, 1), 0);
+
+ write_int (w, 2); /* Variable record. */
+ write_int (w, seg_width); /* Width. */
+ write_int (w, 0); /* No variable label. */
+ write_int (w, 0); /* No missing values. */
+ write_format (w, fmt, seg_width); /* Print format. */
+ write_format (w, fmt, seg_width); /* Write format. */
+ write_utf8_string (w, encoding, var_get_short_name (v, i), 8);
+
+ write_variable_continuation_records (w, seg_width);
}
}
-/* Writes the value labels for variable V having system file
- variable index IDX to system file W. */
+/* Writes the value labels to system file W.
+
+ Value labels for long string variables are written separately,
+ by write_long_string_value_labels. */
static void
-write_value_labels (struct sfm_writer *w, struct variable *v, int idx)
+write_value_labels (struct sfm_writer *w, const struct dictionary *d)
{
- struct value_label_rec
+ struct label_set
{
- int32_t rec_type ;
- int32_t n_labels ;
- flt64 labels[1] ;
- } ATTRIBUTE((packed));
+ struct hmap_node hmap_node;
+ const struct val_labs *val_labs;
+ int *indexes;
+ size_t n_indexes, allocated_indexes;
+ };
+
+ size_t n_sets, allocated_sets;
+ struct label_set **sets;
+ struct hmap same_sets;
+ size_t i;
+ int idx;
- struct var_idx_rec
+ n_sets = allocated_sets = 0;
+ sets = NULL;
+ hmap_init (&same_sets);
+
+ idx = 0;
+ for (i = 0; i < dict_get_n_vars (d); i++)
{
- int32_t rec_type ;
- int32_t n_vars ;
- int32_t vars[1] ;
- } ATTRIBUTE((packed));
-
- const struct val_labs *val_labs;
- struct val_labs_iterator *i;
- struct value_label_rec *vlr;
- struct var_idx_rec vir;
- struct val_lab *vl;
- size_t vlr_size;
- flt64 *loc;
-
- val_labs = var_get_value_labels (v);
- if (val_labs == NULL)
- return;
+ struct variable *v = dict_get_var (d, i);
- /* Pass 1: Count bytes. */
- vlr_size = (sizeof (struct value_label_rec)
- + sizeof (flt64) * (val_labs_count (val_labs) - 1));
- for (vl = val_labs_first (val_labs, &i); vl != NULL;
- vl = val_labs_next (val_labs, &i))
- vlr_size += ROUND_UP (strlen (vl->label) + 1, sizeof (flt64));
-
- /* Pass 2: Copy bytes. */
- vlr = xmalloc (vlr_size);
- vlr->rec_type = 3;
- vlr->n_labels = val_labs_count (val_labs);
- loc = vlr->labels;
- for (vl = val_labs_first_sorted (val_labs, &i); vl != NULL;
- vl = val_labs_next (val_labs, &i))
+ if (var_has_value_labels (v) && var_get_width (v) <= 8)
+ {
+ const struct val_labs *val_labs = var_get_value_labels (v);
+ unsigned int hash = val_labs_hash (val_labs, 0);
+ struct label_set *set;
+
+ HMAP_FOR_EACH_WITH_HASH (set, struct label_set, hmap_node,
+ hash, &same_sets)
+ {
+ if (val_labs_equal (set->val_labs, val_labs))
+ {
+ if (set->n_indexes >= set->allocated_indexes)
+ set->indexes = x2nrealloc (set->indexes,
+ &set->allocated_indexes,
+ sizeof *set->indexes);
+ set->indexes[set->n_indexes++] = idx;
+ goto next_var;
+ }
+ }
+
+ set = xmalloc (sizeof *set);
+ set->val_labs = val_labs;
+ set->indexes = xmalloc (sizeof *set->indexes);
+ set->indexes[0] = idx;
+ set->n_indexes = 1;
+ set->allocated_indexes = 1;
+ hmap_insert (&same_sets, &set->hmap_node, hash);
+
+ if (n_sets >= allocated_sets)
+ sets = x2nrealloc (sets, &allocated_sets, sizeof *sets);
+ sets[n_sets++] = set;
+ }
+
+ next_var:
+ idx += sfm_width_to_octs (var_get_width (v));
+ }
+
+ for (i = 0; i < n_sets; i++)
{
- size_t len = strlen (vl->label);
-
- *loc++ = vl->value.f;
- *(unsigned char *) loc = len;
- memcpy (&((char *) loc)[1], vl->label, len);
- memset (&((char *) loc)[1 + len], ' ',
- REM_RND_UP (len + 1, sizeof (flt64)));
- loc += DIV_RND_UP (len + 1, sizeof (flt64));
+ const struct label_set *set = sets[i];
+ const struct val_labs *val_labs = set->val_labs;
+ size_t n_labels = val_labs_count (val_labs);
+ int width = val_labs_get_width (val_labs);
+ const struct val_lab **labels;
+ size_t j;
+
+ /* Value label record. */
+ write_int (w, 3); /* Record type. */
+ write_int (w, n_labels);
+ labels = val_labs_sorted (val_labs);
+ for (j = 0; j < n_labels; j++)
+ {
+ const struct val_lab *vl = labels[j];
+ char *label = recode_string (dict_get_encoding (d), UTF8,
+ val_lab_get_escaped_label (vl), -1);
+ uint8_t len = MIN (strlen (label), 255);
+
+ write_value (w, val_lab_get_value (vl), width);
+ write_bytes (w, &len, 1);
+ write_bytes (w, label, len);
+ write_zeros (w, REM_RND_UP (len + 1, 8));
+ free (label);
+ }
+ free (labels);
+
+ /* Value label variable record. */
+ write_int (w, 4); /* Record type. */
+ write_int (w, set->n_indexes);
+ for (j = 0; j < set->n_indexes; j++)
+ write_int (w, set->indexes[j] + 1);
}
-
- buf_write (w, vlr, vlr_size);
- free (vlr);
-
- vir.rec_type = 4;
- vir.n_vars = 1;
- vir.vars[0] = idx + 1;
- buf_write (w, &vir, sizeof vir);
+
+ for (i = 0; i < n_sets; i++)
+ {
+ struct label_set *set = sets[i];
+
+ free (set->indexes);
+ free (set);
+ }
+ free (sets);
+ hmap_destroy (&same_sets);
}
/* Writes record type 6, document record. */
static void
write_documents (struct sfm_writer *w, const struct dictionary *d)
{
- struct
- {
- int32_t rec_type ; /* Always 6. */
- int32_t n_lines ; /* Number of lines of documents. */
- } ATTRIBUTE((packed)) rec_6;
-
- const char *documents;
- size_t n_lines;
+ const struct string_array *docs = dict_get_documents (d);
+ const char *enc = dict_get_encoding (d);
+ size_t i;
- documents = dict_get_documents (d);
- n_lines = strlen (documents) / 80;
+ write_int (w, 6); /* Record type. */
+ write_int (w, docs->n);
+ for (i = 0; i < docs->n; i++)
+ {
+ char *s = recode_string (enc, "UTF-8", docs->strings[i], -1);
+ size_t s_len = strlen (s);
+ size_t write_len = MIN (s_len, DOC_LINE_LENGTH);
- rec_6.rec_type = 6;
- rec_6.n_lines = n_lines;
- buf_write (w, &rec_6, sizeof rec_6);
- buf_write (w, documents, 80 * n_lines);
+ write_bytes (w, s, write_len);
+ write_spaces (w, DOC_LINE_LENGTH - write_len);
+ free (s);
+ }
}
-/* Write the alignment, width and scale values */
static void
-write_variable_display_parameters (struct sfm_writer *w,
- const struct dictionary *dict)
+put_attrset (struct string *string, const struct attrset *attrs)
{
- int i;
+ const struct attribute *attr;
+ struct attrset_iterator i;
- struct
- {
- int32_t rec_type ;
- int32_t subtype ;
- int32_t elem_size ;
- int32_t n_elem ;
- } ATTRIBUTE((packed)) vdp_hdr;
+ for (attr = attrset_first (attrs, &i); attr != NULL;
+ attr = attrset_next (attrs, &i))
+ {
+ size_t n_values = attribute_get_n_values (attr);
+ size_t j;
+
+ ds_put_cstr (string, attribute_get_name (attr));
+ ds_put_byte (string, '(');
+ for (j = 0; j < n_values; j++)
+ ds_put_format (string, "'%s'\n", attribute_get_value (attr, j));
+ ds_put_byte (string, ')');
+ }
+}
- vdp_hdr.rec_type = 7;
- vdp_hdr.subtype = 11;
- vdp_hdr.elem_size = 4;
- vdp_hdr.n_elem = w->var_cnt_vls * 3;
+static void
+write_data_file_attributes (struct sfm_writer *w,
+ const struct dictionary *d)
+{
+ struct string s = DS_EMPTY_INITIALIZER;
+ put_attrset (&s, dict_get_attributes (d));
+ write_utf8_record (w, dict_get_encoding (d), &s, 17);
+ ds_destroy (&s);
+}
- buf_write (w, &vdp_hdr, sizeof vdp_hdr);
+static void
+add_role_attribute (enum var_role role, struct attrset *attrs)
+{
+ struct attribute *attr;
+ const char *s;
- for ( i = 0 ; i < w->var_cnt ; ++i )
+ switch (role)
{
- struct variable *v;
- struct
- {
- int32_t measure ;
- int32_t width ;
- int32_t align ;
- } ATTRIBUTE((packed)) params;
-
- v = dict_get_var(dict, i);
-
- params.measure = (var_get_measure (v) == MEASURE_NOMINAL ? 1
- : var_get_measure (v) == MEASURE_ORDINAL ? 2
- : 3);
- params.width = var_get_display_width (v);
- params.align = (var_get_alignment (v) == ALIGN_LEFT ? 0
- : var_get_alignment (v) == ALIGN_RIGHT ? 1
- : 2);
-
- buf_write (w, ¶ms, sizeof(params));
-
- if (var_is_long_string (v))
- {
- int wcount = var_get_width (v) - EFFECTIVE_LONG_STRING_LENGTH ;
-
- while (wcount > 0)
- {
- params.width = wcount >= MIN_VERY_LONG_STRING ? 32 : wcount;
-
- buf_write (w, ¶ms, sizeof(params));
-
- wcount -= EFFECTIVE_LONG_STRING_LENGTH ;
- }
- }
+ case ROLE_INPUT:
+ default:
+ s = "0";
+ break;
+
+ case ROLE_TARGET:
+ s = "1";
+ break;
+
+ case ROLE_BOTH:
+ s = "2";
+ break;
+
+ case ROLE_NONE:
+ s = "3";
+ break;
+
+ case ROLE_PARTITION:
+ s = "4";
+ break;
+
+ case ROLE_SPLIT:
+ s = "5";
+ break;
}
+ attrset_delete (attrs, "$@Role");
+
+ attr = attribute_create ("$@Role");
+ attribute_add_value (attr, s);
+ attrset_add (attrs, attr);
}
-/* Writes the table of lengths for Very Long String Variables */
-static void
-write_vls_length_table (struct sfm_writer *w,
- const struct dictionary *dict)
+static void
+write_variable_attributes (struct sfm_writer *w, const struct dictionary *d)
{
- int i;
- struct
- {
- int32_t rec_type ;
- int32_t subtype ;
- int32_t elem_size ;
- int32_t n_elem ;
- } ATTRIBUTE((packed)) vls_hdr;
+ struct string s = DS_EMPTY_INITIALIZER;
+ size_t n_vars = dict_get_n_vars (d);
+ size_t n_attrsets = 0;
+ size_t i;
- struct string vls_length_map;
+ for (i = 0; i < n_vars; i++)
+ {
+ struct variable *v = dict_get_var (d, i);
+ struct attrset attrs;
- ds_init_empty (&vls_length_map);
+ attrset_clone (&attrs, var_get_attributes (v));
- vls_hdr.rec_type = 7;
- vls_hdr.subtype = 14;
- vls_hdr.elem_size = 1;
+ add_role_attribute (var_get_role (v), &attrs);
+ if (n_attrsets++)
+ ds_put_byte (&s, '/');
+ ds_put_format (&s, "%s:", var_get_name (v));
+ put_attrset (&s, &attrs);
+ attrset_destroy (&attrs);
+ }
+ if (n_attrsets)
+ write_utf8_record (w, dict_get_encoding (d), &s, 18);
+ ds_destroy (&s);
+}
+/* Write multiple response sets. If PRE_V14 is true, writes sets supported by
+ SPSS before release 14, otherwise writes sets supported only by later
+ versions. */
+static void
+write_mrsets (struct sfm_writer *w, const struct dictionary *dict,
+ bool pre_v14)
+{
+ const char *encoding = dict_get_encoding (dict);
+ struct string s = DS_EMPTY_INITIALIZER;
+ size_t n_mrsets;
+ size_t i;
- for (i = 0; i < dict_get_var_cnt (dict); ++i)
+ if (is_encoding_ebcdic_compatible (encoding))
{
- const struct variable *v = dict_get_var (dict, i);
-
- if ( var_get_width (v) < MIN_VERY_LONG_STRING )
- continue;
-
- ds_put_format (&vls_length_map, "%s=%05d",
- var_get_short_name (v), var_get_width (v));
- ds_put_char (&vls_length_map, '\0');
- ds_put_char (&vls_length_map, '\t');
+ /* FIXME. */
+ return;
}
- vls_hdr.n_elem = ds_length (&vls_length_map);
+ n_mrsets = dict_get_n_mrsets (dict);
+ if (n_mrsets == 0)
+ return;
- if ( vls_hdr.n_elem > 0 )
+ for (i = 0; i < n_mrsets; i++)
{
- buf_write (w, &vls_hdr, sizeof vls_hdr);
- buf_write (w, ds_data (&vls_length_map), ds_length (&vls_length_map));
+ const struct mrset *mrset = dict_get_mrset (dict, i);
+ char *name;
+ size_t j;
+
+ if ((mrset->type != MRSET_MD || mrset->cat_source != MRSET_COUNTEDVALUES)
+ != pre_v14)
+ continue;
+
+ name = recode_string (encoding, "UTF-8", mrset->name, -1);
+ ds_put_format (&s, "%s=", name);
+ free (name);
+
+ if (mrset->type == MRSET_MD)
+ {
+ char *counted;
+
+ if (mrset->cat_source == MRSET_COUNTEDVALUES)
+ ds_put_format (&s, "E %d ", mrset->label_from_var_label ? 11 : 1);
+ else
+ ds_put_byte (&s, 'D');
+
+ if (mrset->width == 0)
+ counted = xasprintf ("%.0f", mrset->counted.f);
+ else
+ counted = xmemdup0 (mrset->counted.s, mrset->width);
+ ds_put_format (&s, "%zu %s", strlen (counted), counted);
+ free (counted);
+ }
+ else
+ ds_put_byte (&s, 'C');
+ ds_put_byte (&s, ' ');
+
+ if (mrset->label && !mrset->label_from_var_label)
+ {
+ char *label = recode_string (encoding, "UTF-8", mrset->label, -1);
+ ds_put_format (&s, "%zu %s", strlen (label), label);
+ free (label);
+ }
+ else
+ ds_put_cstr (&s, "0 ");
+
+ for (j = 0; j < mrset->n_vars; j++)
+ {
+ const char *short_name_utf8 = var_get_short_name (mrset->vars[j], 0);
+ char *lower_name_utf8 = utf8_to_lower (short_name_utf8);
+ char *short_name = recode_string (encoding, "UTF-8",
+ lower_name_utf8, -1);
+ ds_put_format (&s, " %s", short_name);
+ free (short_name);
+ free (lower_name_utf8);
+ }
+ ds_put_byte (&s, '\n');
}
- ds_destroy (&vls_length_map);
+ if (!ds_is_empty (&s))
+ write_string_record (w, ds_ss (&s), pre_v14 ? 7 : 19);
+ ds_destroy (&s);
}
-/* Writes the long variable name table */
+/* Write the alignment, width and scale values. */
static void
-write_longvar_table (struct sfm_writer *w, const struct dictionary *dict)
+write_variable_display_parameters (struct sfm_writer *w,
+ const struct dictionary *dict)
{
- struct
- {
- int32_t rec_type ;
- int32_t subtype ;
- int32_t elem_size ;
- int32_t n_elem ;
- } ATTRIBUTE((packed)) lv_hdr;
+ int i;
- struct string long_name_map;
- size_t i;
+ write_int (w, 7); /* Record type. */
+ write_int (w, 11); /* Record subtype. */
+ write_int (w, 4); /* Data item (int32) size. */
+ write_int (w, w->n_segments * 3); /* Number of data items. */
- ds_init_empty (&long_name_map);
- for (i = 0; i < dict_get_var_cnt (dict); i++)
+ for (i = 0; i < dict_get_n_vars (dict); ++i)
{
struct variable *v = dict_get_var (dict, i);
-
- if (i)
- ds_put_char (&long_name_map, '\t');
- ds_put_format (&long_name_map, "%s=%s",
- var_get_short_name (v), var_get_name (v));
- }
-
- lv_hdr.rec_type = 7;
- lv_hdr.subtype = 13;
- lv_hdr.elem_size = 1;
- lv_hdr.n_elem = ds_length (&long_name_map);
-
- buf_write (w, &lv_hdr, sizeof lv_hdr);
- buf_write (w, ds_data (&long_name_map), ds_length (&long_name_map));
+ int width = var_get_width (v);
+ int n_segments = sfm_width_to_segments (width);
+ int measure = (var_get_measure (v) == MEASURE_NOMINAL ? 1
+ : var_get_measure (v) == MEASURE_ORDINAL ? 2
+ : 3);
+ int alignment = (var_get_alignment (v) == ALIGN_LEFT ? 0
+ : var_get_alignment (v) == ALIGN_RIGHT ? 1
+ : 2);
+ int i;
- ds_destroy (&long_name_map);
+ for (i = 0; i < n_segments; i++)
+ {
+ int width_left = width - sfm_segment_effective_offset (width, i);
+ write_int (w, measure);
+ write_int (w, (i == 0 ? var_get_display_width (v)
+ : var_default_display_width (width_left)));
+ write_int (w, alignment);
+ }
+ }
}
-/* Writes record type 7, subtypes 3 and 4. */
+/* Writes the table of lengths for very long string variables. */
static void
-write_rec_7_34 (struct sfm_writer *w)
+write_vls_length_table (struct sfm_writer *w,
+ const struct dictionary *dict)
{
- struct
+ struct string map;
+ int i;
+
+ ds_init_empty (&map);
+ for (i = 0; i < dict_get_n_vars (dict); ++i)
{
- int32_t rec_type_3 ;
- int32_t subtype_3 ;
- int32_t data_type_3 ;
- int32_t n_elem_3 ;
- int32_t elem_3[8] ;
- int32_t rec_type_4 ;
- int32_t subtype_4 ;
- int32_t data_type_4 ;
- int32_t n_elem_4 ;
- flt64 elem_4[3] ;
- } ATTRIBUTE((packed)) rec_7;
-
- /* Components of the version number, from major to minor. */
- int version_component[3];
-
- /* Used to step through the version string. */
- char *p;
-
- /* Parses the version string, which is assumed to be of the form
- #.#x, where each # is a string of digits, and x is a single
- letter. */
- version_component[0] = strtol (bare_version, &p, 10);
- if (*p == '.')
- p++;
- version_component[1] = strtol (bare_version, &p, 10);
- version_component[2] = (isalpha ((unsigned char) *p)
- ? tolower ((unsigned char) *p) - 'a' : 0);
-
- rec_7.rec_type_3 = 7;
- rec_7.subtype_3 = 3;
- rec_7.data_type_3 = sizeof (int32_t);
- rec_7.n_elem_3 = 8;
- rec_7.elem_3[0] = version_component[0];
- rec_7.elem_3[1] = version_component[1];
- rec_7.elem_3[2] = version_component[2];
- rec_7.elem_3[3] = -1;
-
- /* PORTME: 1=IEEE754, 2=IBM 370, 3=DEC VAX E. */
-#ifdef FPREP_IEEE754
- rec_7.elem_3[4] = 1;
-#endif
-
- rec_7.elem_3[5] = 1;
-
- /* PORTME: 1=big-endian, 2=little-endian. */
-#if WORDS_BIGENDIAN
- rec_7.elem_3[6] = 1;
-#else
- rec_7.elem_3[6] = 2;
-#endif
-
- /* PORTME: 1=EBCDIC, 2=7-bit ASCII, 3=8-bit ASCII, 4=DEC Kanji. */
- rec_7.elem_3[7] = 2;
-
- rec_7.rec_type_4 = 7;
- rec_7.subtype_4 = 4;
- rec_7.data_type_4 = sizeof (flt64);
- rec_7.n_elem_4 = 3;
- rec_7.elem_4[0] = -FLT64_MAX;
- rec_7.elem_4[1] = FLT64_MAX;
- rec_7.elem_4[2] = second_lowest_flt64;
-
- buf_write (w, &rec_7, sizeof rec_7);
+ const struct variable *v = dict_get_var (dict, i);
+ if (sfm_width_to_segments (var_get_width (v)) > 1)
+ ds_put_format (&map, "%s=%05d%c\t",
+ var_get_short_name (v, 0), var_get_width (v), 0);
+ }
+ if (!ds_is_empty (&map))
+ write_utf8_record (w, dict_get_encoding (dict), &map, 14);
+ ds_destroy (&map);
}
-/* Write NBYTES starting at BUF to the system file represented by
- H. */
static void
-buf_write (struct sfm_writer *w, const void *buf, size_t nbytes)
+write_long_string_value_labels (struct sfm_writer *w,
+ const struct dictionary *dict)
{
- assert (buf != NULL);
- fwrite (buf, nbytes, 1, w->file);
-}
+ const char *encoding = dict_get_encoding (dict);
+ size_t n_vars = dict_get_n_vars (dict);
+ size_t size, i;
-/* Copies string DEST to SRC with the proviso that DEST does not reach
- byte END; no null terminator is copied. Returns a pointer to the
- byte after the last byte copied. */
-static char *
-append_string_max (char *dest, const char *src, const char *end)
-{
- int nbytes = MIN (end - dest, (int) strlen (src));
- memcpy (dest, src, nbytes);
- return dest + nbytes;
-}
+ /* Figure out the size in advance. */
+ size = 0;
+ for (i = 0; i < n_vars; i++)
+ {
+ struct variable *var = dict_get_var (dict, i);
+ const struct val_labs *val_labs = var_get_value_labels (var);
+ int width = var_get_width (var);
+ const struct val_lab *val_lab;
+
+ if (val_labs_count (val_labs) == 0 || width < 9)
+ continue;
+
+ size += 12;
+ size += recode_string_len (encoding, "UTF-8", var_get_name (var), -1);
+ for (val_lab = val_labs_first (val_labs); val_lab != NULL;
+ val_lab = val_labs_next (val_labs, val_lab))
+ {
+ size += 8 + width;
+ size += recode_string_len (encoding, "UTF-8",
+ val_lab_get_escaped_label (val_lab), -1);
+ }
+ }
+ if (size == 0)
+ return;
-/* Makes certain that the compression buffer of H has room for another
- element. If there's not room, pads out the current instruction
- octet with zero and dumps out the buffer. */
-static void
-ensure_buf_space (struct sfm_writer *w)
-{
- if (w->ptr >= w->end)
+ write_int (w, 7); /* Record type. */
+ write_int (w, 21); /* Record subtype */
+ write_int (w, 1); /* Data item (byte) size. */
+ write_int (w, size); /* Number of data items. */
+
+ for (i = 0; i < n_vars; i++)
{
- memset (w->x, 0, w->y - w->x);
- w->x = w->y;
- w->ptr = w->buf;
- buf_write (w, w->buf, sizeof *w->buf * 128);
+ struct variable *var = dict_get_var (dict, i);
+ const struct val_labs *val_labs = var_get_value_labels (var);
+ int width = var_get_width (var);
+ const struct val_lab *val_lab;
+ char *var_name;
+
+ if (val_labs_count (val_labs) == 0 || width < 9)
+ continue;
+
+ var_name = recode_string (encoding, "UTF-8", var_get_name (var), -1);
+ write_int (w, strlen (var_name));
+ write_bytes (w, var_name, strlen (var_name));
+ free (var_name);
+
+ write_int (w, width);
+ write_int (w, val_labs_count (val_labs));
+ for (val_lab = val_labs_first (val_labs); val_lab != NULL;
+ val_lab = val_labs_next (val_labs, val_lab))
+ {
+ char *label;
+ size_t len;
+
+ write_int (w, width);
+ write_bytes (w, val_lab_get_value (val_lab)->s, width);
+
+ label = recode_string (var_get_encoding (var), "UTF-8",
+ val_lab_get_escaped_label (val_lab), -1);
+ len = strlen (label);
+ write_int (w, len);
+ write_bytes (w, label, len);
+ free (label);
+ }
}
}
-static void write_compressed_data (struct sfm_writer *w, const flt64 *elem);
-
-/* Writes case C to system file W.
- Returns 1 if successful, 0 if an I/O error occurred. */
-int
-sfm_write_case (struct sfm_writer *w, const struct ccase *c)
+static void
+write_long_string_missing_values (struct sfm_writer *w,
+ const struct dictionary *dict)
{
- if (ferror (w->file))
- return 0;
-
- w->case_cnt++;
+ const char *encoding = dict_get_encoding (dict);
+ size_t n_vars = dict_get_n_vars (dict);
+ size_t size, i;
- if (!w->needs_translation && !w->compress
- && sizeof (flt64) == sizeof (union value) && ! w->has_vls )
+ /* Figure out the size in advance. */
+ size = 0;
+ for (i = 0; i < n_vars; i++)
{
- /* Fast path: external and internal representations are the
- same and the dictionary is properly ordered. Write
- directly to file. */
- buf_write (w, case_data_all (c), sizeof (union value) * w->flt64_cnt);
+ struct variable *var = dict_get_var (dict, i);
+ const struct missing_values *mv = var_get_missing_values (var);
+ int width = var_get_width (var);
+
+ if (mv_is_empty (mv) || width < 9)
+ continue;
+
+ size += 4;
+ size += recode_string_len (encoding, "UTF-8", var_get_name (var), -1);
+ size += 1;
+ size += mv_n_values (mv) * (4 + 8);
}
- else
+ if (size == 0)
+ return;
+
+ write_int (w, 7); /* Record type. */
+ write_int (w, 22); /* Record subtype */
+ write_int (w, 1); /* Data item (byte) size. */
+ write_int (w, size); /* Number of data items. */
+
+ for (i = 0; i < n_vars; i++)
{
- /* Slow path: internal and external representations differ.
- Write into a bounce buffer, then write to W. */
- flt64 *bounce;
- flt64 *bounce_cur;
- flt64 *bounce_end;
- size_t bounce_size;
- size_t i;
-
- bounce_size = sizeof *bounce * w->flt64_cnt;
- bounce = bounce_cur = local_alloc (bounce_size);
- bounce_end = bounce + bounce_size;
-
- for (i = 0; i < w->var_cnt; i++)
+ struct variable *var = dict_get_var (dict, i);
+ const struct missing_values *mv = var_get_missing_values (var);
+ int width = var_get_width (var);
+ uint8_t n_missing_values;
+ char *var_name;
+ int j;
+
+ if (mv_is_empty (mv) || width < 9)
+ continue;
+
+ var_name = recode_string (encoding, "UTF-8", var_get_name (var), -1);
+ write_int (w, strlen (var_name));
+ write_bytes (w, var_name, strlen (var_name));
+ free (var_name);
+
+ n_missing_values = mv_n_values (mv);
+ write_bytes (w, &n_missing_values, 1);
+
+ for (j = 0; j < n_missing_values; j++)
{
- struct sfm_var *v = &w->vars[i];
-
- memset(bounce_cur, ' ', v->flt64_cnt * sizeof (flt64));
-
- if (v->width == 0)
- {
- *bounce_cur = case_num_idx (c, v->fv);
- bounce_cur += v->flt64_cnt;
- }
- else
- { int ofs = 0;
- while (ofs < v->width)
- {
- int chunk = MIN (MIN_VERY_LONG_STRING - 1, v->width - ofs);
- int nv = DIV_RND_UP (chunk, sizeof (flt64));
- buf_copy_rpad ((char *) bounce_cur, nv * sizeof (flt64),
- case_data_idx (c, v->fv)->s + ofs, chunk);
- bounce_cur += nv;
- ofs += chunk;
- }
- }
+ const union value *value = mv_get_value (mv, j);
+ write_int (w, 8);
+ write_bytes (w, value->s, 8);
}
+ }
+}
- if (!w->compress)
- buf_write (w, bounce, bounce_size);
- else
- write_compressed_data (w, bounce);
+static void
+write_encoding_record (struct sfm_writer *w,
+ const struct dictionary *d)
+{
+ /* IANA says "...character set names may be up to 40 characters taken
+ from the printable characters of US-ASCII," so character set names
+ don't need to be recoded to be in UTF-8.
+
+ We convert encoding names to uppercase because SPSS writes encoding
+ names in uppercase. */
+ char *encoding = xstrdup (dict_get_encoding (d));
+ str_uppercase (encoding);
+ write_string_record (w, ss_cstr (encoding), 20);
+ free (encoding);
+}
- local_free (bounce);
+/* Writes the long variable name table. */
+static void
+write_longvar_table (struct sfm_writer *w, const struct dictionary *dict)
+{
+ struct string map;
+ size_t i;
+
+ ds_init_empty (&map);
+ for (i = 0; i < dict_get_n_vars (dict); i++)
+ {
+ struct variable *v = dict_get_var (dict, i);
+ if (i)
+ ds_put_byte (&map, '\t');
+ ds_put_format (&map, "%s=%s",
+ var_get_short_name (v, 0), var_get_name (v));
}
-
- return !sfm_write_error (w);
+ write_utf8_record (w, dict_get_encoding (dict), &map, 13);
+ ds_destroy (&map);
}
+/* Write integer information record. */
static void
-put_instruction (struct sfm_writer *w, unsigned char instruction)
+write_integer_info_record (struct sfm_writer *w,
+ const struct dictionary *d)
{
- if (w->x >= w->y)
+ const char *dict_encoding = dict_get_encoding (d);
+ int version_component[3];
+ int float_format;
+ int codepage;
+
+ /* Parse the version string. */
+ memset (version_component, 0, sizeof version_component);
+ sscanf (bare_version, "%d.%d.%d",
+ &version_component[0], &version_component[1], &version_component[2]);
+
+ /* Figure out the floating-point format. */
+ if (FLOAT_NATIVE_64_BIT == FLOAT_IEEE_DOUBLE_LE
+ || FLOAT_NATIVE_64_BIT == FLOAT_IEEE_DOUBLE_BE)
+ float_format = 1;
+ else if (FLOAT_NATIVE_64_BIT == FLOAT_Z_LONG)
+ float_format = 2;
+ else if (FLOAT_NATIVE_64_BIT == FLOAT_VAX_D)
+ float_format = 3;
+ else
+ abort ();
+
+ /* Choose codepage. */
+ codepage = sys_get_codepage_from_encoding (dict_encoding);
+ if (codepage == 0)
{
- ensure_buf_space (w);
- w->x = (unsigned char *) w->ptr++;
- w->y = (unsigned char *) w->ptr;
+ /* The codepage is unknown. Choose a default.
+
+ For an EBCDIC-compatible encoding, use the value for EBCDIC.
+
+ For an ASCII-compatible encoding, default to "7-bit ASCII", because
+ many files use this codepage number regardless of their actual
+ encoding.
+ */
+ if (is_encoding_ascii_compatible (dict_encoding))
+ codepage = 2;
+ else if (is_encoding_ebcdic_compatible (dict_encoding))
+ codepage = 1;
}
- *w->x++ = instruction;
+
+ /* Write record. */
+ write_int (w, 7); /* Record type. */
+ write_int (w, 3); /* Record subtype. */
+ write_int (w, 4); /* Data item (int32) size. */
+ write_int (w, 8); /* Number of data items. */
+ write_int (w, version_component[0]);
+ write_int (w, version_component[1]);
+ write_int (w, version_component[2]);
+ write_int (w, -1); /* Machine code. */
+ write_int (w, float_format);
+ write_int (w, 1); /* Compression code. */
+ write_int (w, INTEGER_NATIVE == INTEGER_MSB_FIRST ? 1 : 2);
+ write_int (w, codepage);
}
+/* Write floating-point information record. */
static void
-put_element (struct sfm_writer *w, const flt64 *elem)
+write_float_info_record (struct sfm_writer *w)
{
- ensure_buf_space (w);
- memcpy (w->ptr++, elem, sizeof *elem);
+ write_int (w, 7); /* Record type. */
+ write_int (w, 4); /* Record subtype. */
+ write_int (w, 8); /* Data item (flt64) size. */
+ write_int (w, 3); /* Number of data items. */
+ write_float (w, SYSMIS); /* System-missing value. */
+ write_float (w, HIGHEST); /* Value used for HIGHEST in missing values. */
+ write_float (w, LOWEST); /* Value used for LOWEST in missing values. */
}
-
+\f
+/* Writes case C to system file W. */
static void
-write_compressed_data (struct sfm_writer *w, const flt64 *elem)
+sys_file_casewriter_write (struct casewriter *writer, void *w_,
+ struct ccase *c)
{
- size_t i;
+ struct sfm_writer *w = w_;
- for (i = 0; i < w->var_cnt; i++)
+ if (ferror (w->file))
{
- struct sfm_var *v = &w->vars[i];
-
- if (v->width == 0)
- {
- if (*elem == -FLT64_MAX)
- put_instruction (w, 255);
- else if (*elem >= 1 - COMPRESSION_BIAS
- && *elem <= 251 - COMPRESSION_BIAS
- && *elem == (int) *elem)
- put_instruction (w, (int) *elem + COMPRESSION_BIAS);
- else
- {
- put_instruction (w, 253);
- put_element (w, elem);
- }
- elem++;
- }
- else
- {
- size_t j;
-
- for (j = 0; j < v->flt64_cnt; j++, elem++)
- {
- if (!memcmp (elem, " ", sizeof (flt64)))
- put_instruction (w, 254);
- else
- {
- put_instruction (w, 253);
- put_element (w, elem);
- }
- }
- }
+ casewriter_force_error (writer);
+ case_unref (c);
+ return;
}
+
+ w->n_cases++;
+
+ if (w->compression == ANY_COMP_NONE)
+ write_case_uncompressed (w, c);
+ else
+ write_case_compressed (w, c);
+
+ case_unref (c);
+}
+
+/* Destroys system file writer W. */
+static void
+sys_file_casewriter_destroy (struct casewriter *writer, void *w_)
+{
+ struct sfm_writer *w = w_;
+ if (!close_writer (w))
+ casewriter_force_error (writer);
}
/* Returns true if an I/O error has occurred on WRITER, false otherwise. */
-bool
-sfm_write_error (const struct sfm_writer *writer)
+static bool
+write_error (const struct sfm_writer *writer)
{
return ferror (writer->file);
}
/* Closes a system file after we're done with it.
Returns true if successful, false if an I/O error occurred. */
-bool
-sfm_close_writer (struct sfm_writer *w)
+static bool
+close_writer (struct sfm_writer *w)
{
bool ok;
-
+
if (w == NULL)
return true;
ok = true;
- if (w->file != NULL)
+ if (w->file != NULL)
{
/* Flush buffer. */
- if (w->buf != NULL && w->ptr > w->buf)
+ flush_compressed (w);
+ if (w->compression == ANY_COMP_ZLIB)
{
- memset (w->x, 0, w->y - w->x);
- buf_write (w, w->buf, (w->ptr - w->buf) * sizeof *w->buf);
+ finish_zstream (w);
+ write_ztrailer (w);
}
fflush (w->file);
- ok = !sfm_write_error (w);
+ ok = !write_error (w);
/* Seek back to the beginning and update the number of cases.
This is just a courtesy to later readers, so there's no need
to check return values or report errors. */
- if (ok && !fseek (w->file, offsetof (struct sysfile_header, case_cnt),
- SEEK_SET))
+ if (ok && w->n_cases <= INT32_MAX && !fseeko (w->file, 80, SEEK_SET))
{
- int32_t case_cnt = w->case_cnt;
- fwrite (&case_cnt, sizeof case_cnt, 1, w->file);
+ write_int (w, w->n_cases);
clearerr (w->file);
}
ok = false;
if (!ok)
- msg (ME, _("An I/O error occurred writing system file \"%s\"."),
+ msg (ME, _("An I/O error occurred writing system file `%s'."),
fh_get_file_name (w->fh));
+
+ if (ok ? !replace_file_commit (w->rf) : !replace_file_abort (w->rf))
+ ok = false;
}
- fh_close (w->fh, "system file", "we");
-
- free (w->buf);
- free (w->vars);
+ free (w->blocks);
+
+ fh_unlock (w->lock);
+ fh_unref (w->fh);
+
+ free (w->sfm_vars);
free (w);
return ok;
}
+
+/* System file writer casewriter class. */
+static const struct casewriter_class sys_file_casewriter_class =
+ {
+ sys_file_casewriter_write,
+ sys_file_casewriter_destroy,
+ NULL,
+ };
+\f
+/* Writes case C to system file W, without compressing it. */
+static void
+write_case_uncompressed (struct sfm_writer *w, const struct ccase *c)
+{
+ size_t i;
+
+ for (i = 0; i < w->sfm_n_vars; i++)
+ {
+ struct sfm_var *v = &w->sfm_vars[i];
+
+ if (v->var_width == 0)
+ write_float (w, case_num_idx (c, v->case_index));
+ else
+ {
+ write_bytes (w, case_str_idx (c, v->case_index) + v->offset,
+ v->segment_width);
+ write_spaces (w, v->padding);
+ }
+ }
+}
+
+/* Writes case C to system file W, with compression. */
+static void
+write_case_compressed (struct sfm_writer *w, const struct ccase *c)
+{
+ size_t i;
+
+ for (i = 0; i < w->sfm_n_vars; i++)
+ {
+ struct sfm_var *v = &w->sfm_vars[i];
+
+ if (v->var_width == 0)
+ {
+ double d = case_num_idx (c, v->case_index);
+ if (d == SYSMIS)
+ put_cmp_opcode (w, 255);
+ else if (d >= 1 - COMPRESSION_BIAS
+ && d <= 251 - COMPRESSION_BIAS
+ && d == (int) d)
+ put_cmp_opcode (w, (int) d + COMPRESSION_BIAS);
+ else
+ put_cmp_number (w, d);
+ }
+ else
+ {
+ int offset = v->offset;
+ int width, padding;
+
+ /* This code properly deals with a width that is not a
+ multiple of 8, by ensuring that the final partial
+ oct (8 byte unit) is treated as padded with spaces
+ on the right. */
+ for (width = v->segment_width; width > 0; width -= 8, offset += 8)
+ {
+ const void *data = case_str_idx (c, v->case_index) + offset;
+ int chunk_size = MIN (width, 8);
+ if (!memcmp (data, " ", chunk_size))
+ put_cmp_opcode (w, 254);
+ else
+ put_cmp_string (w, data, chunk_size);
+ }
+
+ /* This code deals properly with padding that is not a
+ multiple of 8 bytes, by discarding the remainder,
+ which was already effectively padded with spaces in
+ the previous loop. (Note that v->width + v->padding
+ is always a multiple of 8.) */
+ for (padding = v->padding / 8; padding > 0; padding--)
+ put_cmp_opcode (w, 254);
+ }
+ }
+}
+
+static bool
+start_zstream (struct sfm_writer *w)
+{
+ int error;
+
+ error = deflateInit (&w->zstream, 1);
+ if (error != Z_OK)
+ {
+ msg (ME, _("Failed to initialize ZLIB for compression (%s)."),
+ w->zstream.msg);
+ return false;
+ }
+ return true;
+}
+
+static void
+finish_zstream (struct sfm_writer *w)
+{
+ struct zblock *block;
+ int error;
+
+ assert (w->zstream.total_in <= ZBLOCK_SIZE);
+
+ w->zstream.next_in = NULL;
+ w->zstream.avail_in = 0;
+ do
+ {
+ uint8_t buf[4096];
+
+ w->zstream.next_out = buf;
+ w->zstream.avail_out = sizeof buf;
+ error = deflate (&w->zstream, Z_FINISH);
+ write_bytes (w, buf, w->zstream.next_out - buf);
+ }
+ while (error == Z_OK);
+
+ if (error != Z_STREAM_END)
+ msg (ME, _("Failed to complete ZLIB stream compression (%s)."),
+ w->zstream.msg);
+
+ if (w->n_blocks >= w->allocated_blocks)
+ w->blocks = x2nrealloc (w->blocks, &w->allocated_blocks,
+ sizeof *w->blocks);
+ block = &w->blocks[w->n_blocks++];
+ block->uncompressed_size = w->zstream.total_in;
+ block->compressed_size = w->zstream.total_out;
+ deflateEnd (&w->zstream);
+}
+
+static void
+write_zlib (struct sfm_writer *w, const void *data_, unsigned int n)
+{
+ const uint8_t *data = data_;
+
+ while (n > 0)
+ {
+ unsigned int chunk;
+
+ if (w->zstream.total_in >= ZBLOCK_SIZE)
+ {
+ finish_zstream (w);
+ start_zstream (w);
+ }
+
+ chunk = MIN (n, ZBLOCK_SIZE - w->zstream.total_in);
+
+ w->zstream.next_in = CONST_CAST (uint8_t *, data);
+ w->zstream.avail_in = chunk;
+ do
+ {
+ uint8_t buf[4096];
+ int error;
+
+ w->zstream.next_out = buf;
+ w->zstream.avail_out = sizeof buf;
+ error = deflate (&w->zstream, Z_NO_FLUSH);
+ write_bytes (w, buf, w->zstream.next_out - buf);
+ if (error != Z_OK)
+ {
+ msg (ME, _("ZLIB stream compression failed (%s)."),
+ w->zstream.msg);
+ return;
+ }
+ }
+ while (w->zstream.avail_in > 0 || w->zstream.avail_out == 0);
+ data += chunk;
+ n -= chunk;
+ }
+}
+
+static void
+write_ztrailer (struct sfm_writer *w)
+{
+ long long int uncompressed_ofs;
+ long long int compressed_ofs;
+ const struct zblock *block;
+
+ write_int64 (w, -COMPRESSION_BIAS);
+ write_int64 (w, 0);
+ write_int (w, ZBLOCK_SIZE);
+ write_int (w, w->n_blocks);
+
+ uncompressed_ofs = w->zstart;
+ compressed_ofs = w->zstart + 24;
+ for (block = w->blocks; block < &w->blocks[w->n_blocks]; block++)
+ {
+ write_int64 (w, uncompressed_ofs);
+ write_int64 (w, compressed_ofs);
+ write_int (w, block->uncompressed_size);
+ write_int (w, block->compressed_size);
+
+ uncompressed_ofs += block->uncompressed_size;
+ compressed_ofs += block->compressed_size;
+ }
+
+ if (!fseeko (w->file, w->zstart + 8, SEEK_SET))
+ {
+ write_int64 (w, compressed_ofs);
+ write_int64 (w, 24 + (w->n_blocks * 24));
+ }
+ else
+ msg (ME, _("%s: Seek failed (%s)."),
+ fh_get_file_name (w->fh), strerror (errno));
+}
+
+/* Flushes buffered compressed opcodes and data to W. */
+static void
+flush_compressed (struct sfm_writer *w)
+{
+ if (w->n_opcodes)
+ {
+ unsigned int n = 8 * (1 + w->n_elements);
+ if (w->compression == ANY_COMP_SIMPLE)
+ write_bytes (w, w->cbuf, n);
+ else
+ write_zlib (w, w->cbuf, n);
+
+ w->n_opcodes = w->n_elements = 0;
+ memset (w->cbuf[0], 0, 8);
+ }
+}
+
+/* Appends OPCODE to the buffered set of compression opcodes in
+ W. Flushes the compression buffer beforehand if necessary. */
+static void
+put_cmp_opcode (struct sfm_writer *w, uint8_t opcode)
+{
+ if (w->n_opcodes >= 8)
+ flush_compressed (w);
+
+ w->cbuf[0][w->n_opcodes++] = opcode;
+}
+
+/* Appends NUMBER to the buffered compression data in W. */
+static void
+put_cmp_number (struct sfm_writer *w, double number)
+{
+ put_cmp_opcode (w, 253);
+ convert_double_to_output_format (number, w->cbuf[++w->n_elements]);
+}
+
+/* Appends SIZE bytes of DATA to the buffered compression data in
+ W, followed by enough spaces to pad the output data to exactly
+ 8 bytes (thus, SIZE must be no greater than 8). */
+static void
+put_cmp_string (struct sfm_writer *w, const void *data, size_t size)
+{
+ assert (size <= 8);
+
+ put_cmp_opcode (w, 253);
+ w->n_elements++;
+ memset (w->cbuf[w->n_elements], w->space, 8);
+ memcpy (w->cbuf[w->n_elements], data, size);
+}
+\f
+/* Writes 32-bit integer X to the output file for writer W. */
+static void
+write_int (struct sfm_writer *w, int32_t x)
+{
+ write_bytes (w, &x, sizeof x);
+}
+
+/* Writes 64-bit integer X to the output file for writer W. */
+static void
+write_int64 (struct sfm_writer *w, int64_t x)
+{
+ write_bytes (w, &x, sizeof x);
+}
+
+/* Converts NATIVE to the 64-bit format used in output files in
+ OUTPUT. */
+static inline void
+convert_double_to_output_format (double native, uint8_t output[8])
+{
+ /* If "double" is not a 64-bit type, then convert it to a
+ 64-bit type. Otherwise just copy it. */
+ if (FLOAT_NATIVE_DOUBLE != FLOAT_NATIVE_64_BIT)
+ float_convert (FLOAT_NATIVE_DOUBLE, &native, FLOAT_NATIVE_64_BIT, output);
+ else
+ memcpy (output, &native, sizeof native);
+}
+
+/* Writes floating-point number X to the output file for writer
+ W. */
+static void
+write_float (struct sfm_writer *w, double x)
+{
+ uint8_t output[8];
+ convert_double_to_output_format (x, output);
+ write_bytes (w, output, sizeof output);
+}
+
+/* Writes contents of VALUE with the given WIDTH to W, padding
+ with zeros to a multiple of 8 bytes.
+ To avoid a branch, and because we don't actually need to
+ support it, WIDTH must be no bigger than 8. */
+static void
+write_value (struct sfm_writer *w, const union value *value, int width)
+{
+ assert (width <= 8);
+ if (width == 0)
+ write_float (w, value->f);
+ else
+ {
+ write_bytes (w, value->s, width);
+ write_zeros (w, 8 - width);
+ }
+}
+
+/* Writes null-terminated STRING in a field of the given WIDTH to W. If STRING
+ is longer than WIDTH, it is truncated; if STRING is shorter than WIDTH, it
+ is padded on the right with spaces. */
+static void
+write_string (struct sfm_writer *w, const char *string, size_t width)
+{
+ size_t data_bytes = MIN (strlen (string), width);
+ size_t pad_bytes = width - data_bytes;
+ write_bytes (w, string, data_bytes);
+ while (pad_bytes-- > 0)
+ putc (w->space, w->file);
+}
+
+/* Recodes null-terminated UTF-8 encoded STRING into ENCODING, and writes the
+ recoded version in a field of the given WIDTH to W. The string is truncated
+ or padded on the right with spaces to exactly WIDTH bytes. */
+static void
+write_utf8_string (struct sfm_writer *w, const char *encoding,
+ const char *string, size_t width)
+{
+ char *s = recode_string (encoding, "UTF-8", string, -1);
+ write_string (w, s, width);
+ free (s);
+}
+
+/* Writes a record with type 7, subtype SUBTYPE that contains CONTENT recoded
+ from UTF-8 encoded into ENCODING. */
+static void
+write_utf8_record (struct sfm_writer *w, const char *encoding,
+ const struct string *content, int subtype)
+{
+ struct substring s;
+
+ s = recode_substring_pool (encoding, "UTF-8", ds_ss (content), NULL);
+ write_string_record (w, s, subtype);
+ ss_dealloc (&s);
+}
+
+/* Writes a record with type 7, subtype SUBTYPE that contains the string
+ CONTENT. */
+static void
+write_string_record (struct sfm_writer *w,
+ const struct substring content, int subtype)
+{
+ write_int (w, 7);
+ write_int (w, subtype);
+ write_int (w, 1);
+ write_int (w, ss_length (content));
+ write_bytes (w, ss_data (content), ss_length (content));
+}
+
+/* Writes SIZE bytes of DATA to W's output file. */
+static void
+write_bytes (struct sfm_writer *w, const void *data, size_t size)
+{
+ fwrite (data, 1, size, w->file);
+}
+
+/* Writes N zeros to W's output file. */
+static void
+write_zeros (struct sfm_writer *w, size_t n)
+{
+ while (n-- > 0)
+ putc (0, w->file);
+}
+
+/* Writes N spaces to W's output file. */
+static void
+write_spaces (struct sfm_writer *w, size_t n)
+{
+ while (n-- > 0)
+ putc (w->space, w->file);
+}