X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fsys-file-writer.c;h=df5108e2a062758ae79d561f5d1ee51b29a1c152;hb=047d4a8e14cdbb50258bf8c5634db24a88767f24;hp=808a307ec7dc53dc68e26baed0f6fd6c45d41262;hpb=909f10f57c532130c7fb16c103b1328416f5fb06;p=pspp diff --git a/src/data/sys-file-writer.c b/src/data/sys-file-writer.c index 808a307ec7..df5108e2a0 100644 --- a/src/data/sys-file-writer.c +++ b/src/data/sys-file-writer.c @@ -1,109 +1,63 @@ -/* PSPP - computes sample statistics. - Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc. +/* PSPP - a program for statistical analysis. + Copyright (C) 1997-2000, 2006-2014 Free Software Foundation, Inc. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. */ + along with this program. If not, see . */ #include -#include "sys-file-writer.h" -#include "sys-file-private.h" +#include "data/sys-file-writer.h" +#include "data/sys-file-private.h" #include #include -#include +#include #include #include #include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "case.h" -#include "dictionary.h" -#include "file-handle-def.h" -#include "format.h" -#include "missing-values.h" -#include "settings.h" -#include "value-labels.h" -#include "variable.h" - -#include "minmax.h" +#include + +#include "data/attributes.h" +#include "data/case.h" +#include "data/casewriter-provider.h" +#include "data/casewriter.h" +#include "data/dictionary.h" +#include "data/file-handle-def.h" +#include "data/format.h" +#include "data/make-file.h" +#include "data/missing-values.h" +#include "data/mrset.h" +#include "data/settings.h" +#include "data/short-names.h" +#include "data/value-labels.h" +#include "data/variable.h" +#include "libpspp/float-format.h" +#include "libpspp/i18n.h" +#include "libpspp/integer-format.h" +#include "libpspp/message.h" +#include "libpspp/misc.h" +#include "libpspp/str.h" +#include "libpspp/string-array.h" +#include "libpspp/version.h" + +#include "gl/xmemdup0.h" +#include "gl/minmax.h" +#include "gl/unlocked-io.h" +#include "gl/xalloc.h" #include "gettext.h" #define _(msgid) gettext (msgid) - -/* Find 64-bit floating-point type. */ -#if SIZEOF_FLOAT == 8 - #define flt64 float - #define FLT64_MAX FLT_MAX -#elif SIZEOF_DOUBLE == 8 - #define flt64 double - #define FLT64_MAX DBL_MAX -#elif SIZEOF_LONG_DOUBLE == 8 - #define flt64 long double - #define FLT64_MAX LDBL_MAX -#else - #error Which one of your basic types is 64-bit floating point? -#endif - -/* Figure out SYSMIS value for flt64. */ -#include -#if SIZEOF_DOUBLE == 8 -#define second_lowest_flt64 second_lowest_value -#else -#error Must define second_lowest_flt64 for your architecture. -#endif - -/* Record Type 1: General Information. */ -struct sysfile_header - { - char rec_type[4] ; /* 00: Record-type code, "$FL2". */ - char prod_name[60] ; /* 04: Product identification. */ - int32_t layout_code ; /* 40: 2. */ - int32_t nominal_case_size ; /* 44: Number of `value's per case. - Note: some systems set this to -1 */ - int32_t compress ; /* 48: 1=compressed, 0=not compressed. */ - int32_t weight_idx ; /* 4c: 1-based index of weighting var, or 0. */ - int32_t case_cnt ; /* 50: Number of cases, -1 if unknown. */ - flt64 bias ; /* 54: Compression bias (100.0). */ - char creation_date[9] ; /* 5c: `dd mmm yy' creation date of file. */ - char creation_time[8] ; /* 65: `hh:mm:ss' 24-hour creation time. */ - char file_label[64] ; /* 6d: File label. */ - char padding[3] ; /* ad: Ignored padding. */ - } ATTRIBUTE((packed)) ; - -/* Record Type 2: Variable. */ -struct sysfile_variable - { - int32_t rec_type ; /* 2. */ - int32_t type ; /* 0=numeric, 1-255=string width, - -1=continued string. */ - int32_t has_var_label ; /* 1=has a variable label, 0=doesn't. */ - int32_t n_missing_values ; /* Missing value code of -3,-2,0,1,2, or 3. */ - int32_t print ; /* Print format. */ - int32_t write ; /* Write format. */ - char name[SHORT_NAME_LEN] ; /* Variable name. */ - /* The rest of the structure varies. */ - } ATTRIBUTE((packed)) ; +#define N_(msgid) (msgid) /* Compression bias used by PSPP. Values between (1 - COMPRESSION_BIAS) and (251 - COMPRESSION_BIAS) inclusive can be @@ -114,279 +68,264 @@ struct sysfile_variable struct sfm_writer { struct file_handle *fh; /* File handle. */ + struct fh_lock *lock; /* Mutual exclusion for file. */ FILE *file; /* File stream. */ + struct replace_file *rf; /* Ticket for replacing output file. */ + + enum any_compression compression; + casenumber case_cnt; /* Number of cases written so far. */ + uint8_t space; /* ' ' in the file's character encoding. */ - int needs_translation; /* 0=use fast path, 1=translation needed. */ - int compress; /* 1=compressed, 0=not compressed. */ - int case_cnt; /* Number of cases written so far. */ - size_t flt64_cnt; /* Number of flt64 elements in case. */ - bool has_vls; /* Does the dict have very long strings? */ + /* Simple compression buffering. - /* Compression buffering. */ - flt64 *buf; /* Buffered data. */ - flt64 *end; /* Buffer end. */ - flt64 *ptr; /* Current location in buffer. */ - unsigned char *x; /* Location in current instruction octet. */ - unsigned char *y; /* End of instruction octet. */ + Compressed data is output as a series of 8-byte elements, with 1 to 9 + such elements clustered together. The first element in a cluster is 8 + 1-byte opcodes. Some opcodes call for an additional element in the + cluster (hence, if there are eight such opcodes, then the cluster + contains a full 9 elements). + + cbuf[] holds a cluster at a time. */ + uint8_t cbuf[9][8]; + int n_opcodes; /* Number of opcodes in cbuf[0] so far. */ + int n_elements; /* Number of elements in cbuf[] so far. */ + + /* ZLIB compression. */ + z_stream zstream; /* ZLIB deflater. */ + off_t zstart; + struct zblock *blocks; + size_t n_blocks, allocated_blocks; /* Variables. */ - struct sfm_var *vars; /* Variables. */ - size_t var_cnt; /* Number of variables. */ - size_t var_cnt_vls; /* Number of variables including - very long string components. */ + struct sfm_var *sfm_vars; /* Variables. */ + size_t sfm_var_cnt; /* Number of variables. */ + size_t segment_cnt; /* Number of variables including extra segments + for long string variables. */ }; -/* A variable in a system file. */ -struct sfm_var +struct zblock { - int width; /* 0=numeric, otherwise string width. */ - int fv; /* Index into case. */ - size_t flt64_cnt; /* Number of flt64 elements. */ + unsigned int uncompressed_size; + unsigned int compressed_size; }; -static char *append_string_max (char *, const char *, const char *); +static const struct casewriter_class sys_file_casewriter_class; + static void write_header (struct sfm_writer *, const struct dictionary *); -static void buf_write (struct sfm_writer *, const void *, size_t); static void write_variable (struct sfm_writer *, const struct variable *); static void write_value_labels (struct sfm_writer *, - struct variable *, int idx); -static void write_rec_7_34 (struct sfm_writer *); + const struct dictionary *); +static void write_integer_info_record (struct sfm_writer *, + const struct dictionary *); +static void write_float_info_record (struct sfm_writer *); -static void write_longvar_table (struct sfm_writer *w, +static void write_longvar_table (struct sfm_writer *w, const struct dictionary *dict); -static void write_vls_length_table (struct sfm_writer *w, +static void write_encoding_record (struct sfm_writer *w, + const struct dictionary *); + +static void write_vls_length_table (struct sfm_writer *w, const struct dictionary *dict); +static void write_long_string_value_labels (struct sfm_writer *, + const struct dictionary *); +static void write_long_string_missing_values (struct sfm_writer *, + const struct dictionary *); + +static void write_mrsets (struct sfm_writer *, const struct dictionary *, + bool pre_v14); -static void write_variable_display_parameters (struct sfm_writer *w, +static void write_variable_display_parameters (struct sfm_writer *w, const struct dictionary *dict); static void write_documents (struct sfm_writer *, const struct dictionary *); -static inline int -var_flt64_cnt (const struct variable *v) -{ - assert(sizeof(flt64) == MAX_SHORT_STRING); - return sfm_width_to_bytes(var_get_width (v)) / MAX_SHORT_STRING ; -} - -static inline int -var_flt64_cnt_nom (const struct variable *v) -{ - return (var_is_numeric (v) - ? 1 : DIV_RND_UP (var_get_width (v), sizeof (flt64))); -} - +static void write_data_file_attributes (struct sfm_writer *, + const struct dictionary *); +static void write_variable_attributes (struct sfm_writer *, + const struct dictionary *); + +static void write_int (struct sfm_writer *, int32_t); +static void write_int64 (struct sfm_writer *, int64_t); +static inline void convert_double_to_output_format (double, uint8_t[8]); +static void write_float (struct sfm_writer *, double); +static void write_string (struct sfm_writer *, const char *, size_t); +static void write_utf8_string (struct sfm_writer *, const char *encoding, + const char *string, size_t width); +static void write_utf8_record (struct sfm_writer *, const char *encoding, + const struct string *content, int subtype); +static void write_string_record (struct sfm_writer *, + const struct substring content, int subtype); +static void write_bytes (struct sfm_writer *, const void *, size_t); +static void write_zeros (struct sfm_writer *, size_t); +static void write_spaces (struct sfm_writer *, size_t); +static void write_value (struct sfm_writer *, const union value *, int width); + +static void write_case_uncompressed (struct sfm_writer *, + const struct ccase *); +static void write_case_compressed (struct sfm_writer *, const struct ccase *); +static void flush_compressed (struct sfm_writer *); +static void put_cmp_opcode (struct sfm_writer *, uint8_t); +static void put_cmp_number (struct sfm_writer *, double); +static void put_cmp_string (struct sfm_writer *, const void *, size_t); + +static bool start_zstream (struct sfm_writer *); +static void finish_zstream (struct sfm_writer *); +static void write_ztrailer (struct sfm_writer *); + +static bool write_error (const struct sfm_writer *); +static bool close_writer (struct sfm_writer *); /* Returns default options for writing a system file. */ struct sfm_write_options -sfm_writer_default_options (void) +sfm_writer_default_options (void) { struct sfm_write_options opts; + opts.compression = (settings_get_scompression () + ? ANY_COMP_SIMPLE + : ANY_COMP_NONE); opts.create_writeable = true; - opts.compress = get_scompression (); opts.version = 3; return opts; } - -/* Return a short variable name to be used as the continuation of the - variable with the short name SN. - - FIXME: Need to resolve clashes somehow. - - */ -static const char * -cont_var_name(const char *sn, int idx) -{ - static char s[SHORT_NAME_LEN + 1]; - - char abb[SHORT_NAME_LEN + 1 - 3]= {0}; - - strncpy(abb, sn, SHORT_NAME_LEN - 3); - - snprintf(s, SHORT_NAME_LEN + 1, "%s%03d", abb, idx); - - return s; -} - - /* Opens the system file designated by file handle FH for writing - cases from dictionary D according to the given OPTS. If - COMPRESS is nonzero, the system file will be compressed. + cases from dictionary D according to the given OPTS. No reference to D is retained, so it may be modified or destroyed at will after this function returns. D is not modified by this function, except to assign short names. */ -struct sfm_writer * +struct casewriter * sfm_open_writer (struct file_handle *fh, struct dictionary *d, struct sfm_write_options opts) { - struct sfm_writer *w = NULL; + struct encoding_info encoding_info; + struct sfm_writer *w; mode_t mode; - int fd; - int idx; int i; /* Check version. */ - if (opts.version != 2 && opts.version != 3) + if (opts.version != 2 && opts.version != 3) { msg (ME, _("Unknown system file version %d. Treating as version %d."), opts.version, 3); opts.version = 3; } - /* Create file. */ - mode = S_IRUSR | S_IRGRP | S_IROTH; - if (opts.create_writeable) - mode |= S_IWUSR | S_IWGRP | S_IWOTH; - fd = open (fh_get_file_name (fh), O_WRONLY | O_CREAT | O_TRUNC, mode); - if (fd < 0) - goto open_error; - - /* Open file handle. */ - if (!fh_open (fh, FH_REF_FILE, "system file", "we")) - goto error; - /* Create and initialize writer. */ - w = xmalloc (sizeof *w); - w->fh = fh; - w->file = fdopen (fd, "w"); + w = xzalloc (sizeof *w); + w->fh = fh_ref (fh); + w->lock = NULL; + w->file = NULL; + w->rf = NULL; + + /* Use the requested compression, except that no EBCDIC-based ZLIB compressed + files have been observed, so drop back to simple compression for those + files. */ + w->compression = opts.compression; + if (w->compression == ANY_COMP_ZLIB + && is_encoding_ebcdic_compatible (dict_get_encoding (d))) + w->compression = ANY_COMP_SIMPLE; - w->needs_translation = dict_compacting_would_change (d); - w->compress = opts.compress; w->case_cnt = 0; - w->flt64_cnt = 0; - w->has_vls = false; - w->buf = w->end = w->ptr = NULL; - w->x = w->y = NULL; + w->n_opcodes = w->n_elements = 0; + memset (w->cbuf[0], 0, 8); + + /* Figure out how to map in-memory case data to on-disk case + data. Also count the number of segments. Very long strings + occupy multiple segments, otherwise each variable only takes + one segment. */ + w->segment_cnt = sfm_dictionary_to_sfm_vars (d, &w->sfm_vars, + &w->sfm_var_cnt); + + /* Open file handle as an exclusive writer. */ + /* TRANSLATORS: this fragment will be interpolated into + messages in fh_lock() that identify types of files. */ + w->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_WRITE, true); + if (w->lock == NULL) + goto error; - w->var_cnt = dict_get_var_cnt (d); - w->var_cnt_vls = w->var_cnt; - w->vars = xnmalloc (w->var_cnt, sizeof *w->vars); - for (i = 0; i < w->var_cnt; i++) + /* Create the file on disk. */ + mode = 0444; + if (opts.create_writeable) + mode |= 0222; + w->rf = replace_file_start (fh, "wb", mode, &w->file); + if (w->rf == NULL) { - const struct variable *dv = dict_get_var (d, i); - struct sfm_var *sv = &w->vars[i]; - sv->width = var_get_width (dv); - /* spss compatibility nonsense */ - if ( var_get_width (dv) >= MIN_VERY_LONG_STRING ) - w->has_vls = true; - - sv->fv = var_get_case_index (dv); - sv->flt64_cnt = var_flt64_cnt (dv); + msg (ME, _("Error opening `%s' for writing as a system file: %s."), + fh_get_file_name (fh), strerror (errno)); + goto error; } - /* Check that file create succeeded. */ - if (w->file == NULL) - { - close (fd); - goto open_error; - } + get_encoding_info (&encoding_info, dict_get_encoding (d)); + w->space = encoding_info.space[0]; /* Write the file header. */ write_header (w, d); /* Write basic variable info. */ - dict_assign_short_names (d); + short_names_assign (d); for (i = 0; i < dict_get_var_cnt (d); i++) - { - int count = 0; - const struct variable *v = dict_get_var(d, i); - int wcount = var_get_width (v); - - do { - struct variable *var_cont = var_clone (v); - var_set_short_name (var_cont, var_get_short_name (v)); - if ( var_is_alpha (v)) - { - if ( 0 != count ) - { - var_clear_missing_values (var_cont); - var_set_short_name (var_cont, - cont_var_name (var_get_short_name (v), - count)); - var_clear_label (var_cont); - w->var_cnt_vls++; - } - count++; - if ( wcount >= MIN_VERY_LONG_STRING ) - { - var_set_width (var_cont, MIN_VERY_LONG_STRING - 1); - wcount -= EFFECTIVE_LONG_STRING_LENGTH; - } - else - { - var_set_width (var_cont, wcount); - wcount -= var_get_width (var_cont); - } - } - - write_variable (w, var_cont); - var_destroy (var_cont); - } while(wcount > 0); - } - - /* Write out value labels. */ - for (idx = i = 0; i < dict_get_var_cnt (d); i++) - { - struct variable *v = dict_get_var (d, i); + write_variable (w, dict_get_var (d, i)); - write_value_labels (w, v, idx); - idx += var_flt64_cnt (v); - } + write_value_labels (w, d); - if (dict_get_documents (d) != NULL) + if (dict_get_document_line_cnt (d) > 0) write_documents (w, d); - write_rec_7_34 (w); + write_integer_info_record (w, d); + write_float_info_record (w); + + write_mrsets (w, d, true); write_variable_display_parameters (w, d); - if (opts.version >= 3) + if (opts.version >= 3) write_longvar_table (w, d); - write_vls_length_table(w, d); + write_vls_length_table (w, d); - /* Write end-of-headers record. */ - { - struct - { - int32_t rec_type ; - int32_t filler ; - } ATTRIBUTE((packed)) - rec_999; + write_long_string_value_labels (w, d); + write_long_string_missing_values (w, d); - rec_999.rec_type = 999; - rec_999.filler = 0; + if (opts.version >= 3) + { + if (attrset_count (dict_get_attributes (d))) + write_data_file_attributes (w, d); + write_variable_attributes (w, d); + } - buf_write (w, &rec_999, sizeof rec_999); - } + write_mrsets (w, d, false); - if (w->compress) + write_encoding_record (w, d); + + /* Write end-of-headers record. */ + write_int (w, 999); + write_int (w, 0); + + if (w->compression == ANY_COMP_ZLIB) { - w->buf = xnmalloc (128, sizeof *w->buf); - w->ptr = w->buf; - w->end = &w->buf[128]; - w->x = (unsigned char *) w->ptr++; - w->y = (unsigned char *) w->ptr; + w->zstream.zalloc = Z_NULL; + w->zstream.zfree = Z_NULL; + w->zstream.opaque = Z_NULL; + w->zstart = ftello (w->file); + + write_int64 (w, w->zstart); + write_int64 (w, 0); + write_int64 (w, 0); + + start_zstream (w); } - if (sfm_write_error (w)) + if (write_error (w)) goto error; - - return w; - error: - sfm_close_writer (w); - return NULL; + return casewriter_create (dict_get_proto (d), &sys_file_casewriter_class, w); - open_error: - msg (ME, _("Error opening \"%s\" for writing as a system file: %s."), - fh_get_file_name (fh), strerror (errno)); - goto error; +error: + close_writer (w); + return NULL; } /* Returns value of X truncated to two least-significant digits. */ @@ -400,65 +339,83 @@ rerange (int x) return x; } +/* Calculates the offset of data for TARGET_VAR from the + beginning of each case's data for dictionary D. The return + value is in "octs" (8-byte units). */ +static int +calc_oct_idx (const struct dictionary *d, struct variable *target_var) +{ + int oct_idx; + int i; + + oct_idx = 0; + for (i = 0; i < dict_get_var_cnt (d); i++) + { + struct variable *var = dict_get_var (d, i); + if (var == target_var) + break; + oct_idx += sfm_width_to_octs (var_get_width (var)); + } + return oct_idx; +} + /* Write the sysfile_header header to system file W. */ static void write_header (struct sfm_writer *w, const struct dictionary *d) { - struct sysfile_header hdr; - char *p; - int i; + const char *dict_encoding = dict_get_encoding (d); + char prod_name[61]; + const char *file_label; + struct variable *weight; time_t t; - memcpy (hdr.rec_type, "$FL2", 4); + /* Record-type code. */ + if (is_encoding_ebcdic_compatible (dict_encoding)) + write_string (w, EBCDIC_MAGIC, 4); + else if (w->compression == ANY_COMP_ZLIB) + write_string (w, ASCII_ZMAGIC, 4); + else + write_string (w, ASCII_MAGIC, 4); - p = stpcpy (hdr.prod_name, "@(#) SPSS DATA FILE "); - p = append_string_max (p, version, &hdr.prod_name[60]); - p = append_string_max (p, " - ", &hdr.prod_name[60]); - p = append_string_max (p, host_system, &hdr.prod_name[60]); - memset (p, ' ', &hdr.prod_name[60] - p); + /* Product identification. */ + snprintf (prod_name, sizeof prod_name, "@(#) SPSS DATA FILE %s - %s", + version, host_system); + write_utf8_string (w, dict_encoding, prod_name, 60); - hdr.layout_code = 2; + /* Layout code. */ + write_int (w, 2); - w->flt64_cnt = 0; - for (i = 0; i < dict_get_var_cnt (d); i++) - { - w->flt64_cnt += var_flt64_cnt (dict_get_var (d, i)); - } - hdr.nominal_case_size = w->flt64_cnt; + /* Number of `union value's per case. */ + write_int (w, calc_oct_idx (d, NULL)); - hdr.compress = w->compress; + /* Compressed? */ + write_int (w, (w->compression == ANY_COMP_NONE ? 0 + : w->compression == ANY_COMP_SIMPLE ? 1 + : 2)); - if (dict_get_weight (d) != NULL) - { - const struct variable *weight_var; - int recalc_weight_idx = 1; - int i; + /* Weight variable. */ + weight = dict_get_weight (d); + write_int (w, weight != NULL ? calc_oct_idx (d, weight) + 1 : 0); - weight_var = dict_get_weight (d); - for (i = 0; ; i++) - { - struct variable *v = dict_get_var (d, i); - if (v == weight_var) - break; - recalc_weight_idx += var_flt64_cnt (v); - } - hdr.weight_idx = recalc_weight_idx; - } - else - hdr.weight_idx = 0; + /* Number of cases. We don't know this in advance, so we write + -1 to indicate an unknown number of cases. Later we can + come back and overwrite it with the true value. */ + write_int (w, -1); - hdr.case_cnt = -1; - hdr.bias = COMPRESSION_BIAS; + /* Compression bias. */ + write_float (w, COMPRESSION_BIAS); + /* Creation date and time. */ + char *creation_date, *creation_time; if (time (&t) == (time_t) -1) { - memcpy (hdr.creation_date, "01 Jan 70", 9); - memcpy (hdr.creation_time, "00:00:00", 8); + creation_date = xstrdup ("01 Jan 70"); + creation_time = xstrdup ( "00:00:00"); } else { - static const char *month_name[12] = + static const char *const month_name[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", @@ -470,629 +427,847 @@ write_header (struct sfm_writer *w, const struct dictionary *d) int hour = rerange (tmp->tm_hour + 1); int min = rerange (tmp->tm_min + 1); int sec = rerange (tmp->tm_sec + 1); - char buf[10]; - sprintf (buf, "%02d %s %02d", day, month_name[mon - 1], year); - memcpy (hdr.creation_date, buf, sizeof hdr.creation_date); - sprintf (buf, "%02d:%02d:%02d", hour - 1, min - 1, sec - 1); - memcpy (hdr.creation_time, buf, sizeof hdr.creation_time); + creation_date = xasprintf ("%02d %s %02d", + day, month_name[mon - 1], year); + creation_time = xasprintf ("%02d:%02d:%02d", hour - 1, min - 1, sec - 1); } - - { - const char *label = dict_get_label (d); - if (label == NULL) - label = ""; + write_utf8_string (w, dict_encoding, creation_date, 9); + write_utf8_string (w, dict_encoding, creation_time, 8); + free (creation_time); + free (creation_date); + + /* File label. */ + file_label = dict_get_label (d); + if (file_label == NULL) + file_label = ""; + write_utf8_string (w, dict_encoding, file_label, 64); + + /* Padding. */ + write_zeros (w, 3); +} - buf_copy_str_rpad (hdr.file_label, sizeof hdr.file_label, label); - } - - memset (hdr.padding, 0, sizeof hdr.padding); +/* Write format spec FMT to W, after adjusting it to be + compatible with the given WIDTH. */ +static void +write_format (struct sfm_writer *w, struct fmt_spec fmt, int width) +{ + assert (fmt_check_output (&fmt)); + assert (sfm_width_to_segments (width) == 1); - buf_write (w, &hdr, sizeof hdr); + if (width > 0) + fmt_resize (&fmt, width); + write_int (w, (fmt_to_io (fmt.type) << 16) | (fmt.w << 8) | fmt.d); } -/* Translates format spec from internal form in SRC to system file - format in DEST. */ -static inline void -write_format_spec (const struct fmt_spec *src, int32_t *dest) +/* Write a string continuation variable record for each 8-byte + section beyond the initial 8 bytes, for a variable of the + given WIDTH. */ +static void +write_variable_continuation_records (struct sfm_writer *w, int width) { - assert (fmt_check_output (src)); - *dest = (fmt_to_io (src->type) << 16) | (src->w << 8) | src->d; + int position; + + assert (sfm_width_to_segments (width) == 1); + for (position = 8; position < width; position += 8) + { + write_int (w, 2); /* Record type. */ + write_int (w, -1); /* Width. */ + write_int (w, 0); /* No variable label. */ + write_int (w, 0); /* No missing values. */ + write_int (w, 0); /* Print format. */ + write_int (w, 0); /* Write format. */ + write_zeros (w, 8); /* Name. */ + } } -/* Write the variable record(s) for primary variable P and secondary - variable S to system file W. */ +/* Write the variable record(s) for variable V to system file + W. */ static void write_variable (struct sfm_writer *w, const struct variable *v) { - struct sysfile_variable sv; + int width = var_get_width (v); + int segment_cnt = sfm_width_to_segments (width); + int seg0_width = sfm_segment_alloc_width (width, 0); + const char *encoding = var_get_encoding (v); + int i; - /* Missing values. */ - struct missing_values mv; - flt64 m[3]; /* Missing value values. */ - int nm; /* Number of missing values, possibly negative. */ - const char *label = var_get_label (v); + /* Record type. */ + write_int (w, 2); - sv.rec_type = 2; - sv.type = MIN (var_get_width (v), MIN_VERY_LONG_STRING - 1); - sv.has_var_label = label != NULL; + /* Width. */ + write_int (w, seg0_width); - mv_copy (&mv, var_get_missing_values (v)); - nm = 0; - if (mv_has_range (&mv)) - { - double x, y; - mv_pop_range (&mv, &x, &y); - m[nm++] = x == LOWEST ? second_lowest_flt64 : x; - m[nm++] = y == HIGHEST ? FLT64_MAX : y; - } - while (mv_has_value (&mv)) + /* Variable has a variable label? */ + write_int (w, var_has_label (v)); + + /* Number of missing values. If there is a range, then the + range counts as 2 missing values and causes the number to be + negated. + + Missing values for long string variables are written in a separate + record. */ + if (width <= MAX_SHORT_STRING) { - union value value; - mv_pop_value (&mv, &value); - if (var_is_numeric (v)) - m[nm] = value.f; + const struct missing_values *mv = var_get_missing_values (v); + if (mv_has_range (mv)) + write_int (w, -2 - mv_n_values (mv)); else - buf_copy_rpad ((char *) &m[nm], sizeof m[nm], value.s, - var_get_width (v)); - nm++; + write_int (w, mv_n_values (mv)); } - if (mv_has_range (var_get_missing_values (v))) - nm = -nm; - - sv.n_missing_values = nm; - write_format_spec (var_get_print_format (v), &sv.print); - write_format_spec (var_get_write_format (v), &sv.write); - buf_copy_str_rpad (sv.name, sizeof sv.name, var_get_short_name (v)); - buf_write (w, &sv, sizeof sv); + else + write_int (w, 0); - if (label != NULL) - { - struct label - { - int32_t label_len ; - char label[255] ; - } ATTRIBUTE((packed)) - l; + /* Print and write formats. */ + write_format (w, *var_get_print_format (v), seg0_width); + write_format (w, *var_get_write_format (v), seg0_width); - int ext_len; + /* Short name. + The full name is in a translation table written + separately. */ + write_utf8_string (w, encoding, var_get_short_name (v, 0), 8); - l.label_len = MIN (strlen (label), 255); - ext_len = ROUND_UP (l.label_len, sizeof l.label_len); - memcpy (l.label, label, l.label_len); - memset (&l.label[l.label_len], ' ', ext_len - l.label_len); + /* Value label. */ + if (var_has_label (v)) + { + char *label = recode_string (encoding, UTF8, var_get_label (v), -1); + size_t label_len = MIN (strlen (label), 255); + size_t padded_len = ROUND_UP (label_len, 4); + write_int (w, label_len); + write_string (w, label, padded_len); + free (label); + } - buf_write (w, &l, offsetof (struct label, label) + ext_len); + /* Write the missing values, if any, range first. */ + if (width <= MAX_SHORT_STRING) + { + const struct missing_values *mv = var_get_missing_values (v); + if (mv_has_range (mv)) + { + double x, y; + mv_get_range (mv, &x, &y); + write_float (w, x); + write_float (w, y); + } + for (i = 0; i < mv_n_values (mv); i++) + write_value (w, mv_get_value (mv, i), width); } - if (nm) - buf_write (w, m, sizeof *m * abs (nm)); + write_variable_continuation_records (w, seg0_width); - if (var_is_alpha (v) && var_get_width (v) > (int) sizeof (flt64)) + /* Write additional segments for very long string variables. */ + for (i = 1; i < segment_cnt; i++) { - int i; - int pad_count; - - sv.type = -1; - sv.has_var_label = 0; - sv.n_missing_values = 0; - memset (&sv.print, 0, sizeof sv.print); - memset (&sv.write, 0, sizeof sv.write); - memset (&sv.name, 0, sizeof sv.name); - - pad_count = DIV_RND_UP (MIN(var_get_width (v), MIN_VERY_LONG_STRING - 1), - (int) sizeof (flt64)) - 1; - for (i = 0; i < pad_count; i++) - buf_write (w, &sv, sizeof sv); + int seg_width = sfm_segment_alloc_width (width, i); + struct fmt_spec fmt = fmt_for_output (FMT_A, MAX (seg_width, 1), 0); + + write_int (w, 2); /* Variable record. */ + write_int (w, seg_width); /* Width. */ + write_int (w, 0); /* No variable label. */ + write_int (w, 0); /* No missing values. */ + write_format (w, fmt, seg_width); /* Print format. */ + write_format (w, fmt, seg_width); /* Write format. */ + write_utf8_string (w, encoding, var_get_short_name (v, i), 8); + + write_variable_continuation_records (w, seg_width); } } -/* Writes the value labels for variable V having system file - variable index IDX to system file W. */ +/* Writes the value labels to system file W. + + Value labels for long string variables are written separately, + by write_long_string_value_labels. */ static void -write_value_labels (struct sfm_writer *w, struct variable *v, int idx) +write_value_labels (struct sfm_writer *w, const struct dictionary *d) { - struct value_label_rec + struct label_set { - int32_t rec_type ; - int32_t n_labels ; - flt64 labels[1] ; - } ATTRIBUTE((packed)); + struct hmap_node hmap_node; + const struct val_labs *val_labs; + int *indexes; + size_t n_indexes, allocated_indexes; + }; + + size_t n_sets, allocated_sets; + struct label_set **sets; + struct hmap same_sets; + size_t i; + int idx; - struct var_idx_rec + n_sets = allocated_sets = 0; + sets = NULL; + hmap_init (&same_sets); + + idx = 0; + for (i = 0; i < dict_get_var_cnt (d); i++) { - int32_t rec_type ; - int32_t n_vars ; - int32_t vars[1] ; - } ATTRIBUTE((packed)); - - const struct val_labs *val_labs; - struct val_labs_iterator *i; - struct value_label_rec *vlr; - struct var_idx_rec vir; - struct val_lab *vl; - size_t vlr_size; - flt64 *loc; - - val_labs = var_get_value_labels (v); - if (val_labs == NULL) - return; + struct variable *v = dict_get_var (d, i); + + if (var_has_value_labels (v) && var_get_width (v) <= 8) + { + const struct val_labs *val_labs = var_get_value_labels (v); + unsigned int hash = val_labs_hash (val_labs, 0); + struct label_set *set; - /* Pass 1: Count bytes. */ - vlr_size = (sizeof (struct value_label_rec) - + sizeof (flt64) * (val_labs_count (val_labs) - 1)); - for (vl = val_labs_first (val_labs, &i); vl != NULL; - vl = val_labs_next (val_labs, &i)) - vlr_size += ROUND_UP (strlen (vl->label) + 1, sizeof (flt64)); - - /* Pass 2: Copy bytes. */ - vlr = xmalloc (vlr_size); - vlr->rec_type = 3; - vlr->n_labels = val_labs_count (val_labs); - loc = vlr->labels; - for (vl = val_labs_first_sorted (val_labs, &i); vl != NULL; - vl = val_labs_next (val_labs, &i)) + HMAP_FOR_EACH_WITH_HASH (set, struct label_set, hmap_node, + hash, &same_sets) + { + if (val_labs_equal (set->val_labs, val_labs)) + { + if (set->n_indexes >= set->allocated_indexes) + set->indexes = x2nrealloc (set->indexes, + &set->allocated_indexes, + sizeof *set->indexes); + set->indexes[set->n_indexes++] = idx; + goto next_var; + } + } + + set = xmalloc (sizeof *set); + set->val_labs = val_labs; + set->indexes = xmalloc (sizeof *set->indexes); + set->indexes[0] = idx; + set->n_indexes = 1; + set->allocated_indexes = 1; + hmap_insert (&same_sets, &set->hmap_node, hash); + + if (n_sets >= allocated_sets) + sets = x2nrealloc (sets, &allocated_sets, sizeof *sets); + sets[n_sets++] = set; + } + + next_var: + idx += sfm_width_to_octs (var_get_width (v)); + } + + for (i = 0; i < n_sets; i++) + { + const struct label_set *set = sets[i]; + const struct val_labs *val_labs = set->val_labs; + size_t n_labels = val_labs_count (val_labs); + int width = val_labs_get_width (val_labs); + const struct val_lab **labels; + size_t j; + + /* Value label record. */ + write_int (w, 3); /* Record type. */ + write_int (w, n_labels); + labels = val_labs_sorted (val_labs); + for (j = 0; j < n_labels; j++) + { + const struct val_lab *vl = labels[j]; + char *label = recode_string (dict_get_encoding (d), UTF8, + val_lab_get_escaped_label (vl), -1); + uint8_t len = MIN (strlen (label), 255); + + write_value (w, val_lab_get_value (vl), width); + write_bytes (w, &len, 1); + write_bytes (w, label, len); + write_zeros (w, REM_RND_UP (len + 1, 8)); + free (label); + } + free (labels); + + /* Value label variable record. */ + write_int (w, 4); /* Record type. */ + write_int (w, set->n_indexes); + for (j = 0; j < set->n_indexes; j++) + write_int (w, set->indexes[j] + 1); + } + + for (i = 0; i < n_sets; i++) { - size_t len = strlen (vl->label); - - *loc++ = vl->value.f; - *(unsigned char *) loc = len; - memcpy (&((char *) loc)[1], vl->label, len); - memset (&((char *) loc)[1 + len], ' ', - REM_RND_UP (len + 1, sizeof (flt64))); - loc += DIV_RND_UP (len + 1, sizeof (flt64)); + struct label_set *set = sets[i]; + + free (set->indexes); + free (set); } - - buf_write (w, vlr, vlr_size); - free (vlr); - - vir.rec_type = 4; - vir.n_vars = 1; - vir.vars[0] = idx + 1; - buf_write (w, &vir, sizeof vir); + free (sets); + hmap_destroy (&same_sets); } /* Writes record type 6, document record. */ static void write_documents (struct sfm_writer *w, const struct dictionary *d) { - struct - { - int32_t rec_type ; /* Always 6. */ - int32_t n_lines ; /* Number of lines of documents. */ - } ATTRIBUTE((packed)) rec_6; - - const char * documents = dict_get_documents (d); - size_t doc_bytes = strlen (documents); + const struct string_array *docs = dict_get_documents (d); + const char *enc = dict_get_encoding (d); + size_t i; - assert (doc_bytes % 80 == 0); + write_int (w, 6); /* Record type. */ + write_int (w, docs->n); + for (i = 0; i < docs->n; i++) + { + char *s = recode_string (enc, "UTF-8", docs->strings[i], -1); + size_t s_len = strlen (s); + size_t write_len = MIN (s_len, DOC_LINE_LENGTH); - rec_6.rec_type = 6; - rec_6.n_lines = doc_bytes / 80; - buf_write (w, &rec_6, sizeof rec_6); - buf_write (w, documents, 80 * rec_6.n_lines); + write_bytes (w, s, write_len); + write_spaces (w, DOC_LINE_LENGTH - write_len); + free (s); + } } -/* Write the alignment, width and scale values */ static void -write_variable_display_parameters (struct sfm_writer *w, - const struct dictionary *dict) +put_attrset (struct string *string, const struct attrset *attrs) { - int i; + const struct attribute *attr; + struct attrset_iterator i; - struct - { - int32_t rec_type ; - int32_t subtype ; - int32_t elem_size ; - int32_t n_elem ; - } ATTRIBUTE((packed)) vdp_hdr; + for (attr = attrset_first (attrs, &i); attr != NULL; + attr = attrset_next (attrs, &i)) + { + size_t n_values = attribute_get_n_values (attr); + size_t j; + + ds_put_cstr (string, attribute_get_name (attr)); + ds_put_byte (string, '('); + for (j = 0; j < n_values; j++) + ds_put_format (string, "'%s'\n", attribute_get_value (attr, j)); + ds_put_byte (string, ')'); + } +} - vdp_hdr.rec_type = 7; - vdp_hdr.subtype = 11; - vdp_hdr.elem_size = 4; - vdp_hdr.n_elem = w->var_cnt_vls * 3; +static void +write_data_file_attributes (struct sfm_writer *w, + const struct dictionary *d) +{ + struct string s = DS_EMPTY_INITIALIZER; + put_attrset (&s, dict_get_attributes (d)); + write_utf8_record (w, dict_get_encoding (d), &s, 17); + ds_destroy (&s); +} - buf_write (w, &vdp_hdr, sizeof vdp_hdr); +static void +add_role_attribute (enum var_role role, struct attrset *attrs) +{ + struct attribute *attr; + const char *s; - for ( i = 0 ; i < w->var_cnt ; ++i ) + switch (role) { - struct variable *v; - struct - { - int32_t measure ; - int32_t width ; - int32_t align ; - } ATTRIBUTE((packed)) params; - - v = dict_get_var(dict, i); - - params.measure = (var_get_measure (v) == MEASURE_NOMINAL ? 1 - : var_get_measure (v) == MEASURE_ORDINAL ? 2 - : 3); - params.width = var_get_display_width (v); - params.align = (var_get_alignment (v) == ALIGN_LEFT ? 0 - : var_get_alignment (v) == ALIGN_RIGHT ? 1 - : 2); - - buf_write (w, ¶ms, sizeof(params)); - - if (var_is_long_string (v)) - { - int wcount = var_get_width (v) - EFFECTIVE_LONG_STRING_LENGTH ; - - while (wcount > 0) - { - params.width = wcount >= MIN_VERY_LONG_STRING ? 32 : wcount; - - buf_write (w, ¶ms, sizeof(params)); - - wcount -= EFFECTIVE_LONG_STRING_LENGTH ; - } - } + case ROLE_INPUT: + default: + s = "0"; + break; + + case ROLE_TARGET: + s = "1"; + break; + + case ROLE_BOTH: + s = "2"; + break; + + case ROLE_NONE: + s = "3"; + break; + + case ROLE_PARTITION: + s = "4"; + break; + + case ROLE_SPLIT: + s = "5"; + break; } + attrset_delete (attrs, "$@Role"); + + attr = attribute_create ("$@Role"); + attribute_add_value (attr, s); + attrset_add (attrs, attr); } -/* Writes the table of lengths for Very Long String Variables */ -static void -write_vls_length_table (struct sfm_writer *w, - const struct dictionary *dict) +static void +write_variable_attributes (struct sfm_writer *w, const struct dictionary *d) { - int i; - struct - { - int32_t rec_type ; - int32_t subtype ; - int32_t elem_size ; - int32_t n_elem ; - } ATTRIBUTE((packed)) vls_hdr; + struct string s = DS_EMPTY_INITIALIZER; + size_t n_vars = dict_get_var_cnt (d); + size_t n_attrsets = 0; + size_t i; - struct string vls_length_map; + for (i = 0; i < n_vars; i++) + { + struct variable *v = dict_get_var (d, i); + struct attrset attrs; - ds_init_empty (&vls_length_map); + attrset_clone (&attrs, var_get_attributes (v)); - vls_hdr.rec_type = 7; - vls_hdr.subtype = 14; - vls_hdr.elem_size = 1; + add_role_attribute (var_get_role (v), &attrs); + if (n_attrsets++) + ds_put_byte (&s, '/'); + ds_put_format (&s, "%s:", var_get_name (v)); + put_attrset (&s, &attrs); + attrset_destroy (&attrs); + } + if (n_attrsets) + write_utf8_record (w, dict_get_encoding (d), &s, 18); + ds_destroy (&s); +} +/* Write multiple response sets. If PRE_V14 is true, writes sets supported by + SPSS before release 14, otherwise writes sets supported only by later + versions. */ +static void +write_mrsets (struct sfm_writer *w, const struct dictionary *dict, + bool pre_v14) +{ + const char *encoding = dict_get_encoding (dict); + struct string s = DS_EMPTY_INITIALIZER; + size_t n_mrsets; + size_t i; - for (i = 0; i < dict_get_var_cnt (dict); ++i) + if (is_encoding_ebcdic_compatible (encoding)) { - const struct variable *v = dict_get_var (dict, i); - - if ( var_get_width (v) < MIN_VERY_LONG_STRING ) - continue; - - ds_put_format (&vls_length_map, "%s=%05d", - var_get_short_name (v), var_get_width (v)); - ds_put_char (&vls_length_map, '\0'); - ds_put_char (&vls_length_map, '\t'); + /* FIXME. */ + return; } - vls_hdr.n_elem = ds_length (&vls_length_map); + n_mrsets = dict_get_n_mrsets (dict); + if (n_mrsets == 0) + return; - if ( vls_hdr.n_elem > 0 ) + for (i = 0; i < n_mrsets; i++) { - buf_write (w, &vls_hdr, sizeof vls_hdr); - buf_write (w, ds_data (&vls_length_map), ds_length (&vls_length_map)); + const struct mrset *mrset = dict_get_mrset (dict, i); + char *name; + size_t j; + + if ((mrset->type != MRSET_MD || mrset->cat_source != MRSET_COUNTEDVALUES) + != pre_v14) + continue; + + name = recode_string (encoding, "UTF-8", mrset->name, -1); + ds_put_format (&s, "%s=", name); + free (name); + + if (mrset->type == MRSET_MD) + { + char *counted; + + if (mrset->cat_source == MRSET_COUNTEDVALUES) + ds_put_format (&s, "E %d ", mrset->label_from_var_label ? 11 : 1); + else + ds_put_byte (&s, 'D'); + + if (mrset->width == 0) + counted = xasprintf ("%.0f", mrset->counted.f); + else + counted = xmemdup0 (value_str (&mrset->counted, mrset->width), + mrset->width); + ds_put_format (&s, "%zu %s", strlen (counted), counted); + free (counted); + } + else + ds_put_byte (&s, 'C'); + ds_put_byte (&s, ' '); + + if (mrset->label && !mrset->label_from_var_label) + { + char *label = recode_string (encoding, "UTF-8", mrset->label, -1); + ds_put_format (&s, "%zu %s", strlen (label), label); + free (label); + } + else + ds_put_cstr (&s, "0 "); + + for (j = 0; j < mrset->n_vars; j++) + { + const char *short_name_utf8 = var_get_short_name (mrset->vars[j], 0); + char *lower_name_utf8 = utf8_to_lower (short_name_utf8); + char *short_name = recode_string (encoding, "UTF-8", + lower_name_utf8, -1); + ds_put_format (&s, " %s", short_name); + free (short_name); + free (lower_name_utf8); + } + ds_put_byte (&s, '\n'); } - ds_destroy (&vls_length_map); + if (!ds_is_empty (&s)) + write_string_record (w, ds_ss (&s), pre_v14 ? 7 : 19); + ds_destroy (&s); } -/* Writes the long variable name table */ +/* Write the alignment, width and scale values. */ static void -write_longvar_table (struct sfm_writer *w, const struct dictionary *dict) +write_variable_display_parameters (struct sfm_writer *w, + const struct dictionary *dict) { - struct - { - int32_t rec_type ; - int32_t subtype ; - int32_t elem_size ; - int32_t n_elem ; - } ATTRIBUTE((packed)) lv_hdr; + int i; - struct string long_name_map; - size_t i; + write_int (w, 7); /* Record type. */ + write_int (w, 11); /* Record subtype. */ + write_int (w, 4); /* Data item (int32) size. */ + write_int (w, w->segment_cnt * 3); /* Number of data items. */ - ds_init_empty (&long_name_map); - for (i = 0; i < dict_get_var_cnt (dict); i++) + for (i = 0; i < dict_get_var_cnt (dict); ++i) { struct variable *v = dict_get_var (dict, i); - - if (i) - ds_put_char (&long_name_map, '\t'); - ds_put_format (&long_name_map, "%s=%s", - var_get_short_name (v), var_get_name (v)); - } - - lv_hdr.rec_type = 7; - lv_hdr.subtype = 13; - lv_hdr.elem_size = 1; - lv_hdr.n_elem = ds_length (&long_name_map); - - buf_write (w, &lv_hdr, sizeof lv_hdr); - buf_write (w, ds_data (&long_name_map), ds_length (&long_name_map)); + int width = var_get_width (v); + int segment_cnt = sfm_width_to_segments (width); + int measure = (var_get_measure (v) == MEASURE_NOMINAL ? 1 + : var_get_measure (v) == MEASURE_ORDINAL ? 2 + : 3); + int alignment = (var_get_alignment (v) == ALIGN_LEFT ? 0 + : var_get_alignment (v) == ALIGN_RIGHT ? 1 + : 2); + int i; - ds_destroy (&long_name_map); + for (i = 0; i < segment_cnt; i++) + { + int width_left = width - sfm_segment_effective_offset (width, i); + write_int (w, measure); + write_int (w, (i == 0 ? var_get_display_width (v) + : var_default_display_width (width_left))); + write_int (w, alignment); + } + } } -/* Writes record type 7, subtypes 3 and 4. */ +/* Writes the table of lengths for very long string variables. */ static void -write_rec_7_34 (struct sfm_writer *w) +write_vls_length_table (struct sfm_writer *w, + const struct dictionary *dict) { - struct + struct string map; + int i; + + ds_init_empty (&map); + for (i = 0; i < dict_get_var_cnt (dict); ++i) { - int32_t rec_type_3 ; - int32_t subtype_3 ; - int32_t data_type_3 ; - int32_t n_elem_3 ; - int32_t elem_3[8] ; - int32_t rec_type_4 ; - int32_t subtype_4 ; - int32_t data_type_4 ; - int32_t n_elem_4 ; - flt64 elem_4[3] ; - } ATTRIBUTE((packed)) rec_7; - - /* Components of the version number, from major to minor. */ - int version_component[3]; - - /* Used to step through the version string. */ - char *p; - - /* Parses the version string, which is assumed to be of the form - #.#x, where each # is a string of digits, and x is a single - letter. */ - version_component[0] = strtol (bare_version, &p, 10); - if (*p == '.') - p++; - version_component[1] = strtol (bare_version, &p, 10); - version_component[2] = (isalpha ((unsigned char) *p) - ? tolower ((unsigned char) *p) - 'a' : 0); - - rec_7.rec_type_3 = 7; - rec_7.subtype_3 = 3; - rec_7.data_type_3 = sizeof (int32_t); - rec_7.n_elem_3 = 8; - rec_7.elem_3[0] = version_component[0]; - rec_7.elem_3[1] = version_component[1]; - rec_7.elem_3[2] = version_component[2]; - rec_7.elem_3[3] = -1; - - /* PORTME: 1=IEEE754, 2=IBM 370, 3=DEC VAX E. */ -#ifdef FPREP_IEEE754 - rec_7.elem_3[4] = 1; -#endif - - rec_7.elem_3[5] = 1; - - /* PORTME: 1=big-endian, 2=little-endian. */ -#if WORDS_BIGENDIAN - rec_7.elem_3[6] = 1; -#else - rec_7.elem_3[6] = 2; -#endif - - /* PORTME: 1=EBCDIC, 2=7-bit ASCII, 3=8-bit ASCII, 4=DEC Kanji. */ - rec_7.elem_3[7] = 2; - - rec_7.rec_type_4 = 7; - rec_7.subtype_4 = 4; - rec_7.data_type_4 = sizeof (flt64); - rec_7.n_elem_4 = 3; - rec_7.elem_4[0] = -FLT64_MAX; - rec_7.elem_4[1] = FLT64_MAX; - rec_7.elem_4[2] = second_lowest_flt64; - - buf_write (w, &rec_7, sizeof rec_7); + const struct variable *v = dict_get_var (dict, i); + if (sfm_width_to_segments (var_get_width (v)) > 1) + ds_put_format (&map, "%s=%05d%c\t", + var_get_short_name (v, 0), var_get_width (v), 0); + } + if (!ds_is_empty (&map)) + write_utf8_record (w, dict_get_encoding (dict), &map, 14); + ds_destroy (&map); } -/* Write NBYTES starting at BUF to the system file represented by - H. */ static void -buf_write (struct sfm_writer *w, const void *buf, size_t nbytes) +write_long_string_value_labels (struct sfm_writer *w, + const struct dictionary *dict) { - assert (buf != NULL); - fwrite (buf, nbytes, 1, w->file); -} + const char *encoding = dict_get_encoding (dict); + size_t n_vars = dict_get_var_cnt (dict); + size_t size, i; + off_t start UNUSED; + + /* Figure out the size in advance. */ + size = 0; + for (i = 0; i < n_vars; i++) + { + struct variable *var = dict_get_var (dict, i); + const struct val_labs *val_labs = var_get_value_labels (var); + int width = var_get_width (var); + const struct val_lab *val_lab; + + if (val_labs_count (val_labs) == 0 || width < 9) + continue; + + size += 12; + size += recode_string_len (encoding, "UTF-8", var_get_name (var), -1); + for (val_lab = val_labs_first (val_labs); val_lab != NULL; + val_lab = val_labs_next (val_labs, val_lab)) + { + size += 8 + width; + size += recode_string_len (encoding, "UTF-8", + val_lab_get_escaped_label (val_lab), -1); + } + } + if (size == 0) + return; -/* Copies string DEST to SRC with the proviso that DEST does not reach - byte END; no null terminator is copied. Returns a pointer to the - byte after the last byte copied. */ -static char * -append_string_max (char *dest, const char *src, const char *end) -{ - int nbytes = MIN (end - dest, (int) strlen (src)); - memcpy (dest, src, nbytes); - return dest + nbytes; -} + write_int (w, 7); /* Record type. */ + write_int (w, 21); /* Record subtype */ + write_int (w, 1); /* Data item (byte) size. */ + write_int (w, size); /* Number of data items. */ -/* Makes certain that the compression buffer of H has room for another - element. If there's not room, pads out the current instruction - octet with zero and dumps out the buffer. */ -static void -ensure_buf_space (struct sfm_writer *w) -{ - if (w->ptr >= w->end) + start = ftello (w->file); + for (i = 0; i < n_vars; i++) { - memset (w->x, 0, w->y - w->x); - w->x = w->y; - w->ptr = w->buf; - buf_write (w, w->buf, sizeof *w->buf * 128); + struct variable *var = dict_get_var (dict, i); + const struct val_labs *val_labs = var_get_value_labels (var); + int width = var_get_width (var); + const struct val_lab *val_lab; + char *var_name; + + if (val_labs_count (val_labs) == 0 || width < 9) + continue; + + var_name = recode_string (encoding, "UTF-8", var_get_name (var), -1); + write_int (w, strlen (var_name)); + write_bytes (w, var_name, strlen (var_name)); + free (var_name); + + write_int (w, width); + write_int (w, val_labs_count (val_labs)); + for (val_lab = val_labs_first (val_labs); val_lab != NULL; + val_lab = val_labs_next (val_labs, val_lab)) + { + char *label; + size_t len; + + write_int (w, width); + write_bytes (w, value_str (val_lab_get_value (val_lab), width), + width); + + label = recode_string (var_get_encoding (var), "UTF-8", + val_lab_get_escaped_label (val_lab), -1); + len = strlen (label); + write_int (w, len); + write_bytes (w, label, len); + free (label); + } } + assert (ftello (w->file) == start + size); } -static void write_compressed_data (struct sfm_writer *w, const flt64 *elem); - -/* Writes case C to system file W. - Returns 1 if successful, 0 if an I/O error occurred. */ -bool -sfm_write_case (struct sfm_writer *w, const struct ccase *c) +static void +write_long_string_missing_values (struct sfm_writer *w, + const struct dictionary *dict) { - if (ferror (w->file)) - return 0; - - w->case_cnt++; - - if (!w->needs_translation && !w->compress - && sizeof (flt64) == sizeof (union value) && ! w->has_vls ) + const char *encoding = dict_get_encoding (dict); + size_t n_vars = dict_get_var_cnt (dict); + size_t size, i; + off_t start UNUSED; + + /* Figure out the size in advance. */ + size = 0; + for (i = 0; i < n_vars; i++) { - /* Fast path: external and internal representations are the - same and the dictionary is properly ordered. Write - directly to file. */ - buf_write (w, case_data_all (c), sizeof (union value) * w->flt64_cnt); + struct variable *var = dict_get_var (dict, i); + const struct missing_values *mv = var_get_missing_values (var); + int width = var_get_width (var); + + if (mv_is_empty (mv) || width < 9) + continue; + + size += 4; + size += recode_string_len (encoding, "UTF-8", var_get_name (var), -1); + size += 1; + size += mv_n_values (mv) * (4 + 8); } - else + if (size == 0) + return; + + write_int (w, 7); /* Record type. */ + write_int (w, 22); /* Record subtype */ + write_int (w, 1); /* Data item (byte) size. */ + write_int (w, size); /* Number of data items. */ + + start = ftello (w->file); + for (i = 0; i < n_vars; i++) { - /* Slow path: internal and external representations differ. - Write into a bounce buffer, then write to W. */ - flt64 *bounce; - flt64 *bounce_cur; - flt64 *bounce_end; - size_t bounce_size; - size_t i; - - bounce_size = sizeof *bounce * w->flt64_cnt; - bounce = bounce_cur = local_alloc (bounce_size); - bounce_end = bounce + bounce_size; - - for (i = 0; i < w->var_cnt; i++) + struct variable *var = dict_get_var (dict, i); + const struct missing_values *mv = var_get_missing_values (var); + int width = var_get_width (var); + uint8_t n_missing_values; + char *var_name; + int j; + + if (mv_is_empty (mv) || width < 9) + continue; + + var_name = recode_string (encoding, "UTF-8", var_get_name (var), -1); + write_int (w, strlen (var_name)); + write_bytes (w, var_name, strlen (var_name)); + free (var_name); + + n_missing_values = mv_n_values (mv); + write_bytes (w, &n_missing_values, 1); + + for (j = 0; j < n_missing_values; j++) { - struct sfm_var *v = &w->vars[i]; - - memset(bounce_cur, ' ', v->flt64_cnt * sizeof (flt64)); - - if (v->width == 0) - { - *bounce_cur = case_num_idx (c, v->fv); - bounce_cur += v->flt64_cnt; - } - else - { int ofs = 0; - while (ofs < v->width) - { - int chunk = MIN (MIN_VERY_LONG_STRING - 1, v->width - ofs); - int nv = DIV_RND_UP (chunk, sizeof (flt64)); - buf_copy_rpad ((char *) bounce_cur, nv * sizeof (flt64), - case_data_idx (c, v->fv)->s + ofs, chunk); - bounce_cur += nv; - ofs += chunk; - } - } + const union value *value = mv_get_value (mv, j); + write_int (w, 8); + write_bytes (w, value_str (value, width), 8); } + } + assert (ftello (w->file) == start + size); +} - if (!w->compress) - buf_write (w, bounce, bounce_size); - else - write_compressed_data (w, bounce); +static void +write_encoding_record (struct sfm_writer *w, + const struct dictionary *d) +{ + /* IANA says "...character set names may be up to 40 characters taken + from the printable characters of US-ASCII," so character set names + don't need to be recoded to be in UTF-8. + + We convert encoding names to uppercase because SPSS writes encoding + names in uppercase. */ + char *encoding = xstrdup (dict_get_encoding (d)); + str_uppercase (encoding); + write_string_record (w, ss_cstr (encoding), 20); + free (encoding); +} - local_free (bounce); +/* Writes the long variable name table. */ +static void +write_longvar_table (struct sfm_writer *w, const struct dictionary *dict) +{ + struct string map; + size_t i; + + ds_init_empty (&map); + for (i = 0; i < dict_get_var_cnt (dict); i++) + { + struct variable *v = dict_get_var (dict, i); + if (i) + ds_put_byte (&map, '\t'); + ds_put_format (&map, "%s=%s", + var_get_short_name (v, 0), var_get_name (v)); } - - return !sfm_write_error (w); + write_utf8_record (w, dict_get_encoding (dict), &map, 13); + ds_destroy (&map); } +/* Write integer information record. */ static void -put_instruction (struct sfm_writer *w, unsigned char instruction) +write_integer_info_record (struct sfm_writer *w, + const struct dictionary *d) { - if (w->x >= w->y) + const char *dict_encoding = dict_get_encoding (d); + int version_component[3]; + int float_format; + int codepage; + + /* Parse the version string. */ + memset (version_component, 0, sizeof version_component); + sscanf (bare_version, "%d.%d.%d", + &version_component[0], &version_component[1], &version_component[2]); + + /* Figure out the floating-point format. */ + if (FLOAT_NATIVE_64_BIT == FLOAT_IEEE_DOUBLE_LE + || FLOAT_NATIVE_64_BIT == FLOAT_IEEE_DOUBLE_BE) + float_format = 1; + else if (FLOAT_NATIVE_64_BIT == FLOAT_Z_LONG) + float_format = 2; + else if (FLOAT_NATIVE_64_BIT == FLOAT_VAX_D) + float_format = 3; + else + abort (); + + /* Choose codepage. */ + codepage = sys_get_codepage_from_encoding (dict_encoding); + if (codepage == 0) { - ensure_buf_space (w); - w->x = (unsigned char *) w->ptr++; - w->y = (unsigned char *) w->ptr; + /* The codepage is unknown. Choose a default. + + For an EBCDIC-compatible encoding, use the value for EBCDIC. + + For an ASCII-compatible encoding, default to "7-bit ASCII", because + many files use this codepage number regardless of their actual + encoding. + */ + if (is_encoding_ascii_compatible (dict_encoding)) + codepage = 2; + else if (is_encoding_ebcdic_compatible (dict_encoding)) + codepage = 1; } - *w->x++ = instruction; + + /* Write record. */ + write_int (w, 7); /* Record type. */ + write_int (w, 3); /* Record subtype. */ + write_int (w, 4); /* Data item (int32) size. */ + write_int (w, 8); /* Number of data items. */ + write_int (w, version_component[0]); + write_int (w, version_component[1]); + write_int (w, version_component[2]); + write_int (w, -1); /* Machine code. */ + write_int (w, float_format); + write_int (w, 1); /* Compression code. */ + write_int (w, INTEGER_NATIVE == INTEGER_MSB_FIRST ? 1 : 2); + write_int (w, codepage); } +/* Write floating-point information record. */ static void -put_element (struct sfm_writer *w, const flt64 *elem) +write_float_info_record (struct sfm_writer *w) { - ensure_buf_space (w); - memcpy (w->ptr++, elem, sizeof *elem); + write_int (w, 7); /* Record type. */ + write_int (w, 4); /* Record subtype. */ + write_int (w, 8); /* Data item (flt64) size. */ + write_int (w, 3); /* Number of data items. */ + write_float (w, SYSMIS); /* System-missing value. */ + write_float (w, HIGHEST); /* Value used for HIGHEST in missing values. */ + write_float (w, LOWEST); /* Value used for LOWEST in missing values. */ } - + +/* Writes case C to system file W. */ static void -write_compressed_data (struct sfm_writer *w, const flt64 *elem) +sys_file_casewriter_write (struct casewriter *writer, void *w_, + struct ccase *c) { - size_t i; + struct sfm_writer *w = w_; - for (i = 0; i < w->var_cnt; i++) + if (ferror (w->file)) { - struct sfm_var *v = &w->vars[i]; - - if (v->width == 0) - { - if (*elem == -FLT64_MAX) - put_instruction (w, 255); - else if (*elem >= 1 - COMPRESSION_BIAS - && *elem <= 251 - COMPRESSION_BIAS - && *elem == (int) *elem) - put_instruction (w, (int) *elem + COMPRESSION_BIAS); - else - { - put_instruction (w, 253); - put_element (w, elem); - } - elem++; - } - else - { - size_t j; - - for (j = 0; j < v->flt64_cnt; j++, elem++) - { - if (!memcmp (elem, " ", sizeof (flt64))) - put_instruction (w, 254); - else - { - put_instruction (w, 253); - put_element (w, elem); - } - } - } + casewriter_force_error (writer); + case_unref (c); + return; } + + w->case_cnt++; + + if (w->compression == ANY_COMP_NONE) + write_case_uncompressed (w, c); + else + write_case_compressed (w, c); + + case_unref (c); +} + +/* Destroys system file writer W. */ +static void +sys_file_casewriter_destroy (struct casewriter *writer, void *w_) +{ + struct sfm_writer *w = w_; + if (!close_writer (w)) + casewriter_force_error (writer); } /* Returns true if an I/O error has occurred on WRITER, false otherwise. */ -bool -sfm_write_error (const struct sfm_writer *writer) +static bool +write_error (const struct sfm_writer *writer) { return ferror (writer->file); } /* Closes a system file after we're done with it. Returns true if successful, false if an I/O error occurred. */ -bool -sfm_close_writer (struct sfm_writer *w) +static bool +close_writer (struct sfm_writer *w) { bool ok; - + if (w == NULL) return true; ok = true; - if (w->file != NULL) + if (w->file != NULL) { /* Flush buffer. */ - if (w->buf != NULL && w->ptr > w->buf) + flush_compressed (w); + if (w->compression == ANY_COMP_ZLIB) { - memset (w->x, 0, w->y - w->x); - buf_write (w, w->buf, (w->ptr - w->buf) * sizeof *w->buf); + finish_zstream (w); + write_ztrailer (w); } fflush (w->file); - ok = !sfm_write_error (w); + ok = !write_error (w); /* Seek back to the beginning and update the number of cases. This is just a courtesy to later readers, so there's no need to check return values or report errors. */ - if (ok && !fseek (w->file, offsetof (struct sysfile_header, case_cnt), - SEEK_SET)) + if (ok && w->case_cnt <= INT32_MAX && !fseeko (w->file, 80, SEEK_SET)) { - int32_t case_cnt = w->case_cnt; - fwrite (&case_cnt, sizeof case_cnt, 1, w->file); + write_int (w, w->case_cnt); clearerr (w->file); } @@ -1100,15 +1275,404 @@ sfm_close_writer (struct sfm_writer *w) ok = false; if (!ok) - msg (ME, _("An I/O error occurred writing system file \"%s\"."), + msg (ME, _("An I/O error occurred writing system file `%s'."), fh_get_file_name (w->fh)); + + if (ok ? !replace_file_commit (w->rf) : !replace_file_abort (w->rf)) + ok = false; } - fh_close (w->fh, "system file", "we"); - - free (w->buf); - free (w->vars); + free (w->blocks); + + fh_unlock (w->lock); + fh_unref (w->fh); + + free (w->sfm_vars); free (w); return ok; } + +/* System file writer casewriter class. */ +static const struct casewriter_class sys_file_casewriter_class = + { + sys_file_casewriter_write, + sys_file_casewriter_destroy, + NULL, + }; + +/* Writes case C to system file W, without compressing it. */ +static void +write_case_uncompressed (struct sfm_writer *w, const struct ccase *c) +{ + size_t i; + + for (i = 0; i < w->sfm_var_cnt; i++) + { + struct sfm_var *v = &w->sfm_vars[i]; + + if (v->var_width == 0) + write_float (w, case_num_idx (c, v->case_index)); + else + { + write_bytes (w, case_str_idx (c, v->case_index) + v->offset, + v->segment_width); + write_spaces (w, v->padding); + } + } +} + +/* Writes case C to system file W, with compression. */ +static void +write_case_compressed (struct sfm_writer *w, const struct ccase *c) +{ + size_t i; + + for (i = 0; i < w->sfm_var_cnt; i++) + { + struct sfm_var *v = &w->sfm_vars[i]; + + if (v->var_width == 0) + { + double d = case_num_idx (c, v->case_index); + if (d == SYSMIS) + put_cmp_opcode (w, 255); + else if (d >= 1 - COMPRESSION_BIAS + && d <= 251 - COMPRESSION_BIAS + && d == (int) d) + put_cmp_opcode (w, (int) d + COMPRESSION_BIAS); + else + put_cmp_number (w, d); + } + else + { + int offset = v->offset; + int width, padding; + + /* This code properly deals with a width that is not a + multiple of 8, by ensuring that the final partial + oct (8 byte unit) is treated as padded with spaces + on the right. */ + for (width = v->segment_width; width > 0; width -= 8, offset += 8) + { + const void *data = case_str_idx (c, v->case_index) + offset; + int chunk_size = MIN (width, 8); + if (!memcmp (data, " ", chunk_size)) + put_cmp_opcode (w, 254); + else + put_cmp_string (w, data, chunk_size); + } + + /* This code deals properly with padding that is not a + multiple of 8 bytes, by discarding the remainder, + which was already effectively padded with spaces in + the previous loop. (Note that v->width + v->padding + is always a multiple of 8.) */ + for (padding = v->padding / 8; padding > 0; padding--) + put_cmp_opcode (w, 254); + } + } +} + +static bool +start_zstream (struct sfm_writer *w) +{ + int error; + + error = deflateInit (&w->zstream, 1); + if (error != Z_OK) + { + msg (ME, _("Failed to initialize ZLIB for compression (%s)."), + w->zstream.msg); + return false; + } + return true; +} + +static void +finish_zstream (struct sfm_writer *w) +{ + struct zblock *block; + int error; + + assert (w->zstream.total_in <= ZBLOCK_SIZE); + + w->zstream.next_in = NULL; + w->zstream.avail_in = 0; + do + { + uint8_t buf[4096]; + + w->zstream.next_out = buf; + w->zstream.avail_out = sizeof buf; + error = deflate (&w->zstream, Z_FINISH); + write_bytes (w, buf, w->zstream.next_out - buf); + } + while (error == Z_OK); + + if (error != Z_STREAM_END) + msg (ME, _("Failed to complete ZLIB stream compression (%s)."), + w->zstream.msg); + + if (w->n_blocks >= w->allocated_blocks) + w->blocks = x2nrealloc (w->blocks, &w->allocated_blocks, + sizeof *w->blocks); + block = &w->blocks[w->n_blocks++]; + block->uncompressed_size = w->zstream.total_in; + block->compressed_size = w->zstream.total_out; + deflateEnd (&w->zstream); +} + +static void +write_zlib (struct sfm_writer *w, const void *data_, unsigned int n) +{ + const uint8_t *data = data_; + + while (n > 0) + { + unsigned int chunk; + + if (w->zstream.total_in >= ZBLOCK_SIZE) + { + finish_zstream (w); + start_zstream (w); + } + + chunk = MIN (n, ZBLOCK_SIZE - w->zstream.total_in); + + w->zstream.next_in = CONST_CAST (uint8_t *, data); + w->zstream.avail_in = chunk; + do + { + uint8_t buf[4096]; + int error; + + w->zstream.next_out = buf; + w->zstream.avail_out = sizeof buf; + error = deflate (&w->zstream, Z_NO_FLUSH); + write_bytes (w, buf, w->zstream.next_out - buf); + if (error != Z_OK) + { + msg (ME, _("ZLIB stream compression failed (%s)."), + w->zstream.msg); + return; + } + } + while (w->zstream.avail_in > 0 || w->zstream.avail_out == 0); + data += chunk; + n -= chunk; + } +} + +static void +write_ztrailer (struct sfm_writer *w) +{ + long long int uncompressed_ofs; + long long int compressed_ofs; + const struct zblock *block; + + write_int64 (w, -COMPRESSION_BIAS); + write_int64 (w, 0); + write_int (w, ZBLOCK_SIZE); + write_int (w, w->n_blocks); + + uncompressed_ofs = w->zstart; + compressed_ofs = w->zstart + 24; + for (block = w->blocks; block < &w->blocks[w->n_blocks]; block++) + { + write_int64 (w, uncompressed_ofs); + write_int64 (w, compressed_ofs); + write_int (w, block->uncompressed_size); + write_int (w, block->compressed_size); + + uncompressed_ofs += block->uncompressed_size; + compressed_ofs += block->compressed_size; + } + + if (!fseeko (w->file, w->zstart + 8, SEEK_SET)) + { + write_int64 (w, compressed_ofs); + write_int64 (w, 24 + (w->n_blocks * 24)); + } + else + msg (ME, _("%s: Seek failed (%s)."), + fh_get_file_name (w->fh), strerror (errno)); +} + +/* Flushes buffered compressed opcodes and data to W. */ +static void +flush_compressed (struct sfm_writer *w) +{ + if (w->n_opcodes) + { + unsigned int n = 8 * (1 + w->n_elements); + if (w->compression == ANY_COMP_SIMPLE) + write_bytes (w, w->cbuf, n); + else + write_zlib (w, w->cbuf, n); + + w->n_opcodes = w->n_elements = 0; + memset (w->cbuf[0], 0, 8); + } +} + +/* Appends OPCODE to the buffered set of compression opcodes in + W. Flushes the compression buffer beforehand if necessary. */ +static void +put_cmp_opcode (struct sfm_writer *w, uint8_t opcode) +{ + if (w->n_opcodes >= 8) + flush_compressed (w); + + w->cbuf[0][w->n_opcodes++] = opcode; +} + +/* Appends NUMBER to the buffered compression data in W. */ +static void +put_cmp_number (struct sfm_writer *w, double number) +{ + put_cmp_opcode (w, 253); + convert_double_to_output_format (number, w->cbuf[++w->n_elements]); +} + +/* Appends SIZE bytes of DATA to the buffered compression data in + W, followed by enough spaces to pad the output data to exactly + 8 bytes (thus, SIZE must be no greater than 8). */ +static void +put_cmp_string (struct sfm_writer *w, const void *data, size_t size) +{ + assert (size <= 8); + + put_cmp_opcode (w, 253); + w->n_elements++; + memset (w->cbuf[w->n_elements], w->space, 8); + memcpy (w->cbuf[w->n_elements], data, size); +} + +/* Writes 32-bit integer X to the output file for writer W. */ +static void +write_int (struct sfm_writer *w, int32_t x) +{ + write_bytes (w, &x, sizeof x); +} + +/* Writes 64-bit integer X to the output file for writer W. */ +static void +write_int64 (struct sfm_writer *w, int64_t x) +{ + write_bytes (w, &x, sizeof x); +} + +/* Converts NATIVE to the 64-bit format used in output files in + OUTPUT. */ +static inline void +convert_double_to_output_format (double native, uint8_t output[8]) +{ + /* If "double" is not a 64-bit type, then convert it to a + 64-bit type. Otherwise just copy it. */ + if (FLOAT_NATIVE_DOUBLE != FLOAT_NATIVE_64_BIT) + float_convert (FLOAT_NATIVE_DOUBLE, &native, FLOAT_NATIVE_64_BIT, output); + else + memcpy (output, &native, sizeof native); +} + +/* Writes floating-point number X to the output file for writer + W. */ +static void +write_float (struct sfm_writer *w, double x) +{ + uint8_t output[8]; + convert_double_to_output_format (x, output); + write_bytes (w, output, sizeof output); +} + +/* Writes contents of VALUE with the given WIDTH to W, padding + with zeros to a multiple of 8 bytes. + To avoid a branch, and because we don't actually need to + support it, WIDTH must be no bigger than 8. */ +static void +write_value (struct sfm_writer *w, const union value *value, int width) +{ + assert (width <= 8); + if (width == 0) + write_float (w, value->f); + else + { + write_bytes (w, value_str (value, width), width); + write_zeros (w, 8 - width); + } +} + +/* Writes null-terminated STRING in a field of the given WIDTH to W. If STRING + is longer than WIDTH, it is truncated; if STRING is shorter than WIDTH, it + is padded on the right with spaces. */ +static void +write_string (struct sfm_writer *w, const char *string, size_t width) +{ + size_t data_bytes = MIN (strlen (string), width); + size_t pad_bytes = width - data_bytes; + write_bytes (w, string, data_bytes); + while (pad_bytes-- > 0) + putc (w->space, w->file); +} + +/* Recodes null-terminated UTF-8 encoded STRING into ENCODING, and writes the + recoded version in a field of the given WIDTH to W. The string is truncated + or padded on the right with spaces to exactly WIDTH bytes. */ +static void +write_utf8_string (struct sfm_writer *w, const char *encoding, + const char *string, size_t width) +{ + char *s = recode_string (encoding, "UTF-8", string, -1); + write_string (w, s, width); + free (s); +} + +/* Writes a record with type 7, subtype SUBTYPE that contains CONTENT recoded + from UTF-8 encoded into ENCODING. */ +static void +write_utf8_record (struct sfm_writer *w, const char *encoding, + const struct string *content, int subtype) +{ + struct substring s; + + s = recode_substring_pool (encoding, "UTF-8", ds_ss (content), NULL); + write_string_record (w, s, subtype); + ss_dealloc (&s); +} + +/* Writes a record with type 7, subtype SUBTYPE that contains the string + CONTENT. */ +static void +write_string_record (struct sfm_writer *w, + const struct substring content, int subtype) +{ + write_int (w, 7); + write_int (w, subtype); + write_int (w, 1); + write_int (w, ss_length (content)); + write_bytes (w, ss_data (content), ss_length (content)); +} + +/* Writes SIZE bytes of DATA to W's output file. */ +static void +write_bytes (struct sfm_writer *w, const void *data, size_t size) +{ + fwrite (data, 1, size, w->file); +} + +/* Writes N zeros to W's output file. */ +static void +write_zeros (struct sfm_writer *w, size_t n) +{ + while (n-- > 0) + putc (0, w->file); +} + +/* Writes N spaces to W's output file. */ +static void +write_spaces (struct sfm_writer *w, size_t n) +{ + while (n-- > 0) + putc (w->space, w->file); +}