X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fsys-file-writer.c;h=2639c3587aa19011b8d7671990cf2a3806d46ff5;hb=f335cbe3c9591fdc809c197983c8cc3e2d622f1f;hp=5ba525cbfed518edd7af6bcc1cb5ec7c443db429;hpb=77e551d23575da6b89f866612ab39c2b0497c9be;p=pspp-builds.git diff --git a/src/data/sys-file-writer.c b/src/data/sys-file-writer.c index 5ba525cb..2639c358 100644 --- a/src/data/sys-file-writer.c +++ b/src/data/sys-file-writer.c @@ -42,6 +42,7 @@ #include "value-labels.h" #include "variable.h" #include +#include #include "gettext.h" #define _(msgid) gettext (msgid) @@ -61,6 +62,7 @@ struct sfm_writer int compress; /* 1=compressed, 0=not compressed. */ int case_cnt; /* Number of cases written so far. */ size_t flt64_cnt; /* Number of flt64 elements in case. */ + bool has_vls; /* Does the dict have very long strings? */ /* Compression buffering. */ flt64 *buf; /* Buffered data. */ @@ -72,6 +74,8 @@ struct sfm_writer /* Variables. */ struct sfm_var *vars; /* Variables. */ size_t var_cnt; /* Number of variables. */ + size_t var_cnt_vls; /* Number of variables including + very long string components. */ }; /* A variable in a system file. */ @@ -85,7 +89,7 @@ struct sfm_var static char *append_string_max (char *, const char *, const char *); static void write_header (struct sfm_writer *, const struct dictionary *); static void buf_write (struct sfm_writer *, const void *, size_t); -static void write_variable (struct sfm_writer *, struct variable *); +static void write_variable (struct sfm_writer *, const struct variable *); static void write_value_labels (struct sfm_writer *, struct variable *, int idx); static void write_rec_7_34 (struct sfm_writer *); @@ -93,18 +97,29 @@ static void write_rec_7_34 (struct sfm_writer *); static void write_longvar_table (struct sfm_writer *w, const struct dictionary *dict); +static void write_vls_length_table (struct sfm_writer *w, + const struct dictionary *dict); + + static void write_variable_display_parameters (struct sfm_writer *w, const struct dictionary *dict); static void write_documents (struct sfm_writer *, const struct dictionary *); -static int does_dict_need_translation (const struct dictionary *); static inline int var_flt64_cnt (const struct variable *v) +{ + assert(sizeof(flt64) == MAX_SHORT_STRING); + return width_to_bytes(v->width) / MAX_SHORT_STRING ; +} + +static inline int +var_flt64_cnt_nom (const struct variable *v) { return v->type == NUMERIC ? 1 : DIV_RND_UP (v->width, sizeof (flt64)); } + /* Returns default options for writing a system file. */ struct sfm_write_options sfm_writer_default_options (void) @@ -116,6 +131,28 @@ sfm_writer_default_options (void) return opts; } + +/* Return a short variable name to be used as the continuation of the + variable with the short name SN. + + FIXME: Need to resolve clashes somehow. + + */ +static const char * +cont_var_name(const char *sn, int idx) +{ + static char s[SHORT_NAME_LEN + 1]; + + char abb[SHORT_NAME_LEN + 1 - 3]= {0}; + + strncpy(abb, sn, SHORT_NAME_LEN - 3); + + snprintf(s, SHORT_NAME_LEN + 1, "%s%03d", abb, idx); + + return s; +} + + /* Opens the system file designated by file handle FH for writing cases from dictionary D according to the given OPTS. If COMPRESS is nonzero, the system file will be compressed. @@ -145,7 +182,7 @@ sfm_open_writer (struct file_handle *fh, struct dictionary *d, mode = S_IRUSR | S_IRGRP | S_IROTH; if (opts.create_writeable) mode |= S_IWUSR | S_IWGRP | S_IWOTH; - fd = open (fh_get_filename (fh), O_WRONLY | O_CREAT | O_TRUNC, mode); + fd = open (fh_get_file_name (fh), O_WRONLY | O_CREAT | O_TRUNC, mode); if (fd < 0) goto open_error; @@ -158,21 +195,27 @@ sfm_open_writer (struct file_handle *fh, struct dictionary *d, w->fh = fh; w->file = fdopen (fd, "w"); - w->needs_translation = does_dict_need_translation (d); + w->needs_translation = dict_compacting_would_change (d); w->compress = opts.compress; w->case_cnt = 0; w->flt64_cnt = 0; + w->has_vls = false; w->buf = w->end = w->ptr = NULL; w->x = w->y = NULL; w->var_cnt = dict_get_var_cnt (d); + w->var_cnt_vls = w->var_cnt; w->vars = xnmalloc (w->var_cnt, sizeof *w->vars); for (i = 0; i < w->var_cnt; i++) { const struct variable *dv = dict_get_var (d, i); struct sfm_var *sv = &w->vars[i]; sv->width = dv->width; + /* spss compatibility nonsense */ + if ( dv->width > MAX_LONG_STRING ) + w->has_vls = true; + sv->fv = dv->fv; sv->flt64_cnt = var_flt64_cnt (dv); } @@ -190,7 +233,42 @@ sfm_open_writer (struct file_handle *fh, struct dictionary *d, /* Write basic variable info. */ dict_assign_short_names (d); for (i = 0; i < dict_get_var_cnt (d); i++) - write_variable (w, dict_get_var (d, i)); + { + int count = 0; + const struct variable *v = dict_get_var(d, i); + int wcount = v->width; + + do { + struct variable var_cont = *v; + if ( v->type == ALPHA) + { + if ( 0 != count ) + { + mv_init(&var_cont.miss, 0); + strcpy(var_cont.short_name, + cont_var_name(v->short_name, count)); + var_cont.label = NULL; + w->var_cnt_vls++; + } + count++; + if ( wcount > MAX_LONG_STRING ) + { + var_cont.width = MAX_LONG_STRING; + wcount -= EFFECTIVE_LONG_STRING_LENGTH; + } + else + { + var_cont.width = wcount; + wcount -= var_cont.width; + } + + var_cont.write.w = var_cont.width; + var_cont.print.w = var_cont.width; + } + + write_variable (w, &var_cont); + } while(wcount > 0); + } /* Write out value labels. */ for (idx = i = 0; i < dict_get_var_cnt (d); i++) @@ -211,6 +289,8 @@ sfm_open_writer (struct file_handle *fh, struct dictionary *d, if (opts.version >= 3) write_longvar_table (w, d); + write_vls_length_table(w, d); + /* Write end-of-headers record. */ { struct @@ -246,27 +326,10 @@ sfm_open_writer (struct file_handle *fh, struct dictionary *d, open_error: msg (ME, _("Error opening \"%s\" for writing as a system file: %s."), - fh_get_filename (fh), strerror (errno)); + fh_get_file_name (fh), strerror (errno)); goto error; } -static int -does_dict_need_translation (const struct dictionary *d) -{ - size_t case_idx; - size_t i; - - case_idx = 0; - for (i = 0; i < dict_get_var_cnt (d); i++) - { - struct variable *v = dict_get_var (d, i); - if (v->fv != case_idx) - return 0; - case_idx += v->nv; - } - return 1; -} - /* Returns value of X truncated to two least-significant digits. */ static int rerange (int x) @@ -300,8 +363,10 @@ write_header (struct sfm_writer *w, const struct dictionary *d) w->flt64_cnt = 0; for (i = 0; i < dict_get_var_cnt (d); i++) - w->flt64_cnt += var_flt64_cnt (dict_get_var (d, i)); - hdr.case_size = w->flt64_cnt; + { + w->flt64_cnt += var_flt64_cnt (dict_get_var (d, i)); + } + hdr.nominal_case_size = w->flt64_cnt; hdr.compress = w->compress; @@ -370,15 +435,16 @@ write_header (struct sfm_writer *w, const struct dictionary *d) /* Translates format spec from internal form in SRC to system file format in DEST. */ static inline void -write_format_spec (struct fmt_spec *src, int32_t *dest) +write_format_spec (const struct fmt_spec *src, int32_t *dest) { + assert(check_output_specifier(src, true)); *dest = (formats[src->type].spss << 16) | (src->w << 8) | src->d; } /* Write the variable record(s) for primary variable P and secondary variable S to system file W. */ static void -write_variable (struct sfm_writer *w, struct variable *v) +write_variable (struct sfm_writer *w, const struct variable *v) { struct sysfile_variable sv; @@ -388,7 +454,7 @@ write_variable (struct sfm_writer *w, struct variable *v) int nm; /* Number of missing values, possibly negative. */ sv.rec_type = 2; - sv.type = v->width; + sv.type = min(v->width, MAX_LONG_STRING); sv.has_var_label = (v->label != NULL); mv_copy (&mv, &v->miss); @@ -453,7 +519,8 @@ write_variable (struct sfm_writer *w, struct variable *v) memset (&sv.write, 0, sizeof sv.write); memset (&sv.name, 0, sizeof sv.name); - pad_count = DIV_RND_UP (v->width, (int) sizeof (flt64)) - 1; + pad_count = DIV_RND_UP (min(v->width, MAX_LONG_STRING), + (int) sizeof (flt64)) - 1; for (i = 0; i < pad_count; i++) buf_write (w, &sv, sizeof sv); } @@ -563,7 +630,7 @@ write_variable_display_parameters (struct sfm_writer *w, vdp_hdr.rec_type = 7; vdp_hdr.subtype = 11; vdp_hdr.elem_size = 4; - vdp_hdr.n_elem = w->var_cnt * 3; + vdp_hdr.n_elem = w->var_cnt_vls * 3; buf_write (w, &vdp_hdr, sizeof vdp_hdr); @@ -585,7 +652,68 @@ write_variable_display_parameters (struct sfm_writer *w, params.align = v->alignment; buf_write (w, ¶ms, sizeof(params)); + + if ( v->width > MAX_LONG_STRING ) + { + int wcount = v->width - EFFECTIVE_LONG_STRING_LENGTH ; + + while (wcount > 0) + { + params.width = wcount > MAX_LONG_STRING ? 32 : wcount; + + buf_write (w, ¶ms, sizeof(params)); + + wcount -= EFFECTIVE_LONG_STRING_LENGTH ; + } + } + } +} + +/* Writes the table of lengths for Very Long String Variables */ +static void +write_vls_length_table (struct sfm_writer *w, + const struct dictionary *dict) +{ + int i; + struct + { + int32_t rec_type P; + int32_t subtype P; + int32_t elem_size P; + int32_t n_elem P; } + vls_hdr; + + struct string vls_length_map; + + ds_init_empty (&vls_length_map); + + vls_hdr.rec_type = 7; + vls_hdr.subtype = 14; + vls_hdr.elem_size = 1; + + + for (i = 0; i < dict_get_var_cnt (dict); ++i) + { + const struct variable *v = dict_get_var (dict, i); + + if ( v->width <= MAX_LONG_STRING ) + continue; + + ds_put_format (&vls_length_map, "%s=%05d", v->short_name, v->width); + ds_put_char (&vls_length_map, '\0'); + ds_put_char (&vls_length_map, '\t'); + } + + vls_hdr.n_elem = ds_length (&vls_length_map); + + if ( vls_hdr.n_elem > 0 ) + { + buf_write (w, &vls_hdr, sizeof vls_hdr); + buf_write (w, ds_data (&vls_length_map), ds_length (&vls_length_map)); + } + + ds_destroy (&vls_length_map); } /* Writes the long variable name table */ @@ -604,14 +732,14 @@ write_longvar_table (struct sfm_writer *w, const struct dictionary *dict) struct string long_name_map; size_t i; - ds_init (&long_name_map, 10 * dict_get_var_cnt (dict)); + ds_init_empty (&long_name_map); for (i = 0; i < dict_get_var_cnt (dict); i++) { struct variable *v = dict_get_var (dict, i); if (i) - ds_putc (&long_name_map, '\t'); - ds_printf (&long_name_map, "%s=%s", v->short_name, v->name); + ds_put_char (&long_name_map, '\t'); + ds_put_format (&long_name_map, "%s=%s", v->short_name, v->name); } lv_hdr.rec_type = 7; @@ -745,7 +873,7 @@ sfm_write_case (struct sfm_writer *w, const struct ccase *c) w->case_cnt++; if (!w->needs_translation && !w->compress - && sizeof (flt64) == sizeof (union value)) + && sizeof (flt64) == sizeof (union value) && ! w->has_vls ) { /* Fast path: external and internal representations are the same and the dictionary is properly ordered. Write @@ -758,21 +886,38 @@ sfm_write_case (struct sfm_writer *w, const struct ccase *c) Write into a bounce buffer, then write to W. */ flt64 *bounce; flt64 *bounce_cur; + flt64 *bounce_end; size_t bounce_size; size_t i; bounce_size = sizeof *bounce * w->flt64_cnt; bounce = bounce_cur = local_alloc (bounce_size); + bounce_end = bounce + bounce_size; for (i = 0; i < w->var_cnt; i++) { struct sfm_var *v = &w->vars[i]; + memset(bounce_cur, ' ', v->flt64_cnt * sizeof (flt64)); + if (v->width == 0) - *bounce_cur = case_num (c, v->fv); + { + *bounce_cur = case_num (c, v->fv); + bounce_cur += v->flt64_cnt; + } else - memcpy (bounce_cur, case_data (c, v->fv)->s, v->width); - bounce_cur += v->flt64_cnt; + { int ofs = 0; + while (ofs < v->width) + { + int chunk = MIN (MAX_LONG_STRING, v->width - ofs); + int nv = DIV_RND_UP (chunk, sizeof (flt64)); + buf_copy_rpad ((char *) bounce_cur, nv * sizeof (flt64), + case_data (c, v->fv)->s + ofs, chunk); + bounce_cur += nv; + ofs += chunk; + } + } + } if (!w->compress) @@ -893,7 +1038,7 @@ sfm_close_writer (struct sfm_writer *w) if (!ok) msg (ME, _("An I/O error occurred writing system file \"%s\"."), - fh_get_filename (w->fh)); + fh_get_file_name (w->fh)); } fh_close (w->fh, "system file", "we");