From b9a5b6889648f6888b8796a54f03a6f7e60cd728 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Sun, 24 Apr 2011 13:19:15 -0700 Subject: [PATCH 1/1] sys-file-reader, sys-file-writer: Fix encoding problems for mrsets. --- src/data/sys-file-reader.c | 74 +++++++++++++++++++++-------------- src/data/sys-file-writer.c | 46 +++++++++++++++++----- tests/data/sys-file-reader.at | 26 ++++++------ 3 files changed, 94 insertions(+), 52 deletions(-) diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c index aa94e6d6..ee0bfb13 100644 --- a/src/data/sys-file-reader.c +++ b/src/data/sys-file-reader.c @@ -203,7 +203,8 @@ static const char *choose_encoding ( const struct sfm_extension_record *ext_encoding); static struct text_record *open_text_record ( - struct sfm_reader *, const struct sfm_extension_record *); + struct sfm_reader *, const struct sfm_extension_record *, + bool recode_to_utf8); static void close_text_record (struct sfm_reader *, struct text_record *); static bool read_variable_to_value_pair (struct sfm_reader *, @@ -1227,7 +1228,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, struct text_record *text; struct mrset *mrset; - text = open_text_record (r, record); + text = open_text_record (r, record, false); for (;;) { const char *counted = NULL; @@ -1243,12 +1244,12 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, name = text_get_token (text, ss_cstr ("="), NULL); if (name == NULL) break; - mrset->name = xstrdup (name); + mrset->name = recode_string ("UTF-8", r->encoding, name, -1); if (mrset->name[0] != '$') { sys_warn (r, record->pos, - _("`%s' does not begin with `$' at UTF-8 offset %zu " + _("`%s' does not begin with `$' at offset %zu " "in MRSETS record."), mrset->name, text_pos (text)); break; } @@ -1259,7 +1260,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, if (!text_match (text, ' ')) { sys_warn (r, record->pos, - _("Missing space following `%c' at UTF-8 offset %zu " + _("Missing space following `%c' at offset %zu " "in MRSETS record."), 'C', text_pos (text)); break; } @@ -1278,7 +1279,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, if (!text_match (text, ' ')) { sys_warn (r, record->pos, - _("Missing space following `%c' at UTF-8 offset %zu " + _("Missing space following `%c' at offset %zu " "in MRSETS record."), 'E', text_pos (text)); break; } @@ -1289,13 +1290,13 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, else if (strcmp (number, "1")) sys_warn (r, record->pos, _("Unexpected label source value `%s' following `E' " - "at UTF-8 offset %zu in MRSETS record."), + "at offset %zu in MRSETS record."), number, text_pos (text)); } else { sys_warn (r, record->pos, - _("Missing `C', `D', or `E' at UTF-8 offset %zu " + _("Missing `C', `D', or `E' at offset %zu " "in MRSETS record."), text_pos (text)); break; @@ -1311,37 +1312,45 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, label = text_parse_counted_string (r, text); if (label == NULL) break; - mrset->label = label[0] != '\0' ? xstrdup (label) : NULL; + if (label[0] != '\0') + mrset->label = recode_string ("UTF-8", r->encoding, label, -1); stringi_set_init (&var_names); allocated_vars = 0; width = INT_MAX; do { + const char *raw_var_name; struct variable *var; - const char *var_name; + char *var_name; - var_name = text_get_token (text, ss_cstr (" \n"), &delimiter); - if (var_name == NULL) + raw_var_name = text_get_token (text, ss_cstr (" \n"), &delimiter); + if (raw_var_name == NULL) { sys_warn (r, record->pos, _("Missing new-line parsing variable names " - "at UTF-8 offset %zu in MRSETS record."), + "at offset %zu in MRSETS record."), text_pos (text)); break; } + var_name = recode_string ("UTF-8", r->encoding, raw_var_name, -1); var = dict_lookup_var (dict, var_name); if (var == NULL) - continue; + { + free (var_name); + continue; + } if (!stringi_set_insert (&var_names, var_name)) { sys_warn (r, record->pos, _("Duplicate variable name %s " - "at UTF-8 offset %zu in MRSETS record."), + "at offset %zu in MRSETS record."), var_name, text_pos (text)); + free (var_name); continue; } + free (var_name); if (mrset->label == NULL && mrset->label_from_var_label && var_has_label (var)) @@ -1536,7 +1545,7 @@ parse_long_var_name_map (struct sfm_reader *r, system file, this cannot create any intermediate duplicate variable names, because all of the new variable names are longer than any of the old variable names and thus there cannot be any overlaps.) */ - text = open_text_record (r, record); + text = open_text_record (r, record, true); while (read_variable_to_value_pair (r, dict, text, &var, &long_name)) { /* Validate long name. */ @@ -1575,7 +1584,7 @@ parse_long_string_map (struct sfm_reader *r, struct variable *var; char *length_s; - text = open_text_record (r, record); + text = open_text_record (r, record, true); while (read_variable_to_value_pair (r, dict, text, &var, &length_s)) { size_t idx = var_get_dict_index (var); @@ -1803,7 +1812,7 @@ parse_data_file_attributes (struct sfm_reader *r, const struct sfm_extension_record *record, struct dictionary *dict) { - struct text_record *text = open_text_record (r, record); + struct text_record *text = open_text_record (r, record, true); parse_attributes (r, text, dict_get_attributes (dict)); close_text_record (r, text); } @@ -1818,7 +1827,7 @@ parse_variable_attributes (struct sfm_reader *r, struct text_record *text; struct variable *var; - text = open_text_record (r, record); + text = open_text_record (r, record, true); while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var)) parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL); close_text_record (r, text); @@ -2240,15 +2249,17 @@ skip_whole_strings (struct sfm_reader *r, size_t length) /* State. */ struct text_record { - struct substring buffer; /* Record contents, in UTF-8. */ + struct substring buffer; /* Record contents. */ off_t start; /* Starting offset in file. */ size_t pos; /* Current position in buffer. */ int n_warnings; /* Number of warnings issued or suppressed. */ + bool recoded; /* Recoded into UTF-8? */ }; static struct text_record * open_text_record (struct sfm_reader *r, - const struct sfm_extension_record *record) + const struct sfm_extension_record *record, + bool recode_to_utf8) { struct text_record *text; struct substring raw; @@ -2256,9 +2267,12 @@ open_text_record (struct sfm_reader *r, text = pool_alloc (r->pool, sizeof *text); raw = ss_buffer (record->data, record->size * record->count); text->start = record->pos; - text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool); + text->buffer = (recode_to_utf8 + ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool) + : raw); text->pos = 0; text->n_warnings = 0; + text->recoded = recode_to_utf8; return text; } @@ -2271,7 +2285,8 @@ close_text_record (struct sfm_reader *r, struct text_record *text) if (text->n_warnings > MAX_TEXT_WARNINGS) sys_warn (r, -1, _("Suppressed %d additional related warnings."), text->n_warnings - MAX_TEXT_WARNINGS); - pool_free (r->pool, ss_data (text->buffer)); + if (text->recoded) + pool_free (r->pool, ss_data (text->buffer)); } /* Reads a variable=value pair from TEXT. @@ -2393,7 +2408,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) if (start == text->pos) { sys_warn (r, text->start, - _("Expecting digit at UTF-8 offset %zu in MRSETS record."), + _("Expecting digit at offset %zu in MRSETS record."), text->pos); return NULL; } @@ -2401,7 +2416,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) if (!text_match (text, ' ')) { sys_warn (r, text->start, - _("Expecting space at UTF-8 offset %zu in MRSETS record."), + _("Expecting space at offset %zu in MRSETS record."), text->pos); return NULL; } @@ -2409,7 +2424,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) if (text->pos + n > text->buffer.length) { sys_warn (r, text->start, - _("%zu-byte string starting at UTF-8 offset %zu " + _("%zu-byte string starting at offset %zu " "exceeds record length %zu."), n, text->pos, text->buffer.length); return NULL; @@ -2419,8 +2434,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) if (s[n] != ' ') { sys_warn (r, text->start, - _("Expecting space at UTF-8 offset %zu following %zu-byte " - "string."), + _("Expecting space at offset %zu following %zu-byte string."), text->pos + n, n); return NULL; } @@ -2441,8 +2455,8 @@ text_match (struct text_record *text, char c) return false; } -/* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's - string. */ +/* Returns the current byte offset (as converted to UTF-8, if it was converted) + inside the TEXT's string. */ static size_t text_pos (const struct text_record *text) { diff --git a/src/data/sys-file-writer.c b/src/data/sys-file-writer.c index b5e03ecc..d98c0a11 100644 --- a/src/data/sys-file-writer.c +++ b/src/data/sys-file-writer.c @@ -136,6 +136,8 @@ static void write_utf8_string (struct sfm_writer *, const char *encoding, const char *string, size_t width); static void write_utf8_record (struct sfm_writer *, const char *encoding, const struct string *content, int subtype); +static void write_string_record (struct sfm_writer *, + const struct substring content, int subtype); static void write_bytes (struct sfm_writer *, const void *, size_t); static void write_zeros (struct sfm_writer *, size_t); static void write_spaces (struct sfm_writer *, size_t); @@ -639,6 +641,7 @@ static void write_mrsets (struct sfm_writer *w, const struct dictionary *dict, bool pre_v14) { + const char *encoding = dict_get_encoding (dict); struct string s = DS_EMPTY_INITIALIZER; size_t n_mrsets; size_t i; @@ -650,14 +653,17 @@ write_mrsets (struct sfm_writer *w, const struct dictionary *dict, for (i = 0; i < n_mrsets; i++) { const struct mrset *mrset = dict_get_mrset (dict, i); - const char *label; + char *name; size_t j; if ((mrset->type != MRSET_MD || mrset->cat_source != MRSET_COUNTEDVALUES) != pre_v14) continue; - ds_put_format (&s, "%s=", mrset->name); + name = recode_string (encoding, "UTF-8", mrset->name, -1); + ds_put_format (&s, "%s=", name); + free (name); + if (mrset->type == MRSET_MD) { char *counted; @@ -679,14 +685,27 @@ write_mrsets (struct sfm_writer *w, const struct dictionary *dict, ds_put_byte (&s, 'C'); ds_put_byte (&s, ' '); - label = mrset->label && !mrset->label_from_var_label ? mrset->label : ""; - ds_put_format (&s, "%zu %s", strlen (label), label); + if (mrset->label && !mrset->label_from_var_label) + { + char *label = recode_string (encoding, "UTF-8", mrset->label, -1); + ds_put_format (&s, "%zu %s", strlen (label), label); + free (label); + } + else + ds_put_cstr (&s, "0 "); for (j = 0; j < mrset->n_vars; j++) - ds_put_format (&s, " %s", var_get_short_name (mrset->vars[j], 0)); + { + const char *short_name_utf8 = var_get_short_name (mrset->vars[j], 0); + char *short_name = recode_string (encoding, "UTF-8", + short_name_utf8, -1); + ds_put_format (&s, " %s", short_name); + free (short_name); + } ds_put_byte (&s, '\n'); } - write_utf8_record (w, dict_get_encoding (dict), &s, pre_v14 ? 7 : 19); + + write_string_record (w, ds_ss (&s), pre_v14 ? 7 : 19); ds_destroy (&s); } @@ -1234,12 +1253,21 @@ write_utf8_record (struct sfm_writer *w, const char *encoding, struct substring s; s = recode_substring_pool (encoding, "UTF-8", ds_ss (content), NULL); + write_string_record (w, s, subtype); + ss_dealloc (&s); +} + +/* Writes a record with type 7, subtype SUBTYPE that contains the string + CONTENT. */ +static void +write_string_record (struct sfm_writer *w, + const struct substring content, int subtype) +{ write_int (w, 7); write_int (w, subtype); write_int (w, 1); - write_int (w, ss_length (s)); - write_bytes (w, ss_data (s), ss_length (s)); - ss_dealloc (&s); + write_int (w, ss_length (content)); + write_bytes (w, ss_data (content), ss_length (content)); } /* Writes SIZE bytes of DATA to W's output file. */ diff --git a/tests/data/sys-file-reader.at b/tests/data/sys-file-reader.at index df7692f3..2564e7d1 100644 --- a/tests/data/sys-file-reader.at +++ b/tests/data/sys-file-reader.at @@ -2043,9 +2043,9 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: `a' does not begin with `$' at UTF-8 offset 2 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: `a' does not begin with `$' at offset 2 in MRSETS record. -warning: `sys-file.sav' near offset 0xeb: `xyz' does not begin with `$' at UTF-8 offset 4 in MRSETS record. +warning: `sys-file.sav' near offset 0xeb: `xyz' does not begin with `$' at offset 4 in MRSETS record. ]) done AT_CLEANUP @@ -2075,7 +2075,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Missing space following `C' at UTF-8 offset 4 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Missing space following `C' at offset 4 in MRSETS record. ]) done AT_CLEANUP @@ -2105,7 +2105,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Missing space following `E' at UTF-8 offset 4 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Missing space following `E' at offset 4 in MRSETS record. ]) done AT_CLEANUP @@ -2135,9 +2135,9 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Unexpected label source value `2' following `E' at UTF-8 offset 7 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Unexpected label source value `2' following `E' at offset 7 in MRSETS record. -warning: `sys-file.sav' near offset 0xd8: Expecting digit at UTF-8 offset 7 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Expecting digit at offset 7 in MRSETS record. ]) done AT_CLEANUP @@ -2167,7 +2167,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -"warning: `sys-file.sav' near offset 0xd8: Missing `C', `D', or `E' at UTF-8 offset 3 in MRSETS record." +"warning: `sys-file.sav' near offset 0xd8: Missing `C', `D', or `E' at offset 3 in MRSETS record." ]) done AT_CLEANUP @@ -2197,7 +2197,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Expecting digit at UTF-8 offset 4 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Expecting digit at offset 4 in MRSETS record. ]) done AT_CLEANUP @@ -2227,7 +2227,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Expecting space at UTF-8 offset 5 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Expecting space at offset 5 in MRSETS record. ]) done AT_CLEANUP @@ -2257,7 +2257,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: 4-byte string starting at UTF-8 offset 6 exceeds record length 9. +warning: `sys-file.sav' near offset 0xd8: 4-byte string starting at offset 6 exceeds record length 9. ]) done AT_CLEANUP @@ -2287,7 +2287,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Expecting space at UTF-8 offset 9 following 3-byte string. +warning: `sys-file.sav' near offset 0xd8: Expecting space at offset 9 following 3-byte string. ]) done AT_CLEANUP @@ -2317,7 +2317,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Missing new-line parsing variable names at UTF-8 offset 14 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Missing new-line parsing variable names at offset 14 in MRSETS record. warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables. ]) @@ -2349,7 +2349,7 @@ do AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'. ]) AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl -warning: `sys-file.sav' near offset 0xd8: Duplicate variable name NUM1 at UTF-8 offset 18 in MRSETS record. +warning: `sys-file.sav' near offset 0xd8: Duplicate variable name NUM1 at offset 18 in MRSETS record. warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables. ]) -- 2.30.2