const struct sfm_extension_record *ext_encoding);
static struct text_record *open_text_record (
- struct sfm_reader *, const struct sfm_extension_record *);
+ struct sfm_reader *, const struct sfm_extension_record *,
+ bool recode_to_utf8);
static void close_text_record (struct sfm_reader *,
struct text_record *);
static bool read_variable_to_value_pair (struct sfm_reader *,
struct text_record *text;
struct mrset *mrset;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, false);
for (;;)
{
const char *counted = NULL;
name = text_get_token (text, ss_cstr ("="), NULL);
if (name == NULL)
break;
- mrset->name = xstrdup (name);
+ mrset->name = recode_string ("UTF-8", r->encoding, name, -1);
if (mrset->name[0] != '$')
{
sys_warn (r, record->pos,
- _("`%s' does not begin with `$' at UTF-8 offset %zu "
+ _("`%s' does not begin with `$' at offset %zu "
"in MRSETS record."), mrset->name, text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'C', text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'E', text_pos (text));
break;
}
else if (strcmp (number, "1"))
sys_warn (r, record->pos,
_("Unexpected label source value `%s' following `E' "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
number, text_pos (text));
}
else
{
sys_warn (r, record->pos,
- _("Missing `C', `D', or `E' at UTF-8 offset %zu "
+ _("Missing `C', `D', or `E' at offset %zu "
"in MRSETS record."),
text_pos (text));
break;
label = text_parse_counted_string (r, text);
if (label == NULL)
break;
- mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
+ if (label[0] != '\0')
+ mrset->label = recode_string ("UTF-8", r->encoding, label, -1);
stringi_set_init (&var_names);
allocated_vars = 0;
width = INT_MAX;
do
{
+ const char *raw_var_name;
struct variable *var;
- const char *var_name;
+ char *var_name;
- var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
- if (var_name == NULL)
+ raw_var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
+ if (raw_var_name == NULL)
{
sys_warn (r, record->pos,
_("Missing new-line parsing variable names "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
text_pos (text));
break;
}
+ var_name = recode_string ("UTF-8", r->encoding, raw_var_name, -1);
var = dict_lookup_var (dict, var_name);
if (var == NULL)
- continue;
+ {
+ free (var_name);
+ continue;
+ }
if (!stringi_set_insert (&var_names, var_name))
{
sys_warn (r, record->pos,
_("Duplicate variable name %s "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
var_name, text_pos (text));
+ free (var_name);
continue;
}
+ free (var_name);
if (mrset->label == NULL && mrset->label_from_var_label
&& var_has_label (var))
system file, this cannot create any intermediate duplicate variable names,
because all of the new variable names are longer than any of the old
variable names and thus there cannot be any overlaps.) */
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
{
/* Validate long name. */
struct variable *var;
char *length_s;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
{
size_t idx = var_get_dict_index (var);
const struct sfm_extension_record *record,
struct dictionary *dict)
{
- struct text_record *text = open_text_record (r, record);
+ struct text_record *text = open_text_record (r, record, true);
parse_attributes (r, text, dict_get_attributes (dict));
close_text_record (r, text);
}
struct text_record *text;
struct variable *var;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
close_text_record (r, text);
/* State. */
struct text_record
{
- struct substring buffer; /* Record contents, in UTF-8. */
+ struct substring buffer; /* Record contents. */
off_t start; /* Starting offset in file. */
size_t pos; /* Current position in buffer. */
int n_warnings; /* Number of warnings issued or suppressed. */
+ bool recoded; /* Recoded into UTF-8? */
};
static struct text_record *
open_text_record (struct sfm_reader *r,
- const struct sfm_extension_record *record)
+ const struct sfm_extension_record *record,
+ bool recode_to_utf8)
{
struct text_record *text;
struct substring raw;
text = pool_alloc (r->pool, sizeof *text);
raw = ss_buffer (record->data, record->size * record->count);
text->start = record->pos;
- text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool);
+ text->buffer = (recode_to_utf8
+ ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
+ : raw);
text->pos = 0;
text->n_warnings = 0;
+ text->recoded = recode_to_utf8;
return text;
}
if (text->n_warnings > MAX_TEXT_WARNINGS)
sys_warn (r, -1, _("Suppressed %d additional related warnings."),
text->n_warnings - MAX_TEXT_WARNINGS);
- pool_free (r->pool, ss_data (text->buffer));
+ if (text->recoded)
+ pool_free (r->pool, ss_data (text->buffer));
}
/* Reads a variable=value pair from TEXT.
if (start == text->pos)
{
sys_warn (r, text->start,
- _("Expecting digit at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting digit at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (!text_match (text, ' '))
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting space at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (text->pos + n > text->buffer.length)
{
sys_warn (r, text->start,
- _("%zu-byte string starting at UTF-8 offset %zu "
+ _("%zu-byte string starting at offset %zu "
"exceeds record length %zu."),
n, text->pos, text->buffer.length);
return NULL;
if (s[n] != ' ')
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu following %zu-byte "
- "string."),
+ _("Expecting space at offset %zu following %zu-byte string."),
text->pos + n, n);
return NULL;
}
return false;
}
-/* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's
- string. */
+/* Returns the current byte offset (as converted to UTF-8, if it was converted)
+ inside the TEXT's string. */
static size_t
text_pos (const struct text_record *text)
{
const char *string, size_t width);
static void write_utf8_record (struct sfm_writer *, const char *encoding,
const struct string *content, int subtype);
+static void write_string_record (struct sfm_writer *,
+ const struct substring content, int subtype);
static void write_bytes (struct sfm_writer *, const void *, size_t);
static void write_zeros (struct sfm_writer *, size_t);
static void write_spaces (struct sfm_writer *, size_t);
write_mrsets (struct sfm_writer *w, const struct dictionary *dict,
bool pre_v14)
{
+ const char *encoding = dict_get_encoding (dict);
struct string s = DS_EMPTY_INITIALIZER;
size_t n_mrsets;
size_t i;
for (i = 0; i < n_mrsets; i++)
{
const struct mrset *mrset = dict_get_mrset (dict, i);
- const char *label;
+ char *name;
size_t j;
if ((mrset->type != MRSET_MD || mrset->cat_source != MRSET_COUNTEDVALUES)
!= pre_v14)
continue;
- ds_put_format (&s, "%s=", mrset->name);
+ name = recode_string (encoding, "UTF-8", mrset->name, -1);
+ ds_put_format (&s, "%s=", name);
+ free (name);
+
if (mrset->type == MRSET_MD)
{
char *counted;
ds_put_byte (&s, 'C');
ds_put_byte (&s, ' ');
- label = mrset->label && !mrset->label_from_var_label ? mrset->label : "";
- ds_put_format (&s, "%zu %s", strlen (label), label);
+ if (mrset->label && !mrset->label_from_var_label)
+ {
+ char *label = recode_string (encoding, "UTF-8", mrset->label, -1);
+ ds_put_format (&s, "%zu %s", strlen (label), label);
+ free (label);
+ }
+ else
+ ds_put_cstr (&s, "0 ");
for (j = 0; j < mrset->n_vars; j++)
- ds_put_format (&s, " %s", var_get_short_name (mrset->vars[j], 0));
+ {
+ const char *short_name_utf8 = var_get_short_name (mrset->vars[j], 0);
+ char *short_name = recode_string (encoding, "UTF-8",
+ short_name_utf8, -1);
+ ds_put_format (&s, " %s", short_name);
+ free (short_name);
+ }
ds_put_byte (&s, '\n');
}
- write_utf8_record (w, dict_get_encoding (dict), &s, pre_v14 ? 7 : 19);
+
+ write_string_record (w, ds_ss (&s), pre_v14 ? 7 : 19);
ds_destroy (&s);
}
struct substring s;
s = recode_substring_pool (encoding, "UTF-8", ds_ss (content), NULL);
+ write_string_record (w, s, subtype);
+ ss_dealloc (&s);
+}
+
+/* Writes a record with type 7, subtype SUBTYPE that contains the string
+ CONTENT. */
+static void
+write_string_record (struct sfm_writer *w,
+ const struct substring content, int subtype)
+{
write_int (w, 7);
write_int (w, subtype);
write_int (w, 1);
- write_int (w, ss_length (s));
- write_bytes (w, ss_data (s), ss_length (s));
- ss_dealloc (&s);
+ write_int (w, ss_length (content));
+ write_bytes (w, ss_data (content), ss_length (content));
}
/* Writes SIZE bytes of DATA to W's output file. */
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: `a' does not begin with `$' at UTF-8 offset 2 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: `a' does not begin with `$' at offset 2 in MRSETS record.
-warning: `sys-file.sav' near offset 0xeb: `xyz' does not begin with `$' at UTF-8 offset 4 in MRSETS record.
+warning: `sys-file.sav' near offset 0xeb: `xyz' does not begin with `$' at offset 4 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Missing space following `C' at UTF-8 offset 4 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Missing space following `C' at offset 4 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Missing space following `E' at UTF-8 offset 4 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Missing space following `E' at offset 4 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Unexpected label source value `2' following `E' at UTF-8 offset 7 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Unexpected label source value `2' following `E' at offset 7 in MRSETS record.
-warning: `sys-file.sav' near offset 0xd8: Expecting digit at UTF-8 offset 7 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Expecting digit at offset 7 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-"warning: `sys-file.sav' near offset 0xd8: Missing `C', `D', or `E' at UTF-8 offset 3 in MRSETS record."
+"warning: `sys-file.sav' near offset 0xd8: Missing `C', `D', or `E' at offset 3 in MRSETS record."
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Expecting digit at UTF-8 offset 4 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Expecting digit at offset 4 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Expecting space at UTF-8 offset 5 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Expecting space at offset 5 in MRSETS record.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: 4-byte string starting at UTF-8 offset 6 exceeds record length 9.
+warning: `sys-file.sav' near offset 0xd8: 4-byte string starting at offset 6 exceeds record length 9.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Expecting space at UTF-8 offset 9 following 3-byte string.
+warning: `sys-file.sav' near offset 0xd8: Expecting space at offset 9 following 3-byte string.
])
done
AT_CLEANUP
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Missing new-line parsing variable names at UTF-8 offset 14 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Missing new-line parsing variable names at offset 14 in MRSETS record.
warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
])
AT_DATA([sys-file.sps], [GET FILE='sys-file.sav'.
])
AT_CHECK([pspp -O format=csv sys-file.sps], [0], [dnl
-warning: `sys-file.sav' near offset 0xd8: Duplicate variable name NUM1 at UTF-8 offset 18 in MRSETS record.
+warning: `sys-file.sav' near offset 0xd8: Duplicate variable name NUM1 at offset 18 in MRSETS record.
warning: `sys-file.sav' near offset 0xd8: MRSET $a has only 1 variables.
])
projects / pspp-builds.git / commitdiff