const struct sfm_extension_record *ext_encoding);
static struct text_record *open_text_record (
- struct sfm_reader *, const struct sfm_extension_record *);
+ struct sfm_reader *, const struct sfm_extension_record *,
+ bool recode_to_utf8);
static void close_text_record (struct sfm_reader *,
struct text_record *);
static bool read_variable_to_value_pair (struct sfm_reader *,
how the rest of the header data is to be interpreted. */
dict = dict_create (choose_encoding (r, extensions[EXT_INTEGER],
extensions[EXT_ENCODING]));
+ r->encoding = dict_get_encoding (dict);
/* These records don't use variables at all. */
if (document != NULL)
if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars
&& info->version_major != 13)
sys_warn (r, -1, _("File header claims %d variable positions but "
- "%d were read from file."),
+ "%zu were read from file."),
claimed_oct_cnt, n_vars);
/* Create an index of dictionary variable widths for
/* Read up to MAX_LABEL_LEN bytes of label. */
read_len = MIN (MAX_LABEL_LEN, len);
- record->label = xmalloc (read_len + 1);
+ record->label = pool_malloc (r->pool, read_len + 1);
read_string (r, record->label, read_len + 1);
/* Skip unread label bytes. */
record->n_vars = read_int (r);
if (record->n_vars < 1 || record->n_vars > n_vars)
sys_error (r, r->pos - 4,
- _("Number of variables associated with a value label (%d) "
+ _("Number of variables associated with a value label (%zu) "
"is not between 1 and the number of variables (%zu)."),
record->n_vars, n_vars);
record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
utf8_label = recode_string_pool ("UTF-8", dict_encoding,
rec->label, -1, r->pool);
- var_set_label (var, utf8_label, NULL, false);
+ var_set_label (var, utf8_label, false);
}
/* Set missing values. */
uint8_t w = format >> 8;
uint8_t d = format;
struct fmt_spec f;
-
bool ok;
- if (!fmt_from_io (raw_type, &f.type))
- sys_error (r, pos, _("Unknown variable format %"PRIu8"."), raw_type);
f.w = w;
f.d = d;
msg_disable ();
- ok = fmt_check_output (&f) && fmt_check_width_compat (&f, var_get_width (v));
+ ok = (fmt_from_io (raw_type, &f.type)
+ && fmt_check_output (&f)
+ && fmt_check_width_compat (&f, var_get_width (v)));
msg_enable ();
if (ok)
else
var_set_write_format (v, &f);
}
+ else if (format == 0)
+ {
+ /* Actually observed in the wild. No point in warning about it. */
+ }
else if (++*n_warnings <= max_warnings)
{
- char fmt_string[FMT_STRING_LEN_MAX + 1];
- sys_warn (r, pos, _("%s variable %s has invalid %s format %s."),
- var_is_numeric (v) ? _("Numeric") : _("String"),
- var_get_name (v),
- which == PRINT_FORMAT ? _("print") : _("write"),
- fmt_to_string (&f, fmt_string));
+ if (which == PRINT_FORMAT)
+ sys_warn (r, pos, _("Variable %s with width %d has invalid print "
+ "format 0x%x."),
+ var_get_name (v), var_get_width (v), format);
+ else
+ sys_warn (r, pos, _("Variable %s with width %d has invalid write "
+ "format 0x%x."),
+ var_get_name (v), var_get_width (v), format);
if (*n_warnings == max_warnings)
sys_warn (r, -1, _("Suppressing further invalid format warnings."));
if (ext_integer)
{
int codepage = parse_int (r, ext_integer->data, 7 * 4);
+ const char *encoding;
switch (codepage)
{
case 4:
return "MS_KANJI";
- case 65000:
- return "UTF-7";
-
- case 65001:
- return "UTF-8";
-
default:
- return pool_asprintf (r->pool, "CP%d", codepage);
+ encoding = sys_get_encoding_from_codepage (codepage);
+ if (encoding != NULL)
+ return encoding;
+ break;
}
}
struct text_record *text;
struct mrset *mrset;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, false);
for (;;)
{
const char *counted = NULL;
name = text_get_token (text, ss_cstr ("="), NULL);
if (name == NULL)
break;
- mrset->name = xstrdup (name);
+ mrset->name = recode_string ("UTF-8", r->encoding, name, -1);
if (mrset->name[0] != '$')
{
sys_warn (r, record->pos,
- _("`%s' does not begin with `$' at UTF-8 offset %zu "
+ _("`%s' does not begin with `$' at offset %zu "
"in MRSETS record."), mrset->name, text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'C', text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'E', text_pos (text));
break;
}
else if (strcmp (number, "1"))
sys_warn (r, record->pos,
_("Unexpected label source value `%s' following `E' "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
number, text_pos (text));
}
else
{
sys_warn (r, record->pos,
- _("Missing `C', `D', or `E' at UTF-8 offset %zu "
+ _("Missing `C', `D', or `E' at offset %zu "
"in MRSETS record."),
text_pos (text));
break;
label = text_parse_counted_string (r, text);
if (label == NULL)
break;
- mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
+ if (label[0] != '\0')
+ mrset->label = recode_string ("UTF-8", r->encoding, label, -1);
stringi_set_init (&var_names);
allocated_vars = 0;
width = INT_MAX;
do
{
+ const char *raw_var_name;
struct variable *var;
- const char *var_name;
+ char *var_name;
- var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
- if (var_name == NULL)
+ raw_var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
+ if (raw_var_name == NULL)
{
sys_warn (r, record->pos,
_("Missing new-line parsing variable names "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
text_pos (text));
break;
}
+ var_name = recode_string ("UTF-8", r->encoding, raw_var_name, -1);
var = dict_lookup_var (dict, var_name);
if (var == NULL)
- continue;
+ {
+ free (var_name);
+ continue;
+ }
if (!stringi_set_insert (&var_names, var_name))
{
sys_warn (r, record->pos,
_("Duplicate variable name %s "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
var_name, text_pos (text));
+ free (var_name);
continue;
}
+ free (var_name);
if (mrset->label == NULL && mrset->label_from_var_label
&& var_has_label (var))
system file, this cannot create any intermediate duplicate variable names,
because all of the new variable names are longer than any of the old
variable names and thus there cannot be any overlaps.) */
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
{
/* Validate long name. */
struct variable *var;
char *length_s;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
{
size_t idx = var_get_dict_index (var);
if (idx < 1 || idx > n_var_recs)
{
sys_error (r, offset,
- _("Variable index %d not in valid range 1...%d."),
+ _("Variable index %d not in valid range 1...%zu."),
idx, n_var_recs);
return NULL;
}
const struct sfm_extension_record *record,
struct dictionary *dict)
{
- struct text_record *text = open_text_record (r, record);
+ struct text_record *text = open_text_record (r, record, true);
parse_attributes (r, text, dict_get_attributes (dict));
close_text_record (r, text);
}
struct text_record *text;
struct variable *var;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
close_text_record (r, text);
/* State. */
struct text_record
{
- struct substring buffer; /* Record contents, in UTF-8. */
+ struct substring buffer; /* Record contents. */
off_t start; /* Starting offset in file. */
size_t pos; /* Current position in buffer. */
int n_warnings; /* Number of warnings issued or suppressed. */
+ bool recoded; /* Recoded into UTF-8? */
};
static struct text_record *
open_text_record (struct sfm_reader *r,
- const struct sfm_extension_record *record)
+ const struct sfm_extension_record *record,
+ bool recode_to_utf8)
{
struct text_record *text;
struct substring raw;
text = pool_alloc (r->pool, sizeof *text);
raw = ss_buffer (record->data, record->size * record->count);
text->start = record->pos;
- text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool);
+ text->buffer = (recode_to_utf8
+ ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
+ : raw);
text->pos = 0;
text->n_warnings = 0;
+ text->recoded = recode_to_utf8;
return text;
}
if (text->n_warnings > MAX_TEXT_WARNINGS)
sys_warn (r, -1, _("Suppressed %d additional related warnings."),
text->n_warnings - MAX_TEXT_WARNINGS);
- pool_free (r->pool, ss_data (text->buffer));
+ if (text->recoded)
+ pool_free (r->pool, ss_data (text->buffer));
}
/* Reads a variable=value pair from TEXT.
if (start == text->pos)
{
sys_warn (r, text->start,
- _("Expecting digit at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting digit at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (!text_match (text, ' '))
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting space at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (text->pos + n > text->buffer.length)
{
sys_warn (r, text->start,
- _("%zu-byte string starting at UTF-8 offset %zu "
+ _("%zu-byte string starting at offset %zu "
"exceeds record length %zu."),
n, text->pos, text->buffer.length);
return NULL;
if (s[n] != ' ')
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu following %zu-byte "
- "string."),
+ _("Expecting space at offset %zu following %zu-byte string."),
text->pos + n, n);
return NULL;
}
return false;
}
-/* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's
- string. */
+/* Returns the current byte offset (as converted to UTF-8, if it was converted)
+ inside the TEXT's string. */
static size_t
text_pos (const struct text_record *text)
{
projects / pspp-builds.git / blobdiff