size_t sfm_var_cnt; /* Number of variables. */
int case_cnt; /* Number of cases */
const char *encoding; /* String encoding. */
+ bool written_by_readstat; /* From https://github.com/WizardMac/ReadStat? */
/* Decompression. */
enum any_compression compression;
static bool sfm_close (struct any_reader *);
-static struct variable *lookup_var_by_index (struct sfm_reader *, off_t,
- const struct sfm_var_record *,
- size_t n, int idx);
-
static void sys_msg (struct sfm_reader *r, off_t, int class,
const char *format, va_list args)
PRINTF_FORMAT (4, 0);
struct sfm_var_record *);
static bool read_value_label_record (struct sfm_reader *,
struct sfm_value_label_record *);
-static struct sfm_document_record *read_document_record (struct sfm_reader *);
+static bool read_document_record (struct sfm_reader *);
static bool read_extension_record (struct sfm_reader *, int subtype,
struct sfm_extension_record **);
static bool skip_extension_record (struct sfm_reader *, int subtype);
static bool parse_long_string_map (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
-static bool parse_value_labels (struct sfm_reader *, struct dictionary *,
- const struct sfm_var_record *,
- size_t n_var_recs,
- const struct sfm_value_label_record *);
+static void parse_value_labels (struct sfm_reader *, struct dictionary *);
+static struct variable *parse_weight_var (struct sfm_reader *,
+ const struct sfm_var_record *, size_t n_var_recs,
+ int idx);
static void parse_data_file_attributes (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
case 6:
if (r->document != NULL)
- {
- sys_error (r, r->pos, _("Duplicate type 6 (document) record."));
- return false;
- }
- r->document = read_document_record (r);
- return r->document != NULL;
+ sys_warn (r, r->pos, _("Duplicate type 6 (document) record."));
+ return read_document_record (r);
case 7:
if (!read_int (r, &subtype))
18. I'm surprised that SPSS puts up with this. */
struct sfm_extension_record *ext;
bool ok = read_extension_record (r, subtype, &ext);
- if (ok)
+ if (ok && ext)
ll_push_tail (&r->var_attrs, &ext->ll);
return ok;
}
va_end (args);
}
+static const char *
+skip_prefix (const char *s, const char *prefix)
+{
+ size_t prefix_len = strlen (prefix);
+ return !strncmp (s, prefix, prefix_len) ? s + prefix_len : s;
+}
+
/* Retrieves significant string data from R in its raw format, to allow the
caller to try to detect the encoding in use.
add_string (&aux, r->header.creation_date, _("Creation Date"));
add_string (&aux, r->header.creation_time, _("Creation Time"));
- add_string (&aux, r->header.eye_catcher, _("Product"));
+ add_string (&aux, skip_prefix (r->header.eye_catcher, "@(#) "), _("Product"));
add_string (&aux, r->header.file_label, _("File Label"));
if (r->extensions[EXT_PRODUCT_INFO])
{
struct sfm_reader *r = sfm_reader_cast (r_);
struct dictionary *dict;
- size_t i;
if (encoding == NULL)
{
/* Parse value labels and the weight variable immediately after the variable
records. These records use indexes into var_recs[], so we must parse them
before those indexes become invalidated by very long string variables. */
- for (i = 0; i < r->n_labels; i++)
- if (!parse_value_labels (r, dict, r->vars, r->n_vars, &r->labels[i]))
- goto error;
+ parse_value_labels (r, dict);
if (r->header.weight_idx != 0)
- {
- struct variable *weight_var;
-
- weight_var = lookup_var_by_index (r, 76, r->vars, r->n_vars,
- r->header.weight_idx);
- if (weight_var != NULL)
- {
- if (var_is_numeric (weight_var))
- dict_set_weight (dict, weight_var);
- else
- sys_warn (r, -1, _("Ignoring string variable `%s' set "
- "as weighting variable."),
- var_get_name (weight_var));
- }
- }
+ dict_set_weight (dict, parse_weight_var (r, r->vars, r->n_vars,
+ r->header.weight_idx));
if (r->extensions[EXT_DISPLAY] != NULL)
parse_display_parameters (r, r->extensions[EXT_DISPLAY], dict);
amount that the header claims. SPSS version 13 gets this
wrong when very long strings are involved, so don't warn in
that case. */
- if (r->header.nominal_case_size != -1
+ if (r->header.nominal_case_size > 0
&& r->header.nominal_case_size != r->n_vars
&& r->info.version_major != 13)
sys_warn (r, -1, _("File header claims %d variable positions but "
error:
sfm_close (r_);
- dict_destroy (dict);
+ dict_unref (dict);
*dictp = NULL;
return NULL;
}
if (!read_string (r, header->magic, sizeof header->magic)
|| !read_string (r, header->eye_catcher, sizeof header->eye_catcher))
return false;
+ r->written_by_readstat = strstr (header->eye_catcher,
+ "https://github.com/WizardMac/ReadStat");
if (!strcmp (ASCII_MAGIC, header->magic)
|| !strcmp (EBCDIC_MAGIC, header->magic))
if (!read_int (r, &r->case_cnt))
return false;
- if ( r->case_cnt > INT_MAX / 2)
+ if (r->case_cnt > INT_MAX / 2)
r->case_cnt = -1;
/* Identify floating-point format and obtain compression bias. */
return true;
}
-/* Reads a document record from R and returns it. */
-static struct sfm_document_record *
+/* Reads a document record from R. Returns true if successful, false on
+ error. */
+static bool
read_document_record (struct sfm_reader *r)
{
- struct sfm_document_record *record;
int n_lines;
-
- record = pool_malloc (r->pool, sizeof *record);
- record->pos = r->pos;
-
if (!read_int (r, &n_lines))
- return NULL;
- if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
+ return false;
+ else if (n_lines == 0)
+ return true;
+ else if (n_lines < 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
{
- sys_error (r, record->pos,
+ sys_error (r, r->pos,
_("Number of document lines (%d) "
"must be greater than 0 and less than %d."),
n_lines, INT_MAX / DOC_LINE_LENGTH);
- return NULL;
+ return false;
}
+ struct sfm_document_record *record;
+ record = pool_malloc (r->pool, sizeof *record);
+ record->pos = r->pos;
record->n_lines = n_lines;
record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
if (!read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines))
- return NULL;
+ return false;
- return record;
+ r->document = record;
+ return true;
}
static bool
info->product = ss_xstrdup (product);
}
+static struct variable *
+add_var_with_generated_name (struct dictionary *dict, int width)
+{
+ char *name = dict_make_unique_var_name (dict, NULL, NULL);
+ struct variable *var = dict_create_var_assert (dict, name, width);
+ free (name);
+ return var;
+}
+
/* Reads a variable (type 2) record from R and adds the
corresponding variable to DICT.
Also skips past additional variable records for long string
struct sfm_var_record *rec;
int n_warnings = 0;
- for (rec = var_recs; rec < &var_recs[n_var_recs]; )
+ for (rec = var_recs; rec < &var_recs[n_var_recs];)
{
- struct variable *var;
size_t n_values;
char *name;
size_t i;
rec->name, -1, r->pool);
name[strcspn (name, " ")] = '\0';
- if (!dict_id_is_valid (dict, name, false)
- || name[0] == '$' || name[0] == '#')
- {
- sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
- return false;
- }
-
if (rec->width < 0 || rec->width > 255)
{
sys_error (r, rec->pos,
return false;
}
- var = rec->var = dict_create_var (dict, name, rec->width);
- if (var == NULL)
+ struct variable *var;
+ if (!dict_id_is_valid (dict, name, false)
+ || name[0] == '$' || name[0] == '#')
{
- char *new_name = dict_make_unique_var_name (dict, NULL, NULL);
- sys_warn (r, rec->pos, _("Renaming variable with duplicate name "
- "`%s' to `%s'."),
- name, new_name);
- var = rec->var = dict_create_var_assert (dict, new_name, rec->width);
- free (new_name);
+ var = add_var_with_generated_name (dict, rec->width);
+ sys_warn (r, rec->pos, _("Renaming variable with invalid name "
+ "`%s' to `%s'."), name, var_get_name (var));
+ }
+ else
+ {
+ var = dict_create_var (dict, name, rec->width);
+ if (var == NULL)
+ {
+ var = add_var_with_generated_name (dict, rec->width);
+ sys_warn (r, rec->pos, _("Renaming variable with duplicate name "
+ "`%s' to `%s'."),
+ name, var_get_name (var));
+ }
}
+ rec->var = var;
- /* Set the short name the same as the long name. */
- var_set_short_name (var, 0, name);
+ /* Set the short name the same as the long name (even if we renamed
+ it). */
+ var_set_short_name (var, 0, var_get_name (var));
/* Get variable label, if any. */
if (rec->label)
int *n_warnings)
{
const int max_warnings = 8;
- uint8_t raw_type = format >> 16;
- uint8_t w = format >> 8;
- uint8_t d = format;
struct fmt_spec f;
- bool ok;
-
- f.w = w;
- f.d = d;
-
- msg_disable ();
- ok = (fmt_from_io (raw_type, &f.type)
- && fmt_check_output (&f)
- && fmt_check_width_compat (&f, var_get_width (v)));
- msg_enable ();
- if (ok)
+ if (fmt_from_u32 (format, var_get_width (v), false, &f))
{
if (which == PRINT_FORMAT)
var_set_print_format (v, &f);
text = open_text_record (r, record, false);
for (;;)
{
- struct sfm_mrset *mrset;
- size_t allocated_vars;
- char delimiter;
+ struct sfm_mrset *mrset = NULL;
+ size_t allocated_vars = 0;
+ char delimiter = '4';
/* Skip extra line feeds if present. */
while (text_match (text, '\n'))
}
number = text_get_token (text, ss_cstr (" "), NULL);
- if (!strcmp (number, "11"))
+ if (!number)
+ sys_warn (r, record->pos,
+ _("Missing label source value "
+ "following `E' at offset %zu in MRSETS record."),
+ text_pos (text));
+ else if (!strcmp (number, "11"))
mrset->label_from_var_label = true;
else if (strcmp (number, "1"))
sys_warn (r, record->pos,
size_t i;
name = recode_string ("UTF-8", r->encoding, s->name, -1);
- if (name[0] != '$')
+ if (!mrset_is_valid_name (name, dict_get_encoding (dict), false))
{
- sys_warn (r, -1, _("Multiple response set name `%s' does not begin "
- "with `$'."),
+ sys_warn (r, -1, _("Invalid multiple response set name `%s'."),
name);
free (name);
continue;
}
static void
-rename_var_and_save_short_names (struct dictionary *dict, struct variable *var,
- const char *new_name)
+rename_var_and_save_short_names (struct sfm_reader *r, off_t pos,
+ struct dictionary *dict,
+ struct variable *var, const char *new_name)
{
size_t n_short_names;
char **short_names;
for (i = 0; i < n_short_names; i++)
{
const char *s = var_get_short_name (var, i);
- short_names[i] = s != NULL ? xstrdup (s) : NULL;
+ short_names[i] = xstrdup_if_nonnull (s);
}
/* Set long name. */
- dict_rename_var (dict, var, new_name);
+ if (!dict_try_rename_var (dict, var, new_name))
+ sys_warn (r, pos, _("Duplicate long variable name `%s'."), new_name);
/* Restore short names. */
for (i = 0; i < n_short_names; i++)
char *new_name;
new_name = utf8_to_lower (var_get_name (var));
- rename_var_and_save_short_names (dict, var, new_name);
+ rename_var_and_save_short_names (r, -1, dict, var, new_name);
free (new_name);
}
continue;
}
- /* Identify any duplicates. */
- if (utf8_strcasecmp (var_get_short_name (var, 0), long_name)
- && dict_lookup_var (dict, long_name) != NULL)
- {
- sys_warn (r, record->pos,
- _("Duplicate long variable name `%s'."), long_name);
- continue;
- }
-
- rename_var_and_save_short_names (dict, var, long_name);
+ rename_var_and_save_short_names (r, record->pos, dict, var, long_name);
}
close_text_record (r, text);
}
return true;
}
-static bool
-parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
- const struct sfm_var_record *var_recs, size_t n_var_recs,
- const struct sfm_value_label_record *record)
+#define MAX_LABEL_WARNINGS 5
+
+/* Displays a warning for offset OFFSET in the file. */
+static void
+value_label_warning (struct sfm_reader *r, off_t offset, int *n_label_warnings,
+ const char *format, ...)
{
- struct variable **vars;
- char **utf8_labels;
- size_t i;
+ if (++*n_label_warnings > MAX_LABEL_WARNINGS)
+ return;
- utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
- for (i = 0; i < record->n_labels; i++)
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, offset, MW, format, args);
+ va_end (args);
+}
+
+#define MAX_LABEL_WARNINGS 5
+
+static void
+parse_one_value_label_set (struct sfm_reader *r, struct dictionary *dict,
+ const struct sfm_var_record *var_recs,
+ size_t n_var_recs,
+ const struct sfm_value_label_record *record,
+ int *n_label_warnings)
+{
+ char **utf8_labels
+ = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
+ for (size_t i = 0; i < record->n_labels; i++)
utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
record->labels[i].label, -1,
r->pool);
- vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
- for (i = 0; i < record->n_vars; i++)
+ struct variable **vars = pool_nmalloc (r->pool,
+ record->n_vars, sizeof *vars);
+ unsigned int n_vars = 0;
+ for (size_t i = 0; i < record->n_vars; i++)
{
- vars[i] = lookup_var_by_index (r, record->pos,
- var_recs, n_var_recs, record->vars[i]);
- if (vars[i] == NULL)
- return false;
+ int idx = record->vars[i];
+ if (idx < 1 || idx > n_var_recs)
+ {
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value label variable index %d not in valid range 1...%zu."),
+ idx, n_var_recs);
+ continue;
+ }
+
+ const struct sfm_var_record *rec = &var_recs[idx - 1];
+ if (rec->var == NULL)
+ {
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value label variable index %d "
+ "refers to long string continuation."), idx);
+ continue;
+ }
+
+ vars[n_vars++] = rec->var;
}
+ if (!n_vars)
+ return;
- for (i = 1; i < record->n_vars; i++)
+ for (size_t i = 1; i < n_vars; i++)
if (var_get_type (vars[i]) != var_get_type (vars[0]))
{
- sys_error (r, record->pos,
- _("Variables associated with value label are not all of "
- "identical type. Variable %s is %s, but variable "
- "%s is %s."),
- var_get_name (vars[0]),
- var_is_numeric (vars[0]) ? _("numeric") : _("string"),
- var_get_name (vars[i]),
- var_is_numeric (vars[i]) ? _("numeric") : _("string"));
- return false;
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Variables associated with value label are not all of "
+ "identical type. Variable %s is %s, but variable "
+ "%s is %s."),
+ var_get_name (vars[0]),
+ var_is_numeric (vars[0]) ? _("numeric") : _("string"),
+ var_get_name (vars[i]),
+ var_is_numeric (vars[i]) ? _("numeric") : _("string"));
+ return;
}
- for (i = 0; i < record->n_vars; i++)
+ for (size_t i = 0; i < n_vars; i++)
{
struct variable *var = vars[i];
- int width;
- size_t j;
-
- width = var_get_width (var);
+ int width = var_get_width (var);
if (width > 8)
{
- sys_error (r, record->pos,
- _("Value labels may not be added to long string "
- "variables (e.g. %s) using records types 3 and 4."),
- var_get_name (var));
- return false;
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value labels may not be added to long string "
+ "variables (e.g. %s) using records types 3 and 4."),
+ var_get_name (var));
+ continue;
}
- for (j = 0; j < record->n_labels; j++)
+ for (size_t j = 0; j < record->n_labels; j++)
{
struct sfm_value_label *label = &record->labels[j];
union value value;
if (width == 0)
value.f = parse_float (r, label->value, 0);
else
- memcpy (value_str_rw (&value, width), label->value, width);
+ memcpy (value.s, label->value, width);
if (!var_add_value_label (var, &value, utf8_labels[j]))
{
- if (var_is_numeric (var))
- sys_warn (r, record->pos,
- _("Duplicate value label for %g on %s."),
- value.f, var_get_name (var));
+ if (r->written_by_readstat)
+ {
+ /* Ignore the problem. ReadStat is buggy and emits value
+ labels whose values are longer than string variables'
+ widths, that are identical in the actual width of the
+ variable, e.g. both values "ABC123" and "ABC456" for a
+ string variable with width 3. */
+ }
+ else if (var_is_numeric (var))
+ value_label_warning (r, record->pos, n_label_warnings,
+ _("Duplicate value label for %g on %s."),
+ value.f, var_get_name (var));
else
- sys_warn (r, record->pos,
- _("Duplicate value label for `%.*s' on %s."),
- width, value_str (&value, width),
- var_get_name (var));
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value.s, var_get_name (var));
}
value_destroy (&value, width);
}
pool_free (r->pool, vars);
- for (i = 0; i < record->n_labels; i++)
+ for (size_t i = 0; i < record->n_labels; i++)
pool_free (r->pool, utf8_labels[i]);
pool_free (r->pool, utf8_labels);
+}
- return true;
+static void
+parse_value_labels (struct sfm_reader *r, struct dictionary *dict)
+{
+ int n_label_warnings = 0;
+ for (size_t i = 0; i < r->n_labels; i++)
+ parse_one_value_label_set (r, dict, r->vars, r->n_vars, &r->labels[i],
+ &n_label_warnings);
+ if (n_label_warnings > MAX_LABEL_WARNINGS)
+ sys_warn (r, -1,
+ _("Suppressed %d additional warnings for value labels."),
+ n_label_warnings - MAX_LABEL_WARNINGS);
}
static struct variable *
-lookup_var_by_index (struct sfm_reader *r, off_t offset,
- const struct sfm_var_record *var_recs, size_t n_var_recs,
- int idx)
+parse_weight_var (struct sfm_reader *r,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ int idx)
{
- const struct sfm_var_record *rec;
+ off_t offset = 76; /* Offset to variable index in header. */
if (idx < 1 || idx > n_var_recs)
{
- sys_error (r, offset,
- _("Variable index %d not in valid range 1...%zu."),
- idx, n_var_recs);
+ sys_warn (r, offset,
+ _("Weight variable index %d not in valid range 1...%zu. "
+ "Treating file as unweighted."),
+ idx, n_var_recs);
return NULL;
}
- rec = &var_recs[idx - 1];
+ const struct sfm_var_record *rec = &var_recs[idx - 1];
if (rec->var == NULL)
{
- sys_error (r, offset,
- _("Variable index %d refers to long string continuation."),
- idx);
+ sys_warn (r, offset,
+ _("Weight variable index %d refers to long string "
+ "continuation. Treating file as unweighted."), idx);
+ return NULL;
+ }
+
+ struct variable *weight_var = rec->var;
+ if (!var_is_numeric (weight_var))
+ {
+ sys_warn (r, offset, _("Ignoring string variable `%s' set "
+ "as weighting variable."),
+ var_get_name (weight_var));
return NULL;
}
- return rec->var;
+ return weight_var;
}
/* Parses a set of custom attributes from TEXT into ATTRS.
if (text_match (text, ')'))
break;
}
- if (attrs != NULL)
- attrset_add (attrs, attr);
+ if (attrs != NULL && attribute_get_n_values (attr) > 0)
+ {
+ if (!attrset_try_add (attrs, attr))
+ {
+ text_warn (r, text, _("Duplicate attribute %s."),
+ attribute_get_name (attr));
+ attribute_destroy (attr);
+ }
+ }
else
attribute_destroy (attr);
}
struct variable *var = dict_get_var (dict, i);
struct attrset *attrs = var_get_attributes (var);
const struct attribute *attr = attrset_lookup (attrs, "$@Role");
- if (attr != NULL)
+ if (attr != NULL && attribute_get_n_values (attr) > 0)
{
int value = atoi (attribute_get_value (attr, 0));
enum var_role role;
ofs += 4;
/* Parse variable name, width, and number of labels. */
- if (!check_overflow (r, record, ofs, var_name_len + 8))
+ if (!check_overflow (r, record, ofs, var_name_len)
+ || !check_overflow (r, record, ofs, var_name_len + 8))
return;
var_name = recode_string_pool ("UTF-8", dict_encoding,
(const char *) record->data + ofs,
if (!skip)
{
if (value_length == width)
- memcpy (value_str_rw (&value, width),
- (const uint8_t *) record->data + ofs, width);
+ memcpy (value.s, (const uint8_t *) record->data + ofs, width);
else
{
sys_warn (r, record->pos + ofs,
if (!var_add_value_label (var, &value, label))
sys_warn (r, record->pos + ofs,
_("Duplicate value label for `%.*s' on %s."),
- width, value_str (&value, width),
- var_get_name (var));
+ width, value.s, var_get_name (var));
pool_free (r->pool, label);
}
ofs += label_length;
ofs += 4;
/* Parse variable name. */
- if (!check_overflow (r, record, ofs, var_name_len + 1))
+ if (!check_overflow (r, record, ofs, var_name_len)
+ || !check_overflow (r, record, ofs, var_name_len + 1))
return;
var_name = recode_string_pool ("UTF-8", dict_encoding,
(const char *) record->data + ofs,
retval = read_case_number (r, &v->f);
else
{
- uint8_t *s = value_str_rw (v, sv->var_width);
- retval = read_case_string (r, s + sv->offset, sv->segment_width);
+ retval = read_case_string (r, v->s + sv->offset, sv->segment_width);
if (retval == 1)
{
retval = skip_whole_strings (r, ROUND_DOWN (sv->padding, 8));
}
/* Closes TEXT, frees its storage, and issues a final warning
- about suppressed warnings if necesary. */
+ about suppressed warnings if necessary. */
static void
close_text_record (struct sfm_reader *r, struct text_record *text)
{
char *end;
if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
- return NULL;
+ {
+ if (delimiter != NULL)
+ *delimiter = ss_data (text->buffer)[text->pos-1];
+ return NULL;
+ }
end = &ss_data (token)[ss_length (token)];
if (delimiter != NULL)
sys_msg (struct sfm_reader *r, off_t offset,
int class, const char *format, va_list args)
{
- struct msg m;
struct string text;
ds_init_empty (&text);
ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
ds_put_vformat (&text, format, args);
- m.category = msg_class_to_category (class);
- m.severity = msg_class_to_severity (class);
- m.file_name = NULL;
- m.first_line = 0;
- m.last_line = 0;
- m.first_column = 0;
- m.last_column = 0;
- m.text = ds_cstr (&text);
-
+ struct msg m = {
+ .category = msg_class_to_category (class),
+ .severity = msg_class_to_severity (class),
+ .text = ds_cstr (&text),
+ };
msg_emit (&m);
}
if (fstat (fileno (r->file), &s))
{
- sys_error (ME, 0, _("%s: stat failed (%s)."),
+ sys_error (r, 0, _("%s: stat failed (%s)."),
fh_get_file_name (r->fh), strerror (errno));
return false;
}
projects / pspp / blobdiff