/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006, 2007, 2009 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
-#include <data/sys-file-reader.h>
-#include <data/sys-file-private.h>
+#include "data/sys-file-reader.h"
+#include "data/sys-file-private.h"
#include <errno.h>
#include <float.h>
#include <setjmp.h>
#include <stdlib.h>
-#include <libpspp/assertion.h>
-#include <libpspp/message.h>
-#include <libpspp/compiler.h>
-#include <libpspp/misc.h>
-#include <libpspp/pool.h>
-#include <libpspp/str.h>
-#include <libpspp/hash.h>
-#include <libpspp/array.h>
-
-#include <data/attributes.h>
-#include <data/case.h>
-#include <data/casereader-provider.h>
-#include <data/casereader.h>
-#include <data/dictionary.h>
-#include <data/file-handle-def.h>
-#include <data/file-name.h>
-#include <data/format.h>
-#include <data/missing-values.h>
-#include <data/short-names.h>
-#include <data/value-labels.h>
-#include <data/variable.h>
-#include <data/value.h>
-
-#include "c-ctype.h"
-#include "inttostr.h"
-#include "minmax.h"
-#include "unlocked-io.h"
-#include "xalloc.h"
-#include "xsize.h"
+#include "data/attributes.h"
+#include "data/case.h"
+#include "data/casereader-provider.h"
+#include "data/casereader.h"
+#include "data/dictionary.h"
+#include "data/file-handle-def.h"
+#include "data/file-name.h"
+#include "data/format.h"
+#include "data/missing-values.h"
+#include "data/mrset.h"
+#include "data/short-names.h"
+#include "data/value-labels.h"
+#include "data/value.h"
+#include "data/variable.h"
+#include "libpspp/array.h"
+#include "libpspp/assertion.h"
+#include "libpspp/compiler.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "libpspp/misc.h"
+#include "libpspp/pool.h"
+#include "libpspp/str.h"
+#include "libpspp/stringi-set.h"
+
+#include "gl/c-ctype.h"
+#include "gl/inttostr.h"
+#include "gl/minmax.h"
+#include "gl/unlocked-io.h"
+#include "gl/xalloc.h"
+#include "gl/xsize.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
double bias; /* Compression bias, usually 100.0. */
uint8_t opcodes[8]; /* Current block of opcodes. */
size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
+ bool corruption_warning; /* Warned about possible corruption? */
};
static const struct casereader_class sys_file_casereader_class;
static struct variable *lookup_var_by_value_idx (struct sfm_reader *,
struct variable **,
int value_idx);
+static struct variable *lookup_var_by_short_name (struct dictionary *,
+ const char *short_name);
static void sys_msg (struct sfm_reader *r, int class,
const char *format, va_list args)
const char *format, ...)
PRINTF_FORMAT (3, 4);
static char *text_get_token (struct text_record *,
- struct substring delimiters);
+ struct substring delimiters, char *delimiter);
static bool text_match (struct text_record *, char c);
+static bool text_read_variable_name (struct sfm_reader *, struct dictionary *,
+ struct text_record *,
+ struct substring delimiters,
+ struct variable **);
static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
struct text_record *,
struct substring delimiters,
struct variable **);
+static const char *text_parse_counted_string (struct sfm_reader *,
+ struct text_record *);
+static size_t text_pos (const struct text_record *);
static bool close_reader (struct sfm_reader *r);
\f
);
static void read_machine_float_info (struct sfm_reader *,
size_t size, size_t count);
+static void read_mrsets (struct sfm_reader *, size_t size, size_t count,
+ struct dictionary *);
static void read_display_parameters (struct sfm_reader *,
size_t size, size_t count,
struct dictionary *);
size_t size, size_t count,
struct dictionary *);
+/* Convert all the strings in DICT from the dict encoding to UTF8 */
+static void
+recode_strings (struct dictionary *dict)
+{
+ int i;
+
+ const char *enc = dict_get_encoding (dict);
+
+ if ( NULL == enc)
+ enc = get_default_encoding ();
+
+ for (i = 0 ; i < dict_get_var_cnt (dict); ++i)
+ {
+ /* Convert the long variable name */
+ struct variable *var = dict_get_var (dict, i);
+ const char *native_name = var_get_name (var);
+ char *utf8_name = recode_string (UTF8, enc, native_name, -1);
+ if ( 0 != strcmp (utf8_name, native_name))
+ {
+ if ( NULL == dict_lookup_var (dict, utf8_name))
+ dict_rename_var (dict, var, utf8_name);
+ else
+ msg (MW,
+ _("Recoded variable name duplicates an existing `%s' within system file."), utf8_name);
+ }
+
+ free (utf8_name);
+
+ /* Convert the variable label */
+ if (var_has_label (var))
+ {
+ char *utf8_label = recode_string (UTF8, enc, var_get_label (var), -1);
+ var_set_label (var, utf8_label);
+ free (utf8_label);
+ }
+
+ if (var_has_value_labels (var))
+ {
+ const struct val_lab *vl = NULL;
+ const struct val_labs *vlabs = var_get_value_labels (var);
+
+ for (vl = val_labs_first (vlabs); vl != NULL; vl = val_labs_next (vlabs, vl))
+ {
+ const union value *val = val_lab_get_value (vl);
+ const char *label = val_lab_get_label (vl);
+ char *new_label = NULL;
+
+ new_label = recode_string (UTF8, enc, label, -1);
+
+ var_replace_value_label (var, val, new_label);
+ free (new_label);
+ }
+ }
+ }
+}
+
/* Opens the system file designated by file handle FH for
reading. Reads the system file's dictionary into *DICT.
If INFO is non-null, then it receives additional info about the
r->oct_cnt = 0;
r->has_long_var_names = false;
r->opcode_idx = sizeof r->opcodes;
+ r->corruption_warning = false;
/* TRANSLATORS: this fragment will be interpolated into
messages in fh_lock() that identify types of files. */
r->file = fn_open (fh_get_file_name (fh), "rb");
if (r->file == NULL)
{
- msg (ME, _("Error opening \"%s\" for reading as a system file: %s."),
+ msg (ME, _("Error opening `%s' for reading as a system file: %s."),
fh_get_file_name (r->fh), strerror (errno));
goto error;
}
r->has_long_var_names = true;
}
+ recode_strings (*dict);
+
/* Read record 999 data, which is just filler. */
read_int (r);
{
if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)
{
- msg (ME, _("Error closing system file \"%s\": %s."),
+ msg (ME, _("Error closing system file `%s': %s."),
fh_get_file_name (r->fh), strerror (errno));
r->error = true;
}
if ( r->case_cnt > INT_MAX / 2)
r->case_cnt = -1;
-
/* Identify floating-point format and obtain compression bias. */
read_bytes (r, raw_bias, sizeof raw_bias);
if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
{
- sys_warn (r, _("Compression bias is not the usual "
- "value of 100, or system file uses unrecognized "
- "floating-point format."));
+ uint8_t zero_bias[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+ if (memcmp (raw_bias, zero_bias, 8))
+ sys_warn (r, _("Compression bias is not the usual "
+ "value of 100, or system file uses unrecognized "
+ "floating-point format."));
+ else
+ {
+ /* Some software is known to write all-zeros to this
+ field. Such software also writes floating-point
+ numbers in the format that we expect by default
+ (it seems that all software most likely does, in
+ reality), so don't warn in this case. */
+ }
+
if (r->integer_format == INTEGER_MSB_FIRST)
r->float_format = FLOAT_IEEE_DOUBLE_BE;
else
/* Check variable name. */
if (name[0] == '$' || name[0] == '#')
- sys_error (r, "Variable name begins with invalid character `%c'.",
+ sys_error (r, _("Variable name begins with invalid character `%c'."),
name[0]);
if (!var_is_plausible_name (name, false))
sys_error (r, _("Invalid variable name `%s'."), name);
/* Create variable. */
if (width < 0 || width > 255)
- sys_error (r, _("Bad variable width %d."), width);
+ sys_error (r, _("Bad width %d for variable %s."), width, name);
var = dict_create_var (dict, name, width);
if (var == NULL)
- sys_error (r,
- _("Duplicate variable name `%s' within system file."),
- name);
+ sys_error (r, _("Duplicate variable name `%s'."), name);
/* Set the short name the same as the long name. */
var_set_short_name (var, 0, var_get_name (var));
sys_error (r, _("Variable label indicator field is not 0 or 1."));
if (has_variable_label == 1)
{
- size_t len;
+ size_t len, read_len;
char label[255 + 1];
len = read_int (r);
- if (len >= sizeof label)
- sys_error (r, _("Variable %s has label of invalid length %zu."),
- name, len);
- read_string (r, label, len + 1);
+
+ /* Read up to 255 bytes of label. */
+ read_len = MIN (sizeof label - 1, len);
+ read_string (r, label, read_len + 1);
var_set_label (var, label);
+ /* Skip unread label bytes. */
+ skip_bytes (r, len - read_len);
+
+ /* Skip label padding up to multiple of 4 bytes. */
skip_bytes (r, ROUND_UP (len, 4) - len);
}
struct missing_values mv;
int i;
- mv_init (&mv, var_get_width (var));
+ mv_init_pool (r->pool, &mv, var_get_width (var));
if (var_is_numeric (var))
{
if (missing_value_code < -3 || missing_value_code > 3
}
else
{
+ int mv_width = MAX (width, 8);
+ union value value;
+
if (missing_value_code < 1 || missing_value_code > 3)
sys_error (r, _("String missing value indicator field is not "
"0, 1, 2, or 3."));
- if (var_is_long_string (var))
- sys_warn (r, _("Ignoring missing values on long string variable "
- "%s, which PSPP does not yet support."), name);
+
+ value_init (&value, mv_width);
+ value_set_missing (&value, mv_width);
for (i = 0; i < missing_value_code; i++)
{
- char string[9];
- read_string (r, string, sizeof string);
- mv_add_str (&mv, string);
+ uint8_t *s = value_str_rw (&value, mv_width);
+ read_bytes (r, s, 8);
+ mv_add_str (&mv, s);
}
+ value_destroy (&value, mv_width);
}
- if (!var_is_long_string (var))
- var_set_missing_values (var, &mv);
+ var_set_missing_values (var, &mv);
}
/* Set formats. */
else
var_set_write_format (v, &f);
}
- else if (*++format_warning_cnt <= max_format_warnings)
+ else if (++*format_warning_cnt <= max_format_warnings)
{
char fmt_string[FMT_STRING_LEN_MAX + 1];
sys_warn (r, _("%s variable %s has invalid %s format %s."),
if (var_is_numeric (weight_var))
dict_set_weight (dict, weight_var);
else
- sys_error (r, _("Weighting variable must be numeric."));
+ sys_error (r, _("Weighting variable must be numeric "
+ "(not string variable `%s')."),
+ var_get_name (weight_var));
}
}
break;
case 7:
- /* Used by the MRSETS command. */
- break;
+ case 19:
+ read_mrsets (r, size, count, dict);
+ return;
case 8:
/* Used by the SPSS Data Entry software. */
return;
case 16:
- /* New in SPSS v14? Unknown purpose. */
+ /* Extended number of cases. Not important. */
break;
case 17:
return;
default:
- sys_warn (r, _("Unrecognized record type 7, subtype %d. Please send a copy of this file, and the syntax which created it to %s"),
+ sys_warn (r, _("Unrecognized record type 7, subtype %d. Please send "
+ "a copy of this file, and the syntax which created it "
+ "to %s."),
subtype, PACKAGE_BUGREPORT);
break;
}
if (float_representation != expected_float_format)
sys_error (r, _("Floating-point representation indicated by "
"system file (%d) differs from expected (%d)."),
- r->float_format, expected_float_format);
+ float_representation, expected_float_format);
/* Check integer format. */
if (r->integer_format == INTEGER_MSB_FIRST)
else
NOT_REACHED ();
if (integer_representation != expected_integer_format)
- {
- static const char *const endian[] = {N_("little-endian"), N_("big-endian")};
- sys_warn (r, _("Integer format indicated by system file (%s) "
- "differs from expected (%s)."),
- gettext (endian[integer_representation == 1]),
- gettext (endian[expected_integer_format == 1]));
- }
-
+ sys_warn (r, _("Integer format indicated by system file (%d) "
+ "differs from expected (%d)."),
+ integer_representation, expected_integer_format);
/*
Record 7 (20) provides a much more reliable way of
size, count);
if (sysmis != SYSMIS)
- sys_warn (r, _("File specifies unexpected value %g as SYSMIS."), sysmis);
+ sys_warn (r, _("File specifies unexpected value %g as %s."),
+ sysmis, "SYSMIS");
+
if (highest != HIGHEST)
- sys_warn (r, _("File specifies unexpected value %g as HIGHEST."), highest);
+ sys_warn (r, _("File specifies unexpected value %g as %s."),
+ highest, "HIGHEST");
+
if (lowest != LOWEST)
- sys_warn (r, _("File specifies unexpected value %g as LOWEST."), lowest);
+ sys_warn (r, _("File specifies unexpected value %g as %s."),
+ lowest, "LOWEST");
+}
+
+/* Read record type 7, subtype 7 or 19. */
+static void
+read_mrsets (struct sfm_reader *r, size_t size, size_t count,
+ struct dictionary *dict)
+{
+ struct text_record *text;
+ struct mrset *mrset;
+
+ text = open_text_record (r, size * count);
+ for (;;)
+ {
+ const char *name, *label, *counted;
+ struct stringi_set var_names;
+ size_t allocated_vars;
+ char delimiter;
+ int width;
+
+ mrset = xzalloc (sizeof *mrset);
+
+ name = text_get_token (text, ss_cstr ("="), NULL);
+ if (name == NULL)
+ break;
+ mrset->name = xstrdup (name);
+
+ if (mrset->name[0] != '$')
+ {
+ sys_warn (r, _("`%s' does not begin with `$' at offset %zu "
+ "in MRSETS record."), mrset->name, text_pos (text));
+ break;
+ }
+
+ if (text_match (text, 'C'))
+ {
+ mrset->type = MRSET_MC;
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, _("Missing space following `%c' at offset %zu "
+ "in MRSETS record."), 'C', text_pos (text));
+ break;
+ }
+ }
+ else if (text_match (text, 'D'))
+ {
+ mrset->type = MRSET_MD;
+ mrset->cat_source = MRSET_VARLABELS;
+ }
+ else if (text_match (text, 'E'))
+ {
+ char *number;
+
+ mrset->type = MRSET_MD;
+ mrset->cat_source = MRSET_COUNTEDVALUES;
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, _("Missing space following `%c' at offset %zu "
+ "in MRSETS record."), 'E', text_pos (text));
+ break;
+ }
+
+ number = text_get_token (text, ss_cstr (" "), NULL);
+ if (!strcmp (number, "11"))
+ mrset->label_from_var_label = true;
+ else if (strcmp (number, "1"))
+ sys_warn (r, _("Unexpected label source value `%s' "
+ "following `E' at offset %zu in MRSETS record."),
+ number, text_pos (text));
+ }
+ else
+ {
+ sys_warn (r, _("Missing `C', `D', or `E' at offset %zu "
+ "in MRSETS record."),
+ text_pos (text));
+ break;
+ }
+
+ if (mrset->type == MRSET_MD)
+ {
+ counted = text_parse_counted_string (r, text);
+ if (counted == NULL)
+ break;
+ }
+
+ label = text_parse_counted_string (r, text);
+ if (label == NULL)
+ break;
+ mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
+
+ stringi_set_init (&var_names);
+ allocated_vars = 0;
+ width = INT_MAX;
+ do
+ {
+ struct variable *var;
+ const char *var_name;
+
+ var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
+ if (var_name == NULL)
+ {
+ sys_warn (r, _("Missing new-line parsing variable names "
+ "at offset %zu in MRSETS record."),
+ text_pos (text));
+ break;
+ }
+
+ var = lookup_var_by_short_name (dict, var_name);
+ if (var == NULL)
+ continue;
+ if (!stringi_set_insert (&var_names, var_name))
+ {
+ sys_warn (r, _("Duplicate variable name %s "
+ "at offset %zu in MRSETS record."),
+ var_name, text_pos (text));
+ continue;
+ }
+
+ if (mrset->label == NULL && mrset->label_from_var_label
+ && var_has_label (var))
+ mrset->label = xstrdup (var_get_label (var));
+
+ if (mrset->n_vars
+ && var_get_type (var) != var_get_type (mrset->vars[0]))
+ {
+ sys_warn (r, _("MRSET %s contains both string and "
+ "numeric variables."), name);
+ continue;
+ }
+ width = MIN (width, var_get_width (var));
+
+ if (mrset->n_vars >= allocated_vars)
+ mrset->vars = x2nrealloc (mrset->vars, &allocated_vars,
+ sizeof *mrset->vars);
+ mrset->vars[mrset->n_vars++] = var;
+ }
+ while (delimiter != '\n');
+
+ if (mrset->n_vars < 2)
+ {
+ sys_warn (r, _("MRSET %s has only %zu variables."), mrset->name,
+ mrset->n_vars);
+ mrset_destroy (mrset);
+ continue;
+ }
+
+ if (mrset->type == MRSET_MD)
+ {
+ mrset->width = width;
+ value_init (&mrset->counted, width);
+ if (width == 0)
+ mrset->counted.f = strtod (counted, NULL);
+ else
+ value_copy_str_rpad (&mrset->counted, width,
+ (const uint8_t *) counted, ' ');
+ }
+
+ dict_add_mrset (dict, mrset);
+ mrset = NULL;
+ stringi_set_destroy (&var_names);
+ }
+ mrset_destroy (mrset);
+ close_text_record (r, text);
}
/* Read record type 7, subtype 11, which specifies how variables
if (strcasecmp (var_get_short_name (var, 0), long_name)
&& dict_lookup_var (dict, long_name) != NULL)
{
- sys_warn (r, _("Duplicate long variable name `%s' "
- "within system file."), long_name);
+ sys_warn (r, _("Duplicate long variable name `%s'."), long_name);
continue;
}
if (length < 1 || length > MAX_STRING)
{
sys_warn (r, _("%s listed as string of invalid length %s "
- "in very length string record."),
+ "in very long string record."),
var_get_name (var), length_s);
continue;
}
var_set_short_name (var, i, var_get_short_name (seg, 0));
if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
sys_error (r, _("Very long string with width %ld has segment %d "
- "of width %d (expected %d)"),
+ "of width %d (expected %d)."),
length, i, width, alloc_width);
}
dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
struct label
{
- char raw_value[8]; /* Value as uninterpreted bytes. */
+ uint8_t raw_value[8]; /* Value as uninterpreted bytes. */
union value value; /* Value. */
char *label; /* Null-terminated label string. */
};
for (i = 0; i < var_cnt; i++)
{
var[i] = lookup_var_by_value_idx (r, var_by_value_idx, read_int (r));
- if (var_is_long_string (var[i]))
+ if (var_get_width (var[i]) > 8)
sys_error (r, _("Value labels may not be added to long string "
"variables (e.g. %s) using records types 3 and 4."),
var_get_name (var[i]));
value_init_pool (subpool, &label->value, max_width);
if (var_is_alpha (var[0]))
- buf_copy_rpad (value_str_rw (&label->value, max_width), max_width,
+ u8_buf_copy_rpad (value_str_rw (&label->value, max_width), max_width,
label->raw_value, sizeof label->raw_value, ' ');
else
label->value.f = float_get_double (r->float_format, label->raw_value);
sys_warn (r, _("Duplicate value label for %g on %s."),
label->value.f, var_get_name (v));
else
- sys_warn (r, _("Duplicate value label for \"%.*s\" on %s."),
+ sys_warn (r, _("Duplicate value label for `%.*s' on %s."),
max_width, value_str (&label->value, max_width),
var_get_name (v));
}
int index;
/* Parse the key. */
- key = text_get_token (text, ss_cstr ("("));
+ key = text_get_token (text, ss_cstr ("("), NULL);
if (key == NULL)
return;
char *value;
size_t length;
- value = text_get_token (text, ss_cstr ("\n"));
+ value = text_get_token (text, ss_cstr ("\n"), NULL);
if (value == NULL)
{
- text_warn (r, text, _("Error parsing attribute value %s[%d]"),
+ text_warn (r, text, _("Error parsing attribute value %s[%d]."),
key, index);
break;
}
else
{
text_warn (r, text,
- _("Attribute value %s[%d] is not quoted: %s"),
+ _("Attribute value %s[%d] is not quoted: %s."),
key, index, value);
attribute_add_value (attr, value);
}
{
sys_warn (r, _("Ignoring long string value record for variable %s "
"because the record's width (%d) does not match the "
- "variable's width (%d)"),
+ "variable's width (%d)."),
var_name, width, var_get_width (v));
skip_long_string_value_labels (r, n_labels);
continue;
/* Read value. */
value_length = read_int (r);
if (value_length == width)
- read_string (r, value_str_rw (&value, width), width + 1);
+ read_bytes (r, value_str_rw (&value, width), width);
else
{
sys_warn (r, _("Ignoring long string value %zu for variable %s, "
first 255 bytes. The maximum documented length
of a label is 120 bytes so this is more than
generous. */
- skip_bytes (r, sizeof label - (label_length + 1));
+ skip_bytes (r, (label_length + 1) - sizeof label);
}
if (!skip && !var_add_value_label (v, &value, label))
- sys_warn (r, _("Duplicate value label for \"%.*s\" on %s."),
+ sys_warn (r, _("Duplicate value label for `%.*s' on %s."),
width, value_str (&value, width), var_get_name (v));
}
}
for (;;)
{
struct variable *var;
- if (!text_read_short_name (r, dict, text, ss_cstr (":"), &var))
+ if (!text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
break;
read_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
}
static void read_error (struct casereader *, const struct sfm_reader *);
static bool read_case_number (struct sfm_reader *, double *);
-static bool read_case_string (struct sfm_reader *, char *, size_t);
+static bool read_case_string (struct sfm_reader *, uint8_t *, size_t);
static int read_opcode (struct sfm_reader *);
static bool read_compressed_number (struct sfm_reader *, double *);
-static bool read_compressed_string (struct sfm_reader *, char *);
-static bool read_whole_strings (struct sfm_reader *, char *, size_t);
+static bool read_compressed_string (struct sfm_reader *, uint8_t *);
+static bool read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
static bool skip_whole_strings (struct sfm_reader *, size_t);
/* Reads and returns one case from READER's file. Returns a null
}
else
{
- char *s = value_str_rw (v, sv->var_width);
+ uint8_t *s = value_str_rw (v, sv->var_width);
if (!read_case_string (r, s + sv->offset, sv->segment_width))
goto eof;
if (!skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)))
return c;
eof:
- case_unref (c);
if (i != 0)
partial_record (r);
if (r->case_cnt != -1)
read_error (reader, r);
+ case_unref (c);
return NULL;
}
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_case_string (struct sfm_reader *r, char *s, size_t length)
+read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
{
size_t whole = ROUND_DOWN (length, 8);
size_t partial = length % 8;
if (partial)
{
- char bounce[8];
+ uint8_t bounce[8];
if (!read_whole_strings (r, bounce, sizeof bounce))
{
if (whole)
break;
case 254:
- sys_error (r, _("Compressed data is corrupt."));
+ float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
+ if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, _("Possible compressed data corruption: "
+ "compressed spaces appear in numeric field."));
+ }
+ break;
case 255:
*d = SYSMIS;
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_compressed_string (struct sfm_reader *r, char *dst)
+read_compressed_string (struct sfm_reader *r, uint8_t *dst)
{
- switch (read_opcode (r))
+ int opcode = read_opcode (r);
+ switch (opcode)
{
case -1:
case 252:
break;
default:
- sys_error (r, _("Compressed data is corrupt."));
+ {
+ double value = opcode - r->bias;
+ float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
+ if (value == 0.0)
+ {
+ /* This has actually been seen "in the wild". The submitter of the
+ file that showed that the contents decoded as spaces, but they
+ were at the end of the field so it's possible that the null
+ bytes just acted as null terminators. */
+ }
+ else if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, _("Possible compressed data corruption: "
+ "string contains compressed integer (opcode %d)."),
+ opcode);
+ }
+ }
+ break;
}
return true;
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_whole_strings (struct sfm_reader *r, char *s, size_t length)
+read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
{
assert (length % 8 == 0);
if (!r->compressed)
static bool
skip_whole_strings (struct sfm_reader *r, size_t length)
{
- char buffer[1024];
+ uint8_t buffer[1024];
assert (length < sizeof buffer);
return read_whole_strings (r, buffer, length);
}
if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
return false;
- *value = text_get_token (text, ss_buffer ("\t\0", 2));
+ *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
if (*value == NULL)
return false;
}
}
+static bool
+text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text, struct substring delimiters,
+ struct variable **var)
+{
+ char *name;
+
+ name = text_get_token (text, delimiters, NULL);
+ if (name == NULL)
+ return false;
+
+ *var = dict_lookup_var (dict, name);
+ if (*var != NULL)
+ return true;
+
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
+ name);
+ return false;
+}
+
+
static bool
text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
struct text_record *text, struct substring delimiters,
struct variable **var)
{
- char *short_name = text_get_token (text, delimiters);
+ char *short_name = text_get_token (text, delimiters, NULL);
if (short_name == NULL)
return false;
*var = lookup_var_by_short_name (dict, short_name);
if (*var == NULL)
- text_warn (r, text, _("Variable map refers to unknown variable %s."),
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
short_name);
return true;
}
}
static char *
-text_get_token (struct text_record *text, struct substring delimiters)
+text_get_token (struct text_record *text, struct substring delimiters,
+ char *delimiter)
{
struct substring token;
+ char *end;
if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
return NULL;
- ss_data (token)[ss_length (token)] = '\0';
+
+ end = &ss_data (token)[ss_length (token)];
+ if (delimiter != NULL)
+ *delimiter = *end;
+ *end = '\0';
return ss_data (token);
}
+/* Reads a integer value expressed in decimal, then a space, then a string that
+ consists of exactly as many bytes as specified by the integer, then a space,
+ from TEXT. Returns the string, null-terminated, as a subset of TEXT's
+ buffer (so the caller should not free the string). */
+static const char *
+text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
+{
+ size_t start;
+ size_t n;
+ char *s;
+
+ start = text->pos;
+ n = 0;
+ for (;;)
+ {
+ int c = text->buffer.string[text->pos];
+ if (c < '0' || c > '9')
+ break;
+ n = (n * 10) + (c - '0');
+ text->pos++;
+ }
+ if (start == text->pos)
+ {
+ sys_warn (r, _("Expecting digit at offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
+
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, _("Expecting space at offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
+
+ if (text->pos + n > text->buffer.length)
+ {
+ sys_warn (r, _("%zu-byte string starting at offset %zu "
+ "exceeds record length %zu."),
+ n, text->pos, text->buffer.length);
+ return NULL;
+ }
+
+ s = &text->buffer.string[text->pos];
+ if (s[n] != ' ')
+ {
+ sys_warn (r,
+ _("Expecting space at offset %zu following %zu-byte string."),
+ text->pos + n, n);
+ return NULL;
+ }
+ s[n] = '\0';
+ text->pos += n + 1;
+ return s;
+}
+
static bool
text_match (struct text_record *text, char c)
{
else
return false;
}
+
+/* Returns the current byte offset inside the TEXT's string. */
+static size_t
+text_pos (const struct text_record *text)
+{
+ return text->pos;
+}
\f
/* Messages. */
struct string text;
ds_init_empty (&text);
- ds_put_format (&text, "\"%s\" near offset 0x%lx: ",
- fh_get_file_name (r->fh), (unsigned long) ftell (r->file));
+ ds_put_format (&text, "`%s' near offset 0x%llx: ",
+ fh_get_file_name (r->fh), (long long int) ftello (r->file));
ds_put_vformat (&text, format, args);
m.category = msg_class_to_category (class);
m.severity = msg_class_to_severity (class);
m.where.file_name = NULL;
m.where.line_number = 0;
+ m.where.first_column = 0;
+ m.where.last_column = 0;
m.text = ds_cstr (&text);
msg_emit (&m);
projects / pspp-builds.git / blobdiff