EXT_DATE = 6, /* DATE. */
EXT_MRSETS = 7, /* Multiple response sets. */
EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
- /* subtypes 9-10 unknown */
+ /* subtype 9 unknown */
+ EXT_PRODUCT_INFO = 10, /* Extra product info text. */
EXT_DISPLAY = 11, /* Variable display parameters. */
/* subtype 12 unknown */
EXT_LONG_NAMES = 13, /* Long variable names. */
EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
EXT_ENCODING = 20, /* Character encoding. */
EXT_LONG_LABELS = 21, /* Value labels for long strings. */
+ EXT_LONG_MISSING = 22, /* Missing values for long strings. */
EXT_DATAVIEW = 24 /* "Format properties in dataview table". */
};
struct sfm_extension_record
{
+ int subtype; /* Record subtype. */
off_t pos; /* Starting offset in file. */
size_t size; /* Size of data elements. */
size_t count; /* Number of data elements. */
static void read_string (struct sfm_reader *, char *, size_t);
static void skip_bytes (struct sfm_reader *, size_t);
+static char *fix_line_ends (const char *);
+
static int parse_int (struct sfm_reader *, const void *data, size_t ofs);
static double parse_float (struct sfm_reader *, const void *data, size_t ofs);
static const char *text_parse_counted_string (struct sfm_reader *,
struct text_record *);
static size_t text_pos (const struct text_record *);
+static const char *text_get_all (const struct text_record *);
static bool close_reader (struct sfm_reader *r);
\f
struct sfm_read_info *);
static void parse_machine_float_info (struct sfm_reader *,
const struct sfm_extension_record *);
+static void parse_extra_product_info (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct sfm_read_info *);
static void parse_mrsets (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
static void parse_long_string_value_labels (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
+static void parse_long_string_missing_values (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
/* Frees the strings inside INFO. */
void
free (info->creation_date);
free (info->creation_time);
free (info->product);
+ free (info->product_ext);
}
}
if (extensions[EXT_FLOAT] != NULL)
parse_machine_float_info (r, extensions[EXT_FLOAT]);
+ if (extensions[EXT_PRODUCT_INFO] != NULL)
+ parse_extra_product_info (r, extensions[EXT_PRODUCT_INFO], info);
+
if (extensions[EXT_FILE_ATTRS] != NULL)
parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict);
if (extensions[EXT_LONG_LABELS] != NULL)
parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict);
+ if (extensions[EXT_LONG_MISSING] != NULL)
+ parse_long_string_missing_values (r, extensions[EXT_LONG_MISSING], dict);
/* Warn if the actual amount of data per case differs from the
amount that the header claims. SPSS version 13 gets this
read_extension_record_header (struct sfm_reader *r, int subtype,
struct sfm_extension_record *record)
{
+ record->subtype = subtype;
record->pos = r->pos;
record->size = read_int (r);
record->count = read_int (r);
{ EXT_INTEGER, 4, 8 },
{ EXT_FLOAT, 8, 3 },
{ EXT_MRSETS, 1, 0 },
+ { EXT_PRODUCT_INFO, 1, 0 },
{ EXT_DISPLAY, 4, 0 },
{ EXT_LONG_NAMES, 1, 0 },
{ EXT_LONG_STRINGS, 1, 0 },
{ EXT_MRSETS2, 1, 0 },
{ EXT_ENCODING, 1, 0 },
{ EXT_LONG_LABELS, 1, 0 },
+ { EXT_LONG_MISSING, 1, 0 },
/* Ignored record types. */
{ EXT_VAR_SETS, 0, 0 },
const char *dict_encoding = dict_get_encoding (dict);
struct substring product;
struct substring label;
+ char *fixed_label;
/* Convert file label to UTF-8 and put it into DICT. */
label = recode_substring_pool ("UTF-8", dict_encoding,
ss_cstr (header->file_label), r->pool);
ss_trim (&label, ss_cstr (" "));
label.string[label.length] = '\0';
- dict_set_label (dict, label.string);
+ fixed_label = fix_line_ends (label.string);
+ dict_set_label (dict, fixed_label);
+ free (fixed_label);
/* Put creation date and time in UTF-8 into INFO. */
info->creation_date = recode_string ("UTF-8", dict_encoding,
value_init_pool (r->pool, &value, width);
value_set_missing (&value, width);
for (i = 0; i < rec->missing_value_code; i++)
- {
- uint8_t *s = value_str_rw (&value, width);
- memcpy (s, rec->missing + 8 * i, MIN (width, 8));
- mv_add_str (&mv, s);
- }
+ mv_add_str (&mv, rec->missing + 8 * i, MIN (width, 8));
}
var_set_missing_values (var, &mv);
}
lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS);
}
+/* Parses record type 7, subtype 10. */
+static void
+parse_extra_product_info (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct sfm_read_info *info)
+{
+ struct text_record *text;
+
+ text = open_text_record (r, record, true);
+ info->product_ext = fix_line_ends (text_get_all (text));
+ close_text_record (r, text);
+}
+
/* Parses record type 7, subtype 7 or 19. */
static void
parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record,
char **utf8_labels;
size_t i;
- utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels);
+ utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
for (i = 0; i < record->n_labels; i++)
utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
record->labels[i].label, -1,
break;
case 1:
- role = ROLE_OUTPUT;
+ role = ROLE_TARGET;
break;
case 2:
size_t end = record->size * record->count;
if (length >= end || ofs + length > end)
sys_error (r, record->pos + end,
- _("Long string value label record ends unexpectedly."));
+ _("Extension record subtype %d ends unexpectedly."),
+ record->subtype);
}
static void
var = dict_lookup_var (dict, var_name);
if (var == NULL)
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for "
+ _("Ignoring long string value label record for "
"unknown variable %s."), var_name);
else if (var_is_numeric (var))
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for "
+ _("Ignoring long string value label record for "
"numeric variable %s."), var_name);
var = NULL;
}
else if (width != var_get_width (var))
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for variable %s "
- "because the record's width (%d) does not match the "
+ _("Ignoring long string value label record for variable "
+ "%s because the record's width (%d) does not match the "
"variable's width (%d)."),
var_name, width, var_get_width (var));
var = NULL;
else
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value %zu for variable "
- "%s, with width %d, that has bad value "
+ _("Ignoring long string value label %zu for "
+ "variable %s, with width %d, that has bad value "
"width %zu."),
i, var_get_name (var), width, value_length);
skip = true;
}
}
}
+
+static void
+parse_long_string_missing_values (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ size_t end = record->size * record->count;
+ size_t ofs = 0;
+
+ while (ofs < end)
+ {
+ struct missing_values mv;
+ char *var_name;
+ struct variable *var;
+ int n_missing_values;
+ int var_name_len;
+ size_t i;
+
+ /* Parse variable name length. */
+ check_overflow (r, record, ofs, 4);
+ var_name_len = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse variable name. */
+ check_overflow (r, record, ofs, var_name_len + 1);
+ var_name = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ var_name_len, r->pool);
+ ofs += var_name_len;
+
+ /* Parse number of missing values. */
+ n_missing_values = ((const uint8_t *) record->data)[ofs];
+ if (n_missing_values < 1 || n_missing_values > 3)
+ sys_warn (r, record->pos + ofs,
+ _("Long string missing values record says variable %s "
+ "has %d missing values, but only 1 to 3 missing values "
+ "are allowed."),
+ var_name, n_missing_values);
+ ofs++;
+
+ /* Look up 'var' and validate. */
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "unknown variable %s."), var_name);
+ else if (var_is_numeric (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "numeric variable %s."), var_name);
+ var = NULL;
+ }
+
+ /* Parse values. */
+ mv_init_pool (r->pool, &mv, var ? var_get_width (var) : 8);
+ for (i = 0; i < n_missing_values; i++)
+ {
+ size_t value_length;
+
+ /* Parse value length. */
+ check_overflow (r, record, ofs, 4);
+ value_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse value. */
+ check_overflow (r, record, ofs, value_length);
+ if (var != NULL
+ && i < 3
+ && !mv_add_str (&mv, (const uint8_t *) record->data + ofs,
+ value_length))
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value %zu for variable "
+ "%s, with width %d, that has bad value width %zu."),
+ i, var_get_name (var), var_get_width (var),
+ value_length);
+ ofs += value_length;
+ }
+ if (var != NULL)
+ var_set_missing_values (var, &mv);
+ }
+}
\f
/* Case reader. */
{
return text->pos;
}
+
+static const char *
+text_get_all (const struct text_record *text)
+{
+ return text->buffer.string;
+}
\f
/* Messages. */
bytes -= chunk;
}
}
+
+/* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have
+ been replaced by LFs.
+
+ (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system
+ files that use CR-only line ends in the file label and extra product
+ info.) */
+static char *
+fix_line_ends (const char *s)
+{
+ char *dst, *d;
+
+ d = dst = xmalloc (strlen (s) + 1);
+ while (*s != '\0')
+ {
+ if (*s == '\r')
+ {
+ s++;
+ if (*s == '\n')
+ s++;
+ *d++ = '\n';
+ }
+ else
+ *d++ = *s++;
+ }
+ *d = '\0';
+
+ return dst;
+}
\f
static const struct casereader_class sys_file_casereader_class =
{
projects / pspp / blobdiff