/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-2000, 2006-2007, 2009-2011 Free Software Foundation, Inc.
+ Copyright (C) 1997-2000, 2006-2007, 2009-2013 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include "data/file-handle-def.h"
#include "data/file-name.h"
#include "data/format.h"
+#include "data/identifier.h"
#include "data/missing-values.h"
#include "data/mrset.h"
#include "data/short-names.h"
#include "libpspp/str.h"
#include "libpspp/stringi-set.h"
+#include "gl/c-strtod.h"
#include "gl/c-ctype.h"
#include "gl/inttostr.h"
#include "gl/localcharset.h"
EXT_DATE = 6, /* DATE. */
EXT_MRSETS = 7, /* Multiple response sets. */
EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
- /* subtypes 9-10 unknown */
+ /* subtype 9 unknown */
+ EXT_PRODUCT_INFO = 10, /* Extra product info text. */
EXT_DISPLAY = 11, /* Variable display parameters. */
/* subtype 12 unknown */
EXT_LONG_NAMES = 13, /* Long variable names. */
EXT_VAR_ATTRS = 18, /* Variable attributes. */
EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
EXT_ENCODING = 20, /* Character encoding. */
- EXT_LONG_LABELS = 21 /* Value labels for long strings. */
+ EXT_LONG_LABELS = 21, /* Value labels for long strings. */
+ EXT_LONG_MISSING = 22, /* Missing values for long strings. */
+ EXT_DATAVIEW = 24 /* "Format properties in dataview table". */
+ };
+
+/* Fields from the top-level header record. */
+struct sfm_header_record
+ {
+ char magic[5]; /* First 4 bytes of file, then null. */
+ int weight_idx; /* 0 if unweighted, otherwise a var index. */
+ int nominal_case_size; /* Number of var positions. */
+
+ /* These correspond to the members of struct sfm_file_info or a dictionary
+ but in the system file's encoding rather than ASCII. */
+ char creation_date[10]; /* "dd mmm yy". */
+ char creation_time[9]; /* "hh:mm:ss". */
+ char eye_catcher[61]; /* Eye-catcher string, then product name. */
+ char file_label[65]; /* File label. */
};
struct sfm_var_record
struct sfm_extension_record
{
+ int subtype; /* Record subtype. */
off_t pos; /* Starting offset in file. */
size_t size; /* Size of data elements. */
size_t count; /* Number of data elements. */
static void read_string (struct sfm_reader *, char *, size_t);
static void skip_bytes (struct sfm_reader *, size_t);
+static char *fix_line_ends (const char *);
+
static int parse_int (struct sfm_reader *, const void *data, size_t ofs);
static double parse_float (struct sfm_reader *, const void *data, size_t ofs);
static const char *choose_encoding (
struct sfm_reader *,
+ const struct sfm_header_record *,
const struct sfm_extension_record *ext_integer,
const struct sfm_extension_record *ext_encoding);
static struct text_record *open_text_record (
- struct sfm_reader *, const struct sfm_extension_record *);
+ struct sfm_reader *, const struct sfm_extension_record *,
+ bool recode_to_utf8);
static void close_text_record (struct sfm_reader *,
struct text_record *);
static bool read_variable_to_value_pair (struct sfm_reader *,
static const char *text_parse_counted_string (struct sfm_reader *,
struct text_record *);
static size_t text_pos (const struct text_record *);
+static const char *text_get_all (const struct text_record *);
static bool close_reader (struct sfm_reader *r);
\f
WRITE_FORMAT
};
-static void read_header (struct sfm_reader *, int *weight_idx,
- int *claimed_oct_cnt, struct sfm_read_info *,
- char **file_labelp);
-static void parse_file_label (struct sfm_reader *, const char *file_label,
- struct dictionary *);
+static void read_header (struct sfm_reader *, struct sfm_read_info *,
+ struct sfm_header_record *);
+static void parse_header (struct sfm_reader *,
+ const struct sfm_header_record *,
+ struct sfm_read_info *, struct dictionary *);
static void parse_variable_records (struct sfm_reader *, struct dictionary *,
struct sfm_var_record *, size_t n);
static void parse_format_spec (struct sfm_reader *, off_t pos,
struct sfm_read_info *);
static void parse_machine_float_info (struct sfm_reader *,
const struct sfm_extension_record *);
+static void parse_extra_product_info (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct sfm_read_info *);
static void parse_mrsets (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
static void parse_variable_attributes (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
+static void assign_variable_roles (struct sfm_reader *, struct dictionary *);
static void parse_long_string_value_labels (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
+static void parse_long_string_missing_values (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
+
+/* Frees the strings inside INFO. */
+void
+sfm_read_info_destroy (struct sfm_read_info *info)
+{
+ if (info)
+ {
+ free (info->creation_date);
+ free (info->creation_time);
+ free (info->product);
+ free (info->product_ext);
+ }
+}
+
+/* Opens the system file designated by file handle FH for reading. Reads the
+ system file's dictionary into *DICT.
-/* Opens the system file designated by file handle FH for
- reading. Reads the system file's dictionary into *DICT.
- If INFO is non-null, then it receives additional info about the
- system file. */
+ Ordinarily the reader attempts to automatically detect the character
+ encoding based on the file's contents. This isn't always possible,
+ especially for files written by old versions of SPSS or PSPP, so specifying
+ a nonnull ENCODING overrides the choice of character encoding.
+
+ If INFO is non-null, then it receives additional info about the system file,
+ which the caller must eventually free with sfm_read_info_destroy() when it
+ is no longer needed. */
struct casereader *
-sfm_open_reader (struct file_handle *fh, struct dictionary **dictp,
- struct sfm_read_info *volatile info)
+sfm_open_reader (struct file_handle *fh, const char *volatile encoding,
+ struct dictionary **dictp, struct sfm_read_info *infop)
{
struct sfm_reader *volatile r = NULL;
- struct sfm_read_info local_info;
+ struct sfm_read_info *volatile info;
+
+ struct sfm_header_record header;
struct sfm_var_record *vars;
size_t n_vars, allocated_vars;
struct sfm_extension_record *extensions[32];
- int weight_idx;
- int claimed_oct_cnt;
- char *file_label;
-
- struct dictionary *dict = NULL;
+ struct dictionary *volatile dict = NULL;
size_t i;
/* Create and initialize reader. */
r->opcode_idx = sizeof r->opcodes;
r->corruption_warning = false;
+ info = infop ? infop : xmalloc (sizeof *info);
+ memset (info, 0, sizeof *info);
+
/* TRANSLATORS: this fragment will be interpolated into
messages in fh_lock() that identify types of files. */
r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false);
goto error;
}
- /* Initialize info. */
- if (info == NULL)
- info = &local_info;
- memset (info, 0, sizeof *info);
-
if (setjmp (r->bail_out))
goto error;
/* Read header. */
- read_header (r, &weight_idx, &claimed_oct_cnt, info, &file_label);
+ read_header (r, info, &header);
vars = NULL;
n_vars = allocated_vars = 0;
First, figure out the correct character encoding, because this determines
how the rest of the header data is to be interpreted. */
- dict = dict_create ();
- r->encoding = choose_encoding (r, extensions[EXT_INTEGER],
- extensions[EXT_ENCODING]);
- dict_set_encoding (dict, r->encoding);
+ dict = dict_create (encoding
+ ? encoding
+ : choose_encoding (r, &header, extensions[EXT_INTEGER],
+ extensions[EXT_ENCODING]));
+ r->encoding = dict_get_encoding (dict);
/* These records don't use variables at all. */
if (document != NULL)
if (extensions[EXT_FLOAT] != NULL)
parse_machine_float_info (r, extensions[EXT_FLOAT]);
+ if (extensions[EXT_PRODUCT_INFO] != NULL)
+ parse_extra_product_info (r, extensions[EXT_PRODUCT_INFO], info);
+
if (extensions[EXT_FILE_ATTRS] != NULL)
parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict);
- parse_file_label (r, file_label, dict);
+ parse_header (r, &header, info, dict);
/* Parse the variable records, the basis of almost everything else. */
parse_variable_records (r, dict, vars, n_vars);
before those indexes become invalidated by very long string variables. */
for (i = 0; i < n_labels; i++)
parse_value_labels (r, dict, vars, n_vars, &labels[i]);
- if (weight_idx != 0)
+ if (header.weight_idx != 0)
{
struct variable *weight_var;
- weight_var = lookup_var_by_index (r, 76, vars, n_vars, weight_idx);
+ weight_var = lookup_var_by_index (r, 76, vars, n_vars,
+ header.weight_idx);
if (var_is_numeric (weight_var))
dict_set_weight (dict, weight_var);
else
/* The following records use long names, so they need to follow renaming. */
if (extensions[EXT_VAR_ATTRS] != NULL)
- parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict);
+ {
+ parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict);
+
+ /* Roles use the $@Role attribute. */
+ assign_variable_roles (r, dict);
+ }
if (extensions[EXT_LONG_LABELS] != NULL)
parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict);
+ if (extensions[EXT_LONG_MISSING] != NULL)
+ parse_long_string_missing_values (r, extensions[EXT_LONG_MISSING], dict);
/* Warn if the actual amount of data per case differs from the
amount that the header claims. SPSS version 13 gets this
wrong when very long strings are involved, so don't warn in
that case. */
- if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars
+ if (header.nominal_case_size != -1 && header.nominal_case_size != n_vars
&& info->version_major != 13)
sys_warn (r, -1, _("File header claims %d variable positions but "
- "%d were read from file."),
- claimed_oct_cnt, n_vars);
+ "%zu were read from file."),
+ header.nominal_case_size, n_vars);
/* Create an index of dictionary variable widths for
sfm_read_case to use. We cannot use the `struct variable's
r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
*dictp = dict;
+ if (infop != info)
+ {
+ sfm_read_info_destroy (info);
+ free (info);
+ }
+
return casereader_create_sequential
(NULL, r->proto,
r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
&sys_file_casereader_class, r);
error:
+ if (infop != info)
+ {
+ sfm_read_info_destroy (info);
+ free (info);
+ }
+
close_reader (r);
dict_destroy (dict);
*dictp = NULL;
bool
sfm_detect (FILE *file)
{
- char rec_type[5];
+ char magic[5];
- if (fread (rec_type, 4, 1, file) != 1)
+ if (fread (magic, 4, 1, file) != 1)
return false;
- rec_type[4] = '\0';
+ magic[4] = '\0';
- return !strcmp ("$FL2", rec_type);
+ return !strcmp (ASCII_MAGIC, magic) || !strcmp (EBCDIC_MAGIC, magic);
}
\f
-/* Reads the global header of the system file. Sets *WEIGHT_IDX to 0 if the
- system file is unweighted, or to the value index of the weight variable
- otherwise. Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units) per
- case that the file claims to have (although it is not always correct).
- Initializes INFO with header information. Stores the file label as a string
- in dictionary encoding into *FILE_LABELP. */
+/* Reads the global header of the system file. Initializes *HEADER and *INFO,
+ except for the string fields in *INFO, which parse_header() will initialize
+ later once the file's encoding is known. */
static void
-read_header (struct sfm_reader *r, int *weight_idx,
- int *claimed_oct_cnt, struct sfm_read_info *info,
- char **file_labelp)
+read_header (struct sfm_reader *r, struct sfm_read_info *info,
+ struct sfm_header_record *header)
{
- char rec_type[5];
- char eye_catcher[61];
uint8_t raw_layout_code[4];
uint8_t raw_bias[8];
- char creation_date[10];
- char creation_time[9];
- char file_label[65];
- struct substring product;
- read_string (r, rec_type, sizeof rec_type);
- read_string (r, eye_catcher, sizeof eye_catcher);
+ read_string (r, header->magic, sizeof header->magic);
+ read_string (r, header->eye_catcher, sizeof header->eye_catcher);
- if (strcmp ("$FL2", rec_type) != 0)
+ if (strcmp (ASCII_MAGIC, header->magic)
+ && strcmp (EBCDIC_MAGIC, header->magic))
sys_error (r, 0, _("This is not an SPSS system file."));
/* Identify integer format. */
&& r->integer_format != INTEGER_LSB_FIRST))
sys_error (r, 64, _("This is not an SPSS system file."));
- *claimed_oct_cnt = read_int (r);
- if (*claimed_oct_cnt < 0 || *claimed_oct_cnt > INT_MAX / 16)
- *claimed_oct_cnt = -1;
+ header->nominal_case_size = read_int (r);
+ if (header->nominal_case_size < 0
+ || header->nominal_case_size > INT_MAX / 16)
+ header->nominal_case_size = -1;
r->compressed = read_int (r) != 0;
- *weight_idx = read_int (r);
+ header->weight_idx = read_int (r);
r->case_cnt = read_int (r);
if ( r->case_cnt > INT_MAX / 2)
}
float_convert (r->float_format, raw_bias, FLOAT_NATIVE_DOUBLE, &r->bias);
- read_string (r, creation_date, sizeof creation_date);
- read_string (r, creation_time, sizeof creation_time);
- read_string (r, file_label, sizeof file_label);
+ read_string (r, header->creation_date, sizeof header->creation_date);
+ read_string (r, header->creation_time, sizeof header->creation_time);
+ read_string (r, header->file_label, sizeof header->file_label);
skip_bytes (r, 3);
- strcpy (info->creation_date, creation_date);
- strcpy (info->creation_time, creation_time);
info->integer_format = r->integer_format;
info->float_format = r->float_format;
info->compressed = r->compressed;
info->case_cnt = r->case_cnt;
-
- product = ss_cstr (eye_catcher);
- ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE"));
- ss_trim (&product, ss_cstr (" "));
- str_copy_buf_trunc (info->product, sizeof info->product,
- ss_data (product), ss_length (product));
-
- *file_labelp = pool_strdup0 (r->pool, file_label, sizeof file_label - 1);
}
/* Reads a variable (type 2) record from R into RECORD. */
/* Read up to MAX_LABEL_LEN bytes of label. */
read_len = MIN (MAX_LABEL_LEN, len);
- record->label = xmalloc (read_len + 1);
+ record->label = pool_malloc (r->pool, read_len + 1);
read_string (r, record->label, read_len + 1);
/* Skip unread label bytes. */
record->n_vars = read_int (r);
if (record->n_vars < 1 || record->n_vars > n_vars)
sys_error (r, r->pos - 4,
- _("Number of variables associated with a value label (%d) "
+ _("Number of variables associated with a value label (%zu) "
"is not between 1 and the number of variables (%zu)."),
record->n_vars, n_vars);
record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
read_extension_record_header (struct sfm_reader *r, int subtype,
struct sfm_extension_record *record)
{
+ record->subtype = subtype;
record->pos = r->pos;
record->size = read_int (r);
record->count = read_int (r);
{ EXT_INTEGER, 4, 8 },
{ EXT_FLOAT, 8, 3 },
{ EXT_MRSETS, 1, 0 },
+ { EXT_PRODUCT_INFO, 1, 0 },
{ EXT_DISPLAY, 4, 0 },
{ EXT_LONG_NAMES, 1, 0 },
{ EXT_LONG_STRINGS, 1, 0 },
{ EXT_MRSETS2, 1, 0 },
{ EXT_ENCODING, 1, 0 },
{ EXT_LONG_LABELS, 1, 0 },
+ { EXT_LONG_MISSING, 1, 0 },
/* Ignored record types. */
{ EXT_VAR_SETS, 0, 0 },
{ EXT_DATE, 0, 0 },
{ EXT_DATA_ENTRY, 0, 0 },
+ { EXT_DATAVIEW, 0, 0 },
};
const struct extension_record_type *type;
}
static void
-parse_file_label (struct sfm_reader *r, const char *file_label,
- struct dictionary *dict)
+parse_header (struct sfm_reader *r, const struct sfm_header_record *header,
+ struct sfm_read_info *info, struct dictionary *dict)
{
- char *utf8_file_label;
- size_t file_label_len;
-
- utf8_file_label = recode_string_pool ("UTF-8", dict_get_encoding (dict),
- file_label, -1, r->pool);
- file_label_len = strlen (utf8_file_label);
- while (file_label_len > 0 && utf8_file_label[file_label_len - 1] == ' ')
- file_label_len--;
- utf8_file_label[file_label_len] = '\0';
- dict_set_label (dict, utf8_file_label);
+ const char *dict_encoding = dict_get_encoding (dict);
+ struct substring product;
+ struct substring label;
+ char *fixed_label;
+
+ /* Convert file label to UTF-8 and put it into DICT. */
+ label = recode_substring_pool ("UTF-8", dict_encoding,
+ ss_cstr (header->file_label), r->pool);
+ ss_trim (&label, ss_cstr (" "));
+ label.string[label.length] = '\0';
+ fixed_label = fix_line_ends (label.string);
+ dict_set_label (dict, fixed_label);
+ free (fixed_label);
+
+ /* Put creation date and time in UTF-8 into INFO. */
+ info->creation_date = recode_string ("UTF-8", dict_encoding,
+ header->creation_date, -1);
+ info->creation_time = recode_string ("UTF-8", dict_encoding,
+ header->creation_time, -1);
+
+ /* Put product name into INFO, dropping eye-catcher string if present. */
+ product = recode_substring_pool ("UTF-8", dict_encoding,
+ ss_cstr (header->eye_catcher), r->pool);
+ ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE"));
+ ss_trim (&product, ss_cstr (" "));
+ info->product = ss_xstrdup (product);
}
/* Reads a variable (type 2) record from R and adds the
rec->name, 8, r->pool);
name[strcspn (name, " ")] = '\0';
- if (!var_is_valid_name (name, false) || name[0] == '$' || name[0] == '#')
+ if (!dict_id_is_valid (dict, name, false)
+ || name[0] == '$' || name[0] == '#')
sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
if (rec->width < 0 || rec->width > 255)
utf8_label = recode_string_pool ("UTF-8", dict_encoding,
rec->label, -1, r->pool);
- var_set_label (var, utf8_label);
+ var_set_label (var, utf8_label, false);
}
/* Set missing values. */
{
double low = parse_float (r, rec->missing, 0);
double high = parse_float (r, rec->missing, 8);
+
+ /* Deal with SPSS 21 change in representation. */
+ if (low == SYSMIS)
+ low = LOWEST;
+
mv_add_range (&mv, low, high);
ofs += 16;
}
value_init_pool (r->pool, &value, width);
value_set_missing (&value, width);
for (i = 0; i < rec->missing_value_code; i++)
- {
- uint8_t *s = value_str_rw (&value, width);
- memcpy (s, rec->missing + 8 * i, MIN (width, 8));
- mv_add_str (&mv, s);
- }
+ mv_add_str (&mv, rec->missing + 8 * i, MIN (width, 8));
}
var_set_missing_values (var, &mv);
}
uint8_t w = format >> 8;
uint8_t d = format;
struct fmt_spec f;
-
bool ok;
- if (!fmt_from_io (raw_type, &f.type))
- sys_error (r, pos, _("Unknown variable format %"PRIu8"."), raw_type);
f.w = w;
f.d = d;
msg_disable ();
- ok = fmt_check_output (&f) && fmt_check_width_compat (&f, var_get_width (v));
+ ok = (fmt_from_io (raw_type, &f.type)
+ && fmt_check_output (&f)
+ && fmt_check_width_compat (&f, var_get_width (v)));
msg_enable ();
if (ok)
else
var_set_write_format (v, &f);
}
+ else if (format == 0)
+ {
+ /* Actually observed in the wild. No point in warning about it. */
+ }
else if (++*n_warnings <= max_warnings)
{
- char fmt_string[FMT_STRING_LEN_MAX + 1];
- sys_warn (r, pos, _("%s variable %s has invalid %s format %s."),
- var_is_numeric (v) ? _("Numeric") : _("String"),
- var_get_name (v),
- which == PRINT_FORMAT ? _("print") : _("write"),
- fmt_to_string (&f, fmt_string));
+ if (which == PRINT_FORMAT)
+ sys_warn (r, pos, _("Variable %s with width %d has invalid print "
+ "format 0x%x."),
+ var_get_name (v), var_get_width (v), format);
+ else
+ sys_warn (r, pos, _("Variable %s with width %d has invalid write "
+ "format 0x%x."),
+ var_get_name (v), var_get_width (v), format);
if (*n_warnings == max_warnings)
sys_warn (r, -1, _("Suppressing further invalid format warnings."));
ss_rtrim (&line, ss_cstr (" "));
line.string[line.length] = '\0';
- dict_add_document_line (dict, line.string);
+ dict_add_document_line (dict, line.string, false);
ss_dealloc (&line);
}
static const char *
choose_encoding (struct sfm_reader *r,
+ const struct sfm_header_record *header,
const struct sfm_extension_record *ext_integer,
const struct sfm_extension_record *ext_encoding)
{
if (ext_integer)
{
int codepage = parse_int (r, ext_integer->data, 7 * 4);
+ const char *encoding;
switch (codepage)
{
case 4:
return "MS_KANJI";
- case 65000:
- return "UTF-7";
-
- case 65001:
- return "UTF-8";
-
default:
- return pool_asprintf (r->pool, "CP%d", codepage);
+ encoding = sys_get_encoding_from_codepage (codepage);
+ if (encoding != NULL)
+ return encoding;
+ break;
}
}
+ /* If the file magic number is EBCDIC then its character data is too. */
+ if (!strcmp (header->magic, EBCDIC_MAGIC))
+ return "EBCDIC-US";
+
return locale_charset ();
}
double lowest = parse_float (r, record->data, 16);
if (sysmis != SYSMIS)
- sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
- sysmis, "SYSMIS");
+ sys_warn (r, record->pos,
+ _("File specifies unexpected value %g (%a) as %s, "
+ "instead of %g (%a)."),
+ sysmis, sysmis, "SYSMIS", SYSMIS, SYSMIS);
if (highest != HIGHEST)
- sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
- highest, "HIGHEST");
+ sys_warn (r, record->pos,
+ _("File specifies unexpected value %g (%a) as %s, "
+ "instead of %g (%a)."),
+ highest, highest, "HIGHEST", HIGHEST, HIGHEST);
+
+ /* SPSS before version 21 used a unique value just bigger than SYSMIS as
+ LOWEST. SPSS 21 uses SYSMIS for LOWEST, which is OK because LOWEST only
+ appears in a context (missing values) where SYSMIS cannot. */
+ if (lowest != LOWEST && lowest != SYSMIS)
+ sys_warn (r, record->pos,
+ _("File specifies unexpected value %g (%a) as %s, "
+ "instead of %g (%a) or %g (%a)."),
+ lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS);
+}
+
+/* Parses record type 7, subtype 10. */
+static void
+parse_extra_product_info (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct sfm_read_info *info)
+{
+ struct text_record *text;
- if (lowest != LOWEST)
- sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
- lowest, "LOWEST");
+ text = open_text_record (r, record, true);
+ info->product_ext = fix_line_ends (text_get_all (text));
+ close_text_record (r, text);
}
/* Parses record type 7, subtype 7 or 19. */
struct text_record *text;
struct mrset *mrset;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, false);
for (;;)
{
const char *counted = NULL;
name = text_get_token (text, ss_cstr ("="), NULL);
if (name == NULL)
break;
- mrset->name = xstrdup (name);
+ mrset->name = recode_string ("UTF-8", r->encoding, name, -1);
if (mrset->name[0] != '$')
{
sys_warn (r, record->pos,
- _("`%s' does not begin with `$' at UTF-8 offset %zu "
+ _("`%s' does not begin with `$' at offset %zu "
"in MRSETS record."), mrset->name, text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'C', text_pos (text));
break;
}
if (!text_match (text, ' '))
{
sys_warn (r, record->pos,
- _("Missing space following `%c' at UTF-8 offset %zu "
+ _("Missing space following `%c' at offset %zu "
"in MRSETS record."), 'E', text_pos (text));
break;
}
else if (strcmp (number, "1"))
sys_warn (r, record->pos,
_("Unexpected label source value `%s' following `E' "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
number, text_pos (text));
}
else
{
sys_warn (r, record->pos,
- _("Missing `C', `D', or `E' at UTF-8 offset %zu "
+ _("Missing `C', `D', or `E' at offset %zu "
"in MRSETS record."),
text_pos (text));
break;
label = text_parse_counted_string (r, text);
if (label == NULL)
break;
- mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
+ if (label[0] != '\0')
+ mrset->label = recode_string ("UTF-8", r->encoding, label, -1);
stringi_set_init (&var_names);
allocated_vars = 0;
width = INT_MAX;
do
{
+ const char *raw_var_name;
struct variable *var;
- const char *var_name;
+ char *var_name;
- var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
- if (var_name == NULL)
+ raw_var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
+ if (raw_var_name == NULL)
{
sys_warn (r, record->pos,
_("Missing new-line parsing variable names "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
text_pos (text));
break;
}
+ var_name = recode_string ("UTF-8", r->encoding, raw_var_name, -1);
var = dict_lookup_var (dict, var_name);
if (var == NULL)
- continue;
+ {
+ free (var_name);
+ continue;
+ }
if (!stringi_set_insert (&var_names, var_name))
{
sys_warn (r, record->pos,
_("Duplicate variable name %s "
- "at UTF-8 offset %zu in MRSETS record."),
+ "at offset %zu in MRSETS record."),
var_name, text_pos (text));
+ free (var_name);
continue;
}
+ free (var_name);
if (mrset->label == NULL && mrset->label_from_var_label
&& var_has_label (var))
_("MRSET %s has only %zu variables."), mrset->name,
mrset->n_vars);
mrset_destroy (mrset);
+ stringi_set_destroy (&var_names);
continue;
}
mrset->width = width;
value_init (&mrset->counted, width);
if (width == 0)
- mrset->counted.f = strtod (counted, NULL);
+ mrset->counted.f = c_strtod (counted, NULL);
else
value_copy_str_rpad (&mrset->counted, width,
(const uint8_t *) counted, ' ');
align = parse_int (r, record->data, ofs);
ofs += 4;
- /* SPSS 14 sometimes seems to set string variables' measure
- to zero. */
- if (0 == measure && var_is_alpha (v))
+ /* SPSS sometimes seems to set variables' measure to zero. */
+ if (0 == measure)
measure = 1;
if (measure < 1 || measure > 3 || align < 0 || align > 2)
if (record == NULL)
{
- /* Convert variable names to lowercase. */
+ /* There are no long variable names. Use the short variable names,
+ converted to lowercase, as the long variable names. */
size_t i;
for (i = 0; i < dict_get_var_cnt (dict); i++)
struct variable *var = dict_get_var (dict, i);
char *new_name;
- new_name = xstrdup (var_get_name (var));
- str_lowercase (new_name);
-
+ new_name = utf8_to_lower (var_get_name (var));
rename_var_and_save_short_names (dict, var, new_name);
-
free (new_name);
}
system file, this cannot create any intermediate duplicate variable names,
because all of the new variable names are longer than any of the old
variable names and thus there cannot be any overlaps.) */
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
{
/* Validate long name. */
- if (!var_is_valid_name (long_name, false))
+ if (!dict_id_is_valid (dict, long_name, false))
{
sys_warn (r, record->pos,
_("Long variable mapping from %s to invalid "
}
/* Identify any duplicates. */
- if (strcasecmp (var_get_short_name (var, 0), long_name)
+ if (utf8_strcasecmp (var_get_short_name (var, 0), long_name)
&& dict_lookup_var (dict, long_name) != NULL)
{
sys_warn (r, record->pos,
struct variable *var;
char *length_s;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
{
size_t idx = var_get_dict_index (var);
char **utf8_labels;
size_t i;
- utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels);
+ utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
for (i = 0; i < record->n_labels; i++)
utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
record->labels[i].label, -1,
if (idx < 1 || idx > n_var_recs)
{
sys_error (r, offset,
- _("Variable index %d not in valid range 1...%d."),
+ _("Variable index %d not in valid range 1...%zu."),
idx, n_var_recs);
return NULL;
}
const struct sfm_extension_record *record,
struct dictionary *dict)
{
- struct text_record *text = open_text_record (r, record);
+ struct text_record *text = open_text_record (r, record, true);
parse_attributes (r, text, dict_get_attributes (dict));
close_text_record (r, text);
}
struct text_record *text;
struct variable *var;
- text = open_text_record (r, record);
+ text = open_text_record (r, record, true);
while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
close_text_record (r, text);
}
+static void
+assign_variable_roles (struct sfm_reader *r, struct dictionary *dict)
+{
+ size_t n_warnings = 0;
+ size_t i;
+
+ for (i = 0; i < dict_get_var_cnt (dict); i++)
+ {
+ struct variable *var = dict_get_var (dict, i);
+ struct attrset *attrs = var_get_attributes (var);
+ const struct attribute *attr = attrset_lookup (attrs, "$@Role");
+ if (attr != NULL)
+ {
+ int value = atoi (attribute_get_value (attr, 0));
+ enum var_role role;
+
+ switch (value)
+ {
+ case 0:
+ role = ROLE_INPUT;
+ break;
+
+ case 1:
+ role = ROLE_TARGET;
+ break;
+
+ case 2:
+ role = ROLE_BOTH;
+ break;
+
+ case 3:
+ role = ROLE_NONE;
+ break;
+
+ case 4:
+ role = ROLE_PARTITION;
+ break;
+
+ case 5:
+ role = ROLE_SPLIT;
+ break;
+
+ default:
+ role = ROLE_INPUT;
+ if (n_warnings++ == 0)
+ sys_warn (r, -1, _("Invalid role for variable %s."),
+ var_get_name (var));
+ }
+
+ var_set_role (var, role);
+ }
+ }
+
+ if (n_warnings > 1)
+ sys_warn (r, -1, _("%zu other variables had invalid roles."),
+ n_warnings - 1);
+}
+
static void
check_overflow (struct sfm_reader *r,
const struct sfm_extension_record *record,
size_t end = record->size * record->count;
if (length >= end || ofs + length > end)
sys_error (r, record->pos + end,
- _("Long string value label record ends unexpectedly."));
+ _("Extension record subtype %d ends unexpectedly."),
+ record->subtype);
}
static void
var = dict_lookup_var (dict, var_name);
if (var == NULL)
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for "
+ _("Ignoring long string value label record for "
"unknown variable %s."), var_name);
else if (var_is_numeric (var))
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for "
+ _("Ignoring long string value label record for "
"numeric variable %s."), var_name);
var = NULL;
}
else if (width != var_get_width (var))
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value record for variable %s "
- "because the record's width (%d) does not match the "
+ _("Ignoring long string value label record for variable "
+ "%s because the record's width (%d) does not match the "
"variable's width (%d)."),
var_name, width, var_get_width (var));
var = NULL;
else
{
sys_warn (r, record->pos + ofs,
- _("Ignoring long string value %zu for variable "
- "%s, with width %d, that has bad value "
+ _("Ignoring long string value label %zu for "
+ "variable %s, with width %d, that has bad value "
"width %zu."),
i, var_get_name (var), width, value_length);
skip = true;
}
}
}
+
+static void
+parse_long_string_missing_values (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ size_t end = record->size * record->count;
+ size_t ofs = 0;
+
+ while (ofs < end)
+ {
+ struct missing_values mv;
+ char *var_name;
+ struct variable *var;
+ int n_missing_values;
+ int var_name_len;
+ size_t i;
+
+ /* Parse variable name length. */
+ check_overflow (r, record, ofs, 4);
+ var_name_len = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse variable name. */
+ check_overflow (r, record, ofs, var_name_len + 1);
+ var_name = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ var_name_len, r->pool);
+ ofs += var_name_len;
+
+ /* Parse number of missing values. */
+ n_missing_values = ((const uint8_t *) record->data)[ofs];
+ if (n_missing_values < 1 || n_missing_values > 3)
+ sys_warn (r, record->pos + ofs,
+ _("Long string missing values record says variable %s "
+ "has %d missing values, but only 1 to 3 missing values "
+ "are allowed."),
+ var_name, n_missing_values);
+ ofs++;
+
+ /* Look up 'var' and validate. */
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "unknown variable %s."), var_name);
+ else if (var_is_numeric (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "numeric variable %s."), var_name);
+ var = NULL;
+ }
+
+ /* Parse values. */
+ mv_init_pool (r->pool, &mv, var ? var_get_width (var) : 8);
+ for (i = 0; i < n_missing_values; i++)
+ {
+ size_t value_length;
+
+ /* Parse value length. */
+ check_overflow (r, record, ofs, 4);
+ value_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse value. */
+ check_overflow (r, record, ofs, value_length);
+ if (var != NULL
+ && i < 3
+ && !mv_add_str (&mv, (const uint8_t *) record->data + ofs,
+ value_length))
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value %zu for variable "
+ "%s, with width %d, that has bad value width %zu."),
+ i, var_get_name (var), var_get_width (var),
+ value_length);
+ ofs += value_length;
+ }
+ if (var != NULL)
+ var_set_missing_values (var, &mv);
+ }
+}
\f
/* Case reader. */
/* State. */
struct text_record
{
- struct substring buffer; /* Record contents, in UTF-8. */
+ struct substring buffer; /* Record contents. */
off_t start; /* Starting offset in file. */
size_t pos; /* Current position in buffer. */
int n_warnings; /* Number of warnings issued or suppressed. */
+ bool recoded; /* Recoded into UTF-8? */
};
static struct text_record *
open_text_record (struct sfm_reader *r,
- const struct sfm_extension_record *record)
+ const struct sfm_extension_record *record,
+ bool recode_to_utf8)
{
struct text_record *text;
struct substring raw;
text = pool_alloc (r->pool, sizeof *text);
raw = ss_buffer (record->data, record->size * record->count);
text->start = record->pos;
- text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool);
+ text->buffer = (recode_to_utf8
+ ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
+ : raw);
text->pos = 0;
text->n_warnings = 0;
+ text->recoded = recode_to_utf8;
return text;
}
if (text->n_warnings > MAX_TEXT_WARNINGS)
sys_warn (r, -1, _("Suppressed %d additional related warnings."),
text->n_warnings - MAX_TEXT_WARNINGS);
- pool_free (r->pool, ss_data (text->buffer));
+ if (text->recoded)
+ pool_free (r->pool, ss_data (text->buffer));
}
/* Reads a variable=value pair from TEXT.
start = text->pos;
n = 0;
- for (;;)
+ while (text->pos < text->buffer.length)
{
int c = text->buffer.string[text->pos];
if (c < '0' || c > '9')
n = (n * 10) + (c - '0');
text->pos++;
}
- if (start == text->pos)
+ if (text->pos >= text->buffer.length || start == text->pos)
{
sys_warn (r, text->start,
- _("Expecting digit at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting digit at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (!text_match (text, ' '))
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu in MRSETS record."),
+ _("Expecting space at offset %zu in MRSETS record."),
text->pos);
return NULL;
}
if (text->pos + n > text->buffer.length)
{
sys_warn (r, text->start,
- _("%zu-byte string starting at UTF-8 offset %zu "
+ _("%zu-byte string starting at offset %zu "
"exceeds record length %zu."),
n, text->pos, text->buffer.length);
return NULL;
if (s[n] != ' ')
{
sys_warn (r, text->start,
- _("Expecting space at UTF-8 offset %zu following %zu-byte "
- "string."),
+ _("Expecting space at offset %zu following %zu-byte string."),
text->pos + n, n);
return NULL;
}
return false;
}
-/* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's
- string. */
+/* Returns the current byte offset (as converted to UTF-8, if it was converted)
+ inside the TEXT's string. */
static size_t
text_pos (const struct text_record *text)
{
return text->pos;
}
+
+static const char *
+text_get_all (const struct text_record *text)
+{
+ return text->buffer.string;
+}
\f
/* Messages. */
m.category = msg_class_to_category (class);
m.severity = msg_class_to_severity (class);
- m.where.file_name = NULL;
- m.where.line_number = 0;
- m.where.first_column = 0;
- m.where.last_column = 0;
+ m.file_name = NULL;
+ m.first_line = 0;
+ m.last_line = 0;
+ m.first_column = 0;
+ m.last_column = 0;
m.text = ds_cstr (&text);
msg_emit (&m);
bytes -= chunk;
}
}
+
+/* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have
+ been replaced by LFs.
+
+ (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system
+ files that use CR-only line ends in the file label and extra product
+ info.) */
+static char *
+fix_line_ends (const char *s)
+{
+ char *dst, *d;
+
+ d = dst = xmalloc (strlen (s) + 1);
+ while (*s != '\0')
+ {
+ if (*s == '\r')
+ {
+ s++;
+ if (*s == '\n')
+ s++;
+ *d++ = '\n';
+ }
+ else
+ *d++ = *s++;
+ }
+ *d = '\0';
+
+ return dst;
+}
\f
static const struct casereader_class sys_file_casereader_class =
{
projects / pspp / blobdiff