/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-2000, 2006-2007, 2009-2014 Free Software Foundation, Inc.
+ Copyright (C) 1997-2000, 2006-2007, 2009-2016, 2021 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
-#include "data/sys-file-reader.h"
#include "data/sys-file-private.h"
#include <errno.h>
#include <sys/stat.h>
#include <zlib.h>
+#include "data/any-reader.h"
#include "data/attributes.h"
#include "data/case.h"
#include "data/casereader-provider.h"
#include "libpspp/assertion.h"
#include "libpspp/compiler.h"
#include "libpspp/i18n.h"
+#include "libpspp/ll.h"
#include "libpspp/message.h"
#include "libpspp/misc.h"
#include "libpspp/pool.h"
int weight_idx; /* 0 if unweighted, otherwise a var index. */
int nominal_case_size; /* Number of var positions. */
- /* These correspond to the members of struct sfm_file_info or a dictionary
+ /* These correspond to the members of struct any_file_info or a dictionary
but in the system file's encoding rather than ASCII. */
char creation_date[10]; /* "dd mmm yy". */
char creation_time[9]; /* "hh:mm:ss". */
{
off_t pos;
int width;
- char name[8];
+ char name[9];
int print_format;
int write_format;
int missing_value_code;
struct sfm_extension_record
{
+ struct ll ll; /* In struct sfm_reader 'var_attrs' list. */
int subtype; /* Record subtype. */
off_t pos; /* Starting offset in file. */
- size_t size; /* Size of data elements. */
- size_t count; /* Number of data elements. */
+ unsigned int size; /* Size of data elements. */
+ unsigned int count; /* Number of data elements. */
void *data; /* Contents. */
};
/* System file reader. */
struct sfm_reader
{
+ struct any_reader any_reader;
+
/* Resource tracking. */
struct pool *pool; /* All system file state. */
/* File data. */
- struct sfm_read_info info;
+ struct any_read_info info;
struct sfm_header_record header;
struct sfm_var_record *vars;
size_t n_vars;
struct sfm_mrset *mrsets;
size_t n_mrsets;
struct sfm_extension_record *extensions[32];
+ struct ll_list var_attrs; /* Contains "struct sfm_extension_record"s. */
/* File state. */
struct file_handle *fh; /* File handle. */
enum integer_format integer_format; /* On-disk integer format. */
enum float_format float_format; /* On-disk floating point format. */
struct sfm_var *sfm_vars; /* Variables. */
- size_t sfm_var_cnt; /* Number of variables. */
- int case_cnt; /* Number of cases */
+ size_t sfm_n_vars; /* Number of variables. */
+ int n_cases; /* Number of cases */
const char *encoding; /* String encoding. */
+ bool written_by_readstat; /* From https://github.com/WizardMac/ReadStat? */
/* Decompression. */
- enum sfm_compression compression;
+ enum any_compression compression;
double bias; /* Compression bias, usually 100.0. */
uint8_t opcodes[8]; /* Current block of opcodes. */
size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
static const struct casereader_class sys_file_casereader_class;
-static struct variable *lookup_var_by_index (struct sfm_reader *, off_t,
- const struct sfm_var_record *,
- size_t n, int idx);
+static struct sfm_reader *
+sfm_reader_cast (const struct any_reader *r_)
+{
+ assert (r_->klass == &sys_file_reader_class);
+ return UP_CAST (r_, struct sfm_reader, any_reader);
+}
+
+static bool sfm_close (struct any_reader *);
static void sys_msg (struct sfm_reader *r, off_t, int class,
const char *format, va_list args)
struct sfm_var_record *);
static bool read_value_label_record (struct sfm_reader *,
struct sfm_value_label_record *);
-static struct sfm_document_record *read_document_record (struct sfm_reader *);
+static bool read_document_record (struct sfm_reader *);
static bool read_extension_record (struct sfm_reader *, int subtype,
struct sfm_extension_record **);
static bool skip_extension_record (struct sfm_reader *, int subtype);
struct text_record *,
struct variable **var, char **value);
static void text_warn (struct sfm_reader *r, struct text_record *text,
- const char *format, ...)
- PRINTF_FORMAT (3, 4);
+ const char *format, ...) PRINTF_FORMAT (3, 4);
static char *text_get_token (struct text_record *,
struct substring delimiters, char *delimiter);
static bool text_match (struct text_record *, char c);
static bool read_dictionary (struct sfm_reader *);
static bool read_record (struct sfm_reader *, int type,
size_t *allocated_vars, size_t *allocated_labels);
-static bool read_header (struct sfm_reader *, struct sfm_read_info *,
+static bool read_header (struct sfm_reader *, struct any_read_info *,
struct sfm_header_record *);
static void parse_header (struct sfm_reader *,
const struct sfm_header_record *,
- struct sfm_read_info *, struct dictionary *);
+ struct any_read_info *, struct dictionary *);
static bool parse_variable_records (struct sfm_reader *, struct dictionary *,
struct sfm_var_record *, size_t n);
static void parse_format_spec (struct sfm_reader *, off_t pos,
unsigned int format, enum which_format,
- struct variable *, int *format_warning_cnt);
+ struct variable *, int *format_n_warnings);
static void parse_document (struct dictionary *, struct sfm_document_record *);
static void parse_display_parameters (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
static bool parse_machine_integer_info (struct sfm_reader *,
const struct sfm_extension_record *,
- struct sfm_read_info *);
+ struct any_read_info *);
static void parse_machine_float_info (struct sfm_reader *,
const struct sfm_extension_record *);
static void parse_extra_product_info (struct sfm_reader *,
const struct sfm_extension_record *,
- struct sfm_read_info *);
+ struct any_read_info *);
static void parse_mrsets (struct sfm_reader *,
const struct sfm_extension_record *,
size_t *allocated_mrsets);
static bool parse_long_string_map (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
-static bool parse_value_labels (struct sfm_reader *, struct dictionary *,
- const struct sfm_var_record *,
- size_t n_var_recs,
- const struct sfm_value_label_record *);
+static void parse_value_labels (struct sfm_reader *, struct dictionary *);
+static struct variable *parse_weight_var (struct sfm_reader *,
+ const struct sfm_var_record *, size_t n_var_recs,
+ int idx);
static void parse_data_file_attributes (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
const struct sfm_extension_record *,
struct dictionary *);
static void assign_variable_roles (struct sfm_reader *, struct dictionary *);
-static bool parse_long_string_value_labels (struct sfm_reader *,
+static void parse_long_string_value_labels (struct sfm_reader *,
const struct sfm_extension_record *,
struct dictionary *);
-static bool parse_long_string_missing_values (
+static void parse_long_string_missing_values (
struct sfm_reader *, const struct sfm_extension_record *,
struct dictionary *);
/* Frees the strings inside INFO. */
void
-sfm_read_info_destroy (struct sfm_read_info *info)
+any_read_info_destroy (struct any_read_info *info)
{
if (info)
{
/* Tries to open FH for reading as a system file. Returns an sfm_reader if
successful, otherwise NULL. */
-struct sfm_reader *
+static struct any_reader *
sfm_open (struct file_handle *fh)
{
size_t allocated_mrsets = 0;
- struct sfm_reader *r;
/* Create and initialize reader. */
- r = xzalloc (sizeof *r);
+ struct sfm_reader *r = XZALLOC (struct sfm_reader);
+ r->any_reader.klass = &sys_file_reader_class;
r->pool = pool_create ();
pool_register (r->pool, free, r);
r->fh = fh_ref (fh);
r->opcode_idx = sizeof r->opcodes;
+ ll_init (&r->var_attrs);
/* TRANSLATORS: this fragment will be interpolated into
messages in fh_lock() that identify types of files. */
if (r->lock == NULL)
goto error;
- r->file = fn_open (fh_get_file_name (fh), "rb");
+ r->file = fn_open (fh, "rb");
if (r->file == NULL)
{
msg (ME, _("Error opening `%s' for reading as a system file: %s."),
if (r->extensions[EXT_MRSETS2] != NULL)
parse_mrsets (r, r->extensions[EXT_MRSETS2], &allocated_mrsets);
- return r;
+ return &r->any_reader;
+
error:
- sfm_close (r);
+ if (r)
+ sfm_close (&r->any_reader);
return NULL;
}
if (!skip_bytes (r, 4))
return false;
- if (r->compression == SFM_COMP_ZLIB && !read_zheader (r))
+ if (r->compression == ANY_COMP_ZLIB && !read_zheader (r))
return false;
return true;
case 6:
if (r->document != NULL)
- {
- sys_error (r, r->pos, _("Duplicate type 6 (document) record."));
- return false;
- }
- r->document = read_document_record (r);
- return r->document != NULL;
+ sys_warn (r, r->pos, _("Duplicate type 6 (document) record."));
+ return read_document_record (r);
case 7:
if (!read_int (r, &subtype))
|| subtype >= sizeof r->extensions / sizeof *r->extensions)
{
sys_warn (r, r->pos,
- _("Unrecognized record type 7, subtype %d. Please "
- "send a copy of this file, and the syntax which "
- "created it to %s."),
- subtype, PACKAGE_BUGREPORT);
+ _("Unrecognized record type 7, subtype %d. For help, "
+ "please send this file to %s and mention that you were "
+ "using %s."),
+ subtype, PACKAGE_BUGREPORT, PACKAGE_STRING);
return skip_extension_record (r, subtype);
}
+ else if (subtype == 18)
+ {
+ /* System files written by "Stata 14.1/-savespss- 1.77 by S.Radyakin"
+ put each variable attribute into a separate record with subtype
+ 18. I'm surprised that SPSS puts up with this. */
+ struct sfm_extension_record *ext;
+ bool ok = read_extension_record (r, subtype, &ext);
+ if (ok && ext)
+ ll_push_tail (&r->var_attrs, &ext->ll);
+ return ok;
+ }
else if (r->extensions[subtype] != NULL)
{
sys_warn (r, r->pos,
_("Record type 7, subtype %d found here has the same "
- "type as the record found near offset 0x%llx. "
- "Please send a copy of this file, and the syntax "
- "which created it to %s."),
+ "type as the record found near offset 0x%llx. For "
+ "help, please send this file to %s and mention that "
+ "you were using %s."),
subtype, (long long int) r->extensions[subtype]->pos,
- PACKAGE_BUGREPORT);
+ PACKAGE_BUGREPORT, PACKAGE_STRING);
return skip_extension_record (r, subtype);
}
else
/* Returns the character encoding obtained from R, or a null pointer if R
doesn't have an indication of its character encoding. */
-const char *
+static const char *
sfm_get_encoding (const struct sfm_reader *r)
{
/* The EXT_ENCODING record is the best way to determine dictionary
return NULL;
}
+struct get_strings_aux
+ {
+ struct pool *pool;
+ char **titles;
+ char **strings;
+ bool *ids;
+ size_t allocated;
+ size_t n;
+ };
+
+static void
+add_string__ (struct get_strings_aux *aux,
+ const char *string, bool id, char *title)
+{
+ if (aux->n >= aux->allocated)
+ {
+ aux->allocated = 2 * (aux->allocated + 1);
+ aux->titles = pool_realloc (aux->pool, aux->titles,
+ aux->allocated * sizeof *aux->titles);
+ aux->strings = pool_realloc (aux->pool, aux->strings,
+ aux->allocated * sizeof *aux->strings);
+ aux->ids = pool_realloc (aux->pool, aux->ids,
+ aux->allocated * sizeof *aux->ids);
+ }
+
+ aux->titles[aux->n] = title;
+ aux->strings[aux->n] = pool_strdup (aux->pool, string);
+ aux->ids[aux->n] = id;
+ aux->n++;
+}
+
+static void PRINTF_FORMAT (3, 4)
+add_string (struct get_strings_aux *aux,
+ const char *string, const char *title, ...)
+{
+ va_list args;
+
+ va_start (args, title);
+ add_string__ (aux, string, false, pool_vasprintf (aux->pool, title, args));
+ va_end (args);
+}
+
+static void PRINTF_FORMAT (3, 4)
+add_id (struct get_strings_aux *aux, const char *id, const char *title, ...)
+{
+ va_list args;
+
+ va_start (args, title);
+ add_string__ (aux, id, true, pool_vasprintf (aux->pool, title, args));
+ va_end (args);
+}
+
+static const char *
+skip_prefix (const char *s, const char *prefix)
+{
+ size_t prefix_len = strlen (prefix);
+ return !strncmp (s, prefix, prefix_len) ? s + prefix_len : s;
+}
+
+/* Retrieves significant string data from R in its raw format, to allow the
+ caller to try to detect the encoding in use.
+
+ Returns the number of strings retrieved N. Sets each of *TITLESP, *IDSP,
+ and *STRINGSP to an array of N elements allocated from POOL. For each I in
+ 0...N-1, UTF-8 string *TITLESP[I] describes *STRINGSP[I], which is in
+ whatever encoding system file R uses. *IDS[I] is true if *STRINGSP[I] must
+ be a valid PSPP language identifier, false if *STRINGSP[I] is free-form
+ text. */
+static size_t
+sfm_get_strings (const struct any_reader *r_, struct pool *pool,
+ char ***titlesp, bool **idsp, char ***stringsp)
+{
+ struct sfm_reader *r = sfm_reader_cast (r_);
+ const struct sfm_mrset *mrset;
+ struct get_strings_aux aux;
+ size_t var_idx;
+ size_t i, j, k;
+
+ aux.pool = pool;
+ aux.titles = NULL;
+ aux.strings = NULL;
+ aux.ids = NULL;
+ aux.allocated = 0;
+ aux.n = 0;
+
+ var_idx = 0;
+ for (i = 0; i < r->n_vars; i++)
+ if (r->vars[i].width != -1)
+ add_id (&aux, r->vars[i].name, _("Variable %zu"), ++var_idx);
+
+ var_idx = 0;
+ for (i = 0; i < r->n_vars; i++)
+ if (r->vars[i].width != -1)
+ {
+ var_idx++;
+ if (r->vars[i].label)
+ add_string (&aux, r->vars[i].label, _("Variable %zu Label"),
+ var_idx);
+ }
+
+ k = 0;
+ for (i = 0; i < r->n_labels; i++)
+ for (j = 0; j < r->labels[i].n_labels; j++)
+ add_string (&aux, r->labels[i].labels[j].label,
+ _("Value Label %zu"), k++);
+
+ add_string (&aux, r->header.creation_date, _("Creation Date"));
+ add_string (&aux, r->header.creation_time, _("Creation Time"));
+ add_string (&aux, skip_prefix (r->header.eye_catcher, "@(#) "), _("Product"));
+ add_string (&aux, r->header.file_label, _("File Label"));
+
+ if (r->extensions[EXT_PRODUCT_INFO])
+ add_string (&aux, r->extensions[EXT_PRODUCT_INFO]->data,
+ _("Extra Product Info"));
+
+ if (r->document)
+ {
+ size_t i;
+
+ for (i = 0; i < r->document->n_lines; i++)
+ {
+ char line[81];
+
+ memcpy (line, r->document->documents + i * 80, 80);
+ line[80] = '\0';
+
+ add_string (&aux, line, _("Document Line %zu"), i + 1);
+ }
+ }
+
+ for (mrset = r->mrsets; mrset < &r->mrsets[r->n_mrsets]; mrset++)
+ {
+ size_t mrset_idx = mrset - r->mrsets + 1;
+
+ add_id (&aux, mrset->name, _("MRSET %zu"), mrset_idx);
+ if (mrset->label[0])
+ add_string (&aux, mrset->label, _("MRSET %zu Label"), mrset_idx);
+
+ /* Skip the variables because they ought to be duplicates. */
+
+ if (mrset->counted)
+ add_string (&aux, mrset->counted, _("MRSET %zu Counted Value"),
+ mrset_idx);
+ }
+
+ /* data file attributes */
+ /* variable attributes */
+ /* long var map */
+ /* long string value labels */
+ /* long string missing values */
+
+ *titlesp = aux.titles;
+ *idsp = aux.ids;
+ *stringsp = aux.strings;
+ return aux.n;
+}
+
/* Decodes the dictionary read from R, saving it into into *DICT. Character
strings in R are decoded using ENCODING, or an encoding obtained from R if
ENCODING is null, or the locale encoding if R specifies no encoding.
If INFOP is non-null, then it receives additional info about the system
- file, which the caller must eventually free with sfm_read_info_destroy()
+ file, which the caller must eventually free with any_read_info_destroy()
when it is no longer needed.
This function consumes R. The caller must use it again later, even to
destroy it with sfm_close(). */
-struct casereader *
-sfm_decode (struct sfm_reader *r, const char *encoding,
- struct dictionary **dictp, struct sfm_read_info *infop)
+static struct casereader *
+sfm_decode (struct any_reader *r_, const char *encoding,
+ struct dictionary **dictp, struct any_read_info *infop)
{
+ struct sfm_reader *r = sfm_reader_cast (r_);
struct dictionary *dict;
- size_t i;
if (encoding == NULL)
{
encoding = sfm_get_encoding (r);
if (encoding == NULL)
- encoding = locale_charset ();
+ {
+ sys_warn (r, -1, _("This system file does not indicate its own "
+ "character encoding. Using default encoding "
+ "%s. For best results, specify an encoding "
+ "explicitly. Use SYSFILE INFO with "
+ "ENCODING=\"DETECT\" to analyze the possible "
+ "encodings."),
+ locale_charset ());
+ encoding = locale_charset ();
+ }
}
dict = dict_create (encoding);
/* Parse value labels and the weight variable immediately after the variable
records. These records use indexes into var_recs[], so we must parse them
before those indexes become invalidated by very long string variables. */
- for (i = 0; i < r->n_labels; i++)
- if (!parse_value_labels (r, dict, r->vars, r->n_vars, &r->labels[i]))
- goto error;
+ parse_value_labels (r, dict);
if (r->header.weight_idx != 0)
- {
- struct variable *weight_var;
-
- weight_var = lookup_var_by_index (r, 76, r->vars, r->n_vars,
- r->header.weight_idx);
- if (weight_var != NULL)
- {
- if (var_is_numeric (weight_var))
- dict_set_weight (dict, weight_var);
- else
- sys_warn (r, -1, _("Ignoring string variable `%s' set "
- "as weighting variable."),
- var_get_name (weight_var));
- }
- }
+ dict_set_weight (dict, parse_weight_var (r, r->vars, r->n_vars,
+ r->header.weight_idx));
if (r->extensions[EXT_DISPLAY] != NULL)
parse_display_parameters (r, r->extensions[EXT_DISPLAY], dict);
parse_long_var_name_map (r, r->extensions[EXT_LONG_NAMES], dict);
/* The following records use long names, so they need to follow renaming. */
- if (r->extensions[EXT_VAR_ATTRS] != NULL)
+ if (!ll_is_empty (&r->var_attrs))
{
- parse_variable_attributes (r, r->extensions[EXT_VAR_ATTRS], dict);
+ struct sfm_extension_record *ext;
+ ll_for_each (ext, struct sfm_extension_record, ll, &r->var_attrs)
+ parse_variable_attributes (r, ext, dict);
/* Roles use the $@Role attribute. */
assign_variable_roles (r, dict);
}
-
- if (r->extensions[EXT_LONG_LABELS] != NULL
- && !parse_long_string_value_labels (r, r->extensions[EXT_LONG_LABELS],
- dict))
- goto error;
- if (r->extensions[EXT_LONG_MISSING] != NULL
- && !parse_long_string_missing_values (r, r->extensions[EXT_LONG_MISSING],
- dict))
- goto error;
+ if (r->extensions[EXT_LONG_LABELS] != NULL)
+ parse_long_string_value_labels (r, r->extensions[EXT_LONG_LABELS], dict);
+ if (r->extensions[EXT_LONG_MISSING] != NULL)
+ parse_long_string_missing_values (r, r->extensions[EXT_LONG_MISSING],
+ dict);
/* Warn if the actual amount of data per case differs from the
amount that the header claims. SPSS version 13 gets this
wrong when very long strings are involved, so don't warn in
that case. */
- if (r->header.nominal_case_size != -1
+ if (r->header.nominal_case_size > 0
&& r->header.nominal_case_size != r->n_vars
&& r->info.version_major != 13)
sys_warn (r, -1, _("File header claims %d variable positions but "
sfm_read_case to use. We cannot use the `struct variable's
from the dictionary we created, because the caller owns the
dictionary and may destroy or modify its variables. */
- sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_var_cnt);
+ sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_n_vars);
pool_register (r->pool, free, r->sfm_vars);
r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
}
return casereader_create_sequential
- (NULL, r->proto,
- r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
- &sys_file_casereader_class, r);
+ (NULL, r->proto, r->n_cases == -1 ? CASENUMBER_MAX : r->n_cases,
+ &sys_file_casereader_class, r);
error:
- sfm_close (r);
- dict_destroy (dict);
+ sfm_close (r_);
+ dict_unref (dict);
*dictp = NULL;
return NULL;
}
closed with sfm_decode() or this function.
Returns true if an I/O error has occurred on READER, false
otherwise. */
-bool
-sfm_close (struct sfm_reader *r)
+static bool
+sfm_close (struct any_reader *r_)
{
+ struct sfm_reader *r = sfm_reader_cast (r_);
bool error;
- if (r == NULL)
- return true;
-
if (r->file)
{
- if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)
+ if (fn_close (r->fh, r->file) == EOF)
{
msg (ME, _("Error closing system file `%s': %s."),
fh_get_file_name (r->fh), strerror (errno));
r->file = NULL;
}
- sfm_read_info_destroy (&r->info);
+ any_read_info_destroy (&r->info);
fh_unlock (r->lock);
fh_unref (r->fh);
sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
{
struct sfm_reader *r = r_;
- sfm_close (r);
+ sfm_close (&r->any_reader);
}
-/* Returns true if FILE is an SPSS system file,
- false otherwise. */
-bool
+/* Detects whether FILE is an SPSS system file. Returns 1 if so, 0 if not, and
+ a negative errno value if there is an error reading FILE. */
+static int
sfm_detect (FILE *file)
{
char magic[5];
+ if (fseek (file, 0, SEEK_SET) != 0)
+ return -errno;
if (fread (magic, 4, 1, file) != 1)
- return false;
+ return ferror (file) ? -errno : 0;
magic[4] = '\0';
return (!strcmp (ASCII_MAGIC, magic)
except for the string fields in *INFO, which parse_header() will initialize
later once the file's encoding is known. */
static bool
-read_header (struct sfm_reader *r, struct sfm_read_info *info,
+read_header (struct sfm_reader *r, struct any_read_info *info,
struct sfm_header_record *header)
{
uint8_t raw_layout_code[4];
if (!read_string (r, header->magic, sizeof header->magic)
|| !read_string (r, header->eye_catcher, sizeof header->eye_catcher))
return false;
+ r->written_by_readstat = strstr (header->eye_catcher,
+ "https://github.com/WizardMac/ReadStat");
if (!strcmp (ASCII_MAGIC, header->magic)
|| !strcmp (EBCDIC_MAGIC, header->magic))
if (!zmagic)
{
if (compressed == 0)
- r->compression = SFM_COMP_NONE;
+ r->compression = ANY_COMP_NONE;
else if (compressed == 1)
- r->compression = SFM_COMP_SIMPLE;
- else if (compressed != 0)
+ r->compression = ANY_COMP_SIMPLE;
+ else
{
sys_error (r, 0, "System file header has invalid compression "
"value %d.", compressed);
else
{
if (compressed == 2)
- r->compression = SFM_COMP_ZLIB;
+ r->compression = ANY_COMP_ZLIB;
else
{
sys_error (r, 0, "ZLIB-compressed system file header has invalid "
if (!read_int (r, &header->weight_idx))
return false;
- if (!read_int (r, &r->case_cnt))
+ if (!read_int (r, &r->n_cases))
return false;
- if ( r->case_cnt > INT_MAX / 2)
- r->case_cnt = -1;
+ if (r->n_cases > INT_MAX / 2)
+ r->n_cases = -1;
/* Identify floating-point format and obtain compression bias. */
if (!read_bytes (r, raw_bias, sizeof raw_bias))
info->integer_format = r->integer_format;
info->float_format = r->float_format;
info->compression = r->compression;
- info->case_cnt = r->case_cnt;
+ info->n_cases = r->n_cases;
return true;
}
|| !read_int (r, &record->missing_value_code)
|| !read_int (r, &record->print_format)
|| !read_int (r, &record->write_format)
- || !read_bytes (r, record->name, sizeof record->name))
+ || !read_string (r, record->name, sizeof record->name))
return false;
if (has_variable_label == 1)
{
- enum { MAX_LABEL_LEN = 255 };
+ enum { MAX_LABEL_LEN = 65536 };
unsigned int len, read_len;
if (!read_uint (r, &len))
record->pos = r->pos;
if (!read_uint (r, &record->n_labels))
return false;
- if (record->n_labels > SIZE_MAX / sizeof *record->labels)
+ if (record->n_labels > UINT_MAX / sizeof *record->labels)
{
- sys_error (r, r->pos - 4, _("Invalid number of labels %zu."),
+ sys_error (r, r->pos - 4, _("Invalid number of labels %u."),
record->n_labels);
return false;
}
if (record->n_vars < 1 || record->n_vars > r->n_vars)
{
sys_error (r, r->pos - 4,
- _("Number of variables associated with a value label (%zu) "
+ _("Number of variables associated with a value label (%u) "
"is not between 1 and the number of variables (%zu)."),
record->n_vars, r->n_vars);
return false;
return true;
}
-/* Reads a document record from R and returns it. */
-static struct sfm_document_record *
+/* Reads a document record from R. Returns true if successful, false on
+ error. */
+static bool
read_document_record (struct sfm_reader *r)
{
- struct sfm_document_record *record;
int n_lines;
-
- record = pool_malloc (r->pool, sizeof *record);
- record->pos = r->pos;
-
if (!read_int (r, &n_lines))
- return NULL;
- if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
+ return false;
+ else if (n_lines == 0)
+ return true;
+ else if (n_lines < 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
{
- sys_error (r, record->pos,
+ sys_error (r, r->pos,
_("Number of document lines (%d) "
"must be greater than 0 and less than %d."),
n_lines, INT_MAX / DOC_LINE_LENGTH);
- return NULL;
+ return false;
}
+ struct sfm_document_record *record;
+ record = pool_malloc (r->pool, sizeof *record);
+ record->pos = r->pos;
record->n_lines = n_lines;
record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
if (!read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines))
- return NULL;
+ return false;
- return record;
+ r->document = record;
+ return true;
}
static bool
{
if (type->size > 0 && record->size != type->size)
sys_warn (r, record->pos,
- _("Record type 7, subtype %d has bad size %zu "
+ _("Record type 7, subtype %d has bad size %u "
"(expected %d)."), subtype, record->size, type->size);
else if (type->count > 0 && record->count != type->count)
sys_warn (r, record->pos,
- _("Record type 7, subtype %d has bad count %zu "
+ _("Record type 7, subtype %d has bad count %u "
"(expected %d)."), subtype, record->count, type->count);
else if (type->count == 0 && type->size == 0)
{
}
sys_warn (r, record->pos,
- _("Unrecognized record type 7, subtype %d. Please send a "
- "copy of this file, and the syntax which created it to %s."),
- subtype, PACKAGE_BUGREPORT);
+ _("Unrecognized record type 7, subtype %d. For help, please "
+ "send this file to %s and mention that you were using %s."),
+ subtype, PACKAGE_BUGREPORT, PACKAGE_STRING);
skip:
return skip_bytes (r, n_bytes);
static void
parse_header (struct sfm_reader *r, const struct sfm_header_record *header,
- struct sfm_read_info *info, struct dictionary *dict)
+ struct any_read_info *info, struct dictionary *dict)
{
const char *dict_encoding = dict_get_encoding (dict);
struct substring product;
info->product = ss_xstrdup (product);
}
+static struct variable *
+add_var_with_generated_name (struct dictionary *dict, int width)
+{
+ char *name = dict_make_unique_var_name (dict, NULL, NULL);
+ struct variable *var = dict_create_var_assert (dict, name, width);
+ free (name);
+ return var;
+}
+
/* Reads a variable (type 2) record from R and adds the
corresponding variable to DICT.
Also skips past additional variable records for long string
struct sfm_var_record *rec;
int n_warnings = 0;
- for (rec = var_recs; rec < &var_recs[n_var_recs]; )
+ for (rec = var_recs; rec < &var_recs[n_var_recs];)
{
- struct variable *var;
size_t n_values;
char *name;
size_t i;
name = recode_string_pool ("UTF-8", dict_encoding,
- rec->name, 8, r->pool);
+ rec->name, -1, r->pool);
name[strcspn (name, " ")] = '\0';
- if (!dict_id_is_valid (dict, name, false)
- || name[0] == '$' || name[0] == '#')
- {
- sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
- return false;
- }
-
if (rec->width < 0 || rec->width > 255)
{
sys_error (r, rec->pos,
return false;
}
- var = rec->var = dict_create_var (dict, name, rec->width);
- if (var == NULL)
+ struct variable *var;
+ if (!dict_id_is_valid (dict, name, false)
+ || name[0] == '$' || name[0] == '#')
{
- char *new_name = dict_make_unique_var_name (dict, NULL, NULL);
- sys_warn (r, rec->pos, _("Renaming variable with duplicate name "
- "`%s' to `%s'."),
- name, new_name);
- var = rec->var = dict_create_var_assert (dict, new_name, rec->width);
- free (new_name);
+ var = add_var_with_generated_name (dict, rec->width);
+ sys_warn (r, rec->pos, _("Renaming variable with invalid name "
+ "`%s' to `%s'."), name, var_get_name (var));
}
+ else
+ {
+ var = dict_create_var (dict, name, rec->width);
+ if (var == NULL)
+ {
+ var = add_var_with_generated_name (dict, rec->width);
+ sys_warn (r, rec->pos, _("Renaming variable with duplicate name "
+ "`%s' to `%s'."),
+ name, var_get_name (var));
+ }
+ }
+ rec->var = var;
- /* Set the short name the same as the long name. */
- var_set_short_name (var, 0, name);
+ /* Set the short name the same as the long name (even if we renamed
+ it). */
+ var_set_short_name (var, 0, var_get_name (var));
/* Get variable label, if any. */
if (rec->label)
utf8_label = recode_string_pool ("UTF-8", dict_encoding,
rec->label, -1, r->pool);
- var_set_label (var, utf8_label, false);
+ var_set_label (var, utf8_label);
}
/* Set missing values. */
}
}
else
- {
- union value value;
-
- value_init_pool (r->pool, &value, width);
- value_set_missing (&value, width);
- for (i = 0; i < rec->missing_value_code; i++)
- mv_add_str (&mv, rec->missing + 8 * i, MIN (width, 8));
- }
+ for (i = 0; i < rec->missing_value_code; i++)
+ mv_add_str (&mv, rec->missing + 8 * i, MIN (width, 8));
var_set_missing_values (var, &mv);
}
int *n_warnings)
{
const int max_warnings = 8;
- uint8_t raw_type = format >> 16;
- uint8_t w = format >> 8;
- uint8_t d = format;
struct fmt_spec f;
- bool ok;
-
- f.w = w;
- f.d = d;
- msg_disable ();
- ok = (fmt_from_io (raw_type, &f.type)
- && fmt_check_output (&f)
- && fmt_check_width_compat (&f, var_get_width (v)));
- msg_enable ();
-
- if (ok)
+ if (fmt_from_u32 (format, var_get_width (v), false, &f))
{
if (which == PRINT_FORMAT)
var_set_print_format (v, &f);
static bool
parse_machine_integer_info (struct sfm_reader *r,
const struct sfm_extension_record *record,
- struct sfm_read_info *info)
+ struct any_read_info *info)
{
int float_representation, expected_float_format;
int integer_representation, expected_integer_format;
static void
parse_extra_product_info (struct sfm_reader *r,
const struct sfm_extension_record *record,
- struct sfm_read_info *info)
+ struct any_read_info *info)
{
struct text_record *text;
text = open_text_record (r, record, false);
for (;;)
{
- struct sfm_mrset *mrset;
- size_t allocated_vars;
- char delimiter;
+ struct sfm_mrset *mrset = NULL;
+ size_t allocated_vars = 0;
+ char delimiter = '4';
/* Skip extra line feeds if present. */
while (text_match (text, '\n'))
}
number = text_get_token (text, ss_cstr (" "), NULL);
- if (!strcmp (number, "11"))
+ if (!number)
+ sys_warn (r, record->pos,
+ _("Missing label source value "
+ "following `E' at offset %zu in MRSETS record."),
+ text_pos (text));
+ else if (!strcmp (number, "11"))
mrset->label_from_var_label = true;
else if (strcmp (number, "1"))
sys_warn (r, record->pos,
size_t i;
name = recode_string ("UTF-8", r->encoding, s->name, -1);
- if (name[0] != '$')
+ if (!mrset_is_valid_name (name, dict_get_encoding (dict), false))
{
- sys_warn (r, -1, _("Multiple response set name `%s' does not begin "
- "with `$'."),
+ sys_warn (r, -1, _("Invalid multiple response set name `%s'."),
name);
free (name);
continue;
size_t ofs;
size_t i;
- n_vars = dict_get_var_cnt (dict);
+ n_vars = dict_get_n_vars (dict);
if (record->count == 3 * n_vars)
includes_width = true;
else if (record->count == 2 * n_vars)
else
{
sys_warn (r, record->pos,
- _("Extension 11 has bad count %zu (for %zu variables)."),
+ _("Extension 11 has bad count %u (for %zu variables)."),
record->count, n_vars);
return;
}
}
static void
-rename_var_and_save_short_names (struct dictionary *dict, struct variable *var,
- const char *new_name)
+rename_var_and_save_short_names (struct sfm_reader *r, off_t pos,
+ struct dictionary *dict,
+ struct variable *var, const char *new_name)
{
size_t n_short_names;
char **short_names;
/* Renaming a variable may clear its short names, but we
want to retain them, so we save them and re-set them
afterward. */
- n_short_names = var_get_short_name_cnt (var);
+ n_short_names = var_get_n_short_names (var);
short_names = xnmalloc (n_short_names, sizeof *short_names);
for (i = 0; i < n_short_names; i++)
{
const char *s = var_get_short_name (var, i);
- short_names[i] = s != NULL ? xstrdup (s) : NULL;
+ short_names[i] = xstrdup_if_nonnull (s);
}
/* Set long name. */
- dict_rename_var (dict, var, new_name);
+ if (!dict_try_rename_var (dict, var, new_name))
+ sys_warn (r, pos, _("Duplicate long variable name `%s'."), new_name);
/* Restore short names. */
for (i = 0; i < n_short_names; i++)
converted to lowercase, as the long variable names. */
size_t i;
- for (i = 0; i < dict_get_var_cnt (dict); i++)
+ for (i = 0; i < dict_get_n_vars (dict); i++)
{
struct variable *var = dict_get_var (dict, i);
char *new_name;
new_name = utf8_to_lower (var_get_name (var));
- rename_var_and_save_short_names (dict, var, new_name);
+ rename_var_and_save_short_names (r, -1, dict, var, new_name);
free (new_name);
}
while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
{
/* Validate long name. */
- if (!dict_id_is_valid (dict, long_name, false))
+ if (!dict_id_is_valid (dict, long_name, false)
+ || long_name[0] == '$' || long_name[0] == '#')
{
sys_warn (r, record->pos,
_("Long variable mapping from %s to invalid "
continue;
}
- /* Identify any duplicates. */
- if (utf8_strcasecmp (var_get_short_name (var, 0), long_name)
- && dict_lookup_var (dict, long_name) != NULL)
- {
- sys_warn (r, record->pos,
- _("Duplicate long variable name `%s'."), long_name);
- continue;
- }
-
- rename_var_and_save_short_names (dict, var, long_name);
+ rename_var_and_save_short_names (r, record->pos, dict, var, long_name);
}
close_text_record (r, text);
}
{
size_t idx = var_get_dict_index (var);
long int length;
- int segment_cnt;
int i;
/* Get length. */
}
/* Check segments. */
- segment_cnt = sfm_width_to_segments (length);
- if (segment_cnt == 1)
+ int n_segments = sfm_width_to_segments (length);
+ if (n_segments == 1)
{
sys_warn (r, record->pos,
_("%s listed in very long string record with width %s, "
var_get_name (var), length_s);
continue;
}
- if (idx + segment_cnt > dict_get_var_cnt (dict))
+ if (idx + n_segments > dict_get_n_vars (dict))
{
sys_error (r, record->pos,
_("Very long string %s overflows dictionary."),
/* Get the short names from the segments and check their
lengths. */
- for (i = 0; i < segment_cnt; i++)
+ for (i = 0; i < n_segments; i++)
{
struct variable *seg = dict_get_var (dict, idx + i);
int alloc_width = sfm_segment_alloc_width (length, i);
return false;
}
}
- dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
+ dict_delete_consecutive_vars (dict, idx + 1, n_segments - 1);
var_set_width (var, length);
}
close_text_record (r, text);
return true;
}
-static bool
-parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
- const struct sfm_var_record *var_recs, size_t n_var_recs,
- const struct sfm_value_label_record *record)
+#define MAX_LABEL_WARNINGS 5
+
+/* Displays a warning for offset OFFSET in the file. */
+static void
+value_label_warning (struct sfm_reader *r, off_t offset, int *n_label_warnings,
+ const char *format, ...)
{
- struct variable **vars;
- char **utf8_labels;
- size_t i;
+ if (++*n_label_warnings > MAX_LABEL_WARNINGS)
+ return;
- utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
- for (i = 0; i < record->n_labels; i++)
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, offset, MW, format, args);
+ va_end (args);
+}
+
+#define MAX_LABEL_WARNINGS 5
+
+static void
+parse_one_value_label_set (struct sfm_reader *r, struct dictionary *dict,
+ const struct sfm_var_record *var_recs,
+ size_t n_var_recs,
+ const struct sfm_value_label_record *record,
+ int *n_label_warnings)
+{
+ char **utf8_labels
+ = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
+ for (size_t i = 0; i < record->n_labels; i++)
utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
record->labels[i].label, -1,
r->pool);
- vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
- for (i = 0; i < record->n_vars; i++)
+ struct variable **vars = pool_nmalloc (r->pool,
+ record->n_vars, sizeof *vars);
+ unsigned int n_vars = 0;
+ for (size_t i = 0; i < record->n_vars; i++)
{
- vars[i] = lookup_var_by_index (r, record->pos,
- var_recs, n_var_recs, record->vars[i]);
- if (vars[i] == NULL)
- return false;
+ int idx = record->vars[i];
+ if (idx < 1 || idx > n_var_recs)
+ {
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value label variable index %d not in valid range 1...%zu."),
+ idx, n_var_recs);
+ continue;
+ }
+
+ const struct sfm_var_record *rec = &var_recs[idx - 1];
+ if (rec->var == NULL)
+ {
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value label variable index %d "
+ "refers to long string continuation."), idx);
+ continue;
+ }
+
+ vars[n_vars++] = rec->var;
}
+ if (!n_vars)
+ return;
- for (i = 1; i < record->n_vars; i++)
+ for (size_t i = 1; i < n_vars; i++)
if (var_get_type (vars[i]) != var_get_type (vars[0]))
{
- sys_error (r, record->pos,
- _("Variables associated with value label are not all of "
- "identical type. Variable %s is %s, but variable "
- "%s is %s."),
- var_get_name (vars[0]),
- var_is_numeric (vars[0]) ? _("numeric") : _("string"),
- var_get_name (vars[i]),
- var_is_numeric (vars[i]) ? _("numeric") : _("string"));
- return false;
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Variables associated with value label are not all of "
+ "identical type. Variable %s is %s, but variable "
+ "%s is %s."),
+ var_get_name (vars[0]),
+ var_is_numeric (vars[0]) ? _("numeric") : _("string"),
+ var_get_name (vars[i]),
+ var_is_numeric (vars[i]) ? _("numeric") : _("string"));
+ return;
}
- for (i = 0; i < record->n_vars; i++)
+ for (size_t i = 0; i < n_vars; i++)
{
struct variable *var = vars[i];
- int width;
- size_t j;
-
- width = var_get_width (var);
+ int width = var_get_width (var);
if (width > 8)
{
- sys_error (r, record->pos,
- _("Value labels may not be added to long string "
- "variables (e.g. %s) using records types 3 and 4."),
- var_get_name (var));
- return false;
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Value labels may not be added to long string "
+ "variables (e.g. %s) using records types 3 and 4."),
+ var_get_name (var));
+ continue;
}
- for (j = 0; j < record->n_labels; j++)
+ for (size_t j = 0; j < record->n_labels; j++)
{
struct sfm_value_label *label = &record->labels[j];
union value value;
if (width == 0)
value.f = parse_float (r, label->value, 0);
else
- memcpy (value_str_rw (&value, width), label->value, width);
+ memcpy (value.s, label->value, width);
if (!var_add_value_label (var, &value, utf8_labels[j]))
{
- if (var_is_numeric (var))
- sys_warn (r, record->pos,
- _("Duplicate value label for %g on %s."),
- value.f, var_get_name (var));
+ if (r->written_by_readstat)
+ {
+ /* Ignore the problem. ReadStat is buggy and emits value
+ labels whose values are longer than string variables'
+ widths, that are identical in the actual width of the
+ variable, e.g. both values "ABC123" and "ABC456" for a
+ string variable with width 3. */
+ }
+ else if (var_is_numeric (var))
+ value_label_warning (r, record->pos, n_label_warnings,
+ _("Duplicate value label for %g on %s."),
+ value.f, var_get_name (var));
else
- sys_warn (r, record->pos,
- _("Duplicate value label for `%.*s' on %s."),
- width, value_str (&value, width),
- var_get_name (var));
+ value_label_warning (
+ r, record->pos, n_label_warnings,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value.s, var_get_name (var));
}
value_destroy (&value, width);
}
pool_free (r->pool, vars);
- for (i = 0; i < record->n_labels; i++)
+ for (size_t i = 0; i < record->n_labels; i++)
pool_free (r->pool, utf8_labels[i]);
pool_free (r->pool, utf8_labels);
+}
- return true;
+static void
+parse_value_labels (struct sfm_reader *r, struct dictionary *dict)
+{
+ int n_label_warnings = 0;
+ for (size_t i = 0; i < r->n_labels; i++)
+ parse_one_value_label_set (r, dict, r->vars, r->n_vars, &r->labels[i],
+ &n_label_warnings);
+ if (n_label_warnings > MAX_LABEL_WARNINGS)
+ sys_warn (r, -1,
+ _("Suppressed %d additional warnings for value labels."),
+ n_label_warnings - MAX_LABEL_WARNINGS);
}
static struct variable *
-lookup_var_by_index (struct sfm_reader *r, off_t offset,
- const struct sfm_var_record *var_recs, size_t n_var_recs,
- int idx)
+parse_weight_var (struct sfm_reader *r,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ int idx)
{
- const struct sfm_var_record *rec;
+ off_t offset = 76; /* Offset to variable index in header. */
if (idx < 1 || idx > n_var_recs)
{
- sys_error (r, offset,
- _("Variable index %d not in valid range 1...%zu."),
- idx, n_var_recs);
+ sys_warn (r, offset,
+ _("Weight variable index %d not in valid range 1...%zu. "
+ "Treating file as unweighted."),
+ idx, n_var_recs);
return NULL;
}
- rec = &var_recs[idx - 1];
+ const struct sfm_var_record *rec = &var_recs[idx - 1];
if (rec->var == NULL)
{
- sys_error (r, offset,
- _("Variable index %d refers to long string continuation."),
- idx);
+ sys_warn (r, offset,
+ _("Weight variable index %d refers to long string "
+ "continuation. Treating file as unweighted."), idx);
return NULL;
}
- return rec->var;
+ struct variable *weight_var = rec->var;
+ if (!var_is_numeric (weight_var))
+ {
+ sys_warn (r, offset, _("Ignoring string variable `%s' set "
+ "as weighting variable."),
+ var_get_name (weight_var));
+ return NULL;
+ }
+
+ return weight_var;
}
/* Parses a set of custom attributes from TEXT into ATTRS.
text_warn (r, text, _("Error parsing attribute value %s[%d]."),
key, index);
break;
- }
+ }
length = strlen (value);
- if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
+ if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
{
value[length - 1] = '\0';
- attribute_add_value (attr, value + 1);
+ attribute_add_value (attr, value + 1);
}
- else
+ else
{
text_warn (r, text,
_("Attribute value %s[%d] is not quoted: %s."),
key, index, value);
- attribute_add_value (attr, value);
+ attribute_add_value (attr, value);
}
/* Was this the last value for this attribute? */
if (text_match (text, ')'))
break;
}
- if (attrs != NULL)
- attrset_add (attrs, attr);
+ if (attrs != NULL && attribute_get_n_values (attr) > 0)
+ {
+ if (!attrset_try_add (attrs, attr))
+ {
+ text_warn (r, text, _("Duplicate attribute %s."),
+ attribute_get_name (attr));
+ attribute_destroy (attr);
+ }
+ }
else
attribute_destroy (attr);
}
size_t n_warnings = 0;
size_t i;
- for (i = 0; i < dict_get_var_cnt (dict); i++)
+ for (i = 0; i < dict_get_n_vars (dict); i++)
{
struct variable *var = dict_get_var (dict, i);
struct attrset *attrs = var_get_attributes (var);
const struct attribute *attr = attrset_lookup (attrs, "$@Role");
- if (attr != NULL)
+ if (attr != NULL && attribute_get_n_values (attr) > 0)
{
int value = atoi (attribute_get_value (attr, 0));
enum var_role role;
size_t end = record->size * record->count;
if (length >= end || ofs + length > end)
{
- sys_error (r, record->pos + end,
- _("Extension record subtype %d ends unexpectedly."),
- record->subtype);
+ sys_warn (r, record->pos + end,
+ _("Extension record subtype %d ends unexpectedly."),
+ record->subtype);
return false;
}
return true;
}
-static bool
+static void
parse_long_string_value_labels (struct sfm_reader *r,
const struct sfm_extension_record *record,
struct dictionary *dict)
/* Parse variable name length. */
if (!check_overflow (r, record, ofs, 4))
- return false;
+ return;
var_name_len = parse_int (r, record->data, ofs);
ofs += 4;
/* Parse variable name, width, and number of labels. */
- if (!check_overflow (r, record, ofs, var_name_len + 8))
- return false;
+ if (!check_overflow (r, record, ofs, var_name_len)
+ || !check_overflow (r, record, ofs, var_name_len + 8))
+ return;
var_name = recode_string_pool ("UTF-8", dict_encoding,
(const char *) record->data + ofs,
var_name_len, r->pool);
/* Parse value length. */
if (!check_overflow (r, record, ofs, 4))
- return false;
+ return;
value_length = parse_int (r, record->data, ofs);
ofs += 4;
/* Parse value. */
if (!check_overflow (r, record, ofs, value_length))
- return false;
+ return;
if (!skip)
{
if (value_length == width)
- memcpy (value_str_rw (&value, width),
- (const uint8_t *) record->data + ofs, width);
+ memcpy (value.s, (const uint8_t *) record->data + ofs, width);
else
{
sys_warn (r, record->pos + ofs,
/* Parse label length. */
if (!check_overflow (r, record, ofs, 4))
- return false;
+ return;
label_length = parse_int (r, record->data, ofs);
ofs += 4;
/* Parse label. */
if (!check_overflow (r, record, ofs, label_length))
- return false;
+ return;
if (!skip)
{
char *label;
if (!var_add_value_label (var, &value, label))
sys_warn (r, record->pos + ofs,
_("Duplicate value label for `%.*s' on %s."),
- width, value_str (&value, width),
- var_get_name (var));
+ width, value.s, var_get_name (var));
pool_free (r->pool, label);
}
ofs += label_length;
}
}
-
- return true;
}
-static bool
+static void
parse_long_string_missing_values (struct sfm_reader *r,
const struct sfm_extension_record *record,
struct dictionary *dict)
/* Parse variable name length. */
if (!check_overflow (r, record, ofs, 4))
- return false;
+ return;
var_name_len = parse_int (r, record->data, ofs);
ofs += 4;
/* Parse variable name. */
- if (!check_overflow (r, record, ofs, var_name_len + 1))
- return false;
+ if (!check_overflow (r, record, ofs, var_name_len)
+ || !check_overflow (r, record, ofs, var_name_len + 1))
+ return;
var_name = recode_string_pool ("UTF-8", dict_encoding,
(const char *) record->data + ofs,
var_name_len, r->pool);
/* Parse value length. */
if (!check_overflow (r, record, ofs, 4))
- return false;
+ return;
value_length = parse_int (r, record->data, ofs);
ofs += 4;
/* Parse value. */
if (!check_overflow (r, record, ofs, value_length))
- return false;
+ return;
if (var != NULL
&& i < 3
&& !mv_add_str (&mv, (const uint8_t *) record->data + ofs,
if (var != NULL)
var_set_missing_values (var, &mv);
}
-
- return true;
}
\f
/* Case reader. */
int retval;
int i;
- if (r->error)
+ if (r->error || !r->sfm_n_vars)
return NULL;
c = case_create (r->proto);
- for (i = 0; i < r->sfm_var_cnt; i++)
+ for (i = 0; i < r->sfm_n_vars; i++)
{
struct sfm_var *sv = &r->sfm_vars[i];
union value *v = case_data_rw_idx (c, sv->case_index);
retval = read_case_number (r, &v->f);
else
{
- uint8_t *s = value_str_rw (v, sv->var_width);
- retval = read_case_string (r, s + sv->offset, sv->segment_width);
+ retval = read_case_string (r, v->s + sv->offset, sv->segment_width);
if (retval == 1)
{
retval = skip_whole_strings (r, ROUND_DOWN (sv->padding, 8));
eof:
if (i != 0)
partial_record (r);
- if (r->case_cnt != -1)
+ if (r->n_cases != -1)
read_error (reader, r);
case_unref (c);
return NULL;
static bool
read_case_number (struct sfm_reader *r, double *d)
{
- if (r->compression == SFM_COMP_NONE)
+ if (r->compression == ANY_COMP_NONE)
{
uint8_t number[8];
if (!try_read_bytes (r, number, sizeof number))
static int
read_opcode (struct sfm_reader *r)
{
- assert (r->compression != SFM_COMP_NONE);
+ assert (r->compression != ANY_COMP_NONE);
for (;;)
{
int opcode;
read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
{
assert (length % 8 == 0);
- if (r->compression == SFM_COMP_NONE)
+ if (r->compression == ANY_COMP_NONE)
return try_read_bytes (r, s, length);
else
{
}
/* Closes TEXT, frees its storage, and issues a final warning
- about suppressed warnings if necesary. */
+ about suppressed warnings if necessary. */
static void
close_text_record (struct sfm_reader *r, struct text_record *text)
{
{
if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
return false;
-
+
*value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
if (*value == NULL)
return false;
text_warn (struct sfm_reader *r, struct text_record *text,
const char *format, ...)
{
- if (text->n_warnings++ < MAX_TEXT_WARNINGS)
+ if (text->n_warnings++ < MAX_TEXT_WARNINGS)
{
va_list args;
char *end;
if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
- return NULL;
+ {
+ if (delimiter != NULL)
+ *delimiter = ss_data (text->buffer)[text->pos-1];
+ return NULL;
+ }
end = &ss_data (token)[ss_length (token)];
if (delimiter != NULL)
static bool
text_match (struct text_record *text, char c)
{
- if (text->buffer.string[text->pos] == c)
+ if (text->pos >= text->buffer.length)
+ return false;
+
+ if (text->buffer.string[text->pos] == c)
{
text->pos++;
return true;
sys_msg (struct sfm_reader *r, off_t offset,
int class, const char *format, va_list args)
{
- struct msg m;
struct string text;
ds_init_empty (&text);
ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
ds_put_vformat (&text, format, args);
- m.category = msg_class_to_category (class);
- m.severity = msg_class_to_severity (class);
- m.file_name = NULL;
- m.first_line = 0;
- m.last_line = 0;
- m.first_column = 0;
- m.last_column = 0;
- m.text = ds_cstr (&text);
-
- msg_emit (&m);
+ struct msg *m = xmalloc (sizeof *m);
+ *m = (struct msg) {
+ .category = msg_class_to_category (class),
+ .severity = msg_class_to_severity (class),
+ .text = ds_steal_cstr (&text),
+ };
+ msg_emit (m);
}
/* Displays a warning for offset OFFSET in the file. */
va_end (args);
}
-/* Displays an error for the current file position,
- marks it as in an error state,
- and aborts reading it using longjmp. */
+/* Displays an error for the current file position and marks it as in an error
+ state. */
static void
sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
{
an error. */
static inline int
read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
- void *buf, size_t byte_cnt)
+ void *buf, size_t n_bytes)
{
- size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
+ size_t bytes_read = fread (buf, 1, n_bytes, r->file);
r->pos += bytes_read;
- if (bytes_read == byte_cnt)
+ if (bytes_read == n_bytes)
return 1;
else if (ferror (r->file))
{
Returns true if successful.
Returns false upon I/O error or if end-of-file is encountered. */
static bool
-read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
{
- return read_bytes_internal (r, false, buf, byte_cnt) == 1;
+ return read_bytes_internal (r, false, buf, n_bytes) == 1;
}
/* Reads BYTE_CNT bytes into BUF.
Returns 0 if an immediate end-of-file is encountered.
Returns -1 if an I/O error or a partial read occurs. */
static int
-try_read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+try_read_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
{
- return read_bytes_internal (r, true, buf, byte_cnt);
+ return read_bytes_internal (r, true, buf, n_bytes);
}
/* Reads a 32-bit signed integer from R and stores its value in host format in
if (fstat (fileno (r->file), &s))
{
- sys_error (ME, 0, _("%s: stat failed (%s)."),
+ sys_error (r, 0, _("%s: stat failed (%s)."),
fh_get_file_name (r->fh), strerror (errno));
return false;
}
}
static int
-read_bytes_zlib (struct sfm_reader *r, void *buf_, size_t byte_cnt)
+read_bytes_zlib (struct sfm_reader *r, void *buf_, size_t n_bytes)
{
uint8_t *buf = buf_;
- if (byte_cnt == 0)
+ if (n_bytes == 0)
return 1;
for (;;)
/* Use already inflated data if there is any. */
if (r->zout_pos < r->zout_end)
{
- unsigned int n = MIN (byte_cnt, r->zout_end - r->zout_pos);
+ unsigned int n = MIN (n_bytes, r->zout_end - r->zout_pos);
memcpy (buf, &r->zout_buf[r->zout_pos], n);
r->zout_pos += n;
- byte_cnt -= n;
+ n_bytes -= n;
buf += n;
- if (byte_cnt == 0)
+ if (n_bytes == 0)
return 1;
}
}
static int
-read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+read_compressed_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
{
- if (r->compression == SFM_COMP_SIMPLE)
- return read_bytes (r, buf, byte_cnt);
+ if (r->compression == ANY_COMP_SIMPLE)
+ return read_bytes (r, buf, n_bytes);
else
{
- int retval = read_bytes_zlib (r, buf, byte_cnt);
+ int retval = read_bytes_zlib (r, buf, n_bytes);
if (retval == 0)
sys_error (r, r->pos, _("Unexpected end of ZLIB compressed data."));
return retval;
}
static int
-try_read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+try_read_compressed_bytes (struct sfm_reader *r, void *buf, size_t n_bytes)
{
- if (r->compression == SFM_COMP_SIMPLE)
- return try_read_bytes (r, buf, byte_cnt);
+ if (r->compression == ANY_COMP_SIMPLE)
+ return try_read_bytes (r, buf, n_bytes);
else
- return read_bytes_zlib (r, buf, byte_cnt);
+ return read_bytes_zlib (r, buf, n_bytes);
}
/* Reads a 64-bit floating-point number from R and returns its
NULL,
NULL,
};
+
+const struct any_reader_class sys_file_reader_class =
+ {
+ N_("SPSS System File"),
+ sfm_detect,
+ sfm_open,
+ sfm_close,
+ sfm_decode,
+ sfm_get_strings,
+ };
projects / pspp / blobdiff