/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006, 2007 Free Software Foundation, Inc.
+ Copyright (C) 1997-2000, 2006-2007, 2009-2011 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
-#include <data/sys-file-reader.h>
-#include <data/sys-file-private.h>
+#include "data/sys-file-reader.h"
+#include "data/sys-file-private.h"
#include <errno.h>
#include <float.h>
#include <setjmp.h>
#include <stdlib.h>
-#include <libpspp/alloc.h>
-#include <libpspp/assertion.h>
-#include <libpspp/message.h>
-#include <libpspp/compiler.h>
-#include <libpspp/magic.h>
-#include <libpspp/misc.h>
-#include <libpspp/pool.h>
-#include <libpspp/str.h>
-#include <libpspp/hash.h>
-#include <libpspp/array.h>
-
-#include <data/case.h>
-#include <data/casereader-provider.h>
-#include <data/casereader.h>
-#include <data/dictionary.h>
-#include <data/file-handle-def.h>
-#include <data/file-name.h>
-#include <data/format.h>
-#include <data/missing-values.h>
-#include <data/value-labels.h>
-#include <data/variable.h>
-#include <data/value.h>
-
-#include "c-ctype.h"
-#include "inttostr.h"
-#include "minmax.h"
-#include "unlocked-io.h"
-#include "xsize.h"
+#include "data/attributes.h"
+#include "data/case.h"
+#include "data/casereader-provider.h"
+#include "data/casereader.h"
+#include "data/dictionary.h"
+#include "data/file-handle-def.h"
+#include "data/file-name.h"
+#include "data/format.h"
+#include "data/identifier.h"
+#include "data/missing-values.h"
+#include "data/mrset.h"
+#include "data/short-names.h"
+#include "data/value-labels.h"
+#include "data/value.h"
+#include "data/variable.h"
+#include "libpspp/array.h"
+#include "libpspp/assertion.h"
+#include "libpspp/compiler.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "libpspp/misc.h"
+#include "libpspp/pool.h"
+#include "libpspp/str.h"
+#include "libpspp/stringi-set.h"
+
+#include "gl/c-ctype.h"
+#include "gl/inttostr.h"
+#include "gl/localcharset.h"
+#include "gl/minmax.h"
+#include "gl/unlocked-io.h"
+#include "gl/xalloc.h"
+#include "gl/xsize.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
#define N_(msgid) (msgid)
+enum
+ {
+ /* subtypes 0-2 unknown */
+ EXT_INTEGER = 3, /* Machine integer info. */
+ EXT_FLOAT = 4, /* Machine floating-point info. */
+ EXT_VAR_SETS = 5, /* Variable sets. */
+ EXT_DATE = 6, /* DATE. */
+ EXT_MRSETS = 7, /* Multiple response sets. */
+ EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */
+ /* subtypes 9-10 unknown */
+ EXT_DISPLAY = 11, /* Variable display parameters. */
+ /* subtype 12 unknown */
+ EXT_LONG_NAMES = 13, /* Long variable names. */
+ EXT_LONG_STRINGS = 14, /* Long strings. */
+ /* subtype 15 unknown */
+ EXT_NCASES = 16, /* Extended number of cases. */
+ EXT_FILE_ATTRS = 17, /* Data file attributes. */
+ EXT_VAR_ATTRS = 18, /* Variable attributes. */
+ EXT_MRSETS2 = 19, /* Multiple response sets (extended). */
+ EXT_ENCODING = 20, /* Character encoding. */
+ EXT_LONG_LABELS = 21 /* Value labels for long strings. */
+ };
+
+struct sfm_var_record
+ {
+ off_t pos;
+ int width;
+ char name[8];
+ int print_format;
+ int write_format;
+ int missing_value_code;
+ uint8_t missing[24];
+ char *label;
+ struct variable *var;
+ };
+
+struct sfm_value_label
+ {
+ uint8_t value[8];
+ char *label;
+ };
+
+struct sfm_value_label_record
+ {
+ off_t pos;
+ struct sfm_value_label *labels;
+ size_t n_labels;
+
+ int *vars;
+ size_t n_vars;
+ };
+
+struct sfm_document_record
+ {
+ off_t pos;
+ char *documents;
+ size_t n_lines;
+ };
+
+struct sfm_extension_record
+ {
+ off_t pos; /* Starting offset in file. */
+ size_t size; /* Size of data elements. */
+ size_t count; /* Number of data elements. */
+ void *data; /* Contents. */
+ };
+
/* System file reader. */
struct sfm_reader
{
/* File state. */
struct file_handle *fh; /* File handle. */
+ struct fh_lock *lock; /* Mutual exclusion for file handle. */
FILE *file; /* File stream. */
+ off_t pos; /* Position in file. */
bool error; /* I/O or corruption error? */
- size_t value_cnt; /* Number of "union value"s in struct case. */
+ struct caseproto *proto; /* Format of output cases. */
/* File format. */
enum integer_format integer_format; /* On-disk integer format. */
enum float_format float_format; /* On-disk floating point format. */
- int oct_cnt; /* Number of 8-byte units per case. */
struct sfm_var *sfm_vars; /* Variables. */
size_t sfm_var_cnt; /* Number of variables. */
casenumber case_cnt; /* Number of cases */
- bool has_long_var_names; /* File has a long variable name map */
+ const char *encoding; /* String encoding. */
/* Decompression. */
bool compressed; /* File is compressed? */
double bias; /* Compression bias, usually 100.0. */
uint8_t opcodes[8]; /* Current block of opcodes. */
size_t opcode_idx; /* Next opcode to interpret, 8 if none left. */
+ bool corruption_warning; /* Warned about possible corruption? */
};
-static struct casereader_class sys_file_casereader_class;
+static const struct casereader_class sys_file_casereader_class;
static bool close_reader (struct sfm_reader *);
-static struct variable **make_var_by_value_idx (struct sfm_reader *,
- struct dictionary *);
-static struct variable *lookup_var_by_value_idx (struct sfm_reader *,
- struct variable **,
- int value_idx);
-
-static void sys_warn (struct sfm_reader *, const char *, ...)
- PRINTF_FORMAT (2, 3);
-
-static void sys_error (struct sfm_reader *, const char *, ...)
- PRINTF_FORMAT (2, 3)
+static struct variable *lookup_var_by_index (struct sfm_reader *, off_t,
+ const struct sfm_var_record *,
+ size_t n, int idx);
+
+static void sys_msg (struct sfm_reader *r, off_t, int class,
+ const char *format, va_list args)
+ PRINTF_FORMAT (4, 0);
+static void sys_warn (struct sfm_reader *, off_t, const char *, ...)
+ PRINTF_FORMAT (3, 4);
+static void sys_error (struct sfm_reader *, off_t, const char *, ...)
+ PRINTF_FORMAT (3, 4)
NO_RETURN;
static void read_bytes (struct sfm_reader *, void *, size_t);
static void read_string (struct sfm_reader *, char *, size_t);
static void skip_bytes (struct sfm_reader *, size_t);
-static struct variable_to_value_map *open_variable_to_value_map (
- struct sfm_reader *, size_t size);
-static void close_variable_to_value_map (struct sfm_reader *r,
- struct variable_to_value_map *);
-static bool read_variable_to_value_map (struct sfm_reader *,
- struct dictionary *,
- struct variable_to_value_map *,
- struct variable **var, char **value,
- int *warning_cnt);
+static int parse_int (struct sfm_reader *, const void *data, size_t ofs);
+static double parse_float (struct sfm_reader *, const void *data, size_t ofs);
+
+static void read_variable_record (struct sfm_reader *,
+ struct sfm_var_record *);
+static void read_value_label_record (struct sfm_reader *,
+ struct sfm_value_label_record *,
+ size_t n_vars);
+static struct sfm_document_record *read_document_record (struct sfm_reader *);
+static struct sfm_extension_record *read_extension_record (
+ struct sfm_reader *, int subtype);
+static void skip_extension_record (struct sfm_reader *, int subtype);
+
+static const char *choose_encoding (
+ struct sfm_reader *,
+ const struct sfm_extension_record *ext_integer,
+ const struct sfm_extension_record *ext_encoding);
+
+static struct text_record *open_text_record (
+ struct sfm_reader *, const struct sfm_extension_record *);
+static void close_text_record (struct sfm_reader *,
+ struct text_record *);
+static bool read_variable_to_value_pair (struct sfm_reader *,
+ struct dictionary *,
+ struct text_record *,
+ struct variable **var, char **value);
+static void text_warn (struct sfm_reader *r, struct text_record *text,
+ const char *format, ...)
+ PRINTF_FORMAT (3, 4);
+static char *text_get_token (struct text_record *,
+ struct substring delimiters, char *delimiter);
+static bool text_match (struct text_record *, char c);
+static bool text_read_variable_name (struct sfm_reader *, struct dictionary *,
+ struct text_record *,
+ struct substring delimiters,
+ struct variable **);
+static bool text_read_short_name (struct sfm_reader *, struct dictionary *,
+ struct text_record *,
+ struct substring delimiters,
+ struct variable **);
+static const char *text_parse_counted_string (struct sfm_reader *,
+ struct text_record *);
+static size_t text_pos (const struct text_record *);
static bool close_reader (struct sfm_reader *r);
\f
WRITE_FORMAT
};
-static void read_header (struct sfm_reader *, struct dictionary *,
- int *weight_idx, int *claimed_oct_cnt,
- struct sfm_read_info *);
-static void read_variable_record (struct sfm_reader *, struct dictionary *,
- int *format_warning_cnt);
-static void parse_format_spec (struct sfm_reader *, unsigned int,
- enum which_format, struct variable *,
- int *format_warning_cnt);
-static void setup_weight (struct sfm_reader *, int weight_idx,
- struct variable **var_by_value_idx,
+static void read_header (struct sfm_reader *, int *weight_idx,
+ int *claimed_oct_cnt, struct sfm_read_info *,
+ char **file_labelp);
+static void parse_file_label (struct sfm_reader *, const char *file_label,
+ struct dictionary *);
+static void parse_variable_records (struct sfm_reader *, struct dictionary *,
+ struct sfm_var_record *, size_t n);
+static void parse_format_spec (struct sfm_reader *, off_t pos,
+ unsigned int format, enum which_format,
+ struct variable *, int *format_warning_cnt);
+static void parse_document (struct dictionary *, struct sfm_document_record *);
+static void parse_display_parameters (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
+static void parse_machine_integer_info (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct sfm_read_info *);
+static void parse_machine_float_info (struct sfm_reader *,
+ const struct sfm_extension_record *);
+static void parse_mrsets (struct sfm_reader *,
+ const struct sfm_extension_record *,
struct dictionary *);
-static void read_documents (struct sfm_reader *, struct dictionary *);
-static void read_value_labels (struct sfm_reader *, struct dictionary *,
- struct variable **var_by_value_idx);
-
-static void read_extension_record (struct sfm_reader *, struct dictionary *,
- struct sfm_read_info *);
-static void read_machine_integer_info (struct sfm_reader *,
- size_t size, size_t count,
- struct sfm_read_info *);
-static void read_machine_float_info (struct sfm_reader *,
- size_t size, size_t count);
-static void read_display_parameters (struct sfm_reader *,
- size_t size, size_t count,
+static void parse_long_var_name_map (struct sfm_reader *,
+ const struct sfm_extension_record *,
struct dictionary *);
-static void read_long_var_name_map (struct sfm_reader *,
- size_t size, size_t count,
- struct dictionary *);
-static void read_long_string_map (struct sfm_reader *,
- size_t size, size_t count,
- struct dictionary *);
-
+static void parse_long_string_map (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
+static void parse_value_labels (struct sfm_reader *, struct dictionary *,
+ const struct sfm_var_record *,
+ size_t n_var_recs,
+ const struct sfm_value_label_record *);
+static void parse_data_file_attributes (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
+static void parse_variable_attributes (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
+static void parse_long_string_value_labels (struct sfm_reader *,
+ const struct sfm_extension_record *,
+ struct dictionary *);
/* Opens the system file designated by file handle FH for
reading. Reads the system file's dictionary into *DICT.
If INFO is non-null, then it receives additional info about the
system file. */
struct casereader *
-sfm_open_reader (struct file_handle *fh, struct dictionary **dict,
+sfm_open_reader (struct file_handle *fh, struct dictionary **dictp,
struct sfm_read_info *volatile info)
{
struct sfm_reader *volatile r = NULL;
- struct variable **var_by_value_idx;
struct sfm_read_info local_info;
- int format_warning_cnt = 0;
+
+ struct sfm_var_record *vars;
+ size_t n_vars, allocated_vars;
+
+ struct sfm_value_label_record *labels;
+ size_t n_labels, allocated_labels;
+
+ struct sfm_document_record *document;
+
+ struct sfm_extension_record *extensions[32];
+
int weight_idx;
int claimed_oct_cnt;
- int rec_type;
+ char *file_label;
- if (!fh_open (fh, FH_REF_FILE, "system file", "rs"))
- return NULL;
-
- *dict = dict_create ();
+ struct dictionary *dict = NULL;
+ size_t i;
/* Create and initialize reader. */
r = pool_create_container (struct sfm_reader, pool);
- r->fh = fh;
- r->file = fn_open (fh_get_file_name (fh), "rb");
+ r->fh = fh_ref (fh);
+ r->lock = NULL;
+ r->file = NULL;
+ r->pos = 0;
r->error = false;
- r->oct_cnt = 0;
- r->has_long_var_names = false;
r->opcode_idx = sizeof r->opcodes;
+ r->corruption_warning = false;
+
+ /* TRANSLATORS: this fragment will be interpolated into
+ messages in fh_lock() that identify types of files. */
+ r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false);
+ if (r->lock == NULL)
+ goto error;
+
+ r->file = fn_open (fh_get_file_name (fh), "rb");
+ if (r->file == NULL)
+ {
+ msg (ME, _("Error opening `%s' for reading as a system file: %s."),
+ fh_get_file_name (r->fh), strerror (errno));
+ goto error;
+ }
/* Initialize info. */
if (info == NULL)
memset (info, 0, sizeof *info);
if (setjmp (r->bail_out))
- {
- close_reader (r);
- dict_destroy (*dict);
- *dict = NULL;
- return NULL;
- }
-
- if (r->file == NULL)
- {
- msg (ME, _("Error opening \"%s\" for reading as a system file: %s."),
- fh_get_file_name (r->fh), strerror (errno));
- longjmp (r->bail_out, 1);
- }
+ goto error;
/* Read header. */
- read_header (r, *dict, &weight_idx, &claimed_oct_cnt, info);
+ read_header (r, &weight_idx, &claimed_oct_cnt, info, &file_label);
- /* Read all the variable definition records. */
- rec_type = read_int (r);
- while (rec_type == 2)
- {
- read_variable_record (r, *dict, &format_warning_cnt);
- rec_type = read_int (r);
- }
+ vars = NULL;
+ n_vars = allocated_vars = 0;
+
+ labels = NULL;
+ n_labels = allocated_labels = 0;
- /* Figure out the case format. */
- var_by_value_idx = make_var_by_value_idx (r, *dict);
- setup_weight (r, weight_idx, var_by_value_idx, *dict);
+ document = NULL;
- /* Read all the rest of the dictionary records. */
- while (rec_type != 999)
+ memset (extensions, 0, sizeof extensions);
+
+ for (;;)
{
- switch (rec_type)
+ int subtype;
+ int type;
+
+ type = read_int (r);
+ if (type == 999)
{
+ read_int (r); /* Skip filler. */
+ break;
+ }
+
+ switch (type)
+ {
+ case 2:
+ if (n_vars >= allocated_vars)
+ vars = pool_2nrealloc (r->pool, vars, &allocated_vars,
+ sizeof *vars);
+ read_variable_record (r, &vars[n_vars++]);
+ break;
+
case 3:
- read_value_labels (r, *dict, var_by_value_idx);
+ if (n_labels >= allocated_labels)
+ labels = pool_2nrealloc (r->pool, labels, &allocated_labels,
+ sizeof *labels);
+ read_value_label_record (r, &labels[n_labels++], n_vars);
break;
case 4:
- sys_error (r, _("Misplaced type 4 record."));
+ /* A Type 4 record is always immediately after a type 3 record,
+ so the code for type 3 records reads the type 4 record too. */
+ sys_error (r, r->pos, _("Misplaced type 4 record."));
case 6:
- read_documents (r, *dict);
+ if (document != NULL)
+ sys_error (r, r->pos, _("Duplicate type 6 (document) record."));
+ document = read_document_record (r);
break;
case 7:
- read_extension_record (r, *dict, info);
+ subtype = read_int (r);
+ if (subtype < 0 || subtype >= sizeof extensions / sizeof *extensions)
+ {
+ sys_warn (r, r->pos,
+ _("Unrecognized record type 7, subtype %d. Please "
+ "send a copy of this file, and the syntax which "
+ "created it to %s."),
+ subtype, PACKAGE_BUGREPORT);
+ skip_extension_record (r, subtype);
+ }
+ else if (extensions[subtype] != NULL)
+ {
+ sys_warn (r, r->pos,
+ _("Record type 7, subtype %d found here has the same "
+ "type as the record found near offset 0x%llx. "
+ "Please send a copy of this file, and the syntax "
+ "which created it to %s."),
+ subtype, (long long int) extensions[subtype]->pos,
+ PACKAGE_BUGREPORT);
+ skip_extension_record (r, subtype);
+ }
+ else
+ extensions[subtype] = read_extension_record (r, subtype);
break;
default:
- sys_error (r, _("Unrecognized record type %d."), rec_type);
+ sys_error (r, r->pos, _("Unrecognized record type %d."), type);
+ goto error;
}
- rec_type = read_int (r);
}
+ /* Now actually parse what we read.
- if ( ! r->has_long_var_names )
- {
- int i;
- for (i = 0; i < dict_get_var_cnt (*dict); i++)
- {
- struct variable *var = dict_get_var (*dict, i);
- char short_name [SHORT_NAME_LEN + 1];
- char long_name [SHORT_NAME_LEN + 1];
+ First, figure out the correct character encoding, because this determines
+ how the rest of the header data is to be interpreted. */
+ dict = dict_create ();
+ r->encoding = choose_encoding (r, extensions[EXT_INTEGER],
+ extensions[EXT_ENCODING]);
+ dict_set_encoding (dict, r->encoding);
- strcpy (short_name, var_get_name (var));
+ /* These records don't use variables at all. */
+ if (document != NULL)
+ parse_document (dict, document);
- strcpy (long_name, short_name);
- str_lowercase (long_name);
+ if (extensions[EXT_INTEGER] != NULL)
+ parse_machine_integer_info (r, extensions[EXT_INTEGER], info);
- /* Set long name. Renaming a variable may clear the short
- name, but we want to retain it, so re-set it
- explicitly. */
- dict_rename_var (*dict, var, long_name);
- var_set_short_name (var, 0, short_name);
- }
+ if (extensions[EXT_FLOAT] != NULL)
+ parse_machine_float_info (r, extensions[EXT_FLOAT]);
+
+ if (extensions[EXT_FILE_ATTRS] != NULL)
+ parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict);
+
+ parse_file_label (r, file_label, dict);
- r->has_long_var_names = true;
+ /* Parse the variable records, the basis of almost everything else. */
+ parse_variable_records (r, dict, vars, n_vars);
+
+ /* Parse value labels and the weight variable immediately after the variable
+ records. These records use indexes into var_recs[], so we must parse them
+ before those indexes become invalidated by very long string variables. */
+ for (i = 0; i < n_labels; i++)
+ parse_value_labels (r, dict, vars, n_vars, &labels[i]);
+ if (weight_idx != 0)
+ {
+ struct variable *weight_var;
+
+ weight_var = lookup_var_by_index (r, 76, vars, n_vars, weight_idx);
+ if (var_is_numeric (weight_var))
+ dict_set_weight (dict, weight_var);
+ else
+ sys_error (r, -1, _("Weighting variable must be numeric "
+ "(not string variable `%s')."),
+ var_get_name (weight_var));
}
- /* Read record 999 data, which is just filler. */
- read_int (r);
+ if (extensions[EXT_DISPLAY] != NULL)
+ parse_display_parameters (r, extensions[EXT_DISPLAY], dict);
+
+ /* The following records use short names, so they need to be parsed before
+ parse_long_var_name_map() changes short names to long names. */
+ if (extensions[EXT_MRSETS] != NULL)
+ parse_mrsets (r, extensions[EXT_MRSETS], dict);
+
+ if (extensions[EXT_MRSETS2] != NULL)
+ parse_mrsets (r, extensions[EXT_MRSETS2], dict);
+
+ if (extensions[EXT_LONG_STRINGS] != NULL)
+ parse_long_string_map (r, extensions[EXT_LONG_STRINGS], dict);
+
+ /* Now rename variables to their long names. */
+ parse_long_var_name_map (r, extensions[EXT_LONG_NAMES], dict);
+
+ /* The following records use long names, so they need to follow renaming. */
+ if (extensions[EXT_VAR_ATTRS] != NULL)
+ parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict);
+
+ if (extensions[EXT_LONG_LABELS] != NULL)
+ parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict);
/* Warn if the actual amount of data per case differs from the
amount that the header claims. SPSS version 13 gets this
wrong when very long strings are involved, so don't warn in
that case. */
- if (claimed_oct_cnt != -1 && claimed_oct_cnt != r->oct_cnt
+ if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars
&& info->version_major != 13)
- sys_warn (r, _("File header claims %d variable positions but "
- "%d were read from file."),
- claimed_oct_cnt, r->oct_cnt);
+ sys_warn (r, -1, _("File header claims %d variable positions but "
+ "%d were read from file."),
+ claimed_oct_cnt, n_vars);
/* Create an index of dictionary variable widths for
sfm_read_case to use. We cannot use the `struct variable's
from the dictionary we created, because the caller owns the
dictionary and may destroy or modify its variables. */
- sfm_dictionary_to_sfm_vars (*dict, &r->sfm_vars, &r->sfm_var_cnt);
+ sfm_dictionary_to_sfm_vars (dict, &r->sfm_vars, &r->sfm_var_cnt);
pool_register (r->pool, free, r->sfm_vars);
+ r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool);
- pool_free (r->pool, var_by_value_idx);
- r->value_cnt = dict_get_next_value_idx (*dict);
+ *dictp = dict;
return casereader_create_sequential
- (NULL, r->value_cnt,
+ (NULL, r->proto,
r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt,
&sys_file_casereader_class, r);
+
+error:
+ close_reader (r);
+ dict_destroy (dict);
+ *dictp = NULL;
+ return NULL;
}
/* Closes a system file after we're done with it.
{
if (fn_close (fh_get_file_name (r->fh), r->file) == EOF)
{
- msg (ME, _("Error closing system file \"%s\": %s."),
+ msg (ME, _("Error closing system file `%s': %s."),
fh_get_file_name (r->fh), strerror (errno));
r->error = true;
}
r->file = NULL;
}
- if (r->fh != NULL)
- fh_close (r->fh, "system file", "rs");
+ fh_unlock (r->lock);
+ fh_unref (r->fh);
error = r->error;
pool_destroy (r->pool);
return !strcmp ("$FL2", rec_type);
}
\f
-/* Reads the global header of the system file.
- Sets DICT's file label to the system file's label.
- Sets *WEIGHT_IDX to 0 if the system file is unweighted,
- or to the value index of the weight variable otherwise.
- Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units)
- per case that the file claims to have (although it is not
- always correct).
- Initializes INFO with header information. */
+/* Reads the global header of the system file. Sets *WEIGHT_IDX to 0 if the
+ system file is unweighted, or to the value index of the weight variable
+ otherwise. Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units) per
+ case that the file claims to have (although it is not always correct).
+ Initializes INFO with header information. Stores the file label as a string
+ in dictionary encoding into *FILE_LABELP. */
static void
-read_header (struct sfm_reader *r, struct dictionary *dict,
- int *weight_idx, int *claimed_oct_cnt,
- struct sfm_read_info *info)
+read_header (struct sfm_reader *r, int *weight_idx,
+ int *claimed_oct_cnt, struct sfm_read_info *info,
+ char **file_labelp)
{
char rec_type[5];
char eye_catcher[61];
char creation_date[10];
char creation_time[9];
char file_label[65];
- struct substring file_label_ss;
struct substring product;
read_string (r, rec_type, sizeof rec_type);
read_string (r, eye_catcher, sizeof eye_catcher);
if (strcmp ("$FL2", rec_type) != 0)
- sys_error (r, _("This is not an SPSS system file."));
+ sys_error (r, 0, _("This is not an SPSS system file."));
/* Identify integer format. */
read_bytes (r, raw_layout_code, sizeof raw_layout_code);
&r->integer_format))
|| (r->integer_format != INTEGER_MSB_FIRST
&& r->integer_format != INTEGER_LSB_FIRST))
- sys_error (r, _("This is not an SPSS system file."));
+ sys_error (r, 64, _("This is not an SPSS system file."));
*claimed_oct_cnt = read_int (r);
if (*claimed_oct_cnt < 0 || *claimed_oct_cnt > INT_MAX / 16)
if ( r->case_cnt > INT_MAX / 2)
r->case_cnt = -1;
-
/* Identify floating-point format and obtain compression bias. */
read_bytes (r, raw_bias, sizeof raw_bias);
if (float_identify (100.0, raw_bias, sizeof raw_bias, &r->float_format) == 0)
{
- sys_warn (r, _("Compression bias (%g) is not the usual "
- "value of 100, or system file uses unrecognized "
- "floating-point format."),
- r->bias);
+ uint8_t zero_bias[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
+ if (memcmp (raw_bias, zero_bias, 8))
+ sys_warn (r, r->pos - 8,
+ _("Compression bias is not the usual "
+ "value of 100, or system file uses unrecognized "
+ "floating-point format."));
+ else
+ {
+ /* Some software is known to write all-zeros to this
+ field. Such software also writes floating-point
+ numbers in the format that we expect by default
+ (it seems that all software most likely does, in
+ reality), so don't warn in this case. */
+ }
+
if (r->integer_format == INTEGER_MSB_FIRST)
r->float_format = FLOAT_IEEE_DOUBLE_BE;
else
read_string (r, file_label, sizeof file_label);
skip_bytes (r, 3);
- file_label_ss = ss_cstr (file_label);
- ss_trim (&file_label_ss, ss_cstr (" "));
- if (!ss_is_empty (file_label_ss))
- {
- ss_data (file_label_ss)[ss_length (file_label_ss)] = '\0';
- dict_set_label (dict, ss_data (file_label_ss));
- }
-
strcpy (info->creation_date, creation_date);
strcpy (info->creation_time, creation_time);
info->integer_format = r->integer_format;
ss_trim (&product, ss_cstr (" "));
str_copy_buf_trunc (info->product, sizeof info->product,
ss_data (product), ss_length (product));
+
+ *file_labelp = pool_strdup0 (r->pool, file_label, sizeof file_label - 1);
}
-/* Reads a variable (type 2) record from R and adds the
- corresponding variable to DICT.
- Also skips past additional variable records for long string
- variables. */
+/* Reads a variable (type 2) record from R into RECORD. */
static void
-read_variable_record (struct sfm_reader *r, struct dictionary *dict,
- int *format_warning_cnt)
+read_variable_record (struct sfm_reader *r, struct sfm_var_record *record)
{
- int width;
int has_variable_label;
- int missing_value_code;
- int print_format;
- int write_format;
- char name[9];
- struct variable *var;
- int nv;
+ memset (record, 0, sizeof *record);
- width = read_int (r);
+ record->pos = r->pos;
+ record->width = read_int (r);
has_variable_label = read_int (r);
- missing_value_code = read_int (r);
- print_format = read_int (r);
- write_format = read_int (r);
- read_string (r, name, sizeof name);
- name[strcspn (name, " ")] = '\0';
-
- /* Check variable name. */
- if (name[0] == '$' || name[0] == '#')
- sys_error (r, "Variable name begins with invalid character `%c'.",
- name[0]);
- if (!var_is_plausible_name (name, false))
- sys_error (r, _("Invalid variable name `%s'."), name);
-
- /* Create variable. */
- if (width < 0 || width > 255)
- sys_error (r, _("Bad variable width %d."), width);
- var = dict_create_var (dict, name, width);
- if (var == NULL)
- sys_error (r,
- _("Duplicate variable name `%s' within system file."),
- name);
-
- /* Set the short name the same as the long name. */
- var_set_short_name (var, 0, var_get_name (var));
-
- /* Get variable label, if any. */
- if (has_variable_label != 0 && has_variable_label != 1)
- sys_error (r, _("Variable label indicator field is not 0 or 1."));
+ record->missing_value_code = read_int (r);
+ record->print_format = read_int (r);
+ record->write_format = read_int (r);
+ read_bytes (r, record->name, sizeof record->name);
+
if (has_variable_label == 1)
{
- size_t len;
- char label[255 + 1];
+ enum { MAX_LABEL_LEN = 255 };
+ size_t len, read_len;
len = read_int (r);
- if (len >= sizeof label)
- sys_error (r, _("Variable %s has label of invalid length %u."),
- name, (unsigned int) len);
- read_string (r, label, len + 1);
- var_set_label (var, label);
+ /* Read up to MAX_LABEL_LEN bytes of label. */
+ read_len = MIN (MAX_LABEL_LEN, len);
+ record->label = xmalloc (read_len + 1);
+ read_string (r, record->label, read_len + 1);
+
+ /* Skip unread label bytes. */
+ skip_bytes (r, len - read_len);
+
+ /* Skip label padding up to multiple of 4 bytes. */
skip_bytes (r, ROUND_UP (len, 4) - len);
}
+ else if (has_variable_label != 0)
+ sys_error (r, record->pos,
+ _("Variable label indicator field is not 0 or 1."));
/* Set missing values. */
- if (missing_value_code != 0)
+ if (record->missing_value_code != 0)
{
- struct missing_values mv;
- int i;
-
- mv_init (&mv, var_get_width (var));
- if (var_is_numeric (var))
+ int code = record->missing_value_code;
+ if (record->width == 0)
{
- if (missing_value_code < -3 || missing_value_code > 3
- || missing_value_code == -1)
- sys_error (r, _("Numeric missing value indicator field is not "
- "-3, -2, 0, 1, 2, or 3."));
- if (missing_value_code < 0)
- {
- double low = read_float (r);
- double high = read_float (r);
- mv_add_num_range (&mv, low, high);
- missing_value_code = -missing_value_code - 2;
- }
- for (i = 0; i < missing_value_code; i++)
- mv_add_num (&mv, read_float (r));
+ if (code < -3 || code > 3 || code == -1)
+ sys_error (r, record->pos,
+ _("Numeric missing value indicator field is not "
+ "-3, -2, 0, 1, 2, or 3."));
}
- else if (var_get_width (var) <= MAX_SHORT_STRING)
+ else
{
- if (missing_value_code < 1 || missing_value_code > 3)
- sys_error (r, _("String missing value indicator field is not "
- "0, 1, 2, or 3."));
- for (i = 0; i < missing_value_code; i++)
- {
- char string[9];
- read_string (r, string, sizeof string);
- mv_add_str (&mv, string);
- }
+ if (code < 1 || code > 3)
+ sys_error (r, record->pos,
+ _("String missing value indicator field is not "
+ "0, 1, 2, or 3."));
}
- else
- sys_error (r, _("Long string variable %s may not have missing "
- "values."), name);
- var_set_missing_values (var, &mv);
+
+ read_bytes (r, record->missing, 8 * abs (code));
+ }
+}
+
+/* Reads value labels from R into RECORD. */
+static void
+read_value_label_record (struct sfm_reader *r,
+ struct sfm_value_label_record *record,
+ size_t n_vars)
+{
+ size_t i;
+
+ /* Read type 3 record. */
+ record->pos = r->pos;
+ record->n_labels = read_int (r);
+ if (record->n_labels > SIZE_MAX / sizeof *record->labels)
+ sys_error (r, r->pos - 4, _("Invalid number of labels %zu."),
+ record->n_labels);
+ record->labels = pool_nmalloc (r->pool, record->n_labels,
+ sizeof *record->labels);
+ for (i = 0; i < record->n_labels; i++)
+ {
+ struct sfm_value_label *label = &record->labels[i];
+ unsigned char label_len;
+ size_t padded_len;
+
+ read_bytes (r, label->value, sizeof label->value);
+
+ /* Read label length. */
+ read_bytes (r, &label_len, sizeof label_len);
+ padded_len = ROUND_UP (label_len + 1, 8);
+
+ /* Read label, padding. */
+ label->label = pool_malloc (r->pool, padded_len + 1);
+ read_bytes (r, label->label, padded_len - 1);
+ label->label[label_len] = '\0';
}
- /* Set formats. */
- parse_format_spec (r, print_format, PRINT_FORMAT, var, format_warning_cnt);
- parse_format_spec (r, write_format, WRITE_FORMAT, var, format_warning_cnt);
+ /* Read record type of type 4 record. */
+ if (read_int (r) != 4)
+ sys_error (r, r->pos - 4,
+ _("Variable index record (type 4) does not immediately "
+ "follow value label record (type 3) as it should."));
+
+ /* Read number of variables associated with value label from type 4
+ record. */
+ record->n_vars = read_int (r);
+ if (record->n_vars < 1 || record->n_vars > n_vars)
+ sys_error (r, r->pos - 4,
+ _("Number of variables associated with a value label (%d) "
+ "is not between 1 and the number of variables (%zu)."),
+ record->n_vars, n_vars);
+ record->vars = pool_nmalloc (r->pool, record->n_vars, sizeof *record->vars);
+ for (i = 0; i < record->n_vars; i++)
+ record->vars[i] = read_int (r);
+}
+
+/* Reads a document record from R and returns it. */
+static struct sfm_document_record *
+read_document_record (struct sfm_reader *r)
+{
+ struct sfm_document_record *record;
+ int n_lines;
+
+ record = pool_malloc (r->pool, sizeof *record);
+ record->pos = r->pos;
+
+ n_lines = read_int (r);
+ if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH)
+ sys_error (r, record->pos,
+ _("Number of document lines (%d) "
+ "must be greater than 0 and less than %d."),
+ n_lines, INT_MAX / DOC_LINE_LENGTH);
- /* Account for values.
- Skip long string continuation records, if any. */
- nv = width == 0 ? 1 : DIV_RND_UP (width, 8);
- r->oct_cnt += nv;
- if (width > 8)
+ record->n_lines = n_lines;
+ record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines);
+ read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines);
+
+ return record;
+}
+
+static void
+read_extension_record_header (struct sfm_reader *r, int subtype,
+ struct sfm_extension_record *record)
+{
+ record->pos = r->pos;
+ record->size = read_int (r);
+ record->count = read_int (r);
+
+ /* Check that SIZE * COUNT + 1 doesn't overflow. Adding 1
+ allows an extra byte for a null terminator, used by some
+ extension processing routines. */
+ if (record->size != 0
+ && size_overflow_p (xsum (1, xtimes (record->count, record->size))))
+ sys_error (r, record->pos, "Record type 7 subtype %d too large.", subtype);
+}
+
+/* Reads an extension record from R into RECORD. */
+static struct sfm_extension_record *
+read_extension_record (struct sfm_reader *r, int subtype)
+{
+ struct extension_record_type
{
- int i;
+ int subtype;
+ int size;
+ int count;
+ };
- for (i = 1; i < nv; i++)
+ static const struct extension_record_type types[] =
+ {
+ /* Implemented record types. */
+ { EXT_INTEGER, 4, 8 },
+ { EXT_FLOAT, 8, 3 },
+ { EXT_MRSETS, 1, 0 },
+ { EXT_DISPLAY, 4, 0 },
+ { EXT_LONG_NAMES, 1, 0 },
+ { EXT_LONG_STRINGS, 1, 0 },
+ { EXT_NCASES, 8, 2 },
+ { EXT_FILE_ATTRS, 1, 0 },
+ { EXT_VAR_ATTRS, 1, 0 },
+ { EXT_MRSETS2, 1, 0 },
+ { EXT_ENCODING, 1, 0 },
+ { EXT_LONG_LABELS, 1, 0 },
+
+ /* Ignored record types. */
+ { EXT_VAR_SETS, 0, 0 },
+ { EXT_DATE, 0, 0 },
+ { EXT_DATA_ENTRY, 0, 0 },
+ };
+
+ const struct extension_record_type *type;
+ struct sfm_extension_record *record;
+ size_t n_bytes;
+
+ record = pool_malloc (r->pool, sizeof *record);
+ read_extension_record_header (r, subtype, record);
+ n_bytes = record->count * record->size;
+
+ for (type = types; type < &types[sizeof types / sizeof *types]; type++)
+ if (subtype == type->subtype)
+ {
+ if (type->size > 0 && record->size != type->size)
+ sys_warn (r, record->pos,
+ _("Record type 7, subtype %d has bad size %zu "
+ "(expected %d)."), subtype, record->size, type->size);
+ else if (type->count > 0 && record->count != type->count)
+ sys_warn (r, record->pos,
+ _("Record type 7, subtype %d has bad count %zu "
+ "(expected %d)."), subtype, record->count, type->count);
+ else if (type->count == 0 && type->size == 0)
+ {
+ /* Ignore this record. */
+ }
+ else
+ {
+ char *data = pool_malloc (r->pool, n_bytes + 1);
+ data[n_bytes] = '\0';
+
+ record->data = data;
+ read_bytes (r, record->data, n_bytes);
+ return record;
+ }
+
+ goto skip;
+ }
+
+ sys_warn (r, record->pos,
+ _("Unrecognized record type 7, subtype %d. Please send a "
+ "copy of this file, and the syntax which created it to %s."),
+ subtype, PACKAGE_BUGREPORT);
+
+skip:
+ skip_bytes (r, n_bytes);
+ return NULL;
+}
+
+static void
+skip_extension_record (struct sfm_reader *r, int subtype)
+{
+ struct sfm_extension_record record;
+
+ read_extension_record_header (r, subtype, &record);
+ skip_bytes (r, record.count * record.size);
+}
+
+static void
+parse_file_label (struct sfm_reader *r, const char *file_label,
+ struct dictionary *dict)
+{
+ char *utf8_file_label;
+ size_t file_label_len;
+
+ utf8_file_label = recode_string_pool ("UTF-8", dict_get_encoding (dict),
+ file_label, -1, r->pool);
+ file_label_len = strlen (utf8_file_label);
+ while (file_label_len > 0 && utf8_file_label[file_label_len - 1] == ' ')
+ file_label_len--;
+ utf8_file_label[file_label_len] = '\0';
+ dict_set_label (dict, utf8_file_label);
+}
+
+/* Reads a variable (type 2) record from R and adds the
+ corresponding variable to DICT.
+ Also skips past additional variable records for long string
+ variables. */
+static void
+parse_variable_records (struct sfm_reader *r, struct dictionary *dict,
+ struct sfm_var_record *var_recs, size_t n_var_recs)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ struct sfm_var_record *rec;
+ int n_warnings = 0;
+
+ for (rec = var_recs; rec < &var_recs[n_var_recs]; )
+ {
+ struct variable *var;
+ size_t n_values;
+ char *name;
+ size_t i;
+
+ name = recode_string_pool ("UTF-8", dict_encoding,
+ rec->name, 8, r->pool);
+ name[strcspn (name, " ")] = '\0';
+
+ if (!dict_id_is_valid (dict, name, false)
+ || name[0] == '$' || name[0] == '#')
+ sys_error (r, rec->pos, _("Invalid variable name `%s'."), name);
+
+ if (rec->width < 0 || rec->width > 255)
+ sys_error (r, rec->pos,
+ _("Bad width %d for variable %s."), rec->width, name);
+
+ var = rec->var = dict_create_var (dict, name, rec->width);
+ if (var == NULL)
+ sys_error (r, rec->pos, _("Duplicate variable name `%s'."), name);
+
+ /* Set the short name the same as the long name. */
+ var_set_short_name (var, 0, name);
+
+ /* Get variable label, if any. */
+ if (rec->label)
{
- /* Check for record type 2 and width -1. */
- if (read_int (r) != 2 || read_int (r) != -1)
- sys_error (r, _("Missing string continuation record."));
-
- /* Skip and ignore remaining continuation data. */
- has_variable_label = read_int (r);
- missing_value_code = read_int (r);
- print_format = read_int (r);
- write_format = read_int (r);
- read_string (r, name, sizeof name);
-
- /* Variable label fields on continuation records have
- been spotted in system files created by "SPSS Power
- Macintosh Release 6.1". */
- if (has_variable_label)
- skip_bytes (r, ROUND_UP (read_int (r), 4));
+ char *utf8_label;
+
+ utf8_label = recode_string_pool ("UTF-8", dict_encoding,
+ rec->label, -1, r->pool);
+ var_set_label (var, utf8_label, NULL, false);
+ }
+
+ /* Set missing values. */
+ if (rec->missing_value_code != 0)
+ {
+ int width = var_get_width (var);
+ struct missing_values mv;
+
+ mv_init_pool (r->pool, &mv, width);
+ if (var_is_numeric (var))
+ {
+ bool has_range = rec->missing_value_code < 0;
+ int n_discrete = (has_range
+ ? rec->missing_value_code == -3
+ : rec->missing_value_code);
+ int ofs = 0;
+
+ if (has_range)
+ {
+ double low = parse_float (r, rec->missing, 0);
+ double high = parse_float (r, rec->missing, 8);
+ mv_add_range (&mv, low, high);
+ ofs += 16;
+ }
+
+ for (i = 0; i < n_discrete; i++)
+ {
+ mv_add_num (&mv, parse_float (r, rec->missing, ofs));
+ ofs += 8;
+ }
+ }
+ else
+ {
+ union value value;
+
+ value_init_pool (r->pool, &value, width);
+ value_set_missing (&value, width);
+ for (i = 0; i < rec->missing_value_code; i++)
+ {
+ uint8_t *s = value_str_rw (&value, width);
+ memcpy (s, rec->missing + 8 * i, MIN (width, 8));
+ mv_add_str (&mv, s);
+ }
+ }
+ var_set_missing_values (var, &mv);
}
+
+ /* Set formats. */
+ parse_format_spec (r, rec->pos + 12, rec->print_format,
+ PRINT_FORMAT, var, &n_warnings);
+ parse_format_spec (r, rec->pos + 16, rec->write_format,
+ WRITE_FORMAT, var, &n_warnings);
+
+ /* Account for values.
+ Skip long string continuation records, if any. */
+ n_values = rec->width == 0 ? 1 : DIV_RND_UP (rec->width, 8);
+ for (i = 1; i < n_values; i++)
+ if (i + (rec - var_recs) >= n_var_recs || rec[i].width != -1)
+ sys_error (r, rec->pos, _("Missing string continuation record."));
+ rec += n_values;
}
}
/* Translates the format spec from sysfile format to internal
format. */
static void
-parse_format_spec (struct sfm_reader *r, unsigned int s,
+parse_format_spec (struct sfm_reader *r, off_t pos, unsigned int format,
enum which_format which, struct variable *v,
- int *format_warning_cnt)
+ int *n_warnings)
{
- const int max_format_warnings = 8;
+ const int max_warnings = 8;
+ uint8_t raw_type = format >> 16;
+ uint8_t w = format >> 8;
+ uint8_t d = format;
struct fmt_spec f;
- uint8_t raw_type = s >> 16;
- uint8_t w = s >> 8;
- uint8_t d = s;
bool ok;
if (!fmt_from_io (raw_type, &f.type))
- sys_error (r, _("Unknown variable format %d."), (int) raw_type);
+ sys_error (r, pos, _("Unknown variable format %"PRIu8"."), raw_type);
f.w = w;
f.d = d;
else
var_set_write_format (v, &f);
}
- else if (*++format_warning_cnt <= max_format_warnings)
+ else if (++*n_warnings <= max_warnings)
{
char fmt_string[FMT_STRING_LEN_MAX + 1];
- sys_warn (r, _("%s variable %s has invalid %s format %s."),
+ sys_warn (r, pos, _("%s variable %s has invalid %s format %s."),
var_is_numeric (v) ? _("Numeric") : _("String"),
var_get_name (v),
which == PRINT_FORMAT ? _("print") : _("write"),
fmt_to_string (&f, fmt_string));
- if (*format_warning_cnt == max_format_warnings)
- sys_warn (r, _("Suppressing further invalid format warnings."));
+ if (*n_warnings == max_warnings)
+ sys_warn (r, -1, _("Suppressing further invalid format warnings."));
}
}
-/* Sets the weighting variable in DICT to the variable
- corresponding to the given 1-based VALUE_IDX, if VALUE_IDX is
- nonzero. */
static void
-setup_weight (struct sfm_reader *r, int weight_idx,
- struct variable **var_by_value_idx, struct dictionary *dict)
+parse_document (struct dictionary *dict, struct sfm_document_record *record)
{
- if (weight_idx != 0)
+ const char *p;
+
+ for (p = record->documents;
+ p < record->documents + DOC_LINE_LENGTH * record->n_lines;
+ p += DOC_LINE_LENGTH)
{
- struct variable *weight_var
- = lookup_var_by_value_idx (r, var_by_value_idx, weight_idx);
- if (var_is_numeric (weight_var))
- dict_set_weight (dict, weight_var);
- else
- sys_error (r, _("Weighting variable must be numeric."));
+ struct substring line;
+
+ line = recode_substring_pool ("UTF-8", dict_get_encoding (dict),
+ ss_buffer (p, DOC_LINE_LENGTH), NULL);
+ ss_rtrim (&line, ss_cstr (" "));
+ line.string[line.length] = '\0';
+
+ dict_add_document_line (dict, line.string, false);
+
+ ss_dealloc (&line);
}
}
-/* Reads a document record, type 6, from system file R, and sets up
- the documents and n_documents fields in the associated
- dictionary. */
+/* Parses record type 7, subtype 3. */
static void
-read_documents (struct sfm_reader *r, struct dictionary *dict)
+parse_machine_integer_info (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct sfm_read_info *info)
{
- int line_cnt;
- char *documents;
+ int float_representation, expected_float_format;
+ int integer_representation, expected_integer_format;
- if (dict_get_documents (dict) != NULL)
- sys_error (r, _("Multiple type 6 (document) records."));
+ /* Save version info. */
+ info->version_major = parse_int (r, record->data, 0);
+ info->version_minor = parse_int (r, record->data, 4);
+ info->version_revision = parse_int (r, record->data, 8);
- line_cnt = read_int (r);
- if (line_cnt <= 0)
- sys_error (r, _("Number of document lines (%d) "
- "must be greater than 0."), line_cnt);
+ /* Check floating point format. */
+ float_representation = parse_int (r, record->data, 16);
+ if (r->float_format == FLOAT_IEEE_DOUBLE_BE
+ || r->float_format == FLOAT_IEEE_DOUBLE_LE)
+ expected_float_format = 1;
+ else if (r->float_format == FLOAT_Z_LONG)
+ expected_float_format = 2;
+ else if (r->float_format == FLOAT_VAX_G || r->float_format == FLOAT_VAX_D)
+ expected_float_format = 3;
+ else
+ NOT_REACHED ();
+ if (float_representation != expected_float_format)
+ sys_error (r, record->pos, _("Floating-point representation indicated by "
+ "system file (%d) differs from expected (%d)."),
+ float_representation, expected_float_format);
- documents = pool_nmalloc (r->pool, line_cnt + 1, DOC_LINE_LENGTH);
- read_string (r, documents, DOC_LINE_LENGTH * line_cnt + 1);
- if (strlen (documents) == DOC_LINE_LENGTH * line_cnt)
- dict_set_documents (dict, documents);
+ /* Check integer format. */
+ integer_representation = parse_int (r, record->data, 24);
+ if (r->integer_format == INTEGER_MSB_FIRST)
+ expected_integer_format = 1;
+ else if (r->integer_format == INTEGER_LSB_FIRST)
+ expected_integer_format = 2;
else
- sys_error (r, _("Document line contains null byte."));
- pool_free (r->pool, documents);
+ NOT_REACHED ();
+ if (integer_representation != expected_integer_format)
+ sys_warn (r, record->pos,
+ _("Integer format indicated by system file (%d) "
+ "differs from expected (%d)."),
+ integer_representation, expected_integer_format);
+
}
-/* Read a type 7 extension record. */
-static void
-read_extension_record (struct sfm_reader *r, struct dictionary *dict,
- struct sfm_read_info *info)
+static const char *
+choose_encoding (struct sfm_reader *r,
+ const struct sfm_extension_record *ext_integer,
+ const struct sfm_extension_record *ext_encoding)
{
- int subtype = read_int (r);
- size_t size = read_int (r);
- size_t count = read_int (r);
- size_t bytes = size * count;
-
- /* Check that SIZE * COUNT + 1 doesn't overflow. Adding 1
- allows an extra byte for a null terminator, used by some
- extension processing routines. */
- if (size != 0 && size_overflow_p (xsum (1, xtimes (count, size))))
- sys_error (r, "Record type 7 subtype %d too large.", subtype);
+ /* The EXT_ENCODING record is a more reliable way to determine dictionary
+ encoding. */
+ if (ext_encoding)
+ return ext_encoding->data;
- switch (subtype)
+ /* But EXT_INTEGER is better than nothing as a fallback. */
+ if (ext_integer)
{
- case 3:
- read_machine_integer_info (r, size, count, info);
- return;
+ int codepage = parse_int (r, ext_integer->data, 7 * 4);
- case 4:
- read_machine_float_info (r, size, count);
- return;
+ switch (codepage)
+ {
+ case 1:
+ return "EBCDIC-US";
- case 5:
- /* Variable sets information. We don't use these yet.
- They only apply to GUIs; see VARSETS on the APPLY
- DICTIONARY command in SPSS documentation. */
- break;
+ case 2:
+ case 3:
+ /* These ostensibly mean "7-bit ASCII" and "8-bit ASCII"[sic]
+ respectively. However, there are known to be many files in the wild
+ with character code 2, yet have data which are clearly not ASCII.
+ Therefore we ignore these values. */
+ break;
- case 6:
- /* DATE variable information. We don't use it yet, but we
- should. */
- break;
+ case 4:
+ return "MS_KANJI";
- case 7:
- /* Unknown purpose. */
- break;
+ case 65000:
+ return "UTF-7";
- case 11:
- read_display_parameters (r, size, count, dict);
- return;
+ case 65001:
+ return "UTF-8";
- case 13:
- read_long_var_name_map (r, size, count, dict);
- return;
+ default:
+ return pool_asprintf (r->pool, "CP%d", codepage);
+ }
+ }
- case 14:
- read_long_string_map (r, size, count, dict);
- return;
+ return locale_charset ();
+}
- case 16:
- /* New in SPSS v14? Unknown purpose. */
- break;
+/* Parses record type 7, subtype 4. */
+static void
+parse_machine_float_info (struct sfm_reader *r,
+ const struct sfm_extension_record *record)
+{
+ double sysmis = parse_float (r, record->data, 0);
+ double highest = parse_float (r, record->data, 8);
+ double lowest = parse_float (r, record->data, 16);
- case 17:
- /* Text field that defines variable attributes. New in
- SPSS 14. */
- break;
+ if (sysmis != SYSMIS)
+ sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
+ sysmis, "SYSMIS");
- default:
- sys_warn (r, _("Unrecognized record type 7, subtype %d."), subtype);
- break;
- }
+ if (highest != HIGHEST)
+ sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
+ highest, "HIGHEST");
- skip_bytes (r, bytes);
+ if (lowest != LOWEST)
+ sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."),
+ lowest, "LOWEST");
}
-/* Read record type 7, subtype 3. */
+/* Parses record type 7, subtype 7 or 19. */
static void
-read_machine_integer_info (struct sfm_reader *r, size_t size, size_t count,
- struct sfm_read_info *info)
+parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record,
+ struct dictionary *dict)
{
- int version_major = read_int (r);
- int version_minor = read_int (r);
- int version_revision = read_int (r);
- int machine_code UNUSED = read_int (r);
- int float_representation = read_int (r);
- int compression_code UNUSED = read_int (r);
- int integer_representation = read_int (r);
- int character_code UNUSED = read_int (r);
-
- int expected_float_format;
- int expected_integer_format;
-
- if (size != 4 || count != 8)
- sys_error (r, _("Bad size (%u) or count (%u) field on record type 7, "
- "subtype 3."),
- (unsigned int) size, (unsigned int) count);
+ struct text_record *text;
+ struct mrset *mrset;
- /* Save version info. */
- info->version_major = version_major;
- info->version_minor = version_minor;
- info->version_revision = version_revision;
+ text = open_text_record (r, record);
+ for (;;)
+ {
+ const char *counted = NULL;
+ const char *name;
+ const char *label;
+ struct stringi_set var_names;
+ size_t allocated_vars;
+ char delimiter;
+ int width;
+
+ mrset = xzalloc (sizeof *mrset);
+
+ name = text_get_token (text, ss_cstr ("="), NULL);
+ if (name == NULL)
+ break;
+ mrset->name = xstrdup (name);
+
+ if (mrset->name[0] != '$')
+ {
+ sys_warn (r, record->pos,
+ _("`%s' does not begin with `$' at UTF-8 offset %zu "
+ "in MRSETS record."), mrset->name, text_pos (text));
+ break;
+ }
- /* Check floating point format. */
- if (r->float_format == FLOAT_IEEE_DOUBLE_BE
- || r->float_format == FLOAT_IEEE_DOUBLE_LE)
- expected_float_format = 1;
- else if (r->float_format == FLOAT_Z_LONG)
- expected_float_format = 2;
- else if (r->float_format == FLOAT_VAX_G || r->float_format == FLOAT_VAX_D)
- expected_float_format = 3;
- else
- NOT_REACHED ();
- if (float_representation != expected_float_format)
- sys_error (r, _("Floating-point representation indicated by "
- "system file (%d) differs from expected (%d)."),
- r->float_format, expected_float_format);
+ if (text_match (text, 'C'))
+ {
+ mrset->type = MRSET_MC;
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, record->pos,
+ _("Missing space following `%c' at UTF-8 offset %zu "
+ "in MRSETS record."), 'C', text_pos (text));
+ break;
+ }
+ }
+ else if (text_match (text, 'D'))
+ {
+ mrset->type = MRSET_MD;
+ mrset->cat_source = MRSET_VARLABELS;
+ }
+ else if (text_match (text, 'E'))
+ {
+ char *number;
+
+ mrset->type = MRSET_MD;
+ mrset->cat_source = MRSET_COUNTEDVALUES;
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, record->pos,
+ _("Missing space following `%c' at UTF-8 offset %zu "
+ "in MRSETS record."), 'E', text_pos (text));
+ break;
+ }
+
+ number = text_get_token (text, ss_cstr (" "), NULL);
+ if (!strcmp (number, "11"))
+ mrset->label_from_var_label = true;
+ else if (strcmp (number, "1"))
+ sys_warn (r, record->pos,
+ _("Unexpected label source value `%s' following `E' "
+ "at UTF-8 offset %zu in MRSETS record."),
+ number, text_pos (text));
+ }
+ else
+ {
+ sys_warn (r, record->pos,
+ _("Missing `C', `D', or `E' at UTF-8 offset %zu "
+ "in MRSETS record."),
+ text_pos (text));
+ break;
+ }
+
+ if (mrset->type == MRSET_MD)
+ {
+ counted = text_parse_counted_string (r, text);
+ if (counted == NULL)
+ break;
+ }
+
+ label = text_parse_counted_string (r, text);
+ if (label == NULL)
+ break;
+ mrset->label = label[0] != '\0' ? xstrdup (label) : NULL;
+
+ stringi_set_init (&var_names);
+ allocated_vars = 0;
+ width = INT_MAX;
+ do
+ {
+ struct variable *var;
+ const char *var_name;
+
+ var_name = text_get_token (text, ss_cstr (" \n"), &delimiter);
+ if (var_name == NULL)
+ {
+ sys_warn (r, record->pos,
+ _("Missing new-line parsing variable names "
+ "at UTF-8 offset %zu in MRSETS record."),
+ text_pos (text));
+ break;
+ }
+
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ continue;
+ if (!stringi_set_insert (&var_names, var_name))
+ {
+ sys_warn (r, record->pos,
+ _("Duplicate variable name %s "
+ "at UTF-8 offset %zu in MRSETS record."),
+ var_name, text_pos (text));
+ continue;
+ }
- /* Check integer format. */
- if (r->integer_format == INTEGER_MSB_FIRST)
- expected_integer_format = 1;
- else if (r->integer_format == INTEGER_LSB_FIRST)
- expected_integer_format = 2;
- else
- NOT_REACHED ();
- if (integer_representation != expected_integer_format)
- {
- static const char *endian[] = {N_("little-endian"), N_("big-endian")};
- sys_warn (r, _("Integer format indicated by system file (%s) "
- "differs from expected (%s)."),
- gettext (endian[integer_representation == 1]),
- gettext (endian[expected_integer_format == 1]));
- }
-}
+ if (mrset->label == NULL && mrset->label_from_var_label
+ && var_has_label (var))
+ mrset->label = xstrdup (var_get_label (var));
-/* Read record type 7, subtype 4. */
-static void
-read_machine_float_info (struct sfm_reader *r, size_t size, size_t count)
-{
- double sysmis = read_float (r);
- double highest = read_float (r);
- double lowest = read_float (r);
+ if (mrset->n_vars
+ && var_get_type (var) != var_get_type (mrset->vars[0]))
+ {
+ sys_warn (r, record->pos,
+ _("MRSET %s contains both string and "
+ "numeric variables."), name);
+ continue;
+ }
+ width = MIN (width, var_get_width (var));
+
+ if (mrset->n_vars >= allocated_vars)
+ mrset->vars = x2nrealloc (mrset->vars, &allocated_vars,
+ sizeof *mrset->vars);
+ mrset->vars[mrset->n_vars++] = var;
+ }
+ while (delimiter != '\n');
- if (size != 8 || count != 3)
- sys_error (r, _("Bad size (%u) or count (%u) on extension 4."),
- (unsigned int) size, (unsigned int) count);
+ if (mrset->n_vars < 2)
+ {
+ sys_warn (r, record->pos,
+ _("MRSET %s has only %zu variables."), mrset->name,
+ mrset->n_vars);
+ mrset_destroy (mrset);
+ continue;
+ }
- if (sysmis != SYSMIS)
- sys_warn (r, _("File specifies unexpected value %g as SYSMIS."), sysmis);
- if (highest != HIGHEST)
- sys_warn (r, _("File specifies unexpected value %g as HIGHEST."), highest);
- if (lowest != LOWEST)
- sys_warn (r, _("File specifies unexpected value %g as LOWEST."), lowest);
+ if (mrset->type == MRSET_MD)
+ {
+ mrset->width = width;
+ value_init (&mrset->counted, width);
+ if (width == 0)
+ mrset->counted.f = strtod (counted, NULL);
+ else
+ value_copy_str_rpad (&mrset->counted, width,
+ (const uint8_t *) counted, ' ');
+ }
+
+ dict_add_mrset (dict, mrset);
+ mrset = NULL;
+ stringi_set_destroy (&var_names);
+ }
+ mrset_destroy (mrset);
+ close_text_record (r, text);
}
/* Read record type 7, subtype 11, which specifies how variables
should be displayed in GUI environments. */
static void
-read_display_parameters (struct sfm_reader *r, size_t size, size_t count,
+parse_display_parameters (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
struct dictionary *dict)
{
- const size_t n_vars = count / 3 ;
+ bool includes_width;
bool warned = false;
- int i;
+ size_t n_vars;
+ size_t ofs;
+ size_t i;
- if (count % 3 || n_vars != dict_get_var_cnt (dict))
- sys_error (r, _("Bad size (%u) or count (%u) on extension 11."),
- (unsigned int) size, (unsigned int) count);
+ n_vars = dict_get_var_cnt (dict);
+ if (record->count == 3 * n_vars)
+ includes_width = true;
+ else if (record->count == 2 * n_vars)
+ includes_width = false;
+ else
+ {
+ sys_warn (r, record->pos,
+ _("Extension 11 has bad count %zu (for %zu variables)."),
+ record->count, n_vars);
+ return;
+ }
+ ofs = 0;
for (i = 0; i < n_vars; ++i)
{
struct variable *v = dict_get_var (dict, i);
- int measure = read_int (r);
- int width = read_int (r);
- int align = read_int (r);
+ int measure, width, align;
+
+ measure = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ if (includes_width)
+ {
+ width = parse_int (r, record->data, ofs);
+ ofs += 4;
+ }
+ else
+ width = 0;
+
+ align = parse_int (r, record->data, ofs);
+ ofs += 4;
/* SPSS 14 sometimes seems to set string variables' measure
to zero. */
if (0 == measure && var_is_alpha (v))
measure = 1;
- /* Older versions (SPSS 9.0) sometimes set the display width
- to zero. This causes confusion especially in the GUI */
- if (0 == width)
- width = 8;
-
if (measure < 1 || measure > 3 || align < 0 || align > 2)
{
if (!warned)
- sys_warn (r, _("Invalid variable display parameters. "
- "Default parameters substituted."));
+ sys_warn (r, record->pos,
+ _("Invalid variable display parameters for variable "
+ "%zu (%s). Default parameters substituted."),
+ i, var_get_name (v));
warned = true;
continue;
}
var_set_measure (v, (measure == 1 ? MEASURE_NOMINAL
: measure == 2 ? MEASURE_ORDINAL
: MEASURE_SCALE));
- var_set_display_width (v, width);
var_set_alignment (v, (align == 0 ? ALIGN_LEFT
: align == 1 ? ALIGN_RIGHT
: ALIGN_CENTRE));
+
+ /* Older versions (SPSS 9.0) sometimes set the display
+ width to zero. This causes confusion in the GUI, so
+ only set the width if it is nonzero. */
+ if (width > 0)
+ var_set_display_width (v, width);
+ }
+}
+
+static void
+rename_var_and_save_short_names (struct dictionary *dict, struct variable *var,
+ const char *new_name)
+{
+ size_t n_short_names;
+ char **short_names;
+ size_t i;
+
+ /* Renaming a variable may clear its short names, but we
+ want to retain them, so we save them and re-set them
+ afterward. */
+ n_short_names = var_get_short_name_cnt (var);
+ short_names = xnmalloc (n_short_names, sizeof *short_names);
+ for (i = 0; i < n_short_names; i++)
+ {
+ const char *s = var_get_short_name (var, i);
+ short_names[i] = s != NULL ? xstrdup (s) : NULL;
+ }
+
+ /* Set long name. */
+ dict_rename_var (dict, var, new_name);
+
+ /* Restore short names. */
+ for (i = 0; i < n_short_names; i++)
+ {
+ var_set_short_name (var, i, short_names[i]);
+ free (short_names[i]);
}
+ free (short_names);
}
-/* Reads record type 7, subtype 13, which gives the long name
- that corresponds to each short name. Modifies variable names
- in DICT accordingly. */
+/* Parses record type 7, subtype 13, which gives the long name that corresponds
+ to each short name. Modifies variable names in DICT accordingly. */
static void
-read_long_var_name_map (struct sfm_reader *r, size_t size, size_t count,
- struct dictionary *dict)
+parse_long_var_name_map (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
{
- struct variable_to_value_map *map;
+ struct text_record *text;
struct variable *var;
char *long_name;
- int warning_cnt = 0;
- map = open_variable_to_value_map (r, size * count);
- while (read_variable_to_value_map (r, dict, map, &var, &long_name,
- &warning_cnt))
+ if (record == NULL)
{
- char **short_names;
- size_t short_name_cnt;
+ /* Convert variable names to lowercase. */
size_t i;
+ for (i = 0; i < dict_get_var_cnt (dict); i++)
+ {
+ struct variable *var = dict_get_var (dict, i);
+ char *new_name;
+
+ new_name = xstrdup (var_get_name (var));
+ str_lowercase (new_name);
+
+ rename_var_and_save_short_names (dict, var, new_name);
+
+ free (new_name);
+ }
+
+ return;
+ }
+
+ /* Rename each of the variables, one by one. (In a correctly constructed
+ system file, this cannot create any intermediate duplicate variable names,
+ because all of the new variable names are longer than any of the old
+ variable names and thus there cannot be any overlaps.) */
+ text = open_text_record (r, record);
+ while (read_variable_to_value_pair (r, dict, text, &var, &long_name))
+ {
/* Validate long name. */
- if (!var_is_valid_name (long_name, false))
+ /* XXX need to reencode name to UTF-8 */
+ if (!dict_id_is_valid (dict, long_name, false))
{
- sys_warn (r, _("Long variable mapping from %s to invalid "
- "variable name `%s'."),
+ sys_warn (r, record->pos,
+ _("Long variable mapping from %s to invalid "
+ "variable name `%s'."),
var_get_name (var), long_name);
continue;
}
if (strcasecmp (var_get_short_name (var, 0), long_name)
&& dict_lookup_var (dict, long_name) != NULL)
{
- sys_warn (r, _("Duplicate long variable name `%s' "
- "within system file."), long_name);
+ sys_warn (r, record->pos,
+ _("Duplicate long variable name `%s'."), long_name);
continue;
}
- /* Renaming a variable may clear its short names, but we
- want to retain them, so we save them and re-set them
- afterward. */
- short_name_cnt = var_get_short_name_cnt (var);
- short_names = xnmalloc (short_name_cnt, sizeof *short_names);
- for (i = 0; i < short_name_cnt; i++)
- {
- const char *s = var_get_short_name (var, i);
- short_names[i] = s != NULL ? xstrdup (s) : NULL;
- }
-
- /* Set long name. */
- dict_rename_var (dict, var, long_name);
-
- /* Restore short names. */
- for (i = 0; i < short_name_cnt; i++)
- {
- var_set_short_name (var, i, short_names[i]);
- free (short_names[i]);
- }
+ rename_var_and_save_short_names (dict, var, long_name);
}
- close_variable_to_value_map (r, map);
- r->has_long_var_names = true;
+ close_text_record (r, text);
}
/* Reads record type 7, subtype 14, which gives the real length
of each very long string. Rearranges DICT accordingly. */
static void
-read_long_string_map (struct sfm_reader *r, size_t size, size_t count,
- struct dictionary *dict)
+parse_long_string_map (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
{
- struct variable_to_value_map *map;
+ struct text_record *text;
struct variable *var;
char *length_s;
- int warning_cnt = 0;
- map = open_variable_to_value_map (r, size * count);
- while (read_variable_to_value_map (r, dict, map, &var, &length_s,
- &warning_cnt))
+ text = open_text_record (r, record);
+ while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
{
size_t idx = var_get_dict_index (var);
long int length;
length = strtol (length_s, NULL, 10);
if (length < 1 || length > MAX_STRING)
{
- sys_warn (r, _("%s listed as string of invalid length %s "
- "in very length string record."),
+ sys_warn (r, record->pos,
+ _("%s listed as string of invalid length %s "
+ "in very long string record."),
var_get_name (var), length_s);
continue;
}
segment_cnt = sfm_width_to_segments (length);
if (segment_cnt == 1)
{
- sys_warn (r, _("%s listed in very long string record with width %s, "
- "which requires only one segment."),
+ sys_warn (r, record->pos,
+ _("%s listed in very long string record with width %s, "
+ "which requires only one segment."),
var_get_name (var), length_s);
continue;
}
if (idx + segment_cnt > dict_get_var_cnt (dict))
- sys_error (r, _("Very long string %s overflows dictionary."),
+ sys_error (r, record->pos,
+ _("Very long string %s overflows dictionary."),
var_get_name (var));
/* Get the short names from the segments and check their
if (i > 0)
var_set_short_name (var, i, var_get_short_name (seg, 0));
if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
- sys_error (r, _("Very long string with width %ld has segment %d "
- "of width %d (expected %d)"),
+ sys_error (r, record->pos,
+ _("Very long string with width %ld has segment %d "
+ "of width %d (expected %d)."),
length, i, width, alloc_width);
}
dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
var_set_width (var, length);
}
- close_variable_to_value_map (r, map);
+ close_text_record (r, text);
dict_compact_values (dict);
}
-/* Reads value labels from sysfile H and inserts them into the
- associated dictionary. */
static void
-read_value_labels (struct sfm_reader *r,
- struct dictionary *dict, struct variable **var_by_value_idx)
+parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ const struct sfm_value_label_record *record)
{
- struct pool *subpool;
+ struct variable **vars;
+ char **utf8_labels;
+ size_t i;
- struct label
+ utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels);
+ for (i = 0; i < record->n_labels; i++)
+ utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
+ record->labels[i].label, -1,
+ r->pool);
+
+ vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
+ for (i = 0; i < record->n_vars; i++)
+ vars[i] = lookup_var_by_index (r, record->pos,
+ var_recs, n_var_recs, record->vars[i]);
+
+ for (i = 1; i < record->n_vars; i++)
+ if (var_get_type (vars[i]) != var_get_type (vars[0]))
+ sys_error (r, record->pos,
+ _("Variables associated with value label are not all of "
+ "identical type. Variable %s is %s, but variable "
+ "%s is %s."),
+ var_get_name (vars[0]),
+ var_is_numeric (vars[0]) ? _("numeric") : _("string"),
+ var_get_name (vars[i]),
+ var_is_numeric (vars[i]) ? _("numeric") : _("string"));
+
+ for (i = 0; i < record->n_vars; i++)
{
- char raw_value[8]; /* Value as uninterpreted bytes. */
- union value value; /* Value. */
- char *label; /* Null-terminated label string. */
- };
+ struct variable *var = vars[i];
+ int width;
+ size_t j;
+
+ width = var_get_width (var);
+ if (width > 8)
+ sys_error (r, record->pos,
+ _("Value labels may not be added to long string "
+ "variables (e.g. %s) using records types 3 and 4."),
+ var_get_name (var));
- struct label *labels = NULL;
- int label_cnt; /* Number of labels. */
+ for (j = 0; j < record->n_labels; j++)
+ {
+ struct sfm_value_label *label = &record->labels[j];
+ union value value;
- struct variable **var = NULL; /* Associated variables. */
- int var_cnt; /* Number of associated variables. */
+ value_init (&value, width);
+ if (width == 0)
+ value.f = parse_float (r, label->value, 0);
+ else
+ memcpy (value_str_rw (&value, width), label->value, width);
- int i;
+ if (!var_add_value_label (var, &value, utf8_labels[j]))
+ {
+ if (var_is_numeric (var))
+ sys_warn (r, record->pos,
+ _("Duplicate value label for %g on %s."),
+ value.f, var_get_name (var));
+ else
+ sys_warn (r, record->pos,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value_str (&value, width),
+ var_get_name (var));
+ }
- subpool = pool_create_subpool (r->pool);
+ value_destroy (&value, width);
+ }
+ }
- /* Read the type 3 record and record its contents. We can't do
- much with the data yet because we don't know whether it is
- of numeric or string type. */
+ pool_free (r->pool, vars);
+ for (i = 0; i < record->n_labels; i++)
+ pool_free (r->pool, utf8_labels[i]);
+ pool_free (r->pool, utf8_labels);
+}
- /* Read number of labels. */
- label_cnt = read_int (r);
+static struct variable *
+lookup_var_by_index (struct sfm_reader *r, off_t offset,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ int idx)
+{
+ const struct sfm_var_record *rec;
- if (size_overflow_p (xtimes (label_cnt, sizeof *labels)))
+ if (idx < 1 || idx > n_var_recs)
{
- sys_warn (r, _("Invalid number of labels: %d. Ignoring labels."),
- label_cnt);
- label_cnt = 0;
+ sys_error (r, offset,
+ _("Variable index %d not in valid range 1...%d."),
+ idx, n_var_recs);
+ return NULL;
}
- /* Read each value/label tuple into labels[]. */
- labels = pool_nalloc (subpool, label_cnt, sizeof *labels);
- for (i = 0; i < label_cnt; i++)
+ rec = &var_recs[idx - 1];
+ if (rec->var == NULL)
{
- struct label *label = labels + i;
- unsigned char label_len;
- size_t padded_len;
-
- /* Read value. */
- read_bytes (r, label->raw_value, sizeof label->raw_value);
+ sys_error (r, offset,
+ _("Variable index %d refers to long string continuation."),
+ idx);
+ return NULL;
+ }
- /* Read label length. */
- read_bytes (r, &label_len, sizeof label_len);
- padded_len = ROUND_UP (label_len + 1, 8);
+ return rec->var;
+}
- /* Read label, padding. */
- label->label = pool_alloc (subpool, padded_len + 1);
- read_bytes (r, label->label, padded_len - 1);
- label->label[label_len] = 0;
- }
+/* Parses a set of custom attributes from TEXT into ATTRS.
+ ATTRS may be a null pointer, in which case the attributes are
+ read but discarded. */
+static void
+parse_attributes (struct sfm_reader *r, struct text_record *text,
+ struct attrset *attrs)
+{
+ do
+ {
+ struct attribute *attr;
+ char *key;
+ int index;
- /* Now, read the type 4 record that has the list of variables
- to which the value labels are to be applied. */
+ /* Parse the key. */
+ key = text_get_token (text, ss_cstr ("("), NULL);
+ if (key == NULL)
+ return;
- /* Read record type of type 4 record. */
- if (read_int (r) != 4)
- sys_error (r, _("Variable index record (type 4) does not immediately "
- "follow value label record (type 3) as it should."));
+ attr = attribute_create (key);
+ for (index = 1; ; index++)
+ {
+ /* Parse the value. */
+ char *value;
+ size_t length;
- /* Read number of variables associated with value label from type 4
- record. */
- var_cnt = read_int (r);
- if (var_cnt < 1 || var_cnt > dict_get_var_cnt (dict))
- sys_error (r, _("Number of variables associated with a value label (%d) "
- "is not between 1 and the number of variables (%u)."),
- var_cnt, (unsigned int) dict_get_var_cnt (dict));
-
- /* Read the list of variables. */
- var = pool_nalloc (subpool, var_cnt, sizeof *var);
- for (i = 0; i < var_cnt; i++)
- {
- var[i] = lookup_var_by_value_idx (r, var_by_value_idx, read_int (r));
- if (var_is_long_string (var[i]))
- sys_error (r, _("Value labels are not allowed on long string "
- "variables (%s)."), var_get_name (var[i]));
- }
+ value = text_get_token (text, ss_cstr ("\n"), NULL);
+ if (value == NULL)
+ {
+ text_warn (r, text, _("Error parsing attribute value %s[%d]."),
+ key, index);
+ break;
+ }
- /* Type check the variables. */
- for (i = 1; i < var_cnt; i++)
- if (var_get_type (var[i]) != var_get_type (var[0]))
- sys_error (r, _("Variables associated with value label are not all of "
- "identical type. Variable %s is %s, but variable "
- "%s is %s."),
- var_get_name (var[0]),
- var_is_numeric (var[0]) ? _("numeric") : _("string"),
- var_get_name (var[i]),
- var_is_numeric (var[i]) ? _("numeric") : _("string"));
-
- /* Fill in labels[].value, now that we know the desired type. */
- for (i = 0; i < label_cnt; i++)
- {
- struct label *label = labels + i;
+ length = strlen (value);
+ if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
+ {
+ value[length - 1] = '\0';
+ attribute_add_value (attr, value + 1);
+ }
+ else
+ {
+ text_warn (r, text,
+ _("Attribute value %s[%d] is not quoted: %s."),
+ key, index, value);
+ attribute_add_value (attr, value);
+ }
- if (var_is_alpha (var[0]))
- buf_copy_rpad (label->value.s, sizeof label->value.s,
- label->raw_value, sizeof label->raw_value);
+ /* Was this the last value for this attribute? */
+ if (text_match (text, ')'))
+ break;
+ }
+ if (attrs != NULL)
+ attrset_add (attrs, attr);
else
- label->value.f = float_get_double (r->float_format, label->raw_value);
+ attribute_destroy (attr);
}
+ while (!text_match (text, '/'));
+}
+
+/* Reads record type 7, subtype 17, which lists custom
+ attributes on the data file. */
+static void
+parse_data_file_attributes (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ struct text_record *text = open_text_record (r, record);
+ parse_attributes (r, text, dict_get_attributes (dict));
+ close_text_record (r, text);
+}
+
+/* Parses record type 7, subtype 18, which lists custom
+ attributes on individual variables. */
+static void
+parse_variable_attributes (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ struct text_record *text;
+ struct variable *var;
+
+ text = open_text_record (r, record);
+ while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
+ parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
+ close_text_record (r, text);
+}
+
+static void
+check_overflow (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ size_t ofs, size_t length)
+{
+ size_t end = record->size * record->count;
+ if (length >= end || ofs + length > end)
+ sys_error (r, record->pos + end,
+ _("Long string value label record ends unexpectedly."));
+}
- /* Assign the `value_label's to each variable. */
- for (i = 0; i < var_cnt; i++)
+static void
+parse_long_string_value_labels (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ size_t end = record->size * record->count;
+ size_t ofs = 0;
+
+ while (ofs < end)
{
- struct variable *v = var[i];
- int j;
+ char *var_name;
+ size_t n_labels, i;
+ struct variable *var;
+ union value value;
+ int var_name_len;
+ int width;
+
+ /* Parse variable name length. */
+ check_overflow (r, record, ofs, 4);
+ var_name_len = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse variable name, width, and number of labels. */
+ check_overflow (r, record, ofs, var_name_len + 8);
+ var_name = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ var_name_len, r->pool);
+ width = parse_int (r, record->data, ofs + var_name_len);
+ n_labels = parse_int (r, record->data, ofs + var_name_len + 4);
+ ofs += var_name_len + 8;
+
+ /* Look up 'var' and validate. */
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value record for "
+ "unknown variable %s."), var_name);
+ else if (var_is_numeric (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value record for "
+ "numeric variable %s."), var_name);
+ var = NULL;
+ }
+ else if (width != var_get_width (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value record for variable %s "
+ "because the record's width (%d) does not match the "
+ "variable's width (%d)."),
+ var_name, width, var_get_width (var));
+ var = NULL;
+ }
- /* Add each label to the variable. */
- for (j = 0; j < label_cnt; j++)
+ /* Parse values. */
+ value_init_pool (r->pool, &value, width);
+ for (i = 0; i < n_labels; i++)
{
- struct label *label = &labels[j];
- if (!var_add_value_label (v, &label->value, label->label))
+ size_t value_length, label_length;
+ bool skip = var == NULL;
+
+ /* Parse value length. */
+ check_overflow (r, record, ofs, 4);
+ value_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse value. */
+ check_overflow (r, record, ofs, value_length);
+ if (!skip)
{
- if (var_is_numeric (var[0]))
- sys_warn (r, _("Duplicate value label for %g on %s."),
- label->value.f, var_get_name (v));
+ if (value_length == width)
+ memcpy (value_str_rw (&value, width),
+ (const uint8_t *) record->data + ofs, width);
else
- sys_warn (r, _("Duplicate value label for \"%.*s\" on %s."),
- var_get_width (v), label->value.s,
- var_get_name (v));
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value %zu for variable "
+ "%s, with width %d, that has bad value "
+ "width %zu."),
+ i, var_get_name (var), width, value_length);
+ skip = true;
+ }
}
- }
- }
+ ofs += value_length;
+
+ /* Parse label length. */
+ check_overflow (r, record, ofs, 4);
+ label_length = parse_int (r, record->data, ofs);
+ ofs += 4;
- pool_destroy (subpool);
+ /* Parse label. */
+ check_overflow (r, record, ofs, label_length);
+ if (!skip)
+ {
+ char *label;
+
+ label = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ label_length, r->pool);
+ if (!var_add_value_label (var, &value, label))
+ sys_warn (r, record->pos + ofs,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value_str (&value, width),
+ var_get_name (var));
+ pool_free (r->pool, label);
+ }
+ ofs += label_length;
+ }
+ }
}
\f
/* Case reader. */
static void read_error (struct casereader *, const struct sfm_reader *);
static bool read_case_number (struct sfm_reader *, double *);
-static bool read_case_string (struct sfm_reader *, char *, size_t);
+static bool read_case_string (struct sfm_reader *, uint8_t *, size_t);
static int read_opcode (struct sfm_reader *);
static bool read_compressed_number (struct sfm_reader *, double *);
-static bool read_compressed_string (struct sfm_reader *, char *);
-static bool read_whole_strings (struct sfm_reader *, char *, size_t);
+static bool read_compressed_string (struct sfm_reader *, uint8_t *);
+static bool read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
static bool skip_whole_strings (struct sfm_reader *, size_t);
-/* Reads one case from READER's file into C. Returns true only
- if successful. */
-static bool
-sys_file_casereader_read (struct casereader *reader, void *r_,
- struct ccase *c)
+/* Reads and returns one case from READER's file. Returns a null
+ pointer if not successful. */
+static struct ccase *
+sys_file_casereader_read (struct casereader *reader, void *r_)
{
struct sfm_reader *r = r_;
+ struct ccase *volatile c;
int i;
if (r->error)
- return false;
+ return NULL;
- case_create (c, r->value_cnt);
+ c = case_create (r->proto);
if (setjmp (r->bail_out))
{
casereader_force_error (reader);
- case_destroy (c);
- return false;
+ case_unref (c);
+ return NULL;
}
for (i = 0; i < r->sfm_var_cnt; i++)
struct sfm_var *sv = &r->sfm_vars[i];
union value *v = case_data_rw_idx (c, sv->case_index);
- if (sv->width == 0)
+ if (sv->var_width == 0)
{
if (!read_case_number (r, &v->f))
goto eof;
}
else
{
- if (!read_case_string (r, v->s + sv->offset, sv->width))
+ uint8_t *s = value_str_rw (v, sv->var_width);
+ if (!read_case_string (r, s + sv->offset, sv->segment_width))
goto eof;
if (!skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)))
partial_record (r);
}
}
- return true;
+ return c;
eof:
- case_destroy (c);
if (i != 0)
partial_record (r);
if (r->case_cnt != -1)
read_error (reader, r);
- return false;
+ case_unref (c);
+ return NULL;
}
/* Issues an error that R ends in a partial record. */
static void
partial_record (struct sfm_reader *r)
{
- sys_error (r, _("File ends in partial case."));
+ sys_error (r, r->pos, _("File ends in partial case."));
}
/* Issues an error that an unspecified error occurred SFM, and
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_case_string (struct sfm_reader *r, char *s, size_t length)
+read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
{
size_t whole = ROUND_DOWN (length, 8);
size_t partial = length % 8;
if (partial)
{
- char bounce[8];
+ uint8_t bounce[8];
if (!read_whole_strings (r, bounce, sizeof bounce))
{
if (whole)
break;
case 254:
- sys_error (r, _("Compressed data is corrupt."));
+ float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
+ if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, r->pos,
+ _("Possible compressed data corruption: "
+ "compressed spaces appear in numeric field."));
+ }
+ break;
case 255:
*d = SYSMIS;
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_compressed_string (struct sfm_reader *r, char *dst)
+read_compressed_string (struct sfm_reader *r, uint8_t *dst)
{
- switch (read_opcode (r))
+ int opcode = read_opcode (r);
+ switch (opcode)
{
case -1:
case 252:
break;
default:
- sys_error (r, _("Compressed data is corrupt."));
+ {
+ double value = opcode - r->bias;
+ float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
+ if (value == 0.0)
+ {
+ /* This has actually been seen "in the wild". The submitter of the
+ file that showed that the contents decoded as spaces, but they
+ were at the end of the field so it's possible that the null
+ bytes just acted as null terminators. */
+ }
+ else if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, r->pos,
+ _("Possible compressed data corruption: "
+ "string contains compressed integer (opcode %d)."),
+ opcode);
+ }
+ }
+ break;
}
return true;
Returns true if successful, false if end of file is
reached immediately. */
static bool
-read_whole_strings (struct sfm_reader *r, char *s, size_t length)
+read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
{
assert (length % 8 == 0);
if (!r->compressed)
static bool
skip_whole_strings (struct sfm_reader *r, size_t length)
{
- char buffer[1024];
+ uint8_t buffer[1024];
assert (length < sizeof buffer);
return read_whole_strings (r, buffer, length);
}
\f
-/* Creates and returns a table that can be used for translating a value
- index into a case to a "struct variable *" for DICT. Multiple
- system file fields reference variables this way.
-
- This table must be created before processing the very long
- string extension record, because that record causes some
- values to be deleted from the case and the dictionary to be
- compacted. */
-static struct variable **
-make_var_by_value_idx (struct sfm_reader *r, struct dictionary *dict)
-{
- struct variable **var_by_value_idx;
- int value_idx = 0;
- int i;
+/* Helpers for reading records that contain structured text
+ strings. */
- var_by_value_idx = pool_nmalloc (r->pool,
- r->oct_cnt, sizeof *var_by_value_idx);
- for (i = 0; i < dict_get_var_cnt (dict); i++)
- {
- struct variable *v = dict_get_var (dict, i);
- int nv = var_is_numeric (v) ? 1 : DIV_RND_UP (var_get_width (v), 8);
- int j;
+/* Maximum number of warnings to issue for a single text
+ record. */
+#define MAX_TEXT_WARNINGS 5
- var_by_value_idx[value_idx++] = v;
- for (j = 1; j < nv; j++)
- var_by_value_idx[value_idx++] = NULL;
- }
- assert (value_idx == r->oct_cnt);
+/* State. */
+struct text_record
+ {
+ struct substring buffer; /* Record contents, in UTF-8. */
+ off_t start; /* Starting offset in file. */
+ size_t pos; /* Current position in buffer. */
+ int n_warnings; /* Number of warnings issued or suppressed. */
+ };
+
+static struct text_record *
+open_text_record (struct sfm_reader *r,
+ const struct sfm_extension_record *record)
+{
+ struct text_record *text;
+ struct substring raw;
+
+ text = pool_alloc (r->pool, sizeof *text);
+ raw = ss_buffer (record->data, record->size * record->count);
+ text->start = record->pos;
+ text->buffer = recode_substring_pool ("UTF-8", r->encoding, raw, r->pool);
+ text->pos = 0;
+ text->n_warnings = 0;
- return var_by_value_idx;
+ return text;
}
-/* Returns the "struct variable" corresponding to the given
- 1-basd VALUE_IDX in VAR_BY_VALUE_IDX. Verifies that the index
- is valid. */
-static struct variable *
-lookup_var_by_value_idx (struct sfm_reader *r,
- struct variable **var_by_value_idx, int value_idx)
+/* Closes TEXT, frees its storage, and issues a final warning
+ about suppressed warnings if necesary. */
+static void
+close_text_record (struct sfm_reader *r, struct text_record *text)
{
- struct variable *var;
+ if (text->n_warnings > MAX_TEXT_WARNINGS)
+ sys_warn (r, -1, _("Suppressed %d additional related warnings."),
+ text->n_warnings - MAX_TEXT_WARNINGS);
+ pool_free (r->pool, ss_data (text->buffer));
+}
- if (value_idx < 1 || value_idx > r->oct_cnt)
- sys_error (r, _("Variable index %d not in valid range 1...%d."),
- value_idx, r->oct_cnt);
+/* Reads a variable=value pair from TEXT.
+ Looks up the variable in DICT and stores it into *VAR.
+ Stores a null-terminated value into *VALUE. */
+static bool
+read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text,
+ struct variable **var, char **value)
+{
+ for (;;)
+ {
+ if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
+ return false;
+
+ *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
+ if (*value == NULL)
+ return false;
- var = var_by_value_idx[value_idx - 1];
- if (var == NULL)
- sys_error (r, _("Variable index %d refers to long string "
- "continuation."),
- value_idx);
+ text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
+ ss_buffer ("\t\0", 2));
- return var;
+ if (*var != NULL)
+ return true;
+ }
}
-/* Returns the variable in D with the given SHORT_NAME,
- or a null pointer if there is none. */
-static struct variable *
-lookup_var_by_short_name (struct dictionary *d, const char *short_name)
+static bool
+text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text, struct substring delimiters,
+ struct variable **var)
{
- struct variable *var;
- size_t var_cnt;
- size_t i;
+ char *name;
- /* First try looking up by full name. This often succeeds. */
- var = dict_lookup_var (d, short_name);
- if (var != NULL && !strcasecmp (var_get_short_name (var, 0), short_name))
- return var;
+ name = text_get_token (text, delimiters, NULL);
+ if (name == NULL)
+ return false;
- /* Iterate through the whole dictionary as a fallback. */
- var_cnt = dict_get_var_cnt (d);
- for (i = 0; i < var_cnt; i++)
- {
- var = dict_get_var (d, i);
- if (!strcasecmp (var_get_short_name (var, 0), short_name))
- return var;
- }
+ *var = dict_lookup_var (dict, name);
+ if (*var != NULL)
+ return true;
- return NULL;
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
+ name);
+ return false;
}
-\f
-/* Helpers for reading records that contain "variable=value"
- pairs. */
-/* State. */
-struct variable_to_value_map
- {
- struct substring buffer; /* Record contents. */
- size_t pos; /* Current position in buffer. */
- };
-/* Reads SIZE bytes into a "variable=value" map for R,
- and returns the map. */
-static struct variable_to_value_map *
-open_variable_to_value_map (struct sfm_reader *r, size_t size)
+static bool
+text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text, struct substring delimiters,
+ struct variable **var)
{
- struct variable_to_value_map *map = pool_alloc (r->pool, sizeof *map);
- char *buffer = pool_malloc (r->pool, size + 1);
- read_bytes (r, buffer, size);
- map->buffer = ss_buffer (buffer, size);
- map->pos = 0;
- return map;
+ char *short_name = text_get_token (text, delimiters, NULL);
+ if (short_name == NULL)
+ return false;
+
+ *var = dict_lookup_var (dict, short_name);
+ if (*var == NULL)
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
+ short_name);
+ return true;
}
-/* Closes MAP and frees its storage.
- Not really needed, because the pool will free the map anyway,
- but can be used to free it earlier. */
+/* Displays a warning for the current file position, limiting the
+ number to MAX_TEXT_WARNINGS for TEXT. */
static void
-close_variable_to_value_map (struct sfm_reader *r,
- struct variable_to_value_map *map)
+text_warn (struct sfm_reader *r, struct text_record *text,
+ const char *format, ...)
+{
+ if (text->n_warnings++ < MAX_TEXT_WARNINGS)
+ {
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, text->start + text->pos, MW, format, args);
+ va_end (args);
+ }
+}
+
+static char *
+text_get_token (struct text_record *text, struct substring delimiters,
+ char *delimiter)
{
- pool_free (r->pool, ss_data (map->buffer));
+ struct substring token;
+ char *end;
+
+ if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
+ return NULL;
+
+ end = &ss_data (token)[ss_length (token)];
+ if (delimiter != NULL)
+ *delimiter = *end;
+ *end = '\0';
+ return ss_data (token);
}
-/* Reads the next variable=value pair from MAP.
- Looks up the variable in DICT and stores it into *VAR.
- Stores a null-terminated value into *VALUE. */
-static bool
-read_variable_to_value_map (struct sfm_reader *r, struct dictionary *dict,
- struct variable_to_value_map *map,
- struct variable **var, char **value,
- int *warning_cnt)
+/* Reads a integer value expressed in decimal, then a space, then a string that
+ consists of exactly as many bytes as specified by the integer, then a space,
+ from TEXT. Returns the string, null-terminated, as a subset of TEXT's
+ buffer (so the caller should not free the string). */
+static const char *
+text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
{
- int max_warnings = 5;
+ size_t start;
+ size_t n;
+ char *s;
+ start = text->pos;
+ n = 0;
for (;;)
{
- struct substring short_name_ss, value_ss;
-
- if (!ss_tokenize (map->buffer, ss_cstr ("="), &map->pos, &short_name_ss)
- || !ss_tokenize (map->buffer, ss_buffer ("\t\0", 2), &map->pos,
- &value_ss))
- {
- if (*warning_cnt > max_warnings)
- sys_warn (r, _("Suppressed %d additional variable map warnings."),
- *warning_cnt - max_warnings);
- return false;
- }
+ int c = text->buffer.string[text->pos];
+ if (c < '0' || c > '9')
+ break;
+ n = (n * 10) + (c - '0');
+ text->pos++;
+ }
+ if (start == text->pos)
+ {
+ sys_warn (r, text->start,
+ _("Expecting digit at UTF-8 offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
- map->pos += ss_span (ss_substr (map->buffer, map->pos, SIZE_MAX),
- ss_buffer ("\t\0", 2));
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, text->start,
+ _("Expecting space at UTF-8 offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
- ss_data (short_name_ss)[ss_length (short_name_ss)] = '\0';
- *var = lookup_var_by_short_name (dict, ss_data (short_name_ss));
- if (*var == NULL)
- {
- if (++*warning_cnt <= max_warnings)
- sys_warn (r, _("Variable map refers to unknown variable %s."),
- ss_data (short_name_ss));
- continue;
- }
+ if (text->pos + n > text->buffer.length)
+ {
+ sys_warn (r, text->start,
+ _("%zu-byte string starting at UTF-8 offset %zu "
+ "exceeds record length %zu."),
+ n, text->pos, text->buffer.length);
+ return NULL;
+ }
- ss_data (value_ss)[ss_length (value_ss)] = '\0';
- *value = ss_data (value_ss);
+ s = &text->buffer.string[text->pos];
+ if (s[n] != ' ')
+ {
+ sys_warn (r, text->start,
+ _("Expecting space at UTF-8 offset %zu following %zu-byte "
+ "string."),
+ text->pos + n, n);
+ return NULL;
+ }
+ s[n] = '\0';
+ text->pos += n + 1;
+ return s;
+}
+static bool
+text_match (struct text_record *text, char c)
+{
+ if (text->buffer.string[text->pos] == c)
+ {
+ text->pos++;
return true;
}
+ else
+ return false;
+}
+
+/* Returns the current byte offset (as convertd to UTF-8) inside the TEXT's
+ string. */
+static size_t
+text_pos (const struct text_record *text)
+{
+ return text->pos;
}
\f
/* Messages. */
/* Displays a corruption message. */
static void
-sys_msg (struct sfm_reader *r, int class, const char *format, va_list args)
+sys_msg (struct sfm_reader *r, off_t offset,
+ int class, const char *format, va_list args)
{
struct msg m;
struct string text;
ds_init_empty (&text);
- ds_put_format (&text, "\"%s\" near offset 0x%lx: ",
- fh_get_file_name (r->fh), (unsigned long) ftell (r->file));
+ if (offset >= 0)
+ ds_put_format (&text, _("`%s' near offset 0x%llx: "),
+ fh_get_file_name (r->fh), (long long int) offset);
+ else
+ ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
ds_put_vformat (&text, format, args);
m.category = msg_class_to_category (class);
m.severity = msg_class_to_severity (class);
- m.where.file_name = NULL;
- m.where.line_number = 0;
+ m.file_name = NULL;
+ m.first_line = 0;
+ m.last_line = 0;
+ m.first_column = 0;
+ m.last_column = 0;
m.text = ds_cstr (&text);
msg_emit (&m);
}
-/* Displays a warning for the current file position. */
+/* Displays a warning for offset OFFSET in the file. */
static void
-sys_warn (struct sfm_reader *r, const char *format, ...)
+sys_warn (struct sfm_reader *r, off_t offset, const char *format, ...)
{
va_list args;
va_start (args, format);
- sys_msg (r, MW, format, args);
+ sys_msg (r, offset, MW, format, args);
va_end (args);
}
marks it as in an error state,
and aborts reading it using longjmp. */
static void
-sys_error (struct sfm_reader *r, const char *format, ...)
+sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
{
va_list args;
va_start (args, format);
- sys_msg (r, ME, format, args);
+ sys_msg (r, offset, ME, format, args);
va_end (args);
r->error = true;
void *buf, size_t byte_cnt)
{
size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
+ r->pos += bytes_read;
if (bytes_read == byte_cnt)
return true;
else if (ferror (r->file))
- sys_error (r, _("System error: %s."), strerror (errno));
+ sys_error (r, r->pos, _("System error: %s."), strerror (errno));
else if (!eof_is_ok || bytes_read != 0)
- sys_error (r, _("Unexpected end of file."));
+ sys_error (r, r->pos, _("Unexpected end of file."));
else
return false;
}
return float_get_double (r->float_format, number);
}
+static int
+parse_int (struct sfm_reader *r, const void *data, size_t ofs)
+{
+ return integer_get (r->integer_format, (const uint8_t *) data + ofs, 4);
+}
+
+static double
+parse_float (struct sfm_reader *r, const void *data, size_t ofs)
+{
+ return float_get_double (r->float_format, (const uint8_t *) data + ofs);
+}
+
/* Reads exactly SIZE - 1 bytes into BUFFER
and stores a null byte into BUFFER[SIZE - 1]. */
static void
}
}
\f
-static struct casereader_class sys_file_casereader_class =
+static const struct casereader_class sys_file_casereader_class =
{
sys_file_casereader_read,
sys_file_casereader_destroy,
projects / pspp-builds.git / blobdiff