+/* Reads record type 7, subtype 14, which gives the real length
+ of each very long string. Rearranges DICT accordingly. */
+static bool
+parse_long_string_map (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ struct text_record *text;
+ struct variable *var;
+ char *length_s;
+
+ text = open_text_record (r, record, true);
+ while (read_variable_to_value_pair (r, dict, text, &var, &length_s))
+ {
+ size_t idx = var_get_dict_index (var);
+ long int length;
+ int segment_cnt;
+ int i;
+
+ /* Get length. */
+ length = strtol (length_s, NULL, 10);
+ if (length < 1 || length > MAX_STRING)
+ {
+ sys_warn (r, record->pos,
+ _("%s listed as string of invalid length %s "
+ "in very long string record."),
+ var_get_name (var), length_s);
+ continue;
+ }
+
+ /* Check segments. */
+ segment_cnt = sfm_width_to_segments (length);
+ if (segment_cnt == 1)
+ {
+ sys_warn (r, record->pos,
+ _("%s listed in very long string record with width %s, "
+ "which requires only one segment."),
+ var_get_name (var), length_s);
+ continue;
+ }
+ if (idx + segment_cnt > dict_get_var_cnt (dict))
+ {
+ sys_error (r, record->pos,
+ _("Very long string %s overflows dictionary."),
+ var_get_name (var));
+ return false;
+ }
+
+ /* Get the short names from the segments and check their
+ lengths. */
+ for (i = 0; i < segment_cnt; i++)
+ {
+ struct variable *seg = dict_get_var (dict, idx + i);
+ int alloc_width = sfm_segment_alloc_width (length, i);
+ int width = var_get_width (seg);
+
+ if (i > 0)
+ var_set_short_name (var, i, var_get_short_name (seg, 0));
+ if (ROUND_UP (width, 8) != ROUND_UP (alloc_width, 8))
+ {
+ sys_error (r, record->pos,
+ _("Very long string with width %ld has segment %d "
+ "of width %d (expected %d)."),
+ length, i, width, alloc_width);
+ return false;
+ }
+ }
+ dict_delete_consecutive_vars (dict, idx + 1, segment_cnt - 1);
+ var_set_width (var, length);
+ }
+ close_text_record (r, text);
+ dict_compact_values (dict);
+
+ return true;
+}
+
+static bool
+parse_value_labels (struct sfm_reader *r, struct dictionary *dict,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ const struct sfm_value_label_record *record)
+{
+ struct variable **vars;
+ char **utf8_labels;
+ size_t i;
+
+ utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels);
+ for (i = 0; i < record->n_labels; i++)
+ utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict),
+ record->labels[i].label, -1,
+ r->pool);
+
+ vars = pool_nmalloc (r->pool, record->n_vars, sizeof *vars);
+ for (i = 0; i < record->n_vars; i++)
+ {
+ vars[i] = lookup_var_by_index (r, record->pos,
+ var_recs, n_var_recs, record->vars[i]);
+ if (vars[i] == NULL)
+ return false;
+ }
+
+ for (i = 1; i < record->n_vars; i++)
+ if (var_get_type (vars[i]) != var_get_type (vars[0]))
+ {
+ sys_error (r, record->pos,
+ _("Variables associated with value label are not all of "
+ "identical type. Variable %s is %s, but variable "
+ "%s is %s."),
+ var_get_name (vars[0]),
+ var_is_numeric (vars[0]) ? _("numeric") : _("string"),
+ var_get_name (vars[i]),
+ var_is_numeric (vars[i]) ? _("numeric") : _("string"));
+ return false;
+ }
+
+ for (i = 0; i < record->n_vars; i++)
+ {
+ struct variable *var = vars[i];
+ int width;
+ size_t j;
+
+ width = var_get_width (var);
+ if (width > 8)
+ {
+ sys_error (r, record->pos,
+ _("Value labels may not be added to long string "
+ "variables (e.g. %s) using records types 3 and 4."),
+ var_get_name (var));
+ return false;
+ }
+
+ for (j = 0; j < record->n_labels; j++)
+ {
+ struct sfm_value_label *label = &record->labels[j];
+ union value value;
+
+ value_init (&value, width);
+ if (width == 0)
+ value.f = parse_float (r, label->value, 0);
+ else
+ memcpy (value_str_rw (&value, width), label->value, width);
+
+ if (!var_add_value_label (var, &value, utf8_labels[j]))
+ {
+ if (var_is_numeric (var))
+ sys_warn (r, record->pos,
+ _("Duplicate value label for %g on %s."),
+ value.f, var_get_name (var));
+ else
+ sys_warn (r, record->pos,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value_str (&value, width),
+ var_get_name (var));
+ }
+
+ value_destroy (&value, width);
+ }
+ }
+
+ pool_free (r->pool, vars);
+ for (i = 0; i < record->n_labels; i++)
+ pool_free (r->pool, utf8_labels[i]);
+ pool_free (r->pool, utf8_labels);
+
+ return true;
+}
+
+static struct variable *
+lookup_var_by_index (struct sfm_reader *r, off_t offset,
+ const struct sfm_var_record *var_recs, size_t n_var_recs,
+ int idx)
+{
+ const struct sfm_var_record *rec;
+
+ if (idx < 1 || idx > n_var_recs)
+ {
+ sys_error (r, offset,
+ _("Variable index %d not in valid range 1...%zu."),
+ idx, n_var_recs);
+ return NULL;
+ }
+
+ rec = &var_recs[idx - 1];
+ if (rec->var == NULL)
+ {
+ sys_error (r, offset,
+ _("Variable index %d refers to long string continuation."),
+ idx);
+ return NULL;
+ }
+
+ return rec->var;
+}
+
+/* Parses a set of custom attributes from TEXT into ATTRS.
+ ATTRS may be a null pointer, in which case the attributes are
+ read but discarded. */
+static void
+parse_attributes (struct sfm_reader *r, struct text_record *text,
+ struct attrset *attrs)
+{
+ do
+ {
+ struct attribute *attr;
+ char *key;
+ int index;
+
+ /* Parse the key. */
+ key = text_get_token (text, ss_cstr ("("), NULL);
+ if (key == NULL)
+ return;
+
+ attr = attribute_create (key);
+ for (index = 1; ; index++)
+ {
+ /* Parse the value. */
+ char *value;
+ size_t length;
+
+ value = text_get_token (text, ss_cstr ("\n"), NULL);
+ if (value == NULL)
+ {
+ text_warn (r, text, _("Error parsing attribute value %s[%d]."),
+ key, index);
+ break;
+ }
+
+ length = strlen (value);
+ if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'')
+ {
+ value[length - 1] = '\0';
+ attribute_add_value (attr, value + 1);
+ }
+ else
+ {
+ text_warn (r, text,
+ _("Attribute value %s[%d] is not quoted: %s."),
+ key, index, value);
+ attribute_add_value (attr, value);
+ }
+
+ /* Was this the last value for this attribute? */
+ if (text_match (text, ')'))
+ break;
+ }
+ if (attrs != NULL)
+ attrset_add (attrs, attr);
+ else
+ attribute_destroy (attr);
+ }
+ while (!text_match (text, '/'));
+}
+
+/* Reads record type 7, subtype 17, which lists custom
+ attributes on the data file. */
+static void
+parse_data_file_attributes (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ struct text_record *text = open_text_record (r, record, true);
+ parse_attributes (r, text, dict_get_attributes (dict));
+ close_text_record (r, text);
+}
+
+/* Parses record type 7, subtype 18, which lists custom
+ attributes on individual variables. */
+static void
+parse_variable_attributes (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ struct text_record *text;
+ struct variable *var;
+
+ text = open_text_record (r, record, true);
+ while (text_read_variable_name (r, dict, text, ss_cstr (":"), &var))
+ parse_attributes (r, text, var != NULL ? var_get_attributes (var) : NULL);
+ close_text_record (r, text);
+}
+
+static void
+assign_variable_roles (struct sfm_reader *r, struct dictionary *dict)
+{
+ size_t n_warnings = 0;
+ size_t i;
+
+ for (i = 0; i < dict_get_var_cnt (dict); i++)
+ {
+ struct variable *var = dict_get_var (dict, i);
+ struct attrset *attrs = var_get_attributes (var);
+ const struct attribute *attr = attrset_lookup (attrs, "$@Role");
+ if (attr != NULL)
+ {
+ int value = atoi (attribute_get_value (attr, 0));
+ enum var_role role;
+
+ switch (value)
+ {
+ case 0:
+ role = ROLE_INPUT;
+ break;
+
+ case 1:
+ role = ROLE_TARGET;
+ break;
+
+ case 2:
+ role = ROLE_BOTH;
+ break;
+
+ case 3:
+ role = ROLE_NONE;
+ break;
+
+ case 4:
+ role = ROLE_PARTITION;
+ break;
+
+ case 5:
+ role = ROLE_SPLIT;
+ break;
+
+ default:
+ role = ROLE_INPUT;
+ if (n_warnings++ == 0)
+ sys_warn (r, -1, _("Invalid role for variable %s."),
+ var_get_name (var));
+ }
+
+ var_set_role (var, role);
+ }
+ }
+
+ if (n_warnings > 1)
+ sys_warn (r, -1, _("%zu other variables had invalid roles."),
+ n_warnings - 1);
+}
+
+static bool
+check_overflow (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ size_t ofs, size_t length)
+{
+ size_t end = record->size * record->count;
+ if (length >= end || ofs + length > end)
+ {
+ sys_warn (r, record->pos + end,
+ _("Extension record subtype %d ends unexpectedly."),
+ record->subtype);
+ return false;
+ }
+ return true;
+}
+
+static void
+parse_long_string_value_labels (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ size_t end = record->size * record->count;
+ size_t ofs = 0;
+
+ while (ofs < end)
+ {
+ char *var_name;
+ size_t n_labels, i;
+ struct variable *var;
+ union value value;
+ int var_name_len;
+ int width;
+
+ /* Parse variable name length. */
+ if (!check_overflow (r, record, ofs, 4))
+ return;
+ var_name_len = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse variable name, width, and number of labels. */
+ if (!check_overflow (r, record, ofs, var_name_len + 8))
+ return;
+ var_name = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ var_name_len, r->pool);
+ width = parse_int (r, record->data, ofs + var_name_len);
+ n_labels = parse_int (r, record->data, ofs + var_name_len + 4);
+ ofs += var_name_len + 8;
+
+ /* Look up 'var' and validate. */
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value label record for "
+ "unknown variable %s."), var_name);
+ else if (var_is_numeric (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value label record for "
+ "numeric variable %s."), var_name);
+ var = NULL;
+ }
+ else if (width != var_get_width (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value label record for variable "
+ "%s because the record's width (%d) does not match the "
+ "variable's width (%d)."),
+ var_name, width, var_get_width (var));
+ var = NULL;
+ }
+
+ /* Parse values. */
+ value_init_pool (r->pool, &value, width);
+ for (i = 0; i < n_labels; i++)
+ {
+ size_t value_length, label_length;
+ bool skip = var == NULL;
+
+ /* Parse value length. */
+ if (!check_overflow (r, record, ofs, 4))
+ return;
+ value_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse value. */
+ if (!check_overflow (r, record, ofs, value_length))
+ return;
+ if (!skip)
+ {
+ if (value_length == width)
+ memcpy (value_str_rw (&value, width),
+ (const uint8_t *) record->data + ofs, width);
+ else
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string value label %zu for "
+ "variable %s, with width %d, that has bad value "
+ "width %zu."),
+ i, var_get_name (var), width, value_length);
+ skip = true;
+ }
+ }
+ ofs += value_length;
+
+ /* Parse label length. */
+ if (!check_overflow (r, record, ofs, 4))
+ return;
+ label_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse label. */
+ if (!check_overflow (r, record, ofs, label_length))
+ return;
+ if (!skip)
+ {
+ char *label;
+
+ label = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ label_length, r->pool);
+ if (!var_add_value_label (var, &value, label))
+ sys_warn (r, record->pos + ofs,
+ _("Duplicate value label for `%.*s' on %s."),
+ width, value_str (&value, width),
+ var_get_name (var));
+ pool_free (r->pool, label);
+ }
+ ofs += label_length;
+ }
+ }
+}
+
+static void
+parse_long_string_missing_values (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ struct dictionary *dict)
+{
+ const char *dict_encoding = dict_get_encoding (dict);
+ size_t end = record->size * record->count;
+ size_t ofs = 0;
+
+ while (ofs < end)
+ {
+ struct missing_values mv;
+ char *var_name;
+ struct variable *var;
+ int n_missing_values;
+ int var_name_len;
+ size_t i;
+
+ /* Parse variable name length. */
+ if (!check_overflow (r, record, ofs, 4))
+ return;
+ var_name_len = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse variable name. */
+ if (!check_overflow (r, record, ofs, var_name_len + 1))
+ return;
+ var_name = recode_string_pool ("UTF-8", dict_encoding,
+ (const char *) record->data + ofs,
+ var_name_len, r->pool);
+ ofs += var_name_len;
+
+ /* Parse number of missing values. */
+ n_missing_values = ((const uint8_t *) record->data)[ofs];
+ if (n_missing_values < 1 || n_missing_values > 3)
+ sys_warn (r, record->pos + ofs,
+ _("Long string missing values record says variable %s "
+ "has %d missing values, but only 1 to 3 missing values "
+ "are allowed."),
+ var_name, n_missing_values);
+ ofs++;
+
+ /* Look up 'var' and validate. */
+ var = dict_lookup_var (dict, var_name);
+ if (var == NULL)
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "unknown variable %s."), var_name);
+ else if (var_is_numeric (var))
+ {
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value record for "
+ "numeric variable %s."), var_name);
+ var = NULL;
+ }
+
+ /* Parse values. */
+ mv_init_pool (r->pool, &mv, var ? var_get_width (var) : 8);
+ for (i = 0; i < n_missing_values; i++)
+ {
+ size_t value_length;
+
+ /* Parse value length. */
+ if (!check_overflow (r, record, ofs, 4))
+ return;
+ value_length = parse_int (r, record->data, ofs);
+ ofs += 4;
+
+ /* Parse value. */
+ if (!check_overflow (r, record, ofs, value_length))
+ return;
+ if (var != NULL
+ && i < 3
+ && !mv_add_str (&mv, (const uint8_t *) record->data + ofs,
+ value_length))
+ sys_warn (r, record->pos + ofs,
+ _("Ignoring long string missing value %zu for variable "
+ "%s, with width %d, that has bad value width %zu."),
+ i, var_get_name (var), var_get_width (var),
+ value_length);
+ ofs += value_length;
+ }
+ if (var != NULL)
+ var_set_missing_values (var, &mv);
+ }
+}
+\f
+/* Case reader. */
+
+static void partial_record (struct sfm_reader *);
+
+static void read_error (struct casereader *, const struct sfm_reader *);
+
+static bool read_case_number (struct sfm_reader *, double *);
+static int read_case_string (struct sfm_reader *, uint8_t *, size_t);
+static int read_opcode (struct sfm_reader *);
+static bool read_compressed_number (struct sfm_reader *, double *);
+static int read_compressed_string (struct sfm_reader *, uint8_t *);
+static int read_whole_strings (struct sfm_reader *, uint8_t *, size_t);
+static bool skip_whole_strings (struct sfm_reader *, size_t);
+
+/* Reads and returns one case from READER's file. Returns a null
+ pointer if not successful. */
+static struct ccase *
+sys_file_casereader_read (struct casereader *reader, void *r_)
+{
+ struct sfm_reader *r = r_;
+ struct ccase *c;
+ int retval;
+ int i;
+
+ if (r->error || !r->sfm_var_cnt)
+ return NULL;
+
+ c = case_create (r->proto);
+
+ for (i = 0; i < r->sfm_var_cnt; i++)
+ {
+ struct sfm_var *sv = &r->sfm_vars[i];
+ union value *v = case_data_rw_idx (c, sv->case_index);
+
+ if (sv->var_width == 0)
+ retval = read_case_number (r, &v->f);
+ else
+ {
+ uint8_t *s = value_str_rw (v, sv->var_width);
+ retval = read_case_string (r, s + sv->offset, sv->segment_width);
+ if (retval == 1)
+ {
+ retval = skip_whole_strings (r, ROUND_DOWN (sv->padding, 8));
+ if (retval == 0)
+ sys_error (r, r->pos, _("File ends in partial string value."));
+ }
+ }
+
+ if (retval != 1)
+ goto eof;
+ }
+ return c;
+
+eof:
+ if (i != 0)
+ partial_record (r);
+ if (r->case_cnt != -1)
+ read_error (reader, r);
+ case_unref (c);
+ return NULL;
+}
+
+/* Issues an error that R ends in a partial record. */
+static void
+partial_record (struct sfm_reader *r)
+{
+ sys_error (r, r->pos, _("File ends in partial case."));
+}
+
+/* Issues an error that an unspecified error occurred SFM, and
+ marks R tainted. */
+static void
+read_error (struct casereader *r, const struct sfm_reader *sfm)
+{
+ msg (ME, _("Error reading case from file %s."), fh_get_name (sfm->fh));
+ casereader_force_error (r);
+}
+
+/* Reads a number from R and stores its value in *D.
+ If R is compressed, reads a compressed number;
+ otherwise, reads a number in the regular way.
+ Returns true if successful, false if end of file is
+ reached immediately. */
+static bool
+read_case_number (struct sfm_reader *r, double *d)
+{
+ if (r->compression == ANY_COMP_NONE)
+ {
+ uint8_t number[8];
+ if (!try_read_bytes (r, number, sizeof number))
+ return false;
+ float_convert (r->float_format, number, FLOAT_NATIVE_DOUBLE, d);
+ return true;
+ }
+ else
+ return read_compressed_number (r, d);
+}
+
+/* Reads LENGTH string bytes from R into S. Always reads a multiple of 8
+ bytes; if LENGTH is not a multiple of 8, then extra bytes are read and
+ discarded without being written to S. Reads compressed strings if S is
+ compressed. Returns 1 if successful, 0 if end of file is reached
+ immediately, or -1 for some kind of error. */
+static int
+read_case_string (struct sfm_reader *r, uint8_t *s, size_t length)
+{
+ size_t whole = ROUND_DOWN (length, 8);
+ size_t partial = length % 8;
+
+ if (whole)
+ {
+ int retval = read_whole_strings (r, s, whole);
+ if (retval != 1)
+ return retval;
+ }
+
+ if (partial)
+ {
+ uint8_t bounce[8];
+ int retval = read_whole_strings (r, bounce, sizeof bounce);
+ if (retval == -1)
+ return -1;
+ else if (!retval)
+ {
+ if (whole)
+ {
+ partial_record (r);
+ return -1;
+ }
+ return 0;
+ }
+ memcpy (s + whole, bounce, partial);
+ }
+
+ return 1;
+}
+
+/* Reads and returns the next compression opcode from R. */
+static int
+read_opcode (struct sfm_reader *r)
+{
+ assert (r->compression != ANY_COMP_NONE);
+ for (;;)
+ {
+ int opcode;
+ if (r->opcode_idx >= sizeof r->opcodes)
+ {
+
+ int retval = try_read_compressed_bytes (r, r->opcodes,
+ sizeof r->opcodes);
+ if (retval != 1)
+ return -1;
+ r->opcode_idx = 0;
+ }
+ opcode = r->opcodes[r->opcode_idx++];
+
+ if (opcode != 0)
+ return opcode;
+ }
+}
+
+/* Reads a compressed number from R and stores its value in D.
+ Returns true if successful, false if end of file is
+ reached immediately. */
+static bool
+read_compressed_number (struct sfm_reader *r, double *d)
+{
+ int opcode = read_opcode (r);
+ switch (opcode)
+ {
+ case -1:
+ case 252:
+ return false;
+
+ case 253:
+ return read_compressed_float (r, d);
+
+ case 254:
+ float_convert (r->float_format, " ", FLOAT_NATIVE_DOUBLE, d);
+ if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, r->pos,
+ _("Possible compressed data corruption: "
+ "compressed spaces appear in numeric field."));
+ }
+ break;
+
+ case 255:
+ *d = SYSMIS;
+ break;
+
+ default:
+ *d = opcode - r->bias;
+ break;
+ }
+
+ return true;
+}
+
+/* Reads a compressed 8-byte string segment from R and stores it in DST. */
+static int
+read_compressed_string (struct sfm_reader *r, uint8_t *dst)
+{
+ int opcode;
+ int retval;
+
+ opcode = read_opcode (r);
+ switch (opcode)
+ {
+ case -1:
+ case 252:
+ return 0;
+
+ case 253:
+ retval = read_compressed_bytes (r, dst, 8);
+ return retval == 1 ? 1 : -1;
+
+ case 254:
+ memset (dst, ' ', 8);
+ return 1;
+
+ default:
+ {
+ double value = opcode - r->bias;
+ float_convert (FLOAT_NATIVE_DOUBLE, &value, r->float_format, dst);
+ if (value == 0.0)
+ {
+ /* This has actually been seen "in the wild". The submitter of the
+ file that showed that the contents decoded as spaces, but they
+ were at the end of the field so it's possible that the null
+ bytes just acted as null terminators. */
+ }
+ else if (!r->corruption_warning)
+ {
+ r->corruption_warning = true;
+ sys_warn (r, r->pos,
+ _("Possible compressed data corruption: "
+ "string contains compressed integer (opcode %d)."),
+ opcode);
+ }
+ }
+ return 1;
+ }
+}
+
+/* Reads LENGTH string bytes from R into S. LENGTH must be a multiple of 8.
+ Reads compressed strings if S is compressed. Returns 1 if successful, 0 if
+ end of file is reached immediately, or -1 for some kind of error. */
+static int
+read_whole_strings (struct sfm_reader *r, uint8_t *s, size_t length)
+{
+ assert (length % 8 == 0);
+ if (r->compression == ANY_COMP_NONE)
+ return try_read_bytes (r, s, length);
+ else
+ {
+ size_t ofs;
+
+ for (ofs = 0; ofs < length; ofs += 8)
+ {
+ int retval = read_compressed_string (r, s + ofs);
+ if (retval != 1)
+ {
+ if (ofs != 0)
+ {
+ partial_record (r);
+ return -1;
+ }
+ return retval;
+ }
+ }
+ return 1;
+ }
+}
+
+/* Skips LENGTH string bytes from R.
+ LENGTH must be a multiple of 8.
+ (LENGTH is also limited to 1024, but that's only because the
+ current caller never needs more than that many bytes.)
+ Returns true if successful, false if end of file is
+ reached immediately. */
+static bool
+skip_whole_strings (struct sfm_reader *r, size_t length)
+{
+ uint8_t buffer[1024];
+ assert (length < sizeof buffer);
+ return read_whole_strings (r, buffer, length);
+}
+\f
+/* Helpers for reading records that contain structured text
+ strings. */
+
+/* Maximum number of warnings to issue for a single text
+ record. */
+#define MAX_TEXT_WARNINGS 5
+
+/* State. */
+struct text_record
+ {
+ struct substring buffer; /* Record contents. */
+ off_t start; /* Starting offset in file. */
+ size_t pos; /* Current position in buffer. */
+ int n_warnings; /* Number of warnings issued or suppressed. */
+ bool recoded; /* Recoded into UTF-8? */
+ };
+
+static struct text_record *
+open_text_record (struct sfm_reader *r,
+ const struct sfm_extension_record *record,
+ bool recode_to_utf8)
+{
+ struct text_record *text;
+ struct substring raw;
+
+ text = pool_alloc (r->pool, sizeof *text);
+ raw = ss_buffer (record->data, record->size * record->count);
+ text->start = record->pos;
+ text->buffer = (recode_to_utf8
+ ? recode_substring_pool ("UTF-8", r->encoding, raw, r->pool)
+ : raw);
+ text->pos = 0;
+ text->n_warnings = 0;
+ text->recoded = recode_to_utf8;
+
+ return text;
+}
+
+/* Closes TEXT, frees its storage, and issues a final warning
+ about suppressed warnings if necesary. */
+static void
+close_text_record (struct sfm_reader *r, struct text_record *text)
+{
+ if (text->n_warnings > MAX_TEXT_WARNINGS)
+ sys_warn (r, -1, _("Suppressed %d additional related warnings."),
+ text->n_warnings - MAX_TEXT_WARNINGS);
+ if (text->recoded)
+ pool_free (r->pool, ss_data (text->buffer));
+}
+
+/* Reads a variable=value pair from TEXT.
+ Looks up the variable in DICT and stores it into *VAR.
+ Stores a null-terminated value into *VALUE. */
+static bool
+read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text,
+ struct variable **var, char **value)
+{
+ for (;;)
+ {
+ if (!text_read_short_name (r, dict, text, ss_cstr ("="), var))
+ return false;
+
+ *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL);
+ if (*value == NULL)
+ return false;
+
+ text->pos += ss_span (ss_substr (text->buffer, text->pos, SIZE_MAX),
+ ss_buffer ("\t\0", 2));
+
+ if (*var != NULL)
+ return true;
+ }
+}
+
+static bool
+text_read_variable_name (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text, struct substring delimiters,
+ struct variable **var)
+{
+ char *name;
+
+ name = text_get_token (text, delimiters, NULL);
+ if (name == NULL)
+ return false;
+
+ *var = dict_lookup_var (dict, name);
+ if (*var != NULL)
+ return true;
+
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
+ name);
+ return false;
+}
+
+
+static bool
+text_read_short_name (struct sfm_reader *r, struct dictionary *dict,
+ struct text_record *text, struct substring delimiters,
+ struct variable **var)
+{
+ char *short_name = text_get_token (text, delimiters, NULL);
+ if (short_name == NULL)
+ return false;
+
+ *var = dict_lookup_var (dict, short_name);
+ if (*var == NULL)
+ text_warn (r, text, _("Dictionary record refers to unknown variable %s."),
+ short_name);
+ return true;
+}
+
+/* Displays a warning for the current file position, limiting the
+ number to MAX_TEXT_WARNINGS for TEXT. */
+static void
+text_warn (struct sfm_reader *r, struct text_record *text,
+ const char *format, ...)
+{
+ if (text->n_warnings++ < MAX_TEXT_WARNINGS)
+ {
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, text->start + text->pos, MW, format, args);
+ va_end (args);
+ }
+}
+
+static char *
+text_get_token (struct text_record *text, struct substring delimiters,
+ char *delimiter)
+{
+ struct substring token;
+ char *end;
+
+ if (!ss_tokenize (text->buffer, delimiters, &text->pos, &token))
+ return NULL;
+
+ end = &ss_data (token)[ss_length (token)];
+ if (delimiter != NULL)
+ *delimiter = *end;
+ *end = '\0';
+ return ss_data (token);
+}
+
+/* Reads a integer value expressed in decimal, then a space, then a string that
+ consists of exactly as many bytes as specified by the integer, then a space,
+ from TEXT. Returns the string, null-terminated, as a subset of TEXT's
+ buffer (so the caller should not free the string). */
+static const char *
+text_parse_counted_string (struct sfm_reader *r, struct text_record *text)
+{
+ size_t start;
+ size_t n;
+ char *s;
+
+ start = text->pos;
+ n = 0;
+ while (text->pos < text->buffer.length)
+ {
+ int c = text->buffer.string[text->pos];
+ if (c < '0' || c > '9')
+ break;
+ n = (n * 10) + (c - '0');
+ text->pos++;
+ }
+ if (text->pos >= text->buffer.length || start == text->pos)
+ {
+ sys_warn (r, text->start,
+ _("Expecting digit at offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
+
+ if (!text_match (text, ' '))
+ {
+ sys_warn (r, text->start,
+ _("Expecting space at offset %zu in MRSETS record."),
+ text->pos);
+ return NULL;
+ }
+
+ if (text->pos + n > text->buffer.length)
+ {
+ sys_warn (r, text->start,
+ _("%zu-byte string starting at offset %zu "
+ "exceeds record length %zu."),
+ n, text->pos, text->buffer.length);
+ return NULL;
+ }
+
+ s = &text->buffer.string[text->pos];
+ if (s[n] != ' ')
+ {
+ sys_warn (r, text->start,
+ _("Expecting space at offset %zu following %zu-byte string."),
+ text->pos + n, n);
+ return NULL;
+ }
+ s[n] = '\0';
+ text->pos += n + 1;
+ return s;
+}
+
+static bool
+text_match (struct text_record *text, char c)
+{
+ if (text->buffer.string[text->pos] == c)
+ {
+ text->pos++;
+ return true;
+ }
+ else
+ return false;
+}
+
+/* Returns the current byte offset (as converted to UTF-8, if it was converted)
+ inside the TEXT's string. */
+static size_t
+text_pos (const struct text_record *text)
+{
+ return text->pos;
+}
+
+static const char *
+text_get_all (const struct text_record *text)
+{
+ return text->buffer.string;
+}
+\f
+/* Messages. */
+
+/* Displays a corruption message. */
+static void
+sys_msg (struct sfm_reader *r, off_t offset,
+ int class, const char *format, va_list args)
+{
+ struct msg m;
+ struct string text;
+
+ ds_init_empty (&text);
+ if (offset >= 0)
+ ds_put_format (&text, _("`%s' near offset 0x%llx: "),
+ fh_get_file_name (r->fh), (long long int) offset);
+ else
+ ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh));
+ ds_put_vformat (&text, format, args);
+
+ m.category = msg_class_to_category (class);
+ m.severity = msg_class_to_severity (class);
+ m.file_name = NULL;
+ m.first_line = 0;
+ m.last_line = 0;
+ m.first_column = 0;
+ m.last_column = 0;
+ m.text = ds_cstr (&text);
+
+ msg_emit (&m);
+}
+
+/* Displays a warning for offset OFFSET in the file. */
+static void
+sys_warn (struct sfm_reader *r, off_t offset, const char *format, ...)
+{
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, offset, MW, format, args);
+ va_end (args);
+}
+
+/* Displays an error for the current file position and marks it as in an error
+ state. */
+static void
+sys_error (struct sfm_reader *r, off_t offset, const char *format, ...)
+{
+ va_list args;
+
+ va_start (args, format);
+ sys_msg (r, offset, ME, format, args);
+ va_end (args);
+
+ r->error = true;
+}
+\f
+/* Reads BYTE_CNT bytes into BUF.
+ Returns 1 if exactly BYTE_CNT bytes are successfully read.
+ Returns -1 if an I/O error or a partial read occurs.
+ Returns 0 for an immediate end-of-file and, if EOF_IS_OK is false, reports
+ an error. */
+static inline int
+read_bytes_internal (struct sfm_reader *r, bool eof_is_ok,
+ void *buf, size_t byte_cnt)
+{
+ size_t bytes_read = fread (buf, 1, byte_cnt, r->file);
+ r->pos += bytes_read;
+ if (bytes_read == byte_cnt)
+ return 1;
+ else if (ferror (r->file))
+ {
+ sys_error (r, r->pos, _("System error: %s."), strerror (errno));
+ return -1;
+ }
+ else if (!eof_is_ok || bytes_read != 0)
+ {
+ sys_error (r, r->pos, _("Unexpected end of file."));
+ return -1;
+ }
+ else
+ return 0;
+}
+
+/* Reads BYTE_CNT into BUF.
+ Returns true if successful.
+ Returns false upon I/O error or if end-of-file is encountered. */
+static bool
+read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+{
+ return read_bytes_internal (r, false, buf, byte_cnt) == 1;
+}
+
+/* Reads BYTE_CNT bytes into BUF.
+ Returns 1 if exactly BYTE_CNT bytes are successfully read.
+ Returns 0 if an immediate end-of-file is encountered.
+ Returns -1 if an I/O error or a partial read occurs. */
+static int
+try_read_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+{
+ return read_bytes_internal (r, true, buf, byte_cnt);
+}
+
+/* Reads a 32-bit signed integer from R and stores its value in host format in
+ *X. Returns true if successful, otherwise false. */
+static bool
+read_int (struct sfm_reader *r, int *x)
+{
+ uint8_t integer[4];
+ if (read_bytes (r, integer, sizeof integer) != 1)
+ return false;
+ *x = integer_get (r->integer_format, integer, sizeof integer);
+ return true;
+}
+
+static bool
+read_uint (struct sfm_reader *r, unsigned int *x)
+{
+ bool ok;
+ int y;
+
+ ok = read_int (r, &y);
+ *x = y;
+ return ok;
+}
+
+/* Reads a 64-bit signed integer from R and returns its value in
+ host format. */
+static bool
+read_int64 (struct sfm_reader *r, long long int *x)
+{
+ uint8_t integer[8];
+ if (read_bytes (r, integer, sizeof integer) != 1)
+ return false;
+ *x = integer_get (r->integer_format, integer, sizeof integer);
+ return true;
+}
+
+/* Reads a 64-bit signed integer from R and returns its value in
+ host format. */
+static bool
+read_uint64 (struct sfm_reader *r, unsigned long long int *x)
+{
+ long long int y;
+ bool ok;
+
+ ok = read_int64 (r, &y);
+ *x = y;
+ return ok;
+}
+
+static int
+parse_int (const struct sfm_reader *r, const void *data, size_t ofs)
+{
+ return integer_get (r->integer_format, (const uint8_t *) data + ofs, 4);
+}
+
+static double
+parse_float (const struct sfm_reader *r, const void *data, size_t ofs)
+{
+ return float_get_double (r->float_format, (const uint8_t *) data + ofs);
+}
+
+/* Reads exactly SIZE - 1 bytes into BUFFER
+ and stores a null byte into BUFFER[SIZE - 1]. */
+static bool
+read_string (struct sfm_reader *r, char *buffer, size_t size)
+{
+ bool ok;
+
+ assert (size > 0);
+ ok = read_bytes (r, buffer, size - 1);
+ if (ok)
+ buffer[size - 1] = '\0';
+ return ok;
+}
+
+/* Skips BYTES bytes forward in R. */
+static bool
+skip_bytes (struct sfm_reader *r, size_t bytes)
+{
+ while (bytes > 0)
+ {
+ char buffer[1024];
+ size_t chunk = MIN (sizeof buffer, bytes);
+ if (!read_bytes (r, buffer, chunk))
+ return false;
+ bytes -= chunk;
+ }
+
+ return true;
+}
+
+/* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have
+ been replaced by LFs.
+
+ (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system
+ files that use CR-only line ends in the file label and extra product
+ info.) */
+static char *
+fix_line_ends (const char *s)
+{
+ char *dst, *d;
+
+ d = dst = xmalloc (strlen (s) + 1);
+ while (*s != '\0')
+ {
+ if (*s == '\r')
+ {
+ s++;
+ if (*s == '\n')
+ s++;
+ *d++ = '\n';
+ }
+ else
+ *d++ = *s++;
+ }
+ *d = '\0';
+
+ return dst;
+}
+\f
+static bool
+read_ztrailer (struct sfm_reader *r,
+ long long int zheader_ofs,
+ long long int ztrailer_len);
+
+static void *
+zalloc (voidpf pool_, uInt items, uInt size)
+{
+ struct pool *pool = pool_;
+
+ return (!size || xalloc_oversized (items, size)
+ ? Z_NULL
+ : pool_malloc (pool, items * size));
+}
+
+static void
+zfree (voidpf pool_, voidpf address)
+{
+ struct pool *pool = pool_;
+
+ pool_free (pool, address);
+}
+
+static bool
+read_zheader (struct sfm_reader *r)
+{
+ off_t pos = r->pos;
+ long long int zheader_ofs;
+ long long int ztrailer_ofs;
+ long long int ztrailer_len;
+
+ if (!read_int64 (r, &zheader_ofs)
+ || !read_int64 (r, &ztrailer_ofs)
+ || !read_int64 (r, &ztrailer_len))
+ return false;
+
+ if (zheader_ofs != pos)
+ {
+ sys_error (r, pos, _("Wrong ZLIB data header offset %#llx "
+ "(expected %#llx)."),
+ zheader_ofs, (long long int) pos);
+ return false;
+ }
+
+ if (ztrailer_ofs < r->pos)
+ {
+ sys_error (r, pos, _("Impossible ZLIB trailer offset 0x%llx."),
+ ztrailer_ofs);
+ return false;
+ }
+
+ if (ztrailer_len < 24 || ztrailer_len % 24)
+ {
+ sys_error (r, pos, _("Invalid ZLIB trailer length %lld."), ztrailer_len);
+ return false;
+ }
+
+ r->ztrailer_ofs = ztrailer_ofs;
+ if (!read_ztrailer (r, zheader_ofs, ztrailer_len))
+ return false;
+
+ if (r->zin_buf == NULL)
+ {
+ r->zin_buf = pool_malloc (r->pool, ZIN_BUF_SIZE);
+ r->zout_buf = pool_malloc (r->pool, ZOUT_BUF_SIZE);
+ r->zstream.next_in = NULL;
+ r->zstream.avail_in = 0;
+ }
+
+ r->zstream.zalloc = zalloc;
+ r->zstream.zfree = zfree;
+ r->zstream.opaque = r->pool;
+
+ return open_zstream (r);
+}
+
+static void
+seek (struct sfm_reader *r, off_t offset)
+{
+ if (fseeko (r->file, offset, SEEK_SET))
+ sys_error (r, 0, _("%s: seek failed (%s)."),
+ fh_get_file_name (r->fh), strerror (errno));
+ r->pos = offset;
+}
+
+/* Performs some additional consistency checks on the ZLIB compressed data
+ trailer. */
+static bool
+read_ztrailer (struct sfm_reader *r,
+ long long int zheader_ofs,
+ long long int ztrailer_len)
+{
+ long long int expected_uncmp_ofs;
+ long long int expected_cmp_ofs;
+ long long int bias;
+ long long int zero;
+ unsigned int block_size;
+ unsigned int n_blocks;
+ unsigned int i;
+ struct stat s;
+
+ if (fstat (fileno (r->file), &s))
+ {
+ sys_error (ME, 0, _("%s: stat failed (%s)."),
+ fh_get_file_name (r->fh), strerror (errno));
+ return false;
+ }
+
+ if (!S_ISREG (s.st_mode))
+ {
+ /* We can't seek to the trailer and then back to the data in this file,
+ so skip doing extra checks. */
+ return true;
+ }
+
+ if (r->ztrailer_ofs + ztrailer_len != s.st_size)
+ sys_warn (r, r->pos,
+ _("End of ZLIB trailer (0x%llx) is not file size (0x%llx)."),
+ r->ztrailer_ofs + ztrailer_len, (long long int) s.st_size);
+
+ seek (r, r->ztrailer_ofs);
+
+ /* Read fixed header from ZLIB data trailer. */
+ if (!read_int64 (r, &bias))
+ return false;
+ if (-bias != r->bias)
+ {
+ sys_error (r, r->pos, _("ZLIB trailer bias (%lld) differs from "
+ "file header bias (%.2f)."),
+ -bias, r->bias);
+ return false;
+ }
+
+ if (!read_int64 (r, &zero))
+ return false;
+ if (zero != 0)
+ sys_warn (r, r->pos,
+ _("ZLIB trailer \"zero\" field has nonzero value %lld."), zero);
+
+ if (!read_uint (r, &block_size))
+ return false;
+ if (block_size != ZBLOCK_SIZE)
+ sys_warn (r, r->pos,
+ _("ZLIB trailer specifies unexpected %u-byte block size."),
+ block_size);
+
+ if (!read_uint (r, &n_blocks))
+ return false;
+ if (n_blocks != (ztrailer_len - 24) / 24)
+ {
+ sys_error (r, r->pos,
+ _("%lld-byte ZLIB trailer specifies %u data blocks (expected "
+ "%lld)."),
+ ztrailer_len, n_blocks, (ztrailer_len - 24) / 24);
+ return false;
+ }
+
+ expected_uncmp_ofs = zheader_ofs;
+ expected_cmp_ofs = zheader_ofs + 24;
+ for (i = 0; i < n_blocks; i++)
+ {
+ off_t desc_ofs = r->pos;
+ unsigned long long int uncompressed_ofs;
+ unsigned long long int compressed_ofs;
+ unsigned int uncompressed_size;
+ unsigned int compressed_size;
+
+ if (!read_uint64 (r, &uncompressed_ofs)
+ || !read_uint64 (r, &compressed_ofs)
+ || !read_uint (r, &uncompressed_size)
+ || !read_uint (r, &compressed_size))
+ return false;
+
+ if (uncompressed_ofs != expected_uncmp_ofs)
+ {
+ sys_error (r, desc_ofs,
+ _("ZLIB block descriptor %u reported uncompressed data "
+ "offset %#llx, when %#llx was expected."),
+ i, uncompressed_ofs, expected_uncmp_ofs);
+ return false;
+ }
+
+ if (compressed_ofs != expected_cmp_ofs)
+ {
+ sys_error (r, desc_ofs,
+ _("ZLIB block descriptor %u reported compressed data "
+ "offset %#llx, when %#llx was expected."),
+ i, compressed_ofs, expected_cmp_ofs);
+ return false;
+ }
+
+ if (i < n_blocks - 1)
+ {
+ if (uncompressed_size != block_size)
+ sys_warn (r, desc_ofs,
+ _("ZLIB block descriptor %u reported block size %#x, "
+ "when %#x was expected."),
+ i, uncompressed_size, block_size);
+ }
+ else
+ {
+ if (uncompressed_size > block_size)
+ sys_warn (r, desc_ofs,
+ _("ZLIB block descriptor %u reported block size %#x, "
+ "when at most %#x was expected."),
+ i, uncompressed_size, block_size);
+ }
+
+ /* http://www.zlib.net/zlib_tech.html says that the maximum expansion
+ from compression, with worst-case parameters, is 13.5% plus 11 bytes.
+ This code checks for an expansion of more than 14.3% plus 11
+ bytes. */
+ if (compressed_size > uncompressed_size + uncompressed_size / 7 + 11)
+ {
+ sys_error (r, desc_ofs,
+ _("ZLIB block descriptor %u reports compressed size %u "
+ "and uncompressed size %u."),
+ i, compressed_size, uncompressed_size);
+ return false;
+ }
+
+ expected_uncmp_ofs += uncompressed_size;
+ expected_cmp_ofs += compressed_size;
+ }
+
+ if (expected_cmp_ofs != r->ztrailer_ofs)
+ {
+ sys_error (r, r->pos, _("ZLIB trailer is at offset %#llx but %#llx "
+ "would be expected from block descriptors."),
+ r->ztrailer_ofs, expected_cmp_ofs);
+ return false;
+ }
+
+ seek (r, zheader_ofs + 24);
+ return true;
+}
+
+static bool
+open_zstream (struct sfm_reader *r)
+{
+ int error;
+
+ r->zout_pos = r->zout_end = 0;
+ error = inflateInit (&r->zstream);
+ if (error != Z_OK)
+ {
+ sys_error (r, r->pos, _("ZLIB initialization failed (%s)."),
+ r->zstream.msg);
+ return false;
+ }
+ return true;
+}
+
+static bool
+close_zstream (struct sfm_reader *r)
+{
+ int error;
+
+ error = inflateEnd (&r->zstream);
+ if (error != Z_OK)
+ {
+ sys_error (r, r->pos, _("Inconsistency at end of ZLIB stream (%s)."),
+ r->zstream.msg);
+ return false;
+ }
+ return true;
+}
+
+static int
+read_bytes_zlib (struct sfm_reader *r, void *buf_, size_t byte_cnt)
+{
+ uint8_t *buf = buf_;
+
+ if (byte_cnt == 0)
+ return 1;
+
+ for (;;)
+ {
+ int error;
+
+ /* Use already inflated data if there is any. */
+ if (r->zout_pos < r->zout_end)
+ {
+ unsigned int n = MIN (byte_cnt, r->zout_end - r->zout_pos);
+ memcpy (buf, &r->zout_buf[r->zout_pos], n);
+ r->zout_pos += n;
+ byte_cnt -= n;
+ buf += n;
+
+ if (byte_cnt == 0)
+ return 1;
+ }
+
+ /* We need to inflate some more data.
+ Get some more input data if we don't have any. */
+ if (r->zstream.avail_in == 0)
+ {
+ unsigned int n = MIN (ZIN_BUF_SIZE, r->ztrailer_ofs - r->pos);
+ if (n == 0)
+ return 0;
+ else
+ {
+ int retval = try_read_bytes (r, r->zin_buf, n);
+ if (retval != 1)
+ return retval;
+ r->zstream.avail_in = n;
+ r->zstream.next_in = r->zin_buf;
+ }
+ }
+
+ /* Inflate the (remaining) input data. */
+ r->zstream.avail_out = ZOUT_BUF_SIZE;
+ r->zstream.next_out = r->zout_buf;
+ error = inflate (&r->zstream, Z_SYNC_FLUSH);
+ r->zout_pos = 0;
+ r->zout_end = r->zstream.next_out - r->zout_buf;
+ if (r->zout_end == 0)
+ {
+ if (error != Z_STREAM_END)
+ {
+ sys_error (r, r->pos, _("ZLIB stream inconsistency (%s)."),
+ r->zstream.msg);
+ return -1;
+ }
+ else if (!close_zstream (r) || !open_zstream (r))
+ return -1;
+ }
+ else
+ {
+ /* Process the output data and ignore 'error' for now. ZLIB will
+ present it to us again on the next inflate() call. */
+ }
+ }
+}
+
+static int
+read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+{
+ if (r->compression == ANY_COMP_SIMPLE)
+ return read_bytes (r, buf, byte_cnt);
+ else
+ {
+ int retval = read_bytes_zlib (r, buf, byte_cnt);
+ if (retval == 0)
+ sys_error (r, r->pos, _("Unexpected end of ZLIB compressed data."));
+ return retval;
+ }
+}
+
+static int
+try_read_compressed_bytes (struct sfm_reader *r, void *buf, size_t byte_cnt)
+{
+ if (r->compression == ANY_COMP_SIMPLE)
+ return try_read_bytes (r, buf, byte_cnt);
+ else
+ return read_bytes_zlib (r, buf, byte_cnt);
+}
+
+/* Reads a 64-bit floating-point number from R and returns its
+ value in host format. */
+static bool
+read_compressed_float (struct sfm_reader *r, double *d)
+{
+ uint8_t number[8];
+
+ if (!read_compressed_bytes (r, number, sizeof number))
+ return false;
+
+ *d = float_get_double (r->float_format, number);
+ return true;
+}
+\f
+static const struct casereader_class sys_file_casereader_class =
+ {
+ sys_file_casereader_read,
+ sys_file_casereader_destroy,
+ NULL,
+ NULL,
+ };
+
+const struct any_reader_class sys_file_reader_class =
+ {
+ N_("SPSS System File"),
+ sfm_detect,
+ sfm_open,
+ sfm_close,
+ sfm_decode,
+ sfm_get_strings,
+ };