X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fsys-file-reader.c;h=e0b43b9205c59db8b777baacc7c65b1125f1b616;hb=a49b940e58f148bf111c647d9b4822025636ff80;hp=ddc685431b470c335694f98378d97754817ed056;hpb=659254002f2115c30b272cee41870586a9cf0d80;p=pspp diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c index ddc685431b..e0b43b9205 100644 --- a/src/data/sys-file-reader.c +++ b/src/data/sys-file-reader.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-2000, 2006-2007, 2009-2015 Free Software Foundation, Inc. + Copyright (C) 1997-2000, 2006-2007, 2009-2016 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,6 +45,7 @@ #include "libpspp/assertion.h" #include "libpspp/compiler.h" #include "libpspp/i18n.h" +#include "libpspp/ll.h" #include "libpspp/message.h" #include "libpspp/misc.h" #include "libpspp/pool.h" @@ -158,6 +159,7 @@ struct sfm_mrset struct sfm_extension_record { + struct ll ll; /* In struct sfm_reader 'var_attrs' list. */ int subtype; /* Record subtype. */ off_t pos; /* Starting offset in file. */ unsigned int size; /* Size of data elements. */ @@ -184,6 +186,7 @@ struct sfm_reader struct sfm_mrset *mrsets; size_t n_mrsets; struct sfm_extension_record *extensions[32]; + struct ll_list var_attrs; /* Contains "struct sfm_extension_record"s. */ /* File state. */ struct file_handle *fh; /* File handle. */ @@ -200,6 +203,7 @@ struct sfm_reader size_t sfm_var_cnt; /* Number of variables. */ int case_cnt; /* Number of cases */ const char *encoding; /* String encoding. */ + bool written_by_readstat; /* From https://github.com/WizardMac/ReadStat? */ /* Decompression. */ enum any_compression compression; @@ -279,7 +283,7 @@ static bool read_variable_record (struct sfm_reader *, struct sfm_var_record *); static bool read_value_label_record (struct sfm_reader *, struct sfm_value_label_record *); -static struct sfm_document_record *read_document_record (struct sfm_reader *); +static bool read_document_record (struct sfm_reader *); static bool read_extension_record (struct sfm_reader *, int subtype, struct sfm_extension_record **); static bool skip_extension_record (struct sfm_reader *, int subtype); @@ -294,8 +298,7 @@ static bool read_variable_to_value_pair (struct sfm_reader *, struct text_record *, struct variable **var, char **value); static void text_warn (struct sfm_reader *r, struct text_record *text, - const char *format, ...) - PRINTF_FORMAT (3, 4); + const char *format, ...) PRINTF_FORMAT (3, 4); static char *text_get_token (struct text_record *, struct substring delimiters, char *delimiter); static bool text_match (struct text_record *, char c); @@ -401,6 +404,7 @@ sfm_open (struct file_handle *fh) pool_register (r->pool, free, r); r->fh = fh_ref (fh); r->opcode_idx = sizeof r->opcodes; + ll_init (&r->var_attrs); /* TRANSLATORS: this fragment will be interpolated into messages in fh_lock() that identify types of files. */ @@ -408,7 +412,7 @@ sfm_open (struct file_handle *fh) if (r->lock == NULL) goto error; - r->file = fn_open (fh_get_file_name (fh), "rb"); + r->file = fn_open (fh, "rb"); if (r->file == NULL) { msg (ME, _("Error opening `%s' for reading as a system file: %s."), @@ -497,8 +501,7 @@ read_record (struct sfm_reader *r, int type, sys_error (r, r->pos, _("Duplicate type 6 (document) record.")); return false; } - r->document = read_document_record (r); - return r->document != NULL; + return read_document_record (r); case 7: if (!read_int (r, &subtype)) @@ -513,6 +516,17 @@ read_record (struct sfm_reader *r, int type, subtype, PACKAGE_BUGREPORT, PACKAGE_STRING); return skip_extension_record (r, subtype); } + else if (subtype == 18) + { + /* System files written by "Stata 14.1/-savespss- 1.77 by S.Radyakin" + put each variable attribute into a separate record with subtype + 18. I'm surprised that SPSS puts up with this. */ + struct sfm_extension_record *ext; + bool ok = read_extension_record (r, subtype, &ext); + if (ok && ext) + ll_push_tail (&r->var_attrs, &ext->ll); + return ok; + } else if (r->extensions[subtype] != NULL) { sys_warn (r, r->pos, @@ -719,7 +733,6 @@ sfm_get_strings (const struct any_reader *r_, struct pool *pool, mrset_idx); } - /* */ /* data file attributes */ /* variable attributes */ /* long var map */ @@ -830,14 +843,15 @@ sfm_decode (struct any_reader *r_, const char *encoding, parse_long_var_name_map (r, r->extensions[EXT_LONG_NAMES], dict); /* The following records use long names, so they need to follow renaming. */ - if (r->extensions[EXT_VAR_ATTRS] != NULL) + if (!ll_is_empty (&r->var_attrs)) { - parse_variable_attributes (r, r->extensions[EXT_VAR_ATTRS], dict); + struct sfm_extension_record *ext; + ll_for_each (ext, struct sfm_extension_record, ll, &r->var_attrs) + parse_variable_attributes (r, ext, dict); /* Roles use the $@Role attribute. */ assign_variable_roles (r, dict); } - if (r->extensions[EXT_LONG_LABELS] != NULL) parse_long_string_value_labels (r, r->extensions[EXT_LONG_LABELS], dict); if (r->extensions[EXT_LONG_MISSING] != NULL) @@ -848,7 +862,7 @@ sfm_decode (struct any_reader *r_, const char *encoding, amount that the header claims. SPSS version 13 gets this wrong when very long strings are involved, so don't warn in that case. */ - if (r->header.nominal_case_size != -1 + if (r->header.nominal_case_size > 0 && r->header.nominal_case_size != r->n_vars && r->info.version_major != 13) sys_warn (r, -1, _("File header claims %d variable positions but " @@ -877,7 +891,7 @@ sfm_decode (struct any_reader *r_, const char *encoding, error: sfm_close (r_); - dict_destroy (dict); + dict_unref (dict); *dictp = NULL; return NULL; } @@ -894,7 +908,7 @@ sfm_close (struct any_reader *r_) if (r->file) { - if (fn_close (fh_get_file_name (r->fh), r->file) == EOF) + if (fn_close (r->fh, r->file) == EOF) { msg (ME, _("Error closing system file `%s': %s."), fh_get_file_name (r->fh), strerror (errno)); @@ -954,6 +968,8 @@ read_header (struct sfm_reader *r, struct any_read_info *info, if (!read_string (r, header->magic, sizeof header->magic) || !read_string (r, header->eye_catcher, sizeof header->eye_catcher)) return false; + r->written_by_readstat = strstr (header->eye_catcher, + "https://github.com/WizardMac/ReadStat"); if (!strcmp (ASCII_MAGIC, header->magic) || !strcmp (EBCDIC_MAGIC, header->magic)) @@ -1215,33 +1231,35 @@ read_value_label_record (struct sfm_reader *r, return true; } -/* Reads a document record from R and returns it. */ -static struct sfm_document_record * +/* Reads a document record from R. Returns true if successful, false on + error. */ +static bool read_document_record (struct sfm_reader *r) { - struct sfm_document_record *record; int n_lines; - - record = pool_malloc (r->pool, sizeof *record); - record->pos = r->pos; - if (!read_int (r, &n_lines)) - return NULL; - if (n_lines <= 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH) + return false; + else if (n_lines == 0) + return true; + else if (n_lines < 0 || n_lines >= INT_MAX / DOC_LINE_LENGTH) { - sys_error (r, record->pos, + sys_error (r, r->pos, _("Number of document lines (%d) " "must be greater than 0 and less than %d."), n_lines, INT_MAX / DOC_LINE_LENGTH); - return NULL; + return false; } + struct sfm_document_record *record; + record = pool_malloc (r->pool, sizeof *record); + record->pos = r->pos; record->n_lines = n_lines; record->documents = pool_malloc (r->pool, DOC_LINE_LENGTH * n_lines); if (!read_bytes (r, record->documents, DOC_LINE_LENGTH * n_lines)) - return NULL; + return false; - return record; + r->document = record; + return true; } static bool @@ -1439,11 +1457,13 @@ parse_variable_records (struct sfm_reader *r, struct dictionary *dict, "`%s' to `%s'."), name, new_name); var = rec->var = dict_create_var_assert (dict, new_name, rec->width); + var_set_short_name (var, 0, new_name); free (new_name); } - /* Set the short name the same as the long name. */ - var_set_short_name (var, 0, name); + /* Set the short name the same as the long name (even if we renamed + it). */ + var_set_short_name (var, 0, var_get_name (var)); /* Get variable label, if any. */ if (rec->label) @@ -1524,22 +1544,9 @@ parse_format_spec (struct sfm_reader *r, off_t pos, unsigned int format, int *n_warnings) { const int max_warnings = 8; - uint8_t raw_type = format >> 16; - uint8_t w = format >> 8; - uint8_t d = format; struct fmt_spec f; - bool ok; - - f.w = w; - f.d = d; - msg_disable (); - ok = (fmt_from_io (raw_type, &f.type) - && fmt_check_output (&f) - && fmt_check_width_compat (&f, var_get_width (v))); - msg_enable (); - - if (ok) + if (fmt_from_u32 (format, var_get_width (v), false, &f)) { if (which == PRINT_FORMAT) var_set_print_format (v, &f); @@ -1742,7 +1749,12 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, } number = text_get_token (text, ss_cstr (" "), NULL); - if (!strcmp (number, "11")) + if (!number) + sys_warn (r, record->pos, + _("Missing label source value " + "following `E' at offset %zu in MRSETS record."), + text_pos (text)); + else if (!strcmp (number, "11")) mrset->label_from_var_label = true; else if (strcmp (number, "1")) sys_warn (r, record->pos, @@ -1813,10 +1825,9 @@ decode_mrsets (struct sfm_reader *r, struct dictionary *dict) size_t i; name = recode_string ("UTF-8", r->encoding, s->name, -1); - if (name[0] != '$') + if (!mrset_is_valid_name (name, dict_get_encoding (dict), false)) { - sys_warn (r, -1, _("Multiple response set name `%s' does not begin " - "with `$'."), + sys_warn (r, -1, _("Invalid multiple response set name `%s'."), name); free (name); continue; @@ -1978,8 +1989,9 @@ parse_display_parameters (struct sfm_reader *r, } static void -rename_var_and_save_short_names (struct dictionary *dict, struct variable *var, - const char *new_name) +rename_var_and_save_short_names (struct sfm_reader *r, off_t pos, + struct dictionary *dict, + struct variable *var, const char *new_name) { size_t n_short_names; char **short_names; @@ -1997,7 +2009,8 @@ rename_var_and_save_short_names (struct dictionary *dict, struct variable *var, } /* Set long name. */ - dict_rename_var (dict, var, new_name); + if (!dict_try_rename_var (dict, var, new_name)) + sys_warn (r, pos, _("Duplicate long variable name `%s'."), new_name); /* Restore short names. */ for (i = 0; i < n_short_names; i++) @@ -2031,7 +2044,7 @@ parse_long_var_name_map (struct sfm_reader *r, char *new_name; new_name = utf8_to_lower (var_get_name (var)); - rename_var_and_save_short_names (dict, var, new_name); + rename_var_and_save_short_names (r, -1, dict, var, new_name); free (new_name); } @@ -2046,7 +2059,8 @@ parse_long_var_name_map (struct sfm_reader *r, while (read_variable_to_value_pair (r, dict, text, &var, &long_name)) { /* Validate long name. */ - if (!dict_id_is_valid (dict, long_name, false)) + if (!dict_id_is_valid (dict, long_name, false) + || long_name[0] == '$' || long_name[0] == '#') { sys_warn (r, record->pos, _("Long variable mapping from %s to invalid " @@ -2055,16 +2069,7 @@ parse_long_var_name_map (struct sfm_reader *r, continue; } - /* Identify any duplicates. */ - if (utf8_strcasecmp (var_get_short_name (var, 0), long_name) - && dict_lookup_var (dict, long_name) != NULL) - { - sys_warn (r, record->pos, - _("Duplicate long variable name `%s'."), long_name); - continue; - } - - rename_var_and_save_short_names (dict, var, long_name); + rename_var_and_save_short_names (r, record->pos, dict, var, long_name); } close_text_record (r, text); } @@ -2208,19 +2213,26 @@ parse_value_labels (struct sfm_reader *r, struct dictionary *dict, if (width == 0) value.f = parse_float (r, label->value, 0); else - memcpy (value_str_rw (&value, width), label->value, width); + memcpy (value.s, label->value, width); if (!var_add_value_label (var, &value, utf8_labels[j])) { - if (var_is_numeric (var)) + if (r->written_by_readstat) + { + /* Ignore the problem. ReadStat is buggy and emits value + labels whose values are longer than string variables' + widths, that are identical in the actual width of the + variable, e.g. both values "ABC123" and "ABC456" for a + string variable with width 3. */ + } + else if (var_is_numeric (var)) sys_warn (r, record->pos, _("Duplicate value label for %g on %s."), value.f, var_get_name (var)); else sys_warn (r, record->pos, _("Duplicate value label for `%.*s' on %s."), - width, value_str (&value, width), - var_get_name (var)); + width, value.s, var_get_name (var)); } value_destroy (&value, width); @@ -2293,28 +2305,35 @@ parse_attributes (struct sfm_reader *r, struct text_record *text, text_warn (r, text, _("Error parsing attribute value %s[%d]."), key, index); break; - } + } length = strlen (value); - if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'') + if (length >= 2 && value[0] == '\'' && value[length - 1] == '\'') { value[length - 1] = '\0'; - attribute_add_value (attr, value + 1); + attribute_add_value (attr, value + 1); } - else + else { text_warn (r, text, _("Attribute value %s[%d] is not quoted: %s."), key, index, value); - attribute_add_value (attr, value); + attribute_add_value (attr, value); } /* Was this the last value for this attribute? */ if (text_match (text, ')')) break; } - if (attrs != NULL) - attrset_add (attrs, attr); + if (attrs != NULL && attribute_get_n_values (attr) > 0) + { + if (!attrset_try_add (attrs, attr)) + { + text_warn (r, text, _("Duplicate attribute %s."), + attribute_get_name (attr)); + attribute_destroy (attr); + } + } else attribute_destroy (attr); } @@ -2360,7 +2379,7 @@ assign_variable_roles (struct sfm_reader *r, struct dictionary *dict) struct variable *var = dict_get_var (dict, i); struct attrset *attrs = var_get_attributes (var); const struct attribute *attr = attrset_lookup (attrs, "$@Role"); - if (attr != NULL) + if (attr != NULL && attribute_get_n_values (attr) > 0) { int value = atoi (attribute_get_value (attr, 0)); enum var_role role; @@ -2448,7 +2467,8 @@ parse_long_string_value_labels (struct sfm_reader *r, ofs += 4; /* Parse variable name, width, and number of labels. */ - if (!check_overflow (r, record, ofs, var_name_len + 8)) + if (!check_overflow (r, record, ofs, var_name_len) + || !check_overflow (r, record, ofs, var_name_len + 8)) return; var_name = recode_string_pool ("UTF-8", dict_encoding, (const char *) record->data + ofs, @@ -2499,8 +2519,7 @@ parse_long_string_value_labels (struct sfm_reader *r, if (!skip) { if (value_length == width) - memcpy (value_str_rw (&value, width), - (const uint8_t *) record->data + ofs, width); + memcpy (value.s, (const uint8_t *) record->data + ofs, width); else { sys_warn (r, record->pos + ofs, @@ -2532,8 +2551,7 @@ parse_long_string_value_labels (struct sfm_reader *r, if (!var_add_value_label (var, &value, label)) sys_warn (r, record->pos + ofs, _("Duplicate value label for `%.*s' on %s."), - width, value_str (&value, width), - var_get_name (var)); + width, value.s, var_get_name (var)); pool_free (r->pool, label); } ofs += label_length; @@ -2566,7 +2584,8 @@ parse_long_string_missing_values (struct sfm_reader *r, ofs += 4; /* Parse variable name. */ - if (!check_overflow (r, record, ofs, var_name_len + 1)) + if (!check_overflow (r, record, ofs, var_name_len) + || !check_overflow (r, record, ofs, var_name_len + 1)) return; var_name = recode_string_pool ("UTF-8", dict_encoding, (const char *) record->data + ofs, @@ -2666,8 +2685,7 @@ sys_file_casereader_read (struct casereader *reader, void *r_) retval = read_case_number (r, &v->f); else { - uint8_t *s = value_str_rw (v, sv->var_width); - retval = read_case_string (r, s + sv->offset, sv->segment_width); + retval = read_case_string (r, v->s + sv->offset, sv->segment_width); if (retval == 1) { retval = skip_whole_strings (r, ROUND_DOWN (sv->padding, 8)); @@ -2957,7 +2975,7 @@ open_text_record (struct sfm_reader *r, } /* Closes TEXT, frees its storage, and issues a final warning - about suppressed warnings if necesary. */ + about suppressed warnings if necessary. */ static void close_text_record (struct sfm_reader *r, struct text_record *text) { @@ -2980,7 +2998,7 @@ read_variable_to_value_pair (struct sfm_reader *r, struct dictionary *dict, { if (!text_read_short_name (r, dict, text, ss_cstr ("="), var)) return false; - + *value = text_get_token (text, ss_buffer ("\t\0", 2), NULL); if (*value == NULL) return false; @@ -3036,7 +3054,7 @@ static void text_warn (struct sfm_reader *r, struct text_record *text, const char *format, ...) { - if (text->n_warnings++ < MAX_TEXT_WARNINGS) + if (text->n_warnings++ < MAX_TEXT_WARNINGS) { va_list args; @@ -3125,7 +3143,10 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) static bool text_match (struct text_record *text, char c) { - if (text->buffer.string[text->pos] == c) + if (text->pos >= text->buffer.length) + return false; + + if (text->buffer.string[text->pos] == c) { text->pos++; return true; @@ -3155,7 +3176,6 @@ static void sys_msg (struct sfm_reader *r, off_t offset, int class, const char *format, va_list args) { - struct msg m; struct string text; ds_init_empty (&text); @@ -3166,15 +3186,11 @@ sys_msg (struct sfm_reader *r, off_t offset, ds_put_format (&text, _("`%s': "), fh_get_file_name (r->fh)); ds_put_vformat (&text, format, args); - m.category = msg_class_to_category (class); - m.severity = msg_class_to_severity (class); - m.file_name = NULL; - m.first_line = 0; - m.last_line = 0; - m.first_column = 0; - m.last_column = 0; - m.text = ds_cstr (&text); - + struct msg m = { + .category = msg_class_to_category (class), + .severity = msg_class_to_severity (class), + .text = ds_cstr (&text), + }; msg_emit (&m); } @@ -3471,7 +3487,7 @@ read_ztrailer (struct sfm_reader *r, if (fstat (fileno (r->file), &s)) { - sys_error (ME, 0, _("%s: stat failed (%s)."), + sys_error (r, 0, _("%s: stat failed (%s)."), fh_get_file_name (r->fh), strerror (errno)); return false; }