X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;ds=sidebyside;f=src%2Fdata%2Fsys-file-reader.c;h=b3315b1e17381bcd1cbbed241ba6acf4d53ab633;hb=693ac90cdac91d29870f303b88763a3685b3f341;hp=336549b7614248bee9ebaf7077dbcd1b0f2add1d;hpb=261869b71aa60b8974c4a6b98e35b74af5d11de5;p=pspp diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c index 336549b761..b3315b1e17 100644 --- a/src/data/sys-file-reader.c +++ b/src/data/sys-file-reader.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-2000, 2006-2007, 2009-2012 Free Software Foundation, Inc. + Copyright (C) 1997-2000, 2006-2007, 2009-2013 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,6 +50,7 @@ #include "libpspp/str.h" #include "libpspp/stringi-set.h" +#include "gl/c-strtod.h" #include "gl/c-ctype.h" #include "gl/inttostr.h" #include "gl/localcharset.h" @@ -82,12 +83,14 @@ enum EXT_VAR_ATTRS = 18, /* Variable attributes. */ EXT_MRSETS2 = 19, /* Multiple response sets (extended). */ EXT_ENCODING = 20, /* Character encoding. */ - EXT_LONG_LABELS = 21 /* Value labels for long strings. */ + EXT_LONG_LABELS = 21, /* Value labels for long strings. */ + EXT_DATAVIEW = 24 /* "Format properties in dataview table". */ }; /* Fields from the top-level header record. */ struct sfm_header_record { + char magic[5]; /* First 4 bytes of file, then null. */ int weight_idx; /* 0 if unweighted, otherwise a var index. */ int nominal_case_size; /* Number of var positions. */ @@ -213,6 +216,7 @@ static void skip_extension_record (struct sfm_reader *, int subtype); static const char *choose_encoding ( struct sfm_reader *, + const struct sfm_header_record *, const struct sfm_extension_record *ext_integer, const struct sfm_extension_record *ext_encoding); @@ -310,15 +314,20 @@ sfm_read_info_destroy (struct sfm_read_info *info) /* Opens the system file designated by file handle FH for reading. Reads the system file's dictionary into *DICT. + Ordinarily the reader attempts to automatically detect the character + encoding based on the file's contents. This isn't always possible, + especially for files written by old versions of SPSS or PSPP, so specifying + a nonnull ENCODING overrides the choice of character encoding. + If INFO is non-null, then it receives additional info about the system file, which the caller must eventually free with sfm_read_info_destroy() when it is no longer needed. */ struct casereader * -sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, - struct sfm_read_info *infop) +sfm_open_reader (struct file_handle *fh, const char *volatile encoding, + struct dictionary **dictp, struct sfm_read_info *infop) { struct sfm_reader *volatile r = NULL; - struct sfm_read_info info; + struct sfm_read_info *volatile info; struct sfm_header_record header; @@ -332,7 +341,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, struct sfm_extension_record *extensions[32]; - struct dictionary *dict = NULL; + struct dictionary *volatile dict = NULL; size_t i; /* Create and initialize reader. */ @@ -345,7 +354,8 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, r->opcode_idx = sizeof r->opcodes; r->corruption_warning = false; - memset (&info, 0, sizeof info); + info = infop ? infop : xmalloc (sizeof *info); + memset (info, 0, sizeof *info); /* TRANSLATORS: this fragment will be interpolated into messages in fh_lock() that identify types of files. */ @@ -365,7 +375,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, goto error; /* Read header. */ - read_header (r, &info, &header); + read_header (r, info, &header); vars = NULL; n_vars = allocated_vars = 0; @@ -452,8 +462,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, First, figure out the correct character encoding, because this determines how the rest of the header data is to be interpreted. */ - dict = dict_create (choose_encoding (r, extensions[EXT_INTEGER], - extensions[EXT_ENCODING])); + dict = dict_create (encoding + ? encoding + : choose_encoding (r, &header, extensions[EXT_INTEGER], + extensions[EXT_ENCODING])); r->encoding = dict_get_encoding (dict); /* These records don't use variables at all. */ @@ -461,7 +473,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, parse_document (dict, document); if (extensions[EXT_INTEGER] != NULL) - parse_machine_integer_info (r, extensions[EXT_INTEGER], &info); + parse_machine_integer_info (r, extensions[EXT_INTEGER], info); if (extensions[EXT_FLOAT] != NULL) parse_machine_float_info (r, extensions[EXT_FLOAT]); @@ -469,7 +481,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, if (extensions[EXT_FILE_ATTRS] != NULL) parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict); - parse_header (r, &header, &info, dict); + parse_header (r, &header, info, dict); /* Parse the variable records, the basis of almost everything else. */ parse_variable_records (r, dict, vars, n_vars); @@ -522,7 +534,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, wrong when very long strings are involved, so don't warn in that case. */ if (header.nominal_case_size != -1 && header.nominal_case_size != n_vars - && info.version_major != 13) + && info->version_major != 13) sys_warn (r, -1, _("File header claims %d variable positions but " "%zu were read from file."), header.nominal_case_size, n_vars); @@ -536,10 +548,11 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool); *dictp = dict; - if (infop) - *infop = info; - else - sfm_read_info_destroy (&info); + if (infop != info) + { + sfm_read_info_destroy (info); + free (info); + } return casereader_create_sequential (NULL, r->proto, @@ -547,7 +560,12 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, &sys_file_casereader_class, r); error: - sfm_read_info_destroy (&info); + if (infop != info) + { + sfm_read_info_destroy (info); + free (info); + } + close_reader (r); dict_destroy (dict); *dictp = NULL; @@ -598,13 +616,13 @@ sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_) bool sfm_detect (FILE *file) { - char rec_type[5]; + char magic[5]; - if (fread (rec_type, 4, 1, file) != 1) + if (fread (magic, 4, 1, file) != 1) return false; - rec_type[4] = '\0'; + magic[4] = '\0'; - return !strcmp ("$FL2", rec_type); + return !strcmp (ASCII_MAGIC, magic) || !strcmp (EBCDIC_MAGIC, magic); } /* Reads the global header of the system file. Initializes *HEADER and *INFO, @@ -614,14 +632,14 @@ static void read_header (struct sfm_reader *r, struct sfm_read_info *info, struct sfm_header_record *header) { - char rec_type[5]; uint8_t raw_layout_code[4]; uint8_t raw_bias[8]; - read_string (r, rec_type, sizeof rec_type); + read_string (r, header->magic, sizeof header->magic); read_string (r, header->eye_catcher, sizeof header->eye_catcher); - if (strcmp ("$FL2", rec_type) != 0) + if (strcmp (ASCII_MAGIC, header->magic) + && strcmp (EBCDIC_MAGIC, header->magic)) sys_error (r, 0, _("This is not an SPSS system file.")); /* Identify integer format. */ @@ -870,6 +888,7 @@ read_extension_record (struct sfm_reader *r, int subtype) { EXT_VAR_SETS, 0, 0 }, { EXT_DATE, 0, 0 }, { EXT_DATA_ENTRY, 0, 0 }, + { EXT_DATAVIEW, 0, 0 }, }; const struct extension_record_type *type; @@ -1023,6 +1042,11 @@ parse_variable_records (struct sfm_reader *r, struct dictionary *dict, { double low = parse_float (r, rec->missing, 0); double high = parse_float (r, rec->missing, 8); + + /* Deal with SPSS 21 change in representation. */ + if (low == SYSMIS) + low = LOWEST; + mv_add_range (&mv, low, high); ofs += 16; } @@ -1185,6 +1209,7 @@ parse_machine_integer_info (struct sfm_reader *r, static const char * choose_encoding (struct sfm_reader *r, + const struct sfm_header_record *header, const struct sfm_extension_record *ext_integer, const struct sfm_extension_record *ext_encoding) { @@ -1223,6 +1248,10 @@ choose_encoding (struct sfm_reader *r, } } + /* If the file magic number is EBCDIC then its character data is too. */ + if (!strcmp (header->magic, EBCDIC_MAGIC)) + return "EBCDIC-US"; + return locale_charset (); } @@ -1236,16 +1265,25 @@ parse_machine_float_info (struct sfm_reader *r, double lowest = parse_float (r, record->data, 16); if (sysmis != SYSMIS) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - sysmis, "SYSMIS"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + sysmis, sysmis, "SYSMIS", SYSMIS, SYSMIS); if (highest != HIGHEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - highest, "HIGHEST"); - - if (lowest != LOWEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - lowest, "LOWEST"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + highest, highest, "HIGHEST", HIGHEST, HIGHEST); + + /* SPSS before version 21 used a unique value just bigger than SYSMIS as + LOWEST. SPSS 21 uses SYSMIS for LOWEST, which is OK because LOWEST only + appears in a context (missing values) where SYSMIS cannot. */ + if (lowest != LOWEST && lowest != SYSMIS) + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a) or %g (%a)."), + lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS); } /* Parses record type 7, subtype 7 or 19. */ @@ -1407,6 +1445,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, _("MRSET %s has only %zu variables."), mrset->name, mrset->n_vars); mrset_destroy (mrset); + stringi_set_destroy (&var_names); continue; } @@ -1415,7 +1454,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, mrset->width = width; value_init (&mrset->counted, width); if (width == 0) - mrset->counted.f = strtod (counted, NULL); + mrset->counted.f = c_strtod (counted, NULL); else value_copy_str_rpad (&mrset->counted, width, (const uint8_t *) counted, ' '); @@ -1475,9 +1514,8 @@ parse_display_parameters (struct sfm_reader *r, align = parse_int (r, record->data, ofs); ofs += 4; - /* SPSS 14 sometimes seems to set string variables' measure - to zero. */ - if (0 == measure && var_is_alpha (v)) + /* SPSS sometimes seems to set variables' measure to zero. */ + if (0 == measure) measure = 1; if (measure < 1 || measure > 3 || align < 0 || align > 2) @@ -1550,7 +1588,8 @@ parse_long_var_name_map (struct sfm_reader *r, if (record == NULL) { - /* Convert variable names to lowercase. */ + /* There are no long variable names. Use the short variable names, + converted to lowercase, as the long variable names. */ size_t i; for (i = 0; i < dict_get_var_cnt (dict); i++) @@ -1558,11 +1597,8 @@ parse_long_var_name_map (struct sfm_reader *r, struct variable *var = dict_get_var (dict, i); char *new_name; - new_name = xstrdup (var_get_name (var)); - str_lowercase (new_name); - + new_name = utf8_to_lower (var_get_name (var)); rename_var_and_save_short_names (dict, var, new_name); - free (new_name); } @@ -1577,7 +1613,6 @@ parse_long_var_name_map (struct sfm_reader *r, while (read_variable_to_value_pair (r, dict, text, &var, &long_name)) { /* Validate long name. */ - /* XXX need to reencode name to UTF-8 */ if (!dict_id_is_valid (dict, long_name, false)) { sys_warn (r, record->pos, @@ -1588,7 +1623,7 @@ parse_long_var_name_map (struct sfm_reader *r, } /* Identify any duplicates. */ - if (strcasecmp (var_get_short_name (var, 0), long_name) + if (utf8_strcasecmp (var_get_short_name (var, 0), long_name) && dict_lookup_var (dict, long_name) != NULL) { sys_warn (r, record->pos, @@ -2425,7 +2460,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) start = text->pos; n = 0; - for (;;) + while (text->pos < text->buffer.length) { int c = text->buffer.string[text->pos]; if (c < '0' || c > '9') @@ -2433,7 +2468,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) n = (n * 10) + (c - '0'); text->pos++; } - if (start == text->pos) + if (text->pos >= text->buffer.length || start == text->pos) { sys_warn (r, text->start, _("Expecting digit at offset %zu in MRSETS record."),