X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fsys-file-reader.c;h=681e6f531b152a61cc8b7a030893fe715a2c063a;hb=96c73a0044390c4de8b163bb0445ed28c3648792;hp=4a7476ae04840dfadf31103bde26e0f3bf1edd7a;hpb=b50e4949e076515328be62bb431067611eb94ad8;p=pspp diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c index 4a7476ae04..681e6f531b 100644 --- a/src/data/sys-file-reader.c +++ b/src/data/sys-file-reader.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-2000, 2006-2007, 2009-2012 Free Software Foundation, Inc. + Copyright (C) 1997-2000, 2006-2007, 2009-2013 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,6 +50,7 @@ #include "libpspp/str.h" #include "libpspp/stringi-set.h" +#include "gl/c-strtod.h" #include "gl/c-ctype.h" #include "gl/inttostr.h" #include "gl/localcharset.h" @@ -71,7 +72,8 @@ enum EXT_DATE = 6, /* DATE. */ EXT_MRSETS = 7, /* Multiple response sets. */ EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */ - /* subtypes 9-10 unknown */ + /* subtype 9 unknown */ + EXT_PRODUCT_INFO = 10, /* Extra product info text. */ EXT_DISPLAY = 11, /* Variable display parameters. */ /* subtype 12 unknown */ EXT_LONG_NAMES = 13, /* Long variable names. */ @@ -82,7 +84,8 @@ enum EXT_VAR_ATTRS = 18, /* Variable attributes. */ EXT_MRSETS2 = 19, /* Multiple response sets (extended). */ EXT_ENCODING = 20, /* Character encoding. */ - EXT_LONG_LABELS = 21 /* Value labels for long strings. */ + EXT_LONG_LABELS = 21, /* Value labels for long strings. */ + EXT_DATAVIEW = 24 /* "Format properties in dataview table". */ }; /* Fields from the top-level header record. */ @@ -199,6 +202,8 @@ static double read_float (struct sfm_reader *); static void read_string (struct sfm_reader *, char *, size_t); static void skip_bytes (struct sfm_reader *, size_t); +static char *fix_line_ends (const char *); + static int parse_int (struct sfm_reader *, const void *data, size_t ofs); static double parse_float (struct sfm_reader *, const void *data, size_t ofs); @@ -244,6 +249,7 @@ static bool text_read_short_name (struct sfm_reader *, struct dictionary *, static const char *text_parse_counted_string (struct sfm_reader *, struct text_record *); static size_t text_pos (const struct text_record *); +static const char *text_get_all (const struct text_record *); static bool close_reader (struct sfm_reader *r); @@ -274,6 +280,9 @@ static void parse_machine_integer_info (struct sfm_reader *, struct sfm_read_info *); static void parse_machine_float_info (struct sfm_reader *, const struct sfm_extension_record *); +static void parse_extra_product_info (struct sfm_reader *, + const struct sfm_extension_record *, + struct sfm_read_info *); static void parse_mrsets (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); @@ -293,6 +302,7 @@ static void parse_data_file_attributes (struct sfm_reader *, static void parse_variable_attributes (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); +static void assign_variable_roles (struct sfm_reader *, struct dictionary *); static void parse_long_string_value_labels (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); @@ -306,6 +316,7 @@ sfm_read_info_destroy (struct sfm_read_info *info) free (info->creation_date); free (info->creation_time); free (info->product); + free (info->product_ext); } } @@ -476,6 +487,9 @@ sfm_open_reader (struct file_handle *fh, const char *volatile encoding, if (extensions[EXT_FLOAT] != NULL) parse_machine_float_info (r, extensions[EXT_FLOAT]); + if (extensions[EXT_PRODUCT_INFO] != NULL) + parse_extra_product_info (r, extensions[EXT_PRODUCT_INFO], info); + if (extensions[EXT_FILE_ATTRS] != NULL) parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict); @@ -522,7 +536,12 @@ sfm_open_reader (struct file_handle *fh, const char *volatile encoding, /* The following records use long names, so they need to follow renaming. */ if (extensions[EXT_VAR_ATTRS] != NULL) - parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict); + { + parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict); + + /* Roles use the $@Role attribute. */ + assign_variable_roles (r, dict); + } if (extensions[EXT_LONG_LABELS] != NULL) parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict); @@ -872,6 +891,7 @@ read_extension_record (struct sfm_reader *r, int subtype) { EXT_INTEGER, 4, 8 }, { EXT_FLOAT, 8, 3 }, { EXT_MRSETS, 1, 0 }, + { EXT_PRODUCT_INFO, 1, 0 }, { EXT_DISPLAY, 4, 0 }, { EXT_LONG_NAMES, 1, 0 }, { EXT_LONG_STRINGS, 1, 0 }, @@ -886,6 +906,7 @@ read_extension_record (struct sfm_reader *r, int subtype) { EXT_VAR_SETS, 0, 0 }, { EXT_DATE, 0, 0 }, { EXT_DATA_ENTRY, 0, 0 }, + { EXT_DATAVIEW, 0, 0 }, }; const struct extension_record_type *type; @@ -950,13 +971,16 @@ parse_header (struct sfm_reader *r, const struct sfm_header_record *header, const char *dict_encoding = dict_get_encoding (dict); struct substring product; struct substring label; + char *fixed_label; /* Convert file label to UTF-8 and put it into DICT. */ label = recode_substring_pool ("UTF-8", dict_encoding, ss_cstr (header->file_label), r->pool); ss_trim (&label, ss_cstr (" ")); label.string[label.length] = '\0'; - dict_set_label (dict, label.string); + fixed_label = fix_line_ends (label.string); + dict_set_label (dict, fixed_label); + free (fixed_label); /* Put creation date and time in UTF-8 into INFO. */ info->creation_date = recode_string ("UTF-8", dict_encoding, @@ -1039,6 +1063,11 @@ parse_variable_records (struct sfm_reader *r, struct dictionary *dict, { double low = parse_float (r, rec->missing, 0); double high = parse_float (r, rec->missing, 8); + + /* Deal with SPSS 21 change in representation. */ + if (low == SYSMIS) + low = LOWEST; + mv_add_range (&mv, low, high); ofs += 16; } @@ -1257,16 +1286,38 @@ parse_machine_float_info (struct sfm_reader *r, double lowest = parse_float (r, record->data, 16); if (sysmis != SYSMIS) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - sysmis, "SYSMIS"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + sysmis, sysmis, "SYSMIS", SYSMIS, SYSMIS); if (highest != HIGHEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - highest, "HIGHEST"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + highest, highest, "HIGHEST", HIGHEST, HIGHEST); + + /* SPSS before version 21 used a unique value just bigger than SYSMIS as + LOWEST. SPSS 21 uses SYSMIS for LOWEST, which is OK because LOWEST only + appears in a context (missing values) where SYSMIS cannot. */ + if (lowest != LOWEST && lowest != SYSMIS) + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a) or %g (%a)."), + lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS); +} + +/* Parses record type 7, subtype 10. */ +static void +parse_extra_product_info (struct sfm_reader *r, + const struct sfm_extension_record *record, + struct sfm_read_info *info) +{ + struct text_record *text; - if (lowest != LOWEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - lowest, "LOWEST"); + text = open_text_record (r, record, true); + info->product_ext = fix_line_ends (text_get_all (text)); + close_text_record (r, text); } /* Parses record type 7, subtype 7 or 19. */ @@ -1437,7 +1488,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, mrset->width = width; value_init (&mrset->counted, width); if (width == 0) - mrset->counted.f = strtod (counted, NULL); + mrset->counted.f = c_strtod (counted, NULL); else value_copy_str_rpad (&mrset->counted, width, (const uint8_t *) counted, ' '); @@ -1497,9 +1548,8 @@ parse_display_parameters (struct sfm_reader *r, align = parse_int (r, record->data, ofs); ofs += 4; - /* SPSS 14 sometimes seems to set string variables' measure - to zero. */ - if (0 == measure && var_is_alpha (v)) + /* SPSS sometimes seems to set variables' measure to zero. */ + if (0 == measure) measure = 1; if (measure < 1 || measure > 3 || align < 0 || align > 2) @@ -1572,7 +1622,8 @@ parse_long_var_name_map (struct sfm_reader *r, if (record == NULL) { - /* Convert variable names to lowercase. */ + /* There are no long variable names. Use the short variable names, + converted to lowercase, as the long variable names. */ size_t i; for (i = 0; i < dict_get_var_cnt (dict); i++) @@ -1580,11 +1631,8 @@ parse_long_var_name_map (struct sfm_reader *r, struct variable *var = dict_get_var (dict, i); char *new_name; - new_name = xstrdup (var_get_name (var)); - str_lowercase (new_name); - + new_name = utf8_to_lower (var_get_name (var)); rename_var_and_save_short_names (dict, var, new_name); - free (new_name); } @@ -1609,7 +1657,7 @@ parse_long_var_name_map (struct sfm_reader *r, } /* Identify any duplicates. */ - if (strcasecmp (var_get_short_name (var, 0), long_name) + if (utf8_strcasecmp (var_get_short_name (var, 0), long_name) && dict_lookup_var (dict, long_name) != NULL) { sys_warn (r, record->pos, @@ -1699,7 +1747,7 @@ parse_value_labels (struct sfm_reader *r, struct dictionary *dict, char **utf8_labels; size_t i; - utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels); + utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels); for (i = 0; i < record->n_labels; i++) utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict), record->labels[i].label, -1, @@ -1882,6 +1930,64 @@ parse_variable_attributes (struct sfm_reader *r, close_text_record (r, text); } +static void +assign_variable_roles (struct sfm_reader *r, struct dictionary *dict) +{ + size_t n_warnings = 0; + size_t i; + + for (i = 0; i < dict_get_var_cnt (dict); i++) + { + struct variable *var = dict_get_var (dict, i); + struct attrset *attrs = var_get_attributes (var); + const struct attribute *attr = attrset_lookup (attrs, "$@Role"); + if (attr != NULL) + { + int value = atoi (attribute_get_value (attr, 0)); + enum var_role role; + + switch (value) + { + case 0: + role = ROLE_INPUT; + break; + + case 1: + role = ROLE_OUTPUT; + break; + + case 2: + role = ROLE_BOTH; + break; + + case 3: + role = ROLE_NONE; + break; + + case 4: + role = ROLE_PARTITION; + break; + + case 5: + role = ROLE_SPLIT; + break; + + default: + role = ROLE_INPUT; + if (n_warnings++ == 0) + sys_warn (r, -1, _("Invalid role for variable %s."), + var_get_name (var)); + } + + var_set_role (var, role); + } + } + + if (n_warnings > 1) + sys_warn (r, -1, _("%zu other variables had invalid roles."), + n_warnings - 1); +} + static void check_overflow (struct sfm_reader *r, const struct sfm_extension_record *record, @@ -2511,6 +2617,12 @@ text_pos (const struct text_record *text) { return text->pos; } + +static const char * +text_get_all (const struct text_record *text) +{ + return text->buffer.string; +} /* Messages. */ @@ -2663,6 +2775,35 @@ skip_bytes (struct sfm_reader *r, size_t bytes) bytes -= chunk; } } + +/* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have + been replaced by LFs. + + (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system + files that use CR-only line ends in the file label and extra product + info.) */ +static char * +fix_line_ends (const char *s) +{ + char *dst, *d; + + d = dst = xmalloc (strlen (s) + 1); + while (*s != '\0') + { + if (*s == '\r') + { + s++; + if (*s == '\n') + s++; + *d++ = '\n'; + } + else + *d++ = *s++; + } + *d = '\0'; + + return dst; +} static const struct casereader_class sys_file_casereader_class = {