X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fsys-file-reader.c;h=681e6f531b152a61cc8b7a030893fe715a2c063a;hb=96c73a0044390c4de8b163bb0445ed28c3648792;hp=9ec176f09f494f9562548918db80b61b31fa36f1;hpb=fe8dc2171009e90d2335f159d05f7e6660e24780;p=pspp diff --git a/src/data/sys-file-reader.c b/src/data/sys-file-reader.c index 9ec176f09f..681e6f531b 100644 --- a/src/data/sys-file-reader.c +++ b/src/data/sys-file-reader.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-2000, 2006-2007, 2009-2011 Free Software Foundation, Inc. + Copyright (C) 1997-2000, 2006-2007, 2009-2013 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,6 +50,7 @@ #include "libpspp/str.h" #include "libpspp/stringi-set.h" +#include "gl/c-strtod.h" #include "gl/c-ctype.h" #include "gl/inttostr.h" #include "gl/localcharset.h" @@ -71,7 +72,8 @@ enum EXT_DATE = 6, /* DATE. */ EXT_MRSETS = 7, /* Multiple response sets. */ EXT_DATA_ENTRY = 8, /* SPSS Data Entry. */ - /* subtypes 9-10 unknown */ + /* subtype 9 unknown */ + EXT_PRODUCT_INFO = 10, /* Extra product info text. */ EXT_DISPLAY = 11, /* Variable display parameters. */ /* subtype 12 unknown */ EXT_LONG_NAMES = 13, /* Long variable names. */ @@ -82,7 +84,23 @@ enum EXT_VAR_ATTRS = 18, /* Variable attributes. */ EXT_MRSETS2 = 19, /* Multiple response sets (extended). */ EXT_ENCODING = 20, /* Character encoding. */ - EXT_LONG_LABELS = 21 /* Value labels for long strings. */ + EXT_LONG_LABELS = 21, /* Value labels for long strings. */ + EXT_DATAVIEW = 24 /* "Format properties in dataview table". */ + }; + +/* Fields from the top-level header record. */ +struct sfm_header_record + { + char magic[5]; /* First 4 bytes of file, then null. */ + int weight_idx; /* 0 if unweighted, otherwise a var index. */ + int nominal_case_size; /* Number of var positions. */ + + /* These correspond to the members of struct sfm_file_info or a dictionary + but in the system file's encoding rather than ASCII. */ + char creation_date[10]; /* "dd mmm yy". */ + char creation_time[9]; /* "hh:mm:ss". */ + char eye_catcher[61]; /* Eye-catcher string, then product name. */ + char file_label[65]; /* File label. */ }; struct sfm_var_record @@ -184,6 +202,8 @@ static double read_float (struct sfm_reader *); static void read_string (struct sfm_reader *, char *, size_t); static void skip_bytes (struct sfm_reader *, size_t); +static char *fix_line_ends (const char *); + static int parse_int (struct sfm_reader *, const void *data, size_t ofs); static double parse_float (struct sfm_reader *, const void *data, size_t ofs); @@ -199,6 +219,7 @@ static void skip_extension_record (struct sfm_reader *, int subtype); static const char *choose_encoding ( struct sfm_reader *, + const struct sfm_header_record *, const struct sfm_extension_record *ext_integer, const struct sfm_extension_record *ext_encoding); @@ -228,6 +249,7 @@ static bool text_read_short_name (struct sfm_reader *, struct dictionary *, static const char *text_parse_counted_string (struct sfm_reader *, struct text_record *); static size_t text_pos (const struct text_record *); +static const char *text_get_all (const struct text_record *); static bool close_reader (struct sfm_reader *r); @@ -239,11 +261,11 @@ enum which_format WRITE_FORMAT }; -static void read_header (struct sfm_reader *, int *weight_idx, - int *claimed_oct_cnt, struct sfm_read_info *, - char **file_labelp); -static void parse_file_label (struct sfm_reader *, const char *file_label, - struct dictionary *); +static void read_header (struct sfm_reader *, struct sfm_read_info *, + struct sfm_header_record *); +static void parse_header (struct sfm_reader *, + const struct sfm_header_record *, + struct sfm_read_info *, struct dictionary *); static void parse_variable_records (struct sfm_reader *, struct dictionary *, struct sfm_var_record *, size_t n); static void parse_format_spec (struct sfm_reader *, off_t pos, @@ -258,6 +280,9 @@ static void parse_machine_integer_info (struct sfm_reader *, struct sfm_read_info *); static void parse_machine_float_info (struct sfm_reader *, const struct sfm_extension_record *); +static void parse_extra_product_info (struct sfm_reader *, + const struct sfm_extension_record *, + struct sfm_read_info *); static void parse_mrsets (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); @@ -277,20 +302,43 @@ static void parse_data_file_attributes (struct sfm_reader *, static void parse_variable_attributes (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); +static void assign_variable_roles (struct sfm_reader *, struct dictionary *); static void parse_long_string_value_labels (struct sfm_reader *, const struct sfm_extension_record *, struct dictionary *); -/* Opens the system file designated by file handle FH for - reading. Reads the system file's dictionary into *DICT. - If INFO is non-null, then it receives additional info about the - system file. */ +/* Frees the strings inside INFO. */ +void +sfm_read_info_destroy (struct sfm_read_info *info) +{ + if (info) + { + free (info->creation_date); + free (info->creation_time); + free (info->product); + free (info->product_ext); + } +} + +/* Opens the system file designated by file handle FH for reading. Reads the + system file's dictionary into *DICT. + + Ordinarily the reader attempts to automatically detect the character + encoding based on the file's contents. This isn't always possible, + especially for files written by old versions of SPSS or PSPP, so specifying + a nonnull ENCODING overrides the choice of character encoding. + + If INFO is non-null, then it receives additional info about the system file, + which the caller must eventually free with sfm_read_info_destroy() when it + is no longer needed. */ struct casereader * -sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, - struct sfm_read_info *volatile info) +sfm_open_reader (struct file_handle *fh, const char *volatile encoding, + struct dictionary **dictp, struct sfm_read_info *infop) { struct sfm_reader *volatile r = NULL; - struct sfm_read_info local_info; + struct sfm_read_info *volatile info; + + struct sfm_header_record header; struct sfm_var_record *vars; size_t n_vars, allocated_vars; @@ -302,11 +350,7 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, struct sfm_extension_record *extensions[32]; - int weight_idx; - int claimed_oct_cnt; - char *file_label; - - struct dictionary *dict = NULL; + struct dictionary *volatile dict = NULL; size_t i; /* Create and initialize reader. */ @@ -319,6 +363,9 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, r->opcode_idx = sizeof r->opcodes; r->corruption_warning = false; + info = infop ? infop : xmalloc (sizeof *info); + memset (info, 0, sizeof *info); + /* TRANSLATORS: this fragment will be interpolated into messages in fh_lock() that identify types of files. */ r->lock = fh_lock (fh, FH_REF_FILE, N_("system file"), FH_ACC_READ, false); @@ -333,16 +380,11 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, goto error; } - /* Initialize info. */ - if (info == NULL) - info = &local_info; - memset (info, 0, sizeof *info); - if (setjmp (r->bail_out)) goto error; /* Read header. */ - read_header (r, &weight_idx, &claimed_oct_cnt, info, &file_label); + read_header (r, info, &header); vars = NULL; n_vars = allocated_vars = 0; @@ -429,8 +471,10 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, First, figure out the correct character encoding, because this determines how the rest of the header data is to be interpreted. */ - dict = dict_create (choose_encoding (r, extensions[EXT_INTEGER], - extensions[EXT_ENCODING])); + dict = dict_create (encoding + ? encoding + : choose_encoding (r, &header, extensions[EXT_INTEGER], + extensions[EXT_ENCODING])); r->encoding = dict_get_encoding (dict); /* These records don't use variables at all. */ @@ -443,10 +487,13 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, if (extensions[EXT_FLOAT] != NULL) parse_machine_float_info (r, extensions[EXT_FLOAT]); + if (extensions[EXT_PRODUCT_INFO] != NULL) + parse_extra_product_info (r, extensions[EXT_PRODUCT_INFO], info); + if (extensions[EXT_FILE_ATTRS] != NULL) parse_data_file_attributes (r, extensions[EXT_FILE_ATTRS], dict); - parse_file_label (r, file_label, dict); + parse_header (r, &header, info, dict); /* Parse the variable records, the basis of almost everything else. */ parse_variable_records (r, dict, vars, n_vars); @@ -456,11 +503,12 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, before those indexes become invalidated by very long string variables. */ for (i = 0; i < n_labels; i++) parse_value_labels (r, dict, vars, n_vars, &labels[i]); - if (weight_idx != 0) + if (header.weight_idx != 0) { struct variable *weight_var; - weight_var = lookup_var_by_index (r, 76, vars, n_vars, weight_idx); + weight_var = lookup_var_by_index (r, 76, vars, n_vars, + header.weight_idx); if (var_is_numeric (weight_var)) dict_set_weight (dict, weight_var); else @@ -488,7 +536,12 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, /* The following records use long names, so they need to follow renaming. */ if (extensions[EXT_VAR_ATTRS] != NULL) - parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict); + { + parse_variable_attributes (r, extensions[EXT_VAR_ATTRS], dict); + + /* Roles use the $@Role attribute. */ + assign_variable_roles (r, dict); + } if (extensions[EXT_LONG_LABELS] != NULL) parse_long_string_value_labels (r, extensions[EXT_LONG_LABELS], dict); @@ -497,11 +550,11 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, amount that the header claims. SPSS version 13 gets this wrong when very long strings are involved, so don't warn in that case. */ - if (claimed_oct_cnt != -1 && claimed_oct_cnt != n_vars + if (header.nominal_case_size != -1 && header.nominal_case_size != n_vars && info->version_major != 13) sys_warn (r, -1, _("File header claims %d variable positions but " "%zu were read from file."), - claimed_oct_cnt, n_vars); + header.nominal_case_size, n_vars); /* Create an index of dictionary variable widths for sfm_read_case to use. We cannot use the `struct variable's @@ -512,12 +565,24 @@ sfm_open_reader (struct file_handle *fh, struct dictionary **dictp, r->proto = caseproto_ref_pool (dict_get_proto (dict), r->pool); *dictp = dict; + if (infop != info) + { + sfm_read_info_destroy (info); + free (info); + } + return casereader_create_sequential (NULL, r->proto, r->case_cnt == -1 ? CASENUMBER_MAX: r->case_cnt, &sys_file_casereader_class, r); error: + if (infop != info) + { + sfm_read_info_destroy (info); + free (info); + } + close_reader (r); dict_destroy (dict); *dictp = NULL; @@ -568,39 +633,30 @@ sys_file_casereader_destroy (struct casereader *reader UNUSED, void *r_) bool sfm_detect (FILE *file) { - char rec_type[5]; + char magic[5]; - if (fread (rec_type, 4, 1, file) != 1) + if (fread (magic, 4, 1, file) != 1) return false; - rec_type[4] = '\0'; + magic[4] = '\0'; - return !strcmp ("$FL2", rec_type); + return !strcmp (ASCII_MAGIC, magic) || !strcmp (EBCDIC_MAGIC, magic); } -/* Reads the global header of the system file. Sets *WEIGHT_IDX to 0 if the - system file is unweighted, or to the value index of the weight variable - otherwise. Sets *CLAIMED_OCT_CNT to the number of "octs" (8-byte units) per - case that the file claims to have (although it is not always correct). - Initializes INFO with header information. Stores the file label as a string - in dictionary encoding into *FILE_LABELP. */ +/* Reads the global header of the system file. Initializes *HEADER and *INFO, + except for the string fields in *INFO, which parse_header() will initialize + later once the file's encoding is known. */ static void -read_header (struct sfm_reader *r, int *weight_idx, - int *claimed_oct_cnt, struct sfm_read_info *info, - char **file_labelp) +read_header (struct sfm_reader *r, struct sfm_read_info *info, + struct sfm_header_record *header) { - char rec_type[5]; - char eye_catcher[61]; uint8_t raw_layout_code[4]; uint8_t raw_bias[8]; - char creation_date[10]; - char creation_time[9]; - char file_label[65]; - struct substring product; - read_string (r, rec_type, sizeof rec_type); - read_string (r, eye_catcher, sizeof eye_catcher); + read_string (r, header->magic, sizeof header->magic); + read_string (r, header->eye_catcher, sizeof header->eye_catcher); - if (strcmp ("$FL2", rec_type) != 0) + if (strcmp (ASCII_MAGIC, header->magic) + && strcmp (EBCDIC_MAGIC, header->magic)) sys_error (r, 0, _("This is not an SPSS system file.")); /* Identify integer format. */ @@ -613,13 +669,14 @@ read_header (struct sfm_reader *r, int *weight_idx, && r->integer_format != INTEGER_LSB_FIRST)) sys_error (r, 64, _("This is not an SPSS system file.")); - *claimed_oct_cnt = read_int (r); - if (*claimed_oct_cnt < 0 || *claimed_oct_cnt > INT_MAX / 16) - *claimed_oct_cnt = -1; + header->nominal_case_size = read_int (r); + if (header->nominal_case_size < 0 + || header->nominal_case_size > INT_MAX / 16) + header->nominal_case_size = -1; r->compressed = read_int (r) != 0; - *weight_idx = read_int (r); + header->weight_idx = read_int (r); r->case_cnt = read_int (r); if ( r->case_cnt > INT_MAX / 2) @@ -652,25 +709,15 @@ read_header (struct sfm_reader *r, int *weight_idx, } float_convert (r->float_format, raw_bias, FLOAT_NATIVE_DOUBLE, &r->bias); - read_string (r, creation_date, sizeof creation_date); - read_string (r, creation_time, sizeof creation_time); - read_string (r, file_label, sizeof file_label); + read_string (r, header->creation_date, sizeof header->creation_date); + read_string (r, header->creation_time, sizeof header->creation_time); + read_string (r, header->file_label, sizeof header->file_label); skip_bytes (r, 3); - strcpy (info->creation_date, creation_date); - strcpy (info->creation_time, creation_time); info->integer_format = r->integer_format; info->float_format = r->float_format; info->compressed = r->compressed; info->case_cnt = r->case_cnt; - - product = ss_cstr (eye_catcher); - ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE")); - ss_trim (&product, ss_cstr (" ")); - str_copy_buf_trunc (info->product, sizeof info->product, - ss_data (product), ss_length (product)); - - *file_labelp = pool_strdup0 (r->pool, file_label, sizeof file_label - 1); } /* Reads a variable (type 2) record from R into RECORD. */ @@ -698,7 +745,7 @@ read_variable_record (struct sfm_reader *r, struct sfm_var_record *record) /* Read up to MAX_LABEL_LEN bytes of label. */ read_len = MIN (MAX_LABEL_LEN, len); - record->label = xmalloc (read_len + 1); + record->label = pool_malloc (r->pool, read_len + 1); read_string (r, record->label, read_len + 1); /* Skip unread label bytes. */ @@ -844,6 +891,7 @@ read_extension_record (struct sfm_reader *r, int subtype) { EXT_INTEGER, 4, 8 }, { EXT_FLOAT, 8, 3 }, { EXT_MRSETS, 1, 0 }, + { EXT_PRODUCT_INFO, 1, 0 }, { EXT_DISPLAY, 4, 0 }, { EXT_LONG_NAMES, 1, 0 }, { EXT_LONG_STRINGS, 1, 0 }, @@ -858,6 +906,7 @@ read_extension_record (struct sfm_reader *r, int subtype) { EXT_VAR_SETS, 0, 0 }, { EXT_DATE, 0, 0 }, { EXT_DATA_ENTRY, 0, 0 }, + { EXT_DATAVIEW, 0, 0 }, }; const struct extension_record_type *type; @@ -916,19 +965,35 @@ skip_extension_record (struct sfm_reader *r, int subtype) } static void -parse_file_label (struct sfm_reader *r, const char *file_label, - struct dictionary *dict) +parse_header (struct sfm_reader *r, const struct sfm_header_record *header, + struct sfm_read_info *info, struct dictionary *dict) { - char *utf8_file_label; - size_t file_label_len; - - utf8_file_label = recode_string_pool ("UTF-8", dict_get_encoding (dict), - file_label, -1, r->pool); - file_label_len = strlen (utf8_file_label); - while (file_label_len > 0 && utf8_file_label[file_label_len - 1] == ' ') - file_label_len--; - utf8_file_label[file_label_len] = '\0'; - dict_set_label (dict, utf8_file_label); + const char *dict_encoding = dict_get_encoding (dict); + struct substring product; + struct substring label; + char *fixed_label; + + /* Convert file label to UTF-8 and put it into DICT. */ + label = recode_substring_pool ("UTF-8", dict_encoding, + ss_cstr (header->file_label), r->pool); + ss_trim (&label, ss_cstr (" ")); + label.string[label.length] = '\0'; + fixed_label = fix_line_ends (label.string); + dict_set_label (dict, fixed_label); + free (fixed_label); + + /* Put creation date and time in UTF-8 into INFO. */ + info->creation_date = recode_string ("UTF-8", dict_encoding, + header->creation_date, -1); + info->creation_time = recode_string ("UTF-8", dict_encoding, + header->creation_time, -1); + + /* Put product name into INFO, dropping eye-catcher string if present. */ + product = recode_substring_pool ("UTF-8", dict_encoding, + ss_cstr (header->eye_catcher), r->pool); + ss_match_string (&product, ss_cstr ("@(#) SPSS DATA FILE")); + ss_trim (&product, ss_cstr (" ")); + info->product = ss_xstrdup (product); } /* Reads a variable (type 2) record from R and adds the @@ -998,6 +1063,11 @@ parse_variable_records (struct sfm_reader *r, struct dictionary *dict, { double low = parse_float (r, rec->missing, 0); double high = parse_float (r, rec->missing, 8); + + /* Deal with SPSS 21 change in representation. */ + if (low == SYSMIS) + low = LOWEST; + mv_add_range (&mv, low, high); ofs += 16; } @@ -1160,6 +1230,7 @@ parse_machine_integer_info (struct sfm_reader *r, static const char * choose_encoding (struct sfm_reader *r, + const struct sfm_header_record *header, const struct sfm_extension_record *ext_integer, const struct sfm_extension_record *ext_encoding) { @@ -1198,6 +1269,10 @@ choose_encoding (struct sfm_reader *r, } } + /* If the file magic number is EBCDIC then its character data is too. */ + if (!strcmp (header->magic, EBCDIC_MAGIC)) + return "EBCDIC-US"; + return locale_charset (); } @@ -1211,16 +1286,38 @@ parse_machine_float_info (struct sfm_reader *r, double lowest = parse_float (r, record->data, 16); if (sysmis != SYSMIS) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - sysmis, "SYSMIS"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + sysmis, sysmis, "SYSMIS", SYSMIS, SYSMIS); if (highest != HIGHEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - highest, "HIGHEST"); + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a)."), + highest, highest, "HIGHEST", HIGHEST, HIGHEST); + + /* SPSS before version 21 used a unique value just bigger than SYSMIS as + LOWEST. SPSS 21 uses SYSMIS for LOWEST, which is OK because LOWEST only + appears in a context (missing values) where SYSMIS cannot. */ + if (lowest != LOWEST && lowest != SYSMIS) + sys_warn (r, record->pos, + _("File specifies unexpected value %g (%a) as %s, " + "instead of %g (%a) or %g (%a)."), + lowest, lowest, "LOWEST", LOWEST, LOWEST, SYSMIS, SYSMIS); +} + +/* Parses record type 7, subtype 10. */ +static void +parse_extra_product_info (struct sfm_reader *r, + const struct sfm_extension_record *record, + struct sfm_read_info *info) +{ + struct text_record *text; - if (lowest != LOWEST) - sys_warn (r, record->pos, _("File specifies unexpected value %g as %s."), - lowest, "LOWEST"); + text = open_text_record (r, record, true); + info->product_ext = fix_line_ends (text_get_all (text)); + close_text_record (r, text); } /* Parses record type 7, subtype 7 or 19. */ @@ -1382,6 +1479,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, _("MRSET %s has only %zu variables."), mrset->name, mrset->n_vars); mrset_destroy (mrset); + stringi_set_destroy (&var_names); continue; } @@ -1390,7 +1488,7 @@ parse_mrsets (struct sfm_reader *r, const struct sfm_extension_record *record, mrset->width = width; value_init (&mrset->counted, width); if (width == 0) - mrset->counted.f = strtod (counted, NULL); + mrset->counted.f = c_strtod (counted, NULL); else value_copy_str_rpad (&mrset->counted, width, (const uint8_t *) counted, ' '); @@ -1450,9 +1548,8 @@ parse_display_parameters (struct sfm_reader *r, align = parse_int (r, record->data, ofs); ofs += 4; - /* SPSS 14 sometimes seems to set string variables' measure - to zero. */ - if (0 == measure && var_is_alpha (v)) + /* SPSS sometimes seems to set variables' measure to zero. */ + if (0 == measure) measure = 1; if (measure < 1 || measure > 3 || align < 0 || align > 2) @@ -1525,7 +1622,8 @@ parse_long_var_name_map (struct sfm_reader *r, if (record == NULL) { - /* Convert variable names to lowercase. */ + /* There are no long variable names. Use the short variable names, + converted to lowercase, as the long variable names. */ size_t i; for (i = 0; i < dict_get_var_cnt (dict); i++) @@ -1533,11 +1631,8 @@ parse_long_var_name_map (struct sfm_reader *r, struct variable *var = dict_get_var (dict, i); char *new_name; - new_name = xstrdup (var_get_name (var)); - str_lowercase (new_name); - + new_name = utf8_to_lower (var_get_name (var)); rename_var_and_save_short_names (dict, var, new_name); - free (new_name); } @@ -1552,7 +1647,6 @@ parse_long_var_name_map (struct sfm_reader *r, while (read_variable_to_value_pair (r, dict, text, &var, &long_name)) { /* Validate long name. */ - /* XXX need to reencode name to UTF-8 */ if (!dict_id_is_valid (dict, long_name, false)) { sys_warn (r, record->pos, @@ -1563,7 +1657,7 @@ parse_long_var_name_map (struct sfm_reader *r, } /* Identify any duplicates. */ - if (strcasecmp (var_get_short_name (var, 0), long_name) + if (utf8_strcasecmp (var_get_short_name (var, 0), long_name) && dict_lookup_var (dict, long_name) != NULL) { sys_warn (r, record->pos, @@ -1653,7 +1747,7 @@ parse_value_labels (struct sfm_reader *r, struct dictionary *dict, char **utf8_labels; size_t i; - utf8_labels = pool_nmalloc (r->pool, sizeof *utf8_labels, record->n_labels); + utf8_labels = pool_nmalloc (r->pool, record->n_labels, sizeof *utf8_labels); for (i = 0; i < record->n_labels; i++) utf8_labels[i] = recode_string_pool ("UTF-8", dict_get_encoding (dict), record->labels[i].label, -1, @@ -1836,6 +1930,64 @@ parse_variable_attributes (struct sfm_reader *r, close_text_record (r, text); } +static void +assign_variable_roles (struct sfm_reader *r, struct dictionary *dict) +{ + size_t n_warnings = 0; + size_t i; + + for (i = 0; i < dict_get_var_cnt (dict); i++) + { + struct variable *var = dict_get_var (dict, i); + struct attrset *attrs = var_get_attributes (var); + const struct attribute *attr = attrset_lookup (attrs, "$@Role"); + if (attr != NULL) + { + int value = atoi (attribute_get_value (attr, 0)); + enum var_role role; + + switch (value) + { + case 0: + role = ROLE_INPUT; + break; + + case 1: + role = ROLE_OUTPUT; + break; + + case 2: + role = ROLE_BOTH; + break; + + case 3: + role = ROLE_NONE; + break; + + case 4: + role = ROLE_PARTITION; + break; + + case 5: + role = ROLE_SPLIT; + break; + + default: + role = ROLE_INPUT; + if (n_warnings++ == 0) + sys_warn (r, -1, _("Invalid role for variable %s."), + var_get_name (var)); + } + + var_set_role (var, role); + } + } + + if (n_warnings > 1) + sys_warn (r, -1, _("%zu other variables had invalid roles."), + n_warnings - 1); +} + static void check_overflow (struct sfm_reader *r, const struct sfm_extension_record *record, @@ -2400,7 +2552,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) start = text->pos; n = 0; - for (;;) + while (text->pos < text->buffer.length) { int c = text->buffer.string[text->pos]; if (c < '0' || c > '9') @@ -2408,7 +2560,7 @@ text_parse_counted_string (struct sfm_reader *r, struct text_record *text) n = (n * 10) + (c - '0'); text->pos++; } - if (start == text->pos) + if (text->pos >= text->buffer.length || start == text->pos) { sys_warn (r, text->start, _("Expecting digit at offset %zu in MRSETS record."), @@ -2465,6 +2617,12 @@ text_pos (const struct text_record *text) { return text->pos; } + +static const char * +text_get_all (const struct text_record *text) +{ + return text->buffer.string; +} /* Messages. */ @@ -2617,6 +2775,35 @@ skip_bytes (struct sfm_reader *r, size_t bytes) bytes -= chunk; } } + +/* Returns a malloc()'d copy of S in which all lone CRs and CR LF pairs have + been replaced by LFs. + + (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system + files that use CR-only line ends in the file label and extra product + info.) */ +static char * +fix_line_ends (const char *s) +{ + char *dst, *d; + + d = dst = xmalloc (strlen (s) + 1); + while (*s != '\0') + { + if (*s == '\r') + { + s++; + if (*s == '\n') + s++; + *d++ = '\n'; + } + else + *d++ = *s++; + } + *d = '\0'; + + return dst; +} static const struct casereader_class sys_file_casereader_class = {