X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fdata-io%2Fget-data.c;h=2bf419659d18e52ae99a91455c05dcd8a5b698df;hb=1b5c2d8fe129a5f8db76c42250a9c199deccb773;hp=e4ab76a9992739c78a36f6ad2a98c4f6e0b48d83;hpb=9e0e4996fad6563f0a1ce628b80db5c23ef8279e;p=pspp diff --git a/src/language/data-io/get-data.c b/src/language/data-io/get-data.c index e4ab76a999..2bf419659d 100644 --- a/src/language/data-io/get-data.c +++ b/src/language/data-io/get-data.c @@ -1,5 +1,6 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2007, 2008, 2009 Free Software Foundation, Inc. + Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, + 2013, 2015, 2016 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,50 +19,140 @@ #include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "xalloc.h" +#include + +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/format.h" +#include "data/gnumeric-reader.h" +#include "data/ods-reader.h" +#include "data/spreadsheet-reader.h" +#include "data/psql-reader.h" +#include "data/settings.h" +#include "language/command.h" +#include "language/data-io/data-parser.h" +#include "language/data-io/data-reader.h" +#include "language/data-io/file-handle.h" +#include "language/data-io/placement-parser.h" +#include "language/lexer/format-parser.h" +#include "language/lexer/lexer.h" +#include "libpspp/cast.h" +#include "libpspp/i18n.h" +#include "libpspp/message.h" + +#include "gl/xalloc.h" #include "gettext.h" #define _(msgid) gettext (msgid) #define N_(msgid) (msgid) -static int parse_get_gnm (struct lexer *lexer, struct dataset *); + +#ifdef ODF_READ_SUPPORT +static const bool odf_read_support = true; +#else +static const bool odf_read_support = false; +struct spreadsheet *ods_probe (const char *filename, bool report_errors){} +struct casereader * ods_make_reader (struct spreadsheet *spreadsheet, + const struct spreadsheet_read_options *opts){} +void ods_unref (struct spreadsheet *r){} +#endif + +#ifdef GNM_READ_SUPPORT +static const bool gnm_read_support = true; +#else +static const bool gnm_read_support = false; +struct spreadsheet *gnumeric_probe (const char *filename, bool report_errors){} +struct casereader * gnumeric_make_reader (struct spreadsheet *spreadsheet, + const struct spreadsheet_read_options *opts){} +void gnumeric_unref (struct spreadsheet *r){} + +#endif + +static bool parse_spreadsheet (struct lexer *lexer, char **filename, + struct spreadsheet_read_options *opts); + +static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *); + static int parse_get_txt (struct lexer *lexer, struct dataset *); static int parse_get_psql (struct lexer *lexer, struct dataset *); int cmd_get_data (struct lexer *lexer, struct dataset *ds) { - lex_force_match (lexer, '/'); + char *tok = NULL; + struct spreadsheet_read_options opts; + + opts.sheet_name = NULL; + opts.sheet_index = -1; + opts.cell_range = NULL; + opts.read_names = false; + opts.asw = -1; + + lex_force_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "TYPE")) - return CMD_FAILURE; + goto error; - lex_force_match (lexer, '='); + lex_force_match (lexer, T_EQUALS); - if (lex_match_id (lexer, "GNM")) - return parse_get_gnm (lexer, ds); - else if (lex_match_id (lexer, "TXT")) - return parse_get_txt (lexer, ds); + tok = strdup (lex_tokcstr (lexer)); + if (lex_match_id (lexer, "TXT")) + { + free (tok); + return parse_get_txt (lexer, ds); + } else if (lex_match_id (lexer, "PSQL")) - return parse_get_psql (lexer, ds); + { + free (tok); + return parse_get_psql (lexer, ds); + } + else if (lex_match_id (lexer, "GNM") || + lex_match_id (lexer, "ODS")) + { + char *filename = NULL; + struct casereader *reader = NULL; + struct dictionary *dict = NULL; + + if (!parse_spreadsheet (lexer, &filename, &opts)) + goto error; + + if ( gnm_read_support && 0 == strncasecmp (tok, "GNM", 3)) + { + struct spreadsheet *spreadsheet = gnumeric_probe (filename, true); + if (spreadsheet == NULL) + goto error; + reader = gnumeric_make_reader (spreadsheet, &opts); + dict = spreadsheet->dict; + gnumeric_unref (spreadsheet); + } + else if ( odf_read_support && 0 == strncasecmp (tok, "ODS", 3)) + { + struct spreadsheet *spreadsheet = ods_probe (filename, true); + if (spreadsheet == NULL) + goto error; + reader = ods_make_reader (spreadsheet, &opts); + dict = spreadsheet->dict; + ods_unref (spreadsheet); + } + + free (filename); + + if (reader) + { + dataset_set_dict (ds, dict); + dataset_set_source (ds, reader); + free (tok); + destroy_spreadsheet_read_info (&opts); + return CMD_SUCCESS; + } + } + else + msg (SE, _("Unsupported TYPE %s."), tok); - msg (SE, _("Unsupported TYPE %s"), lex_tokid (lexer)); + + error: + destroy_spreadsheet_read_info (&opts); + free (tok); return CMD_FAILURE; } @@ -75,31 +166,31 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds) psql.bsize = -1; ds_init_empty (&psql.sql); - lex_force_match (lexer, '/'); + lex_force_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "CONNECT")) goto error; - lex_force_match (lexer, '='); + lex_force_match (lexer, T_EQUALS); if (!lex_force_string (lexer)) goto error; - psql.conninfo = xstrdup (ds_cstr (lex_tokstr (lexer))); + psql.conninfo = ss_xstrdup (lex_tokss (lexer)); lex_get (lexer); - while (lex_match (lexer, '/') ) + while (lex_match (lexer, T_SLASH) ) { if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); psql.str_width = lex_integer (lexer); lex_get (lexer); } else if ( lex_match_id (lexer, "BSIZE")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); psql.bsize = lex_integer (lexer); lex_get (lexer); } @@ -109,11 +200,11 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "SQL")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if ( ! lex_force_string (lexer) ) goto error; - ds_put_substring (&psql.sql, lex_tokstr (lexer)->ss); + ds_put_substring (&psql.sql, lex_tokss (lexer)); lex_get (lexer); } } @@ -122,7 +213,10 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds) struct casereader *reader = psql_open_reader (&psql, &dict); if ( reader ) - proc_set_active_file (ds, reader, dict); + { + dataset_set_dict (ds, dict); + dataset_set_source (ds, reader); + } } ds_destroy (&psql.sql); @@ -138,114 +232,124 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds) return CMD_FAILURE; } -static int -parse_get_gnm (struct lexer *lexer, struct dataset *ds) +static bool +parse_spreadsheet (struct lexer *lexer, char **filename, + struct spreadsheet_read_options *opts) { - struct gnumeric_read_info gri = {NULL, NULL, NULL, 1, true, -1}; + opts->sheet_index = 1; + opts->sheet_name = NULL; + opts->cell_range = NULL; + opts->read_names = true; + opts->asw = -1; - lex_force_match (lexer, '/'); + lex_force_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "FILE")) goto error; - lex_force_match (lexer, '='); + lex_force_match (lexer, T_EQUALS); if (!lex_force_string (lexer)) goto error; - gri.file_name = xstrdup (ds_cstr (lex_tokstr (lexer))); + *filename = utf8_to_filename (lex_tokcstr (lexer)); lex_get (lexer); - while (lex_match (lexer, '/') ) + while (lex_match (lexer, T_SLASH) ) { if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH")) { - lex_match (lexer, '='); - gri.asw = lex_integer (lexer); + lex_match (lexer, T_EQUALS); + opts->asw = lex_integer (lexer); + lex_get (lexer); } else if (lex_match_id (lexer, "SHEET")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "NAME")) { if ( ! lex_force_string (lexer) ) goto error; - gri.sheet_name = xstrdup (ds_cstr (lex_tokstr (lexer))); - gri.sheet_index = -1; + opts->sheet_name = ss_xstrdup (lex_tokss (lexer)); + opts->sheet_index = -1; + + lex_get (lexer); } else if (lex_match_id (lexer, "INDEX")) { - gri.sheet_index = lex_integer (lexer); + opts->sheet_index = lex_integer (lexer); + if (opts->sheet_index <= 0) + { + msg (SE, _("The sheet index must be greater than or equal to 1")); + goto error; + } + lex_get (lexer); } else - goto error; + { + msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."), + "/SHEET", "NAME", "INDEX"); + goto error; + } } else if (lex_match_id (lexer, "CELLRANGE")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "FULL")) { - gri.cell_range = NULL; - lex_put_back (lexer, T_ID); + opts->cell_range = NULL; } else if (lex_match_id (lexer, "RANGE")) { if ( ! lex_force_string (lexer) ) goto error; - gri.cell_range = xstrdup (ds_cstr (lex_tokstr (lexer))); + opts->cell_range = ss_xstrdup (lex_tokss (lexer)); + lex_get (lexer); } else - goto error; + { + msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."), + "/CELLRANGE", "FULL", "RANGE"); + goto error; + } } else if (lex_match_id (lexer, "READNAMES")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if ( lex_match_id (lexer, "ON")) { - gri.read_names = true; + opts->read_names = true; } else if (lex_match_id (lexer, "OFF")) { - gri.read_names = false; + opts->read_names = false; } else - goto error; - lex_put_back (lexer, T_ID); + { + msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."), + "/READNAMES", "ON", "OFF"); + goto error; + } } else { - printf ("Unknown data file type \"\%s\"\n", lex_tokid (lexer)); + lex_error (lexer, NULL); goto error; } - lex_get (lexer); } - { - struct dictionary *dict = NULL; - struct casereader *reader = gnumeric_open_reader (&gri, &dict); - - if ( reader ) - proc_set_active_file (ds, reader, dict); - } - - free (gri.file_name); - free (gri.sheet_name); - free (gri.cell_range); - return CMD_SUCCESS; + return true; error: - - free (gri.file_name); - free (gri.sheet_name); - free (gri.cell_range); - return CMD_FAILURE; + return false; } + static bool set_type (struct data_parser *parser, const char *subcommand, enum data_parser_type type, bool *has_type) @@ -271,40 +375,54 @@ static int parse_get_txt (struct lexer *lexer, struct dataset *ds) { struct data_parser *parser = NULL; - struct dictionary *dict = NULL; + struct dictionary *dict = dict_create (get_default_encoding ()); struct file_handle *fh = NULL; struct dfm_reader *reader = NULL; + char *encoding = NULL; + char *name = NULL; int record; enum data_parser_type type; bool has_type; - lex_force_match (lexer, '/'); + lex_force_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "FILE")) goto error; - lex_force_match (lexer, '='); - fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE); + lex_force_match (lexer, T_EQUALS); + fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL); if (fh == NULL) goto error; - parser = data_parser_create (); + parser = data_parser_create (dict); has_type = false; data_parser_set_type (parser, DP_DELIMITED); data_parser_set_span (parser, false); data_parser_set_quotes (parser, ss_empty ()); + data_parser_set_quote_escape (parser, true); data_parser_set_empty_line_has_field (parser, true); for (;;) { - if (!lex_force_match (lexer, '/')) + if (!lex_force_match (lexer, T_SLASH)) goto error; - if (lex_match_id (lexer, "ARRANGEMENT")) + if (lex_match_id (lexer, "ENCODING")) + { + lex_match (lexer, T_EQUALS); + if (!lex_force_string (lexer)) + goto error; + + free (encoding); + encoding = ss_xstrdup (lex_tokss (lexer)); + + lex_get (lexer); + } + else if (lex_match_id (lexer, "ARRANGEMENT")) { bool ok; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "FIXED")) ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type); else if (lex_match_id (lexer, "DELIMITED")) @@ -312,7 +430,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) DP_DELIMITED, &has_type); else { - lex_error (lexer, _("expecting FIXED or DELIMITED")); + lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL); goto error; } if (!ok) @@ -320,12 +438,12 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "FIRSTCASE")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!lex_force_int (lexer)) goto error; if (lex_integer (lexer) < 1) { - msg (SE, _("Value of FIRSTCASE must be 1 or greater.")); + msg (SE, _("Value of %s must be 1 or greater."), "FIRSTCASE"); goto error; } data_parser_set_skip (parser, lex_integer (lexer) - 1); @@ -335,7 +453,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) { if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type)) goto error; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "LINE")) data_parser_set_span (parser, false); else if (lex_match_id (lexer, "VARIABLES")) @@ -350,7 +468,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) } else { - lex_error (lexer, _("expecting LINE or VARIABLES")); + lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL); goto error; } } @@ -358,12 +476,12 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) { if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type)) goto error; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!lex_force_int (lexer)) goto error; if (lex_integer (lexer) < 1) { - msg (SE, _("Value of FIXCASE must be at least 1.")); + msg (SE, _("Value of %s must be 1 or greater."), "FIXCASE"); goto error; } data_parser_set_records (parser, lex_integer (lexer)); @@ -371,36 +489,25 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "IMPORTCASES")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match (lexer, T_ALL)) { - data_parser_set_case_limit (parser, -1); - data_parser_set_case_percent (parser, 100); + /* Nothing to do. */ } else if (lex_match_id (lexer, "FIRST")) { if (!lex_force_int (lexer)) goto error; - if (lex_integer (lexer) < 1) - { - msg (SE, _("Value of FIRST must be at least 1.")); - goto error; - } - data_parser_set_case_limit (parser, lex_integer (lexer)); lex_get (lexer); } else if (lex_match_id (lexer, "PERCENT")) { if (!lex_force_int (lexer)) goto error; - if (lex_integer (lexer) < 1 || lex_integer (lexer) > 100) - { - msg (SE, _("Value of PERCENT must be between 1 and 100.")); - goto error; - } - data_parser_set_case_percent (parser, lex_integer (lexer)); lex_get (lexer); } + msg (SW, _("Ignoring obsolete IMPORTCASES subcommand. (N OF CASES " + "or SAMPLE may be used to substitute.)")); } else if (lex_match_id_n (lexer, "DELIMITERS", 4)) { @@ -411,21 +518,22 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type)) goto error; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!lex_force_string (lexer)) goto error; - s = ds_ss (lex_tokstr (lexer)); + /* XXX should support multibyte UTF-8 characters */ + s = lex_tokss (lexer); if (ss_match_string (&s, ss_cstr ("\\t"))) ds_put_cstr (&hard_seps, "\t"); if (ss_match_string (&s, ss_cstr ("\\\\"))) ds_put_cstr (&hard_seps, "\\"); - while ((c = ss_get_char (&s)) != EOF) + while ((c = ss_get_byte (&s)) != EOF) if (c == ' ') soft_seps = " "; else - ds_put_char (&hard_seps, c); + ds_put_byte (&hard_seps, c); data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps)); data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps)); ds_destroy (&hard_seps); @@ -436,46 +544,42 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) { if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type)) goto error; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!lex_force_string (lexer)) goto error; + /* XXX should support multibyte UTF-8 characters */ if (settings_get_syntax () == COMPATIBLE - && ds_length (lex_tokstr (lexer)) != 1) + && ss_length (lex_tokss (lexer)) != 1) { msg (SE, _("In compatible syntax mode, the QUALIFIER string " "must contain exactly one character.")); goto error; } - data_parser_set_quotes (parser, ds_ss (lex_tokstr (lexer))); + data_parser_set_quotes (parser, lex_tokss (lexer)); lex_get (lexer); } - else if (settings_get_syntax () == ENHANCED - && lex_match_id (lexer, "ESCAPE")) - data_parser_set_quote_escape (parser, true); else if (lex_match_id (lexer, "VARIABLES")) break; else { - lex_error (lexer, _("expecting VARIABLES")); + lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL); goto error; } } - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); - dict = dict_create (); record = 1; type = data_parser_get_type (parser); do { - char name[VAR_NAME_LEN + 1]; struct fmt_spec input, output; - int fc, lc; struct variable *v; + int fc, lc; - while (type == DP_FIXED && lex_match (lexer, '/')) + while (type == DP_FIXED && lex_match (lexer, T_SLASH)) { if (!lex_force_int (lexer)) goto error; @@ -500,9 +604,10 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) lex_get (lexer); } - if (!lex_force_id (lexer)) + if (!lex_force_id (lexer) + || !dict_id_is_valid (dict, lex_tokcstr (lexer), true)) goto error; - strcpy (name, lex_tokid (lexer)); + name = xstrdup (lex_tokcstr (lexer)); lex_get (lexer); if (type == DP_DELIMITED) @@ -510,19 +615,46 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) if (!parse_format_specifier (lexer, &input) || !fmt_check_input (&input)) goto error; + + output = fmt_for_output_from_input (&input); } else { + char fmt_type_name[FMT_TYPE_LEN_MAX + 1]; + enum fmt_type fmt_type; + int w, d; + if (!parse_column_range (lexer, 0, &fc, &lc, NULL)) goto error; - if (!parse_format_specifier_name (lexer, &input.type)) + + /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */ + if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d)) goto error; + if (!fmt_from_name (fmt_type_name, &fmt_type)) + { + msg (SE, _("Unknown format type `%s'."), fmt_type_name); + goto error; + } + + /* Compose input format. */ + input.type = fmt_type; input.w = lc - fc + 1; input.d = 0; if (!fmt_check_input (&input)) goto error; + + /* Compose output format. */ + if (w != 0) + { + output.type = fmt_type; + output.w = w; + output.d = d; + if (!fmt_check_output (&output)) + goto error; + } + else + output = fmt_for_output_from_input (&input); } - output = fmt_for_output_from_input (&input); v = dict_create_var (dict, name, fmt_var_width (&input)); if (v == NULL) @@ -539,20 +671,33 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds) else data_parser_add_fixed_field (parser, &input, var_get_case_index (v), name, record, fc); + free (name); + name = NULL; } - while (lex_token (lexer) != '.'); + while (lex_token (lexer) != T_ENDCMD); - reader = dfm_open_reader (fh, lexer); + reader = dfm_open_reader (fh, lexer, encoding); if (reader == NULL) goto error; data_parser_make_active_file (parser, ds, reader, dict); fh_unref (fh); + free (encoding); return CMD_SUCCESS; error: data_parser_destroy (parser); dict_destroy (dict); fh_unref (fh); + free (name); + free (encoding); return CMD_CASCADING_FAILURE; } + + +static void +destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts) +{ + free (opts->cell_range); + free (opts->sheet_name); +}