1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
3 2013, 2015, 2016 Free Software Foundation, Inc.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
24 #include "data/dataset.h"
25 #include "data/dictionary.h"
26 #include "data/format.h"
27 #include "data/gnumeric-reader.h"
28 #include "data/ods-reader.h"
29 #include "data/spreadsheet-reader.h"
30 #include "data/psql-reader.h"
31 #include "data/settings.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/placement-parser.h"
37 #include "language/lexer/format-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "libpspp/cast.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
50 #ifdef ODF_READ_SUPPORT
51 static const bool odf_read_support = true;
53 static const bool odf_read_support = false;
54 struct spreadsheet *ods_probe (const char *filename, bool report_errors){}
55 struct casereader * ods_make_reader (struct spreadsheet *spreadsheet,
56 const struct spreadsheet_read_options *opts){}
57 void ods_unref (struct spreadsheet *r){}
60 #ifdef GNM_READ_SUPPORT
61 static const bool gnm_read_support = true;
63 static const bool gnm_read_support = false;
64 struct spreadsheet *gnumeric_probe (const char *filename, bool report_errors){}
65 struct casereader * gnumeric_make_reader (struct spreadsheet *spreadsheet,
66 const struct spreadsheet_read_options *opts){}
67 void gnumeric_unref (struct spreadsheet *r){}
71 static bool parse_spreadsheet (struct lexer *lexer, char **filename,
72 struct spreadsheet_read_options *opts);
74 static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *);
76 static int parse_get_txt (struct lexer *lexer, struct dataset *);
77 static int parse_get_psql (struct lexer *lexer, struct dataset *);
80 cmd_get_data (struct lexer *lexer, struct dataset *ds)
83 struct spreadsheet_read_options opts;
85 opts.sheet_name = NULL;
86 opts.sheet_index = -1;
87 opts.cell_range = NULL;
88 opts.read_names = false;
91 if (! lex_force_match (lexer, T_SLASH))
94 if (!lex_force_match_id (lexer, "TYPE"))
97 if (!lex_force_match (lexer, T_EQUALS))
100 const char *s = lex_tokcstr (lexer);
105 if (lex_match_id (lexer, "TXT"))
108 return parse_get_txt (lexer, ds);
110 else if (lex_match_id (lexer, "PSQL"))
113 return parse_get_psql (lexer, ds);
115 else if (lex_match_id (lexer, "GNM") ||
116 lex_match_id (lexer, "ODS"))
118 char *filename = NULL;
119 if (!parse_spreadsheet (lexer, &filename, &opts))
122 struct spreadsheet *spreadsheet = NULL;
123 if ( gnm_read_support && 0 == strncasecmp (tok, "GNM", 3))
124 spreadsheet = gnumeric_probe (filename, true);
125 else if ( odf_read_support && 0 == strncasecmp (tok, "ODS", 3))
126 spreadsheet = ods_probe (filename, true);
129 if (spreadsheet == NULL)
132 struct casereader *reader = spreadsheet_make_reader (spreadsheet, &opts);
135 dataset_set_dict (ds, dict_clone (spreadsheet->dict));
136 dataset_set_source (ds, reader);
138 destroy_spreadsheet_read_info (&opts);
139 spreadsheet_unref (spreadsheet);
142 spreadsheet_unref (spreadsheet);
145 msg (SE, _("Unsupported TYPE %s."), tok);
149 destroy_spreadsheet_read_info (&opts);
155 parse_get_psql (struct lexer *lexer, struct dataset *ds)
157 struct psql_read_info psql;
158 psql.allow_clear = false;
159 psql.conninfo = NULL;
162 ds_init_empty (&psql.sql);
164 if (! lex_force_match (lexer, T_SLASH))
167 if (!lex_force_match_id (lexer, "CONNECT"))
170 if (! lex_force_match (lexer, T_EQUALS))
173 if (!lex_force_string (lexer))
176 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
180 while (lex_match (lexer, T_SLASH) )
182 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
184 lex_match (lexer, T_EQUALS);
185 if (lex_force_int (lexer))
187 psql.str_width = lex_integer (lexer);
191 else if ( lex_match_id (lexer, "BSIZE"))
193 lex_match (lexer, T_EQUALS);
194 if (lex_force_int (lexer))
196 psql.bsize = lex_integer (lexer);
200 else if ( lex_match_id (lexer, "UNENCRYPTED"))
202 psql.allow_clear = true;
204 else if (lex_match_id (lexer, "SQL"))
206 lex_match (lexer, T_EQUALS);
207 if ( ! lex_force_string (lexer) )
210 ds_put_substring (&psql.sql, lex_tokss (lexer));
215 struct dictionary *dict = NULL;
216 struct casereader *reader = psql_open_reader (&psql, &dict);
220 dataset_set_dict (ds, dict);
221 dataset_set_source (ds, reader);
225 ds_destroy (&psql.sql);
226 free (psql.conninfo);
232 ds_destroy (&psql.sql);
233 free (psql.conninfo);
239 parse_spreadsheet (struct lexer *lexer, char **filename,
240 struct spreadsheet_read_options *opts)
242 opts->sheet_index = 1;
243 opts->sheet_name = NULL;
244 opts->cell_range = NULL;
245 opts->read_names = true;
248 if (! lex_force_match (lexer, T_SLASH))
251 if (!lex_force_match_id (lexer, "FILE"))
254 if (! lex_force_match (lexer, T_EQUALS))
257 if (!lex_force_string (lexer))
260 *filename = utf8_to_filename (lex_tokcstr (lexer));
264 while (lex_match (lexer, T_SLASH) )
266 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
268 lex_match (lexer, T_EQUALS);
269 if (lex_force_int (lexer))
271 opts->asw = lex_integer (lexer);
275 else if (lex_match_id (lexer, "SHEET"))
277 lex_match (lexer, T_EQUALS);
278 if (lex_match_id (lexer, "NAME"))
280 if ( ! lex_force_string (lexer) )
283 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
284 opts->sheet_index = -1;
288 else if (lex_match_id (lexer, "INDEX"))
290 if (!lex_force_int (lexer))
293 opts->sheet_index = lex_integer (lexer);
294 if (opts->sheet_index <= 0)
296 msg (SE, _("The sheet index must be greater than or equal to 1"));
303 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
304 "/SHEET", "NAME", "INDEX");
308 else if (lex_match_id (lexer, "CELLRANGE"))
310 lex_match (lexer, T_EQUALS);
312 if (lex_match_id (lexer, "FULL"))
314 opts->cell_range = NULL;
316 else if (lex_match_id (lexer, "RANGE"))
318 if ( ! lex_force_string (lexer) )
321 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
326 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
327 "/CELLRANGE", "FULL", "RANGE");
331 else if (lex_match_id (lexer, "READNAMES"))
333 lex_match (lexer, T_EQUALS);
335 if ( lex_match_id (lexer, "ON"))
337 opts->read_names = true;
339 else if (lex_match_id (lexer, "OFF"))
341 opts->read_names = false;
345 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
346 "/READNAMES", "ON", "OFF");
352 lex_error (lexer, NULL);
365 set_type (struct data_parser *parser, const char *subcommand,
366 enum data_parser_type type, bool *has_type)
370 data_parser_set_type (parser, type);
373 else if (type != data_parser_get_type (parser))
375 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
376 "was stated or implied earlier in this command."),
378 type == DP_FIXED ? "FIXED" : "DELIMITED",
379 type == DP_FIXED ? "DELIMITED" : "FIXED");
386 parse_get_txt (struct lexer *lexer, struct dataset *ds)
388 struct data_parser *parser = NULL;
389 struct dictionary *dict = dict_create (get_default_encoding ());
390 struct file_handle *fh = NULL;
391 struct dfm_reader *reader = NULL;
392 char *encoding = NULL;
396 enum data_parser_type type;
399 if (! lex_force_match (lexer, T_SLASH))
402 if (!lex_force_match_id (lexer, "FILE"))
404 if (! lex_force_match (lexer, T_EQUALS))
406 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
410 parser = data_parser_create (dict);
412 data_parser_set_type (parser, DP_DELIMITED);
413 data_parser_set_span (parser, false);
414 data_parser_set_quotes (parser, ss_empty ());
415 data_parser_set_quote_escape (parser, true);
416 data_parser_set_empty_line_has_field (parser, true);
420 if (!lex_force_match (lexer, T_SLASH))
423 if (lex_match_id (lexer, "ENCODING"))
425 lex_match (lexer, T_EQUALS);
426 if (!lex_force_string (lexer))
430 encoding = ss_xstrdup (lex_tokss (lexer));
434 else if (lex_match_id (lexer, "ARRANGEMENT"))
438 lex_match (lexer, T_EQUALS);
439 if (lex_match_id (lexer, "FIXED"))
440 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
441 else if (lex_match_id (lexer, "DELIMITED"))
442 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
443 DP_DELIMITED, &has_type);
446 lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL);
452 else if (lex_match_id (lexer, "FIRSTCASE"))
454 lex_match (lexer, T_EQUALS);
455 if (!lex_force_int (lexer))
457 if (lex_integer (lexer) < 1)
459 msg (SE, _("Value of %s must be 1 or greater."), "FIRSTCASE");
462 data_parser_set_skip (parser, lex_integer (lexer) - 1);
465 else if (lex_match_id_n (lexer, "DELCASE", 4))
467 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
469 lex_match (lexer, T_EQUALS);
470 if (lex_match_id (lexer, "LINE"))
471 data_parser_set_span (parser, false);
472 else if (lex_match_id (lexer, "VARIABLES"))
474 data_parser_set_span (parser, true);
476 /* VARIABLES takes an integer argument, but for no
477 good reason. We just ignore it. */
478 if (!lex_force_int (lexer))
484 lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL);
488 else if (lex_match_id (lexer, "FIXCASE"))
490 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
492 lex_match (lexer, T_EQUALS);
493 if (!lex_force_int (lexer))
495 if (lex_integer (lexer) < 1)
497 msg (SE, _("Value of %s must be 1 or greater."), "FIXCASE");
500 data_parser_set_records (parser, lex_integer (lexer));
503 else if (lex_match_id (lexer, "IMPORTCASES"))
505 lex_match (lexer, T_EQUALS);
506 if (lex_match (lexer, T_ALL))
510 else if (lex_match_id (lexer, "FIRST"))
512 if (!lex_force_int (lexer))
516 else if (lex_match_id (lexer, "PERCENT"))
518 if (!lex_force_int (lexer))
522 msg (SW, _("Ignoring obsolete IMPORTCASES subcommand. (N OF CASES "
523 "or SAMPLE may be used to substitute.)"));
525 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
527 struct string hard_seps = DS_EMPTY_INITIALIZER;
528 const char *soft_seps = "";
532 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
534 lex_match (lexer, T_EQUALS);
536 if (!lex_force_string (lexer))
539 /* XXX should support multibyte UTF-8 characters */
540 s = lex_tokss (lexer);
541 if (ss_match_string (&s, ss_cstr ("\\t")))
542 ds_put_cstr (&hard_seps, "\t");
543 if (ss_match_string (&s, ss_cstr ("\\\\")))
544 ds_put_cstr (&hard_seps, "\\");
545 while ((c = ss_get_byte (&s)) != EOF)
549 ds_put_byte (&hard_seps, c);
550 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
551 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
552 ds_destroy (&hard_seps);
556 else if (lex_match_id (lexer, "QUALIFIERS"))
558 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
560 lex_match (lexer, T_EQUALS);
562 if (!lex_force_string (lexer))
565 /* XXX should support multibyte UTF-8 characters */
566 if (settings_get_syntax () == COMPATIBLE
567 && ss_length (lex_tokss (lexer)) != 1)
569 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
570 "must contain exactly one character."));
574 data_parser_set_quotes (parser, lex_tokss (lexer));
577 else if (lex_match_id (lexer, "VARIABLES"))
581 lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL);
585 lex_match (lexer, T_EQUALS);
588 type = data_parser_get_type (parser);
591 struct fmt_spec input, output;
595 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
597 if (!lex_force_int (lexer))
599 if (lex_integer (lexer) < record)
601 msg (SE, _("The record number specified, %ld, is at or "
602 "before the previous record, %d. Data "
603 "fields must be listed in order of "
604 "increasing record number."),
605 lex_integer (lexer), record);
608 if (lex_integer (lexer) > data_parser_get_records (parser))
610 msg (SE, _("The record number specified, %ld, exceeds "
611 "the number of records per case specified "
613 lex_integer (lexer), data_parser_get_records (parser));
616 record = lex_integer (lexer);
620 if (!lex_force_id (lexer)
621 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
623 name = xstrdup (lex_tokcstr (lexer));
626 if (type == DP_DELIMITED)
628 if (!parse_format_specifier (lexer, &input)
629 || !fmt_check_input (&input))
632 output = fmt_for_output_from_input (&input);
636 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
637 enum fmt_type fmt_type;
640 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
643 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
644 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
646 if (!fmt_from_name (fmt_type_name, &fmt_type))
648 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
652 /* Compose input format. */
653 input.type = fmt_type;
654 input.w = lc - fc + 1;
656 if (!fmt_check_input (&input))
659 /* Compose output format. */
662 output.type = fmt_type;
665 if (!fmt_check_output (&output))
669 output = fmt_for_output_from_input (&input);
672 v = dict_create_var (dict, name, fmt_var_width (&input));
675 msg (SE, _("%s is a duplicate variable name."), name);
678 var_set_both_formats (v, &output);
680 if (type == DP_DELIMITED)
681 data_parser_add_delimited_field (parser, &input,
682 var_get_case_index (v),
685 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
690 while (lex_token (lexer) != T_ENDCMD);
692 reader = dfm_open_reader (fh, lexer, encoding);
696 data_parser_make_active_file (parser, ds, reader, dict);
702 data_parser_destroy (parser);
707 return CMD_CASCADING_FAILURE;
712 destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts)
714 free (opts->cell_range);
715 free (opts->sheet_name);