1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
3 2013, 2015, 2016 Free Software Foundation, Inc.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
24 #include "data/dataset.h"
25 #include "data/dictionary.h"
26 #include "data/format.h"
27 #include "data/gnumeric-reader.h"
28 #include "data/ods-reader.h"
29 #include "data/spreadsheet-reader.h"
30 #include "data/psql-reader.h"
31 #include "data/settings.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/placement-parser.h"
37 #include "language/lexer/format-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "libpspp/cast.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
49 static bool parse_spreadsheet (struct lexer *lexer, char **filename,
50 struct spreadsheet_read_options *opts);
52 static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *);
54 static int parse_get_txt (struct lexer *lexer, struct dataset *);
55 static int parse_get_psql (struct lexer *lexer, struct dataset *);
58 cmd_get_data (struct lexer *lexer, struct dataset *ds)
61 struct spreadsheet_read_options opts;
63 opts.sheet_name = NULL;
64 opts.sheet_index = -1;
65 opts.cell_range = NULL;
66 opts.read_names = false;
69 if (! lex_force_match (lexer, T_SLASH))
72 if (!lex_force_match_id (lexer, "TYPE"))
75 if (!lex_force_match (lexer, T_EQUALS))
78 const char *s = lex_tokcstr (lexer);
83 if (lex_match_id (lexer, "TXT"))
86 return parse_get_txt (lexer, ds);
88 else if (lex_match_id (lexer, "PSQL"))
91 return parse_get_psql (lexer, ds);
93 else if (lex_match_id (lexer, "GNM") ||
94 lex_match_id (lexer, "ODS"))
96 char *filename = NULL;
97 if (!parse_spreadsheet (lexer, &filename, &opts))
100 struct spreadsheet *spreadsheet = NULL;
101 if (0 == strncasecmp (tok, "GNM", 3))
102 spreadsheet = gnumeric_probe (filename, true);
103 else if (0 == strncasecmp (tok, "ODS", 3))
104 spreadsheet = ods_probe (filename, true);
106 if (spreadsheet == NULL)
108 msg (SE, _("error reading file `%s'"), filename);
114 struct casereader *reader = spreadsheet_make_reader (spreadsheet, &opts);
117 dataset_set_dict (ds, dict_clone (spreadsheet->dict));
118 dataset_set_source (ds, reader);
120 destroy_spreadsheet_read_info (&opts);
121 spreadsheet_unref (spreadsheet);
124 spreadsheet_unref (spreadsheet);
127 lex_error_expecting (lexer, "TXT", "PSQL", "GNM", "ODS");
130 destroy_spreadsheet_read_info (&opts);
136 parse_get_psql (struct lexer *lexer, struct dataset *ds)
138 struct psql_read_info psql;
139 psql.allow_clear = false;
140 psql.conninfo = NULL;
143 ds_init_empty (&psql.sql);
145 if (! lex_force_match (lexer, T_SLASH))
148 if (!lex_force_match_id (lexer, "CONNECT"))
151 if (! lex_force_match (lexer, T_EQUALS))
154 if (!lex_force_string (lexer))
157 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
161 while (lex_match (lexer, T_SLASH))
163 if (lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
165 lex_match (lexer, T_EQUALS);
166 if (lex_force_int_range (lexer, "ASSUMEDSTRWIDTH", 1, 32767))
168 psql.str_width = lex_integer (lexer);
172 else if (lex_match_id (lexer, "BSIZE"))
174 lex_match (lexer, T_EQUALS);
175 if (lex_force_int_range (lexer, "BSIZE", 1, INT_MAX))
177 psql.bsize = lex_integer (lexer);
181 else if (lex_match_id (lexer, "UNENCRYPTED"))
183 psql.allow_clear = true;
185 else if (lex_match_id (lexer, "SQL"))
187 lex_match (lexer, T_EQUALS);
188 if (! lex_force_string (lexer))
191 ds_put_substring (&psql.sql, lex_tokss (lexer));
196 struct dictionary *dict = NULL;
197 struct casereader *reader = psql_open_reader (&psql, &dict);
201 dataset_set_dict (ds, dict);
202 dataset_set_source (ds, reader);
206 ds_destroy (&psql.sql);
207 free (psql.conninfo);
213 ds_destroy (&psql.sql);
214 free (psql.conninfo);
220 parse_spreadsheet (struct lexer *lexer, char **filename,
221 struct spreadsheet_read_options *opts)
223 opts->sheet_index = 1;
224 opts->sheet_name = NULL;
225 opts->cell_range = NULL;
226 opts->read_names = true;
229 if (! lex_force_match (lexer, T_SLASH))
232 if (!lex_force_match_id (lexer, "FILE"))
235 if (! lex_force_match (lexer, T_EQUALS))
238 if (!lex_force_string (lexer))
241 *filename = utf8_to_filename (lex_tokcstr (lexer));
245 while (lex_match (lexer, T_SLASH))
247 if (lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
249 lex_match (lexer, T_EQUALS);
250 if (lex_force_int_range (lexer, "ASSUMEDSTRWIDTH", 1, 32767))
252 opts->asw = lex_integer (lexer);
256 else if (lex_match_id (lexer, "SHEET"))
258 lex_match (lexer, T_EQUALS);
259 if (lex_match_id (lexer, "NAME"))
261 if (! lex_force_string (lexer))
264 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
265 opts->sheet_index = -1;
269 else if (lex_match_id (lexer, "INDEX"))
271 if (!lex_force_int_range (lexer, "INDEX", 1, INT_MAX))
273 opts->sheet_index = lex_integer (lexer);
278 lex_error_expecting (lexer, "NAME", "INDEX");
282 else if (lex_match_id (lexer, "CELLRANGE"))
284 lex_match (lexer, T_EQUALS);
286 if (lex_match_id (lexer, "FULL"))
288 opts->cell_range = NULL;
290 else if (lex_match_id (lexer, "RANGE"))
292 if (! lex_force_string (lexer))
295 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
300 lex_error_expecting (lexer, "FULL", "RANGE");
304 else if (lex_match_id (lexer, "READNAMES"))
306 lex_match (lexer, T_EQUALS);
308 if (lex_match_id (lexer, "ON"))
310 opts->read_names = true;
312 else if (lex_match_id (lexer, "OFF"))
314 opts->read_names = false;
318 lex_error_expecting (lexer, "ON", "OFF");
324 lex_error (lexer, NULL);
337 set_type (struct data_parser *parser, const char *subcommand,
338 enum data_parser_type type, bool *has_type)
342 data_parser_set_type (parser, type);
345 else if (type != data_parser_get_type (parser))
347 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
348 "was stated or implied earlier in this command."),
350 type == DP_FIXED ? "FIXED" : "DELIMITED",
351 type == DP_FIXED ? "DELIMITED" : "FIXED");
358 parse_get_txt (struct lexer *lexer, struct dataset *ds)
360 struct data_parser *parser = NULL;
361 struct dictionary *dict = dict_create (get_default_encoding ());
362 struct file_handle *fh = NULL;
363 struct dfm_reader *reader = NULL;
364 char *encoding = NULL;
368 enum data_parser_type type;
371 if (! lex_force_match (lexer, T_SLASH))
374 if (!lex_force_match_id (lexer, "FILE"))
376 if (! lex_force_match (lexer, T_EQUALS))
378 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
382 parser = data_parser_create ();
384 data_parser_set_type (parser, DP_DELIMITED);
385 data_parser_set_span (parser, false);
386 data_parser_set_quotes (parser, ss_empty ());
387 data_parser_set_quote_escape (parser, true);
388 data_parser_set_empty_line_has_field (parser, true);
392 if (!lex_force_match (lexer, T_SLASH))
395 if (lex_match_id (lexer, "ENCODING"))
397 lex_match (lexer, T_EQUALS);
398 if (!lex_force_string (lexer))
402 encoding = ss_xstrdup (lex_tokss (lexer));
406 else if (lex_match_id (lexer, "ARRANGEMENT"))
410 lex_match (lexer, T_EQUALS);
411 if (lex_match_id (lexer, "FIXED"))
412 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
413 else if (lex_match_id (lexer, "DELIMITED"))
414 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
415 DP_DELIMITED, &has_type);
418 lex_error_expecting (lexer, "FIXED", "DELIMITED");
424 else if (lex_match_id (lexer, "FIRSTCASE"))
426 lex_match (lexer, T_EQUALS);
427 if (!lex_force_int_range (lexer, "FIRSTCASE", 1, INT_MAX))
429 data_parser_set_skip (parser, lex_integer (lexer) - 1);
432 else if (lex_match_id_n (lexer, "DELCASE", 4))
434 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
436 lex_match (lexer, T_EQUALS);
437 if (lex_match_id (lexer, "LINE"))
438 data_parser_set_span (parser, false);
439 else if (lex_match_id (lexer, "VARIABLES"))
441 data_parser_set_span (parser, true);
443 /* VARIABLES takes an integer argument, but for no
444 good reason. We just ignore it. */
445 if (!lex_force_int (lexer))
451 lex_error_expecting (lexer, "LINE", "VARIABLES");
455 else if (lex_match_id (lexer, "FIXCASE"))
457 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
459 lex_match (lexer, T_EQUALS);
460 if (!lex_force_int_range (lexer, "FIXCASE", 1, INT_MAX))
462 data_parser_set_records (parser, lex_integer (lexer));
465 else if (lex_match_id (lexer, "IMPORTCASES"))
467 int start_ofs = lex_ofs (lexer) - 1;
468 lex_match (lexer, T_EQUALS);
469 if (lex_match (lexer, T_ALL))
473 else if (lex_match_id (lexer, "FIRST"))
475 if (!lex_force_int (lexer))
479 else if (lex_match_id (lexer, "PERCENT"))
481 if (!lex_force_int (lexer))
485 lex_ofs_msg (lexer, SW, start_ofs, lex_ofs (lexer) - 1,
486 _("Ignoring obsolete IMPORTCASES subcommand. (N OF "
487 "CASES or SAMPLE may be used to substitute.)"));
489 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
491 struct string hard_seps = DS_EMPTY_INITIALIZER;
492 const char *soft_seps = "";
496 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
498 lex_match (lexer, T_EQUALS);
500 if (!lex_force_string (lexer))
503 /* XXX should support multibyte UTF-8 characters */
504 s = lex_tokss (lexer);
505 if (ss_match_string (&s, ss_cstr ("\\t")))
506 ds_put_cstr (&hard_seps, "\t");
507 if (ss_match_string (&s, ss_cstr ("\\\\")))
508 ds_put_cstr (&hard_seps, "\\");
509 while ((c = ss_get_byte (&s)) != EOF)
513 ds_put_byte (&hard_seps, c);
514 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
515 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
516 ds_destroy (&hard_seps);
520 else if (lex_match_id (lexer, "QUALIFIERS"))
522 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
524 lex_match (lexer, T_EQUALS);
526 if (!lex_force_string (lexer))
529 /* XXX should support multibyte UTF-8 characters */
530 if (settings_get_syntax () == COMPATIBLE
531 && ss_length (lex_tokss (lexer)) != 1)
533 lex_error (lexer, _("In compatible syntax mode, the QUALIFIER "
534 "string must contain exactly one character."));
538 data_parser_set_quotes (parser, lex_tokss (lexer));
541 else if (lex_match_id (lexer, "VARIABLES"))
545 lex_error_expecting (lexer, "VARIABLES");
549 lex_match (lexer, T_EQUALS);
552 type = data_parser_get_type (parser);
555 struct fmt_spec input, output;
559 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
561 if (!lex_force_int_range (lexer, NULL, record,
562 data_parser_get_records (parser)))
564 record = lex_integer (lexer);
568 int name_ofs = lex_ofs (lexer);
569 if (!lex_force_id (lexer))
571 name = xstrdup (lex_tokcstr (lexer));
572 char *error = dict_id_is_valid__ (dict, name);
575 lex_error (lexer, "%s", error);
580 if (type == DP_DELIMITED)
582 if (!parse_format_specifier (lexer, &input))
584 error = fmt_check_input__ (&input);
587 lex_next_error (lexer, -1, -1, "%s", error);
591 output = fmt_for_output_from_input (&input,
592 settings_get_fmt_settings ());
596 int start_ofs = lex_ofs (lexer);
597 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
600 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
601 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
604 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
607 enum fmt_type fmt_type;
608 if (!fmt_from_name (fmt_type_name, &fmt_type))
610 lex_next_error (lexer, -1, -1,
611 _("Unknown format type `%s'."), fmt_type_name);
614 int end_ofs = lex_ofs (lexer) - 1;
616 /* Compose input format. */
617 input = (struct fmt_spec) { .type = fmt_type, .w = lc - fc + 1 };
618 error = fmt_check_input__ (&input);
621 lex_next_error (lexer, start_ofs, end_ofs, "%s", error);
626 /* Compose output format. */
629 output = (struct fmt_spec) { .type = fmt_type, .w = w, .d = d };
630 error = fmt_check_output__ (&output);
633 lex_next_error (lexer, start_ofs, end_ofs, "%s", error);
639 output = fmt_for_output_from_input (&input,
640 settings_get_fmt_settings ());
642 v = dict_create_var (dict, name, fmt_var_width (&input));
645 lex_ofs_error (lexer, name_ofs, name_ofs,
646 _("%s is a duplicate variable name."), name);
649 var_set_both_formats (v, &output);
650 if (type == DP_DELIMITED)
651 data_parser_add_delimited_field (parser, &input,
652 var_get_case_index (v),
655 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
660 while (lex_token (lexer) != T_ENDCMD);
662 reader = dfm_open_reader (fh, lexer, encoding);
666 data_parser_make_active_file (parser, ds, reader, dict, NULL, NULL);
672 data_parser_destroy (parser);
677 return CMD_CASCADING_FAILURE;
682 destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts)
684 free (opts->cell_range);
685 free (opts->sheet_name);