1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
3 2013, 2015 Free Software Foundation, Inc.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
24 #include "data/dataset.h"
25 #include "data/dictionary.h"
26 #include "data/format.h"
27 #include "data/gnumeric-reader.h"
28 #include "data/ods-reader.h"
29 #include "data/spreadsheet-reader.h"
30 #include "data/psql-reader.h"
31 #include "data/settings.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/placement-parser.h"
37 #include "language/lexer/format-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "libpspp/cast.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
50 #ifdef ODF_READ_SUPPORT
51 static const bool odf_read_support = true;
53 static const bool odf_read_support = false;
56 #ifdef GNM_READ_SUPPORT
57 static const bool gnm_read_support = true;
59 static const bool gnm_read_support = false;
62 static bool parse_spreadsheet (struct lexer *lexer, char **filename,
63 struct spreadsheet_read_options *opts);
65 static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *);
67 static int parse_get_txt (struct lexer *lexer, struct dataset *);
68 static int parse_get_psql (struct lexer *lexer, struct dataset *);
71 cmd_get_data (struct lexer *lexer, struct dataset *ds)
74 struct spreadsheet_read_options opts;
76 opts.sheet_name = NULL;
77 opts.sheet_index = -1;
78 opts.cell_range = NULL;
79 opts.read_names = false;
82 lex_force_match (lexer, T_SLASH);
84 if (!lex_force_match_id (lexer, "TYPE"))
87 lex_force_match (lexer, T_EQUALS);
89 tok = strdup (lex_tokcstr (lexer));
90 if (lex_match_id (lexer, "TXT"))
93 return parse_get_txt (lexer, ds);
95 else if (lex_match_id (lexer, "PSQL"))
98 return parse_get_psql (lexer, ds);
100 else if (lex_match_id (lexer, "GNM") ||
101 lex_match_id (lexer, "ODS"))
103 char *filename = NULL;
104 struct casereader *reader = NULL;
105 struct dictionary *dict = NULL;
107 if (!parse_spreadsheet (lexer, &filename, &opts))
110 if ( gnm_read_support && 0 == strncasecmp (tok, "GNM", 3))
112 struct spreadsheet *spreadsheet = gnumeric_probe (filename, true);
113 if (spreadsheet == NULL)
115 reader = gnumeric_make_reader (spreadsheet, &opts);
116 dict = spreadsheet->dict;
117 gnumeric_destroy (spreadsheet);
119 else if ( odf_read_support && 0 == strncasecmp (tok, "ODS", 3))
121 struct spreadsheet *spreadsheet = ods_probe (filename, true);
122 if (spreadsheet == NULL)
124 reader = ods_make_reader (spreadsheet, &opts);
125 dict = spreadsheet->dict;
126 ods_destroy (spreadsheet);
133 dataset_set_dict (ds, dict);
134 dataset_set_source (ds, reader);
136 destroy_spreadsheet_read_info (&opts);
141 msg (SE, _("Unsupported TYPE %s."), tok);
145 destroy_spreadsheet_read_info (&opts);
151 parse_get_psql (struct lexer *lexer, struct dataset *ds)
153 struct psql_read_info psql;
154 psql.allow_clear = false;
155 psql.conninfo = NULL;
158 ds_init_empty (&psql.sql);
160 lex_force_match (lexer, T_SLASH);
162 if (!lex_force_match_id (lexer, "CONNECT"))
165 lex_force_match (lexer, T_EQUALS);
167 if (!lex_force_string (lexer))
170 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
174 while (lex_match (lexer, T_SLASH) )
176 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
178 lex_match (lexer, T_EQUALS);
179 psql.str_width = lex_integer (lexer);
182 else if ( lex_match_id (lexer, "BSIZE"))
184 lex_match (lexer, T_EQUALS);
185 psql.bsize = lex_integer (lexer);
188 else if ( lex_match_id (lexer, "UNENCRYPTED"))
190 psql.allow_clear = true;
192 else if (lex_match_id (lexer, "SQL"))
194 lex_match (lexer, T_EQUALS);
195 if ( ! lex_force_string (lexer) )
198 ds_put_substring (&psql.sql, lex_tokss (lexer));
203 struct dictionary *dict = NULL;
204 struct casereader *reader = psql_open_reader (&psql, &dict);
208 dataset_set_dict (ds, dict);
209 dataset_set_source (ds, reader);
213 ds_destroy (&psql.sql);
214 free (psql.conninfo);
220 ds_destroy (&psql.sql);
221 free (psql.conninfo);
227 parse_spreadsheet (struct lexer *lexer, char **filename,
228 struct spreadsheet_read_options *opts)
230 opts->sheet_index = 1;
231 opts->sheet_name = NULL;
232 opts->cell_range = NULL;
233 opts->read_names = true;
236 lex_force_match (lexer, T_SLASH);
238 if (!lex_force_match_id (lexer, "FILE"))
241 lex_force_match (lexer, T_EQUALS);
243 if (!lex_force_string (lexer))
246 *filename = utf8_to_filename (lex_tokcstr (lexer));
250 while (lex_match (lexer, T_SLASH) )
252 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
254 lex_match (lexer, T_EQUALS);
255 opts->asw = lex_integer (lexer);
258 else if (lex_match_id (lexer, "SHEET"))
260 lex_match (lexer, T_EQUALS);
261 if (lex_match_id (lexer, "NAME"))
263 if ( ! lex_force_string (lexer) )
266 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
267 opts->sheet_index = -1;
271 else if (lex_match_id (lexer, "INDEX"))
273 opts->sheet_index = lex_integer (lexer);
274 if (opts->sheet_index <= 0)
276 msg (SE, _("The sheet index must be greater than or equal to 1"));
283 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
284 "/SHEET", "NAME", "INDEX");
288 else if (lex_match_id (lexer, "CELLRANGE"))
290 lex_match (lexer, T_EQUALS);
292 if (lex_match_id (lexer, "FULL"))
294 opts->cell_range = NULL;
296 else if (lex_match_id (lexer, "RANGE"))
298 if ( ! lex_force_string (lexer) )
301 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
306 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
307 "/CELLRANGE", "FULL", "RANGE");
311 else if (lex_match_id (lexer, "READNAMES"))
313 lex_match (lexer, T_EQUALS);
315 if ( lex_match_id (lexer, "ON"))
317 opts->read_names = true;
319 else if (lex_match_id (lexer, "OFF"))
321 opts->read_names = false;
325 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
326 "/READNAMES", "ON", "OFF");
332 lex_error (lexer, NULL);
345 set_type (struct data_parser *parser, const char *subcommand,
346 enum data_parser_type type, bool *has_type)
350 data_parser_set_type (parser, type);
353 else if (type != data_parser_get_type (parser))
355 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
356 "was stated or implied earlier in this command."),
358 type == DP_FIXED ? "FIXED" : "DELIMITED",
359 type == DP_FIXED ? "DELIMITED" : "FIXED");
366 parse_get_txt (struct lexer *lexer, struct dataset *ds)
368 struct data_parser *parser = NULL;
369 struct dictionary *dict = dict_create (get_default_encoding ());
370 struct file_handle *fh = NULL;
371 struct dfm_reader *reader = NULL;
372 char *encoding = NULL;
376 enum data_parser_type type;
379 lex_force_match (lexer, T_SLASH);
381 if (!lex_force_match_id (lexer, "FILE"))
383 lex_force_match (lexer, T_EQUALS);
384 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
388 parser = data_parser_create (dict);
390 data_parser_set_type (parser, DP_DELIMITED);
391 data_parser_set_span (parser, false);
392 data_parser_set_quotes (parser, ss_empty ());
393 data_parser_set_quote_escape (parser, true);
394 data_parser_set_empty_line_has_field (parser, true);
398 if (!lex_force_match (lexer, T_SLASH))
401 if (lex_match_id (lexer, "ENCODING"))
403 lex_match (lexer, T_EQUALS);
404 if (!lex_force_string (lexer))
408 encoding = ss_xstrdup (lex_tokss (lexer));
412 else if (lex_match_id (lexer, "ARRANGEMENT"))
416 lex_match (lexer, T_EQUALS);
417 if (lex_match_id (lexer, "FIXED"))
418 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
419 else if (lex_match_id (lexer, "DELIMITED"))
420 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
421 DP_DELIMITED, &has_type);
424 lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL);
430 else if (lex_match_id (lexer, "FIRSTCASE"))
432 lex_match (lexer, T_EQUALS);
433 if (!lex_force_int (lexer))
435 if (lex_integer (lexer) < 1)
437 msg (SE, _("Value of %s must be 1 or greater."), "FIRSTCASE");
440 data_parser_set_skip (parser, lex_integer (lexer) - 1);
443 else if (lex_match_id_n (lexer, "DELCASE", 4))
445 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
447 lex_match (lexer, T_EQUALS);
448 if (lex_match_id (lexer, "LINE"))
449 data_parser_set_span (parser, false);
450 else if (lex_match_id (lexer, "VARIABLES"))
452 data_parser_set_span (parser, true);
454 /* VARIABLES takes an integer argument, but for no
455 good reason. We just ignore it. */
456 if (!lex_force_int (lexer))
462 lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL);
466 else if (lex_match_id (lexer, "FIXCASE"))
468 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
470 lex_match (lexer, T_EQUALS);
471 if (!lex_force_int (lexer))
473 if (lex_integer (lexer) < 1)
475 msg (SE, _("Value of %s must be 1 or greater."), "FIXCASE");
478 data_parser_set_records (parser, lex_integer (lexer));
481 else if (lex_match_id (lexer, "IMPORTCASES"))
483 lex_match (lexer, T_EQUALS);
484 if (lex_match (lexer, T_ALL))
486 data_parser_set_case_limit (parser, -1);
487 data_parser_set_case_percent (parser, 100);
489 else if (lex_match_id (lexer, "FIRST"))
491 if (!lex_force_int (lexer))
493 if (lex_integer (lexer) < 1)
495 msg (SE, _("Value of %s must be 1 or greater."), "FIRST");
498 data_parser_set_case_limit (parser, lex_integer (lexer));
501 else if (lex_match_id (lexer, "PERCENT"))
503 if (!lex_force_int (lexer))
505 if (lex_integer (lexer) < 1 || lex_integer (lexer) > 100)
507 msg (SE, _("Value of %s must be between 1 and 100."), "PERCENT");
510 data_parser_set_case_percent (parser, lex_integer (lexer));
514 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
516 struct string hard_seps = DS_EMPTY_INITIALIZER;
517 const char *soft_seps = "";
521 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
523 lex_match (lexer, T_EQUALS);
525 if (!lex_force_string (lexer))
528 /* XXX should support multibyte UTF-8 characters */
529 s = lex_tokss (lexer);
530 if (ss_match_string (&s, ss_cstr ("\\t")))
531 ds_put_cstr (&hard_seps, "\t");
532 if (ss_match_string (&s, ss_cstr ("\\\\")))
533 ds_put_cstr (&hard_seps, "\\");
534 while ((c = ss_get_byte (&s)) != EOF)
538 ds_put_byte (&hard_seps, c);
539 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
540 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
541 ds_destroy (&hard_seps);
545 else if (lex_match_id (lexer, "QUALIFIERS"))
547 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
549 lex_match (lexer, T_EQUALS);
551 if (!lex_force_string (lexer))
554 /* XXX should support multibyte UTF-8 characters */
555 if (settings_get_syntax () == COMPATIBLE
556 && ss_length (lex_tokss (lexer)) != 1)
558 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
559 "must contain exactly one character."));
563 data_parser_set_quotes (parser, lex_tokss (lexer));
566 else if (lex_match_id (lexer, "VARIABLES"))
570 lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL);
574 lex_match (lexer, T_EQUALS);
577 type = data_parser_get_type (parser);
580 struct fmt_spec input, output;
584 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
586 if (!lex_force_int (lexer))
588 if (lex_integer (lexer) < record)
590 msg (SE, _("The record number specified, %ld, is at or "
591 "before the previous record, %d. Data "
592 "fields must be listed in order of "
593 "increasing record number."),
594 lex_integer (lexer), record);
597 if (lex_integer (lexer) > data_parser_get_records (parser))
599 msg (SE, _("The record number specified, %ld, exceeds "
600 "the number of records per case specified "
602 lex_integer (lexer), data_parser_get_records (parser));
605 record = lex_integer (lexer);
609 if (!lex_force_id (lexer)
610 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
612 name = xstrdup (lex_tokcstr (lexer));
615 if (type == DP_DELIMITED)
617 if (!parse_format_specifier (lexer, &input)
618 || !fmt_check_input (&input))
621 output = fmt_for_output_from_input (&input);
625 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
626 enum fmt_type fmt_type;
629 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
632 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
633 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
635 if (!fmt_from_name (fmt_type_name, &fmt_type))
637 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
641 /* Compose input format. */
642 input.type = fmt_type;
643 input.w = lc - fc + 1;
645 if (!fmt_check_input (&input))
648 /* Compose output format. */
651 output.type = fmt_type;
654 if (!fmt_check_output (&output))
658 output = fmt_for_output_from_input (&input);
661 v = dict_create_var (dict, name, fmt_var_width (&input));
664 msg (SE, _("%s is a duplicate variable name."), name);
667 var_set_both_formats (v, &output);
669 if (type == DP_DELIMITED)
670 data_parser_add_delimited_field (parser, &input,
671 var_get_case_index (v),
674 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
679 while (lex_token (lexer) != T_ENDCMD);
681 reader = dfm_open_reader (fh, lexer, encoding);
685 data_parser_make_active_file (parser, ds, reader, dict);
691 data_parser_destroy (parser);
696 return CMD_CASCADING_FAILURE;
701 destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts)
703 free (opts->cell_range);
704 free (opts->sheet_name);