1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012,
3 2013 Free Software Foundation, Inc.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>. */
24 #include "data/dataset.h"
25 #include "data/dictionary.h"
26 #include "data/format.h"
27 #include "data/gnumeric-reader.h"
28 #include "data/ods-reader.h"
29 #include "data/spreadsheet-reader.h"
30 #include "data/psql-reader.h"
31 #include "data/settings.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/placement-parser.h"
37 #include "language/lexer/format-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "libpspp/cast.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
49 static bool parse_spreadsheet (struct lexer *lexer, char **filename,
50 struct spreadsheet_read_options *opts);
52 static void destroy_spreadsheet_read_info (struct spreadsheet_read_options *);
54 static int parse_get_txt (struct lexer *lexer, struct dataset *);
55 static int parse_get_psql (struct lexer *lexer, struct dataset *);
58 cmd_get_data (struct lexer *lexer, struct dataset *ds)
60 struct spreadsheet_read_options opts;
62 lex_force_match (lexer, T_SLASH);
64 if (!lex_force_match_id (lexer, "TYPE"))
67 lex_force_match (lexer, T_EQUALS);
69 tok = strdup (lex_tokcstr (lexer));
70 if (lex_match_id (lexer, "TXT"))
73 return parse_get_txt (lexer, ds);
75 else if (lex_match_id (lexer, "PSQL"))
78 return parse_get_psql (lexer, ds);
80 else if (lex_match_id (lexer, "GNM") ||
81 lex_match_id (lexer, "ODS"))
83 char *filename = NULL;
84 struct casereader *reader = NULL;
85 struct dictionary *dict = NULL;
87 if (!parse_spreadsheet (lexer, &filename, &opts))
90 if ( 0 == strncasecmp (tok, "GNM", 3))
92 struct spreadsheet *spreadsheet = gnumeric_probe (filename, true);
93 if (spreadsheet == NULL)
95 reader = gnumeric_make_reader (spreadsheet, &opts);
96 dict = spreadsheet->dict;
98 else if (0 == strncasecmp (tok, "ODS", 3))
100 struct spreadsheet *spreadsheet = ods_probe (filename, true);
101 if (spreadsheet == NULL)
103 reader = ods_make_reader (spreadsheet, &opts);
104 dict = spreadsheet->dict;
105 ods_destroy (spreadsheet);
112 dataset_set_dict (ds, dict);
113 dataset_set_source (ds, reader);
115 destroy_spreadsheet_read_info (&opts);
120 msg (SE, _("Unsupported TYPE %s."), tok);
124 destroy_spreadsheet_read_info (&opts);
130 parse_get_psql (struct lexer *lexer, struct dataset *ds)
132 struct psql_read_info psql;
133 psql.allow_clear = false;
134 psql.conninfo = NULL;
137 ds_init_empty (&psql.sql);
139 lex_force_match (lexer, T_SLASH);
141 if (!lex_force_match_id (lexer, "CONNECT"))
144 lex_force_match (lexer, T_EQUALS);
146 if (!lex_force_string (lexer))
149 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
153 while (lex_match (lexer, T_SLASH) )
155 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
157 lex_match (lexer, T_EQUALS);
158 psql.str_width = lex_integer (lexer);
161 else if ( lex_match_id (lexer, "BSIZE"))
163 lex_match (lexer, T_EQUALS);
164 psql.bsize = lex_integer (lexer);
167 else if ( lex_match_id (lexer, "UNENCRYPTED"))
169 psql.allow_clear = true;
171 else if (lex_match_id (lexer, "SQL"))
173 lex_match (lexer, T_EQUALS);
174 if ( ! lex_force_string (lexer) )
177 ds_put_substring (&psql.sql, lex_tokss (lexer));
182 struct dictionary *dict = NULL;
183 struct casereader *reader = psql_open_reader (&psql, &dict);
187 dataset_set_dict (ds, dict);
188 dataset_set_source (ds, reader);
192 ds_destroy (&psql.sql);
193 free (psql.conninfo);
199 ds_destroy (&psql.sql);
200 free (psql.conninfo);
206 parse_spreadsheet (struct lexer *lexer, char **filename,
207 struct spreadsheet_read_options *opts)
209 opts->sheet_index = 1;
210 opts->sheet_name = NULL;
211 opts->cell_range = NULL;
212 opts->read_names = true;
215 lex_force_match (lexer, T_SLASH);
217 if (!lex_force_match_id (lexer, "FILE"))
220 lex_force_match (lexer, T_EQUALS);
222 if (!lex_force_string (lexer))
225 *filename = utf8_to_filename (lex_tokcstr (lexer));
229 while (lex_match (lexer, T_SLASH) )
231 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
233 lex_match (lexer, T_EQUALS);
234 opts->asw = lex_integer (lexer);
237 else if (lex_match_id (lexer, "SHEET"))
239 lex_match (lexer, T_EQUALS);
240 if (lex_match_id (lexer, "NAME"))
242 if ( ! lex_force_string (lexer) )
245 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
246 opts->sheet_index = -1;
250 else if (lex_match_id (lexer, "INDEX"))
252 opts->sheet_index = lex_integer (lexer);
253 if (opts->sheet_index <= 0)
255 msg (SE, _("The sheet index must be greater than or equal to 1"));
262 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
263 "/SHEET", "NAME", "INDEX");
267 else if (lex_match_id (lexer, "CELLRANGE"))
269 lex_match (lexer, T_EQUALS);
271 if (lex_match_id (lexer, "FULL"))
273 opts->cell_range = NULL;
275 else if (lex_match_id (lexer, "RANGE"))
277 if ( ! lex_force_string (lexer) )
280 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
285 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
286 "/CELLRANGE", "FULL", "RANGE");
290 else if (lex_match_id (lexer, "READNAMES"))
292 lex_match (lexer, T_EQUALS);
294 if ( lex_match_id (lexer, "ON"))
296 opts->read_names = true;
298 else if (lex_match_id (lexer, "OFF"))
300 opts->read_names = false;
304 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
305 "/READNAMES", "ON", "OFF");
311 lex_error (lexer, NULL);
324 set_type (struct data_parser *parser, const char *subcommand,
325 enum data_parser_type type, bool *has_type)
329 data_parser_set_type (parser, type);
332 else if (type != data_parser_get_type (parser))
334 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
335 "was stated or implied earlier in this command."),
337 type == DP_FIXED ? "FIXED" : "DELIMITED",
338 type == DP_FIXED ? "DELIMITED" : "FIXED");
345 parse_get_txt (struct lexer *lexer, struct dataset *ds)
347 struct data_parser *parser = NULL;
348 struct dictionary *dict = dict_create (get_default_encoding ());
349 struct file_handle *fh = NULL;
350 struct dfm_reader *reader = NULL;
351 char *encoding = NULL;
355 enum data_parser_type type;
358 lex_force_match (lexer, T_SLASH);
360 if (!lex_force_match_id (lexer, "FILE"))
362 lex_force_match (lexer, T_EQUALS);
363 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
367 parser = data_parser_create (dict);
369 data_parser_set_type (parser, DP_DELIMITED);
370 data_parser_set_span (parser, false);
371 data_parser_set_quotes (parser, ss_empty ());
372 data_parser_set_empty_line_has_field (parser, true);
376 if (!lex_force_match (lexer, T_SLASH))
379 if (lex_match_id (lexer, "ENCODING"))
381 lex_match (lexer, T_EQUALS);
382 if (!lex_force_string (lexer))
386 encoding = ss_xstrdup (lex_tokss (lexer));
390 else if (lex_match_id (lexer, "ARRANGEMENT"))
394 lex_match (lexer, T_EQUALS);
395 if (lex_match_id (lexer, "FIXED"))
396 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
397 else if (lex_match_id (lexer, "DELIMITED"))
398 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
399 DP_DELIMITED, &has_type);
402 lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL);
408 else if (lex_match_id (lexer, "FIRSTCASE"))
410 lex_match (lexer, T_EQUALS);
411 if (!lex_force_int (lexer))
413 if (lex_integer (lexer) < 1)
415 msg (SE, _("Value of FIRSTCASE must be 1 or greater."));
418 data_parser_set_skip (parser, lex_integer (lexer) - 1);
421 else if (lex_match_id_n (lexer, "DELCASE", 4))
423 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
425 lex_match (lexer, T_EQUALS);
426 if (lex_match_id (lexer, "LINE"))
427 data_parser_set_span (parser, false);
428 else if (lex_match_id (lexer, "VARIABLES"))
430 data_parser_set_span (parser, true);
432 /* VARIABLES takes an integer argument, but for no
433 good reason. We just ignore it. */
434 if (!lex_force_int (lexer))
440 lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL);
444 else if (lex_match_id (lexer, "FIXCASE"))
446 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
448 lex_match (lexer, T_EQUALS);
449 if (!lex_force_int (lexer))
451 if (lex_integer (lexer) < 1)
453 msg (SE, _("Value of FIXCASE must be at least 1."));
456 data_parser_set_records (parser, lex_integer (lexer));
459 else if (lex_match_id (lexer, "IMPORTCASES"))
461 lex_match (lexer, T_EQUALS);
462 if (lex_match (lexer, T_ALL))
464 data_parser_set_case_limit (parser, -1);
465 data_parser_set_case_percent (parser, 100);
467 else if (lex_match_id (lexer, "FIRST"))
469 if (!lex_force_int (lexer))
471 if (lex_integer (lexer) < 1)
473 msg (SE, _("Value of FIRST must be at least 1."));
476 data_parser_set_case_limit (parser, lex_integer (lexer));
479 else if (lex_match_id (lexer, "PERCENT"))
481 if (!lex_force_int (lexer))
483 if (lex_integer (lexer) < 1 || lex_integer (lexer) > 100)
485 msg (SE, _("Value of PERCENT must be between 1 and 100."));
488 data_parser_set_case_percent (parser, lex_integer (lexer));
492 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
494 struct string hard_seps = DS_EMPTY_INITIALIZER;
495 const char *soft_seps = "";
499 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
501 lex_match (lexer, T_EQUALS);
503 if (!lex_force_string (lexer))
506 /* XXX should support multibyte UTF-8 characters */
507 s = lex_tokss (lexer);
508 if (ss_match_string (&s, ss_cstr ("\\t")))
509 ds_put_cstr (&hard_seps, "\t");
510 if (ss_match_string (&s, ss_cstr ("\\\\")))
511 ds_put_cstr (&hard_seps, "\\");
512 while ((c = ss_get_byte (&s)) != EOF)
516 ds_put_byte (&hard_seps, c);
517 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
518 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
519 ds_destroy (&hard_seps);
523 else if (lex_match_id (lexer, "QUALIFIERS"))
525 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
527 lex_match (lexer, T_EQUALS);
529 if (!lex_force_string (lexer))
532 /* XXX should support multibyte UTF-8 characters */
533 if (settings_get_syntax () == COMPATIBLE
534 && ss_length (lex_tokss (lexer)) != 1)
536 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
537 "must contain exactly one character."));
541 data_parser_set_quotes (parser, lex_tokss (lexer));
544 else if (settings_get_syntax () == ENHANCED
545 && lex_match_id (lexer, "ESCAPE"))
546 data_parser_set_quote_escape (parser, true);
547 else if (lex_match_id (lexer, "VARIABLES"))
551 lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL);
555 lex_match (lexer, T_EQUALS);
558 type = data_parser_get_type (parser);
561 struct fmt_spec input, output;
565 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
567 if (!lex_force_int (lexer))
569 if (lex_integer (lexer) < record)
571 msg (SE, _("The record number specified, %ld, is at or "
572 "before the previous record, %d. Data "
573 "fields must be listed in order of "
574 "increasing record number."),
575 lex_integer (lexer), record);
578 if (lex_integer (lexer) > data_parser_get_records (parser))
580 msg (SE, _("The record number specified, %ld, exceeds "
581 "the number of records per case specified "
583 lex_integer (lexer), data_parser_get_records (parser));
586 record = lex_integer (lexer);
590 if (!lex_force_id (lexer)
591 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
593 name = xstrdup (lex_tokcstr (lexer));
596 if (type == DP_DELIMITED)
598 if (!parse_format_specifier (lexer, &input)
599 || !fmt_check_input (&input))
602 output = fmt_for_output_from_input (&input);
606 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
607 enum fmt_type fmt_type;
610 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
613 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
614 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
616 if (!fmt_from_name (fmt_type_name, &fmt_type))
618 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
622 /* Compose input format. */
623 input.type = fmt_type;
624 input.w = lc - fc + 1;
626 if (!fmt_check_input (&input))
629 /* Compose output format. */
632 output.type = fmt_type;
635 if (!fmt_check_output (&output))
639 output = fmt_for_output_from_input (&input);
642 v = dict_create_var (dict, name, fmt_var_width (&input));
645 msg (SE, _("%s is a duplicate variable name."), name);
648 var_set_both_formats (v, &output);
650 if (type == DP_DELIMITED)
651 data_parser_add_delimited_field (parser, &input,
652 var_get_case_index (v),
655 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
660 while (lex_token (lexer) != T_ENDCMD);
662 reader = dfm_open_reader (fh, lexer, encoding);
666 data_parser_make_active_file (parser, ds, reader, dict);
672 data_parser_destroy (parser);
677 return CMD_CASCADING_FAILURE;
682 destroy_spreadsheet_read_info (struct spreadsheet_read_options *opts)
684 free (opts->cell_range);
685 free (opts->sheet_name);