1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
23 #include "data/dataset.h"
24 #include "data/dictionary.h"
25 #include "data/format.h"
26 #include "data/gnumeric-reader.h"
27 #include "data/ods-reader.h"
28 #include "data/spreadsheet-reader.h"
29 #include "data/psql-reader.h"
30 #include "data/settings.h"
31 #include "language/command.h"
32 #include "language/data-io/data-parser.h"
33 #include "language/data-io/data-reader.h"
34 #include "language/data-io/file-handle.h"
35 #include "language/data-io/placement-parser.h"
36 #include "language/lexer/format-parser.h"
37 #include "language/lexer/lexer.h"
38 #include "libpspp/cast.h"
39 #include "libpspp/i18n.h"
40 #include "libpspp/message.h"
42 #include "gl/xalloc.h"
45 #define _(msgid) gettext (msgid)
46 #define N_(msgid) (msgid)
48 static bool parse_spreadsheet (struct lexer *lexer, struct spreadsheet_read_info *sri,
49 struct spreadsheet_read_options *opts);
51 static void destroy_spreadsheet_read_info (struct spreadsheet_read_info *, struct spreadsheet_read_options *);
53 static int parse_get_txt (struct lexer *lexer, struct dataset *);
54 static int parse_get_psql (struct lexer *lexer, struct dataset *);
57 cmd_get_data (struct lexer *lexer, struct dataset *ds)
60 lex_force_match (lexer, T_SLASH);
62 if (!lex_force_match_id (lexer, "TYPE"))
65 lex_force_match (lexer, T_EQUALS);
67 tok = strdup (lex_tokcstr (lexer));
68 if (lex_match_id (lexer, "TXT"))
71 return parse_get_txt (lexer, ds);
73 else if (lex_match_id (lexer, "PSQL"))
76 return parse_get_psql (lexer, ds);
78 else if (lex_match_id (lexer, "GNM") ||
79 lex_match_id (lexer, "ODS"))
81 struct casereader *reader = NULL;
82 struct dictionary *dict = NULL;
83 struct spreadsheet_read_info sri;
84 struct spreadsheet_read_options opts;
85 if (!parse_spreadsheet (lexer, &sri, &opts))
88 if ( 0 == strncasecmp (tok, "GNM", 3))
90 struct spreadsheet *spreadsheet = gnumeric_probe (sri.file_name);
91 reader = gnumeric_make_reader (spreadsheet, &sri, &opts);
92 dict = spreadsheet->dict;
94 else if (0 == strncasecmp (tok, "ODS", 3))
96 struct spreadsheet *spreadsheet = ods_probe (sri.file_name);
97 reader = ods_make_reader (spreadsheet, &sri, &opts);
98 dict = spreadsheet->dict;
103 dataset_set_dict (ds, dict);
104 dataset_set_source (ds, reader);
105 destroy_spreadsheet_read_info (&sri, &opts);
109 destroy_spreadsheet_read_info (&sri, &opts);
112 msg (SE, _("Unsupported TYPE %s."), tok);
120 parse_get_psql (struct lexer *lexer, struct dataset *ds)
122 struct psql_read_info psql;
123 psql.allow_clear = false;
124 psql.conninfo = NULL;
127 ds_init_empty (&psql.sql);
129 lex_force_match (lexer, T_SLASH);
131 if (!lex_force_match_id (lexer, "CONNECT"))
134 lex_force_match (lexer, T_EQUALS);
136 if (!lex_force_string (lexer))
139 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
143 while (lex_match (lexer, T_SLASH) )
145 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
147 lex_match (lexer, T_EQUALS);
148 psql.str_width = lex_integer (lexer);
151 else if ( lex_match_id (lexer, "BSIZE"))
153 lex_match (lexer, T_EQUALS);
154 psql.bsize = lex_integer (lexer);
157 else if ( lex_match_id (lexer, "UNENCRYPTED"))
159 psql.allow_clear = true;
161 else if (lex_match_id (lexer, "SQL"))
163 lex_match (lexer, T_EQUALS);
164 if ( ! lex_force_string (lexer) )
167 ds_put_substring (&psql.sql, lex_tokss (lexer));
172 struct dictionary *dict = NULL;
173 struct casereader *reader = psql_open_reader (&psql, &dict);
177 dataset_set_dict (ds, dict);
178 dataset_set_source (ds, reader);
182 ds_destroy (&psql.sql);
183 free (psql.conninfo);
189 ds_destroy (&psql.sql);
190 free (psql.conninfo);
196 parse_spreadsheet (struct lexer *lexer, struct spreadsheet_read_info *sri,
197 struct spreadsheet_read_options *opts)
199 opts->sheet_index = 1;
200 opts->sheet_name = NULL;
201 opts->cell_range = NULL;
202 sri->read_names = true;
204 sri->file_name = NULL;
206 lex_force_match (lexer, T_SLASH);
208 if (!lex_force_match_id (lexer, "FILE"))
211 lex_force_match (lexer, T_EQUALS);
213 if (!lex_force_string (lexer))
216 sri->file_name = utf8_to_filename (lex_tokcstr (lexer));
220 while (lex_match (lexer, T_SLASH) )
222 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
224 lex_match (lexer, T_EQUALS);
225 sri->asw = lex_integer (lexer);
228 else if (lex_match_id (lexer, "SHEET"))
230 lex_match (lexer, T_EQUALS);
231 if (lex_match_id (lexer, "NAME"))
233 if ( ! lex_force_string (lexer) )
236 opts->sheet_name = ss_xstrdup (lex_tokss (lexer));
237 opts->sheet_index = -1;
241 else if (lex_match_id (lexer, "INDEX"))
243 opts->sheet_index = lex_integer (lexer);
244 if (opts->sheet_index <= 0)
246 msg (SE, _("The sheet index must be greater than or equal to 1"));
253 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
254 "/SHEET", "NAME", "INDEX");
258 else if (lex_match_id (lexer, "CELLRANGE"))
260 lex_match (lexer, T_EQUALS);
262 if (lex_match_id (lexer, "FULL"))
264 opts->cell_range = NULL;
266 else if (lex_match_id (lexer, "RANGE"))
268 if ( ! lex_force_string (lexer) )
271 opts->cell_range = ss_xstrdup (lex_tokss (lexer));
276 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
277 "/CELLRANGE", "FULL", "RANGE");
281 else if (lex_match_id (lexer, "READNAMES"))
283 lex_match (lexer, T_EQUALS);
285 if ( lex_match_id (lexer, "ON"))
287 sri->read_names = true;
289 else if (lex_match_id (lexer, "OFF"))
291 sri->read_names = false;
295 msg (SE, _("%s must be followed by either \"%s\" or \"%s\"."),
296 "/READNAMES", "ON", "OFF");
302 lex_error (lexer, NULL);
310 destroy_spreadsheet_read_info (sri, opts);
316 set_type (struct data_parser *parser, const char *subcommand,
317 enum data_parser_type type, bool *has_type)
321 data_parser_set_type (parser, type);
324 else if (type != data_parser_get_type (parser))
326 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
327 "was stated or implied earlier in this command."),
329 type == DP_FIXED ? "FIXED" : "DELIMITED",
330 type == DP_FIXED ? "DELIMITED" : "FIXED");
337 parse_get_txt (struct lexer *lexer, struct dataset *ds)
339 struct data_parser *parser = NULL;
340 struct dictionary *dict = dict_create (get_default_encoding ());
341 struct file_handle *fh = NULL;
342 struct dfm_reader *reader = NULL;
343 char *encoding = NULL;
347 enum data_parser_type type;
350 lex_force_match (lexer, T_SLASH);
352 if (!lex_force_match_id (lexer, "FILE"))
354 lex_force_match (lexer, T_EQUALS);
355 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
359 parser = data_parser_create (dict);
361 data_parser_set_type (parser, DP_DELIMITED);
362 data_parser_set_span (parser, false);
363 data_parser_set_quotes (parser, ss_empty ());
364 data_parser_set_empty_line_has_field (parser, true);
368 if (!lex_force_match (lexer, T_SLASH))
371 if (lex_match_id (lexer, "ENCODING"))
373 lex_match (lexer, T_EQUALS);
374 if (!lex_force_string (lexer))
378 encoding = ss_xstrdup (lex_tokss (lexer));
382 else if (lex_match_id (lexer, "ARRANGEMENT"))
386 lex_match (lexer, T_EQUALS);
387 if (lex_match_id (lexer, "FIXED"))
388 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
389 else if (lex_match_id (lexer, "DELIMITED"))
390 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
391 DP_DELIMITED, &has_type);
394 lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL);
400 else if (lex_match_id (lexer, "FIRSTCASE"))
402 lex_match (lexer, T_EQUALS);
403 if (!lex_force_int (lexer))
405 if (lex_integer (lexer) < 1)
407 msg (SE, _("Value of FIRSTCASE must be 1 or greater."));
410 data_parser_set_skip (parser, lex_integer (lexer) - 1);
413 else if (lex_match_id_n (lexer, "DELCASE", 4))
415 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
417 lex_match (lexer, T_EQUALS);
418 if (lex_match_id (lexer, "LINE"))
419 data_parser_set_span (parser, false);
420 else if (lex_match_id (lexer, "VARIABLES"))
422 data_parser_set_span (parser, true);
424 /* VARIABLES takes an integer argument, but for no
425 good reason. We just ignore it. */
426 if (!lex_force_int (lexer))
432 lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL);
436 else if (lex_match_id (lexer, "FIXCASE"))
438 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
440 lex_match (lexer, T_EQUALS);
441 if (!lex_force_int (lexer))
443 if (lex_integer (lexer) < 1)
445 msg (SE, _("Value of FIXCASE must be at least 1."));
448 data_parser_set_records (parser, lex_integer (lexer));
451 else if (lex_match_id (lexer, "IMPORTCASES"))
453 lex_match (lexer, T_EQUALS);
454 if (lex_match (lexer, T_ALL))
456 data_parser_set_case_limit (parser, -1);
457 data_parser_set_case_percent (parser, 100);
459 else if (lex_match_id (lexer, "FIRST"))
461 if (!lex_force_int (lexer))
463 if (lex_integer (lexer) < 1)
465 msg (SE, _("Value of FIRST must be at least 1."));
468 data_parser_set_case_limit (parser, lex_integer (lexer));
471 else if (lex_match_id (lexer, "PERCENT"))
473 if (!lex_force_int (lexer))
475 if (lex_integer (lexer) < 1 || lex_integer (lexer) > 100)
477 msg (SE, _("Value of PERCENT must be between 1 and 100."));
480 data_parser_set_case_percent (parser, lex_integer (lexer));
484 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
486 struct string hard_seps = DS_EMPTY_INITIALIZER;
487 const char *soft_seps = "";
491 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
493 lex_match (lexer, T_EQUALS);
495 if (!lex_force_string (lexer))
498 /* XXX should support multibyte UTF-8 characters */
499 s = lex_tokss (lexer);
500 if (ss_match_string (&s, ss_cstr ("\\t")))
501 ds_put_cstr (&hard_seps, "\t");
502 if (ss_match_string (&s, ss_cstr ("\\\\")))
503 ds_put_cstr (&hard_seps, "\\");
504 while ((c = ss_get_byte (&s)) != EOF)
508 ds_put_byte (&hard_seps, c);
509 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
510 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
511 ds_destroy (&hard_seps);
515 else if (lex_match_id (lexer, "QUALIFIERS"))
517 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
519 lex_match (lexer, T_EQUALS);
521 if (!lex_force_string (lexer))
524 /* XXX should support multibyte UTF-8 characters */
525 if (settings_get_syntax () == COMPATIBLE
526 && ss_length (lex_tokss (lexer)) != 1)
528 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
529 "must contain exactly one character."));
533 data_parser_set_quotes (parser, lex_tokss (lexer));
536 else if (settings_get_syntax () == ENHANCED
537 && lex_match_id (lexer, "ESCAPE"))
538 data_parser_set_quote_escape (parser, true);
539 else if (lex_match_id (lexer, "VARIABLES"))
543 lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL);
547 lex_match (lexer, T_EQUALS);
550 type = data_parser_get_type (parser);
553 struct fmt_spec input, output;
557 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
559 if (!lex_force_int (lexer))
561 if (lex_integer (lexer) < record)
563 msg (SE, _("The record number specified, %ld, is at or "
564 "before the previous record, %d. Data "
565 "fields must be listed in order of "
566 "increasing record number."),
567 lex_integer (lexer), record);
570 if (lex_integer (lexer) > data_parser_get_records (parser))
572 msg (SE, _("The record number specified, %ld, exceeds "
573 "the number of records per case specified "
575 lex_integer (lexer), data_parser_get_records (parser));
578 record = lex_integer (lexer);
582 if (!lex_force_id (lexer)
583 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
585 name = xstrdup (lex_tokcstr (lexer));
588 if (type == DP_DELIMITED)
590 if (!parse_format_specifier (lexer, &input)
591 || !fmt_check_input (&input))
594 output = fmt_for_output_from_input (&input);
598 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
599 enum fmt_type fmt_type;
602 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
605 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
606 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
608 if (!fmt_from_name (fmt_type_name, &fmt_type))
610 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
614 /* Compose input format. */
615 input.type = fmt_type;
616 input.w = lc - fc + 1;
618 if (!fmt_check_input (&input))
621 /* Compose output format. */
624 output.type = fmt_type;
627 if (!fmt_check_output (&output))
631 output = fmt_for_output_from_input (&input);
634 v = dict_create_var (dict, name, fmt_var_width (&input));
637 msg (SE, _("%s is a duplicate variable name."), name);
640 var_set_both_formats (v, &output);
642 if (type == DP_DELIMITED)
643 data_parser_add_delimited_field (parser, &input,
644 var_get_case_index (v),
647 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
652 while (lex_token (lexer) != T_ENDCMD);
654 reader = dfm_open_reader (fh, lexer, encoding);
658 data_parser_make_active_file (parser, ds, reader, dict);
664 data_parser_destroy (parser);
669 return CMD_CASCADING_FAILURE;
674 destroy_spreadsheet_read_info (struct spreadsheet_read_info *sri,
675 struct spreadsheet_read_options *opts)
677 free (opts->sheet_name);
678 free (opts->cell_range);
679 free (sri->file_name);