1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
23 #include "data/dataset.h"
24 #include "data/dictionary.h"
25 #include "data/format.h"
26 #include "data/gnumeric-reader.h"
27 #include "data/ods-reader.h"
28 #include "data/spreadsheet-reader.h"
29 #include "data/psql-reader.h"
30 #include "data/settings.h"
31 #include "language/command.h"
32 #include "language/data-io/data-parser.h"
33 #include "language/data-io/data-reader.h"
34 #include "language/data-io/file-handle.h"
35 #include "language/data-io/placement-parser.h"
36 #include "language/lexer/format-parser.h"
37 #include "language/lexer/lexer.h"
38 #include "libpspp/cast.h"
39 #include "libpspp/i18n.h"
40 #include "libpspp/message.h"
42 #include "gl/xalloc.h"
45 #define _(msgid) gettext (msgid)
46 #define N_(msgid) (msgid)
48 static struct spreadsheet_read_info *parse_spreadsheet (struct lexer *lexer);
49 static void destroy_spreadsheet_read_info (struct spreadsheet_read_info *);
51 static int parse_get_txt (struct lexer *lexer, struct dataset *);
52 static int parse_get_psql (struct lexer *lexer, struct dataset *);
55 cmd_get_data (struct lexer *lexer, struct dataset *ds)
58 lex_force_match (lexer, T_SLASH);
60 if (!lex_force_match_id (lexer, "TYPE"))
63 lex_force_match (lexer, T_EQUALS);
65 tok = strdup (lex_tokcstr (lexer));
66 if (lex_match_id (lexer, "TXT"))
68 return parse_get_txt (lexer, ds);
70 else if (lex_match_id (lexer, "PSQL"))
72 return parse_get_psql (lexer, ds);
74 else if (lex_match_id (lexer, "GNM") ||
75 lex_match_id (lexer, "ODS"))
77 struct casereader *reader = NULL;
78 struct dictionary *dict = NULL;
79 struct spreadsheet_read_info *sri = parse_spreadsheet (lexer);
83 if ( 0 == strncasecmp (tok, "GNM", 3))
84 reader = gnumeric_open_reader (sri, &dict);
85 else if (0 == strncasecmp (tok, "ODS", 3))
86 reader = ods_open_reader (sri, &dict);
90 dataset_set_dict (ds, dict);
91 dataset_set_source (ds, reader);
92 destroy_spreadsheet_read_info (sri);
96 destroy_spreadsheet_read_info (sri);
99 msg (SE, _("Unsupported TYPE %s."), tok);
107 parse_get_psql (struct lexer *lexer, struct dataset *ds)
109 struct psql_read_info psql;
110 psql.allow_clear = false;
111 psql.conninfo = NULL;
114 ds_init_empty (&psql.sql);
116 lex_force_match (lexer, T_SLASH);
118 if (!lex_force_match_id (lexer, "CONNECT"))
121 lex_force_match (lexer, T_EQUALS);
123 if (!lex_force_string (lexer))
126 psql.conninfo = ss_xstrdup (lex_tokss (lexer));
130 while (lex_match (lexer, T_SLASH) )
132 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
134 lex_match (lexer, T_EQUALS);
135 psql.str_width = lex_integer (lexer);
138 else if ( lex_match_id (lexer, "BSIZE"))
140 lex_match (lexer, T_EQUALS);
141 psql.bsize = lex_integer (lexer);
144 else if ( lex_match_id (lexer, "UNENCRYPTED"))
146 psql.allow_clear = true;
148 else if (lex_match_id (lexer, "SQL"))
150 lex_match (lexer, T_EQUALS);
151 if ( ! lex_force_string (lexer) )
154 ds_put_substring (&psql.sql, lex_tokss (lexer));
159 struct dictionary *dict = NULL;
160 struct casereader *reader = psql_open_reader (&psql, &dict);
164 dataset_set_dict (ds, dict);
165 dataset_set_source (ds, reader);
169 ds_destroy (&psql.sql);
170 free (psql.conninfo);
176 ds_destroy (&psql.sql);
177 free (psql.conninfo);
182 static struct spreadsheet_read_info *
183 parse_spreadsheet (struct lexer *lexer)
185 struct spreadsheet_read_info *sri = xzalloc (sizeof *sri);
186 sri->sheet_index = 1;
187 sri->read_names = true;
190 lex_force_match (lexer, T_SLASH);
192 if (!lex_force_match_id (lexer, "FILE"))
195 lex_force_match (lexer, T_EQUALS);
197 if (!lex_force_string (lexer))
200 sri->file_name = utf8_to_filename (lex_tokcstr (lexer));
204 while (lex_match (lexer, T_SLASH) )
206 if ( lex_match_id (lexer, "ASSUMEDSTRWIDTH"))
208 lex_match (lexer, T_EQUALS);
209 sri->asw = lex_integer (lexer);
212 else if (lex_match_id (lexer, "SHEET"))
214 lex_match (lexer, T_EQUALS);
215 if (lex_match_id (lexer, "NAME"))
217 if ( ! lex_force_string (lexer) )
220 sri->sheet_name = ss_xstrdup (lex_tokss (lexer));
221 sri->sheet_index = -1;
225 else if (lex_match_id (lexer, "INDEX"))
227 sri->sheet_index = lex_integer (lexer);
233 else if (lex_match_id (lexer, "CELLRANGE"))
235 lex_match (lexer, T_EQUALS);
237 if (lex_match_id (lexer, "FULL"))
239 sri->cell_range = NULL;
241 else if (lex_match_id (lexer, "RANGE"))
243 if ( ! lex_force_string (lexer) )
246 sri->cell_range = ss_xstrdup (lex_tokss (lexer));
252 else if (lex_match_id (lexer, "READNAMES"))
254 lex_match (lexer, T_EQUALS);
256 if ( lex_match_id (lexer, "ON"))
258 sri->read_names = true;
260 else if (lex_match_id (lexer, "OFF"))
262 sri->read_names = false;
269 lex_error (lexer, NULL);
277 destroy_spreadsheet_read_info (sri);
283 set_type (struct data_parser *parser, const char *subcommand,
284 enum data_parser_type type, bool *has_type)
288 data_parser_set_type (parser, type);
291 else if (type != data_parser_get_type (parser))
293 msg (SE, _("%s is allowed only with %s arrangement, but %s arrangement "
294 "was stated or implied earlier in this command."),
296 type == DP_FIXED ? "FIXED" : "DELIMITED",
297 type == DP_FIXED ? "DELIMITED" : "FIXED");
304 parse_get_txt (struct lexer *lexer, struct dataset *ds)
306 struct data_parser *parser = NULL;
307 struct dictionary *dict = dict_create (get_default_encoding ());
308 struct file_handle *fh = NULL;
309 struct dfm_reader *reader = NULL;
313 enum data_parser_type type;
316 lex_force_match (lexer, T_SLASH);
318 if (!lex_force_match_id (lexer, "FILE"))
320 lex_force_match (lexer, T_EQUALS);
321 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
325 parser = data_parser_create (dict);
327 data_parser_set_type (parser, DP_DELIMITED);
328 data_parser_set_span (parser, false);
329 data_parser_set_quotes (parser, ss_empty ());
330 data_parser_set_empty_line_has_field (parser, true);
334 if (!lex_force_match (lexer, T_SLASH))
337 if (lex_match_id (lexer, "ARRANGEMENT"))
341 lex_match (lexer, T_EQUALS);
342 if (lex_match_id (lexer, "FIXED"))
343 ok = set_type (parser, "ARRANGEMENT=FIXED", DP_FIXED, &has_type);
344 else if (lex_match_id (lexer, "DELIMITED"))
345 ok = set_type (parser, "ARRANGEMENT=DELIMITED",
346 DP_DELIMITED, &has_type);
349 lex_error_expecting (lexer, "FIXED", "DELIMITED", NULL_SENTINEL);
355 else if (lex_match_id (lexer, "FIRSTCASE"))
357 lex_match (lexer, T_EQUALS);
358 if (!lex_force_int (lexer))
360 if (lex_integer (lexer) < 1)
362 msg (SE, _("Value of FIRSTCASE must be 1 or greater."));
365 data_parser_set_skip (parser, lex_integer (lexer) - 1);
368 else if (lex_match_id_n (lexer, "DELCASE", 4))
370 if (!set_type (parser, "DELCASE", DP_DELIMITED, &has_type))
372 lex_match (lexer, T_EQUALS);
373 if (lex_match_id (lexer, "LINE"))
374 data_parser_set_span (parser, false);
375 else if (lex_match_id (lexer, "VARIABLES"))
377 data_parser_set_span (parser, true);
379 /* VARIABLES takes an integer argument, but for no
380 good reason. We just ignore it. */
381 if (!lex_force_int (lexer))
387 lex_error_expecting (lexer, "LINE", "VARIABLES", NULL_SENTINEL);
391 else if (lex_match_id (lexer, "FIXCASE"))
393 if (!set_type (parser, "FIXCASE", DP_FIXED, &has_type))
395 lex_match (lexer, T_EQUALS);
396 if (!lex_force_int (lexer))
398 if (lex_integer (lexer) < 1)
400 msg (SE, _("Value of FIXCASE must be at least 1."));
403 data_parser_set_records (parser, lex_integer (lexer));
406 else if (lex_match_id (lexer, "IMPORTCASES"))
408 lex_match (lexer, T_EQUALS);
409 if (lex_match (lexer, T_ALL))
411 data_parser_set_case_limit (parser, -1);
412 data_parser_set_case_percent (parser, 100);
414 else if (lex_match_id (lexer, "FIRST"))
416 if (!lex_force_int (lexer))
418 if (lex_integer (lexer) < 1)
420 msg (SE, _("Value of FIRST must be at least 1."));
423 data_parser_set_case_limit (parser, lex_integer (lexer));
426 else if (lex_match_id (lexer, "PERCENT"))
428 if (!lex_force_int (lexer))
430 if (lex_integer (lexer) < 1 || lex_integer (lexer) > 100)
432 msg (SE, _("Value of PERCENT must be between 1 and 100."));
435 data_parser_set_case_percent (parser, lex_integer (lexer));
439 else if (lex_match_id_n (lexer, "DELIMITERS", 4))
441 struct string hard_seps = DS_EMPTY_INITIALIZER;
442 const char *soft_seps = "";
446 if (!set_type (parser, "DELIMITERS", DP_DELIMITED, &has_type))
448 lex_match (lexer, T_EQUALS);
450 if (!lex_force_string (lexer))
453 /* XXX should support multibyte UTF-8 characters */
454 s = lex_tokss (lexer);
455 if (ss_match_string (&s, ss_cstr ("\\t")))
456 ds_put_cstr (&hard_seps, "\t");
457 if (ss_match_string (&s, ss_cstr ("\\\\")))
458 ds_put_cstr (&hard_seps, "\\");
459 while ((c = ss_get_byte (&s)) != EOF)
463 ds_put_byte (&hard_seps, c);
464 data_parser_set_soft_delimiters (parser, ss_cstr (soft_seps));
465 data_parser_set_hard_delimiters (parser, ds_ss (&hard_seps));
466 ds_destroy (&hard_seps);
470 else if (lex_match_id (lexer, "QUALIFIERS"))
472 if (!set_type (parser, "QUALIFIERS", DP_DELIMITED, &has_type))
474 lex_match (lexer, T_EQUALS);
476 if (!lex_force_string (lexer))
479 /* XXX should support multibyte UTF-8 characters */
480 if (settings_get_syntax () == COMPATIBLE
481 && ss_length (lex_tokss (lexer)) != 1)
483 msg (SE, _("In compatible syntax mode, the QUALIFIER string "
484 "must contain exactly one character."));
488 data_parser_set_quotes (parser, lex_tokss (lexer));
491 else if (settings_get_syntax () == ENHANCED
492 && lex_match_id (lexer, "ESCAPE"))
493 data_parser_set_quote_escape (parser, true);
494 else if (lex_match_id (lexer, "VARIABLES"))
498 lex_error_expecting (lexer, "VARIABLES", NULL_SENTINEL);
502 lex_match (lexer, T_EQUALS);
505 type = data_parser_get_type (parser);
508 struct fmt_spec input, output;
512 while (type == DP_FIXED && lex_match (lexer, T_SLASH))
514 if (!lex_force_int (lexer))
516 if (lex_integer (lexer) < record)
518 msg (SE, _("The record number specified, %ld, is at or "
519 "before the previous record, %d. Data "
520 "fields must be listed in order of "
521 "increasing record number."),
522 lex_integer (lexer), record);
525 if (lex_integer (lexer) > data_parser_get_records (parser))
527 msg (SE, _("The record number specified, %ld, exceeds "
528 "the number of records per case specified "
530 lex_integer (lexer), data_parser_get_records (parser));
533 record = lex_integer (lexer);
537 if (!lex_force_id (lexer)
538 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
540 name = xstrdup (lex_tokcstr (lexer));
543 if (type == DP_DELIMITED)
545 if (!parse_format_specifier (lexer, &input)
546 || !fmt_check_input (&input))
549 output = fmt_for_output_from_input (&input);
553 char fmt_type_name[FMT_TYPE_LEN_MAX + 1];
554 enum fmt_type fmt_type;
557 if (!parse_column_range (lexer, 0, &fc, &lc, NULL))
560 /* Accept a format (e.g. F8.2) or just a type name (e.g. DOLLAR). */
561 if (!parse_abstract_format_specifier (lexer, fmt_type_name, &w, &d))
563 if (!fmt_from_name (fmt_type_name, &fmt_type))
565 msg (SE, _("Unknown format type `%s'."), fmt_type_name);
569 /* Compose input format. */
570 input.type = fmt_type;
571 input.w = lc - fc + 1;
573 if (!fmt_check_input (&input))
576 /* Compose output format. */
579 output.type = fmt_type;
582 if (!fmt_check_output (&output))
586 output = fmt_for_output_from_input (&input);
589 v = dict_create_var (dict, name, fmt_var_width (&input));
592 msg (SE, _("%s is a duplicate variable name."), name);
595 var_set_both_formats (v, &output);
597 if (type == DP_DELIMITED)
598 data_parser_add_delimited_field (parser, &input,
599 var_get_case_index (v),
602 data_parser_add_fixed_field (parser, &input, var_get_case_index (v),
607 while (lex_token (lexer) != T_ENDCMD);
609 reader = dfm_open_reader (fh, lexer);
613 data_parser_make_active_file (parser, ds, reader, dict);
618 data_parser_destroy (parser);
622 return CMD_CASCADING_FAILURE;
627 destroy_spreadsheet_read_info (struct spreadsheet_read_info *sri)
632 free (sri->sheet_name);
633 free (sri->cell_range);
634 free (sri->file_name);