1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
25 #include "data/case.h"
26 #include "data/casereader.h"
27 #include "data/data-in.h"
28 #include "data/dataset.h"
29 #include "data/dictionary.h"
30 #include "data/format.h"
31 #include "data/settings.h"
32 #include "data/transformations.h"
33 #include "data/variable.h"
34 #include "language/command.h"
35 #include "language/data-io/data-parser.h"
36 #include "language/data-io/data-reader.h"
37 #include "language/data-io/file-handle.h"
38 #include "language/data-io/inpt-pgm.h"
39 #include "language/data-io/placement-parser.h"
40 #include "language/lexer/format-parser.h"
41 #include "language/lexer/lexer.h"
42 #include "language/lexer/variable-parser.h"
43 #include "libpspp/assertion.h"
44 #include "libpspp/compiler.h"
45 #include "libpspp/i18n.h"
46 #include "libpspp/message.h"
47 #include "libpspp/misc.h"
48 #include "libpspp/pool.h"
49 #include "libpspp/str.h"
52 #include "gl/xalloc.h"
55 #define _(msgid) gettext (msgid)
57 /* DATA LIST transformation data. */
60 struct data_parser *parser; /* Parser. */
61 struct dfm_reader *reader; /* Data file reader. */
62 struct variable *end; /* Variable specified on END subcommand. */
65 static bool parse_fixed (struct lexer *, struct dictionary *,
66 struct pool *, struct data_parser *);
67 static bool parse_free (struct lexer *, struct dictionary *,
68 struct pool *, struct data_parser *);
70 static const struct trns_class data_list_trns_class;
73 cmd_data_list (struct lexer *lexer, struct dataset *ds)
75 struct dictionary *dict;
76 struct data_parser *parser;
77 struct dfm_reader *reader;
78 struct variable *end = NULL;
79 struct file_handle *fh = NULL;
80 char *encoding = NULL;
83 enum data_parser_type type;
85 struct pool *tmp_pool;
88 dict = (in_input_program ()
90 : dict_create (get_default_encoding ()));
91 parser = data_parser_create (dict);
94 table = -1; /* Print table if nonzero, -1=undecided. */
97 while (lex_token (lexer) != T_SLASH)
99 if (lex_match_id (lexer, "FILE"))
101 lex_match (lexer, T_EQUALS);
103 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
107 else if (lex_match_id (lexer, "ENCODING"))
109 lex_match (lexer, T_EQUALS);
110 if (!lex_force_string (lexer))
114 encoding = ss_xstrdup (lex_tokss (lexer));
118 else if (lex_match_id (lexer, "RECORDS"))
120 if (data_parser_get_records (parser) > 0)
122 lex_sbc_only_once ("RECORDS");
125 lex_match (lexer, T_EQUALS);
126 lex_match (lexer, T_LPAREN);
127 if (!lex_force_int_range (lexer, "RECORDS", 0, INT_MAX))
129 data_parser_set_records (parser, lex_integer (lexer));
131 lex_match (lexer, T_RPAREN);
133 else if (lex_match_id (lexer, "SKIP"))
135 lex_match (lexer, T_EQUALS);
136 if (!lex_force_int_range (lexer, "SKIP", 0, INT_MAX))
138 data_parser_set_skip (parser, lex_integer (lexer));
141 else if (lex_match_id (lexer, "END"))
143 if (!in_input_program ())
145 msg (SE, _("The %s subcommand may only be used within %s."), "END", "INPUT PROGRAM");
150 lex_sbc_only_once ("END");
154 lex_match (lexer, T_EQUALS);
155 if (!lex_force_id (lexer))
157 end = dict_lookup_var (dict, lex_tokcstr (lexer));
159 end = dict_create_var_assert (dict, lex_tokcstr (lexer), 0);
162 else if (lex_match_id (lexer, "NOTABLE"))
164 else if (lex_match_id (lexer, "TABLE"))
166 else if (lex_token (lexer) == T_ID)
168 if (lex_match_id (lexer, "FIXED"))
169 data_parser_set_type (parser, DP_FIXED);
170 else if (lex_match_id (lexer, "FREE"))
172 data_parser_set_type (parser, DP_DELIMITED);
173 data_parser_set_span (parser, true);
175 else if (lex_match_id (lexer, "LIST"))
177 data_parser_set_type (parser, DP_DELIMITED);
178 data_parser_set_span (parser, false);
182 lex_error (lexer, NULL);
188 msg (SE, _("Only one of FIXED, FREE, or LIST may "
194 if (data_parser_get_type (parser) == DP_DELIMITED)
196 if (lex_match (lexer, T_LPAREN))
198 struct string delims = DS_EMPTY_INITIALIZER;
200 while (!lex_match (lexer, T_RPAREN))
204 if (lex_match_id (lexer, "TAB"))
206 else if (lex_is_string (lexer)
207 && ss_length (lex_tokss (lexer)) == 1)
209 delim = ss_first (lex_tokss (lexer));
214 /* XXX should support multibyte UTF-8 characters */
215 lex_error (lexer, NULL);
216 ds_destroy (&delims);
219 ds_put_byte (&delims, delim);
221 lex_match (lexer, T_COMMA);
224 data_parser_set_empty_line_has_field (parser, true);
225 data_parser_set_quotes (parser, ss_empty ());
226 data_parser_set_soft_delimiters (parser, ss_empty ());
227 data_parser_set_hard_delimiters (parser, ds_ss (&delims));
228 ds_destroy (&delims);
232 data_parser_set_empty_line_has_field (parser, false);
233 data_parser_set_quotes (parser, ss_cstr ("'\""));
234 data_parser_set_soft_delimiters (parser,
235 ss_cstr (CC_SPACES));
236 const char decimal = settings_get_fmt_settings ()->decimal;
237 data_parser_set_hard_delimiters (parser,
238 ss_buffer (",", (decimal == '.') ? 1 : 0));
244 lex_error (lexer, NULL);
248 type = data_parser_get_type (parser);
250 if (encoding && NULL == fh)
251 msg (MW, _("Encoding should not be specified for inline data. It will be "
255 fh = fh_inline_file ();
256 fh_set_default_handle (fh);
258 if (type != DP_FIXED && end != NULL)
260 msg (SE, _("The %s subcommand may be used only with %s."), "END", "DATA LIST FIXED");
264 tmp_pool = pool_create ();
265 if (type == DP_FIXED)
266 ok = parse_fixed (lexer, dict, tmp_pool, parser);
268 ok = parse_free (lexer, dict, tmp_pool, parser);
269 pool_destroy (tmp_pool);
273 if (!data_parser_any_fields (parser))
275 msg (SE, _("At least one variable must be specified."));
279 if (lex_end_of_command (lexer) != CMD_SUCCESS)
283 table = type == DP_FIXED || !data_parser_get_span (parser);
285 data_parser_output_description (parser, fh);
287 reader = dfm_open_reader (fh, lexer, encoding);
291 if (in_input_program ())
293 struct data_list_trns *trns = xmalloc (sizeof *trns);
294 trns->parser = parser;
295 trns->reader = reader;
297 add_transformation (ds, &data_list_trns_class, trns);
300 data_parser_make_active_file (parser, ds, reader, dict, NULL, NULL);
310 data_parser_destroy (parser);
311 if (!in_input_program ())
315 return CMD_CASCADING_FAILURE;
318 /* Fixed-format parsing. */
320 /* Parses all the variable specifications for DATA LIST FIXED,
321 storing them into DLS. Uses TMP_POOL for temporary storage;
322 the caller may destroy it. Returns true only if
325 parse_fixed (struct lexer *lexer, struct dictionary *dict,
326 struct pool *tmp_pool, struct data_parser *parser)
328 int max_records = data_parser_get_records (parser);
332 while (lex_token (lexer) != T_ENDCMD)
335 size_t n_names, name_idx;
336 struct fmt_spec *formats, *f;
339 /* Parse everything. */
340 if (!parse_record_placement (lexer, &record, &column)
341 || !parse_DATA_LIST_vars_pool (lexer, dict, tmp_pool,
342 &names, &n_names, PV_NONE)
343 || !parse_var_placements (lexer, tmp_pool, n_names, FMT_FOR_INPUT,
344 &formats, &n_formats))
347 /* Create variables and var specs. */
349 for (f = formats; f < &formats[n_formats]; f++)
350 if (!execute_placement_format (f, &record, &column))
356 name = names[name_idx++];
358 /* Create variable. */
359 width = fmt_var_width (f);
360 v = dict_create_var (dict, name, width);
364 struct fmt_spec output = fmt_for_output_from_input (
365 f, settings_get_fmt_settings ());
366 var_set_both_formats (v, &output);
371 This can be acceptable if we're in INPUT
372 PROGRAM, but only if the existing variable has
373 the same width as the one we would have
375 if (!in_input_program ())
377 msg (SE, _("%s is a duplicate variable name."), name);
381 v = dict_lookup_var_assert (dict, name);
382 if ((width != 0) != (var_get_width (v) != 0))
384 msg (SE, _("There is already a variable %s of a "
389 if (width != 0 && width != var_get_width (v))
391 msg (SE, _("There is already a string variable %s of a "
392 "different width."), name);
397 if (max_records && record > max_records)
399 msg (SE, _("Cannot place variable %s on record %d when "
400 "RECORDS=%d is specified."),
401 var_get_name (v), record,
402 data_parser_get_records (parser));
405 data_parser_add_fixed_field (parser, f,
406 var_get_case_index (v),
407 var_get_name (v), record, column);
411 assert (name_idx == n_names);
417 /* Free-format parsing. */
419 /* Parses variable specifications for DATA LIST FREE and adds
420 them to DLS. Uses TMP_POOL for temporary storage; the caller
421 may destroy it. Returns true only if successful. */
423 parse_free (struct lexer *lexer, struct dictionary *dict,
424 struct pool *tmp_pool, struct data_parser *parser)
427 while (lex_token (lexer) != T_ENDCMD)
429 struct fmt_spec input, output;
434 if (!parse_DATA_LIST_vars_pool (lexer, dict, tmp_pool,
435 &name, &n_names, PV_NONE))
438 if (lex_match (lexer, T_LPAREN))
440 char type[FMT_TYPE_LEN_MAX + 1];
442 if (!parse_abstract_format_specifier (lexer, type, &input.w,
445 if (!fmt_from_name (type, &input.type))
447 msg (SE, _("Unknown format type `%s'."), type);
451 /* If no width was included, use the minimum width for the type.
452 This isn't quite right, because DATETIME by itself seems to become
453 DATETIME20 (see bug #30690), whereas this will become
454 DATETIME17. The correct behavior is not documented. */
457 input.w = fmt_min_input_width (input.type);
461 if (!fmt_check_input (&input) || !lex_force_match (lexer, T_RPAREN))
464 /* As a special case, N format is treated as F format
465 for free-field input. */
466 if (input.type == FMT_N)
469 output = fmt_for_output_from_input (&input,
470 settings_get_fmt_settings ());
474 lex_match (lexer, T_ASTERISK);
475 input = fmt_for_input (FMT_F, 8, 0);
476 output = *settings_get_format ();
479 for (i = 0; i < n_names; i++)
483 v = dict_create_var (dict, name[i], fmt_var_width (&input));
486 msg (SE, _("%s is a duplicate variable name."), name[i]);
489 var_set_both_formats (v, &output);
491 data_parser_add_delimited_field (parser,
492 &input, var_get_case_index (v),
500 /* Input procedure. */
502 /* Destroys DATA LIST transformation TRNS.
503 Returns true if successful, false if an I/O error occurred. */
505 data_list_trns_free (void *trns_)
507 struct data_list_trns *trns = trns_;
508 data_parser_destroy (trns->parser);
509 dfm_close_reader (trns->reader);
514 /* Handle DATA LIST transformation TRNS, parsing data into *C. */
515 static enum trns_result
516 data_list_trns_proc (void *trns_, struct ccase **c, casenumber case_num UNUSED)
518 struct data_list_trns *trns = trns_;
519 enum trns_result retval;
521 *c = case_unshare (*c);
522 if (data_parser_parse (trns->parser, trns->reader, *c))
523 retval = TRNS_CONTINUE;
524 else if (dfm_reader_error (trns->reader) || dfm_eof (trns->reader) > 1)
526 /* An I/O error, or encountering end of file for a second
527 time, should be escalated into a more serious error. */
531 retval = TRNS_END_FILE;
533 /* If there was an END subcommand handle it. */
534 if (trns->end != NULL)
536 double *end = case_num_rw (*c, trns->end);
537 if (retval == TRNS_END_FILE)
540 retval = TRNS_CONTINUE;
549 static const struct trns_class data_list_trns_class = {
551 .execute = data_list_trns_proc,
552 .destroy = data_list_trns_free,