1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2017 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <gsl/gsl_matrix.h>
20 #include <gsl/gsl_vector.h>
22 #include "data/case.h"
23 #include "data/casereader.h"
24 #include "data/casewriter.h"
25 #include "data/data-in.h"
26 #include "data/dataset.h"
27 #include "data/dictionary.h"
28 #include "data/format.h"
29 #include "data/short-names.h"
30 #include "data/transformations.h"
31 #include "data/variable.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/inpt-pgm.h"
37 #include "language/data-io/placement-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "language/lexer/variable-parser.h"
40 #include "libpspp/assertion.h"
41 #include "libpspp/i18n.h"
42 #include "libpspp/message.h"
43 #include "libpspp/str.h"
45 #include "gl/c-ctype.h"
46 #include "gl/minmax.h"
48 #include "gl/xalloc.h"
51 #define _(msgid) gettext (msgid)
54 /* Matrix row types. */ \
61 /* Vector row types. */ \
69 /* Scalar row types. */ \
74 #define RT(NAME, DIMS) C_##NAME,
81 #define RT(NAME, DIMS) +1
85 verify (N_ROWTYPES < 32);
87 /* Returns the number of dimensions in the indexes for row type RT. A matrix
88 has 2 dimensions, a vector has 1, a scalar has 0. */
90 rowtype_dimensions (enum rowtype rt)
92 static const int rowtype_dims[N_ROWTYPES] = {
93 #define RT(NAME, DIMS) [C_##NAME] = DIMS,
97 return rowtype_dims[rt];
100 static struct substring
101 rowtype_name (enum rowtype rt)
103 static const struct substring rowtype_names[N_ROWTYPES] = {
104 #define RT(NAME, DIMS) [C_##NAME] = SS_LITERAL_INITIALIZER (#NAME),
109 return rowtype_names[rt];
113 rowtype_from_string (struct substring token, enum rowtype *rt)
115 ss_trim (&token, ss_cstr (CC_SPACES));
116 for (size_t i = 0; i < N_ROWTYPES; i++)
117 if (lex_id_match (rowtype_name (i), token))
123 if (lex_id_match (ss_cstr ("N_VECTOR"), token))
128 else if (lex_id_match (ss_cstr ("SD"), token))
138 rowtype_parse (struct lexer *lexer, enum rowtype *rt)
140 bool parsed = (lex_token (lexer) == T_ID
141 && rowtype_from_string (lex_tokss (lexer), rt));
165 struct variable **input_vars;
168 /* How to read matrices with each possible number of dimensions (0=scalar,
169 1=vector, 2=matrix). */
172 /* Number of rows and columns in the matrix: (1,1) for a scalar, (1,n) for
173 a vector, (n,n) for a matrix. */
176 /* Rows of data to read and the number of columns in each. Because we
177 often read just a triangle and sometimes omit the diagonal, 'n_rp' can
178 be less than 'nr' and 'rp[i]->y' isn't always 'y'. */
181 /* The y-value of the row inside the matrix. */
184 /* first and last (exclusive) columns to read in this row. */
192 struct variable *rowtype;
193 struct variable *varname;
194 struct variable **cvars;
196 struct variable **svars;
197 size_t *svar_indexes;
199 struct variable **fvars;
200 size_t *fvar_indexes;
205 unsigned int pooled_rowtype_mask;
206 unsigned int factor_rowtype_mask;
211 enum rowtype rowtype;
219 matrix_format_uninit (struct matrix_format *mf)
221 free (mf->input_vars);
222 for (int i = 0; i < 3; i++)
226 free (mf->svar_indexes);
228 free (mf->fvar_indexes);
233 set_string (struct ccase *outcase, const struct variable *var,
234 struct substring src)
236 struct substring dst = case_ss (outcase, var);
237 for (size_t i = 0; i < dst.length; i++)
238 dst.string[i] = i < src.length ? src.string[i] : ' ';
242 parse_msg (struct dfm_reader *reader, const struct substring *token,
243 char *text, enum msg_severity severity)
245 int first_column = 0;
248 struct substring line = dfm_get_record (reader);
249 if (token->string >= line.string && token->string < ss_end (line))
250 first_column = ss_pointer_to_position (line, token->string) + 1;
253 int line_number = dfm_get_line_number (reader);
254 struct msg_location *location = xmalloc (sizeof *location);
255 *location = (struct msg_location) {
256 .file_name = xstrdup (dfm_get_file_name (reader)),
257 .first_line = line_number,
258 .last_line = line_number + 1,
259 .first_column = first_column,
260 .last_column = first_column ? first_column + token->length : 0,
262 struct msg *m = xmalloc (sizeof *m);
264 .category = MSG_C_DATA,
265 .severity = severity,
266 .location = location,
272 static void PRINTF_FORMAT (3, 4)
273 parse_warning (struct dfm_reader *reader, const struct substring *token,
274 const char *format, ...)
277 va_start (args, format);
278 parse_msg (reader, token, xvasprintf (format, args), MSG_S_WARNING);
282 static void PRINTF_FORMAT (3, 4)
283 parse_error (struct dfm_reader *reader, const struct substring *token,
284 const char *format, ...)
287 va_start (args, format);
288 parse_msg (reader, token, xvasprintf (format, args), MSG_S_ERROR);
292 /* Advance to beginning of next token. */
294 more_tokens (struct substring *p, struct dfm_reader *r)
298 ss_ltrim (p, ss_cstr (CC_SPACES ","));
302 dfm_forward_record (r);
305 *p = dfm_get_record (r);
310 next_token (struct substring *p, struct dfm_reader *r, struct substring *token)
312 if (!more_tokens (p, r))
316 int c = ss_first (*p);
317 if (c == '\'' || c == '"')
320 ss_get_until (p, c, token);
329 || ss_find_byte (ss_cstr (CC_SPACES ","), c) != SIZE_MAX
330 || ((c == '+' || c == '-')
331 && ss_find_byte (ss_cstr ("dDeE"),
332 ss_at (*p, n - 1)) == SIZE_MAX))
336 ss_get_bytes (p, n, token);
342 next_number (struct substring *p, struct dfm_reader *r, double *d)
344 struct substring token;
345 if (!next_token (p, r, &token))
349 char *error = data_in (token, dfm_reader_get_encoding (r), FMT_F,
350 settings_get_fmt_settings (), &v, 0, NULL);
353 parse_error (r, &token, "%s", error);
361 next_rowtype (struct substring *p, struct dfm_reader *r, enum rowtype *rt)
363 struct substring token;
364 if (!next_token (p, r, &token))
367 if (rowtype_from_string (token, rt))
370 parse_error (r, &token, _("Unknown row type \"%.*s\"."),
371 (int) token.length, token.string);
375 struct read_matrix_params
377 /* Adjustments to first and last row to read. */
380 /* Left and right columns to read in first row, inclusive.
381 For x1, INT_MAX is the rightmost column. */
384 /* Adjustment to x0 and x1 for each subsequent row we read. Each of these
385 is 0 to keep it the same or -1 or +1 to adjust it by that much. */
389 static const struct read_matrix_params *
390 get_read_matrix_params (const struct matrix_format *mf)
392 if (mf->triangle == FULL)
398 static const struct read_matrix_params rmp = { 0, 0, 0, INT_MAX, 0, 0 };
401 else if (mf->triangle == LOWER)
403 if (mf->diagonal == DIAGONAL)
409 static const struct read_matrix_params rmp = { 0, 0, 0, 0, 0, 1 };
418 static const struct read_matrix_params rmp = { 1, 0, 0, 0, 0, 1 };
422 else if (mf->triangle == UPPER)
424 if (mf->diagonal == DIAGONAL)
430 static const struct read_matrix_params rmp = { 0, 0, 0, INT_MAX, 1, 0 };
439 static const struct read_matrix_params rmp = { 0, -1, 1, INT_MAX, 1, 0 };
448 schedule_matrices (struct matrix_format *mf)
450 struct matrix_sched *ms0 = &mf->ms[0];
453 ms0->rp = xmalloc (sizeof *ms0->rp);
454 ms0->rp[0] = (struct row_sched) { .y = 0, .x0 = 0, .x1 = 1 };
457 struct matrix_sched *ms1 = &mf->ms[1];
459 ms1->nc = mf->n_cvars;
460 ms1->rp = xmalloc (sizeof *ms1->rp);
461 ms1->rp[0] = (struct row_sched) { .y = 0, .x0 = 0, .x1 = mf->n_cvars };
464 struct matrix_sched *ms2 = &mf->ms[2];
465 ms2->nr = mf->n_cvars;
466 ms2->nc = mf->n_cvars;
467 ms2->rp = xmalloc (mf->n_cvars * sizeof *ms2->rp);
470 const struct read_matrix_params *rmp = get_read_matrix_params (mf);
472 int x1 = rmp->x1 < mf->n_cvars ? rmp->x1 : mf->n_cvars - 1;
474 int y1 = (int) mf->n_cvars + rmp->dy1;
475 for (int y = y0; y < y1; y++)
477 assert (x0 >= 0 && x0 < mf->n_cvars);
478 assert (x1 >= 0 && x1 < mf->n_cvars);
481 ms2->rp[ms2->n_rp++] = (struct row_sched) {
482 .y = y, .x0 = x0, .x1 = x1 + 1
491 read_id_columns (const struct matrix_format *mf,
492 struct substring *p, struct dfm_reader *r,
493 double *d, enum rowtype *rt)
495 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
496 if (!(mf->input_vars[i] == mf->rowtype
497 ? next_rowtype (p, r, rt)
498 : next_number (p, r, &d[i])))
504 equal_id_columns (const struct matrix_format *mf,
505 const double *a, const double *b)
507 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
508 if (mf->input_vars[i] != mf->rowtype && a[i] != b[i])
514 equal_split_columns (const struct matrix_format *mf,
515 const double *a, const double *b)
517 for (size_t i = 0; i < mf->n_svars; i++)
519 size_t idx = mf->svar_indexes[i];
520 if (a[idx] != b[idx])
527 is_pooled (const struct matrix_format *mf, const double *d)
529 for (size_t i = 0; i < mf->n_fvars; i++)
530 if (d[mf->fvar_indexes[i]] != SYSMIS)
536 matrix_sched_init (const struct matrix_format *mf, enum rowtype rt,
539 int n_dims = rowtype_dimensions (rt);
540 const struct matrix_sched *ms = &mf->ms[n_dims];
541 double diagonal = n_dims < 2 || rt != C_CORR ? SYSMIS : 1.0;
542 for (size_t y = 0; y < ms->nr; y++)
543 for (size_t x = 0; x < ms->nc; x++)
544 gsl_matrix_set (m, y, x, y == x ? diagonal : SYSMIS);
548 matrix_sched_output (const struct matrix_format *mf, enum rowtype rt,
549 gsl_matrix *m, const double *d, int split_num,
550 struct casewriter *w)
552 int n_dims = rowtype_dimensions (rt);
553 const struct matrix_sched *ms = &mf->ms[n_dims];
555 if (rt == C_N_SCALAR)
557 for (size_t x = 1; x < mf->n_cvars; x++)
558 gsl_matrix_set (m, 0, x, gsl_matrix_get (m, 0, 0));
562 for (int y = 0; y < ms->nr; y++)
564 struct ccase *c = case_create (casewriter_get_proto (w));
565 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
566 if (mf->input_vars[i] != mf->rowtype)
567 *case_num_rw (c, mf->input_vars[i]) = d[i];
568 if (mf->n_svars && !mf->svar_indexes)
569 *case_num_rw (c, mf->svars[0]) = split_num;
570 set_string (c, mf->rowtype, rowtype_name (rt));
571 const char *varname = n_dims == 2 ? var_get_name (mf->cvars[y]) : "";
572 set_string (c, mf->varname, ss_cstr (varname));
573 for (int x = 0; x < mf->n_cvars; x++)
574 *case_num_rw (c, mf->cvars[x]) = gsl_matrix_get (m, y, x);
575 casewriter_write (w, c);
580 matrix_sched_output_n (const struct matrix_format *mf, double n,
581 gsl_matrix *m, const double *d, int split_num,
582 struct casewriter *w)
584 gsl_matrix_set (m, 0, 0, n);
585 matrix_sched_output (mf, C_N_SCALAR, m, d, split_num, w);
589 check_eol (const struct matrix_format *mf, struct substring *p,
590 struct dfm_reader *r)
594 ss_ltrim (p, ss_cstr (CC_SPACES ","));
597 parse_error (r, p, _("Extraneous data expecting end of line."));
604 parse_data_with_rowtype (const struct matrix_format *mf,
605 struct dfm_reader *r, struct casewriter *w)
609 struct substring p = dfm_get_record (r);
612 gsl_matrix *m = gsl_matrix_alloc (mf->n_cvars, mf->n_cvars);
614 double *d = xnmalloc (mf->n_input_vars, sizeof *d);
617 double *d_next = xnmalloc (mf->n_input_vars, sizeof *d_next);
619 if (!read_id_columns (mf, &p, r, d, &rt))
623 /* If this has rowtype N but there was an N subcommand, then the
624 subcommand takes precedence, so we will suppress outputting this
625 record. We still need to parse it, though, so we can't skip other
627 bool suppress_output = mf->n >= 0 && (rt == C_N || rt == C_N_SCALAR);
629 parse_error (r, NULL, _("N record is not allowed with N subcommand. "
630 "Ignoring N record."));
632 /* If there's an N subcommand, and this is a new split, then output an N
634 if (mf->n >= 0 && (!prev || !equal_split_columns (mf, prev, d)))
636 matrix_sched_output_n (mf, mf->n, m, d, 0, w);
639 prev = xnmalloc (mf->n_input_vars, sizeof *prev);
640 memcpy (prev, d, mf->n_input_vars * sizeof *prev);
643 /* Usually users don't provide the CONTENTS subcommand with ROWTYPE_, but
644 if they did then warn if ROWTYPE_ is an unexpected type. */
645 if (mf->factor_rowtype_mask || mf->pooled_rowtype_mask)
647 const char *name = rowtype_name (rt).string;
648 if (is_pooled (mf, d))
650 if (!((1u << rt) & mf->pooled_rowtype_mask))
651 parse_warning (r, NULL, _("Data contains pooled row type %s not "
652 "included in CONTENTS."), name);
656 if (!((1u << rt) & mf->factor_rowtype_mask))
657 parse_warning (r, NULL, _("Data contains with-factors row type "
658 "%s not included in CONTENTS."), name);
662 /* Initialize the matrix to be filled-in. */
663 int n_dims = rowtype_dimensions (rt);
664 const struct matrix_sched *ms = &mf->ms[n_dims];
665 matrix_sched_init (mf, rt, m);
667 enum rowtype rt_next;
671 for (n_rows = 1; ; n_rows++)
673 if (n_rows <= ms->n_rp)
675 const struct row_sched *rs = &ms->rp[n_rows - 1];
677 for (size_t x = rs->x0; x < rs->x1; x++)
680 if (!next_number (&p, r, &e))
682 gsl_matrix_set (m, y, x, e);
683 if (n_dims == 2 && mf->triangle != FULL)
684 gsl_matrix_set (m, x, y, e);
686 check_eol (mf, &p, r);
690 /* Suppress bad input data. We'll issue an error later. */
694 eof = (!more_tokens (&p, r)
695 || !read_id_columns (mf, &p, r, d_next, &rt_next));
699 if (!equal_id_columns (mf, d, d_next) || rt_next != rt)
702 if (!suppress_output)
703 matrix_sched_output (mf, rt, m, d, 0, w);
705 if (n_rows != ms->n_rp)
706 parse_error (r, NULL,
707 _("Matrix %s had %zu rows but %zu rows were expected."),
708 rowtype_name (rt).string, n_rows, ms->n_rp);
727 parse_matrix_without_rowtype (const struct matrix_format *mf,
728 struct substring *p, struct dfm_reader *r,
729 gsl_matrix *m, enum rowtype rowtype, bool pooled,
730 int split_num, struct casewriter *w)
732 int n_dims = rowtype_dimensions (rowtype);
733 const struct matrix_sched *ms = &mf->ms[n_dims];
735 double *d = xnmalloc (mf->n_input_vars, sizeof *d);
736 matrix_sched_init (mf, rowtype, m);
737 for (size_t i = 0; i < ms->n_rp; i++)
742 for (size_t j = 0; j < mf->n_input_vars; j++)
744 const struct variable *iv = mf->input_vars[j];
745 if (k < mf->n_cvars && iv == mf->cvars[k])
747 if (k < ms->rp[i].x1 - ms->rp[i].x0)
750 if (!next_number (p, r, &e))
753 int x = k + ms->rp[i].x0;
754 gsl_matrix_set (m, y, x, e);
755 if (n_dims == 2 && mf->triangle != FULL)
756 gsl_matrix_set (m, x, y, e);
761 if (h < mf->n_fvars && iv == mf->fvars[h])
772 if (!next_number (p, r, &e))
776 check_eol (mf, p, r);
779 matrix_sched_output (mf, rowtype, m, d, split_num, w);
785 parse_data_without_rowtype (const struct matrix_format *mf,
786 struct dfm_reader *r, struct casewriter *w)
790 struct substring p = dfm_get_record (r);
792 gsl_matrix *m = gsl_matrix_alloc (mf->n_cvars, mf->n_cvars);
797 for (size_t i = 0; i < mf->n_contents; )
800 if (mf->contents[i].open)
801 while (!mf->contents[j].close)
804 if (mf->contents[i].open)
806 for (size_t k = 0; k < mf->cells; k++)
807 for (size_t h = i; h <= j; h++)
808 parse_matrix_without_rowtype (mf, &p, r, m,
809 mf->contents[h].rowtype, false,
813 parse_matrix_without_rowtype (mf, &p, r, m, mf->contents[i].rowtype,
820 while (more_tokens (&p, r));
825 /* Parses VARIABLES=varnames for MATRIX DATA and returns a dictionary with the
826 named variables in it. */
827 static struct dictionary *
828 parse_matrix_data_variables (struct lexer *lexer)
830 if (!lex_force_match_id (lexer, "VARIABLES"))
832 lex_match (lexer, T_EQUALS);
834 struct dictionary *dict = dict_create (get_default_encoding ());
838 if (!parse_DATA_LIST_vars (lexer, dict, &names, &n_names, PV_NO_DUPLICATE))
844 for (size_t i = 0; i < n_names; i++)
845 if (!strcasecmp (names[i], "ROWTYPE_"))
846 dict_create_var_assert (dict, "ROWTYPE_", 8);
848 dict_create_var_assert (dict, names[i], 0);
850 for (size_t i = 0; i < n_names; ++i)
854 if (dict_lookup_var (dict, "VARNAME_"))
856 msg (SE, _("VARIABLES may not include VARNAME_."));
864 parse_matrix_data_subvars (struct lexer *lexer, struct dictionary *dict,
866 struct variable ***vars, size_t **indexes,
869 if (!parse_variables (lexer, dict, vars, n_vars, 0))
872 *indexes = xnmalloc (*n_vars, sizeof **indexes);
873 for (size_t i = 0; i < *n_vars; i++)
875 struct variable *v = (*vars)[i];
876 if (!strcasecmp (var_get_name (v), "ROWTYPE_"))
878 msg (SE, _("ROWTYPE_ is not allowed on SPLIT or FACTORS."));
881 (*indexes)[i] = var_get_dict_index (v);
883 bool *tv = &taken_vars[var_get_dict_index (v)];
886 msg (SE, _("%s may not appear on both SPLIT and FACTORS."),
892 var_set_both_formats (v, &(struct fmt_spec) { .type = FMT_F, .w = 4 });
906 cmd_matrix_data (struct lexer *lexer, struct dataset *ds)
908 struct dictionary *dict = parse_matrix_data_variables (lexer);
912 size_t n_input_vars = dict_get_var_cnt (dict);
913 struct variable **input_vars = xnmalloc (n_input_vars, sizeof *input_vars);
914 for (size_t i = 0; i < n_input_vars; i++)
915 input_vars[i] = dict_get_var (dict, i);
917 int varname_width = 8;
918 for (size_t i = 0; i < n_input_vars; i++)
920 int w = strlen (var_get_name (input_vars[i]));
921 varname_width = MAX (w, varname_width);
924 struct variable *rowtype = dict_lookup_var (dict, "ROWTYPE_");
925 bool input_rowtype = rowtype != NULL;
927 rowtype = dict_create_var_assert (dict, "ROWTYPE_", 8);
929 struct matrix_format mf = {
930 .input_rowtype = input_rowtype,
931 .input_vars = input_vars,
932 .n_input_vars = n_input_vars,
935 .varname = dict_create_var_assert (dict, "VARNAME_", varname_width),
938 .diagonal = DIAGONAL,
943 bool *taken_vars = XCALLOC (n_input_vars, bool);
945 taken_vars[var_get_dict_index (rowtype)] = true;
947 struct file_handle *fh = NULL;
948 while (lex_token (lexer) != T_ENDCMD)
950 if (!lex_force_match (lexer, T_SLASH))
953 if (lex_match_id (lexer, "N"))
955 lex_match (lexer, T_EQUALS);
957 if (!lex_force_int_range (lexer, "N", 0, INT_MAX))
960 mf.n = lex_integer (lexer);
963 else if (lex_match_id (lexer, "FORMAT"))
965 lex_match (lexer, T_EQUALS);
967 while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
969 if (lex_match_id (lexer, "LIST"))
971 else if (lex_match_id (lexer, "FREE"))
973 else if (lex_match_id (lexer, "UPPER"))
975 else if (lex_match_id (lexer, "LOWER"))
977 else if (lex_match_id (lexer, "FULL"))
979 else if (lex_match_id (lexer, "DIAGONAL"))
980 mf.diagonal = DIAGONAL;
981 else if (lex_match_id (lexer, "NODIAGONAL"))
982 mf.diagonal = NO_DIAGONAL;
985 lex_error (lexer, NULL);
990 else if (lex_match_id (lexer, "FILE"))
992 lex_match (lexer, T_EQUALS);
994 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
998 else if (!mf.n_svars && lex_match_id (lexer, "SPLIT"))
1000 lex_match (lexer, T_EQUALS);
1001 if (!mf.input_rowtype
1002 && lex_token (lexer) == T_ID
1003 && !dict_lookup_var (dict, lex_tokcstr (lexer)))
1005 mf.svars = xmalloc (sizeof *mf.svars);
1006 mf.svars[0] = dict_create_var_assert (dict, lex_tokcstr (lexer),
1008 var_set_both_formats (
1009 mf.svars[0], &(struct fmt_spec) { .type = FMT_F, .w = 4 });
1013 else if (!parse_matrix_data_subvars (lexer, dict, taken_vars,
1014 &mf.svars, &mf.svar_indexes,
1018 else if (!mf.n_fvars && lex_match_id (lexer, "FACTORS"))
1020 lex_match (lexer, T_EQUALS);
1021 if (!parse_matrix_data_subvars (lexer, dict, taken_vars,
1022 &mf.fvars, &mf.fvar_indexes,
1026 else if (lex_match_id (lexer, "CELLS"))
1028 if (mf.input_rowtype)
1029 msg (SW, _("CELLS is ignored when VARIABLES includes ROWTYPE_"));
1031 lex_match (lexer, T_EQUALS);
1033 if (!lex_force_int_range (lexer, "CELLS", 0, INT_MAX))
1036 mf.cells = lex_integer (lexer);
1039 else if (lex_match_id (lexer, "CONTENTS"))
1041 lex_match (lexer, T_EQUALS);
1043 size_t allocated_contents = mf.n_contents;
1044 bool in_parens = false;
1047 bool open = !in_parens && lex_match (lexer, T_LPAREN);
1049 if (!rowtype_parse (lexer, &rt))
1051 if (open || in_parens || (lex_token (lexer) != T_ENDCMD
1052 && lex_token (lexer) != T_SLASH))
1054 lex_error (lexer, _("Row type keyword expected."));
1064 mf.factor_rowtype_mask |= 1u << rt;
1066 mf.pooled_rowtype_mask |= 1u << rt;
1068 bool close = in_parens && lex_match (lexer, T_RPAREN);
1072 if (mf.n_contents >= allocated_contents)
1073 mf.contents = x2nrealloc (mf.contents, &allocated_contents,
1074 sizeof *mf.contents);
1075 mf.contents[mf.n_contents++] = (struct content) {
1076 .open = open, .rowtype = rt, .close = close
1082 lex_error (lexer, NULL);
1086 if (mf.diagonal == NO_DIAGONAL && mf.triangle == FULL)
1088 msg (SE, _("FORMAT=FULL and FORMAT=NODIAGONAL are mutually exclusive."));
1091 if (!mf.input_rowtype)
1097 msg (SE, _("CELLS is required when factor variables are specified "
1098 "and VARIABLES does not include ROWTYPE_."));
1106 msg (SW, _("CONTENTS was not specified and VARIABLES does not "
1107 "include ROWTYPE_. Assuming CONTENTS=CORR."));
1110 mf.contents = xmalloc (sizeof *mf.contents);
1111 *mf.contents = (struct content) { .rowtype = C_CORR };
1114 mf.cvars = xmalloc (mf.n_input_vars * sizeof *mf.cvars);
1115 for (size_t i = 0; i < mf.n_input_vars; i++)
1118 struct variable *v = input_vars[i];
1119 mf.cvars[mf.n_cvars++] = v;
1120 var_set_both_formats (v, &(struct fmt_spec) { .type = FMT_F, .w = 10,
1125 msg (SE, _("At least one continuous variable is required."));
1128 if (mf.input_rowtype)
1130 for (size_t i = 0; i < mf.n_cvars; i++)
1131 if (mf.cvars[i] != input_vars[n_input_vars - mf.n_cvars + i])
1133 msg (SE, _("VARIABLES includes ROWTYPE_ but the continuous "
1134 "variables are not the last ones on VARIABLES."));
1138 unsigned int rowtype_mask = mf.pooled_rowtype_mask | mf.factor_rowtype_mask;
1139 if (rowtype_mask & (1u << C_N) && mf.n >= 0)
1141 msg (SE, _("Cannot specify N on CONTENTS along with the N subcommand."));
1145 struct variable **order = xnmalloc (dict_get_var_cnt (dict), sizeof *order);
1147 for (size_t i = 0; i < mf.n_svars; i++)
1148 order[n_order++] = mf.svars[i];
1149 order[n_order++] = mf.rowtype;
1150 for (size_t i = 0; i < mf.n_fvars; i++)
1151 order[n_order++] = mf.fvars[i];
1152 order[n_order++] = mf.varname;
1153 for (size_t i = 0; i < mf.n_cvars; i++)
1154 order[n_order++] = mf.cvars[i];
1155 assert (n_order == dict_get_var_cnt (dict));
1156 dict_reorder_vars (dict, order, n_order);
1159 dict_set_split_vars (dict, mf.svars, mf.n_svars);
1161 schedule_matrices (&mf);
1164 fh = fh_inline_file ();
1166 if (lex_end_of_command (lexer) != CMD_SUCCESS)
1169 struct dfm_reader *reader = dfm_open_reader (fh, lexer, NULL);
1173 struct casewriter *writer = autopaging_writer_create (dict_get_proto (dict));
1174 if (mf.input_rowtype)
1175 parse_data_with_rowtype (&mf, reader, writer);
1177 parse_data_without_rowtype (&mf, reader, writer);
1178 dfm_close_reader (reader);
1180 dataset_set_dict (ds, dict);
1181 dataset_set_source (ds, casewriter_make_reader (writer));
1183 matrix_format_uninit (&mf);
1190 matrix_format_uninit (&mf);