1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2017 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <gsl/gsl_matrix.h>
20 #include <gsl/gsl_vector.h>
22 #include "data/case.h"
23 #include "data/casereader.h"
24 #include "data/casewriter.h"
25 #include "data/data-in.h"
26 #include "data/dataset.h"
27 #include "data/dictionary.h"
28 #include "data/format.h"
29 #include "data/short-names.h"
30 #include "data/transformations.h"
31 #include "data/variable.h"
32 #include "language/command.h"
33 #include "language/data-io/data-parser.h"
34 #include "language/data-io/data-reader.h"
35 #include "language/data-io/file-handle.h"
36 #include "language/data-io/inpt-pgm.h"
37 #include "language/data-io/placement-parser.h"
38 #include "language/lexer/lexer.h"
39 #include "language/lexer/variable-parser.h"
40 #include "libpspp/assertion.h"
41 #include "libpspp/i18n.h"
42 #include "libpspp/intern.h"
43 #include "libpspp/message.h"
44 #include "libpspp/str.h"
46 #include "gl/c-ctype.h"
47 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
52 #define _(msgid) gettext (msgid)
55 /* Matrix row types. */ \
62 /* Vector row types. */ \
70 /* Scalar row types. */ \
75 #define RT(NAME, DIMS) C_##NAME,
82 #define RT(NAME, DIMS) +1
86 verify (N_ROWTYPES < 32);
88 /* Returns the number of dimensions in the indexes for row type RT. A matrix
89 has 2 dimensions, a vector has 1, a scalar has 0. */
91 rowtype_dimensions (enum rowtype rt)
93 static const int rowtype_dims[N_ROWTYPES] = {
94 #define RT(NAME, DIMS) [C_##NAME] = DIMS,
98 return rowtype_dims[rt];
101 static struct substring
102 rowtype_name (enum rowtype rt)
104 static const struct substring rowtype_names[N_ROWTYPES] = {
105 #define RT(NAME, DIMS) [C_##NAME] = SS_LITERAL_INITIALIZER (#NAME),
110 return rowtype_names[rt];
114 rowtype_from_string (struct substring token, enum rowtype *rt)
116 ss_trim (&token, ss_cstr (CC_SPACES));
117 for (size_t i = 0; i < N_ROWTYPES; i++)
118 if (lex_id_match (rowtype_name (i), token))
124 if (lex_id_match (ss_cstr ("N_VECTOR"), token))
129 else if (lex_id_match (ss_cstr ("SD"), token))
139 rowtype_parse (struct lexer *lexer, enum rowtype *rt)
141 bool parsed = (lex_token (lexer) == T_ID
142 && rowtype_from_string (lex_tokss (lexer), rt));
166 struct variable **input_vars;
169 /* How to read matrices with each possible number of dimensions (0=scalar,
170 1=vector, 2=matrix). */
173 /* Number of rows and columns in the matrix: (1,1) for a scalar, (1,n) for
174 a vector, (n,n) for a matrix. */
177 /* Rows of data to read and the number of columns in each. Because we
178 often read just a triangle and sometimes omit the diagonal, 'n_rp' can
179 be less than 'nr' and 'rp[i]->y' isn't always 'y'. */
182 /* The y-value of the row inside the matrix. */
185 /* first and last (exclusive) columns to read in this row. */
193 struct variable *rowtype;
194 struct variable *varname;
195 struct variable **cvars;
197 struct variable **svars;
198 size_t *svar_indexes;
200 struct variable **fvars;
201 size_t *fvar_indexes;
206 unsigned int pooled_rowtype_mask;
207 unsigned int factor_rowtype_mask;
212 enum rowtype rowtype;
220 matrix_format_uninit (struct matrix_format *mf)
222 free (mf->input_vars);
223 for (int i = 0; i < 3; i++)
227 free (mf->svar_indexes);
229 free (mf->fvar_indexes);
234 set_string (struct ccase *outcase, const struct variable *var,
235 struct substring src)
237 struct substring dst = case_ss (outcase, var);
238 for (size_t i = 0; i < dst.length; i++)
239 dst.string[i] = i < src.length ? src.string[i] : ' ';
243 parse_msg (struct dfm_reader *reader, const struct substring *token,
244 char *text, enum msg_severity severity)
246 int first_column = 0;
249 struct substring line = dfm_get_record (reader);
250 if (token->string >= line.string && token->string < ss_end (line))
251 first_column = ss_pointer_to_position (line, token->string) + 1;
254 int line_number = dfm_get_line_number (reader);
255 struct msg_location *location = xmalloc (sizeof *location);
256 *location = (struct msg_location) {
257 .file_name = intern_new (dfm_get_file_name (reader)),
258 .first_line = line_number,
259 .last_line = line_number + 1,
260 .first_column = first_column,
261 .last_column = first_column ? first_column + token->length : 0,
263 struct msg *m = xmalloc (sizeof *m);
265 .category = MSG_C_DATA,
266 .severity = severity,
267 .location = location,
273 static void PRINTF_FORMAT (3, 4)
274 parse_warning (struct dfm_reader *reader, const struct substring *token,
275 const char *format, ...)
278 va_start (args, format);
279 parse_msg (reader, token, xvasprintf (format, args), MSG_S_WARNING);
283 static void PRINTF_FORMAT (3, 4)
284 parse_error (struct dfm_reader *reader, const struct substring *token,
285 const char *format, ...)
288 va_start (args, format);
289 parse_msg (reader, token, xvasprintf (format, args), MSG_S_ERROR);
293 /* Advance to beginning of next token. */
295 more_tokens (struct substring *p, struct dfm_reader *r)
299 ss_ltrim (p, ss_cstr (CC_SPACES ","));
303 dfm_forward_record (r);
306 *p = dfm_get_record (r);
311 next_token (struct substring *p, struct dfm_reader *r, struct substring *token)
313 if (!more_tokens (p, r))
317 int c = ss_first (*p);
318 if (c == '\'' || c == '"')
321 ss_get_until (p, c, token);
330 || ss_find_byte (ss_cstr (CC_SPACES ","), c) != SIZE_MAX
331 || ((c == '+' || c == '-')
332 && ss_find_byte (ss_cstr ("dDeE"),
333 ss_at (*p, n - 1)) == SIZE_MAX))
337 ss_get_bytes (p, n, token);
343 next_number (struct substring *p, struct dfm_reader *r, double *d)
345 struct substring token;
346 if (!next_token (p, r, &token))
350 char *error = data_in (token, dfm_reader_get_encoding (r), FMT_F,
351 settings_get_fmt_settings (), &v, 0, NULL);
354 parse_error (r, &token, "%s", error);
362 next_rowtype (struct substring *p, struct dfm_reader *r, enum rowtype *rt)
364 struct substring token;
365 if (!next_token (p, r, &token))
368 if (rowtype_from_string (token, rt))
371 parse_error (r, &token, _("Unknown row type \"%.*s\"."),
372 (int) token.length, token.string);
376 struct read_matrix_params
378 /* Adjustments to first and last row to read. */
381 /* Left and right columns to read in first row, inclusive.
382 For x1, INT_MAX is the rightmost column. */
385 /* Adjustment to x0 and x1 for each subsequent row we read. Each of these
386 is 0 to keep it the same or -1 or +1 to adjust it by that much. */
390 static const struct read_matrix_params *
391 get_read_matrix_params (const struct matrix_format *mf)
393 if (mf->triangle == FULL)
399 static const struct read_matrix_params rmp = { 0, 0, 0, INT_MAX, 0, 0 };
402 else if (mf->triangle == LOWER)
404 if (mf->diagonal == DIAGONAL)
410 static const struct read_matrix_params rmp = { 0, 0, 0, 0, 0, 1 };
419 static const struct read_matrix_params rmp = { 1, 0, 0, 0, 0, 1 };
423 else if (mf->triangle == UPPER)
425 if (mf->diagonal == DIAGONAL)
431 static const struct read_matrix_params rmp = { 0, 0, 0, INT_MAX, 1, 0 };
440 static const struct read_matrix_params rmp = { 0, -1, 1, INT_MAX, 1, 0 };
449 schedule_matrices (struct matrix_format *mf)
451 struct matrix_sched *ms0 = &mf->ms[0];
454 ms0->rp = xmalloc (sizeof *ms0->rp);
455 ms0->rp[0] = (struct row_sched) { .y = 0, .x0 = 0, .x1 = 1 };
458 struct matrix_sched *ms1 = &mf->ms[1];
460 ms1->nc = mf->n_cvars;
461 ms1->rp = xmalloc (sizeof *ms1->rp);
462 ms1->rp[0] = (struct row_sched) { .y = 0, .x0 = 0, .x1 = mf->n_cvars };
465 struct matrix_sched *ms2 = &mf->ms[2];
466 ms2->nr = mf->n_cvars;
467 ms2->nc = mf->n_cvars;
468 ms2->rp = xmalloc (mf->n_cvars * sizeof *ms2->rp);
471 const struct read_matrix_params *rmp = get_read_matrix_params (mf);
473 int x1 = rmp->x1 < mf->n_cvars ? rmp->x1 : mf->n_cvars - 1;
475 int y1 = (int) mf->n_cvars + rmp->dy1;
476 for (int y = y0; y < y1; y++)
478 assert (x0 >= 0 && x0 < mf->n_cvars);
479 assert (x1 >= 0 && x1 < mf->n_cvars);
482 ms2->rp[ms2->n_rp++] = (struct row_sched) {
483 .y = y, .x0 = x0, .x1 = x1 + 1
492 read_id_columns (const struct matrix_format *mf,
493 struct substring *p, struct dfm_reader *r,
494 double *d, enum rowtype *rt)
496 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
497 if (!(mf->input_vars[i] == mf->rowtype
498 ? next_rowtype (p, r, rt)
499 : next_number (p, r, &d[i])))
505 equal_id_columns (const struct matrix_format *mf,
506 const double *a, const double *b)
508 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
509 if (mf->input_vars[i] != mf->rowtype && a[i] != b[i])
515 equal_split_columns (const struct matrix_format *mf,
516 const double *a, const double *b)
518 for (size_t i = 0; i < mf->n_svars; i++)
520 size_t idx = mf->svar_indexes[i];
521 if (a[idx] != b[idx])
528 is_pooled (const struct matrix_format *mf, const double *d)
530 for (size_t i = 0; i < mf->n_fvars; i++)
531 if (d[mf->fvar_indexes[i]] != SYSMIS)
537 matrix_sched_init (const struct matrix_format *mf, enum rowtype rt,
540 int n_dims = rowtype_dimensions (rt);
541 const struct matrix_sched *ms = &mf->ms[n_dims];
542 double diagonal = n_dims < 2 || rt != C_CORR ? SYSMIS : 1.0;
543 for (size_t y = 0; y < ms->nr; y++)
544 for (size_t x = 0; x < ms->nc; x++)
545 gsl_matrix_set (m, y, x, y == x ? diagonal : SYSMIS);
549 matrix_sched_output (const struct matrix_format *mf, enum rowtype rt,
550 gsl_matrix *m, const double *d, int split_num,
551 struct casewriter *w)
553 int n_dims = rowtype_dimensions (rt);
554 const struct matrix_sched *ms = &mf->ms[n_dims];
556 if (rt == C_N_SCALAR)
558 for (size_t x = 1; x < mf->n_cvars; x++)
559 gsl_matrix_set (m, 0, x, gsl_matrix_get (m, 0, 0));
563 for (int y = 0; y < ms->nr; y++)
565 struct ccase *c = case_create (casewriter_get_proto (w));
566 for (size_t i = 0; mf->input_vars[i] != mf->cvars[0]; i++)
567 if (mf->input_vars[i] != mf->rowtype)
568 *case_num_rw (c, mf->input_vars[i]) = d[i];
569 if (mf->n_svars && !mf->svar_indexes)
570 *case_num_rw (c, mf->svars[0]) = split_num;
571 set_string (c, mf->rowtype, rowtype_name (rt));
572 const char *varname = n_dims == 2 ? var_get_name (mf->cvars[y]) : "";
573 set_string (c, mf->varname, ss_cstr (varname));
574 for (int x = 0; x < mf->n_cvars; x++)
575 *case_num_rw (c, mf->cvars[x]) = gsl_matrix_get (m, y, x);
576 casewriter_write (w, c);
581 matrix_sched_output_n (const struct matrix_format *mf, double n,
582 gsl_matrix *m, const double *d, int split_num,
583 struct casewriter *w)
585 gsl_matrix_set (m, 0, 0, n);
586 matrix_sched_output (mf, C_N_SCALAR, m, d, split_num, w);
590 check_eol (const struct matrix_format *mf, struct substring *p,
591 struct dfm_reader *r)
595 ss_ltrim (p, ss_cstr (CC_SPACES ","));
598 parse_error (r, p, _("Extraneous data expecting end of line."));
605 parse_data_with_rowtype (const struct matrix_format *mf,
606 struct dfm_reader *r, struct casewriter *w)
610 struct substring p = dfm_get_record (r);
613 gsl_matrix *m = gsl_matrix_alloc (mf->n_cvars, mf->n_cvars);
615 double *d = xnmalloc (mf->n_input_vars, sizeof *d);
618 double *d_next = xnmalloc (mf->n_input_vars, sizeof *d_next);
620 if (!read_id_columns (mf, &p, r, d, &rt))
624 /* If this has rowtype N but there was an N subcommand, then the
625 subcommand takes precedence, so we will suppress outputting this
626 record. We still need to parse it, though, so we can't skip other
628 bool suppress_output = mf->n >= 0 && (rt == C_N || rt == C_N_SCALAR);
630 parse_error (r, NULL, _("N record is not allowed with N subcommand. "
631 "Ignoring N record."));
633 /* If there's an N subcommand, and this is a new split, then output an N
635 if (mf->n >= 0 && (!prev || !equal_split_columns (mf, prev, d)))
637 matrix_sched_output_n (mf, mf->n, m, d, 0, w);
640 prev = xnmalloc (mf->n_input_vars, sizeof *prev);
641 memcpy (prev, d, mf->n_input_vars * sizeof *prev);
644 /* Usually users don't provide the CONTENTS subcommand with ROWTYPE_, but
645 if they did then warn if ROWTYPE_ is an unexpected type. */
646 if (mf->factor_rowtype_mask || mf->pooled_rowtype_mask)
648 const char *name = rowtype_name (rt).string;
649 if (is_pooled (mf, d))
651 if (!((1u << rt) & mf->pooled_rowtype_mask))
652 parse_warning (r, NULL, _("Data contains pooled row type %s not "
653 "included in CONTENTS."), name);
657 if (!((1u << rt) & mf->factor_rowtype_mask))
658 parse_warning (r, NULL, _("Data contains with-factors row type "
659 "%s not included in CONTENTS."), name);
663 /* Initialize the matrix to be filled-in. */
664 int n_dims = rowtype_dimensions (rt);
665 const struct matrix_sched *ms = &mf->ms[n_dims];
666 matrix_sched_init (mf, rt, m);
668 enum rowtype rt_next;
672 for (n_rows = 1; ; n_rows++)
674 if (n_rows <= ms->n_rp)
676 const struct row_sched *rs = &ms->rp[n_rows - 1];
678 for (size_t x = rs->x0; x < rs->x1; x++)
681 if (!next_number (&p, r, &e))
683 gsl_matrix_set (m, y, x, e);
684 if (n_dims == 2 && mf->triangle != FULL)
685 gsl_matrix_set (m, x, y, e);
687 check_eol (mf, &p, r);
691 /* Suppress bad input data. We'll issue an error later. */
695 eof = (!more_tokens (&p, r)
696 || !read_id_columns (mf, &p, r, d_next, &rt_next));
700 if (!equal_id_columns (mf, d, d_next) || rt_next != rt)
703 if (!suppress_output)
704 matrix_sched_output (mf, rt, m, d, 0, w);
706 if (n_rows != ms->n_rp)
707 parse_error (r, NULL,
708 _("Matrix %s had %zu rows but %zu rows were expected."),
709 rowtype_name (rt).string, n_rows, ms->n_rp);
728 parse_matrix_without_rowtype (const struct matrix_format *mf,
729 struct substring *p, struct dfm_reader *r,
730 gsl_matrix *m, enum rowtype rowtype, bool pooled,
731 int split_num, struct casewriter *w)
733 int n_dims = rowtype_dimensions (rowtype);
734 const struct matrix_sched *ms = &mf->ms[n_dims];
736 double *d = xnmalloc (mf->n_input_vars, sizeof *d);
737 matrix_sched_init (mf, rowtype, m);
738 for (size_t i = 0; i < ms->n_rp; i++)
743 for (size_t j = 0; j < mf->n_input_vars; j++)
745 const struct variable *iv = mf->input_vars[j];
746 if (k < mf->n_cvars && iv == mf->cvars[k])
748 if (k < ms->rp[i].x1 - ms->rp[i].x0)
751 if (!next_number (p, r, &e))
754 int x = k + ms->rp[i].x0;
755 gsl_matrix_set (m, y, x, e);
756 if (n_dims == 2 && mf->triangle != FULL)
757 gsl_matrix_set (m, x, y, e);
762 if (h < mf->n_fvars && iv == mf->fvars[h])
773 if (!next_number (p, r, &e))
777 check_eol (mf, p, r);
780 matrix_sched_output (mf, rowtype, m, d, split_num, w);
786 parse_data_without_rowtype (const struct matrix_format *mf,
787 struct dfm_reader *r, struct casewriter *w)
791 struct substring p = dfm_get_record (r);
793 gsl_matrix *m = gsl_matrix_alloc (mf->n_cvars, mf->n_cvars);
798 for (size_t i = 0; i < mf->n_contents; )
801 if (mf->contents[i].open)
802 while (!mf->contents[j].close)
805 if (mf->contents[i].open)
807 for (size_t k = 0; k < mf->cells; k++)
808 for (size_t h = i; h <= j; h++)
809 parse_matrix_without_rowtype (mf, &p, r, m,
810 mf->contents[h].rowtype, false,
814 parse_matrix_without_rowtype (mf, &p, r, m, mf->contents[i].rowtype,
821 while (more_tokens (&p, r));
826 /* Parses VARIABLES=varnames for MATRIX DATA and returns a dictionary with the
827 named variables in it. */
828 static struct dictionary *
829 parse_matrix_data_variables (struct lexer *lexer)
831 if (!lex_force_match_id (lexer, "VARIABLES"))
833 lex_match (lexer, T_EQUALS);
835 struct dictionary *dict = dict_create (get_default_encoding ());
839 if (!parse_DATA_LIST_vars (lexer, dict, &names, &n_names, PV_NO_DUPLICATE))
845 for (size_t i = 0; i < n_names; i++)
846 if (!strcasecmp (names[i], "ROWTYPE_"))
847 dict_create_var_assert (dict, "ROWTYPE_", 8);
849 dict_create_var_assert (dict, names[i], 0);
851 for (size_t i = 0; i < n_names; ++i)
855 if (dict_lookup_var (dict, "VARNAME_"))
857 msg (SE, _("VARIABLES may not include VARNAME_."));
865 parse_matrix_data_subvars (struct lexer *lexer, struct dictionary *dict,
867 struct variable ***vars, size_t **indexes,
870 if (!parse_variables (lexer, dict, vars, n_vars, 0))
873 *indexes = xnmalloc (*n_vars, sizeof **indexes);
874 for (size_t i = 0; i < *n_vars; i++)
876 struct variable *v = (*vars)[i];
877 if (!strcasecmp (var_get_name (v), "ROWTYPE_"))
879 msg (SE, _("ROWTYPE_ is not allowed on SPLIT or FACTORS."));
882 (*indexes)[i] = var_get_dict_index (v);
884 bool *tv = &taken_vars[var_get_dict_index (v)];
887 msg (SE, _("%s may not appear on both SPLIT and FACTORS."),
893 var_set_both_formats (v, &(struct fmt_spec) { .type = FMT_F, .w = 4 });
907 cmd_matrix_data (struct lexer *lexer, struct dataset *ds)
909 struct dictionary *dict = parse_matrix_data_variables (lexer);
913 size_t n_input_vars = dict_get_var_cnt (dict);
914 struct variable **input_vars = xnmalloc (n_input_vars, sizeof *input_vars);
915 for (size_t i = 0; i < n_input_vars; i++)
916 input_vars[i] = dict_get_var (dict, i);
918 int varname_width = 8;
919 for (size_t i = 0; i < n_input_vars; i++)
921 int w = strlen (var_get_name (input_vars[i]));
922 varname_width = MAX (w, varname_width);
925 struct variable *rowtype = dict_lookup_var (dict, "ROWTYPE_");
926 bool input_rowtype = rowtype != NULL;
928 rowtype = dict_create_var_assert (dict, "ROWTYPE_", 8);
930 struct matrix_format mf = {
931 .input_rowtype = input_rowtype,
932 .input_vars = input_vars,
933 .n_input_vars = n_input_vars,
936 .varname = dict_create_var_assert (dict, "VARNAME_", varname_width),
939 .diagonal = DIAGONAL,
944 bool *taken_vars = XCALLOC (n_input_vars, bool);
946 taken_vars[var_get_dict_index (rowtype)] = true;
948 struct file_handle *fh = NULL;
949 while (lex_token (lexer) != T_ENDCMD)
951 if (!lex_force_match (lexer, T_SLASH))
954 if (lex_match_id (lexer, "N"))
956 lex_match (lexer, T_EQUALS);
958 if (!lex_force_int_range (lexer, "N", 0, INT_MAX))
961 mf.n = lex_integer (lexer);
964 else if (lex_match_id (lexer, "FORMAT"))
966 lex_match (lexer, T_EQUALS);
968 while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
970 if (lex_match_id (lexer, "LIST"))
972 else if (lex_match_id (lexer, "FREE"))
974 else if (lex_match_id (lexer, "UPPER"))
976 else if (lex_match_id (lexer, "LOWER"))
978 else if (lex_match_id (lexer, "FULL"))
980 else if (lex_match_id (lexer, "DIAGONAL"))
981 mf.diagonal = DIAGONAL;
982 else if (lex_match_id (lexer, "NODIAGONAL"))
983 mf.diagonal = NO_DIAGONAL;
986 lex_error (lexer, NULL);
991 else if (lex_match_id (lexer, "FILE"))
993 lex_match (lexer, T_EQUALS);
995 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE, NULL);
999 else if (!mf.n_svars && lex_match_id (lexer, "SPLIT"))
1001 lex_match (lexer, T_EQUALS);
1002 if (!mf.input_rowtype
1003 && lex_token (lexer) == T_ID
1004 && !dict_lookup_var (dict, lex_tokcstr (lexer)))
1006 mf.svars = xmalloc (sizeof *mf.svars);
1007 mf.svars[0] = dict_create_var_assert (dict, lex_tokcstr (lexer),
1009 var_set_both_formats (
1010 mf.svars[0], &(struct fmt_spec) { .type = FMT_F, .w = 4 });
1014 else if (!parse_matrix_data_subvars (lexer, dict, taken_vars,
1015 &mf.svars, &mf.svar_indexes,
1019 else if (!mf.n_fvars && lex_match_id (lexer, "FACTORS"))
1021 lex_match (lexer, T_EQUALS);
1022 if (!parse_matrix_data_subvars (lexer, dict, taken_vars,
1023 &mf.fvars, &mf.fvar_indexes,
1027 else if (lex_match_id (lexer, "CELLS"))
1029 if (mf.input_rowtype)
1030 msg (SW, _("CELLS is ignored when VARIABLES includes ROWTYPE_"));
1032 lex_match (lexer, T_EQUALS);
1034 if (!lex_force_int_range (lexer, "CELLS", 0, INT_MAX))
1037 mf.cells = lex_integer (lexer);
1040 else if (lex_match_id (lexer, "CONTENTS"))
1042 lex_match (lexer, T_EQUALS);
1044 size_t allocated_contents = mf.n_contents;
1045 bool in_parens = false;
1048 bool open = !in_parens && lex_match (lexer, T_LPAREN);
1050 if (!rowtype_parse (lexer, &rt))
1052 if (open || in_parens || (lex_token (lexer) != T_ENDCMD
1053 && lex_token (lexer) != T_SLASH))
1055 lex_error (lexer, _("Row type keyword expected."));
1065 mf.factor_rowtype_mask |= 1u << rt;
1067 mf.pooled_rowtype_mask |= 1u << rt;
1069 bool close = in_parens && lex_match (lexer, T_RPAREN);
1073 if (mf.n_contents >= allocated_contents)
1074 mf.contents = x2nrealloc (mf.contents, &allocated_contents,
1075 sizeof *mf.contents);
1076 mf.contents[mf.n_contents++] = (struct content) {
1077 .open = open, .rowtype = rt, .close = close
1083 lex_error (lexer, NULL);
1087 if (mf.diagonal == NO_DIAGONAL && mf.triangle == FULL)
1089 msg (SE, _("FORMAT=FULL and FORMAT=NODIAGONAL are mutually exclusive."));
1092 if (!mf.input_rowtype)
1098 msg (SE, _("CELLS is required when factor variables are specified "
1099 "and VARIABLES does not include ROWTYPE_."));
1107 msg (SW, _("CONTENTS was not specified and VARIABLES does not "
1108 "include ROWTYPE_. Assuming CONTENTS=CORR."));
1111 mf.contents = xmalloc (sizeof *mf.contents);
1112 *mf.contents = (struct content) { .rowtype = C_CORR };
1115 mf.cvars = xmalloc (mf.n_input_vars * sizeof *mf.cvars);
1116 for (size_t i = 0; i < mf.n_input_vars; i++)
1119 struct variable *v = input_vars[i];
1120 mf.cvars[mf.n_cvars++] = v;
1121 var_set_both_formats (v, &(struct fmt_spec) { .type = FMT_F, .w = 10,
1126 msg (SE, _("At least one continuous variable is required."));
1129 if (mf.input_rowtype)
1131 for (size_t i = 0; i < mf.n_cvars; i++)
1132 if (mf.cvars[i] != input_vars[n_input_vars - mf.n_cvars + i])
1134 msg (SE, _("VARIABLES includes ROWTYPE_ but the continuous "
1135 "variables are not the last ones on VARIABLES."));
1139 unsigned int rowtype_mask = mf.pooled_rowtype_mask | mf.factor_rowtype_mask;
1140 if (rowtype_mask & (1u << C_N) && mf.n >= 0)
1142 msg (SE, _("Cannot specify N on CONTENTS along with the N subcommand."));
1146 struct variable **order = xnmalloc (dict_get_var_cnt (dict), sizeof *order);
1148 for (size_t i = 0; i < mf.n_svars; i++)
1149 order[n_order++] = mf.svars[i];
1150 order[n_order++] = mf.rowtype;
1151 for (size_t i = 0; i < mf.n_fvars; i++)
1152 order[n_order++] = mf.fvars[i];
1153 order[n_order++] = mf.varname;
1154 for (size_t i = 0; i < mf.n_cvars; i++)
1155 order[n_order++] = mf.cvars[i];
1156 assert (n_order == dict_get_var_cnt (dict));
1157 dict_reorder_vars (dict, order, n_order);
1160 dict_set_split_vars (dict, mf.svars, mf.n_svars);
1162 schedule_matrices (&mf);
1165 fh = fh_inline_file ();
1167 if (lex_end_of_command (lexer) != CMD_SUCCESS)
1170 struct dfm_reader *reader = dfm_open_reader (fh, lexer, NULL);
1174 struct casewriter *writer = autopaging_writer_create (dict_get_proto (dict));
1175 if (mf.input_rowtype)
1176 parse_data_with_rowtype (&mf, reader, writer);
1178 parse_data_without_rowtype (&mf, reader, writer);
1179 dfm_close_reader (reader);
1181 dataset_set_dict (ds, dict);
1182 dataset_set_source (ds, casewriter_make_reader (writer));
1184 matrix_format_uninit (&mf);
1191 matrix_format_uninit (&mf);