1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 #include <data/case-source.h>
27 #include <data/case.h>
28 #include <data/data-in.h>
29 #include <data/dictionary.h>
30 #include <data/procedure.h>
31 #include <data/variable.h>
32 #include <language/command.h>
33 #include <language/data-io/data-reader.h>
34 #include <language/data-io/file-handle.h>
35 #include <language/lexer/lexer.h>
36 #include <language/lexer/variable-parser.h>
37 #include <libpspp/alloc.h>
38 #include <libpspp/array.h>
39 #include <libpspp/assertion.h>
40 #include <libpspp/compiler.h>
41 #include <libpspp/message.h>
42 #include <libpspp/message.h>
43 #include <libpspp/misc.h>
44 #include <libpspp/pool.h>
45 #include <libpspp/str.h>
50 #define _(msgid) gettext (msgid)
52 /* FIXME: /N subcommand not implemented. It should be pretty simple,
55 /* Different types of variables for MATRIX DATA procedure. Order is
56 important: these are used for sort keys. */
59 MXD_SPLIT, /* SPLIT FILE variables. */
60 MXD_ROWTYPE, /* ROWTYPE_. */
61 MXD_FACTOR, /* Factor variables. */
62 MXD_VARNAME, /* VARNAME_. */
63 MXD_CONTINUOUS, /* Continuous variables. */
68 /* Format type enums. */
75 /* Matrix section enums. */
83 /* Diagonal inclusion enums. */
111 /* 0=vector, 1=matrix, 2=scalar. */
112 static const int content_type[PROX + 1] =
114 0, 2, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,
117 /* Name of each content type. */
118 static const char *const content_names[PROX + 1] =
120 "N", "N", "N_MATRIX", "MEAN", "STDDEV", "COUNT", "MSE",
121 "DFE", "MAT", "COV", "CORR", "PROX",
124 /* A MATRIX DATA input program. */
125 struct matrix_data_pgm
127 struct pool *container; /* Arena used for all allocations. */
128 struct dfm_reader *reader; /* Data file to read. */
131 enum format_type fmt; /* LIST or FREE. */
132 enum matrix_section section;/* LOWER or UPPER or FULL. */
133 enum include_diagonal diag; /* DIAGONAL or NODIAGONAL. */
135 int explicit_rowtype; /* ROWTYPE_ specified explicitly in data? */
136 struct variable *rowtype_, *varname_; /* ROWTYPE_, VARNAME_ variables. */
138 struct variable *single_split; /* Single SPLIT FILE variable. */
140 /* Factor variables. */
141 size_t n_factors; /* Number of factor variables. */
142 struct variable **factors; /* Factor variables. */
143 int is_per_factor[PROX + 1]; /* Is there per-factor data? */
145 int cells; /* Number of cells, or -1 if none. */
147 int pop_n; /* Population N specified by user. */
149 /* CONTENTS subcommand. */
150 int contents[EOC * 3 + 1]; /* Contents. */
151 int n_contents; /* Number of entries. */
153 /* Continuous variables. */
154 int n_continuous; /* Number of continuous variables. */
155 int first_continuous; /* Index into dictionary of
156 first continuous variable. */
159 /* Auxiliary data attached to MATRIX DATA variables. */
162 int var_type; /* Variable type. */
163 int sub_type; /* Subtype. */
166 static const struct case_source_class matrix_data_with_rowtype_source_class;
167 static const struct case_source_class matrix_data_without_rowtype_source_class;
169 static int compare_variables_by_mxd_var_type (const void *pa,
171 static bool read_matrices_without_rowtype (struct dataset *ds, struct matrix_data_pgm *);
172 static bool read_matrices_with_rowtype (struct dataset *ds, struct matrix_data_pgm *);
173 static int string_to_content_type (const char *, int *);
174 static void attach_mxd_aux (struct variable *, int var_type, int sub_type);
177 cmd_matrix_data (struct lexer *lexer, struct dataset *ds)
180 struct matrix_data_pgm *mx;
181 struct file_handle *fh = fh_inline_file ();
186 discard_variables (ds);
188 pool = pool_create ();
189 mx = pool_alloc (pool, sizeof *mx);
190 mx->container = pool;
195 mx->explicit_rowtype = 0;
198 mx->single_split = NULL;
201 memset (mx->is_per_factor, 0, sizeof mx->is_per_factor);
205 mx->n_continuous = 0;
206 mx->first_continuous = 0;
207 while (lex_token (lexer) != '.')
209 lex_match (lexer, '/');
211 if (lex_match_id (lexer, "VARIABLES"))
218 msg (SE, _("VARIABLES subcommand multiply specified."));
223 lex_match (lexer, '=');
224 if (!parse_DATA_LIST_vars (lexer, &v, &nv, PV_NO_DUPLICATE))
230 for (i = 0; i < nv; i++)
231 if (!strcasecmp (v[i], "VARNAME_"))
233 msg (SE, _("VARNAME_ cannot be explicitly specified on "
235 for (i = 0; i < nv; i++)
245 for (i = 0; i < nv; i++)
247 struct variable *new_var;
249 if (strcasecmp (v[i], "ROWTYPE_"))
251 new_var = dict_create_var_assert (dataset_dict (ds), v[i], 0);
252 attach_mxd_aux (new_var, MXD_CONTINUOUS, i);
255 mx->explicit_rowtype = 1;
261 mx->rowtype_ = dict_create_var_assert (dataset_dict (ds),
263 attach_mxd_aux (mx->rowtype_, MXD_ROWTYPE, 0);
265 else if (lex_match_id (lexer, "FILE"))
267 lex_match (lexer, '=');
268 fh = fh_parse (lexer, FH_REF_FILE | FH_REF_INLINE);
272 else if (lex_match_id (lexer, "FORMAT"))
274 lex_match (lexer, '=');
276 while (lex_token (lexer) == T_ID)
278 if (lex_match_id (lexer, "LIST"))
280 else if (lex_match_id (lexer, "FREE"))
282 else if (lex_match_id (lexer, "LOWER"))
284 else if (lex_match_id (lexer, "UPPER"))
286 else if (lex_match_id (lexer, "FULL"))
288 else if (lex_match_id (lexer, "DIAGONAL"))
290 else if (lex_match_id (lexer, "NODIAGONAL"))
291 mx->diag = NODIAGONAL;
294 lex_error (lexer, _("in FORMAT subcommand"));
299 else if (lex_match_id (lexer, "SPLIT"))
301 lex_match (lexer, '=');
305 msg (SE, _("SPLIT subcommand multiply specified."));
310 if (lex_token (lexer) != T_ID)
312 lex_error (lexer, _("in SPLIT subcommand"));
316 if (dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL
317 && (lex_look_ahead (lexer) == '.' || lex_look_ahead (lexer) == '/'))
319 if (!strcasecmp (lex_tokid (lexer), "ROWTYPE_")
320 || !strcasecmp (lex_tokid (lexer), "VARNAME_"))
322 msg (SE, _("Split variable may not be named ROWTYPE_ "
327 mx->single_split = dict_create_var_assert (dataset_dict (ds),
328 lex_tokid (lexer), 0);
329 attach_mxd_aux (mx->single_split, MXD_CONTINUOUS, 0);
332 dict_set_split_vars (dataset_dict (ds), &mx->single_split, 1);
336 struct variable **split;
339 if (!parse_variables (lexer, dataset_dict (ds),
340 &split, &n, PV_NO_DUPLICATE))
343 dict_set_split_vars (dataset_dict (ds), split, n);
347 struct variable *const *split = dict_get_split_vars (dataset_dict (ds));
348 size_t split_cnt = dict_get_split_cnt (dataset_dict (ds));
351 for (i = 0; i < split_cnt; i++)
353 struct mxd_var *mv = split[i]->aux;
355 if (mv->var_type != MXD_CONTINUOUS)
357 msg (SE, _("Split variable %s is already another type."),
361 var_clear_aux (split[i]);
362 attach_mxd_aux (split[i], MXD_SPLIT, i);
366 else if (lex_match_id (lexer, "FACTORS"))
368 lex_match (lexer, '=');
372 msg (SE, _("FACTORS subcommand multiply specified."));
377 if (!parse_variables (lexer, dataset_dict (ds), &mx->factors, &mx->n_factors,
384 for (i = 0; i < mx->n_factors; i++)
386 struct variable *v = mx->factors[i];
387 struct mxd_var *mv = v->aux;
389 if (mv->var_type != MXD_CONTINUOUS)
391 msg (SE, _("Factor variable %s is already another type."),
396 attach_mxd_aux (v, MXD_FACTOR, i);
400 else if (lex_match_id (lexer, "CELLS"))
402 lex_match (lexer, '=');
406 msg (SE, _("CELLS subcommand multiply specified."));
410 if (!lex_is_integer (lexer) || lex_integer (lexer) < 1)
412 lex_error (lexer, _("expecting positive integer"));
416 mx->cells = lex_integer (lexer);
419 else if (lex_match_id (lexer, "N"))
421 lex_match (lexer, '=');
425 msg (SE, _("N subcommand multiply specified."));
429 if (!lex_is_integer (lexer) || lex_integer (lexer) < 1)
431 lex_error (lexer, _("expecting positive integer"));
435 mx->pop_n = lex_integer (lexer);
438 else if (lex_match_id (lexer, "CONTENTS"))
440 int inside_parens = 0;
441 unsigned collide = 0;
446 msg (SE, _("CONTENTS subcommand multiply specified."));
451 lex_match (lexer, '=');
456 for (i = 0; i <= PROX; i++)
457 mx->is_per_factor[i] = 0;
462 if (lex_match (lexer, '('))
466 msg (SE, _("Nested parentheses not allowed."));
472 else if (lex_match (lexer, ')'))
476 msg (SE, _("Mismatched right parenthesis (`(')."));
479 if (mx->contents[mx->n_contents - 1] == LPAREN)
481 msg (SE, _("Empty parentheses not allowed."));
492 if (lex_token (lexer) != T_ID)
494 lex_error (lexer, _("in CONTENTS subcommand"));
498 content_type = string_to_content_type (lex_tokid (lexer),
500 if (content_type == -1)
502 lex_error (lexer, _("in CONTENTS subcommand"));
507 if (collide & (1 << collide_index))
509 msg (SE, _("Content multiply specified for %s."),
510 content_names[content_type]);
513 collide |= (1 << collide_index);
516 mx->is_per_factor[item] = inside_parens;
518 mx->contents[mx->n_contents++] = item;
520 if (lex_token (lexer) == '/' || lex_token (lexer) == '.')
526 msg (SE, _("Missing right parenthesis."));
529 mx->contents[mx->n_contents] = EOC;
533 lex_error (lexer, NULL);
538 if (lex_token (lexer) != '.')
540 lex_error (lexer, _("expecting end of command"));
546 msg (SE, _("Missing VARIABLES subcommand."));
550 if (!mx->n_contents && !mx->explicit_rowtype)
552 msg (SW, _("CONTENTS subcommand not specified: assuming file "
553 "contains only CORR matrix."));
555 mx->contents[0] = CORR;
556 mx->contents[1] = EOC;
560 if (mx->n_factors && !mx->explicit_rowtype && mx->cells == -1)
562 msg (SE, _("Missing CELLS subcommand. CELLS is required "
563 "when ROWTYPE_ is not given in the data and "
564 "factors are present."));
568 if (mx->explicit_rowtype && mx->single_split)
570 msg (SE, _("Split file values must be present in the data when "
571 "ROWTYPE_ is present."));
575 /* Create VARNAME_. */
576 mx->varname_ = dict_create_var_assert (dataset_dict (ds), "VARNAME_", 8);
577 attach_mxd_aux (mx->varname_, MXD_VARNAME, 0);
579 /* Sort the dictionary variables into the desired order for the
580 system file output. */
585 dict_get_vars (dataset_dict (ds), &v, &nv, 0);
586 qsort (v, nv, sizeof *v, compare_variables_by_mxd_var_type);
587 dict_reorder_vars (dataset_dict (ds), v, nv);
593 static const struct fmt_spec fmt_tab[MXD_COUNT] =
604 mx->first_continuous = -1;
605 for (i = 0; i < dict_get_var_cnt (dataset_dict (ds)); i++)
607 struct variable *v = dict_get_var (dataset_dict (ds), i);
608 struct mxd_var *mv = v->aux;
609 int type = mv->var_type;
611 assert (type >= 0 && type < MXD_COUNT);
612 v->print = v->write = fmt_tab[type];
614 if (type == MXD_CONTINUOUS)
616 if (mx->first_continuous == -1 && type == MXD_CONTINUOUS)
617 mx->first_continuous = i;
621 if (mx->n_continuous == 0)
623 msg (SE, _("No continuous variables specified."));
627 mx->reader = dfm_open_reader (fh, lexer);
628 if (mx->reader == NULL)
631 if (mx->explicit_rowtype)
632 ok = read_matrices_with_rowtype (ds, mx);
634 ok = read_matrices_without_rowtype (ds, mx);
636 dfm_close_reader (mx->reader);
638 pool_destroy (mx->container);
640 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
643 discard_variables (ds);
645 pool_destroy (mx->container);
646 return CMD_CASCADING_FAILURE;
649 /* Look up string S as a content-type name and return the
650 corresponding enumerated value, or -1 if there is no match. If
651 COLLIDE is non-NULL then *COLLIDE returns a value (suitable for use
652 as a bit-index) which can be used for determining whether a related
653 statistic has already been used. */
655 string_to_content_type (const char *s, int *collide)
666 {N_VECTOR, 0, "N_VECTOR"},
668 {N_SCALAR, 0, "N_SCALAR"},
669 {N_MATRIX, 1, "N_MATRIX"},
671 {STDDEV, 3, "STDDEV"},
683 for (tp = tab; tp->value != -1; tp++)
684 if (!strcasecmp (s, tp->string))
687 *collide = tp->collide;
694 /* Compare two variables using p.mxd.var_type and p.mxd.sub_type
697 compare_variables_by_mxd_var_type (const void *a_, const void *b_)
699 struct variable *const *pa = a_;
700 struct variable *const *pb = b_;
701 const struct mxd_var *a = (*pa)->aux;
702 const struct mxd_var *b = (*pb)->aux;
704 if (a->var_type != b->var_type)
705 return a->var_type > b->var_type ? 1 : -1;
707 return a->sub_type < b->sub_type ? -1 : a->sub_type > b->sub_type;
710 /* Attaches a struct mxd_var with the specific member values to
713 attach_mxd_aux (struct variable *v, int var_type, int sub_type)
717 assert (v->aux == NULL);
718 mv = xmalloc (sizeof *mv);
719 mv->var_type = var_type;
720 mv->sub_type = sub_type;
721 var_attach_aux (v, mv, var_dtor_free);
724 /* Matrix tokenizer. */
726 /* Matrix token types. */
727 enum matrix_token_type
733 /* A MATRIX DATA parsing token. */
736 enum matrix_token_type type;
737 double number; /* MNUM: token value. */
738 char *string; /* MSTR: token string; not null-terminated. */
739 int length; /* MSTR: tokstr length. */
742 static int mget_token (struct matrix_token *, struct dfm_reader *);
745 #define mget_token(TOKEN, READER) mget_token_dump(TOKEN, READER)
748 mdump_token (const struct matrix_token *token)
753 printf (" #%g", token->number);
756 printf (" '%.*s'", token->length, token->string);
765 mget_token_dump (struct matrix_token *token, struct dfm_reader *reader)
767 int result = (mget_token) (token, reader);
773 /* Return the current position in READER. */
775 context (struct dfm_reader *reader)
777 static struct string buf = DS_EMPTY_INITIALIZER;
780 if (dfm_eof (reader))
781 ds_assign_cstr (&buf, "at end of file");
786 p = dfm_get_record (reader);
787 ss_ltrim (&p, ss_cstr (CC_SPACES));
789 ds_assign_cstr (&buf, "at end of line");
791 ds_put_format (&buf, "before `%.*s'",
792 (int) ss_cspan (p, ss_cstr (CC_SPACES)), ss_data (p));
795 return ds_cstr (&buf);
798 /* Is there at least one token left in the data file? */
800 another_token (struct dfm_reader *reader)
807 if (dfm_eof (reader))
810 p = dfm_get_record (reader);
811 space_cnt = ss_span (p, ss_cstr (CC_SPACES));
812 if (space_cnt < ss_length (p))
814 dfm_forward_columns (reader, space_cnt);
818 dfm_forward_record (reader);
823 /* Parse a MATRIX DATA token from READER into TOKEN. */
825 (mget_token) (struct matrix_token *token, struct dfm_reader *reader)
827 struct substring line, p;
831 if (!another_token (reader))
834 line = p = dfm_get_record (reader);
836 /* Three types of fields: quoted with ', quoted with ", unquoted. */
838 if (c == '\'' || c == '"')
841 if (!ss_get_until (&p, c, &s))
842 msg (SW, _("Scope of string exceeds line."));
846 bool is_num = isdigit (c) || c == '.';
847 const char *start = ss_data (p);
852 if (strchr (CC_SPACES ",-+", c) != NULL)
857 if (strchr ("deDE", c) && strchr ("+-", ss_at (p, 1)))
865 s = ss_buffer (start, ss_data (p) - start);
873 di.v = (union value *) &token->number;
874 di.f1 = dfm_get_column (reader, di.s);
875 di.format = fmt_for_output (FMT_F, token->length, 0);
882 token->string = ss_data (s);
883 token->length = ss_length (s);
885 dfm_reread_record (reader, dfm_get_column (reader, ss_end (s)));
890 /* Forcibly skip the end of a line for content type CONTENT in
893 force_eol (struct dfm_reader *reader, const char *content)
897 if (dfm_eof (reader))
900 p = dfm_get_record (reader);
901 if (ss_span (p, ss_cstr (CC_SPACES)) != ss_length (p))
903 msg (SE, _("End of line expected %s while reading %s."),
904 context (reader), content);
908 dfm_forward_record (reader);
912 /* Back end, omitting ROWTYPE_. */
916 const struct dictionary *dict; /* The dictionary */
917 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
918 double ***data; /* MATRIX DATA data. */
919 double *factor_values; /* Factor values. */
920 int max_cell_idx; /* Max-numbered cell that we have
921 read so far, plus one. */
922 double *split_values; /* SPLIT FILE variable values. */
925 static bool nr_read_splits (struct nr_aux_data *, int compare);
926 static bool nr_read_factors (struct nr_aux_data *, int cell);
927 static bool nr_output_data (struct nr_aux_data *, struct ccase *,
928 write_case_func *, write_case_data);
929 static bool matrix_data_read_without_rowtype (struct case_source *source,
934 /* Read from the data file and write it to the active file.
935 Returns true if successful, false if an I/O error occurred. */
937 read_matrices_without_rowtype (struct dataset *ds, struct matrix_data_pgm *mx)
939 struct nr_aux_data nr;
946 nr.dict = dataset_dict (ds);
948 nr.factor_values = xnmalloc (mx->n_factors * mx->cells,
949 sizeof *nr.factor_values);
951 nr.split_values = xnmalloc (dict_get_split_cnt (dataset_dict (ds)),
952 sizeof *nr.split_values);
954 proc_set_source (ds, create_case_source (
955 &matrix_data_without_rowtype_source_class, &nr));
957 ok = procedure (ds, NULL, NULL);
959 free (nr.split_values);
960 free (nr.factor_values);
965 /* Mirror data across the diagonal of matrix CP which contains
966 CONTENT type data. */
968 fill_matrix (struct matrix_data_pgm *mx, int content, double *cp)
970 int type = content_type[content];
972 if (type == 1 && mx->section != FULL)
974 if (mx->diag == NODIAGONAL)
976 const double fill = content == CORR ? 1.0 : SYSMIS;
979 for (i = 0; i < mx->n_continuous; i++)
980 cp[i * (1 + mx->n_continuous)] = fill;
986 if (mx->section == LOWER)
988 int n_lines = mx->n_continuous;
989 if (mx->section != FULL && mx->diag == NODIAGONAL)
992 for (r = 1; r < n_lines; r++)
993 for (c = 0; c < r; c++)
994 cp[r + c * mx->n_continuous] = cp[c + r * mx->n_continuous];
998 assert (mx->section == UPPER);
999 for (r = 1; r < mx->n_continuous; r++)
1000 for (c = 0; c < r; c++)
1001 cp[c + r * mx->n_continuous] = cp[r + c * mx->n_continuous];
1009 for (c = 1; c < mx->n_continuous; c++)
1014 /* Read data lines for content type CONTENT from the data file.
1015 If PER_FACTOR is nonzero, then factor information is read from
1016 the data file. Data is for cell number CELL. */
1018 nr_read_data_lines (struct nr_aux_data *nr,
1019 int per_factor, int cell, int content, int compare)
1021 struct matrix_data_pgm *mx = nr->mx;
1022 const int type = content_type[content]; /* Content type. */
1023 int n_lines; /* Number of lines to parse from data file for this type. */
1024 double *cp; /* Current position in vector or matrix. */
1031 n_lines = mx->n_continuous;
1032 if (mx->section != FULL && mx->diag == NODIAGONAL)
1036 cp = nr->data[content][cell];
1037 if (type == 1 && mx->section == LOWER && mx->diag == NODIAGONAL)
1038 cp += mx->n_continuous;
1040 for (i = 0; i < n_lines; i++)
1044 if (!nr_read_splits (nr, 1))
1046 if (per_factor && !nr_read_factors (nr, cell))
1053 n_cols = mx->n_continuous;
1056 switch (mx->section)
1063 n_cols = mx->n_continuous - i;
1064 if (mx->diag == NODIAGONAL)
1071 n_cols = mx->n_continuous;
1087 for (j = 0; j < n_cols; j++)
1089 struct matrix_token token;
1090 if (!mget_token (&token, mx->reader))
1092 if (token.type != MNUM)
1094 msg (SE, _("expecting value for %s %s"),
1095 dict_get_var (nr->dict, j)->name,
1096 context (mx->reader));
1100 *cp++ = token.number;
1103 && !force_eol (mx->reader, content_names[content]))
1107 if (mx->section == LOWER)
1108 cp += mx->n_continuous - n_cols;
1111 fill_matrix (mx, content, nr->data[content][cell]);
1116 /* When ROWTYPE_ does not appear in the data, reads the matrices and
1117 writes them to the output file.
1118 Returns true if successful, false if an I/O error occurred. */
1120 matrix_data_read_without_rowtype (struct case_source *source,
1122 write_case_func *write_case,
1123 write_case_data wc_data)
1125 struct nr_aux_data *nr = source->aux;
1126 struct matrix_data_pgm *mx = nr->mx;
1131 nr->data = pool_nalloc (mx->container, PROX + 1, sizeof *nr->data);
1136 for (i = 0; i <= PROX; i++)
1140 for (cp = mx->contents; *cp != EOC; cp++)
1141 if (*cp != LPAREN && *cp != RPAREN)
1143 int per_factor = mx->is_per_factor[*cp];
1146 n_entries = mx->n_continuous;
1147 if (content_type[*cp] == 1)
1148 n_entries *= mx->n_continuous;
1151 int n_vectors = per_factor ? mx->cells : 1;
1154 nr->data[*cp] = pool_nalloc (mx->container,
1155 n_vectors, sizeof **nr->data);
1157 for (i = 0; i < n_vectors; i++)
1158 nr->data[*cp][i] = pool_nalloc (mx->container,
1159 n_entries, sizeof ***nr->data);
1168 if (!nr_read_splits (nr, 0))
1171 for (bp = mx->contents; *bp != EOC; bp = np)
1175 /* Trap the CONTENTS that we should parse in this pass
1176 between bp and ep. Set np to the starting bp for next
1181 while (*ep != RPAREN)
1189 while (*ep != EOC && *ep != LPAREN)
1198 for (i = 0; i < (per_factor ? mx->cells : 1); i++)
1202 for (cp = bp; cp < ep; cp++)
1203 if (!nr_read_data_lines (nr, per_factor, i, *cp, cp != bp))
1209 if (!nr_output_data (nr, c, write_case, wc_data))
1212 if (dict_get_split_cnt (nr->dict) == 0
1213 || !another_token (mx->reader))
1218 /* Read the split file variables. If COMPARE is 1, compares the
1219 values read to the last values read and returns true if they're equal,
1222 nr_read_splits (struct nr_aux_data *nr, int compare)
1224 struct matrix_data_pgm *mx = nr->mx;
1225 static int just_read = 0; /* FIXME: WTF? */
1229 if (compare && just_read)
1235 if (dict_get_split_vars (nr->dict) == NULL)
1238 if (mx->single_split)
1242 struct mxd_var *mv = dict_get_split_vars (nr->dict)[0]->aux;
1243 nr->split_values[0] = ++mv->sub_type;
1251 split_cnt = dict_get_split_cnt (nr->dict);
1252 for (i = 0; i < split_cnt; i++)
1254 struct matrix_token token;
1255 if (!mget_token (&token, mx->reader))
1257 if (token.type != MNUM)
1259 msg (SE, _("Syntax error expecting SPLIT FILE value %s."),
1260 context (mx->reader));
1265 nr->split_values[i] = token.number;
1266 else if (nr->split_values[i] != token.number)
1268 msg (SE, _("Expecting value %g for %s."),
1269 nr->split_values[i],
1270 dict_get_split_vars (nr->dict)[i]->name);
1278 /* Read the factors for cell CELL. If COMPARE is 1, compares the
1279 values read to the last values read and returns true if they're equal,
1282 nr_read_factors (struct nr_aux_data *nr, int cell)
1284 struct matrix_data_pgm *mx = nr->mx;
1287 if (mx->n_factors == 0)
1290 assert (nr->max_cell_idx >= cell);
1291 if (cell != nr->max_cell_idx)
1302 for (i = 0; i < mx->n_factors; i++)
1304 struct matrix_token token;
1305 if (!mget_token (&token, mx->reader))
1307 if (token.type != MNUM)
1309 msg (SE, _("Syntax error expecting factor value %s."),
1310 context (mx->reader));
1315 nr->factor_values[i + mx->n_factors * cell] = token.number;
1316 else if (nr->factor_values[i + mx->n_factors * cell] != token.number)
1318 msg (SE, _("Syntax error expecting value %g for %s %s."),
1319 nr->factor_values[i + mx->n_factors * cell],
1320 mx->factors[i]->name, context (mx->reader));
1329 /* Write the contents of a cell having content type CONTENT and data
1330 CP to the active file.
1331 Returns true if successful, false if an I/O error occurred. */
1333 dump_cell_content (const struct dictionary *dict,
1334 struct matrix_data_pgm *mx, int content, double *cp,
1336 write_case_func *write_case, write_case_data wc_data)
1338 int type = content_type[content];
1341 buf_copy_str_rpad (case_data_rw (c, mx->rowtype_->fv)->s, 8,
1342 content_names[content]);
1345 memset (case_data_rw (c, mx->varname_->fv)->s, ' ', 8);
1349 int n_lines = (type == 1) ? mx->n_continuous : 1;
1352 for (i = 0; i < n_lines; i++)
1356 for (j = 0; j < mx->n_continuous; j++)
1358 int fv = dict_get_var (dict, mx->first_continuous + j)->fv;
1359 case_data_rw (c, fv)->f = *cp;
1363 buf_copy_str_rpad (case_data_rw (c, mx->varname_->fv)->s, 8,
1365 mx->first_continuous + i)->name);
1366 if (!write_case (wc_data))
1373 /* Finally dump out everything from nr_data[] to the output file. */
1375 nr_output_data (struct nr_aux_data *nr, struct ccase *c,
1376 write_case_func *write_case, write_case_data wc_data)
1378 struct matrix_data_pgm *mx = nr->mx;
1381 struct variable *const *split;
1385 split_cnt = dict_get_split_cnt (nr->dict);
1386 split = dict_get_split_vars (nr->dict);
1387 for (i = 0; i < split_cnt; i++)
1388 case_data_rw (c, split[i]->fv)->f = nr->split_values[i];
1395 for (cell = 0; cell < mx->cells; cell++)
1400 for (factor = 0; factor < mx->n_factors; factor++)
1401 case_data_rw (c, mx->factors[factor]->fv)->f
1402 = nr->factor_values[factor + cell * mx->n_factors];
1408 for (content = 0; content <= PROX; content++)
1409 if (mx->is_per_factor[content])
1411 assert (nr->data[content] != NULL
1412 && nr->data[content][cell] != NULL);
1414 if (!dump_cell_content (nr->dict, mx,
1415 content, nr->data[content][cell],
1416 c, write_case, wc_data))
1429 for (factor = 0; factor < mx->n_factors; factor++)
1430 case_data_rw (c, mx->factors[factor]->fv)->f = SYSMIS;
1433 for (content = 0; content <= PROX; content++)
1434 if (!mx->is_per_factor[content] && nr->data[content] != NULL)
1436 if (!dump_cell_content (nr->dict, mx, content, nr->data[content][0],
1437 c, write_case, wc_data))
1445 /* Back end, with ROWTYPE_. */
1447 /* All the data for one set of factor values. */
1451 int n_rows[PROX + 1];
1452 double *data[PROX + 1];
1453 struct factor_data *next;
1456 /* With ROWTYPE_ auxiliary data. */
1459 const struct dictionary *dict; /* The dictionary */
1460 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1461 int content; /* Type of current row. */
1462 double *split_values; /* SPLIT FILE variable values. */
1463 struct factor_data *data; /* All the data. */
1464 struct factor_data *current; /* Current factor. */
1467 static bool wr_read_splits (struct wr_aux_data *, struct ccase *,
1468 write_case_func *, write_case_data);
1469 static bool wr_output_data (struct wr_aux_data *, struct ccase *,
1470 write_case_func *, write_case_data);
1471 static bool wr_read_rowtype (struct wr_aux_data *,
1472 const struct matrix_token *, struct dfm_reader *);
1473 static bool wr_read_factors (struct wr_aux_data *);
1474 static bool wr_read_indeps (struct wr_aux_data *);
1475 static bool matrix_data_read_with_rowtype (struct case_source *,
1480 /* When ROWTYPE_ appears in the data, reads the matrices and writes
1481 them to the output file.
1482 Returns true if successful, false if an I/O error occurred. */
1484 read_matrices_with_rowtype (struct dataset *ds, struct matrix_data_pgm *mx)
1486 struct wr_aux_data wr;
1491 wr.split_values = NULL;
1494 wr.dict = dataset_dict (ds);
1497 proc_set_source (ds,
1498 create_case_source (&matrix_data_with_rowtype_source_class,
1500 ok = procedure (ds, NULL, NULL);
1502 free (wr.split_values);
1506 /* Read from the data file and write it to the active file.
1507 Returns true if successful, false if an I/O error occurred. */
1509 matrix_data_read_with_rowtype (struct case_source *source,
1511 write_case_func *write_case,
1512 write_case_data wc_data)
1514 struct wr_aux_data *wr = source->aux;
1515 struct matrix_data_pgm *mx = wr->mx;
1519 if (!wr_read_splits (wr, c, write_case, wc_data))
1522 if (!wr_read_factors (wr))
1525 if (!wr_read_indeps (wr))
1528 while (another_token (mx->reader));
1530 return wr_output_data (wr, c, write_case, wc_data);
1533 /* Read the split file variables. If they differ from the previous
1534 set of split variables then output the data. Returns success. */
1536 wr_read_splits (struct wr_aux_data *wr,
1538 write_case_func *write_case, write_case_data wc_data)
1540 struct matrix_data_pgm *mx = wr->mx;
1544 split_cnt = dict_get_split_cnt (wr->dict);
1548 if (wr->split_values)
1553 wr->split_values = xnmalloc (split_cnt, sizeof *wr->split_values);
1557 bool different = false;
1560 for (i = 0; i < split_cnt; i++)
1562 struct matrix_token token;
1563 if (!mget_token (&token, mx->reader))
1565 if (token.type != MNUM)
1567 msg (SE, _("Syntax error %s expecting SPLIT FILE value."),
1568 context (mx->reader));
1572 if (compare && wr->split_values[i] != token.number && !different)
1574 if (!wr_output_data (wr, c, write_case, wc_data))
1579 wr->split_values[i] = token.number;
1586 /* Compares doubles A and B, treating SYSMIS as greatest. */
1588 compare_doubles (const void *a_, const void *b_, const void *aux UNUSED)
1590 const double *a = a_;
1591 const double *b = b_;
1595 else if (*a == SYSMIS)
1597 else if (*b == SYSMIS)
1605 /* Return strcmp()-type comparison of the MX->n_factors factors at _A and
1606 _B. Sort missing values toward the end. */
1608 compare_factors (const void *a_, const void *b_, const void *mx_)
1610 const struct matrix_data_pgm *mx = mx_;
1611 struct factor_data *const *pa = a_;
1612 struct factor_data *const *pb = b_;
1613 const double *a = (*pa)->factors;
1614 const double *b = (*pb)->factors;
1616 return lexicographical_compare_3way (a, mx->n_factors,
1619 compare_doubles, NULL);
1622 /* Write out the data for the current split file to the active
1624 Returns true if successful, false if an I/O error occurred. */
1626 wr_output_data (struct wr_aux_data *wr,
1628 write_case_func *write_case, write_case_data wc_data)
1630 struct matrix_data_pgm *mx = wr->mx;
1634 struct variable *const *split;
1638 split_cnt = dict_get_split_cnt (wr->dict);
1639 split = dict_get_split_vars (wr->dict);
1640 for (i = 0; i < split_cnt; i++)
1641 case_data_rw (c, split[i]->fv)->f = wr->split_values[i];
1644 /* Sort the wr->data list. */
1646 struct factor_data **factors;
1647 struct factor_data *iter;
1650 factors = xnmalloc (mx->cells, sizeof *factors);
1652 for (i = 0, iter = wr->data; iter; iter = iter->next, i++)
1655 sort (factors, mx->cells, sizeof *factors, compare_factors, mx);
1657 wr->data = factors[0];
1658 for (i = 0; i < mx->cells - 1; i++)
1659 factors[i]->next = factors[i + 1];
1660 factors[mx->cells - 1]->next = NULL;
1665 /* Write out records for every set of factor values. */
1667 struct factor_data *iter;
1669 for (iter = wr->data; iter; iter = iter->next)
1674 for (factor = 0; factor < mx->n_factors; factor++)
1675 case_data_rw (c, mx->factors[factor]->fv)->f
1676 = iter->factors[factor];
1682 for (content = 0; content <= PROX; content++)
1684 if (!iter->n_rows[content])
1688 int type = content_type[content];
1689 int n_lines = (type == 1
1691 - (mx->section != FULL && mx->diag == NODIAGONAL))
1694 if (n_lines != iter->n_rows[content])
1696 msg (SE, _("Expected %d lines of data for %s content; "
1697 "actually saw %d lines. No data will be "
1698 "output for this content."),
1699 n_lines, content_names[content],
1700 iter->n_rows[content]);
1705 fill_matrix (mx, content, iter->data[content]);
1707 ok = dump_cell_content (wr->dict, mx, content,
1708 iter->data[content],
1709 c, write_case, wc_data);
1717 pool_destroy (mx->container);
1718 mx->container = pool_create ();
1720 wr->data = wr->current = NULL;
1725 /* Sets ROWTYPE_ based on the given TOKEN read from READER.
1728 wr_read_rowtype (struct wr_aux_data *wr,
1729 const struct matrix_token *token,
1730 struct dfm_reader *reader)
1732 if (wr->content != -1)
1734 msg (SE, _("Multiply specified ROWTYPE_ %s."), context (reader));
1737 if (token->type != MSTR)
1739 msg (SE, _("Syntax error %s expecting ROWTYPE_ string."),
1748 memcpy (s, token->string, min (15, token->length));
1749 s[min (15, token->length)] = 0;
1751 for (cp = s; *cp; cp++)
1752 *cp = toupper ((unsigned char) *cp);
1754 wr->content = string_to_content_type (s, NULL);
1757 if (wr->content == -1)
1759 msg (SE, _("Syntax error %s."), context (reader));
1766 /* Read the factors for the current row. Select a set of factors and
1767 point wr_current to it. */
1769 wr_read_factors (struct wr_aux_data *wr)
1771 struct matrix_data_pgm *mx = wr->mx;
1772 double *factor_values = local_alloc (sizeof *factor_values * mx->n_factors);
1778 for (i = 0; i < mx->n_factors; i++)
1780 struct matrix_token token;
1781 if (!mget_token (&token, mx->reader))
1783 if (token.type == MSTR)
1785 if (!wr_read_rowtype (wr, &token, mx->reader))
1787 if (!mget_token (&token, mx->reader))
1790 if (token.type != MNUM)
1792 msg (SE, _("Syntax error expecting factor value %s."),
1793 context (mx->reader));
1797 factor_values[i] = token.number;
1800 if (wr->content == -1)
1802 struct matrix_token token;
1803 if (!mget_token (&token, mx->reader))
1805 if (!wr_read_rowtype (wr, &token, mx->reader))
1809 /* Try the most recent factor first as a simple caching
1815 for (i = 0; i < mx->n_factors; i++)
1816 if (factor_values[i] != wr->current->factors[i])
1821 /* Linear search through the list. */
1824 struct factor_data *iter;
1826 for (iter = wr->data; iter; iter = iter->next)
1830 for (i = 0; i < mx->n_factors; i++)
1831 if (factor_values[i] != iter->factors[i])
1841 /* Not found. Make a new item. */
1843 struct factor_data *new = pool_alloc (mx->container, sizeof *new);
1845 new->factors = pool_nalloc (mx->container,
1846 mx->n_factors, sizeof *new->factors);
1851 for (i = 0; i < mx->n_factors; i++)
1852 new->factors[i] = factor_values[i];
1858 for (i = 0; i <= PROX; i++)
1861 new->data[i] = NULL;
1865 new->next = wr->data;
1866 wr->data = wr->current = new;
1871 local_free (factor_values);
1875 local_free (factor_values);
1879 /* Read the independent variables into wr->current. */
1881 wr_read_indeps (struct wr_aux_data *wr)
1883 struct matrix_data_pgm *mx = wr->mx;
1884 struct factor_data *c = wr->current;
1885 const int type = content_type[wr->content];
1886 const int n_rows = c->n_rows[wr->content];
1890 /* Allocate room for data if necessary. */
1891 if (c->data[wr->content] == NULL)
1893 int n_items = mx->n_continuous;
1895 n_items *= mx->n_continuous;
1897 c->data[wr->content] = pool_nalloc (mx->container,
1898 n_items, sizeof **c->data);
1901 cp = &c->data[wr->content][n_rows * mx->n_continuous];
1903 /* Figure out how much to read from this line. */
1910 msg (SE, _("Duplicate specification for %s."),
1911 content_names[wr->content]);
1915 n_cols = mx->n_continuous;
1920 if (n_rows >= mx->n_continuous - (mx->section != FULL && mx->diag == NODIAGONAL))
1922 msg (SE, _("Too many rows of matrix data for %s."),
1923 content_names[wr->content]);
1927 switch (mx->section)
1930 n_cols = n_rows + 1;
1931 if (mx->diag == NODIAGONAL)
1932 cp += mx->n_continuous;
1936 n_cols = mx->n_continuous - n_rows;
1937 if (mx->diag == NODIAGONAL)
1944 n_cols = mx->n_continuous;
1953 c->n_rows[wr->content]++;
1955 /* Read N_COLS items at CP. */
1959 for (j = 0; j < n_cols; j++)
1961 struct matrix_token token;
1962 if (!mget_token (&token, mx->reader))
1964 if (token.type != MNUM)
1966 msg (SE, _("Syntax error expecting value for %s %s."),
1967 dict_get_var (wr->dict, mx->first_continuous + j)->name,
1968 context (mx->reader));
1972 *cp++ = token.number;
1975 && !force_eol (mx->reader, content_names[wr->content]))
1982 /* Matrix source. */
1984 static const struct case_source_class matrix_data_with_rowtype_source_class =
1988 matrix_data_read_with_rowtype,
1992 static const struct case_source_class
1993 matrix_data_without_rowtype_source_class =
1997 matrix_data_read_without_rowtype,