1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 #include <data/case-source.h>
27 #include <data/case.h>
28 #include <data/data-in.h>
29 #include <data/dictionary.h>
30 #include <data/procedure.h>
31 #include <data/variable.h>
32 #include <language/command.h>
33 #include <language/data-io/data-reader.h>
34 #include <language/data-io/file-handle.h>
35 #include <language/lexer/lexer.h>
36 #include <libpspp/alloc.h>
37 #include <libpspp/array.h>
38 #include <libpspp/compiler.h>
39 #include <libpspp/message.h>
40 #include <libpspp/message.h>
41 #include <libpspp/misc.h>
42 #include <libpspp/pool.h>
43 #include <libpspp/str.h>
48 #define _(msgid) gettext (msgid)
50 /* FIXME: /N subcommand not implemented. It should be pretty simple,
53 /* Different types of variables for MATRIX DATA procedure. Order is
54 important: these are used for sort keys. */
57 MXD_SPLIT, /* SPLIT FILE variables. */
58 MXD_ROWTYPE, /* ROWTYPE_. */
59 MXD_FACTOR, /* Factor variables. */
60 MXD_VARNAME, /* VARNAME_. */
61 MXD_CONTINUOUS, /* Continuous variables. */
66 /* Format type enums. */
73 /* Matrix section enums. */
81 /* Diagonal inclusion enums. */
109 /* 0=vector, 1=matrix, 2=scalar. */
110 static const int content_type[PROX + 1] =
112 0, 2, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,
115 /* Name of each content type. */
116 static const char *content_names[PROX + 1] =
118 "N", "N", "N_MATRIX", "MEAN", "STDDEV", "COUNT", "MSE",
119 "DFE", "MAT", "COV", "CORR", "PROX",
122 /* A MATRIX DATA input program. */
123 struct matrix_data_pgm
125 struct pool *container; /* Arena used for all allocations. */
126 struct dfm_reader *reader; /* Data file to read. */
129 enum format_type fmt; /* LIST or FREE. */
130 enum matrix_section section;/* LOWER or UPPER or FULL. */
131 enum include_diagonal diag; /* DIAGONAL or NODIAGONAL. */
133 int explicit_rowtype; /* ROWTYPE_ specified explicitly in data? */
134 struct variable *rowtype_, *varname_; /* ROWTYPE_, VARNAME_ variables. */
136 struct variable *single_split; /* Single SPLIT FILE variable. */
138 /* Factor variables. */
139 size_t n_factors; /* Number of factor variables. */
140 struct variable **factors; /* Factor variables. */
141 int is_per_factor[PROX + 1]; /* Is there per-factor data? */
143 int cells; /* Number of cells, or -1 if none. */
145 int pop_n; /* Population N specified by user. */
147 /* CONTENTS subcommand. */
148 int contents[EOC * 3 + 1]; /* Contents. */
149 int n_contents; /* Number of entries. */
151 /* Continuous variables. */
152 int n_continuous; /* Number of continuous variables. */
153 int first_continuous; /* Index into default_dict.var of
154 first continuous variable. */
157 /* Auxiliary data attached to MATRIX DATA variables. */
160 int var_type; /* Variable type. */
161 int sub_type; /* Subtype. */
164 static const struct case_source_class matrix_data_with_rowtype_source_class;
165 static const struct case_source_class matrix_data_without_rowtype_source_class;
167 static int compare_variables_by_mxd_var_type (const void *pa,
169 static bool read_matrices_without_rowtype (struct matrix_data_pgm *);
170 static bool read_matrices_with_rowtype (struct matrix_data_pgm *);
171 static int string_to_content_type (char *, int *);
172 static void attach_mxd_aux (struct variable *, int var_type, int sub_type);
175 cmd_matrix_data (void)
178 struct matrix_data_pgm *mx;
179 struct file_handle *fh = fh_inline_file ();
184 discard_variables ();
186 pool = pool_create ();
187 mx = pool_alloc (pool, sizeof *mx);
188 mx->container = pool;
193 mx->explicit_rowtype = 0;
196 mx->single_split = NULL;
199 memset (mx->is_per_factor, 0, sizeof mx->is_per_factor);
203 mx->n_continuous = 0;
204 mx->first_continuous = 0;
209 if (lex_match_id ("VARIABLES"))
216 msg (SE, _("VARIABLES subcommand multiply specified."));
222 if (!parse_DATA_LIST_vars (&v, &nv, PV_NO_DUPLICATE))
228 for (i = 0; i < nv; i++)
229 if (!strcasecmp (v[i], "VARNAME_"))
231 msg (SE, _("VARNAME_ cannot be explicitly specified on "
233 for (i = 0; i < nv; i++)
243 for (i = 0; i < nv; i++)
245 struct variable *new_var;
247 if (strcasecmp (v[i], "ROWTYPE_"))
249 new_var = dict_create_var_assert (default_dict, v[i], 0);
250 attach_mxd_aux (new_var, MXD_CONTINUOUS, i);
253 mx->explicit_rowtype = 1;
259 mx->rowtype_ = dict_create_var_assert (default_dict,
261 attach_mxd_aux (mx->rowtype_, MXD_ROWTYPE, 0);
263 else if (lex_match_id ("FILE"))
266 fh = fh_parse (FH_REF_FILE | FH_REF_INLINE);
270 else if (lex_match_id ("FORMAT"))
274 while (token == T_ID)
276 if (lex_match_id ("LIST"))
278 else if (lex_match_id ("FREE"))
280 else if (lex_match_id ("LOWER"))
282 else if (lex_match_id ("UPPER"))
284 else if (lex_match_id ("FULL"))
286 else if (lex_match_id ("DIAGONAL"))
288 else if (lex_match_id ("NODIAGONAL"))
289 mx->diag = NODIAGONAL;
292 lex_error (_("in FORMAT subcommand"));
297 else if (lex_match_id ("SPLIT"))
303 msg (SE, _("SPLIT subcommand multiply specified."));
310 lex_error (_("in SPLIT subcommand"));
314 if (dict_lookup_var (default_dict, tokid) == NULL
315 && (lex_look_ahead () == '.' || lex_look_ahead () == '/'))
317 if (!strcasecmp (tokid, "ROWTYPE_")
318 || !strcasecmp (tokid, "VARNAME_"))
320 msg (SE, _("Split variable may not be named ROWTYPE_ "
325 mx->single_split = dict_create_var_assert (default_dict,
327 attach_mxd_aux (mx->single_split, MXD_CONTINUOUS, 0);
330 dict_set_split_vars (default_dict, &mx->single_split, 1);
334 struct variable **split;
337 if (!parse_variables (default_dict, &split, &n, PV_NO_DUPLICATE))
340 dict_set_split_vars (default_dict, split, n);
344 struct variable *const *split = dict_get_split_vars (default_dict);
345 size_t split_cnt = dict_get_split_cnt (default_dict);
348 for (i = 0; i < split_cnt; i++)
350 struct mxd_var *mv = split[i]->aux;
352 if (mv->var_type != MXD_CONTINUOUS)
354 msg (SE, _("Split variable %s is already another type."),
358 var_clear_aux (split[i]);
359 attach_mxd_aux (split[i], MXD_SPLIT, i);
363 else if (lex_match_id ("FACTORS"))
369 msg (SE, _("FACTORS subcommand multiply specified."));
374 if (!parse_variables (default_dict, &mx->factors, &mx->n_factors,
381 for (i = 0; i < mx->n_factors; i++)
383 struct variable *v = mx->factors[i];
384 struct mxd_var *mv = v->aux;
386 if (mv->var_type != MXD_CONTINUOUS)
388 msg (SE, _("Factor variable %s is already another type."),
393 attach_mxd_aux (v, MXD_FACTOR, i);
397 else if (lex_match_id ("CELLS"))
403 msg (SE, _("CELLS subcommand multiply specified."));
407 if (!lex_is_integer () || lex_integer () < 1)
409 lex_error (_("expecting positive integer"));
413 mx->cells = lex_integer ();
416 else if (lex_match_id ("N"))
422 msg (SE, _("N subcommand multiply specified."));
426 if (!lex_is_integer () || lex_integer () < 1)
428 lex_error (_("expecting positive integer"));
432 mx->pop_n = lex_integer ();
435 else if (lex_match_id ("CONTENTS"))
437 int inside_parens = 0;
438 unsigned collide = 0;
443 msg (SE, _("CONTENTS subcommand multiply specified."));
453 for (i = 0; i <= PROX; i++)
454 mx->is_per_factor[i] = 0;
463 msg (SE, _("Nested parentheses not allowed."));
469 else if (lex_match (')'))
473 msg (SE, _("Mismatched right parenthesis (`(')."));
476 if (mx->contents[mx->n_contents - 1] == LPAREN)
478 msg (SE, _("Empty parentheses not allowed."));
491 lex_error (_("in CONTENTS subcommand"));
495 content_type = string_to_content_type (tokid,
497 if (content_type == -1)
499 lex_error (_("in CONTENTS subcommand"));
504 if (collide & (1 << collide_index))
506 msg (SE, _("Content multiply specified for %s."),
507 content_names[content_type]);
510 collide |= (1 << collide_index);
513 mx->is_per_factor[item] = inside_parens;
515 mx->contents[mx->n_contents++] = item;
517 if (token == '/' || token == '.')
523 msg (SE, _("Missing right parenthesis."));
526 mx->contents[mx->n_contents] = EOC;
537 lex_error (_("expecting end of command"));
543 msg (SE, _("Missing VARIABLES subcommand."));
547 if (!mx->n_contents && !mx->explicit_rowtype)
549 msg (SW, _("CONTENTS subcommand not specified: assuming file "
550 "contains only CORR matrix."));
552 mx->contents[0] = CORR;
553 mx->contents[1] = EOC;
557 if (mx->n_factors && !mx->explicit_rowtype && mx->cells == -1)
559 msg (SE, _("Missing CELLS subcommand. CELLS is required "
560 "when ROWTYPE_ is not given in the data and "
561 "factors are present."));
565 if (mx->explicit_rowtype && mx->single_split)
567 msg (SE, _("Split file values must be present in the data when "
568 "ROWTYPE_ is present."));
572 /* Create VARNAME_. */
573 mx->varname_ = dict_create_var_assert (default_dict, "VARNAME_", 8);
574 attach_mxd_aux (mx->varname_, MXD_VARNAME, 0);
576 /* Sort the dictionary variables into the desired order for the
577 system file output. */
582 dict_get_vars (default_dict, &v, &nv, 0);
583 qsort (v, nv, sizeof *v, compare_variables_by_mxd_var_type);
584 dict_reorder_vars (default_dict, v, nv);
590 static const struct fmt_spec fmt_tab[MXD_COUNT] =
601 mx->first_continuous = -1;
602 for (i = 0; i < dict_get_var_cnt (default_dict); i++)
604 struct variable *v = dict_get_var (default_dict, i);
605 struct mxd_var *mv = v->aux;
606 int type = mv->var_type;
608 assert (type >= 0 && type < MXD_COUNT);
609 v->print = v->write = fmt_tab[type];
611 if (type == MXD_CONTINUOUS)
613 if (mx->first_continuous == -1 && type == MXD_CONTINUOUS)
614 mx->first_continuous = i;
618 if (mx->n_continuous == 0)
620 msg (SE, _("No continuous variables specified."));
624 mx->reader = dfm_open_reader (fh);
625 if (mx->reader == NULL)
628 if (mx->explicit_rowtype)
629 ok = read_matrices_with_rowtype (mx);
631 ok = read_matrices_without_rowtype (mx);
633 dfm_close_reader (mx->reader);
635 pool_destroy (mx->container);
637 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
640 discard_variables ();
642 pool_destroy (mx->container);
643 return CMD_CASCADING_FAILURE;
646 /* Look up string S as a content-type name and return the
647 corresponding enumerated value, or -1 if there is no match. If
648 COLLIDE is non-NULL then *COLLIDE returns a value (suitable for use
649 as a bit-index) which can be used for determining whether a related
650 statistic has already been used. */
652 string_to_content_type (char *s, int *collide)
663 {N_VECTOR, 0, "N_VECTOR"},
665 {N_SCALAR, 0, "N_SCALAR"},
666 {N_MATRIX, 1, "N_MATRIX"},
668 {STDDEV, 3, "STDDEV"},
680 for (tp = tab; tp->value != -1; tp++)
681 if (!strcasecmp (s, tp->string))
684 *collide = tp->collide;
691 /* Compare two variables using p.mxd.var_type and p.mxd.sub_type
694 compare_variables_by_mxd_var_type (const void *a_, const void *b_)
696 struct variable *const *pa = a_;
697 struct variable *const *pb = b_;
698 const struct mxd_var *a = (*pa)->aux;
699 const struct mxd_var *b = (*pb)->aux;
701 if (a->var_type != b->var_type)
702 return a->var_type > b->var_type ? 1 : -1;
704 return a->sub_type < b->sub_type ? -1 : a->sub_type > b->sub_type;
707 /* Attaches a struct mxd_var with the specific member values to
710 attach_mxd_aux (struct variable *v, int var_type, int sub_type)
714 assert (v->aux == NULL);
715 mv = xmalloc (sizeof *mv);
716 mv->var_type = var_type;
717 mv->sub_type = sub_type;
718 var_attach_aux (v, mv, var_dtor_free);
721 /* Matrix tokenizer. */
723 /* Matrix token types. */
724 enum matrix_token_type
730 /* A MATRIX DATA parsing token. */
733 enum matrix_token_type type;
734 double number; /* MNUM: token value. */
735 char *string; /* MSTR: token string; not null-terminated. */
736 int length; /* MSTR: tokstr length. */
739 static int mget_token (struct matrix_token *, struct dfm_reader *);
742 #define mget_token(TOKEN, READER) mget_token_dump(TOKEN, READER)
745 mdump_token (const struct matrix_token *token)
750 printf (" #%g", token->number);
753 printf (" '%.*s'", token->length, token->string);
762 mget_token_dump (struct matrix_token *token, struct dfm_reader *reader)
764 int result = (mget_token) (token, reader);
770 /* Return the current position in READER. */
772 context (struct dfm_reader *reader)
774 static struct string buf = DS_EMPTY_INITIALIZER;
777 if (dfm_eof (reader))
778 ds_assign_cstr (&buf, "at end of file");
783 p = dfm_get_record (reader);
784 ss_ltrim (&p, ss_cstr (CC_SPACES));
786 ds_assign_cstr (&buf, "at end of line");
788 ds_put_format (&buf, "before `%.*s'",
789 (int) ss_cspan (p, ss_cstr (CC_SPACES)), ss_data (p));
792 return ds_cstr (&buf);
795 /* Is there at least one token left in the data file? */
797 another_token (struct dfm_reader *reader)
804 if (dfm_eof (reader))
807 p = dfm_get_record (reader);
808 space_cnt = ss_span (p, ss_cstr (CC_SPACES));
809 if (space_cnt < ss_length (p))
811 dfm_forward_columns (reader, space_cnt);
815 dfm_forward_record (reader);
819 /* Parse a MATRIX DATA token from READER into TOKEN. */
821 (mget_token) (struct matrix_token *token, struct dfm_reader *reader)
823 struct substring line, p;
827 if (!another_token (reader))
830 line = p = dfm_get_record (reader);
832 /* Three types of fields: quoted with ', quoted with ", unquoted. */
834 if (c == '\'' || c == '"')
837 if (!ss_get_until (&p, c, &s))
838 msg (SW, _("Scope of string exceeds line."));
842 bool is_num = isdigit (c) || c == '.';
843 const char *start = ss_data (p);
848 if (strchr (CC_SPACES ",-+", c) != NULL)
853 if (strchr ("deDE", c) && strchr ("+-", ss_at (p, 1)))
861 s = ss_buffer (start, ss_data (p) - start);
869 di.v = (union value *) &token->number;
870 di.f1 = dfm_get_column (reader, di.s);
871 di.format = make_output_format (FMT_F, token->length, 0);
878 token->string = ss_data (s);
879 token->length = ss_length (s);
881 dfm_reread_record (reader, dfm_get_column (reader, ss_end (s)));
886 /* Forcibly skip the end of a line for content type CONTENT in
889 force_eol (struct dfm_reader *reader, const char *content)
893 if (dfm_eof (reader))
896 p = dfm_get_record (reader);
897 if (ss_span (p, ss_cstr (CC_SPACES)) != ss_length (p))
899 msg (SE, _("End of line expected %s while reading %s."),
900 context (reader), content);
904 dfm_forward_record (reader);
908 /* Back end, omitting ROWTYPE_. */
912 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
913 double ***data; /* MATRIX DATA data. */
914 double *factor_values; /* Factor values. */
915 int max_cell_idx; /* Max-numbered cell that we have
916 read so far, plus one. */
917 double *split_values; /* SPLIT FILE variable values. */
920 static int nr_read_splits (struct nr_aux_data *, int compare);
921 static int nr_read_factors (struct nr_aux_data *, int cell);
922 static bool nr_output_data (struct nr_aux_data *, struct ccase *,
923 write_case_func *, write_case_data);
924 static bool matrix_data_read_without_rowtype (struct case_source *source,
929 /* Read from the data file and write it to the active file.
930 Returns true if successful, false if an I/O error occurred. */
932 read_matrices_without_rowtype (struct matrix_data_pgm *mx)
934 struct nr_aux_data nr;
942 nr.factor_values = xnmalloc (mx->n_factors * mx->cells,
943 sizeof *nr.factor_values);
945 nr.split_values = xnmalloc (dict_get_split_cnt (default_dict),
946 sizeof *nr.split_values);
948 proc_set_source (create_case_source (
949 &matrix_data_without_rowtype_source_class, &nr));
951 ok = procedure (NULL, NULL);
953 free (nr.split_values);
954 free (nr.factor_values);
959 /* Mirror data across the diagonal of matrix CP which contains
960 CONTENT type data. */
962 fill_matrix (struct matrix_data_pgm *mx, int content, double *cp)
964 int type = content_type[content];
966 if (type == 1 && mx->section != FULL)
968 if (mx->diag == NODIAGONAL)
970 const double fill = content == CORR ? 1.0 : SYSMIS;
973 for (i = 0; i < mx->n_continuous; i++)
974 cp[i * (1 + mx->n_continuous)] = fill;
980 if (mx->section == LOWER)
982 int n_lines = mx->n_continuous;
983 if (mx->section != FULL && mx->diag == NODIAGONAL)
986 for (r = 1; r < n_lines; r++)
987 for (c = 0; c < r; c++)
988 cp[r + c * mx->n_continuous] = cp[c + r * mx->n_continuous];
992 assert (mx->section == UPPER);
993 for (r = 1; r < mx->n_continuous; r++)
994 for (c = 0; c < r; c++)
995 cp[c + r * mx->n_continuous] = cp[r + c * mx->n_continuous];
1003 for (c = 1; c < mx->n_continuous; c++)
1008 /* Read data lines for content type CONTENT from the data file.
1009 If PER_FACTOR is nonzero, then factor information is read from
1010 the data file. Data is for cell number CELL. */
1012 nr_read_data_lines (struct nr_aux_data *nr,
1013 int per_factor, int cell, int content, int compare)
1015 struct matrix_data_pgm *mx = nr->mx;
1016 const int type = content_type[content]; /* Content type. */
1017 int n_lines; /* Number of lines to parse from data file for this type. */
1018 double *cp; /* Current position in vector or matrix. */
1025 n_lines = mx->n_continuous;
1026 if (mx->section != FULL && mx->diag == NODIAGONAL)
1030 cp = nr->data[content][cell];
1031 if (type == 1 && mx->section == LOWER && mx->diag == NODIAGONAL)
1032 cp += mx->n_continuous;
1034 for (i = 0; i < n_lines; i++)
1038 if (!nr_read_splits (nr, 1))
1040 if (per_factor && !nr_read_factors (nr, cell))
1047 n_cols = mx->n_continuous;
1050 switch (mx->section)
1057 n_cols = mx->n_continuous - i;
1058 if (mx->diag == NODIAGONAL)
1065 n_cols = mx->n_continuous;
1083 for (j = 0; j < n_cols; j++)
1085 struct matrix_token token;
1086 if (!mget_token (&token, mx->reader))
1088 if (token.type != MNUM)
1090 msg (SE, _("expecting value for %s %s"),
1091 dict_get_var (default_dict, j)->name,
1092 context (mx->reader));
1096 *cp++ = token.number;
1099 && !force_eol (mx->reader, content_names[content]))
1103 if (mx->section == LOWER)
1104 cp += mx->n_continuous - n_cols;
1107 fill_matrix (mx, content, nr->data[content][cell]);
1112 /* When ROWTYPE_ does not appear in the data, reads the matrices and
1113 writes them to the output file.
1114 Returns true if successful, false if an I/O error occurred. */
1116 matrix_data_read_without_rowtype (struct case_source *source,
1118 write_case_func *write_case,
1119 write_case_data wc_data)
1121 struct nr_aux_data *nr = source->aux;
1122 struct matrix_data_pgm *mx = nr->mx;
1127 nr->data = pool_nalloc (mx->container, PROX + 1, sizeof *nr->data);
1132 for (i = 0; i <= PROX; i++)
1136 for (cp = mx->contents; *cp != EOC; cp++)
1137 if (*cp != LPAREN && *cp != RPAREN)
1139 int per_factor = mx->is_per_factor[*cp];
1142 n_entries = mx->n_continuous;
1143 if (content_type[*cp] == 1)
1144 n_entries *= mx->n_continuous;
1147 int n_vectors = per_factor ? mx->cells : 1;
1150 nr->data[*cp] = pool_nalloc (mx->container,
1151 n_vectors, sizeof **nr->data);
1153 for (i = 0; i < n_vectors; i++)
1154 nr->data[*cp][i] = pool_nalloc (mx->container,
1155 n_entries, sizeof ***nr->data);
1164 if (!nr_read_splits (nr, 0))
1167 for (bp = mx->contents; *bp != EOC; bp = np)
1171 /* Trap the CONTENTS that we should parse in this pass
1172 between bp and ep. Set np to the starting bp for next
1177 while (*ep != RPAREN)
1185 while (*ep != EOC && *ep != LPAREN)
1194 for (i = 0; i < (per_factor ? mx->cells : 1); i++)
1198 for (cp = bp; cp < ep; cp++)
1199 if (!nr_read_data_lines (nr, per_factor, i, *cp, cp != bp))
1205 if (!nr_output_data (nr, c, write_case, wc_data))
1208 if (dict_get_split_cnt (default_dict) == 0
1209 || !another_token (mx->reader))
1214 /* Read the split file variables. If COMPARE is 1, compares the
1215 values read to the last values read and returns 1 if they're equal,
1218 nr_read_splits (struct nr_aux_data *nr, int compare)
1220 struct matrix_data_pgm *mx = nr->mx;
1221 static int just_read = 0; /* FIXME: WTF? */
1225 if (compare && just_read)
1231 if (dict_get_split_vars (default_dict) == NULL)
1234 if (mx->single_split)
1238 struct mxd_var *mv = dict_get_split_vars (default_dict)[0]->aux;
1239 nr->split_values[0] = ++mv->sub_type;
1247 split_cnt = dict_get_split_cnt (default_dict);
1248 for (i = 0; i < split_cnt; i++)
1250 struct matrix_token token;
1251 if (!mget_token (&token, mx->reader))
1253 if (token.type != MNUM)
1255 msg (SE, _("Syntax error expecting SPLIT FILE value %s."),
1256 context (mx->reader));
1261 nr->split_values[i] = token.number;
1262 else if (nr->split_values[i] != token.number)
1264 msg (SE, _("Expecting value %g for %s."),
1265 nr->split_values[i],
1266 dict_get_split_vars (default_dict)[i]->name);
1274 /* Read the factors for cell CELL. If COMPARE is 1, compares the
1275 values read to the last values read and returns 1 if they're equal,
1278 nr_read_factors (struct nr_aux_data *nr, int cell)
1280 struct matrix_data_pgm *mx = nr->mx;
1283 if (mx->n_factors == 0)
1286 assert (nr->max_cell_idx >= cell);
1287 if (cell != nr->max_cell_idx)
1298 for (i = 0; i < mx->n_factors; i++)
1300 struct matrix_token token;
1301 if (!mget_token (&token, mx->reader))
1303 if (token.type != MNUM)
1305 msg (SE, _("Syntax error expecting factor value %s."),
1306 context (mx->reader));
1311 nr->factor_values[i + mx->n_factors * cell] = token.number;
1312 else if (nr->factor_values[i + mx->n_factors * cell] != token.number)
1314 msg (SE, _("Syntax error expecting value %g for %s %s."),
1315 nr->factor_values[i + mx->n_factors * cell],
1316 mx->factors[i]->name, context (mx->reader));
1325 /* Write the contents of a cell having content type CONTENT and data
1326 CP to the active file.
1327 Returns true if successful, false if an I/O error occurred. */
1329 dump_cell_content (struct matrix_data_pgm *mx, int content, double *cp,
1331 write_case_func *write_case, write_case_data wc_data)
1333 int type = content_type[content];
1336 buf_copy_str_rpad (case_data_rw (c, mx->rowtype_->fv)->s, 8,
1337 content_names[content]);
1340 memset (case_data_rw (c, mx->varname_->fv)->s, ' ', 8);
1344 int n_lines = (type == 1) ? mx->n_continuous : 1;
1347 for (i = 0; i < n_lines; i++)
1351 for (j = 0; j < mx->n_continuous; j++)
1353 int fv = dict_get_var (default_dict, mx->first_continuous + j)->fv;
1354 case_data_rw (c, fv)->f = *cp;
1358 buf_copy_str_rpad (case_data_rw (c, mx->varname_->fv)->s, 8,
1359 dict_get_var (default_dict,
1360 mx->first_continuous + i)->name);
1361 if (!write_case (wc_data))
1368 /* Finally dump out everything from nr_data[] to the output file. */
1370 nr_output_data (struct nr_aux_data *nr, struct ccase *c,
1371 write_case_func *write_case, write_case_data wc_data)
1373 struct matrix_data_pgm *mx = nr->mx;
1376 struct variable *const *split;
1380 split_cnt = dict_get_split_cnt (default_dict);
1381 split = dict_get_split_vars (default_dict);
1382 for (i = 0; i < split_cnt; i++)
1383 case_data_rw (c, split[i]->fv)->f = nr->split_values[i];
1390 for (cell = 0; cell < mx->cells; cell++)
1395 for (factor = 0; factor < mx->n_factors; factor++)
1396 case_data_rw (c, mx->factors[factor]->fv)->f
1397 = nr->factor_values[factor + cell * mx->n_factors];
1403 for (content = 0; content <= PROX; content++)
1404 if (mx->is_per_factor[content])
1406 assert (nr->data[content] != NULL
1407 && nr->data[content][cell] != NULL);
1409 if (!dump_cell_content (mx, content, nr->data[content][cell],
1410 c, write_case, wc_data))
1423 for (factor = 0; factor < mx->n_factors; factor++)
1424 case_data_rw (c, mx->factors[factor]->fv)->f = SYSMIS;
1427 for (content = 0; content <= PROX; content++)
1428 if (!mx->is_per_factor[content] && nr->data[content] != NULL)
1430 if (!dump_cell_content (mx, content, nr->data[content][0],
1431 c, write_case, wc_data))
1439 /* Back end, with ROWTYPE_. */
1441 /* All the data for one set of factor values. */
1445 int n_rows[PROX + 1];
1446 double *data[PROX + 1];
1447 struct factor_data *next;
1450 /* With ROWTYPE_ auxiliary data. */
1453 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1454 int content; /* Type of current row. */
1455 double *split_values; /* SPLIT FILE variable values. */
1456 struct factor_data *data; /* All the data. */
1457 struct factor_data *current; /* Current factor. */
1460 static int wr_read_splits (struct wr_aux_data *, struct ccase *,
1461 write_case_func *, write_case_data);
1462 static bool wr_output_data (struct wr_aux_data *, struct ccase *,
1463 write_case_func *, write_case_data);
1464 static int wr_read_rowtype (struct wr_aux_data *,
1465 const struct matrix_token *, struct dfm_reader *);
1466 static int wr_read_factors (struct wr_aux_data *);
1467 static int wr_read_indeps (struct wr_aux_data *);
1468 static bool matrix_data_read_with_rowtype (struct case_source *,
1473 /* When ROWTYPE_ appears in the data, reads the matrices and writes
1474 them to the output file.
1475 Returns true if successful, false if an I/O error occurred. */
1477 read_matrices_with_rowtype (struct matrix_data_pgm *mx)
1479 struct wr_aux_data wr;
1484 wr.split_values = NULL;
1489 proc_set_source (create_case_source (&matrix_data_with_rowtype_source_class,
1491 ok = procedure (NULL, NULL);
1493 free (wr.split_values);
1497 /* Read from the data file and write it to the active file.
1498 Returns true if successful, false if an I/O error occurred. */
1500 matrix_data_read_with_rowtype (struct case_source *source,
1502 write_case_func *write_case,
1503 write_case_data wc_data)
1505 struct wr_aux_data *wr = source->aux;
1506 struct matrix_data_pgm *mx = wr->mx;
1510 if (!wr_read_splits (wr, c, write_case, wc_data))
1513 if (!wr_read_factors (wr))
1516 if (!wr_read_indeps (wr))
1519 while (another_token (mx->reader));
1521 return wr_output_data (wr, c, write_case, wc_data);
1524 /* Read the split file variables. If they differ from the previous
1525 set of split variables then output the data. Returns success. */
1527 wr_read_splits (struct wr_aux_data *wr,
1529 write_case_func *write_case, write_case_data wc_data)
1531 struct matrix_data_pgm *mx = wr->mx;
1535 split_cnt = dict_get_split_cnt (default_dict);
1539 if (wr->split_values)
1544 wr->split_values = xnmalloc (split_cnt, sizeof *wr->split_values);
1551 for (i = 0; i < split_cnt; i++)
1553 struct matrix_token token;
1554 if (!mget_token (&token, mx->reader))
1556 if (token.type != MNUM)
1558 msg (SE, _("Syntax error %s expecting SPLIT FILE value."),
1559 context (mx->reader));
1563 if (compare && wr->split_values[i] != token.number && !different)
1565 if (!wr_output_data (wr, c, write_case, wc_data))
1570 wr->split_values[i] = token.number;
1577 /* Compares doubles A and B, treating SYSMIS as greatest. */
1579 compare_doubles (const void *a_, const void *b_, void *aux UNUSED)
1581 const double *a = a_;
1582 const double *b = b_;
1586 else if (*a == SYSMIS)
1588 else if (*b == SYSMIS)
1596 /* Return strcmp()-type comparison of the MX->n_factors factors at _A and
1597 _B. Sort missing values toward the end. */
1599 compare_factors (const void *a_, const void *b_, void *mx_)
1601 struct matrix_data_pgm *mx = mx_;
1602 struct factor_data *const *pa = a_;
1603 struct factor_data *const *pb = b_;
1604 const double *a = (*pa)->factors;
1605 const double *b = (*pb)->factors;
1607 return lexicographical_compare_3way (a, mx->n_factors,
1610 compare_doubles, NULL);
1613 /* Write out the data for the current split file to the active
1615 Returns true if successful, false if an I/O error occurred. */
1617 wr_output_data (struct wr_aux_data *wr,
1619 write_case_func *write_case, write_case_data wc_data)
1621 struct matrix_data_pgm *mx = wr->mx;
1625 struct variable *const *split;
1629 split_cnt = dict_get_split_cnt (default_dict);
1630 split = dict_get_split_vars (default_dict);
1631 for (i = 0; i < split_cnt; i++)
1632 case_data_rw (c, split[i]->fv)->f = wr->split_values[i];
1635 /* Sort the wr->data list. */
1637 struct factor_data **factors;
1638 struct factor_data *iter;
1641 factors = xnmalloc (mx->cells, sizeof *factors);
1643 for (i = 0, iter = wr->data; iter; iter = iter->next, i++)
1646 sort (factors, mx->cells, sizeof *factors, compare_factors, mx);
1648 wr->data = factors[0];
1649 for (i = 0; i < mx->cells - 1; i++)
1650 factors[i]->next = factors[i + 1];
1651 factors[mx->cells - 1]->next = NULL;
1656 /* Write out records for every set of factor values. */
1658 struct factor_data *iter;
1660 for (iter = wr->data; iter; iter = iter->next)
1665 for (factor = 0; factor < mx->n_factors; factor++)
1666 case_data_rw (c, mx->factors[factor]->fv)->f
1667 = iter->factors[factor];
1673 for (content = 0; content <= PROX; content++)
1675 if (!iter->n_rows[content])
1679 int type = content_type[content];
1680 int n_lines = (type == 1
1682 - (mx->section != FULL && mx->diag == NODIAGONAL))
1685 if (n_lines != iter->n_rows[content])
1687 msg (SE, _("Expected %d lines of data for %s content; "
1688 "actually saw %d lines. No data will be "
1689 "output for this content."),
1690 n_lines, content_names[content],
1691 iter->n_rows[content]);
1696 fill_matrix (mx, content, iter->data[content]);
1698 ok = dump_cell_content (mx, content, iter->data[content],
1699 c, write_case, wc_data);
1707 pool_destroy (mx->container);
1708 mx->container = pool_create ();
1710 wr->data = wr->current = NULL;
1715 /* Sets ROWTYPE_ based on the given TOKEN read from READER.
1718 wr_read_rowtype (struct wr_aux_data *wr,
1719 const struct matrix_token *token,
1720 struct dfm_reader *reader)
1722 if (wr->content != -1)
1724 msg (SE, _("Multiply specified ROWTYPE_ %s."), context (reader));
1727 if (token->type != MSTR)
1729 msg (SE, _("Syntax error %s expecting ROWTYPE_ string."),
1738 memcpy (s, token->string, min (15, token->length));
1739 s[min (15, token->length)] = 0;
1741 for (cp = s; *cp; cp++)
1742 *cp = toupper ((unsigned char) *cp);
1744 wr->content = string_to_content_type (s, NULL);
1747 if (wr->content == -1)
1749 msg (SE, _("Syntax error %s."), context (reader));
1756 /* Read the factors for the current row. Select a set of factors and
1757 point wr_current to it. */
1759 wr_read_factors (struct wr_aux_data *wr)
1761 struct matrix_data_pgm *mx = wr->mx;
1762 double *factor_values = local_alloc (sizeof *factor_values * mx->n_factors);
1768 for (i = 0; i < mx->n_factors; i++)
1770 struct matrix_token token;
1771 if (!mget_token (&token, mx->reader))
1773 if (token.type == MSTR)
1775 if (!wr_read_rowtype (wr, &token, mx->reader))
1777 if (!mget_token (&token, mx->reader))
1780 if (token.type != MNUM)
1782 msg (SE, _("Syntax error expecting factor value %s."),
1783 context (mx->reader));
1787 factor_values[i] = token.number;
1790 if (wr->content == -1)
1792 struct matrix_token token;
1793 if (!mget_token (&token, mx->reader))
1795 if (!wr_read_rowtype (wr, &token, mx->reader))
1799 /* Try the most recent factor first as a simple caching
1805 for (i = 0; i < mx->n_factors; i++)
1806 if (factor_values[i] != wr->current->factors[i])
1811 /* Linear search through the list. */
1814 struct factor_data *iter;
1816 for (iter = wr->data; iter; iter = iter->next)
1820 for (i = 0; i < mx->n_factors; i++)
1821 if (factor_values[i] != iter->factors[i])
1831 /* Not found. Make a new item. */
1833 struct factor_data *new = pool_alloc (mx->container, sizeof *new);
1835 new->factors = pool_nalloc (mx->container,
1836 mx->n_factors, sizeof *new->factors);
1841 for (i = 0; i < mx->n_factors; i++)
1842 new->factors[i] = factor_values[i];
1848 for (i = 0; i <= PROX; i++)
1851 new->data[i] = NULL;
1855 new->next = wr->data;
1856 wr->data = wr->current = new;
1861 local_free (factor_values);
1865 local_free (factor_values);
1869 /* Read the independent variables into wr->current. */
1871 wr_read_indeps (struct wr_aux_data *wr)
1873 struct matrix_data_pgm *mx = wr->mx;
1874 struct factor_data *c = wr->current;
1875 const int type = content_type[wr->content];
1876 const int n_rows = c->n_rows[wr->content];
1880 /* Allocate room for data if necessary. */
1881 if (c->data[wr->content] == NULL)
1883 int n_items = mx->n_continuous;
1885 n_items *= mx->n_continuous;
1887 c->data[wr->content] = pool_nalloc (mx->container,
1888 n_items, sizeof **c->data);
1891 cp = &c->data[wr->content][n_rows * mx->n_continuous];
1893 /* Figure out how much to read from this line. */
1900 msg (SE, _("Duplicate specification for %s."),
1901 content_names[wr->content]);
1905 n_cols = mx->n_continuous;
1910 if (n_rows >= mx->n_continuous - (mx->section != FULL && mx->diag == NODIAGONAL))
1912 msg (SE, _("Too many rows of matrix data for %s."),
1913 content_names[wr->content]);
1917 switch (mx->section)
1920 n_cols = n_rows + 1;
1921 if (mx->diag == NODIAGONAL)
1922 cp += mx->n_continuous;
1926 n_cols = mx->n_continuous - n_rows;
1927 if (mx->diag == NODIAGONAL)
1934 n_cols = mx->n_continuous;
1945 c->n_rows[wr->content]++;
1947 /* Read N_COLS items at CP. */
1951 for (j = 0; j < n_cols; j++)
1953 struct matrix_token token;
1954 if (!mget_token (&token, mx->reader))
1956 if (token.type != MNUM)
1958 msg (SE, _("Syntax error expecting value for %s %s."),
1959 dict_get_var (default_dict, mx->first_continuous + j)->name,
1960 context (mx->reader));
1964 *cp++ = token.number;
1967 && !force_eol (mx->reader, content_names[wr->content]))
1974 /* Matrix source. */
1976 static const struct case_source_class matrix_data_with_rowtype_source_class =
1980 matrix_data_read_with_rowtype,
1984 static const struct case_source_class
1985 matrix_data_without_rowtype_source_class =
1989 matrix_data_read_without_rowtype,