1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
25 #include "algorithm.h"
31 #include "file-handle.h"
39 #include "debug-print.h"
41 /* FIXME: /N subcommand not implemented. It should be pretty simple,
44 /* Format type enums. */
51 /* Matrix section enums. */
59 /* Diagonal inclusion enums. */
87 /* 0=vector, 1=matrix, 2=scalar. */
88 static const int content_type[PROX + 1] =
90 0, 2, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,
93 /* Name of each content type. */
94 static const char *content_names[PROX + 1] =
96 "N", "N", "N_MATRIX", "MEAN", "STDDEV", "COUNT", "MSE",
97 "DFE", "MAT", "COV", "CORR", "PROX",
100 /* A MATRIX DATA input program. */
101 struct matrix_data_pgm
103 struct pool *container; /* Arena used for all allocations. */
104 struct file_handle *data_file; /* The data file to be read. */
107 enum format_type fmt; /* LIST or FREE. */
108 enum matrix_section section;/* LOWER or UPPER or FULL. */
109 enum include_diagonal diag; /* DIAGONAL or NODIAGONAL. */
111 int explicit_rowtype; /* ROWTYPE_ specified explicitly in data? */
112 struct variable *rowtype_, *varname_; /* ROWTYPE_, VARNAME_ variables. */
114 struct variable *single_split; /* Single SPLIT FILE variable. */
116 /* Factor variables. */
117 int n_factors; /* Number of factor variables. */
118 struct variable **factors; /* Factor variables. */
119 int is_per_factor[PROX + 1]; /* Is there per-factor data? */
121 int cells; /* Number of cells, or -1 if none. */
123 int pop_n; /* Population N specified by user. */
125 /* CONTENTS subcommand. */
126 int contents[EOC * 3 + 1]; /* Contents. */
127 int n_contents; /* Number of entries. */
129 /* Continuous variables. */
130 int n_continuous; /* Number of continuous variables. */
131 int first_continuous; /* Index into default_dict.var of
132 first continuous variable. */
135 static const struct case_source_class matrix_data_with_rowtype_source_class;
136 static const struct case_source_class matrix_data_without_rowtype_source_class;
138 static int compare_variables_by_mxd_vartype (const void *pa,
140 static void read_matrices_without_rowtype (struct matrix_data_pgm *);
141 static void read_matrices_with_rowtype (struct matrix_data_pgm *);
142 static int string_to_content_type (char *, int *);
145 static void debug_print (void);
149 cmd_matrix_data (void)
152 struct matrix_data_pgm *mx;
156 discard_variables ();
158 pool = pool_create ();
159 mx = pool_alloc (pool, sizeof *mx);
160 mx->container = pool;
161 mx->data_file = inline_file;
165 mx->explicit_rowtype = 0;
168 mx->single_split = NULL;
171 memset (mx->is_per_factor, 0, sizeof mx->is_per_factor);
175 mx->n_continuous = 0;
176 mx->first_continuous = 0;
181 if (lex_match_id ("VARIABLES"))
188 msg (SE, _("VARIABLES subcommand multiply specified."));
194 if (!parse_DATA_LIST_vars (&v, &nv, PV_NO_DUPLICATE))
200 for (i = 0; i < nv; i++)
201 if (!strcmp (v[i], "VARNAME_"))
203 msg (SE, _("VARNAME_ cannot be explicitly specified on "
205 for (i = 0; i < nv; i++)
215 for (i = 0; i < nv; i++)
217 struct variable *new_var;
219 if (strcmp (v[i], "ROWTYPE_"))
221 new_var = dict_create_var_assert (default_dict, v[i], 0);
222 new_var->p.mxd.vartype = MXD_CONTINUOUS;
223 new_var->p.mxd.subtype = i;
226 mx->explicit_rowtype = 1;
233 mx->rowtype_ = dict_create_var_assert (default_dict,
235 mx->rowtype_->p.mxd.vartype = MXD_ROWTYPE;
236 mx->rowtype_->p.mxd.subtype = 0;
239 else if (lex_match_id ("FILE"))
242 mx->data_file = fh_parse_file_handle ();
243 if (mx->data_file == NULL)
246 else if (lex_match_id ("FORMAT"))
250 while (token == T_ID)
252 if (lex_match_id ("LIST"))
254 else if (lex_match_id ("FREE"))
256 else if (lex_match_id ("LOWER"))
258 else if (lex_match_id ("UPPER"))
260 else if (lex_match_id ("FULL"))
262 else if (lex_match_id ("DIAGONAL"))
264 else if (lex_match_id ("NODIAGONAL"))
265 mx->diag = NODIAGONAL;
268 lex_error (_("in FORMAT subcommand"));
273 else if (lex_match_id ("SPLIT"))
279 msg (SE, _("SPLIT subcommand multiply specified."));
286 lex_error (_("in SPLIT subcommand"));
290 if (dict_lookup_var (default_dict, tokid) == NULL
291 && (lex_look_ahead () == '.' || lex_look_ahead () == '/'))
293 if (!strcmp (tokid, "ROWTYPE_") || !strcmp (tokid, "VARNAME_"))
295 msg (SE, _("Split variable may not be named ROWTYPE_ "
300 mx->single_split = dict_create_var_assert (default_dict,
304 mx->single_split->p.mxd.vartype = MXD_CONTINUOUS;
306 dict_set_split_vars (default_dict, &mx->single_split, 1);
310 struct variable **split;
313 if (!parse_variables (default_dict, &split, &n, PV_NO_DUPLICATE))
316 dict_set_split_vars (default_dict, split, n);
320 struct variable *const *split = dict_get_split_vars (default_dict);
321 size_t split_cnt = dict_get_split_cnt (default_dict);
324 for (i = 0; i < split_cnt; i++)
326 if (split[i]->p.mxd.vartype != MXD_CONTINUOUS)
328 msg (SE, _("Split variable %s is already another type."),
332 split[i]->p.mxd.vartype = MXD_SPLIT;
333 split[i]->p.mxd.subtype = i;
337 else if (lex_match_id ("FACTORS"))
343 msg (SE, _("FACTORS subcommand multiply specified."));
348 if (!parse_variables (default_dict, &mx->factors, &mx->n_factors, PV_NONE))
354 for (i = 0; i < mx->n_factors; i++)
356 if (mx->factors[i]->p.mxd.vartype != MXD_CONTINUOUS)
358 msg (SE, _("Factor variable %s is already another type."),
362 mx->factors[i]->p.mxd.vartype = MXD_FACTOR;
363 mx->factors[i]->p.mxd.subtype = i;
367 else if (lex_match_id ("CELLS"))
373 msg (SE, _("CELLS subcommand multiply specified."));
377 if (!lex_integer_p () || lex_integer () < 1)
379 lex_error (_("expecting positive integer"));
383 mx->cells = lex_integer ();
386 else if (lex_match_id ("N"))
392 msg (SE, _("N subcommand multiply specified."));
396 if (!lex_integer_p () || lex_integer () < 1)
398 lex_error (_("expecting positive integer"));
402 mx->pop_n = lex_integer ();
405 else if (lex_match_id ("CONTENTS"))
407 int inside_parens = 0;
408 unsigned collide = 0;
413 msg (SE, _("CONTENTS subcommand multiply specified."));
423 for (i = 0; i <= PROX; i++)
424 mx->is_per_factor[i] = 0;
433 msg (SE, _("Nested parentheses not allowed."));
439 else if (lex_match (')'))
443 msg (SE, _("Mismatched right parenthesis (`(')."));
446 if (mx->contents[mx->n_contents - 1] == LPAREN)
448 msg (SE, _("Empty parentheses not allowed."));
461 lex_error (_("in CONTENTS subcommand"));
465 content_type = string_to_content_type (tokid,
467 if (content_type == -1)
469 lex_error (_("in CONTENTS subcommand"));
474 if (collide & (1 << collide_index))
476 msg (SE, _("Content multiply specified for %s."),
477 content_names[content_type]);
480 collide |= (1 << collide_index);
483 mx->is_per_factor[item] = inside_parens;
485 mx->contents[mx->n_contents++] = item;
487 if (token == '/' || token == '.')
493 msg (SE, _("Missing right parenthesis."));
496 mx->contents[mx->n_contents] = EOC;
507 lex_error (_("expecting end of command"));
513 msg (SE, _("Missing VARIABLES subcommand."));
517 if (!mx->n_contents && !mx->explicit_rowtype)
519 msg (SW, _("CONTENTS subcommand not specified: assuming file "
520 "contains only CORR matrix."));
522 mx->contents[0] = CORR;
523 mx->contents[1] = EOC;
527 if (mx->n_factors && !mx->explicit_rowtype && mx->cells == -1)
529 msg (SE, _("Missing CELLS subcommand. CELLS is required "
530 "when ROWTYPE_ is not given in the data and "
531 "factors are present."));
535 if (mx->explicit_rowtype && mx->single_split)
537 msg (SE, _("Split file values must be present in the data when "
538 "ROWTYPE_ is present."));
542 /* Create VARNAME_. */
544 mx->varname_ = dict_create_var_assert (default_dict, "VARNAME_", 8);
545 mx->varname_->p.mxd.vartype = MXD_VARNAME;
546 mx->varname_->p.mxd.subtype = 0;
549 /* Sort the dictionary variables into the desired order for the
550 system file output. */
555 dict_get_vars (default_dict, &v, &nv, 0);
556 qsort (v, nv, sizeof *v, compare_variables_by_mxd_vartype);
557 dict_reorder_vars (default_dict, v, nv);
563 static const struct fmt_spec fmt_tab[MXD_COUNT] =
574 mx->first_continuous = -1;
575 for (i = 0; i < dict_get_var_cnt (default_dict); i++)
577 struct variable *v = dict_get_var (default_dict, i);
578 int type = v->p.mxd.vartype;
580 assert (type >= 0 && type < MXD_COUNT);
581 v->print = v->write = fmt_tab[type];
583 if (type == MXD_CONTINUOUS)
585 if (mx->first_continuous == -1 && type == MXD_CONTINUOUS)
586 mx->first_continuous = i;
590 if (mx->n_continuous == 0)
592 msg (SE, _("No continuous variables specified."));
600 if (!dfm_open_for_reading (mx->data_file))
603 if (mx->explicit_rowtype)
604 read_matrices_with_rowtype (mx);
606 read_matrices_without_rowtype (mx);
608 pool_destroy (mx->container);
613 discard_variables ();
615 pool_destroy (mx->container);
619 /* Look up string S as a content-type name and return the
620 corresponding enumerated value, or -1 if there is no match. If
621 COLLIDE is non-NULL then *COLLIDE returns a value (suitable for use
622 as a bit-index) which can be used for determining whether a related
623 statistic has already been used. */
625 string_to_content_type (char *s, int *collide)
636 {N_VECTOR, 0, "N_VECTOR"},
638 {N_SCALAR, 0, "N_SCALAR"},
639 {N_MATRIX, 1, "N_MATRIX"},
641 {STDDEV, 3, "STDDEV"},
653 for (tp = tab; tp->value != -1; tp++)
654 if (!strcmp (s, tp->string))
657 *collide = tp->collide;
664 /* Compare two variables using p.mxd.vartype and p.mxd.subtype
667 compare_variables_by_mxd_vartype (const void *a_, const void *b_)
669 struct variable *const *pa = a_;
670 struct variable *const *pb = b_;
671 const struct matrix_data_proc *a = &(*pa)->p.mxd;
672 const struct matrix_data_proc *b = &(*pb)->p.mxd;
674 if (a->vartype != b->vartype)
675 return a->vartype > b->vartype ? 1 : -1;
677 return a->subtype < b->subtype ? -1 : a->subtype > b->subtype;
681 /* Print out the command as input. */
685 printf ("MATRIX DATA\n\t/VARIABLES=");
690 for (i = 0; i < default_dict.nvar; i++)
691 printf ("%s ", default_dict.var[i]->name);
695 printf ("\t/FORMAT=");
698 else if (fmt == FREE)
702 if (section == LOWER)
704 else if (section == UPPER)
706 else if (section == FULL)
710 if (diag == DIAGONAL)
711 printf (" DIAGONAL\n");
712 else if (diag == NODIAGONAL)
713 printf (" NODIAGONAL\n");
717 if (dict_get_split_cnt (default_dict) != 0)
721 printf ("\t/SPLIT=");
722 for (i = 0; i < dict_get_split_cnt (default_dict); i++)
723 printf ("%s ", dict_get_split_vars (default_dict)[i]->name);
725 printf ("\t/* single split");
733 printf ("\t/FACTORS=");
734 for (i = 0; i < n_factors; i++)
735 printf ("%s ", factors[i]->name);
740 printf ("\t/CELLS=%d\n", cells);
743 printf ("\t/N=%d\n", mx->pop_n);
750 printf ("\t/CONTENTS=");
751 for (i = 0; i < mx->n_contents; i++)
753 if (mx->contents[i] == LPAREN)
760 else if (mx->contents[i] == RPAREN)
768 assert (mx->contents[i] >= 0 && mx->contents[i] <= PROX);
771 printf ("%s", content_names[mx->contents[i]]);
778 #endif /* DEBUGGING */
780 /* Matrix tokenizer. */
782 /* Matrix token types. */
783 enum matrix_token_type
789 /* A MATRIX DATA parsing token. */
792 enum matrix_token_type type;
793 double number; /* MNUM: token value. */
794 char *string; /* MSTR: token string; not null-terminated. */
795 int length; /* MSTR: tokstr length. */
798 static int mget_token (struct matrix_token *, struct file_handle *);
801 #define mget_token(TOKEN, HANDLE) mget_token_dump(TOKEN, HANDLE)
804 mdump_token (const struct matrix_token *token)
809 printf (" #%g", token->number);
812 printf (" '%.*s'", token->length, token->string);
821 mget_token_dump (struct matrix_token *token, struct file_handle *data_file)
823 int result = (mget_token) (token, data_file);
829 /* Return the current position in DATA_FILE. */
831 context (struct file_handle *data_file)
835 if (dfm_eof (data_file))
836 strcpy (buf, "at end of file");
839 struct len_string line;
842 dfm_get_record (data_file, &line);
843 sp = ls_c_str (&line);
844 while (sp < ls_end (&line) && isspace ((unsigned char) *sp))
846 if (sp >= ls_end (&line))
847 strcpy (buf, "at end of line");
853 dp = stpcpy (buf, "before `");
854 while (sp < ls_end (&line) && !isspace ((unsigned char) *sp)
867 /* Is there at least one token left in the data file? */
869 another_token (struct file_handle *data_file)
873 struct len_string line;
876 if (dfm_eof (data_file))
878 dfm_get_record (data_file, &line);
880 cp = ls_c_str (&line);
881 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
884 if (cp < ls_end (&line))
886 dfm_forward_columns (data_file, cp - ls_c_str (&line));
890 dfm_forward_record (data_file);
894 /* Parse a MATRIX DATA token from mx->data_file into TOKEN. */
896 (mget_token) (struct matrix_token *token, struct file_handle *data_file)
898 struct len_string line;
902 if (!another_token (data_file))
905 dfm_get_record (data_file, &line);
906 first_column = dfm_column_start (data_file);
908 /* Three types of fields: quoted with ', quoted with ", unquoted. */
909 cp = ls_c_str (&line);
910 if (*cp == '\'' || *cp == '"')
915 token->string = ++cp;
916 while (cp < ls_end (&line) && *cp != quote)
918 token->length = cp - token->string;
919 if (cp < ls_end (&line))
922 msg (SW, _("Scope of string exceeds line."));
926 int is_num = isdigit ((unsigned char) *cp) || *cp == '.';
928 token->string = cp++;
929 while (cp < ls_end (&line)
930 && !isspace ((unsigned char) *cp) && *cp != ','
931 && *cp != '-' && *cp != '+')
933 if (isdigit ((unsigned char) *cp))
936 if ((tolower ((unsigned char) *cp) == 'd'
937 || tolower ((unsigned char) *cp) == 'e')
938 && (cp[1] == '+' || cp[1] == '-'))
944 token->length = cp - token->string;
945 assert (token->length);
951 di.s = token->string;
952 di.e = token->string + token->length;
953 di.v = (union value *) &token->number;
954 di.f1 = first_column;
955 di.format.type = FMT_F;
956 di.format.w = token->length;
966 dfm_forward_columns (data_file, cp - ls_c_str (&line));
971 /* Forcibly skip the end of a line for content type CONTENT in
974 force_eol (struct file_handle *data_file, const char *content)
976 struct len_string line;
979 if (dfm_eof (data_file))
981 dfm_get_record (data_file, &line);
983 cp = ls_c_str (&line);
984 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
987 if (cp < ls_end (&line))
989 msg (SE, _("End of line expected %s while reading %s."),
990 context (data_file), content);
994 dfm_forward_record (data_file);
998 /* Back end, omitting ROWTYPE_. */
1002 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1003 double ***data; /* MATRIX DATA data. */
1004 double *factor_values; /* Factor values. */
1005 int max_cell_idx; /* Max-numbered cell that we have
1006 read so far, plus one. */
1007 double *split_values; /* SPLIT FILE variable values. */
1010 static int nr_read_splits (struct nr_aux_data *, int compare);
1011 static int nr_read_factors (struct nr_aux_data *, int cell);
1012 static void nr_output_data (struct nr_aux_data *, struct ccase *,
1013 write_case_func *, write_case_data);
1014 static void matrix_data_read_without_rowtype (struct case_source *source,
1019 /* Read from the data file and write it to the active file. */
1021 read_matrices_without_rowtype (struct matrix_data_pgm *mx)
1023 struct nr_aux_data nr;
1025 if (mx->cells == -1)
1030 nr.factor_values = xmalloc (sizeof *nr.factor_values * mx->n_factors * mx->cells);
1031 nr.max_cell_idx = 0;
1032 nr.split_values = xmalloc (sizeof *nr.split_values
1033 * dict_get_split_cnt (default_dict));
1035 vfm_source = create_case_source (&matrix_data_without_rowtype_source_class,
1038 procedure (NULL, &nr);
1040 free (nr.split_values);
1041 free (nr.factor_values);
1043 fh_close_handle (mx->data_file);
1046 /* Mirror data across the diagonal of matrix CP which contains
1047 CONTENT type data. */
1049 fill_matrix (struct matrix_data_pgm *mx, int content, double *cp)
1051 int type = content_type[content];
1053 if (type == 1 && mx->section != FULL)
1055 if (mx->diag == NODIAGONAL)
1057 const double fill = content == CORR ? 1.0 : SYSMIS;
1060 for (i = 0; i < mx->n_continuous; i++)
1061 cp[i * (1 + mx->n_continuous)] = fill;
1067 if (mx->section == LOWER)
1069 int n_lines = mx->n_continuous;
1070 if (mx->section != FULL && mx->diag == NODIAGONAL)
1073 for (r = 1; r < n_lines; r++)
1074 for (c = 0; c < r; c++)
1075 cp[r + c * mx->n_continuous] = cp[c + r * mx->n_continuous];
1079 assert (mx->section == UPPER);
1080 for (r = 1; r < mx->n_continuous; r++)
1081 for (c = 0; c < r; c++)
1082 cp[c + r * mx->n_continuous] = cp[r + c * mx->n_continuous];
1090 for (c = 1; c < mx->n_continuous; c++)
1095 /* Read data lines for content type CONTENT from the data file.
1096 If PER_FACTOR is nonzero, then factor information is read from
1097 the data file. Data is for cell number CELL. */
1099 nr_read_data_lines (struct nr_aux_data *nr,
1100 int per_factor, int cell, int content, int compare)
1102 struct matrix_data_pgm *mx = nr->mx;
1103 const int type = content_type[content]; /* Content type. */
1104 int n_lines; /* Number of lines to parse from data file for this type. */
1105 double *cp; /* Current position in vector or matrix. */
1112 n_lines = mx->n_continuous;
1113 if (mx->section != FULL && mx->diag == NODIAGONAL)
1117 cp = nr->data[content][cell];
1118 if (type == 1 && mx->section == LOWER && mx->diag == NODIAGONAL)
1119 cp += mx->n_continuous;
1121 for (i = 0; i < n_lines; i++)
1125 if (!nr_read_splits (nr, 1))
1127 if (per_factor && !nr_read_factors (nr, cell))
1134 n_cols = mx->n_continuous;
1137 switch (mx->section)
1144 n_cols = mx->n_continuous - i;
1145 if (mx->diag == NODIAGONAL)
1152 n_cols = mx->n_continuous;
1170 for (j = 0; j < n_cols; j++)
1172 struct matrix_token token;
1173 if (!mget_token (&token, mx->data_file))
1175 if (token.type != MNUM)
1177 msg (SE, _("expecting value for %s %s"),
1178 dict_get_var (default_dict, j)->name,
1179 context (mx->data_file));
1183 *cp++ = token.number;
1186 && !force_eol (mx->data_file, content_names[content]))
1188 debug_printf (("\n"));
1191 if (mx->section == LOWER)
1192 cp += mx->n_continuous - n_cols;
1195 fill_matrix (mx, content, nr->data[content][cell]);
1200 /* When ROWTYPE_ does not appear in the data, reads the matrices and
1201 writes them to the output file. Returns success. */
1203 matrix_data_read_without_rowtype (struct case_source *source,
1205 write_case_func *write_case,
1206 write_case_data wc_data)
1208 struct nr_aux_data *nr = source->aux;
1209 struct matrix_data_pgm *mx = nr->mx;
1214 nr->data = pool_alloc (mx->container, (PROX + 1) * sizeof *nr->data);
1219 for (i = 0; i <= PROX; i++)
1223 for (cp = mx->contents; *cp != EOC; cp++)
1224 if (*cp != LPAREN && *cp != RPAREN)
1226 int per_factor = mx->is_per_factor[*cp];
1229 n_entries = mx->n_continuous;
1230 if (content_type[*cp] == 1)
1231 n_entries *= mx->n_continuous;
1234 int n_vectors = per_factor ? mx->cells : 1;
1237 nr->data[*cp] = pool_alloc (mx->container,
1238 n_vectors * sizeof **nr->data);
1240 for (i = 0; i < n_vectors; i++)
1241 nr->data[*cp][i] = pool_alloc (mx->container,
1242 n_entries * sizeof ***nr->data);
1251 if (!nr_read_splits (nr, 0))
1254 for (bp = mx->contents; *bp != EOC; bp = np)
1258 /* Trap the CONTENTS that we should parse in this pass
1259 between bp and ep. Set np to the starting bp for next
1264 while (*ep != RPAREN)
1272 while (*ep != EOC && *ep != LPAREN)
1281 for (i = 0; i < (per_factor ? mx->cells : 1); i++)
1285 for (cp = bp; cp < ep; cp++)
1286 if (!nr_read_data_lines (nr, per_factor, i, *cp, cp != bp))
1292 nr_output_data (nr, c, write_case, wc_data);
1294 if (dict_get_split_cnt (default_dict) == 0
1295 || !another_token (mx->data_file))
1300 /* Read the split file variables. If COMPARE is 1, compares the
1301 values read to the last values read and returns 1 if they're equal,
1304 nr_read_splits (struct nr_aux_data *nr, int compare)
1306 struct matrix_data_pgm *mx = nr->mx;
1307 static int just_read = 0; /* FIXME: WTF? */
1311 if (compare && just_read)
1317 if (dict_get_split_vars (default_dict) == NULL)
1320 if (mx->single_split)
1324 = ++dict_get_split_vars (default_dict)[0]->p.mxd.subtype;
1331 split_cnt = dict_get_split_cnt (default_dict);
1332 for (i = 0; i < split_cnt; i++)
1334 struct matrix_token token;
1335 if (!mget_token (&token, mx->data_file))
1337 if (token.type != MNUM)
1339 msg (SE, _("Syntax error expecting SPLIT FILE value %s."),
1340 context (mx->data_file));
1345 nr->split_values[i] = token.number;
1346 else if (nr->split_values[i] != token.number)
1348 msg (SE, _("Expecting value %g for %s."),
1349 nr->split_values[i],
1350 dict_get_split_vars (default_dict)[i]->name);
1358 /* Read the factors for cell CELL. If COMPARE is 1, compares the
1359 values read to the last values read and returns 1 if they're equal,
1362 nr_read_factors (struct nr_aux_data *nr, int cell)
1364 struct matrix_data_pgm *mx = nr->mx;
1367 if (mx->n_factors == 0)
1370 assert (nr->max_cell_idx >= cell);
1371 if (cell != nr->max_cell_idx)
1382 for (i = 0; i < mx->n_factors; i++)
1384 struct matrix_token token;
1385 if (!mget_token (&token, mx->data_file))
1387 if (token.type != MNUM)
1389 msg (SE, _("Syntax error expecting factor value %s."),
1390 context (mx->data_file));
1395 nr->factor_values[i + mx->n_factors * cell] = token.number;
1396 else if (nr->factor_values[i + mx->n_factors * cell] != token.number)
1398 msg (SE, _("Syntax error expecting value %g for %s %s."),
1399 nr->factor_values[i + mx->n_factors * cell],
1400 mx->factors[i]->name, context (mx->data_file));
1409 /* Write the contents of a cell having content type CONTENT and data
1410 CP to the active file. */
1412 dump_cell_content (struct matrix_data_pgm *mx, int content, double *cp,
1414 write_case_func *write_case, write_case_data wc_data)
1416 int type = content_type[content];
1419 st_bare_pad_copy (c->data[mx->rowtype_->fv].s,
1420 content_names[content], 8);
1423 memset (&c->data[mx->varname_->fv].s, ' ', 8);
1427 int n_lines = (type == 1) ? mx->n_continuous : 1;
1430 for (i = 0; i < n_lines; i++)
1434 for (j = 0; j < mx->n_continuous; j++)
1436 int fv = dict_get_var (default_dict, mx->first_continuous + j)->fv;
1437 c->data[fv].f = *cp;
1441 st_bare_pad_copy (c->data[mx->varname_->fv].s,
1442 dict_get_var (default_dict,
1443 mx->first_continuous + i)->name,
1445 write_case (wc_data);
1450 /* Finally dump out everything from nr_data[] to the output file. */
1452 nr_output_data (struct nr_aux_data *nr, struct ccase *c,
1453 write_case_func *write_case, write_case_data wc_data)
1455 struct matrix_data_pgm *mx = nr->mx;
1458 struct variable *const *split;
1462 split_cnt = dict_get_split_cnt (default_dict);
1463 split = dict_get_split_vars (default_dict);
1464 for (i = 0; i < split_cnt; i++)
1465 c->data[split[i]->fv].f = nr->split_values[i];
1472 for (cell = 0; cell < mx->cells; cell++)
1477 for (factor = 0; factor < mx->n_factors; factor++)
1479 c->data[mx->factors[factor]->fv].f
1480 = nr->factor_values[factor + cell * mx->n_factors];
1481 debug_printf (("f:%s ", mx->factors[factor]->name));
1488 for (content = 0; content <= PROX; content++)
1489 if (mx->is_per_factor[content])
1491 assert (nr->data[content] != NULL
1492 && nr->data[content][cell] != NULL);
1494 dump_cell_content (mx, content, nr->data[content][cell],
1495 c, write_case, wc_data);
1507 for (factor = 0; factor < mx->n_factors; factor++)
1508 c->data[mx->factors[factor]->fv].f = SYSMIS;
1511 for (content = 0; content <= PROX; content++)
1512 if (!mx->is_per_factor[content] && nr->data[content] != NULL)
1513 dump_cell_content (mx, content, nr->data[content][0],
1514 c, write_case, wc_data);
1518 /* Back end, with ROWTYPE_. */
1520 /* All the data for one set of factor values. */
1524 int n_rows[PROX + 1];
1525 double *data[PROX + 1];
1526 struct factor_data *next;
1529 /* With ROWTYPE_ auxiliary data. */
1532 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1533 int content; /* Type of current row. */
1534 double *split_values; /* SPLIT FILE variable values. */
1535 struct factor_data *data; /* All the data. */
1536 struct factor_data *current; /* Current factor. */
1539 static int wr_read_splits (struct wr_aux_data *, struct ccase *,
1540 write_case_func *, write_case_data);
1541 static int wr_output_data (struct wr_aux_data *, struct ccase *,
1542 write_case_func *, write_case_data);
1543 static int wr_read_rowtype (struct wr_aux_data *,
1544 const struct matrix_token *, struct file_handle *);
1545 static int wr_read_factors (struct wr_aux_data *);
1546 static int wr_read_indeps (struct wr_aux_data *);
1547 static void matrix_data_read_with_rowtype (struct case_source *,
1552 /* When ROWTYPE_ appears in the data, reads the matrices and writes
1553 them to the output file. */
1555 read_matrices_with_rowtype (struct matrix_data_pgm *mx)
1557 struct wr_aux_data wr;
1561 wr.split_values = NULL;
1566 vfm_source = create_case_source (&matrix_data_with_rowtype_source_class,
1568 procedure (NULL, &wr);
1570 free (wr.split_values);
1571 fh_close_handle (mx->data_file);
1574 /* Read from the data file and write it to the active file. */
1576 matrix_data_read_with_rowtype (struct case_source *source,
1578 write_case_func *write_case,
1579 write_case_data wc_data)
1581 struct wr_aux_data *wr = source->aux;
1582 struct matrix_data_pgm *mx = wr->mx;
1586 if (!wr_read_splits (wr, c, write_case, wc_data))
1589 if (!wr_read_factors (wr))
1592 if (!wr_read_indeps (wr))
1595 while (another_token (mx->data_file));
1597 wr_output_data (wr, c, write_case, wc_data);
1600 /* Read the split file variables. If they differ from the previous
1601 set of split variables then output the data. Returns success. */
1603 wr_read_splits (struct wr_aux_data *wr,
1605 write_case_func *write_case, write_case_data wc_data)
1607 struct matrix_data_pgm *mx = wr->mx;
1611 split_cnt = dict_get_split_cnt (default_dict);
1615 if (wr->split_values)
1620 wr->split_values = xmalloc (split_cnt * sizeof *wr->split_values);
1627 for (i = 0; i < split_cnt; i++)
1629 struct matrix_token token;
1630 if (!mget_token (&token, mx->data_file))
1632 if (token.type != MNUM)
1634 msg (SE, _("Syntax error %s expecting SPLIT FILE value."),
1635 context (mx->data_file));
1639 if (compare && wr->split_values[i] != token.number && !different)
1641 if (!wr_output_data (wr, c, write_case, wc_data))
1646 wr->split_values[i] = token.number;
1653 /* Compares doubles A and B, treating SYSMIS as greatest. */
1655 compare_doubles (const void *a_, const void *b_, void *aux UNUSED)
1657 const double *a = a_;
1658 const double *b = b_;
1662 else if (*a == SYSMIS)
1664 else if (*b == SYSMIS)
1672 /* Return strcmp()-type comparison of the MX->n_factors factors at _A and
1673 _B. Sort missing values toward the end. */
1675 compare_factors (const void *a_, const void *b_, void *mx_)
1677 struct matrix_data_pgm *mx = mx_;
1678 struct factor_data *const *pa = a_;
1679 struct factor_data *const *pb = b_;
1680 const double *a = (*pa)->factors;
1681 const double *b = (*pb)->factors;
1683 return lexicographical_compare_3way (a, mx->n_factors,
1686 compare_doubles, NULL);
1689 /* Write out the data for the current split file to the active
1692 wr_output_data (struct wr_aux_data *wr,
1694 write_case_func *write_case, write_case_data wc_data)
1696 struct matrix_data_pgm *mx = wr->mx;
1699 struct variable *const *split;
1703 split_cnt = dict_get_split_cnt (default_dict);
1704 split = dict_get_split_vars (default_dict);
1705 for (i = 0; i < split_cnt; i++)
1706 c->data[split[i]->fv].f = wr->split_values[i];
1709 /* Sort the wr->data list. */
1711 struct factor_data **factors;
1712 struct factor_data *iter;
1715 factors = xmalloc (sizeof *factors * mx->cells);
1717 for (i = 0, iter = wr->data; iter; iter = iter->next, i++)
1720 sort (factors, mx->cells, sizeof *factors, compare_factors, mx);
1722 wr->data = factors[0];
1723 for (i = 0; i < mx->cells - 1; i++)
1724 factors[i]->next = factors[i + 1];
1725 factors[mx->cells - 1]->next = NULL;
1730 /* Write out records for every set of factor values. */
1732 struct factor_data *iter;
1734 for (iter = wr->data; iter; iter = iter->next)
1739 for (factor = 0; factor < mx->n_factors; factor++)
1741 c->data[mx->factors[factor]->fv].f
1742 = iter->factors[factor];
1743 debug_printf (("f:%s ", factors[factor]->name));
1750 for (content = 0; content <= PROX; content++)
1752 if (!iter->n_rows[content])
1756 int type = content_type[content];
1757 int n_lines = (type == 1
1759 - (mx->section != FULL && mx->diag == NODIAGONAL))
1762 if (n_lines != iter->n_rows[content])
1764 msg (SE, _("Expected %d lines of data for %s content; "
1765 "actually saw %d lines. No data will be "
1766 "output for this content."),
1767 n_lines, content_names[content],
1768 iter->n_rows[content]);
1773 fill_matrix (mx, content, iter->data[content]);
1775 dump_cell_content (mx, content, iter->data[content],
1776 c, write_case, wc_data);
1782 pool_destroy (mx->container);
1783 mx->container = pool_create ();
1785 wr->data = wr->current = NULL;
1790 /* Sets ROWTYPE_ based on the given TOKEN read from DATA_FILE.
1793 wr_read_rowtype (struct wr_aux_data *wr,
1794 const struct matrix_token *token,
1795 struct file_handle *data_file)
1797 if (wr->content != -1)
1799 msg (SE, _("Multiply specified ROWTYPE_ %s."), context (data_file));
1802 if (token->type != MSTR)
1804 msg (SE, _("Syntax error %s expecting ROWTYPE_ string."),
1805 context (data_file));
1813 memcpy (s, token->string, min (15, token->length));
1814 s[min (15, token->length)] = 0;
1816 for (cp = s; *cp; cp++)
1817 *cp = toupper ((unsigned char) *cp);
1819 wr->content = string_to_content_type (s, NULL);
1822 if (wr->content == -1)
1824 msg (SE, _("Syntax error %s."), context (data_file));
1831 /* Read the factors for the current row. Select a set of factors and
1832 point wr_current to it. */
1834 wr_read_factors (struct wr_aux_data *wr)
1836 struct matrix_data_pgm *mx = wr->mx;
1837 double *factor_values = local_alloc (sizeof *factor_values * mx->n_factors);
1843 for (i = 0; i < mx->n_factors; i++)
1845 struct matrix_token token;
1846 if (!mget_token (&token, mx->data_file))
1848 if (token.type == MSTR)
1850 if (!wr_read_rowtype (wr, &token, mx->data_file))
1852 if (!mget_token (&token, mx->data_file))
1855 if (token.type != MNUM)
1857 msg (SE, _("Syntax error expecting factor value %s."),
1858 context (mx->data_file));
1862 factor_values[i] = token.number;
1865 if (wr->content == -1)
1867 struct matrix_token token;
1868 if (!mget_token (&token, mx->data_file))
1870 if (!wr_read_rowtype (wr, &token, mx->data_file))
1874 /* Try the most recent factor first as a simple caching
1880 for (i = 0; i < mx->n_factors; i++)
1881 if (factor_values[i] != wr->current->factors[i])
1886 /* Linear search through the list. */
1889 struct factor_data *iter;
1891 for (iter = wr->data; iter; iter = iter->next)
1895 for (i = 0; i < mx->n_factors; i++)
1896 if (factor_values[i] != iter->factors[i])
1906 /* Not found. Make a new item. */
1908 struct factor_data *new = pool_alloc (mx->container, sizeof *new);
1910 new->factors = pool_alloc (mx->container, sizeof *new->factors * mx->n_factors);
1915 for (i = 0; i < mx->n_factors; i++)
1916 new->factors[i] = factor_values[i];
1922 for (i = 0; i <= PROX; i++)
1925 new->data[i] = NULL;
1929 new->next = wr->data;
1930 wr->data = wr->current = new;
1935 local_free (factor_values);
1939 local_free (factor_values);
1943 /* Read the independent variables into wr->current. */
1945 wr_read_indeps (struct wr_aux_data *wr)
1947 struct matrix_data_pgm *mx = wr->mx;
1948 struct factor_data *c = wr->current;
1949 const int type = content_type[wr->content];
1950 const int n_rows = c->n_rows[wr->content];
1954 /* Allocate room for data if necessary. */
1955 if (c->data[wr->content] == NULL)
1957 int n_items = mx->n_continuous;
1959 n_items *= mx->n_continuous;
1961 c->data[wr->content] = pool_alloc (mx->container,
1962 sizeof **c->data * n_items);
1965 cp = &c->data[wr->content][n_rows * mx->n_continuous];
1967 /* Figure out how much to read from this line. */
1974 msg (SE, _("Duplicate specification for %s."),
1975 content_names[wr->content]);
1979 n_cols = mx->n_continuous;
1984 if (n_rows >= mx->n_continuous - (mx->section != FULL && mx->diag == NODIAGONAL))
1986 msg (SE, _("Too many rows of matrix data for %s."),
1987 content_names[wr->content]);
1991 switch (mx->section)
1994 n_cols = n_rows + 1;
1995 if (mx->diag == NODIAGONAL)
1996 cp += mx->n_continuous;
2000 n_cols = mx->n_continuous - n_rows;
2001 if (mx->diag == NODIAGONAL)
2008 n_cols = mx->n_continuous;
2019 c->n_rows[wr->content]++;
2021 debug_printf ((" (c=%p,r=%d,n=%d)", c, n_rows + 1, n_cols));
2023 /* Read N_COLS items at CP. */
2027 for (j = 0; j < n_cols; j++)
2029 struct matrix_token token;
2030 if (!mget_token (&token, mx->data_file))
2032 if (token.type != MNUM)
2034 msg (SE, _("Syntax error expecting value for %s %s."),
2035 dict_get_var (default_dict, mx->first_continuous + j)->name,
2036 context (mx->data_file));
2040 *cp++ = token.number;
2043 && !force_eol (mx->data_file, content_names[wr->content]))
2045 debug_printf (("\n"));
2051 /* Matrix source. */
2053 static const struct case_source_class matrix_data_with_rowtype_source_class =
2057 matrix_data_read_with_rowtype,
2061 static const struct case_source_class
2062 matrix_data_without_rowtype_source_class =
2066 matrix_data_read_without_rowtype,