1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
25 #include "algorithm.h"
31 #include "dictionary.h"
33 #include "file-handle.h"
41 #include "debug-print.h"
43 /* FIXME: /N subcommand not implemented. It should be pretty simple,
46 /* Different types of variables for MATRIX DATA procedure. Order is
47 important: these are used for sort keys. */
50 MXD_SPLIT, /* SPLIT FILE variables. */
51 MXD_ROWTYPE, /* ROWTYPE_. */
52 MXD_FACTOR, /* Factor variables. */
53 MXD_VARNAME, /* VARNAME_. */
54 MXD_CONTINUOUS, /* Continuous variables. */
59 /* Format type enums. */
66 /* Matrix section enums. */
74 /* Diagonal inclusion enums. */
102 /* 0=vector, 1=matrix, 2=scalar. */
103 static const int content_type[PROX + 1] =
105 0, 2, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,
108 /* Name of each content type. */
109 static const char *content_names[PROX + 1] =
111 "N", "N", "N_MATRIX", "MEAN", "STDDEV", "COUNT", "MSE",
112 "DFE", "MAT", "COV", "CORR", "PROX",
115 /* A MATRIX DATA input program. */
116 struct matrix_data_pgm
118 struct pool *container; /* Arena used for all allocations. */
119 struct dfm_reader *reader; /* Data file to read. */
122 enum format_type fmt; /* LIST or FREE. */
123 enum matrix_section section;/* LOWER or UPPER or FULL. */
124 enum include_diagonal diag; /* DIAGONAL or NODIAGONAL. */
126 int explicit_rowtype; /* ROWTYPE_ specified explicitly in data? */
127 struct variable *rowtype_, *varname_; /* ROWTYPE_, VARNAME_ variables. */
129 struct variable *single_split; /* Single SPLIT FILE variable. */
131 /* Factor variables. */
132 int n_factors; /* Number of factor variables. */
133 struct variable **factors; /* Factor variables. */
134 int is_per_factor[PROX + 1]; /* Is there per-factor data? */
136 int cells; /* Number of cells, or -1 if none. */
138 int pop_n; /* Population N specified by user. */
140 /* CONTENTS subcommand. */
141 int contents[EOC * 3 + 1]; /* Contents. */
142 int n_contents; /* Number of entries. */
144 /* Continuous variables. */
145 int n_continuous; /* Number of continuous variables. */
146 int first_continuous; /* Index into default_dict.var of
147 first continuous variable. */
150 /* Auxiliary data attached to MATRIX DATA variables. */
153 int var_type; /* Variable type. */
154 int sub_type; /* Subtype. */
157 static const struct case_source_class matrix_data_with_rowtype_source_class;
158 static const struct case_source_class matrix_data_without_rowtype_source_class;
160 static int compare_variables_by_mxd_var_type (const void *pa,
162 static void read_matrices_without_rowtype (struct matrix_data_pgm *);
163 static void read_matrices_with_rowtype (struct matrix_data_pgm *);
164 static int string_to_content_type (char *, int *);
165 static void attach_mxd_aux (struct variable *, int var_type, int sub_type);
168 cmd_matrix_data (void)
171 struct matrix_data_pgm *mx;
172 struct file_handle *fh = NULL;
176 discard_variables ();
178 pool = pool_create ();
179 mx = pool_alloc (pool, sizeof *mx);
180 mx->container = pool;
185 mx->explicit_rowtype = 0;
188 mx->single_split = NULL;
191 memset (mx->is_per_factor, 0, sizeof mx->is_per_factor);
195 mx->n_continuous = 0;
196 mx->first_continuous = 0;
201 if (lex_match_id ("VARIABLES"))
208 msg (SE, _("VARIABLES subcommand multiply specified."));
214 if (!parse_DATA_LIST_vars (&v, &nv, PV_NO_DUPLICATE))
220 for (i = 0; i < nv; i++)
221 if (!strcasecmp (v[i], "VARNAME_"))
223 msg (SE, _("VARNAME_ cannot be explicitly specified on "
225 for (i = 0; i < nv; i++)
235 for (i = 0; i < nv; i++)
237 struct variable *new_var;
239 if (strcasecmp (v[i], "ROWTYPE_"))
241 new_var = dict_create_var_assert (default_dict, v[i], 0);
242 attach_mxd_aux (new_var, MXD_CONTINUOUS, i);
245 mx->explicit_rowtype = 1;
251 mx->rowtype_ = dict_create_var_assert (default_dict,
253 attach_mxd_aux (mx->rowtype_, MXD_ROWTYPE, 0);
255 else if (lex_match_id ("FILE"))
262 else if (lex_match_id ("FORMAT"))
266 while (token == T_ID)
268 if (lex_match_id ("LIST"))
270 else if (lex_match_id ("FREE"))
272 else if (lex_match_id ("LOWER"))
274 else if (lex_match_id ("UPPER"))
276 else if (lex_match_id ("FULL"))
278 else if (lex_match_id ("DIAGONAL"))
280 else if (lex_match_id ("NODIAGONAL"))
281 mx->diag = NODIAGONAL;
284 lex_error (_("in FORMAT subcommand"));
289 else if (lex_match_id ("SPLIT"))
295 msg (SE, _("SPLIT subcommand multiply specified."));
302 lex_error (_("in SPLIT subcommand"));
306 if (dict_lookup_var (default_dict, tokid) == NULL
307 && (lex_look_ahead () == '.' || lex_look_ahead () == '/'))
309 if (!strcasecmp (tokid, "ROWTYPE_")
310 || !strcasecmp (tokid, "VARNAME_"))
312 msg (SE, _("Split variable may not be named ROWTYPE_ "
317 mx->single_split = dict_create_var_assert (default_dict,
319 attach_mxd_aux (mx->single_split, MXD_CONTINUOUS, 0);
322 dict_set_split_vars (default_dict, &mx->single_split, 1);
326 struct variable **split;
329 if (!parse_variables (default_dict, &split, &n, PV_NO_DUPLICATE))
332 dict_set_split_vars (default_dict, split, n);
336 struct variable *const *split = dict_get_split_vars (default_dict);
337 size_t split_cnt = dict_get_split_cnt (default_dict);
340 for (i = 0; i < split_cnt; i++)
342 struct mxd_var *mv = split[i]->aux;
344 if (mv->var_type != MXD_CONTINUOUS)
346 msg (SE, _("Split variable %s is already another type."),
350 var_clear_aux (split[i]);
351 attach_mxd_aux (split[i], MXD_SPLIT, i);
355 else if (lex_match_id ("FACTORS"))
361 msg (SE, _("FACTORS subcommand multiply specified."));
366 if (!parse_variables (default_dict, &mx->factors, &mx->n_factors, PV_NONE))
372 for (i = 0; i < mx->n_factors; i++)
374 struct variable *v = mx->factors[i];
375 struct mxd_var *mv = v->aux;
377 if (mv->var_type != MXD_CONTINUOUS)
379 msg (SE, _("Factor variable %s is already another type."),
384 attach_mxd_aux (v, MXD_FACTOR, i);
388 else if (lex_match_id ("CELLS"))
394 msg (SE, _("CELLS subcommand multiply specified."));
398 if (!lex_is_integer () || lex_integer () < 1)
400 lex_error (_("expecting positive integer"));
404 mx->cells = lex_integer ();
407 else if (lex_match_id ("N"))
413 msg (SE, _("N subcommand multiply specified."));
417 if (!lex_is_integer () || lex_integer () < 1)
419 lex_error (_("expecting positive integer"));
423 mx->pop_n = lex_integer ();
426 else if (lex_match_id ("CONTENTS"))
428 int inside_parens = 0;
429 unsigned collide = 0;
434 msg (SE, _("CONTENTS subcommand multiply specified."));
444 for (i = 0; i <= PROX; i++)
445 mx->is_per_factor[i] = 0;
454 msg (SE, _("Nested parentheses not allowed."));
460 else if (lex_match (')'))
464 msg (SE, _("Mismatched right parenthesis (`(')."));
467 if (mx->contents[mx->n_contents - 1] == LPAREN)
469 msg (SE, _("Empty parentheses not allowed."));
482 lex_error (_("in CONTENTS subcommand"));
486 content_type = string_to_content_type (tokid,
488 if (content_type == -1)
490 lex_error (_("in CONTENTS subcommand"));
495 if (collide & (1 << collide_index))
497 msg (SE, _("Content multiply specified for %s."),
498 content_names[content_type]);
501 collide |= (1 << collide_index);
504 mx->is_per_factor[item] = inside_parens;
506 mx->contents[mx->n_contents++] = item;
508 if (token == '/' || token == '.')
514 msg (SE, _("Missing right parenthesis."));
517 mx->contents[mx->n_contents] = EOC;
528 lex_error (_("expecting end of command"));
534 msg (SE, _("Missing VARIABLES subcommand."));
538 if (!mx->n_contents && !mx->explicit_rowtype)
540 msg (SW, _("CONTENTS subcommand not specified: assuming file "
541 "contains only CORR matrix."));
543 mx->contents[0] = CORR;
544 mx->contents[1] = EOC;
548 if (mx->n_factors && !mx->explicit_rowtype && mx->cells == -1)
550 msg (SE, _("Missing CELLS subcommand. CELLS is required "
551 "when ROWTYPE_ is not given in the data and "
552 "factors are present."));
556 if (mx->explicit_rowtype && mx->single_split)
558 msg (SE, _("Split file values must be present in the data when "
559 "ROWTYPE_ is present."));
563 /* Create VARNAME_. */
564 mx->varname_ = dict_create_var_assert (default_dict, "VARNAME_", 8);
565 attach_mxd_aux (mx->varname_, MXD_VARNAME, 0);
567 /* Sort the dictionary variables into the desired order for the
568 system file output. */
573 dict_get_vars (default_dict, &v, &nv, 0);
574 qsort (v, nv, sizeof *v, compare_variables_by_mxd_var_type);
575 dict_reorder_vars (default_dict, v, nv);
581 static const struct fmt_spec fmt_tab[MXD_COUNT] =
592 mx->first_continuous = -1;
593 for (i = 0; i < dict_get_var_cnt (default_dict); i++)
595 struct variable *v = dict_get_var (default_dict, i);
596 struct mxd_var *mv = v->aux;
597 int type = mv->var_type;
599 assert (type >= 0 && type < MXD_COUNT);
600 v->print = v->write = fmt_tab[type];
602 if (type == MXD_CONTINUOUS)
604 if (mx->first_continuous == -1 && type == MXD_CONTINUOUS)
605 mx->first_continuous = i;
609 if (mx->n_continuous == 0)
611 msg (SE, _("No continuous variables specified."));
615 mx->reader = dfm_open_reader (fh);
616 if (mx->reader == NULL)
619 if (mx->explicit_rowtype)
620 read_matrices_with_rowtype (mx);
622 read_matrices_without_rowtype (mx);
624 dfm_close_reader (mx->reader);
626 pool_destroy (mx->container);
631 discard_variables ();
633 pool_destroy (mx->container);
637 /* Look up string S as a content-type name and return the
638 corresponding enumerated value, or -1 if there is no match. If
639 COLLIDE is non-NULL then *COLLIDE returns a value (suitable for use
640 as a bit-index) which can be used for determining whether a related
641 statistic has already been used. */
643 string_to_content_type (char *s, int *collide)
654 {N_VECTOR, 0, "N_VECTOR"},
656 {N_SCALAR, 0, "N_SCALAR"},
657 {N_MATRIX, 1, "N_MATRIX"},
659 {STDDEV, 3, "STDDEV"},
671 for (tp = tab; tp->value != -1; tp++)
672 if (!strcasecmp (s, tp->string))
675 *collide = tp->collide;
682 /* Compare two variables using p.mxd.var_type and p.mxd.sub_type
685 compare_variables_by_mxd_var_type (const void *a_, const void *b_)
687 struct variable *const *pa = a_;
688 struct variable *const *pb = b_;
689 const struct mxd_var *a = (*pa)->aux;
690 const struct mxd_var *b = (*pb)->aux;
692 if (a->var_type != b->var_type)
693 return a->var_type > b->var_type ? 1 : -1;
695 return a->sub_type < b->sub_type ? -1 : a->sub_type > b->sub_type;
698 /* Attaches a struct mxd_var with the specific member values to
701 attach_mxd_aux (struct variable *v, int var_type, int sub_type)
705 assert (v->aux == NULL);
706 mv = xmalloc (sizeof *mv);
707 mv->var_type = var_type;
708 mv->sub_type = sub_type;
709 var_attach_aux (v, mv, var_dtor_free);
712 /* Matrix tokenizer. */
714 /* Matrix token types. */
715 enum matrix_token_type
721 /* A MATRIX DATA parsing token. */
724 enum matrix_token_type type;
725 double number; /* MNUM: token value. */
726 char *string; /* MSTR: token string; not null-terminated. */
727 int length; /* MSTR: tokstr length. */
730 static int mget_token (struct matrix_token *, struct dfm_reader *);
733 #define mget_token(TOKEN, READER) mget_token_dump(TOKEN, READER)
736 mdump_token (const struct matrix_token *token)
741 printf (" #%g", token->number);
744 printf (" '%.*s'", token->length, token->string);
753 mget_token_dump (struct matrix_token *token, struct dfm_reader *reader)
755 int result = (mget_token) (token, reader);
761 /* Return the current position in READER. */
763 context (struct dfm_reader *reader)
767 if (dfm_eof (reader))
768 strcpy (buf, "at end of file");
771 struct fixed_string line;
774 dfm_get_record (reader, &line);
775 sp = ls_c_str (&line);
776 while (sp < ls_end (&line) && isspace ((unsigned char) *sp))
778 if (sp >= ls_end (&line))
779 strcpy (buf, "at end of line");
785 dp = stpcpy (buf, "before `");
786 while (sp < ls_end (&line) && !isspace ((unsigned char) *sp)
799 /* Is there at least one token left in the data file? */
801 another_token (struct dfm_reader *reader)
805 struct fixed_string line;
808 if (dfm_eof (reader))
810 dfm_get_record (reader, &line);
812 cp = ls_c_str (&line);
813 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
816 if (cp < ls_end (&line))
818 dfm_forward_columns (reader, cp - ls_c_str (&line));
822 dfm_forward_record (reader);
826 /* Parse a MATRIX DATA token from READER into TOKEN. */
828 (mget_token) (struct matrix_token *token, struct dfm_reader *reader)
830 struct fixed_string line;
834 if (!another_token (reader))
837 dfm_get_record (reader, &line);
838 first_column = dfm_column_start (reader);
840 /* Three types of fields: quoted with ', quoted with ", unquoted. */
841 cp = ls_c_str (&line);
842 if (*cp == '\'' || *cp == '"')
847 token->string = ++cp;
848 while (cp < ls_end (&line) && *cp != quote)
850 token->length = cp - token->string;
851 if (cp < ls_end (&line))
854 msg (SW, _("Scope of string exceeds line."));
858 int is_num = isdigit ((unsigned char) *cp) || *cp == '.';
860 token->string = cp++;
861 while (cp < ls_end (&line)
862 && !isspace ((unsigned char) *cp) && *cp != ','
863 && *cp != '-' && *cp != '+')
865 if (isdigit ((unsigned char) *cp))
868 if ((tolower ((unsigned char) *cp) == 'd'
869 || tolower ((unsigned char) *cp) == 'e')
870 && (cp[1] == '+' || cp[1] == '-'))
876 token->length = cp - token->string;
877 assert (token->length);
883 di.s = token->string;
884 di.e = token->string + token->length;
885 di.v = (union value *) &token->number;
886 di.f1 = first_column;
887 di.format.type = FMT_F;
888 di.format.w = token->length;
898 dfm_forward_columns (reader, cp - ls_c_str (&line));
903 /* Forcibly skip the end of a line for content type CONTENT in
906 force_eol (struct dfm_reader *reader, const char *content)
908 struct fixed_string line;
911 if (dfm_eof (reader))
913 dfm_get_record (reader, &line);
915 cp = ls_c_str (&line);
916 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
919 if (cp < ls_end (&line))
921 msg (SE, _("End of line expected %s while reading %s."),
922 context (reader), content);
926 dfm_forward_record (reader);
930 /* Back end, omitting ROWTYPE_. */
934 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
935 double ***data; /* MATRIX DATA data. */
936 double *factor_values; /* Factor values. */
937 int max_cell_idx; /* Max-numbered cell that we have
938 read so far, plus one. */
939 double *split_values; /* SPLIT FILE variable values. */
942 static int nr_read_splits (struct nr_aux_data *, int compare);
943 static int nr_read_factors (struct nr_aux_data *, int cell);
944 static void nr_output_data (struct nr_aux_data *, struct ccase *,
945 write_case_func *, write_case_data);
946 static void matrix_data_read_without_rowtype (struct case_source *source,
951 /* Read from the data file and write it to the active file. */
953 read_matrices_without_rowtype (struct matrix_data_pgm *mx)
955 struct nr_aux_data nr;
962 nr.factor_values = xmalloc (sizeof *nr.factor_values * mx->n_factors * mx->cells);
964 nr.split_values = xmalloc (sizeof *nr.split_values
965 * dict_get_split_cnt (default_dict));
967 vfm_source = create_case_source (&matrix_data_without_rowtype_source_class, &nr);
969 procedure (NULL, NULL);
971 free (nr.split_values);
972 free (nr.factor_values);
975 /* Mirror data across the diagonal of matrix CP which contains
976 CONTENT type data. */
978 fill_matrix (struct matrix_data_pgm *mx, int content, double *cp)
980 int type = content_type[content];
982 if (type == 1 && mx->section != FULL)
984 if (mx->diag == NODIAGONAL)
986 const double fill = content == CORR ? 1.0 : SYSMIS;
989 for (i = 0; i < mx->n_continuous; i++)
990 cp[i * (1 + mx->n_continuous)] = fill;
996 if (mx->section == LOWER)
998 int n_lines = mx->n_continuous;
999 if (mx->section != FULL && mx->diag == NODIAGONAL)
1002 for (r = 1; r < n_lines; r++)
1003 for (c = 0; c < r; c++)
1004 cp[r + c * mx->n_continuous] = cp[c + r * mx->n_continuous];
1008 assert (mx->section == UPPER);
1009 for (r = 1; r < mx->n_continuous; r++)
1010 for (c = 0; c < r; c++)
1011 cp[c + r * mx->n_continuous] = cp[r + c * mx->n_continuous];
1019 for (c = 1; c < mx->n_continuous; c++)
1024 /* Read data lines for content type CONTENT from the data file.
1025 If PER_FACTOR is nonzero, then factor information is read from
1026 the data file. Data is for cell number CELL. */
1028 nr_read_data_lines (struct nr_aux_data *nr,
1029 int per_factor, int cell, int content, int compare)
1031 struct matrix_data_pgm *mx = nr->mx;
1032 const int type = content_type[content]; /* Content type. */
1033 int n_lines; /* Number of lines to parse from data file for this type. */
1034 double *cp; /* Current position in vector or matrix. */
1041 n_lines = mx->n_continuous;
1042 if (mx->section != FULL && mx->diag == NODIAGONAL)
1046 cp = nr->data[content][cell];
1047 if (type == 1 && mx->section == LOWER && mx->diag == NODIAGONAL)
1048 cp += mx->n_continuous;
1050 for (i = 0; i < n_lines; i++)
1054 if (!nr_read_splits (nr, 1))
1056 if (per_factor && !nr_read_factors (nr, cell))
1063 n_cols = mx->n_continuous;
1066 switch (mx->section)
1073 n_cols = mx->n_continuous - i;
1074 if (mx->diag == NODIAGONAL)
1081 n_cols = mx->n_continuous;
1099 for (j = 0; j < n_cols; j++)
1101 struct matrix_token token;
1102 if (!mget_token (&token, mx->reader))
1104 if (token.type != MNUM)
1106 msg (SE, _("expecting value for %s %s"),
1107 dict_get_var (default_dict, j)->name,
1108 context (mx->reader));
1112 *cp++ = token.number;
1115 && !force_eol (mx->reader, content_names[content]))
1117 debug_printf (("\n"));
1120 if (mx->section == LOWER)
1121 cp += mx->n_continuous - n_cols;
1124 fill_matrix (mx, content, nr->data[content][cell]);
1129 /* When ROWTYPE_ does not appear in the data, reads the matrices and
1130 writes them to the output file. Returns success. */
1132 matrix_data_read_without_rowtype (struct case_source *source,
1134 write_case_func *write_case,
1135 write_case_data wc_data)
1137 struct nr_aux_data *nr = source->aux;
1138 struct matrix_data_pgm *mx = nr->mx;
1143 nr->data = pool_alloc (mx->container, (PROX + 1) * sizeof *nr->data);
1148 for (i = 0; i <= PROX; i++)
1152 for (cp = mx->contents; *cp != EOC; cp++)
1153 if (*cp != LPAREN && *cp != RPAREN)
1155 int per_factor = mx->is_per_factor[*cp];
1158 n_entries = mx->n_continuous;
1159 if (content_type[*cp] == 1)
1160 n_entries *= mx->n_continuous;
1163 int n_vectors = per_factor ? mx->cells : 1;
1166 nr->data[*cp] = pool_alloc (mx->container,
1167 n_vectors * sizeof **nr->data);
1169 for (i = 0; i < n_vectors; i++)
1170 nr->data[*cp][i] = pool_alloc (mx->container,
1171 n_entries * sizeof ***nr->data);
1180 if (!nr_read_splits (nr, 0))
1183 for (bp = mx->contents; *bp != EOC; bp = np)
1187 /* Trap the CONTENTS that we should parse in this pass
1188 between bp and ep. Set np to the starting bp for next
1193 while (*ep != RPAREN)
1201 while (*ep != EOC && *ep != LPAREN)
1210 for (i = 0; i < (per_factor ? mx->cells : 1); i++)
1214 for (cp = bp; cp < ep; cp++)
1215 if (!nr_read_data_lines (nr, per_factor, i, *cp, cp != bp))
1221 nr_output_data (nr, c, write_case, wc_data);
1223 if (dict_get_split_cnt (default_dict) == 0
1224 || !another_token (mx->reader))
1229 /* Read the split file variables. If COMPARE is 1, compares the
1230 values read to the last values read and returns 1 if they're equal,
1233 nr_read_splits (struct nr_aux_data *nr, int compare)
1235 struct matrix_data_pgm *mx = nr->mx;
1236 static int just_read = 0; /* FIXME: WTF? */
1240 if (compare && just_read)
1246 if (dict_get_split_vars (default_dict) == NULL)
1249 if (mx->single_split)
1253 struct mxd_var *mv = dict_get_split_vars (default_dict)[0]->aux;
1254 nr->split_values[0] = ++mv->sub_type;
1262 split_cnt = dict_get_split_cnt (default_dict);
1263 for (i = 0; i < split_cnt; i++)
1265 struct matrix_token token;
1266 if (!mget_token (&token, mx->reader))
1268 if (token.type != MNUM)
1270 msg (SE, _("Syntax error expecting SPLIT FILE value %s."),
1271 context (mx->reader));
1276 nr->split_values[i] = token.number;
1277 else if (nr->split_values[i] != token.number)
1279 msg (SE, _("Expecting value %g for %s."),
1280 nr->split_values[i],
1281 dict_get_split_vars (default_dict)[i]->name);
1289 /* Read the factors for cell CELL. If COMPARE is 1, compares the
1290 values read to the last values read and returns 1 if they're equal,
1293 nr_read_factors (struct nr_aux_data *nr, int cell)
1295 struct matrix_data_pgm *mx = nr->mx;
1298 if (mx->n_factors == 0)
1301 assert (nr->max_cell_idx >= cell);
1302 if (cell != nr->max_cell_idx)
1313 for (i = 0; i < mx->n_factors; i++)
1315 struct matrix_token token;
1316 if (!mget_token (&token, mx->reader))
1318 if (token.type != MNUM)
1320 msg (SE, _("Syntax error expecting factor value %s."),
1321 context (mx->reader));
1326 nr->factor_values[i + mx->n_factors * cell] = token.number;
1327 else if (nr->factor_values[i + mx->n_factors * cell] != token.number)
1329 msg (SE, _("Syntax error expecting value %g for %s %s."),
1330 nr->factor_values[i + mx->n_factors * cell],
1331 mx->factors[i]->name, context (mx->reader));
1340 /* Write the contents of a cell having content type CONTENT and data
1341 CP to the active file. */
1343 dump_cell_content (struct matrix_data_pgm *mx, int content, double *cp,
1345 write_case_func *write_case, write_case_data wc_data)
1347 int type = content_type[content];
1350 st_bare_pad_copy (case_data_rw (c, mx->rowtype_->fv)->s,
1351 content_names[content], 8);
1354 memset (case_data_rw (c, mx->varname_->fv)->s, ' ', 8);
1358 int n_lines = (type == 1) ? mx->n_continuous : 1;
1361 for (i = 0; i < n_lines; i++)
1365 for (j = 0; j < mx->n_continuous; j++)
1367 int fv = dict_get_var (default_dict, mx->first_continuous + j)->fv;
1368 case_data_rw (c, fv)->f = *cp;
1372 st_bare_pad_copy (case_data_rw (c, mx->varname_->fv)->s,
1373 dict_get_var (default_dict,
1374 mx->first_continuous + i)->name,
1376 write_case (wc_data);
1381 /* Finally dump out everything from nr_data[] to the output file. */
1383 nr_output_data (struct nr_aux_data *nr, struct ccase *c,
1384 write_case_func *write_case, write_case_data wc_data)
1386 struct matrix_data_pgm *mx = nr->mx;
1389 struct variable *const *split;
1393 split_cnt = dict_get_split_cnt (default_dict);
1394 split = dict_get_split_vars (default_dict);
1395 for (i = 0; i < split_cnt; i++)
1396 case_data_rw (c, split[i]->fv)->f = nr->split_values[i];
1403 for (cell = 0; cell < mx->cells; cell++)
1408 for (factor = 0; factor < mx->n_factors; factor++)
1410 case_data_rw (c, mx->factors[factor]->fv)->f
1411 = nr->factor_values[factor + cell * mx->n_factors];
1412 debug_printf (("f:%s ", mx->factors[factor]->name));
1419 for (content = 0; content <= PROX; content++)
1420 if (mx->is_per_factor[content])
1422 assert (nr->data[content] != NULL
1423 && nr->data[content][cell] != NULL);
1425 dump_cell_content (mx, content, nr->data[content][cell],
1426 c, write_case, wc_data);
1438 for (factor = 0; factor < mx->n_factors; factor++)
1439 case_data_rw (c, mx->factors[factor]->fv)->f = SYSMIS;
1442 for (content = 0; content <= PROX; content++)
1443 if (!mx->is_per_factor[content] && nr->data[content] != NULL)
1444 dump_cell_content (mx, content, nr->data[content][0],
1445 c, write_case, wc_data);
1449 /* Back end, with ROWTYPE_. */
1451 /* All the data for one set of factor values. */
1455 int n_rows[PROX + 1];
1456 double *data[PROX + 1];
1457 struct factor_data *next;
1460 /* With ROWTYPE_ auxiliary data. */
1463 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1464 int content; /* Type of current row. */
1465 double *split_values; /* SPLIT FILE variable values. */
1466 struct factor_data *data; /* All the data. */
1467 struct factor_data *current; /* Current factor. */
1470 static int wr_read_splits (struct wr_aux_data *, struct ccase *,
1471 write_case_func *, write_case_data);
1472 static int wr_output_data (struct wr_aux_data *, struct ccase *,
1473 write_case_func *, write_case_data);
1474 static int wr_read_rowtype (struct wr_aux_data *,
1475 const struct matrix_token *, struct dfm_reader *);
1476 static int wr_read_factors (struct wr_aux_data *);
1477 static int wr_read_indeps (struct wr_aux_data *);
1478 static void matrix_data_read_with_rowtype (struct case_source *,
1483 /* When ROWTYPE_ appears in the data, reads the matrices and writes
1484 them to the output file. */
1486 read_matrices_with_rowtype (struct matrix_data_pgm *mx)
1488 struct wr_aux_data wr;
1492 wr.split_values = NULL;
1497 vfm_source = create_case_source (&matrix_data_with_rowtype_source_class,
1499 procedure (NULL, NULL);
1501 free (wr.split_values);
1504 /* Read from the data file and write it to the active file. */
1506 matrix_data_read_with_rowtype (struct case_source *source,
1508 write_case_func *write_case,
1509 write_case_data wc_data)
1511 struct wr_aux_data *wr = source->aux;
1512 struct matrix_data_pgm *mx = wr->mx;
1516 if (!wr_read_splits (wr, c, write_case, wc_data))
1519 if (!wr_read_factors (wr))
1522 if (!wr_read_indeps (wr))
1525 while (another_token (mx->reader));
1527 wr_output_data (wr, c, write_case, wc_data);
1530 /* Read the split file variables. If they differ from the previous
1531 set of split variables then output the data. Returns success. */
1533 wr_read_splits (struct wr_aux_data *wr,
1535 write_case_func *write_case, write_case_data wc_data)
1537 struct matrix_data_pgm *mx = wr->mx;
1541 split_cnt = dict_get_split_cnt (default_dict);
1545 if (wr->split_values)
1550 wr->split_values = xmalloc (split_cnt * sizeof *wr->split_values);
1557 for (i = 0; i < split_cnt; i++)
1559 struct matrix_token token;
1560 if (!mget_token (&token, mx->reader))
1562 if (token.type != MNUM)
1564 msg (SE, _("Syntax error %s expecting SPLIT FILE value."),
1565 context (mx->reader));
1569 if (compare && wr->split_values[i] != token.number && !different)
1571 if (!wr_output_data (wr, c, write_case, wc_data))
1576 wr->split_values[i] = token.number;
1583 /* Compares doubles A and B, treating SYSMIS as greatest. */
1585 compare_doubles (const void *a_, const void *b_, void *aux UNUSED)
1587 const double *a = a_;
1588 const double *b = b_;
1592 else if (*a == SYSMIS)
1594 else if (*b == SYSMIS)
1602 /* Return strcmp()-type comparison of the MX->n_factors factors at _A and
1603 _B. Sort missing values toward the end. */
1605 compare_factors (const void *a_, const void *b_, void *mx_)
1607 struct matrix_data_pgm *mx = mx_;
1608 struct factor_data *const *pa = a_;
1609 struct factor_data *const *pb = b_;
1610 const double *a = (*pa)->factors;
1611 const double *b = (*pb)->factors;
1613 return lexicographical_compare_3way (a, mx->n_factors,
1616 compare_doubles, NULL);
1619 /* Write out the data for the current split file to the active
1622 wr_output_data (struct wr_aux_data *wr,
1624 write_case_func *write_case, write_case_data wc_data)
1626 struct matrix_data_pgm *mx = wr->mx;
1629 struct variable *const *split;
1633 split_cnt = dict_get_split_cnt (default_dict);
1634 split = dict_get_split_vars (default_dict);
1635 for (i = 0; i < split_cnt; i++)
1636 case_data_rw (c, split[i]->fv)->f = wr->split_values[i];
1639 /* Sort the wr->data list. */
1641 struct factor_data **factors;
1642 struct factor_data *iter;
1645 factors = xmalloc (sizeof *factors * mx->cells);
1647 for (i = 0, iter = wr->data; iter; iter = iter->next, i++)
1650 sort (factors, mx->cells, sizeof *factors, compare_factors, mx);
1652 wr->data = factors[0];
1653 for (i = 0; i < mx->cells - 1; i++)
1654 factors[i]->next = factors[i + 1];
1655 factors[mx->cells - 1]->next = NULL;
1660 /* Write out records for every set of factor values. */
1662 struct factor_data *iter;
1664 for (iter = wr->data; iter; iter = iter->next)
1669 for (factor = 0; factor < mx->n_factors; factor++)
1670 case_data_rw (c, mx->factors[factor]->fv)->f
1671 = iter->factors[factor];
1677 for (content = 0; content <= PROX; content++)
1679 if (!iter->n_rows[content])
1683 int type = content_type[content];
1684 int n_lines = (type == 1
1686 - (mx->section != FULL && mx->diag == NODIAGONAL))
1689 if (n_lines != iter->n_rows[content])
1691 msg (SE, _("Expected %d lines of data for %s content; "
1692 "actually saw %d lines. No data will be "
1693 "output for this content."),
1694 n_lines, content_names[content],
1695 iter->n_rows[content]);
1700 fill_matrix (mx, content, iter->data[content]);
1702 dump_cell_content (mx, content, iter->data[content],
1703 c, write_case, wc_data);
1709 pool_destroy (mx->container);
1710 mx->container = pool_create ();
1712 wr->data = wr->current = NULL;
1717 /* Sets ROWTYPE_ based on the given TOKEN read from READER.
1720 wr_read_rowtype (struct wr_aux_data *wr,
1721 const struct matrix_token *token,
1722 struct dfm_reader *reader)
1724 if (wr->content != -1)
1726 msg (SE, _("Multiply specified ROWTYPE_ %s."), context (reader));
1729 if (token->type != MSTR)
1731 msg (SE, _("Syntax error %s expecting ROWTYPE_ string."),
1740 memcpy (s, token->string, min (15, token->length));
1741 s[min (15, token->length)] = 0;
1743 for (cp = s; *cp; cp++)
1744 *cp = toupper ((unsigned char) *cp);
1746 wr->content = string_to_content_type (s, NULL);
1749 if (wr->content == -1)
1751 msg (SE, _("Syntax error %s."), context (reader));
1758 /* Read the factors for the current row. Select a set of factors and
1759 point wr_current to it. */
1761 wr_read_factors (struct wr_aux_data *wr)
1763 struct matrix_data_pgm *mx = wr->mx;
1764 double *factor_values = local_alloc (sizeof *factor_values * mx->n_factors);
1770 for (i = 0; i < mx->n_factors; i++)
1772 struct matrix_token token;
1773 if (!mget_token (&token, mx->reader))
1775 if (token.type == MSTR)
1777 if (!wr_read_rowtype (wr, &token, mx->reader))
1779 if (!mget_token (&token, mx->reader))
1782 if (token.type != MNUM)
1784 msg (SE, _("Syntax error expecting factor value %s."),
1785 context (mx->reader));
1789 factor_values[i] = token.number;
1792 if (wr->content == -1)
1794 struct matrix_token token;
1795 if (!mget_token (&token, mx->reader))
1797 if (!wr_read_rowtype (wr, &token, mx->reader))
1801 /* Try the most recent factor first as a simple caching
1807 for (i = 0; i < mx->n_factors; i++)
1808 if (factor_values[i] != wr->current->factors[i])
1813 /* Linear search through the list. */
1816 struct factor_data *iter;
1818 for (iter = wr->data; iter; iter = iter->next)
1822 for (i = 0; i < mx->n_factors; i++)
1823 if (factor_values[i] != iter->factors[i])
1833 /* Not found. Make a new item. */
1835 struct factor_data *new = pool_alloc (mx->container, sizeof *new);
1837 new->factors = pool_alloc (mx->container, sizeof *new->factors * mx->n_factors);
1842 for (i = 0; i < mx->n_factors; i++)
1843 new->factors[i] = factor_values[i];
1849 for (i = 0; i <= PROX; i++)
1852 new->data[i] = NULL;
1856 new->next = wr->data;
1857 wr->data = wr->current = new;
1862 local_free (factor_values);
1866 local_free (factor_values);
1870 /* Read the independent variables into wr->current. */
1872 wr_read_indeps (struct wr_aux_data *wr)
1874 struct matrix_data_pgm *mx = wr->mx;
1875 struct factor_data *c = wr->current;
1876 const int type = content_type[wr->content];
1877 const int n_rows = c->n_rows[wr->content];
1881 /* Allocate room for data if necessary. */
1882 if (c->data[wr->content] == NULL)
1884 int n_items = mx->n_continuous;
1886 n_items *= mx->n_continuous;
1888 c->data[wr->content] = pool_alloc (mx->container,
1889 sizeof **c->data * n_items);
1892 cp = &c->data[wr->content][n_rows * mx->n_continuous];
1894 /* Figure out how much to read from this line. */
1901 msg (SE, _("Duplicate specification for %s."),
1902 content_names[wr->content]);
1906 n_cols = mx->n_continuous;
1911 if (n_rows >= mx->n_continuous - (mx->section != FULL && mx->diag == NODIAGONAL))
1913 msg (SE, _("Too many rows of matrix data for %s."),
1914 content_names[wr->content]);
1918 switch (mx->section)
1921 n_cols = n_rows + 1;
1922 if (mx->diag == NODIAGONAL)
1923 cp += mx->n_continuous;
1927 n_cols = mx->n_continuous - n_rows;
1928 if (mx->diag == NODIAGONAL)
1935 n_cols = mx->n_continuous;
1946 c->n_rows[wr->content]++;
1948 debug_printf ((" (c=%p,r=%d,n=%d)", c, n_rows + 1, n_cols));
1950 /* Read N_COLS items at CP. */
1954 for (j = 0; j < n_cols; j++)
1956 struct matrix_token token;
1957 if (!mget_token (&token, mx->reader))
1959 if (token.type != MNUM)
1961 msg (SE, _("Syntax error expecting value for %s %s."),
1962 dict_get_var (default_dict, mx->first_continuous + j)->name,
1963 context (mx->reader));
1967 *cp++ = token.number;
1970 && !force_eol (mx->reader, content_names[wr->content]))
1972 debug_printf (("\n"));
1978 /* Matrix source. */
1980 static const struct case_source_class matrix_data_with_rowtype_source_class =
1984 matrix_data_read_with_rowtype,
1988 static const struct case_source_class
1989 matrix_data_without_rowtype_source_class =
1993 matrix_data_read_without_rowtype,