1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
30 #include "data-reader.h"
31 #include "dictionary.h"
33 #include "file-handle.h"
39 #include "procedure.h"
42 #define _(msgid) gettext (msgid)
44 #include "debug-print.h"
46 /* FIXME: /N subcommand not implemented. It should be pretty simple,
49 /* Different types of variables for MATRIX DATA procedure. Order is
50 important: these are used for sort keys. */
53 MXD_SPLIT, /* SPLIT FILE variables. */
54 MXD_ROWTYPE, /* ROWTYPE_. */
55 MXD_FACTOR, /* Factor variables. */
56 MXD_VARNAME, /* VARNAME_. */
57 MXD_CONTINUOUS, /* Continuous variables. */
62 /* Format type enums. */
69 /* Matrix section enums. */
77 /* Diagonal inclusion enums. */
105 /* 0=vector, 1=matrix, 2=scalar. */
106 static const int content_type[PROX + 1] =
108 0, 2, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1,
111 /* Name of each content type. */
112 static const char *content_names[PROX + 1] =
114 "N", "N", "N_MATRIX", "MEAN", "STDDEV", "COUNT", "MSE",
115 "DFE", "MAT", "COV", "CORR", "PROX",
118 /* A MATRIX DATA input program. */
119 struct matrix_data_pgm
121 struct pool *container; /* Arena used for all allocations. */
122 struct dfm_reader *reader; /* Data file to read. */
125 enum format_type fmt; /* LIST or FREE. */
126 enum matrix_section section;/* LOWER or UPPER or FULL. */
127 enum include_diagonal diag; /* DIAGONAL or NODIAGONAL. */
129 int explicit_rowtype; /* ROWTYPE_ specified explicitly in data? */
130 struct variable *rowtype_, *varname_; /* ROWTYPE_, VARNAME_ variables. */
132 struct variable *single_split; /* Single SPLIT FILE variable. */
134 /* Factor variables. */
135 size_t n_factors; /* Number of factor variables. */
136 struct variable **factors; /* Factor variables. */
137 int is_per_factor[PROX + 1]; /* Is there per-factor data? */
139 int cells; /* Number of cells, or -1 if none. */
141 int pop_n; /* Population N specified by user. */
143 /* CONTENTS subcommand. */
144 int contents[EOC * 3 + 1]; /* Contents. */
145 int n_contents; /* Number of entries. */
147 /* Continuous variables. */
148 int n_continuous; /* Number of continuous variables. */
149 int first_continuous; /* Index into default_dict.var of
150 first continuous variable. */
153 /* Auxiliary data attached to MATRIX DATA variables. */
156 int var_type; /* Variable type. */
157 int sub_type; /* Subtype. */
160 static const struct case_source_class matrix_data_with_rowtype_source_class;
161 static const struct case_source_class matrix_data_without_rowtype_source_class;
163 static int compare_variables_by_mxd_var_type (const void *pa,
165 static bool read_matrices_without_rowtype (struct matrix_data_pgm *);
166 static bool read_matrices_with_rowtype (struct matrix_data_pgm *);
167 static int string_to_content_type (char *, int *);
168 static void attach_mxd_aux (struct variable *, int var_type, int sub_type);
171 cmd_matrix_data (void)
174 struct matrix_data_pgm *mx;
175 struct file_handle *fh = fh_inline_file ();
180 discard_variables ();
182 pool = pool_create ();
183 mx = pool_alloc (pool, sizeof *mx);
184 mx->container = pool;
189 mx->explicit_rowtype = 0;
192 mx->single_split = NULL;
195 memset (mx->is_per_factor, 0, sizeof mx->is_per_factor);
199 mx->n_continuous = 0;
200 mx->first_continuous = 0;
205 if (lex_match_id ("VARIABLES"))
212 msg (SE, _("VARIABLES subcommand multiply specified."));
218 if (!parse_DATA_LIST_vars (&v, &nv, PV_NO_DUPLICATE))
224 for (i = 0; i < nv; i++)
225 if (!strcasecmp (v[i], "VARNAME_"))
227 msg (SE, _("VARNAME_ cannot be explicitly specified on "
229 for (i = 0; i < nv; i++)
239 for (i = 0; i < nv; i++)
241 struct variable *new_var;
243 if (strcasecmp (v[i], "ROWTYPE_"))
245 new_var = dict_create_var_assert (default_dict, v[i], 0);
246 attach_mxd_aux (new_var, MXD_CONTINUOUS, i);
249 mx->explicit_rowtype = 1;
255 mx->rowtype_ = dict_create_var_assert (default_dict,
257 attach_mxd_aux (mx->rowtype_, MXD_ROWTYPE, 0);
259 else if (lex_match_id ("FILE"))
262 fh = fh_parse (FH_REF_FILE | FH_REF_INLINE);
266 else if (lex_match_id ("FORMAT"))
270 while (token == T_ID)
272 if (lex_match_id ("LIST"))
274 else if (lex_match_id ("FREE"))
276 else if (lex_match_id ("LOWER"))
278 else if (lex_match_id ("UPPER"))
280 else if (lex_match_id ("FULL"))
282 else if (lex_match_id ("DIAGONAL"))
284 else if (lex_match_id ("NODIAGONAL"))
285 mx->diag = NODIAGONAL;
288 lex_error (_("in FORMAT subcommand"));
293 else if (lex_match_id ("SPLIT"))
299 msg (SE, _("SPLIT subcommand multiply specified."));
306 lex_error (_("in SPLIT subcommand"));
310 if (dict_lookup_var (default_dict, tokid) == NULL
311 && (lex_look_ahead () == '.' || lex_look_ahead () == '/'))
313 if (!strcasecmp (tokid, "ROWTYPE_")
314 || !strcasecmp (tokid, "VARNAME_"))
316 msg (SE, _("Split variable may not be named ROWTYPE_ "
321 mx->single_split = dict_create_var_assert (default_dict,
323 attach_mxd_aux (mx->single_split, MXD_CONTINUOUS, 0);
326 dict_set_split_vars (default_dict, &mx->single_split, 1);
330 struct variable **split;
333 if (!parse_variables (default_dict, &split, &n, PV_NO_DUPLICATE))
336 dict_set_split_vars (default_dict, split, n);
340 struct variable *const *split = dict_get_split_vars (default_dict);
341 size_t split_cnt = dict_get_split_cnt (default_dict);
344 for (i = 0; i < split_cnt; i++)
346 struct mxd_var *mv = split[i]->aux;
348 if (mv->var_type != MXD_CONTINUOUS)
350 msg (SE, _("Split variable %s is already another type."),
354 var_clear_aux (split[i]);
355 attach_mxd_aux (split[i], MXD_SPLIT, i);
359 else if (lex_match_id ("FACTORS"))
365 msg (SE, _("FACTORS subcommand multiply specified."));
370 if (!parse_variables (default_dict, &mx->factors, &mx->n_factors,
377 for (i = 0; i < mx->n_factors; i++)
379 struct variable *v = mx->factors[i];
380 struct mxd_var *mv = v->aux;
382 if (mv->var_type != MXD_CONTINUOUS)
384 msg (SE, _("Factor variable %s is already another type."),
389 attach_mxd_aux (v, MXD_FACTOR, i);
393 else if (lex_match_id ("CELLS"))
399 msg (SE, _("CELLS subcommand multiply specified."));
403 if (!lex_is_integer () || lex_integer () < 1)
405 lex_error (_("expecting positive integer"));
409 mx->cells = lex_integer ();
412 else if (lex_match_id ("N"))
418 msg (SE, _("N subcommand multiply specified."));
422 if (!lex_is_integer () || lex_integer () < 1)
424 lex_error (_("expecting positive integer"));
428 mx->pop_n = lex_integer ();
431 else if (lex_match_id ("CONTENTS"))
433 int inside_parens = 0;
434 unsigned collide = 0;
439 msg (SE, _("CONTENTS subcommand multiply specified."));
449 for (i = 0; i <= PROX; i++)
450 mx->is_per_factor[i] = 0;
459 msg (SE, _("Nested parentheses not allowed."));
465 else if (lex_match (')'))
469 msg (SE, _("Mismatched right parenthesis (`(')."));
472 if (mx->contents[mx->n_contents - 1] == LPAREN)
474 msg (SE, _("Empty parentheses not allowed."));
487 lex_error (_("in CONTENTS subcommand"));
491 content_type = string_to_content_type (tokid,
493 if (content_type == -1)
495 lex_error (_("in CONTENTS subcommand"));
500 if (collide & (1 << collide_index))
502 msg (SE, _("Content multiply specified for %s."),
503 content_names[content_type]);
506 collide |= (1 << collide_index);
509 mx->is_per_factor[item] = inside_parens;
511 mx->contents[mx->n_contents++] = item;
513 if (token == '/' || token == '.')
519 msg (SE, _("Missing right parenthesis."));
522 mx->contents[mx->n_contents] = EOC;
533 lex_error (_("expecting end of command"));
539 msg (SE, _("Missing VARIABLES subcommand."));
543 if (!mx->n_contents && !mx->explicit_rowtype)
545 msg (SW, _("CONTENTS subcommand not specified: assuming file "
546 "contains only CORR matrix."));
548 mx->contents[0] = CORR;
549 mx->contents[1] = EOC;
553 if (mx->n_factors && !mx->explicit_rowtype && mx->cells == -1)
555 msg (SE, _("Missing CELLS subcommand. CELLS is required "
556 "when ROWTYPE_ is not given in the data and "
557 "factors are present."));
561 if (mx->explicit_rowtype && mx->single_split)
563 msg (SE, _("Split file values must be present in the data when "
564 "ROWTYPE_ is present."));
568 /* Create VARNAME_. */
569 mx->varname_ = dict_create_var_assert (default_dict, "VARNAME_", 8);
570 attach_mxd_aux (mx->varname_, MXD_VARNAME, 0);
572 /* Sort the dictionary variables into the desired order for the
573 system file output. */
578 dict_get_vars (default_dict, &v, &nv, 0);
579 qsort (v, nv, sizeof *v, compare_variables_by_mxd_var_type);
580 dict_reorder_vars (default_dict, v, nv);
586 static const struct fmt_spec fmt_tab[MXD_COUNT] =
597 mx->first_continuous = -1;
598 for (i = 0; i < dict_get_var_cnt (default_dict); i++)
600 struct variable *v = dict_get_var (default_dict, i);
601 struct mxd_var *mv = v->aux;
602 int type = mv->var_type;
604 assert (type >= 0 && type < MXD_COUNT);
605 v->print = v->write = fmt_tab[type];
607 if (type == MXD_CONTINUOUS)
609 if (mx->first_continuous == -1 && type == MXD_CONTINUOUS)
610 mx->first_continuous = i;
614 if (mx->n_continuous == 0)
616 msg (SE, _("No continuous variables specified."));
620 mx->reader = dfm_open_reader (fh);
621 if (mx->reader == NULL)
624 if (mx->explicit_rowtype)
625 ok = read_matrices_with_rowtype (mx);
627 ok = read_matrices_without_rowtype (mx);
629 dfm_close_reader (mx->reader);
631 pool_destroy (mx->container);
633 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;
636 discard_variables ();
638 pool_destroy (mx->container);
639 return CMD_CASCADING_FAILURE;
642 /* Look up string S as a content-type name and return the
643 corresponding enumerated value, or -1 if there is no match. If
644 COLLIDE is non-NULL then *COLLIDE returns a value (suitable for use
645 as a bit-index) which can be used for determining whether a related
646 statistic has already been used. */
648 string_to_content_type (char *s, int *collide)
659 {N_VECTOR, 0, "N_VECTOR"},
661 {N_SCALAR, 0, "N_SCALAR"},
662 {N_MATRIX, 1, "N_MATRIX"},
664 {STDDEV, 3, "STDDEV"},
676 for (tp = tab; tp->value != -1; tp++)
677 if (!strcasecmp (s, tp->string))
680 *collide = tp->collide;
687 /* Compare two variables using p.mxd.var_type and p.mxd.sub_type
690 compare_variables_by_mxd_var_type (const void *a_, const void *b_)
692 struct variable *const *pa = a_;
693 struct variable *const *pb = b_;
694 const struct mxd_var *a = (*pa)->aux;
695 const struct mxd_var *b = (*pb)->aux;
697 if (a->var_type != b->var_type)
698 return a->var_type > b->var_type ? 1 : -1;
700 return a->sub_type < b->sub_type ? -1 : a->sub_type > b->sub_type;
703 /* Attaches a struct mxd_var with the specific member values to
706 attach_mxd_aux (struct variable *v, int var_type, int sub_type)
710 assert (v->aux == NULL);
711 mv = xmalloc (sizeof *mv);
712 mv->var_type = var_type;
713 mv->sub_type = sub_type;
714 var_attach_aux (v, mv, var_dtor_free);
717 /* Matrix tokenizer. */
719 /* Matrix token types. */
720 enum matrix_token_type
726 /* A MATRIX DATA parsing token. */
729 enum matrix_token_type type;
730 double number; /* MNUM: token value. */
731 char *string; /* MSTR: token string; not null-terminated. */
732 int length; /* MSTR: tokstr length. */
735 static int mget_token (struct matrix_token *, struct dfm_reader *);
738 #define mget_token(TOKEN, READER) mget_token_dump(TOKEN, READER)
741 mdump_token (const struct matrix_token *token)
746 printf (" #%g", token->number);
749 printf (" '%.*s'", token->length, token->string);
758 mget_token_dump (struct matrix_token *token, struct dfm_reader *reader)
760 int result = (mget_token) (token, reader);
766 /* Return the current position in READER. */
768 context (struct dfm_reader *reader)
772 if (dfm_eof (reader))
773 strcpy (buf, "at end of file");
776 struct fixed_string line;
779 dfm_get_record (reader, &line);
780 sp = ls_c_str (&line);
781 while (sp < ls_end (&line) && isspace ((unsigned char) *sp))
783 if (sp >= ls_end (&line))
784 strcpy (buf, "at end of line");
790 dp = stpcpy (buf, "before `");
791 while (sp < ls_end (&line) && !isspace ((unsigned char) *sp)
804 /* Is there at least one token left in the data file? */
806 another_token (struct dfm_reader *reader)
810 struct fixed_string line;
813 if (dfm_eof (reader))
815 dfm_get_record (reader, &line);
817 cp = ls_c_str (&line);
818 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
821 if (cp < ls_end (&line))
823 dfm_forward_columns (reader, cp - ls_c_str (&line));
827 dfm_forward_record (reader);
831 /* Parse a MATRIX DATA token from READER into TOKEN. */
833 (mget_token) (struct matrix_token *token, struct dfm_reader *reader)
835 struct fixed_string line;
839 if (!another_token (reader))
842 dfm_get_record (reader, &line);
843 first_column = dfm_column_start (reader);
845 /* Three types of fields: quoted with ', quoted with ", unquoted. */
846 cp = ls_c_str (&line);
847 if (*cp == '\'' || *cp == '"')
852 token->string = ++cp;
853 while (cp < ls_end (&line) && *cp != quote)
855 token->length = cp - token->string;
856 if (cp < ls_end (&line))
859 msg (SW, _("Scope of string exceeds line."));
863 int is_num = isdigit ((unsigned char) *cp) || *cp == '.';
865 token->string = cp++;
866 while (cp < ls_end (&line)
867 && !isspace ((unsigned char) *cp) && *cp != ','
868 && *cp != '-' && *cp != '+')
870 if (isdigit ((unsigned char) *cp))
873 if ((tolower ((unsigned char) *cp) == 'd'
874 || tolower ((unsigned char) *cp) == 'e')
875 && (cp[1] == '+' || cp[1] == '-'))
881 token->length = cp - token->string;
882 assert (token->length);
888 di.s = token->string;
889 di.e = token->string + token->length;
890 di.v = (union value *) &token->number;
891 di.f1 = first_column;
892 di.format = make_output_format (FMT_F, token->length, 0);
901 dfm_forward_columns (reader, cp - ls_c_str (&line));
906 /* Forcibly skip the end of a line for content type CONTENT in
909 force_eol (struct dfm_reader *reader, const char *content)
911 struct fixed_string line;
914 if (dfm_eof (reader))
916 dfm_get_record (reader, &line);
918 cp = ls_c_str (&line);
919 while (isspace ((unsigned char) *cp) && cp < ls_end (&line))
922 if (cp < ls_end (&line))
924 msg (SE, _("End of line expected %s while reading %s."),
925 context (reader), content);
929 dfm_forward_record (reader);
933 /* Back end, omitting ROWTYPE_. */
937 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
938 double ***data; /* MATRIX DATA data. */
939 double *factor_values; /* Factor values. */
940 int max_cell_idx; /* Max-numbered cell that we have
941 read so far, plus one. */
942 double *split_values; /* SPLIT FILE variable values. */
945 static int nr_read_splits (struct nr_aux_data *, int compare);
946 static int nr_read_factors (struct nr_aux_data *, int cell);
947 static bool nr_output_data (struct nr_aux_data *, struct ccase *,
948 write_case_func *, write_case_data);
949 static bool matrix_data_read_without_rowtype (struct case_source *source,
954 /* Read from the data file and write it to the active file.
955 Returns true if successful, false if an I/O error occurred. */
957 read_matrices_without_rowtype (struct matrix_data_pgm *mx)
959 struct nr_aux_data nr;
967 nr.factor_values = xnmalloc (mx->n_factors * mx->cells,
968 sizeof *nr.factor_values);
970 nr.split_values = xnmalloc (dict_get_split_cnt (default_dict),
971 sizeof *nr.split_values);
973 vfm_source = create_case_source (&matrix_data_without_rowtype_source_class, &nr);
975 ok = procedure (NULL, NULL);
977 free (nr.split_values);
978 free (nr.factor_values);
983 /* Mirror data across the diagonal of matrix CP which contains
984 CONTENT type data. */
986 fill_matrix (struct matrix_data_pgm *mx, int content, double *cp)
988 int type = content_type[content];
990 if (type == 1 && mx->section != FULL)
992 if (mx->diag == NODIAGONAL)
994 const double fill = content == CORR ? 1.0 : SYSMIS;
997 for (i = 0; i < mx->n_continuous; i++)
998 cp[i * (1 + mx->n_continuous)] = fill;
1004 if (mx->section == LOWER)
1006 int n_lines = mx->n_continuous;
1007 if (mx->section != FULL && mx->diag == NODIAGONAL)
1010 for (r = 1; r < n_lines; r++)
1011 for (c = 0; c < r; c++)
1012 cp[r + c * mx->n_continuous] = cp[c + r * mx->n_continuous];
1016 assert (mx->section == UPPER);
1017 for (r = 1; r < mx->n_continuous; r++)
1018 for (c = 0; c < r; c++)
1019 cp[c + r * mx->n_continuous] = cp[r + c * mx->n_continuous];
1027 for (c = 1; c < mx->n_continuous; c++)
1032 /* Read data lines for content type CONTENT from the data file.
1033 If PER_FACTOR is nonzero, then factor information is read from
1034 the data file. Data is for cell number CELL. */
1036 nr_read_data_lines (struct nr_aux_data *nr,
1037 int per_factor, int cell, int content, int compare)
1039 struct matrix_data_pgm *mx = nr->mx;
1040 const int type = content_type[content]; /* Content type. */
1041 int n_lines; /* Number of lines to parse from data file for this type. */
1042 double *cp; /* Current position in vector or matrix. */
1049 n_lines = mx->n_continuous;
1050 if (mx->section != FULL && mx->diag == NODIAGONAL)
1054 cp = nr->data[content][cell];
1055 if (type == 1 && mx->section == LOWER && mx->diag == NODIAGONAL)
1056 cp += mx->n_continuous;
1058 for (i = 0; i < n_lines; i++)
1062 if (!nr_read_splits (nr, 1))
1064 if (per_factor && !nr_read_factors (nr, cell))
1071 n_cols = mx->n_continuous;
1074 switch (mx->section)
1081 n_cols = mx->n_continuous - i;
1082 if (mx->diag == NODIAGONAL)
1089 n_cols = mx->n_continuous;
1107 for (j = 0; j < n_cols; j++)
1109 struct matrix_token token;
1110 if (!mget_token (&token, mx->reader))
1112 if (token.type != MNUM)
1114 msg (SE, _("expecting value for %s %s"),
1115 dict_get_var (default_dict, j)->name,
1116 context (mx->reader));
1120 *cp++ = token.number;
1123 && !force_eol (mx->reader, content_names[content]))
1125 debug_printf (("\n"));
1128 if (mx->section == LOWER)
1129 cp += mx->n_continuous - n_cols;
1132 fill_matrix (mx, content, nr->data[content][cell]);
1137 /* When ROWTYPE_ does not appear in the data, reads the matrices and
1138 writes them to the output file.
1139 Returns true if successful, false if an I/O error occurred. */
1141 matrix_data_read_without_rowtype (struct case_source *source,
1143 write_case_func *write_case,
1144 write_case_data wc_data)
1146 struct nr_aux_data *nr = source->aux;
1147 struct matrix_data_pgm *mx = nr->mx;
1152 nr->data = pool_nalloc (mx->container, PROX + 1, sizeof *nr->data);
1157 for (i = 0; i <= PROX; i++)
1161 for (cp = mx->contents; *cp != EOC; cp++)
1162 if (*cp != LPAREN && *cp != RPAREN)
1164 int per_factor = mx->is_per_factor[*cp];
1167 n_entries = mx->n_continuous;
1168 if (content_type[*cp] == 1)
1169 n_entries *= mx->n_continuous;
1172 int n_vectors = per_factor ? mx->cells : 1;
1175 nr->data[*cp] = pool_nalloc (mx->container,
1176 n_vectors, sizeof **nr->data);
1178 for (i = 0; i < n_vectors; i++)
1179 nr->data[*cp][i] = pool_nalloc (mx->container,
1180 n_entries, sizeof ***nr->data);
1189 if (!nr_read_splits (nr, 0))
1192 for (bp = mx->contents; *bp != EOC; bp = np)
1196 /* Trap the CONTENTS that we should parse in this pass
1197 between bp and ep. Set np to the starting bp for next
1202 while (*ep != RPAREN)
1210 while (*ep != EOC && *ep != LPAREN)
1219 for (i = 0; i < (per_factor ? mx->cells : 1); i++)
1223 for (cp = bp; cp < ep; cp++)
1224 if (!nr_read_data_lines (nr, per_factor, i, *cp, cp != bp))
1230 if (!nr_output_data (nr, c, write_case, wc_data))
1233 if (dict_get_split_cnt (default_dict) == 0
1234 || !another_token (mx->reader))
1239 /* Read the split file variables. If COMPARE is 1, compares the
1240 values read to the last values read and returns 1 if they're equal,
1243 nr_read_splits (struct nr_aux_data *nr, int compare)
1245 struct matrix_data_pgm *mx = nr->mx;
1246 static int just_read = 0; /* FIXME: WTF? */
1250 if (compare && just_read)
1256 if (dict_get_split_vars (default_dict) == NULL)
1259 if (mx->single_split)
1263 struct mxd_var *mv = dict_get_split_vars (default_dict)[0]->aux;
1264 nr->split_values[0] = ++mv->sub_type;
1272 split_cnt = dict_get_split_cnt (default_dict);
1273 for (i = 0; i < split_cnt; i++)
1275 struct matrix_token token;
1276 if (!mget_token (&token, mx->reader))
1278 if (token.type != MNUM)
1280 msg (SE, _("Syntax error expecting SPLIT FILE value %s."),
1281 context (mx->reader));
1286 nr->split_values[i] = token.number;
1287 else if (nr->split_values[i] != token.number)
1289 msg (SE, _("Expecting value %g for %s."),
1290 nr->split_values[i],
1291 dict_get_split_vars (default_dict)[i]->name);
1299 /* Read the factors for cell CELL. If COMPARE is 1, compares the
1300 values read to the last values read and returns 1 if they're equal,
1303 nr_read_factors (struct nr_aux_data *nr, int cell)
1305 struct matrix_data_pgm *mx = nr->mx;
1308 if (mx->n_factors == 0)
1311 assert (nr->max_cell_idx >= cell);
1312 if (cell != nr->max_cell_idx)
1323 for (i = 0; i < mx->n_factors; i++)
1325 struct matrix_token token;
1326 if (!mget_token (&token, mx->reader))
1328 if (token.type != MNUM)
1330 msg (SE, _("Syntax error expecting factor value %s."),
1331 context (mx->reader));
1336 nr->factor_values[i + mx->n_factors * cell] = token.number;
1337 else if (nr->factor_values[i + mx->n_factors * cell] != token.number)
1339 msg (SE, _("Syntax error expecting value %g for %s %s."),
1340 nr->factor_values[i + mx->n_factors * cell],
1341 mx->factors[i]->name, context (mx->reader));
1350 /* Write the contents of a cell having content type CONTENT and data
1351 CP to the active file.
1352 Returns true if successful, false if an I/O error occurred. */
1354 dump_cell_content (struct matrix_data_pgm *mx, int content, double *cp,
1356 write_case_func *write_case, write_case_data wc_data)
1358 int type = content_type[content];
1361 buf_copy_str_rpad (case_data_rw (c, mx->rowtype_->fv)->s, 8,
1362 content_names[content]);
1365 memset (case_data_rw (c, mx->varname_->fv)->s, ' ', 8);
1369 int n_lines = (type == 1) ? mx->n_continuous : 1;
1372 for (i = 0; i < n_lines; i++)
1376 for (j = 0; j < mx->n_continuous; j++)
1378 int fv = dict_get_var (default_dict, mx->first_continuous + j)->fv;
1379 case_data_rw (c, fv)->f = *cp;
1383 buf_copy_str_rpad (case_data_rw (c, mx->varname_->fv)->s, 8,
1384 dict_get_var (default_dict,
1385 mx->first_continuous + i)->name);
1386 if (!write_case (wc_data))
1393 /* Finally dump out everything from nr_data[] to the output file. */
1395 nr_output_data (struct nr_aux_data *nr, struct ccase *c,
1396 write_case_func *write_case, write_case_data wc_data)
1398 struct matrix_data_pgm *mx = nr->mx;
1401 struct variable *const *split;
1405 split_cnt = dict_get_split_cnt (default_dict);
1406 split = dict_get_split_vars (default_dict);
1407 for (i = 0; i < split_cnt; i++)
1408 case_data_rw (c, split[i]->fv)->f = nr->split_values[i];
1415 for (cell = 0; cell < mx->cells; cell++)
1420 for (factor = 0; factor < mx->n_factors; factor++)
1422 case_data_rw (c, mx->factors[factor]->fv)->f
1423 = nr->factor_values[factor + cell * mx->n_factors];
1424 debug_printf (("f:%s ", mx->factors[factor]->name));
1431 for (content = 0; content <= PROX; content++)
1432 if (mx->is_per_factor[content])
1434 assert (nr->data[content] != NULL
1435 && nr->data[content][cell] != NULL);
1437 if (!dump_cell_content (mx, content, nr->data[content][cell],
1438 c, write_case, wc_data))
1451 for (factor = 0; factor < mx->n_factors; factor++)
1452 case_data_rw (c, mx->factors[factor]->fv)->f = SYSMIS;
1455 for (content = 0; content <= PROX; content++)
1456 if (!mx->is_per_factor[content] && nr->data[content] != NULL)
1458 if (!dump_cell_content (mx, content, nr->data[content][0],
1459 c, write_case, wc_data))
1467 /* Back end, with ROWTYPE_. */
1469 /* All the data for one set of factor values. */
1473 int n_rows[PROX + 1];
1474 double *data[PROX + 1];
1475 struct factor_data *next;
1478 /* With ROWTYPE_ auxiliary data. */
1481 struct matrix_data_pgm *mx; /* MATRIX DATA program. */
1482 int content; /* Type of current row. */
1483 double *split_values; /* SPLIT FILE variable values. */
1484 struct factor_data *data; /* All the data. */
1485 struct factor_data *current; /* Current factor. */
1488 static int wr_read_splits (struct wr_aux_data *, struct ccase *,
1489 write_case_func *, write_case_data);
1490 static bool wr_output_data (struct wr_aux_data *, struct ccase *,
1491 write_case_func *, write_case_data);
1492 static int wr_read_rowtype (struct wr_aux_data *,
1493 const struct matrix_token *, struct dfm_reader *);
1494 static int wr_read_factors (struct wr_aux_data *);
1495 static int wr_read_indeps (struct wr_aux_data *);
1496 static bool matrix_data_read_with_rowtype (struct case_source *,
1501 /* When ROWTYPE_ appears in the data, reads the matrices and writes
1502 them to the output file.
1503 Returns true if successful, false if an I/O error occurred. */
1505 read_matrices_with_rowtype (struct matrix_data_pgm *mx)
1507 struct wr_aux_data wr;
1512 wr.split_values = NULL;
1517 vfm_source = create_case_source (&matrix_data_with_rowtype_source_class,
1519 ok = procedure (NULL, NULL);
1521 free (wr.split_values);
1525 /* Read from the data file and write it to the active file.
1526 Returns true if successful, false if an I/O error occurred. */
1528 matrix_data_read_with_rowtype (struct case_source *source,
1530 write_case_func *write_case,
1531 write_case_data wc_data)
1533 struct wr_aux_data *wr = source->aux;
1534 struct matrix_data_pgm *mx = wr->mx;
1538 if (!wr_read_splits (wr, c, write_case, wc_data))
1541 if (!wr_read_factors (wr))
1544 if (!wr_read_indeps (wr))
1547 while (another_token (mx->reader));
1549 return wr_output_data (wr, c, write_case, wc_data);
1552 /* Read the split file variables. If they differ from the previous
1553 set of split variables then output the data. Returns success. */
1555 wr_read_splits (struct wr_aux_data *wr,
1557 write_case_func *write_case, write_case_data wc_data)
1559 struct matrix_data_pgm *mx = wr->mx;
1563 split_cnt = dict_get_split_cnt (default_dict);
1567 if (wr->split_values)
1572 wr->split_values = xnmalloc (split_cnt, sizeof *wr->split_values);
1579 for (i = 0; i < split_cnt; i++)
1581 struct matrix_token token;
1582 if (!mget_token (&token, mx->reader))
1584 if (token.type != MNUM)
1586 msg (SE, _("Syntax error %s expecting SPLIT FILE value."),
1587 context (mx->reader));
1591 if (compare && wr->split_values[i] != token.number && !different)
1593 if (!wr_output_data (wr, c, write_case, wc_data))
1598 wr->split_values[i] = token.number;
1605 /* Compares doubles A and B, treating SYSMIS as greatest. */
1607 compare_doubles (const void *a_, const void *b_, void *aux UNUSED)
1609 const double *a = a_;
1610 const double *b = b_;
1614 else if (*a == SYSMIS)
1616 else if (*b == SYSMIS)
1624 /* Return strcmp()-type comparison of the MX->n_factors factors at _A and
1625 _B. Sort missing values toward the end. */
1627 compare_factors (const void *a_, const void *b_, void *mx_)
1629 struct matrix_data_pgm *mx = mx_;
1630 struct factor_data *const *pa = a_;
1631 struct factor_data *const *pb = b_;
1632 const double *a = (*pa)->factors;
1633 const double *b = (*pb)->factors;
1635 return lexicographical_compare_3way (a, mx->n_factors,
1638 compare_doubles, NULL);
1641 /* Write out the data for the current split file to the active
1643 Returns true if successful, false if an I/O error occurred. */
1645 wr_output_data (struct wr_aux_data *wr,
1647 write_case_func *write_case, write_case_data wc_data)
1649 struct matrix_data_pgm *mx = wr->mx;
1653 struct variable *const *split;
1657 split_cnt = dict_get_split_cnt (default_dict);
1658 split = dict_get_split_vars (default_dict);
1659 for (i = 0; i < split_cnt; i++)
1660 case_data_rw (c, split[i]->fv)->f = wr->split_values[i];
1663 /* Sort the wr->data list. */
1665 struct factor_data **factors;
1666 struct factor_data *iter;
1669 factors = xnmalloc (mx->cells, sizeof *factors);
1671 for (i = 0, iter = wr->data; iter; iter = iter->next, i++)
1674 sort (factors, mx->cells, sizeof *factors, compare_factors, mx);
1676 wr->data = factors[0];
1677 for (i = 0; i < mx->cells - 1; i++)
1678 factors[i]->next = factors[i + 1];
1679 factors[mx->cells - 1]->next = NULL;
1684 /* Write out records for every set of factor values. */
1686 struct factor_data *iter;
1688 for (iter = wr->data; iter; iter = iter->next)
1693 for (factor = 0; factor < mx->n_factors; factor++)
1694 case_data_rw (c, mx->factors[factor]->fv)->f
1695 = iter->factors[factor];
1701 for (content = 0; content <= PROX; content++)
1703 if (!iter->n_rows[content])
1707 int type = content_type[content];
1708 int n_lines = (type == 1
1710 - (mx->section != FULL && mx->diag == NODIAGONAL))
1713 if (n_lines != iter->n_rows[content])
1715 msg (SE, _("Expected %d lines of data for %s content; "
1716 "actually saw %d lines. No data will be "
1717 "output for this content."),
1718 n_lines, content_names[content],
1719 iter->n_rows[content]);
1724 fill_matrix (mx, content, iter->data[content]);
1726 ok = dump_cell_content (mx, content, iter->data[content],
1727 c, write_case, wc_data);
1735 pool_destroy (mx->container);
1736 mx->container = pool_create ();
1738 wr->data = wr->current = NULL;
1743 /* Sets ROWTYPE_ based on the given TOKEN read from READER.
1746 wr_read_rowtype (struct wr_aux_data *wr,
1747 const struct matrix_token *token,
1748 struct dfm_reader *reader)
1750 if (wr->content != -1)
1752 msg (SE, _("Multiply specified ROWTYPE_ %s."), context (reader));
1755 if (token->type != MSTR)
1757 msg (SE, _("Syntax error %s expecting ROWTYPE_ string."),
1766 memcpy (s, token->string, min (15, token->length));
1767 s[min (15, token->length)] = 0;
1769 for (cp = s; *cp; cp++)
1770 *cp = toupper ((unsigned char) *cp);
1772 wr->content = string_to_content_type (s, NULL);
1775 if (wr->content == -1)
1777 msg (SE, _("Syntax error %s."), context (reader));
1784 /* Read the factors for the current row. Select a set of factors and
1785 point wr_current to it. */
1787 wr_read_factors (struct wr_aux_data *wr)
1789 struct matrix_data_pgm *mx = wr->mx;
1790 double *factor_values = local_alloc (sizeof *factor_values * mx->n_factors);
1796 for (i = 0; i < mx->n_factors; i++)
1798 struct matrix_token token;
1799 if (!mget_token (&token, mx->reader))
1801 if (token.type == MSTR)
1803 if (!wr_read_rowtype (wr, &token, mx->reader))
1805 if (!mget_token (&token, mx->reader))
1808 if (token.type != MNUM)
1810 msg (SE, _("Syntax error expecting factor value %s."),
1811 context (mx->reader));
1815 factor_values[i] = token.number;
1818 if (wr->content == -1)
1820 struct matrix_token token;
1821 if (!mget_token (&token, mx->reader))
1823 if (!wr_read_rowtype (wr, &token, mx->reader))
1827 /* Try the most recent factor first as a simple caching
1833 for (i = 0; i < mx->n_factors; i++)
1834 if (factor_values[i] != wr->current->factors[i])
1839 /* Linear search through the list. */
1842 struct factor_data *iter;
1844 for (iter = wr->data; iter; iter = iter->next)
1848 for (i = 0; i < mx->n_factors; i++)
1849 if (factor_values[i] != iter->factors[i])
1859 /* Not found. Make a new item. */
1861 struct factor_data *new = pool_alloc (mx->container, sizeof *new);
1863 new->factors = pool_nalloc (mx->container,
1864 mx->n_factors, sizeof *new->factors);
1869 for (i = 0; i < mx->n_factors; i++)
1870 new->factors[i] = factor_values[i];
1876 for (i = 0; i <= PROX; i++)
1879 new->data[i] = NULL;
1883 new->next = wr->data;
1884 wr->data = wr->current = new;
1889 local_free (factor_values);
1893 local_free (factor_values);
1897 /* Read the independent variables into wr->current. */
1899 wr_read_indeps (struct wr_aux_data *wr)
1901 struct matrix_data_pgm *mx = wr->mx;
1902 struct factor_data *c = wr->current;
1903 const int type = content_type[wr->content];
1904 const int n_rows = c->n_rows[wr->content];
1908 /* Allocate room for data if necessary. */
1909 if (c->data[wr->content] == NULL)
1911 int n_items = mx->n_continuous;
1913 n_items *= mx->n_continuous;
1915 c->data[wr->content] = pool_nalloc (mx->container,
1916 n_items, sizeof **c->data);
1919 cp = &c->data[wr->content][n_rows * mx->n_continuous];
1921 /* Figure out how much to read from this line. */
1928 msg (SE, _("Duplicate specification for %s."),
1929 content_names[wr->content]);
1933 n_cols = mx->n_continuous;
1938 if (n_rows >= mx->n_continuous - (mx->section != FULL && mx->diag == NODIAGONAL))
1940 msg (SE, _("Too many rows of matrix data for %s."),
1941 content_names[wr->content]);
1945 switch (mx->section)
1948 n_cols = n_rows + 1;
1949 if (mx->diag == NODIAGONAL)
1950 cp += mx->n_continuous;
1954 n_cols = mx->n_continuous - n_rows;
1955 if (mx->diag == NODIAGONAL)
1962 n_cols = mx->n_continuous;
1973 c->n_rows[wr->content]++;
1975 debug_printf ((" (c=%p,r=%d,n=%d)", c, n_rows + 1, n_cols));
1977 /* Read N_COLS items at CP. */
1981 for (j = 0; j < n_cols; j++)
1983 struct matrix_token token;
1984 if (!mget_token (&token, mx->reader))
1986 if (token.type != MNUM)
1988 msg (SE, _("Syntax error expecting value for %s %s."),
1989 dict_get_var (default_dict, mx->first_continuous + j)->name,
1990 context (mx->reader));
1994 *cp++ = token.number;
1997 && !force_eol (mx->reader, content_names[wr->content]))
1999 debug_printf (("\n"));
2005 /* Matrix source. */
2007 static const struct case_source_class matrix_data_with_rowtype_source_class =
2011 matrix_data_read_with_rowtype,
2015 static const struct case_source_class
2016 matrix_data_without_rowtype_source_class =
2020 matrix_data_read_without_rowtype,