1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2009 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <math/design-matrix.h>
20 #include <gsl/gsl_matrix.h>
21 #include <data/casegrouper.h>
22 #include <data/casereader.h>
23 #include <data/dictionary.h>
24 #include <data/procedure.h>
25 #include <data/variable.h>
26 #include <language/command.h>
27 #include <language/dictionary/split-file.h>
28 #include <language/lexer/lexer.h>
29 #include <language/lexer/variable-parser.h>
30 #include <output/manager.h>
31 #include <output/table.h>
32 #include <libpspp/message.h>
33 #include <data/format.h>
38 #include <libpspp/misc.h>
39 #include <gsl/gsl_cdf.h>
42 #define _(msgid) gettext (msgid)
43 #define N_(msgid) msgid
45 /* Returns the correlation matrix corresponding to the covariance
46 matrix COV. The return value must be freed with gsl_matrix_free
47 when no longer required.
50 covariance_to_correlation (const gsl_matrix *cov)
53 gsl_matrix *corr = gsl_matrix_alloc (cov->size1, cov->size2);
55 for (r = 0 ; r < cov->size1; ++r)
57 for (c = 0 ; c < cov->size2 ; ++c)
59 double x = gsl_matrix_get (cov, r, c);
60 x /= sqrt (gsl_matrix_get (cov, r, r)
61 * gsl_matrix_get (cov, c, c) );
62 gsl_matrix_set (corr, r, c, x);
70 significance_of_correlation (double rho, double w)
73 t /= 1 - MIN (1, pow2 (rho));
78 return gsl_cdf_tdist_Q (t, w - 2);
80 return gsl_cdf_tdist_P (t, w - 2);
89 const struct variable **vars;
93 /* Handling of missing values. */
94 enum corr_missing_type
96 CORR_PAIRWISE, /* Handle missing values on a per-variable-pair basis. */
97 CORR_LISTWISE /* Discard entire case if any variable is missing. */
102 enum corr_missing_type missing_type;
103 enum mv_class exclude; /* Classes of missing values to exclude. */
105 bool sig; /* Flag significant values or not */
106 int tails; /* Report significance with how many tails ? */
108 const struct variable *wv; /* The weight variable (if any) */
113 output_correlation (const struct corr *corr, const struct corr_opts *opts,
114 const gsl_matrix *cm, const gsl_matrix *samples)
119 int nr = corr->n_vars1;
120 int nc = matrix_cols = corr->n_vars_total > corr->n_vars1 ?
121 corr->n_vars_total - corr->n_vars1 : corr->n_vars1;
123 const struct fmt_spec *wfmt = opts->wv ? var_get_print_format (opts->wv) : & F_8_0;
125 const int heading_columns = 2;
126 const int heading_rows = 1;
128 const int rows_per_variable = opts->missing_type == CORR_LISTWISE ? 2 : 3;
130 /* Two header columns */
131 nc += heading_columns;
133 /* Three data per variable */
134 nr *= rows_per_variable;
139 t = tab_create (nc, nr, 0);
140 tab_title (t, _("Correlations"));
141 tab_dim (t, tab_natural_dimensions, NULL);
143 tab_headers (t, heading_columns, 0, heading_rows, 0);
145 /* Outline the box */
159 tab_vline (t, TAL_2, heading_columns, 0, nr - 1);
160 tab_vline (t, TAL_1, 1, heading_rows, nr - 1);
162 for (r = 0 ; r < corr->n_vars1 ; ++r)
164 tab_text (t, 0, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE,
165 var_to_string (corr->vars[r]));
167 tab_text (t, 1, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Pearson Correlation"));
168 tab_text (t, 1, 2 + r * rows_per_variable, TAB_LEFT | TAT_TITLE,
169 (opts->tails == 2) ? _("Sig. (2-tailed)") : _("Sig. (1-tailed)"));
170 if ( opts->missing_type != CORR_LISTWISE )
171 tab_text (t, 1, 3 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("N"));
172 tab_hline (t, TAL_1, 0, nc - 1, r * rows_per_variable + 1);
175 for (c = 0 ; c < matrix_cols ; ++c)
177 const struct variable *v = corr->n_vars_total > corr->n_vars1 ? corr->vars[corr->n_vars_total - corr->n_vars1 + c] : corr->vars[c];
178 tab_text (t, heading_columns + c, 0, TAB_LEFT | TAT_TITLE, var_to_string (v));
181 for (r = 0 ; r < corr->n_vars1 ; ++r)
183 const int row = r * rows_per_variable + heading_rows;
184 for (c = 0 ; c < matrix_cols ; ++c)
186 unsigned char flags = 0;
187 int col_index = corr->n_vars_total - corr->n_vars1 + c;
188 double pearson = gsl_matrix_get (cm, r, col_index);
189 double w = gsl_matrix_get (samples, r, col_index);
190 double sig = opts->tails * significance_of_correlation (pearson, w);
192 if ( opts->missing_type != CORR_LISTWISE )
193 tab_double (t, c + heading_columns, row + 2, 0, w, wfmt);
196 tab_double (t, c + heading_columns, row + 1, 0, sig, NULL);
198 if ( opts->sig && c != r && sig < 0.05)
201 tab_double (t, c + heading_columns, row, flags, pearson, NULL);
209 run_corr (struct casereader *r, const struct corr_opts *opts, const struct corr *corr)
212 const struct design_matrix *cov_matrix;
213 const gsl_matrix *samples_matrix;
215 for ( ; (c = casereader_read (r) ); case_unref (c))
221 output_correlation (corr, opts,
222 covariance_to_correlation (cov_matrix->m),
227 cmd_correlation (struct lexer *lexer, struct dataset *ds)
229 int n_all_vars = 0; /* Total number of variables involved in this command */
230 const struct dictionary *dict = dataset_dict (ds);
233 struct casegrouper *grouper;
234 struct casereader *group;
236 struct corr *corr = NULL;
239 struct corr_opts opts;
240 opts.missing_type = CORR_PAIRWISE;
241 opts.wv = dict_get_weight (dict);
244 opts.exclude = MV_ANY;
246 /* Parse CORRELATIONS. */
247 while (lex_token (lexer) != '.')
249 lex_match (lexer, '/');
250 if (lex_match_id (lexer, "MISSING"))
252 lex_match (lexer, '=');
253 while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
255 if (lex_match_id (lexer, "PAIRWISE"))
256 opts.missing_type = CORR_PAIRWISE;
257 else if (lex_match_id (lexer, "LISTWISE"))
258 opts.missing_type = CORR_LISTWISE;
260 else if (lex_match_id (lexer, "INCLUDE"))
261 opts.exclude = MV_SYSTEM;
262 else if (lex_match_id (lexer, "EXCLUDE"))
263 opts.exclude = MV_ANY;
266 lex_error (lexer, NULL);
269 lex_match (lexer, ',');
272 else if (lex_match_id (lexer, "PRINT"))
274 lex_match (lexer, '=');
275 while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
277 if ( lex_match_id (lexer, "TWOTAIL"))
279 else if (lex_match_id (lexer, "ONETAIL"))
281 else if (lex_match_id (lexer, "SIG"))
283 else if (lex_match_id (lexer, "NOSIG"))
287 lex_error (lexer, NULL);
291 lex_match (lexer, ',');
296 if (lex_match_id (lexer, "VARIABLES"))
298 lex_match (lexer, '=');
301 corr = xrealloc (corr, sizeof (*corr) * (n_corrs + 1));
302 corr[n_corrs].n_vars_total = corr[n_corrs].n_vars1 = 0;
304 if ( ! parse_variables_const (lexer, dict, &corr[n_corrs].vars,
305 &corr[n_corrs].n_vars_total,
313 corr[n_corrs].n_vars1 = corr[n_corrs].n_vars_total;
315 if ( lex_match (lexer, T_WITH))
317 if ( ! parse_variables_const (lexer, dict,
318 &corr[n_corrs].vars, &corr[n_corrs].n_vars_total,
319 PV_NUMERIC | PV_APPEND))
326 n_all_vars += corr[n_corrs].n_vars_total;
334 msg (SE, _("No variables specified."));
339 const struct variable **all_vars = xmalloc (sizeof (*all_vars) * n_all_vars);
343 /* FIXME: Using a hash here would make more sense */
344 const struct variable **vv = all_vars;
346 for (i = 0 ; i < n_corrs; ++i)
349 const struct corr *c = &corr[i];
350 for (v = 0 ; v < c->n_vars_total; ++v)
355 grouper = casegrouper_create_splits (proc_open (ds), dict);
357 while (casegrouper_get_next_group (grouper, &group))
359 for (i = 0 ; i < n_corrs; ++i)
361 /* FIXME: No need to iterate the data multiple times */
362 struct casereader *r = casereader_clone (group);
364 if ( opts.missing_type == CORR_LISTWISE)
365 r = casereader_create_filter_missing (r, all_vars, n_all_vars,
366 opts.exclude, NULL, NULL);
368 run_corr (r, &opts, &corr[i]);
369 casereader_destroy (r);
371 casereader_destroy (group);
374 ok = casegrouper_destroy (grouper);
375 ok = proc_commit (ds) && ok;
382 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;