1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2009 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <libpspp/assertion.h>
20 #include <math/covariance.h>
21 #include <math/design-matrix.h>
22 #include <gsl/gsl_matrix.h>
23 #include <data/casegrouper.h>
24 #include <data/casereader.h>
25 #include <data/dictionary.h>
26 #include <data/procedure.h>
27 #include <data/variable.h>
28 #include <language/command.h>
29 #include <language/dictionary/split-file.h>
30 #include <language/lexer/lexer.h>
31 #include <language/lexer/variable-parser.h>
32 #include <output/manager.h>
33 #include <output/table.h>
34 #include <libpspp/message.h>
35 #include <data/format.h>
36 #include <math/moments.h>
41 #include <libpspp/misc.h>
42 #include <gsl/gsl_cdf.h>
45 #define _(msgid) gettext (msgid)
46 #define N_(msgid) msgid
50 significance_of_correlation (double rho, double w)
53 t /= 1 - MIN (1, pow2 (rho));
58 return gsl_cdf_tdist_Q (t, w - 2);
60 return gsl_cdf_tdist_P (t, w - 2);
69 const struct variable **vars;
73 /* Handling of missing values. */
74 enum corr_missing_type
76 CORR_PAIRWISE, /* Handle missing values on a per-variable-pair basis. */
77 CORR_LISTWISE /* Discard entire case if any variable is missing. */
82 STATS_DESCRIPTIVES = 0x01,
84 STATS_ALL = STATS_XPROD | STATS_DESCRIPTIVES
89 enum corr_missing_type missing_type;
90 enum mv_class exclude; /* Classes of missing values to exclude. */
92 bool sig; /* Flag significant values or not */
93 int tails; /* Report significance with how many tails ? */
94 enum stats_opts statistics;
96 const struct variable *wv; /* The weight variable (if any) */
101 output_descriptives (const struct corr *corr, const gsl_matrix *means,
102 const gsl_matrix *vars, const gsl_matrix *ns)
104 const int nr = corr->n_vars_total + 1;
108 const int heading_columns = 1;
109 const int heading_rows = 1;
111 struct tab_table *t = tab_create (nc, nr, 0);
112 tab_title (t, _("Descriptive Statistics"));
113 tab_dim (t, tab_natural_dimensions, NULL);
115 tab_headers (t, heading_columns, 0, heading_rows, 0);
117 /* Outline the box */
131 tab_vline (t, TAL_2, heading_columns, 0, nr - 1);
132 tab_hline (t, TAL_1, 0, nc - 1, heading_rows);
134 tab_text (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Mean"));
135 tab_text (t, 2, 0, TAB_CENTER | TAT_TITLE, _("Std. Deviation"));
136 tab_text (t, 3, 0, TAB_CENTER | TAT_TITLE, _("N"));
138 for (r = 0 ; r < corr->n_vars_total ; ++r)
140 const struct variable *v = corr->vars[r];
141 tab_text (t, 0, r + heading_rows, TAB_LEFT | TAT_TITLE, var_to_string (v));
143 for (c = 1 ; c < nc ; ++c)
150 x = gsl_matrix_get (means, r, 0);
153 x = gsl_matrix_get (vars, r, 0);
155 /* Here we want to display the non-biased estimator */
156 n = gsl_matrix_get (ns, r, 0);
162 x = gsl_matrix_get (ns, r, 0);
168 tab_double (t, c, r + heading_rows, 0, x, NULL);
176 output_correlation (const struct corr *corr, const struct corr_opts *opts,
177 const gsl_matrix *cm, const gsl_matrix *samples,
178 const gsl_matrix *cv)
183 int nr = corr->n_vars1;
184 int nc = matrix_cols = corr->n_vars_total > corr->n_vars1 ?
185 corr->n_vars_total - corr->n_vars1 : corr->n_vars1;
187 const struct fmt_spec *wfmt = opts->wv ? var_get_print_format (opts->wv) : & F_8_0;
189 const int heading_columns = 2;
190 const int heading_rows = 1;
192 int rows_per_variable = opts->missing_type == CORR_LISTWISE ? 2 : 3;
194 if (opts->statistics & STATS_XPROD)
195 rows_per_variable += 2;
197 /* Two header columns */
198 nc += heading_columns;
200 /* Three data per variable */
201 nr *= rows_per_variable;
206 t = tab_create (nc, nr, 0);
207 tab_title (t, _("Correlations"));
208 tab_dim (t, tab_natural_dimensions, NULL);
210 tab_headers (t, heading_columns, 0, heading_rows, 0);
212 /* Outline the box */
226 tab_vline (t, TAL_2, heading_columns, 0, nr - 1);
227 tab_vline (t, TAL_1, 1, heading_rows, nr - 1);
229 for (r = 0 ; r < corr->n_vars1 ; ++r)
231 tab_text (t, 0, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE,
232 var_to_string (corr->vars[r]));
234 tab_text (t, 1, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Pearson Correlation"));
235 tab_text (t, 1, 2 + r * rows_per_variable, TAB_LEFT | TAT_TITLE,
236 (opts->tails == 2) ? _("Sig. (2-tailed)") : _("Sig. (1-tailed)"));
238 if (opts->statistics & STATS_XPROD)
240 tab_text (t, 1, 3 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Cross-products"));
241 tab_text (t, 1, 4 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Covariance"));
244 if ( opts->missing_type != CORR_LISTWISE )
245 tab_text (t, 1, rows_per_variable + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("N"));
247 tab_hline (t, TAL_1, 0, nc - 1, r * rows_per_variable + 1);
250 for (c = 0 ; c < matrix_cols ; ++c)
252 const struct variable *v = corr->n_vars_total > corr->n_vars1 ? corr->vars[corr->n_vars_total - corr->n_vars1 + c] : corr->vars[c];
253 tab_text (t, heading_columns + c, 0, TAB_LEFT | TAT_TITLE, var_to_string (v));
256 for (r = 0 ; r < corr->n_vars1 ; ++r)
258 const int row = r * rows_per_variable + heading_rows;
259 for (c = 0 ; c < matrix_cols ; ++c)
261 unsigned char flags = 0;
262 const int col_index = corr->n_vars_total - corr->n_vars1 + c;
263 double pearson = gsl_matrix_get (cm, r, col_index);
264 double w = gsl_matrix_get (samples, r, col_index);
265 double sig = opts->tails * significance_of_correlation (pearson, w);
267 if ( opts->missing_type != CORR_LISTWISE )
268 tab_double (t, c + heading_columns, row + rows_per_variable - 1, 0, w, wfmt);
271 tab_double (t, c + heading_columns, row + 1, 0, sig, NULL);
273 if ( opts->sig && c != r && sig < 0.05)
276 tab_double (t, c + heading_columns, row, flags, pearson, NULL);
278 if (opts->statistics & STATS_XPROD)
280 double cov = gsl_matrix_get (cv, r, col_index);
281 const double xprod_dev = cov * w;
282 cov *= w / (w - 1.0);
284 tab_double (t, c + heading_columns, row + 2, 0, xprod_dev, NULL);
285 tab_double (t, c + heading_columns, row + 3, 0, cov, NULL);
295 correlation_from_covariance (const gsl_matrix *cv, const gsl_matrix *v)
298 gsl_matrix *corr = gsl_matrix_calloc (cv->size1, cv->size2);
300 for (i = 0 ; i < cv->size1; ++i)
302 for (j = 0 ; j < cv->size2; ++j)
304 double rho = gsl_matrix_get (cv, i, j);
306 rho /= sqrt (gsl_matrix_get (v, i, j))
308 sqrt (gsl_matrix_get (v, j, i));
310 gsl_matrix_set (corr, i, j, rho);
321 run_corr (struct casereader *r, const struct corr_opts *opts, const struct corr *corr)
324 const gsl_matrix *var_matrix, *samples_matrix, *mean_matrix;
325 const gsl_matrix *cov_matrix;
326 gsl_matrix *corr_matrix;
327 struct covariance *cov = covariance_2pass_create (corr->n_vars_total, corr->vars,
329 opts->wv, opts->exclude);
331 struct casereader *rc = casereader_clone (r);
332 for ( ; (c = casereader_read (r) ); case_unref (c))
334 covariance_accumulate_pass1 (cov, c);
337 for ( ; (c = casereader_read (rc) ); case_unref (c))
339 covariance_accumulate_pass2 (cov, c);
342 cov_matrix = covariance_calculate (cov);
344 casereader_destroy (rc);
346 samples_matrix = covariance_moments (cov, MOMENT_NONE);
347 var_matrix = covariance_moments (cov, MOMENT_VARIANCE);
348 mean_matrix = covariance_moments (cov, MOMENT_MEAN);
350 corr_matrix = correlation_from_covariance (cov_matrix, var_matrix);
352 if ( opts->statistics & STATS_DESCRIPTIVES)
353 output_descriptives (corr, mean_matrix, var_matrix, samples_matrix);
355 output_correlation (corr, opts,
360 covariance_destroy (cov);
361 gsl_matrix_free (corr_matrix);
365 cmd_correlation (struct lexer *lexer, struct dataset *ds)
368 int n_all_vars = 0; /* Total number of variables involved in this command */
369 const struct variable **all_vars ;
370 const struct dictionary *dict = dataset_dict (ds);
373 struct casegrouper *grouper;
374 struct casereader *group;
376 struct corr *corr = NULL;
379 struct corr_opts opts;
380 opts.missing_type = CORR_PAIRWISE;
381 opts.wv = dict_get_weight (dict);
384 opts.exclude = MV_ANY;
387 /* Parse CORRELATIONS. */
388 while (lex_token (lexer) != '.')
390 lex_match (lexer, '/');
391 if (lex_match_id (lexer, "MISSING"))
393 lex_match (lexer, '=');
394 while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
396 if (lex_match_id (lexer, "PAIRWISE"))
397 opts.missing_type = CORR_PAIRWISE;
398 else if (lex_match_id (lexer, "LISTWISE"))
399 opts.missing_type = CORR_LISTWISE;
401 else if (lex_match_id (lexer, "INCLUDE"))
402 opts.exclude = MV_SYSTEM;
403 else if (lex_match_id (lexer, "EXCLUDE"))
404 opts.exclude = MV_ANY;
407 lex_error (lexer, NULL);
410 lex_match (lexer, ',');
413 else if (lex_match_id (lexer, "PRINT"))
415 lex_match (lexer, '=');
416 while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
418 if ( lex_match_id (lexer, "TWOTAIL"))
420 else if (lex_match_id (lexer, "ONETAIL"))
422 else if (lex_match_id (lexer, "SIG"))
424 else if (lex_match_id (lexer, "NOSIG"))
428 lex_error (lexer, NULL);
432 lex_match (lexer, ',');
435 else if (lex_match_id (lexer, "STATISTICS"))
437 lex_match (lexer, '=');
438 while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
440 if ( lex_match_id (lexer, "DESCRIPTIVES"))
441 opts.statistics = STATS_DESCRIPTIVES;
442 else if (lex_match_id (lexer, "XPROD"))
443 opts.statistics = STATS_XPROD;
444 else if (lex_token (lexer) == T_ALL)
446 opts.statistics = STATS_ALL;
451 lex_error (lexer, NULL);
455 lex_match (lexer, ',');
460 if (lex_match_id (lexer, "VARIABLES"))
462 lex_match (lexer, '=');
465 corr = xrealloc (corr, sizeof (*corr) * (n_corrs + 1));
466 corr[n_corrs].n_vars_total = corr[n_corrs].n_vars1 = 0;
468 if ( ! parse_variables_const (lexer, dict, &corr[n_corrs].vars,
469 &corr[n_corrs].n_vars_total,
477 corr[n_corrs].n_vars1 = corr[n_corrs].n_vars_total;
479 if ( lex_match (lexer, T_WITH))
481 if ( ! parse_variables_const (lexer, dict,
482 &corr[n_corrs].vars, &corr[n_corrs].n_vars_total,
483 PV_NUMERIC | PV_APPEND))
490 n_all_vars += corr[n_corrs].n_vars_total;
498 msg (SE, _("No variables specified."));
503 all_vars = xmalloc (sizeof (*all_vars) * n_all_vars);
506 /* FIXME: Using a hash here would make more sense */
507 const struct variable **vv = all_vars;
509 for (i = 0 ; i < n_corrs; ++i)
512 const struct corr *c = &corr[i];
513 for (v = 0 ; v < c->n_vars_total; ++v)
518 grouper = casegrouper_create_splits (proc_open (ds), dict);
520 while (casegrouper_get_next_group (grouper, &group))
522 for (i = 0 ; i < n_corrs; ++i)
524 /* FIXME: No need to iterate the data multiple times */
525 struct casereader *r = casereader_clone (group);
527 if ( opts.missing_type == CORR_LISTWISE)
528 r = casereader_create_filter_missing (r, all_vars, n_all_vars,
529 opts.exclude, NULL, NULL);
532 run_corr (r, &opts, &corr[i]);
533 casereader_destroy (r);
535 casereader_destroy (group);
538 ok = casegrouper_destroy (grouper);
539 ok = proc_commit (ds) && ok;
546 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;