X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fcorrelations.c;h=b33baf5758372a4a50d869cda1cecd9a4224a0a6;hb=6776d19d2b602afa75fd65ba7f08d90a23691645;hp=ad90ef409051f47d12d8fcc2e48f5280fadf75aa;hpb=cb586666724d5fcbdb658ce471b85484f0a7babe;p=pspp diff --git a/src/language/stats/correlations.c b/src/language/stats/correlations.c index ad90ef4090..b33baf5758 100644 --- a/src/language/stats/correlations.c +++ b/src/language/stats/correlations.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2009 Free Software Foundation, Inc. + Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,51 +16,36 @@ #include -#include -#include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include "xalloc.h" -#include "minmax.h" -#include -#include + +#include "data/casegrouper.h" +#include "data/casereader.h" +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/format.h" +#include "data/variable.h" +#include "language/command.h" +#include "language/dictionary/split-file.h" +#include "language/lexer/lexer.h" +#include "language/lexer/variable-parser.h" +#include "libpspp/assertion.h" +#include "libpspp/message.h" +#include "libpspp/misc.h" +#include "math/correlation.h" +#include "math/covariance.h" +#include "math/moments.h" +#include "output/pivot-table.h" + +#include "gl/xalloc.h" +#include "gl/minmax.h" #include "gettext.h" #define _(msgid) gettext (msgid) #define N_(msgid) msgid -static double -significance_of_correlation (double rho, double w) -{ - double t = w - 2; - t /= 1 - MIN (1, pow2 (rho)); - t = sqrt (t); - t *= rho; - - if (t > 0) - return gsl_cdf_tdist_Q (t, w - 2); - else - return gsl_cdf_tdist_P (t, w - 2); -} - - struct corr { size_t n_vars_total; @@ -98,78 +83,39 @@ struct corr_opts static void -output_descriptives (const struct corr *corr, const gsl_matrix *means, +output_descriptives (const struct corr *corr, const struct corr_opts *opts, + const gsl_matrix *means, const gsl_matrix *vars, const gsl_matrix *ns) { - const int nr = corr->n_vars_total + 1; - const int nc = 4; - int c, r; - - const int heading_columns = 1; - const int heading_rows = 1; + struct pivot_table *table = pivot_table_create ( + N_("Descriptive Statistics")); + pivot_table_set_weight_var (table, opts->wv); - struct tab_table *t = tab_create (nc, nr); - tab_title (t, _("Descriptive Statistics")); - tab_dim (t, tab_natural_dimensions, NULL, NULL); + pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Statistics"), + N_("Mean"), PIVOT_RC_OTHER, + N_("Std. Deviation"), PIVOT_RC_OTHER, + N_("N"), PIVOT_RC_COUNT); - tab_headers (t, heading_columns, 0, heading_rows, 0); + struct pivot_dimension *variables = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Variable")); - /* Outline the box */ - tab_box (t, - TAL_2, TAL_2, - -1, -1, - 0, 0, - nc - 1, nr - 1); - - /* Vertical lines */ - tab_box (t, - -1, -1, - -1, TAL_1, - heading_columns, 0, - nc - 1, nr - 1); - - tab_vline (t, TAL_2, heading_columns, 0, nr - 1); - tab_hline (t, TAL_1, 0, nc - 1, heading_rows); - - tab_text (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Mean")); - tab_text (t, 2, 0, TAB_CENTER | TAT_TITLE, _("Std. Deviation")); - tab_text (t, 3, 0, TAB_CENTER | TAT_TITLE, _("N")); - - for (r = 0 ; r < corr->n_vars_total ; ++r) + for (size_t r = 0 ; r < corr->n_vars_total ; ++r) { const struct variable *v = corr->vars[r]; - tab_text (t, 0, r + heading_rows, TAB_LEFT | TAT_TITLE, var_to_string (v)); - for (c = 1 ; c < nc ; ++c) - { - double x ; - double n; - switch (c) - { - case 1: - x = gsl_matrix_get (means, r, 0); - break; - case 2: - x = gsl_matrix_get (vars, r, 0); + int row = pivot_category_create_leaf (variables->root, + pivot_value_new_variable (v)); - /* Here we want to display the non-biased estimator */ - n = gsl_matrix_get (ns, r, 0); - x *= n / (n -1); - - x = sqrt (x); - break; - case 3: - x = gsl_matrix_get (ns, r, 0); - break; - default: - NOT_REACHED (); - }; - - tab_double (t, c, r + heading_rows, 0, x, NULL); - } + double mean = gsl_matrix_get (means, r, 0); + /* Here we want to display the non-biased estimator */ + double n = gsl_matrix_get (ns, r, 0); + double stddev = sqrt (gsl_matrix_get (vars, r, 0) * n / (n - 1)); + double entries[] = { mean, stddev, n }; + for (size_t i = 0; i < sizeof entries / sizeof *entries; i++) + pivot_table_put2 (table, i, row, pivot_value_new_number (entries[i])); } - tab_submit (t); + pivot_table_submit (table); } static void @@ -177,162 +123,117 @@ output_correlation (const struct corr *corr, const struct corr_opts *opts, const gsl_matrix *cm, const gsl_matrix *samples, const gsl_matrix *cv) { - int r, c; - struct tab_table *t; - int matrix_cols; - int nr = corr->n_vars1; - int nc = matrix_cols = corr->n_vars_total > corr->n_vars1 ? - corr->n_vars_total - corr->n_vars1 : corr->n_vars1; - - const struct fmt_spec *wfmt = opts->wv ? var_get_print_format (opts->wv) : & F_8_0; - - const int heading_columns = 2; - const int heading_rows = 1; - - int rows_per_variable = opts->missing_type == CORR_LISTWISE ? 2 : 3; - - if (opts->statistics & STATS_XPROD) - rows_per_variable += 2; - - /* Two header columns */ - nc += heading_columns; - - /* Three data per variable */ - nr *= rows_per_variable; + struct pivot_table *table = pivot_table_create (N_("Correlations")); + pivot_table_set_weight_var (table, opts->wv); - /* One header row */ - nr += heading_rows; + /* Column variable dimension. */ + struct pivot_dimension *columns = pivot_dimension_create ( + table, PIVOT_AXIS_COLUMN, N_("Variables")); - t = tab_create (nc, nr); - tab_title (t, _("Correlations")); - tab_dim (t, tab_natural_dimensions, NULL, NULL); - - tab_headers (t, heading_columns, 0, heading_rows, 0); - - /* Outline the box */ - tab_box (t, - TAL_2, TAL_2, - -1, -1, - 0, 0, - nc - 1, nr - 1); - - /* Vertical lines */ - tab_box (t, - -1, -1, - -1, TAL_1, - heading_columns, 0, - nc - 1, nr - 1); - - tab_vline (t, TAL_2, heading_columns, 0, nr - 1); - tab_vline (t, TAL_1, 1, heading_rows, nr - 1); - - for (r = 0 ; r < corr->n_vars1 ; ++r) + int matrix_cols = (corr->n_vars_total > corr->n_vars1 + ? corr->n_vars_total - corr->n_vars1 + : corr->n_vars1); + for (int c = 0; c < matrix_cols; c++) { - tab_text (t, 0, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, - var_to_string (corr->vars[r])); - - tab_text (t, 1, 1 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Pearson Correlation")); - tab_text (t, 1, 2 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, - (opts->tails == 2) ? _("Sig. (2-tailed)") : _("Sig. (1-tailed)")); - - if (opts->statistics & STATS_XPROD) - { - tab_text (t, 1, 3 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Cross-products")); - tab_text (t, 1, 4 + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("Covariance")); - } - - if ( opts->missing_type != CORR_LISTWISE ) - tab_text (t, 1, rows_per_variable + r * rows_per_variable, TAB_LEFT | TAT_TITLE, _("N")); - - tab_hline (t, TAL_1, 0, nc - 1, r * rows_per_variable + 1); - } - - for (c = 0 ; c < matrix_cols ; ++c) - { - const struct variable *v = corr->n_vars_total > corr->n_vars1 ? corr->vars[corr->n_vars_total - corr->n_vars1 + c] : corr->vars[c]; - tab_text (t, heading_columns + c, 0, TAB_LEFT | TAT_TITLE, var_to_string (v)); + const struct variable *v = corr->n_vars_total > corr->n_vars1 ? + corr->vars[corr->n_vars1 + c] : corr->vars[c]; + pivot_category_create_leaf (columns->root, pivot_value_new_variable (v)); } - for (r = 0 ; r < corr->n_vars1 ; ++r) - { - const int row = r * rows_per_variable + heading_rows; - for (c = 0 ; c < matrix_cols ; ++c) - { - unsigned char flags = 0; - const int col_index = corr->n_vars_total - corr->n_vars1 + c; - double pearson = gsl_matrix_get (cm, r, col_index); - double w = gsl_matrix_get (samples, r, col_index); - double sig = opts->tails * significance_of_correlation (pearson, w); - - if ( opts->missing_type != CORR_LISTWISE ) - tab_double (t, c + heading_columns, row + rows_per_variable - 1, 0, w, wfmt); - - if ( c != r) - tab_double (t, c + heading_columns, row + 1, 0, sig, NULL); + /* Statistics dimension. */ + struct pivot_dimension *statistics = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Statistics"), + N_("Pearson Correlation"), PIVOT_RC_CORRELATION, + opts->tails == 2 ? N_("Sig. (2-tailed)") : N_("Sig. (1-tailed)"), + PIVOT_RC_SIGNIFICANCE); - if ( opts->sig && c != r && sig < 0.05) - flags = TAB_EMPH; - - tab_double (t, c + heading_columns, row, flags, pearson, NULL); + if (opts->statistics & STATS_XPROD) + pivot_category_create_leaves (statistics->root, N_("Cross-products"), + N_("Covariance")); - if (opts->statistics & STATS_XPROD) - { - double cov = gsl_matrix_get (cv, r, col_index); - const double xprod_dev = cov * w; - cov *= w / (w - 1.0); + if (opts->missing_type != CORR_LISTWISE) + pivot_category_create_leaves (statistics->root, N_("N"), PIVOT_RC_COUNT); - tab_double (t, c + heading_columns, row + 2, 0, xprod_dev, NULL); - tab_double (t, c + heading_columns, row + 3, 0, cov, NULL); - } - } - } + /* Row variable dimension. */ + struct pivot_dimension *rows = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Variables")); + for (size_t r = 0; r < corr->n_vars1; r++) + pivot_category_create_leaf (rows->root, + pivot_value_new_variable (corr->vars[r])); - tab_submit (t); -} + struct pivot_footnote *sig_footnote = pivot_table_create_footnote ( + table, pivot_value_new_text (N_("Significant at .05 level"))); + for (int r = 0; r < corr->n_vars1; r++) + for (int c = 0; c < matrix_cols; c++) + { + const int col_index = (corr->n_vars_total > corr->n_vars1 + ? corr->n_vars1 + c + : c); + double pearson = gsl_matrix_get (cm, r, col_index); + double w = gsl_matrix_get (samples, r, col_index); + double sig = opts->tails * significance_of_correlation (pearson, w); + + double entries[5]; + int n = 0; + entries[n++] = pearson; + entries[n++] = col_index != r ? sig : SYSMIS; + if (opts->statistics & STATS_XPROD) + { + double cov = gsl_matrix_get (cv, r, col_index); + const double xprod_dev = cov * w; + cov *= w / (w - 1.0); + + entries[n++] = xprod_dev; + entries[n++] = cov; + } + if (opts->missing_type != CORR_LISTWISE) + entries[n++] = w; + + for (int i = 0; i < n; i++) + if (entries[i] != SYSMIS) + { + struct pivot_value *v = pivot_value_new_number (entries[i]); + if (!i && opts->sig && col_index != r && sig < 0.05) + pivot_value_add_footnote (v, sig_footnote); + pivot_table_put3 (table, c, i, r, v); + } + } -static gsl_matrix * -correlation_from_covariance (const gsl_matrix *cv, const gsl_matrix *v) -{ - size_t i, j; - gsl_matrix *corr = gsl_matrix_calloc (cv->size1, cv->size2); - - for (i = 0 ; i < cv->size1; ++i) - { - for (j = 0 ; j < cv->size2; ++j) - { - double rho = gsl_matrix_get (cv, i, j); - - rho /= sqrt (gsl_matrix_get (v, i, j)) - * - sqrt (gsl_matrix_get (v, j, i)); - - gsl_matrix_set (corr, i, j, rho); - } - } - - return corr; + pivot_table_submit (table); } - - static void run_corr (struct casereader *r, const struct corr_opts *opts, const struct corr *corr) { struct ccase *c; const gsl_matrix *var_matrix, *samples_matrix, *mean_matrix; - const gsl_matrix *cov_matrix; - gsl_matrix *corr_matrix; - struct covariance *cov = covariance_create (corr->n_vars_total, corr->vars, - opts->wv, opts->exclude); + gsl_matrix *cov_matrix = NULL; + gsl_matrix *corr_matrix = NULL; + struct covariance *cov = covariance_2pass_create (corr->n_vars_total, corr->vars, + NULL, + opts->wv, opts->exclude, + true); + + struct casereader *rc = casereader_clone (r); + for (; (c = casereader_read (r)); case_unref (c)) + { + covariance_accumulate_pass1 (cov, c); + } - for ( ; (c = casereader_read (r) ); case_unref (c)) + for (; (c = casereader_read (rc)); case_unref (c)) { - covariance_accumulate (cov, c); + covariance_accumulate_pass2 (cov, c); } + casereader_destroy (rc); cov_matrix = covariance_calculate (cov); + if (! cov_matrix) + { + msg (SE, _("The data for the chosen variables are all missing or empty.")); + goto error; + } samples_matrix = covariance_moments (cov, MOMENT_NONE); var_matrix = covariance_moments (cov, MOMENT_VARIANCE); @@ -340,16 +241,16 @@ run_corr (struct casereader *r, const struct corr_opts *opts, const struct corr corr_matrix = correlation_from_covariance (cov_matrix, var_matrix); - if ( opts->statistics & STATS_DESCRIPTIVES) - output_descriptives (corr, mean_matrix, var_matrix, samples_matrix); + if (opts->statistics & STATS_DESCRIPTIVES) + output_descriptives (corr, opts, mean_matrix, var_matrix, samples_matrix); - output_correlation (corr, opts, - corr_matrix, - samples_matrix, - cov_matrix); + output_correlation (corr, opts, corr_matrix, + samples_matrix, cov_matrix); + error: covariance_destroy (cov); gsl_matrix_free (corr_matrix); + gsl_matrix_free (cov_matrix); } int @@ -376,13 +277,13 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) opts.statistics = 0; /* Parse CORRELATIONS. */ - while (lex_token (lexer) != '.') + while (lex_token (lexer) != T_ENDCMD) { - lex_match (lexer, '/'); + lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "MISSING")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "PAIRWISE")) opts.missing_type = CORR_PAIRWISE; @@ -398,15 +299,15 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_match (lexer, ','); + lex_match (lexer, T_COMMA); } } else if (lex_match_id (lexer, "PRINT")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { - if ( lex_match_id (lexer, "TWOTAIL")) + if (lex_match_id (lexer, "TWOTAIL")) opts.tails = 2; else if (lex_match_id (lexer, "ONETAIL")) opts.tails = 1; @@ -420,15 +321,15 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) goto error; } - lex_match (lexer, ','); + lex_match (lexer, T_COMMA); } } else if (lex_match_id (lexer, "STATISTICS")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { - if ( lex_match_id (lexer, "DESCRIPTIVES")) + if (lex_match_id (lexer, "DESCRIPTIVES")) opts.statistics = STATS_DESCRIPTIVES; else if (lex_match_id (lexer, "XPROD")) opts.statistics = STATS_XPROD; @@ -437,26 +338,26 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) opts.statistics = STATS_ALL; lex_get (lexer); } - else + else { lex_error (lexer, NULL); goto error; } - lex_match (lexer, ','); + lex_match (lexer, T_COMMA); } } else { if (lex_match_id (lexer, "VARIABLES")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); } corr = xrealloc (corr, sizeof (*corr) * (n_corrs + 1)); corr[n_corrs].n_vars_total = corr[n_corrs].n_vars1 = 0; - - if ( ! parse_variables_const (lexer, dict, &corr[n_corrs].vars, + + if (! parse_variables_const (lexer, dict, &corr[n_corrs].vars, &corr[n_corrs].n_vars_total, PV_NUMERIC)) { @@ -467,9 +368,9 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) corr[n_corrs].n_vars1 = corr[n_corrs].n_vars_total; - if ( lex_match (lexer, T_WITH)) + if (lex_match (lexer, T_WITH)) { - if ( ! parse_variables_const (lexer, dict, + if (! parse_variables_const (lexer, dict, &corr[n_corrs].vars, &corr[n_corrs].n_vars_total, PV_NUMERIC | PV_APPEND)) { @@ -515,7 +416,7 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) /* FIXME: No need to iterate the data multiple times */ struct casereader *r = casereader_clone (group); - if ( opts.missing_type == CORR_LISTWISE) + if (opts.missing_type == CORR_LISTWISE) r = casereader_create_filter_missing (r, all_vars, n_all_vars, opts.exclude, NULL, NULL); @@ -533,10 +434,14 @@ cmd_correlation (struct lexer *lexer, struct dataset *ds) /* Done. */ + free (corr->vars); free (corr); + return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE; error: + if (corr) + free (corr->vars); free (corr); return CMD_FAILURE; }