X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fglm.c;h=74e918b886b21f26d05b1fadf2d245faa60096bb;hb=edd5c738dfef01c90d02e06a33b93fc9d38320b8;hp=28f61bc68bc7dd21e6f931aea8bf631d069eb1d7;hpb=0cab34a28a09856c4aff9ce432c2c53350d1a501;p=pspp diff --git a/src/language/stats/glm.c b/src/language/stats/glm.c index 28f61bc68b..74e918b886 100644 --- a/src/language/stats/glm.c +++ b/src/language/stats/glm.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2010, 2011 Free Software Foundation, Inc. + Copyright (C) 2010, 2011, 2012 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -81,7 +81,7 @@ struct glm_workspace struct categoricals *cats; - /* + /* Sums of squares due to different variables. Element 0 is the SSE for the entire model. For i > 0, element i is the SS due to variable i. @@ -155,7 +155,8 @@ cmd_glm (struct lexer *lexer, struct dataset *ds) PV_NO_DUPLICATE | PV_NUMERIC)) goto error; - lex_force_match (lexer, T_BY); + if (! lex_force_match (lexer, T_BY)) + goto error; if (!parse_variables_const (lexer, glm.dict, &glm.factor_vars, &glm.n_factor_vars, @@ -229,7 +230,7 @@ cmd_glm (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - + glm.alpha = lex_number (lexer); lex_get (lexer); if ( ! lex_force_match (lexer, T_RPAREN)) @@ -267,9 +268,9 @@ cmd_glm (struct lexer *lexer, struct dataset *ds) } glm.ss_type = lex_integer (lexer); - if (1 != glm.ss_type && 2 != glm.ss_type ) + if (1 > glm.ss_type || 3 < glm.ss_type ) { - msg (ME, _("Only types 1 & 2 sum of squares are currently implemented")); + msg (ME, _("Only types 1, 2 & 3 sums of squares are currently implemented")); goto error; } @@ -326,6 +327,7 @@ cmd_glm (struct lexer *lexer, struct dataset *ds) free (glm.factor_vars); for (i = 0 ; i < glm.n_interactions; ++i) interaction_destroy (glm.interactions[i]); + free (glm.interactions); free (glm.dep_vars); @@ -345,9 +347,6 @@ error: return CMD_FAILURE; } -static void get_ssq (struct covariance *, gsl_vector *, - const struct glm_spec *); - static inline bool not_dropped (size_t j, const bool *ff) { @@ -361,11 +360,11 @@ fill_submatrix (const gsl_matrix * cov, gsl_matrix * submatrix, bool *dropped_f) size_t j; size_t n = 0; size_t m = 0; - + for (i = 0; i < cov->size1; i++) { if (not_dropped (i, dropped_f)) - { + { m = 0; for (j = 0; j < cov->size2; j++) { @@ -374,17 +373,91 @@ fill_submatrix (const gsl_matrix * cov, gsl_matrix * submatrix, bool *dropped_f) gsl_matrix_set (submatrix, n, m, gsl_matrix_get (cov, i, j)); m++; - } + } } n++; } } } - + + +/* + Type 1 sums of squares. + Populate SSQ with the Type 1 sums of squares according to COV + */ +static void +ssq_type1 (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) +{ + const gsl_matrix *cm = covariance_calculate_unnormalized (cov); + size_t i; + size_t k; + bool *model_dropped = xcalloc (covariance_dim (cov), sizeof (*model_dropped)); + bool *submodel_dropped = xcalloc (covariance_dim (cov), sizeof (*submodel_dropped)); + const struct categoricals *cats = covariance_get_categoricals (cov); + + size_t n_dropped_model = 0; + size_t n_dropped_submodel = 0; + + for (i = cmd->n_dep_vars; i < covariance_dim (cov); i++) + { + n_dropped_model++; + n_dropped_submodel++; + model_dropped[i] = true; + submodel_dropped[i] = true; + } + + for (k = 0; k < cmd->n_interactions; k++) + { + gsl_matrix *model_cov = NULL; + gsl_matrix *submodel_cov = NULL; + + n_dropped_submodel = n_dropped_model; + for (i = cmd->n_dep_vars; i < covariance_dim (cov); i++) + { + submodel_dropped[i] = model_dropped[i]; + } + + for (i = cmd->n_dep_vars; i < covariance_dim (cov); i++) + { + const struct interaction * x = + categoricals_get_interaction_by_subscript (cats, i - cmd->n_dep_vars); + + if ( x == cmd->interactions [k]) + { + model_dropped[i] = false; + n_dropped_model--; + } + } + + model_cov = gsl_matrix_alloc (cm->size1 - n_dropped_model, cm->size2 - n_dropped_model); + submodel_cov = gsl_matrix_alloc (cm->size1 - n_dropped_submodel, cm->size2 - n_dropped_submodel); + + fill_submatrix (cm, model_cov, model_dropped); + fill_submatrix (cm, submodel_cov, submodel_dropped); + + reg_sweep (model_cov, 0); + reg_sweep (submodel_cov, 0); + + gsl_vector_set (ssq, k + 1, + gsl_matrix_get (submodel_cov, 0, 0) - gsl_matrix_get (model_cov, 0, 0) + ); + + gsl_matrix_free (model_cov); + gsl_matrix_free (submodel_cov); + } + + free (model_dropped); + free (submodel_dropped); +} + +/* + Type 2 sums of squares. + Populate SSQ with the Type 2 sums of squares according to COV + */ static void -get_ssq (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) +ssq_type2 (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) { - gsl_matrix *cm = covariance_calculate_unnormalized (cov); + const gsl_matrix *cm = covariance_calculate_unnormalized (cov); size_t i; size_t k; bool *model_dropped = xcalloc (covariance_dim (cov), sizeof (*model_dropped)); @@ -399,7 +472,7 @@ get_ssq (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) size_t n_dropped_submodel = 0; for (i = cmd->n_dep_vars; i < covariance_dim (cov); i++) { - const struct interaction * x = + const struct interaction * x = categoricals_get_interaction_by_subscript (cats, i - cmd->n_dep_vars); model_dropped[i] = false; @@ -438,9 +511,66 @@ get_ssq (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) free (model_dropped); free (submodel_dropped); - gsl_matrix_free (cm); } +/* + Type 3 sums of squares. + Populate SSQ with the Type 2 sums of squares according to COV + */ +static void +ssq_type3 (struct covariance *cov, gsl_vector *ssq, const struct glm_spec *cmd) +{ + const gsl_matrix *cm = covariance_calculate_unnormalized (cov); + size_t i; + size_t k; + bool *model_dropped = xcalloc (covariance_dim (cov), sizeof (*model_dropped)); + bool *submodel_dropped = xcalloc (covariance_dim (cov), sizeof (*submodel_dropped)); + const struct categoricals *cats = covariance_get_categoricals (cov); + + double ss0; + gsl_matrix *submodel_cov = gsl_matrix_alloc (cm->size1, cm->size2); + fill_submatrix (cm, submodel_cov, submodel_dropped); + reg_sweep (submodel_cov, 0); + ss0 = gsl_matrix_get (submodel_cov, 0, 0); + gsl_matrix_free (submodel_cov); + free (submodel_dropped); + + for (k = 0; k < cmd->n_interactions; k++) + { + gsl_matrix *model_cov = NULL; + size_t n_dropped_model = 0; + + for (i = cmd->n_dep_vars; i < covariance_dim (cov); i++) + { + const struct interaction * x = + categoricals_get_interaction_by_subscript (cats, i - cmd->n_dep_vars); + + model_dropped[i] = false; + + if ( cmd->interactions [k] == x) + { + assert (n_dropped_model < covariance_dim (cov)); + n_dropped_model++; + model_dropped[i] = true; + } + } + + model_cov = gsl_matrix_alloc (cm->size1 - n_dropped_model, cm->size2 - n_dropped_model); + + fill_submatrix (cm, model_cov, model_dropped); + + reg_sweep (model_cov, 0); + + gsl_vector_set (ssq, k + 1, + gsl_matrix_get (model_cov, 0, 0) - ss0); + + gsl_matrix_free (model_cov); + } + free (model_dropped); +} + + + //static void dump_matrix (const gsl_matrix *m); static void @@ -457,12 +587,21 @@ run_glm (struct glm_spec *cmd, struct casereader *input, struct glm_workspace ws; struct covariance *cov; + input = casereader_create_filter_missing (input, + cmd->dep_vars, cmd->n_dep_vars, + cmd->exclude, + NULL, NULL); + + input = casereader_create_filter_missing (input, + cmd->factor_vars, cmd->n_factor_vars, + cmd->exclude, + NULL, NULL); + ws.cats = categoricals_create (cmd->interactions, cmd->n_interactions, - cmd->wv, cmd->exclude, - NULL, NULL, NULL, NULL); + cmd->wv, cmd->exclude, MV_ANY); cov = covariance_2pass_create (cmd->n_dep_vars, cmd->dep_vars, - ws.cats, cmd->wv, cmd->exclude); + ws.cats, cmd->wv, cmd->exclude, true); c = casereader_peek (input, 0); @@ -525,7 +664,9 @@ run_glm (struct glm_spec *cmd, struct casereader *input, } { - gsl_matrix *cm = covariance_calculate_unnormalized (cov); + const gsl_matrix *ucm = covariance_calculate_unnormalized (cov); + gsl_matrix *cm = gsl_matrix_alloc (ucm->size1, ucm->size2); + gsl_matrix_memcpy (cm, ucm); // dump_matrix (cm); @@ -541,17 +682,19 @@ run_glm (struct glm_spec *cmd, struct casereader *input, switch (cmd->ss_type) { case 1: + ssq_type1 (cov, ws.ssq, cmd); break; case 2: + ssq_type2 (cov, ws.ssq, cmd); + break; case 3: - get_ssq (cov, ws.ssq, cmd); + ssq_type3 (cov, ws.ssq, cmd); break; default: NOT_REACHED (); break; } // dump_matrix (cm); - gsl_matrix_free (cm); } @@ -566,7 +709,7 @@ run_glm (struct glm_spec *cmd, struct casereader *input, taint_destroy (taint); } -static const char *roman[] = +static const char *roman[] = { "", /* The Romans had no concept of zero */ "I", @@ -581,8 +724,10 @@ output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) const struct fmt_spec *wfmt = cmd->wv ? var_get_print_format (cmd->wv) : &F_8_0; + double intercept_ssq; + double ssq_effects; double n_total, mean; - double df_corr = 0.0; + double df_corr = 1.0; double mse = 0; int f; @@ -592,12 +737,12 @@ output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) struct tab_table *t; const int nc = 6; - int nr = heading_rows + 4 + cmd->n_interactions; + int nr = heading_rows + 3 + cmd->n_interactions; if (cmd->intercept) - nr++; + nr += 2; - msg (MW, "GLM is experimental. Do not rely on these results."); t = tab_create (nc, nr); + tab_set_format (t, RC_WEIGHT, wfmt); tab_title (t, _("Tests of Between-Subjects Effects")); tab_headers (t, heading_columns, 0, heading_rows, 0); @@ -611,7 +756,7 @@ output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) /* TRANSLATORS: The parameter is a roman numeral */ tab_text_format (t, 1, 0, TAB_CENTER | TAT_TITLE, - _("Type %s Sum of Squares"), + _("Type %s Sum of Squares"), roman[cmd->ss_type]); tab_text (t, 2, 0, TAB_CENTER | TAT_TITLE, _("df")); tab_text (t, 3, 0, TAB_CENTER | TAT_TITLE, _("Mean Square")); @@ -620,65 +765,90 @@ output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) moments_calculate (ws->totals, &n_total, &mean, NULL, NULL, NULL); - if (cmd->intercept) - df_corr += 1.0; - df_corr += categoricals_df_total (ws->cats); - mse = gsl_vector_get (ws->ssq, 0) / (n_total - df_corr); - r = heading_rows; - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Corrected Model")); + if (cmd->intercept) + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Corrected Model")); + else + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Model")); r++; + mse = gsl_vector_get (ws->ssq, 0) / (n_total - df_corr); + + intercept_ssq = pow2 (mean * n_total) / n_total; + + ssq_effects = 0.0; if (cmd->intercept) { - const double intercept = pow2 (mean * n_total) / n_total; const double df = 1.0; - const double F = intercept / df / mse; + const double F = intercept_ssq / df / mse; tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Intercept")); - tab_double (t, 1, r, 0, intercept, NULL); - tab_double (t, 2, r, 0, 1.00, wfmt); - tab_double (t, 3, r, 0, intercept / df, NULL); - tab_double (t, 4, r, 0, F, NULL); - tab_double (t, 5, r, 0, gsl_cdf_fdist_Q (F, df, n_total - df_corr), - NULL); + /* The intercept for unbalanced models is of limited use and + nobody knows how to calculate it properly */ + if (categoricals_isbalanced (ws->cats)) + { + tab_double (t, 1, r, 0, intercept_ssq, NULL, RC_OTHER); + tab_double (t, 2, r, 0, 1.00, NULL, RC_WEIGHT); + tab_double (t, 3, r, 0, intercept_ssq / df, NULL, RC_OTHER); + tab_double (t, 4, r, 0, F, NULL, RC_OTHER); + tab_double (t, 5, r, 0, gsl_cdf_fdist_Q (F, df, n_total - df_corr), + NULL, RC_PVALUE); + } r++; } for (f = 0; f < cmd->n_interactions; ++f) { struct string str = DS_EMPTY_INITIALIZER; - const double df = categoricals_df (ws->cats, f); - const double ssq = gsl_vector_get (ws->ssq, f + 1); - const double F = ssq / df / mse; + double df = categoricals_df (ws->cats, f); + + double ssq = gsl_vector_get (ws->ssq, f + 1); + double F; + + ssq_effects += ssq; + + if (! cmd->intercept) + { + df++; + ssq += intercept_ssq; + } + + F = ssq / df / mse; interaction_to_string (cmd->interactions[f], &str); tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, ds_cstr (&str)); ds_destroy (&str); - tab_double (t, 1, r, 0, ssq, NULL); - tab_double (t, 2, r, 0, df, wfmt); - tab_double (t, 3, r, 0, ssq / df, NULL); - tab_double (t, 4, r, 0, F, NULL); + tab_double (t, 1, r, 0, ssq, NULL, RC_OTHER); + tab_double (t, 2, r, 0, df, NULL, RC_WEIGHT); + tab_double (t, 3, r, 0, ssq / df, NULL, RC_OTHER); + tab_double (t, 4, r, 0, F, NULL, RC_OTHER); tab_double (t, 5, r, 0, gsl_cdf_fdist_Q (F, df, n_total - df_corr), - NULL); + NULL, RC_PVALUE); r++; } { - /* Corrected Model */ - const double df = df_corr - 1.0; - const double ssq = ws->total_ssq - gsl_vector_get (ws->ssq, 0); - const double F = ssq / df / mse; - tab_double (t, 1, heading_rows, 0, ssq, NULL); - tab_double (t, 2, heading_rows, 0, df, wfmt); - tab_double (t, 3, heading_rows, 0, ssq / df, NULL); - tab_double (t, 4, heading_rows, 0, F, NULL); + /* Model / Corrected Model */ + double df = df_corr; + double ssq = ws->total_ssq - gsl_vector_get (ws->ssq, 0); + double F; + + if ( cmd->intercept ) + df --; + else + ssq += intercept_ssq; + + F = ssq / df / mse; + tab_double (t, 1, heading_rows, 0, ssq, NULL, RC_OTHER); + tab_double (t, 2, heading_rows, 0, df, NULL, RC_WEIGHT); + tab_double (t, 3, heading_rows, 0, ssq / df, NULL, RC_OTHER); + tab_double (t, 4, heading_rows, 0, F, NULL, RC_OTHER); tab_double (t, 5, heading_rows, 0, - gsl_cdf_fdist_Q (F, df, n_total - df_corr), NULL); + gsl_cdf_fdist_Q (F, df, n_total - df_corr), NULL, RC_PVALUE); } { @@ -686,29 +856,26 @@ output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) const double ssq = gsl_vector_get (ws->ssq, 0); const double mse = ssq / df; tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Error")); - tab_double (t, 1, r, 0, ssq, NULL); - tab_double (t, 2, r, 0, df, wfmt); - tab_double (t, 3, r++, 0, mse, NULL); + tab_double (t, 1, r, 0, ssq, NULL, RC_OTHER); + tab_double (t, 2, r, 0, df, NULL, RC_WEIGHT); + tab_double (t, 3, r++, 0, mse, NULL, RC_OTHER); } - if (cmd->intercept) - { - const double intercept = pow2 (mean * n_total) / n_total; - const double ssq = intercept + ws->total_ssq; + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Total")); + tab_double (t, 1, r, 0, ws->total_ssq + intercept_ssq, NULL, RC_OTHER); + tab_double (t, 2, r, 0, n_total, NULL, RC_WEIGHT); - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Total")); - tab_double (t, 1, r, 0, ssq, NULL); - tab_double (t, 2, r, 0, n_total, wfmt); + r++; + } - r++; + if (cmd->intercept) + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Corrected Total")); + tab_double (t, 1, r, 0, ws->total_ssq, NULL, RC_OTHER); + tab_double (t, 2, r, 0, n_total - 1.0, NULL, RC_WEIGHT); } - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Corrected Total")); - - - tab_double (t, 1, r, 0, ws->total_ssq, NULL); - tab_double (t, 2, r, 0, n_total - 1.0, wfmt); - tab_submit (t); } @@ -732,71 +899,11 @@ dump_matrix (const gsl_matrix * m) - -/* Match a variable. - If the match succeeds, the variable will be placed in VAR. - Returns true if successful */ -static bool -lex_match_variable (struct lexer *lexer, const struct glm_spec *glm, const struct variable **var) -{ - if (lex_token (lexer) != T_ID) - return false; - - *var = parse_variable_const (lexer, glm->dict); - - if ( *var == NULL) - return false; - return true; -} - -/* An interaction is a variable followed by {*, BY} followed by an interaction */ -static bool -parse_design_interaction (struct lexer *lexer, struct glm_spec *glm, struct interaction **iact) -{ - const struct variable *v = NULL; - assert (iact); - - switch (lex_next_token (lexer, 1)) - { - case T_ENDCMD: - case T_SLASH: - case T_COMMA: - case T_ID: - case T_BY: - case T_ASTERISK: - break; - default: - return false; - break; - } - - if (! lex_match_variable (lexer, glm, &v)) - { - interaction_destroy (*iact); - *iact = NULL; - return false; - } - - assert (v); - - if ( *iact == NULL) - *iact = interaction_create (v); - else - interaction_add_variable (*iact, v); - - if ( lex_match (lexer, T_ASTERISK) || lex_match (lexer, T_BY)) - { - return parse_design_interaction (lexer, glm, iact); - } - - return true; -} - static bool parse_nested_variable (struct lexer *lexer, struct glm_spec *glm) { const struct variable *v = NULL; - if ( ! lex_match_variable (lexer, glm, &v)) + if ( ! lex_match_variable (lexer, glm->dict, &v)) return false; if (lex_match (lexer, T_LPAREN)) @@ -808,7 +915,7 @@ parse_nested_variable (struct lexer *lexer, struct glm_spec *glm) return false; } - lex_error (lexer, "Nested variables are not yet implemented"); return false; + lex_error (lexer, "Nested variables are not yet implemented"); return false; return true; } @@ -817,7 +924,7 @@ static bool parse_design_term (struct lexer *lexer, struct glm_spec *glm) { struct interaction *iact = NULL; - if (parse_design_interaction (lexer, glm, &iact)) + if (parse_design_interaction (lexer, glm->dict, &iact)) { /* Interaction parsing successful. Add to list of interactions */ glm->interactions = xrealloc (glm->interactions, sizeof *glm->interactions * ++glm->n_interactions);