From: John Darrington Date: Wed, 29 Sep 2010 20:02:28 +0000 (+0200) Subject: Replace glm with a new implementation. X-Git-Tag: v0.7.6~138 X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4399391aa1efe1cf6cbbcfde90ff52d7f88c9c40;p=pspp-builds.git Replace glm with a new implementation. The code is still not ready for use. It's awaiting a working type 3 sum of squares. --- diff --git a/src/language/stats/.gitignore b/src/language/stats/.gitignore index c52a3ab9..dddbdb32 100644 --- a/src/language/stats/.gitignore +++ b/src/language/stats/.gitignore @@ -1,7 +1,6 @@ crosstabs.c examine.c frequencies.c -glm.c npar.c rank.c regression.c diff --git a/src/language/stats/automake.mk b/src/language/stats/automake.mk index 6242d9c0..eb0a6eb3 100644 --- a/src/language/stats/automake.mk +++ b/src/language/stats/automake.mk @@ -6,7 +6,6 @@ src_language_stats_built_sources = \ src/language/stats/crosstabs.c \ src/language/stats/examine.c \ src/language/stats/frequencies.c \ - src/language/stats/glm.c \ src/language/stats/npar.c \ src/language/stats/rank.c \ src/language/stats/regression.c \ @@ -30,6 +29,7 @@ language_stats_sources = \ src/language/stats/flip.c \ src/language/stats/freq.c \ src/language/stats/freq.h \ + src/language/stats/glm.c \ src/language/stats/npar-summary.c \ src/language/stats/npar-summary.h \ src/language/stats/oneway.c \ diff --git a/src/language/stats/glm.c b/src/language/stats/glm.c new file mode 100644 index 00000000..1acf03ff --- /dev/null +++ b/src/language/stats/glm.c @@ -0,0 +1,372 @@ +/* PSPP - a program for statistical analysis. + Copyright (C) 2010 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . */ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include "gettext.h" +#define _(msgid) gettext (msgid) + +struct glm_spec +{ + size_t n_dep_vars; + const struct variable **dep_vars; + + size_t n_factor_vars; + const struct variable **factor_vars; + + enum mv_class exclude; + + /* The weight variable */ + const struct variable *wv; + + bool intercept; +}; + +struct glm_workspace +{ + double total_ssq; + struct moments *totals; +}; + +static void output_glm (const struct glm_spec *, const struct glm_workspace *ws); +static void run_glm (const struct glm_spec *cmd, struct casereader *input, const struct dataset *ds); + +int +cmd_glm (struct lexer *lexer, struct dataset *ds) +{ + const struct dictionary *dict = dataset_dict (ds); + struct glm_spec glm ; + glm.n_dep_vars = 0; + glm.n_factor_vars = 0; + glm.dep_vars = NULL; + glm.factor_vars = NULL; + glm.exclude = MV_ANY; + glm.intercept = true; + glm.wv = dict_get_weight (dict); + + + if (!parse_variables_const (lexer, dict, + &glm.dep_vars, &glm.n_dep_vars, + PV_NO_DUPLICATE | PV_NUMERIC)) + goto error; + + lex_force_match (lexer, T_BY); + + if (!parse_variables_const (lexer, dict, + &glm.factor_vars, &glm.n_factor_vars, + PV_NO_DUPLICATE | PV_NUMERIC)) + goto error; + + if ( glm.n_dep_vars > 1) + { + msg (ME, _("Multivariate analysis is not yet implemented")); + return CMD_FAILURE; + } + + struct const_var_set *factors = const_var_set_create_from_array (glm.factor_vars, glm.n_factor_vars); + + + while (lex_token (lexer) != '.') + { + lex_match (lexer, '/'); + + if (lex_match_id (lexer, "MISSING")) + { + lex_match (lexer, '='); + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + { + if (lex_match_id (lexer, "INCLUDE")) + { + glm.exclude = MV_SYSTEM; + } + else if (lex_match_id (lexer, "EXCLUDE")) + { + glm.exclude = MV_ANY; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else if (lex_match_id (lexer, "INTERCEPT")) + { + lex_match (lexer, '='); + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + { + if (lex_match_id (lexer, "INCLUDE")) + { + glm.intercept = true; + } + else if (lex_match_id (lexer, "EXCLUDE")) + { + glm.intercept = false; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else if (lex_match_id (lexer, "DESIGN")) + { + size_t n_des; + const struct variable **des; + lex_match (lexer, '='); + + parse_const_var_set_vars (lexer, factors, &des, &n_des, 0); + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + + + { + struct casegrouper *grouper; + struct casereader *group; + bool ok; + + grouper = casegrouper_create_splits (proc_open (ds), dict); + while (casegrouper_get_next_group (grouper, &group)) + run_glm (&glm, group, ds); + ok = casegrouper_destroy (grouper); + ok = proc_commit (ds) && ok; + } + + return CMD_SUCCESS; + + error: + return CMD_FAILURE; +} + +static void dump_matrix (const gsl_matrix *m); + +static void +run_glm (const struct glm_spec *cmd, struct casereader *input, const struct dataset *ds) +{ + int v; + struct taint *taint; + struct dictionary *dict = dataset_dict (ds); + struct casereader *reader; + struct ccase *c; + + struct glm_workspace ws; + + struct categoricals *cats = categoricals_create (cmd->factor_vars, cmd->n_factor_vars, + cmd->wv, cmd->exclude, + NULL, NULL, + NULL, NULL); + + struct covariance *cov = covariance_2pass_create (cmd->n_dep_vars, cmd->dep_vars, + cats, + cmd->wv, cmd->exclude); + + + c = casereader_peek (input, 0); + if (c == NULL) + { + casereader_destroy (input); + return; + } + output_split_file_values (ds, c); + case_unref (c); + + taint = taint_clone (casereader_get_taint (input)); + + ws.totals = moments_create (MOMENT_VARIANCE); + + bool warn_bad_weight = true; + for (reader = casereader_clone (input); + (c = casereader_read (reader)) != NULL; case_unref (c)) + { + double weight = dict_get_case_weight (dict, c, &warn_bad_weight); + + for ( v = 0; v < cmd->n_dep_vars; ++v) + moments_pass_one (ws.totals, case_data (c, cmd->dep_vars[v])->f, weight); + + covariance_accumulate_pass1 (cov, c); + } + casereader_destroy (reader); + + categoricals_done (cats); + + for (reader = casereader_clone (input); + (c = casereader_read (reader)) != NULL; case_unref (c)) + { + double weight = dict_get_case_weight (dict, c, &warn_bad_weight); + + for ( v = 0; v < cmd->n_dep_vars; ++v) + moments_pass_two (ws.totals, case_data (c, cmd->dep_vars[v])->f, weight); + + covariance_accumulate_pass2 (cov, c); + } + casereader_destroy (reader); + + { + gsl_matrix *cm = covariance_calculate_unnormalized (cov); + + dump_matrix (cm); + + ws.total_ssq = gsl_matrix_get (cm, 0, 0); + + reg_sweep (cm, 0); + + dump_matrix (cm); + } + + if (!taint_has_tainted_successor (taint)) + output_glm (cmd, &ws); + + taint_destroy (taint); +} + +static void +output_glm (const struct glm_spec *cmd, const struct glm_workspace *ws) +{ + const struct fmt_spec *wfmt = cmd->wv ? var_get_print_format (cmd->wv) : &F_8_0; + + int f; + int r; + const int heading_columns = 1; + const int heading_rows = 1; + struct tab_table *t ; + + const int nc = 6; + int nr = heading_rows + 4 + cmd->n_factor_vars; + if (cmd->intercept) + nr++; + + t = tab_create (nc, nr); + tab_title (t, _("Tests of Between-Subjects Effects")); + + tab_headers (t, heading_columns, 0, heading_rows, 0); + + tab_box (t, + TAL_2, TAL_2, + -1, TAL_1, + 0, 0, + nc - 1, nr - 1); + + tab_hline (t, TAL_2, 0, nc - 1, heading_rows); + tab_vline (t, TAL_2, heading_columns, 0, nr - 1); + + tab_text (t, 0, 0, TAB_CENTER | TAT_TITLE, _("Source")); + + /* TRANSLATORS: The parameter is a roman numeral */ + tab_text_format (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Type %s Sum of Squares"), "III"); + tab_text (t, 2, 0, TAB_CENTER | TAT_TITLE, _("df")); + tab_text (t, 3, 0, TAB_CENTER | TAT_TITLE, _("Mean Square")); + tab_text (t, 4, 0, TAB_CENTER | TAT_TITLE, _("F")); + tab_text (t, 5, 0, TAB_CENTER | TAT_TITLE, _("Sig.")); + + r = heading_rows; + tab_text (t, 0, r++, TAB_LEFT | TAT_TITLE, _("Corrected Model")); + + double intercept, n_total; + if (cmd->intercept) + { + double mean; + moments_calculate (ws->totals, &n_total, &mean, NULL, NULL, NULL); + intercept = pow2 (mean * n_total) / n_total; + + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Intercept")); + tab_double (t, 1, r, 0, intercept, NULL); + tab_double (t, 2, r, 0, 1.00, wfmt); + + tab_double (t, 3, r, 0, intercept / 1.0 , NULL); + r++; + } + + for (f = 0; f < cmd->n_factor_vars; ++f) + { + tab_text (t, 0, r++, TAB_LEFT | TAT_TITLE, + var_to_string (cmd->factor_vars[f])); + } + + tab_text (t, 0, r++, TAB_LEFT | TAT_TITLE, _("Error")); + + if (cmd->intercept) + { + double ssq = intercept + ws->total_ssq; + double mse = ssq / n_total; + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Total")); + tab_double (t, 1, r, 0, ssq, NULL); + tab_double (t, 2, r, 0, n_total, wfmt); + + r++; + } + + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Corrected Total")); + + tab_double (t, 1, r, 0, ws->total_ssq, NULL); + tab_double (t, 2, r, 0, n_total - 1.0, wfmt); + + tab_submit (t); +} + +static +void dump_matrix (const gsl_matrix *m) +{ + size_t i, j; + for (i = 0; i < m->size1; ++i) + { + for (j = 0; j < m->size2; ++j) + { + double x = gsl_matrix_get (m, i, j); + printf ("%.3f ", x); + } + printf ("\n"); + } + printf ("\n"); +} diff --git a/src/language/stats/glm.q b/src/language/stats/glm.q deleted file mode 100644 index 4de00e3b..00000000 --- a/src/language/stats/glm.q +++ /dev/null @@ -1,405 +0,0 @@ -/* PSPP - a program for statistical analysis. - Copyright (C) 2007, 2009 Free Software Foundation, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . */ - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "xalloc.h" -#include "gettext.h" - -/* (headers) */ - -/* (specification) - "GLM" (glm_): - *dependent=custom; - design=custom; - by=varlist; - with=varlist. -*/ -/* (declarations) */ -/* (functions) */ -static struct cmd_glm cmd; - - -/* - Moments for each of the variables used. - */ -struct moments_var -{ - struct moments1 *m; - double *weight; - double *mean; - double *variance; - const struct variable *v; -}; - - -/* - Dependent variable used. - */ -static const struct variable **v_dependent; - -/* - Number of dependent variables. - */ -static size_t n_dependent; - -size_t n_inter; /* Number of interactions. */ -size_t n_members; /* Number of memebr variables in an interaction. */ - -struct interaction_variable **interactions; - -int cmd_glm (struct lexer *lexer, struct dataset *ds); - -static bool run_glm (struct casereader *, - struct cmd_glm *, - const struct dataset *); -/* - If the DESIGN subcommand was not specified, specify all possible - two-way interactions. - */ -static void -check_interactions (struct dataset *ds, struct cmd_glm *cmd) -{ - size_t i; - size_t j; - size_t k = 0; - struct variable **interaction_vars; - - /* - User did not specify the design matrix, so we - specify it here. - */ - n_inter = (cmd->n_by + cmd->n_with) * (cmd->n_by + cmd->n_with - 1) / 2; - interactions = xnmalloc (n_inter, sizeof (*interactions)); - interaction_vars = xnmalloc (2, sizeof (*interaction_vars)); - for (i = 0; i < cmd->n_by; i++) - { - for (j = i + 1; j < cmd->n_by; j++) - { - interaction_vars[0] = cmd->v_by[i]; - interaction_vars[1] = cmd->v_by[j]; - interactions[k] = interaction_variable_create (interaction_vars, 2); - k++; - } - for (j = 0; j < cmd->n_with; j++) - { - interaction_vars[0] = cmd->v_by[i]; - interaction_vars[1] = cmd->v_with[j]; - interactions[k] = interaction_variable_create (interaction_vars, 2); - k++; - } - } - for (i = 0; i < cmd->n_with; i++) - { - for (j = i + 1; j < cmd->n_with; j++) - { - interaction_vars[0] = cmd->v_with[i]; - interaction_vars[1] = cmd->v_with[j]; - interactions[k] = interaction_variable_create (interaction_vars, 2); - k++; - } - } -} -int -cmd_glm (struct lexer *lexer, struct dataset *ds) -{ - struct casegrouper *grouper; - struct casereader *group; - - bool ok; - - if (!parse_glm (lexer, ds, &cmd, NULL)) - return CMD_FAILURE; - - if (!lex_match_id (lexer, "DESIGN")) - { - check_interactions (ds, &cmd); - } - /* Data pass. */ - grouper = casegrouper_create_splits (proc_open (ds), dataset_dict (ds)); - while (casegrouper_get_next_group (grouper, &group)) - { - run_glm (group, &cmd, ds); - } - ok = casegrouper_destroy (grouper); - ok = proc_commit (ds) && ok; - - free (v_dependent); - return ok ? CMD_SUCCESS : CMD_FAILURE; -} - -static int -parse_interactions (struct lexer *lexer, const struct variable **interaction_vars, int n_members, - int max_members, struct dataset *ds) -{ - if (lex_match (lexer, '*')) - { - if (n_members > max_members) - { - max_members *= 2; - xnrealloc (interaction_vars, max_members, sizeof (*interaction_vars)); - } - interaction_vars[n_members] = parse_variable (lexer, dataset_dict (ds)); - parse_interactions (lexer, interaction_vars, n_members++, max_members, ds); - } - return n_members; -} -/* Parser for the design subcommand. */ -static int -glm_custom_design (struct lexer *lexer, struct dataset *ds, - struct cmd_glm *cmd UNUSED, void *aux UNUSED) -{ - size_t n_allocated = 2; - size_t n_members; - struct variable **interaction_vars; - struct variable *this_var; - - interactions = xnmalloc (n_allocated, sizeof (*interactions)); - n_inter = 1; - while (lex_token (lexer) != T_STOP && lex_token (lexer) != '.') - { - this_var = parse_variable (lexer, dataset_dict (ds)); - if (lex_match (lexer, '(')) - { - lex_force_match (lexer, ')'); - } - else if (lex_match (lexer, '*')) - { - interaction_vars = xnmalloc (2 * n_inter, sizeof (*interaction_vars)); - n_members = parse_interactions (lexer, interaction_vars, 1, 2 * n_inter, ds); - if (n_allocated < n_inter) - { - n_allocated *= 2; - xnrealloc (interactions, n_allocated, sizeof (*interactions)); - } - interactions [n_inter - 1] = - interaction_variable_create (interaction_vars, n_members); - n_inter++; - free (interaction_vars); - } - } - return 1; -} -/* Parser for the dependent sub command */ -static int -glm_custom_dependent (struct lexer *lexer, struct dataset *ds, - struct cmd_glm *cmd UNUSED, void *aux UNUSED) -{ - const struct dictionary *dict = dataset_dict (ds); - size_t i; - - if ((lex_token (lexer) != T_ID - || dict_lookup_var (dict, lex_tokid (lexer)) == NULL) - && lex_token (lexer) != T_ALL) - return 2; - - if (!parse_variables_const (lexer, dict, &v_dependent, &n_dependent, PV_NONE)) - { - free (v_dependent); - return 0; - } - for (i = 0; i < n_dependent; i++) - { - assert (var_is_numeric (v_dependent[i])); - } - assert (n_dependent); - if (n_dependent > 1) - msg (SE, _("Multivariate GLM not yet supported")); - n_dependent = 1; /* Drop this line after adding support for multivariate GLM. */ - - return 1; -} - -static linreg * -fit_model (const struct covariance *cov, - const struct variable *dep_var, - const struct variable ** indep_vars, - size_t n_data, size_t n_indep) -{ - linreg *result = NULL; - - return result; -} - -static bool -run_glm (struct casereader *input, - struct cmd_glm *cmd, - const struct dataset *ds) -{ - casenumber row; - const struct variable **numerics = NULL; - const struct variable **categoricals = NULL; - int n_indep = 0; - linreg *model = NULL; - pspp_linreg_opts lopts; - struct ccase *c; - size_t i; - size_t n_data; /* Number of valid cases. */ - size_t n_categoricals = 0; - size_t n_numerics; - struct casereader *reader; - struct covariance *cov; - - c = casereader_peek (input, 0); - if (c == NULL) - { - casereader_destroy (input); - return true; - } - output_split_file_values (ds, c); - case_unref (c); - - if (!v_dependent) - { - dict_get_vars (dataset_dict (ds), &v_dependent, &n_dependent, - 1u << DC_SYSTEM); - } - - lopts.get_depvar_mean_std = 1; - - lopts.get_indep_mean_std = xnmalloc (n_dependent, sizeof (int)); - - - n_numerics = n_dependent; - for (i = 0; i < cmd->n_with; i++) - { - if (var_is_alpha (cmd->v_with[i])) - { - n_categoricals++; - } - else - { - n_numerics++; - } - } - for (i = 0; i < cmd->n_by; i++) - { - if (var_is_alpha (cmd->v_by[i])) - { - n_categoricals++; - } - else - { - n_numerics++; - } - } - numerics = xnmalloc (n_numerics, sizeof *numerics); - categoricals = xnmalloc (n_categoricals, sizeof (*categoricals)); - size_t j = 0; - size_t k = 0; - for (i = 0; i < cmd->n_by; i++) - { - if (var_is_alpha (cmd->v_by[i])) - { - categoricals[j] = cmd->v_by[i]; - j++; - } - else - { - numerics[k] = cmd->v_by[i]; - k++; - } - } - for (i = 0; i < cmd->n_with; i++) - { - if (var_is_alpha (cmd->v_with[i])) - { - categoricals[j] = cmd->v_with[i]; - j++; - } - else - { - numerics[k] = cmd->v_with[i]; - k++; - } - } - for (i = 0; i < n_dependent; i++) - { - numerics[k] = v_dependent[i]; - k++; - } - - struct categoricals *cats = - categoricals_create (categoricals, n_categoricals, - NULL, MV_NEVER, - NULL, NULL, NULL, NULL); - - cov = covariance_2pass_create (n_numerics, numerics, - cats, - NULL, MV_NEVER); - - reader = casereader_clone (input); - reader = casereader_create_filter_missing (reader, numerics, n_numerics, - MV_ANY, NULL, NULL); - reader = casereader_create_filter_missing (reader, categoricals, n_categoricals, - MV_ANY, NULL, NULL); - struct casereader *r = casereader_clone (reader); - - reader = casereader_create_counter (reader, &row, -1); - - for (; (c = casereader_read (reader)) != NULL; case_unref (c)) - { - covariance_accumulate_pass1 (cov, c); - } - for (; (c = casereader_read (r)) != NULL; case_unref (c)) - { - covariance_accumulate_pass2 (cov, c); - } - - covariance_destroy (cov); - casereader_destroy (reader); - casereader_destroy (r); - - free (numerics); - free (categoricals); - free (lopts.get_indep_mean_std); - casereader_destroy (input); - - return true; -} - -/* - Local Variables: - mode: c - End: -*/