X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fdescriptives.c;h=7b30a6013908c6692a5ba0b6977b3046a534f33c;hb=2cf38ce51a9f34961d68a75e0b312a591b5c9abf;hp=1f7fed891edc9de5b78889130aff6ca6a426501b;hpb=9f650fc3d2946c216e6cd3c7922a8a63d0f97117;p=pspp-builds.git diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c index 1f7fed89..7b30a601 100644 --- a/src/language/stats/descriptives.c +++ b/src/language/stats/descriptives.c @@ -1,22 +1,18 @@ -/* PSPP - computes sample statistics. - Copyright (C) 1997-9, 2000 Free Software Foundation, Inc. +/* PSPP - a program for statistical analysis. + Copyright (C) 1997-9, 2000, 2009 Free Software Foundation, Inc. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. */ - -/* FIXME: Many possible optimizations. */ + along with this program. If not, see . */ #include @@ -24,8 +20,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -34,16 +30,16 @@ #include #include #include -#include #include #include -#include #include #include #include #include #include +#include "xalloc.h" + #include "gettext.h" #define _(msgid) gettext (msgid) #define N_(msgid) msgid @@ -63,7 +59,7 @@ enum dsc_missing_type calculating a Z-score. */ struct dsc_z_score { - struct variable *src_var; /* Variable on which z-score is based. */ + const struct variable *src_var; /* Variable on which z-score is based. */ struct variable *z_var; /* New z-score variable. */ double mean; /* Distribution mean. */ double std_dev; /* Distribution standard deviation. */ @@ -74,7 +70,7 @@ struct dsc_trns { struct dsc_z_score *z_scores; /* Array of Z-scores. */ int z_score_cnt; /* Number of Z-scores. */ - struct variable **vars; /* Variables for listwise missing checks. */ + const struct variable **vars; /* Variables for listwise missing checks. */ size_t var_cnt; /* Number of variables. */ enum dsc_missing_type missing_type; /* Treatment of missing values. */ enum mv_class exclude; /* Classes of missing values to exclude. */ @@ -122,12 +118,12 @@ static const struct dsc_statistic_info dsc_info[DSC_N_STATS] = #define DEFAULT_STATS \ ((1ul << DSC_MEAN) | (1ul << DSC_STDDEV) | (1ul << DSC_MIN) \ | (1ul << DSC_MAX)) - + /* A variable specified on DESCRIPTIVES. */ struct dsc_var { - struct variable *v; /* Variable to calculate on. */ - char z_name[LONG_NAME_LEN + 1]; /* Name for z-score variable. */ + const struct variable *v; /* Variable to calculate on. */ + char z_name[VAR_NAME_LEN + 1]; /* Name for z-score variable. */ double valid, missing; /* Valid, missing counts. */ struct moments *moments; /* Moments. */ double min, max; /* Maximum and mimimum values. */ @@ -135,14 +131,14 @@ struct dsc_var }; /* Output format. */ -enum dsc_format +enum dsc_format { DSC_LINE, /* Abbreviated format. */ DSC_SERIAL /* Long format. */ }; /* A DESCRIPTIVES procedure. */ -struct dsc_proc +struct dsc_proc { /* Per-variable info. */ struct dsc_var *vars; /* Variables. */ @@ -171,18 +167,17 @@ static enum dsc_statistic match_statistic (struct lexer *); static void free_dsc_proc (struct dsc_proc *); /* Z-score functions. */ -static bool try_name (const struct dictionary *dict, +static bool try_name (const struct dictionary *dict, struct dsc_proc *dsc, const char *name); -static bool generate_z_varname (const struct dictionary *dict, +static bool generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z_name, - const char *name, size_t *z_cnt); + const char *name, int *z_cnt); static void dump_z_table (struct dsc_proc *); static void setup_z_trns (struct dsc_proc *, struct dataset *); /* Procedure execution functions. */ -static bool calc_descriptives (const struct ccase *first, - const struct casefile *, void *dsc_, - const struct dataset *); +static void calc_descriptives (struct dsc_proc *, struct casereader *, + struct dataset *); static void display (struct dsc_proc *dsc); /* Parser and outline. */ @@ -193,13 +188,16 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) { struct dictionary *dict = dataset_dict (ds); struct dsc_proc *dsc; - struct variable **vars = NULL; + const struct variable **vars = NULL; size_t var_cnt = 0; int save_z_scores = 0; - size_t z_cnt = 0; + int z_cnt = 0; size_t i; bool ok; + struct casegrouper *grouper; + struct casereader *group; + /* Create and initialize dsc. */ dsc = xmalloc (sizeof *dsc); dsc->vars = NULL; @@ -217,12 +215,12 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) dsc->show_stats = dsc->calc_stats = DEFAULT_STATS; /* Parse DESCRIPTIVES. */ - while (lex_token (lexer) != '.') + while (lex_token (lexer) != '.') { if (lex_match_id (lexer, "MISSING")) { lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') { if (lex_match_id (lexer, "VARIABLE")) dsc->missing_type = DSC_VARIABLE; @@ -240,10 +238,10 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "SAVE")) save_z_scores = 1; - else if (lex_match_id (lexer, "FORMAT")) + else if (lex_match_id (lexer, "FORMAT")) { lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') { if (lex_match_id (lexer, "LABELS")) dsc->show_var_labels = 1; @@ -265,13 +263,13 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) lex_match (lexer, ','); } } - else if (lex_match_id (lexer, "STATISTICS")) + else if (lex_match_id (lexer, "STATISTICS")) { lex_match (lexer, '='); dsc->show_stats = 0; - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') { - if (lex_match (lexer, T_ALL)) + if (lex_match (lexer, T_ALL)) dsc->show_stats |= (1ul << DSC_N_STATS) - 1; else if (lex_match_id (lexer, "DEFAULT")) dsc->show_stats |= DEFAULT_STATS; @@ -282,18 +280,18 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) if (dsc->show_stats == 0) dsc->show_stats = DEFAULT_STATS; } - else if (lex_match_id (lexer, "SORT")) + else if (lex_match_id (lexer, "SORT")) { lex_match (lexer, '='); if (lex_match_id (lexer, "NAME")) dsc->sort_by_stat = DSC_NAME; - else + else { dsc->sort_by_stat = match_statistic (lexer); if (dsc->sort_by_stat == DSC_NONE ) dsc->sort_by_stat = DSC_MEAN; } - if (lex_match (lexer, '(')) + if (lex_match (lexer, '(')) { if (lex_match_id (lexer, "A")) dsc->sort_ascending = 1; @@ -306,21 +304,21 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) } else if (var_cnt == 0) { - if (lex_look_ahead (lexer) == '=') + if (lex_look_ahead (lexer) == '=') { lex_match_id (lexer, "VARIABLES"); lex_match (lexer, '='); } - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + while (lex_token (lexer) != '.' && lex_token (lexer) != '/') { int i; - - if (!parse_variables (lexer, dataset_dict (ds), &vars, &var_cnt, + + if (!parse_variables_const (lexer, dict, &vars, &var_cnt, PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC)) goto error; - dsc->vars = xnrealloc (dsc->vars, var_cnt, sizeof *dsc->vars); + dsc->vars = xnrealloc ((void *)dsc->vars, var_cnt, sizeof *dsc->vars); for (i = dsc->var_cnt; i < var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; @@ -330,14 +328,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) } dsc->var_cnt = var_cnt; - if (lex_match (lexer, '(')) + if (lex_match (lexer, '(')) { - if (lex_token (lexer) != T_ID) + if (lex_token (lexer) != T_ID) { lex_error (lexer, NULL); goto error; } - if (try_name (dict, dsc, lex_tokid (lexer))) + if (try_name (dict, dsc, lex_tokid (lexer))) { strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer)); z_cnt++; @@ -351,10 +349,10 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) } } } - else + else { lex_error (lexer, NULL); - goto error; + goto error; } lex_match (lexer, '/'); @@ -368,19 +366,19 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) /* Construct z-score varnames, show translation table. */ if (z_cnt || save_z_scores) { - if (save_z_scores) + if (save_z_scores) { - size_t gen_cnt = 0; + int gen_cnt = 0; for (i = 0; i < dsc->var_cnt; i++) - if (dsc->vars[i].z_name[0] == 0) + if (dsc->vars[i].z_name[0] == 0) { if (!generate_z_varname (dict, dsc, dsc->vars[i].z_name, var_get_name (dsc->vars[i].v), &gen_cnt)) goto error; z_cnt++; - } + } } dump_z_table (dsc); } @@ -405,7 +403,7 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) /* Figure out maximum moment needed and allocate moments for the variables. */ dsc->max_moment = MOMENT_NONE; - for (i = 0; i < DSC_N_STATS; i++) + for (i = 0; i < DSC_N_STATS; i++) if (dsc->calc_stats & (1ul << i) && dsc_info[i].moment > dsc->max_moment) dsc->max_moment = dsc_info[i].moment; if (dsc->max_moment != MOMENT_NONE) @@ -413,7 +411,11 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) dsc->vars[i].moments = moments_create (dsc->max_moment); /* Data pass. */ - ok = multipass_procedure_with_splits (ds, calc_descriptives, dsc); + grouper = casegrouper_create_splits (proc_open (ds), dict); + while (casegrouper_get_next_group (grouper, &group)) + calc_descriptives (dsc, group, ds); + ok = casegrouper_destroy (grouper); + ok = proc_commit (ds) && ok; /* Z-scoring! */ if (ok && z_cnt) @@ -435,14 +437,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) specifiers). Emits an error if the current token ID does not name a statistic. */ static enum dsc_statistic -match_statistic (struct lexer *lexer) +match_statistic (struct lexer *lexer) { - if (lex_token (lexer) == T_ID) + if (lex_token (lexer) == T_ID) { enum dsc_statistic stat; for (stat = 0; stat < DSC_N_STATS; stat++) - if (lex_match_id (lexer, dsc_info[stat].identifier)) + if (lex_match_id (lexer, dsc_info[stat].identifier)) return stat; lex_get (lexer); @@ -460,7 +462,7 @@ free_dsc_proc (struct dsc_proc *dsc) if (dsc == NULL) return; - + for (i = 0; i < dsc->var_cnt; i++) moments_destroy (dsc->vars[i].moments); free (dsc->vars); @@ -472,7 +474,7 @@ free_dsc_proc (struct dsc_proc *dsc) /* Returns false if NAME is a duplicate of any existing variable name or of any previously-declared z-var name; otherwise returns true. */ static bool -try_name (const struct dictionary *dict, struct dsc_proc *dsc, +try_name (const struct dictionary *dict, struct dsc_proc *dsc, const char *name) { size_t i; @@ -491,9 +493,9 @@ try_name (const struct dictionary *dict, struct dsc_proc *dsc, copies the new name into Z_NAME. On failure, returns false. */ static bool generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z_name, - const char *var_name, size_t *z_cnt) + const char *var_name, int *z_cnt) { - char name[LONG_NAME_LEN + 1]; + char name[VAR_NAME_LEN + 1]; /* Try a name based on the original variable name. */ name[0] = 'Z'; @@ -524,7 +526,7 @@ generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z "STDZ01-STDZ09, ZZZZ01-ZZZZ09, ZQZQ01-ZQZQ09.")); return false; } - + if (try_name (dict, dsc, name)) { strcpy (z_name, name); @@ -541,15 +543,15 @@ dump_z_table (struct dsc_proc *dsc) { size_t cnt = 0; struct tab_table *t; - + { size_t i; - + for (i = 0; i < dsc->var_cnt; i++) if (dsc->vars[i].z_name[0] != '\0') cnt++; } - + t = tab_create (2, cnt + 1, 0); tab_title (t, _("Mapping of variables to corresponding Z-scores.")); tab_columns (t, SOM_COL_DOWN, 1); @@ -558,11 +560,11 @@ dump_z_table (struct dsc_proc *dsc) tab_hline (t, TAL_2, 0, 1, 1); tab_text (t, 0, 0, TAB_CENTER | TAT_TITLE, _("Source")); tab_text (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Target")); - tab_dim (t, tab_natural_dimensions); + tab_dim (t, tab_natural_dimensions, NULL); { size_t i, y; - + for (i = 0, y = 1; i < dsc->var_cnt; i++) if (dsc->vars[i].z_name[0] != '\0') { @@ -570,7 +572,7 @@ dump_z_table (struct dsc_proc *dsc) tab_text (t, 1, y++, TAB_LEFT, dsc->vars[i].z_name); } } - + tab_submit (t); } @@ -581,12 +583,12 @@ dump_z_table (struct dsc_proc *dsc) (either system or user-missing values that weren't included). */ static int -descriptives_trns_proc (void *trns_, struct ccase * c, +descriptives_trns_proc (void *trns_, struct ccase **c, casenumber case_idx UNUSED) { struct dsc_trns *t = trns_; struct dsc_z_score *z; - struct variable **vars; + const struct variable **vars; int all_sysmis = 0; if (t->missing_type == DSC_LISTWISE) @@ -594,7 +596,7 @@ descriptives_trns_proc (void *trns_, struct ccase * c, assert(t->vars); for (vars = t->vars; vars < t->vars + t->var_cnt; vars++) { - double score = case_num (c, *vars); + double score = case_num (*c, *vars); if (var_is_num_missing (*vars, score, t->exclude)) { all_sysmis = 1; @@ -602,11 +604,12 @@ descriptives_trns_proc (void *trns_, struct ccase * c, } } } - + + *c = case_unshare (*c); for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) { - double input = case_num (c, z->src_var); - double *output = &case_data_rw (c, z->z_var)->f; + double input = case_num (*c, z->src_var); + double *output = &case_data_rw (*c, z->z_var)->f; if (z->mean == SYSMIS || z->std_dev == SYSMIS || all_sysmis || var_is_num_missing (z->src_var, input, t->exclude)) @@ -688,22 +691,33 @@ static bool listwise_missing (struct dsc_proc *dsc, const struct ccase *c); /* Calculates and displays descriptive statistics for the cases in CF. */ -static bool -calc_descriptives (const struct ccase *first, - const struct casefile *cf, void *dsc_, - const struct dataset *ds) +static void +calc_descriptives (struct dsc_proc *dsc, struct casereader *group, + struct dataset *ds) { - struct dsc_proc *dsc = dsc_; - struct casereader *reader; - struct ccase c; + struct casereader *pass1, *pass2; + struct ccase *c; size_t i; - output_split_file_values (ds, first); + c = casereader_peek (group, 0); + if (c == NULL) + { + casereader_destroy (group); + return; + } + output_split_file_values (ds, c); + case_unref (c); + + group = casereader_create_filter_weight (group, dataset_dict (ds), + NULL, NULL); + + pass1 = group; + pass2 = dsc->max_moment <= MOMENT_MEAN ? NULL : casereader_clone (pass1); for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - + dv->valid = dv->missing = 0.0; if (dv->moments != NULL) moments_clear (dv->moments); @@ -714,36 +728,31 @@ calc_descriptives (const struct ccase *first, dsc->valid = 0.; /* First pass to handle most of the work. */ - for (reader = casefile_get_reader (cf, NULL); - casereader_read (reader, &c); - case_destroy (&c)) + for (; (c = casereader_read (pass1)) != NULL; case_unref (c)) { - double weight = dict_get_case_weight (dataset_dict (ds), &c, &dsc->bad_warn); - if (weight <= 0.0) - continue; - + double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + /* Check for missing values. */ - if (listwise_missing (dsc, &c)) + if (listwise_missing (dsc, c)) { dsc->missing_listwise += weight; if (dsc->missing_type == DSC_LISTWISE) - continue; + continue; } dsc->valid += weight; - for (i = 0; i < dsc->var_cnt; i++) + for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - double x = case_num (&c, dv->v); - - if (dsc->missing_type != DSC_LISTWISE - && var_is_num_missing (dv->v, x, dsc->exclude)) + double x = case_num (c, dv->v); + + if (var_is_num_missing (dv->v, x, dsc->exclude)) { dv->missing += weight; continue; } - if (dv->moments != NULL) + if (dv->moments != NULL) moments_pass_one (dv->moments, x, weight); if (x < dv->min) @@ -752,40 +761,39 @@ calc_descriptives (const struct ccase *first, dv->max = x; } } - casereader_destroy (reader); + if (!casereader_destroy (pass1)) + { + casereader_destroy (pass2); + return; + } /* Second pass for higher-order moments. */ - if (dsc->max_moment > MOMENT_MEAN) + if (dsc->max_moment > MOMENT_MEAN) { - for (reader = casefile_get_reader (cf, NULL); - casereader_read (reader, &c); - case_destroy (&c)) + for (; (c = casereader_read (pass2)) != NULL; case_unref (c)) { - double weight = dict_get_case_weight (dataset_dict (ds), &c, - &dsc->bad_warn); - if (weight <= 0.0) - continue; - + double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + /* Check for missing values. */ - if (dsc->missing_type == DSC_LISTWISE && listwise_missing (dsc, &c)) - continue; + if (dsc->missing_type == DSC_LISTWISE && listwise_missing (dsc, c)) + continue; - for (i = 0; i < dsc->var_cnt; i++) + for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - double x = case_num (&c, dv->v); - - if (dsc->missing_type != DSC_LISTWISE - && var_is_num_missing (dv->v, x, dsc->exclude)) + double x = case_num (c, dv->v); + + if (var_is_num_missing (dv->v, x, dsc->exclude)) continue; if (dv->moments != NULL) moments_pass_two (dv->moments, x, weight); } } - casereader_destroy (reader); + if (!casereader_destroy (pass2)) + return; } - + /* Calculate results. */ for (i = 0; i < dsc->var_cnt; i++) { @@ -808,7 +816,7 @@ calc_descriptives (const struct ccase *first, if (dsc->calc_stats & (1ul << DSC_STDDEV) && dv->stats[DSC_VARIANCE] != SYSMIS) dv->stats[DSC_STDDEV] = sqrt (dv->stats[DSC_VARIANCE]); - if (dsc->calc_stats & (1ul << DSC_SEKURT)) + if (dsc->calc_stats & (1ul << DSC_SEKURT)) if (dv->stats[DSC_KURTOSIS] != SYSMIS) dv->stats[DSC_SEKURT] = calc_sekurt (W); if (dsc->calc_stats & (1ul << DSC_SESKEW) @@ -824,14 +832,12 @@ calc_descriptives (const struct ccase *first, /* Output results. */ display (dsc); - - return true; } /* Returns true if any of the descriptives variables in DSC's variable list have missing values in case C, false otherwise. */ static bool -listwise_missing (struct dsc_proc *dsc, const struct ccase *c) +listwise_missing (struct dsc_proc *dsc, const struct ccase *c) { size_t i; @@ -873,7 +879,7 @@ display (struct dsc_proc *dsc) tab_box (t, -1, -1, -1, TAL_1, 1, 0, nc - 1, dsc->var_cnt); tab_hline (t, TAL_2, 0, nc - 1, 1); tab_vline (t, TAL_2, 1, 0, dsc->var_cnt); - tab_dim (t, tab_natural_dimensions); + tab_dim (t, tab_natural_dimensions, NULL); nc = 0; tab_text (t, nc++, 0, TAB_LEFT | TAT_TITLE, _("Variable")); @@ -899,12 +905,13 @@ display (struct dsc_proc *dsc) nc = 0; tab_text (t, nc++, i + 1, TAB_LEFT, var_get_name (dv->v)); - tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->valid); + tab_text_format (t, nc++, i + 1, 0, "%g", dv->valid); if (dsc->format == DSC_SERIAL) - tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->missing); + tab_text_format (t, nc++, i + 1, 0, "%g", dv->missing); + for (j = 0; j < DSC_N_STATS; j++) if (dsc->show_stats & (1ul << j)) - tab_float (t, nc++, i + 1, TAB_NONE, dv->stats[j], 10, 3); + tab_double (t, nc++, i + 1, TAB_NONE, dv->stats[j], NULL); } tab_title (t, _("Valid cases = %g; cases with missing value(s) = %g."), @@ -926,7 +933,7 @@ descriptives_compare_dsc_vars (const void *a_, const void *b_, const void *dsc_) if (dsc->sort_by_stat == DSC_NAME) result = strcasecmp (var_get_name (a->v), var_get_name (b->v)); - else + else { double as = a->stats[dsc->sort_by_stat]; double bs = b->stats[dsc->sort_by_stat];