X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fdescriptives.c;h=11a9f7f31e50fad3bf8fe540a046c9af57eaadd4;hb=047d4a8e14cdbb50258bf8c5634db24a88767f24;hp=de0e417c81fdb59b397159e9bf549e8ea2455747;hpb=f43378497b8400e9c22a3485c534693dc1bc9554;p=pspp diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c index de0e417c81..11a9f7f31e 100644 --- a/src/language/stats/descriptives.c +++ b/src/language/stats/descriptives.c @@ -1,49 +1,46 @@ -/* PSPP - computes sample statistics. - Copyright (C) 1997-9, 2000 Free Software Foundation, Inc. - Written by Ben Pfaff . +/* PSPP - a program for statistical analysis. + Copyright (C) 1997-2000, 2009-2014 Free Software Foundation, Inc. - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301, USA. */ - -/* FIXME: Many possible optimizations. */ + along with this program. If not, see . */ #include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "data/casegrouper.h" +#include "data/casereader.h" +#include "data/casewriter.h" +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/transformations.h" +#include "data/variable.h" +#include "language/command.h" +#include "language/dictionary/split-file.h" +#include "language/lexer/lexer.h" +#include "language/lexer/variable-parser.h" +#include "libpspp/array.h" +#include "libpspp/assertion.h" +#include "libpspp/compiler.h" +#include "libpspp/i18n.h" +#include "libpspp/message.h" +#include "math/moments.h" +#include "output/tab.h" + +#include "gl/xalloc.h" #include "gettext.h" #define _(msgid) gettext (msgid) @@ -64,11 +61,10 @@ enum dsc_missing_type calculating a Z-score. */ struct dsc_z_score { - int src_idx; /* Source index into case data. */ - int dst_idx; /* Destination index into case data. */ + const struct variable *src_var; /* Variable on which z-score is based. */ + struct variable *z_var; /* New z-score variable. */ double mean; /* Distribution mean. */ double std_dev; /* Distribution standard deviation. */ - struct variable *v; /* Variable on which z-score is based. */ }; /* DESCRIPTIVES transformation (for calculating Z-scores). */ @@ -76,10 +72,14 @@ struct dsc_trns { struct dsc_z_score *z_scores; /* Array of Z-scores. */ int z_score_cnt; /* Number of Z-scores. */ - struct variable **vars; /* Variables for listwise missing checks. */ + const struct variable **vars; /* Variables for listwise missing checks. */ size_t var_cnt; /* Number of variables. */ enum dsc_missing_type missing_type; /* Treatment of missing values. */ - int include_user_missing; /* Nonzero to include user-missing values. */ + enum mv_class exclude; /* Classes of missing values to exclude. */ + struct variable *filter; /* Dictionary FILTER BY variable. */ + struct casereader *z_reader; /* Reader for count, mean, stddev. */ + casenumber count; /* Number left in this SPLIT FILE group.*/ + bool ok; }; /* Statistics. Used as bit indexes, so must be 32 or fewer. */ @@ -106,13 +106,13 @@ struct dsc_statistic_info static const struct dsc_statistic_info dsc_info[DSC_N_STATS] = { {"MEAN", N_("Mean"), MOMENT_MEAN}, - {"SEMEAN", N_("S E Mean"), MOMENT_VARIANCE}, + {"SEMEAN", N_("S.E. Mean"), MOMENT_VARIANCE}, {"STDDEV", N_("Std Dev"), MOMENT_VARIANCE}, {"VARIANCE", N_("Variance"), MOMENT_VARIANCE}, {"KURTOSIS", N_("Kurtosis"), MOMENT_KURTOSIS}, - {"SEKURTOSIS", N_("S E Kurt"), MOMENT_NONE}, + {"SEKURTOSIS", N_("S.E. Kurt"), MOMENT_NONE}, {"SKEWNESS", N_("Skewness"), MOMENT_SKEWNESS}, - {"SESKEWNESS", N_("S E Skew"), MOMENT_NONE}, + {"SESKEWNESS", N_("S.E. Skew"), MOMENT_NONE}, {"RANGE", N_("Range"), MOMENT_NONE}, {"MINIMUM", N_("Minimum"), MOMENT_NONE}, {"MAXIMUM", N_("Maximum"), MOMENT_NONE}, @@ -124,12 +124,12 @@ static const struct dsc_statistic_info dsc_info[DSC_N_STATS] = #define DEFAULT_STATS \ ((1ul << DSC_MEAN) | (1ul << DSC_STDDEV) | (1ul << DSC_MIN) \ | (1ul << DSC_MAX)) - + /* A variable specified on DESCRIPTIVES. */ struct dsc_var { - struct variable *v; /* Variable to calculate on. */ - char z_name[LONG_NAME_LEN + 1]; /* Name for z-score variable. */ + const struct variable *v; /* Variable to calculate on. */ + char *z_name; /* Name for z-score variable. */ double valid, missing; /* Valid, missing counts. */ struct moments *moments; /* Moments. */ double min, max; /* Maximum and mimimum values. */ @@ -137,14 +137,14 @@ struct dsc_var }; /* Output format. */ -enum dsc_format +enum dsc_format { DSC_LINE, /* Abbreviated format. */ DSC_SERIAL /* Long format. */ }; /* A DESCRIPTIVES procedure. */ -struct dsc_proc +struct dsc_proc { /* Per-variable info. */ struct dsc_var *vars; /* Variables. */ @@ -152,7 +152,7 @@ struct dsc_proc /* User options. */ enum dsc_missing_type missing_type; /* Treatment of missing values. */ - int include_user_missing; /* Nonzero to include user-missing values. */ + enum mv_class exclude; /* Classes of missing values to exclude. */ int show_var_labels; /* Nonzero to show variable labels. */ int show_index; /* Nonzero to show variable index. */ enum dsc_format format; /* Output format. */ @@ -166,48 +166,53 @@ struct dsc_proc unsigned long show_stats; /* Statistics to display. */ unsigned long calc_stats; /* Statistics to calculate. */ enum moment max_moment; /* Highest moment needed for stats. */ + + /* Z scores. */ + struct casewriter *z_writer; /* Mean and stddev per SPLIT FILE group. */ }; /* Parsing. */ -static enum dsc_statistic match_statistic (void); +static enum dsc_statistic match_statistic (struct lexer *); static void free_dsc_proc (struct dsc_proc *); /* Z-score functions. */ -static bool try_name (const struct dictionary *dict, - struct dsc_proc *dsc, char *name); -static bool generate_z_varname (const struct dictionary *dict, - struct dsc_proc *dsc, char *z_name, - const char *name, size_t *z_cnt); +static bool try_name (const struct dictionary *dict, + struct dsc_proc *dsc, const char *name); +static char *generate_z_varname (const struct dictionary *dict, + struct dsc_proc *dsc, + const char *name, int *z_cnt); static void dump_z_table (struct dsc_proc *); static void setup_z_trns (struct dsc_proc *, struct dataset *); /* Procedure execution functions. */ -static bool calc_descriptives (const struct ccase *first, - const struct casefile *, void *dsc_, - const struct dataset *); +static void calc_descriptives (struct dsc_proc *, struct casereader *, + struct dataset *); static void display (struct dsc_proc *dsc); /* Parser and outline. */ /* Handles DESCRIPTIVES. */ int -cmd_descriptives (struct dataset *ds) +cmd_descriptives (struct lexer *lexer, struct dataset *ds) { struct dictionary *dict = dataset_dict (ds); struct dsc_proc *dsc; - struct variable **vars = NULL; + const struct variable **vars = NULL; size_t var_cnt = 0; int save_z_scores = 0; - size_t z_cnt = 0; + int z_cnt = 0; size_t i; bool ok; + struct casegrouper *grouper; + struct casereader *group; + /* Create and initialize dsc. */ dsc = xmalloc (sizeof *dsc); dsc->vars = NULL; dsc->var_cnt = 0; dsc->missing_type = DSC_VARIABLE; - dsc->include_user_missing = 0; + dsc->exclude = MV_ANY; dsc->show_var_labels = 1; dsc->show_index = 0; dsc->format = DSC_LINE; @@ -217,149 +222,160 @@ cmd_descriptives (struct dataset *ds) dsc->sort_by_stat = DSC_NONE; dsc->sort_ascending = 1; dsc->show_stats = dsc->calc_stats = DEFAULT_STATS; + dsc->z_writer = NULL; /* Parse DESCRIPTIVES. */ - while (token != '.') + while (lex_token (lexer) != T_ENDCMD) { - if (lex_match_id ("MISSING")) + if (lex_match_id (lexer, "MISSING")) { - lex_match ('='); - while (token != '.' && token != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { - if (lex_match_id ("VARIABLE")) + if (lex_match_id (lexer, "VARIABLE")) dsc->missing_type = DSC_VARIABLE; - else if (lex_match_id ("LISTWISE")) + else if (lex_match_id (lexer, "LISTWISE")) dsc->missing_type = DSC_LISTWISE; - else if (lex_match_id ("INCLUDE")) - dsc->include_user_missing = 1; + else if (lex_match_id (lexer, "INCLUDE")) + dsc->exclude = MV_SYSTEM; else { - lex_error (NULL); + lex_error (lexer, NULL); goto error; } - lex_match (','); + lex_match (lexer, T_COMMA); } } - else if (lex_match_id ("SAVE")) + else if (lex_match_id (lexer, "SAVE")) save_z_scores = 1; - else if (lex_match_id ("FORMAT")) + else if (lex_match_id (lexer, "FORMAT")) { - lex_match ('='); - while (token != '.' && token != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { - if (lex_match_id ("LABELS")) + if (lex_match_id (lexer, "LABELS")) dsc->show_var_labels = 1; - else if (lex_match_id ("NOLABELS")) + else if (lex_match_id (lexer, "NOLABELS")) dsc->show_var_labels = 0; - else if (lex_match_id ("INDEX")) + else if (lex_match_id (lexer, "INDEX")) dsc->show_index = 1; - else if (lex_match_id ("NOINDEX")) + else if (lex_match_id (lexer, "NOINDEX")) dsc->show_index = 0; - else if (lex_match_id ("LINE")) + else if (lex_match_id (lexer, "LINE")) dsc->format = DSC_LINE; - else if (lex_match_id ("SERIAL")) + else if (lex_match_id (lexer, "SERIAL")) dsc->format = DSC_SERIAL; else { - lex_error (NULL); + lex_error (lexer, NULL); goto error; } - lex_match (','); + lex_match (lexer, T_COMMA); } } - else if (lex_match_id ("STATISTICS")) + else if (lex_match_id (lexer, "STATISTICS")) { - lex_match ('='); + lex_match (lexer, T_EQUALS); dsc->show_stats = 0; - while (token != '.' && token != '/') + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { - if (lex_match (T_ALL)) + if (lex_match (lexer, T_ALL)) dsc->show_stats |= (1ul << DSC_N_STATS) - 1; - else if (lex_match_id ("DEFAULT")) + else if (lex_match_id (lexer, "DEFAULT")) dsc->show_stats |= DEFAULT_STATS; else - dsc->show_stats |= 1ul << (match_statistic ()); - lex_match (','); + { + enum dsc_statistic s = match_statistic (lexer); + if (s == DSC_NONE ) + { + lex_error (lexer, NULL); + goto error; + } + dsc->show_stats |= 1ul << s; + } + lex_match (lexer, T_COMMA); } if (dsc->show_stats == 0) dsc->show_stats = DEFAULT_STATS; } - else if (lex_match_id ("SORT")) + else if (lex_match_id (lexer, "SORT")) { - lex_match ('='); - if (lex_match_id ("NAME")) + lex_match (lexer, T_EQUALS); + if (lex_match_id (lexer, "NAME")) dsc->sort_by_stat = DSC_NAME; - else + else { - dsc->sort_by_stat = match_statistic (); + dsc->sort_by_stat = match_statistic (lexer); if (dsc->sort_by_stat == DSC_NONE ) dsc->sort_by_stat = DSC_MEAN; } - if (lex_match ('(')) + if (lex_match (lexer, T_LPAREN)) { - if (lex_match_id ("A")) + if (lex_match_id (lexer, "A")) dsc->sort_ascending = 1; - else if (lex_match_id ("D")) + else if (lex_match_id (lexer, "D")) dsc->sort_ascending = 0; else - lex_error (NULL); - lex_force_match (')'); + lex_error (lexer, NULL); + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } } else if (var_cnt == 0) { - if (lex_look_ahead () == '=') + if (lex_next_token (lexer, 1) == T_EQUALS) { - lex_match_id ("VARIABLES"); - lex_match ('='); + lex_match_id (lexer, "VARIABLES"); + lex_match (lexer, T_EQUALS); } - while (token != '.' && token != '/') + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { int i; - - if (!parse_variables (dataset_dict (ds), &vars, &var_cnt, + + if (!parse_variables_const (lexer, dict, &vars, &var_cnt, PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC)) goto error; - dsc->vars = xnrealloc (dsc->vars, var_cnt, sizeof *dsc->vars); + dsc->vars = xnrealloc ((void *)dsc->vars, var_cnt, sizeof *dsc->vars); for (i = dsc->var_cnt; i < var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; dv->v = vars[i]; - dv->z_name[0] = '\0'; + dv->z_name = NULL; dv->moments = NULL; } dsc->var_cnt = var_cnt; - if (lex_match ('(')) + if (lex_match (lexer, T_LPAREN)) { - if (token != T_ID) + if (lex_token (lexer) != T_ID) { - lex_error (NULL); + lex_error (lexer, NULL); goto error; } - if (try_name (dict, dsc, tokid)) + if (try_name (dict, dsc, lex_tokcstr (lexer))) { - strcpy (dsc->vars[dsc->var_cnt - 1].z_name, tokid); + struct dsc_var *dsc_var = &dsc->vars[dsc->var_cnt - 1]; + dsc_var->z_name = xstrdup (lex_tokcstr (lexer)); z_cnt++; } else msg (SE, _("Z-score variable name %s would be" - " a duplicate variable name."), tokid); - lex_get (); - if (!lex_force_match (')')) + " a duplicate variable name."), lex_tokcstr (lexer)); + lex_get (lexer); + if (!lex_force_match (lexer, T_RPAREN)) goto error; } } } - else + else { - lex_error (NULL); - goto error; + lex_error (lexer, NULL); + goto error; } - lex_match ('/'); + lex_match (lexer, T_SLASH); } if (var_cnt == 0) { @@ -370,19 +386,41 @@ cmd_descriptives (struct dataset *ds) /* Construct z-score varnames, show translation table. */ if (z_cnt || save_z_scores) { - if (save_z_scores) + struct caseproto *proto; + + if (save_z_scores) { - size_t gen_cnt = 0; + int gen_cnt = 0; for (i = 0; i < dsc->var_cnt; i++) - if (dsc->vars[i].z_name[0] == 0) - { - if (!generate_z_varname (dict, dsc, dsc->vars[i].z_name, - dsc->vars[i].v->name, &gen_cnt)) - goto error; - z_cnt++; - } + { + struct dsc_var *dsc_var = &dsc->vars[i]; + if (dsc_var->z_name == NULL) + { + const char *name = var_get_name (dsc_var->v); + dsc_var->z_name = generate_z_varname (dict, dsc, name, + &gen_cnt); + if (dsc_var->z_name == NULL) + goto error; + + z_cnt++; + } + } } + + /* It would be better to handle Z scores correctly (however we define + that) when TEMPORARY is in effect, but in the meantime this at least + prevents a use-after-free error. See bug #38786. */ + if (proc_make_temporary_transformations_permanent (ds)) + msg (SW, _("DESCRIPTIVES with Z scores ignores TEMPORARY. " + "Temporary transformations will be made permanent.")); + + proto = caseproto_create (); + for (i = 0; i < 1 + 2 * z_cnt; i++) + proto = caseproto_add_width (proto, 0); + dsc->z_writer = autopaging_writer_create (proto); + caseproto_unref (proto); + dump_z_table (dsc); } @@ -406,7 +444,7 @@ cmd_descriptives (struct dataset *ds) /* Figure out maximum moment needed and allocate moments for the variables. */ dsc->max_moment = MOMENT_NONE; - for (i = 0; i < DSC_N_STATS; i++) + for (i = 0; i < DSC_N_STATS; i++) if (dsc->calc_stats & (1ul << i) && dsc_info[i].moment > dsc->max_moment) dsc->max_moment = dsc_info[i].moment; if (dsc->max_moment != MOMENT_NONE) @@ -414,7 +452,11 @@ cmd_descriptives (struct dataset *ds) dsc->vars[i].moments = moments_create (dsc->max_moment); /* Data pass. */ - ok = multipass_procedure_with_splits (ds, calc_descriptives, dsc); + grouper = casegrouper_create_splits (proc_open_filtering (ds, false), dict); + while (casegrouper_get_next_group (grouper, &group)) + calc_descriptives (dsc, group, ds); + ok = casegrouper_destroy (grouper); + ok = proc_commit (ds) && ok; /* Z-scoring! */ if (ok && z_cnt) @@ -436,18 +478,18 @@ cmd_descriptives (struct dataset *ds) specifiers). Emits an error if the current token ID does not name a statistic. */ static enum dsc_statistic -match_statistic (void) +match_statistic (struct lexer *lexer) { - if (token == T_ID) + if (lex_token (lexer) == T_ID) { enum dsc_statistic stat; for (stat = 0; stat < DSC_N_STATS; stat++) - if (lex_match_id (dsc_info[stat].identifier)) + if (lex_match_id (lexer, dsc_info[stat].identifier)) return stat; - lex_get(); - lex_error (_("expecting statistic name: reverting to default")); + lex_get (lexer); + lex_error (lexer, _("expecting statistic name: reverting to default")); } return DSC_NONE; @@ -461,9 +503,14 @@ free_dsc_proc (struct dsc_proc *dsc) if (dsc == NULL) return; - + for (i = 0; i < dsc->var_cnt; i++) - moments_destroy (dsc->vars[i].moments); + { + struct dsc_var *dsc_var = &dsc->vars[i]; + free (dsc_var->z_name); + moments_destroy (dsc_var->moments); + } + casewriter_destroy (dsc->z_writer); free (dsc->vars); free (dsc); } @@ -473,40 +520,46 @@ free_dsc_proc (struct dsc_proc *dsc) /* Returns false if NAME is a duplicate of any existing variable name or of any previously-declared z-var name; otherwise returns true. */ static bool -try_name (const struct dictionary *dict, struct dsc_proc *dsc, char *name) +try_name (const struct dictionary *dict, struct dsc_proc *dsc, + const char *name) { size_t i; if (dict_lookup_var (dict, name) != NULL) return false; for (i = 0; i < dsc->var_cnt; i++) - if (!strcasecmp (dsc->vars[i].z_name, name)) - return false; + { + struct dsc_var *dsc_var = &dsc->vars[i]; + if (dsc_var->z_name != NULL && !utf8_strcasecmp (dsc_var->z_name, name)) + return false; + } return true; } /* Generates a name for a Z-score variable based on a variable named VAR_NAME, given that *Z_CNT generated variable names are - known to already exist. If successful, returns true and - copies the new name into Z_NAME. On failure, returns false. */ -static bool -generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z_name, - const char *var_name, size_t *z_cnt) + known to already exist. If successful, returns the new name + as a dynamically allocated string. On failure, returns NULL. */ +static char * +generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, + const char *var_name, int *z_cnt) { - char name[LONG_NAME_LEN + 1]; + char *z_name, *trunc_name; /* Try a name based on the original variable name. */ - name[0] = 'Z'; - str_copy_trunc (name + 1, sizeof name - 1, var_name); - if (try_name (dict, dsc, name)) - { - strcpy (z_name, name); - return true; - } + z_name = xasprintf ("Z%s", var_name); + trunc_name = utf8_encoding_trunc (z_name, dict_get_encoding (dict), + ID_MAX_LEN); + free (z_name); + if (try_name (dict, dsc, trunc_name)) + return trunc_name; + free (trunc_name); /* Generate a synthetic name. */ for (;;) { + char name[16]; + (*z_cnt)++; if (*z_cnt <= 99) @@ -522,14 +575,11 @@ generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z msg (SE, _("Ran out of generic names for Z-score variables. " "There are only 126 generic names: ZSC001-ZSC0999, " "STDZ01-STDZ09, ZZZZ01-ZZZZ09, ZQZQ01-ZQZQ09.")); - return false; + return NULL; } - + if (try_name (dict, dsc, name)) - { - strcpy (z_name, name); - return true; - } + return xstrdup (name); } NOT_REACHED(); } @@ -541,39 +591,46 @@ dump_z_table (struct dsc_proc *dsc) { size_t cnt = 0; struct tab_table *t; - + { size_t i; - + for (i = 0; i < dsc->var_cnt; i++) - if (dsc->vars[i].z_name[0] != '\0') + if (dsc->vars[i].z_name != NULL) cnt++; } - - t = tab_create (2, cnt + 1, 0); + + t = tab_create (2, cnt + 1); tab_title (t, _("Mapping of variables to corresponding Z-scores.")); - tab_columns (t, SOM_COL_DOWN, 1); tab_headers (t, 0, 0, 1, 0); tab_box (t, TAL_1, TAL_1, TAL_0, TAL_1, 0, 0, 1, cnt); tab_hline (t, TAL_2, 0, 1, 1); tab_text (t, 0, 0, TAB_CENTER | TAT_TITLE, _("Source")); tab_text (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Target")); - tab_dim (t, tab_natural_dimensions); { size_t i, y; - + for (i = 0, y = 1; i < dsc->var_cnt; i++) - if (dsc->vars[i].z_name[0] != '\0') + if (dsc->vars[i].z_name != NULL) { - tab_text (t, 0, y, TAB_LEFT, dsc->vars[i].v->name); + tab_text (t, 0, y, TAB_LEFT, var_to_string (dsc->vars[i].v)); tab_text (t, 1, y++, TAB_LEFT, dsc->vars[i].z_name); } } - + tab_submit (t); } +static void +descriptives_set_all_sysmis_zscores (const struct dsc_trns *t, struct ccase *c) +{ + const struct dsc_z_score *z; + + for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) + case_data_rw (c, z->z_var)->f = SYSMIS; +} + /* Transformation function to calculate Z-scores. Will return SYSMIS if any of the following are true: 1) mean or standard deviation is SYSMIS 2) score is SYSMIS 3) score is user missing and they were not included in the original @@ -581,39 +638,76 @@ dump_z_table (struct dsc_proc *dsc) (either system or user-missing values that weren't included). */ static int -descriptives_trns_proc (void *trns_, struct ccase * c, +descriptives_trns_proc (void *trns_, struct ccase **c, casenumber case_idx UNUSED) { struct dsc_trns *t = trns_; struct dsc_z_score *z; - struct variable **vars; - int all_sysmis = 0; + const struct variable **vars; + + *c = case_unshare (*c); + + if (t->filter) + { + double f = case_num (*c, t->filter); + if (f == 0.0 || var_is_num_missing (t->filter, f, MV_ANY)) + { + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; + } + } + + if (t->count <= 0) + { + struct ccase *z_case; + + z_case = casereader_read (t->z_reader); + if (z_case) + { + size_t z_idx = 0; + + t->count = case_num_idx (z_case, z_idx++); + for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) + { + z->mean = case_num_idx (z_case, z_idx++); + z->std_dev = case_num_idx (z_case, z_idx++); + } + case_unref (z_case); + } + else + { + if (t->ok) + { + msg (SE, _("Internal error processing Z scores")); + t->ok = false; + } + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; + } + } + t->count--; if (t->missing_type == DSC_LISTWISE) { assert(t->vars); for (vars = t->vars; vars < t->vars + t->var_cnt; vars++) { - double score = case_num (c, (*vars)->fv); - if ( score == SYSMIS - || (!t->include_user_missing - && mv_is_num_user_missing (&(*vars)->miss, score))) + double score = case_num (*c, *vars); + if (var_is_num_missing (*vars, score, t->exclude)) { - all_sysmis = 1; - break; + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; } } } - + for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) { - double input = case_num (c, z->src_idx); - double *output = &case_data_rw (c, z->dst_idx)->f; + double input = case_num (*c, z->src_var); + double *output = &case_data_rw (*c, z->z_var)->f; - if (z->mean == SYSMIS || z->std_dev == SYSMIS - || all_sysmis || input == SYSMIS - || (!t->include_user_missing - && mv_is_num_user_missing (&z->v->miss, input))) + if (z->mean == SYSMIS || z->std_dev == SYSMIS + || var_is_num_missing (z->src_var, input, t->exclude)) *output = SYSMIS; else *output = (input - z->mean) / z->std_dev; @@ -626,11 +720,15 @@ static bool descriptives_trns_free (void *trns_) { struct dsc_trns *t = trns_; + bool ok = t->ok && !casereader_error (t->z_reader); free (t->z_scores); + casereader_destroy (t->z_reader); assert((t->missing_type != DSC_LISTWISE) ^ (t->vars != NULL)); free (t->vars); - return true; + free (t); + + return ok; } /* Sets up a transformation to calculate Z scores. */ @@ -641,14 +739,14 @@ setup_z_trns (struct dsc_proc *dsc, struct dataset *ds) size_t cnt, i; for (cnt = i = 0; i < dsc->var_cnt; i++) - if (dsc->vars[i].z_name[0] != '\0') + if (dsc->vars[i].z_name != NULL) cnt++; t = xmalloc (sizeof *t); t->z_scores = xnmalloc (cnt, sizeof *t->z_scores); t->z_score_cnt = cnt; t->missing_type = dsc->missing_type; - t->include_user_missing = dsc->include_user_missing; + t->exclude = dsc->exclude; if ( t->missing_type == DSC_LISTWISE ) { t->var_cnt = dsc->var_cnt; @@ -661,36 +759,30 @@ setup_z_trns (struct dsc_proc *dsc, struct dataset *ds) t->var_cnt = 0; t->vars = NULL; } + t->filter = dict_get_filter (dataset_dict (ds)); + t->z_reader = casewriter_make_reader (dsc->z_writer); + t->count = 0; + t->ok = true; + dsc->z_writer = NULL; for (cnt = i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - if (dv->z_name[0] != '\0') + if (dv->z_name != NULL) { struct dsc_z_score *z; - char *cp; struct variable *dst_var; + char *label; dst_var = dict_create_var_assert (dataset_dict (ds), dv->z_name, 0); - if (dv->v->label) - { - dst_var->label = xmalloc (strlen (dv->v->label) + 12); - cp = stpcpy (dst_var->label, _("Z-score of ")); - strcpy (cp, dv->v->label); - } - else - { - dst_var->label = xmalloc (strlen (dv->v->name) + 12); - cp = stpcpy (dst_var->label, _("Z-score of ")); - strcpy (cp, dv->v->name); - } + + label = xasprintf (_("Z-score of %s"),var_to_string (dv->v)); + var_set_label (dst_var, label); + free (label); z = &t->z_scores[cnt++]; - z->src_idx = dv->v->fv; - z->dst_idx = dst_var->fv; - z->mean = dv->stats[DSC_MEAN]; - z->std_dev = dv->stats[DSC_STDDEV]; - z->v = dv->v; + z->src_var = dv->v; + z->z_var = dst_var; } } @@ -704,22 +796,36 @@ static bool listwise_missing (struct dsc_proc *dsc, const struct ccase *c); /* Calculates and displays descriptive statistics for the cases in CF. */ -static bool -calc_descriptives (const struct ccase *first, - const struct casefile *cf, void *dsc_, - const struct dataset *ds) +static void +calc_descriptives (struct dsc_proc *dsc, struct casereader *group, + struct dataset *ds) { - struct dsc_proc *dsc = dsc_; - struct casereader *reader; - struct ccase c; + struct variable *filter = dict_get_filter (dataset_dict (ds)); + struct casereader *pass1, *pass2; + casenumber count; + struct ccase *c; + size_t z_idx; size_t i; - output_split_file_values (ds, first); + c = casereader_peek (group, 0); + if (c == NULL) + { + casereader_destroy (group); + return; + } + output_split_file_values (ds, c); + case_unref (c); + + group = casereader_create_filter_weight (group, dataset_dict (ds), + NULL, NULL); + + pass1 = group; + pass2 = dsc->max_moment <= MOMENT_MEAN ? NULL : casereader_clone (pass1); for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - + dv->valid = dv->missing = 0.0; if (dv->moments != NULL) moments_clear (dv->moments); @@ -730,38 +836,39 @@ calc_descriptives (const struct ccase *first, dsc->valid = 0.; /* First pass to handle most of the work. */ - for (reader = casefile_get_reader (cf, NULL); - casereader_read (reader, &c); - case_destroy (&c)) + count = 0; + for (; (c = casereader_read (pass1)) != NULL; case_unref (c)) { - double weight = dict_get_case_weight (dataset_dict (ds), &c, &dsc->bad_warn); - if (weight <= 0.0) - continue; - + double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + + if (filter) + { + double f = case_num (c, filter); + if (f == 0.0 || var_is_num_missing (filter, f, MV_ANY)) + continue; + } + /* Check for missing values. */ - if (listwise_missing (dsc, &c)) + if (listwise_missing (dsc, c)) { dsc->missing_listwise += weight; if (dsc->missing_type == DSC_LISTWISE) - continue; + continue; } dsc->valid += weight; - for (i = 0; i < dsc->var_cnt; i++) + for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - double x = case_num (&c, dv->v->fv); - - if (dsc->missing_type != DSC_LISTWISE - && (x == SYSMIS - || (!dsc->include_user_missing - && mv_is_num_user_missing (&dv->v->miss, x)))) + double x = case_num (c, dv->v); + + if (var_is_num_missing (dv->v, x, dsc->exclude)) { dv->missing += weight; continue; } - if (dv->moments != NULL) + if (dv->moments != NULL) moments_pass_one (dv->moments, x, weight); if (x < dv->min) @@ -769,45 +876,59 @@ calc_descriptives (const struct ccase *first, if (x > dv->max) dv->max = x; } + + count++; + } + if (!casereader_destroy (pass1)) + { + casereader_destroy (pass2); + return; } - casereader_destroy (reader); /* Second pass for higher-order moments. */ - if (dsc->max_moment > MOMENT_MEAN) + if (dsc->max_moment > MOMENT_MEAN) { - for (reader = casefile_get_reader (cf, NULL); - casereader_read (reader, &c); - case_destroy (&c)) + for (; (c = casereader_read (pass2)) != NULL; case_unref (c)) { - double weight = dict_get_case_weight (dataset_dict (ds), &c, - &dsc->bad_warn); - if (weight <= 0.0) - continue; - + double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + + if (filter) + { + double f = case_num (c, filter); + if (f == 0.0 || var_is_num_missing (filter, f, MV_ANY)) + continue; + } + /* Check for missing values. */ - if (listwise_missing (dsc, &c) - && dsc->missing_type == DSC_LISTWISE) - continue; + if (dsc->missing_type == DSC_LISTWISE && listwise_missing (dsc, c)) + continue; - for (i = 0; i < dsc->var_cnt; i++) + for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - double x = case_num (&c, dv->v->fv); - - if (dsc->missing_type != DSC_LISTWISE - && (x == SYSMIS - || (!dsc->include_user_missing - && mv_is_num_user_missing (&dv->v->miss, x)))) + double x = case_num (c, dv->v); + + if (var_is_num_missing (dv->v, x, dsc->exclude)) continue; if (dv->moments != NULL) moments_pass_two (dv->moments, x, weight); } } - casereader_destroy (reader); + if (!casereader_destroy (pass2)) + return; } - + /* Calculate results. */ + if (dsc->z_writer && count > 0) + { + c = case_create (casewriter_get_proto (dsc->z_writer)); + z_idx = 0; + case_data_rw_idx (c, z_idx++)->f = count; + } + else + c = NULL; + for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; @@ -829,7 +950,7 @@ calc_descriptives (const struct ccase *first, if (dsc->calc_stats & (1ul << DSC_STDDEV) && dv->stats[DSC_VARIANCE] != SYSMIS) dv->stats[DSC_STDDEV] = sqrt (dv->stats[DSC_VARIANCE]); - if (dsc->calc_stats & (1ul << DSC_SEKURT)) + if (dsc->calc_stats & (1ul << DSC_SEKURT)) if (dv->stats[DSC_KURTOSIS] != SYSMIS) dv->stats[DSC_SEKURT] = calc_sekurt (W); if (dsc->calc_stats & (1ul << DSC_SESKEW) @@ -841,29 +962,34 @@ calc_descriptives (const struct ccase *first, dv->stats[DSC_MAX] = dv->max == -DBL_MAX ? SYSMIS : dv->max; if (dsc->calc_stats & (1ul << DSC_SUM)) dv->stats[DSC_SUM] = W * dv->stats[DSC_MEAN]; + + if (dv->z_name && c != NULL) + { + case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_MEAN]; + case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_STDDEV]; + } } + if (c != NULL) + casewriter_write (dsc->z_writer, c); + /* Output results. */ display (dsc); - - return true; } /* Returns true if any of the descriptives variables in DSC's variable list have missing values in case C, false otherwise. */ static bool -listwise_missing (struct dsc_proc *dsc, const struct ccase *c) +listwise_missing (struct dsc_proc *dsc, const struct ccase *c) { size_t i; for (i = 0; i < dsc->var_cnt; i++) { struct dsc_var *dv = &dsc->vars[i]; - double x = case_num (c, dv->v->fv); + double x = case_num (c, dv->v); - if (x == SYSMIS - || (!dsc->include_user_missing - && mv_is_num_user_missing (&dv->v->miss, x))) + if (var_is_num_missing (dv->v, x, dsc->exclude)) return true; } return false; @@ -890,13 +1016,12 @@ display (struct dsc_proc *dsc) sort (dsc->vars, dsc->var_cnt, sizeof *dsc->vars, descriptives_compare_dsc_vars, dsc); - t = tab_create (nc, dsc->var_cnt + 1, 0); + t = tab_create (nc, dsc->var_cnt + 1); tab_headers (t, 1, 0, 1, 0); tab_box (t, TAL_1, TAL_1, -1, -1, 0, 0, nc - 1, dsc->var_cnt); tab_box (t, -1, -1, -1, TAL_1, 1, 0, nc - 1, dsc->var_cnt); tab_hline (t, TAL_2, 0, nc - 1, 1); tab_vline (t, TAL_2, 1, 0, dsc->var_cnt); - tab_dim (t, tab_natural_dimensions); nc = 0; tab_text (t, nc++, 0, TAB_LEFT | TAT_TITLE, _("Variable")); @@ -921,17 +1046,19 @@ display (struct dsc_proc *dsc) size_t j; nc = 0; - tab_text (t, nc++, i + 1, TAB_LEFT, dv->v->name); - tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->valid); + tab_text (t, nc++, i + 1, TAB_LEFT, var_to_string (dv->v)); + tab_text_format (t, nc++, i + 1, 0, "%.*g", DBL_DIG + 1, dv->valid); if (dsc->format == DSC_SERIAL) - tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->missing); + tab_text_format (t, nc++, i + 1, 0, "%.*g", DBL_DIG + 1, dv->missing); + for (j = 0; j < DSC_N_STATS; j++) if (dsc->show_stats & (1ul << j)) - tab_float (t, nc++, i + 1, TAB_NONE, dv->stats[j], 10, 3); + tab_double (t, nc++, i + 1, TAB_NONE, dv->stats[j], NULL, RC_OTHER); } - tab_title (t, _("Valid cases = %g; cases with missing value(s) = %g."), - dsc->valid, dsc->missing_listwise); + tab_title (t, _("Valid cases = %.*g; cases with missing value(s) = %.*g."), + DBL_DIG + 1, dsc->valid, + DBL_DIG + 1, dsc->missing_listwise); tab_submit (t); } @@ -948,8 +1075,8 @@ descriptives_compare_dsc_vars (const void *a_, const void *b_, const void *dsc_) int result; if (dsc->sort_by_stat == DSC_NAME) - result = strcasecmp (a->v->name, b->v->name); - else + result = utf8_strcasecmp (var_get_name (a->v), var_get_name (b->v)); + else { double as = a->stats[dsc->sort_by_stat]; double bs = b->stats[dsc->sort_by_stat];