From dbb294a59a93358b911ae3c3982301842c18e66b Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Fri, 4 Apr 2014 15:32:32 -0700 Subject: [PATCH] DESCRIPTIVES: Fix treatment of FILTER in presence of Z scores. Bug #42012. Reported by Mindaugus. --- NEWS | 3 ++ src/language/stats/descriptives.c | 57 ++++++++++++++++++++++------ tests/language/stats/descriptives.at | 56 +++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS index 6cf918b8db..23167296e5 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,9 @@ Changes since 0.8.2: * REGRESSION now recognises /STATISTICS=CI(x) which causes confidence intervals for the coefficients to be printed. + * When DESCRIPTIVES calculates Z scores, it now omits cases filtered + by the current FILTER settings. + * PSPPIRE graphical user interface improvements: - Dialog boxes with source variable lists can now choose the sort diff --git a/src/language/stats/descriptives.c b/src/language/stats/descriptives.c index a35878b75a..beb5ec0b41 100644 --- a/src/language/stats/descriptives.c +++ b/src/language/stats/descriptives.c @@ -76,6 +76,7 @@ struct dsc_trns size_t var_cnt; /* Number of variables. */ enum dsc_missing_type missing_type; /* Treatment of missing values. */ enum mv_class exclude; /* Classes of missing values to exclude. */ + struct variable *filter; /* Dictionary FILTER BY variable. */ struct casereader *z_reader; /* Reader for count, mean, stddev. */ casenumber count; /* Number left in this SPLIT FILE group.*/ bool ok; @@ -442,8 +443,7 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds) dsc->vars[i].moments = moments_create (dsc->max_moment); /* Data pass. */ - grouper = casegrouper_create_splits (proc_open_filtering (ds, z_cnt == 0), - dict); + grouper = casegrouper_create_splits (proc_open_filtering (ds, false), dict); while (casegrouper_get_next_group (grouper, &group)) calc_descriptives (dsc, group, ds); ok = casegrouper_destroy (grouper); @@ -613,6 +613,15 @@ dump_z_table (struct dsc_proc *dsc) tab_submit (t); } +static void +descriptives_set_all_sysmis_zscores (const struct dsc_trns *t, struct ccase *c) +{ + const struct dsc_z_score *z; + + for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) + case_data_rw (c, z->z_var)->f = SYSMIS; +} + /* Transformation function to calculate Z-scores. Will return SYSMIS if any of the following are true: 1) mean or standard deviation is SYSMIS 2) score is SYSMIS 3) score is user missing and they were not included in the original @@ -626,7 +635,18 @@ descriptives_trns_proc (void *trns_, struct ccase **c, struct dsc_trns *t = trns_; struct dsc_z_score *z; const struct variable **vars; - int all_sysmis = 0; + + *c = case_unshare (*c); + + if (t->filter) + { + double f = case_num (*c, t->filter); + if (f == 0.0 || var_is_num_missing (t->filter, f, MV_ANY)) + { + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; + } + } if (t->count <= 0) { @@ -652,8 +672,8 @@ descriptives_trns_proc (void *trns_, struct ccase **c, msg (SE, _("Internal error processing Z scores")); t->ok = false; } - for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) - z->mean = z->std_dev = SYSMIS; + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; } } t->count--; @@ -666,19 +686,18 @@ descriptives_trns_proc (void *trns_, struct ccase **c, double score = case_num (*c, *vars); if (var_is_num_missing (*vars, score, t->exclude)) { - all_sysmis = 1; - break; + descriptives_set_all_sysmis_zscores (t, *c); + return TRNS_CONTINUE; } } } - *c = case_unshare (*c); for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++) { double input = case_num (*c, z->src_var); double *output = &case_data_rw (*c, z->z_var)->f; - if (z->mean == SYSMIS || z->std_dev == SYSMIS || all_sysmis + if (z->mean == SYSMIS || z->std_dev == SYSMIS || var_is_num_missing (z->src_var, input, t->exclude)) *output = SYSMIS; else @@ -731,6 +750,7 @@ setup_z_trns (struct dsc_proc *dsc, struct dataset *ds) t->var_cnt = 0; t->vars = NULL; } + t->filter = dict_get_filter (dataset_dict (ds)); t->z_reader = casewriter_make_reader (dsc->z_writer); t->count = 0; t->ok = true; @@ -771,6 +791,7 @@ static void calc_descriptives (struct dsc_proc *dsc, struct casereader *group, struct dataset *ds) { + struct variable *filter = dict_get_filter (dataset_dict (ds)); struct casereader *pass1, *pass2; casenumber count; struct ccase *c; @@ -811,6 +832,13 @@ calc_descriptives (struct dsc_proc *dsc, struct casereader *group, { double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + if (filter) + { + double f = case_num (c, filter); + if (f == 0.0 || var_is_num_missing (filter, f, MV_ANY)) + continue; + } + /* Check for missing values. */ if (listwise_missing (dsc, c)) { @@ -855,6 +883,13 @@ calc_descriptives (struct dsc_proc *dsc, struct casereader *group, { double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + if (filter) + { + double f = case_num (c, filter); + if (f == 0.0 || var_is_num_missing (filter, f, MV_ANY)) + continue; + } + /* Check for missing values. */ if (dsc->missing_type == DSC_LISTWISE && listwise_missing (dsc, c)) continue; @@ -876,7 +911,7 @@ calc_descriptives (struct dsc_proc *dsc, struct casereader *group, } /* Calculate results. */ - if (dsc->z_writer) + if (dsc->z_writer && count > 0) { c = case_create (casewriter_get_proto (dsc->z_writer)); z_idx = 0; @@ -919,7 +954,7 @@ calc_descriptives (struct dsc_proc *dsc, struct casereader *group, if (dsc->calc_stats & (1ul << DSC_SUM)) dv->stats[DSC_SUM] = W * dv->stats[DSC_MEAN]; - if (dv->z_name) + if (dv->z_name && c != NULL) { case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_MEAN]; case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_STDDEV]; diff --git a/tests/language/stats/descriptives.at b/tests/language/stats/descriptives.at index 37b7d5a813..9725cdae43 100644 --- a/tests/language/stats/descriptives.at +++ b/tests/language/stats/descriptives.at @@ -321,3 +321,59 @@ id,abc,Zabc 6.00,4.00,1.20 ]) AT_CLEANUP + +dnl This test was supplied by Mindaugus as part of the report for bug #42012. +AT_SETUP([DESCRIPTIVES -- Z scores with FILTER]) +AT_DATA([descriptives.sps], [dnl +DATA LIST LIST/filter1 filter2 x. +BEGIN DATA. +0,0,300 +0,1,200 +0,1,100 +1,0,5 +1,0,4 +1,1,3 +1,1,2 +1,1,1 +END DATA. + +FILTER OFF. +SPLIT FILE OFF. +DESCRIPTIVES /VARIABLES=X /SAVE. + +FILTER BY filter1. +SPLIT FILE OFF. +DESCRIPTIVES /VARIABLES=X /SAVE. + +FILTER OFF. +SORT CASES BY filter1. +SPLIT FILE BY filter1. +DESCRIPTIVES /VARIABLES=X /SAVE. + +FILTER BY filter2. +SPLIT FILE BY filter1. +DESCRIPTIVES /VARIABLES=X /SAVE. + +FILTER OFF. +SORT CASES BY filter1 filter2. +SPLIT FILE BY filter1 filter2. +DESCRIPTIVES /VARIABLES=X /SAVE. +EXECUTE. + +SPLIT FILE OFF. +LIST. +]) +AT_CHECK([pspp -o pspp.csv descriptives.sps]) +AT_CHECK([sed -n '/Table: Data List/,$p' < pspp.csv], [0], [dnl +Table: Data List +filter1,filter2,x,Zx,ZSC001,ZSC002,ZSC003,ZSC004 +.00,.00,300.00,1.94,. ,1.00,. ,. @&t@ +.00,1.00,200.00,1.07,. ,.00,.71,.71 +.00,1.00,100.00,.20,. ,-1.00,-.71,-.71 +1.00,.00,5.00,-.62,1.26,1.26,. ,.71 +1.00,.00,4.00,-.63,.63,.63,. ,-.71 +1.00,1.00,3.00,-.64,.00,.00,1.00,1.00 +1.00,1.00,2.00,-.65,-.63,-.63,.00,.00 +1.00,1.00,1.00,-.66,-1.26,-1.26,-1.00,-1.00 +]) +AT_CLEANUP -- 2.30.2