X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Ffrequencies.c;h=d20d8392c176d4827a2f4f0e28d3f4e53b73bf0f;hb=3facf6314da231910f60a53805916e879b1782e9;hp=675a02319aaebf1ca6f8d3b91cb323106eb1e929;hpb=f9b2322d7b0662bd313d9c63450638c39b88be70;p=pspp diff --git a/src/language/stats/frequencies.c b/src/language/stats/frequencies.c index 675a02319a..d20d8392c1 100644 --- a/src/language/stats/frequencies.c +++ b/src/language/stats/frequencies.c @@ -1,7 +1,7 @@ /* PSPP - a program for statistical analysis. Copyright (C) 1997-9, 2000, 2007, 2009, 2010, 2011, 2014, 2015 Free Software Foundation, Inc. - + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or @@ -11,7 +11,7 @@ but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + You should have received a copy of the GNU General Public License along with this program. If not, see . */ @@ -45,18 +45,16 @@ #include "libpspp/hmap.h" #include "libpspp/message.h" #include "libpspp/misc.h" -#include "libpspp/pool.h" #include "math/histogram.h" #include "math/moments.h" #include "math/chart-geometry.h" -#include "output/chart-item.h" #include "output/charts/barchart.h" #include "output/charts/piechart.h" #include "output/charts/plot-hist.h" -#include "output/tab.h" +#include "output/pivot-table.h" #include "gl/minmax.h" #include "gl/xalloc.h" @@ -70,7 +68,6 @@ struct percentile { double p; /* the %ile to be calculated */ - double value; /* the %ile's value */ bool show; /* True to show this percentile in the statistics box. */ }; @@ -107,7 +104,7 @@ enum FRQ_PERCENT }; -enum sortprops +enum sortprops { FRQ_AFREQ, FRQ_DFREQ, @@ -191,12 +188,9 @@ struct var_freqs /* Freqency table. */ struct freq_tab tab; /* Frequencies table to use. */ - /* Percentiles. */ - int n_groups; /* Number of groups. */ - double *groups; /* Groups. */ - /* Statistics. */ double stat[FRQ_ST_count]; + double *percentiles; /* Variable attributes. */ int width; @@ -204,14 +198,13 @@ struct var_freqs struct frq_proc { - struct pool *pool; - struct var_freqs *vars; size_t n_vars; /* Percentiles to calculate and possibly display. */ struct percentile *percentiles; - int n_percentiles, n_show_percentiles; + size_t median_idx; + size_t n_percentiles; /* Frequency table display. */ long int max_categories; /* Maximum categories to show. */ @@ -224,6 +217,8 @@ struct frq_proc /* Histogram and pie chart settings. */ struct frq_chart *hist, *pie, *bar; + + bool warn; }; @@ -236,9 +231,10 @@ struct freq_compare_aux bool ascending_value; }; -static void calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]); +static void calc_stats (const struct frq_proc *, + const struct var_freqs *, double d[FRQ_ST_count]); -static void do_piechart(const struct frq_chart *pie, +static void do_piechart(const struct frq_chart *pie, const struct variable *var, const struct freq_tab *frq_tab); @@ -246,9 +242,11 @@ static void do_barchart(const struct frq_chart *bar, const struct variable **var, const struct freq_tab *frq_tab); -static void dump_statistics (const struct frq_proc *frq, - const struct var_freqs *vf, - const struct variable *wv); +static struct frq_stats_table *frq_stats_table_submit ( + struct frq_stats_table *, const struct frq_proc *, + const struct dictionary *, const struct variable *wv, + const struct ccase *example); +static void frq_stats_table_destroy (struct frq_stats_table *); static int compare_freq (const void *a_, const void *b_, const void *aux_) @@ -270,93 +268,77 @@ compare_freq (const void *a_, const void *b_, const void *aux_) } /* Create a gsl_histogram from a freq_tab */ -static struct histogram * -freq_tab_to_hist (const struct frq_proc *frq, const struct freq_tab *ft, - const struct variable *var); +static struct histogram *freq_tab_to_hist (const struct frq_proc *, + const struct var_freqs *); +static void +put_freq_row (struct pivot_table *table, int var_idx, + double frequency, double percent, + double valid_percent, double cum_percent) +{ + double entries[] = { frequency, percent, valid_percent, cum_percent }; + for (size_t i = 0; i < sizeof entries / sizeof *entries; i++) + if (entries[i] != SYSMIS) + pivot_table_put2 (table, i, var_idx, + pivot_value_new_number (entries[i])); +} /* Displays a full frequency table for variable V. */ static void dump_freq_table (const struct var_freqs *vf, const struct variable *wv) { - const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0; const struct freq_tab *ft = &vf->tab; - int n_categories; - struct freq *f; - struct tab_table *t; - int r, x; - double cum_total = 0.0; - double cum_freq = 0.0; - static const char *headings[] = { - N_("Value Label"), - N_("Value"), - N_("Frequency"), - N_("Percent"), - N_("Valid Percent"), - N_("Cum Percent") - }; + struct pivot_table *table = pivot_table_create__ (pivot_value_new_variable ( + vf->var), "Frequencies"); + pivot_table_set_weight_var (table, wv); - n_categories = ft->n_valid + ft->n_missing; - t = tab_create (6, n_categories + 2); - tab_set_format (t, RC_WEIGHT, wfmt); - tab_headers (t, 0, 0, 1, 0); + pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Statistics"), + N_("Frequency"), PIVOT_RC_COUNT, + N_("Percent"), PIVOT_RC_PERCENT, + N_("Valid Percent"), PIVOT_RC_PERCENT, + N_("Cumulative Percent"), PIVOT_RC_PERCENT); - for (x = 0; x < 6; x++) - tab_text (t, x, 0, TAB_CENTER | TAT_TITLE, gettext (headings[x])); + struct pivot_dimension *variable = pivot_dimension_create__ ( + table, PIVOT_AXIS_ROW, pivot_value_new_variable (vf->var)); - r = 1; - for (f = ft->valid; f < ft->missing; f++) + double cum_freq = 0.0; + double cum_percent = 0.0; + struct pivot_category *valid = NULL; + for (const struct freq *f = ft->valid; f < ft->missing; f++) { - const char *label; - double percent, valid_percent; - cum_freq += f->count; - - percent = f->count / ft->total_cases * 100.0; - valid_percent = f->count / ft->valid_cases * 100.0; - cum_total += valid_percent; - - label = var_lookup_value_label (vf->var, f->values); - if (label != NULL) - tab_text (t, 0, r, TAB_LEFT, label); - - tab_value (t, 1, r, TAB_NONE, f->values, vf->var, NULL); - tab_double (t, 2, r, TAB_NONE, f->count, NULL, RC_WEIGHT); - tab_double (t, 3, r, TAB_NONE, percent, NULL, RC_OTHER); - tab_double (t, 4, r, TAB_NONE, valid_percent, NULL, RC_OTHER); - tab_double (t, 5, r, TAB_NONE, cum_total, NULL, RC_OTHER); - r++; + double valid_percent = f->count / ft->valid_cases * 100.0; + cum_percent += valid_percent; + + if (!valid) + valid = pivot_category_create_group (variable->root, N_("Valid")); + int var_idx = pivot_category_create_leaf ( + valid, pivot_value_new_var_value (vf->var, &f->values[0])); + put_freq_row (table, var_idx, f->count, + f->count / ft->total_cases * 100.0, + valid_percent, cum_percent); } - for (; f < &ft->valid[n_categories]; f++) - { - const char *label; + struct pivot_category *missing = NULL; + size_t n_categories = ft->n_valid + ft->n_missing; + for (const struct freq *f = ft->missing; f < &ft->valid[n_categories]; f++) + { cum_freq += f->count; - label = var_lookup_value_label (vf->var, f->values); - if (label != NULL) - tab_text (t, 0, r, TAB_LEFT, label); - - tab_value (t, 1, r, TAB_NONE, f->values, vf->var, NULL); - tab_double (t, 2, r, TAB_NONE, f->count, NULL, RC_WEIGHT); - tab_double (t, 3, r, TAB_NONE, - f->count / ft->total_cases * 100.0, NULL, RC_OTHER); - tab_text (t, 4, r, TAB_NONE, _("Missing")); - r++; + if (!missing) + missing = pivot_category_create_group (variable->root, N_("Missing")); + int var_idx = pivot_category_create_leaf ( + missing, pivot_value_new_var_value (vf->var, &f->values[0])); + put_freq_row (table, var_idx, f->count, + f->count / ft->total_cases * 100.0, SYSMIS, SYSMIS); } - tab_box (t, TAL_1, TAL_1, -1, TAL_1, 0, 0, 5, r); - tab_hline (t, TAL_2, 0, 5, 1); - tab_hline (t, TAL_2, 0, 5, r); - tab_joint_text (t, 0, r, 1, r, TAB_RIGHT | TAT_TITLE, _("Total")); - tab_vline (t, TAL_0, 1, r, r); - tab_double (t, 2, r, TAB_NONE, cum_freq, NULL, RC_WEIGHT); - tab_double (t, 3, r, TAB_NONE, 100.0, &F_5_1, RC_OTHER); - tab_double (t, 4, r, TAB_NONE, 100.0, &F_5_1, RC_OTHER); - - tab_title (t, "%s", var_to_string (vf->var)); - tab_submit (t); + int var_idx = pivot_category_create_leaf ( + variable->root, pivot_value_new_text (N_("Total"))); + put_freq_row (table, var_idx, cum_freq, cum_percent, SYSMIS, SYSMIS); + + pivot_table_submit (table); } /* Statistical display. */ @@ -375,20 +357,25 @@ calc_percentile (double p, double valid_cases, double x1, double x2) /* Calculates all of the percentiles for VF within FRQ. */ static void -calc_percentiles (const struct frq_proc *frq, const struct var_freqs *vf) +calc_percentiles (const struct frq_proc *frq, struct var_freqs *vf) { + if (!frq->n_percentiles) + return; + + if (!vf->percentiles) + vf->percentiles = xnmalloc (frq->n_percentiles, sizeof *vf->percentiles); + const struct freq_tab *ft = &vf->tab; - double W = ft->valid_cases; - const struct freq *f; - int percentile_idx = 0; - double rank = 0; + const double W = ft->valid_cases; + size_t idx = 0; - for (f = ft->valid; f < ft->missing; f++) + double rank = 0; + for (const struct freq *f = ft->valid; f < ft->missing; f++) { rank += f->count; - for (; percentile_idx < frq->n_percentiles; percentile_idx++) + for (; idx < frq->n_percentiles; idx++) { - struct percentile *pc = &frq->percentiles[percentile_idx]; + struct percentile *pc = &frq->percentiles[idx]; double tp; tp = (settings_get_algorithm () == ENHANCED @@ -399,16 +386,16 @@ calc_percentiles (const struct frq_proc *frq, const struct var_freqs *vf) break; if (tp + 1 < rank || f + 1 >= ft->missing) - pc->value = f->values[0].f; + vf->percentiles[idx] = f->values[0].f; else - pc->value = calc_percentile (pc->p, W, f->values[0].f, f[1].values[0].f); + vf->percentiles[idx] = calc_percentile (pc->p, W, f->values[0].f, + f[1].values[0].f); } } - for (; percentile_idx < frq->n_percentiles; percentile_idx++) - { - struct percentile *pc = &frq->percentiles[percentile_idx]; - pc->value = ft->valid[ft->n_valid - 1].values[0].f; - } + for (; idx < frq->n_percentiles; idx++) + vf->percentiles[idx] = (ft->n_valid > 0 + ? ft->valid[ft->n_valid - 1].values[0].f + : SYSMIS); } /* Returns true iff the value in struct freq F is non-missing @@ -419,7 +406,7 @@ not_missing (const void *f_, const void *v_) const struct freq *f = f_; const struct variable *v = v_; - return !var_is_value_missing (v, f->values, MV_ANY); + return !var_is_value_missing (v, f->values); } @@ -453,7 +440,7 @@ postprocess_freq_tab (const struct frq_proc *frq, struct var_freqs *vf) /* Summary statistics. */ ft->valid_cases = 0.0; - for(i = 0 ; i < ft->n_valid ; ++i ) + for(i = 0 ; i < ft->n_valid ; ++i) { f = &ft->valid[i]; ft->valid_cases += f->count; @@ -461,7 +448,7 @@ postprocess_freq_tab (const struct frq_proc *frq, struct var_freqs *vf) } ft->total_cases = ft->valid_cases ; - for(i = 0 ; i < ft->n_missing ; ++i ) + for(i = 0 ; i < ft->n_missing ; ++i) { f = &ft->missing[i]; ft->total_cases += f->count; @@ -469,19 +456,11 @@ postprocess_freq_tab (const struct frq_proc *frq, struct var_freqs *vf) } -/* Frees the frequency table for variable V. */ -static void -cleanup_freq_tab (struct var_freqs *vf) -{ - free (vf->tab.valid); - freq_hmap_destroy (&vf->tab.data, vf->width); -} - /* Add data from case C to the frequency table. */ static void calc (struct frq_proc *frq, const struct ccase *c, const struct dataset *ds) { - double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + double weight = dict_get_case_weight (dataset_dict (ds), c, &frq->warn); size_t i; for (i = 0; i < frq->n_vars; i++) @@ -499,80 +478,120 @@ calc (struct frq_proc *frq, const struct ccase *c, const struct dataset *ds) } } -/* Prepares each variable that is the target of FREQUENCIES by setting - up its hash table. */ static void -precalc (struct frq_proc *frq, struct casereader *input, struct dataset *ds) +output_splits_once (bool *need_splits, const struct dataset *ds, + const struct ccase *c) { - struct ccase *c; - size_t i; - - c = casereader_peek (input, 0); - if (c != NULL) + if (*need_splits) { output_split_file_values (ds, c); - case_unref (c); + *need_splits = false; } - - for (i = 0; i < frq->n_vars; i++) - hmap_init (&frq->vars[i].tab.data); } /* Finishes up with the variables after frequencies have been calculated. Displays statistics, percentiles, ... */ -static void -postcalc (struct frq_proc *frq, const struct dataset *ds) +static struct frq_stats_table * +postcalc (struct frq_proc *frq, const struct dataset *ds, + struct ccase *example, struct frq_stats_table *fst) { const struct dictionary *dict = dataset_dict (ds); const struct variable *wv = dict_get_weight (dict); - size_t i; - for (i = 0; i < frq->n_vars; i++) + for (size_t i = 0; i < frq->n_vars; i++) { struct var_freqs *vf = &frq->vars[i]; - postprocess_freq_tab (frq, vf); + calc_percentiles (frq, vf); + } - /* Frequencies tables. */ - if (vf->tab.n_valid + vf->tab.n_missing <= frq->max_categories) - dump_freq_table (vf, wv); + enum split_type st = dict_get_split_type (dict); + bool need_splits = true; + if (frq->n_stats) + { + if (st != SPLIT_LAYERED) + output_splits_once (&need_splits, ds, example); + fst = frq_stats_table_submit (fst, frq, dict, wv, example); + } - calc_percentiles (frq, vf); + for (size_t i = 0; i < frq->n_vars; i++) + { + struct var_freqs *vf = &frq->vars[i]; - /* Statistics. */ - if (frq->n_stats) - dump_statistics (frq, vf, wv); + /* Frequencies tables. */ + if (vf->tab.n_valid + vf->tab.n_missing <= frq->max_categories) + { + output_splits_once (&need_splits, ds, example); + dump_freq_table (vf, wv); + } if (frq->hist && var_is_numeric (vf->var) && vf->tab.n_valid > 0) { double d[FRQ_ST_count]; struct histogram *histogram; - calc_stats (vf, d); + calc_stats (frq, vf, d); - histogram = freq_tab_to_hist (frq, &vf->tab, vf->var); + histogram = freq_tab_to_hist (frq, vf); - if ( histogram) + if (histogram) { - chart_item_submit (histogram_chart_create ( - histogram->gsl_hist, var_to_string(vf->var), - vf->tab.valid_cases, - d[FRQ_ST_MEAN], - d[FRQ_ST_STDDEV], - frq->hist->draw_normal)); + output_splits_once (&need_splits, ds, example); + chart_submit (histogram_chart_create ( + histogram->gsl_hist, var_to_string(vf->var), + vf->tab.valid_cases, + d[FRQ_ST_MEAN], + d[FRQ_ST_STDDEV], + frq->hist->draw_normal)); statistic_destroy (&histogram->parent); } } if (frq->pie) - do_piechart(frq->pie, vf->var, &vf->tab); + { + output_splits_once (&need_splits, ds, example); + do_piechart(frq->pie, vf->var, &vf->tab); + } if (frq->bar) - do_barchart(frq->bar, &vf->var, &vf->tab); + { + output_splits_once (&need_splits, ds, example); + do_barchart(frq->bar, &vf->var, &vf->tab); + } + + free (vf->tab.valid); + freq_hmap_destroy (&vf->tab.data, vf->width); + } - cleanup_freq_tab (vf); + return fst; +} + +static void +frq_run (struct frq_proc *frq, struct dataset *ds) +{ + struct frq_stats_table *fst = NULL; + struct casegrouper *grouper = casegrouper_create_splits (proc_open (ds), + dataset_dict (ds)); + struct casereader *group; + while (casegrouper_get_next_group (grouper, &group)) + { + for (size_t i = 0; i < frq->n_vars; i++) + hmap_init (&frq->vars[i].tab.data); + + struct ccase *example = casereader_peek (group, 0); + + struct ccase *c; + for (; (c = casereader_read (group)) != NULL; case_unref (c)) + calc (frq, c, ds); + fst = postcalc (frq, ds, example, fst); + casereader_destroy (group); + + case_unref (example); } + frq_stats_table_destroy (fst); + casegrouper_destroy (grouper); + proc_commit (ds); } int @@ -601,14 +620,13 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) int hi_pcnt = INT_MIN; int hi_norm = FRQ_NONORMAL; - frq.pool = pool_create (); frq.sort = FRQ_AVALUE; frq.vars = NULL; frq.n_vars = 0; - - frq.stats = BIT_INDEX (FRQ_ST_MEAN) - | BIT_INDEX (FRQ_ST_STDDEV) + + frq.stats = BIT_INDEX (FRQ_ST_MEAN) + | BIT_INDEX (FRQ_ST_STDDEV) | BIT_INDEX (FRQ_ST_MINIMUM) | BIT_INDEX (FRQ_ST_MAXIMUM); @@ -618,18 +636,18 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) frq.percentiles = NULL; frq.n_percentiles = 0; - frq.n_show_percentiles = 0; frq.hist = NULL; frq.pie = NULL; frq.bar = NULL; + frq.warn = true; /* Accept an optional, completely pointless "/VARIABLES=" */ lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "VARIABLES")) { - if (! lex_force_match (lexer, T_EQUALS) ) + if (! lex_force_match (lexer, T_EQUALS)) goto error; } @@ -639,7 +657,7 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) PV_NO_DUPLICATE)) goto error; - frq.vars = xzalloc (frq.n_vars * sizeof (*frq.vars)); + frq.vars = xcalloc (frq.n_vars, sizeof (*frq.vars)); for (i = 0; i < frq.n_vars; ++i) { frq.vars[i].var = vars[i]; @@ -652,11 +670,11 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) if (lex_match_id (lexer, "STATISTICS")) { - frq.stats = BIT_INDEX (FRQ_ST_MEAN) - | BIT_INDEX (FRQ_ST_STDDEV) + frq.stats = BIT_INDEX (FRQ_ST_MEAN) + | BIT_INDEX (FRQ_ST_STDDEV) | BIT_INDEX (FRQ_ST_MINIMUM) | BIT_INDEX (FRQ_ST_MAXIMUM); - + frq.n_stats = 4; if (lex_match (lexer, T_EQUALS)) @@ -670,8 +688,8 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) { if (lex_match_id (lexer, "DEFAULT")) { - frq.stats = BIT_INDEX (FRQ_ST_MEAN) - | BIT_INDEX (FRQ_ST_STDDEV) + frq.stats = BIT_INDEX (FRQ_ST_MEAN) + | BIT_INDEX (FRQ_ST_STDDEV) | BIT_INDEX (FRQ_ST_MINIMUM) | BIT_INDEX (FRQ_ST_MAXIMUM); @@ -704,7 +722,7 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "VARIANCE")) { - frq.stats |= BIT_INDEX (FRQ_ST_MEAN); + frq.stats |= BIT_INDEX (FRQ_ST_VARIANCE); frq.n_stats++; } else if (lex_match_id (lexer, "KURTOSIS")) @@ -773,14 +791,13 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) if (lex_force_num (lexer)) { frq.percentiles = - xrealloc (frq.percentiles, + xrealloc (frq.percentiles, (frq.n_percentiles + 1) * sizeof (*frq.percentiles)); frq.percentiles[frq.n_percentiles].p = lex_number (lexer) / 100.0; frq.percentiles[frq.n_percentiles].show = true; lex_get (lexer); frq.n_percentiles++; - frq.n_show_percentiles++; } else { @@ -806,7 +823,7 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) else if (lex_match_id (lexer, "LIMIT")) { if (!lex_force_match (lexer, T_LPAREN) - || !lex_force_int (lexer)) + || !lex_force_int_range (lexer, "LIMIT", 0, INT_MAX)) goto error; frq.max_categories = lex_integer (lexer); @@ -842,15 +859,14 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) { lex_match (lexer, T_EQUALS); - if (lex_force_int (lexer)) + if (lex_force_int_range (lexer, "NTILES", 0, INT_MAX)) { - int i; int n = lex_integer (lexer); lex_get (lexer); - for (i = 0; i < n + 1; ++i) + for (int i = 0; i < n + 1; ++i) { frq.percentiles = - xrealloc (frq.percentiles, + xrealloc (frq.percentiles, (frq.n_percentiles + 1) * sizeof (*frq.percentiles)); frq.percentiles[frq.n_percentiles].p = @@ -858,7 +874,6 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) frq.percentiles[frq.n_percentiles].show = true; frq.n_percentiles++; - frq.n_show_percentiles++; } } else @@ -906,13 +921,9 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) hi_scale = FRQ_FREQ; if (lex_match (lexer, T_LPAREN)) { - if (lex_force_int (lexer)) + if (lex_force_int_range (lexer, "FREQ", 1, INT_MAX)) { hi_freq = lex_integer (lexer); - if (hi_freq <= 0) - { - lex_error (lexer, _("Histogram frequency must be greater than zero.")); - } lex_get (lexer); if (! lex_force_match (lexer, T_RPAREN)) goto error; @@ -924,13 +935,9 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) hi_scale = FRQ_PERCENT; if (lex_match (lexer, T_LPAREN)) { - if (lex_force_int (lexer)) + if (lex_force_int_range (lexer, "PERCENT", 1, INT_MAX)) { hi_pcnt = lex_integer (lexer); - if (hi_pcnt <= 0) - { - lex_error (lexer, _("Histogram percentage must be greater than zero.")); - } lex_get (lexer); if (! lex_force_match (lexer, T_RPAREN)) goto error; @@ -1046,7 +1053,7 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "FREQ")) { - if ( lex_match (lexer, T_LPAREN)) + if (lex_match (lexer, T_LPAREN)) { if (lex_force_num (lexer)) { @@ -1060,7 +1067,7 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "PERCENT")) { - if ( lex_match (lexer, T_LPAREN)) + if (lex_match (lexer, T_LPAREN)) { if (lex_force_num (lexer)) { @@ -1116,15 +1123,14 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) if (frq.stats & BIT_INDEX (FRQ_ST_MEDIAN)) { frq.percentiles = - xrealloc (frq.percentiles, + xrealloc (frq.percentiles, (frq.n_percentiles + 1) * sizeof (*frq.percentiles)); - + frq.percentiles[frq.n_percentiles].p = 0.50; - frq.percentiles[frq.n_percentiles].show = true; + frq.percentiles[frq.n_percentiles].show = false; frq.n_percentiles++; - frq.n_show_percentiles++; } @@ -1149,24 +1155,24 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) msg (SE, _("%s for histogram must be greater than or equal to %s, " "but %s was specified as %.15g and %s as %.15g. " "%s and %s will be ignored."), - "MAX", "MIN", - "MIN", hist->x_min, + "MAX", "MIN", + "MIN", hist->x_min, "MAX", hist->x_max, "MIN", "MAX"); hist->x_min = hist->x_max = SYSMIS; } frq.percentiles = - xrealloc (frq.percentiles, + xrealloc (frq.percentiles, (frq.n_percentiles + 2) * sizeof (*frq.percentiles)); - + frq.percentiles[frq.n_percentiles].p = 0.25; frq.percentiles[frq.n_percentiles].show = false; frq.percentiles[frq.n_percentiles + 1].p = 0.75; frq.percentiles[frq.n_percentiles + 1].show = false; - + frq.n_percentiles+=2; } @@ -1193,8 +1199,8 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) { msg (SE, _("%s for pie chart must be greater than or equal to %s, " "but %s was specified as %.15g and %s as %.15g. " - "%s and %s will be ignored."), - "MAX", "MIN", + "%s and %s will be ignored."), + "MAX", "MIN", "MIN", pie->x_min, "MAX", pie->x_max, "MIN", "MAX"); @@ -1207,78 +1213,65 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds) int i,o; double previous_p = -1; qsort (frq.percentiles, frq.n_percentiles, - sizeof (*frq.percentiles), + sizeof (*frq.percentiles), ptile_3way); - frq.n_show_percentiles = 0; for (i = o = 0; i < frq.n_percentiles; ++i) { if (frq.percentiles[i].p != previous_p) { frq.percentiles[o].p = frq.percentiles[i].p; frq.percentiles[o].show = frq.percentiles[i].show; - if (frq.percentiles[i].show) - frq.n_show_percentiles++; o++; } else if (frq.percentiles[i].show && !frq.percentiles[o].show) { frq.percentiles[o].show = true; - frq.n_show_percentiles++; } previous_p = frq.percentiles[i].p; } frq.n_percentiles = o; - } - { - struct casegrouper *grouper; - struct casereader *group; - bool ok; - - grouper = casegrouper_create_splits (proc_open (ds), dataset_dict (ds)); - while (casegrouper_get_next_group (grouper, &group)) - { - struct ccase *c; - precalc (&frq, group, ds); - - for (; (c = casereader_read (group)) != NULL; case_unref (c)) - calc (&frq, c, ds); - postcalc (&frq, ds); - casereader_destroy (group); - } - ok = casegrouper_destroy (grouper); - ok = proc_commit (ds) && ok; + frq.median_idx = SIZE_MAX; + for (i = 0; i < frq.n_percentiles; i++) + if (frq.percentiles[i].p == 0.5) + { + frq.median_idx = i; + break; + } } + frq_run (&frq, ds); free (vars); + for (size_t i = 0; i < frq.n_vars; i++) + free (frq.vars[i].percentiles); free (frq.vars); free (frq.bar); free (frq.pie); free (frq.hist); free (frq.percentiles); - pool_destroy (frq.pool); return CMD_SUCCESS; error: free (vars); + for (size_t i = 0; i < frq.n_vars; i++) + free (frq.vars[i].percentiles); free (frq.vars); free (frq.bar); free (frq.pie); free (frq.hist); free (frq.percentiles); - pool_destroy (frq.pool); return CMD_FAILURE; } static double -calculate_iqr (const struct frq_proc *frq) +calculate_iqr (const struct frq_proc *frq, const struct var_freqs *vf) { double q1 = SYSMIS; double q3 = SYSMIS; @@ -1291,9 +1284,9 @@ calculate_iqr (const struct frq_proc *frq) struct percentile *pc = &frq->percentiles[i]; if (fabs (0.25 - pc->p) < DBL_EPSILON) - q1 = pc->value; + q1 = vf->percentiles[i]; else if (fabs (0.75 - pc->p) < DBL_EPSILON) - q3 = pc->value; + q3 = vf->percentiles[i]; } return q1 == SYSMIS || q3 == SYSMIS ? SYSMIS : q3 - q1; @@ -1304,7 +1297,7 @@ chart_includes_value (const struct frq_chart *chart, const struct variable *var, const union value *value) { - if (!chart->include_missing && var_is_value_missing (var, value, MV_ANY)) + if (!chart->include_missing && var_is_value_missing (var, value)) return false; if (var_is_numeric (var) @@ -1317,24 +1310,17 @@ chart_includes_value (const struct frq_chart *chart, /* Create a gsl_histogram from a freq_tab */ static struct histogram * -freq_tab_to_hist (const struct frq_proc *frq, const struct freq_tab *ft, - const struct variable *var) +freq_tab_to_hist (const struct frq_proc *frq, const struct var_freqs *vf) { - double x_min, x_max, valid_freq; - int i; - double bin_width; - struct histogram *histogram; - double iqr; - /* Find out the extremes of the x value, within the range to be included in the histogram, and sum the total frequency of those values. */ - x_min = DBL_MAX; - x_max = -DBL_MAX; - valid_freq = 0; - for (i = 0; i < ft->n_valid; i++) + double x_min = DBL_MAX; + double x_max = -DBL_MAX; + double valid_freq = 0; + for (int i = 0; i < vf->tab.n_valid; i++) { - const struct freq *f = &ft->valid[i]; - if (chart_includes_value (frq->hist, var, f->values)) + const struct freq *f = &vf->tab.valid[i]; + if (chart_includes_value (frq->hist, vf->var, f->values)) { x_min = MIN (x_min, f->values[0].f); x_max = MAX (x_max, f->values[0].f); @@ -1345,25 +1331,21 @@ freq_tab_to_hist (const struct frq_proc *frq, const struct freq_tab *ft, if (valid_freq <= 0) return NULL; - iqr = calculate_iqr (frq); - - if (iqr > 0) - /* Freedman-Diaconis' choice of bin width. */ - bin_width = 2 * iqr / pow (valid_freq, 1.0 / 3.0); + double iqr = calculate_iqr (frq, vf); - else - /* Sturges Rule */ - bin_width = (x_max - x_min) / (1 + log2 (valid_freq)); + double bin_width = + (iqr > 0 + ? 2 * iqr / pow (valid_freq, 1.0 / 3.0) /* Freedman-Diaconis. */ + : (x_max - x_min) / (1 + log2 (valid_freq))); /* Sturges */ - histogram = histogram_create (bin_width, x_min, x_max); - - if ( histogram == NULL) + struct histogram *histogram = histogram_create (bin_width, x_min, x_max); + if (histogram == NULL) return NULL; - for (i = 0; i < ft->n_valid; i++) + for (int i = 0; i < vf->tab.n_valid; i++) { - const struct freq *f = &ft->valid[i]; - if (chart_includes_value (frq->hist, var, f->values)) + const struct freq *f = &vf->tab.valid[i]; + if (chart_includes_value (frq->hist, vf->var, f->values)) histogram_add (histogram, f->values[0].f, f->count); } @@ -1384,7 +1366,7 @@ pick_cat_counts (const struct frq_chart *catchart, int n_slices = 0; int i; struct freq *slices = xnmalloc (frq_tab->n_valid + frq_tab->n_missing, sizeof *slices); - + for (i = 0; i < frq_tab->n_valid; i++) { const struct freq *f = &frq_tab->valid[i]; @@ -1393,9 +1375,9 @@ pick_cat_counts (const struct frq_chart *catchart, if (f->count < catchart->x_min) continue; - + slices[n_slices] = *f; - + n_slices++; } @@ -1405,11 +1387,11 @@ pick_cat_counts (const struct frq_chart *catchart, { const struct freq *f = &frq_tab->missing[i]; slices[n_slices].count += f->count; - + if (i == 0) slices[n_slices].values[0] = f->values[0]; } - + if (frq_tab->n_missing > 0) n_slices++; } @@ -1432,7 +1414,7 @@ pick_cat_counts_ptr (const struct frq_chart *catchart, int n_slices = 0; int i; struct freq **slices = xnmalloc (frq_tab->n_valid + frq_tab->n_missing, sizeof *slices); - + for (i = 0; i < frq_tab->n_valid; i++) { struct freq *f = &frq_tab->valid[i]; @@ -1441,9 +1423,9 @@ pick_cat_counts_ptr (const struct frq_chart *catchart, if (f->count < catchart->x_min) continue; - + slices[n_slices] = f; - + n_slices++; } @@ -1459,7 +1441,7 @@ pick_cat_counts_ptr (const struct frq_chart *catchart, } slices[n_slices]->count += f->count; - + } } @@ -1483,7 +1465,7 @@ do_piechart(const struct frq_chart *pie, const struct variable *var, msg (SW, _("Omitting pie chart for %s, which has over 50 unique values."), var_get_name (var)); else - chart_item_submit (piechart_create (var, slices, n_slices)); + chart_submit (piechart_create (var, slices, n_slices)); free (slices); } @@ -1496,10 +1478,13 @@ do_barchart(const struct frq_chart *bar, const struct variable **var, int n_slices; struct freq **slices = pick_cat_counts_ptr (bar, frq_tab, &n_slices); - chart_item_submit (barchart_create (var, 1, - (bar->y_scale == FRQ_FREQ) ? _("Count") : _("Percent"), - (bar->y_scale == FRQ_PERCENT), - slices, n_slices)); + if (n_slices < 1) + msg (SW, _("Omitting bar chart, which has no values.")); + else + chart_submit (barchart_create (var, 1, + (bar->y_scale == FRQ_FREQ) ? _("Count") : _("Percent"), + (bar->y_scale == FRQ_PERCENT), + slices, n_slices)); free (slices); } @@ -1507,7 +1492,8 @@ do_barchart(const struct frq_chart *bar, const struct variable **var, /* Calculates all the pertinent statistics for VF, putting them in array D[]. */ static void -calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]) +calc_stats (const struct frq_proc *frq, const struct var_freqs *vf, + double d[FRQ_ST_count]) { const struct freq_tab *ft = &vf->tab; double W = ft->valid_cases; @@ -1516,21 +1502,14 @@ calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]) int most_often = -1; double X_mode = SYSMIS; - /* Calculate the mode. */ + /* Calculate the mode. If there is more than one mode, we take the + smallest. */ for (f = ft->valid; f < ft->missing; f++) - { - if (most_often < f->count) - { - most_often = f->count; - X_mode = f->values[0].f; - } - else if (most_often == f->count) - { - /* A duplicate mode is undefined. - FIXME: keep track of *all* the modes. */ - X_mode = SYSMIS; - } - } + if (most_often < f->count) + { + most_often = f->count; + X_mode = f->values[0].f; + } /* Calculate moments. */ m = moments_create (MOMENT_KURTOSIS); @@ -1543,98 +1522,175 @@ calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]) moments_destroy (m); /* Formulae below are taken from _SPSS Statistical Algorithms_. */ - d[FRQ_ST_MINIMUM] = ft->valid[0].values[0].f; - d[FRQ_ST_MAXIMUM] = ft->valid[ft->n_valid - 1].values[0].f; + if (ft->n_valid > 0) + { + d[FRQ_ST_MINIMUM] = ft->valid[0].values[0].f; + d[FRQ_ST_MAXIMUM] = ft->valid[ft->n_valid - 1].values[0].f; + d[FRQ_ST_RANGE] = d[FRQ_ST_MAXIMUM] - d[FRQ_ST_MINIMUM]; + } + else + { + d[FRQ_ST_MINIMUM] = SYSMIS; + d[FRQ_ST_MAXIMUM] = SYSMIS; + d[FRQ_ST_RANGE] = SYSMIS; + } d[FRQ_ST_MODE] = X_mode; - d[FRQ_ST_RANGE] = d[FRQ_ST_MAXIMUM] - d[FRQ_ST_MINIMUM]; d[FRQ_ST_SUM] = d[FRQ_ST_MEAN] * W; d[FRQ_ST_STDDEV] = sqrt (d[FRQ_ST_VARIANCE]); d[FRQ_ST_SEMEAN] = d[FRQ_ST_STDDEV] / sqrt (W); d[FRQ_ST_SESKEWNESS] = calc_seskew (W); d[FRQ_ST_SEKURTOSIS] = calc_sekurt (W); + d[FRQ_ST_MEDIAN] = (frq->median_idx != SIZE_MAX + ? vf->percentiles[frq->median_idx] + : SYSMIS); } -/* Displays a table of all the statistics requested for variable V. */ -static void -dump_statistics (const struct frq_proc *frq, const struct var_freqs *vf, - const struct variable *wv) +static bool +all_string_variables (const struct frq_proc *frq) { - const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0; - const struct freq_tab *ft = &vf->tab; - double stat_value[FRQ_ST_count]; - struct tab_table *t; - int i, r = 2; /* N missing and N valid are always dumped */ + for (size_t i = 0; i < frq->n_vars; i++) + if (var_is_numeric (frq->vars[i].var)) + return false; - if (var_is_alpha (vf->var)) - return; - - calc_stats (vf, stat_value); - - t = tab_create (3, ((frq->stats & BIT_INDEX (FRQ_ST_MEDIAN)) ? frq->n_stats - 1 : frq->n_stats) - + frq->n_show_percentiles + 2); - - tab_set_format (t, RC_WEIGHT, wfmt); - tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ; + return true; +} + +struct frq_stats_table + { + struct pivot_table *table; + struct pivot_splits *splits; + }; - tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1); - tab_vline (t, TAL_GAP , 1, 0, tab_nr(t) - 1 ) ; +/* Displays a table of all the statistics requested. */ +static struct frq_stats_table * +frq_stats_table_create (const struct frq_proc *frq, + const struct dictionary *dict, + const struct variable *wv) +{ + if (all_string_variables (frq)) + return NULL; - for (i = 0; i < FRQ_ST_count; i++) + struct pivot_table *table = pivot_table_create (N_("Statistics")); + pivot_table_set_weight_var (table, wv); + + struct pivot_dimension *variables + = pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Variables")); + for (size_t i = 0; i < frq->n_vars; i++) + if (!var_is_alpha (frq->vars[i].var)) + pivot_category_create_leaf (variables->root, + pivot_value_new_variable (frq->vars[i].var)); + + struct pivot_dimension *statistics = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Statistics")); + struct pivot_category *n = pivot_category_create_group ( + statistics->root, N_("N")); + pivot_category_create_leaves (n, + N_("Valid"), PIVOT_RC_COUNT, + N_("Missing"), PIVOT_RC_COUNT); + for (int i = 0; i < FRQ_ST_count; i++) + if (frq->stats & BIT_INDEX (i)) + pivot_category_create_leaf (statistics->root, + pivot_value_new_text (st_name[i])); + struct pivot_category *percentiles = NULL; + for (size_t i = 0; i < frq->n_percentiles; i++) { - if (FRQ_ST_MEDIAN == i) - continue; + const struct percentile *pc = &frq->percentiles[i]; - if (frq->stats & BIT_INDEX (i)) - { - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, - gettext (st_name[i])); - - if (vf->tab.n_valid <= 0 && r >= 2) - tab_text (t, 2, r, 0, "."); - else - tab_double (t, 2, r, TAB_NONE, stat_value[i], NULL, RC_OTHER); - r++; - } + if (!pc->show) + continue; + + if (!percentiles) + percentiles = pivot_category_create_group ( + statistics->root, N_("Percentiles")); + pivot_category_create_leaf (percentiles, pivot_value_new_integer ( + pc->p * 100.0)); } - tab_text (t, 0, 0, TAB_LEFT | TAT_TITLE, _("N")); - tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid")); - tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing")); + struct pivot_splits *splits = pivot_splits_create (table, PIVOT_AXIS_COLUMN, + dict); - tab_double (t, 2, 0, TAB_NONE, ft->valid_cases, NULL, RC_WEIGHT); - tab_double (t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, NULL, RC_WEIGHT); + struct frq_stats_table *fst = xmalloc (sizeof *fst); + *fst = (struct frq_stats_table) { .table = table, .splits = splits }; + return fst; +} - for (i = 0; i < frq->n_percentiles; i++) +static struct frq_stats_table * +frq_stats_table_submit (struct frq_stats_table *fst, + const struct frq_proc *frq, + const struct dictionary *dict, + const struct variable *wv, + const struct ccase *example) +{ + if (!fst) { - const struct percentile *pc = &frq->percentiles[i]; + fst = frq_stats_table_create (frq, dict, wv); + if (!fst) + return NULL; + } + pivot_splits_new_split (fst->splits, example); - if (!pc->show) + int var_idx = 0; + for (size_t i = 0; i < frq->n_vars; i++) + { + struct var_freqs *vf = &frq->vars[i]; + if (var_is_alpha (vf->var)) continue; - if ( i == 0 ) - { - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles")); - } + const struct freq_tab *ft = &vf->tab; - if (vf->tab.n_valid <= 0) - { - tab_text (t, 2, r, 0, "."); - ++r; - continue; - } + int row = 0; + pivot_splits_put2 (fst->splits, fst->table, var_idx, row++, + pivot_value_new_number (ft->valid_cases)); + pivot_splits_put2 (fst->splits, fst->table, var_idx, row++, + pivot_value_new_number ( + ft->total_cases - ft->valid_cases)); - if (pc->p == 0.5) - tab_text (t, 1, r, TAB_LEFT, _("50 (Median)")); - else - tab_double (t, 1, r, TAB_LEFT, pc->p * 100, NULL, RC_INTEGER); - tab_double (t, 2, r, TAB_NONE, pc->value, - var_get_print_format (vf->var), RC_OTHER); + double stat_values[FRQ_ST_count]; + calc_stats (frq, vf, stat_values); + for (int j = 0; j < FRQ_ST_count; j++) + { + if (!(frq->stats & BIT_INDEX (j))) + continue; + + union value v = { .f = vf->tab.n_valid ? stat_values[j] : SYSMIS }; + struct pivot_value *pv + = (j == FRQ_ST_MODE || j == FRQ_ST_MINIMUM || j == FRQ_ST_MAXIMUM + ? pivot_value_new_var_value (vf->var, &v) + : pivot_value_new_number (v.f)); + pivot_splits_put2 (fst->splits, fst->table, var_idx, row++, pv); + } - ++r; - } + for (size_t j = 0; j < frq->n_percentiles; j++) + { + const struct percentile *pc = &frq->percentiles[j]; + if (!pc->show) + continue; + + union value v = { + .f = vf->tab.n_valid ? vf->percentiles[j] : SYSMIS + }; + pivot_splits_put2 (fst->splits, fst->table, var_idx, row++, + pivot_value_new_var_value (vf->var, &v)); + } - tab_title (t, "%s", var_to_string (vf->var)); + var_idx++; + } - tab_submit (t); + if (!fst->splits) + { + frq_stats_table_destroy (fst); + return NULL; + } + return fst; } +static void +frq_stats_table_destroy (struct frq_stats_table *fst) +{ + if (!fst) + return; + + pivot_table_submit (fst->table); + pivot_splits_destroy (fst->splits); + free (fst); +}