X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Ffrequencies.q;h=9eb171724565e65338b51d1c79de87256acabbbd;hb=2c9a5954d98d4dd508d8fbf2496f2bb819527a46;hp=e46bb1dc3839415d9d03bde87de3367b47e4321c;hpb=14e7292894533c5491a774a2d749386362660812;p=pspp diff --git a/src/frequencies.q b/src/frequencies.q index e46bb1dc38..9eb1717245 100644 --- a/src/frequencies.q +++ b/src/frequencies.q @@ -24,20 +24,21 @@ */ #include -#include +#include "error.h" #include #include #include "alloc.h" #include "bitvector.h" +#include "case.h" #include "hash.h" #include "pool.h" #include "command.h" #include "lexer.h" +#include "moments.h" #include "error.h" #include "algorithm.h" #include "magic.h" #include "misc.h" -#include "stats.h" #include "output.h" #include "som.h" #include "str.h" @@ -45,6 +46,8 @@ #include "value-labels.h" #include "var.h" #include "vfm.h" +#include "settings.h" +#include "chart.h" #include "debug-print.h" @@ -61,6 +64,9 @@ barchart(ba_)=:minimum(d:min), :maximum(d:max), scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"); + piechart(pie_)=:minimum(d:min), + :maximum(d:max), + missing:missing/!nomissing; histogram(hi_)=:minimum(d:min), :maximum(d:max), scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"), @@ -109,10 +115,22 @@ static struct frq_info st_name[frq_n_stats + 1] = }; /* Percentiles to calculate. */ -static double *percentiles; -static double *percentile_values; + +struct percentile +{ + double p; /* the %ile to be calculated */ + double value; /* the %ile's value */ + double x1; /* The datum value <= the percentile */ + double x2; /* The datum value >= the percentile */ + int flag; + int flag2; /* Set to 1 if this percentile value has been found */ +}; + +static struct percentile *percentiles; static int n_percentiles; +static int implicit_50th ; + /* Groups of statistics. */ #define BI BIT_INDEX #define frq_default \ @@ -134,6 +152,7 @@ enum GFT_NONE, /* Don't draw graphs. */ GFT_BAR, /* Draw bar charts. */ GFT_HIST, /* Draw histograms. */ + GFT_PIE, /* Draw piechart */ GFT_HBAR /* Draw bar charts or histograms at our discretion. */ }; @@ -141,7 +160,8 @@ enum static struct cmd_frequencies cmd; /* Summary of the barchart, histogram, and hbar subcommands. */ -static int chart; /* NONE/BAR/HIST/HBAR. */ +/* FIXME: These should not be mututally exclusive */ +static int chart; /* NONE/BAR/HIST/HBAR/PIE. */ static double min, max; /* Minimum, maximum on y axis. */ static int format; /* FREQ/PERCENT: Scaling of y axis. */ static double scale, incr; /* FIXME */ @@ -160,6 +180,8 @@ static struct pool *gen_pool; /* General mode. */ static void determine_charts (void); +static void calc_stats (struct variable * v, double d[frq_n_stats]); + static void precalc (void *); static int calc (struct ccase *, void *); static void postcalc (void *); @@ -202,7 +224,6 @@ internal_cmd_frequencies (void) int i; n_percentiles = 0; - percentile_values = NULL; percentiles = NULL; n_variables = 0; @@ -211,7 +232,6 @@ internal_cmd_frequencies (void) for (i = 0; i < dict_get_var_cnt (default_dict); i++) dict_get_var(default_dict, i)->p.frq.used = 0; - lex_match_id ("FREQUENCIES"); if (!parse_frequencies (&cmd)) return CMD_FAILURE; @@ -255,7 +275,8 @@ internal_cmd_frequencies (void) static void determine_charts (void) { - int count = (!!cmd.sbc_histogram) + (!!cmd.sbc_barchart) + (!!cmd.sbc_hbar); + int count = (!!cmd.sbc_histogram) + (!!cmd.sbc_barchart) + + (!!cmd.sbc_hbar) + (!!cmd.sbc_piechart); if (!count) { @@ -273,6 +294,8 @@ determine_charts (void) chart = GFT_HIST; else if (cmd.sbc_barchart) chart = GFT_BAR; + else if (cmd.sbc_piechart) + chart = GFT_PIE; else chart = GFT_HBAR; @@ -316,7 +339,7 @@ determine_charts (void) format = FRQ_PERCENT; scale = cmd.ba_pcnt; } - if (cmd.hi_norm) + if (cmd.hi_norm != FRQ_NONORMAL ) normal = 1; if (cmd.hi_incr == FRQ_INCREMENT) incr = cmd.hi_inc; @@ -359,22 +382,24 @@ calc (struct ccase *c, void *aux UNUSED) { double weight; int i; + int bad_warn = 1; - weight = dict_get_case_weight (default_dict, c); + weight = dict_get_case_weight (default_dict, c, &bad_warn); for (i = 0; i < n_variables; i++) { struct variable *v = v_variables[i]; - union value *val = &c->data[v->fv]; + const union value *val = case_data (c, v->fv); struct freq_tab *ft = &v->p.frq.tab; switch (v->p.frq.tab.mode) { case FRQM_GENERAL: { + /* General mode. */ struct freq **fpp = (struct freq **) hsh_probe (ft->data, val); - + if (*fpp != NULL) (*fpp)->c += weight; else @@ -491,7 +516,40 @@ postcalc (void *aux UNUSED) if (n_stats) dump_statistics (v, !dumped_freq_tab); + + if ( chart == GFT_HIST) + { + struct chart ch; + double d[frq_n_stats]; + struct frequencies_proc *frq = &v->p.frq; + + struct normal_curve norm; + norm.N = frq->tab.total_cases ; + + calc_stats(v,d); + norm.mean = d[frq_mean]; + norm.stddev = d[frq_stddev]; + + chart_initialise(&ch); + draw_histogram(&ch, v_variables[i], "HISTOGRAM",&norm,normal); + chart_finalise(&ch); + } + + + if ( chart == GFT_PIE) + { + struct chart ch; + + chart_initialise(&ch); + + draw_piechart(&ch, v_variables[i]); + + chart_finalise(&ch); + } + + cleanup_freq_tab (v); + } } @@ -549,7 +607,7 @@ postprocess_freq_tab (struct variable *v) data = hsh_data (ft->data); /* Copy dereferenced data into freqs. */ - freqs = xmalloc (count* sizeof *freqs); + freqs = xmalloc (count * sizeof *freqs); for (i = 0; i < count; i++) { struct freq *f = data[i]; @@ -567,15 +625,21 @@ postprocess_freq_tab (struct variable *v) sort (ft->missing, ft->n_missing, sizeof *ft->missing, compare, v); /* Summary statistics. */ - ft->total_cases = ft->valid_cases = 0.0; - for (f = ft->valid; f < ft->valid + ft->n_valid; f++) + ft->valid_cases = 0.0; + for(i = 0 ; i < ft->n_valid ; ++i ) { - ft->total_cases += f->c; + f = &ft->valid[i]; + ft->valid_cases += f->c; + + } - if ((v->type != NUMERIC || f->v.f != SYSMIS) - && (cmd.miss != FRQ_EXCLUDE || !is_user_missing (&f->v, v))) - ft->valid_cases += f->c; + ft->total_cases = ft->valid_cases ; + for(i = 0 ; i < ft->n_missing ; ++i ) + { + f = &ft->missing[i]; + ft->total_cases += f->c; } + } /* Frees the frequency table for variable V. */ @@ -762,24 +826,20 @@ add_percentile (double x) int i; for (i = 0; i < n_percentiles; i++) - if (x <= percentiles[i]) + if (x <= percentiles[i].p) break; - if (i >= n_percentiles || tokval != percentiles[i]) + + if (i >= n_percentiles || tokval != percentiles[i].p) { percentiles = pool_realloc (int_pool, percentiles, - (n_percentiles + 1) * sizeof *percentiles); - percentile_values - = pool_realloc (int_pool, percentile_values, - (n_percentiles + 1) * sizeof *percentile_values); - if (i < n_percentiles) - { + (n_percentiles + 1) * sizeof (struct percentile )); + + if (i < n_percentiles) memmove (&percentiles[i + 1], &percentiles[i], - (n_percentiles - i) * sizeof *percentiles); - memmove (&percentile_values[i + 1], &percentile_values[i], - (n_percentiles - i) * sizeof *percentile_values); - } - percentiles[i] = x; + (n_percentiles - i) * sizeof (struct percentile) ); + + percentiles[i].p = x; n_percentiles++; } } @@ -798,10 +858,9 @@ frq_custom_percentiles (struct cmd_frequencies *cmd UNUSED) do { - if (tokval <= 0 || tokval >= 100) + if (tokval < 0 || tokval > 100) { - msg (SE, _("Percentiles must be greater than " - "0 and less than 100.")); + msg (SE, _("Percentiles must be between 0 and 100.")); return 0; } @@ -1088,6 +1147,7 @@ dump_full (struct variable * v) tab_title (t, 1, "%s: %s", v->name, v->label ? v->label : ""); tab_submit (t); + } /* Sets the widths of all the columns and heights of all the rows in @@ -1170,36 +1230,121 @@ static void calc_stats (struct variable * v, double d[frq_n_stats]) { double W = v->p.frq.tab.valid_cases; - double X_bar, X_mode, M2, M3, M4; - struct freq *f; + struct moments *m; + struct freq *f=0; int most_often; + double X_mode; - double cum_total; + double rank; int i = 0; - double previous_value; - - /* Calculate the mean. */ - X_bar = 0.0; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) - X_bar += f->v.f * f->c; - X_bar /= W; + int idx; + double *median_value; /* Calculate percentiles. */ - cum_total = 0; - previous_value = SYSMIS; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) + + /* If the 50th percentile was not explicitly requested then we must + calculate it anyway --- it's the median */ + median_value = 0 ; + for (i = 0; i < n_percentiles; i++) { - cum_total += f->c ; - for (; i < n_percentiles; i++) + if (percentiles[i].p == 0.5) + { + median_value = &percentiles[i].value; + break; + } + } + + if ( 0 == median_value ) + { + add_percentile (0.5); + implicit_50th = 1; + } + + for (i = 0; i < n_percentiles; i++) + { + percentiles[i].flag = 0; + percentiles[i].flag2 = 0; + } + + rank = 0; + for (idx = 0; idx < v->p.frq.tab.n_valid; ++idx) + { + static double prev_value = SYSMIS; + f = &v->p.frq.tab.valid[idx]; + rank += f->c ; + for (i = 0; i < n_percentiles; i++) { - if (cum_total / v->p.frq.tab.valid_cases < percentiles[i]) - break; + double tp; + if ( percentiles[i].flag2 ) continue ; + + if ( get_algorithm() != COMPATIBLE ) + tp = + (v->p.frq.tab.valid_cases - 1) * percentiles[i].p; + else + tp = + (v->p.frq.tab.valid_cases + 1) * percentiles[i].p - 1; - percentile_values[i] = previous_value; + if ( percentiles[i].flag ) + { + percentiles[i].x2 = f->v.f; + percentiles[i].x1 = prev_value; + percentiles[i].flag2 = 1; + continue; + } + + if (rank > tp ) + { + if ( f->c > 1 && rank - (f->c - 1) > tp ) + { + percentiles[i].x2 = percentiles[i].x1 = f->v.f; + percentiles[i].flag2 = 1; + } + else + { + percentiles[i].flag=1; + } + + continue; + } } - previous_value = f->v.f; + prev_value = f->v.f; } + for (i = 0; i < n_percentiles; i++) + { + /* Catches the case when p == 100% */ + if ( ! percentiles[i].flag2 ) + percentiles[i].x1 = percentiles[i].x2 = f->v.f; + + /* + printf("percentile %d (p==%.2f); X1 = %g; X2 = %g\n", + i,percentiles[i].p,percentiles[i].x1,percentiles[i].x2); + */ + } + + for (i = 0; i < n_percentiles; i++) + { + struct freq_tab *ft = &v->p.frq.tab; + double s; + + double dummy; + if ( get_algorithm() != COMPATIBLE ) + { + s = modf((ft->valid_cases - 1) * percentiles[i].p , &dummy); + } + else + { + s = modf((ft->valid_cases + 1) * percentiles[i].p -1, &dummy); + } + + percentiles[i].value = percentiles[i].x1 + + ( percentiles[i].x2 - percentiles[i].x1) * s ; + + if ( percentiles[i].p == 0.50) + median_value = &percentiles[i].value; + } + + /* Calculate the mode. */ most_often = -1; X_mode = SYSMIS; @@ -1218,56 +1363,27 @@ calc_stats (struct variable * v, double d[frq_n_stats]) } } - /* Calculate moments about the mean. */ - M2 = M3 = M4 = 0.0; + /* Calculate moments. */ + m = moments_create (MOMENT_KURTOSIS); for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) - { - double dev = f->v.f - X_bar; - double tmp; - tmp = dev * dev; - M2 += f->c * tmp; - tmp *= dev; - M3 += f->c * tmp; - tmp *= dev; - M4 += f->c * tmp; - } - + moments_pass_one (m, f->v.f, f->c); + for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) + moments_pass_two (m, f->v.f, f->c); + moments_calculate (m, NULL, &d[frq_mean], &d[frq_variance], + &d[frq_skew], &d[frq_kurt]); + moments_destroy (m); + /* Formulas below are taken from _SPSS Statistical Algorithms_. */ d[frq_min] = v->p.frq.tab.valid[0].v.f; d[frq_max] = v->p.frq.tab.valid[v->p.frq.tab.n_valid - 1].v.f; d[frq_mode] = X_mode; d[frq_range] = d[frq_max] - d[frq_min]; - d[frq_median] = SYSMIS; - d[frq_mean] = X_bar; - d[frq_sum] = X_bar * W; - d[frq_variance] = M2 / (W - 1); + d[frq_median] = *median_value; + d[frq_sum] = d[frq_mean] * W; d[frq_stddev] = sqrt (d[frq_variance]); d[frq_semean] = d[frq_stddev] / sqrt (W); - if (W >= 3.0 && d[frq_variance] > 0) - { - double S = d[frq_stddev]; - d[frq_skew] = (W * M3 / ((W - 1.0) * (W - 2.0) * S * S * S)); - d[frq_seskew] = sqrt (6.0 * W * (W - 1.0) - / ((W - 2.0) * (W + 1.0) * (W + 3.0))); - } - else - { - d[frq_skew] = d[frq_seskew] = SYSMIS; - } - if (W >= 4.0 && d[frq_variance] > 0) - { - double S2 = d[frq_variance]; - double SE_g1 = d[frq_seskew]; - - d[frq_kurt] = ((W * (W + 1.0) * M4 - 3.0 * M2 * M2 * (W - 1.0)) - / ((W - 1.0) * (W - 2.0) * (W - 3.0) * S2 * S2)); - d[frq_sekurt] = sqrt ((4.0 * (W * W - 1.0) * SE_g1 * SE_g1) - / ((W - 3.0) * (W + 5.0))); - } - else - { - d[frq_kurt] = d[frq_sekurt] = SYSMIS; - } + d[frq_seskew] = calc_seskew (W); + d[frq_sekurt] = calc_sekurt (W); } /* Displays a table of all the statistics requested for variable V. */ @@ -1278,6 +1394,11 @@ dump_statistics (struct variable * v, int show_varname) struct tab_table *t; int i, r; + int n_explicit_percentiles = n_percentiles; + + if ( implicit_50th && n_percentiles > 0 ) + --n_percentiles; + if (v->type == ALPHA) return; if (v->p.frq.tab.n_valid == 0) @@ -1288,29 +1409,45 @@ dump_statistics (struct variable * v, int show_varname) } calc_stats (v, stat_value); - t = tab_create (2, n_stats + n_percentiles, 0); + t = tab_create (3, n_stats + n_explicit_percentiles + 2, 0); tab_dim (t, tab_natural_dimensions); - tab_vline (t, TAL_1 | TAL_SPACING, 1, 0, n_stats - 1); - for (i = r = 0; i < frq_n_stats; i++) + + tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ; + + + tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1); + tab_vline (t, TAL_1 | TAL_SPACING , 1, 0, tab_nr(t) - 1 ) ; + + r=2; /* N missing and N valid are always dumped */ + + for (i = 0; i < frq_n_stats; i++) if (stats & BIT_INDEX (i)) { tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, gettext (st_name[i].s10)); - tab_float (t, 1, r, TAB_NONE, stat_value[i], 11, 3); + tab_float (t, 2, r, TAB_NONE, stat_value[i], 11, 3); r++; } - for (i = 0; i < n_percentiles; i++, r++) - { - struct string ds; + tab_text (t, 0, 0, TAB_LEFT | TAT_TITLE, _("N")); + tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid")); + tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing")); - ds_init (gen_pool, &ds, 20); - ds_printf (&ds, "%s %d", _("Percentile"), (int) (percentiles[i] * 100)); + tab_float(t, 2, 0, TAB_NONE, v->p.frq.tab.valid_cases, 11, 0); + tab_float(t, 2, 1, TAB_NONE, + v->p.frq.tab.total_cases - v->p.frq.tab.valid_cases, 11, 0); - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, ds.string); - tab_float (t, 1, r, TAB_NONE, percentile_values[i], 11, 3); - ds_destroy (&ds); + for (i = 0; i < n_explicit_percentiles; i++, r++) + { + if ( i == 0 ) + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles")); + } + + tab_float (t, 1, r, TAB_LEFT, percentiles[i].p * 100, 3, 0 ); + tab_float (t, 2, r, TAB_NONE, percentiles[i].value, 11, 3); + } tab_columns (t, SOM_COL_DOWN, 1); @@ -1323,7 +1460,8 @@ dump_statistics (struct variable * v, int show_varname) } else tab_flags (t, SOMF_NO_TITLE); - + + tab_submit (t); } /*