From 171bd1341565bf922cd3361e564d3d02b18c9547 Mon Sep 17 00:00:00 2001 From: John Darrington Date: Mon, 21 Apr 2014 17:03:44 +0200 Subject: [PATCH] FREQUENCIES: Replaced frequencies.q with frequencies.c --- src/language/stats/.gitignore | 1 - src/language/stats/automake.mk | 4 +- .../stats/{frequencies.q => frequencies.c} | 1812 +++++++++-------- 3 files changed, 995 insertions(+), 822 deletions(-) rename src/language/stats/{frequencies.q => frequencies.c} (53%) diff --git a/src/language/stats/.gitignore b/src/language/stats/.gitignore index 10f6baf208..48d8eaa3ca 100644 --- a/src/language/stats/.gitignore +++ b/src/language/stats/.gitignore @@ -1,2 +1 @@ crosstabs.c -frequencies.c diff --git a/src/language/stats/automake.mk b/src/language/stats/automake.mk index cdd2b8dc82..d923a7d134 100644 --- a/src/language/stats/automake.mk +++ b/src/language/stats/automake.mk @@ -3,8 +3,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/language/stats src_language_stats_built_sources = \ - src/language/stats/crosstabs.c \ - src/language/stats/frequencies.c + src/language/stats/crosstabs.c language_stats_sources = \ src/language/stats/aggregate.c \ @@ -23,6 +22,7 @@ language_stats_sources = \ src/language/stats/flip.c \ src/language/stats/freq.c \ src/language/stats/freq.h \ + src/language/stats/frequencies.c \ src/language/stats/friedman.c \ src/language/stats/friedman.h \ src/language/stats/glm.c \ diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.c similarity index 53% rename from src/language/stats/frequencies.q rename to src/language/stats/frequencies.c index 8a8ac6c4bd..1f8290cf2c 100644 --- a/src/language/stats/frequencies.q +++ b/src/language/stats/frequencies.c @@ -1,38 +1,44 @@ -/* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2007, 2009, 2010, 2011 Free Software Foundation, Inc. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . */ +/* + PSPP - a program for statistical analysis. + Copyright (C) 1997-9, 2000, 2007, 2009, 2010, 2011, 2014 Free Software Foundation, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ #include - -#include #include #include + #include "data/case.h" #include "data/casegrouper.h" #include "data/casereader.h" #include "data/dataset.h" #include "data/dictionary.h" #include "data/format.h" +#include "data/missing-values.h" #include "data/settings.h" #include "data/value-labels.h" #include "data/variable.h" -#include "language/command.h" + #include "language/dictionary/split-file.h" + +#include "language/command.h" #include "language/lexer/lexer.h" +#include "language/lexer/variable-parser.h" #include "language/stats/freq.h" + #include "libpspp/array.h" #include "libpspp/bit-vector.h" #include "libpspp/compiler.h" @@ -40,11 +46,12 @@ #include "libpspp/message.h" #include "libpspp/misc.h" #include "libpspp/pool.h" -#include "libpspp/str.h" + #include "math/histogram.h" #include "math/moments.h" #include "math/chart-geometry.h" + #include "output/chart-item.h" #include "output/charts/piechart.h" #include "output/charts/plot-hist.h" @@ -57,70 +64,6 @@ #define _(msgid) gettext (msgid) #define N_(msgid) msgid -/* (headers) */ - -/* (specification) - FREQUENCIES (frq_): - *+variables=custom; - +format=table:limit(n:limit,"%s>0")/notable/!table, - sort:!avalue/dvalue/afreq/dfreq; - missing=miss:include/!exclude; - barchart(ba_)=:minimum(d:min), - :maximum(d:max), - scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"); - piechart(pie_)=:minimum(d:min), - :maximum(d:max), - missing:missing/!nomissing, - scale:!freq/percent; - histogram(hi_)=:minimum(d:min), - :maximum(d:max), - scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"), - norm:!nonormal/normal; - +grouped=custom; - +ntiles=integer; - +percentiles = double list; - +statistics[st_]=mean,semean,median,mode,stddev,variance, - kurtosis,skewness,range,minimum,maximum,sum, - default,seskewness,sekurtosis,all,none. -*/ -/* (declarations) */ -/* (functions) */ - -/* Statistics. */ -enum - { - FRQ_MEAN, FRQ_SEMEAN, FRQ_MEDIAN, FRQ_MODE, FRQ_STDDEV, FRQ_VARIANCE, - FRQ_KURT, FRQ_SEKURT, FRQ_SKEW, FRQ_SESKEW, FRQ_RANGE, FRQ_MIN, FRQ_MAX, - FRQ_SUM, FRQ_N_STATS - }; - -/* Description of a statistic. */ -struct frq_info - { - int st_indx; /* Index into a_statistics[]. */ - const char *s10; /* Identifying string. */ - }; - -/* Table of statistics, indexed by dsc_*. */ -static const struct frq_info st_name[FRQ_N_STATS + 1] = -{ - {FRQ_ST_MEAN, N_("Mean")}, - {FRQ_ST_SEMEAN, N_("S.E. Mean")}, - {FRQ_ST_MEDIAN, N_("Median")}, - {FRQ_ST_MODE, N_("Mode")}, - {FRQ_ST_STDDEV, N_("Std Dev")}, - {FRQ_ST_VARIANCE, N_("Variance")}, - {FRQ_ST_KURTOSIS, N_("Kurtosis")}, - {FRQ_ST_SEKURTOSIS, N_("S.E. Kurt")}, - {FRQ_ST_SKEWNESS, N_("Skewness")}, - {FRQ_ST_SESKEWNESS, N_("S.E. Skew")}, - {FRQ_ST_RANGE, N_("Range")}, - {FRQ_ST_MINIMUM, N_("Minimum")}, - {FRQ_ST_MAXIMUM, N_("Maximum")}, - {FRQ_ST_SUM, N_("Sum")}, - {-1, 0}, -}; - /* Percentiles to calculate. */ struct percentile @@ -130,34 +73,78 @@ struct percentile bool show; /* True to show this percentile in the statistics box. */ }; -/* Groups of statistics. */ -#define BI BIT_INDEX -#define FRQ_DEFAULT \ - (BI (FRQ_MEAN) | BI (FRQ_STDDEV) | BI (FRQ_MIN) | BI (FRQ_MAX)) -#define FRQ_ALL \ - (BI (FRQ_SUM) | BI(FRQ_MIN) | BI(FRQ_MAX) \ - | BI(FRQ_MEAN) | BI(FRQ_SEMEAN) | BI(FRQ_STDDEV) \ - | BI(FRQ_VARIANCE) | BI(FRQ_KURT) | BI(FRQ_SEKURT) \ - | BI(FRQ_SKEW) | BI(FRQ_SESKEW) | BI(FRQ_RANGE) \ - | BI(FRQ_RANGE) | BI(FRQ_MODE) | BI(FRQ_MEDIAN)) +static int +ptile_3way (const void *_p1, const void *_p2) +{ + const struct percentile *p1 = _p1; + const struct percentile *p2 = _p2; -struct frq_chart + if (p1->p < p2->p) + return -1; + + return (p1->p > p2->p); +} + + +enum { - double x_min; /* X axis minimum value. */ - double x_max; /* X axis maximum value. */ - int y_scale; /* Y axis scale: FRQ_FREQ or FRQ_PERCENT. */ + FRQ_NONORMAL, + FRQ_NORMAL + }; - /* Histograms only. */ - double y_max; /* Y axis maximum value. */ - bool draw_normal; /* Whether to draw normal curve. */ +enum + { + FRQ_FREQ, + FRQ_PERCENT + }; - /* Pie charts only. */ - bool include_missing; /* Whether to include missing values. */ +enum sortprops + { + FRQ_AFREQ, + FRQ_DFREQ, + FRQ_AVALUE, + FRQ_DVALUE }; -/* Frequency tables. */ +/* Array indices for STATISTICS subcommand. */ +enum + { + FRQ_ST_MEAN, + FRQ_ST_SEMEAN, + FRQ_ST_MEDIAN, + FRQ_ST_MODE, + FRQ_ST_STDDEV, + FRQ_ST_VARIANCE, + FRQ_ST_KURTOSIS, + FRQ_ST_SEKURTOSIS, + FRQ_ST_SKEWNESS, + FRQ_ST_SESKEWNESS, + FRQ_ST_RANGE, + FRQ_ST_MINIMUM, + FRQ_ST_MAXIMUM, + FRQ_ST_SUM, + FRQ_ST_count + }; + +/* Description of statistics. */ +static const char *st_name[FRQ_ST_count] = +{ + N_("Mean"), + N_("S.E. Mean"), + N_("Median"), + N_("Mode"), + N_("Std Dev"), + N_("Variance"), + N_("Kurtosis"), + N_("S.E. Kurt"), + N_("Skewness"), + N_("S.E. Skew"), + N_("Range"), + N_("Minimum"), + N_("Maximum"), + N_("Sum") +}; -/* Entire frequency table. */ struct freq_tab { struct hmap data; /* Hash table for accumulating counts. */ @@ -173,10 +160,24 @@ struct freq_tab double valid_cases; /* Sum of weights of valid cases. */ }; +struct frq_chart + { + double x_min; /* X axis minimum value. */ + double x_max; /* X axis maximum value. */ + int y_scale; /* Y axis scale: FRQ_FREQ or FRQ_PERCENT. */ + + /* Histograms only. */ + double y_max; /* Y axis maximum value. */ + bool draw_normal; /* Whether to draw normal curve. */ + + /* Pie charts only. */ + bool include_missing; /* Whether to include missing values. */ + }; + /* Per-variable frequency data. */ struct var_freqs { - struct variable *var; + const struct variable *var; /* Freqency table. */ struct freq_tab tab; /* Frequencies table to use. */ @@ -185,635 +186,81 @@ struct var_freqs int n_groups; /* Number of groups. */ double *groups; /* Groups. */ - /* Statistics. */ - double stat[FRQ_N_STATS]; - - /* Variable attributes. */ - int width; - }; - -struct frq_proc - { - struct pool *pool; - - struct var_freqs *vars; - size_t n_vars; - - /* Percentiles to calculate and possibly display. */ - struct percentile *percentiles; - int n_percentiles, n_show_percentiles; - - /* Frequency table display. */ - int max_categories; /* Maximum categories to show. */ - int sort; /* FRQ_AVALUE or FRQ_DVALUE - or FRQ_ACOUNT or FRQ_DCOUNT. */ - - /* Statistics; number of statistics. */ - unsigned long stats; - int n_stats; - - /* Histogram and pie chart settings. */ - struct frq_chart *hist, *pie; - }; - -static void determine_charts (struct frq_proc *, - const struct cmd_frequencies *); - -static void calc_stats (const struct var_freqs *, double d[FRQ_N_STATS]); -static void calc_percentiles (const struct frq_proc *, - const struct var_freqs *); - -static void precalc (struct frq_proc *, struct casereader *, struct dataset *); -static void calc (struct frq_proc *, const struct ccase *, - const struct dataset *); -static void postcalc (struct frq_proc *, const struct dataset *); - -static void postprocess_freq_tab (const struct frq_proc *, struct var_freqs *); -static void dump_freq_table (const struct var_freqs *, - const struct variable *weight_var); -static void dump_statistics (const struct frq_proc *, const struct var_freqs *, - const struct variable *weight_var); -static void cleanup_freq_tab (struct var_freqs *); - -static void add_percentile (struct frq_proc *, double x, bool show, - size_t *allocated_percentiles); - -static void do_piechart(const struct frq_chart *, const struct variable *, - const struct freq_tab *); - -struct histogram *freq_tab_to_hist(const struct frq_proc *, - const struct freq_tab *, - const struct variable *); - -/* Parser and outline. */ - -int -cmd_frequencies (struct lexer *lexer, struct dataset *ds) -{ - struct cmd_frequencies cmd; - struct frq_proc frq; - struct casegrouper *grouper; - struct casereader *input, *group; - size_t allocated_percentiles; - bool ok; - int i; - - frq.pool = pool_create (); - - frq.vars = NULL; - frq.n_vars = 0; - - frq.percentiles = NULL; - frq.n_percentiles = 0; - frq.n_show_percentiles = 0; - - frq.hist = NULL; - frq.pie = NULL; - - allocated_percentiles = 0; - - if (!parse_frequencies (lexer, ds, &cmd, &frq)) - { - pool_destroy (frq.pool); - return CMD_FAILURE; - } - - /* Figure out when to show frequency tables. */ - frq.max_categories = (cmd.table == FRQ_NOTABLE ? -1 - : cmd.table == FRQ_TABLE ? INT_MAX - : cmd.limit); - frq.sort = cmd.sort; - - /* Figure out statistics to calculate. */ - frq.stats = 0; - if (cmd.a_statistics[FRQ_ST_DEFAULT] || !cmd.sbc_statistics) - frq.stats |= FRQ_DEFAULT; - if (cmd.a_statistics[FRQ_ST_ALL]) - frq.stats |= FRQ_ALL; - if (cmd.sort != FRQ_AVALUE && cmd.sort != FRQ_DVALUE) - frq.stats &= ~BIT_INDEX (FRQ_MEDIAN); - for (i = 0; i < FRQ_N_STATS; i++) - if (cmd.a_statistics[st_name[i].st_indx]) - frq.stats |= BIT_INDEX (i); - if (frq.stats & FRQ_KURT) - frq.stats |= BIT_INDEX (FRQ_SEKURT); - if (frq.stats & FRQ_SKEW) - frq.stats |= BIT_INDEX (FRQ_SESKEW); - - /* Calculate n_stats. */ - frq.n_stats = 0; - for (i = 0; i < FRQ_N_STATS; i++) - if ((frq.stats & BIT_INDEX (i))) - frq.n_stats++; - - /* Charting. */ - determine_charts (&frq, &cmd); - if (cmd.sbc_histogram || cmd.sbc_piechart || cmd.sbc_ntiles) - cmd.sort = FRQ_AVALUE; - - /* Work out what percentiles need to be calculated */ - if ( cmd.sbc_percentiles ) - { - for ( i = 0 ; i < MAXLISTS ; ++i ) - { - int pl; - subc_list_double *ptl_list = &cmd.dl_percentiles[i]; - for ( pl = 0 ; pl < subc_list_double_count(ptl_list); ++pl) - add_percentile (&frq, subc_list_double_at(ptl_list, pl) / 100.0, - true, &allocated_percentiles); - } - } - if ( cmd.sbc_ntiles ) - { - for ( i = 0 ; i < cmd.sbc_ntiles ; ++i ) - { - int j; - for (j = 0; j <= cmd.n_ntiles[i]; ++j ) - add_percentile (&frq, j / (double) cmd.n_ntiles[i], true, - &allocated_percentiles); - } - } - if (frq.stats & BIT_INDEX (FRQ_MEDIAN)) - { - /* Treat the median as the 50% percentile. - We output it in the percentiles table as "50 (Median)." */ - add_percentile (&frq, 0.5, true, &allocated_percentiles); - frq.stats &= ~BIT_INDEX (FRQ_MEDIAN); - frq.n_stats--; - } - if (cmd.sbc_histogram) - { - add_percentile (&frq, 0.25, false, &allocated_percentiles); - add_percentile (&frq, 0.75, false, &allocated_percentiles); - } - - /* Do it! */ - input = casereader_create_filter_weight (proc_open (ds), dataset_dict (ds), - NULL, NULL); - grouper = casegrouper_create_splits (input, dataset_dict (ds)); - for (; casegrouper_get_next_group (grouper, &group); - casereader_destroy (group)) - { - struct ccase *c; - - precalc (&frq, group, ds); - for (; (c = casereader_read (group)) != NULL; case_unref (c)) - calc (&frq, c, ds); - postcalc (&frq, ds); - } - ok = casegrouper_destroy (grouper); - ok = proc_commit (ds) && ok; - - free_frequencies(&cmd); - - pool_destroy (frq.pool); - free (frq.vars); - free (frq.percentiles); - free (frq.hist); - free (frq.pie); - - return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE; -} - -/* Figure out which charts the user requested. */ -static void -determine_charts (struct frq_proc *frq, const struct cmd_frequencies *cmd) -{ - if (cmd->sbc_barchart) - msg (SW, _("Bar charts are not implemented.")); - - if (cmd->sbc_histogram) - { - struct frq_chart *hist; - - hist = frq->hist = xmalloc (sizeof *frq->hist); - hist->x_min = cmd->hi_min; - hist->x_max = cmd->hi_max; - hist->y_scale = cmd->hi_scale; - hist->y_max = cmd->hi_scale == FRQ_FREQ ? cmd->hi_freq : cmd->hi_pcnt; - hist->draw_normal = cmd->hi_norm != FRQ_NONORMAL; - hist->include_missing = false; - - if (hist->x_min != SYSMIS && hist->x_max != SYSMIS - && hist->x_min >= hist->x_max) - { - msg (SE, _("%s for histogram must be greater than or equal to %s, " - "but %s was specified as %.15g and %s as %.15g. " - "%s and %s will be ignored."), - "MAX", "MIN", - "MIN", hist->x_min, - "MAX", hist->x_max, - "MIN", "MAX"); - hist->x_min = hist->x_max = SYSMIS; - } - } - - if (cmd->sbc_piechart) - { - struct frq_chart *pie; - - pie = frq->pie = xmalloc (sizeof *frq->pie); - pie->x_min = cmd->pie_min; - pie->x_max = cmd->pie_max; - pie->y_scale = cmd->pie_scale; - pie->include_missing = cmd->pie_missing == FRQ_MISSING; - - if (pie->x_min != SYSMIS && pie->x_max != SYSMIS - && pie->x_min >= pie->x_max) - { - msg (SE, _("%s for pie chart must be greater than or equal to %s, " - "but %s was specified as %.15g and %s as %.15g. " - "%s and %s will be ignored."), - "MAX", "MIN", - "MIN", pie->x_min, - "MAX", pie->x_max, - "MIN", "MAX"); - pie->x_min = pie->x_max = SYSMIS; - } - } -} - -/* Add data from case C to the frequency table. */ -static void -calc (struct frq_proc *frq, const struct ccase *c, const struct dataset *ds) -{ - double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); - size_t i; - - for (i = 0; i < frq->n_vars; i++) - { - struct var_freqs *vf = &frq->vars[i]; - const union value *value = case_data (c, vf->var); - size_t hash = value_hash (value, vf->width, 0); - struct freq *f; - - f = freq_hmap_search (&vf->tab.data, value, vf->width, hash); - if (f == NULL) - f = freq_hmap_insert (&vf->tab.data, value, vf->width, hash); - - f->count += weight; - } -} - -/* Prepares each variable that is the target of FREQUENCIES by setting - up its hash table. */ -static void -precalc (struct frq_proc *frq, struct casereader *input, struct dataset *ds) -{ - struct ccase *c; - size_t i; - - c = casereader_peek (input, 0); - if (c != NULL) - { - output_split_file_values (ds, c); - case_unref (c); - } - - for (i = 0; i < frq->n_vars; i++) - hmap_init (&frq->vars[i].tab.data); -} - -/* Finishes up with the variables after frequencies have been - calculated. Displays statistics, percentiles, ... */ -static void -postcalc (struct frq_proc *frq, const struct dataset *ds) -{ - const struct dictionary *dict = dataset_dict (ds); - const struct variable *wv = dict_get_weight (dict); - size_t i; - - for (i = 0; i < frq->n_vars; i++) - { - struct var_freqs *vf = &frq->vars[i]; - - postprocess_freq_tab (frq, vf); - - /* Frequencies tables. */ - if (vf->tab.n_valid + vf->tab.n_missing <= frq->max_categories) - dump_freq_table (vf, wv); - - /* Statistics. */ - if (frq->n_stats) - dump_statistics (frq, vf, wv); - - if (frq->hist && var_is_numeric (vf->var) && vf->tab.n_valid > 0) - { - double d[FRQ_N_STATS]; - struct histogram *histogram; - - calc_stats (vf, d); - - histogram = freq_tab_to_hist (frq, &vf->tab, vf->var); - - if ( histogram) - { - chart_item_submit (histogram_chart_create ( - histogram->gsl_hist, var_to_string(vf->var), - vf->tab.valid_cases, - d[FRQ_MEAN], - d[FRQ_STDDEV], - frq->hist->draw_normal)); - - statistic_destroy (&histogram->parent); - } - } - - if (frq->pie) - do_piechart(frq->pie, vf->var, &vf->tab); - - cleanup_freq_tab (vf); - - } -} - -/* Returns true iff the value in struct freq F is non-missing - for variable V. */ -static bool -not_missing (const void *f_, const void *v_) -{ - const struct freq *f = f_; - const struct variable *v = v_; - - return !var_is_value_missing (v, &f->value, MV_ANY); -} - -struct freq_compare_aux - { - bool by_freq; - bool ascending_freq; - - int width; - bool ascending_value; - }; - -static int -compare_freq (const void *a_, const void *b_, const void *aux_) -{ - const struct freq_compare_aux *aux = aux_; - const struct freq *a = a_; - const struct freq *b = b_; - - if (aux->by_freq && a->count != b->count) - { - int cmp = a->count > b->count ? 1 : -1; - return aux->ascending_freq ? cmp : -cmp; - } - else - { - int cmp = value_compare_3way (&a->value, &b->value, aux->width); - return aux->ascending_value ? cmp : -cmp; - } -} -/* Summarizes the frequency table data for variable V. */ -static void -postprocess_freq_tab (const struct frq_proc *frq, struct var_freqs *vf) -{ - struct freq_tab *ft = &vf->tab; - struct freq_compare_aux aux; - size_t count; - struct freq *freqs, *f; - size_t i; - - /* Extract data from hash table. */ - count = hmap_count (&ft->data); - freqs = freq_hmap_extract (&ft->data); - - /* Put data into ft. */ - ft->valid = freqs; - ft->n_valid = partition (freqs, count, sizeof *freqs, not_missing, vf->var); - ft->missing = freqs + ft->n_valid; - ft->n_missing = count - ft->n_valid; - - /* Sort data. */ - aux.by_freq = frq->sort == FRQ_AFREQ || frq->sort == FRQ_DFREQ; - aux.ascending_freq = frq->sort != FRQ_DFREQ; - aux.width = vf->width; - aux.ascending_value = frq->sort != FRQ_DVALUE; - sort (ft->valid, ft->n_valid, sizeof *ft->valid, compare_freq, &aux); - sort (ft->missing, ft->n_missing, sizeof *ft->missing, compare_freq, &aux); - - /* Summary statistics. */ - ft->valid_cases = 0.0; - for(i = 0 ; i < ft->n_valid ; ++i ) - { - f = &ft->valid[i]; - ft->valid_cases += f->count; - - } - - ft->total_cases = ft->valid_cases ; - for(i = 0 ; i < ft->n_missing ; ++i ) - { - f = &ft->missing[i]; - ft->total_cases += f->count; - } - -} - -/* Frees the frequency table for variable V. */ -static void -cleanup_freq_tab (struct var_freqs *vf) -{ - free (vf->tab.valid); - freq_hmap_destroy (&vf->tab.data, vf->width); -} - -/* Parses the VARIABLES subcommand. */ -static int -frq_custom_variables (struct lexer *lexer, struct dataset *ds, - struct cmd_frequencies *cmd UNUSED, void *frq_ UNUSED) -{ - struct frq_proc *frq = frq_; - struct variable **vars; - size_t n_vars; - size_t i; - - lex_match (lexer, T_EQUALS); - if (lex_token (lexer) != T_ALL - && (lex_token (lexer) != T_ID - || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)) - return 2; + /* Statistics. */ + double stat[FRQ_ST_count]; - /* Get list of current variables, to avoid duplicates. */ - vars = xmalloc (frq->n_vars * sizeof *vars); - n_vars = frq->n_vars; - for (i = 0; i < frq->n_vars; i++) - vars[i] = frq->vars[i].var; + /* Variable attributes. */ + int width; + }; - if (!parse_variables (lexer, dataset_dict (ds), &vars, &n_vars, - PV_APPEND | PV_NO_SCRATCH)) - return 0; +struct frq_proc + { + struct pool *pool; - frq->vars = xrealloc (frq->vars, n_vars * sizeof *frq->vars); - for (i = frq->n_vars; i < n_vars; i++) - { - struct variable *var = vars[i]; - struct var_freqs *vf = &frq->vars[i]; + struct var_freqs *vars; + size_t n_vars; - vf->var = var; - vf->tab.valid = vf->tab.missing = NULL; - vf->tab.dict = dataset_dict (ds); - vf->n_groups = 0; - vf->groups = NULL; - vf->width = var_get_width (var); - } - frq->n_vars = n_vars; + /* Percentiles to calculate and possibly display. */ + struct percentile *percentiles; + int n_percentiles, n_show_percentiles; - free (vars); + /* Frequency table display. */ + int max_categories; /* Maximum categories to show. */ + int sort; /* FRQ_AVALUE or FRQ_DVALUE + or FRQ_AFREQ or FRQ_DFREQ. */ - return 1; -} + /* Statistics; number of statistics. */ + unsigned long stats; + int n_stats; -/* Parses the GROUPED subcommand, setting the n_grouped, grouped - fields of specified variables. */ -static int -frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *frq_ UNUSED) -{ - struct frq_proc *frq = frq_; + /* Histogram and pie chart settings. */ + struct frq_chart *hist, *pie; + }; - lex_match (lexer, T_EQUALS); - if ((lex_token (lexer) == T_ID - && dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL) - || lex_token (lexer) == T_ID) - for (;;) - { - size_t i; - /* Max, current size of list; list itself. */ - int nl, ml; - double *dl; +struct freq_compare_aux + { + bool by_freq; + bool ascending_freq; + + int width; + bool ascending_value; + }; - /* Variable list. */ - size_t n; - const struct variable **v; +static void calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]); - if (!parse_variables_const (lexer, dataset_dict (ds), &v, &n, - PV_NO_DUPLICATE | PV_NUMERIC)) - return 0; - if (lex_match (lexer, T_LPAREN)) - { - nl = ml = 0; - dl = NULL; - while (lex_integer (lexer)) - { - if (nl >= ml) - { - ml += 16; - dl = pool_nrealloc (frq->pool, dl, ml, sizeof *dl); - } - dl[nl++] = lex_tokval (lexer); - lex_get (lexer); - lex_match (lexer, T_COMMA); - } - /* Note that nl might still be 0 and dl might still be - NULL. That's okay. */ - if (!lex_match (lexer, T_RPAREN)) - { - free (v); - lex_error_expecting (lexer, "`)'", NULL_SENTINEL); - return 0; - } - } - else - { - nl = 0; - dl = NULL; - } - - for (i = 0; i < n; i++) - { - size_t j; - - for (j = 0; j < frq->n_vars; j++) - { - struct var_freqs *vf = &frq->vars[j]; - if (vf->var == v[i]) - { - if (vf->groups != NULL) - msg (SE, _("Variables %s specified multiple times on " - "%s subcommand."), var_get_name (v[i]), "GROUPED"); - else - { - vf->n_groups = nl; - vf->groups = dl; - } - goto found; - } - } - msg (SE, _("Variables %s specified on %s but not on " - "%s."), var_get_name (v[i]), "GROUPED", "VARIABLES"); - - found:; - } - - free (v); - if (lex_token (lexer) != T_SLASH) - break; - - if ((lex_next_token (lexer, 1) == T_ID - && dict_lookup_var (dataset_dict (ds), - lex_next_tokcstr (lexer, 1))) - || lex_next_token (lexer, 1) == T_ALL) - { - /* The token after the slash is a variable name. Keep parsing. */ - lex_get (lexer); - } - else - { - /* The token after the slash must be the start of a new - subcommand. Let the caller see the slash. */ - break; - } - } +static void do_piechart(const struct frq_chart *pie, + const struct variable *var, + const struct freq_tab *frq_tab); - return 1; -} +static void dump_statistics (const struct frq_proc *frq, + const struct var_freqs *vf, + const struct variable *wv); -/* Adds X to the list of percentiles, keeping the list in proper - order. If SHOW is true, the percentile will be shown in the statistics - box, otherwise it will be hidden. */ -static void -add_percentile (struct frq_proc *frq, double x, bool show, - size_t *allocated_percentiles) +static int +compare_freq (const void *a_, const void *b_, const void *aux_) { - int i; + const struct freq_compare_aux *aux = aux_; + const struct freq *a = a_; + const struct freq *b = b_; - /* Do nothing if it's already in the list */ - for (i = 0; i < frq->n_percentiles; i++) + if (aux->by_freq && a->count != b->count) { - struct percentile *pc = &frq->percentiles[i]; - - if ( fabs(x - pc->p) < DBL_EPSILON ) - { - if (show && !pc->show) - { - frq->n_show_percentiles++; - pc->show = true; - } - return; - } - - if (x < pc->p) - break; + int cmp = a->count > b->count ? 1 : -1; + return aux->ascending_freq ? cmp : -cmp; + } + else + { + int cmp = value_compare_3way (&a->value, &b->value, aux->width); + return aux->ascending_value ? cmp : -cmp; } - - if (frq->n_percentiles >= *allocated_percentiles) - frq->percentiles = x2nrealloc (frq->percentiles, allocated_percentiles, - sizeof *frq->percentiles); - insert_element (frq->percentiles, frq->n_percentiles, - sizeof *frq->percentiles, i); - frq->percentiles[i].p = x; - frq->percentiles[i].show = show; - frq->n_percentiles++; - if (show) - frq->n_show_percentiles++; } -/* Comparison functions. */ +/* Create a gsl_histogram from a freq_tab */ +static struct histogram * +freq_tab_to_hist (const struct frq_proc *frq, const struct freq_tab *ft, + const struct variable *var); - -/* Frequency table display. */ /* Displays a full frequency table for variable V. */ static void @@ -954,133 +401,733 @@ calc_percentiles (const struct frq_proc *frq, const struct var_freqs *vf) } } -/* Calculates all the pertinent statistics for VF, putting them in array - D[]. */ +/* Returns true iff the value in struct freq F is non-missing + for variable V. */ +static bool +not_missing (const void *f_, const void *v_) +{ + const struct freq *f = f_; + const struct variable *v = v_; + + return !var_is_value_missing (v, &f->value, MV_ANY); +} + + +/* Summarizes the frequency table data for variable V. */ static void -calc_stats (const struct var_freqs *vf, double d[FRQ_N_STATS]) +postprocess_freq_tab (const struct frq_proc *frq, struct var_freqs *vf) { - const struct freq_tab *ft = &vf->tab; - double W = ft->valid_cases; - const struct freq *f; - struct moments *m; - int most_often; - double X_mode; + struct freq_tab *ft = &vf->tab; + struct freq_compare_aux aux; + size_t count; + struct freq *freqs, *f; + size_t i; + + /* Extract data from hash table. */ + count = hmap_count (&ft->data); + freqs = freq_hmap_extract (&ft->data); + + /* Put data into ft. */ + ft->valid = freqs; + ft->n_valid = partition (freqs, count, sizeof *freqs, not_missing, vf->var); + ft->missing = freqs + ft->n_valid; + ft->n_missing = count - ft->n_valid; + + /* Sort data. */ + aux.by_freq = frq->sort == FRQ_AFREQ || frq->sort == FRQ_DFREQ; + aux.ascending_freq = frq->sort != FRQ_DFREQ; + aux.width = vf->width; + aux.ascending_value = frq->sort != FRQ_DVALUE; + sort (ft->valid, ft->n_valid, sizeof *ft->valid, compare_freq, &aux); + sort (ft->missing, ft->n_missing, sizeof *ft->missing, compare_freq, &aux); + + /* Summary statistics. */ + ft->valid_cases = 0.0; + for(i = 0 ; i < ft->n_valid ; ++i ) + { + f = &ft->valid[i]; + ft->valid_cases += f->count; + + } + + ft->total_cases = ft->valid_cases ; + for(i = 0 ; i < ft->n_missing ; ++i ) + { + f = &ft->missing[i]; + ft->total_cases += f->count; + } + +} + +/* Frees the frequency table for variable V. */ +static void +cleanup_freq_tab (struct var_freqs *vf) +{ + free (vf->tab.valid); + freq_hmap_destroy (&vf->tab.data, vf->width); +} + +/* Add data from case C to the frequency table. */ +static void +calc (struct frq_proc *frq, const struct ccase *c, const struct dataset *ds) +{ + double weight = dict_get_case_weight (dataset_dict (ds), c, NULL); + size_t i; + + for (i = 0; i < frq->n_vars; i++) + { + struct var_freqs *vf = &frq->vars[i]; + const union value *value = case_data (c, vf->var); + size_t hash = value_hash (value, vf->width, 0); + struct freq *f; + + f = freq_hmap_search (&vf->tab.data, value, vf->width, hash); + if (f == NULL) + f = freq_hmap_insert (&vf->tab.data, value, vf->width, hash); + + f->count += weight; + } +} + +/* Prepares each variable that is the target of FREQUENCIES by setting + up its hash table. */ +static void +precalc (struct frq_proc *frq, struct casereader *input, struct dataset *ds) +{ + struct ccase *c; + size_t i; + + c = casereader_peek (input, 0); + if (c != NULL) + { + output_split_file_values (ds, c); + case_unref (c); + } + + for (i = 0; i < frq->n_vars; i++) + hmap_init (&frq->vars[i].tab.data); +} + +/* Finishes up with the variables after frequencies have been + calculated. Displays statistics, percentiles, ... */ +static void +postcalc (struct frq_proc *frq, const struct dataset *ds) +{ + const struct dictionary *dict = dataset_dict (ds); + const struct variable *wv = dict_get_weight (dict); + size_t i; + + for (i = 0; i < frq->n_vars; i++) + { + struct var_freqs *vf = &frq->vars[i]; + + postprocess_freq_tab (frq, vf); + + /* Frequencies tables. */ + if (vf->tab.n_valid + vf->tab.n_missing <= frq->max_categories) + dump_freq_table (vf, wv); + + calc_percentiles (frq, vf); + + /* Statistics. */ + if (frq->n_stats) + dump_statistics (frq, vf, wv); + + if (frq->hist && var_is_numeric (vf->var) && vf->tab.n_valid > 0) + { + double d[FRQ_ST_count]; + struct histogram *histogram; + + calc_stats (vf, d); + + histogram = freq_tab_to_hist (frq, &vf->tab, vf->var); + + if ( histogram) + { + chart_item_submit (histogram_chart_create ( + histogram->gsl_hist, var_to_string(vf->var), + vf->tab.valid_cases, + d[FRQ_ST_MEAN], + d[FRQ_ST_STDDEV], + frq->hist->draw_normal)); + + statistic_destroy (&histogram->parent); + } + } + + if (frq->pie) + do_piechart(frq->pie, vf->var, &vf->tab); + + cleanup_freq_tab (vf); + } +} + +int +cmd_frequencies (struct lexer *lexer, struct dataset *ds) +{ + int i; + struct frq_proc frq; + const struct variable **vars; + + bool sbc_barchart = false; + bool sbc_piechart = false; + bool sbc_histogram = false; + + double pie_min = -DBL_MAX; + double pie_max = DBL_MAX; + bool pie_missing = false; + + double hi_min = -DBL_MAX; + double hi_max = DBL_MAX; + int hi_scale = FRQ_FREQ; + int hi_freq = INT_MIN; + int hi_pcnt = INT_MIN; + int hi_norm = FRQ_NONORMAL; + + frq.pool = pool_create (); + frq.sort = FRQ_AVALUE; + + frq.vars = NULL; + frq.n_vars = 0; + + frq.stats = BIT_INDEX (FRQ_ST_MEAN) + | BIT_INDEX (FRQ_ST_STDDEV) + | BIT_INDEX (FRQ_ST_MINIMUM) + | BIT_INDEX (FRQ_ST_MAXIMUM); + + frq.n_stats = 4; + + frq.max_categories = INT_MAX; + + frq.percentiles = NULL; + frq.n_percentiles = 0; + frq.n_show_percentiles = 0; + + frq.hist = NULL; + frq.pie = NULL; + - assert (ft->n_valid > 0); + /* Accept an optional, completely pointless "/VARIABLES=" */ + lex_match (lexer, T_SLASH); + if (lex_match_id (lexer, "VARIABLES")) + { + if (! lex_force_match (lexer, T_EQUALS) ) + goto error; + } - /* Calculate the mode. */ - most_often = -1; - X_mode = SYSMIS; - for (f = ft->valid; f < ft->missing; f++) + if (!parse_variables_const (lexer, dataset_dict (ds), + &vars, + &frq.n_vars, + PV_NO_DUPLICATE)) + goto error; + + frq.vars = xzalloc (frq.n_vars * sizeof (*frq.vars)); + for (i = 0; i < frq.n_vars; ++i) { - if (most_often < f->count) - { - most_often = f->count; - X_mode = f->value.f; + frq.vars[i].var = vars[i]; + } + + while (lex_token (lexer) != T_ENDCMD) + { + lex_match (lexer, T_SLASH); + + if (lex_match_id (lexer, "STATISTICS")) + { + lex_match (lexer, T_EQUALS); + + frq.stats = 0; + frq.n_stats = 0; + + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_match_id (lexer, "DEFAULT")) + { + frq.stats = BIT_INDEX (FRQ_ST_MEAN) + | BIT_INDEX (FRQ_ST_STDDEV) + | BIT_INDEX (FRQ_ST_MINIMUM) + | BIT_INDEX (FRQ_ST_MAXIMUM); + + frq.n_stats = 4; + } + else if (lex_match_id (lexer, "MEAN")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MEAN); + frq.n_stats++; + } + else if (lex_match_id (lexer, "SEMEAN")) + { + frq.stats |= BIT_INDEX (FRQ_ST_SEMEAN); + frq.n_stats++; + } + else if (lex_match_id (lexer, "MEDIAN")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MEDIAN); + frq.n_stats++; + } + else if (lex_match_id (lexer, "MODE")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MODE); + frq.n_stats++; + } + else if (lex_match_id (lexer, "STDDEV")) + { + frq.stats |= BIT_INDEX (FRQ_ST_STDDEV); + frq.n_stats++; + } + else if (lex_match_id (lexer, "VARIANCE")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MEAN); + frq.n_stats++; + } + else if (lex_match_id (lexer, "KURTOSIS")) + { + frq.stats |= BIT_INDEX (FRQ_ST_KURTOSIS); + frq.n_stats++; + } + else if (lex_match_id (lexer, "SKEWNESS")) + { + frq.stats |= BIT_INDEX (FRQ_ST_SKEWNESS); + frq.n_stats++; + } + else if (lex_match_id (lexer, "RANGE")) + { + frq.stats |= BIT_INDEX (FRQ_ST_RANGE); + frq.n_stats++; + } + else if (lex_match_id (lexer, "MINIMUM")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MINIMUM); + frq.n_stats++; + } + else if (lex_match_id (lexer, "MAXIMUM")) + { + frq.stats |= BIT_INDEX (FRQ_ST_MAXIMUM); + frq.n_stats++; + } + else if (lex_match_id (lexer, "SUM")) + { + frq.stats |= BIT_INDEX (FRQ_ST_SUM); + frq.n_stats++; + } + else if (lex_match_id (lexer, "SESKEWNESS")) + { + frq.stats |= BIT_INDEX (FRQ_ST_SESKEWNESS); + frq.n_stats++; + } + else if (lex_match_id (lexer, "SEKURTOSIS")) + { + frq.stats |= BIT_INDEX (FRQ_ST_SEKURTOSIS); + frq.n_stats++; + } + else if (lex_match_id (lexer, "NONE")) + { + frq.stats = 0; + frq.n_stats = 0; + } + else if (lex_match (lexer, T_ALL)) + { + frq.stats = ~0; + frq.n_stats = FRQ_ST_count; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } } - else if (most_often == f->count) + else if (lex_match_id (lexer, "PERCENTILES")) { - /* A duplicate mode is undefined. - FIXME: keep track of *all* the modes. */ - X_mode = SYSMIS; - } - } + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_force_num (lexer)) + { + frq.percentiles = + xrealloc (frq.percentiles, + (frq.n_percentiles + 1) + * sizeof (*frq.percentiles)); + frq.percentiles[frq.n_percentiles].p = lex_number (lexer) / 100.0; + frq.percentiles[frq.n_percentiles].show = true; + lex_get (lexer); + frq.n_percentiles++; + frq.n_show_percentiles++; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else if (lex_match_id (lexer, "FORMAT")) + { + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_match_id (lexer, "TABLE")) + { + + } + else if (lex_match_id (lexer, "NOTABLE")) + { + frq.max_categories = 0; + } + else if (lex_match_id (lexer, "AVALUE")) + { + frq.sort = FRQ_AVALUE; + } + else if (lex_match_id (lexer, "DVALUE")) + { + frq.sort = FRQ_DVALUE; + } + else if (lex_match_id (lexer, "AFREQ")) + { + frq.sort = FRQ_AFREQ; + } + else if (lex_match_id (lexer, "DFREQ")) + { + frq.sort = FRQ_DFREQ; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else if (lex_match_id (lexer, "NTILES")) + { + lex_match (lexer, T_EQUALS); - /* Calculate moments. */ - m = moments_create (MOMENT_KURTOSIS); - for (f = ft->valid; f < ft->missing; f++) - moments_pass_one (m, f->value.f, f->count); - for (f = ft->valid; f < ft->missing; f++) - moments_pass_two (m, f->value.f, f->count); - moments_calculate (m, NULL, &d[FRQ_MEAN], &d[FRQ_VARIANCE], - &d[FRQ_SKEW], &d[FRQ_KURT]); - moments_destroy (m); + if (lex_force_int (lexer)) + { + int i; + int n = lex_integer (lexer); + lex_get (lexer); + for (i = 0; i < n + 1; ++i) + { + frq.percentiles = + xrealloc (frq.percentiles, + (frq.n_percentiles + 1) + * sizeof (*frq.percentiles)); + frq.percentiles[frq.n_percentiles].p = + i / (double) n ; + frq.percentiles[frq.n_percentiles].show = true; + + frq.n_percentiles++; + frq.n_show_percentiles++; + } + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + else if (lex_match_id (lexer, "ALGORITHM")) + { + lex_match (lexer, T_EQUALS); - /* Formulas below are taken from _SPSS Statistical Algorithms_. */ - d[FRQ_MIN] = ft->valid[0].value.f; - d[FRQ_MAX] = ft->valid[ft->n_valid - 1].value.f; - d[FRQ_MODE] = X_mode; - d[FRQ_RANGE] = d[FRQ_MAX] - d[FRQ_MIN]; - d[FRQ_SUM] = d[FRQ_MEAN] * W; - d[FRQ_STDDEV] = sqrt (d[FRQ_VARIANCE]); - d[FRQ_SEMEAN] = d[FRQ_STDDEV] / sqrt (W); - d[FRQ_SESKEW] = calc_seskew (W); - d[FRQ_SEKURT] = calc_sekurt (W); -} + if (lex_match_id (lexer, "COMPATIBLE")) + { + settings_set_cmd_algorithm (COMPATIBLE); + } + else if (lex_match_id (lexer, "ENHANCED")) + { + settings_set_cmd_algorithm (ENHANCED); + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + else if (lex_match_id (lexer, "HISTOGRAM")) + { + lex_match (lexer, T_EQUALS); + sbc_histogram = true; -/* Displays a table of all the statistics requested for variable V. */ -static void -dump_statistics (const struct frq_proc *frq, const struct var_freqs *vf, - const struct variable *wv) -{ - const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0; - const struct freq_tab *ft = &vf->tab; - double stat_value[FRQ_N_STATS]; - struct tab_table *t; - int i, r; + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_match_id (lexer, "NORMAL")) + { + hi_norm = FRQ_NORMAL; + } + else if (lex_match_id (lexer, "NONORMAL")) + { + hi_norm = FRQ_NONORMAL; + } + else if (lex_match_id (lexer, "FREQ")) + { + hi_scale = FRQ_FREQ; + if (lex_match (lexer, T_LPAREN)) + { + if (lex_force_int (lexer)) + { + hi_freq = lex_integer (lexer); + if (hi_freq <= 0) + { + lex_error (lexer, _("Histogram frequency must be greater than zero.")); + } + lex_get (lexer); + lex_force_match (lexer, T_RPAREN); + } + } + } + else if (lex_match_id (lexer, "PERCENT")) + { + hi_scale = FRQ_PERCENT; + if (lex_match (lexer, T_LPAREN)) + { + if (lex_force_int (lexer)) + { + hi_pcnt = lex_integer (lexer); + if (hi_pcnt <= 0) + { + lex_error (lexer, _("Histogram percentaage must be greater than zero.")); + } + lex_get (lexer); + lex_force_match (lexer, T_RPAREN); + } + } + } + else if (lex_match_id (lexer, "MINIMUM")) + { + lex_force_match (lexer, T_LPAREN); + if (lex_force_num (lexer)) + { + hi_min = lex_number (lexer); + lex_get (lexer); + } + lex_force_match (lexer, T_RPAREN); + } + else if (lex_match_id (lexer, "MAXIMUM")) + { + lex_force_match (lexer, T_LPAREN); + if (lex_force_num (lexer)) + { + hi_max = lex_number (lexer); + lex_get (lexer); + } + lex_force_match (lexer, T_RPAREN); + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else if (lex_match_id (lexer, "PIECHART")) + { + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_match_id (lexer, "MINIMUM")) + { + lex_force_match (lexer, T_LPAREN); + if (lex_force_num (lexer)) + { + pie_min = lex_number (lexer); + lex_get (lexer); + } + lex_force_match (lexer, T_RPAREN); + } + else if (lex_match_id (lexer, "MAXIMUM")) + { + lex_force_match (lexer, T_LPAREN); + if (lex_force_num (lexer)) + { + pie_max = lex_number (lexer); + lex_get (lexer); + } + lex_force_match (lexer, T_RPAREN); + } + else if (lex_match_id (lexer, "MISSING")) + { + pie_missing = true; + } + else if (lex_match_id (lexer, "NOMISSING")) + { + pie_missing = false; + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + sbc_piechart = true; + } + else if (lex_match_id (lexer, "MISSING")) + { + lex_match (lexer, T_EQUALS); - if (var_is_alpha (vf->var)) - return; + while (lex_token (lexer) != T_ENDCMD + && lex_token (lexer) != T_SLASH) + { + if (lex_match_id (lexer, "EXCLUDE")) + { + } + else if (lex_match_id (lexer, "INCLUDE")) + { + } + else + { + lex_error (lexer, NULL); + goto error; + } + } + } + else + { + lex_error (lexer, NULL); + goto error; + } + } - if (ft->n_valid == 0) + if (frq.stats & FRQ_ST_MEDIAN) { - msg (SW, _("No valid data for variable %s; statistics not displayed."), - var_get_name (vf->var)); - return; + frq.percentiles = + xrealloc (frq.percentiles, + (frq.n_percentiles + 1) + * sizeof (*frq.percentiles)); + + frq.percentiles[frq.n_percentiles].p = 0.50; + frq.percentiles[frq.n_percentiles].show = true; + + frq.n_percentiles++; } - calc_stats (vf, stat_value); - calc_percentiles (frq, vf); - t = tab_create (3, frq->n_stats + frq->n_show_percentiles + 2); - tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ; +/* Figure out which charts the user requested. */ + { + if (sbc_barchart) + msg (SW, _("Bar charts are not implemented.")); - tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1); - tab_vline (t, TAL_GAP , 1, 0, tab_nr(t) - 1 ) ; + if (sbc_histogram) + { + struct frq_chart *hist; + + hist = frq.hist = xmalloc (sizeof *frq.hist); + hist->x_min = hi_min; + hist->x_max = hi_max; + hist->y_scale = hi_scale; + hist->y_max = hi_scale == FRQ_FREQ ? hi_freq : hi_pcnt; + hist->draw_normal = hi_norm != FRQ_NONORMAL; + hist->include_missing = false; + + if (hist->x_min != SYSMIS && hist->x_max != SYSMIS + && hist->x_min >= hist->x_max) + { + msg (SE, _("%s for histogram must be greater than or equal to %s, " + "but %s was specified as %.15g and %s as %.15g. " + "%s and %s will be ignored."), + "MAX", "MIN", + "MIN", hist->x_min, + "MAX", hist->x_max, + "MIN", "MAX"); + hist->x_min = hist->x_max = SYSMIS; + } - r=2; /* N missing and N valid are always dumped */ + frq.percentiles = + xrealloc (frq.percentiles, + (frq.n_percentiles + 2) + * sizeof (*frq.percentiles)); + + frq.percentiles[frq.n_percentiles].p = 0.25; + frq.percentiles[frq.n_percentiles].show = false; + + frq.percentiles[frq.n_percentiles + 1].p = 0.75; + frq.percentiles[frq.n_percentiles + 1].show = false; + + frq.n_percentiles+=2; + } - for (i = 0; i < FRQ_N_STATS; i++) - if (frq->stats & BIT_INDEX (i)) + if (sbc_piechart) { - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, - gettext (st_name[i].s10)); - tab_double (t, 2, r, TAB_NONE, stat_value[i], NULL); - r++; + struct frq_chart *pie; + + pie = frq.pie = xmalloc (sizeof *frq.pie); + pie->x_min = pie_min; + pie->x_max = pie_max; + pie->include_missing = pie_missing; + + if (pie->x_min != SYSMIS && pie->x_max != SYSMIS + && pie->x_min >= pie->x_max) + { + msg (SE, _("%s for pie chart must be greater than or equal to %s, " + "but %s was specified as %.15g and %s as %.15g. " + "%s and %s will be ignored."), + "MAX", "MIN", + "MIN", pie->x_min, + "MAX", pie->x_max, + "MIN", "MAX"); + pie->x_min = pie->x_max = SYSMIS; + } } + } - tab_text (t, 0, 0, TAB_LEFT | TAT_TITLE, _("N")); - tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid")); - tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing")); + { + int i,o; + double previous_p = -1; + qsort (frq.percentiles, frq.n_percentiles, + sizeof (*frq.percentiles), + ptile_3way); + + frq.n_show_percentiles = 0; + for (i = o = 0; i < frq.n_percentiles; ++i) + { + frq.percentiles[o].p = frq.percentiles[i].p; - tab_double (t, 2, 0, TAB_NONE, ft->valid_cases, wfmt); - tab_double (t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, wfmt); + if (frq.percentiles[i].show) + frq.percentiles[o].show = true; - for (i = 0; i < frq->n_percentiles; i++) - { - struct percentile *pc = &frq->percentiles[i]; + if (frq.percentiles[i].p != previous_p) + { + if (frq.percentiles[i].show) + frq.n_show_percentiles++; - if (!pc->show) - continue; + o++; + } - if ( i == 0 ) - { - tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles")); - } + previous_p = frq.percentiles[i].p; + } - if (pc->p == 0.5) - tab_text (t, 1, r, TAB_LEFT, _("50 (Median)")); - else - tab_fixed (t, 1, r, TAB_LEFT, pc->p * 100, 3, 0); - tab_double (t, 2, r, TAB_NONE, pc->value, - var_get_print_format (vf->var)); - r++; - } + frq.n_percentiles = o; + } - tab_title (t, "%s", var_to_string (vf->var)); + { + struct casegrouper *grouper; + struct casereader *group; + bool ok; - tab_submit (t); + grouper = casegrouper_create_splits (proc_open (ds), dataset_dict (ds)); + while (casegrouper_get_next_group (grouper, &group)) + { + struct ccase *c; + precalc (&frq, group, ds); + for (; (c = casereader_read (group)) != NULL; case_unref (c)) + calc (&frq, c, ds); + postcalc (&frq, ds); + } + ok = casegrouper_destroy (grouper); + ok = proc_commit (ds) && ok; + } + + + return CMD_SUCCESS; + + error: + + return CMD_FAILURE; } static double @@ -1090,6 +1137,8 @@ calculate_iqr (const struct frq_proc *frq) double q3 = SYSMIS; int i; + /* This cannot work unless the 25th and 75th percentile are calculated */ + assert (frq->n_percentiles >= 2); for (i = 0; i < frq->n_percentiles; i++) { struct percentile *pc = &frq->percentiles[i]; @@ -1120,7 +1169,7 @@ chart_includes_value (const struct frq_chart *chart, } /* Create a gsl_histogram from a freq_tab */ -struct histogram * +static struct histogram * freq_tab_to_hist (const struct frq_proc *frq, const struct freq_tab *ft, const struct variable *var) { @@ -1207,8 +1256,6 @@ freq_tab_to_slice_array(const struct frq_chart *pie, } - - static void do_piechart(const struct frq_chart *pie, const struct variable *var, const struct freq_tab *frq_tab) @@ -1232,9 +1279,136 @@ do_piechart(const struct frq_chart *pie, const struct variable *var, free (slices); } +/* Calculates all the pertinent statistics for VF, putting them in array + D[]. */ +static void +calc_stats (const struct var_freqs *vf, double d[FRQ_ST_count]) +{ + const struct freq_tab *ft = &vf->tab; + double W = ft->valid_cases; + const struct freq *f; + struct moments *m; + int most_often; + double X_mode; + + assert (ft->n_valid > 0); + + /* Calculate the mode. */ + most_often = -1; + X_mode = SYSMIS; + for (f = ft->valid; f < ft->missing; f++) + { + if (most_often < f->count) + { + most_often = f->count; + X_mode = f->value.f; + } + else if (most_often == f->count) + { + /* A duplicate mode is undefined. + FIXME: keep track of *all* the modes. */ + X_mode = SYSMIS; + } + } + + /* Calculate moments. */ + m = moments_create (MOMENT_KURTOSIS); + for (f = ft->valid; f < ft->missing; f++) + moments_pass_one (m, f->value.f, f->count); + for (f = ft->valid; f < ft->missing; f++) + moments_pass_two (m, f->value.f, f->count); + moments_calculate (m, NULL, &d[FRQ_ST_MEAN], &d[FRQ_ST_VARIANCE], + &d[FRQ_ST_SKEWNESS], &d[FRQ_ST_KURTOSIS]); + moments_destroy (m); + + /* Formulas below are taken from _SPSS Statistical Algorithms_. */ + d[FRQ_ST_MINIMUM] = ft->valid[0].value.f; + d[FRQ_ST_MAXIMUM] = ft->valid[ft->n_valid - 1].value.f; + d[FRQ_ST_MODE] = X_mode; + d[FRQ_ST_RANGE] = d[FRQ_ST_MAXIMUM] - d[FRQ_ST_MINIMUM]; + d[FRQ_ST_SUM] = d[FRQ_ST_MEAN] * W; + d[FRQ_ST_STDDEV] = sqrt (d[FRQ_ST_VARIANCE]); + d[FRQ_ST_SEMEAN] = d[FRQ_ST_STDDEV] / sqrt (W); + d[FRQ_ST_SESKEWNESS] = calc_seskew (W); + d[FRQ_ST_SEKURTOSIS] = calc_sekurt (W); +} + +/* Displays a table of all the statistics requested for variable V. */ +static void +dump_statistics (const struct frq_proc *frq, const struct var_freqs *vf, + const struct variable *wv) +{ + const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0; + const struct freq_tab *ft = &vf->tab; + double stat_value[FRQ_ST_count]; + struct tab_table *t; + int i, r; + + if (var_is_alpha (vf->var)) + return; + + if (ft->n_valid == 0) + { + msg (SW, _("No valid data for variable %s; statistics not displayed."), + var_get_name (vf->var)); + return; + } + calc_stats (vf, stat_value); + + t = tab_create (3, ((frq->stats & FRQ_ST_MEDIAN) ? frq->n_stats - 1 : frq->n_stats) + frq->n_show_percentiles + 2); + + tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ; + + + tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1); + tab_vline (t, TAL_GAP , 1, 0, tab_nr(t) - 1 ) ; + + r = 2; /* N missing and N valid are always dumped */ + + for (i = 0; i < FRQ_ST_count; i++) + { + if (FRQ_ST_MEDIAN == i) + continue; + + if (frq->stats & BIT_INDEX (i)) + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, + gettext (st_name[i])); + tab_double (t, 2, r, TAB_NONE, stat_value[i], NULL); + r++; + } + } + + tab_text (t, 0, 0, TAB_LEFT | TAT_TITLE, _("N")); + tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid")); + tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing")); + + tab_double (t, 2, 0, TAB_NONE, ft->valid_cases, wfmt); + tab_double (t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, wfmt); + + for (i = 0; i < frq->n_percentiles; i++) + { + const struct percentile *pc = &frq->percentiles[i]; + + if (!pc->show) + continue; + + if ( i == 0 ) + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles")); + } + + if (pc->p == 0.5) + tab_text (t, 1, r, TAB_LEFT, _("50 (Median)")); + else + tab_fixed (t, 1, r, TAB_LEFT, pc->p * 100, 3, 0); + tab_double (t, 2, r, TAB_NONE, pc->value, + var_get_print_format (vf->var)); + r++; + } + + tab_title (t, "%s", var_to_string (vf->var)); + + tab_submit (t); +} -/* - Local Variables: - mode: c - End: -*/ -- 2.30.2