X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Ffrequencies.q;h=fa380dea0fef49b18a78a4b94b93da1f483a6759;hb=fad826ff86720f76220b05e00dc7dfa46e418859;hp=d40267e26bac7b92d555f01625e749e6c746bd29;hpb=2e0595dd8e344dbdcab740d7d2a3b67d153d6b39;p=pspp-builds.git diff --git a/src/frequencies.q b/src/frequencies.q index d40267e2..fa380dea 100644 --- a/src/frequencies.q +++ b/src/frequencies.q @@ -24,29 +24,35 @@ */ #include -#include +#include "error.h" #include #include +#include + #include "alloc.h" -#include "avl.h" #include "bitvector.h" +#include "case.h" +#include "dictionary.h" #include "hash.h" #include "pool.h" #include "command.h" #include "lexer.h" +#include "moments.h" #include "error.h" -#include "approx.h" +#include "algorithm.h" #include "magic.h" #include "misc.h" -#include "stats.h" #include "output.h" #include "som.h" +#include "str.h" #include "tab.h" +#include "value-labels.h" #include "var.h" #include "vfm.h" +#include "settings.h" +#include "chart.h" +/* (headers) */ -#undef DEBUGGING -/*#define DEBUGGING 1 */ #include "debug-print.h" /* (specification) @@ -62,6 +68,9 @@ barchart(ba_)=:minimum(d:min), :maximum(d:max), scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"); + piechart(pie_)=:minimum(d:min), + :maximum(d:max), + missing:missing/!nomissing; histogram(hi_)=:minimum(d:min), :maximum(d:max), scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"), @@ -73,8 +82,8 @@ norm:!nonormal/normal, incr:increment(d:inc,"%s>0"); grouped=custom; - ntiles=custom; - percentiles=custom; + ntiles=integer; + +percentiles = double list; statistics[st_]=1|mean,2|semean,3|median,4|mode,5|stddev,6|variance, 7|kurtosis,8|skewness,9|range,10|minimum,11|maximum,12|sum, 13|default,14|seskewness,15|sekurtosis,all,none. @@ -82,6 +91,14 @@ /* (declarations) */ /* (functions) */ +/* Statistics. */ +enum + { + frq_mean = 0, frq_semean, frq_median, frq_mode, frq_stddev, frq_variance, + frq_kurt, frq_sekurt, frq_skew, frq_seskew, frq_range, frq_min, frq_max, + frq_sum, frq_n_stats + }; + /* Description of a statistic. */ struct frq_info { @@ -110,9 +127,25 @@ static struct frq_info st_name[frq_n_stats + 1] = }; /* Percentiles to calculate. */ -static double *percentiles; + +struct percentile +{ + double p; /* the %ile to be calculated */ + double value; /* the %ile's value */ + double x1; /* The datum value <= the percentile */ + double x2; /* The datum value >= the percentile */ + int flag; + int flag2; /* Set to 1 if this percentile value has been found */ +}; + + +static void add_percentile (double x) ; + +static struct percentile *percentiles; static int n_percentiles; +static int implicit_50th ; + /* Groups of statistics. */ #define BI BIT_INDEX #define frq_default \ @@ -134,6 +167,7 @@ enum GFT_NONE, /* Don't draw graphs. */ GFT_BAR, /* Draw bar charts. */ GFT_HIST, /* Draw histograms. */ + GFT_PIE, /* Draw piechart */ GFT_HBAR /* Draw bar charts or histograms at our discretion. */ }; @@ -141,7 +175,8 @@ enum static struct cmd_frequencies cmd; /* Summary of the barchart, histogram, and hbar subcommands. */ -static int chart; /* NONE/BAR/HIST/HBAR. */ +/* FIXME: These should not be mututally exclusive */ +static int chart; /* NONE/BAR/HIST/HBAR/PIE. */ static double min, max; /* Minimum, maximum on y axis. */ static int format; /* FREQ/PERCENT: Scaling of y axis. */ static double scale, incr; /* FIXME */ @@ -155,15 +190,78 @@ static struct variable **v_variables; static struct pool *int_pool; /* Integer mode. */ static struct pool *gen_pool; /* General mode. */ -/* Easier access to a_statistics. */ -#define stat cmd.a_statistics +/* Frequency tables. */ + +/* Frequency table entry. */ +struct freq + { + union value v; /* The value. */ + double c; /* The number of occurrences of the value. */ + }; + +/* Types of frequency tables. */ +enum + { + FRQM_GENERAL, + FRQM_INTEGER + }; + +/* Entire frequency table. */ +struct freq_tab + { + int mode; /* FRQM_GENERAL or FRQM_INTEGER. */ + + /* General mode. */ + struct hsh_table *data; /* Undifferentiated data. */ + + /* Integer mode. */ + double *vector; /* Frequencies proper. */ + int min, max; /* The boundaries of the table. */ + double out_of_range; /* Sum of weights of out-of-range values. */ + double sysmis; /* Sum of weights of SYSMIS values. */ + + /* All modes. */ + struct freq *valid; /* Valid freqs. */ + int n_valid; /* Number of total freqs. */ + + struct freq *missing; /* Missing freqs. */ + int n_missing; /* Number of missing freqs. */ + + /* Statistics. */ + double total_cases; /* Sum of weights of all cases. */ + double valid_cases; /* Sum of weights of valid cases. */ + }; + + +/* Per-variable frequency data. */ +struct var_freqs + { + /* Freqency table. */ + struct freq_tab tab; /* Frequencies table to use. */ + + /* Percentiles. */ + int n_groups; /* Number of groups. */ + double *groups; /* Groups. */ + + /* Statistics. */ + double stat[frq_n_stats]; + }; + +static inline struct var_freqs * +get_var_freqs (struct variable *v) +{ + assert (v != NULL); + assert (v->aux != NULL); + return v->aux; +} static void determine_charts (void); -static void precalc (void); -static int calc_weighting (struct ccase *); -static int calc_no_weight (struct ccase *); -static void postcalc (void); +static void calc_stats (struct variable *v, double d[frq_n_stats]); + +static void precalc (void *); +static int calc (struct ccase *, void *); +static void postcalc (void *); static void postprocess_freq_tab (struct variable *); static void dump_full (struct variable *); @@ -171,14 +269,20 @@ static void dump_condensed (struct variable *); static void dump_statistics (struct variable *, int show_varname); static void cleanup_freq_tab (struct variable *); -static int compare_value_numeric_a (const void *, const void *, void *); -static int compare_value_alpha_a (const void *, const void *, void *); -static int compare_value_numeric_d (const void *, const void *, void *); -static int compare_value_alpha_d (const void *, const void *, void *); -static int compare_freq_numeric_a (const void *, const void *, void *); -static int compare_freq_alpha_a (const void *, const void *, void *); -static int compare_freq_numeric_d (const void *, const void *, void *); -static int compare_freq_alpha_d (const void *, const void *, void *); +static hsh_hash_func hash_value_numeric, hash_value_alpha; +static hsh_compare_func compare_value_numeric_a, compare_value_alpha_a; +static hsh_compare_func compare_value_numeric_d, compare_value_alpha_d; +static hsh_compare_func compare_freq_numeric_a, compare_freq_alpha_a; +static hsh_compare_func compare_freq_numeric_d, compare_freq_alpha_d; + + +static void do_piechart(const struct variable *var, + const struct freq_tab *frq_tab); + +gsl_histogram * +freq_tab_to_hist(const struct freq_tab *ft, const struct variable *var); + + /* Parser and outline. */ @@ -192,15 +296,17 @@ cmd_frequencies (void) int_pool = pool_create (); result = internal_cmd_frequencies (); pool_destroy (int_pool); + int_pool=0; pool_destroy (gen_pool); + gen_pool=0; free (v_variables); + v_variables=0; return result; } static int internal_cmd_frequencies (void) { - int (*calc) (struct ccase *); int i; n_percentiles = 0; @@ -209,10 +315,6 @@ internal_cmd_frequencies (void) n_variables = 0; v_variables = NULL; - for (i = 0; i < default_dict.nvar; i++) - default_dict.var[i]->foo = 0; - - lex_match_id ("FREQUENCIES"); if (!parse_frequencies (&cmd)) return CMD_FAILURE; @@ -221,14 +323,14 @@ internal_cmd_frequencies (void) /* Figure out statistics to calculate. */ stats = 0; - if (stat[FRQ_ST_DEFAULT] || !cmd.sbc_statistics) + if (cmd.a_statistics[FRQ_ST_DEFAULT] || !cmd.sbc_statistics) stats |= frq_default; - if (stat[FRQ_ST_ALL]) + if (cmd.a_statistics[FRQ_ST_ALL]) stats |= frq_all; if (cmd.sort != FRQ_AVALUE && cmd.sort != FRQ_DVALUE) stats &= ~frq_median; for (i = 0; i < frq_n_stats; i++) - if (stat[st_name[i].st_indx]) + if (cmd.a_statistics[st_name[i].st_indx]) stats |= BIT_INDEX (i); if (stats & frq_kurt) stats |= frq_sekurt; @@ -246,10 +348,32 @@ internal_cmd_frequencies (void) if (chart != GFT_NONE || cmd.sbc_ntiles) cmd.sort = FRQ_AVALUE; + /* Work out what percentiles need to be calculated */ + if ( cmd.sbc_percentiles ) + { + for ( i = 0 ; i < MAXLISTS ; ++i ) + { + int pl; + subc_list_double *ptl_list = &cmd.dl_percentiles[i]; + for ( pl = 0 ; pl < subc_list_double_count(ptl_list); ++pl) + add_percentile(subc_list_double_at(ptl_list,pl) / 100.0 ); + } + } + if ( cmd.sbc_ntiles ) + { + for ( i = 0 ; i < cmd.sbc_ntiles ; ++i ) + { + int j; + for (j = 0; j <= cmd.n_ntiles[i]; ++j ) + add_percentile(j / (double) cmd.n_ntiles[i]); + } + } + + /* Do it! */ - update_weighting (&default_dict); - calc = default_dict.weight_index == -1 ? calc_no_weight : calc_weighting; - procedure (precalc, calc, postcalc); + procedure_with_splits (precalc, calc, postcalc, NULL); + + free_frequencies(&cmd); return CMD_SUCCESS; } @@ -258,7 +382,8 @@ internal_cmd_frequencies (void) static void determine_charts (void) { - int count = (!!cmd.sbc_histogram) + (!!cmd.sbc_barchart) + (!!cmd.sbc_hbar); + int count = (!!cmd.sbc_histogram) + (!!cmd.sbc_barchart) + + (!!cmd.sbc_hbar) + (!!cmd.sbc_piechart); if (!count) { @@ -276,6 +401,8 @@ determine_charts (void) chart = GFT_HIST; else if (cmd.sbc_barchart) chart = GFT_BAR; + else if (cmd.sbc_piechart) + chart = GFT_PIE; else chart = GFT_HBAR; @@ -319,7 +446,7 @@ determine_charts (void) format = FRQ_PERCENT; scale = cmd.ba_pcnt; } - if (cmd.hi_norm) + if (cmd.hi_norm != FRQ_NONORMAL ) normal = 1; if (cmd.hi_incr == FRQ_INCREMENT) incr = cmd.hi_inc; @@ -356,17 +483,64 @@ determine_charts (void) } } -/* Generate each calc_*(). */ -#define WEIGHTING 0 -#include "frequencies.g" +/* Add data from case C to the frequency table. */ +static int +calc (struct ccase *c, void *aux UNUSED) +{ + double weight; + int i; + int bad_warn = 1; + + weight = dict_get_case_weight (default_dict, c, &bad_warn); -#define WEIGHTING 1 -#include "frequencies.g" + for (i = 0; i < n_variables; i++) + { + struct variable *v = v_variables[i]; + const union value *val = case_data (c, v->fv); + struct freq_tab *ft = &get_var_freqs (v)->tab; + + switch (ft->mode) + { + case FRQM_GENERAL: + { + + /* General mode. */ + struct freq **fpp = (struct freq **) hsh_probe (ft->data, val); + + if (*fpp != NULL) + (*fpp)->c += weight; + else + { + struct freq *fp = *fpp = pool_alloc (gen_pool, sizeof *fp); + fp->v = *val; + fp->c = weight; + } + } + break; + case FRQM_INTEGER: + /* Integer mode. */ + if (val->f == SYSMIS) + ft->sysmis += weight; + else if (val->f > INT_MIN+1 && val->f < INT_MAX-1) + { + int i = val->f; + if (i >= ft->min && i <= ft->max) + ft->vector[i - ft->min] += weight; + } + else + ft->out_of_range += weight; + break; + default: + assert (0); + } + } + return 1; +} /* Prepares each variable that is the target of FREQUENCIES by setting up its hash table. */ static void -precalc (void) +precalc (void *aux UNUSED) { int i; @@ -376,26 +550,33 @@ precalc (void) for (i = 0; i < n_variables; i++) { struct variable *v = v_variables[i]; + struct freq_tab *ft = &get_var_freqs (v)->tab; - if (v->p.frq.tab.mode == FRQM_GENERAL) + if (ft->mode == FRQM_GENERAL) { - avl_comparison_func compare; - if (v->type == NUMERIC) - compare = compare_value_numeric_a; - else - compare = compare_value_alpha_a; - v->p.frq.tab.tree = avl_create (gen_pool, compare, - (void *) v->width); - v->p.frq.tab.n_missing = 0; + hsh_hash_func *hash; + hsh_compare_func *compare; + + if (v->type == NUMERIC) + { + hash = hash_value_numeric; + compare = compare_value_numeric_a; + } + else + { + hash = hash_value_alpha; + compare = compare_value_alpha_a; + } + ft->data = hsh_create (16, compare, hash, NULL, v); } else { int j; - for (j = (v->p.frq.tab.max - v->p.frq.tab.min); j >= 0; j--) - v->p.frq.tab.vector[j] = 0.0; - v->p.frq.tab.out_of_range = 0.0; - v->p.frq.tab.sysmis = 0.0; + for (j = (ft->max - ft->min); j >= 0; j--) + ft->vector[j] = 0.0; + ft->out_of_range = 0.0; + ft->sysmis = 0.0; } } } @@ -403,20 +584,22 @@ precalc (void) /* Finishes up with the variables after frequencies have been calculated. Displays statistics, percentiles, ... */ static void -postcalc (void) +postcalc (void *aux UNUSED) { int i; for (i = 0; i < n_variables; i++) { struct variable *v = v_variables[i]; + struct var_freqs *vf = get_var_freqs (v); + struct freq_tab *ft = &vf->tab; int n_categories; int dumped_freq_tab = 1; postprocess_freq_tab (v); /* Frequencies tables. */ - n_categories = v->p.frq.tab.n_valid + v->p.frq.tab.n_missing; + n_categories = ft->n_valid + ft->n_missing; if (cmd.table == FRQ_TABLE || (cmd.table == FRQ_LIMIT && n_categories <= cmd.limit)) switch (cmd.cond) @@ -443,143 +626,160 @@ postcalc (void) if (n_stats) dump_statistics (v, !dumped_freq_tab); + + + if ( chart == GFT_HIST) + { + double d[frq_n_stats]; + struct normal_curve norm; + gsl_histogram *hist ; + + + norm.N = vf->tab.valid_cases; + + calc_stats(v,d); + norm.mean = d[frq_mean]; + norm.stddev = d[frq_stddev]; + + hist = freq_tab_to_hist(ft,v); + + histogram_plot(hist, var_to_string(v), &norm, normal); + + gsl_histogram_free(hist); + } + + + if ( chart == GFT_PIE) + { + do_piechart(v_variables[i], ft); + } + + + cleanup_freq_tab (v); + } } -/* Comparison function called by comparison_helper(). */ -static avl_comparison_func comparison_func; - -/* Passed to comparison function by comparison_helper(). */ -static void *comparison_param; - -/* Used by postprocess_freq_tab to re-sort frequency tables. */ -static int -comparison_helper (const void *a, const void *b) +/* Returns the comparison function that should be used for + sorting a frequency table by FRQ_SORT using VAR_TYPE + variables. */ +static hsh_compare_func * +get_freq_comparator (int frq_sort, int var_type) { - return comparison_func (&((struct freq *) a)->v, - &((struct freq *) b)->v, comparison_param); + /* Note that q2c generates tags beginning with 1000. */ + switch (frq_sort | (var_type << 16)) + { + case FRQ_AVALUE | (NUMERIC << 16): return compare_value_numeric_a; + case FRQ_AVALUE | (ALPHA << 16): return compare_value_alpha_a; + case FRQ_DVALUE | (NUMERIC << 16): return compare_value_numeric_d; + case FRQ_DVALUE | (ALPHA << 16): return compare_value_alpha_d; + case FRQ_AFREQ | (NUMERIC << 16): return compare_freq_numeric_a; + case FRQ_AFREQ | (ALPHA << 16): return compare_freq_alpha_a; + case FRQ_DFREQ | (NUMERIC << 16): return compare_freq_numeric_d; + case FRQ_DFREQ | (ALPHA << 16): return compare_freq_alpha_d; + default: assert (0); + } + + return 0; } -/* Used by postprocess_freq_tab to construct the array members valid, - missing of freq_tab. */ -static void -add_freq (void *data, void *param) +/* Returns nonzero iff the value in struct freq F is non-missing + for variable V. */ +static int +not_missing (const void *f_, void *v_) { - struct freq *f = data; - struct variable *v = param; - - v->p.frq.tab.total_cases += f->c; + const struct freq *f = f_; + struct variable *v = v_; - if ((v->type == NUMERIC && f->v.f == SYSMIS) - || (cmd.miss == FRQ_EXCLUDE && is_user_missing (&f->v, v))) - { - *v->p.frq.tab.missing++ = *f; - v->p.frq.tab.valid_cases -= f->c; - } - else - *v->p.frq.tab.valid++ = *f; + return !is_missing (&f->v, v); } +/* Summarizes the frequency table data for variable V. */ static void -postprocess_freq_tab (struct variable * v) +postprocess_freq_tab (struct variable *v) { - avl_comparison_func compare; - - switch (cmd.sort | (v->type << 16)) + hsh_compare_func *compare; + struct freq_tab *ft; + size_t count; + void **data; + struct freq *freqs, *f; + size_t i; + + ft = &get_var_freqs (v)->tab; + assert (ft->mode == FRQM_GENERAL); + compare = get_freq_comparator (cmd.sort, v->type); + + /* Extract data from hash table. */ + count = hsh_count (ft->data); + data = hsh_data (ft->data); + + /* Copy dereferenced data into freqs. */ + freqs = xmalloc (count * sizeof *freqs); + for (i = 0; i < count; i++) { - /* Note that q2c generates tags beginning with 1000. */ - case FRQ_AVALUE | (NUMERIC << 16): - compare = NULL; - break; - case FRQ_AVALUE | (ALPHA << 16): - compare = NULL; - break; - case FRQ_DVALUE | (NUMERIC << 16): - comparison_func = compare_value_numeric_d; - break; - case FRQ_DVALUE | (ALPHA << 16): - compare = compare_value_alpha_d; - break; - case FRQ_AFREQ | (NUMERIC << 16): - compare = compare_freq_numeric_a; - break; - case FRQ_AFREQ | (ALPHA << 16): - compare = compare_freq_alpha_a; - break; - case FRQ_DFREQ | (NUMERIC << 16): - compare = compare_freq_numeric_d; - break; - case FRQ_DFREQ | (ALPHA << 16): - compare = compare_freq_alpha_d; - break; - default: - assert (0); + struct freq *f = data[i]; + freqs[i] = *f; } - comparison_func = compare; - if (v->p.frq.tab.mode == FRQM_GENERAL) - { - int total; - struct freq_tab *ft = &v->p.frq.tab; + /* Put data into ft. */ + ft->valid = freqs; + ft->n_valid = partition (freqs, count, sizeof *freqs, not_missing, v); + ft->missing = freqs + ft->n_valid; + ft->n_missing = count - ft->n_valid; - total = avl_count (ft->tree); - ft->n_valid = total - ft->n_missing; - ft->valid = xmalloc (sizeof (struct freq) * total); - ft->missing = &ft->valid[ft->n_valid]; - ft->valid_cases = ft->total_cases = 0.0; + /* Sort data. */ + sort (ft->valid, ft->n_valid, sizeof *ft->valid, compare, v); + sort (ft->missing, ft->n_missing, sizeof *ft->missing, compare, v); - avl_walk (ft->tree, add_freq, (void *) v); + /* Summary statistics. */ + ft->valid_cases = 0.0; + for(i = 0 ; i < ft->n_valid ; ++i ) + { + f = &ft->valid[i]; + ft->valid_cases += f->c; - ft->valid -= ft->n_valid; - ft->missing -= ft->n_missing; - ft->valid_cases += ft->total_cases; + } - if (compare) - { - qsort (ft->valid, ft->n_valid, sizeof (struct freq), comparison_helper); - qsort (ft->missing, ft->n_missing, sizeof (struct freq), comparison_helper); - } + ft->total_cases = ft->valid_cases ; + for(i = 0 ; i < ft->n_missing ; ++i ) + { + f = &ft->missing[i]; + ft->total_cases += f->c; } - else - assert (0); + } +/* Frees the frequency table for variable V. */ static void -cleanup_freq_tab (struct variable * v) +cleanup_freq_tab (struct variable *v) { - if (v->p.frq.tab.mode == FRQM_GENERAL) - { - struct freq_tab *ft = &v->p.frq.tab; - - free (ft->valid); - } - else - assert (0); + struct freq_tab *ft = &get_var_freqs (v)->tab; + assert (ft->mode == FRQM_GENERAL); + free (ft->valid); + hsh_destroy (ft->data); } /* Parses the VARIABLES subcommand, adding to {n_variables,v_variables}. */ static int -frq_custom_variables (struct cmd_frequencies *cmd unused) +frq_custom_variables (struct cmd_frequencies *cmd UNUSED) { int mode; - int min, max; + int min = 0, max = 0; int old_n_variables = n_variables; int i; lex_match ('='); - if (token != T_ALL && (token != T_ID || !is_varname (tokid))) + if (token != T_ALL && (token != T_ID + || dict_lookup_var (default_dict, tokid) == NULL)) return 2; - if (!parse_variables (NULL, &v_variables, &n_variables, + if (!parse_variables (default_dict, &v_variables, &n_variables, PV_APPEND | PV_NO_SCRATCH)) return 0; - for (i = old_n_variables; i < n_variables; i++) - v_variables[i]->p.frq.tab.mode = FRQM_GENERAL; - if (!lex_match ('(')) mode = FRQM_GENERAL; else @@ -608,48 +808,47 @@ frq_custom_variables (struct cmd_frequencies *cmd unused) for (i = old_n_variables; i < n_variables; i++) { struct variable *v = v_variables[i]; + struct var_freqs *vf; - if (v->foo != 0) + if (v->aux != NULL) { msg (SE, _("Variable %s specified multiple times on VARIABLES " "subcommand."), v->name); return 0; } - - v->foo = 1; /* Used simply as a marker. */ - - v->p.frq.tab.valid = v->p.frq.tab.missing = NULL; - + if (mode == FRQM_INTEGER && v->type != NUMERIC) + { + msg (SE, _("Integer mode specified, but %s is not a numeric " + "variable."), v->name); + return 0; + } + + vf = var_attach_aux (v, xmalloc (sizeof *vf), var_dtor_free); + vf->tab.mode = mode; + vf->tab.valid = vf->tab.missing = NULL; if (mode == FRQM_INTEGER) { - if (v->type != NUMERIC) - { - msg (SE, _("Integer mode specified, but %s is not a numeric " - "variable."), v->name); - return 0; - } - - v->p.frq.tab.min = min; - v->p.frq.tab.max = max; - v->p.frq.tab.vector = pool_alloc (int_pool, - sizeof (struct freq) * (max - min + 1)); + vf->tab.min = min; + vf->tab.max = max; + vf->tab.vector = pool_alloc (int_pool, + sizeof (struct freq) * (max - min + 1)); } else - v->p.frq.tab.vector = NULL; - - v->p.frq.n_groups = 0; - v->p.frq.groups = NULL; + vf->tab.vector = NULL; + vf->n_groups = 0; + vf->groups = NULL; } return 1; } -/* Parses the GROUPED subcommand, setting the frq.{n_grouped,grouped} +/* Parses the GROUPED subcommand, setting the n_grouped, grouped fields of specified variables. */ static int -frq_custom_grouped (struct cmd_frequencies *cmd unused) +frq_custom_grouped (struct cmd_frequencies *cmd UNUSED) { lex_match ('='); - if ((token == T_ID && is_varname (tokid)) || token == T_ID) + if ((token == T_ID && dict_lookup_var (default_dict, tokid) != NULL) + || token == T_ID) for (;;) { int i; @@ -662,13 +861,14 @@ frq_custom_grouped (struct cmd_frequencies *cmd unused) int n; struct variable **v; - if (!parse_variables (NULL, &v, &n, PV_NO_DUPLICATE | PV_NUMERIC)) + if (!parse_variables (default_dict, &v, &n, + PV_NO_DUPLICATE | PV_NUMERIC)) return 0; if (lex_match ('(')) { nl = ml = 0; dl = NULL; - while (token == T_NUM) + while (lex_integer ()) { if (nl >= ml) { @@ -688,27 +888,34 @@ frq_custom_grouped (struct cmd_frequencies *cmd unused) return 0; } } - else - nl = 0; + else + { + nl = 0; + dl = NULL; + } for (i = 0; i < n; i++) - { - if (v[i]->foo == 0) - msg (SE, _("Variables %s specified on GROUPED but not on " - "VARIABLES."), v[i]->name); - if (v[i]->p.frq.groups != NULL) - msg (SE, _("Variables %s specified multiple times on GROUPED " - "subcommand."), v[i]->name); - else - { - v[i]->p.frq.n_groups = nl; - v[i]->p.frq.groups = dl; - } - } + if (v[i]->aux == NULL) + msg (SE, _("Variables %s specified on GROUPED but not on " + "VARIABLES."), v[i]->name); + else + { + struct var_freqs *vf = get_var_freqs (v[i]); + + if (vf->groups != NULL) + msg (SE, _("Variables %s specified multiple times on GROUPED " + "subcommand."), v[i]->name); + else + { + vf->n_groups = nl; + vf->groups = dl; + } + } free (v); if (!lex_match ('/')) break; - if ((token != T_ID || !is_varname (tokid)) && token != T_ALL) + if ((token != T_ID || dict_lookup_var (default_dict, tokid) != NULL) + && token != T_ALL) { lex_put_back ('/'); break; @@ -726,129 +933,164 @@ add_percentile (double x) int i; for (i = 0; i < n_percentiles; i++) - if (x <= percentiles[i]) - break; - if (i >= n_percentiles || tokval != percentiles[i]) { - percentiles = pool_realloc (int_pool, percentiles, - (n_percentiles + 1) * sizeof (double)); + /* Do nothing if it's already in the list */ + if ( fabs(x - percentiles[i].p) < DBL_EPSILON ) + return; + + if (x < percentiles[i].p) + break; + } + + if (i >= n_percentiles || tokval != percentiles[i].p) + { + percentiles + = pool_realloc (int_pool, percentiles, + (n_percentiles + 1) * sizeof (struct percentile )); + if (i < n_percentiles) - memmove (&percentiles[i + 1], &percentiles[i], - (n_percentiles - i) * sizeof (double)); - percentiles[i] = x; + memmove (&percentiles[i + 1], &percentiles[i], + (n_percentiles - i) * sizeof (struct percentile) ); + + percentiles[i].p = x; n_percentiles++; } } -/* Parses the PERCENTILES subcommand, adding user-specified - percentiles to the list. */ -static int -frq_custom_percentiles (struct cmd_frequencies *cmd unused) +/* Comparison functions. */ + +/* Hash of numeric values. */ +static unsigned +hash_value_numeric (const void *value_, void *foo UNUSED) { - lex_match ('='); - if (token != T_NUM) - { - msg (SE, _("Percentile list expected after PERCENTILES.")); - return 0; - } - - do - { - if (tokval <= 0 || tokval >= 100) - { - msg (SE, _("Percentiles must be greater than " - "0 and less than 100.")); - return 0; - } - - add_percentile (tokval / 100.0); - lex_get (); - lex_match (','); - } - while (token == T_NUM); - return 1; + const struct freq *value = value_; + return hsh_hash_double (value->v.f); } -/* Parses the NTILES subcommand, adding the percentiles that - correspond to the specified evenly-distributed ntiles. */ -static int -frq_custom_ntiles (struct cmd_frequencies *cmd unused) +/* Hash of string values. */ +static unsigned +hash_value_alpha (const void *value_, void *v_) { - int i; + const struct freq *value = value_; + struct variable *v = v_; - lex_match ('='); - if (!lex_force_int ()) - return 0; - for (i = 1; i < lex_integer (); i++) - add_percentile (1.0 / lex_integer () * i); - lex_get (); - return 1; + return hsh_hash_bytes (value->v.s, v->width); } - -/* Comparison functions. */ /* Ascending numeric compare of values. */ static int -compare_value_numeric_a (const void *a, const void *b, void *foo unused) +compare_value_numeric_a (const void *a_, const void *b_, void *foo UNUSED) { - return approx_compare (((struct freq *) a)->v.f, ((struct freq *) b)->v.f); + const struct freq *a = a_; + const struct freq *b = b_; + + if (a->v.f > b->v.f) + return 1; + else if (a->v.f < b->v.f) + return -1; + else + return 0; } /* Ascending string compare of values. */ static int -compare_value_alpha_a (const void *a, const void *b, void *len) +compare_value_alpha_a (const void *a_, const void *b_, void *v_) { - return memcmp (((struct freq *) a)->v.s, ((struct freq *) b)->v.s, (int) len); + const struct freq *a = a_; + const struct freq *b = b_; + const struct variable *v = v_; + + return memcmp (a->v.s, b->v.s, v->width); } /* Descending numeric compare of values. */ static int -compare_value_numeric_d (const void *a, const void *b, void *foo unused) +compare_value_numeric_d (const void *a, const void *b, void *foo UNUSED) { - return approx_compare (((struct freq *) b)->v.f, ((struct freq *) a)->v.f); + return -compare_value_numeric_a (a, b, foo); } /* Descending string compare of values. */ static int -compare_value_alpha_d (const void *a, const void *b, void *len) +compare_value_alpha_d (const void *a, const void *b, void *v) { - return memcmp (((struct freq *) b)->v.s, ((struct freq *) a)->v.s, (int) len); + return -compare_value_alpha_a (a, b, v); } /* Ascending numeric compare of frequency; secondary key on ascending numeric value. */ static int -compare_freq_numeric_a (const void *a, const void *b, void *foo unused) +compare_freq_numeric_a (const void *a_, const void *b_, void *foo UNUSED) { - int x = approx_compare (((struct freq *) a)->c, ((struct freq *) b)->c); - return x ? x : approx_compare (((struct freq *) a)->v.f, ((struct freq *) b)->v.f); + const struct freq *a = a_; + const struct freq *b = b_; + + if (a->c > b->c) + return 1; + else if (a->c < b->c) + return -1; + + if (a->v.f > b->v.f) + return 1; + else if (a->v.f < b->v.f) + return -1; + else + return 0; } /* Ascending numeric compare of frequency; secondary key on ascending string value. */ static int -compare_freq_alpha_a (const void *a, const void *b, void *len) +compare_freq_alpha_a (const void *a_, const void *b_, void *v_) { - int x = approx_compare (((struct freq *) a)->c, ((struct freq *) b)->c); - return x ? x : memcmp (((struct freq *) a)->v.s, ((struct freq *) b)->v.s, (int) len); + const struct freq *a = a_; + const struct freq *b = b_; + const struct variable *v = v_; + + if (a->c > b->c) + return 1; + else if (a->c < b->c) + return -1; + else + return memcmp (a->v.s, b->v.s, v->width); } /* Descending numeric compare of frequency; secondary key on ascending numeric value. */ static int -compare_freq_numeric_d (const void *a, const void *b, void *foo unused) +compare_freq_numeric_d (const void *a_, const void *b_, void *foo UNUSED) { - int x = approx_compare (((struct freq *) b)->c, ((struct freq *) a)->c); - return x ? x : approx_compare (((struct freq *) a)->v.f, ((struct freq *) b)->v.f); + const struct freq *a = a_; + const struct freq *b = b_; + + if (a->c > b->c) + return -1; + else if (a->c < b->c) + return 1; + + if (a->v.f > b->v.f) + return 1; + else if (a->v.f < b->v.f) + return -1; + else + return 0; } /* Descending numeric compare of frequency; secondary key on ascending string value. */ static int -compare_freq_alpha_d (const void *a, const void *b, void *len) +compare_freq_alpha_d (const void *a_, const void *b_, void *v_) { - int x = approx_compare (((struct freq *) b)->c, ((struct freq *) a)->c); - return x ? x : memcmp (((struct freq *) a)->v.s, ((struct freq *) b)->v.s, (int) len); + const struct freq *a = a_; + const struct freq *b = b_; + const struct variable *v = v_; + + if (a->c > b->c) + return -1; + else if (a->c < b->c) + return 1; + else + return memcmp (a->v.s, b->v.s, v->width); } /* Frequency table display. */ @@ -871,13 +1113,14 @@ full_dim (struct tab_table *t, struct outp_driver *d) /* Displays a full frequency table for variable V. */ static void -dump_full (struct variable * v) +dump_full (struct variable *v) { int n_categories; + struct freq_tab *ft; struct freq *f; struct tab_table *t; int r; - double cum_percent = 0.0; + double cum_total = 0.0; double cum_freq = 0.0; struct init @@ -906,7 +1149,8 @@ dump_full (struct variable * v) int lab = cmd.labels == FRQ_LABELS; - n_categories = v->p.frq.tab.n_valid + v->p.frq.tab.n_missing; + ft = &get_var_freqs (v)->tab; + n_categories = ft->n_valid + ft->n_missing; t = tab_create (5 + lab, n_categories + 3, 0); tab_headers (t, 0, 0, 2, 0); tab_dim (t, full_dim); @@ -918,19 +1162,19 @@ dump_full (struct variable * v) TAB_CENTER | TAT_TITLE, gettext (p->s)); r = 2; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) + for (f = ft->valid; f < ft->missing; f++) { double percent, valid_percent; cum_freq += f->c; - percent = f->c / v->p.frq.tab.total_cases * 100.0; - valid_percent = f->c / v->p.frq.tab.valid_cases * 100.0; - cum_percent += valid_percent; + percent = f->c / ft->total_cases * 100.0; + valid_percent = f->c / ft->valid_cases * 100.0; + cum_total += valid_percent; if (lab) { - char *label = get_val_lab (v, f->v, 0); + const char *label = val_labs_find (v->val_labs, f->v); if (label != NULL) tab_text (t, 0, r, TAB_LEFT, label); } @@ -939,16 +1183,16 @@ dump_full (struct variable * v) tab_float (t, 1 + lab, r, TAB_NONE, f->c, 8, 0); tab_float (t, 2 + lab, r, TAB_NONE, percent, 5, 1); tab_float (t, 3 + lab, r, TAB_NONE, valid_percent, 5, 1); - tab_float (t, 4 + lab, r, TAB_NONE, cum_percent, 5, 1); + tab_float (t, 4 + lab, r, TAB_NONE, cum_total, 5, 1); r++; } - for (; f < &v->p.frq.tab.valid[n_categories]; f++) + for (; f < &ft->valid[n_categories]; f++) { cum_freq += f->c; if (lab) { - char *label = get_val_lab (v, f->v, 0); + const char *label = val_labs_find (v->val_labs, f->v); if (label != NULL) tab_text (t, 0, r, TAB_LEFT, label); } @@ -956,7 +1200,7 @@ dump_full (struct variable * v) tab_value (t, 0 + lab, r, TAB_NONE, &f->v, &v->print); tab_float (t, 1 + lab, r, TAB_NONE, f->c, 8, 0); tab_float (t, 2 + lab, r, TAB_NONE, - f->c / v->p.frq.tab.total_cases * 100.0, 5, 1); + f->c / ft->total_cases * 100.0, 5, 1); tab_text (t, 3 + lab, r, TAB_NONE, _("Missing")); r++; } @@ -974,6 +1218,7 @@ dump_full (struct variable * v) tab_title (t, 1, "%s: %s", v->name, v->label ? v->label : ""); tab_submit (t); + } /* Sets the widths of all the columns and heights of all the rows in @@ -997,15 +1242,17 @@ condensed_dim (struct tab_table *t, struct outp_driver *d) /* Display condensed frequency table for variable V. */ static void -dump_condensed (struct variable * v) +dump_condensed (struct variable *v) { int n_categories; + struct freq_tab *ft; struct freq *f; struct tab_table *t; int r; - double cum_percent = 0.0; + double cum_total = 0.0; - n_categories = v->p.frq.tab.n_valid + v->p.frq.tab.n_missing; + ft = &get_var_freqs (v)->tab; + n_categories = ft->n_valid + ft->n_missing; t = tab_create (4, n_categories + 2, 0); tab_headers (t, 0, 0, 2, 0); @@ -1017,25 +1264,25 @@ dump_condensed (struct variable * v) tab_dim (t, condensed_dim); r = 2; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) + for (f = ft->valid; f < ft->missing; f++) { double percent; - percent = f->c / v->p.frq.tab.total_cases * 100.0; - cum_percent += f->c / v->p.frq.tab.valid_cases * 100.0; + percent = f->c / ft->total_cases * 100.0; + cum_total += f->c / ft->valid_cases * 100.0; tab_value (t, 0, r, TAB_NONE, &f->v, &v->print); tab_float (t, 1, r, TAB_NONE, f->c, 8, 0); tab_float (t, 2, r, TAB_NONE, percent, 3, 0); - tab_float (t, 3, r, TAB_NONE, cum_percent, 3, 0); + tab_float (t, 3, r, TAB_NONE, cum_total, 3, 0); r++; } - for (; f < &v->p.frq.tab.valid[n_categories]; f++) + for (; f < &ft->valid[n_categories]; f++) { tab_value (t, 0, r, TAB_NONE, &f->v, &v->print); tab_float (t, 1, r, TAB_NONE, f->c, 8, 0); tab_float (t, 2, r, TAB_NONE, - f->c / v->p.frq.tab.total_cases * 100.0, 3, 0); + f->c / ft->total_cases * 100.0, 3, 0); r++; } @@ -1053,81 +1300,184 @@ dump_condensed (struct variable * v) /* Calculates all the pertinent statistics for variable V, putting them in array D[]. FIXME: This could be made much more optimal. */ static void -calc_stats (struct variable * v, double d[frq_n_stats]) +calc_stats (struct variable *v, double d[frq_n_stats]) { - double W = v->p.frq.tab.valid_cases; - double X_bar, M2, M3, M4; - struct freq *f; + struct freq_tab *ft = &get_var_freqs (v)->tab; + double W = ft->valid_cases; + struct moments *m; + struct freq *f=0; + int most_often; + double X_mode; + + double rank; + int i = 0; + int idx; + double *median_value; + + /* Calculate percentiles. */ + + /* If the 50th percentile was not explicitly requested then we must + calculate it anyway --- it's the median */ + median_value = 0 ; + for (i = 0; i < n_percentiles; i++) + { + if (percentiles[i].p == 0.5) + { + median_value = &percentiles[i].value; + break; + } + } - /* Calculate the mean. */ - X_bar = 0.0; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) - X_bar += f->v.f * f->c; - X_bar /= W; + if ( 0 == median_value ) + { + add_percentile (0.5); + implicit_50th = 1; + } - /* Calculate moments about the mean. */ - M2 = M3 = M4 = 0.0; - for (f = v->p.frq.tab.valid; f < v->p.frq.tab.missing; f++) + for (i = 0; i < n_percentiles; i++) { - double dev = f->v.f - X_bar; - double tmp; - tmp = dev * dev; - M2 += f->c * tmp; - tmp *= dev; - M3 += f->c * tmp; - tmp *= dev; - M4 += f->c * tmp; + percentiles[i].flag = 0; + percentiles[i].flag2 = 0; } - /* Formulas below are taken from _SPSS Statistical Algorithms_. */ - d[frq_min] = v->p.frq.tab.valid[0].v.f; - d[frq_max] = v->p.frq.tab.missing[-1].v.f; - d[frq_mode] = 0.0; - d[frq_range] = d[frq_max] - d[frq_min]; - d[frq_median] = 0.0; - d[frq_mean] = X_bar; - d[frq_sum] = X_bar * W; - d[frq_variance] = M2 / (W - 1); - d[frq_stddev] = sqrt (d[frq_variance]); - d[frq_semean] = d[frq_stddev] / sqrt (W); - if (W >= 3.0 && d[frq_variance] > 0) + rank = 0; + for (idx = 0; idx < ft->n_valid; ++idx) { - double S = d[frq_stddev]; - d[frq_skew] = (W * M3 / ((W - 1.0) * (W - 2.0) * S * S * S)); - d[frq_seskew] = sqrt (6.0 * W * (W - 1.0) - / ((W - 2.0) * (W + 1.0) * (W + 3.0))); + static double prev_value = SYSMIS; + f = &ft->valid[idx]; + rank += f->c ; + for (i = 0; i < n_percentiles; i++) + { + double tp; + if ( percentiles[i].flag2 ) continue ; + + if ( get_algorithm() != COMPATIBLE ) + tp = + (ft->valid_cases - 1) * percentiles[i].p; + else + tp = + (ft->valid_cases + 1) * percentiles[i].p - 1; + + if ( percentiles[i].flag ) + { + percentiles[i].x2 = f->v.f; + percentiles[i].x1 = prev_value; + percentiles[i].flag2 = 1; + continue; + } + + if (rank > tp ) + { + if ( f->c > 1 && rank - (f->c - 1) > tp ) + { + percentiles[i].x2 = percentiles[i].x1 = f->v.f; + percentiles[i].flag2 = 1; + } + else + { + percentiles[i].flag=1; + } + + continue; + } + } + prev_value = f->v.f; } - else + + for (i = 0; i < n_percentiles; i++) { - d[frq_skew] = d[frq_seskew] = SYSMIS; + /* Catches the case when p == 100% */ + if ( ! percentiles[i].flag2 ) + percentiles[i].x1 = percentiles[i].x2 = f->v.f; + + /* + printf("percentile %d (p==%.2f); X1 = %g; X2 = %g\n", + i,percentiles[i].p,percentiles[i].x1,percentiles[i].x2); + */ } - if (W >= 4.0 && d[frq_variance] > 0) + + for (i = 0; i < n_percentiles; i++) { - double S2 = d[frq_variance]; - double SE_g1 = d[frq_seskew]; + struct freq_tab *ft = &get_var_freqs (v)->tab; + double s; + + double dummy; + if ( get_algorithm() != COMPATIBLE ) + { + s = modf((ft->valid_cases - 1) * percentiles[i].p , &dummy); + } + else + { + s = modf((ft->valid_cases + 1) * percentiles[i].p -1, &dummy); + } + + percentiles[i].value = percentiles[i].x1 + + ( percentiles[i].x2 - percentiles[i].x1) * s ; - d[frq_kurt] = ((W * (W + 1.0) * M4 - 3.0 * M2 * M2 * (W - 1.0)) - / ((W - 1.0) * (W - 2.0) * (W - 3.0) * S2 * S2)); - d[frq_sekurt] = sqrt ((4.0 * (W * W - 1.0) * SE_g1 * SE_g1) - / ((W - 3.0) * (W + 5.0))); + if ( percentiles[i].p == 0.50) + median_value = &percentiles[i].value; } - else + + + /* Calculate the mode. */ + most_often = -1; + X_mode = SYSMIS; + for (f = ft->valid; f < ft->missing; f++) { - d[frq_kurt] = d[frq_sekurt] = SYSMIS; + if (most_often < f->c) + { + most_often = f->c; + X_mode = f->v.f; + } + else if (most_often == f->c) + { + /* A duplicate mode is undefined. + FIXME: keep track of *all* the modes. */ + X_mode = SYSMIS; + } } + + /* Calculate moments. */ + m = moments_create (MOMENT_KURTOSIS); + for (f = ft->valid; f < ft->missing; f++) + moments_pass_one (m, f->v.f, f->c); + for (f = ft->valid; f < ft->missing; f++) + moments_pass_two (m, f->v.f, f->c); + moments_calculate (m, NULL, &d[frq_mean], &d[frq_variance], + &d[frq_skew], &d[frq_kurt]); + moments_destroy (m); + + /* Formulas below are taken from _SPSS Statistical Algorithms_. */ + d[frq_min] = ft->valid[0].v.f; + d[frq_max] = ft->valid[ft->n_valid - 1].v.f; + d[frq_mode] = X_mode; + d[frq_range] = d[frq_max] - d[frq_min]; + d[frq_median] = *median_value; + d[frq_sum] = d[frq_mean] * W; + d[frq_stddev] = sqrt (d[frq_variance]); + d[frq_semean] = d[frq_stddev] / sqrt (W); + d[frq_seskew] = calc_seskew (W); + d[frq_sekurt] = calc_sekurt (W); } /* Displays a table of all the statistics requested for variable V. */ static void -dump_statistics (struct variable * v, int show_varname) +dump_statistics (struct variable *v, int show_varname) { + struct freq_tab *ft; double stat_value[frq_n_stats]; struct tab_table *t; int i, r; + int n_explicit_percentiles = n_percentiles; + + if ( implicit_50th && n_percentiles > 0 ) + --n_percentiles; + if (v->type == ALPHA) return; - if (v->p.frq.tab.n_valid == 0) + ft = &get_var_freqs (v)->tab; + if (ft->n_valid == 0) { msg (SW, _("No valid data for variable %s; statistics not displayed."), v->name); @@ -1135,18 +1485,46 @@ dump_statistics (struct variable * v, int show_varname) } calc_stats (v, stat_value); - t = tab_create (2, n_stats, 0); + t = tab_create (3, n_stats + n_explicit_percentiles + 2, 0); tab_dim (t, tab_natural_dimensions); - tab_vline (t, TAL_1 | TAL_SPACING, 1, 0, n_stats - 1); - for (i = r = 0; i < frq_n_stats; i++) + + tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ; + + + tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1); + tab_vline (t, TAL_1 | TAL_SPACING , 1, 0, tab_nr(t) - 1 ) ; + + r=2; /* N missing and N valid are always dumped */ + + for (i = 0; i < frq_n_stats; i++) if (stats & BIT_INDEX (i)) { tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, gettext (st_name[i].s10)); - tab_float (t, 1, r, TAB_NONE, stat_value[i], 11, 3); + tab_float (t, 2, r, TAB_NONE, stat_value[i], 11, 3); r++; } + tab_text (t, 0, 0, TAB_LEFT | TAT_TITLE, _("N")); + tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid")); + tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing")); + + tab_float(t, 2, 0, TAB_NONE, ft->valid_cases, 11, 0); + tab_float(t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, 11, 0); + + + for (i = 0; i < n_explicit_percentiles; i++, r++) + { + if ( i == 0 ) + { + tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles")); + } + + tab_float (t, 1, r, TAB_LEFT, percentiles[i].p * 100, 3, 0 ); + tab_float (t, 2, r, TAB_NONE, percentiles[i].value, 11, 3); + + } + tab_columns (t, SOM_COL_DOWN, 1); if (show_varname) { @@ -1157,659 +1535,99 @@ dump_statistics (struct variable * v, int show_varname) } else tab_flags (t, SOMF_NO_TITLE); - + + tab_submit (t); } - -#if 0 -/* Statistical calculation. */ - -static int degree[6]; -static int maxdegree, minmax; - -static void stat_func (struct freq *, VISIT, int); -static void calc_stats (int); -static void display_stats (int); - -/* mapping of data[]: - * 0=>8 - * 1=>9 - * 2=>10 - * index 3: number of modes found (detects multiple modes) - * index 4: number of nodes processed, for calculation of median - * 5=>11 - * - * mapping of dbl[]: - * index 0-3: sum of X**i - * index 4: minimum - * index 5: maximum - * index 6: mode - * index 7: median - * index 8: number of cases, valid and missing - * index 9: number of valid cases - * index 10: maximum frequency found, for calculation of mode - * index 11: maximum frequency - */ -static void -out_stats (int i) -{ - int j; - if (cur_var->type == ALPHA) - return; - for (j = 0; j < 8; j++) - cur_var->dbl[j] = 0.; - cur_var->dbl[10] = 0; - cur_var->dbl[4] = DBL_MAX; - cur_var->dbl[5] = -DBL_MAX; - for (j = 2; j < 5; j++) - cur_var->data[j] = 0; - cur_var->p.frq.median_ncases = cur_var->p.frq.t.valid_cases / 2; - avlwalk (cur_var->p.frq.t.f, stat_func, LEFT_TO_RIGHT); - calc_stats (i); - display_stats (i); -} -static void -calc_stats (int i) +/* Create a gsl_histogram from a freq_tab */ +gsl_histogram * +freq_tab_to_hist(const struct freq_tab *ft, const struct variable *var) { - struct variable *v; - double n; - double *d; - - v = v_variables[i]; - n = v->p.frq.t.valid_cases; - d = v->dbl; + int i; + double x_min = DBL_MAX; + double x_max = -DBL_MAX; - if (n < 2 || (n < 3 && stat[FRQ_ST_7])) - { - warn (_("only %g case%s for variable %s, statistics not " - "computed"), n, n == 1 ? "" : "s", v->name); - return; - } - if (stat[FRQ_ST_9]) - v->res[FRQ_ST_9] = d[5] - d[4]; - if (stat[FRQ_ST_10]) - v->res[FRQ_ST_10] = d[4]; - if (stat[FRQ_ST_11]) - v->res[FRQ_ST_11] = d[5]; - if (stat[FRQ_ST_12]) - v->res[FRQ_ST_12] = d[0]; - if (stat[FRQ_ST_1] || stat[FRQ_ST_2] || stat[FRQ_ST_5] || stat[FRQ_ST_6] || stat[FRQ_ST_7]) - { - v->res[FRQ_ST_1] = calc_mean (d, n); - v->res[FRQ_ST_6] = calc_variance (d, n); - } - if (stat[FRQ_ST_2] || stat[FRQ_ST_5] || stat[FRQ_ST_7]) - v->res[FRQ_ST_5] = calc_stddev (v->res[FRQ_ST_6]); - if (stat[FRQ_ST_2]) - v->res[FRQ_ST_2] = calc_semean (v->res[FRQ_ST_5], n); - if (stat[FRQ_ST_7]) - { - v->res[FRQ_ST_7] = calc_kurt (d, n, v->res[FRQ_ST_6]); - v->res[FRQ_ST_14] = calc_sekurt (n); - } - if (stat[FRQ_ST_8]) - { - v->res[FRQ_ST_8] = calc_skew (d, n, v->res[FRQ_ST_5]); - v->res[FRQ_ST_15] = calc_seskew (n); - } - if (stat[FRQ_ST_MODE]) - { - v->res[FRQ_ST_MODE] = v->dbl[6]; - if (v->data[3] > 1) - warn (_("The variable %s has %d modes. The lowest of these " - "is the one given in the table."), v->name, v->data[3]); - } - if (stat[FRQ_ST_MEDIAN]) - v->res[FRQ_ST_MEDIAN] = v->dbl[7]; -} + gsl_histogram *hist; + const double bins = 11; -static void -stat_func (struct freq * x, VISIT order, int param) -{ - double d, f; + struct hsh_iterator hi; + struct hsh_table *fh = ft->data; + struct freq *frq; - if (order != INORDER) - return; - f = d = x->v.f; - cur_var->dbl[0] += (d * x->c); - switch (maxdegree) - { - case 1: - f *= d; - cur_var->dbl[1] += (f * x->c); - break; - case 2: - f *= d; - cur_var->dbl[1] += (f * x->c); - f *= d; - cur_var->dbl[2] += (f * x->c); - break; - case 3: - f *= d; - cur_var->dbl[1] += (f * x->c); - f *= d; - cur_var->dbl[2] += (f * x->c); - f *= d; - cur_var->dbl[3] += (f * x->c); - break; - } - if (minmax) - { - if (d < cur_var->dbl[4]) - cur_var->dbl[4] = d; - if (d > cur_var->dbl[5]) - cur_var->dbl[5] = d; - } - if (x->c > cur_var->dbl[10]) + /* Find out the extremes of the x value */ + for ( frq = hsh_first(fh, &hi); frq != 0; frq = hsh_next(fh, &hi) ) { - cur_var->data[3] = 1; - cur_var->dbl[10] = x->c; - cur_var->dbl[6] = x->v.f; - } - else if (x->c == cur_var->dbl[10]) - cur_var->data[3]++; - if (cur_var->data[4] < cur_var->p.frq.median_ncases - && cur_var->data[4] + x->c >= cur_var->p.frq.median_ncases) - cur_var->dbl[7] = x->v.f; - cur_var->data[4] += x->c; -} - -/* Statistical display. */ -static int column, ncolumns; - -static void outstat (char *, double); - -static void -display_stats (int i) -{ - statname *sp; - struct variable *v; - int nlines; - - v = v_variables[i]; - ncolumns = (margin_width + 3) / 26; - if (ncolumns < 1) - ncolumns = 1; - nlines = sc / ncolumns + (sc % ncolumns > 0); - if (nlines == 2 && sc == 4) - ncolumns = 2; - if (nlines == 3 && sc == 9) - ncolumns = 3; - if (nlines == 4 && sc == 12) - ncolumns = 3; - column = 0; - for (sp = st_name; sp->s != -1; sp++) - if (stat[sp->s] == 1) - outstat (gettext (sp->s10), v->res[sp->s]); - if (column) - out_eol (); - blank_line (); -} + if ( is_missing(&frq->v, var)) + continue; -static void -outstat (char *label, double value) -{ - char buf[128], *cp; - int dw, n; - - cp = &buf[0]; - if (!column) - out_header (); - else - { - memset (buf, ' ', 3); - cp = &buf[3]; + if ( frq->v.f < x_min ) x_min = frq->v.f ; + if ( frq->v.f > x_max ) x_max = frq->v.f ; } - dw = 4; - n = nsprintf (cp, "%-10s %12.4f", label, value); - while (n > 23 && dw > 0) - n = nsprintf (cp, "%-10s %12.*f", label, --dw, value); - outs (buf); - column++; - if (column == ncolumns) - { - column = 0; - out_eol (); - } -} - -/* Graphs. */ - -static rect pb, gb; /* Page border, graph border. */ -static int px, py; /* Page width, height. */ -static int ix, iy; /* Inch width, height. */ - -static void draw_bar_chart (int); -static void draw_histogram (int); -static int scale_dep_axis (int); -static void -out_graphs (int i) -{ - struct variable *v; + hist = histogram_create(bins, x_min, x_max); - v = v_variables[i]; - if (avlcount (cur_var->p.frq.t.f) < 2 - || (chart == HIST && v_variables[i]->type == ALPHA)) - return; - if (driver_id && set_highres == 1) + for( i = 0 ; i < ft->n_valid ; ++i ) { - char *text; - - graf_page_size (&px, &py, &ix, &iy); - graf_feed_page (); - - /* Calculate borders. */ - pb.x1 = ix; - pb.y1 = iy; - pb.x2 = px - ix; - pb.y2 = py - iy; - gb.x1 = pb.x1 + ix; - gb.y1 = pb.y1 + iy; - gb.x2 = pb.x2 - ix / 2; - gb.y2 = pb.y2 - iy; - - /* Draw borders. */ - graf_frame_rect (COMPONENTS (pb)); - graf_frame_rect (COMPONENTS (gb)); - - /* Draw axis labels. */ - graf_font_size (iy / 4); /* 18-point text */ - text = format == PERCENT ? _("Percentage") : _("Frequency"); - graf_text (pb.x1 + max (ix, iy) / 4 + max (ix, iy) / 16, gb.y2, text, - SIDEWAYS); - text = v->label ? v->label : v->name; - graf_text (gb.x1, pb.y2 - iy / 4, text, UPRIGHT); - - /* Draw axes, chart proper. */ - if (chart == BAR || - (chart == HBAR - && (avlcount (cur_var->p.frq.t.f) || v_variables[i]->type == ALPHA))) - draw_bar_chart (i); - else - draw_histogram (i); - - graf_eject_page (); + frq = &ft->valid[i]; + gsl_histogram_accumulate(hist, frq->v.f, frq->c); } - if (set_lowres == 1 || (set_lowres == 2 && (!driver_id || !set_highres))) - { - static warned; - /* Do character-based graphs. */ - if (!warned) - { - warn (_("low-res graphs not implemented")); - warned = 1; - } - } + return hist; } -#if __GNUC__ && !__CHECKER__ -#define BIG_TYPE long long -#else /* !__GNUC__ */ -#define BIG_TYPE double -#endif /* !__GNUC__ */ -static void -draw_bar_chart (int i) -{ - int bar_width, bar_spacing; - int w, max, row; - double val; - struct freq *f; - rect r; - AVLtraverser *t = NULL; - - w = (px - ix * 7 / 2) / avlcount (cur_var->p.frq.t.f); - bar_width = w * 2 / 3; - bar_spacing = w - bar_width; - -#if !ALLOW_HUGE_BARS - if (bar_width > ix / 2) - bar_width = ix / 2; -#endif /* !ALLOW_HUGE_BARS */ - - max = scale_dep_axis (cur_var->p.frq.t.max_freq); - - row = 0; - r.x1 = gb.x1 + bar_spacing / 2; - r.x2 = r.x1 + bar_width; - r.y2 = gb.y2; - graf_fill_color (255, 0, 0); - for (f = avltrav (cur_var->p.frq.t.f, &t); f; - f = avltrav (cur_var->p.frq.t.f, &t)) - { - char buf2[64]; - char *buf; - - val = f->c; - if (format == PERCENT) - val = val * 100 / cur_var->p.frq.t.valid_cases; - r.y1 = r.y2 - val * (height (gb) - 1) / max; - graf_fill_rect (COMPONENTS (r)); - graf_frame_rect (COMPONENTS (r)); - buf = get_val_lab (cur_var, f->v, 0); - if (!buf) - if (cur_var->type == ALPHA) - buf = f->v.s; - else - { - sprintf (buf2, "%g", f->v.f); - buf = buf2; - } - graf_text (r.x1 + bar_width / 2, - gb.y2 + iy / 32 + row * iy / 9, buf, TCJUST); - row ^= 1; - r.x1 += bar_width + bar_spacing; - r.x2 += bar_width + bar_spacing; - } - graf_fill_color (0, 0, 0); -} +static struct slice * +freq_tab_to_slice_array(const struct freq_tab *frq_tab, + const struct variable *var, + int *n_slices); -#define round_down(X, V) \ - (floor ((X) / (V)) * (V)) -#define round_up(X, V) \ - (ceil ((X) / (V)) * (V)) -static void -draw_histogram (int i) +/* Allocate an array of slices and fill them from the data in frq_tab + n_slices will contain the number of slices allocated. + The caller is responsible for freeing slices +*/ +static struct slice * +freq_tab_to_slice_array(const struct freq_tab *frq_tab, + const struct variable *var, + int *n_slices) { - double lower, upper, interval; - int bars[MAX_HIST_BARS + 1], top, j; - int err, addend, rem, nbars, row, max_freq; - char buf[25]; - rect r; - struct freq *f; - AVLtraverser *t = NULL; - - lower = min == SYSMIS ? cur_var->dbl[4] : min; - upper = max == SYSMIS ? cur_var->dbl[5] : max; - if (upper - lower >= 10) - { - double l, u; + int i; + struct slice *slices; - u = round_up (upper, 5); - l = round_down (lower, 5); - nbars = (u - l) / 5; - if (nbars * 2 + 1 <= MAX_HIST_BARS) - { - nbars *= 2; - u = round_up (upper, 2.5); - l = round_down (lower, 2.5); - if (l + 1.25 <= lower && u - 1.25 >= upper) - nbars--, lower = l + 1.25, upper = u - 1.25; - else if (l + 1.25 <= lower) - lower = l + 1.25, upper = u + 1.25; - else if (u - 1.25 >= upper) - lower = l - 1.25, upper = u - 1.25; - else - nbars++, lower = l - 1.25, upper = u + 1.25; - } - else if (nbars < MAX_HIST_BARS) - { - if (l + 2.5 <= lower && u - 2.5 >= upper) - nbars--, lower = l + 2.5, upper = u - 2.5; - else if (l + 2.5 <= lower) - lower = l + 2.5, upper = u + 2.5; - else if (u - 2.5 >= upper) - lower = l - 2.5, upper = u - 2.5; - else - nbars++, lower = l - 2.5, upper = u + 2.5; - } - else - nbars = MAX_HIST_BARS; - } - else - { - nbars = avlcount (cur_var->p.frq.t.f); - if (nbars > MAX_HIST_BARS) - nbars = MAX_HIST_BARS; - } - if (nbars < MIN_HIST_BARS) - nbars = MIN_HIST_BARS; - interval = (upper - lower) / nbars; + *n_slices = frq_tab->n_valid; + + slices = xmalloc ( *n_slices * sizeof (struct slice ) ); - memset (bars, 0, sizeof (int[nbars + 1])); - if (lower >= upper) - { - msg (SE, _("Could not make histogram for %s for specified " - "minimum %g and maximum %g; please discard graph."), cur_var->name, - lower, upper); - return; - } - for (f = avltrav (cur_var->p.frq.t.f, &t); f; - f = avltrav (cur_var->p.frq.t.f, &t)) - if (f->v.f == upper) - bars[nbars - 1] += f->c; - else if (f->v.f >= lower && f->v.f < upper) - bars[(int) ((f->v.f - lower) / interval)] += f->c; - bars[nbars - 1] += bars[nbars]; - for (j = top = 0; j < nbars; j++) - if (bars[j] > top) - top = bars[j]; - max_freq = top; - top = scale_dep_axis (top); - - err = row = 0; - addend = width (gb) / nbars; - rem = width (gb) % nbars; - r.x1 = gb.x1; - r.x2 = r.x1 + addend; - r.y2 = gb.y2; - err += rem; - graf_fill_color (255, 0, 0); - for (j = 0; j < nbars; j++) - { - int w; - - r.y1 = r.y2 - (BIG_TYPE) bars[j] * (height (gb) - 1) / top; - graf_fill_rect (COMPONENTS (r)); - graf_frame_rect (COMPONENTS (r)); - sprintf (buf, "%g", lower + interval / 2 + interval * j); - graf_text (r.x1 + addend / 2, - gb.y2 + iy / 32 + row * iy / 9, buf, TCJUST); - row ^= 1; - w = addend; - err += rem; - while (err >= addend) - { - w++; - err -= addend; - } - r.x1 = r.x2; - r.x2 = r.x1 + w; - } - if (normal) + for (i = 0 ; i < *n_slices ; ++i ) { - double x, y, variance, mean, step, factor; - - variance = cur_var->res[FRQ_ST_VARIANCE]; - mean = cur_var->res[FRQ_ST_MEAN]; - factor = (1. / (sqrt (2. * PI * variance)) - * cur_var->p.frq.t.valid_cases * interval); - graf_polyline_begin (); - for (x = lower, step = (upper - lower) / (POLYLINE_DENSITY); - x <= upper; x += step) - { - y = factor * exp (-square (x - mean) / (2. * variance)); - debug_printf (("(%20.10f, %20.10f)\n", x, y)); - graf_polyline_point (gb.x1 + (x - lower) / (upper - lower) * width (gb), - gb.y2 - y * (height (gb) - 1) / top); - } - graf_polyline_end (); - } - graf_fill_color (0, 0, 0); -} + const struct freq *frq = &frq_tab->valid[i]; -static int -scale_dep_axis (int max) -{ - int j, s, x, y, ty, by; - char buf[10]; + slices[i].label = value_to_string(&frq->v, var); - x = 10, s = 2; - if (scale != SYSMIS && max < scale) - x = scale, s = scale / 5; - else if (format == PERCENT) - { - max = ((BIG_TYPE) 100 * cur_var->p.frq.t.max_freq - / cur_var->p.frq.t.valid_cases + 1); - if (max < 5) - x = 5, s = 1; - else if (max < 10) - x = 10, s = 2; - else if (max < 25) - x = 25, s = 5; - else if (max < 50) - x = 50, s = 10; - else - max = 100, s = 20; + slices[i].magnetude = frq->c; } - else /* format==FREQ */ - /* Uses a progression of 10, 20, 50, 100, 200, 500, ... */ - for (;;) - { - if (x > max) - break; - x *= 2; - s *= 2; - if (x > max) - break; - x = x / 2 * 5; - s = s / 2 * 5; - if (x > max) - break; - x *= 2; - s *= 2; - } - graf_font_size (iy / 9); /* 8-pt text */ - for (j = 0; j <= x; j += s) - { - y = gb.y2 - (BIG_TYPE) j *(height (gb) - 1) / x; - ty = y - iy / 64; - by = y + iy / 64; - if (ty < gb.y1) - ty += iy / 64, by += iy / 64; - else if (by > gb.y2) - ty -= iy / 64, by -= iy / 64; - graf_fill_rect (gb.x1 - ix / 16, ty, gb.x1, by); - sprintf (buf, "%d", j); - graf_text (gb.x1 - ix / 8, (ty + by) / 2, buf, CRJUST); - } - return x; + + return slices; } - -/* Percentiles. */ -static void ungrouped_pcnt (int i); -static int grouped_interval_pcnt (int i); -static void out_pcnt (double, double); -static void -out_percentiles (int i) -{ - if (cur_var->type == ALPHA || !n_percentiles) - return; - outs_line (_("Percentile Value " - "Percentile Value " - "Percentile Value")); - blank_line (); - - column = 0; - if (!g_var[i]) - ungrouped_pcnt (i); - else if (g_var[i] == 1) - grouped_interval_pcnt (i); -#if 0 - else if (g_var[i] == -1) - grouped_pcnt (i); - else - grouped_boundaries_pcnt (i); -#else /* !0 */ - else - warn (_("this form of percentiles not supported")); -#endif - if (column) - out_eol (); -} static void -out_pcnt (double pcnt, double value) +do_piechart(const struct variable *var, const struct freq_tab *frq_tab) { - if (!column) - out_header (); - else - outs (" "); - out ("%7.2f%13.3f", pcnt * 100., value); - column++; - if (column == 3) - { - out_eol (); - column = 0; - } -} + struct slice *slices; + int n_slices; -static void -ungrouped_pcnt (int i) -{ - AVLtraverser *t = NULL; - struct freq *f; - double *p, *e; - int sum; - - p = percentiles; - e = &percentiles[n_percentiles]; - sum = 0; - for (f = avltrav (cur_var->p.frq.t.f, &t); - f && p < e; f = avltrav (cur_var->p.frq.t.f, &t)) - { - sum += f->c; - while (sum >= p[0] * cur_var->p.frq.t.valid_cases && p < e) - out_pcnt (*p++, f->v.f); - } -} + slices = freq_tab_to_slice_array(frq_tab, var, &n_slices); + piechart_plot(var_to_string(var), slices, n_slices); -static int -grouped_interval_pcnt (int i) -{ - AVLtraverser * t = NULL; - struct freq * f, *fp; - double *p, *e, w; - int sum, psum; - - p = percentiles; - e = &percentiles[n_percentiles]; - w = gl_var[i][0]; - sum = psum = 0; - for (fp = 0, f = avltrav (cur_var->p.frq.t.f, &t); - f && p < e; - fp = f, f = avltrav (cur_var->p.frq.t.f, &t)) - { - if (fp) - if (fabs (f->v.f - fp->v.f) < w) - { - out_eol (); - column = 0; - return msg (SE, _("Difference between %g and %g is " - "too small for grouping interval %g."), f->v.f, - fp->v.f, w); - } - psum = sum; - sum += f->c; - while (sum >= p[0] * cur_var->p.frq.t.valid_cases && p < e) - { - out_pcnt (p[0], (((p[0] * cur_var->p.frq.t.valid_cases) - psum) * w / f->c - + (f->v.f - w / 2))); - p++; - } - } - return 1; + free(slices); } -#endif + /* Local Variables: