X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fcrosstabs.q;h=ab73f96ad5bea6b37bb481b488b39fdf33c79202;hb=979ad20e18c60327ccb55215bed7b9d08be1aeae;hp=e8f6aa5833f8ba150f65baf6a666867c735a5d44;hpb=24c05c4eafa2fa462bae17b45c9f58d0fb2a09c7;p=pspp diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q index e8f6aa5833..ab73f96ad5 100644 --- a/src/language/stats/crosstabs.q +++ b/src/language/stats/crosstabs.q @@ -16,11 +16,10 @@ /* FIXME: - - How to calculate significance of symmetric and directional measures? - - Asymmetric ASEs and T values for lambda are wrong. - - ASE of Goodman and Kruskal's tau is not calculated. - - ASE of symmetric somers' d is wrong. - - Approx. T of uncertainty coefficient is wrong. + - How to calculate significance of some symmetric and directional measures? + - How to calculate ASE for symmetric Somers ' d? + - How to calculate ASE for Goodman and Kruskal's tau? + - How to calculate approx. T of symmetric uncertainty coefficient? */ @@ -42,6 +41,7 @@ #include "data/value-labels.h" #include "data/variable.h" #include "language/command.h" +#include "language/stats/freq.h" #include "language/dictionary/split-file.h" #include "language/lexer/lexer.h" #include "language/lexer/variable-parser.h" @@ -56,6 +56,8 @@ #include "libpspp/pool.h" #include "libpspp/str.h" #include "output/tab.h" +#include "output/chart-item.h" +#include "output/charts/barchart.h" #include "gl/minmax.h" #include "gl/xalloc.h" @@ -78,6 +80,7 @@ tabl:!tables/notables, box:!box/nobox, pivot:!pivot/nopivot; + +barchart=; +cells[cl_]=count,expected,row,column,total,residual,sresidual, asresidual,all,none; +statistics[st_]=chisq,phi,cc,lambda,uc,none,btau,ctau,risk,gamma,d, @@ -95,20 +98,6 @@ /* Number of directional statistics. */ #define N_DIRECTIONAL 13 -/* A single table entry for general mode. */ -struct table_entry - { - struct hmap_node node; /* Entry in hash table. */ - double freq; /* Frequency count. */ - union value values[1]; /* Values. */ - }; - -static size_t -table_entry_size (size_t n_values) -{ - return (offsetof (struct table_entry, values) - + n_values * sizeof (union value)); -} /* Indexes into the 'vars' member of struct pivot_table and struct crosstab member. */ @@ -137,7 +126,7 @@ struct pivot_table /* Data. */ struct hmap data; - struct table_entry **entries; + struct freq **entries; size_t n_entries; /* Column values, number of columns. */ @@ -175,6 +164,7 @@ struct crosstabs_proc enum { INTEGER, GENERAL } mode; enum mv_class exclude; bool pivot; + bool barchart; bool bad_warn; struct fmt_spec weight_format; @@ -242,7 +232,7 @@ cmd_crosstabs (struct lexer *lexer, struct dataset *ds) } proc.mode = proc.n_variables ? INTEGER : GENERAL; - + proc.barchart = cmd.sbc_barchart > 0; proc.descending = cmd.val == CRS_DVALUE; @@ -591,7 +581,7 @@ static void tabulate_integer_case (struct pivot_table *pt, const struct ccase *c, double weight) { - struct table_entry *te; + struct freq *te; size_t hash; int j; @@ -602,14 +592,14 @@ tabulate_integer_case (struct pivot_table *pt, const struct ccase *c, hash = hash_int (case_num (c, pt->vars[j]), hash); } - HMAP_FOR_EACH_WITH_HASH (te, struct table_entry, node, hash, &pt->data) + HMAP_FOR_EACH_WITH_HASH (te, struct freq, node, hash, &pt->data) { for (j = 0; j < pt->n_vars; j++) if ((int) case_num (c, pt->vars[j]) != (int) te->values[j].f) goto no_match; /* Found an existing entry. */ - te->freq += weight; + te->count += weight; return; no_match: ; @@ -617,7 +607,7 @@ tabulate_integer_case (struct pivot_table *pt, const struct ccase *c, /* No existing entry. Create a new one. */ te = xmalloc (table_entry_size (pt->n_vars)); - te->freq = weight; + te->count = weight; for (j = 0; j < pt->n_vars; j++) te->values[j].f = (int) case_num (c, pt->vars[j]); hmap_insert (&pt->data, &te->node, hash); @@ -627,7 +617,7 @@ static void tabulate_general_case (struct pivot_table *pt, const struct ccase *c, double weight) { - struct table_entry *te; + struct freq *te; size_t hash; int j; @@ -638,7 +628,7 @@ tabulate_general_case (struct pivot_table *pt, const struct ccase *c, hash = value_hash (case_data (c, var), var_get_width (var), hash); } - HMAP_FOR_EACH_WITH_HASH (te, struct table_entry, node, hash, &pt->data) + HMAP_FOR_EACH_WITH_HASH (te, struct freq, node, hash, &pt->data) { for (j = 0; j < pt->n_vars; j++) { @@ -649,7 +639,7 @@ tabulate_general_case (struct pivot_table *pt, const struct ccase *c, } /* Found an existing entry. */ - te->freq += weight; + te->count += weight; return; no_match: ; @@ -657,7 +647,7 @@ tabulate_general_case (struct pivot_table *pt, const struct ccase *c, /* No existing entry. Create a new one. */ te = xmalloc (table_entry_size (pt->n_vars)); - te->freq = weight; + te->count = weight; for (j = 0; j < pt->n_vars; j++) { const struct variable *var = pt->vars[j]; @@ -668,8 +658,8 @@ tabulate_general_case (struct pivot_table *pt, const struct ccase *c, /* Post-data reading calculations. */ -static int compare_table_entry_vars_3way (const struct table_entry *a, - const struct table_entry *b, +static int compare_table_entry_vars_3way (const struct freq *a, + const struct freq *b, const struct pivot_table *pt, int idx0, int idx1); static int compare_table_entry_3way (const void *ap_, const void *bp_, @@ -695,19 +685,20 @@ postcalc (struct crosstabs_proc *proc) /* Convert hash tables into sorted arrays of entries. */ for (pt = &proc->pivots[0]; pt < &proc->pivots[proc->n_pivots]; pt++) { - struct table_entry *e; + struct freq *e; size_t i; pt->n_entries = hmap_count (&pt->data); pt->entries = xnmalloc (pt->n_entries, sizeof *pt->entries); i = 0; - HMAP_FOR_EACH (e, struct table_entry, node, &pt->data) + HMAP_FOR_EACH (e, struct freq, node, &pt->data) pt->entries[i++] = e; hmap_destroy (&pt->data); sort (pt->entries, pt->n_entries, sizeof *pt->entries, proc->descending ? compare_table_entry_3way_inv : compare_table_entry_3way, pt); + } make_summary_table (proc); @@ -727,6 +718,9 @@ postcalc (struct crosstabs_proc *proc) output_pivot_table (proc, &subset); } } + if (proc->barchart) + chart_item_submit + (barchart_create (pt->vars, pt->n_vars, _("Count"), pt->entries, pt->n_entries)); } /* Free output and prepare for next split file. */ @@ -781,8 +775,8 @@ make_pivot_table_subset (struct pivot_table *pt, size_t row0, size_t row1, } static int -compare_table_entry_var_3way (const struct table_entry *a, - const struct table_entry *b, +compare_table_entry_var_3way (const struct freq *a, + const struct freq *b, const struct pivot_table *pt, int idx) { @@ -791,8 +785,8 @@ compare_table_entry_var_3way (const struct table_entry *a, } static int -compare_table_entry_vars_3way (const struct table_entry *a, - const struct table_entry *b, +compare_table_entry_vars_3way (const struct freq *a, + const struct freq *b, const struct pivot_table *pt, int idx0, int idx1) { @@ -807,15 +801,15 @@ compare_table_entry_vars_3way (const struct table_entry *a, return 0; } -/* Compare the struct table_entry at *AP to the one at *BP and +/* Compare the struct freq at *AP to the one at *BP and return a strcmp()-type result. */ static int compare_table_entry_3way (const void *ap_, const void *bp_, const void *pt_) { - const struct table_entry *const *ap = ap_; - const struct table_entry *const *bp = bp_; - const struct table_entry *a = *ap; - const struct table_entry *b = *bp; + const struct freq *const *ap = ap_; + const struct freq *const *bp = bp_; + const struct freq *a = *ap; + const struct freq *b = *bp; const struct pivot_table *pt = pt_; int cmp; @@ -844,8 +838,8 @@ find_first_difference (const struct pivot_table *pt, size_t row) return pt->n_vars - 1; else { - const struct table_entry *a = pt->entries[row]; - const struct table_entry *b = pt->entries[row - 1]; + const struct freq *a = pt->entries[row]; + const struct freq *b = pt->entries[row - 1]; int col; for (col = pt->n_vars - 1; col >= 0; col--) @@ -903,7 +897,7 @@ make_summary_table (struct crosstabs_proc *proc) valid = 0.; for (i = 0; i < pt->n_entries; i++) - valid += pt->entries[i]->freq; + valid += pt->entries[i]->count; n[0] = valid; n[1] = pt->missing; @@ -1089,13 +1083,13 @@ build_matrix (struct pivot_table *x) const int row_var_width = var_get_width (x->vars[ROW_VAR]); int col, row; double *mp; - struct table_entry **p; + struct freq **p; mp = x->mat; col = row = 0; for (p = x->entries; p < &x->entries[x->n_entries]; p++) { - const struct table_entry *te = *p; + const struct freq *te = *p; while (!value_equal (&x->rows[row], &te->values[ROW_VAR], row_var_width)) { @@ -1111,7 +1105,7 @@ build_matrix (struct pivot_table *x) col++; } - *mp++ = te->freq; + *mp++ = te->count; if (++col >= x->n_cols) { col = 0; @@ -1431,8 +1425,8 @@ find_crosstab (struct pivot_table *pt, size_t *row0p, size_t *row1p) for (row1 = row0 + 1; row1 < pt->n_entries; row1++) { - struct table_entry *a = pt->entries[row0]; - struct table_entry *b = pt->entries[row1]; + struct freq *a = pt->entries[row0]; + struct freq *b = pt->entries[row1]; if (compare_table_entry_vars_3way (a, b, pt, 2, pt->n_vars) != 0) break; } @@ -1496,7 +1490,7 @@ enum_var_values (const struct pivot_table *pt, int var_idx, hmapx_init (&set); for (i = 0; i < pt->n_entries; i++) { - const struct table_entry *te = pt->entries[i]; + const struct freq *te = pt->entries[i]; const union value *value = &te->values[var_idx]; size_t hash = value_hash (value, width, 0); @@ -1975,7 +1969,7 @@ display_risk (struct pivot_table *pt, struct tab_table *risk) static int calc_directional (struct crosstabs_proc *, struct pivot_table *, double[N_DIRECTIONAL], double[N_DIRECTIONAL], - double[N_DIRECTIONAL]); + double[N_DIRECTIONAL], double[N_DIRECTIONAL]); /* Display directional measures. */ static void @@ -2042,10 +2036,11 @@ display_directional (struct crosstabs_proc *proc, struct pivot_table *pt, double direct_v[N_DIRECTIONAL]; double direct_ase[N_DIRECTIONAL]; double direct_t[N_DIRECTIONAL]; + double sig[N_DIRECTIONAL]; int i; - if (!calc_directional (proc, pt, direct_v, direct_ase, direct_t)) + if (!calc_directional (proc, pt, direct_v, direct_ase, direct_t, sig)) return; tab_offset (direct, pt->n_consts + pt->n_vars - 2, -1); @@ -2087,7 +2082,7 @@ display_directional (struct crosstabs_proc *proc, struct pivot_table *pt, tab_double (direct, 4, 0, TAB_RIGHT, direct_ase[i], NULL, RC_OTHER); if (direct_t[i] != SYSMIS) tab_double (direct, 5, 0, TAB_RIGHT, direct_t[i], NULL, RC_OTHER); - /*tab_double (direct, 6, 0, TAB_RIGHT, normal_sig (direct_v[i]), NULL, RC_PVALUE);*/ + tab_double (direct, 6, 0, TAB_RIGHT, sig[i], NULL, RC_PVALUE); tab_next_row (direct); } @@ -2535,7 +2530,7 @@ calc_symmetric (struct crosstabs_proc *proc, struct pivot_table *pt, if (proc->statistics & (1u << CRS_ST_D)) { somers_d_v[0] = (P - Q) / (.5 * (Dc + Dr)); - somers_d_ase[0] = 2. * btau_var / (Dr + Dc) * sqrt (Dr * Dc); + somers_d_ase[0] = SYSMIS; somers_d_t[0] = (somers_d_v[0] / (4 / (Dc + Dr) * sqrt (ctau_cum - pow2 (P - Q) / pt->total))); @@ -2605,6 +2600,7 @@ calc_symmetric (struct crosstabs_proc *proc, struct pivot_table *pt, /* Cohen's kappa. */ if (proc->statistics & (1u << CRS_ST_KAPPA) && pt->ns_rows == pt->ns_cols) { + double ase_under_h0; double sum_fii, sum_rici, sum_fiiri_ci, sum_fijri_ci2, sum_riciri_ci; int i, j; @@ -2633,24 +2629,23 @@ calc_symmetric (struct crosstabs_proc *proc, struct pivot_table *pt, v[8] = (pt->total * sum_fii - sum_rici) / (pow2 (pt->total) - sum_rici); - ase[8] = sqrt ((pow2 (pt->total) * sum_rici - + sum_rici * sum_rici - - pt->total * sum_riciri_ci) - / (pt->total * (pow2 (pt->total) - sum_rici) * (pow2 (pt->total) - sum_rici))); -#if 0 - t[8] = v[8] / sqrt (pt->total * (((sum_fii * (pt->total - sum_fii)) + ase_under_h0 = sqrt ((pow2 (pt->total) * sum_rici + + sum_rici * sum_rici + - pt->total * sum_riciri_ci) + / (pt->total * (pow2 (pt->total) - sum_rici) * (pow2 (pt->total) - sum_rici))); + + ase[8] = sqrt (pt->total * (((sum_fii * (pt->total - sum_fii)) / pow2 (pow2 (pt->total) - sum_rici)) + ((2. * (pt->total - sum_fii) * (2. * sum_fii * sum_rici - pt->total * sum_fiiri_ci)) - / cube (pow2 (pt->total) - sum_rici)) + / pow3 (pow2 (pt->total) - sum_rici)) + (pow2 (pt->total - sum_fii) * (pt->total * sum_fijri_ci2 - 4. * sum_rici * sum_rici) / pow4 (pow2 (pt->total) - sum_rici)))); -#else - t[8] = v[8] / ase[8]; -#endif + + t[8] = v[8] / ase_under_h0; } return 1; @@ -2721,13 +2716,13 @@ calc_risk (struct pivot_table *pt, static int calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, double v[N_DIRECTIONAL], double ase[N_DIRECTIONAL], - double t[N_DIRECTIONAL]) + double t[N_DIRECTIONAL], double sig[N_DIRECTIONAL]) { { int i; for (i = 0; i < N_DIRECTIONAL; i++) - v[i] = ase[i] = t[i] = SYSMIS; + v[i] = ase[i] = t[i] = sig[i] = SYSMIS; } /* Lambda. */ @@ -2802,19 +2797,14 @@ calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, /* ASE1 for Y given PT. */ { - double accum; - - for (accum = 0., i = 0; i < pt->n_rows; i++) - for (j = 0; j < pt->n_cols; j++) - { - const int deltaj = j == cm_index; - accum += (pt->mat[j + i * pt->n_cols] - * pow2 ((j == fim_index[i]) - - deltaj - + v[0] * deltaj)); - } + double accum; - ase[2] = sqrt (accum - pt->total * v[0]) / (pt->total - cm); + accum = 0.; + for (i = 0; i < pt->n_rows; i++) + if (cm_index == fim_index[i]) + accum += fim[i]; + ase[2] = sqrt ((pt->total - sum_fim) * (sum_fim + cm - 2. * accum) + / pow3 (pt->total - cm)); } /* ASE0 for Y given PT. */ @@ -2830,19 +2820,14 @@ calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, /* ASE1 for PT given Y. */ { - double accum; - - for (accum = 0., i = 0; i < pt->n_rows; i++) - for (j = 0; j < pt->n_cols; j++) - { - const int deltaj = i == rm_index; - accum += (pt->mat[j + i * pt->n_cols] - * pow2 ((i == fmj_index[j]) - - deltaj - + v[0] * deltaj)); - } + double accum; - ase[1] = sqrt (accum - pt->total * v[0]) / (pt->total - rm); + accum = 0.; + for (j = 0; j < pt->n_cols; j++) + if (rm_index == fmj_index[j]) + accum += fmj[j]; + ase[1] = sqrt ((pt->total - sum_fmj) * (sum_fmj + rm - 2. * accum) + / pow3 (pt->total - rm)); } /* ASE0 for PT given Y. */ @@ -2871,15 +2856,19 @@ calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, * pow2 (temp0 + (v[0] - 1.) * temp1)); } ase[0] = sqrt (accum1 - 4. * pt->total * v[0] * v[0]) / (2. * pt->total - rm - cm); - t[0] = v[0] / (sqrt (accum0 - pow2 ((sum_fim + sum_fmj - cm - rm) / pt->total)) + t[0] = v[0] / (sqrt (accum0 - pow2 (sum_fim + sum_fmj - cm - rm) / pt->total) / (2. * pt->total - rm - cm)); } + for (i = 0; i < 3; i++) + sig[i] = 2 * gsl_cdf_ugaussian_Q (t[i]); + free (fim); free (fim_index); free (fmj); free (fmj_index); + /* Tau. */ { double sum_fij2_ri, sum_fij2_ci; double sum_ri2, sum_cj2; @@ -2948,8 +2937,7 @@ calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, v[5] = 2. * ((UX + UY - UXY) / (UX + UY)); ase[5] = (2. / (pt->total * pow2 (UX + UY))) * sqrt (ase1_sym); - t[5] = v[5] / ((2. / (pt->total * (UX + UY))) - * sqrt (P - pow2 (UX + UY - UXY) / pt->total)); + t[5] = SYSMIS; v[6] = (UX + UY - UXY) / UX; ase[6] = sqrt (ase1_xy) / (pt->total * UX * UX); @@ -2979,6 +2967,7 @@ calc_directional (struct crosstabs_proc *proc, struct pivot_table *pt, v[8 + i] = somers_d_v[i]; ase[8 + i] = somers_d_ase[i]; t[8 + i] = somers_d_t[i]; + sig[8 + i] = 2 * gsl_cdf_ugaussian_Q (fabs (somers_d_t[i])); } } }