X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Froc.c;h=b3f8e91e47da59645d04f83cfa75ae66a4f3ff88;hb=29917c4f5908454803e663d2ad78bca4bc35e805;hp=86aee1f9d1f5e6a9662f1a605ebf560db52b19b7;hpb=3bbb4370239deb29ebbf813d258aef6249e2a431;p=pspp diff --git a/src/language/stats/roc.c b/src/language/stats/roc.c index 86aee1f9d1..b3f8e91e47 100644 --- a/src/language/stats/roc.c +++ b/src/language/stats/roc.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2009 Free Software Foundation, Inc. + Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,28 +16,25 @@ #include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - - -#include +#include "language/stats/roc.h" #include -#include -#include -#include +#include "data/casegrouper.h" +#include "data/casereader.h" +#include "data/casewriter.h" +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/format.h" +#include "data/subcase.h" +#include "language/command.h" +#include "language/lexer/lexer.h" +#include "language/lexer/value-parser.h" +#include "language/lexer/variable-parser.h" +#include "libpspp/misc.h" +#include "math/sort.h" +#include "output/charts/roc-chart.h" +#include "output/pivot-table.h" #include "gettext.h" #define _(msgid) gettext (msgid) @@ -49,8 +46,9 @@ struct cmd_roc const struct variable **vars; const struct dictionary *dict; - const struct variable *state_var ; + const struct variable *state_var; union value state_value; + size_t state_var_width; /* Plot the roc curve */ bool curve; @@ -95,40 +93,47 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) roc.pos = roc.pos_weighted = 0; roc.neg = roc.neg_weighted = 0; roc.dict = dataset_dict (ds); + roc.state_var = NULL; + roc.state_var_width = -1; + lex_match (lexer, T_SLASH); if (!parse_variables_const (lexer, dict, &roc.vars, &roc.n_vars, PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC)) goto error; - if ( ! lex_force_match (lexer, T_BY)) + if (! lex_force_match (lexer, T_BY)) { goto error; } roc.state_var = parse_variable (lexer, dict); + if (! roc.state_var) + { + goto error; + } - if ( !lex_force_match (lexer, '(')) + if (!lex_force_match (lexer, T_LPAREN)) { goto error; } - value_init (&roc.state_value, var_get_width (roc.state_var)); - parse_value (lexer, &roc.state_value, var_get_width (roc.state_var)); + roc.state_var_width = var_get_width (roc.state_var); + value_init (&roc.state_value, roc.state_var_width); + parse_value (lexer, &roc.state_value, roc.state_var); - if ( !lex_force_match (lexer, ')')) + if (!lex_force_match (lexer, T_RPAREN)) { goto error; } - - while (lex_token (lexer) != '.') + while (lex_token (lexer) != T_ENDCMD) { - lex_match (lexer, '/'); + lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "MISSING")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "INCLUDE")) { @@ -147,15 +152,17 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "PLOT")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "CURVE")) { roc.curve = true; - if (lex_match (lexer, '(')) + if (lex_match (lexer, T_LPAREN)) { roc.reference = true; - lex_force_match_id (lexer, "REFERENCE"); - lex_force_match (lexer, ')'); + if (! lex_force_match_id (lexer, "REFERENCE")) + goto error; + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } } else if (lex_match_id (lexer, "NONE")) @@ -170,8 +177,8 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "PRINT")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "SE")) { @@ -190,12 +197,13 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "CRITERIA")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "CUTOFF")) { - lex_force_match (lexer, '('); + if (! lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "INCLUDE")) { roc.exclude = MV_SYSTEM; @@ -209,11 +217,13 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "TESTPOS")) { - lex_force_match (lexer, '('); + if (! lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "LARGE")) { roc.invert = false; @@ -227,19 +237,24 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "CI")) { - lex_force_match (lexer, '('); - lex_force_num (lexer); + if (!lex_force_match (lexer, T_LPAREN)) + goto error; + if (! lex_force_num (lexer)) + goto error; roc.ci = lex_number (lexer); lex_get (lexer); - lex_force_match (lexer, ')'); + if (!lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "DISTRIBUTION")) { - lex_force_match (lexer, '('); + if (!lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "FREE")) { roc.bi_neg_exp = false; @@ -253,7 +268,8 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (!lex_force_match (lexer, T_RPAREN)) + goto error; } else { @@ -269,15 +285,17 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } } - if ( ! run_roc (ds, &roc)) + if (! run_roc (ds, &roc)) goto error; - value_destroy (&roc.state_value, var_get_width (roc.state_var)); + if (roc.state_var) + value_destroy (&roc.state_value, roc.state_var_width); free (roc.vars); return CMD_SUCCESS; error: - value_destroy (&roc.state_value, var_get_width (roc.state_var)); + if (roc.state_var) + value_destroy (&roc.state_value, roc.state_var_width); free (roc.vars); return CMD_FAILURE; } @@ -314,7 +332,7 @@ dump_casereader (struct casereader *reader) struct ccase *c; struct casereader *r = casereader_clone (reader); - for ( ; (c = casereader_read (r) ); case_unref (c)) + for (; (c = casereader_read (r)); case_unref (c)) { int i; for (i = 0 ; i < case_get_value_cnt (c); ++i) @@ -329,10 +347,10 @@ dump_casereader (struct casereader *reader) #endif -/* +/* Return true iff the state variable indicates that C has positive actual state. - As a side effect, this function also accumulates the roc->{pos,neg} and + As a side effect, this function also accumulates the roc->{pos,neg} and roc->{pos,neg}_weighted counts. */ static bool @@ -343,10 +361,10 @@ match_positives (const struct ccase *c, void *aux) const double weight = wv ? case_data (c, wv)->f : 1.0; const bool positive = - ( 0 == value_compare_3way (case_data (c, roc->state_var), &roc->state_value, + (0 == value_compare_3way (case_data (c, roc->state_var), &roc->state_value, var_get_width (roc->state_var))); - if ( positive ) + if (positive) { roc->pos++; roc->pos_weighted += weight; @@ -365,7 +383,7 @@ match_positives (const struct ccase *c, void *aux) #define N_EQ 1 #define N_PRED 2 -/* Some intermediate state for calculating the cutpoints and the +/* Some intermediate state for calculating the cutpoints and the standard error values */ struct roc_state { @@ -375,7 +393,7 @@ struct roc_state double n2; /* total weight of negatives */ /* intermediates for standard error */ - double q1hat; + double q1hat; double q2hat; /* intermediates for cutpoints */ @@ -386,49 +404,41 @@ struct roc_state double max; }; -#define CUTPOINT 0 -#define TP 1 -#define FN 2 -#define TN 3 -#define FP 4 - - -/* +/* Return a new casereader based upon CUTPOINT_RDR. The number of "positive" cases are placed into the position TRUE_INDEX, and the number of "negative" cases into FALSE_INDEX. - POS_COND and RESULT determine the semantics of what is + POS_COND and RESULT determine the semantics of what is "positive". WEIGHT is the value of a single count. */ static struct casereader * -accumulate_counts (struct casereader *cutpoint_rdr, - double result, double weight, +accumulate_counts (struct casereader *input, + double result, double weight, bool (*pos_cond) (double, double), int true_index, int false_index) { - const struct caseproto *proto = casereader_get_proto (cutpoint_rdr); + const struct caseproto *proto = casereader_get_proto (input); struct casewriter *w = autopaging_writer_create (proto); - struct casereader *r = casereader_clone (cutpoint_rdr); struct ccase *cpc; double prev_cp = SYSMIS; - for ( ; (cpc = casereader_read (r) ); case_unref (cpc)) + for (; (cpc = casereader_read (input)); case_unref (cpc)) { struct ccase *new_case; - const double cp = case_data_idx (cpc, CUTPOINT)->f; + const double cp = case_data_idx (cpc, ROC_CUTPOINT)->f; assert (cp != SYSMIS); /* We don't want duplicates here */ - if ( cp == prev_cp ) + if (cp == prev_cp) continue; new_case = case_clone (cpc); - if ( pos_cond (result, cp)) + if (pos_cond (result, cp)) case_data_rw_idx (new_case, true_index)->f += weight; else case_data_rw_idx (new_case, false_index)->f += weight; @@ -437,7 +447,7 @@ accumulate_counts (struct casereader *cutpoint_rdr, casewriter_write (w, new_case); } - casereader_destroy (r); + casereader_destroy (input); return casewriter_make_reader (w); } @@ -458,7 +468,7 @@ static void output_roc (struct roc_state *rs, const struct cmd_roc *roc); CUTPOINT_RDR accordingly. TRUE_INDEX and FALSE_INDEX are the indices which receive these values. POS_COND is the condition defining true and false. - + 3. CC is filled with the cumulative weight of all cases of READER. */ static struct casereader * @@ -466,7 +476,7 @@ process_group (const struct variable *var, struct casereader *reader, bool (*pred) (double, double), const struct dictionary *dict, double *cc, - struct casereader **cutpoint_rdr, + struct casereader **cutpoint_rdr, bool (*pos_cond) (double, double), int true_index, int false_index) @@ -478,7 +488,7 @@ process_group (const struct variable *var, struct casereader *reader, const int weight_idx = w ? var_get_case_index (w) : caseproto_get_n_widths (casereader_get_proto (r1)) - 1; - + struct ccase *c1; struct casereader *rclone = casereader_clone (r1); @@ -489,11 +499,11 @@ process_group (const struct variable *var, struct casereader *reader, proto = caseproto_add_width (proto, 0); proto = caseproto_add_width (proto, 0); - wtr = autopaging_writer_create (proto); + wtr = autopaging_writer_create (proto); *cc = 0; - for ( ; (c1 = casereader_read (r1) ); case_unref (c1)) + for (; (c1 = casereader_read (r1)); case_unref (c1)) { struct ccase *new_case = case_create (proto); struct ccase *c2; @@ -510,17 +520,17 @@ process_group (const struct variable *var, struct casereader *reader, *cc += weight1; - for ( ; (c2 = casereader_read (r2) ); case_unref (c2)) + for (; (c2 = casereader_read (r2)); case_unref (c2)) { const double d2 = case_data (c2, var)->f; const double weight2 = case_data_idx (c2, weight_idx)->f; - if ( d1 == d2 ) + if (d1 == d2) { n_eq += weight2; continue; } - else if ( pred (d2, d1)) + else if (pred (d2, d1)) { n_pred += weight2; } @@ -535,15 +545,18 @@ process_group (const struct variable *var, struct casereader *reader, casereader_destroy (r2); } + casereader_destroy (r1); casereader_destroy (rclone); + caseproto_unref (proto); + return casewriter_make_reader (wtr); } /* Some more indeces into case data */ #define N_POS_EQ 1 /* number of positive cases with values equal to n */ -#define N_POS_GT 2 /* number of postive cases with values greater than n */ +#define N_POS_GT 2 /* number of positive cases with values greater than n */ #define N_NEG_EQ 3 /* number of negative cases with values equal to n */ #define N_NEG_LT 4 /* number of negative cases with values less than n */ @@ -582,7 +595,7 @@ process_positive_group (const struct variable *var, struct casereader *reader, return process_group (var, reader, gt, dict, &rs->n1, &rs->cutpoint_rdr, ge, - TP, FN); + ROC_TP, ROC_FN); } /* @@ -600,7 +613,7 @@ process_negative_group (const struct variable *var, struct casereader *reader, return process_group (var, reader, lt, dict, &rs->n2, &rs->cutpoint_rdr, lt, - TN, FP); + ROC_TN, ROC_FP); } @@ -611,21 +624,21 @@ append_cutpoint (struct casewriter *writer, double cutpoint) { struct ccase *cc = case_create (casewriter_get_proto (writer)); - case_data_rw_idx (cc, CUTPOINT)->f = cutpoint; - case_data_rw_idx (cc, TP)->f = 0; - case_data_rw_idx (cc, FN)->f = 0; - case_data_rw_idx (cc, TN)->f = 0; - case_data_rw_idx (cc, FP)->f = 0; + case_data_rw_idx (cc, ROC_CUTPOINT)->f = cutpoint; + case_data_rw_idx (cc, ROC_TP)->f = 0; + case_data_rw_idx (cc, ROC_FN)->f = 0; + case_data_rw_idx (cc, ROC_TN)->f = 0; + case_data_rw_idx (cc, ROC_FP)->f = 0; casewriter_write (writer, cc); } -/* +/* Create and initialise the rs[x].cutpoint_rdr casereaders. That is, the readers will - be created with width 5, ready to take the values (cutpoint, TP, FN, TN, FP), and the + be created with width 5, ready to take the values (cutpoint, ROC_TP, ROC_FN, ROC_TN, ROC_FP), and the reader will be populated with its final number of cases. - However on exit from this function, only CUTPOINT entries will be set to their final + However on exit from this function, only ROC_CUTPOINT entries will be set to their final value. The other entries will be initialised to zero. */ static void @@ -634,41 +647,46 @@ prepare_cutpoints (struct cmd_roc *roc, struct roc_state *rs, struct casereader int i; struct casereader *r = casereader_clone (input); struct ccase *c; - struct caseproto *proto = caseproto_create (); - struct subcase ordering; - subcase_init (&ordering, CUTPOINT, 0, SC_ASCEND); + { + struct caseproto *proto = caseproto_create (); + struct subcase ordering; + subcase_init (&ordering, ROC_CUTPOINT, 0, SC_ASCEND); - proto = caseproto_add_width (proto, 0); /* cutpoint */ - proto = caseproto_add_width (proto, 0); /* TP */ - proto = caseproto_add_width (proto, 0); /* FN */ - proto = caseproto_add_width (proto, 0); /* TN */ - proto = caseproto_add_width (proto, 0); /* FP */ + proto = caseproto_add_width (proto, 0); /* cutpoint */ + proto = caseproto_add_width (proto, 0); /* ROC_TP */ + proto = caseproto_add_width (proto, 0); /* ROC_FN */ + proto = caseproto_add_width (proto, 0); /* ROC_TN */ + proto = caseproto_add_width (proto, 0); /* ROC_FP */ - for (i = 0 ; i < roc->n_vars; ++i) - { - rs[i].cutpoint_wtr = sort_create_writer (&ordering, proto); - rs[i].prev_result = SYSMIS; - rs[i].max = -DBL_MAX; - rs[i].min = DBL_MAX; - } + for (i = 0 ; i < roc->n_vars; ++i) + { + rs[i].cutpoint_wtr = sort_create_writer (&ordering, proto); + rs[i].prev_result = SYSMIS; + rs[i].max = -DBL_MAX; + rs[i].min = DBL_MAX; + } + + caseproto_unref (proto); + subcase_destroy (&ordering); + } for (; (c = casereader_read (r)) != NULL; case_unref (c)) { for (i = 0 ; i < roc->n_vars; ++i) { - const union value *v = case_data (c, roc->vars[i]); + const union value *v = case_data (c, roc->vars[i]); const double result = v->f; - if ( mv_is_value_missing (var_get_missing_values (roc->vars[i]), v, roc->exclude)) + if (mv_is_value_missing (var_get_missing_values (roc->vars[i]), v, roc->exclude)) continue; minimize (&rs[i].min, result); maximize (&rs[i].max, result); - if ( rs[i].prev_result != SYSMIS && rs[i].prev_result != result ) + if (rs[i].prev_result != SYSMIS && rs[i].prev_result != result) { - const double mean = (result + rs[i].prev_result ) / 2.0; + const double mean = (result + rs[i].prev_result) / 2.0; append_cutpoint (rs[i].cutpoint_wtr, mean); } @@ -693,12 +711,12 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) { int i; - struct roc_state *rs = xcalloc (roc->n_vars, sizeof *rs); + struct roc_state *rs = XCALLOC (roc->n_vars, struct roc_state); struct casereader *negatives = NULL; struct casereader *positives = NULL; - struct caseproto *n_proto = caseproto_create (); + struct caseproto *n_proto = NULL; struct subcase up_ordering; struct subcase down_ordering; @@ -723,7 +741,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) /* Separate the positive actual state cases from the negative ones */ - positives = + positives = casereader_create_filter_func (input, match_positives, NULL, @@ -731,7 +749,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) neg_wtr); n_proto = caseproto_create (); - + n_proto = caseproto_add_width (n_proto, 0); n_proto = caseproto_add_width (n_proto, 0); n_proto = caseproto_add_width (n_proto, 0); @@ -749,35 +767,33 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) struct ccase *c; struct ccase *cpos; - struct casereader *n_neg ; + struct casereader *n_neg_reader ; const struct variable *var = roc->vars[i]; struct casereader *neg ; struct casereader *pos = casereader_clone (positives); - - struct casereader *n_pos = + struct casereader *n_pos_reader = process_positive_group (var, pos, dict, &rs[i]); - if ( negatives == NULL) + if (negatives == NULL) { negatives = casewriter_make_reader (neg_wtr); } neg = casereader_clone (negatives); - n_neg = process_negative_group (var, neg, dict, &rs[i]); - + n_neg_reader = process_negative_group (var, neg, dict, &rs[i]); /* Merge the n_pos and n_neg casereaders */ w = sort_create_writer (&up_ordering, n_proto); - for ( ; (cpos = casereader_read (n_pos) ); case_unref (cpos)) + for (; (cpos = casereader_read (n_pos_reader)); case_unref (cpos)) { struct ccase *pos_case = case_create (n_proto); struct ccase *cneg; const double jpos = case_data_idx (cpos, VALUE)->f; - while ((cneg = casereader_read (n_neg))) + while ((cneg = casereader_read (n_neg_reader))) { struct ccase *nc = case_create (n_proto); @@ -794,7 +810,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) casewriter_write (w, nc); case_unref (cneg); - if ( jneg > jpos) + if (jneg > jpos) break; } @@ -807,6 +823,9 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) casewriter_write (w, pos_case); } + casereader_destroy (n_pos_reader); + casereader_destroy (n_neg_reader); + /* These aren't used anymore */ #undef N_EQ #undef N_PRED @@ -819,21 +838,22 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) double prev_pos_gt = rs[i].n1; w = sort_create_writer (&down_ordering, n_proto); - for ( ; (c = casereader_read (r) ); case_unref (c)) + for (; (c = casereader_read (r)); case_unref (c)) { double n_pos_gt = case_data_idx (c, N_POS_GT)->f; struct ccase *nc = case_clone (c); - if ( n_pos_gt == SYSMIS) + if (n_pos_gt == SYSMIS) { n_pos_gt = prev_pos_gt; case_data_rw_idx (nc, N_POS_GT)->f = n_pos_gt; } - + casewriter_write (w, nc); prev_pos_gt = n_pos_gt; } + casereader_destroy (r); r = casewriter_make_reader (w); } @@ -843,29 +863,30 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) double prev_neg_lt = rs[i].n2; w = sort_create_writer (&up_ordering, n_proto); - for ( ; (c = casereader_read (r) ); case_unref (c)) + for (; (c = casereader_read (r)); case_unref (c)) { double n_neg_lt = case_data_idx (c, N_NEG_LT)->f; struct ccase *nc = case_clone (c); - if ( n_neg_lt == SYSMIS) + if (n_neg_lt == SYSMIS) { n_neg_lt = prev_neg_lt; case_data_rw_idx (nc, N_NEG_LT)->f = n_neg_lt; } - + casewriter_write (w, nc); prev_neg_lt = n_neg_lt; } + casereader_destroy (r); r = casewriter_make_reader (w); } { struct ccase *prev_case = NULL; - for ( ; (c = casereader_read (r) ); case_unref (c)) + for (; (c = casereader_read (r)); case_unref (c)) { - const struct ccase *next_case = casereader_peek (r, 0); + struct ccase *next_case = casereader_peek (r, 0); const double j = case_data_idx (c, VALUE)->f; double n_pos_eq = case_data_idx (c, N_POS_EQ)->f; @@ -873,44 +894,47 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) double n_neg_eq = case_data_idx (c, N_NEG_EQ)->f; double n_neg_lt = case_data_idx (c, N_NEG_LT)->f; - if ( prev_case && j == case_data_idx (prev_case, VALUE)->f) + if (prev_case && j == case_data_idx (prev_case, VALUE)->f) { - if ( 0 == case_data_idx (c, N_POS_EQ)->f) + if (0 == case_data_idx (c, N_POS_EQ)->f) { n_pos_eq = case_data_idx (prev_case, N_POS_EQ)->f; n_pos_gt = case_data_idx (prev_case, N_POS_GT)->f; } - if ( 0 == case_data_idx (c, N_NEG_EQ)->f) + if (0 == case_data_idx (c, N_NEG_EQ)->f) { n_neg_eq = case_data_idx (prev_case, N_NEG_EQ)->f; n_neg_lt = case_data_idx (prev_case, N_NEG_LT)->f; } } - if ( NULL == next_case || j != case_data_idx (next_case, VALUE)->f) + if (NULL == next_case || j != case_data_idx (next_case, VALUE)->f) { rs[i].auc += n_pos_gt * n_neg_eq + (n_pos_eq * n_neg_eq) / 2.0; rs[i].q1hat += - n_neg_eq * ( pow2 (n_pos_gt) + n_pos_gt * n_pos_eq + pow2 (n_pos_eq) / 3.0); + n_neg_eq * (pow2 (n_pos_gt) + n_pos_gt * n_pos_eq + pow2 (n_pos_eq) / 3.0); rs[i].q2hat += - n_pos_eq * ( pow2 (n_neg_lt) + n_neg_lt * n_neg_eq + pow2 (n_neg_eq) / 3.0); + n_pos_eq * (pow2 (n_neg_lt) + n_neg_lt * n_neg_eq + pow2 (n_neg_eq) / 3.0); } + case_unref (next_case); case_unref (prev_case); prev_case = case_clone (c); } + casereader_destroy (r); + case_unref (prev_case); - rs[i].auc /= rs[i].n1 * rs[i].n2; - if ( roc->invert ) + rs[i].auc /= rs[i].n1 * rs[i].n2; + if (roc->invert) rs[i].auc = 1 - rs[i].auc; - if ( roc->bi_neg_exp ) + if (roc->bi_neg_exp) { - rs[i].q1hat = rs[i].auc / ( 2 - rs[i].auc); - rs[i].q2hat = 2 * pow2 (rs[i].auc) / ( 1 + rs[i].auc); + rs[i].q1hat = rs[i].auc / (2 - rs[i].auc); + rs[i].q2hat = 2 * pow2 (rs[i].auc) / (1 + rs[i].auc); } else { @@ -923,293 +947,175 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) casereader_destroy (positives); casereader_destroy (negatives); + caseproto_unref (n_proto); + subcase_destroy (&up_ordering); + subcase_destroy (&down_ordering); + output_roc (rs, roc); + for (i = 0 ; i < roc->n_vars; ++i) + casereader_destroy (rs[i].cutpoint_rdr); + free (rs); } static void show_auc (struct roc_state *rs, const struct cmd_roc *roc) { - int i; - const int n_fields = roc->print_se ? 5 : 1; - const int n_cols = roc->n_vars > 1 ? n_fields + 1: n_fields; - const int n_rows = 2 + roc->n_vars; - struct tab_table *tbl = tab_create (n_cols, n_rows, 0); - - if ( roc->n_vars > 1) - tab_title (tbl, _("Area Under the Curve")); - else - tab_title (tbl, _("Area Under the Curve (%s)"), var_to_string (roc->vars[0])); - - tab_headers (tbl, n_cols - n_fields, 0, 1, 0); - - tab_dim (tbl, tab_natural_dimensions, NULL); - - tab_text (tbl, n_cols - n_fields, 1, TAT_TITLE, _("Area")); + struct pivot_table *table = pivot_table_create (N_("Area Under the Curve")); - tab_hline (tbl, TAL_2, 0, n_cols - 1, 2); - - tab_box (tbl, - TAL_2, TAL_2, - -1, TAL_1, - 0, 0, - n_cols - 1, - n_rows - 1); - - if ( roc->print_se ) + struct pivot_dimension *statistics = pivot_dimension_create ( + table, PIVOT_AXIS_COLUMN, N_("Statistics"), + N_("Area"), PIVOT_RC_OTHER); + if (roc->print_se) { - tab_text (tbl, n_cols - 4, 1, TAT_TITLE, _("Std. Error")); - tab_text (tbl, n_cols - 3, 1, TAT_TITLE, _("Asymptotic Sig.")); - - tab_text (tbl, n_cols - 2, 1, TAT_TITLE, _("Lower Bound")); - tab_text (tbl, n_cols - 1, 1, TAT_TITLE, _("Upper Bound")); - - tab_joint_text_format (tbl, n_cols - 2, 0, 4, 0, - TAT_TITLE | TAB_CENTER, - _("Asymp. %g%% Confidence Interval"), roc->ci); - tab_vline (tbl, 0, n_cols - 1, 0, 0); - tab_hline (tbl, TAL_1, n_cols - 2, n_cols - 1, 1); + pivot_category_create_leaves ( + statistics->root, + N_("Std. Error"), PIVOT_RC_OTHER, + N_("Asymptotic Sig."), PIVOT_RC_SIGNIFICANCE); + struct pivot_category *interval = pivot_category_create_group__ ( + statistics->root, + pivot_value_new_text_format (N_("Asymp. %g%% Confidence Interval"), + roc->ci)); + pivot_category_create_leaves (interval, + N_("Lower Bound"), PIVOT_RC_OTHER, + N_("Upper Bound"), PIVOT_RC_OTHER); } - if ( roc->n_vars > 1) - tab_text (tbl, 0, 1, TAT_TITLE, _("Variable under test")); - - if ( roc->n_vars > 1) - tab_vline (tbl, TAL_2, 1, 0, n_rows - 1); + struct pivot_dimension *variables = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Variable under test")); + variables->root->show_label = true; - - for ( i = 0 ; i < roc->n_vars ; ++i ) + for (size_t i = 0 ; i < roc->n_vars ; ++i) { - tab_text (tbl, 0, 2 + i, TAT_TITLE, var_to_string (roc->vars[i])); + int var_idx = pivot_category_create_leaf ( + variables->root, pivot_value_new_variable (roc->vars[i])); - tab_double (tbl, n_cols - n_fields, 2 + i, 0, rs[i].auc, NULL); + pivot_table_put2 (table, 0, var_idx, pivot_value_new_number (rs[i].auc)); - if ( roc->print_se ) + if (roc->print_se) { - double se ; - const double sd_0_5 = sqrt ((rs[i].n1 + rs[i].n2 + 1) / - (12 * rs[i].n1 * rs[i].n2)); - double ci ; - double yy ; - - se = rs[i].auc * (1 - rs[i].auc) + (rs[i].n1 - 1) * (rs[i].q1hat - pow2 (rs[i].auc)) + - (rs[i].n2 - 1) * (rs[i].q2hat - pow2 (rs[i].auc)); - + double se = (rs[i].auc * (1 - rs[i].auc) + + (rs[i].n1 - 1) * (rs[i].q1hat - pow2 (rs[i].auc)) + + (rs[i].n2 - 1) * (rs[i].q2hat - pow2 (rs[i].auc))); se /= rs[i].n1 * rs[i].n2; - se = sqrt (se); - tab_double (tbl, n_cols - 4, 2 + i, 0, - se, - NULL); - - ci = 1 - roc->ci / 100.0; - yy = gsl_cdf_gaussian_Qinv (ci, se) ; - - tab_double (tbl, n_cols - 2, 2 + i, 0, - rs[i].auc - yy, - NULL); - - tab_double (tbl, n_cols - 1, 2 + i, 0, - rs[i].auc + yy, - NULL); - - tab_double (tbl, n_cols - 3, 2 + i, 0, - 2.0 * gsl_cdf_ugaussian_Q (fabs ((rs[i].auc - 0.5 ) / sd_0_5)), - NULL); + double ci = 1 - roc->ci / 100.0; + double yy = gsl_cdf_gaussian_Qinv (ci, se); + + double sd_0_5 = sqrt ((rs[i].n1 + rs[i].n2 + 1) / + (12 * rs[i].n1 * rs[i].n2)); + double sig = 2.0 * gsl_cdf_ugaussian_Q (fabs ((rs[i].auc - 0.5) + / sd_0_5)); + double entries[] = { se, sig, rs[i].auc - yy, rs[i].auc + yy }; + for (size_t i = 0; i < sizeof entries / sizeof *entries; i++) + pivot_table_put2 (table, i + 1, var_idx, + pivot_value_new_number (entries[i])); } } - tab_submit (tbl); + pivot_table_submit (table); } static void show_summary (const struct cmd_roc *roc) { - const int n_cols = 3; - const int n_rows = 4; - struct tab_table *tbl = tab_create (n_cols, n_rows, 0); - - tab_title (tbl, _("Case Summary")); - - tab_headers (tbl, 1, 0, 2, 0); - - tab_dim (tbl, tab_natural_dimensions, NULL); - - tab_box (tbl, - TAL_2, TAL_2, - -1, -1, - 0, 0, - n_cols - 1, - n_rows - 1); + struct pivot_table *table = pivot_table_create (N_("Case Summary")); - tab_hline (tbl, TAL_2, 0, n_cols - 1, 2); - tab_vline (tbl, TAL_2, 1, 0, n_rows - 1); + struct pivot_dimension *statistics = pivot_dimension_create ( + table, PIVOT_AXIS_COLUMN, N_("Valid N (listwise)"), + N_("Unweighted"), PIVOT_RC_INTEGER, + N_("Weighted"), PIVOT_RC_OTHER); + statistics->root->show_label = true; + struct pivot_dimension *cases = pivot_dimension_create__ ( + table, PIVOT_AXIS_ROW, pivot_value_new_variable (roc->state_var)); + cases->root->show_label = true; + pivot_category_create_leaves (cases->root, N_("Positive"), N_("Negative")); - tab_hline (tbl, TAL_2, 1, n_cols - 1, 1); - tab_vline (tbl, TAL_1, 2, 1, n_rows - 1); - - - tab_text (tbl, 0, 1, TAT_TITLE | TAB_LEFT, var_to_string (roc->state_var)); - tab_text (tbl, 1, 1, TAT_TITLE, _("Unweighted")); - tab_text (tbl, 2, 1, TAT_TITLE, _("Weighted")); - - tab_joint_text (tbl, 1, 0, 2, 0, - TAT_TITLE | TAB_CENTER, - _("Valid N (listwise)")); - - - tab_text (tbl, 0, 2, TAB_LEFT, _("Positive")); - tab_text (tbl, 0, 3, TAB_LEFT, _("Negative")); - - - tab_double (tbl, 1, 2, 0, roc->pos, &F_8_0); - tab_double (tbl, 1, 3, 0, roc->neg, &F_8_0); - - tab_double (tbl, 2, 2, 0, roc->pos_weighted, 0); - tab_double (tbl, 2, 3, 0, roc->neg_weighted, 0); - - tab_submit (tbl); + struct entry + { + int stat_idx; + int case_idx; + double x; + } + entries[] = { + { 0, 0, roc->pos }, + { 0, 1, roc->neg }, + { 1, 0, roc->pos_weighted }, + { 1, 1, roc->neg_weighted }, + }; + for (size_t i = 0; i < sizeof entries / sizeof *entries; i++) + { + const struct entry *e = &entries[i]; + pivot_table_put2 (table, e->stat_idx, e->case_idx, + pivot_value_new_number (e->x)); + } + pivot_table_submit (table); } - static void show_coords (struct roc_state *rs, const struct cmd_roc *roc) { - int x = 1; - int i; - const int n_cols = roc->n_vars > 1 ? 4 : 3; - int n_rows = 1; - struct tab_table *tbl ; - - for (i = 0; i < roc->n_vars; ++i) - n_rows += casereader_count_cases (rs[i].cutpoint_rdr); - - tbl = tab_create (n_cols, n_rows, 0); - - if ( roc->n_vars > 1) - tab_title (tbl, _("Coordinates of the Curve")); - else - tab_title (tbl, _("Coordinates of the Curve (%s)"), var_to_string (roc->vars[0])); - - - tab_headers (tbl, 1, 0, 1, 0); + struct pivot_table *table = pivot_table_create ( + N_("Coordinates of the Curve")); - tab_dim (tbl, tab_natural_dimensions, NULL); + pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Statistics"), + N_("Positive if greater than or equal to"), + N_("Sensitivity"), N_("1 - Specificity")); - tab_hline (tbl, TAL_2, 0, n_cols - 1, 1); + struct pivot_dimension *coordinates = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Coordinates")); + coordinates->hide_all_labels = true; - if ( roc->n_vars > 1) - tab_text (tbl, 0, 0, TAT_TITLE, _("Test variable")); + struct pivot_dimension *variables = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Test variable")); + variables->root->show_label = true; - tab_text (tbl, n_cols - 3, 0, TAT_TITLE, _("Positive if greater than or equal to")); - tab_text (tbl, n_cols - 2, 0, TAT_TITLE, _("Sensitivity")); - tab_text (tbl, n_cols - 1, 0, TAT_TITLE, _("1 - Specificity")); - tab_box (tbl, - TAL_2, TAL_2, - -1, TAL_1, - 0, 0, - n_cols - 1, - n_rows - 1); - - if ( roc->n_vars > 1) - tab_vline (tbl, TAL_2, 1, 0, n_rows - 1); - - for (i = 0; i < roc->n_vars; ++i) + int n_coords = 0; + for (size_t i = 0; i < roc->n_vars; ++i) { - struct ccase *cc; struct casereader *r = casereader_clone (rs[i].cutpoint_rdr); - if ( roc->n_vars > 1) - tab_text (tbl, 0, x, TAT_TITLE, var_to_string (roc->vars[i])); - - if ( i > 0) - tab_hline (tbl, TAL_1, 0, n_cols - 1, x); + int var_idx = pivot_category_create_leaf ( + variables->root, pivot_value_new_variable (roc->vars[i])); - - for (; (cc = casereader_read (r)) != NULL; - case_unref (cc), x++) - { - const double se = case_data_idx (cc, TP)->f / - ( - case_data_idx (cc, TP)->f - + - case_data_idx (cc, FN)->f - ); - - const double sp = case_data_idx (cc, TN)->f / - ( - case_data_idx (cc, TN)->f - + - case_data_idx (cc, FP)->f - ); - - tab_double (tbl, n_cols - 3, x, 0, case_data_idx (cc, CUTPOINT)->f, - var_get_print_format (roc->vars[i])); - - tab_double (tbl, n_cols - 2, x, 0, se, NULL); - tab_double (tbl, n_cols - 1, x, 0, 1 - sp, NULL); - } - - casereader_destroy (r); - } - - tab_submit (tbl); -} - - -static void -draw_roc (struct roc_state *rs, const struct cmd_roc *roc) -{ - int i; - - struct chart *roc_chart = chart_create (); - - chart_write_title (roc_chart, _("ROC Curve")); - chart_write_xlabel (roc_chart, _("1 - Specificity")); - chart_write_ylabel (roc_chart, _("Sensitivity")); - - chart_write_xscale (roc_chart, 0, 1, 5); - chart_write_yscale (roc_chart, 0, 1, 5); - - if ( roc->reference ) - { - chart_line (roc_chart, 1.0, 0, - 0.0, 1.0, - CHART_DIM_X); - } - - for (i = 0; i < roc->n_vars; ++i) - { struct ccase *cc; - struct casereader *r = casereader_clone (rs[i].cutpoint_rdr); - - chart_vector_start (roc_chart, var_get_name (roc->vars[i])); - for (; (cc = casereader_read (r)) != NULL; - case_unref (cc)) + int coord_idx = 0; + for (; (cc = casereader_read (r)) != NULL; case_unref (cc)) { - double se = case_data_idx (cc, TP)->f; - double sp = case_data_idx (cc, TN)->f; - - se /= case_data_idx (cc, FN)->f + - case_data_idx (cc, TP)->f ; + const double se = case_data_idx (cc, ROC_TP)->f / + (case_data_idx (cc, ROC_TP)->f + case_data_idx (cc, ROC_FN)->f); - sp /= case_data_idx (cc, TN)->f + - case_data_idx (cc, FP)->f ; + const double sp = case_data_idx (cc, ROC_TN)->f / + (case_data_idx (cc, ROC_TN)->f + case_data_idx (cc, ROC_FP)->f); - chart_vector (roc_chart, 1 - sp, se); + if (coord_idx >= n_coords) + { + assert (coord_idx == n_coords); + pivot_category_create_leaf ( + coordinates->root, pivot_value_new_integer (++n_coords)); + } + + pivot_table_put3 ( + table, 0, coord_idx, var_idx, + pivot_value_new_var_value (roc->vars[i], + case_data_idx (cc, ROC_CUTPOINT))); + + pivot_table_put3 (table, 1, coord_idx, var_idx, + pivot_value_new_number (se)); + pivot_table_put3 (table, 2, coord_idx, var_idx, + pivot_value_new_number (1 - sp)); + coord_idx++; } - chart_vector_end (roc_chart); + casereader_destroy (r); } - chart_write_legend (roc_chart); - - chart_submit (roc_chart); + pivot_table_submit (table); } @@ -1218,13 +1124,21 @@ output_roc (struct roc_state *rs, const struct cmd_roc *roc) { show_summary (roc); - if ( roc->curve ) - draw_roc (rs, roc); + if (roc->curve) + { + struct roc_chart *rc; + size_t i; + + rc = roc_chart_create (roc->reference); + for (i = 0; i < roc->n_vars; i++) + roc_chart_add_var (rc, var_get_name (roc->vars[i]), + rs[i].cutpoint_rdr); + roc_chart_submit (rc); + } show_auc (rs, roc); - - if ( roc->print_coords ) + if (roc->print_coords) show_coords (rs, roc); }