X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Froc.c;h=7d13c2c2cb7cf32c13c9d5c5051cf1328441517a;hb=f790dbda9d498eef9c9c0a49078adbeecf768d56;hp=1d21a4f6a1221ed6b83f3c2bbb8d4e25c457c95a;hpb=cb72db62c20ecab427229110820c5b053d0663c4;p=pspp diff --git a/src/language/stats/roc.c b/src/language/stats/roc.c index 1d21a4f6a1..7d13c2c2cb 100644 --- a/src/language/stats/roc.c +++ b/src/language/stats/roc.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2009 Free Software Foundation, Inc. + Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,30 +16,26 @@ #include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - - -#include +#include "language/stats/roc.h" #include -#include -#include -#include +#include "data/casegrouper.h" +#include "data/casereader.h" +#include "data/casewriter.h" +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/format.h" +#include "data/subcase.h" +#include "language/command.h" +#include "language/lexer/lexer.h" +#include "language/lexer/value-parser.h" +#include "language/lexer/variable-parser.h" +#include "libpspp/misc.h" +#include "math/sort.h" +#include "output/chart-item.h" +#include "output/charts/roc-chart.h" +#include "output/tab.h" #include "gettext.h" #define _(msgid) gettext (msgid) @@ -51,8 +47,9 @@ struct cmd_roc const struct variable **vars; const struct dictionary *dict; - const struct variable *state_var ; + const struct variable *state_var; union value state_value; + size_t state_var_width; /* Plot the roc curve */ bool curve; @@ -97,7 +94,10 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) roc.pos = roc.pos_weighted = 0; roc.neg = roc.neg_weighted = 0; roc.dict = dataset_dict (ds); + roc.state_var = NULL; + roc.state_var_width = -1; + lex_match (lexer, T_SLASH); if (!parse_variables_const (lexer, dict, &roc.vars, &roc.n_vars, PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC)) goto error; @@ -108,28 +108,33 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } roc.state_var = parse_variable (lexer, dict); + if (! roc.state_var) + { + goto error; + } - if ( !lex_force_match (lexer, '(')) + if ( !lex_force_match (lexer, T_LPAREN)) { goto error; } - parse_value (lexer, &roc.state_value, var_get_width (roc.state_var)); + roc.state_var_width = var_get_width (roc.state_var); + value_init (&roc.state_value, roc.state_var_width); + parse_value (lexer, &roc.state_value, roc.state_var); - if ( !lex_force_match (lexer, ')')) + if ( !lex_force_match (lexer, T_RPAREN)) { goto error; } - - while (lex_token (lexer) != '.') + while (lex_token (lexer) != T_ENDCMD) { - lex_match (lexer, '/'); + lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "MISSING")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "INCLUDE")) { @@ -148,15 +153,17 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "PLOT")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (lex_match_id (lexer, "CURVE")) { roc.curve = true; - if (lex_match (lexer, '(')) + if (lex_match (lexer, T_LPAREN)) { roc.reference = true; - lex_force_match_id (lexer, "REFERENCE"); - lex_force_match (lexer, ')'); + if (! lex_force_match_id (lexer, "REFERENCE")) + goto error; + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } } else if (lex_match_id (lexer, "NONE")) @@ -171,8 +178,8 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "PRINT")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "SE")) { @@ -191,12 +198,13 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } else if (lex_match_id (lexer, "CRITERIA")) { - lex_match (lexer, '='); - while (lex_token (lexer) != '.' && lex_token (lexer) != '/') + lex_match (lexer, T_EQUALS); + while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH) { if (lex_match_id (lexer, "CUTOFF")) { - lex_force_match (lexer, '('); + if (! lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "INCLUDE")) { roc.exclude = MV_SYSTEM; @@ -210,11 +218,13 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "TESTPOS")) { - lex_force_match (lexer, '('); + if (! lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "LARGE")) { roc.invert = false; @@ -228,19 +238,24 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (! lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "CI")) { - lex_force_match (lexer, '('); - lex_force_num (lexer); + if (!lex_force_match (lexer, T_LPAREN)) + goto error; + if (! lex_force_num (lexer)) + goto error; roc.ci = lex_number (lexer); lex_get (lexer); - lex_force_match (lexer, ')'); + if (!lex_force_match (lexer, T_RPAREN)) + goto error; } else if (lex_match_id (lexer, "DISTRIBUTION")) { - lex_force_match (lexer, '('); + if (!lex_force_match (lexer, T_LPAREN)) + goto error; if (lex_match_id (lexer, "FREE")) { roc.bi_neg_exp = false; @@ -254,7 +269,8 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) lex_error (lexer, NULL); goto error; } - lex_force_match (lexer, ')'); + if (!lex_force_match (lexer, T_RPAREN)) + goto error; } else { @@ -270,13 +286,17 @@ cmd_roc (struct lexer *lexer, struct dataset *ds) } } - if ( ! run_roc (ds, &roc)) + if ( ! run_roc (ds, &roc)) goto error; + if ( roc.state_var) + value_destroy (&roc.state_value, roc.state_var_width); free (roc.vars); return CMD_SUCCESS; error: + if ( roc.state_var) + value_destroy (&roc.state_value, roc.state_var_width); free (roc.vars); return CMD_FAILURE; } @@ -328,10 +348,10 @@ dump_casereader (struct casereader *reader) #endif -/* +/* Return true iff the state variable indicates that C has positive actual state. - As a side effect, this function also accumulates the roc->{pos,neg} and + As a side effect, this function also accumulates the roc->{pos,neg} and roc->{pos,neg}_weighted counts. */ static bool @@ -364,7 +384,7 @@ match_positives (const struct ccase *c, void *aux) #define N_EQ 1 #define N_PRED 2 -/* Some intermediate state for calculating the cutpoints and the +/* Some intermediate state for calculating the cutpoints and the standard error values */ struct roc_state { @@ -374,7 +394,7 @@ struct roc_state double n2; /* total weight of negatives */ /* intermediates for standard error */ - double q1hat; + double q1hat; double q2hat; /* intermediates for cutpoints */ @@ -385,29 +405,28 @@ struct roc_state double max; }; -/* +/* Return a new casereader based upon CUTPOINT_RDR. The number of "positive" cases are placed into the position TRUE_INDEX, and the number of "negative" cases into FALSE_INDEX. - POS_COND and RESULT determine the semantics of what is + POS_COND and RESULT determine the semantics of what is "positive". WEIGHT is the value of a single count. */ static struct casereader * -accumulate_counts (struct casereader *cutpoint_rdr, - double result, double weight, +accumulate_counts (struct casereader *input, + double result, double weight, bool (*pos_cond) (double, double), int true_index, int false_index) { - const struct caseproto *proto = casereader_get_proto (cutpoint_rdr); + const struct caseproto *proto = casereader_get_proto (input); struct casewriter *w = autopaging_writer_create (proto); - struct casereader *r = casereader_clone (cutpoint_rdr); struct ccase *cpc; double prev_cp = SYSMIS; - for ( ; (cpc = casereader_read (r) ); case_unref (cpc)) + for ( ; (cpc = casereader_read (input) ); case_unref (cpc)) { struct ccase *new_case; const double cp = case_data_idx (cpc, ROC_CUTPOINT)->f; @@ -429,7 +448,7 @@ accumulate_counts (struct casereader *cutpoint_rdr, casewriter_write (w, new_case); } - casereader_destroy (r); + casereader_destroy (input); return casewriter_make_reader (w); } @@ -450,7 +469,7 @@ static void output_roc (struct roc_state *rs, const struct cmd_roc *roc); CUTPOINT_RDR accordingly. TRUE_INDEX and FALSE_INDEX are the indices which receive these values. POS_COND is the condition defining true and false. - + 3. CC is filled with the cumulative weight of all cases of READER. */ static struct casereader * @@ -458,7 +477,7 @@ process_group (const struct variable *var, struct casereader *reader, bool (*pred) (double, double), const struct dictionary *dict, double *cc, - struct casereader **cutpoint_rdr, + struct casereader **cutpoint_rdr, bool (*pos_cond) (double, double), int true_index, int false_index) @@ -470,7 +489,7 @@ process_group (const struct variable *var, struct casereader *reader, const int weight_idx = w ? var_get_case_index (w) : caseproto_get_n_widths (casereader_get_proto (r1)) - 1; - + struct ccase *c1; struct casereader *rclone = casereader_clone (r1); @@ -481,7 +500,7 @@ process_group (const struct variable *var, struct casereader *reader, proto = caseproto_add_width (proto, 0); proto = caseproto_add_width (proto, 0); - wtr = autopaging_writer_create (proto); + wtr = autopaging_writer_create (proto); *cc = 0; @@ -527,15 +546,18 @@ process_group (const struct variable *var, struct casereader *reader, casereader_destroy (r2); } + casereader_destroy (r1); casereader_destroy (rclone); + caseproto_unref (proto); + return casewriter_make_reader (wtr); } /* Some more indeces into case data */ #define N_POS_EQ 1 /* number of positive cases with values equal to n */ -#define N_POS_GT 2 /* number of postive cases with values greater than n */ +#define N_POS_GT 2 /* number of positive cases with values greater than n */ #define N_NEG_EQ 3 /* number of negative cases with values equal to n */ #define N_NEG_LT 4 /* number of negative cases with values less than n */ @@ -613,7 +635,7 @@ append_cutpoint (struct casewriter *writer, double cutpoint) } -/* +/* Create and initialise the rs[x].cutpoint_rdr casereaders. That is, the readers will be created with width 5, ready to take the values (cutpoint, ROC_TP, ROC_FN, ROC_TN, ROC_FP), and the reader will be populated with its final number of cases. @@ -626,30 +648,35 @@ prepare_cutpoints (struct cmd_roc *roc, struct roc_state *rs, struct casereader int i; struct casereader *r = casereader_clone (input); struct ccase *c; - struct caseproto *proto = caseproto_create (); - struct subcase ordering; - subcase_init (&ordering, ROC_CUTPOINT, 0, SC_ASCEND); + { + struct caseproto *proto = caseproto_create (); + struct subcase ordering; + subcase_init (&ordering, ROC_CUTPOINT, 0, SC_ASCEND); - proto = caseproto_add_width (proto, 0); /* cutpoint */ - proto = caseproto_add_width (proto, 0); /* ROC_TP */ - proto = caseproto_add_width (proto, 0); /* ROC_FN */ - proto = caseproto_add_width (proto, 0); /* ROC_TN */ - proto = caseproto_add_width (proto, 0); /* ROC_FP */ + proto = caseproto_add_width (proto, 0); /* cutpoint */ + proto = caseproto_add_width (proto, 0); /* ROC_TP */ + proto = caseproto_add_width (proto, 0); /* ROC_FN */ + proto = caseproto_add_width (proto, 0); /* ROC_TN */ + proto = caseproto_add_width (proto, 0); /* ROC_FP */ - for (i = 0 ; i < roc->n_vars; ++i) - { - rs[i].cutpoint_wtr = sort_create_writer (&ordering, proto); - rs[i].prev_result = SYSMIS; - rs[i].max = -DBL_MAX; - rs[i].min = DBL_MAX; - } + for (i = 0 ; i < roc->n_vars; ++i) + { + rs[i].cutpoint_wtr = sort_create_writer (&ordering, proto); + rs[i].prev_result = SYSMIS; + rs[i].max = -DBL_MAX; + rs[i].min = DBL_MAX; + } + + caseproto_unref (proto); + subcase_destroy (&ordering); + } for (; (c = casereader_read (r)) != NULL; case_unref (c)) { for (i = 0 ; i < roc->n_vars; ++i) { - const union value *v = case_data (c, roc->vars[i]); + const union value *v = case_data (c, roc->vars[i]); const double result = v->f; if ( mv_is_value_missing (var_get_missing_values (roc->vars[i]), v, roc->exclude)) @@ -690,7 +717,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) struct casereader *negatives = NULL; struct casereader *positives = NULL; - struct caseproto *n_proto = caseproto_create (); + struct caseproto *n_proto = NULL; struct subcase up_ordering; struct subcase down_ordering; @@ -715,7 +742,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) /* Separate the positive actual state cases from the negative ones */ - positives = + positives = casereader_create_filter_func (input, match_positives, NULL, @@ -723,7 +750,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) neg_wtr); n_proto = caseproto_create (); - + n_proto = caseproto_add_width (n_proto, 0); n_proto = caseproto_add_width (n_proto, 0); n_proto = caseproto_add_width (n_proto, 0); @@ -741,14 +768,13 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) struct ccase *c; struct ccase *cpos; - struct casereader *n_neg ; + struct casereader *n_neg_reader ; const struct variable *var = roc->vars[i]; struct casereader *neg ; struct casereader *pos = casereader_clone (positives); - - struct casereader *n_pos = + struct casereader *n_pos_reader = process_positive_group (var, pos, dict, &rs[i]); if ( negatives == NULL) @@ -758,18 +784,17 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) neg = casereader_clone (negatives); - n_neg = process_negative_group (var, neg, dict, &rs[i]); - + n_neg_reader = process_negative_group (var, neg, dict, &rs[i]); /* Merge the n_pos and n_neg casereaders */ w = sort_create_writer (&up_ordering, n_proto); - for ( ; (cpos = casereader_read (n_pos) ); case_unref (cpos)) + for ( ; (cpos = casereader_read (n_pos_reader) ); case_unref (cpos)) { struct ccase *pos_case = case_create (n_proto); struct ccase *cneg; const double jpos = case_data_idx (cpos, VALUE)->f; - while ((cneg = casereader_read (n_neg))) + while ((cneg = casereader_read (n_neg_reader))) { struct ccase *nc = case_create (n_proto); @@ -799,6 +824,9 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) casewriter_write (w, pos_case); } + casereader_destroy (n_pos_reader); + casereader_destroy (n_neg_reader); + /* These aren't used anymore */ #undef N_EQ #undef N_PRED @@ -821,11 +849,12 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) n_pos_gt = prev_pos_gt; case_data_rw_idx (nc, N_POS_GT)->f = n_pos_gt; } - + casewriter_write (w, nc); prev_pos_gt = n_pos_gt; } + casereader_destroy (r); r = casewriter_make_reader (w); } @@ -845,11 +874,12 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) n_neg_lt = prev_neg_lt; case_data_rw_idx (nc, N_NEG_LT)->f = n_neg_lt; } - + casewriter_write (w, nc); prev_neg_lt = n_neg_lt; } + casereader_destroy (r); r = casewriter_make_reader (w); } @@ -857,7 +887,7 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) struct ccase *prev_case = NULL; for ( ; (c = casereader_read (r) ); case_unref (c)) { - const struct ccase *next_case = casereader_peek (r, 0); + struct ccase *next_case = casereader_peek (r, 0); const double j = case_data_idx (c, VALUE)->f; double n_pos_eq = case_data_idx (c, N_POS_EQ)->f; @@ -891,12 +921,15 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) } + case_unref (next_case); case_unref (prev_case); prev_case = case_clone (c); } + casereader_destroy (r); + case_unref (prev_case); - rs[i].auc /= rs[i].n1 * rs[i].n2; - if ( roc->invert ) + rs[i].auc /= rs[i].n1 * rs[i].n2; + if ( roc->invert ) rs[i].auc = 1 - rs[i].auc; if ( roc->bi_neg_exp ) @@ -915,8 +948,15 @@ do_roc (struct cmd_roc *roc, struct casereader *reader, struct dictionary *dict) casereader_destroy (positives); casereader_destroy (negatives); + caseproto_unref (n_proto); + subcase_destroy (&up_ordering); + subcase_destroy (&down_ordering); + output_roc (rs, roc); + for (i = 0 ; i < roc->n_vars; ++i) + casereader_destroy (rs[i].cutpoint_rdr); + free (rs); } @@ -936,7 +976,6 @@ show_auc (struct roc_state *rs, const struct cmd_roc *roc) tab_headers (tbl, n_cols - n_fields, 0, 1, 0); - tab_dim (tbl, tab_natural_dimensions, NULL, NULL); tab_text (tbl, n_cols - n_fields, 1, TAT_TITLE, _("Area")); @@ -975,7 +1014,7 @@ show_auc (struct roc_state *rs, const struct cmd_roc *roc) { tab_text (tbl, 0, 2 + i, TAT_TITLE, var_to_string (roc->vars[i])); - tab_double (tbl, n_cols - n_fields, 2 + i, 0, rs[i].auc, NULL); + tab_double (tbl, n_cols - n_fields, 2 + i, 0, rs[i].auc, NULL, RC_OTHER); if ( roc->print_se ) { @@ -994,22 +1033,22 @@ show_auc (struct roc_state *rs, const struct cmd_roc *roc) tab_double (tbl, n_cols - 4, 2 + i, 0, se, - NULL); + NULL, RC_OTHER); ci = 1 - roc->ci / 100.0; yy = gsl_cdf_gaussian_Qinv (ci, se) ; tab_double (tbl, n_cols - 2, 2 + i, 0, rs[i].auc - yy, - NULL); + NULL, RC_OTHER); tab_double (tbl, n_cols - 1, 2 + i, 0, rs[i].auc + yy, - NULL); + NULL, RC_OTHER); tab_double (tbl, n_cols - 3, 2 + i, 0, 2.0 * gsl_cdf_ugaussian_Q (fabs ((rs[i].auc - 0.5 ) / sd_0_5)), - NULL); + NULL, RC_PVALUE); } } @@ -1028,8 +1067,6 @@ show_summary (const struct cmd_roc *roc) tab_headers (tbl, 1, 0, 2, 0); - tab_dim (tbl, tab_natural_dimensions, NULL, NULL); - tab_box (tbl, TAL_2, TAL_2, -1, -1, @@ -1058,11 +1095,11 @@ show_summary (const struct cmd_roc *roc) tab_text (tbl, 0, 3, TAB_LEFT, _("Negative")); - tab_double (tbl, 1, 2, 0, roc->pos, &F_8_0); - tab_double (tbl, 1, 3, 0, roc->neg, &F_8_0); + tab_double (tbl, 1, 2, 0, roc->pos, NULL, RC_INTEGER); + tab_double (tbl, 1, 3, 0, roc->neg, NULL, RC_INTEGER); - tab_double (tbl, 2, 2, 0, roc->pos_weighted, 0); - tab_double (tbl, 2, 3, 0, roc->neg_weighted, 0); + tab_double (tbl, 2, 2, 0, roc->pos_weighted, NULL, RC_OTHER); + tab_double (tbl, 2, 3, 0, roc->neg_weighted, NULL, RC_OTHER); tab_submit (tbl); } @@ -1090,8 +1127,6 @@ show_coords (struct roc_state *rs, const struct cmd_roc *roc) tab_headers (tbl, 1, 0, 1, 0); - tab_dim (tbl, tab_natural_dimensions, NULL, NULL); - tab_hline (tbl, TAL_2, 0, n_cols - 1, 1); if ( roc->n_vars > 1) @@ -1141,10 +1176,10 @@ show_coords (struct roc_state *rs, const struct cmd_roc *roc) ); tab_double (tbl, n_cols - 3, x, 0, case_data_idx (cc, ROC_CUTPOINT)->f, - var_get_print_format (roc->vars[i])); + var_get_print_format (roc->vars[i]), RC_OTHER); - tab_double (tbl, n_cols - 2, x, 0, se, NULL); - tab_double (tbl, n_cols - 1, x, 0, 1 - sp, NULL); + tab_double (tbl, n_cols - 2, x, 0, se, NULL, RC_OTHER); + tab_double (tbl, n_cols - 1, x, 0, 1 - sp, NULL, RC_OTHER); } casereader_destroy (r); @@ -1168,7 +1203,7 @@ output_roc (struct roc_state *rs, const struct cmd_roc *roc) for (i = 0; i < roc->n_vars; i++) roc_chart_add_var (rc, var_get_name (roc->vars[i]), rs[i].cutpoint_rdr); - chart_submit (roc_chart_get_chart (rc)); + roc_chart_submit (rc); } show_auc (rs, roc);