X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Faggregate.c;h=0d181bd45e3999ab1c45b8267e713f6cf1aeb797;hb=5c3291dc396b795696e94f47780308fd7ace6fc4;hp=c9d68d35da70867fccfaf92f0b7f5a1b08322f86;hpb=9f4661992f4b481c6dafa6fd53c94ecfe7b3af8c;p=pspp-builds.git diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c index c9d68d35..0d181bd4 100644 --- a/src/language/stats/aggregate.c +++ b/src/language/stats/aggregate.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc. + Copyright (C) 1997-9, 2000, 2006, 2008, 2009 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -29,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +43,8 @@ #include #include #include +#include +#include #include "minmax.h" #include "xalloc.h" @@ -75,12 +77,17 @@ struct agr_var char *string; bool saw_missing; struct moments1 *moments; + double cc; + + struct variable *subject; + struct variable *weight; + struct casewriter *writer; }; /* Aggregation functions. */ enum { - NONE, SUM, MEAN, SD, MAX, MIN, PGT, PLT, PIN, POUT, FGT, FLT, FIN, + NONE, SUM, MEAN, MEDIAN, SD, MAX, MIN, PGT, PLT, PIN, POUT, FGT, FLT, FIN, FOUT, N, NU, NMISS, NUMISS, FIRST, LAST, N_AGR_FUNCS, N_NO_VARS, NU_NO_VARS, FUNC = 0x1f, /* Function mask. */ @@ -92,7 +99,7 @@ struct agr_func { const char *name; /* Aggregation function name. */ size_t n_args; /* Number of arguments. */ - enum var_type alpha_type; /* When given ALPHA arguments, output type. */ + enum val_type alpha_type; /* When given ALPHA arguments, output type. */ struct fmt_spec format; /* Format spec if alpha_type != ALPHA. */ }; @@ -102,26 +109,27 @@ static const struct agr_func agr_func_tab[] = {"", 0, -1, {0, 0, 0}}, {"SUM", 0, -1, {FMT_F, 8, 2}}, {"MEAN", 0, -1, {FMT_F, 8, 2}}, + {"MEDIAN", 0, -1, {FMT_F, 8, 2}}, {"SD", 0, -1, {FMT_F, 8, 2}}, - {"MAX", 0, VAR_STRING, {-1, -1, -1}}, - {"MIN", 0, VAR_STRING, {-1, -1, -1}}, - {"PGT", 1, VAR_NUMERIC, {FMT_F, 5, 1}}, - {"PLT", 1, VAR_NUMERIC, {FMT_F, 5, 1}}, - {"PIN", 2, VAR_NUMERIC, {FMT_F, 5, 1}}, - {"POUT", 2, VAR_NUMERIC, {FMT_F, 5, 1}}, - {"FGT", 1, VAR_NUMERIC, {FMT_F, 5, 3}}, - {"FLT", 1, VAR_NUMERIC, {FMT_F, 5, 3}}, - {"FIN", 2, VAR_NUMERIC, {FMT_F, 5, 3}}, - {"FOUT", 2, VAR_NUMERIC, {FMT_F, 5, 3}}, - {"N", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, - {"NU", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, - {"NMISS", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, - {"NUMISS", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, - {"FIRST", 0, VAR_STRING, {-1, -1, -1}}, - {"LAST", 0, VAR_STRING, {-1, -1, -1}}, + {"MAX", 0, VAL_STRING, {-1, -1, -1}}, + {"MIN", 0, VAL_STRING, {-1, -1, -1}}, + {"PGT", 1, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"PLT", 1, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"PIN", 2, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"POUT", 2, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"FGT", 1, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FLT", 1, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FIN", 2, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FOUT", 2, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"N", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NU", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NMISS", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NUMISS", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"FIRST", 0, VAL_STRING, {-1, -1, -1}}, + {"LAST", 0, VAL_STRING, {-1, -1, -1}}, {NULL, 0, -1, {-1, -1, -1}}, - {"N", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, - {"NU", 0, VAR_NUMERIC, {FMT_F, 7, 0}}, + {"N", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NU", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, }; /* Missing value types. */ @@ -135,10 +143,10 @@ enum missing_treatment struct agr_proc { /* Break variables. */ - struct case_ordering *sort; /* Sort criteria. */ + struct subcase sort; /* Sort criteria (break variables). */ const struct variable **break_vars; /* Break variables. */ size_t break_var_cnt; /* Number of break variables. */ - struct ccase break_case; /* Last values of break variables. */ + struct ccase *break_case; /* Last values of break variables. */ enum missing_treatment missing; /* How to treat missing values. */ struct agr_var *agr_vars; /* First aggregate variable. */ @@ -149,6 +157,7 @@ struct agr_proc static void initialize_aggregate_info (struct agr_proc *, const struct ccase *); + static void accumulate_aggregate_info (struct agr_proc *, const struct ccase *); /* Prototypes. */ @@ -178,10 +187,11 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) memset(&agr, 0 , sizeof (agr)); agr.missing = ITEMWISE; - case_nullify (&agr.break_case); + agr.break_case = NULL; agr.dict = dict_create (); agr.src_dict = dict; + subcase_init_empty (&agr.sort); dict_set_label (agr.dict, dict_get_label (dict)); dict_set_documents (agr.dict, dict_get_documents (dict)); @@ -220,13 +230,10 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) int i; lex_match (lexer, '='); - agr.sort = parse_case_ordering (lexer, dict, - - &saw_direction); - if (agr.sort == NULL) + if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars, + &saw_direction)) goto error; - case_ordering_get_vars (agr.sort, - &agr.break_vars, &agr.break_var_cnt); + agr.break_var_cnt = subcase_get_n_fields (&agr.sort); for (i = 0; i < agr.break_var_cnt; i++) dict_clone_var_assert (agr.dict, agr.break_vars[i], @@ -267,7 +274,7 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) so TEMPORARY is moot. */ proc_cancel_temporary_transformations (ds); proc_discard_output (ds); - output = autopaging_writer_create (dict_get_next_value_idx (agr.dict)); + output = autopaging_writer_create (dict_get_proto (agr.dict)); } else { @@ -277,10 +284,10 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) } input = proc_open (ds); - if (agr.sort != NULL && !presorted) + if (!subcase_is_empty (&agr.sort) && !presorted) { - input = sort_execute (input, agr.sort); - agr.sort = NULL; + input = sort_execute (input, &agr.sort); + subcase_clear (&agr.sort); } for (grouper = casegrouper_create_vars (input, agr.break_vars, @@ -288,18 +295,17 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) casegrouper_get_next_group (grouper, &group); casereader_destroy (group)) { - struct ccase c; - - if (!casereader_peek (group, 0, &c)) + struct ccase *c = casereader_peek (group, 0); + if (c == NULL) { casereader_destroy (group); continue; } - initialize_aggregate_info (&agr, &c); - case_destroy (&c); + initialize_aggregate_info (&agr, c); + case_unref (c); - for (; casereader_read (group, &c); case_destroy (&c)) - accumulate_aggregate_info (&agr, &c); + for (; (c = casereader_read (group)) != NULL; case_unref (c)) + accumulate_aggregate_info (&agr, c); dump_aggregate_info (&agr, output); } if (!casegrouper_destroy (grouper)) @@ -330,6 +336,7 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) } agr_destroy (&agr); + fh_unref (out_file); return CMD_SUCCESS; error: @@ -337,12 +344,14 @@ error: proc_commit (ds); casewriter_destroy (output); agr_destroy (&agr); + fh_unref (out_file); return CMD_CASCADING_FAILURE; } /* Parse all the aggregate functions. */ static bool -parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, struct agr_proc *agr) +parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, + struct agr_proc *agr) { struct agr_var *tail; /* Tail of linked list starting at agr->vars. */ @@ -476,12 +485,12 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, s if (lex_token (lexer) == T_STRING) { arg[i].c = ds_xstrdup (lex_tokstr (lexer)); - type = VAR_STRING; + type = VAL_STRING; } else if (lex_is_number (lexer)) { arg[i].f = lex_tokval (lexer); - type = VAR_NUMERIC; + type = VAL_NUMERIC; } else { @@ -543,7 +552,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, s variables. */ for (i = 0; i < n_dest; i++) { - struct agr_var *v = xmalloc (sizeof *v); + struct agr_var *v = xzalloc (sizeof *v); /* Add variable to chain. */ if (agr->agr_vars != NULL) @@ -571,12 +580,12 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, s v->string = xmalloc (var_get_width (src[i])); } - if (function->alpha_type == VAR_STRING) + if (function->alpha_type == VAL_STRING) destvar = dict_clone_var (agr->dict, v->src, dest[i]); else { assert (var_is_numeric (v->src) - || function->alpha_type == VAR_NUMERIC); + || function->alpha_type == VAL_NUMERIC); destvar = dict_create_var (agr->dict, dest[i], 0); if (destvar != NULL) { @@ -682,9 +691,9 @@ agr_destroy (struct agr_proc *agr) { struct agr_var *iter, *next; - case_ordering_destroy (agr->sort); + subcase_destroy (&agr->sort); free (agr->break_vars); - case_destroy (&agr->break_case); + case_unref (agr->break_case); for (iter = agr->agr_vars; iter; iter = next) { next = iter->next; @@ -701,6 +710,10 @@ agr_destroy (struct agr_proc *agr) } else if (iter->function == SD) moments1_destroy (iter->moments); + + var_destroy (iter->subject); + var_destroy (iter->weight); + free (iter); } if (agr->dict != NULL) @@ -753,6 +766,25 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) iter->dbl[0] += v->f * weight; iter->dbl[1] += weight; break; + case MEDIAN: + { + double wv ; + struct ccase *cout; + + cout = case_create (casewriter_get_proto (iter->writer)); + + case_data_rw (cout, iter->subject)->f + = case_data (input, iter->src)->f; + + wv = dict_get_case_weight (agr->src_dict, input, NULL); + + case_data_rw (cout, iter->weight)->f = wv; + + iter->cc += wv; + + casewriter_write (iter->writer, cout); + } + break; case SD: moments1_add (iter->moments, v->f, weight); break; @@ -761,8 +793,8 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) iter->int1 = 1; break; case MAX | FSTRING: - if (memcmp (iter->string, v->s, src_width) < 0) - memcpy (iter->string, v->s, src_width); + if (memcmp (iter->string, value_str (v, src_width), src_width) < 0) + memcpy (iter->string, value_str (v, src_width), src_width); iter->int1 = 1; break; case MIN: @@ -770,8 +802,8 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) iter->int1 = 1; break; case MIN | FSTRING: - if (memcmp (iter->string, v->s, src_width) > 0) - memcpy (iter->string, v->s, src_width); + if (memcmp (iter->string, value_str (v, src_width), src_width) > 0) + memcpy (iter->string, value_str (v, src_width), src_width); iter->int1 = 1; break; case FGT: @@ -782,7 +814,8 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) break; case FGT | FSTRING: case PGT | FSTRING: - if (memcmp (iter->arg[0].c, v->s, src_width) < 0) + if (memcmp (iter->arg[0].c, + value_str (v, src_width), src_width) < 0) iter->dbl[0] += weight; iter->dbl[1] += weight; break; @@ -794,7 +827,8 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) break; case FLT | FSTRING: case PLT | FSTRING: - if (memcmp (iter->arg[0].c, v->s, src_width) > 0) + if (memcmp (iter->arg[0].c, + value_str (v, src_width), src_width) > 0) iter->dbl[0] += weight; iter->dbl[1] += weight; break; @@ -806,8 +840,10 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) break; case FIN | FSTRING: case PIN | FSTRING: - if (memcmp (iter->arg[0].c, v->s, src_width) <= 0 - && memcmp (iter->arg[1].c, v->s, src_width) >= 0) + if (memcmp (iter->arg[0].c, + value_str (v, src_width), src_width) <= 0 + && memcmp (iter->arg[1].c, + value_str (v, src_width), src_width) >= 0) iter->dbl[0] += weight; iter->dbl[1] += weight; break; @@ -819,8 +855,10 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) break; case FOUT | FSTRING: case POUT | FSTRING: - if (memcmp (iter->arg[0].c, v->s, src_width) > 0 - || memcmp (iter->arg[1].c, v->s, src_width) < 0) + if (memcmp (iter->arg[0].c, + value_str (v, src_width), src_width) > 0 + || memcmp (iter->arg[1].c, + value_str (v, src_width), src_width) < 0) iter->dbl[0] += weight; iter->dbl[1] += weight; break; @@ -842,7 +880,7 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) case FIRST | FSTRING: if (iter->int1 == 0) { - memcpy (iter->string, v->s, src_width); + memcpy (iter->string, value_str (v, src_width), src_width); iter->int1 = 1; } break; @@ -851,7 +889,7 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) iter->int1 = 1; break; case LAST | FSTRING: - memcpy (iter->string, v->s, src_width); + memcpy (iter->string, value_str (v, src_width), src_width); iter->int1 = 1; break; case NMISS: @@ -883,9 +921,7 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) static void dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) { - struct ccase c; - - case_create (&c, dict_get_next_value_idx (agr->dict)); + struct ccase *c = case_create (dict_get_proto (agr->dict)); { int value_idx = 0; @@ -894,11 +930,10 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) for (i = 0; i < agr->break_var_cnt; i++) { const struct variable *v = agr->break_vars[i]; - size_t value_cnt = var_get_value_cnt (v); - memcpy (case_data_rw_idx (&c, value_idx), - case_data (&agr->break_case, v), - sizeof (union value) * value_cnt); - value_idx += value_cnt; + value_copy (case_data_rw_idx (c, value_idx), + case_data (agr->break_case, v), + var_get_width (v)); + value_idx++; } } @@ -907,16 +942,15 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) for (i = agr->agr_vars; i; i = i->next) { - union value *v = case_data_rw (&c, i->dest); + union value *v = case_data_rw (c, i->dest); + int width = var_get_width (i->dest); if (agr->missing == COLUMNWISE && i->saw_missing && (i->function & FUNC) != N && (i->function & FUNC) != NU && (i->function & FUNC) != NMISS && (i->function & FUNC) != NUMISS) { - if (var_is_alpha (i->dest)) - memset (v->s, ' ', var_get_width (i->dest)); - else - v->f = SYSMIS; + value_set_missing (v, width); + casewriter_destroy (i->writer); continue; } @@ -928,6 +962,25 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) case MEAN: v->f = i->dbl[1] != 0.0 ? i->dbl[0] / i->dbl[1] : SYSMIS; break; + case MEDIAN: + { + struct casereader *sorted_reader; + struct order_stats *median = percentile_create (0.5, i->cc); + + sorted_reader = casewriter_make_reader (i->writer); + + order_stats_accumulate (&median, 1, + sorted_reader, + i->weight, + i->subject, + i->exclude); + + v->f = percentile_calculate ((struct percentile *) median, + PC_HAVERAGE); + + statistic_destroy ((struct statistic *) median); + } + break; case SD: { double variance; @@ -948,9 +1001,9 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) case MAX | FSTRING: case MIN | FSTRING: if (i->int1) - memcpy (v->s, i->string, var_get_width (i->dest)); + memcpy (value_str_rw (v, width), i->string, width); else - memset (v->s, ' ', var_get_width (i->dest)); + value_set_missing (v, width); break; case FGT: case FGT | FSTRING: @@ -987,9 +1040,9 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) case FIRST | FSTRING: case LAST | FSTRING: if (i->int1) - memcpy (v->s, i->string, var_get_width (i->dest)); + memcpy (value_str_rw (v, width), i->string, width); else - memset (v->s, ' ', var_get_width (i->dest)); + value_set_missing (v, width); break; case N_NO_VARS: v->f = i->dbl[0]; @@ -1011,7 +1064,7 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) } } - casewriter_write (output, &c); + casewriter_write (output, c); } /* Resets the state for all the aggregate functions. */ @@ -1020,8 +1073,8 @@ initialize_aggregate_info (struct agr_proc *agr, const struct ccase *input) { struct agr_var *iter; - case_destroy (&agr->break_case); - case_clone (&agr->break_case, input); + case_unref (agr->break_case); + agr->break_case = case_ref (input); for (iter = agr->agr_vars; iter; iter = iter->next) { @@ -1042,6 +1095,29 @@ initialize_aggregate_info (struct agr_proc *agr, const struct ccase *input) case MAX | FSTRING: memset (iter->string, 0, var_get_width (iter->src)); break; + case MEDIAN: + { + struct caseproto *proto; + struct subcase ordering; + + proto = caseproto_create (); + proto = caseproto_add_width (proto, 0); + proto = caseproto_add_width (proto, 0); + + if ( ! iter->subject) + iter->subject = var_create_internal (0); + + if ( ! iter->weight) + iter->weight = var_create_internal (1); + + subcase_init_var (&ordering, iter->subject, SC_ASCEND); + iter->writer = sort_create_writer (&ordering, proto); + subcase_destroy (&ordering); + caseproto_unref (proto); + + iter->cc = 0; + } + break; case SD: if (iter->moments == NULL) iter->moments = moments1_create (MOMENT_VARIANCE);