X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Faggregate.c;h=b0df0b8e29ae702a9d0aedaf555730a4353690ce;hb=2be9bee9da6a2ce27715e58128569594319abfa2;hp=3af2cd8d749a833a55b1a437b9aede885772c1dd;hpb=b46b794dfb9f0758aafec83f50993d1930894099;p=pspp-builds.git diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c index 3af2cd8d..b0df0b8e 100644 --- a/src/language/stats/aggregate.c +++ b/src/language/stats/aggregate.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2006, 2008, 2009 Free Software Foundation, Inc. + Copyright (C) 1997-9, 2000, 2006, 2008, 2009, 2010, 2011 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,41 +16,45 @@ #include +#include "language/stats/aggregate.h" + #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "minmax.h" -#include "xalloc.h" +#include "data/any-writer.h" +#include "data/case.h" +#include "data/casegrouper.h" +#include "data/casereader.h" +#include "data/casewriter.h" +#include "data/dataset.h" +#include "data/dictionary.h" +#include "data/file-handle-def.h" +#include "data/format.h" +#include "data/settings.h" +#include "data/subcase.h" +#include "data/sys-file-writer.h" +#include "data/variable.h" +#include "language/command.h" +#include "language/data-io/file-handle.h" +#include "language/lexer/lexer.h" +#include "language/lexer/variable-parser.h" +#include "language/stats/sort-criteria.h" +#include "libpspp/assertion.h" +#include "libpspp/i18n.h" +#include "libpspp/message.h" +#include "libpspp/misc.h" +#include "libpspp/pool.h" +#include "libpspp/str.h" +#include "math/moments.h" +#include "math/percentiles.h" +#include "math/sort.h" +#include "math/statistic.h" + +#include "gl/minmax.h" +#include "gl/xalloc.h" #include "gettext.h" #define _(msgid) gettext (msgid) +#define N_(msgid) msgid /* Argument for AGGREGATE function. */ union agr_argument @@ -84,52 +88,31 @@ struct agr_var struct casewriter *writer; }; -/* Aggregation functions. */ -enum - { - NONE, SUM, MEAN, MEDIAN, SD, MAX, MIN, PGT, PLT, PIN, POUT, FGT, FLT, FIN, - FOUT, N, NU, NMISS, NUMISS, FIRST, LAST, - N_AGR_FUNCS, N_NO_VARS, NU_NO_VARS, - FUNC = 0x1f, /* Function mask. */ - FSTRING = 1<<5, /* String function bit. */ - }; - -/* Attributes of an aggregation function. */ -struct agr_func - { - const char *name; /* Aggregation function name. */ - size_t n_args; /* Number of arguments. */ - enum val_type alpha_type; /* When given ALPHA arguments, output type. */ - struct fmt_spec format; /* Format spec if alpha_type != ALPHA. */ - }; /* Attributes of aggregation functions. */ -static const struct agr_func agr_func_tab[] = +const struct agr_func agr_func_tab[] = { - {"", 0, -1, {0, 0, 0}}, - {"SUM", 0, -1, {FMT_F, 8, 2}}, - {"MEAN", 0, -1, {FMT_F, 8, 2}}, - {"MEDIAN", 0, -1, {FMT_F, 8, 2}}, - {"SD", 0, -1, {FMT_F, 8, 2}}, - {"MAX", 0, VAL_STRING, {-1, -1, -1}}, - {"MIN", 0, VAL_STRING, {-1, -1, -1}}, - {"PGT", 1, VAL_NUMERIC, {FMT_F, 5, 1}}, - {"PLT", 1, VAL_NUMERIC, {FMT_F, 5, 1}}, - {"PIN", 2, VAL_NUMERIC, {FMT_F, 5, 1}}, - {"POUT", 2, VAL_NUMERIC, {FMT_F, 5, 1}}, - {"FGT", 1, VAL_NUMERIC, {FMT_F, 5, 3}}, - {"FLT", 1, VAL_NUMERIC, {FMT_F, 5, 3}}, - {"FIN", 2, VAL_NUMERIC, {FMT_F, 5, 3}}, - {"FOUT", 2, VAL_NUMERIC, {FMT_F, 5, 3}}, - {"N", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, - {"NU", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, - {"NMISS", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, - {"NUMISS", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, - {"FIRST", 0, VAL_STRING, {-1, -1, -1}}, - {"LAST", 0, VAL_STRING, {-1, -1, -1}}, - {NULL, 0, -1, {-1, -1, -1}}, - {"N", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, - {"NU", 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"SUM", N_("Sum of values"), AGR_SV_YES, 0, -1, {FMT_F, 8, 2}}, + {"MEAN", N_("Mean average"), AGR_SV_YES, 0, -1, {FMT_F, 8, 2}}, + {"MEDIAN", N_("Median average"), AGR_SV_YES, 0, -1, {FMT_F, 8, 2}}, + {"SD", N_("Standard deviation"), AGR_SV_YES, 0, -1, {FMT_F, 8, 2}}, + {"MAX", N_("Maximum value"), AGR_SV_YES, 0, VAL_STRING, {-1, -1, -1}}, + {"MIN", N_("Minimum value"), AGR_SV_YES, 0, VAL_STRING, {-1, -1, -1}}, + {"PGT", N_("Percentage greater than"), AGR_SV_YES, 1, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"PLT", N_("Percentage less than"), AGR_SV_YES, 1, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"PIN", N_("Percentage included in range"), AGR_SV_YES, 2, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"POUT", N_("Percentage excluded from range"), AGR_SV_YES, 2, VAL_NUMERIC, {FMT_F, 5, 1}}, + {"FGT", N_("Fraction greater than"), AGR_SV_YES, 1, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FLT", N_("Fraction less than"), AGR_SV_YES, 1, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FIN", N_("Fraction included in range"), AGR_SV_YES, 2, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"FOUT", N_("Fraction excluded from range"), AGR_SV_YES, 2, VAL_NUMERIC, {FMT_F, 5, 3}}, + {"N", N_("Number of cases"), AGR_SV_NO, 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NU", N_("Number of cases (unweighted)"), AGR_SV_OPT, 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NMISS", N_("Number of missing values"), AGR_SV_YES, 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"NUMISS", N_("Number of missing values (unweighted)"), AGR_SV_YES, 0, VAL_NUMERIC, {FMT_F, 7, 0}}, + {"FIRST", N_("First non-missing value"), AGR_SV_YES, 0, VAL_STRING, {-1, -1, -1}}, + {"LAST", N_("Last non-missing value"), AGR_SV_YES, 0, VAL_STRING, {-1, -1, -1}}, + {NULL, NULL, AGR_SV_NO, 0, -1, {-1, -1, -1}}, }; /* Missing value types. */ @@ -146,17 +129,18 @@ struct agr_proc struct subcase sort; /* Sort criteria (break variables). */ const struct variable **break_vars; /* Break variables. */ size_t break_var_cnt; /* Number of break variables. */ - struct ccase *break_case; /* Last values of break variables. */ enum missing_treatment missing; /* How to treat missing values. */ struct agr_var *agr_vars; /* First aggregate variable. */ struct dictionary *dict; /* Aggregate dictionary. */ const struct dictionary *src_dict; /* Dict of the source */ int case_cnt; /* Counts aggregated cases. */ + + bool add_variables; /* True iff the aggregated variables should + be appended to the existing dictionary */ }; -static void initialize_aggregate_info (struct agr_proc *, - const struct ccase *); +static void initialize_aggregate_info (struct agr_proc *); static void accumulate_aggregate_info (struct agr_proc *, const struct ccase *); @@ -164,8 +148,9 @@ static void accumulate_aggregate_info (struct agr_proc *, static bool parse_aggregate_functions (struct lexer *, const struct dictionary *, struct agr_proc *); static void agr_destroy (struct agr_proc *); -static void dump_aggregate_info (struct agr_proc *agr, - struct casewriter *output); +static void dump_aggregate_info (const struct agr_proc *agr, + struct casewriter *output, + const struct ccase *break_case); /* Parsing. */ @@ -187,36 +172,58 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) memset(&agr, 0 , sizeof (agr)); agr.missing = ITEMWISE; - agr.break_case = NULL; - - agr.dict = dict_create (); agr.src_dict = dict; subcase_init_empty (&agr.sort); - dict_set_label (agr.dict, dict_get_label (dict)); - dict_set_documents (agr.dict, dict_get_documents (dict)); /* OUTFILE subcommand must be first. */ + lex_match (lexer, T_SLASH); if (!lex_force_match_id (lexer, "OUTFILE")) goto error; - lex_match (lexer, '='); - if (!lex_match (lexer, '*')) + lex_match (lexer, T_EQUALS); + if (!lex_match (lexer, T_ASTERISK)) { out_file = fh_parse (lexer, FH_REF_FILE | FH_REF_SCRATCH); if (out_file == NULL) goto error; } + if (out_file == NULL && lex_match_id (lexer, "MODE")) + { + lex_match (lexer, T_EQUALS); + if (lex_match_id (lexer, "ADDVARIABLES")) + { + agr.add_variables = true; + + /* presorted is assumed in ADDVARIABLES mode */ + presorted = true; + } + else if (lex_match_id (lexer, "REPLACE")) + { + agr.add_variables = false; + } + else + goto error; + } + + if ( agr.add_variables ) + agr.dict = dict_clone (dict); + else + agr.dict = dict_create (); + + dict_set_label (agr.dict, dict_get_label (dict)); + dict_set_documents (agr.dict, dict_get_documents (dict)); + /* Read most of the subcommands. */ for (;;) { - lex_match (lexer, '/'); + lex_match (lexer, T_SLASH); if (lex_match_id (lexer, "MISSING")) { - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!lex_match_id (lexer, "COLUMNWISE")) { - lex_error (lexer, _("while expecting COLUMNWISE")); + lex_error (lexer, _("expecting %s"), "COLUMNWISE"); goto error; } agr.missing = COLUMNWISE; @@ -225,28 +232,26 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) copy_documents = true; else if (lex_match_id (lexer, "PRESORTED")) presorted = true; - else if (lex_match_id (lexer, "BREAK")) + else if (lex_force_match_id (lexer, "BREAK")) { int i; - lex_match (lexer, '='); + lex_match (lexer, T_EQUALS); if (!parse_sort_criteria (lexer, dict, &agr.sort, &agr.break_vars, &saw_direction)) goto error; agr.break_var_cnt = subcase_get_n_fields (&agr.sort); - for (i = 0; i < agr.break_var_cnt; i++) - dict_clone_var_assert (agr.dict, agr.break_vars[i], - var_get_name (agr.break_vars[i])); + if (! agr.add_variables) + for (i = 0; i < agr.break_var_cnt; i++) + dict_clone_var_assert (agr.dict, agr.break_vars[i]); /* BREAK must follow the options. */ break; } else - { - lex_error (lexer, _("expecting BREAK")); - goto error; - } + goto error; + } if (presorted && saw_direction) msg (SW, _("When PRESORTED is specified, specifying sorting directions " @@ -254,7 +259,7 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) "the same way as the input data.")); /* Read in the aggregate functions. */ - lex_match (lexer, '/'); + lex_match (lexer, T_SLASH); if (!parse_aggregate_functions (lexer, dict, &agr)) goto error; @@ -295,18 +300,40 @@ cmd_aggregate (struct lexer *lexer, struct dataset *ds) casegrouper_get_next_group (grouper, &group); casereader_destroy (group)) { + struct casereader *placeholder = NULL; struct ccase *c = casereader_peek (group, 0); + if (c == NULL) { casereader_destroy (group); continue; } - initialize_aggregate_info (&agr, c); - case_unref (c); - for (; (c = casereader_read (group)) != NULL; case_unref (c)) - accumulate_aggregate_info (&agr, c); - dump_aggregate_info (&agr, output); + initialize_aggregate_info (&agr); + + if ( agr.add_variables ) + placeholder = casereader_clone (group); + + { + struct ccase *cg; + for (; (cg = casereader_read (group)) != NULL; case_unref (cg)) + accumulate_aggregate_info (&agr, cg); + } + + + if (agr.add_variables) + { + struct ccase *cg; + for (; (cg = casereader_read (placeholder)) != NULL; case_unref (cg)) + dump_aggregate_info (&agr, output, cg); + + casereader_destroy (placeholder); + } + else + { + dump_aggregate_info (&agr, output, c); + case_unref (c); + } } if (!casegrouper_destroy (grouper)) goto error; @@ -386,12 +413,13 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, ds_init_empty (&function_name); /* Parse the list of target variables. */ - while (!lex_match (lexer, '=')) + while (!lex_match (lexer, T_EQUALS)) { size_t n_dest_prev = n_dest; - if (!parse_DATA_LIST_vars (lexer, &dest, &n_dest, - PV_APPEND | PV_SINGLE | PV_NO_SCRATCH)) + if (!parse_DATA_LIST_vars (lexer, dict, &dest, &n_dest, + (PV_APPEND | PV_SINGLE | PV_NO_SCRATCH + | PV_NO_DUPLICATE))) goto error; /* Assign empty labels. */ @@ -405,15 +433,10 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, - if (lex_token (lexer) == T_STRING) + if (lex_is_string (lexer)) { - struct string label; - ds_init_string (&label, lex_tokstr (lexer)); - - ds_truncate (&label, 255); - dest_label[n_dest - 1] = ds_xstrdup (&label); + dest_label[n_dest - 1] = xstrdup (lex_tokcstr (lexer)); lex_get (lexer); - ds_destroy (&label); } } @@ -424,14 +447,8 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, goto error; } - exclude = MV_ANY; - - ds_assign_string (&function_name, lex_tokstr (lexer)); - - ds_chomp (&function_name, '.'); - - if (lex_tokid(lexer)[strlen (lex_tokid (lexer)) - 1] == '.') - exclude = MV_SYSTEM; + ds_assign_substring (&function_name, lex_tokss (lexer)); + exclude = ds_chomp_byte (&function_name, '.') ? MV_SYSTEM : MV_ANY; for (function = agr_func_tab; function->name; function++) if (!strcasecmp (function->name, ds_cstr (&function_name))) @@ -447,15 +464,11 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, lex_get (lexer); /* Check for leading lparen. */ - if (!lex_match (lexer, '(')) + if (!lex_match (lexer, T_LPAREN)) { - if (func_index == N) - func_index = N_NO_VARS; - else if (func_index == NU) - func_index = NU_NO_VARS; - else + if (function->src_vars == AGR_SV_YES) { - lex_error (lexer, _("expecting `('")); + lex_force_match (lexer, T_LPAREN); goto error; } } @@ -481,10 +494,12 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, { int type; - lex_match (lexer, ','); - if (lex_token (lexer) == T_STRING) + lex_match (lexer, T_COMMA); + if (lex_is_string (lexer)) { - arg[i].c = ds_xstrdup (lex_tokstr (lexer)); + arg[i].c = recode_string (dict_get_encoding (agr->dict), + "UTF-8", lex_tokcstr (lexer), + -1); type = VAL_STRING; } else if (lex_is_number (lexer)) @@ -511,11 +526,8 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, } /* Trailing rparen. */ - if (!lex_match (lexer, ')')) - { - lex_error (lexer, _("expecting `)'")); - goto error; - } + if (!lex_force_match (lexer, T_RPAREN)) + goto error; /* Now check that the number of source variables match the number of target variables. If we check earlier @@ -581,7 +593,7 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, } if (function->alpha_type == VAL_STRING) - destvar = dict_clone_var (agr->dict, v->src, dest[i]); + destvar = dict_clone_var_as (agr->dict, v->src, dest[i]); else { assert (var_is_numeric (v->src) @@ -602,12 +614,16 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, struct fmt_spec f; v->src = NULL; destvar = dict_create_var (agr->dict, dest[i], 0); - if (func_index == N_NO_VARS && dict_get_weight (dict) != NULL) - f = fmt_for_output (FMT_F, 8, 2); - else - f = function->format; - var_set_both_formats (destvar, &f); - } + if (destvar != NULL) + { + if ((func_index == N || func_index == NMISS) + && dict_get_weight (dict) != NULL) + f = fmt_for_output (FMT_F, 8, 2); + else + f = function->format; + var_set_both_formats (destvar, &f); + } + } if (!destvar) { @@ -621,7 +637,8 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, free (dest[i]); if (dest_label[i]) - var_set_label (destvar, dest_label[i]); + var_set_label (destvar, dest_label[i], + dict_get_encoding (agr->dict), true); v->dest = destvar; } @@ -652,9 +669,9 @@ parse_aggregate_functions (struct lexer *lexer, const struct dictionary *dict, free (dest); free (dest_label); - if (!lex_match (lexer, '/')) + if (!lex_match (lexer, T_SLASH)) { - if (lex_token (lexer) == '.') + if (lex_token (lexer) == T_ENDCMD) return true; lex_error (lexer, "expecting end of command"); @@ -693,7 +710,6 @@ agr_destroy (struct agr_proc *agr) subcase_destroy (&agr->sort); free (agr->break_vars); - case_unref (agr->break_case); for (iter = agr->agr_vars; iter; iter = next) { next = iter->next; @@ -711,8 +727,8 @@ agr_destroy (struct agr_proc *agr) else if (iter->function == SD) moments1_destroy (iter->moments); - var_destroy (iter->subject); - var_destroy (iter->weight); + dict_destroy_internal_var (iter->subject); + dict_destroy_internal_var (iter->weight); free (iter); } @@ -793,6 +809,7 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) iter->int1 = 1; break; case MAX | FSTRING: + /* Need to do some kind of Unicode collation thingy here */ if (memcmp (iter->string, value_str (v, src_width), src_width) < 0) memcpy (iter->string, value_str (v, src_width), src_width); iter->int1 = 1; @@ -902,13 +919,13 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) default: NOT_REACHED (); } - } else { + } else { switch (iter->function) { - case N_NO_VARS: + case N: iter->dbl[0] += weight; break; - case NU_NO_VARS: + case NU: iter->int1++; break; default: @@ -919,23 +936,28 @@ accumulate_aggregate_info (struct agr_proc *agr, const struct ccase *input) /* Writes an aggregated record to OUTPUT. */ static void -dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) +dump_aggregate_info (const struct agr_proc *agr, struct casewriter *output, const struct ccase *break_case) { struct ccase *c = case_create (dict_get_proto (agr->dict)); - { - int value_idx = 0; - int i; + if ( agr->add_variables) + { + case_copy (c, 0, break_case, 0, dict_get_var_cnt (agr->src_dict)); + } + else + { + int value_idx = 0; + int i; - for (i = 0; i < agr->break_var_cnt; i++) - { - const struct variable *v = agr->break_vars[i]; - value_copy (case_data_rw_idx (c, value_idx), - case_data (agr->break_case, v), - var_get_width (v)); - value_idx++; - } - } + for (i = 0; i < agr->break_var_cnt; i++) + { + const struct variable *v = agr->break_vars[i]; + value_copy (case_data_rw_idx (c, value_idx), + case_data (break_case, v), + var_get_width (v)); + value_idx++; + } + } { struct agr_var *i; @@ -964,21 +986,22 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) break; case MEDIAN: { - struct casereader *sorted_reader; - struct percentile *median = percentile_create (0.5, i->cc); - struct order_stats *os = &median->parent; - - sorted_reader = casewriter_make_reader (i->writer); - - order_stats_accumulate (&os, 1, - sorted_reader, - i->weight, - i->subject, - i->exclude); - - v->f = percentile_calculate (median, PC_HAVERAGE); - - statistic_destroy (&median->parent.parent); + if ( i->writer) + { + struct percentile *median = percentile_create (0.5, i->cc); + struct order_stats *os = &median->parent; + struct casereader *sorted_reader = casewriter_make_reader (i->writer); + i->writer = NULL; + + order_stats_accumulate (&os, 1, + sorted_reader, + i->weight, + i->subject, + i->exclude); + i->dbl[0] = percentile_calculate (median, PC_HAVERAGE); + statistic_destroy (&median->parent.parent); + } + v->f = i->dbl[0]; } break; case SD: @@ -1027,7 +1050,7 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) break; case N: case N | FSTRING: - v->f = i->dbl[0]; + v->f = i->dbl[0]; break; case NU: case NU | FSTRING: @@ -1044,12 +1067,6 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) else value_set_missing (v, width); break; - case N_NO_VARS: - v->f = i->dbl[0]; - break; - case NU_NO_VARS: - v->f = i->int1; - break; case NMISS: case NMISS | FSTRING: v->f = i->dbl[0]; @@ -1069,13 +1086,10 @@ dump_aggregate_info (struct agr_proc *agr, struct casewriter *output) /* Resets the state for all the aggregate functions. */ static void -initialize_aggregate_info (struct agr_proc *agr, const struct ccase *input) +initialize_aggregate_info (struct agr_proc *agr) { struct agr_var *iter; - case_unref (agr->break_case); - agr->break_case = case_ref (input); - for (iter = agr->agr_vars; iter; iter = iter->next) { iter->saw_missing = false; @@ -1105,10 +1119,10 @@ initialize_aggregate_info (struct agr_proc *agr, const struct ccase *input) proto = caseproto_add_width (proto, 0); if ( ! iter->subject) - iter->subject = var_create_internal (0); + iter->subject = dict_create_internal_var (0, 0); if ( ! iter->weight) - iter->weight = var_create_internal (1); + iter->weight = dict_create_internal_var (1, 0); subcase_init_var (&ordering, iter->subject, SC_ASCEND); iter->writer = sort_create_writer (&ordering, proto);