1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2005, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include <gsl/gsl_cdf.h>
23 #include "data/case.h"
24 #include "data/casegrouper.h"
25 #include "data/casereader.h"
26 #include "data/casewriter.h"
27 #include "data/dictionary.h"
28 #include "data/format.h"
29 #include "data/missing-values.h"
30 #include "data/procedure.h"
31 #include "data/short-names.h"
32 #include "data/subcase.h"
33 #include "data/variable.h"
34 #include "language/command.h"
35 #include "language/stats/sort-criteria.h"
36 #include "libpspp/compiler.h"
37 #include "libpspp/taint.h"
38 #include "math/sort.h"
39 #include "output/tab.h"
42 #define _(msgid) gettext (msgid)
58 +fraction=fraction:!blom/tukey/vw/rankit;
59 +ties=ties:!mean/low/high/condense;
60 missing=miss:!exclude/include.
65 typedef double (*rank_function_t) (double c, double cc, double cc_1,
68 static double rank_proportion (double c, double cc, double cc_1,
71 static double rank_normal (double c, double cc, double cc_1,
74 static double rank_percent (double c, double cc, double cc_1,
77 static double rank_rfraction (double c, double cc, double cc_1,
80 static double rank_rank (double c, double cc, double cc_1,
83 static double rank_n (double c, double cc, double cc_1,
86 static double rank_savage (double c, double cc, double cc_1,
89 static double rank_ntiles (double c, double cc, double cc_1,
106 static const struct fmt_spec dest_format[n_RANK_FUNCS] = {
107 {FMT_F, 9, 3}, /* rank */
108 {FMT_F, 6, 4}, /* normal */
109 {FMT_F, 6, 2}, /* percent */
110 {FMT_F, 6, 4}, /* rfraction */
111 {FMT_F, 6, 4}, /* proportion */
112 {FMT_F, 6, 0}, /* n */
113 {FMT_F, 3, 0}, /* ntiles */
114 {FMT_F, 8, 4} /* savage */
117 static const char * const function_name[n_RANK_FUNCS] = {
128 static const rank_function_t rank_func[n_RANK_FUNCS] = {
142 enum RANK_FUNC rfunc;
143 struct variable **destvars;
147 /* Categories of missing values to exclude. */
148 static enum mv_class exclude_values;
150 static struct rank_spec *rank_specs;
151 static size_t n_rank_specs;
153 static struct subcase sc;
155 static const struct variable **group_vars;
156 static size_t n_group_vars;
158 static const struct variable **src_vars;
159 static size_t n_src_vars;
164 static struct cmd_rank cmd;
166 static void rank_sorted_file (struct casereader *,
168 const struct dictionary *,
169 const struct rank_spec *rs,
172 const struct variable *rank_var);
177 static char name[10];
178 switch ( cmd.fraction )
181 strcpy (name, "BLOM");
184 strcpy (name, "RANKIT");
187 strcpy (name, "TUKEY");
198 /* Create a label on DEST_VAR, describing its derivation from SRC_VAR and F */
200 create_var_label (struct variable *dest_var,
201 const struct variable *src_var, enum RANK_FUNC f,
202 const char *dict_encoding)
205 ds_init_empty (&label);
207 if ( n_group_vars > 0 )
209 struct string group_var_str;
212 ds_init_empty (&group_var_str);
214 for (g = 0 ; g < n_group_vars ; ++g )
216 if ( g > 0 ) ds_put_cstr (&group_var_str, " ");
217 ds_put_cstr (&group_var_str, var_get_name (group_vars[g]));
220 ds_put_format (&label, _("%s of %s by %s"), function_name[f],
221 var_get_name (src_var), ds_cstr (&group_var_str));
222 ds_destroy (&group_var_str);
225 ds_put_format (&label, _("%s of %s"),
226 function_name[f], var_get_name (src_var));
228 var_set_label (dest_var, ds_cstr (&label), dict_encoding, false);
235 rank_cmd (struct dataset *ds, const struct subcase *sc,
236 const struct rank_spec *rank_specs, int n_rank_specs)
238 struct dictionary *d = dataset_dict (ds);
242 for (i = 0 ; i < subcase_get_n_fields (sc) ; ++i )
244 /* Rank variable at index I in SC. */
245 struct casegrouper *split_grouper;
246 struct casereader *split_group;
247 struct casewriter *output;
249 proc_discard_output (ds);
250 split_grouper = casegrouper_create_splits (proc_open (ds), d);
251 output = autopaging_writer_create (dict_get_proto (d));
253 while (casegrouper_get_next_group (split_grouper, &split_group))
255 struct subcase ordering;
256 struct casereader *ordered;
257 struct casegrouper *by_grouper;
258 struct casereader *by_group;
260 /* Sort this split group by the BY variables as primary
261 keys and the rank variable as secondary key. */
262 subcase_init_vars (&ordering, group_vars, n_group_vars);
263 subcase_add_var (&ordering, src_vars[i],
264 subcase_get_direction (sc, i));
265 ordered = sort_execute (split_group, &ordering);
266 subcase_destroy (&ordering);
268 /* Rank the rank variable within this split group. */
269 by_grouper = casegrouper_create_vars (ordered,
270 group_vars, n_group_vars);
271 while (casegrouper_get_next_group (by_grouper, &by_group))
273 /* Rank the rank variable within this BY group
274 within the split group. */
276 rank_sorted_file (by_group, output, d, rank_specs, n_rank_specs,
279 ok = casegrouper_destroy (by_grouper) && ok;
281 ok = casegrouper_destroy (split_grouper);
282 ok = proc_commit (ds) && ok;
283 ok = (proc_set_active_file_data (ds, casewriter_make_reader (output))
292 /* Hardly a rank function !! */
294 rank_n (double c UNUSED, double cc UNUSED, double cc_1 UNUSED,
295 int i UNUSED, double w)
302 rank_rank (double c, double cc, double cc_1,
303 int i, double w UNUSED)
318 rank = cc_1 + (c + 1.0)/ 2.0;
338 rank = cc_1 + c / 2.0 ;
353 rank_rfraction (double c, double cc, double cc_1,
356 return rank_rank (c, cc, cc_1, i, w) / w ;
361 rank_percent (double c, double cc, double cc_1,
364 return rank_rank (c, cc, cc_1, i, w) * 100.0 / w ;
369 rank_proportion (double c, double cc, double cc_1,
372 const double r = rank_rank (c, cc, cc_1, i, w) ;
376 switch ( cmd.fraction )
379 f = (r - 3.0/8.0) / (w + 0.25);
385 f = (r - 1.0/3.0) / (w + 1.0/3.0);
395 return (f > 0) ? f : SYSMIS;
399 rank_normal (double c, double cc, double cc_1,
402 double f = rank_proportion (c, cc, cc_1, i, w);
404 return gsl_cdf_ugaussian_Pinv (f);
408 rank_ntiles (double c, double cc, double cc_1,
411 double r = rank_rank (c, cc, cc_1, i, w);
414 return ( floor (( r * k_ntiles) / ( w + 1) ) + 1);
417 /* Expected value of the order statistics from an exponential distribution */
419 ee (int j, double w_star)
424 for (k = 1 ; k <= j; k++)
425 sum += 1.0 / ( w_star + 1 - k );
432 rank_savage (double c, double cc, double cc_1,
433 int i UNUSED, double w)
436 const int i_1 = floor (cc_1);
437 const int i_2 = floor (cc);
439 const double w_star = (modf (w, &int_part) == 0 ) ? w : floor (w) + 1;
441 const double g_1 = cc_1 - i_1;
442 const double g_2 = cc - i_2;
444 /* The second factor is infinite, when the first is zero.
445 Therefore, evaluate the second, only when the first is non-zero */
446 const double expr1 = (1 - g_1) ? (1 - g_1) * ee(i_1+1, w_star) : ( 1 - g_1);
447 const double expr2 = g_2 ? g_2 * ee (i_2+1, w_star) : g_2 ;
450 return ee (i_1 + 1, w_star) - 1;
452 if ( i_1 + 1 == i_2 )
453 return ( ( expr1 + expr2 )/c ) - 1;
455 if ( i_1 + 2 <= i_2 )
459 for (j = i_1 + 2 ; j <= i_2; ++j )
460 sigma += ee (j, w_star);
461 return ( (expr1 + expr2 + sigma) / c) -1;
468 rank_sorted_file (struct casereader *input,
469 struct casewriter *output,
470 const struct dictionary *dict,
471 const struct rank_spec *rs,
474 const struct variable *rank_var)
476 struct casereader *pass1, *pass2, *pass2_1;
477 struct casegrouper *tie_grouper;
484 input = casereader_create_filter_missing (input, &rank_var, 1,
485 exclude_values, NULL, output);
486 input = casereader_create_filter_weight (input, dict, NULL, output);
488 casereader_split (input, &pass1, &pass2);
490 /* Pass 1: Get total group weight. */
491 for (; (c = casereader_read (pass1)) != NULL; case_unref (c))
492 w += dict_get_case_weight (dict, c, NULL);
493 casereader_destroy (pass1);
495 /* Pass 2: Do ranking. */
496 tie_grouper = casegrouper_create_vars (pass2, &rank_var, 1);
497 while (casegrouper_get_next_group (tie_grouper, &pass2_1))
499 struct casereader *pass2_2;
504 pass2_2 = casereader_clone (pass2_1);
505 taint_propagate (casereader_get_taint (pass2_2),
506 casewriter_get_taint (output));
508 /* Pass 2.1: Sum up weight for tied cases. */
509 for (; (c = casereader_read (pass2_1)) != NULL; case_unref (c))
510 tw += dict_get_case_weight (dict, c, NULL);
512 casereader_destroy (pass2_1);
514 /* Pass 2.2: Rank tied cases. */
515 while ((c = casereader_read (pass2_2)) != NULL)
517 c = case_unshare (c);
518 for (i = 0; i < n_rank_specs; ++i)
520 const struct variable *dst_var = rs[i].destvars[dest_idx];
521 double *dst_value = &case_data_rw (c, dst_var)->f;
522 *dst_value = rank_func[rs[i].rfunc] (tw, cc, cc_1, tie_group, w);
524 casewriter_write (output, c);
526 casereader_destroy (pass2_2);
530 casegrouper_destroy (tie_grouper);
533 /* Transformation function to enumerate all the cases */
535 create_resort_key (void *key_var_, struct ccase **cc, casenumber case_num)
537 struct variable *key_var = key_var_;
539 *cc = case_unshare (*cc);
540 case_data_rw (*cc, key_var)->f = case_num;
542 return TRNS_CONTINUE;
546 /* Create and return a new variable in which to store the ranks of SRC_VAR
547 accoring to the rank function F.
548 VNAME is the name of the variable to be created.
549 If VNAME is NULL, then a name will be automatically chosen.
551 static struct variable *
552 create_rank_variable (struct dictionary *dict, enum RANK_FUNC f,
553 const struct variable *src_var,
557 struct variable *var = NULL;
558 char name[SHORT_NAME_LEN + 1];
561 var = dict_create_var(dict, vname, 0);
565 snprintf (name, SHORT_NAME_LEN + 1, "%c%s",
566 function_name[f][0], var_get_name (src_var));
568 var = dict_create_var(dict, name, 0);
575 snprintf(func_abb, 4, "%s", function_name[f]);
576 snprintf(name, SHORT_NAME_LEN + 1, "%s%03d", func_abb,
579 var = dict_create_var(dict, name, 0);
585 while ( NULL == var )
588 snprintf(func_abb, 3, "%s", function_name[f]);
590 snprintf(name, SHORT_NAME_LEN + 1,
591 "RNK%s%02d", func_abb, i);
593 var = dict_create_var(dict, name, 0);
600 msg(ME, _("Cannot create new rank variable. All candidates in use."));
604 var_set_both_formats (var, &dest_format[f]);
619 for (i = 0 ; i < n_rank_specs ; ++i )
620 free (rank_specs[i].destvars);
626 subcase_destroy (&sc);
634 cmd_rank (struct lexer *lexer, struct dataset *ds)
637 struct variable *order;
641 subcase_init_empty (&sc);
642 if ( !parse_rank (lexer, ds, &cmd, NULL) )
648 /* If /MISSING=INCLUDE is set, then user missing values are ignored */
649 exclude_values = cmd.miss == RANK_INCLUDE ? MV_SYSTEM : MV_ANY;
651 /* Default to /RANK if no function subcommands are given */
652 if ( !( cmd.sbc_normal || cmd.sbc_ntiles || cmd.sbc_proportion ||
653 cmd.sbc_rfraction || cmd.sbc_savage || cmd.sbc_n ||
654 cmd.sbc_percent || cmd.sbc_rank ) )
656 assert ( n_rank_specs == 0 );
658 rank_specs = xmalloc (sizeof (*rank_specs));
659 rank_specs[0].rfunc = RANK;
660 rank_specs[0].destvars =
661 xcalloc (subcase_get_n_fields (&sc), sizeof (struct variable *));
666 assert ( subcase_get_n_fields (&sc) == n_src_vars);
668 /* Create variables for all rank destinations which haven't
669 already been created with INTO.
670 Add labels to all the destination variables.
672 for (i = 0 ; i < n_rank_specs ; ++i )
675 for ( v = 0 ; v < n_src_vars ; v ++ )
677 struct dictionary *dict = dataset_dict (ds);
679 if ( rank_specs[i].destvars[v] == NULL )
681 rank_specs[i].destvars[v] =
682 create_rank_variable (dict, rank_specs[i].rfunc, src_vars[v], NULL);
685 create_var_label ( rank_specs[i].destvars[v],
688 dict_get_encoding (dict));
692 if ( cmd.print == RANK_YES )
696 tab_output_text (0, _("Variables Created By RANK"));
697 tab_output_text (0, "");
699 for (i = 0 ; i < n_rank_specs ; ++i )
701 for ( v = 0 ; v < n_src_vars ; v ++ )
703 if ( n_group_vars > 0 )
705 struct string varlist;
708 ds_init_empty (&varlist);
709 for ( g = 0 ; g < n_group_vars ; ++g )
711 ds_put_cstr (&varlist, var_get_name (group_vars[g]));
713 if ( g < n_group_vars - 1)
714 ds_put_cstr (&varlist, " ");
717 if ( rank_specs[i].rfunc == NORMAL ||
718 rank_specs[i].rfunc == PROPORTION )
719 tab_output_text_format (0,
720 _("%s into %s(%s of %s using %s BY %s)"),
721 var_get_name (src_vars[v]),
722 var_get_name (rank_specs[i].destvars[v]),
723 function_name[rank_specs[i].rfunc],
724 var_get_name (src_vars[v]),
729 tab_output_text_format (0,
730 _("%s into %s(%s of %s BY %s)"),
731 var_get_name (src_vars[v]),
732 var_get_name (rank_specs[i].destvars[v]),
733 function_name[rank_specs[i].rfunc],
734 var_get_name (src_vars[v]),
736 ds_destroy (&varlist);
740 if ( rank_specs[i].rfunc == NORMAL ||
741 rank_specs[i].rfunc == PROPORTION )
742 tab_output_text_format (0,
743 _("%s into %s(%s of %s using %s)"),
744 var_get_name (src_vars[v]),
745 var_get_name (rank_specs[i].destvars[v]),
746 function_name[rank_specs[i].rfunc],
747 var_get_name (src_vars[v]),
751 tab_output_text_format (0,
752 _("%s into %s(%s of %s)"),
753 var_get_name (src_vars[v]),
754 var_get_name (rank_specs[i].destvars[v]),
755 function_name[rank_specs[i].rfunc],
756 var_get_name (src_vars[v]));
762 if ( cmd.sbc_fraction &&
763 ( ! cmd.sbc_normal && ! cmd.sbc_proportion) )
764 msg(MW, _("FRACTION has been specified, but NORMAL and PROPORTION rank functions have not been requested. The FRACTION subcommand will be ignored.") );
766 /* Add a variable which we can sort by to get back the original
768 order = dict_create_var_assert (dataset_dict (ds), "$ORDER_", 0);
770 add_transformation (ds, create_resort_key, 0, order);
773 result = rank_cmd (ds, &sc, rank_specs, n_rank_specs);
775 /* Put the active file back in its original order. Delete
776 our sort key, which we don't need anymore. */
778 struct casereader *sorted;
780 /* FIXME: loses error conditions. */
782 proc_discard_output (ds);
783 sorted = sort_execute_1var (proc_open (ds), order);
784 result = proc_commit (ds) && result;
786 dict_delete_var (dataset_dict (ds), order);
787 result = proc_set_active_file_data (ds, sorted) && result;
793 return (result ? CMD_SUCCESS : CMD_CASCADING_FAILURE);
797 /* Parser for the variables sub command
798 Returns 1 on success */
800 rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd UNUSED, void *aux UNUSED)
802 lex_match (lexer, T_EQUALS);
804 if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
805 && lex_token (lexer) != T_ALL)
808 if (!parse_sort_criteria (lexer, dataset_dict (ds), &sc, &src_vars, NULL))
810 n_src_vars = subcase_get_n_fields (&sc);
812 if ( lex_match (lexer, T_BY) )
814 if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
819 if (!parse_variables_const (lexer, dataset_dict (ds),
820 &group_vars, &n_group_vars,
821 PV_NO_DUPLICATE | PV_NO_SCRATCH) )
832 /* Parse the [/rank INTO var1 var2 ... varN ] clause */
834 parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum RANK_FUNC f)
839 rank_specs = xnrealloc(rank_specs, n_rank_specs, sizeof *rank_specs);
840 rank_specs[n_rank_specs - 1].rfunc = f;
841 rank_specs[n_rank_specs - 1].destvars = NULL;
843 rank_specs[n_rank_specs - 1].destvars =
844 xcalloc (subcase_get_n_fields (&sc), sizeof (struct variable *));
846 if (lex_match_id (lexer, "INTO"))
848 struct variable *destvar;
850 while( lex_token (lexer) == T_ID )
853 if ( dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL )
855 msg(SE, _("Variable %s already exists."), lex_tokcstr (lexer));
858 if ( var_count >= subcase_get_n_fields (&sc) )
860 msg(SE, _("Too many variables in INTO clause."));
864 destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokcstr (lexer));
865 rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
877 rank_custom_rank (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
879 struct dictionary *dict = dataset_dict (ds);
881 return parse_rank_function (lexer, dict, cmd, RANK);
885 rank_custom_normal (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
887 struct dictionary *dict = dataset_dict (ds);
889 return parse_rank_function (lexer, dict, cmd, NORMAL);
893 rank_custom_percent (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
895 struct dictionary *dict = dataset_dict (ds);
897 return parse_rank_function (lexer, dict, cmd, PERCENT);
901 rank_custom_rfraction (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
903 struct dictionary *dict = dataset_dict (ds);
905 return parse_rank_function (lexer, dict, cmd, RFRACTION);
909 rank_custom_proportion (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
911 struct dictionary *dict = dataset_dict (ds);
913 return parse_rank_function (lexer, dict, cmd, PROPORTION);
917 rank_custom_n (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
919 struct dictionary *dict = dataset_dict (ds);
921 return parse_rank_function (lexer, dict, cmd, N);
925 rank_custom_savage (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
927 struct dictionary *dict = dataset_dict (ds);
929 return parse_rank_function (lexer, dict, cmd, SAVAGE);
934 rank_custom_ntiles (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
936 struct dictionary *dict = dataset_dict (ds);
938 if ( lex_force_match (lexer, T_LPAREN) )
940 if ( lex_force_int (lexer) )
942 k_ntiles = lex_integer (lexer);
944 lex_force_match (lexer, T_RPAREN);
952 return parse_rank_function (lexer, dict, cmd, NTILES);