1 /* PSPP - RANK. -*-c-*-
2 Copyright (C) 2005, 2006, 2007 Free Software Foundation, Inc.
4 This program is free software; you can redistribute it and/or
5 modify it under the terms of the GNU General Public License as
6 published by the Free Software Foundation; either version 2 of the
7 License, or (at your option) any later version.
9 This program is distributed in the hope that it will be useful, but
10 WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
24 #include <data/dictionary.h>
25 #include <data/format.h>
26 #include <data/missing-values.h>
27 #include <data/procedure.h>
28 #include <data/variable.h>
29 #include <data/case-ordering.h>
30 #include <data/case.h>
31 #include <data/casegrouper.h>
32 #include <data/casereader.h>
33 #include <data/casewriter.h>
34 #include <language/command.h>
35 #include <language/stats/sort-criteria.h>
36 #include <libpspp/compiler.h>
37 #include <libpspp/taint.h>
38 #include <math/sort.h>
39 #include <output/table.h>
40 #include <output/manager.h>
42 #include <gsl/gsl_cdf.h>
45 #define _(msgid) gettext (msgid)
61 +fraction=fraction:!blom/tukey/vw/rankit;
62 +ties=ties:!mean/low/high/condense;
63 missing=miss:!exclude/include.
68 typedef double (*rank_function_t) (double c, double cc, double cc_1,
71 static double rank_proportion (double c, double cc, double cc_1,
74 static double rank_normal (double c, double cc, double cc_1,
77 static double rank_percent (double c, double cc, double cc_1,
80 static double rank_rfraction (double c, double cc, double cc_1,
83 static double rank_rank (double c, double cc, double cc_1,
86 static double rank_n (double c, double cc, double cc_1,
89 static double rank_savage (double c, double cc, double cc_1,
92 static double rank_ntiles (double c, double cc, double cc_1,
109 static const struct fmt_spec dest_format[n_RANK_FUNCS] = {
110 {FMT_F, 9, 3}, /* rank */
111 {FMT_F, 6, 4}, /* normal */
112 {FMT_F, 6, 2}, /* percent */
113 {FMT_F, 6, 4}, /* rfraction */
114 {FMT_F, 6, 4}, /* proportion */
115 {FMT_F, 6, 0}, /* n */
116 {FMT_F, 3, 0}, /* ntiles */
117 {FMT_F, 8, 4} /* savage */
120 static const char * const function_name[n_RANK_FUNCS] = {
131 static const rank_function_t rank_func[n_RANK_FUNCS] = {
145 enum RANK_FUNC rfunc;
146 struct variable **destvars;
150 /* Categories of missing values to exclude. */
151 static enum mv_class exclude_values;
153 static struct rank_spec *rank_specs;
154 static size_t n_rank_specs;
156 static struct case_ordering *sc;
158 static const struct variable **group_vars;
159 static size_t n_group_vars;
161 static const struct variable **src_vars;
162 static size_t n_src_vars;
167 static struct cmd_rank cmd;
169 static void rank_sorted_file (struct casereader *,
171 const struct dictionary *,
172 const struct rank_spec *rs,
175 struct variable *rank_var);
180 static char name[10];
181 switch ( cmd.fraction )
184 strcpy (name, "BLOM");
187 strcpy (name, "RANKIT");
190 strcpy (name, "TUKEY");
201 /* Create a label on DEST_VAR, describing its derivation from SRC_VAR and F */
203 create_var_label (struct variable *dest_var,
204 const struct variable *src_var, enum RANK_FUNC f)
207 ds_init_empty (&label);
209 if ( n_group_vars > 0 )
211 struct string group_var_str;
214 ds_init_empty (&group_var_str);
216 for (g = 0 ; g < n_group_vars ; ++g )
218 if ( g > 0 ) ds_put_cstr (&group_var_str, " ");
219 ds_put_cstr (&group_var_str, var_get_name (group_vars[g]));
222 ds_put_format (&label, _("%s of %s by %s"), function_name[f],
223 var_get_name (src_var), ds_cstr (&group_var_str));
224 ds_destroy (&group_var_str);
227 ds_put_format (&label, _("%s of %s"),
228 function_name[f], var_get_name (src_var));
230 var_set_label (dest_var, ds_cstr (&label));
237 rank_cmd (struct dataset *ds, const struct case_ordering *sc,
238 const struct rank_spec *rank_specs, int n_rank_specs)
240 struct case_ordering *base_ordering;
243 const int n_splits = dict_get_split_cnt (dataset_dict (ds));
245 base_ordering = case_ordering_create (dataset_dict (ds));
246 for (i = 0; i < n_splits ; i++)
247 case_ordering_add_var (base_ordering,
248 dict_get_split_vars (dataset_dict (ds))[i],
251 for (i = 0; i < n_group_vars; i++)
252 case_ordering_add_var (base_ordering, group_vars[i], SRT_ASCEND);
253 for (i = 0 ; i < case_ordering_get_var_cnt (sc) ; ++i )
255 struct case_ordering *ordering;
256 struct casegrouper *grouper;
257 struct casereader *group;
258 struct casewriter *output;
259 struct casereader *ranked_file;
261 ordering = case_ordering_clone (base_ordering);
262 case_ordering_add_var (ordering,
263 case_ordering_get_var (sc, i),
264 case_ordering_get_direction (sc, i));
266 proc_discard_output (ds);
267 grouper = casegrouper_create_case_ordering (sort_execute (proc_open (ds),
270 output = autopaging_writer_create (dict_get_next_value_idx (
272 while (casegrouper_get_next_group (grouper, &group))
273 rank_sorted_file (group, output, dataset_dict (ds),
274 rank_specs, n_rank_specs,
276 ok = casegrouper_destroy (grouper);
277 ok = proc_commit (ds) && ok;
278 ranked_file = casewriter_make_reader (output);
279 ok = proc_set_active_file_data (ds, ranked_file) && ok;
283 case_ordering_destroy (base_ordering);
288 /* Hardly a rank function !! */
290 rank_n (double c UNUSED, double cc UNUSED, double cc_1 UNUSED,
291 int i UNUSED, double w)
298 rank_rank (double c, double cc, double cc_1,
299 int i, double w UNUSED)
314 rank = cc_1 + (c + 1.0)/ 2.0;
334 rank = cc_1 + c / 2.0 ;
349 rank_rfraction (double c, double cc, double cc_1,
352 return rank_rank (c, cc, cc_1, i, w) / w ;
357 rank_percent (double c, double cc, double cc_1,
360 return rank_rank (c, cc, cc_1, i, w) * 100.0 / w ;
365 rank_proportion (double c, double cc, double cc_1,
368 const double r = rank_rank (c, cc, cc_1, i, w) ;
372 switch ( cmd.fraction )
375 f = (r - 3.0/8.0) / (w + 0.25);
381 f = (r - 1.0/3.0) / (w + 1.0/3.0);
391 return (f > 0) ? f : SYSMIS;
395 rank_normal (double c, double cc, double cc_1,
398 double f = rank_proportion (c, cc, cc_1, i, w);
400 return gsl_cdf_ugaussian_Pinv (f);
404 rank_ntiles (double c, double cc, double cc_1,
407 double r = rank_rank (c, cc, cc_1, i, w);
410 return ( floor (( r * k_ntiles) / ( w + 1) ) + 1);
413 /* Expected value of the order statistics from an exponential distribution */
415 ee (int j, double w_star)
420 for (k = 1 ; k <= j; k++)
421 sum += 1.0 / ( w_star + 1 - k );
428 rank_savage (double c, double cc, double cc_1,
429 int i UNUSED, double w)
432 const int i_1 = floor (cc_1);
433 const int i_2 = floor (cc);
435 const double w_star = (modf (w, &int_part) == 0 ) ? w : floor (w) + 1;
437 const double g_1 = cc_1 - i_1;
438 const double g_2 = cc - i_2;
440 /* The second factor is infinite, when the first is zero.
441 Therefore, evaluate the second, only when the first is non-zero */
442 const double expr1 = (1 - g_1) ? (1 - g_1) * ee(i_1+1, w_star) : ( 1 - g_1);
443 const double expr2 = g_2 ? g_2 * ee (i_2+1, w_star) : g_2 ;
446 return ee (i_1 + 1, w_star) - 1;
448 if ( i_1 + 1 == i_2 )
449 return ( ( expr1 + expr2 )/c ) - 1;
451 if ( i_1 + 2 <= i_2 )
455 for (j = i_1 + 2 ; j <= i_2; ++j )
456 sigma += ee (j, w_star);
457 return ( (expr1 + expr2 + sigma) / c) -1;
464 rank_sorted_file (struct casereader *input,
465 struct casewriter *output,
466 const struct dictionary *dict,
467 const struct rank_spec *rs,
470 struct variable *rank_var)
472 struct casereader *pass1, *pass2, *pass2_1;
473 struct casegrouper *tie_grouper;
480 input = casereader_create_filter_missing (input, &rank_var, 1,
481 exclude_values, output);
482 input = casereader_create_filter_weight (input, dict, NULL, output);
484 casereader_split (input, &pass1, &pass2);
486 /* Pass 1: Get total group weight. */
487 for (; casereader_read (pass1, &c); case_destroy (&c))
488 w += dict_get_case_weight (dict, &c, NULL);
489 casereader_destroy (pass1);
491 /* Pass 2: Do ranking. */
492 tie_grouper = casegrouper_create_vars (pass2, &rank_var, 1);
493 while (casegrouper_get_next_group (tie_grouper, &pass2_1))
495 struct casereader *pass2_2;
500 pass2_2 = casereader_clone (pass2_1);
501 taint_propagate (casereader_get_taint (pass2_2),
502 casewriter_get_taint (output));
504 /* Pass 2.1: Sum up weight for tied cases. */
505 for (; casereader_read (pass2_1, &c); case_destroy (&c))
506 tw += dict_get_case_weight (dict, &c, NULL);
508 casereader_destroy (pass2_1);
510 /* Pass 2.2: Rank tied cases. */
511 while (casereader_read (pass2_2, &c))
513 for (i = 0; i < n_rank_specs; ++i)
515 const struct variable *dst_var = rs[i].destvars[dest_idx];
516 double *dst_value = &case_data_rw (&c, dst_var)->f;
517 *dst_value = rank_func[rs[i].rfunc] (tw, cc, cc_1, tie_group, w);
519 casewriter_write (output, &c);
521 casereader_destroy (pass2_2);
525 casegrouper_destroy (tie_grouper);
528 /* Transformation function to enumerate all the cases */
530 create_resort_key (void *key_var_, struct ccase *cc, casenumber case_num)
532 struct variable *key_var = key_var_;
534 case_data_rw(cc, key_var)->f = case_num;
536 return TRNS_CONTINUE;
540 /* Create and return a new variable in which to store the ranks of SRC_VAR
541 accoring to the rank function F.
542 VNAME is the name of the variable to be created.
543 If VNAME is NULL, then a name will be automatically chosen.
545 static struct variable *
546 create_rank_variable (struct dictionary *dict, enum RANK_FUNC f,
547 const struct variable *src_var,
551 struct variable *var = NULL;
552 char name[SHORT_NAME_LEN + 1];
555 var = dict_create_var(dict, vname, 0);
559 snprintf (name, SHORT_NAME_LEN + 1, "%c%s",
560 function_name[f][0], var_get_name (src_var));
562 var = dict_create_var(dict, name, 0);
569 snprintf(func_abb, 4, "%s", function_name[f]);
570 snprintf(name, SHORT_NAME_LEN + 1, "%s%03d", func_abb,
573 var = dict_create_var(dict, name, 0);
579 while ( NULL == var )
582 snprintf(func_abb, 3, "%s", function_name[f]);
584 snprintf(name, SHORT_NAME_LEN + 1,
585 "RNK%s%02d", func_abb, i);
587 var = dict_create_var(dict, name, 0);
594 msg(ME, _("Cannot create new rank variable. All candidates in use."));
598 var_set_both_formats (var, &dest_format[f]);
613 for (i = 0 ; i < n_rank_specs ; ++i )
614 free (rank_specs[i].destvars);
620 case_ordering_destroy (sc);
629 cmd_rank (struct lexer *lexer, struct dataset *ds)
632 struct variable *order;
636 if ( !parse_rank (lexer, ds, &cmd, NULL) )
642 /* If /MISSING=INCLUDE is set, then user missing values are ignored */
643 exclude_values = cmd.miss == RANK_INCLUDE ? MV_SYSTEM : MV_ANY;
645 /* Default to /RANK if no function subcommands are given */
646 if ( !( cmd.sbc_normal || cmd.sbc_ntiles || cmd.sbc_proportion ||
647 cmd.sbc_rfraction || cmd.sbc_savage || cmd.sbc_n ||
648 cmd.sbc_percent || cmd.sbc_rank ) )
650 assert ( n_rank_specs == 0 );
652 rank_specs = xmalloc (sizeof (*rank_specs));
653 rank_specs[0].rfunc = RANK;
654 rank_specs[0].destvars =
655 xcalloc (case_ordering_get_var_cnt (sc), sizeof (struct variable *));
660 assert ( case_ordering_get_var_cnt (sc) == n_src_vars);
662 /* Create variables for all rank destinations which haven't
663 already been created with INTO.
664 Add labels to all the destination variables.
666 for (i = 0 ; i < n_rank_specs ; ++i )
669 for ( v = 0 ; v < n_src_vars ; v ++ )
671 if ( rank_specs[i].destvars[v] == NULL )
673 rank_specs[i].destvars[v] =
674 create_rank_variable (dataset_dict(ds), rank_specs[i].rfunc, src_vars[v], NULL);
677 create_var_label ( rank_specs[i].destvars[v],
679 rank_specs[i].rfunc);
683 if ( cmd.print == RANK_YES )
687 tab_output_text (0, _("Variables Created By RANK"));
688 tab_output_text (0, "\n");
690 for (i = 0 ; i < n_rank_specs ; ++i )
692 for ( v = 0 ; v < n_src_vars ; v ++ )
694 if ( n_group_vars > 0 )
696 struct string varlist;
699 ds_init_empty (&varlist);
700 for ( g = 0 ; g < n_group_vars ; ++g )
702 ds_put_cstr (&varlist, var_get_name (group_vars[g]));
704 if ( g < n_group_vars - 1)
705 ds_put_cstr (&varlist, " ");
708 if ( rank_specs[i].rfunc == NORMAL ||
709 rank_specs[i].rfunc == PROPORTION )
710 tab_output_text (TAT_PRINTF,
711 _("%s into %s(%s of %s using %s BY %s)"),
712 var_get_name (src_vars[v]),
713 var_get_name (rank_specs[i].destvars[v]),
714 function_name[rank_specs[i].rfunc],
715 var_get_name (src_vars[v]),
721 tab_output_text (TAT_PRINTF,
722 _("%s into %s(%s of %s BY %s)"),
723 var_get_name (src_vars[v]),
724 var_get_name (rank_specs[i].destvars[v]),
725 function_name[rank_specs[i].rfunc],
726 var_get_name (src_vars[v]),
729 ds_destroy (&varlist);
733 if ( rank_specs[i].rfunc == NORMAL ||
734 rank_specs[i].rfunc == PROPORTION )
735 tab_output_text (TAT_PRINTF,
736 _("%s into %s(%s of %s using %s)"),
737 var_get_name (src_vars[v]),
738 var_get_name (rank_specs[i].destvars[v]),
739 function_name[rank_specs[i].rfunc],
740 var_get_name (src_vars[v]),
745 tab_output_text (TAT_PRINTF,
746 _("%s into %s(%s of %s)"),
747 var_get_name (src_vars[v]),
748 var_get_name (rank_specs[i].destvars[v]),
749 function_name[rank_specs[i].rfunc],
750 var_get_name (src_vars[v])
757 if ( cmd.sbc_fraction &&
758 ( ! cmd.sbc_normal && ! cmd.sbc_proportion) )
759 msg(MW, _("FRACTION has been specified, but NORMAL and PROPORTION rank functions have not been requested. The FRACTION subcommand will be ignored.") );
761 /* Add a variable which we can sort by to get back the original
763 order = dict_create_var_assert (dataset_dict (ds), "$ORDER_", 0);
765 add_transformation (ds, create_resort_key, 0, order);
768 result = rank_cmd (ds, sc, rank_specs, n_rank_specs);
770 /* Put the active file back in its original order. Delete
771 our sort key, which we don't need anymore. */
773 struct case_ordering *ordering = case_ordering_create (dataset_dict (ds));
774 struct casereader *sorted;
775 case_ordering_add_var (ordering, order, SRT_ASCEND);
776 /* FIXME: loses error conditions. */
777 proc_discard_output (ds);
778 sorted = sort_execute (proc_open (ds), ordering);
779 result = proc_commit (ds) && result;
781 dict_delete_var (dataset_dict (ds), order);
782 result = proc_set_active_file_data (ds, sorted) && result;
788 return (result ? CMD_SUCCESS : CMD_CASCADING_FAILURE);
792 /* Parser for the variables sub command
793 Returns 1 on success */
795 rank_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd UNUSED, void *aux UNUSED)
797 lex_match (lexer, '=');
799 if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
800 && lex_token (lexer) != T_ALL)
803 sc = parse_case_ordering (lexer, dataset_dict (ds), NULL);
806 case_ordering_get_vars (sc, &src_vars, &n_src_vars);
808 if ( lex_match (lexer, T_BY) )
810 if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
815 if (!parse_variables_const (lexer, dataset_dict (ds),
816 &group_vars, &n_group_vars,
817 PV_NO_DUPLICATE | PV_NO_SCRATCH) )
828 /* Parse the [/rank INTO var1 var2 ... varN ] clause */
830 parse_rank_function (struct lexer *lexer, struct dictionary *dict, struct cmd_rank *cmd UNUSED, enum RANK_FUNC f)
835 rank_specs = xnrealloc(rank_specs, n_rank_specs, sizeof *rank_specs);
836 rank_specs[n_rank_specs - 1].rfunc = f;
837 rank_specs[n_rank_specs - 1].destvars = NULL;
839 rank_specs[n_rank_specs - 1].destvars =
840 xcalloc (case_ordering_get_var_cnt (sc),
841 sizeof (struct variable *));
843 if (lex_match_id (lexer, "INTO"))
845 struct variable *destvar;
847 while( lex_token (lexer) == T_ID )
850 if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
852 msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
855 if ( var_count >= case_ordering_get_var_cnt (sc) )
857 msg(SE, _("Too many variables in INTO clause."));
861 destvar = create_rank_variable (dict, f, src_vars[var_count], lex_tokid (lexer));
862 rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
874 rank_custom_rank (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
876 struct dictionary *dict = dataset_dict (ds);
878 return parse_rank_function (lexer, dict, cmd, RANK);
882 rank_custom_normal (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
884 struct dictionary *dict = dataset_dict (ds);
886 return parse_rank_function (lexer, dict, cmd, NORMAL);
890 rank_custom_percent (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
892 struct dictionary *dict = dataset_dict (ds);
894 return parse_rank_function (lexer, dict, cmd, PERCENT);
898 rank_custom_rfraction (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
900 struct dictionary *dict = dataset_dict (ds);
902 return parse_rank_function (lexer, dict, cmd, RFRACTION);
906 rank_custom_proportion (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
908 struct dictionary *dict = dataset_dict (ds);
910 return parse_rank_function (lexer, dict, cmd, PROPORTION);
914 rank_custom_n (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
916 struct dictionary *dict = dataset_dict (ds);
918 return parse_rank_function (lexer, dict, cmd, N);
922 rank_custom_savage (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
924 struct dictionary *dict = dataset_dict (ds);
926 return parse_rank_function (lexer, dict, cmd, SAVAGE);
931 rank_custom_ntiles (struct lexer *lexer, struct dataset *ds, struct cmd_rank *cmd, void *aux UNUSED )
933 struct dictionary *dict = dataset_dict (ds);
935 if ( lex_force_match (lexer, '(') )
937 if ( lex_force_int (lexer) )
939 k_ntiles = lex_integer (lexer);
941 lex_force_match (lexer, ')');
949 return parse_rank_function (lexer, dict, cmd, NTILES);