1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2005, 2006, 2007, 2009, 2010, 2011, 2012 Free Software Foundation, Inc
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
20 #include <gsl/gsl_cdf.h>
22 #include "data/case.h"
23 #include "data/casegrouper.h"
24 #include "data/casereader.h"
25 #include "data/dataset.h"
26 #include "data/dictionary.h"
27 #include "data/format.h"
28 #include "data/variable.h"
29 #include "data/subcase.h"
30 #include "data/casewriter.h"
31 #include "data/short-names.h"
32 #include "language/command.h"
33 #include "language/lexer/lexer.h"
34 #include "language/lexer/variable-parser.h"
35 #include "language/stats/sort-criteria.h"
36 #include "math/sort.h"
37 #include "libpspp/assertion.h"
38 #include "libpspp/message.h"
39 #include "libpspp/misc.h"
40 #include "libpspp/pool.h"
41 #include "libpspp/taint.h"
43 #include "output/tab.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
51 typedef double (*rank_function_t) (const struct rank*, double c, double cc, double cc_1,
54 static double rank_proportion (const struct rank *, double c, double cc, double cc_1,
57 static double rank_normal (const struct rank *, double c, double cc, double cc_1,
60 static double rank_percent (const struct rank *, double c, double cc, double cc_1,
63 static double rank_rfraction (const struct rank *, double c, double cc, double cc_1,
66 static double rank_rank (const struct rank *, double c, double cc, double cc_1,
69 static double rank_n (const struct rank *, double c, double cc, double cc_1,
72 static double rank_savage (const struct rank *, double c, double cc, double cc_1,
75 static double rank_ntiles (const struct rank *, double c, double cc, double cc_1,
92 static const struct fmt_spec dest_format[n_RANK_FUNCS] = {
93 {FMT_F, 9, 3}, /* rank */
94 {FMT_F, 6, 4}, /* normal */
95 {FMT_F, 6, 2}, /* percent */
96 {FMT_F, 6, 4}, /* rfraction */
97 {FMT_F, 6, 4}, /* proportion */
98 {FMT_F, 6, 0}, /* n */
99 {FMT_F, 3, 0}, /* ntiles */
100 {FMT_F, 8, 4} /* savage */
103 static const char * const function_name[n_RANK_FUNCS] = {
114 static const rank_function_t rank_func[n_RANK_FUNCS] = {
144 enum rank_func rfunc;
145 struct variable **destvars;
149 /* Create and return a new variable in which to store the ranks of SRC_VAR
150 accoring to the rank function F.
151 VNAME is the name of the variable to be created.
152 If VNAME is NULL, then a name will be automatically chosen.
154 static struct variable *
155 create_rank_variable (struct dictionary *dict, enum rank_func f,
156 const struct variable *src_var,
160 struct variable *var = NULL;
161 char name[SHORT_NAME_LEN + 1];
164 var = dict_create_var(dict, vname, 0);
168 snprintf (name, SHORT_NAME_LEN + 1, "%c%s",
169 function_name[f][0], var_get_name (src_var));
171 var = dict_create_var(dict, name, 0);
177 snprintf(func_abb, 4, "%s", function_name[f]);
178 snprintf(name, SHORT_NAME_LEN + 1, "%s%03d", func_abb,
181 var = dict_create_var(dict, name, 0);
187 while ( NULL == var )
190 snprintf(func_abb, 3, "%s", function_name[f]);
192 snprintf(name, SHORT_NAME_LEN + 1,
193 "RNK%s%02d", func_abb, i);
195 var = dict_create_var(dict, name, 0);
202 msg(ME, _("Cannot create new rank variable. All candidates in use."));
206 var_set_both_formats (var, &dest_format[f]);
213 struct dictionary *dict;
217 const struct variable **vars;
220 const struct variable **group_vars;
224 enum mv_class exclude;
226 struct rank_spec *rs;
231 enum fraction fraction;
236 /* Pool on which cell functions may allocate data */
242 destroy_rank (struct rank *rank)
245 free (rank->group_vars);
246 subcase_destroy (&rank->sc);
247 pool_destroy (rank->pool);
251 parse_into (struct lexer *lexer, struct rank *cmd)
254 struct rank_spec *rs = NULL;
256 cmd->rs = pool_realloc (cmd->pool, cmd->rs, sizeof (*cmd->rs) * (cmd->n_rs + 1));
257 rs = &cmd->rs[cmd->n_rs];
259 if (lex_match_id (lexer, "RANK"))
263 else if (lex_match_id (lexer, "NORMAL"))
267 else if (lex_match_id (lexer, "RFRACTION"))
269 rs->rfunc = RFRACTION;
271 else if (lex_match_id (lexer, "N"))
275 else if (lex_match_id (lexer, "SAVAGE"))
279 else if (lex_match_id (lexer, "PERCENT"))
283 else if (lex_match_id (lexer, "PROPORTION"))
285 rs->rfunc = PROPORTION;
287 else if (lex_match_id (lexer, "NTILES"))
289 if ( !lex_force_match (lexer, T_LPAREN))
292 if (! lex_force_int (lexer) )
295 cmd->k_ntiles = lex_integer (lexer);
298 if ( !lex_force_match (lexer, T_RPAREN))
310 rs->destvars = pool_calloc (cmd->pool, cmd->n_vars, sizeof (*rs->destvars));
312 if (lex_match_id (lexer, "INTO"))
314 while( lex_token (lexer) == T_ID )
316 const char *name = lex_tokcstr (lexer);
317 if ( dict_lookup_var (cmd->dict, name) != NULL )
319 msg (SE, _("Variable %s already exists."), name);
323 if ( var_count >= subcase_get_n_fields (&cmd->sc) )
325 msg (SE, _("Too many variables in INTO clause."));
328 rs->destvars[var_count] =
329 create_rank_variable (cmd->dict, rs->rfunc, cmd->vars[var_count], name);
338 /* Hardly a rank function !! */
340 rank_n (const struct rank *cmd UNUSED, double c UNUSED, double cc UNUSED, double cc_1 UNUSED,
341 int i UNUSED, double w)
348 rank_rank (const struct rank *cmd, double c, double cc, double cc_1,
349 int i, double w UNUSED)
364 rank = cc_1 + (c + 1.0)/ 2.0;
384 rank = cc_1 + c / 2.0 ;
399 rank_rfraction (const struct rank *cmd, double c, double cc, double cc_1,
402 return rank_rank (cmd, c, cc, cc_1, i, w) / w ;
407 rank_percent (const struct rank *cmd, double c, double cc, double cc_1,
410 return rank_rank (cmd, c, cc, cc_1, i, w) * 100.0 / w ;
415 rank_proportion (const struct rank *cmd, double c, double cc, double cc_1,
418 const double r = rank_rank (cmd, c, cc, cc_1, i, w) ;
422 switch ( cmd->fraction )
425 f = (r - 3.0/8.0) / (w + 0.25);
431 f = (r - 1.0/3.0) / (w + 1.0/3.0);
441 return (f > 0) ? f : SYSMIS;
445 rank_normal (const struct rank *cmd, double c, double cc, double cc_1,
448 double f = rank_proportion (cmd, c, cc, cc_1, i, w);
450 return gsl_cdf_ugaussian_Pinv (f);
454 rank_ntiles (const struct rank *cmd, double c, double cc, double cc_1,
457 double r = rank_rank (cmd, c, cc, cc_1, i, w);
460 return ( floor (( r * cmd->k_ntiles) / ( w + 1) ) + 1);
463 /* Expected value of the order statistics from an exponential distribution */
465 ee (int j, double w_star)
470 for (k = 1 ; k <= j; k++)
471 sum += 1.0 / ( w_star + 1 - k );
478 rank_savage (const struct rank *cmd UNUSED, double c, double cc, double cc_1,
479 int i UNUSED, double w)
482 const int i_1 = floor (cc_1);
483 const int i_2 = floor (cc);
485 const double w_star = (modf (w, &int_part) == 0 ) ? w : floor (w) + 1;
487 const double g_1 = cc_1 - i_1;
488 const double g_2 = cc - i_2;
490 /* The second factor is infinite, when the first is zero.
491 Therefore, evaluate the second, only when the first is non-zero */
492 const double expr1 = (1 - g_1) ? (1 - g_1) * ee(i_1+1, w_star) : ( 1 - g_1);
493 const double expr2 = g_2 ? g_2 * ee (i_2+1, w_star) : g_2 ;
496 return ee (i_1 + 1, w_star) - 1;
498 if ( i_1 + 1 == i_2 )
499 return ( ( expr1 + expr2 )/c ) - 1;
501 if ( i_1 + 2 <= i_2 )
505 for (j = i_1 + 2 ; j <= i_2; ++j )
506 sigma += ee (j, w_star);
507 return ( (expr1 + expr2 + sigma) / c) -1;
515 rank_sorted_file (struct casereader *input,
516 struct casewriter *output,
517 const struct dictionary *dict,
519 const struct rank *cmd
522 struct casereader *pass1, *pass2, *pass2_1;
523 struct casegrouper *tie_grouper;
529 input = casereader_create_filter_missing (input, &cmd->vars[dest_idx], 1,
530 cmd->exclude, NULL, output);
531 input = casereader_create_filter_weight (input, dict, NULL, output);
533 casereader_split (input, &pass1, &pass2);
535 /* Pass 1: Get total group weight. */
536 for (; (c = casereader_read (pass1)) != NULL; case_unref (c))
537 w += dict_get_case_weight (dict, c, NULL);
538 casereader_destroy (pass1);
540 /* Pass 2: Do ranking. */
541 tie_grouper = casegrouper_create_vars (pass2, &cmd->vars[dest_idx], 1);
542 while (casegrouper_get_next_group (tie_grouper, &pass2_1))
544 struct casereader *pass2_2;
549 pass2_2 = casereader_clone (pass2_1);
550 taint_propagate (casereader_get_taint (pass2_2),
551 casewriter_get_taint (output));
553 /* Pass 2.1: Sum up weight for tied cases. */
554 for (; (c = casereader_read (pass2_1)) != NULL; case_unref (c))
555 tw += dict_get_case_weight (dict, c, NULL);
557 casereader_destroy (pass2_1);
559 /* Pass 2.2: Rank tied cases. */
560 while ((c = casereader_read (pass2_2)) != NULL)
562 c = case_unshare (c);
563 for (i = 0; i < cmd->n_rs; ++i)
565 const struct variable *dst_var = cmd->rs[i].destvars[dest_idx];
566 double *dst_value = &case_data_rw (c, dst_var)->f;
567 *dst_value = rank_func[cmd->rs[i].rfunc] (cmd, tw, cc, cc_1, tie_group, w);
569 casewriter_write (output, c);
571 casereader_destroy (pass2_2);
575 casegrouper_destroy (tie_grouper);
579 /* Transformation function to enumerate all the cases */
581 create_resort_key (void *key_var_, struct ccase **cc, casenumber case_num)
583 struct variable *key_var = key_var_;
585 *cc = case_unshare (*cc);
586 case_data_rw (*cc, key_var)->f = case_num;
588 return TRNS_CONTINUE;
592 rank_cmd (struct dataset *ds, const struct rank *cmd);
596 fraction_name (const struct rank *cmd)
598 switch (cmd->fraction )
600 case FRAC_BLOM: return "BLOM";
601 case FRAC_RANKIT: return "RANKIT";
602 case FRAC_TUKEY: return "TUKEY";
603 case FRAC_VW: return "VW";
604 default: NOT_REACHED ();
608 /* Create a label on DEST_VAR, describing its derivation from SRC_VAR and F */
610 create_var_label (struct rank *cmd, struct variable *dest_var,
611 const struct variable *src_var, enum rank_func f)
614 ds_init_empty (&label);
616 if ( cmd->n_group_vars > 0 )
618 struct string group_var_str;
621 ds_init_empty (&group_var_str);
623 for (g = 0 ; g < cmd->n_group_vars ; ++g )
625 if ( g > 0 ) ds_put_cstr (&group_var_str, " ");
626 ds_put_cstr (&group_var_str, var_get_name (cmd->group_vars[g]));
629 ds_put_format (&label, _("%s of %s by %s"), function_name[f],
630 var_get_name (src_var), ds_cstr (&group_var_str));
631 ds_destroy (&group_var_str);
634 ds_put_format (&label, _("%s of %s"),
635 function_name[f], var_get_name (src_var));
637 var_set_label (dest_var, ds_cstr (&label), false);
643 cmd_rank (struct lexer *lexer, struct dataset *ds)
646 struct variable *order;
650 subcase_init_empty (&rank.sc);
654 rank.exclude = MV_ANY;
655 rank.n_group_vars = 0;
656 rank.group_vars = NULL;
657 rank.dict = dataset_dict (ds);
658 rank.ties = TIES_MEAN;
659 rank.fraction = FRAC_BLOM;
661 rank.pool = pool_create ();
663 if (lex_match_id (lexer, "VARIABLES"))
664 lex_force_match (lexer, T_EQUALS);
666 if (!parse_sort_criteria (lexer, rank.dict,
671 rank.n_vars = rank.sc.n_fields;
673 if (lex_match (lexer, T_BY) )
675 if ( ! parse_variables_const (lexer, rank.dict,
676 &rank.group_vars, &rank.n_group_vars,
677 PV_NO_DUPLICATE | PV_NO_SCRATCH))
682 while (lex_token (lexer) != T_ENDCMD )
684 lex_force_match (lexer, T_SLASH);
685 if (lex_match_id (lexer, "TIES"))
687 lex_force_match (lexer, T_EQUALS);
688 if (lex_match_id (lexer, "MEAN"))
690 rank.ties = TIES_MEAN;
692 else if (lex_match_id (lexer, "LOW"))
694 rank.ties = TIES_LOW;
696 else if (lex_match_id (lexer, "HIGH"))
698 rank.ties = TIES_HIGH;
700 else if (lex_match_id (lexer, "CONDENSE"))
702 rank.ties = TIES_CONDENSE;
706 lex_error (lexer, NULL);
710 else if (lex_match_id (lexer, "FRACTION"))
712 lex_force_match (lexer, T_EQUALS);
713 if (lex_match_id (lexer, "BLOM"))
715 rank.fraction = FRAC_BLOM;
717 else if (lex_match_id (lexer, "TUKEY"))
719 rank.fraction = FRAC_TUKEY;
721 else if (lex_match_id (lexer, "VW"))
723 rank.fraction = FRAC_VW;
725 else if (lex_match_id (lexer, "RANKIT"))
727 rank.fraction = FRAC_RANKIT;
731 lex_error (lexer, NULL);
735 else if (lex_match_id (lexer, "PRINT"))
737 lex_force_match (lexer, T_EQUALS);
738 if (lex_match_id (lexer, "YES"))
742 else if (lex_match_id (lexer, "NO"))
748 lex_error (lexer, NULL);
752 else if (lex_match_id (lexer, "MISSING"))
754 lex_force_match (lexer, T_EQUALS);
755 if (lex_match_id (lexer, "INCLUDE"))
757 rank.exclude = MV_SYSTEM;
759 else if (lex_match_id (lexer, "EXCLUDE"))
761 rank.exclude = MV_ANY;
765 lex_error (lexer, NULL);
769 else if (! parse_into (lexer, &rank))
774 /* If no rank specs are given, then apply a default */
777 rank.rs = pool_calloc (rank.pool, 1, sizeof (*rank.rs));
779 rank.rs[0].rfunc = RANK;
780 rank.rs[0].destvars = pool_calloc (rank.pool, rank.n_vars, sizeof (*rank.rs[0].destvars));
783 /* Create variables for all rank destinations which haven't
784 already been created with INTO.
785 Add labels to all the destination variables.
787 for (i = 0 ; i < rank.n_rs ; ++i )
790 struct rank_spec *rs = &rank.rs[i];
792 for ( v = 0 ; v < rank.n_vars ; v ++ )
794 if ( rs->destvars[v] == NULL )
797 create_rank_variable (rank.dict, rs->rfunc, rank.vars[v], NULL);
800 create_var_label (&rank, rs->destvars[v],
810 tab_output_text (0, _("Variables Created By RANK"));
811 tab_output_text (0, "");
813 for (i = 0 ; i < rank.n_rs ; ++i )
815 for ( v = 0 ; v < rank.n_vars ; v ++ )
817 if ( rank.n_group_vars > 0 )
819 struct string varlist;
822 ds_init_empty (&varlist);
823 for ( g = 0 ; g < rank.n_group_vars ; ++g )
825 ds_put_cstr (&varlist, var_get_name (rank.group_vars[g]));
827 if ( g < rank.n_group_vars - 1)
828 ds_put_cstr (&varlist, " ");
831 if ( rank.rs[i].rfunc == NORMAL ||
832 rank.rs[i].rfunc == PROPORTION )
833 tab_output_text_format (0,
834 _("%s into %s(%s of %s using %s BY %s)"),
835 var_get_name (rank.vars[v]),
836 var_get_name (rank.rs[i].destvars[v]),
837 function_name[rank.rs[i].rfunc],
838 var_get_name (rank.vars[v]),
839 fraction_name (&rank),
843 tab_output_text_format (0,
844 _("%s into %s(%s of %s BY %s)"),
845 var_get_name (rank.vars[v]),
846 var_get_name (rank.rs[i].destvars[v]),
847 function_name[rank.rs[i].rfunc],
848 var_get_name (rank.vars[v]),
850 ds_destroy (&varlist);
854 if ( rank.rs[i].rfunc == NORMAL ||
855 rank.rs[i].rfunc == PROPORTION )
856 tab_output_text_format (0,
857 _("%s into %s(%s of %s using %s)"),
858 var_get_name (rank.vars[v]),
859 var_get_name (rank.rs[i].destvars[v]),
860 function_name[rank.rs[i].rfunc],
861 var_get_name (rank.vars[v]),
862 fraction_name (&rank));
865 tab_output_text_format (0,
866 _("%s into %s(%s of %s)"),
867 var_get_name (rank.vars[v]),
868 var_get_name (rank.rs[i].destvars[v]),
869 function_name[rank.rs[i].rfunc],
870 var_get_name (rank.vars[v]));
876 /* Add a variable which we can sort by to get back the original
878 order = dict_create_var_assert (dataset_dict (ds), "$ORDER_", 0);
880 add_transformation (ds, create_resort_key, 0, order);
883 result = rank_cmd (ds, &rank);
885 /* Put the active dataset back in its original order. Delete
886 our sort key, which we don't need anymore. */
888 struct casereader *sorted;
891 /* FIXME: loses error conditions. */
893 proc_discard_output (ds);
894 sorted = sort_execute_1var (proc_open (ds), order);
895 result = proc_commit (ds) && result;
897 dict_delete_var (dataset_dict (ds), order);
898 result = dataset_set_source (ds, sorted) && result;
903 destroy_rank (&rank);
908 destroy_rank (&rank);
915 rank_cmd (struct dataset *ds, const struct rank *cmd)
917 struct dictionary *d = dataset_dict (ds);
921 for (i = 0 ; i < subcase_get_n_fields (&cmd->sc) ; ++i )
923 /* Rank variable at index I in SC. */
924 struct casegrouper *split_grouper;
925 struct casereader *split_group;
926 struct casewriter *output;
928 proc_discard_output (ds);
929 split_grouper = casegrouper_create_splits (proc_open (ds), d);
930 output = autopaging_writer_create (dict_get_proto (d));
932 while (casegrouper_get_next_group (split_grouper, &split_group))
934 struct subcase ordering;
935 struct casereader *ordered;
936 struct casegrouper *by_grouper;
937 struct casereader *by_group;
939 /* Sort this split group by the BY variables as primary
940 keys and the rank variable as secondary key. */
941 subcase_init_vars (&ordering, cmd->group_vars, cmd->n_group_vars);
942 subcase_add_var (&ordering, cmd->vars[i],
943 subcase_get_direction (&cmd->sc, i));
944 ordered = sort_execute (split_group, &ordering);
945 subcase_destroy (&ordering);
947 /* Rank the rank variable within this split group. */
948 by_grouper = casegrouper_create_vars (ordered,
949 cmd->group_vars, cmd->n_group_vars);
950 while (casegrouper_get_next_group (by_grouper, &by_group))
952 /* Rank the rank variable within this BY group
953 within the split group. */
955 rank_sorted_file (by_group, output, d, i, cmd);
958 ok = casegrouper_destroy (by_grouper) && ok;
960 ok = casegrouper_destroy (split_grouper);
961 ok = proc_commit (ds) && ok;
962 ok = (dataset_set_source (ds, casewriter_make_reader (output))