2 PSPP - a program for statistical analysis.
3 Copyright (C) 2012, 2013, 2015 Free Software Foundation, Inc.
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
20 * This module implements the graph command
26 #include "gl/xalloc.h"
27 #include <gsl/gsl_cdf.h>
29 #include "libpspp/assertion.h"
30 #include "libpspp/message.h"
31 #include "libpspp/pool.h"
34 #include "data/dataset.h"
35 #include "data/dictionary.h"
36 #include "data/casegrouper.h"
37 #include "data/casereader.h"
38 #include "data/casewriter.h"
39 #include "data/caseproto.h"
40 #include "data/subcase.h"
43 #include "data/format.h"
45 #include "math/chart-geometry.h"
46 #include "math/histogram.h"
47 #include "math/moments.h"
48 #include "math/sort.h"
49 #include "math/order-stats.h"
50 #include "output/charts/plot-hist.h"
51 #include "output/charts/scatterplot.h"
52 #include "output/charts/barchart.h"
54 #include "language/command.h"
55 #include "language/lexer/lexer.h"
56 #include "language/lexer/value-parser.h"
57 #include "language/lexer/variable-parser.h"
58 #include "language/stats/freq.h"
59 #include "language/stats/chart-category.h"
62 #define _(msgid) gettext (msgid)
63 #define N_(msgid) msgid
95 /* Variable index for histogram case */
102 struct exploratory_stats
115 /* The minimum weight */
125 const struct variable **dep_vars;
126 struct exploratory_stats *es;
128 enum mv_class dep_excl;
129 enum mv_class fctr_excl;
131 const struct dictionary *dict;
135 /* ------------ Graph ---------------- */
136 bool normal; /* For histograms, draw the normal curve */
138 enum chart_type chart_type;
139 enum scatter_type scatter_type;
140 enum bar_type bar_type;
141 const struct variable *by_var[2];
144 struct subcase ordering; /* Ordering for aggregation */
145 int agr; /* Index into ag_func */
147 /* A caseproto that contains the plot data */
148 struct caseproto *gr_proto;
155 calc_mom1 (double acc, double x, double w)
161 calc_mom0 (double acc, double x UNUSED, double w)
167 pre_low_extreme (void)
173 calc_max (double acc, double x, double w UNUSED)
175 return (acc > x) ? acc : x;
179 pre_high_extreme (void)
185 calc_min (double acc, double x, double w UNUSED)
187 return (acc < x) ? acc : x;
191 post_normalise (double acc, double cc)
197 post_percentage (double acc, double ccc)
199 return acc / ccc * 100.0;
203 const struct ag_func ag_func[] =
205 {"COUNT", N_("Count"), 0, 0, NULL, calc_mom0, 0, 0},
206 {"PCT", N_("Percentage"), 0, 0, NULL, calc_mom0, 0, post_percentage},
207 {"CUFREQ", N_("Cumulative Count"), 0, 1, NULL, calc_mom0, 0, 0},
208 {"CUPCT", N_("Cumulative Percent"), 0, 1, NULL, calc_mom0, 0, post_percentage},
210 {"MEAN", N_("Mean"), 1, 0, NULL, calc_mom1, post_normalise, 0},
211 {"SUM", N_("Sum"), 1, 0, NULL, calc_mom1, 0, 0},
212 {"MAXIMUM", N_("Maximum"), 1, 0, pre_low_extreme, calc_max, 0, 0},
213 {"MINIMUM", N_("Minimum"), 1, 0, pre_high_extreme, calc_min, 0, 0},
216 const int N_AG_FUNCS = sizeof (ag_func) / sizeof (ag_func[0]);
219 parse_function (struct lexer *lexer, struct graph *graph)
222 for (i = 0 ; i < N_AG_FUNCS; ++i)
224 if (lex_match_id (lexer, ag_func[i].name))
235 graph->n_dep_vars = ag_func[i].arity;
236 if (ag_func[i].arity > 0)
239 if (!lex_force_match (lexer, T_LPAREN))
242 graph->dep_vars = xzalloc (sizeof (graph->dep_vars) * graph->n_dep_vars);
243 for (v = 0; v < ag_func[i].arity; ++v)
245 graph->dep_vars[v] = parse_variable (lexer, graph->dict);
246 if (! graph->dep_vars[v])
250 if (!lex_force_match (lexer, T_RPAREN))
254 if (!lex_force_match (lexer, T_BY))
257 graph->by_var[0] = parse_variable (lexer, graph->dict);
258 if (!graph->by_var[0])
262 subcase_add_var (&graph->ordering, graph->by_var[0], SC_ASCEND);
265 if (lex_match (lexer, T_BY))
267 graph->by_var[1] = parse_variable (lexer, graph->dict);
268 if (!graph->by_var[1])
272 subcase_add_var (&graph->ordering, graph->by_var[1], SC_ASCEND);
279 lex_error (lexer, NULL);
285 show_scatterplot (const struct graph *cmd, struct casereader *input)
288 struct scatterplot_chart *scatterplot;
289 bool byvar_overflow = false;
291 ds_init_empty (&title);
293 if (cmd->n_by_vars > 0)
295 ds_put_format (&title, _("%s vs. %s by %s"),
296 var_to_string (cmd->dep_vars[1]),
297 var_to_string (cmd->dep_vars[0]),
298 var_to_string (cmd->by_var[0]));
302 ds_put_format (&title, _("%s vs. %s"),
303 var_to_string (cmd->dep_vars[1]),
304 var_to_string (cmd->dep_vars[0]));
307 scatterplot = scatterplot_create (input,
308 var_to_string(cmd->dep_vars[0]),
309 var_to_string(cmd->dep_vars[1]),
310 (cmd->n_by_vars > 0) ? cmd->by_var[0] : NULL,
313 cmd->es[0].minimum, cmd->es[0].maximum,
314 cmd->es[1].minimum, cmd->es[1].maximum);
315 scatterplot_chart_submit (scatterplot);
320 msg (MW, _("Maximum number of scatterplot categories reached. "
321 "Your BY variable has too many distinct values. "
322 "The coloring of the plot will not be correct."));
327 show_histogr (const struct graph *cmd, struct casereader *input)
329 struct histogram *histogram;
332 if (cmd->es[0].cc <= 0)
334 casereader_destroy (input);
340 double bin_width = fabs (cmd->es[0].minimum - cmd->es[0].maximum)
341 / (1 + log2 (cmd->es[0].cc))
345 histogram_create (bin_width, cmd->es[0].minimum, cmd->es[0].maximum);
348 if (NULL == histogram)
350 casereader_destroy (input);
354 for (;(c = casereader_read (input)) != NULL; case_unref (c))
356 const double x = case_data_idx (c, HG_IDX_X)->f;
357 const double weight = case_data_idx (c, HG_IDX_WT)->f;
358 moments_pass_two (cmd->es[0].mom, x, weight);
359 histogram_add (histogram, x, weight);
361 casereader_destroy (input);
369 ds_init_cstr (&label,
370 var_to_string (cmd->dep_vars[0]));
372 moments_calculate (cmd->es[0].mom, &n, &mean, &var, NULL, NULL);
375 ( histogram_chart_create (histogram->gsl_hist,
376 ds_cstr (&label), n, mean,
377 sqrt (var), cmd->normal));
379 statistic_destroy (&histogram->parent);
385 cleanup_exploratory_stats (struct graph *cmd)
389 for (v = 0; v < cmd->n_dep_vars; ++v)
391 moments_destroy (cmd->es[v].mom);
397 run_barchart (struct graph *cmd, struct casereader *input)
399 struct casegrouper *grouper;
400 struct casereader *group;
403 if ( cmd->missing_pw == false)
404 input = casereader_create_filter_missing (input,
412 input = sort_execute (input, &cmd->ordering);
414 struct freq **freqs = NULL;
417 for (grouper = casegrouper_create_vars (input, cmd->by_var,
419 casegrouper_get_next_group (grouper, &group);
420 casereader_destroy (group))
423 struct ccase *c = casereader_peek (group, 0);
425 /* Deal with missing values in the categorical variables */
426 for (v = 0; v < cmd->n_by_vars; ++v)
428 if (var_is_value_missing (cmd->by_var[v], case_data (c, cmd->by_var[v]), cmd->fctr_excl) )
432 if (v < cmd->n_by_vars)
438 freqs = xrealloc (freqs, sizeof (*freqs) * ++n_freqs);
439 freqs[n_freqs - 1] = xzalloc (sizeof (**freqs) +
440 sizeof (union value) * (cmd->n_by_vars - 1) );
442 if (ag_func[cmd->agr].cumulative && n_freqs >= 2)
443 freqs[n_freqs - 1]->count = freqs[n_freqs - 2]->count;
445 freqs[n_freqs - 1]->count = 0;
446 if (ag_func[cmd->agr].pre)
447 freqs[n_freqs - 1]->count = ag_func[cmd->agr].pre();
450 for (v = 0; v < cmd->n_by_vars; ++v)
452 value_clone (&freqs[n_freqs - 1]->values[v], case_data (c, cmd->by_var[v]),
453 var_get_width (cmd->by_var[v])
459 for (;(c = casereader_read (group)) != NULL; case_unref (c))
461 const double weight = dict_get_case_weight (cmd->dict,c,NULL);
462 const double x = (cmd->n_dep_vars > 0) ? case_data (c, cmd->dep_vars[0])->f : SYSMIS;
466 freqs[n_freqs - 1]->count
467 = ag_func[cmd->agr].calc (freqs[n_freqs - 1]->count, x, weight);
470 if (ag_func[cmd->agr].post)
471 freqs[n_freqs - 1]->count
472 = ag_func[cmd->agr].post (freqs[n_freqs - 1]->count, cc);
477 casegrouper_destroy (grouper);
479 for (int i = 0; i < n_freqs; ++i)
481 if (ag_func[cmd->agr].ppost)
482 freqs[i]->count = ag_func[cmd->agr].ppost (freqs[i]->count, ccc);
488 ds_init_empty (&label);
490 if (cmd->n_dep_vars > 0)
491 ds_put_format (&label, _("%s of %s"),
492 ag_func[cmd->agr].description,
493 var_get_name (cmd->dep_vars[0]));
496 ag_func[cmd->agr].description);
498 chart_item_submit (barchart_create (cmd->by_var, cmd->n_by_vars,
499 ds_cstr (&label), false,
505 for (int i = 0; i < n_freqs; ++i)
513 run_graph (struct graph *cmd, struct casereader *input)
516 struct casereader *reader;
517 struct casewriter *writer;
519 cmd->es = pool_calloc (cmd->pool,cmd->n_dep_vars,sizeof(struct exploratory_stats));
520 for(int v=0;v<cmd->n_dep_vars;v++)
522 cmd->es[v].mom = moments_create (MOMENT_KURTOSIS);
523 cmd->es[v].cmin = DBL_MAX;
524 cmd->es[v].maximum = -DBL_MAX;
525 cmd->es[v].minimum = DBL_MAX;
527 /* Always remove cases listwise. This is correct for */
528 /* the histogram because there is only one variable */
529 /* and a simple bivariate scatterplot */
530 /* if ( cmd->missing_pw == false) */
531 input = casereader_create_filter_missing (input,
538 writer = autopaging_writer_create (cmd->gr_proto);
540 /* The case data is copied to a new writer */
541 /* The setup of the case depends on the Charttype */
542 /* For Scatterplot x is assumed in dep_vars[0] */
543 /* y is assumed in dep_vars[1] */
544 /* For Histogram x is assumed in dep_vars[0] */
545 assert(SP_IDX_X == 0 && SP_IDX_Y == 1 && HG_IDX_X == 0);
547 for (;(c = casereader_read (input)) != NULL; case_unref (c))
549 struct ccase *outcase = case_create (cmd->gr_proto);
550 const double weight = dict_get_case_weight (cmd->dict,c,NULL);
551 if (cmd->chart_type == CT_HISTOGRAM)
552 case_data_rw_idx (outcase, HG_IDX_WT)->f = weight;
553 if (cmd->chart_type == CT_SCATTERPLOT && cmd->n_by_vars > 0)
554 value_copy (case_data_rw_idx (outcase, SP_IDX_BY),
555 case_data (c, cmd->by_var[0]),
556 var_get_width (cmd->by_var[0]));
557 for(int v=0;v<cmd->n_dep_vars;v++)
559 const struct variable *var = cmd->dep_vars[v];
560 const double x = case_data (c, var)->f;
562 if (var_is_value_missing (var, case_data (c, var), cmd->dep_excl))
564 cmd->es[v].missing += weight;
567 /* Magically v value fits to SP_IDX_X, SP_IDX_Y, HG_IDX_X */
568 case_data_rw_idx (outcase, v)->f = x;
570 if (x > cmd->es[v].maximum)
571 cmd->es[v].maximum = x;
573 if (x < cmd->es[v].minimum)
574 cmd->es[v].minimum = x;
576 cmd->es[v].non_missing += weight;
578 moments_pass_one (cmd->es[v].mom, x, weight);
580 cmd->es[v].cc += weight;
582 if (cmd->es[v].cmin > weight)
583 cmd->es[v].cmin = weight;
585 casewriter_write (writer,outcase);
588 reader = casewriter_make_reader (writer);
590 switch (cmd->chart_type)
593 show_histogr (cmd,reader);
596 show_scatterplot (cmd,reader);
603 casereader_destroy (input);
604 cleanup_exploratory_stats (cmd);
609 cmd_graph (struct lexer *lexer, struct dataset *ds)
613 graph.missing_pw = false;
615 graph.pool = pool_create ();
617 graph.dep_excl = MV_ANY;
618 graph.fctr_excl = MV_ANY;
620 graph.dict = dataset_dict (ds);
622 graph.dep_vars = NULL;
623 graph.chart_type = CT_NONE;
624 graph.scatter_type = ST_BIVARIATE;
626 graph.gr_proto = caseproto_create ();
628 subcase_init_empty (&graph.ordering);
630 while (lex_token (lexer) != T_ENDCMD)
632 lex_match (lexer, T_SLASH);
634 if (lex_match_id (lexer, "HISTOGRAM"))
636 if (graph.chart_type != CT_NONE)
638 lex_error (lexer, _("Only one chart type is allowed."));
641 graph.normal = false;
642 if (lex_match (lexer, T_LPAREN))
644 if (!lex_force_match_id (lexer, "NORMAL"))
647 if (!lex_force_match (lexer, T_RPAREN))
652 if (!lex_force_match (lexer, T_EQUALS))
654 graph.chart_type = CT_HISTOGRAM;
655 if (!parse_variables_const (lexer, graph.dict,
656 &graph.dep_vars, &graph.n_dep_vars,
657 PV_NO_DUPLICATE | PV_NUMERIC))
659 if (graph.n_dep_vars > 1)
661 lex_error (lexer, _("Only one variable is allowed."));
665 else if (lex_match_id (lexer, "BAR"))
667 if (graph.chart_type != CT_NONE)
669 lex_error (lexer, _("Only one chart type is allowed."));
672 graph.chart_type = CT_BAR;
673 graph.bar_type = CBT_SIMPLE;
675 if (lex_match (lexer, T_LPAREN))
677 if (lex_match_id (lexer, "SIMPLE"))
679 /* This is the default anyway */
681 else if (lex_match_id (lexer, "GROUPED"))
683 graph.bar_type = CBT_GROUPED;
686 else if (lex_match_id (lexer, "STACKED"))
688 graph.bar_type = CBT_STACKED;
689 lex_error (lexer, _("%s is not yet implemented."), "STACKED");
692 else if (lex_match_id (lexer, "RANGE"))
694 graph.bar_type = CBT_RANGE;
695 lex_error (lexer, _("%s is not yet implemented."), "RANGE");
700 lex_error (lexer, NULL);
703 if (!lex_force_match (lexer, T_RPAREN))
707 if (!lex_force_match (lexer, T_EQUALS))
710 if (! parse_function (lexer, &graph))
713 else if (lex_match_id (lexer, "SCATTERPLOT"))
715 if (graph.chart_type != CT_NONE)
717 lex_error (lexer, _("Only one chart type is allowed."));
720 graph.chart_type = CT_SCATTERPLOT;
721 if (lex_match (lexer, T_LPAREN))
723 if (lex_match_id (lexer, "BIVARIATE"))
725 /* This is the default anyway */
727 else if (lex_match_id (lexer, "OVERLAY"))
729 lex_error (lexer, _("%s is not yet implemented."),"OVERLAY");
732 else if (lex_match_id (lexer, "MATRIX"))
734 lex_error (lexer, _("%s is not yet implemented."),"MATRIX");
737 else if (lex_match_id (lexer, "XYZ"))
739 lex_error(lexer, _("%s is not yet implemented."),"XYZ");
744 lex_error_expecting (lexer, "BIVARIATE", NULL);
747 if (!lex_force_match (lexer, T_RPAREN))
750 if (!lex_force_match (lexer, T_EQUALS))
753 if (!parse_variables_const (lexer, graph.dict,
754 &graph.dep_vars, &graph.n_dep_vars,
755 PV_NO_DUPLICATE | PV_NUMERIC))
758 if (graph.scatter_type == ST_BIVARIATE && graph.n_dep_vars != 1)
760 lex_error(lexer, _("Only one variable is allowed."));
764 if (!lex_force_match (lexer, T_WITH))
767 if (!parse_variables_const (lexer, graph.dict,
768 &graph.dep_vars, &graph.n_dep_vars,
769 PV_NO_DUPLICATE | PV_NUMERIC | PV_APPEND))
772 if (graph.scatter_type == ST_BIVARIATE && graph.n_dep_vars != 2)
774 lex_error (lexer, _("Only one variable is allowed."));
778 if (lex_match (lexer, T_BY))
780 const struct variable *v = NULL;
781 if (!lex_match_variable (lexer,graph.dict,&v))
783 lex_error (lexer, _("Variable expected"));
790 else if (lex_match_id (lexer, "LINE"))
792 lex_error (lexer, _("%s is not yet implemented."),"LINE");
795 else if (lex_match_id (lexer, "PIE"))
797 lex_error (lexer, _("%s is not yet implemented."),"PIE");
800 else if (lex_match_id (lexer, "ERRORBAR"))
802 lex_error (lexer, _("%s is not yet implemented."),"ERRORBAR");
805 else if (lex_match_id (lexer, "PARETO"))
807 lex_error (lexer, _("%s is not yet implemented."),"PARETO");
810 else if (lex_match_id (lexer, "TITLE"))
812 lex_error (lexer, _("%s is not yet implemented."),"TITLE");
815 else if (lex_match_id (lexer, "SUBTITLE"))
817 lex_error (lexer, _("%s is not yet implemented."),"SUBTITLE");
820 else if (lex_match_id (lexer, "FOOTNOTE"))
822 lex_error (lexer, _("%s is not yet implemented."),"FOOTNOTE");
823 lex_error (lexer, _("FOOTNOTE is not implemented yet for GRAPH"));
826 else if (lex_match_id (lexer, "MISSING"))
828 lex_match (lexer, T_EQUALS);
830 while (lex_token (lexer) != T_ENDCMD
831 && lex_token (lexer) != T_SLASH)
833 if (lex_match_id (lexer, "LISTWISE"))
835 graph.missing_pw = false;
837 else if (lex_match_id (lexer, "VARIABLE"))
839 graph.missing_pw = true;
841 else if (lex_match_id (lexer, "EXCLUDE"))
843 graph.dep_excl = MV_ANY;
845 else if (lex_match_id (lexer, "INCLUDE"))
847 graph.dep_excl = MV_SYSTEM;
849 else if (lex_match_id (lexer, "REPORT"))
851 graph.fctr_excl = MV_NEVER;
853 else if (lex_match_id (lexer, "NOREPORT"))
855 graph.fctr_excl = MV_ANY;
859 lex_error (lexer, NULL);
866 lex_error (lexer, NULL);
871 switch (graph.chart_type)
874 /* See scatterplot.h for the setup of the case prototype */
875 graph.gr_proto = caseproto_add_width (graph.gr_proto, 0); /* x value - SP_IDX_X*/
876 graph.gr_proto = caseproto_add_width (graph.gr_proto, 0); /* y value - SP_IDX_Y*/
877 /* The by_var contains the plot categories for the different xy plot colors */
878 if (graph.n_by_vars > 0) /* SP_IDX_BY */
879 graph.gr_proto = caseproto_add_width (graph.gr_proto, var_get_width(graph.by_var[0]));
882 graph.gr_proto = caseproto_add_width (graph.gr_proto, 0); /* x value */
883 graph.gr_proto = caseproto_add_width (graph.gr_proto, 0); /* weight value */
888 lex_error_expecting (lexer, "HISTOGRAM", "SCATTERPLOT", "BAR", NULL);
896 struct casegrouper *grouper;
897 struct casereader *group;
900 grouper = casegrouper_create_splits (proc_open (ds), graph.dict);
901 while (casegrouper_get_next_group (grouper, &group))
903 if (graph.chart_type == CT_BAR)
904 run_barchart (&graph, group);
906 run_graph (&graph, group);
908 ok = casegrouper_destroy (grouper);
909 ok = proc_commit (ds) && ok;
912 subcase_destroy (&graph.ordering);
913 free (graph.dep_vars);
914 pool_destroy (graph.pool);
915 caseproto_unref (graph.gr_proto);
920 subcase_destroy (&graph.ordering);
921 caseproto_unref (graph.gr_proto);
922 free (graph.dep_vars);
923 pool_destroy (graph.pool);