1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2021 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
22 #include "data/casegrouper.h"
23 #include "data/casereader.h"
24 #include "data/casewriter.h"
25 #include "data/data-in.h"
26 #include "data/data-out.h"
27 #include "data/dataset.h"
28 #include "data/dictionary.h"
29 #include "data/mrset.h"
30 #include "data/subcase.h"
31 #include "data/value-labels.h"
32 #include "language/command.h"
33 #include "language/dictionary/split-file.h"
34 #include "language/lexer/format-parser.h"
35 #include "language/lexer/lexer.h"
36 #include "language/lexer/token.h"
37 #include "language/lexer/variable-parser.h"
38 #include "libpspp/array.h"
39 #include "libpspp/assertion.h"
40 #include "libpspp/hash-functions.h"
41 #include "libpspp/hmap.h"
42 #include "libpspp/i18n.h"
43 #include "libpspp/message.h"
44 #include "libpspp/string-array.h"
45 #include "math/mode.h"
46 #include "math/moments.h"
47 #include "math/percentiles.h"
48 #include "math/sort.h"
49 #include "output/pivot-table.h"
51 #include "gl/minmax.h"
52 #include "gl/xalloc.h"
55 #define _(msgid) gettext (msgid)
56 #define N_(msgid) (msgid)
60 CTVL_NONE = SETTINGS_VALUE_SHOW_DEFAULT,
61 CTVL_NAME = SETTINGS_VALUE_SHOW_VALUE,
62 CTVL_LABEL = SETTINGS_VALUE_SHOW_LABEL,
63 CTVL_BOTH = SETTINGS_VALUE_SHOW_BOTH,
66 enum ctables_weighting
74 enum ctables_function_type
76 /* A function that operates on data in a single cell. It operates on
77 effective weights. It does not have an unweighted version. */
80 /* A function that operates on data in a single cell. The function
81 operates on effective weights and has a U-prefixed unweighted
85 /* A function that operates on data in a single cell. It operates on
86 dictionary weights, and has U-prefixed unweighted version and an
87 E-prefixed effective weight version. */
90 /* A function that operates on an area of cells. It operates on effective
91 weights and has a U-prefixed unweighted version. */
102 enum ctables_function_availability
104 CTFA_ALL, /* Any variables. */
105 CTFA_SCALE, /* Only scale variables, totals, and subtotals. */
106 //CTFA_MRSETS, /* Only multiple-response sets */
109 enum ctables_summary_function
111 #define S(ENUM, NAME, TYPE, FORMAT, AVAILABILITY) ENUM,
112 #include "ctables.inc"
117 #define S(ENUM, NAME, TYPE, FORMAT, AVAILABILITY) +1
119 #include "ctables.inc"
123 struct ctables_function_info
125 struct substring basename;
126 enum ctables_function_type type;
127 enum ctables_format format;
128 enum ctables_function_availability availability;
130 bool u_prefix; /* Accepts a 'U' prefix (for unweighted)? */
131 bool e_prefix; /* Accepts an 'E' prefix (for effective)? */
132 bool is_area; /* Needs an area prefix. */
134 static const struct ctables_function_info ctables_function_info[N_CTSF_FUNCTIONS] = {
135 #define S(ENUM, NAME, TYPE, FORMAT, AVAILABILITY) \
137 .basename = SS_LITERAL_INITIALIZER (NAME), \
140 .availability = AVAILABILITY, \
141 .u_prefix = (TYPE) == CTFT_UCELL || (TYPE) == CTFT_UECELL || (TYPE) == CTFT_AREA, \
142 .e_prefix = (TYPE) == CTFT_UECELL, \
143 .is_area = (TYPE) == CTFT_AREA \
145 #include "ctables.inc"
149 enum ctables_area_type
151 /* Within a section, where stacked variables divide one section from
154 Keep CTAT_LAYER after CTAT_LAYERROW and CTAT_LAYERCOL so that
155 parse_ctables_summary_function() parses correctly. */
156 CTAT_TABLE, /* All layers of a whole section. */
157 CTAT_LAYERROW, /* Row in one layer within a section. */
158 CTAT_LAYERCOL, /* Column in one layer within a section. */
159 CTAT_LAYER, /* One layer within a section. */
161 /* Within a subtable, where a subtable pairs an innermost row variable with
162 an innermost column variable within a single layer. */
163 CTAT_SUBTABLE, /* Whole subtable. */
164 CTAT_ROW, /* Row within a subtable. */
165 CTAT_COL, /* Column within a subtable. */
169 static const char *ctables_area_type_name[N_CTATS] = {
170 [CTAT_TABLE] = "TABLE",
171 [CTAT_LAYER] = "LAYER",
172 [CTAT_LAYERROW] = "LAYERROW",
173 [CTAT_LAYERCOL] = "LAYERCOL",
174 [CTAT_SUBTABLE] = "SUBTABLE",
181 struct hmap_node node;
183 const struct ctables_cell *example;
186 double count[N_CTWS];
187 double valid[N_CTWS];
188 double total[N_CTWS];
189 struct ctables_sum *sums;
197 enum ctables_summary_variant
206 /* In struct ctables_section's 'cells' hmap. Indexed by all the values in
207 all the axes (except the scalar variable, if any). */
208 struct hmap_node node;
210 /* The areas that contain this cell. */
212 struct ctables_area *areas[N_CTATS];
217 enum ctables_summary_variant sv;
219 struct ctables_cell_axis
221 struct ctables_cell_value
223 const struct ctables_category *category;
231 union ctables_summary *summaries;
236 const struct dictionary *dict;
237 struct pivot_table_look *look;
239 /* CTABLES has a number of extra formats that we implement via custom
240 currency specifications on an alternate fmt_settings. */
241 #define CTEF_NEGPAREN FMT_CCA
242 #define CTEF_NEQUAL FMT_CCB
243 #define CTEF_PAREN FMT_CCC
244 #define CTEF_PCTPAREN FMT_CCD
245 struct fmt_settings ctables_formats;
247 /* If this is NULL, zeros are displayed using the normal print format.
248 Otherwise, this string is displayed. */
251 /* If this is NULL, missing values are displayed using the normal print
252 format. Otherwise, this string is displayed. */
255 /* Indexed by variable dictionary index. */
256 enum ctables_vlabel *vlabels;
258 struct hmap postcomputes; /* Contains "struct ctables_postcompute"s. */
260 bool mrsets_count_duplicates; /* MRSETS. */
261 bool smissing_listwise; /* SMISSING. */
262 struct variable *e_weight; /* WEIGHT. */
263 int hide_threshold; /* HIDESMALLCOUNTS. */
265 struct ctables_table **tables;
269 static struct ctables_postcompute *ctables_find_postcompute (struct ctables *,
272 struct ctables_postcompute
274 struct hmap_node hmap_node; /* In struct ctables's 'pcompute' hmap. */
275 char *name; /* Name, without leading &. */
277 struct msg_location *location; /* Location of definition. */
278 struct ctables_pcexpr *expr;
280 struct ctables_summary_spec_set *specs;
281 bool hide_source_cats;
284 struct ctables_pcexpr
294 enum ctables_postcompute_op
297 CTPO_CONSTANT, /* 5 */
298 CTPO_CAT_NUMBER, /* [5] */
299 CTPO_CAT_STRING, /* ["STRING"] */
300 CTPO_CAT_NRANGE, /* [LO THRU 5] */
301 CTPO_CAT_SRANGE, /* ["A" THRU "B"] */
302 CTPO_CAT_MISSING, /* MISSING */
303 CTPO_CAT_OTHERNM, /* OTHERNM */
304 CTPO_CAT_SUBTOTAL, /* SUBTOTAL */
305 CTPO_CAT_TOTAL, /* TOTAL */
319 /* CTPO_CAT_NUMBER. */
322 /* CTPO_CAT_STRING, in dictionary encoding. */
323 struct substring string;
325 /* CTPO_CAT_NRANGE. */
328 /* CTPO_CAT_SRANGE. */
329 struct substring srange[2];
331 /* CTPO_CAT_SUBTOTAL. */
332 size_t subtotal_index;
334 /* Two elements: CTPO_ADD, CTPO_SUB, CTPO_MUL, CTPO_DIV, CTPO_POW.
335 One element: CTPO_NEG. */
336 struct ctables_pcexpr *subs[2];
339 /* Source location. */
340 struct msg_location *location;
343 static void ctables_pcexpr_destroy (struct ctables_pcexpr *);
344 static struct ctables_pcexpr *ctables_pcexpr_allocate_binary (
345 enum ctables_postcompute_op, struct ctables_pcexpr *sub0,
346 struct ctables_pcexpr *sub1);
348 struct ctables_summary_spec_set
350 struct ctables_summary_spec *specs;
354 /* The variable to which the summary specs are applied. */
355 struct variable *var;
357 /* Whether the variable to which the summary specs are applied is a scale
358 variable for the purpose of summarization.
360 (VALIDN and TOTALN act differently for summarizing scale and categorical
364 /* If any of these optional additional scale variables are missing, then
365 treat 'var' as if it's missing too. This is for implementing
366 SMISSING=LISTWISE. */
367 struct variable **listwise_vars;
368 size_t n_listwise_vars;
371 static void ctables_summary_spec_set_clone (struct ctables_summary_spec_set *,
372 const struct ctables_summary_spec_set *);
373 static void ctables_summary_spec_set_uninit (struct ctables_summary_spec_set *);
375 /* A nested sequence of variables, e.g. a > b > c. */
378 struct variable **vars;
382 size_t *areas[N_CTATS];
383 size_t n_areas[N_CTATS];
386 struct ctables_summary_spec_set specs[N_CSVS];
389 /* A stack of nestings, e.g. nest1 + nest2 + ... + nestN. */
392 struct ctables_nest *nests;
396 static void ctables_stack_uninit (struct ctables_stack *);
400 struct hmap_node node;
405 struct ctables_occurrence
407 struct hmap_node node;
411 struct ctables_section
414 struct ctables_table *table;
415 struct ctables_nest *nests[PIVOT_N_AXES];
418 struct hmap *occurrences[PIVOT_N_AXES]; /* "struct ctables_occurrence"s. */
419 struct hmap cells; /* Contains "struct ctables_cell"s. */
420 struct hmap areas[N_CTATS]; /* Contains "struct ctables_area"s. */
423 static void ctables_section_uninit (struct ctables_section *);
427 struct ctables *ctables;
428 struct ctables_axis *axes[PIVOT_N_AXES];
429 struct ctables_stack stacks[PIVOT_N_AXES];
430 struct ctables_section *sections;
432 enum pivot_axis_type summary_axis;
433 struct ctables_summary_spec_set summary_specs;
434 struct variable **sum_vars;
437 enum pivot_axis_type slabels_axis;
438 bool slabels_visible;
440 /* The innermost category labels for axis 'a' appear on axis label_axis[a].
442 Most commonly, label_axis[a] == a, and in particular we always have
443 label_axis{PIVOT_AXIS_LAYER] == PIVOT_AXIS_LAYER.
445 If ROWLABELS or COLLABELS is specified, then one of
446 label_axis[PIVOT_AXIS_ROW] or label_axis[PIVOT_AXIS_COLUMN] can be the
447 opposite axis or PIVOT_AXIS_LAYER. Only one of them will differ.
449 If any category labels are moved, then 'clabels_example' is one of the
450 variables being moved (and it is otherwise NULL). All of the variables
451 being moved have the same width, value labels, and categories, so this
452 example variable can be used to find those out.
454 The remaining members in this group are relevant only if category labels
457 'clabels_values_map' holds a "struct ctables_value" for all the values
458 that appear in all of the variables in the moved categories. It is
459 accumulated as the data is read. Once the data is fully read, its
460 sorted values are put into 'clabels_values' and 'n_clabels_values'.
462 enum pivot_axis_type label_axis[PIVOT_N_AXES];
463 enum pivot_axis_type clabels_from_axis;
464 enum pivot_axis_type clabels_to_axis;
465 const struct variable *clabels_example;
466 struct hmap clabels_values_map;
467 struct ctables_value **clabels_values;
468 size_t n_clabels_values;
470 /* Indexed by variable dictionary index. */
471 struct ctables_categories **categories;
480 struct ctables_chisq *chisq;
481 struct ctables_pairwise *pairwise;
484 struct ctables_categories
487 struct ctables_category *cats;
492 struct ctables_category
494 enum ctables_category_type
496 /* Explicit category lists. */
499 CCT_NRANGE, /* Numerical range. */
500 CCT_SRANGE, /* String range. */
505 /* Totals and subtotals. */
509 /* Implicit category lists. */
514 /* For contributing to TOTALN. */
515 CCT_EXCLUDED_MISSING,
519 struct ctables_category *subtotal;
525 double number; /* CCT_NUMBER. */
526 struct substring string; /* CCT_STRING, in dictionary encoding. */
527 double nrange[2]; /* CCT_NRANGE. */
528 struct substring srange[2]; /* CCT_SRANGE. */
532 char *total_label; /* CCT_SUBTOTAL, CCT_TOTAL. */
533 bool hide_subcategories; /* CCT_SUBTOTAL. */
536 /* CCT_POSTCOMPUTE. */
539 const struct ctables_postcompute *pc;
540 enum fmt_type parse_format;
543 /* CCT_VALUE, CCT_LABEL, CCT_FUNCTION. */
546 bool include_missing;
550 enum ctables_summary_function sort_function;
551 enum ctables_weighting weighting;
552 enum ctables_area_type area;
553 struct variable *sort_var;
558 /* Source location. This is null for CCT_TOTAL, CCT_VALUE, CCT_LABEL,
559 CCT_FUNCTION, CCT_EXCLUDED_MISSING. */
560 struct msg_location *location;
564 ctables_category_uninit (struct ctables_category *cat)
569 msg_location_destroy (cat->location);
576 case CCT_POSTCOMPUTE:
580 ss_dealloc (&cat->string);
584 ss_dealloc (&cat->srange[0]);
585 ss_dealloc (&cat->srange[1]);
590 free (cat->total_label);
598 case CCT_EXCLUDED_MISSING:
604 nullable_substring_equal (const struct substring *a,
605 const struct substring *b)
607 return !a->string ? !b->string : b->string && ss_equals (*a, *b);
611 ctables_category_equal (const struct ctables_category *a,
612 const struct ctables_category *b)
614 if (a->type != b->type)
620 return a->number == b->number;
623 return ss_equals (a->string, b->string);
626 return a->nrange[0] == b->nrange[0] && a->nrange[1] == b->nrange[1];
629 return (nullable_substring_equal (&a->srange[0], &b->srange[0])
630 && nullable_substring_equal (&a->srange[1], &b->srange[1]));
636 case CCT_POSTCOMPUTE:
637 return a->pc == b->pc;
641 return !strcmp (a->total_label, b->total_label);
646 return (a->include_missing == b->include_missing
647 && a->sort_ascending == b->sort_ascending
648 && a->sort_function == b->sort_function
649 && a->sort_var == b->sort_var
650 && a->percentile == b->percentile);
652 case CCT_EXCLUDED_MISSING:
660 ctables_categories_unref (struct ctables_categories *c)
665 assert (c->n_refs > 0);
669 for (size_t i = 0; i < c->n_cats; i++)
670 ctables_category_uninit (&c->cats[i]);
676 ctables_categories_equal (const struct ctables_categories *a,
677 const struct ctables_categories *b)
679 if (a->n_cats != b->n_cats || a->show_empty != b->show_empty)
682 for (size_t i = 0; i < a->n_cats; i++)
683 if (!ctables_category_equal (&a->cats[i], &b->cats[i]))
689 /* Chi-square test (SIGTEST). */
697 /* Pairwise comparison test (COMPARETEST). */
698 struct ctables_pairwise
700 enum { PROP, MEAN } type;
703 bool meansvariance_allcats;
705 enum { BONFERRONI = 1, BH } adjust;
729 struct variable *var;
731 struct ctables_summary_spec_set specs[N_CSVS];
735 struct ctables_axis *subs[2];
738 struct msg_location *loc;
741 static void ctables_axis_destroy (struct ctables_axis *);
743 struct ctables_summary_spec
745 /* The calculation to be performed.
747 'function' is the function to calculate. 'weighted' specifies whether
748 to use weighted or unweighted data (for functions that do not support a
749 choice, it must be true). 'calc_area' is the area over which the
750 calculation takes place (for functions that target only an individual
751 cell, it must be 0). For CTSF_PTILE only, 'percentile' is the
752 percentile between 0 and 100 (for other functions it must be 0). */
753 enum ctables_summary_function function;
754 enum ctables_weighting weighting;
755 enum ctables_area_type calc_area;
756 double percentile; /* CTSF_PTILE only. */
758 /* How to display the result of the calculation.
760 'label' is a user-specified label, NULL if the user didn't specify
763 'user_area' is usually the same as 'calc_area', but when category labels
764 are rotated from one axis to another it swaps rows and columns.
766 'format' is the format for displaying the output. If
767 'is_ctables_format' is true, then 'format.type' is one of the special
768 CTEF_* formats instead of the standard ones. */
770 enum ctables_area_type user_area;
771 struct fmt_spec format;
772 bool is_ctables_format; /* Is 'format' one of CTEF_*? */
779 ctables_summary_spec_clone (struct ctables_summary_spec *dst,
780 const struct ctables_summary_spec *src)
783 dst->label = xstrdup_if_nonnull (src->label);
787 ctables_summary_spec_uninit (struct ctables_summary_spec *s)
794 ctables_summary_spec_set_clone (struct ctables_summary_spec_set *dst,
795 const struct ctables_summary_spec_set *src)
797 struct ctables_summary_spec *specs
798 = (src->n ? xnmalloc (src->n, sizeof *specs) : NULL);
799 for (size_t i = 0; i < src->n; i++)
800 ctables_summary_spec_clone (&specs[i], &src->specs[i]);
802 *dst = (struct ctables_summary_spec_set) {
807 .is_scale = src->is_scale,
812 ctables_summary_spec_set_uninit (struct ctables_summary_spec_set *set)
814 for (size_t i = 0; i < set->n; i++)
815 ctables_summary_spec_uninit (&set->specs[i]);
816 free (set->listwise_vars);
821 parse_col_width (struct lexer *lexer, const char *name, double *width)
823 lex_match (lexer, T_EQUALS);
824 if (lex_match_id (lexer, "DEFAULT"))
826 else if (lex_force_num_range_closed (lexer, name, 0, DBL_MAX))
828 *width = lex_number (lexer);
838 parse_bool (struct lexer *lexer, bool *b)
840 if (lex_match_id (lexer, "NO"))
842 else if (lex_match_id (lexer, "YES"))
846 lex_error_expecting (lexer, "YES", "NO");
852 static enum ctables_function_availability
853 ctables_function_availability (enum ctables_summary_function f)
855 static enum ctables_function_availability availability[] = {
856 #define S(ENUM, NAME, TYPE, FORMAT, AVAILABILITY) [ENUM] = AVAILABILITY,
857 #include "ctables.inc"
861 return availability[f];
865 parse_ctables_summary_function (struct lexer *lexer,
866 enum ctables_summary_function *function,
867 enum ctables_weighting *weighting,
868 enum ctables_area_type *area)
870 if (!lex_force_id (lexer))
873 struct substring name = lex_tokss (lexer);
874 if (ss_ends_with_case (name, ss_cstr (".LCL"))
875 || ss_ends_with_case (name, ss_cstr (".UCL"))
876 || ss_ends_with_case (name, ss_cstr (".SE")))
878 lex_error (lexer, _("Support for LCL, UCL, and SE summary functions "
879 "is not yet implemented."));
883 bool u = ss_match_byte (&name, 'U') || ss_match_byte (&name, 'u');
884 bool e = !u && (ss_match_byte (&name, 'E') || ss_match_byte (&name, 'e'));
886 bool has_area = false;
888 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
889 if (ss_match_string_case (&name, ss_cstr (ctables_area_type_name[at])))
894 if (ss_equals_case (name, ss_cstr ("PCT")))
896 /* Special case where .COUNT suffix is omitted. */
897 *function = CTSF_areaPCT_COUNT;
898 *weighting = CTW_EFFECTIVE;
905 for (int f = 0; f < N_CTSF_FUNCTIONS; f++)
907 const struct ctables_function_info *cfi = &ctables_function_info[f];
908 if (ss_equals_case (cfi->basename, name))
911 if ((u && !cfi->u_prefix) || (e && !cfi->e_prefix) || (has_area != cfi->is_area))
914 *weighting = (e ? CTW_EFFECTIVE
916 : cfi->e_prefix ? CTW_DICTIONARY
923 lex_error (lexer, _("Expecting summary function name."));
928 ctables_axis_destroy (struct ctables_axis *axis)
936 for (size_t i = 0; i < N_CSVS; i++)
937 ctables_summary_spec_set_uninit (&axis->specs[i]);
942 ctables_axis_destroy (axis->subs[0]);
943 ctables_axis_destroy (axis->subs[1]);
946 msg_location_destroy (axis->loc);
950 static struct ctables_axis *
951 ctables_axis_new_nonterminal (enum ctables_axis_op op,
952 struct ctables_axis *sub0,
953 struct ctables_axis *sub1,
954 struct lexer *lexer, int start_ofs)
956 struct ctables_axis *axis = xmalloc (sizeof *axis);
957 *axis = (struct ctables_axis) {
959 .subs = { sub0, sub1 },
960 .loc = lex_ofs_location (lexer, start_ofs, lex_ofs (lexer) - 1),
965 struct ctables_axis_parse_ctx
968 struct dictionary *dict;
970 struct ctables_table *t;
973 static struct fmt_spec
974 ctables_summary_default_format (enum ctables_summary_function function,
975 const struct variable *var)
977 static const enum ctables_format default_formats[] = {
978 #define S(ENUM, NAME, TYPE, FORMAT, AVAILABILITY) [ENUM] = FORMAT,
979 #include "ctables.inc"
982 switch (default_formats[function])
985 return (struct fmt_spec) { .type = FMT_F, .w = 40 };
988 return (struct fmt_spec) { .type = FMT_PCT, .w = 40, .d = 1 };
991 return *var_get_print_format (var);
999 ctables_summary_label__ (const struct ctables_summary_spec *spec)
1001 bool w = spec->weighting != CTW_UNWEIGHTED;
1002 bool d = spec->weighting == CTW_DICTIONARY;
1003 enum ctables_area_type a = spec->user_area;
1004 switch (spec->function)
1007 return (d ? N_("Count")
1008 : w ? N_("Adjusted Count")
1009 : N_("Unweighted Count"));
1011 case CTSF_areaPCT_COUNT:
1014 case CTAT_TABLE: return w ? N_("Table %") : N_("Unweighted Table %");
1015 case CTAT_LAYER: return w ? N_("Layer %") : N_("Unweighted Layer %");
1016 case CTAT_LAYERROW: return w ? N_("Layer Row %") : N_("Unweighted Layer Row %");
1017 case CTAT_LAYERCOL: return w ? N_("Layer Column %") : N_("Unweighted Layer Column %");
1018 case CTAT_SUBTABLE: return w ? N_("Subtable %") : N_("Unweighted Subtable %");
1019 case CTAT_ROW: return w ? N_("Row %") : N_("Unweighted Row %");
1020 case CTAT_COL: return w ? N_("Column %") : N_("Unweighted Column %");
1024 case CTSF_areaPCT_VALIDN:
1027 case CTAT_TABLE: return w ? N_("Table Valid N %") : N_("Unweighted Table Valid N %");
1028 case CTAT_LAYER: return w ? N_("Layer Valid N %") : N_("Unweighted Layer Valid N %");
1029 case CTAT_LAYERROW: return w ? N_("Layer Row Valid N %") : N_("Unweighted Layer Row Valid N %");
1030 case CTAT_LAYERCOL: return w ? N_("Layer Column Valid N %") : N_("Unweighted Layer Column Valid N %");
1031 case CTAT_SUBTABLE: return w ? N_("Subtable Valid N %") : N_("Unweighted Subtable Valid N %");
1032 case CTAT_ROW: return w ? N_("Row Valid N %") : N_("Unweighted Row Valid N %");
1033 case CTAT_COL: return w ? N_("Column Valid N %") : N_("Unweighted Column Valid N %");
1037 case CTSF_areaPCT_TOTALN:
1040 case CTAT_TABLE: return w ? N_("Table Total N %") : N_("Unweighted Table Total N %");
1041 case CTAT_LAYER: return w ? N_("Layer Total N %") : N_("Unweighted Layer Total N %");
1042 case CTAT_LAYERROW: return w ? N_("Layer Row Total N %") : N_("Unweighted Layer Row Total N %");
1043 case CTAT_LAYERCOL: return w ? N_("Layer Column Total N %") : N_("Unweighted Layer Column Total N %");
1044 case CTAT_SUBTABLE: return w ? N_("Subtable Total N %") : N_("Unweighted Subtable Total N %");
1045 case CTAT_ROW: return w ? N_("Row Total N %") : N_("Unweighted Row Total N %");
1046 case CTAT_COL: return w ? N_("Column Total N %") : N_("Unweighted Column Total N %");
1050 case CTSF_MAXIMUM: return N_("Maximum");
1051 case CTSF_MEAN: return w ? N_("Mean") : N_("Unweighted Mean");
1052 case CTSF_MEDIAN: return w ? N_("Median") : N_("Unweighted Median");
1053 case CTSF_MINIMUM: return N_("Minimum");
1054 case CTSF_MISSING: return w ? N_("Missing") : N_("Unweighted Missing");
1055 case CTSF_MODE: return w ? N_("Mode") : N_("Unweighted Mode");
1056 case CTSF_PTILE: NOT_REACHED ();
1057 case CTSF_RANGE: return N_("Range");
1058 case CTSF_SEMEAN: return w ? N_("Std Error of Mean") : N_("Unweighted Std Error of Mean");
1059 case CTSF_STDDEV: return w ? N_("Std Deviation") : N_("Unweighted Std Deviation");
1060 case CTSF_SUM: return w ? N_("Sum") : N_("Unweighted Sum");
1061 case CTSF_TOTALN: return (d ? N_("Total N")
1062 : w ? N_("Adjusted Total N")
1063 : N_("Unweighted Total N"));
1064 case CTSF_VALIDN: return (d ? N_("Valid N")
1065 : w ? N_("Adjusted Valid N")
1066 : N_("Unweighted Valid N"));
1067 case CTSF_VARIANCE: return w ? N_("Variance") : N_("Unweighted Variance");
1068 case CTSF_areaPCT_SUM:
1071 case CTAT_TABLE: return w ? N_("Table Sum %") : N_("Unweighted Table Sum %");
1072 case CTAT_LAYER: return w ? N_("Layer Sum %") : N_("Unweighted Layer Sum %");
1073 case CTAT_LAYERROW: return w ? N_("Layer Row Sum %") : N_("Unweighted Layer Row Sum %");
1074 case CTAT_LAYERCOL: return w ? N_("Layer Column Sum %") : N_("Unweighted Layer Column Sum %");
1075 case CTAT_SUBTABLE: return w ? N_("Subtable Sum %") : N_("Unweighted Subtable Sum %");
1076 case CTAT_ROW: return w ? N_("Row Sum %") : N_("Unweighted Row Sum %");
1077 case CTAT_COL: return w ? N_("Column Sum %") : N_("Unweighted Column Sum %");
1084 /* Don't bother translating these: they are for developers only. */
1085 case CTAT_TABLE: return "Table ID";
1086 case CTAT_LAYER: return "Layer ID";
1087 case CTAT_LAYERROW: return "Layer Row ID";
1088 case CTAT_LAYERCOL: return "Layer Column ID";
1089 case CTAT_SUBTABLE: return "Subtable ID";
1090 case CTAT_ROW: return "Row ID";
1091 case CTAT_COL: return "Column ID";
1099 static struct pivot_value *
1100 ctables_summary_label (const struct ctables_summary_spec *spec, double cilevel)
1104 if (spec->function == CTSF_PTILE)
1106 double p = spec->percentile;
1107 char *s = (spec->weighting != CTW_UNWEIGHTED
1108 ? xasprintf (_("Percentile %.2f"), p)
1109 : xasprintf (_("Unweighted Percentile %.2f"), p));
1110 return pivot_value_new_user_text_nocopy (s);
1113 return pivot_value_new_text (ctables_summary_label__ (spec));
1117 struct substring in = ss_cstr (spec->label);
1118 struct substring target = ss_cstr (")CILEVEL");
1120 struct string out = DS_EMPTY_INITIALIZER;
1123 size_t chunk = ss_find_substring (in, target);
1124 ds_put_substring (&out, ss_head (in, chunk));
1125 ss_advance (&in, chunk);
1127 return pivot_value_new_user_text_nocopy (ds_steal_cstr (&out));
1129 ss_advance (&in, target.length);
1130 ds_put_format (&out, "%g", cilevel);
1136 ctables_summary_function_name (enum ctables_summary_function function,
1137 enum ctables_weighting weighting,
1138 enum ctables_area_type area,
1139 char *buffer, size_t bufsize)
1141 const struct ctables_function_info *cfi = &ctables_function_info[function];
1142 snprintf (buffer, bufsize, "%s%s%s",
1143 (weighting == CTW_UNWEIGHTED ? "U"
1144 : weighting == CTW_DICTIONARY ? ""
1145 : cfi->e_prefix ? "E"
1147 cfi->is_area ? ctables_area_type_name[area] : "",
1148 cfi->basename.string);
1153 add_summary_spec (struct ctables_axis *axis,
1154 enum ctables_summary_function function,
1155 enum ctables_weighting weighting,
1156 enum ctables_area_type area, double percentile,
1157 const char *label, const struct fmt_spec *format,
1158 bool is_ctables_format, const struct msg_location *loc,
1159 enum ctables_summary_variant sv)
1161 if (axis->op == CTAO_VAR)
1163 char function_name[128];
1164 ctables_summary_function_name (function, weighting, area,
1165 function_name, sizeof function_name);
1166 const char *var_name = var_get_name (axis->var);
1167 switch (ctables_function_availability (function))
1171 msg_at (SE, loc, _("Summary function %s applies only to multiple "
1172 "response sets."), function_name);
1173 msg_at (SN, axis->loc, _("'%s' is not a multiple response set."),
1179 if (!axis->scale && sv != CSV_TOTAL)
1182 _("Summary function %s applies only to scale variables."),
1184 msg_at (SN, axis->loc, _("'%s' is not a scale variable."),
1194 struct ctables_summary_spec_set *set = &axis->specs[sv];
1195 if (set->n >= set->allocated)
1196 set->specs = x2nrealloc (set->specs, &set->allocated,
1197 sizeof *set->specs);
1199 struct ctables_summary_spec *dst = &set->specs[set->n++];
1200 *dst = (struct ctables_summary_spec) {
1201 .function = function,
1202 .weighting = weighting,
1205 .percentile = percentile,
1206 .label = xstrdup_if_nonnull (label),
1207 .format = (format ? *format
1208 : ctables_summary_default_format (function, axis->var)),
1209 .is_ctables_format = is_ctables_format,
1215 for (size_t i = 0; i < 2; i++)
1216 if (!add_summary_spec (axis->subs[i], function, weighting, area,
1217 percentile, label, format, is_ctables_format,
1224 static struct ctables_axis *ctables_axis_parse_stack (
1225 struct ctables_axis_parse_ctx *);
1227 static struct ctables_axis *
1228 ctables_axis_parse_primary (struct ctables_axis_parse_ctx *ctx)
1230 if (lex_match (ctx->lexer, T_LPAREN))
1232 struct ctables_axis *sub = ctables_axis_parse_stack (ctx);
1233 if (!sub || !lex_force_match (ctx->lexer, T_RPAREN))
1235 ctables_axis_destroy (sub);
1241 if (!lex_force_id (ctx->lexer))
1244 if (lex_tokcstr (ctx->lexer)[0] == '$')
1246 lex_error (ctx->lexer,
1247 _("Multiple response set support not implemented."));
1251 int start_ofs = lex_ofs (ctx->lexer);
1252 struct variable *var = parse_variable (ctx->lexer, ctx->dict);
1256 struct ctables_axis *axis = xmalloc (sizeof *axis);
1257 *axis = (struct ctables_axis) { .op = CTAO_VAR, .var = var };
1259 axis->scale = (lex_match_phrase (ctx->lexer, "[S]") ? true
1260 : lex_match_phrase (ctx->lexer, "[C]") ? false
1261 : var_get_measure (var) == MEASURE_SCALE);
1262 axis->loc = lex_ofs_location (ctx->lexer, start_ofs,
1263 lex_ofs (ctx->lexer) - 1);
1264 if (axis->scale && var_is_alpha (var))
1266 msg_at (SE, axis->loc, _("Cannot use string variable %s as a scale "
1268 var_get_name (var));
1269 ctables_axis_destroy (axis);
1277 has_digit (const char *s)
1279 return s[strcspn (s, "0123456789")] != '\0';
1283 parse_ctables_format_specifier (struct lexer *lexer, struct fmt_spec *format,
1284 bool *is_ctables_format)
1286 char type[FMT_TYPE_LEN_MAX + 1];
1287 if (!parse_abstract_format_specifier__ (lexer, type, &format->w, &format->d))
1290 if (!strcasecmp (type, "NEGPAREN"))
1291 format->type = CTEF_NEGPAREN;
1292 else if (!strcasecmp (type, "NEQUAL"))
1293 format->type = CTEF_NEQUAL;
1294 else if (!strcasecmp (type, "PAREN"))
1295 format->type = CTEF_PAREN;
1296 else if (!strcasecmp (type, "PCTPAREN"))
1297 format->type = CTEF_PCTPAREN;
1300 *is_ctables_format = false;
1301 return (parse_format_specifier (lexer, format)
1302 && fmt_check_output (format)
1303 && fmt_check_type_compat (format, VAL_NUMERIC));
1309 lex_next_error (lexer, -1, -1,
1310 _("Output format %s requires width 2 or greater."), type);
1313 else if (format->d > format->w - 1)
1315 lex_next_error (lexer, -1, -1, _("Output format %s requires width "
1316 "greater than decimals."), type);
1321 *is_ctables_format = true;
1326 static struct ctables_axis *
1327 ctables_axis_parse_postfix (struct ctables_axis_parse_ctx *ctx)
1329 struct ctables_axis *sub = ctables_axis_parse_primary (ctx);
1330 if (!sub || !lex_match (ctx->lexer, T_LBRACK))
1333 enum ctables_summary_variant sv = CSV_CELL;
1336 int start_ofs = lex_ofs (ctx->lexer);
1338 /* Parse function. */
1339 enum ctables_summary_function function;
1340 enum ctables_weighting weighting;
1341 enum ctables_area_type area;
1342 if (!parse_ctables_summary_function (ctx->lexer, &function, &weighting,
1346 /* Parse percentile. */
1347 double percentile = 0;
1348 if (function == CTSF_PTILE)
1350 if (!lex_force_num_range_closed (ctx->lexer, "PTILE", 0, 100))
1352 percentile = lex_number (ctx->lexer);
1353 lex_get (ctx->lexer);
1358 if (lex_is_string (ctx->lexer))
1360 label = ss_xstrdup (lex_tokss (ctx->lexer));
1361 lex_get (ctx->lexer);
1365 struct fmt_spec format;
1366 const struct fmt_spec *formatp;
1367 bool is_ctables_format = false;
1368 if (lex_token (ctx->lexer) == T_ID
1369 && has_digit (lex_tokcstr (ctx->lexer)))
1371 if (!parse_ctables_format_specifier (ctx->lexer, &format,
1372 &is_ctables_format))
1382 struct msg_location *loc = lex_ofs_location (ctx->lexer, start_ofs,
1383 lex_ofs (ctx->lexer) - 1);
1384 add_summary_spec (sub, function, weighting, area, percentile, label,
1385 formatp, is_ctables_format, loc, sv);
1387 msg_location_destroy (loc);
1389 lex_match (ctx->lexer, T_COMMA);
1390 if (sv == CSV_CELL && lex_match_id (ctx->lexer, "TOTALS"))
1392 if (!lex_force_match (ctx->lexer, T_LBRACK))
1396 else if (lex_match (ctx->lexer, T_RBRACK))
1398 if (sv == CSV_TOTAL && !lex_force_match (ctx->lexer, T_RBRACK))
1405 ctables_axis_destroy (sub);
1409 static const struct ctables_axis *
1410 find_scale (const struct ctables_axis *axis)
1414 else if (axis->op == CTAO_VAR)
1415 return axis->scale ? axis : NULL;
1418 for (size_t i = 0; i < 2; i++)
1420 const struct ctables_axis *scale = find_scale (axis->subs[i]);
1428 static const struct ctables_axis *
1429 find_categorical_summary_spec (const struct ctables_axis *axis)
1433 else if (axis->op == CTAO_VAR)
1434 return !axis->scale && axis->specs[CSV_CELL].n ? axis : NULL;
1437 for (size_t i = 0; i < 2; i++)
1439 const struct ctables_axis *sum
1440 = find_categorical_summary_spec (axis->subs[i]);
1448 static struct ctables_axis *
1449 ctables_axis_parse_nest (struct ctables_axis_parse_ctx *ctx)
1451 int start_ofs = lex_ofs (ctx->lexer);
1452 struct ctables_axis *lhs = ctables_axis_parse_postfix (ctx);
1456 while (lex_match (ctx->lexer, T_GT))
1458 struct ctables_axis *rhs = ctables_axis_parse_postfix (ctx);
1461 ctables_axis_destroy (lhs);
1465 struct ctables_axis *nest = ctables_axis_new_nonterminal (
1466 CTAO_NEST, lhs, rhs, ctx->lexer, start_ofs);
1468 const struct ctables_axis *outer_scale = find_scale (lhs);
1469 const struct ctables_axis *inner_scale = find_scale (rhs);
1470 if (outer_scale && inner_scale)
1472 msg_at (SE, nest->loc, _("Cannot nest scale variables."));
1473 msg_at (SN, outer_scale->loc, _("This is an outer scale variable."));
1474 msg_at (SN, inner_scale->loc, _("This is an inner scale variable."));
1475 ctables_axis_destroy (nest);
1479 const struct ctables_axis *outer_sum = find_categorical_summary_spec (lhs);
1482 msg_at (SE, nest->loc,
1483 _("Summaries may only be requested for categorical variables "
1484 "at the innermost nesting level."));
1485 msg_at (SN, outer_sum->loc,
1486 _("This outer categorical variable has a summary."));
1487 ctables_axis_destroy (nest);
1497 static struct ctables_axis *
1498 ctables_axis_parse_stack (struct ctables_axis_parse_ctx *ctx)
1500 int start_ofs = lex_ofs (ctx->lexer);
1501 struct ctables_axis *lhs = ctables_axis_parse_nest (ctx);
1505 while (lex_match (ctx->lexer, T_PLUS))
1507 struct ctables_axis *rhs = ctables_axis_parse_nest (ctx);
1510 ctables_axis_destroy (lhs);
1514 lhs = ctables_axis_new_nonterminal (CTAO_STACK, lhs, rhs,
1515 ctx->lexer, start_ofs);
1522 ctables_axis_parse (struct lexer *lexer, struct dictionary *dict,
1523 struct ctables *ct, struct ctables_table *t,
1524 enum pivot_axis_type a)
1526 if (lex_token (lexer) == T_BY
1527 || lex_token (lexer) == T_SLASH
1528 || lex_token (lexer) == T_ENDCMD)
1531 struct ctables_axis_parse_ctx ctx = {
1537 t->axes[a] = ctables_axis_parse_stack (&ctx);
1538 return t->axes[a] != NULL;
1542 ctables_chisq_destroy (struct ctables_chisq *chisq)
1548 ctables_pairwise_destroy (struct ctables_pairwise *pairwise)
1554 ctables_table_destroy (struct ctables_table *t)
1559 for (size_t i = 0; i < t->n_sections; i++)
1560 ctables_section_uninit (&t->sections[i]);
1563 for (size_t i = 0; i < t->n_categories; i++)
1564 ctables_categories_unref (t->categories[i]);
1565 free (t->categories);
1567 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
1569 ctables_axis_destroy (t->axes[a]);
1570 ctables_stack_uninit (&t->stacks[a]);
1572 free (t->summary_specs.specs);
1574 struct ctables_value *ctv, *next_ctv;
1575 HMAP_FOR_EACH_SAFE (ctv, next_ctv, struct ctables_value, node,
1576 &t->clabels_values_map)
1578 value_destroy (&ctv->value, var_get_width (t->clabels_example));
1579 hmap_delete (&t->clabels_values_map, &ctv->node);
1582 hmap_destroy (&t->clabels_values_map);
1583 free (t->clabels_values);
1589 ctables_chisq_destroy (t->chisq);
1590 ctables_pairwise_destroy (t->pairwise);
1595 ctables_destroy (struct ctables *ct)
1600 struct ctables_postcompute *pc, *next_pc;
1601 HMAP_FOR_EACH_SAFE (pc, next_pc, struct ctables_postcompute, hmap_node,
1605 msg_location_destroy (pc->location);
1606 ctables_pcexpr_destroy (pc->expr);
1610 ctables_summary_spec_set_uninit (pc->specs);
1613 hmap_delete (&ct->postcomputes, &pc->hmap_node);
1616 hmap_destroy (&ct->postcomputes);
1618 fmt_settings_uninit (&ct->ctables_formats);
1619 pivot_table_look_unref (ct->look);
1623 for (size_t i = 0; i < ct->n_tables; i++)
1624 ctables_table_destroy (ct->tables[i]);
1629 static struct ctables_category
1630 cct_nrange (double low, double high)
1632 return (struct ctables_category) {
1634 .nrange = { low, high }
1638 static struct ctables_category
1639 cct_srange (struct substring low, struct substring high)
1641 return (struct ctables_category) {
1643 .srange = { low, high }
1648 ctables_table_parse_subtotal (struct lexer *lexer, bool hide_subcategories,
1649 struct ctables_category *cat)
1652 if (lex_match (lexer, T_EQUALS))
1654 if (!lex_force_string (lexer))
1657 total_label = ss_xstrdup (lex_tokss (lexer));
1661 total_label = xstrdup (_("Subtotal"));
1663 *cat = (struct ctables_category) {
1664 .type = CCT_SUBTOTAL,
1665 .hide_subcategories = hide_subcategories,
1666 .total_label = total_label
1671 static struct substring
1672 parse_substring (struct lexer *lexer, struct dictionary *dict)
1674 struct substring s = recode_substring_pool (
1675 dict_get_encoding (dict), "UTF-8", lex_tokss (lexer), NULL);
1676 ss_rtrim (&s, ss_cstr (" "));
1682 ctables_table_parse_explicit_category (struct lexer *lexer,
1683 struct dictionary *dict,
1685 struct ctables_category *cat)
1687 if (lex_match_id (lexer, "OTHERNM"))
1688 *cat = (struct ctables_category) { .type = CCT_OTHERNM };
1689 else if (lex_match_id (lexer, "MISSING"))
1690 *cat = (struct ctables_category) { .type = CCT_MISSING };
1691 else if (lex_match_id (lexer, "SUBTOTAL"))
1692 return ctables_table_parse_subtotal (lexer, false, cat);
1693 else if (lex_match_id (lexer, "HSUBTOTAL"))
1694 return ctables_table_parse_subtotal (lexer, true, cat);
1695 else if (lex_match_id (lexer, "LO"))
1697 if (!lex_force_match_id (lexer, "THRU"))
1699 if (lex_is_string (lexer))
1701 struct substring sr0 = { .string = NULL };
1702 struct substring sr1 = parse_substring (lexer, dict);
1703 *cat = cct_srange (sr0, sr1);
1705 else if (lex_force_num (lexer))
1707 *cat = cct_nrange (-DBL_MAX, lex_number (lexer));
1713 else if (lex_is_number (lexer))
1715 double number = lex_number (lexer);
1717 if (lex_match_id (lexer, "THRU"))
1719 if (lex_match_id (lexer, "HI"))
1720 *cat = cct_nrange (number, DBL_MAX);
1723 if (!lex_force_num (lexer))
1725 *cat = cct_nrange (number, lex_number (lexer));
1730 *cat = (struct ctables_category) {
1735 else if (lex_is_string (lexer))
1737 struct substring s = parse_substring (lexer, dict);
1738 if (lex_match_id (lexer, "THRU"))
1740 if (lex_match_id (lexer, "HI"))
1742 struct substring sr1 = { .string = NULL };
1743 *cat = cct_srange (s, sr1);
1747 if (!lex_force_string (lexer))
1752 struct substring sr1 = parse_substring (lexer, dict);
1753 *cat = cct_srange (s, sr1);
1757 *cat = (struct ctables_category) { .type = CCT_STRING, .string = s };
1759 else if (lex_match (lexer, T_AND))
1761 if (!lex_force_id (lexer))
1763 struct ctables_postcompute *pc = ctables_find_postcompute (
1764 ct, lex_tokcstr (lexer));
1767 struct msg_location *loc = lex_get_location (lexer, -1, 0);
1768 msg_at (SE, loc, _("Unknown postcompute &%s."),
1769 lex_tokcstr (lexer));
1770 msg_location_destroy (loc);
1775 *cat = (struct ctables_category) { .type = CCT_POSTCOMPUTE, .pc = pc };
1779 lex_error (lexer, NULL);
1787 parse_category_string (struct msg_location *location,
1788 struct substring s, const struct dictionary *dict,
1789 enum fmt_type format, double *n)
1792 char *error = data_in (s, dict_get_encoding (dict), format,
1793 settings_get_fmt_settings (), &v, 0, NULL);
1796 msg_at (SE, location,
1797 _("Failed to parse category specification as format %s: %s."),
1798 fmt_name (format), error);
1807 static struct ctables_category *
1808 ctables_find_category_for_postcompute__ (const struct ctables_categories *cats,
1809 const struct ctables_pcexpr *e)
1811 struct ctables_category *best = NULL;
1812 size_t n_subtotals = 0;
1813 for (size_t i = 0; i < cats->n_cats; i++)
1815 struct ctables_category *cat = &cats->cats[i];
1818 case CTPO_CAT_NUMBER:
1819 if (cat->type == CCT_NUMBER && cat->number == e->number)
1823 case CTPO_CAT_STRING:
1824 if (cat->type == CCT_STRING && ss_equals (cat->string, e->string))
1828 case CTPO_CAT_NRANGE:
1829 if (cat->type == CCT_NRANGE
1830 && cat->nrange[0] == e->nrange[0]
1831 && cat->nrange[1] == e->nrange[1])
1835 case CTPO_CAT_SRANGE:
1836 if (cat->type == CCT_SRANGE
1837 && nullable_substring_equal (&cat->srange[0], &e->srange[0])
1838 && nullable_substring_equal (&cat->srange[1], &e->srange[1]))
1842 case CTPO_CAT_MISSING:
1843 if (cat->type == CCT_MISSING)
1847 case CTPO_CAT_OTHERNM:
1848 if (cat->type == CCT_OTHERNM)
1852 case CTPO_CAT_SUBTOTAL:
1853 if (cat->type == CCT_SUBTOTAL)
1856 if (e->subtotal_index == n_subtotals)
1858 else if (e->subtotal_index == 0)
1863 case CTPO_CAT_TOTAL:
1864 if (cat->type == CCT_TOTAL)
1878 if (e->op == CTPO_CAT_SUBTOTAL && e->subtotal_index == 0 && n_subtotals > 1)
1883 static struct ctables_category *
1884 ctables_find_category_for_postcompute (const struct dictionary *dict,
1885 const struct ctables_categories *cats,
1886 enum fmt_type parse_format,
1887 const struct ctables_pcexpr *e)
1889 if (parse_format != FMT_F)
1891 if (e->op == CTPO_CAT_STRING)
1894 if (!parse_category_string (e->location, e->string, dict,
1895 parse_format, &number))
1898 struct ctables_pcexpr e2 = {
1899 .op = CTPO_CAT_NUMBER,
1901 .location = e->location,
1903 return ctables_find_category_for_postcompute__ (cats, &e2);
1905 else if (e->op == CTPO_CAT_SRANGE)
1908 if (!e->srange[0].string)
1909 nrange[0] = -DBL_MAX;
1910 else if (!parse_category_string (e->location, e->srange[0], dict,
1911 parse_format, &nrange[0]))
1914 if (!e->srange[1].string)
1915 nrange[1] = DBL_MAX;
1916 else if (!parse_category_string (e->location, e->srange[1], dict,
1917 parse_format, &nrange[1]))
1920 struct ctables_pcexpr e2 = {
1921 .op = CTPO_CAT_NRANGE,
1922 .nrange = { nrange[0], nrange[1] },
1923 .location = e->location,
1925 return ctables_find_category_for_postcompute__ (cats, &e2);
1928 return ctables_find_category_for_postcompute__ (cats, e);
1932 ctables_recursive_check_postcompute (struct dictionary *dict,
1933 const struct ctables_pcexpr *e,
1934 struct ctables_category *pc_cat,
1935 const struct ctables_categories *cats,
1936 const struct msg_location *cats_location)
1940 case CTPO_CAT_NUMBER:
1941 case CTPO_CAT_STRING:
1942 case CTPO_CAT_NRANGE:
1943 case CTPO_CAT_SRANGE:
1944 case CTPO_CAT_MISSING:
1945 case CTPO_CAT_OTHERNM:
1946 case CTPO_CAT_SUBTOTAL:
1947 case CTPO_CAT_TOTAL:
1949 struct ctables_category *cat = ctables_find_category_for_postcompute (
1950 dict, cats, pc_cat->parse_format, e);
1953 if (e->op == CTPO_CAT_SUBTOTAL && e->subtotal_index == 0)
1955 size_t n_subtotals = 0;
1956 for (size_t i = 0; i < cats->n_cats; i++)
1957 n_subtotals += cats->cats[i].type == CCT_SUBTOTAL;
1958 if (n_subtotals > 1)
1960 msg_at (SE, cats_location,
1961 ngettext ("These categories include %zu instance "
1962 "of SUBTOTAL or HSUBTOTAL, so references "
1963 "from computed categories must refer to "
1964 "subtotals by position, "
1965 "e.g. SUBTOTAL[1].",
1966 "These categories include %zu instances "
1967 "of SUBTOTAL or HSUBTOTAL, so references "
1968 "from computed categories must refer to "
1969 "subtotals by position, "
1970 "e.g. SUBTOTAL[1].",
1973 msg_at (SN, e->location,
1974 _("This is the reference that lacks a position."));
1979 msg_at (SE, pc_cat->location,
1980 _("Computed category &%s references a category not included "
1981 "in the category list."),
1983 msg_at (SN, e->location, _("This is the missing category."));
1984 if (e->op == CTPO_CAT_SUBTOTAL)
1985 msg_at (SN, cats_location,
1986 _("To fix the problem, add subtotals to the "
1987 "list of categories here."));
1988 else if (e->op == CTPO_CAT_TOTAL)
1989 msg (SN, _("To fix the problem, add TOTAL=YES to the variable's "
1990 "CATEGORIES specification."));
1992 msg_at (SN, cats_location,
1993 _("To fix the problem, add the missing category to the "
1994 "list of categories here."));
1997 if (pc_cat->pc->hide_source_cats)
2011 for (size_t i = 0; i < 2; i++)
2012 if (e->subs[i] && !ctables_recursive_check_postcompute (
2013 dict, e->subs[i], pc_cat, cats, cats_location))
2022 all_strings (struct variable **vars, size_t n_vars,
2023 const struct ctables_category *cat)
2025 for (size_t j = 0; j < n_vars; j++)
2026 if (var_is_numeric (vars[j]))
2028 msg_at (SE, cat->location,
2029 _("This category specification may be applied only to string "
2030 "variables, but this subcommand tries to apply it to "
2031 "numeric variable %s."),
2032 var_get_name (vars[j]));
2039 ctables_table_parse_categories (struct lexer *lexer, struct dictionary *dict,
2040 struct ctables *ct, struct ctables_table *t)
2042 if (!lex_match_id (lexer, "VARIABLES"))
2044 lex_match (lexer, T_EQUALS);
2046 struct variable **vars;
2048 if (!parse_variables (lexer, dict, &vars, &n_vars, PV_NO_SCRATCH))
2051 const struct fmt_spec *common_format = var_get_print_format (vars[0]);
2052 for (size_t i = 1; i < n_vars; i++)
2054 const struct fmt_spec *f = var_get_print_format (vars[i]);
2055 if (f->type != common_format->type)
2057 common_format = NULL;
2063 && (fmt_get_category (common_format->type)
2064 & (FMT_CAT_DATE | FMT_CAT_TIME | FMT_CAT_DATE_COMPONENT)));
2066 struct ctables_categories *c = xmalloc (sizeof *c);
2067 *c = (struct ctables_categories) { .n_refs = n_vars, .show_empty = true };
2068 for (size_t i = 0; i < n_vars; i++)
2070 struct ctables_categories **cp
2071 = &t->categories[var_get_dict_index (vars[i])];
2072 ctables_categories_unref (*cp);
2076 size_t allocated_cats = 0;
2077 int cats_start_ofs = -1;
2078 int cats_end_ofs = -1;
2079 if (lex_match (lexer, T_LBRACK))
2081 cats_start_ofs = lex_ofs (lexer);
2084 if (c->n_cats >= allocated_cats)
2085 c->cats = x2nrealloc (c->cats, &allocated_cats, sizeof *c->cats);
2087 int start_ofs = lex_ofs (lexer);
2088 struct ctables_category *cat = &c->cats[c->n_cats];
2089 if (!ctables_table_parse_explicit_category (lexer, dict, ct, cat))
2091 cat->location = lex_ofs_location (lexer, start_ofs, lex_ofs (lexer) - 1);
2094 lex_match (lexer, T_COMMA);
2096 while (!lex_match (lexer, T_RBRACK));
2097 cats_end_ofs = lex_ofs (lexer) - 1;
2100 struct ctables_category cat = {
2102 .include_missing = false,
2103 .sort_ascending = true,
2105 bool show_totals = false;
2106 char *total_label = NULL;
2107 bool totals_before = false;
2108 while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
2110 if (!c->n_cats && lex_match_id (lexer, "ORDER"))
2112 lex_match (lexer, T_EQUALS);
2113 if (lex_match_id (lexer, "A"))
2114 cat.sort_ascending = true;
2115 else if (lex_match_id (lexer, "D"))
2116 cat.sort_ascending = false;
2119 lex_error_expecting (lexer, "A", "D");
2123 else if (!c->n_cats && lex_match_id (lexer, "KEY"))
2125 int start_ofs = lex_ofs (lexer) - 1;
2126 lex_match (lexer, T_EQUALS);
2127 if (lex_match_id (lexer, "VALUE"))
2128 cat.type = CCT_VALUE;
2129 else if (lex_match_id (lexer, "LABEL"))
2130 cat.type = CCT_LABEL;
2133 cat.type = CCT_FUNCTION;
2134 if (!parse_ctables_summary_function (lexer, &cat.sort_function,
2135 &cat.weighting, &cat.area))
2138 if (lex_match (lexer, T_LPAREN))
2140 cat.sort_var = parse_variable (lexer, dict);
2144 if (cat.sort_function == CTSF_PTILE)
2146 lex_match (lexer, T_COMMA);
2147 if (!lex_force_num_range_closed (lexer, "PTILE", 0, 100))
2149 cat.percentile = lex_number (lexer);
2153 if (!lex_force_match (lexer, T_RPAREN))
2156 else if (ctables_function_availability (cat.sort_function)
2159 bool UNUSED b = lex_force_match (lexer, T_LPAREN);
2163 lex_ofs_error (lexer, start_ofs, lex_ofs (lexer) - 1,
2164 _("Data-dependent sorting is not implemented."));
2168 else if (!c->n_cats && lex_match_id (lexer, "MISSING"))
2170 lex_match (lexer, T_EQUALS);
2171 if (lex_match_id (lexer, "INCLUDE"))
2172 cat.include_missing = true;
2173 else if (lex_match_id (lexer, "EXCLUDE"))
2174 cat.include_missing = false;
2177 lex_error_expecting (lexer, "INCLUDE", "EXCLUDE");
2181 else if (lex_match_id (lexer, "TOTAL"))
2183 lex_match (lexer, T_EQUALS);
2184 if (!parse_bool (lexer, &show_totals))
2187 else if (lex_match_id (lexer, "LABEL"))
2189 lex_match (lexer, T_EQUALS);
2190 if (!lex_force_string (lexer))
2193 total_label = ss_xstrdup (lex_tokss (lexer));
2196 else if (lex_match_id (lexer, "POSITION"))
2198 lex_match (lexer, T_EQUALS);
2199 if (lex_match_id (lexer, "BEFORE"))
2200 totals_before = true;
2201 else if (lex_match_id (lexer, "AFTER"))
2202 totals_before = false;
2205 lex_error_expecting (lexer, "BEFORE", "AFTER");
2209 else if (lex_match_id (lexer, "EMPTY"))
2211 lex_match (lexer, T_EQUALS);
2212 if (lex_match_id (lexer, "INCLUDE"))
2213 c->show_empty = true;
2214 else if (lex_match_id (lexer, "EXCLUDE"))
2215 c->show_empty = false;
2218 lex_error_expecting (lexer, "INCLUDE", "EXCLUDE");
2225 lex_error_expecting (lexer, "ORDER", "KEY", "MISSING",
2226 "TOTAL", "LABEL", "POSITION", "EMPTY");
2228 lex_error_expecting (lexer, "TOTAL", "LABEL", "POSITION", "EMPTY");
2235 if (c->n_cats >= allocated_cats)
2236 c->cats = x2nrealloc (c->cats, &allocated_cats, sizeof *c->cats);
2237 c->cats[c->n_cats++] = cat;
2242 if (c->n_cats >= allocated_cats)
2243 c->cats = x2nrealloc (c->cats, &allocated_cats, sizeof *c->cats);
2245 struct ctables_category *totals;
2248 insert_element (c->cats, c->n_cats, sizeof *c->cats, 0);
2249 totals = &c->cats[0];
2252 totals = &c->cats[c->n_cats];
2255 *totals = (struct ctables_category) {
2257 .total_label = total_label ? total_label : xstrdup (_("Total")),
2261 struct ctables_category *subtotal = NULL;
2262 for (size_t i = totals_before ? 0 : c->n_cats;
2263 totals_before ? i < c->n_cats : i-- > 0;
2264 totals_before ? i++ : 0)
2266 struct ctables_category *cat = &c->cats[i];
2275 cat->subtotal = subtotal;
2278 case CCT_POSTCOMPUTE:
2289 case CCT_EXCLUDED_MISSING:
2294 if (cats_start_ofs != -1)
2296 for (size_t i = 0; i < c->n_cats; i++)
2298 struct ctables_category *cat = &c->cats[i];
2301 case CCT_POSTCOMPUTE:
2302 cat->parse_format = parse_strings ? common_format->type : FMT_F;
2303 struct msg_location *cats_location
2304 = lex_ofs_location (lexer, cats_start_ofs, cats_end_ofs);
2305 bool ok = ctables_recursive_check_postcompute (
2306 dict, cat->pc->expr, cat, c, cats_location);
2307 msg_location_destroy (cats_location);
2314 for (size_t j = 0; j < n_vars; j++)
2315 if (var_is_alpha (vars[j]))
2317 msg_at (SE, cat->location,
2318 _("This category specification may be applied "
2319 "only to numeric variables, but this "
2320 "subcommand tries to apply it to string "
2322 var_get_name (vars[j]));
2331 if (!parse_category_string (cat->location, cat->string, dict,
2332 common_format->type, &n))
2335 ss_dealloc (&cat->string);
2337 cat->type = CCT_NUMBER;
2340 else if (!all_strings (vars, n_vars, cat))
2349 if (!cat->srange[0].string)
2351 else if (!parse_category_string (cat->location,
2352 cat->srange[0], dict,
2353 common_format->type, &n[0]))
2356 if (!cat->srange[1].string)
2358 else if (!parse_category_string (cat->location,
2359 cat->srange[1], dict,
2360 common_format->type, &n[1]))
2363 ss_dealloc (&cat->srange[0]);
2364 ss_dealloc (&cat->srange[1]);
2366 cat->type = CCT_NRANGE;
2367 cat->nrange[0] = n[0];
2368 cat->nrange[1] = n[1];
2370 else if (!all_strings (vars, n_vars, cat))
2381 case CCT_EXCLUDED_MISSING:
2396 ctables_nest_uninit (struct ctables_nest *nest)
2399 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
2400 ctables_summary_spec_set_uninit (&nest->specs[sv]);
2401 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
2402 free (nest->areas[at]);
2406 ctables_stack_uninit (struct ctables_stack *stack)
2410 for (size_t i = 0; i < stack->n; i++)
2411 ctables_nest_uninit (&stack->nests[i]);
2412 free (stack->nests);
2416 static struct ctables_stack
2417 nest_fts (struct ctables_stack s0, struct ctables_stack s1)
2424 struct ctables_stack stack = { .nests = xnmalloc (s0.n, s1.n * sizeof *stack.nests) };
2425 for (size_t i = 0; i < s0.n; i++)
2426 for (size_t j = 0; j < s1.n; j++)
2428 const struct ctables_nest *a = &s0.nests[i];
2429 const struct ctables_nest *b = &s1.nests[j];
2431 size_t allocate = a->n + b->n;
2432 struct variable **vars = xnmalloc (allocate, sizeof *vars);
2434 for (size_t k = 0; k < a->n; k++)
2435 vars[n++] = a->vars[k];
2436 for (size_t k = 0; k < b->n; k++)
2437 vars[n++] = b->vars[k];
2438 assert (n == allocate);
2440 const struct ctables_nest *summary_src;
2441 if (!a->specs[CSV_CELL].var)
2443 else if (!b->specs[CSV_CELL].var)
2448 struct ctables_nest *new = &stack.nests[stack.n++];
2449 *new = (struct ctables_nest) {
2451 .scale_idx = (a->scale_idx != SIZE_MAX ? a->scale_idx
2452 : b->scale_idx != SIZE_MAX ? a->n + b->scale_idx
2454 .summary_idx = (a->summary_idx != SIZE_MAX ? a->summary_idx
2455 : b->summary_idx != SIZE_MAX ? a->n + b->summary_idx
2459 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
2460 ctables_summary_spec_set_clone (&new->specs[sv], &summary_src->specs[sv]);
2462 ctables_stack_uninit (&s0);
2463 ctables_stack_uninit (&s1);
2467 static struct ctables_stack
2468 stack_fts (struct ctables_stack s0, struct ctables_stack s1)
2470 struct ctables_stack stack = { .nests = xnmalloc (s0.n + s1.n, sizeof *stack.nests) };
2471 for (size_t i = 0; i < s0.n; i++)
2472 stack.nests[stack.n++] = s0.nests[i];
2473 for (size_t i = 0; i < s1.n; i++)
2475 stack.nests[stack.n] = s1.nests[i];
2476 stack.nests[stack.n].group_head += s0.n;
2479 assert (stack.n == s0.n + s1.n);
2485 static struct ctables_stack
2486 var_fts (const struct ctables_axis *a)
2488 struct variable **vars = xmalloc (sizeof *vars);
2491 bool is_summary = a->specs[CSV_CELL].n || a->scale;
2492 struct ctables_nest *nest = xmalloc (sizeof *nest);
2493 *nest = (struct ctables_nest) {
2496 .scale_idx = a->scale ? 0 : SIZE_MAX,
2497 .summary_idx = is_summary ? 0 : SIZE_MAX,
2500 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
2502 ctables_summary_spec_set_clone (&nest->specs[sv], &a->specs[sv]);
2503 nest->specs[sv].var = a->var;
2504 nest->specs[sv].is_scale = a->scale;
2506 return (struct ctables_stack) { .nests = nest, .n = 1 };
2509 static struct ctables_stack
2510 enumerate_fts (enum pivot_axis_type axis_type, const struct ctables_axis *a)
2513 return (struct ctables_stack) { .n = 0 };
2521 return stack_fts (enumerate_fts (axis_type, a->subs[0]),
2522 enumerate_fts (axis_type, a->subs[1]));
2525 /* This should consider any of the scale variables found in the result to
2526 be linked to each other listwise for SMISSING=LISTWISE. */
2527 return nest_fts (enumerate_fts (axis_type, a->subs[0]),
2528 enumerate_fts (axis_type, a->subs[1]));
2534 union ctables_summary
2536 /* COUNT, VALIDN, TOTALN. */
2539 /* MINIMUM, MAXIMUM, RANGE. */
2546 /* MEAN, SEMEAN, STDDEV, SUM, VARIANCE, *.SUM. */
2547 struct moments1 *moments;
2549 /* MEDIAN, MODE, PTILE. */
2552 struct casewriter *writer;
2559 ctables_summary_init (union ctables_summary *s,
2560 const struct ctables_summary_spec *ss)
2562 switch (ss->function)
2565 case CTSF_areaPCT_COUNT:
2566 case CTSF_areaPCT_VALIDN:
2567 case CTSF_areaPCT_TOTALN:
2580 s->min = s->max = SYSMIS;
2585 case CTSF_areaPCT_SUM:
2586 s->moments = moments1_create (MOMENT_MEAN);
2592 s->moments = moments1_create (MOMENT_VARIANCE);
2599 struct caseproto *proto = caseproto_create ();
2600 proto = caseproto_add_width (proto, 0);
2601 proto = caseproto_add_width (proto, 0);
2603 struct subcase ordering;
2604 subcase_init (&ordering, 0, 0, SC_ASCEND);
2605 s->writer = sort_create_writer (&ordering, proto);
2606 subcase_uninit (&ordering);
2607 caseproto_unref (proto);
2617 ctables_summary_uninit (union ctables_summary *s,
2618 const struct ctables_summary_spec *ss)
2620 switch (ss->function)
2623 case CTSF_areaPCT_COUNT:
2624 case CTSF_areaPCT_VALIDN:
2625 case CTSF_areaPCT_TOTALN:
2644 case CTSF_areaPCT_SUM:
2645 moments1_destroy (s->moments);
2651 casewriter_destroy (s->writer);
2657 ctables_summary_add (union ctables_summary *s,
2658 const struct ctables_summary_spec *ss,
2659 const union value *value,
2660 bool is_scale, bool is_scale_missing,
2661 bool is_missing, bool is_included,
2664 /* To determine whether a case is included in a given table for a particular
2665 kind of summary, consider the following charts for the variable being
2666 summarized. Only if "yes" appears is the case counted.
2668 Categorical variables: VALIDN other TOTALN
2669 Valid values in included categories yes yes yes
2670 Missing values in included categories --- yes yes
2671 Missing values in excluded categories --- --- yes
2672 Valid values in excluded categories --- --- ---
2674 Scale variables: VALIDN other TOTALN
2675 Valid value yes yes yes
2676 Missing value --- yes yes
2678 Missing values include both user- and system-missing. (The system-missing
2679 value is always in an excluded category.)
2681 One way to interpret the above table is that scale variables are like
2682 categorical variables in which all values are in included categories.
2684 switch (ss->function)
2687 case CTSF_areaPCT_TOTALN:
2692 case CTSF_areaPCT_COUNT:
2698 case CTSF_areaPCT_VALIDN:
2718 if (!is_scale_missing)
2720 if (s->min == SYSMIS || value->f < s->min)
2722 if (s->max == SYSMIS || value->f > s->max)
2732 if (!is_scale_missing)
2733 moments1_add (s->moments, value->f, weight);
2736 case CTSF_areaPCT_SUM:
2737 if (!is_missing && !is_scale_missing)
2738 moments1_add (s->moments, value->f, weight);
2744 if (!is_scale_missing)
2746 s->ovalid += weight;
2748 struct ccase *c = case_create (casewriter_get_proto (s->writer));
2749 *case_num_rw_idx (c, 0) = value->f;
2750 *case_num_rw_idx (c, 1) = weight;
2751 casewriter_write (s->writer, c);
2758 ctables_summary_value (const struct ctables_cell *cell,
2759 union ctables_summary *s,
2760 const struct ctables_summary_spec *ss)
2762 switch (ss->function)
2768 return cell->areas[ss->calc_area]->sequence;
2770 case CTSF_areaPCT_COUNT:
2772 const struct ctables_area *a = cell->areas[ss->calc_area];
2773 double a_count = a->count[ss->weighting];
2774 return a_count ? s->count / a_count * 100 : SYSMIS;
2777 case CTSF_areaPCT_VALIDN:
2779 const struct ctables_area *a = cell->areas[ss->calc_area];
2780 double a_valid = a->valid[ss->weighting];
2781 return a_valid ? s->count / a_valid * 100 : SYSMIS;
2784 case CTSF_areaPCT_TOTALN:
2786 const struct ctables_area *a = cell->areas[ss->calc_area];
2787 double a_total = a->total[ss->weighting];
2788 return a_total ? s->count / a_total * 100 : SYSMIS;
2803 return s->max != SYSMIS && s->min != SYSMIS ? s->max - s->min : SYSMIS;
2808 moments1_calculate (s->moments, NULL, &mean, NULL, NULL, NULL);
2814 double weight, variance;
2815 moments1_calculate (s->moments, &weight, NULL, &variance, NULL, NULL);
2816 return calc_semean (variance, weight);
2822 moments1_calculate (s->moments, NULL, NULL, &variance, NULL, NULL);
2823 return variance != SYSMIS ? sqrt (variance) : SYSMIS;
2828 double weight, mean;
2829 moments1_calculate (s->moments, &weight, &mean, NULL, NULL, NULL);
2830 return weight != SYSMIS && mean != SYSMIS ? weight * mean : SYSMIS;
2836 moments1_calculate (s->moments, NULL, NULL, &variance, NULL, NULL);
2840 case CTSF_areaPCT_SUM:
2842 double weight, mean;
2843 moments1_calculate (s->moments, &weight, &mean, NULL, NULL, NULL);
2844 if (weight == SYSMIS || mean == SYSMIS)
2847 const struct ctables_area *a = cell->areas[ss->calc_area];
2848 const struct ctables_sum *sum = &a->sums[ss->sum_var_idx];
2849 double denom = sum->sum[ss->weighting];
2850 return denom != 0 ? weight * mean / denom * 100 : SYSMIS;
2857 struct casereader *reader = casewriter_make_reader (s->writer);
2860 struct percentile *ptile = percentile_create (
2861 ss->function == CTSF_PTILE ? ss->percentile : 0.5, s->ovalid);
2862 struct order_stats *os = &ptile->parent;
2863 order_stats_accumulate_idx (&os, 1, reader, 1, 0);
2864 s->ovalue = percentile_calculate (ptile, PC_HAVERAGE);
2865 statistic_destroy (&ptile->parent.parent);
2872 struct casereader *reader = casewriter_make_reader (s->writer);
2875 struct mode *mode = mode_create ();
2876 struct order_stats *os = &mode->parent;
2877 order_stats_accumulate_idx (&os, 1, reader, 1, 0);
2878 s->ovalue = mode->mode;
2879 statistic_destroy (&mode->parent.parent);
2887 struct ctables_cell_sort_aux
2889 const struct ctables_nest *nest;
2890 enum pivot_axis_type a;
2894 ctables_cell_compare_3way (const void *a_, const void *b_, const void *aux_)
2896 const struct ctables_cell_sort_aux *aux = aux_;
2897 struct ctables_cell *const *ap = a_;
2898 struct ctables_cell *const *bp = b_;
2899 const struct ctables_cell *a = *ap;
2900 const struct ctables_cell *b = *bp;
2902 const struct ctables_nest *nest = aux->nest;
2903 for (size_t i = 0; i < nest->n; i++)
2904 if (i != nest->scale_idx)
2906 const struct variable *var = nest->vars[i];
2907 const struct ctables_cell_value *a_cv = &a->axes[aux->a].cvs[i];
2908 const struct ctables_cell_value *b_cv = &b->axes[aux->a].cvs[i];
2909 if (a_cv->category != b_cv->category)
2910 return a_cv->category > b_cv->category ? 1 : -1;
2912 const union value *a_val = &a_cv->value;
2913 const union value *b_val = &b_cv->value;
2914 switch (a_cv->category->type)
2920 case CCT_POSTCOMPUTE:
2921 case CCT_EXCLUDED_MISSING:
2922 /* Must be equal. */
2930 int cmp = value_compare_3way (a_val, b_val, var_get_width (var));
2938 int cmp = value_compare_3way (a_val, b_val, var_get_width (var));
2940 return a_cv->category->sort_ascending ? cmp : -cmp;
2946 const char *a_label = var_lookup_value_label (var, a_val);
2947 const char *b_label = var_lookup_value_label (var, b_val);
2953 cmp = strcmp (a_label, b_label);
2959 cmp = value_compare_3way (a_val, b_val, var_get_width (var));
2962 return a_cv->category->sort_ascending ? cmp : -cmp;
2974 ctables_cell_compare_leaf_3way (const void *a_, const void *b_,
2975 const void *aux UNUSED)
2977 struct ctables_cell *const *ap = a_;
2978 struct ctables_cell *const *bp = b_;
2979 const struct ctables_cell *a = *ap;
2980 const struct ctables_cell *b = *bp;
2982 for (enum pivot_axis_type axis = 0; axis < PIVOT_N_AXES; axis++)
2984 int al = a->axes[axis].leaf;
2985 int bl = b->axes[axis].leaf;
2987 return al > bl ? 1 : -1;
2992 static struct ctables_area *
2993 ctables_area_insert (struct ctables_section *s, struct ctables_cell *cell,
2994 enum ctables_area_type area)
2997 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
2999 const struct ctables_nest *nest = s->nests[a];
3000 for (size_t i = 0; i < nest->n_areas[area]; i++)
3002 size_t v_idx = nest->areas[area][i];
3003 struct ctables_cell_value *cv = &cell->axes[a].cvs[v_idx];
3004 hash = hash_pointer (cv->category, hash);
3005 if (cv->category->type != CCT_TOTAL
3006 && cv->category->type != CCT_SUBTOTAL
3007 && cv->category->type != CCT_POSTCOMPUTE)
3008 hash = value_hash (&cv->value,
3009 var_get_width (nest->vars[v_idx]), hash);
3013 struct ctables_area *a;
3014 HMAP_FOR_EACH_WITH_HASH (a, struct ctables_area, node, hash, &s->areas[area])
3016 const struct ctables_cell *df = a->example;
3017 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3019 const struct ctables_nest *nest = s->nests[a];
3020 for (size_t i = 0; i < nest->n_areas[area]; i++)
3022 size_t v_idx = nest->areas[area][i];
3023 struct ctables_cell_value *cv1 = &df->axes[a].cvs[v_idx];
3024 struct ctables_cell_value *cv2 = &cell->axes[a].cvs[v_idx];
3025 if (cv1->category != cv2->category
3026 || (cv1->category->type != CCT_TOTAL
3027 && cv1->category->type != CCT_SUBTOTAL
3028 && cv1->category->type != CCT_POSTCOMPUTE
3029 && !value_equal (&cv1->value, &cv2->value,
3030 var_get_width (nest->vars[v_idx]))))
3039 struct ctables_sum *sums = (s->table->n_sum_vars
3040 ? xzalloc (s->table->n_sum_vars * sizeof *sums)
3043 a = xmalloc (sizeof *a);
3044 *a = (struct ctables_area) { .example = cell, .sums = sums };
3045 hmap_insert (&s->areas[area], &a->node, hash);
3049 static struct substring
3050 rtrim_value (const union value *v, const struct variable *var)
3052 struct substring s = ss_buffer (CHAR_CAST (char *, v->s),
3053 var_get_width (var));
3054 ss_rtrim (&s, ss_cstr (" "));
3059 in_string_range (const union value *v, const struct variable *var,
3060 const struct substring *srange)
3062 struct substring s = rtrim_value (v, var);
3063 return ((!srange[0].string || ss_compare (s, srange[0]) >= 0)
3064 && (!srange[1].string || ss_compare (s, srange[1]) <= 0));
3067 static const struct ctables_category *
3068 ctables_categories_match (const struct ctables_categories *c,
3069 const union value *v, const struct variable *var)
3071 if (var_is_numeric (var) && v->f == SYSMIS)
3074 const struct ctables_category *othernm = NULL;
3075 for (size_t i = c->n_cats; i-- > 0; )
3077 const struct ctables_category *cat = &c->cats[i];
3081 if (cat->number == v->f)
3086 if (ss_equals (cat->string, rtrim_value (v, var)))
3091 if ((cat->nrange[0] == -DBL_MAX || v->f >= cat->nrange[0])
3092 && (cat->nrange[1] == DBL_MAX || v->f <= cat->nrange[1]))
3097 if (in_string_range (v, var, cat->srange))
3102 if (var_is_value_missing (var, v))
3106 case CCT_POSTCOMPUTE:
3121 return (cat->include_missing || !var_is_value_missing (var, v) ? cat
3124 case CCT_EXCLUDED_MISSING:
3129 return var_is_value_missing (var, v) ? NULL : othernm;
3132 static const struct ctables_category *
3133 ctables_categories_total (const struct ctables_categories *c)
3135 const struct ctables_category *first = &c->cats[0];
3136 const struct ctables_category *last = &c->cats[c->n_cats - 1];
3137 return (first->type == CCT_TOTAL ? first
3138 : last->type == CCT_TOTAL ? last
3142 static struct ctables_cell *
3143 ctables_cell_insert__ (struct ctables_section *s, const struct ccase *c,
3144 const struct ctables_category **cats[PIVOT_N_AXES])
3147 enum ctables_summary_variant sv = CSV_CELL;
3148 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3150 const struct ctables_nest *nest = s->nests[a];
3151 for (size_t i = 0; i < nest->n; i++)
3152 if (i != nest->scale_idx)
3154 hash = hash_pointer (cats[a][i], hash);
3155 if (cats[a][i]->type != CCT_TOTAL
3156 && cats[a][i]->type != CCT_SUBTOTAL
3157 && cats[a][i]->type != CCT_POSTCOMPUTE)
3158 hash = value_hash (case_data (c, nest->vars[i]),
3159 var_get_width (nest->vars[i]), hash);
3165 struct ctables_cell *cell;
3166 HMAP_FOR_EACH_WITH_HASH (cell, struct ctables_cell, node, hash, &s->cells)
3168 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3170 const struct ctables_nest *nest = s->nests[a];
3171 for (size_t i = 0; i < nest->n; i++)
3172 if (i != nest->scale_idx
3173 && (cats[a][i] != cell->axes[a].cvs[i].category
3174 || (cats[a][i]->type != CCT_TOTAL
3175 && cats[a][i]->type != CCT_SUBTOTAL
3176 && cats[a][i]->type != CCT_POSTCOMPUTE
3177 && !value_equal (case_data (c, nest->vars[i]),
3178 &cell->axes[a].cvs[i].value,
3179 var_get_width (nest->vars[i])))))
3188 cell = xmalloc (sizeof *cell);
3191 cell->omit_areas = 0;
3192 cell->postcompute = false;
3193 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3195 const struct ctables_nest *nest = s->nests[a];
3196 cell->axes[a].cvs = (nest->n
3197 ? xnmalloc (nest->n, sizeof *cell->axes[a].cvs)
3199 for (size_t i = 0; i < nest->n; i++)
3201 const struct ctables_category *cat = cats[a][i];
3202 const struct variable *var = nest->vars[i];
3203 const union value *value = case_data (c, var);
3204 if (i != nest->scale_idx)
3206 const struct ctables_category *subtotal = cat->subtotal;
3207 if (cat->hide || (subtotal && subtotal->hide_subcategories))
3210 if (cat->type == CCT_TOTAL
3211 || cat->type == CCT_SUBTOTAL
3212 || cat->type == CCT_POSTCOMPUTE)
3216 case PIVOT_AXIS_COLUMN:
3217 cell->omit_areas |= ((1u << CTAT_TABLE) |
3218 (1u << CTAT_LAYER) |
3219 (1u << CTAT_LAYERCOL) |
3220 (1u << CTAT_SUBTABLE) |
3223 case PIVOT_AXIS_ROW:
3224 cell->omit_areas |= ((1u << CTAT_TABLE) |
3225 (1u << CTAT_LAYER) |
3226 (1u << CTAT_LAYERROW) |
3227 (1u << CTAT_SUBTABLE) |
3230 case PIVOT_AXIS_LAYER:
3231 cell->omit_areas |= ((1u << CTAT_TABLE) |
3232 (1u << CTAT_LAYER));
3236 if (cat->type == CCT_POSTCOMPUTE)
3237 cell->postcompute = true;
3240 cell->axes[a].cvs[i].category = cat;
3241 value_clone (&cell->axes[a].cvs[i].value, value, var_get_width (var));
3245 const struct ctables_nest *ss = s->nests[s->table->summary_axis];
3246 const struct ctables_summary_spec_set *specs = &ss->specs[cell->sv];
3247 cell->summaries = xmalloc (specs->n * sizeof *cell->summaries);
3248 for (size_t i = 0; i < specs->n; i++)
3249 ctables_summary_init (&cell->summaries[i], &specs->specs[i]);
3250 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
3251 cell->areas[at] = ctables_area_insert (s, cell, at);
3252 hmap_insert (&s->cells, &cell->node, hash);
3257 is_listwise_missing (const struct ctables_summary_spec_set *specs,
3258 const struct ccase *c)
3260 for (size_t i = 0; i < specs->n_listwise_vars; i++)
3262 const struct variable *var = specs->listwise_vars[i];
3263 if (var_is_num_missing (var, case_num (c, var)))
3271 add_weight (double dst[N_CTWS], const double src[N_CTWS])
3273 for (enum ctables_weighting wt = 0; wt < N_CTWS; wt++)
3278 ctables_cell_add__ (struct ctables_section *s, const struct ccase *c,
3279 const struct ctables_category **cats[PIVOT_N_AXES],
3280 bool is_included, double weight[N_CTWS])
3282 struct ctables_cell *cell = ctables_cell_insert__ (s, c, cats);
3283 const struct ctables_nest *ss = s->nests[s->table->summary_axis];
3285 const struct ctables_summary_spec_set *specs = &ss->specs[cell->sv];
3286 const union value *value = case_data (c, specs->var);
3287 bool is_missing = var_is_value_missing (specs->var, value);
3288 bool is_scale_missing
3289 = specs->is_scale && (is_missing || is_listwise_missing (specs, c));
3291 for (size_t i = 0; i < specs->n; i++)
3292 ctables_summary_add (&cell->summaries[i], &specs->specs[i], value,
3293 specs->is_scale, is_scale_missing, is_missing,
3294 is_included, weight[specs->specs[i].weighting]);
3295 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
3296 if (!(cell->omit_areas && (1u << at)))
3298 struct ctables_area *a = cell->areas[at];
3300 add_weight (a->total, weight);
3302 add_weight (a->count, weight);
3305 add_weight (a->valid, weight);
3307 if (!is_scale_missing)
3308 for (size_t i = 0; i < s->table->n_sum_vars; i++)
3310 const struct variable *var = s->table->sum_vars[i];
3311 double addend = case_num (c, var);
3312 if (!var_is_num_missing (var, addend))
3313 for (enum ctables_weighting wt = 0; wt < N_CTWS; wt++)
3314 a->sums[i].sum[wt] += addend * weight[wt];
3321 recurse_totals (struct ctables_section *s, const struct ccase *c,
3322 const struct ctables_category **cats[PIVOT_N_AXES],
3323 bool is_included, double weight[N_CTWS],
3324 enum pivot_axis_type start_axis, size_t start_nest)
3326 for (enum pivot_axis_type a = start_axis; a < PIVOT_N_AXES; a++)
3328 const struct ctables_nest *nest = s->nests[a];
3329 for (size_t i = start_nest; i < nest->n; i++)
3331 if (i == nest->scale_idx)
3334 const struct variable *var = nest->vars[i];
3336 const struct ctables_category *total = ctables_categories_total (
3337 s->table->categories[var_get_dict_index (var)]);
3340 const struct ctables_category *save = cats[a][i];
3342 ctables_cell_add__ (s, c, cats, is_included, weight);
3343 recurse_totals (s, c, cats, is_included, weight, a, i + 1);
3352 recurse_subtotals (struct ctables_section *s, const struct ccase *c,
3353 const struct ctables_category **cats[PIVOT_N_AXES],
3354 bool is_included, double weight[N_CTWS],
3355 enum pivot_axis_type start_axis, size_t start_nest)
3357 for (enum pivot_axis_type a = start_axis; a < PIVOT_N_AXES; a++)
3359 const struct ctables_nest *nest = s->nests[a];
3360 for (size_t i = start_nest; i < nest->n; i++)
3362 if (i == nest->scale_idx)
3365 const struct ctables_category *save = cats[a][i];
3368 cats[a][i] = save->subtotal;
3369 ctables_cell_add__ (s, c, cats, is_included, weight);
3370 recurse_subtotals (s, c, cats, is_included, weight, a, i + 1);
3379 ctables_add_occurrence (const struct variable *var,
3380 const union value *value,
3381 struct hmap *occurrences)
3383 int width = var_get_width (var);
3384 unsigned int hash = value_hash (value, width, 0);
3386 struct ctables_occurrence *o;
3387 HMAP_FOR_EACH_WITH_HASH (o, struct ctables_occurrence, node, hash,
3389 if (value_equal (value, &o->value, width))
3392 o = xmalloc (sizeof *o);
3393 value_clone (&o->value, value, width);
3394 hmap_insert (occurrences, &o->node, hash);
3398 ctables_cell_insert (struct ctables_section *s, const struct ccase *c,
3399 double weight[N_CTWS])
3401 const struct ctables_category *layer_cats[s->nests[PIVOT_AXIS_LAYER]->n];
3402 const struct ctables_category *row_cats[s->nests[PIVOT_AXIS_ROW]->n];
3403 const struct ctables_category *column_cats[s->nests[PIVOT_AXIS_COLUMN]->n];
3404 const struct ctables_category **cats[PIVOT_N_AXES] =
3406 [PIVOT_AXIS_LAYER] = layer_cats,
3407 [PIVOT_AXIS_ROW] = row_cats,
3408 [PIVOT_AXIS_COLUMN] = column_cats,
3411 bool is_included = true;
3413 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3415 const struct ctables_nest *nest = s->nests[a];
3416 for (size_t i = 0; i < nest->n; i++)
3417 if (i != nest->scale_idx)
3419 const struct variable *var = nest->vars[i];
3420 const union value *value = case_data (c, var);
3422 cats[a][i] = ctables_categories_match (
3423 s->table->categories[var_get_dict_index (var)], value, var);
3426 if (i != nest->summary_idx)
3429 if (!var_is_value_missing (var, value))
3432 static const struct ctables_category cct_excluded_missing = {
3433 .type = CCT_EXCLUDED_MISSING,
3436 cats[a][i] = &cct_excluded_missing;
3437 is_included = false;
3443 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3445 const struct ctables_nest *nest = s->nests[a];
3446 for (size_t i = 0; i < nest->n; i++)
3447 if (i != nest->scale_idx)
3449 const struct variable *var = nest->vars[i];
3450 const union value *value = case_data (c, var);
3451 ctables_add_occurrence (var, value, &s->occurrences[a][i]);
3455 ctables_cell_add__ (s, c, cats, is_included, weight);
3456 recurse_totals (s, c, cats, is_included, weight, 0, 0);
3457 recurse_subtotals (s, c, cats, is_included, weight, 0, 0);
3462 const struct ctables_summary_spec_set *set;
3467 merge_item_compare_3way (const struct merge_item *a, const struct merge_item *b)
3469 const struct ctables_summary_spec *as = &a->set->specs[a->ofs];
3470 const struct ctables_summary_spec *bs = &b->set->specs[b->ofs];
3471 if (as->function != bs->function)
3472 return as->function > bs->function ? 1 : -1;
3473 else if (as->weighting != bs->weighting)
3474 return as->weighting > bs->weighting ? 1 : -1;
3475 else if (as->calc_area != bs->calc_area)
3476 return as->calc_area > bs->calc_area ? 1 : -1;
3477 else if (as->percentile != bs->percentile)
3478 return as->percentile < bs->percentile ? 1 : -1;
3480 const char *as_label = as->label ? as->label : "";
3481 const char *bs_label = bs->label ? bs->label : "";
3482 return strcmp (as_label, bs_label);
3486 ctables_category_format_number (double number, const struct variable *var,
3489 struct pivot_value *pv = pivot_value_new_var_value (
3490 var, &(union value) { .f = number });
3491 pivot_value_format (pv, NULL, s);
3492 pivot_value_destroy (pv);
3496 ctables_category_format_string (struct substring string,
3497 const struct variable *var, struct string *out)
3499 int width = var_get_width (var);
3500 char *s = xmalloc (width);
3501 buf_copy_rpad (s, width, string.string, string.length, ' ');
3502 struct pivot_value *pv = pivot_value_new_var_value (
3503 var, &(union value) { .s = CHAR_CAST (uint8_t *, s) });
3504 pivot_value_format (pv, NULL, out);
3505 pivot_value_destroy (pv);
3510 ctables_category_format_label (const struct ctables_category *cat,
3511 const struct variable *var,
3517 ctables_category_format_number (cat->number, var, s);
3521 ctables_category_format_string (cat->string, var, s);
3525 ctables_category_format_number (cat->nrange[0], var, s);
3526 ds_put_format (s, " THRU ");
3527 ctables_category_format_number (cat->nrange[1], var, s);
3531 ctables_category_format_string (cat->srange[0], var, s);
3532 ds_put_format (s, " THRU ");
3533 ctables_category_format_string (cat->srange[1], var, s);
3537 ds_put_cstr (s, "MISSING");
3541 ds_put_cstr (s, "OTHERNM");
3544 case CCT_POSTCOMPUTE:
3545 ds_put_format (s, "&%s", cat->pc->name);
3550 ds_put_cstr (s, cat->total_label);
3556 case CCT_EXCLUDED_MISSING:
3563 static struct pivot_value *
3564 ctables_postcompute_label (const struct ctables_categories *cats,
3565 const struct ctables_category *cat,
3566 const struct variable *var)
3568 struct substring in = ss_cstr (cat->pc->label);
3569 struct substring target = ss_cstr (")LABEL[");
3571 struct string out = DS_EMPTY_INITIALIZER;
3574 size_t chunk = ss_find_substring (in, target);
3575 if (chunk == SIZE_MAX)
3577 if (ds_is_empty (&out))
3578 return pivot_value_new_user_text (in.string, in.length);
3581 ds_put_substring (&out, in);
3582 return pivot_value_new_user_text_nocopy (ds_steal_cstr (&out));
3586 ds_put_substring (&out, ss_head (in, chunk));
3587 ss_advance (&in, chunk + target.length);
3589 struct substring idx_s;
3590 if (!ss_get_until (&in, ']', &idx_s))
3593 long int idx = strtol (idx_s.string, &tail, 10);
3594 if (idx < 1 || idx > cats->n_cats || tail != ss_end (idx_s))
3597 struct ctables_category *cat2 = &cats->cats[idx - 1];
3598 if (!ctables_category_format_label (cat2, var, &out))
3604 return pivot_value_new_user_text (cat->pc->label, SIZE_MAX);
3607 static struct pivot_value *
3608 ctables_category_create_value_label (const struct ctables_categories *cats,
3609 const struct ctables_category *cat,
3610 const struct variable *var,
3611 const union value *value)
3613 return (cat->type == CCT_POSTCOMPUTE && cat->pc->label
3614 ? ctables_postcompute_label (cats, cat, var)
3615 : cat->type == CCT_TOTAL || cat->type == CCT_SUBTOTAL
3616 ? pivot_value_new_user_text (cat->total_label, SIZE_MAX)
3617 : pivot_value_new_var_value (var, value));
3620 static struct ctables_value *
3621 ctables_value_find__ (struct ctables_table *t, const union value *value,
3622 int width, unsigned int hash)
3624 struct ctables_value *clv;
3625 HMAP_FOR_EACH_WITH_HASH (clv, struct ctables_value, node,
3626 hash, &t->clabels_values_map)
3627 if (value_equal (value, &clv->value, width))
3633 ctables_value_insert (struct ctables_table *t, const union value *value,
3636 unsigned int hash = value_hash (value, width, 0);
3637 struct ctables_value *clv = ctables_value_find__ (t, value, width, hash);
3640 clv = xmalloc (sizeof *clv);
3641 value_clone (&clv->value, value, width);
3642 hmap_insert (&t->clabels_values_map, &clv->node, hash);
3646 static struct ctables_value *
3647 ctables_value_find (struct ctables_table *t,
3648 const union value *value, int width)
3650 return ctables_value_find__ (t, value, width,
3651 value_hash (value, width, 0));
3655 ctables_table_add_section (struct ctables_table *t, enum pivot_axis_type a,
3656 size_t ix[PIVOT_N_AXES])
3658 if (a < PIVOT_N_AXES)
3660 size_t limit = MAX (t->stacks[a].n, 1);
3661 for (ix[a] = 0; ix[a] < limit; ix[a]++)
3662 ctables_table_add_section (t, a + 1, ix);
3666 struct ctables_section *s = &t->sections[t->n_sections++];
3667 *s = (struct ctables_section) {
3669 .cells = HMAP_INITIALIZER (s->cells),
3671 for (a = 0; a < PIVOT_N_AXES; a++)
3674 struct ctables_nest *nest = &t->stacks[a].nests[ix[a]];
3676 s->occurrences[a] = xnmalloc (nest->n, sizeof *s->occurrences[a]);
3677 for (size_t i = 0; i < nest->n; i++)
3678 hmap_init (&s->occurrences[a][i]);
3680 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
3681 hmap_init (&s->areas[at]);
3686 ctpo_add (double a, double b)
3692 ctpo_sub (double a, double b)
3698 ctpo_mul (double a, double b)
3704 ctpo_div (double a, double b)
3706 return b ? a / b : SYSMIS;
3710 ctpo_pow (double a, double b)
3712 int save_errno = errno;
3714 double result = pow (a, b);
3722 ctpo_neg (double a, double b UNUSED)
3727 struct ctables_pcexpr_evaluate_ctx
3729 const struct ctables_cell *cell;
3730 const struct ctables_section *section;
3731 const struct ctables_categories *cats;
3732 enum pivot_axis_type pc_a;
3735 enum fmt_type parse_format;
3738 static double ctables_pcexpr_evaluate (
3739 const struct ctables_pcexpr_evaluate_ctx *, const struct ctables_pcexpr *);
3742 ctables_pcexpr_evaluate_nonterminal (
3743 const struct ctables_pcexpr_evaluate_ctx *ctx,
3744 const struct ctables_pcexpr *e, size_t n_args,
3745 double evaluate (double, double))
3747 double args[2] = { 0, 0 };
3748 for (size_t i = 0; i < n_args; i++)
3750 args[i] = ctables_pcexpr_evaluate (ctx, e->subs[i]);
3751 if (!isfinite (args[i]) || args[i] == SYSMIS)
3754 return evaluate (args[0], args[1]);
3758 ctables_pcexpr_evaluate_category (const struct ctables_pcexpr_evaluate_ctx *ctx,
3759 const struct ctables_cell_value *pc_cv)
3761 const struct ctables_section *s = ctx->section;
3764 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3766 const struct ctables_nest *nest = s->nests[a];
3767 for (size_t i = 0; i < nest->n; i++)
3768 if (i != nest->scale_idx)
3770 const struct ctables_cell_value *cv
3771 = (a == ctx->pc_a && i == ctx->pc_a_idx ? pc_cv
3772 : &ctx->cell->axes[a].cvs[i]);
3773 hash = hash_pointer (cv->category, hash);
3774 if (cv->category->type != CCT_TOTAL
3775 && cv->category->type != CCT_SUBTOTAL
3776 && cv->category->type != CCT_POSTCOMPUTE)
3777 hash = value_hash (&cv->value,
3778 var_get_width (nest->vars[i]), hash);
3782 struct ctables_cell *tc;
3783 HMAP_FOR_EACH_WITH_HASH (tc, struct ctables_cell, node, hash, &s->cells)
3785 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
3787 const struct ctables_nest *nest = s->nests[a];
3788 for (size_t i = 0; i < nest->n; i++)
3789 if (i != nest->scale_idx)
3791 const struct ctables_cell_value *p_cv
3792 = (a == ctx->pc_a && i == ctx->pc_a_idx ? pc_cv
3793 : &ctx->cell->axes[a].cvs[i]);
3794 const struct ctables_cell_value *t_cv = &tc->axes[a].cvs[i];
3795 if (p_cv->category != t_cv->category
3796 || (p_cv->category->type != CCT_TOTAL
3797 && p_cv->category->type != CCT_SUBTOTAL
3798 && p_cv->category->type != CCT_POSTCOMPUTE
3799 && !value_equal (&p_cv->value,
3801 var_get_width (nest->vars[i]))))
3813 const struct ctables_table *t = s->table;
3814 const struct ctables_nest *specs_nest = s->nests[t->summary_axis];
3815 const struct ctables_summary_spec_set *specs = &specs_nest->specs[tc->sv];
3816 return ctables_summary_value (tc, &tc->summaries[ctx->summary_idx],
3817 &specs->specs[ctx->summary_idx]);
3821 ctables_pcexpr_evaluate (const struct ctables_pcexpr_evaluate_ctx *ctx,
3822 const struct ctables_pcexpr *e)
3829 case CTPO_CAT_NRANGE:
3830 case CTPO_CAT_SRANGE:
3831 case CTPO_CAT_MISSING:
3832 case CTPO_CAT_OTHERNM:
3834 struct ctables_cell_value cv = {
3835 .category = ctables_find_category_for_postcompute (ctx->section->table->ctables->dict, ctx->cats, ctx->parse_format, e)
3837 assert (cv.category != NULL);
3839 struct hmap *occurrences = &ctx->section->occurrences[ctx->pc_a][ctx->pc_a_idx];
3840 const struct ctables_occurrence *o;
3843 const struct variable *var = ctx->section->nests[ctx->pc_a]->vars[ctx->pc_a_idx];
3844 HMAP_FOR_EACH (o, struct ctables_occurrence, node, occurrences)
3845 if (ctables_categories_match (ctx->cats, &o->value, var) == cv.category)
3847 cv.value = o->value;
3848 sum += ctables_pcexpr_evaluate_category (ctx, &cv);
3853 case CTPO_CAT_NUMBER:
3854 case CTPO_CAT_SUBTOTAL:
3855 case CTPO_CAT_TOTAL:
3857 struct ctables_cell_value cv = {
3858 .category = ctables_find_category_for_postcompute (ctx->section->table->ctables->dict, ctx->cats, ctx->parse_format, e),
3859 .value = { .f = e->number },
3861 assert (cv.category != NULL);
3862 return ctables_pcexpr_evaluate_category (ctx, &cv);
3865 case CTPO_CAT_STRING:
3867 int width = var_get_width (ctx->section->nests[ctx->pc_a]->vars[ctx->pc_a_idx]);
3869 if (width > e->string.length)
3871 s = xmalloc (width);
3872 buf_copy_rpad (s, width, e->string.string, e->string.length, ' ');
3875 const struct ctables_category *category
3876 = ctables_find_category_for_postcompute (
3877 ctx->section->table->ctables->dict,
3878 ctx->cats, ctx->parse_format, e);
3879 assert (category != NULL);
3881 struct ctables_cell_value cv = { .category = category };
3882 if (category->type == CCT_NUMBER)
3883 cv.value.f = category->number;
3884 else if (category->type == CCT_STRING)
3885 cv.value.s = CHAR_CAST (uint8_t *, s ? s : e->string.string);
3889 double retval = ctables_pcexpr_evaluate_category (ctx, &cv);
3895 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 2, ctpo_add);
3898 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 2, ctpo_sub);
3901 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 2, ctpo_mul);
3904 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 2, ctpo_div);
3907 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 2, ctpo_pow);
3910 return ctables_pcexpr_evaluate_nonterminal (ctx, e, 1, ctpo_neg);
3916 static const struct ctables_category *
3917 ctables_cell_postcompute (const struct ctables_section *s,
3918 const struct ctables_cell *cell,
3919 enum pivot_axis_type *pc_a_p,
3922 assert (cell->postcompute);
3923 const struct ctables_category *pc_cat = NULL;
3924 for (enum pivot_axis_type pc_a = 0; pc_a < PIVOT_N_AXES; pc_a++)
3925 for (size_t pc_a_idx = 0; pc_a_idx < s->nests[pc_a]->n; pc_a_idx++)
3927 const struct ctables_cell_value *cv = &cell->axes[pc_a].cvs[pc_a_idx];
3928 if (cv->category->type == CCT_POSTCOMPUTE)
3932 /* Multiple postcomputes cross each other. The value is
3937 pc_cat = cv->category;
3941 *pc_a_idx_p = pc_a_idx;
3945 assert (pc_cat != NULL);
3950 ctables_cell_calculate_postcompute (const struct ctables_section *s,
3951 const struct ctables_cell *cell,
3952 const struct ctables_summary_spec *ss,
3953 struct fmt_spec *format,
3954 bool *is_ctables_format,
3957 enum pivot_axis_type pc_a = 0;
3958 size_t pc_a_idx = 0;
3959 const struct ctables_category *pc_cat = ctables_cell_postcompute (
3960 s, cell, &pc_a, &pc_a_idx);
3964 const struct ctables_postcompute *pc = pc_cat->pc;
3967 for (size_t i = 0; i < pc->specs->n; i++)
3969 const struct ctables_summary_spec *ss2 = &pc->specs->specs[i];
3970 if (ss->function == ss2->function
3971 && ss->weighting == ss2->weighting
3972 && ss->calc_area == ss2->calc_area
3973 && ss->percentile == ss2->percentile)
3975 *format = ss2->format;
3976 *is_ctables_format = ss2->is_ctables_format;
3982 const struct variable *var = s->nests[pc_a]->vars[pc_a_idx];
3983 const struct ctables_categories *cats = s->table->categories[
3984 var_get_dict_index (var)];
3985 struct ctables_pcexpr_evaluate_ctx ctx = {
3990 .pc_a_idx = pc_a_idx,
3991 .summary_idx = summary_idx,
3992 .parse_format = pc_cat->parse_format,
3994 return ctables_pcexpr_evaluate (&ctx, pc->expr);
3998 ctables_format (double d, const struct fmt_spec *format,
3999 const struct fmt_settings *settings)
4001 const union value v = { .f = d };
4002 char *s = data_out_stretchy (&v, "UTF-8", format, settings, NULL);
4004 /* The custom-currency specifications for NEQUAL, PAREN, and PCTPAREN don't
4005 produce the results we want for negative numbers, putting the negative
4006 sign in the wrong spot, before the prefix instead of after it. We can't,
4007 in fact, produce the desired results using a custom-currency
4008 specification. Instead, we postprocess the output, moving the negative
4011 NEQUAL: "-N=3" => "N=-3"
4012 PAREN: "-(3)" => "(-3)"
4013 PCTPAREN: "-(3%)" => "(-3%)"
4015 This transformation doesn't affect NEGPAREN. */
4016 char *minus_src = strchr (s, '-');
4017 if (minus_src && (minus_src == s || minus_src[-1] != 'E'))
4019 char *n_equals = strstr (s, "N=");
4020 char *lparen = strchr (s, '(');
4021 char *minus_dst = n_equals ? n_equals + 1 : lparen;
4023 move_element (s, minus_dst - s + 1, 1, minus_src - s, minus_dst - s);
4029 all_hidden_vlabels (const struct ctables_table *t, enum pivot_axis_type a)
4031 for (size_t i = 0; i < t->stacks[a].n; i++)
4033 struct ctables_nest *nest = &t->stacks[a].nests[i];
4034 if (nest->n != 1 || nest->scale_idx != 0)
4037 enum ctables_vlabel vlabel
4038 = t->ctables->vlabels[var_get_dict_index (nest->vars[0])];
4039 if (vlabel != CTVL_NONE)
4046 ctables_table_output (struct ctables *ct, struct ctables_table *t)
4048 struct pivot_table *pt = pivot_table_create__ (
4050 ? pivot_value_new_user_text (t->title, SIZE_MAX)
4051 : pivot_value_new_text (N_("Custom Tables"))),
4054 pivot_table_set_caption (
4055 pt, pivot_value_new_user_text (t->caption, SIZE_MAX));
4057 pivot_table_set_corner_text (
4058 pt, pivot_value_new_user_text (t->corner, SIZE_MAX));
4060 bool summary_dimension = (t->summary_axis != t->slabels_axis
4061 || (!t->slabels_visible
4062 && t->summary_specs.n > 1));
4063 if (summary_dimension)
4065 struct pivot_dimension *d = pivot_dimension_create (
4066 pt, t->slabels_axis, N_("Statistics"));
4067 const struct ctables_summary_spec_set *specs = &t->summary_specs;
4068 if (!t->slabels_visible)
4069 d->hide_all_labels = true;
4070 for (size_t i = 0; i < specs->n; i++)
4071 pivot_category_create_leaf (
4072 d->root, ctables_summary_label (&specs->specs[i], t->cilevel));
4075 bool categories_dimension = t->clabels_example != NULL;
4076 if (categories_dimension)
4078 struct pivot_dimension *d = pivot_dimension_create (
4079 pt, t->label_axis[t->clabels_from_axis],
4080 t->clabels_from_axis == PIVOT_AXIS_ROW
4081 ? N_("Row Categories")
4082 : N_("Column Categories"));
4083 const struct variable *var = t->clabels_example;
4084 const struct ctables_categories *c = t->categories[var_get_dict_index (var)];
4085 for (size_t i = 0; i < t->n_clabels_values; i++)
4087 const struct ctables_value *value = t->clabels_values[i];
4088 const struct ctables_category *cat = ctables_categories_match (c, &value->value, var);
4089 assert (cat != NULL);
4090 pivot_category_create_leaf (
4091 d->root, ctables_category_create_value_label (c, cat,
4097 pivot_table_set_look (pt, ct->look);
4098 struct pivot_dimension *d[PIVOT_N_AXES];
4099 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
4101 static const char *names[] = {
4102 [PIVOT_AXIS_ROW] = N_("Rows"),
4103 [PIVOT_AXIS_COLUMN] = N_("Columns"),
4104 [PIVOT_AXIS_LAYER] = N_("Layers"),
4106 d[a] = (t->axes[a] || a == t->summary_axis
4107 ? pivot_dimension_create (pt, a, names[a])
4112 assert (t->axes[a]);
4114 for (size_t i = 0; i < t->stacks[a].n; i++)
4116 struct ctables_nest *nest = &t->stacks[a].nests[i];
4117 struct ctables_section **sections = xnmalloc (t->n_sections,
4119 size_t n_sections = 0;
4121 size_t n_total_cells = 0;
4122 size_t max_depth = 0;
4123 for (size_t j = 0; j < t->n_sections; j++)
4124 if (t->sections[j].nests[a] == nest)
4126 struct ctables_section *s = &t->sections[j];
4127 sections[n_sections++] = s;
4128 n_total_cells += hmap_count (&s->cells);
4130 size_t depth = s->nests[a]->n;
4131 max_depth = MAX (depth, max_depth);
4134 struct ctables_cell **sorted = xnmalloc (n_total_cells,
4136 size_t n_sorted = 0;
4138 for (size_t j = 0; j < n_sections; j++)
4140 struct ctables_section *s = sections[j];
4142 struct ctables_cell *cell;
4143 HMAP_FOR_EACH (cell, struct ctables_cell, node, &s->cells)
4145 sorted[n_sorted++] = cell;
4146 assert (n_sorted <= n_total_cells);
4149 struct ctables_cell_sort_aux aux = { .nest = nest, .a = a };
4150 sort (sorted, n_sorted, sizeof *sorted, ctables_cell_compare_3way, &aux);
4152 struct ctables_level
4154 enum ctables_level_type
4156 CTL_VAR, /* Variable label for nest->vars[var_idx]. */
4157 CTL_CATEGORY, /* Category for nest->vars[var_idx]. */
4158 CTL_SUMMARY, /* Summary functions. */
4162 enum settings_value_show vlabel; /* CTL_VAR only. */
4165 struct ctables_level *levels = xnmalloc (1 + 2 * max_depth, sizeof *levels);
4166 size_t n_levels = 0;
4167 for (size_t k = 0; k < nest->n; k++)
4169 enum ctables_vlabel vlabel = ct->vlabels[var_get_dict_index (nest->vars[k])];
4170 if (vlabel == CTVL_NONE && nest->scale_idx == k)
4172 if (vlabel != CTVL_NONE)
4174 levels[n_levels++] = (struct ctables_level) {
4176 .vlabel = (enum settings_value_show) vlabel,
4181 if (nest->scale_idx != k
4182 && (k != nest->n - 1 || t->label_axis[a] == a))
4184 levels[n_levels++] = (struct ctables_level) {
4185 .type = CTL_CATEGORY,
4191 if (!summary_dimension && a == t->slabels_axis)
4193 levels[n_levels++] = (struct ctables_level) {
4194 .type = CTL_SUMMARY,
4195 .var_idx = SIZE_MAX,
4199 /* Pivot categories:
4201 - variable label for nest->vars[0], if vlabel != CTVL_NONE
4202 - category for nest->vars[0], if nest->scale_idx != 0
4203 - variable label for nest->vars[1], if vlabel != CTVL_NONE
4204 - category for nest->vars[1], if nest->scale_idx != 1
4206 - variable label for nest->vars[n - 1], if vlabel != CTVL_NONE
4207 - category for nest->vars[n - 1], if t->label_axis[a] == a && nest->scale_idx != n - 1.
4208 - summary function, if 'a == t->slabels_axis && a ==
4211 Additional dimensions:
4213 - If 'a == t->slabels_axis && a != t->summary_axis', add a summary
4215 - If 't->label_axis[b] == a' for some 'b != a', add a category
4220 struct pivot_category **groups = xnmalloc (1 + 2 * max_depth, sizeof *groups);
4222 for (size_t j = 0; j < n_sorted; j++)
4224 struct ctables_cell *cell = sorted[j];
4225 struct ctables_cell *prev = j > 0 ? sorted[j - 1] : NULL;
4227 size_t n_common = 0;
4230 for (; n_common < n_levels; n_common++)
4232 const struct ctables_level *level = &levels[n_common];
4233 if (level->type == CTL_CATEGORY)
4235 size_t var_idx = level->var_idx;
4236 const struct ctables_category *c = cell->axes[a].cvs[var_idx].category;
4237 if (prev->axes[a].cvs[var_idx].category != c)
4239 else if (c->type != CCT_SUBTOTAL
4240 && c->type != CCT_TOTAL
4241 && c->type != CCT_POSTCOMPUTE
4242 && !value_equal (&prev->axes[a].cvs[var_idx].value,
4243 &cell->axes[a].cvs[var_idx].value,
4244 var_get_type (nest->vars[var_idx])))
4250 for (size_t k = n_common; k < n_levels; k++)
4252 const struct ctables_level *level = &levels[k];
4253 struct pivot_category *parent = k ? groups[k - 1] : d[a]->root;
4254 if (level->type == CTL_SUMMARY)
4256 assert (k == n_levels - 1);
4258 const struct ctables_summary_spec_set *specs = &t->summary_specs;
4259 for (size_t m = 0; m < specs->n; m++)
4261 int leaf = pivot_category_create_leaf (
4262 parent, ctables_summary_label (&specs->specs[m],
4270 const struct variable *var = nest->vars[level->var_idx];
4271 struct pivot_value *label;
4272 if (level->type == CTL_VAR)
4274 label = pivot_value_new_variable (var);
4275 label->variable.show = level->vlabel;
4277 else if (level->type == CTL_CATEGORY)
4279 const struct ctables_cell_value *cv = &cell->axes[a].cvs[level->var_idx];
4280 label = ctables_category_create_value_label (
4281 t->categories[var_get_dict_index (var)],
4282 cv->category, var, &cv->value);
4287 if (k == n_levels - 1)
4288 prev_leaf = pivot_category_create_leaf (parent, label);
4290 groups[k] = pivot_category_create_group__ (parent, label);
4294 cell->axes[a].leaf = prev_leaf;
4303 d[a]->hide_all_labels = all_hidden_vlabels (t, a);
4307 size_t n_total_cells = 0;
4308 for (size_t j = 0; j < t->n_sections; j++)
4309 n_total_cells += hmap_count (&t->sections[j].cells);
4311 struct ctables_cell **sorted = xnmalloc (n_total_cells, sizeof *sorted);
4312 size_t n_sorted = 0;
4313 for (size_t j = 0; j < t->n_sections; j++)
4315 const struct ctables_section *s = &t->sections[j];
4316 struct ctables_cell *cell;
4317 HMAP_FOR_EACH (cell, struct ctables_cell, node, &s->cells)
4319 sorted[n_sorted++] = cell;
4321 assert (n_sorted <= n_total_cells);
4322 sort (sorted, n_sorted, sizeof *sorted, ctables_cell_compare_leaf_3way,
4324 size_t ids[N_CTATS];
4325 memset (ids, 0, sizeof ids);
4326 for (size_t j = 0; j < n_sorted; j++)
4328 struct ctables_cell *cell = sorted[j];
4329 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
4331 struct ctables_area *area = cell->areas[at];
4332 if (!area->sequence)
4333 area->sequence = ++ids[at];
4340 for (size_t i = 0; i < t->n_sections; i++)
4342 struct ctables_section *s = &t->sections[i];
4344 struct ctables_cell *cell;
4345 HMAP_FOR_EACH (cell, struct ctables_cell, node, &s->cells)
4350 const struct ctables_nest *specs_nest = s->nests[t->summary_axis];
4351 const struct ctables_summary_spec_set *specs = &specs_nest->specs[cell->sv];
4352 for (size_t j = 0; j < specs->n; j++)
4355 size_t n_dindexes = 0;
4357 if (summary_dimension)
4358 dindexes[n_dindexes++] = specs->specs[j].axis_idx;
4360 if (categories_dimension)
4362 const struct ctables_nest *clabels_nest = s->nests[t->clabels_from_axis];
4363 const struct variable *var = clabels_nest->vars[clabels_nest->n - 1];
4364 const union value *value = &cell->axes[t->clabels_from_axis].cvs[clabels_nest->n - 1].value;
4365 const struct ctables_value *ctv = ctables_value_find (t, value, var_get_width (var));
4368 dindexes[n_dindexes++] = ctv->leaf;
4371 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
4374 int leaf = cell->axes[a].leaf;
4375 if (a == t->summary_axis && !summary_dimension)
4377 dindexes[n_dindexes++] = leaf;
4380 const struct ctables_summary_spec *ss = &specs->specs[j];
4382 struct fmt_spec format = specs->specs[j].format;
4383 bool is_ctables_format = ss->is_ctables_format;
4384 double d = (cell->postcompute
4385 ? ctables_cell_calculate_postcompute (
4386 s, cell, ss, &format, &is_ctables_format, j)
4387 : ctables_summary_value (cell, &cell->summaries[j],
4390 struct pivot_value *value;
4391 if (ct->hide_threshold != 0
4392 && d < ct->hide_threshold
4393 && ss->function == CTSF_COUNT)
4395 value = pivot_value_new_user_text_nocopy (
4396 xasprintf ("<%d", ct->hide_threshold));
4398 else if (d == 0 && ct->zero)
4399 value = pivot_value_new_user_text (ct->zero, SIZE_MAX);
4400 else if (d == SYSMIS && ct->missing)
4401 value = pivot_value_new_user_text (ct->missing, SIZE_MAX);
4402 else if (is_ctables_format)
4403 value = pivot_value_new_user_text_nocopy (
4404 ctables_format (d, &format, &ct->ctables_formats));
4407 value = pivot_value_new_number (d);
4408 value->numeric.format = format;
4410 /* XXX should text values be right-justified? */
4411 pivot_table_put (pt, dindexes, n_dindexes, value);
4416 pivot_table_submit (pt);
4420 ctables_check_label_position (struct ctables_table *t, enum pivot_axis_type a)
4422 enum pivot_axis_type label_pos = t->label_axis[a];
4426 const char *subcommand_name = a == PIVOT_AXIS_ROW ? "ROWLABELS" : "COLLABELS";
4427 const char *pos_name = label_pos == PIVOT_AXIS_LAYER ? "LAYER" : "OPPOSITE";
4429 const struct ctables_stack *stack = &t->stacks[a];
4433 const struct ctables_nest *n0 = &stack->nests[0];
4436 assert (stack->n == 1);
4440 const struct variable *v0 = n0->vars[n0->n - 1];
4441 struct ctables_categories *c0 = t->categories[var_get_dict_index (v0)];
4442 t->clabels_example = v0;
4444 for (size_t i = 0; i < c0->n_cats; i++)
4445 if (c0->cats[i].type == CCT_FUNCTION)
4447 msg (SE, _("%s=%s is not allowed with sorting based "
4448 "on a summary function."),
4449 subcommand_name, pos_name);
4452 if (n0->n - 1 == n0->scale_idx)
4454 msg (SE, _("%s=%s requires the variables to be moved to be categorical, "
4455 "but %s is a scale variable."),
4456 subcommand_name, pos_name, var_get_name (v0));
4460 for (size_t i = 1; i < stack->n; i++)
4462 const struct ctables_nest *ni = &stack->nests[i];
4464 const struct variable *vi = ni->vars[ni->n - 1];
4465 struct ctables_categories *ci = t->categories[var_get_dict_index (vi)];
4467 if (ni->n - 1 == ni->scale_idx)
4469 msg (SE, _("%s=%s requires the variables to be moved to be "
4470 "categorical, but %s is a scale variable."),
4471 subcommand_name, pos_name, var_get_name (vi));
4474 if (var_get_width (v0) != var_get_width (vi))
4476 msg (SE, _("%s=%s requires the variables to be "
4477 "moved to have the same width, but %s has "
4478 "width %d and %s has width %d."),
4479 subcommand_name, pos_name,
4480 var_get_name (v0), var_get_width (v0),
4481 var_get_name (vi), var_get_width (vi));
4484 if (!val_labs_equal (var_get_value_labels (v0),
4485 var_get_value_labels (vi)))
4487 msg (SE, _("%s=%s requires the variables to be "
4488 "moved to have the same value labels, but %s "
4489 "and %s have different value labels."),
4490 subcommand_name, pos_name,
4491 var_get_name (v0), var_get_name (vi));
4494 if (!ctables_categories_equal (c0, ci))
4496 msg (SE, _("%s=%s requires the variables to be "
4497 "moved to have the same category "
4498 "specifications, but %s and %s have different "
4499 "category specifications."),
4500 subcommand_name, pos_name,
4501 var_get_name (v0), var_get_name (vi));
4510 add_sum_var (struct variable *var,
4511 struct variable ***sum_vars, size_t *n, size_t *allocated)
4513 for (size_t i = 0; i < *n; i++)
4514 if (var == (*sum_vars)[i])
4517 if (*n >= *allocated)
4518 *sum_vars = x2nrealloc (*sum_vars, allocated, sizeof **sum_vars);
4519 (*sum_vars)[*n] = var;
4523 static enum ctables_area_type
4524 rotate_area (enum ctables_area_type area)
4535 return CTAT_LAYERCOL;
4538 return CTAT_LAYERROW;
4551 enumerate_sum_vars (const struct ctables_axis *a,
4552 struct variable ***sum_vars, size_t *n, size_t *allocated)
4560 for (size_t i = 0; i < N_CSVS; i++)
4561 for (size_t j = 0; j < a->specs[i].n; j++)
4563 struct ctables_summary_spec *spec = &a->specs[i].specs[j];
4564 if (spec->function == CTSF_areaPCT_SUM)
4565 spec->sum_var_idx = add_sum_var (a->var, sum_vars, n, allocated);
4571 for (size_t i = 0; i < 2; i++)
4572 enumerate_sum_vars (a->subs[i], sum_vars, n, allocated);
4578 ctables_prepare_table (struct ctables_table *t)
4580 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
4583 t->stacks[a] = enumerate_fts (a, t->axes[a]);
4585 for (size_t j = 0; j < t->stacks[a].n; j++)
4587 struct ctables_nest *nest = &t->stacks[a].nests[j];
4588 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
4590 nest->areas[at] = xmalloc (nest->n * sizeof *nest->areas[at]);
4591 nest->n_areas[at] = 0;
4593 enum pivot_axis_type ata, atb;
4594 if (at == CTAT_ROW || at == CTAT_LAYERROW)
4596 ata = PIVOT_AXIS_ROW;
4597 atb = PIVOT_AXIS_COLUMN;
4599 else if (at == CTAT_COL || at == CTAT_LAYERCOL)
4601 ata = PIVOT_AXIS_COLUMN;
4602 atb = PIVOT_AXIS_ROW;
4605 if (at == CTAT_LAYER
4606 ? a != PIVOT_AXIS_LAYER && t->label_axis[a] == PIVOT_AXIS_LAYER
4607 : at == CTAT_LAYERCOL || at == CTAT_LAYERROW
4608 ? a == atb && t->label_axis[a] != a
4611 for (size_t k = nest->n - 1; k < nest->n; k--)
4612 if (k != nest->scale_idx)
4614 nest->areas[at][nest->n_areas[at]++] = k;
4620 if (at == CTAT_LAYER ? a != PIVOT_AXIS_LAYER
4621 : at == CTAT_LAYERROW || at == CTAT_LAYERCOL ? a == atb
4622 : at == CTAT_TABLE ? true
4626 for (size_t k = 0; k < nest->n; k++)
4627 if (k != nest->scale_idx)
4628 nest->areas[at][nest->n_areas[at]++] = k;
4634 #define L PIVOT_AXIS_LAYER
4635 n_drop = (t->clabels_from_axis == L ? a != L
4636 : t->clabels_to_axis == L ? (t->clabels_from_axis == a ? -1 : a != L)
4637 : t->clabels_from_axis == a ? 2
4644 n_drop = a == ata && t->label_axis[ata] == atb;
4649 n_drop = (a == ata ? t->label_axis[ata] == atb
4651 : t->clabels_from_axis == atb ? -1
4652 : t->clabels_to_axis != atb ? 1
4664 size_t n = nest->n_areas[at];
4667 nest->areas[at][n - 2] = nest->areas[at][n - 1];
4668 nest->n_areas[at]--;
4673 for (int i = 0; i < n_drop; i++)
4674 if (nest->n_areas[at] > 0)
4675 nest->n_areas[at]--;
4682 struct ctables_nest *nest = xmalloc (sizeof *nest);
4683 *nest = (struct ctables_nest) {
4685 .scale_idx = SIZE_MAX,
4686 .summary_idx = SIZE_MAX
4688 t->stacks[a] = (struct ctables_stack) { .nests = nest, .n = 1 };
4690 /* There's no point in moving labels away from an axis that has no
4691 labels, so avoid dealing with the special cases around that. */
4692 t->label_axis[a] = a;
4695 struct ctables_stack *stack = &t->stacks[t->summary_axis];
4696 for (size_t i = 0; i < stack->n; i++)
4698 struct ctables_nest *nest = &stack->nests[i];
4699 if (!nest->specs[CSV_CELL].n)
4701 struct ctables_summary_spec_set *ss = &nest->specs[CSV_CELL];
4702 ss->specs = xmalloc (sizeof *ss->specs);
4705 enum ctables_summary_function function
4706 = ss->is_scale ? CTSF_MEAN : CTSF_COUNT;
4710 nest->summary_idx = nest->n - 1;
4711 ss->var = nest->vars[nest->summary_idx];
4713 *ss->specs = (struct ctables_summary_spec) {
4714 .function = function,
4715 .weighting = ss->is_scale ? CTW_EFFECTIVE : CTW_DICTIONARY,
4716 .format = ctables_summary_default_format (function, ss->var),
4719 ctables_summary_spec_set_clone (&nest->specs[CSV_TOTAL],
4720 &nest->specs[CSV_CELL]);
4722 else if (!nest->specs[CSV_TOTAL].n)
4723 ctables_summary_spec_set_clone (&nest->specs[CSV_TOTAL],
4724 &nest->specs[CSV_CELL]);
4726 if (t->label_axis[PIVOT_AXIS_ROW] == PIVOT_AXIS_COLUMN
4727 || t->label_axis[PIVOT_AXIS_COLUMN] == PIVOT_AXIS_ROW)
4729 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
4730 for (size_t i = 0; i < nest->specs[sv].n; i++)
4732 struct ctables_summary_spec *ss = &nest->specs[sv].specs[i];
4733 const struct ctables_function_info *cfi =
4734 &ctables_function_info[ss->function];
4736 ss->calc_area = rotate_area (ss->calc_area);
4740 if (t->ctables->smissing_listwise)
4742 struct variable **listwise_vars = NULL;
4744 size_t allocated = 0;
4746 for (size_t j = nest->group_head; j < stack->n; j++)
4748 const struct ctables_nest *other_nest = &stack->nests[j];
4749 if (other_nest->group_head != nest->group_head)
4752 if (nest != other_nest && other_nest->scale_idx < other_nest->n)
4755 listwise_vars = x2nrealloc (listwise_vars, &allocated,
4756 sizeof *listwise_vars);
4757 listwise_vars[n++] = other_nest->vars[other_nest->scale_idx];
4760 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
4763 listwise_vars = xmemdup (listwise_vars,
4764 n * sizeof *listwise_vars);
4765 nest->specs[sv].listwise_vars = listwise_vars;
4766 nest->specs[sv].n_listwise_vars = n;
4771 struct ctables_summary_spec_set *merged = &t->summary_specs;
4772 struct merge_item *items = xnmalloc (N_CSVS * stack->n, sizeof *items);
4774 for (size_t j = 0; j < stack->n; j++)
4776 const struct ctables_nest *nest = &stack->nests[j];
4778 for (enum ctables_summary_variant sv = 0; sv < N_CSVS; sv++)
4779 items[n_left++] = (struct merge_item) { .set = &nest->specs[sv] };
4784 struct merge_item min = items[0];
4785 for (size_t j = 1; j < n_left; j++)
4786 if (merge_item_compare_3way (&items[j], &min) < 0)
4789 if (merged->n >= merged->allocated)
4790 merged->specs = x2nrealloc (merged->specs, &merged->allocated,
4791 sizeof *merged->specs);
4792 merged->specs[merged->n++] = min.set->specs[min.ofs];
4794 for (size_t j = 0; j < n_left; )
4796 if (merge_item_compare_3way (&items[j], &min) == 0)
4798 struct merge_item *item = &items[j];
4799 item->set->specs[item->ofs].axis_idx = merged->n - 1;
4800 if (++item->ofs >= item->set->n)
4802 items[j] = items[--n_left];
4811 size_t allocated_sum_vars = 0;
4812 enumerate_sum_vars (t->axes[t->summary_axis],
4813 &t->sum_vars, &t->n_sum_vars, &allocated_sum_vars);
4815 return (ctables_check_label_position (t, PIVOT_AXIS_ROW)
4816 && ctables_check_label_position (t, PIVOT_AXIS_COLUMN));
4820 ctables_insert_clabels_values (struct ctables_table *t, const struct ccase *c,
4821 enum pivot_axis_type a)
4823 struct ctables_stack *stack = &t->stacks[a];
4824 for (size_t i = 0; i < stack->n; i++)
4826 const struct ctables_nest *nest = &stack->nests[i];
4827 const struct variable *var = nest->vars[nest->n - 1];
4828 const union value *value = case_data (c, var);
4830 if (var_is_numeric (var) && value->f == SYSMIS)
4833 if (ctables_categories_match (t->categories [var_get_dict_index (var)],
4835 ctables_value_insert (t, value, var_get_width (var));
4840 compare_clabels_values_3way (const void *a_, const void *b_, const void *width_)
4842 const struct ctables_value *const *ap = a_;
4843 const struct ctables_value *const *bp = b_;
4844 const struct ctables_value *a = *ap;
4845 const struct ctables_value *b = *bp;
4846 const int *width = width_;
4847 return value_compare_3way (&a->value, &b->value, *width);
4851 ctables_sort_clabels_values (struct ctables_table *t)
4853 const struct variable *v0 = t->clabels_example;
4854 int width = var_get_width (v0);
4856 struct ctables_categories *c0 = t->categories[var_get_dict_index (v0)];
4859 const struct val_labs *val_labs = var_get_value_labels (v0);
4860 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4861 vl = val_labs_next (val_labs, vl))
4862 if (ctables_categories_match (c0, &vl->value, v0))
4863 ctables_value_insert (t, &vl->value, width);
4866 size_t n = hmap_count (&t->clabels_values_map);
4867 t->clabels_values = xnmalloc (n, sizeof *t->clabels_values);
4869 struct ctables_value *clv;
4871 HMAP_FOR_EACH (clv, struct ctables_value, node, &t->clabels_values_map)
4872 t->clabels_values[i++] = clv;
4873 t->n_clabels_values = n;
4876 sort (t->clabels_values, n, sizeof *t->clabels_values,
4877 compare_clabels_values_3way, &width);
4879 for (size_t i = 0; i < n; i++)
4880 t->clabels_values[i]->leaf = i;
4884 ctables_add_category_occurrences (const struct variable *var,
4885 struct hmap *occurrences,
4886 const struct ctables_categories *cats)
4888 const struct val_labs *val_labs = var_get_value_labels (var);
4890 for (size_t i = 0; i < cats->n_cats; i++)
4892 const struct ctables_category *c = &cats->cats[i];
4896 ctables_add_occurrence (var, &(const union value) { .f = c->number },
4902 int width = var_get_width (var);
4904 value_init (&value, width);
4905 value_copy_buf_rpad (&value, width,
4906 CHAR_CAST (uint8_t *, c->string.string),
4907 c->string.length, ' ');
4908 ctables_add_occurrence (var, &value, occurrences);
4909 value_destroy (&value, width);
4914 assert (var_is_numeric (var));
4915 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4916 vl = val_labs_next (val_labs, vl))
4917 if (vl->value.f >= c->nrange[0] && vl->value.f <= c->nrange[1])
4918 ctables_add_occurrence (var, &vl->value, occurrences);
4922 assert (var_is_alpha (var));
4923 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4924 vl = val_labs_next (val_labs, vl))
4925 if (in_string_range (&vl->value, var, c->srange))
4926 ctables_add_occurrence (var, &vl->value, occurrences);
4930 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4931 vl = val_labs_next (val_labs, vl))
4932 if (var_is_value_missing (var, &vl->value))
4933 ctables_add_occurrence (var, &vl->value, occurrences);
4937 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4938 vl = val_labs_next (val_labs, vl))
4939 ctables_add_occurrence (var, &vl->value, occurrences);
4942 case CCT_POSTCOMPUTE:
4952 for (const struct val_lab *vl = val_labs_first (val_labs); vl;
4953 vl = val_labs_next (val_labs, vl))
4954 if (c->include_missing || !var_is_value_missing (var, &vl->value))
4955 ctables_add_occurrence (var, &vl->value, occurrences);
4958 case CCT_EXCLUDED_MISSING:
4965 ctables_section_recurse_add_empty_categories (
4966 struct ctables_section *s,
4967 const struct ctables_category **cats[PIVOT_N_AXES], struct ccase *c,
4968 enum pivot_axis_type a, size_t a_idx)
4970 if (a >= PIVOT_N_AXES)
4971 ctables_cell_insert__ (s, c, cats);
4972 else if (!s->nests[a] || a_idx >= s->nests[a]->n)
4973 ctables_section_recurse_add_empty_categories (s, cats, c, a + 1, 0);
4976 const struct variable *var = s->nests[a]->vars[a_idx];
4977 const struct ctables_categories *categories = s->table->categories[
4978 var_get_dict_index (var)];
4979 int width = var_get_width (var);
4980 const struct hmap *occurrences = &s->occurrences[a][a_idx];
4981 const struct ctables_occurrence *o;
4982 HMAP_FOR_EACH (o, struct ctables_occurrence, node, occurrences)
4984 union value *value = case_data_rw (c, var);
4985 value_destroy (value, width);
4986 value_clone (value, &o->value, width);
4987 cats[a][a_idx] = ctables_categories_match (categories, value, var);
4988 assert (cats[a][a_idx] != NULL);
4989 ctables_section_recurse_add_empty_categories (s, cats, c, a, a_idx + 1);
4992 for (size_t i = 0; i < categories->n_cats; i++)
4994 const struct ctables_category *cat = &categories->cats[i];
4995 if (cat->type == CCT_POSTCOMPUTE)
4997 cats[a][a_idx] = cat;
4998 ctables_section_recurse_add_empty_categories (s, cats, c, a, a_idx + 1);
5005 ctables_section_add_empty_categories (struct ctables_section *s)
5007 bool show_empty = false;
5008 for (size_t a = 0; a < PIVOT_N_AXES; a++)
5010 for (size_t k = 0; k < s->nests[a]->n; k++)
5011 if (k != s->nests[a]->scale_idx)
5013 const struct variable *var = s->nests[a]->vars[k];
5014 const struct ctables_categories *cats = s->table->categories[
5015 var_get_dict_index (var)];
5016 if (cats->show_empty)
5019 ctables_add_category_occurrences (var, &s->occurrences[a][k], cats);
5025 const struct ctables_category *layer_cats[s->nests[PIVOT_AXIS_LAYER]->n];
5026 const struct ctables_category *row_cats[s->nests[PIVOT_AXIS_ROW]->n];
5027 const struct ctables_category *column_cats[s->nests[PIVOT_AXIS_COLUMN]->n];
5028 const struct ctables_category **cats[PIVOT_N_AXES] =
5030 [PIVOT_AXIS_LAYER] = layer_cats,
5031 [PIVOT_AXIS_ROW] = row_cats,
5032 [PIVOT_AXIS_COLUMN] = column_cats,
5034 struct ccase *c = case_create (dict_get_proto (s->table->ctables->dict));
5035 ctables_section_recurse_add_empty_categories (s, cats, c, 0, 0);
5040 ctables_section_clear (struct ctables_section *s)
5042 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
5044 const struct ctables_nest *nest = s->nests[a];
5045 for (size_t i = 0; i < nest->n; i++)
5046 if (i != nest->scale_idx)
5048 const struct variable *var = nest->vars[i];
5049 int width = var_get_width (var);
5050 struct ctables_occurrence *o, *next;
5051 struct hmap *map = &s->occurrences[a][i];
5052 HMAP_FOR_EACH_SAFE (o, next, struct ctables_occurrence, node, map)
5054 value_destroy (&o->value, width);
5055 hmap_delete (map, &o->node);
5062 struct ctables_cell *cell, *next_cell;
5063 HMAP_FOR_EACH_SAFE (cell, next_cell, struct ctables_cell, node, &s->cells)
5065 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
5067 const struct ctables_nest *nest = s->nests[a];
5068 for (size_t i = 0; i < nest->n; i++)
5069 if (i != nest->scale_idx)
5070 value_destroy (&cell->axes[a].cvs[i].value,
5071 var_get_width (nest->vars[i]));
5072 free (cell->axes[a].cvs);
5075 const struct ctables_nest *ss = s->nests[s->table->summary_axis];
5076 const struct ctables_summary_spec_set *specs = &ss->specs[cell->sv];
5077 for (size_t i = 0; i < specs->n; i++)
5078 ctables_summary_uninit (&cell->summaries[i], &specs->specs[i]);
5079 free (cell->summaries);
5081 hmap_delete (&s->cells, &cell->node);
5084 hmap_shrink (&s->cells);
5086 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
5088 struct ctables_area *area, *next_area;
5089 HMAP_FOR_EACH_SAFE (area, next_area, struct ctables_area, node,
5093 hmap_delete (&s->areas[at], &area->node);
5096 hmap_shrink (&s->areas[at]);
5101 ctables_section_uninit (struct ctables_section *s)
5103 ctables_section_clear (s);
5105 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
5107 struct ctables_nest *nest = s->nests[a];
5108 for (size_t i = 0; i < nest->n; i++)
5109 hmap_destroy (&s->occurrences[a][i]);
5110 free (s->occurrences[a]);
5113 hmap_destroy (&s->cells);
5114 for (enum ctables_area_type at = 0; at < N_CTATS; at++)
5115 hmap_destroy (&s->areas[at]);
5119 ctables_table_clear (struct ctables_table *t)
5121 for (size_t i = 0; i < t->n_sections; i++)
5122 ctables_section_clear (&t->sections[i]);
5124 if (t->clabels_example)
5126 int width = var_get_width (t->clabels_example);
5127 struct ctables_value *value, *next_value;
5128 HMAP_FOR_EACH_SAFE (value, next_value, struct ctables_value, node,
5129 &t->clabels_values_map)
5131 value_destroy (&value->value, width);
5132 hmap_delete (&t->clabels_values_map, &value->node);
5135 hmap_shrink (&t->clabels_values_map);
5137 free (t->clabels_values);
5138 t->clabels_values = NULL;
5139 t->n_clabels_values = 0;
5144 ctables_execute (struct dataset *ds, struct casereader *input,
5147 for (size_t i = 0; i < ct->n_tables; i++)
5149 struct ctables_table *t = ct->tables[i];
5150 t->sections = xnmalloc (MAX (1, t->stacks[PIVOT_AXIS_ROW].n) *
5151 MAX (1, t->stacks[PIVOT_AXIS_COLUMN].n) *
5152 MAX (1, t->stacks[PIVOT_AXIS_LAYER].n),
5153 sizeof *t->sections);
5154 size_t ix[PIVOT_N_AXES];
5155 ctables_table_add_section (t, 0, ix);
5158 struct dictionary *dict = dataset_dict (ds);
5160 bool splitting = dict_get_split_type (dict) == SPLIT_SEPARATE;
5161 struct casegrouper *grouper
5163 ? casegrouper_create_splits (input, dict)
5164 : casegrouper_create_vars (input, NULL, 0));
5165 struct casereader *group;
5166 while (casegrouper_get_next_group (grouper, &group))
5170 struct ccase *c = casereader_peek (group, 0);
5173 output_split_file_values (ds, c);
5178 bool warn_on_invalid = true;
5179 for (struct ccase *c = casereader_read (group); c;
5180 case_unref (c), c = casereader_read (group))
5182 double d_weight = dict_get_rounded_case_weight (dict, c, &warn_on_invalid);
5183 double e_weight = (ct->e_weight
5184 ? var_force_valid_weight (ct->e_weight,
5185 case_num (c, ct->e_weight),
5189 [CTW_DICTIONARY] = d_weight,
5190 [CTW_EFFECTIVE] = e_weight,
5191 [CTW_UNWEIGHTED] = 1.0,
5194 for (size_t i = 0; i < ct->n_tables; i++)
5196 struct ctables_table *t = ct->tables[i];
5198 for (size_t j = 0; j < t->n_sections; j++)
5199 ctables_cell_insert (&t->sections[j], c, weight);
5201 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
5202 if (t->label_axis[a] != a)
5203 ctables_insert_clabels_values (t, c, a);
5206 casereader_destroy (group);
5208 for (size_t i = 0; i < ct->n_tables; i++)
5210 struct ctables_table *t = ct->tables[i];
5212 if (t->clabels_example)
5213 ctables_sort_clabels_values (t);
5215 for (size_t j = 0; j < t->n_sections; j++)
5216 ctables_section_add_empty_categories (&t->sections[j]);
5218 ctables_table_output (ct, t);
5219 ctables_table_clear (t);
5222 return casegrouper_destroy (grouper);
5227 typedef struct ctables_pcexpr *parse_recursively_func (struct lexer *,
5228 struct dictionary *);
5231 ctables_pcexpr_destroy (struct ctables_pcexpr *e)
5237 case CTPO_CAT_STRING:
5238 ss_dealloc (&e->string);
5241 case CTPO_CAT_SRANGE:
5242 for (size_t i = 0; i < 2; i++)
5243 ss_dealloc (&e->srange[i]);
5252 for (size_t i = 0; i < 2; i++)
5253 ctables_pcexpr_destroy (e->subs[i]);
5257 case CTPO_CAT_NUMBER:
5258 case CTPO_CAT_NRANGE:
5259 case CTPO_CAT_MISSING:
5260 case CTPO_CAT_OTHERNM:
5261 case CTPO_CAT_SUBTOTAL:
5262 case CTPO_CAT_TOTAL:
5266 msg_location_destroy (e->location);
5271 static struct ctables_pcexpr *
5272 ctables_pcexpr_allocate_binary (enum ctables_postcompute_op op,
5273 struct ctables_pcexpr *sub0,
5274 struct ctables_pcexpr *sub1)
5276 struct ctables_pcexpr *e = xmalloc (sizeof *e);
5277 *e = (struct ctables_pcexpr) {
5279 .subs = { sub0, sub1 },
5280 .location = msg_location_merged (sub0->location, sub1->location),
5285 /* How to parse an operator. */
5288 enum token_type token;
5289 enum ctables_postcompute_op op;
5292 static const struct operator *
5293 ctables_pcexpr_match_operator (struct lexer *lexer,
5294 const struct operator ops[], size_t n_ops)
5296 for (const struct operator *op = ops; op < ops + n_ops; op++)
5297 if (lex_token (lexer) == op->token)
5299 if (op->token != T_NEG_NUM)
5308 static struct ctables_pcexpr *
5309 ctables_pcexpr_parse_binary_operators__ (
5310 struct lexer *lexer, struct dictionary *dict,
5311 const struct operator ops[], size_t n_ops,
5312 parse_recursively_func *parse_next_level,
5313 const char *chain_warning, struct ctables_pcexpr *lhs)
5315 for (int op_count = 0; ; op_count++)
5317 const struct operator *op
5318 = ctables_pcexpr_match_operator (lexer, ops, n_ops);
5321 if (op_count > 1 && chain_warning)
5322 msg_at (SW, lhs->location, "%s", chain_warning);
5327 struct ctables_pcexpr *rhs = parse_next_level (lexer, dict);
5330 ctables_pcexpr_destroy (lhs);
5334 lhs = ctables_pcexpr_allocate_binary (op->op, lhs, rhs);
5338 static struct ctables_pcexpr *
5339 ctables_pcexpr_parse_binary_operators (
5340 struct lexer *lexer, struct dictionary *dict,
5341 const struct operator ops[], size_t n_ops,
5342 parse_recursively_func *parse_next_level, const char *chain_warning)
5344 struct ctables_pcexpr *lhs = parse_next_level (lexer, dict);
5348 return ctables_pcexpr_parse_binary_operators__ (lexer, dict, ops, n_ops,
5350 chain_warning, lhs);
5353 static struct ctables_pcexpr *ctables_pcexpr_parse_add (struct lexer *,
5354 struct dictionary *);
5356 static struct ctables_pcexpr
5357 ctpo_cat_nrange (double low, double high)
5359 return (struct ctables_pcexpr) {
5360 .op = CTPO_CAT_NRANGE,
5361 .nrange = { low, high },
5365 static struct ctables_pcexpr
5366 ctpo_cat_srange (struct substring low, struct substring high)
5368 return (struct ctables_pcexpr) {
5369 .op = CTPO_CAT_SRANGE,
5370 .srange = { low, high },
5374 static struct ctables_pcexpr *
5375 ctables_pcexpr_parse_primary (struct lexer *lexer, struct dictionary *dict)
5377 int start_ofs = lex_ofs (lexer);
5378 struct ctables_pcexpr e;
5379 if (lex_is_number (lexer))
5381 e = (struct ctables_pcexpr) { .op = CTPO_CONSTANT,
5382 .number = lex_number (lexer) };
5385 else if (lex_match_id (lexer, "MISSING"))
5386 e = (struct ctables_pcexpr) { .op = CTPO_CAT_MISSING };
5387 else if (lex_match_id (lexer, "OTHERNM"))
5388 e = (struct ctables_pcexpr) { .op = CTPO_CAT_OTHERNM };
5389 else if (lex_match_id (lexer, "TOTAL"))
5390 e = (struct ctables_pcexpr) { .op = CTPO_CAT_TOTAL };
5391 else if (lex_match_id (lexer, "SUBTOTAL"))
5393 size_t subtotal_index = 0;
5394 if (lex_match (lexer, T_LBRACK))
5396 if (!lex_force_int_range (lexer, "SUBTOTAL", 1, LONG_MAX))
5398 subtotal_index = lex_integer (lexer);
5400 if (!lex_force_match (lexer, T_RBRACK))
5403 e = (struct ctables_pcexpr) { .op = CTPO_CAT_SUBTOTAL,
5404 .subtotal_index = subtotal_index };
5406 else if (lex_match (lexer, T_LBRACK))
5408 if (lex_match_id (lexer, "LO"))
5410 if (!lex_force_match_id (lexer, "THRU"))
5413 if (lex_is_string (lexer))
5415 struct substring low = { .string = NULL };
5416 struct substring high = parse_substring (lexer, dict);
5417 e = ctpo_cat_srange (low, high);
5421 if (!lex_force_num (lexer))
5423 e = ctpo_cat_nrange (-DBL_MAX, lex_number (lexer));
5427 else if (lex_is_number (lexer))
5429 double number = lex_number (lexer);
5431 if (lex_match_id (lexer, "THRU"))
5433 if (lex_match_id (lexer, "HI"))
5434 e = ctpo_cat_nrange (number, DBL_MAX);
5437 if (!lex_force_num (lexer))
5439 e = ctpo_cat_nrange (number, lex_number (lexer));
5444 e = (struct ctables_pcexpr) { .op = CTPO_CAT_NUMBER,
5447 else if (lex_is_string (lexer))
5449 struct substring s = parse_substring (lexer, dict);
5451 if (lex_match_id (lexer, "THRU"))
5453 struct substring high;
5455 if (lex_match_id (lexer, "HI"))
5456 high = (struct substring) { .string = NULL };
5459 if (!lex_force_string (lexer))
5464 high = parse_substring (lexer, dict);
5467 e = ctpo_cat_srange (s, high);
5470 e = (struct ctables_pcexpr) { .op = CTPO_CAT_STRING, .string = s };
5474 lex_error (lexer, NULL);
5478 if (!lex_force_match (lexer, T_RBRACK))
5480 if (e.op == CTPO_CAT_STRING)
5481 ss_dealloc (&e.string);
5482 else if (e.op == CTPO_CAT_SRANGE)
5484 ss_dealloc (&e.srange[0]);
5485 ss_dealloc (&e.srange[1]);
5490 else if (lex_match (lexer, T_LPAREN))
5492 struct ctables_pcexpr *ep = ctables_pcexpr_parse_add (lexer, dict);
5495 if (!lex_force_match (lexer, T_RPAREN))
5497 ctables_pcexpr_destroy (ep);
5504 lex_error (lexer, NULL);
5508 e.location = lex_ofs_location (lexer, start_ofs, lex_ofs (lexer) - 1);
5509 return xmemdup (&e, sizeof e);
5512 static struct ctables_pcexpr *
5513 ctables_pcexpr_allocate_neg (struct ctables_pcexpr *sub,
5514 struct lexer *lexer, int start_ofs)
5516 struct ctables_pcexpr *e = xmalloc (sizeof *e);
5517 *e = (struct ctables_pcexpr) {
5520 .location = lex_ofs_location (lexer, start_ofs, lex_ofs (lexer) - 1),
5525 static struct ctables_pcexpr *
5526 ctables_pcexpr_parse_exp (struct lexer *lexer, struct dictionary *dict)
5528 static const struct operator op = { T_EXP, CTPO_POW };
5530 const char *chain_warning =
5531 _("The exponentiation operator (`**') is left-associative: "
5532 "`a**b**c' equals `(a**b)**c', not `a**(b**c)'. "
5533 "To disable this warning, insert parentheses.");
5535 if (lex_token (lexer) != T_NEG_NUM || lex_next_token (lexer, 1) != T_EXP)
5536 return ctables_pcexpr_parse_binary_operators (lexer, dict, &op, 1,
5537 ctables_pcexpr_parse_primary,
5540 /* Special case for situations like "-5**6", which must be parsed as
5543 int start_ofs = lex_ofs (lexer);
5544 struct ctables_pcexpr *lhs = xmalloc (sizeof *lhs);
5545 *lhs = (struct ctables_pcexpr) {
5546 .op = CTPO_CONSTANT,
5547 .number = -lex_tokval (lexer),
5548 .location = lex_ofs_location (lexer, start_ofs, lex_ofs (lexer)),
5552 struct ctables_pcexpr *node = ctables_pcexpr_parse_binary_operators__ (
5553 lexer, dict, &op, 1,
5554 ctables_pcexpr_parse_primary, chain_warning, lhs);
5558 return ctables_pcexpr_allocate_neg (node, lexer, start_ofs);
5561 /* Parses the unary minus level. */
5562 static struct ctables_pcexpr *
5563 ctables_pcexpr_parse_neg (struct lexer *lexer, struct dictionary *dict)
5565 int start_ofs = lex_ofs (lexer);
5566 if (!lex_match (lexer, T_DASH))
5567 return ctables_pcexpr_parse_exp (lexer, dict);
5569 struct ctables_pcexpr *inner = ctables_pcexpr_parse_neg (lexer, dict);
5573 return ctables_pcexpr_allocate_neg (inner, lexer, start_ofs);
5576 /* Parses the multiplication and division level. */
5577 static struct ctables_pcexpr *
5578 ctables_pcexpr_parse_mul (struct lexer *lexer, struct dictionary *dict)
5580 static const struct operator ops[] =
5582 { T_ASTERISK, CTPO_MUL },
5583 { T_SLASH, CTPO_DIV },
5586 return ctables_pcexpr_parse_binary_operators (lexer, dict, ops,
5587 sizeof ops / sizeof *ops,
5588 ctables_pcexpr_parse_neg, NULL);
5591 /* Parses the addition and subtraction level. */
5592 static struct ctables_pcexpr *
5593 ctables_pcexpr_parse_add (struct lexer *lexer, struct dictionary *dict)
5595 static const struct operator ops[] =
5597 { T_PLUS, CTPO_ADD },
5598 { T_DASH, CTPO_SUB },
5599 { T_NEG_NUM, CTPO_ADD },
5602 return ctables_pcexpr_parse_binary_operators (lexer, dict,
5603 ops, sizeof ops / sizeof *ops,
5604 ctables_pcexpr_parse_mul, NULL);
5607 static struct ctables_postcompute *
5608 ctables_find_postcompute (struct ctables *ct, const char *name)
5610 struct ctables_postcompute *pc;
5611 HMAP_FOR_EACH_WITH_HASH (pc, struct ctables_postcompute, hmap_node,
5612 utf8_hash_case_string (name, 0), &ct->postcomputes)
5613 if (!utf8_strcasecmp (pc->name, name))
5619 ctables_parse_pcompute (struct lexer *lexer, struct dictionary *dict,
5622 int pcompute_start = lex_ofs (lexer) - 1;
5624 if (!lex_match (lexer, T_AND))
5626 lex_error_expecting (lexer, "&");
5629 if (!lex_force_id (lexer))
5632 char *name = ss_xstrdup (lex_tokss (lexer));
5635 if (!lex_force_match (lexer, T_EQUALS)
5636 || !lex_force_match_id (lexer, "EXPR")
5637 || !lex_force_match (lexer, T_LPAREN))
5643 int expr_start = lex_ofs (lexer);
5644 struct ctables_pcexpr *expr = ctables_pcexpr_parse_add (lexer, dict);
5645 int expr_end = lex_ofs (lexer) - 1;
5646 if (!expr || !lex_force_match (lexer, T_RPAREN))
5648 ctables_pcexpr_destroy (expr);
5652 int pcompute_end = lex_ofs (lexer) - 1;
5654 struct msg_location *location = lex_ofs_location (lexer, pcompute_start,
5657 struct ctables_postcompute *pc = ctables_find_postcompute (ct, name);
5660 msg_at (SW, location, _("New definition of &%s will override the "
5661 "previous definition."),
5663 msg_at (SN, pc->location, _("This is the previous definition."));
5665 ctables_pcexpr_destroy (pc->expr);
5666 msg_location_destroy (pc->location);
5671 pc = xmalloc (sizeof *pc);
5672 *pc = (struct ctables_postcompute) { .name = name };
5673 hmap_insert (&ct->postcomputes, &pc->hmap_node,
5674 utf8_hash_case_string (pc->name, 0));
5677 pc->location = location;
5679 pc->label = lex_ofs_representation (lexer, expr_start, expr_end);
5684 ctables_parse_pproperties_format (struct lexer *lexer,
5685 struct ctables_summary_spec_set *sss)
5687 *sss = (struct ctables_summary_spec_set) { .n = 0 };
5689 while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH
5690 && !(lex_token (lexer) == T_ID
5691 && (lex_id_match (ss_cstr ("LABEL"), lex_tokss (lexer))
5692 || lex_id_match (ss_cstr ("HIDESOURCECATS"),
5693 lex_tokss (lexer)))))
5695 /* Parse function. */
5696 enum ctables_summary_function function;
5697 enum ctables_weighting weighting;
5698 enum ctables_area_type area;
5699 if (!parse_ctables_summary_function (lexer, &function, &weighting, &area))
5702 /* Parse percentile. */
5703 double percentile = 0;
5704 if (function == CTSF_PTILE)
5706 if (!lex_force_num_range_closed (lexer, "PTILE", 0, 100))
5708 percentile = lex_number (lexer);
5713 struct fmt_spec format;
5714 bool is_ctables_format;
5715 if (!parse_ctables_format_specifier (lexer, &format, &is_ctables_format))
5718 if (sss->n >= sss->allocated)
5719 sss->specs = x2nrealloc (sss->specs, &sss->allocated,
5720 sizeof *sss->specs);
5721 sss->specs[sss->n++] = (struct ctables_summary_spec) {
5722 .function = function,
5723 .weighting = weighting,
5726 .percentile = percentile,
5728 .is_ctables_format = is_ctables_format,
5734 ctables_summary_spec_set_uninit (sss);
5739 ctables_parse_pproperties (struct lexer *lexer, struct ctables *ct)
5741 struct ctables_postcompute **pcs = NULL;
5743 size_t allocated_pcs = 0;
5745 while (lex_match (lexer, T_AND))
5747 if (!lex_force_id (lexer))
5749 struct ctables_postcompute *pc
5750 = ctables_find_postcompute (ct, lex_tokcstr (lexer));
5753 msg (SE, _("Unknown computed category &%s."), lex_tokcstr (lexer));
5758 if (n_pcs >= allocated_pcs)
5759 pcs = x2nrealloc (pcs, &allocated_pcs, sizeof *pcs);
5763 while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
5765 if (lex_match_id (lexer, "LABEL"))
5767 lex_match (lexer, T_EQUALS);
5768 if (!lex_force_string (lexer))
5771 for (size_t i = 0; i < n_pcs; i++)
5773 free (pcs[i]->label);
5774 pcs[i]->label = ss_xstrdup (lex_tokss (lexer));
5779 else if (lex_match_id (lexer, "FORMAT"))
5781 lex_match (lexer, T_EQUALS);
5783 struct ctables_summary_spec_set sss;
5784 if (!ctables_parse_pproperties_format (lexer, &sss))
5787 for (size_t i = 0; i < n_pcs; i++)
5790 ctables_summary_spec_set_uninit (pcs[i]->specs);
5792 pcs[i]->specs = xmalloc (sizeof *pcs[i]->specs);
5793 ctables_summary_spec_set_clone (pcs[i]->specs, &sss);
5795 ctables_summary_spec_set_uninit (&sss);
5797 else if (lex_match_id (lexer, "HIDESOURCECATS"))
5799 lex_match (lexer, T_EQUALS);
5800 bool hide_source_cats;
5801 if (!parse_bool (lexer, &hide_source_cats))
5803 for (size_t i = 0; i < n_pcs; i++)
5804 pcs[i]->hide_source_cats = hide_source_cats;
5808 lex_error_expecting (lexer, "LABEL", "FORMAT", "HIDESOURCECATS");
5821 put_strftime (struct string *out, time_t now, const char *format)
5823 const struct tm *tm = localtime (&now);
5825 strftime (value, sizeof value, format, tm);
5826 ds_put_cstr (out, value);
5830 skip_prefix (struct substring *s, struct substring prefix)
5832 if (ss_starts_with (*s, prefix))
5834 ss_advance (s, prefix.length);
5842 put_table_expression (struct string *out, struct lexer *lexer,
5843 struct dictionary *dict, int expr_start, int expr_end)
5846 for (int ofs = expr_start; ofs < expr_end; ofs++)
5848 const struct token *t = lex_ofs_token (lexer, ofs);
5849 if (t->type == T_LBRACK)
5851 else if (t->type == T_RBRACK && nest > 0)
5857 else if (t->type == T_ID)
5859 const struct variable *var
5860 = dict_lookup_var (dict, t->string.string);
5861 const char *label = var ? var_get_label (var) : NULL;
5862 ds_put_cstr (out, label ? label : t->string.string);
5866 if (ofs != expr_start && t->type != T_RPAREN && ds_last (out) != ' ')
5867 ds_put_byte (out, ' ');
5869 char *repr = lex_ofs_representation (lexer, ofs, ofs);
5870 ds_put_cstr (out, repr);
5873 if (ofs + 1 != expr_end && t->type != T_LPAREN)
5874 ds_put_byte (out, ' ');
5880 put_title_text (struct string *out, struct substring in, time_t now,
5881 struct lexer *lexer, struct dictionary *dict,
5882 int expr_start, int expr_end)
5886 size_t chunk = ss_find_byte (in, ')');
5887 ds_put_substring (out, ss_head (in, chunk));
5888 ss_advance (&in, chunk);
5889 if (ss_is_empty (in))
5892 if (skip_prefix (&in, ss_cstr (")DATE")))
5893 put_strftime (out, now, "%x");
5894 else if (skip_prefix (&in, ss_cstr (")TIME")))
5895 put_strftime (out, now, "%X");
5896 else if (skip_prefix (&in, ss_cstr (")TABLE")))
5897 put_table_expression (out, lexer, dict, expr_start, expr_end);
5900 ds_put_byte (out, ')');
5901 ss_advance (&in, 1);
5907 cmd_ctables (struct lexer *lexer, struct dataset *ds)
5909 struct casereader *input = NULL;
5911 struct measure_guesser *mg = measure_guesser_create (ds);
5914 input = proc_open (ds);
5915 measure_guesser_run (mg, input);
5916 measure_guesser_destroy (mg);
5919 size_t n_vars = dict_get_n_vars (dataset_dict (ds));
5920 enum ctables_vlabel *vlabels = xnmalloc (n_vars, sizeof *vlabels);
5921 enum settings_value_show tvars = settings_get_show_variables ();
5922 for (size_t i = 0; i < n_vars; i++)
5923 vlabels[i] = (enum ctables_vlabel) tvars;
5925 struct pivot_table_look *look = pivot_table_look_unshare (
5926 pivot_table_look_ref (pivot_table_look_get_default ()));
5927 look->omit_empty = false;
5929 struct ctables *ct = xmalloc (sizeof *ct);
5930 *ct = (struct ctables) {
5931 .dict = dataset_dict (ds),
5933 .ctables_formats = FMT_SETTINGS_INIT,
5935 .postcomputes = HMAP_INITIALIZER (ct->postcomputes),
5938 time_t now = time (NULL);
5943 const char *dot_string;
5944 const char *comma_string;
5946 static const struct ctf ctfs[4] = {
5947 { CTEF_NEGPAREN, "(,,,)", "(...)" },
5948 { CTEF_NEQUAL, "-,N=,,", "-.N=.." },
5949 { CTEF_PAREN, "-,(,),", "-.(.)." },
5950 { CTEF_PCTPAREN, "-,(,%),", "-.(.%)." },
5952 bool is_dot = settings_get_fmt_settings ()->decimal == '.';
5953 for (size_t i = 0; i < 4; i++)
5955 const char *s = is_dot ? ctfs[i].dot_string : ctfs[i].comma_string;
5956 fmt_settings_set_cc (&ct->ctables_formats, ctfs[i].type,
5957 fmt_number_style_from_string (s));
5960 if (!lex_force_match (lexer, T_SLASH))
5963 while (!lex_match_id (lexer, "TABLE"))
5965 if (lex_match_id (lexer, "FORMAT"))
5967 double widths[2] = { SYSMIS, SYSMIS };
5968 double units_per_inch = 72.0;
5970 while (lex_token (lexer) != T_SLASH)
5972 if (lex_match_id (lexer, "MINCOLWIDTH"))
5974 if (!parse_col_width (lexer, "MINCOLWIDTH", &widths[0]))
5977 else if (lex_match_id (lexer, "MAXCOLWIDTH"))
5979 if (!parse_col_width (lexer, "MAXCOLWIDTH", &widths[1]))
5982 else if (lex_match_id (lexer, "UNITS"))
5984 lex_match (lexer, T_EQUALS);
5985 if (lex_match_id (lexer, "POINTS"))
5986 units_per_inch = 72.0;
5987 else if (lex_match_id (lexer, "INCHES"))
5988 units_per_inch = 1.0;
5989 else if (lex_match_id (lexer, "CM"))
5990 units_per_inch = 2.54;
5993 lex_error_expecting (lexer, "POINTS", "INCHES", "CM");
5997 else if (lex_match_id (lexer, "EMPTY"))
6002 lex_match (lexer, T_EQUALS);
6003 if (lex_match_id (lexer, "ZERO"))
6005 /* Nothing to do. */
6007 else if (lex_match_id (lexer, "BLANK"))
6008 ct->zero = xstrdup ("");
6009 else if (lex_force_string (lexer))
6011 ct->zero = ss_xstrdup (lex_tokss (lexer));
6017 else if (lex_match_id (lexer, "MISSING"))
6019 lex_match (lexer, T_EQUALS);
6020 if (!lex_force_string (lexer))
6024 ct->missing = (strcmp (lex_tokcstr (lexer), ".")
6025 ? ss_xstrdup (lex_tokss (lexer))
6031 lex_error_expecting (lexer, "MINCOLWIDTH", "MAXCOLWIDTH",
6032 "UNITS", "EMPTY", "MISSING");
6037 if (widths[0] != SYSMIS && widths[1] != SYSMIS
6038 && widths[0] > widths[1])
6040 msg (SE, _("MINCOLWIDTH must not be greater than MAXCOLWIDTH."));
6044 for (size_t i = 0; i < 2; i++)
6045 if (widths[i] != SYSMIS)
6047 int *wr = ct->look->width_ranges[TABLE_HORZ];
6048 wr[i] = widths[i] / units_per_inch * 96.0;
6053 else if (lex_match_id (lexer, "VLABELS"))
6055 if (!lex_force_match_id (lexer, "VARIABLES"))
6057 lex_match (lexer, T_EQUALS);
6059 struct variable **vars;
6061 if (!parse_variables (lexer, dataset_dict (ds), &vars, &n_vars,
6065 if (!lex_force_match_id (lexer, "DISPLAY"))
6070 lex_match (lexer, T_EQUALS);
6072 enum ctables_vlabel vlabel;
6073 if (lex_match_id (lexer, "DEFAULT"))
6074 vlabel = (enum ctables_vlabel) settings_get_show_variables ();
6075 else if (lex_match_id (lexer, "NAME"))
6077 else if (lex_match_id (lexer, "LABEL"))
6078 vlabel = CTVL_LABEL;
6079 else if (lex_match_id (lexer, "BOTH"))
6081 else if (lex_match_id (lexer, "NONE"))
6085 lex_error_expecting (lexer, "DEFAULT", "NAME", "LABEL",
6091 for (size_t i = 0; i < n_vars; i++)
6092 ct->vlabels[var_get_dict_index (vars[i])] = vlabel;
6095 else if (lex_match_id (lexer, "MRSETS"))
6097 if (!lex_force_match_id (lexer, "COUNTDUPLICATES"))
6099 lex_match (lexer, T_EQUALS);
6100 if (!parse_bool (lexer, &ct->mrsets_count_duplicates))
6103 else if (lex_match_id (lexer, "SMISSING"))
6105 if (lex_match_id (lexer, "VARIABLE"))
6106 ct->smissing_listwise = false;
6107 else if (lex_match_id (lexer, "LISTWISE"))
6108 ct->smissing_listwise = true;
6111 lex_error_expecting (lexer, "VARIABLE", "LISTWISE");
6115 else if (lex_match_id (lexer, "PCOMPUTE"))
6117 if (!ctables_parse_pcompute (lexer, dataset_dict (ds), ct))
6120 else if (lex_match_id (lexer, "PPROPERTIES"))
6122 if (!ctables_parse_pproperties (lexer, ct))
6125 else if (lex_match_id (lexer, "WEIGHT"))
6127 if (!lex_force_match_id (lexer, "VARIABLE"))
6129 lex_match (lexer, T_EQUALS);
6130 ct->e_weight = parse_variable (lexer, dataset_dict (ds));
6134 else if (lex_match_id (lexer, "HIDESMALLCOUNTS"))
6136 if (lex_match_id (lexer, "COUNT"))
6138 lex_match (lexer, T_EQUALS);
6139 if (!lex_force_int_range (lexer, "HIDESMALLCOUNTS COUNT",
6142 ct->hide_threshold = lex_integer (lexer);
6145 else if (ct->hide_threshold == 0)
6146 ct->hide_threshold = 5;
6150 lex_error_expecting (lexer, "FORMAT", "VLABELS", "MRSETS",
6151 "SMISSING", "PCOMPUTE", "PPROPERTIES",
6152 "WEIGHT", "HIDESMALLCOUNTS", "TABLE");
6156 if (!lex_force_match (lexer, T_SLASH))
6160 size_t allocated_tables = 0;
6163 if (ct->n_tables >= allocated_tables)
6164 ct->tables = x2nrealloc (ct->tables, &allocated_tables,
6165 sizeof *ct->tables);
6167 struct ctables_category *cat = xmalloc (sizeof *cat);
6168 *cat = (struct ctables_category) {
6170 .include_missing = false,
6171 .sort_ascending = true,
6174 struct ctables_categories *c = xmalloc (sizeof *c);
6175 size_t n_vars = dict_get_n_vars (dataset_dict (ds));
6176 *c = (struct ctables_categories) {
6183 struct ctables_categories **categories = xnmalloc (n_vars,
6184 sizeof *categories);
6185 for (size_t i = 0; i < n_vars; i++)
6188 struct ctables_table *t = xmalloc (sizeof *t);
6189 *t = (struct ctables_table) {
6191 .slabels_axis = PIVOT_AXIS_COLUMN,
6192 .slabels_visible = true,
6193 .clabels_values_map = HMAP_INITIALIZER (t->clabels_values_map),
6195 [PIVOT_AXIS_ROW] = PIVOT_AXIS_ROW,
6196 [PIVOT_AXIS_COLUMN] = PIVOT_AXIS_COLUMN,
6197 [PIVOT_AXIS_LAYER] = PIVOT_AXIS_LAYER,
6199 .clabels_from_axis = PIVOT_AXIS_LAYER,
6200 .clabels_to_axis = PIVOT_AXIS_LAYER,
6201 .categories = categories,
6202 .n_categories = n_vars,
6205 ct->tables[ct->n_tables++] = t;
6207 lex_match (lexer, T_EQUALS);
6208 int expr_start = lex_ofs (lexer);
6209 if (!ctables_axis_parse (lexer, dataset_dict (ds), ct, t, PIVOT_AXIS_ROW))
6211 if (lex_match (lexer, T_BY))
6213 if (!ctables_axis_parse (lexer, dataset_dict (ds),
6214 ct, t, PIVOT_AXIS_COLUMN))
6217 if (lex_match (lexer, T_BY))
6219 if (!ctables_axis_parse (lexer, dataset_dict (ds),
6220 ct, t, PIVOT_AXIS_LAYER))
6224 int expr_end = lex_ofs (lexer);
6226 if (!t->axes[PIVOT_AXIS_ROW] && !t->axes[PIVOT_AXIS_COLUMN]
6227 && !t->axes[PIVOT_AXIS_LAYER])
6229 lex_error (lexer, _("At least one variable must be specified."));
6233 const struct ctables_axis *scales[PIVOT_N_AXES];
6234 size_t n_scales = 0;
6235 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
6237 scales[a] = find_scale (t->axes[a]);
6243 msg (SE, _("Scale variables may appear only on one axis."));
6244 if (scales[PIVOT_AXIS_ROW])
6245 msg_at (SN, scales[PIVOT_AXIS_ROW]->loc,
6246 _("This scale variable appears on the rows axis."));
6247 if (scales[PIVOT_AXIS_COLUMN])
6248 msg_at (SN, scales[PIVOT_AXIS_COLUMN]->loc,
6249 _("This scale variable appears on the columns axis."));
6250 if (scales[PIVOT_AXIS_LAYER])
6251 msg_at (SN, scales[PIVOT_AXIS_LAYER]->loc,
6252 _("This scale variable appears on the layer axis."));
6256 const struct ctables_axis *summaries[PIVOT_N_AXES];
6257 size_t n_summaries = 0;
6258 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
6260 summaries[a] = (scales[a]
6262 : find_categorical_summary_spec (t->axes[a]));
6266 if (n_summaries > 1)
6268 msg (SE, _("Summaries may appear only on one axis."));
6269 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
6272 msg_at (SN, summaries[a]->loc,
6274 ? _("This variable on the rows axis has a summary.")
6275 : a == PIVOT_AXIS_COLUMN
6276 ? _("This variable on the columns axis has a summary.")
6277 : _("This variable on the layers axis has a summary."));
6279 msg_at (SN, summaries[a]->loc,
6280 _("This is a scale variable, so it always has a "
6281 "summary even if the syntax does not explicitly "
6286 for (enum pivot_axis_type a = 0; a < PIVOT_N_AXES; a++)
6287 if (n_summaries ? summaries[a] : t->axes[a])
6289 t->summary_axis = a;
6293 if (lex_token (lexer) == T_ENDCMD)
6295 if (!ctables_prepare_table (t))
6299 if (!lex_force_match (lexer, T_SLASH))
6302 while (!lex_match_id (lexer, "TABLE") && lex_token (lexer) != T_ENDCMD)
6304 if (lex_match_id (lexer, "SLABELS"))
6306 while (lex_token (lexer) != T_SLASH && lex_token (lexer) != T_ENDCMD)
6308 if (lex_match_id (lexer, "POSITION"))
6310 lex_match (lexer, T_EQUALS);
6311 if (lex_match_id (lexer, "COLUMN"))
6312 t->slabels_axis = PIVOT_AXIS_COLUMN;
6313 else if (lex_match_id (lexer, "ROW"))
6314 t->slabels_axis = PIVOT_AXIS_ROW;
6315 else if (lex_match_id (lexer, "LAYER"))
6316 t->slabels_axis = PIVOT_AXIS_LAYER;
6319 lex_error_expecting (lexer, "COLUMN", "ROW", "LAYER");
6323 else if (lex_match_id (lexer, "VISIBLE"))
6325 lex_match (lexer, T_EQUALS);
6326 if (!parse_bool (lexer, &t->slabels_visible))
6331 lex_error_expecting (lexer, "POSITION", "VISIBLE");
6336 else if (lex_match_id (lexer, "CLABELS"))
6338 if (lex_match_id (lexer, "AUTO"))
6340 t->label_axis[PIVOT_AXIS_ROW] = PIVOT_AXIS_ROW;
6341 t->label_axis[PIVOT_AXIS_COLUMN] = PIVOT_AXIS_COLUMN;
6343 else if (lex_match_id (lexer, "ROWLABELS"))
6345 lex_match (lexer, T_EQUALS);
6346 if (lex_match_id (lexer, "OPPOSITE"))
6347 t->label_axis[PIVOT_AXIS_ROW] = PIVOT_AXIS_COLUMN;
6348 else if (lex_match_id (lexer, "LAYER"))
6349 t->label_axis[PIVOT_AXIS_ROW] = PIVOT_AXIS_LAYER;
6352 lex_error_expecting (lexer, "OPPOSITE", "LAYER");
6356 else if (lex_match_id (lexer, "COLLABELS"))
6358 lex_match (lexer, T_EQUALS);
6359 if (lex_match_id (lexer, "OPPOSITE"))
6360 t->label_axis[PIVOT_AXIS_COLUMN] = PIVOT_AXIS_ROW;
6361 else if (lex_match_id (lexer, "LAYER"))
6362 t->label_axis[PIVOT_AXIS_COLUMN] = PIVOT_AXIS_LAYER;
6365 lex_error_expecting (lexer, "OPPOSITE", "LAYER");
6371 lex_error_expecting (lexer, "AUTO", "ROWLABELS",
6376 else if (lex_match_id (lexer, "CRITERIA"))
6378 if (!lex_force_match_id (lexer, "CILEVEL"))
6380 lex_match (lexer, T_EQUALS);
6382 if (!lex_force_num_range_halfopen (lexer, "CILEVEL", 0, 100))
6384 t->cilevel = lex_number (lexer);
6387 else if (lex_match_id (lexer, "CATEGORIES"))
6389 if (!ctables_table_parse_categories (lexer, dataset_dict (ds),
6393 else if (lex_match_id (lexer, "TITLES"))
6398 if (lex_match_id (lexer, "CAPTION"))
6399 textp = &t->caption;
6400 else if (lex_match_id (lexer, "CORNER"))
6402 else if (lex_match_id (lexer, "TITLE"))
6406 lex_error_expecting (lexer, "CAPTION", "CORNER", "TITLE");
6409 lex_match (lexer, T_EQUALS);
6411 struct string s = DS_EMPTY_INITIALIZER;
6412 while (lex_is_string (lexer))
6414 if (!ds_is_empty (&s))
6415 ds_put_byte (&s, ' ');
6416 put_title_text (&s, lex_tokss (lexer), now,
6417 lexer, dataset_dict (ds),
6418 expr_start, expr_end);
6422 *textp = ds_steal_cstr (&s);
6424 while (lex_token (lexer) != T_SLASH
6425 && lex_token (lexer) != T_ENDCMD);
6427 else if (lex_match_id (lexer, "SIGTEST"))
6429 int start_ofs = lex_ofs (lexer) - 1;
6432 t->chisq = xmalloc (sizeof *t->chisq);
6433 *t->chisq = (struct ctables_chisq) {
6435 .include_mrsets = true,
6436 .all_visible = true,
6442 if (lex_match_id (lexer, "TYPE"))
6444 lex_match (lexer, T_EQUALS);
6445 if (!lex_force_match_id (lexer, "CHISQUARE"))
6448 else if (lex_match_id (lexer, "ALPHA"))
6450 lex_match (lexer, T_EQUALS);
6451 if (!lex_force_num_range_halfopen (lexer, "ALPHA", 0, 1))
6453 t->chisq->alpha = lex_number (lexer);
6456 else if (lex_match_id (lexer, "INCLUDEMRSETS"))
6458 lex_match (lexer, T_EQUALS);
6459 if (!parse_bool (lexer, &t->chisq->include_mrsets))
6462 else if (lex_match_id (lexer, "CATEGORIES"))
6464 lex_match (lexer, T_EQUALS);
6465 if (lex_match_id (lexer, "ALLVISIBLE"))
6466 t->chisq->all_visible = true;
6467 else if (lex_match_id (lexer, "SUBTOTALS"))
6468 t->chisq->all_visible = false;
6471 lex_error_expecting (lexer,
6472 "ALLVISIBLE", "SUBTOTALS");
6478 lex_error_expecting (lexer, "TYPE", "ALPHA",
6479 "INCLUDEMRSETS", "CATEGORIES");
6483 while (lex_token (lexer) != T_SLASH
6484 && lex_token (lexer) != T_ENDCMD);
6486 lex_ofs_error (lexer, start_ofs, lex_ofs (lexer) - 1,
6487 _("Support for SIGTEST not yet implemented."));
6490 else if (lex_match_id (lexer, "COMPARETEST"))
6492 int start_ofs = lex_ofs (lexer);
6495 t->pairwise = xmalloc (sizeof *t->pairwise);
6496 *t->pairwise = (struct ctables_pairwise) {
6498 .alpha = { .05, .05 },
6499 .adjust = BONFERRONI,
6500 .include_mrsets = true,
6501 .meansvariance_allcats = true,
6502 .all_visible = true,
6511 if (lex_match_id (lexer, "TYPE"))
6513 lex_match (lexer, T_EQUALS);
6514 if (lex_match_id (lexer, "PROP"))
6515 t->pairwise->type = PROP;
6516 else if (lex_match_id (lexer, "MEAN"))
6517 t->pairwise->type = MEAN;
6520 lex_error_expecting (lexer, "PROP", "MEAN");
6524 else if (lex_match_id (lexer, "ALPHA"))
6526 lex_match (lexer, T_EQUALS);
6528 if (!lex_force_num_range_open (lexer, "ALPHA", 0, 1))
6530 double a0 = lex_number (lexer);
6533 lex_match (lexer, T_COMMA);
6534 if (lex_is_number (lexer))
6536 if (!lex_force_num_range_open (lexer, "ALPHA", 0, 1))
6538 double a1 = lex_number (lexer);
6541 t->pairwise->alpha[0] = MIN (a0, a1);
6542 t->pairwise->alpha[1] = MAX (a0, a1);
6545 t->pairwise->alpha[0] = t->pairwise->alpha[1] = a0;
6547 else if (lex_match_id (lexer, "ADJUST"))
6549 lex_match (lexer, T_EQUALS);
6550 if (lex_match_id (lexer, "BONFERRONI"))
6551 t->pairwise->adjust = BONFERRONI;
6552 else if (lex_match_id (lexer, "BH"))
6553 t->pairwise->adjust = BH;
6554 else if (lex_match_id (lexer, "NONE"))
6555 t->pairwise->adjust = 0;
6558 lex_error_expecting (lexer, "BONFERRONI", "BH",
6563 else if (lex_match_id (lexer, "INCLUDEMRSETS"))
6565 lex_match (lexer, T_EQUALS);
6566 if (!parse_bool (lexer, &t->pairwise->include_mrsets))
6569 else if (lex_match_id (lexer, "MEANSVARIANCE"))
6571 lex_match (lexer, T_EQUALS);
6572 if (lex_match_id (lexer, "ALLCATS"))
6573 t->pairwise->meansvariance_allcats = true;
6574 else if (lex_match_id (lexer, "TESTEDCATS"))
6575 t->pairwise->meansvariance_allcats = false;
6578 lex_error_expecting (lexer, "ALLCATS", "TESTEDCATS");
6582 else if (lex_match_id (lexer, "CATEGORIES"))
6584 lex_match (lexer, T_EQUALS);
6585 if (lex_match_id (lexer, "ALLVISIBLE"))
6586 t->pairwise->all_visible = true;
6587 else if (lex_match_id (lexer, "SUBTOTALS"))
6588 t->pairwise->all_visible = false;
6591 lex_error_expecting (lexer, "ALLVISIBLE",
6596 else if (lex_match_id (lexer, "MERGE"))
6598 lex_match (lexer, T_EQUALS);
6599 if (!parse_bool (lexer, &t->pairwise->merge))
6602 else if (lex_match_id (lexer, "STYLE"))
6604 lex_match (lexer, T_EQUALS);
6605 if (lex_match_id (lexer, "APA"))
6606 t->pairwise->apa_style = true;
6607 else if (lex_match_id (lexer, "SIMPLE"))
6608 t->pairwise->apa_style = false;
6611 lex_error_expecting (lexer, "APA", "SIMPLE");
6615 else if (lex_match_id (lexer, "SHOWSIG"))
6617 lex_match (lexer, T_EQUALS);
6618 if (!parse_bool (lexer, &t->pairwise->show_sig))
6623 lex_error_expecting (lexer, "TYPE", "ALPHA", "ADJUST",
6624 "INCLUDEMRSETS", "MEANSVARIANCE",
6625 "CATEGORIES", "MERGE", "STYLE",
6630 while (lex_token (lexer) != T_SLASH
6631 && lex_token (lexer) != T_ENDCMD);
6633 lex_ofs_error (lexer, start_ofs, lex_ofs (lexer) - 1,
6634 _("Support for COMPARETEST not yet implemented."));
6639 lex_error_expecting (lexer, "TABLE", "SLABELS", "CLABELS",
6640 "CRITERIA", "CATEGORIES", "TITLES",
6641 "SIGTEST", "COMPARETEST");
6645 if (!lex_match (lexer, T_SLASH))
6649 if (t->label_axis[PIVOT_AXIS_ROW] != PIVOT_AXIS_ROW)
6651 t->clabels_from_axis = PIVOT_AXIS_ROW;
6652 if (t->label_axis[PIVOT_AXIS_COLUMN] != PIVOT_AXIS_COLUMN)
6654 msg (SE, _("ROWLABELS and COLLABELS may not both be specified."));
6658 else if (t->label_axis[PIVOT_AXIS_COLUMN] != PIVOT_AXIS_COLUMN)
6659 t->clabels_from_axis = PIVOT_AXIS_COLUMN;
6660 t->clabels_to_axis = t->label_axis[t->clabels_from_axis];
6662 if (!ctables_prepare_table (t))
6665 while (lex_token (lexer) != T_ENDCMD);
6668 input = proc_open (ds);
6669 bool ok = ctables_execute (ds, input, ct);
6670 ok = proc_commit (ds) && ok;
6672 ctables_destroy (ct);
6673 return ok ? CMD_SUCCESS : CMD_FAILURE;
6678 ctables_destroy (ct);