1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/lexer/variable-parser.h"
26 #include "data/dataset.h"
27 #include "data/dictionary.h"
28 #include "data/variable.h"
29 #include "language/lexer/lexer.h"
30 #include "libpspp/assertion.h"
31 #include "libpspp/cast.h"
32 #include "libpspp/hash-functions.h"
33 #include "libpspp/hmapx.h"
34 #include "libpspp/message.h"
35 #include "libpspp/misc.h"
36 #include "libpspp/pool.h"
37 #include "libpspp/str.h"
38 #include "libpspp/stringi-set.h"
40 #include "math/interaction.h"
42 #include "gl/c-ctype.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
48 static struct variable * var_set_get_var (const struct var_set *, size_t );
50 static struct variable *var_set_lookup_var (const struct var_set *,
53 static bool var_set_lookup_var_idx (const struct var_set *, const char *,
58 /* Parses a name as a variable within VS. Sets *IDX to the
59 variable's index and returns true if successful. On failure
60 emits an error message and returns false. */
62 parse_vs_variable_idx (struct lexer *lexer, const struct var_set *vs,
67 if (lex_token (lexer) != T_ID)
69 lex_error (lexer, _("expecting variable name"));
72 else if (var_set_lookup_var_idx (vs, lex_tokcstr (lexer), idx))
79 msg (SE, _("%s is not a variable name."), lex_tokcstr (lexer));
84 /* Parses a name as a variable within VS and returns the variable
85 if successful. On failure emits an error message and returns
87 static struct variable *
88 parse_vs_variable (struct lexer *lexer, const struct var_set *vs)
91 return parse_vs_variable_idx (lexer, vs, &idx) ? var_set_get_var (vs, idx) : NULL;
94 /* Parses a variable name in dictionary D and returns the
95 variable if successful. On failure emits an error message and
96 returns a null pointer. */
98 parse_variable (struct lexer *lexer, const struct dictionary *d)
100 struct var_set *vs = var_set_create_from_dict (d);
101 struct variable *var = parse_vs_variable (lexer, vs);
102 var_set_destroy (vs);
106 /* Parses a set of variables from dictionary D given options
107 OPTS. Resulting list of variables stored in *VAR and the
108 number of variables into *CNT. Returns true only if
111 parse_variables (struct lexer *lexer, const struct dictionary *d,
112 struct variable ***var,
113 size_t *cnt, int opts)
119 assert (var != NULL);
120 assert (cnt != NULL);
122 vs = var_set_create_from_dict (d);
123 success = parse_var_set_vars (lexer, vs, var, cnt, opts);
124 var_set_destroy (vs);
128 /* Parses a set of variables from dictionary D given options
129 OPTS. Resulting list of variables stored in *VARS and the
130 number of variables into *VAR_CNT. Returns true only if
131 successful. Same behavior as parse_variables, except that all
132 allocations are taken from the given POOL. */
134 parse_variables_pool (struct lexer *lexer, struct pool *pool,
135 const struct dictionary *dict,
136 struct variable ***vars, size_t *var_cnt, int opts)
140 /* PV_APPEND is unsafe because parse_variables would free the
141 existing names on failure, but those names are presumably
142 already in the pool, which would attempt to re-free it
144 assert (!(opts & PV_APPEND));
146 retval = parse_variables (lexer, dict, vars, var_cnt, opts);
148 pool_register (pool, free, *vars);
152 /* Parses a variable name from VS. If successful, sets *IDX to
153 the variable's index in VS, *CLASS to the variable's
154 dictionary class, and returns true. Returns false on
157 parse_var_idx_class (struct lexer *lexer, const struct var_set *vs,
159 enum dict_class *class)
161 if (!parse_vs_variable_idx (lexer, vs, idx))
164 *class = dict_class_from_id (var_get_name (var_set_get_var (vs, *idx)));
168 /* Add the variable from VS with index IDX to the list of
169 variables V that has *NV elements and room for *MV.
170 Uses and updates INCLUDED to avoid duplicates if indicated by
171 PV_OPTS, which also affects what variables are allowed in
174 add_variable (struct variable ***v, size_t *nv, size_t *mv,
175 char *included, int pv_opts,
176 const struct var_set *vs, size_t idx)
178 struct variable *add = var_set_get_var (vs, idx);
179 const char *add_name = var_get_name (add);
181 if ((pv_opts & PV_NUMERIC) && !var_is_numeric (add))
182 msg (SW, _("%s is not a numeric variable. It will not be "
183 "included in the variable list."), add_name);
184 else if ((pv_opts & PV_STRING) && !var_is_alpha (add))
185 msg (SE, _("%s is not a string variable. It will not be "
186 "included in the variable list."), add_name);
187 else if ((pv_opts & PV_NO_SCRATCH)
188 && dict_class_from_id (add_name) == DC_SCRATCH)
189 msg (SE, _("Scratch variables (such as %s) are not allowed "
191 else if ((pv_opts & (PV_SAME_TYPE | PV_SAME_WIDTH)) && *nv
192 && var_get_type (add) != var_get_type ((*v)[0]))
193 msg (SE, _("%s and %s are not the same type. All variables in "
194 "this variable list must be of the same type. %s "
195 "will be omitted from the list."),
196 var_get_name ((*v)[0]), add_name, add_name);
197 else if ((pv_opts & PV_SAME_WIDTH) && *nv
198 && var_get_width (add) != var_get_width ((*v)[0]))
199 msg (SE, _("%s and %s are string variables with different widths. "
200 "All variables in this variable list must have the "
201 "same width. %s will be omitted from the list."),
202 var_get_name ((*v)[0]), add_name, add_name);
203 else if ((pv_opts & PV_NO_DUPLICATE) && included[idx])
204 msg (SE, _("Variable %s appears twice in variable list."), add_name);
205 else if ((pv_opts & PV_DUPLICATE) || !included[idx])
210 *v = xnrealloc (*v, *mv, sizeof **v);
213 if (included != NULL)
218 /* Adds the variables in VS with indexes FIRST_IDX through
219 LAST_IDX, inclusive, to the list of variables V that has *NV
220 elements and room for *MV. Uses and updates INCLUDED to avoid
221 duplicates if indicated by PV_OPTS, which also affects what
222 variables are allowed in appropriate ways. */
224 add_variables (struct variable ***v, size_t *nv, size_t *mv, char *included,
226 const struct var_set *vs, int first_idx, int last_idx,
227 enum dict_class class)
231 for (i = first_idx; i <= last_idx; i++)
232 if (dict_class_from_id (var_get_name (var_set_get_var (vs, i))) == class)
233 add_variable (v, nv, mv, included, pv_opts, vs, i);
236 /* Note that if parse_variables() returns false, *v is free()'d.
237 Conversely, if parse_variables() returns true, then *nv is
238 nonzero and *v is non-NULL. */
240 parse_var_set_vars (struct lexer *lexer, const struct var_set *vs,
241 struct variable ***v, size_t *nv,
251 /* At most one of PV_NUMERIC, PV_STRING, PV_SAME_TYPE,
252 PV_SAME_WIDTH may be specified. */
253 assert (((pv_opts & PV_NUMERIC) != 0)
254 + ((pv_opts & PV_STRING) != 0)
255 + ((pv_opts & PV_SAME_TYPE) != 0)
256 + ((pv_opts & PV_SAME_WIDTH) != 0) <= 1);
258 /* PV_DUPLICATE and PV_NO_DUPLICATE are incompatible. */
259 assert (!(pv_opts & PV_DUPLICATE) || !(pv_opts & PV_NO_DUPLICATE));
261 if (!(pv_opts & PV_APPEND))
270 if (!(pv_opts & PV_DUPLICATE))
274 included = xcalloc (var_set_get_cnt (vs), sizeof *included);
275 for (i = 0; i < *nv; i++)
278 if (!var_set_lookup_var_idx (vs, var_get_name ((*v)[i]), &index))
288 if (lex_match (lexer, T_ALL))
289 add_variables (v, nv, &mv, included, pv_opts,
290 vs, 0, var_set_get_cnt (vs) - 1, DC_ORDINARY);
293 enum dict_class class;
296 if (!parse_var_idx_class (lexer, vs, &first_idx, &class))
299 if (!lex_match (lexer, T_TO))
300 add_variable (v, nv, &mv, included, pv_opts, vs, first_idx);
304 enum dict_class last_class;
305 struct variable *first_var, *last_var;
307 if (!parse_var_idx_class (lexer, vs, &last_idx, &last_class))
310 first_var = var_set_get_var (vs, first_idx);
311 last_var = var_set_get_var (vs, last_idx);
313 if (last_idx < first_idx)
315 const char *first_name = var_get_name (first_var);
316 const char *last_name = var_get_name (last_var);
317 msg (SE, _("%s TO %s is not valid syntax since %s "
318 "precedes %s in the dictionary."),
319 first_name, last_name, first_name, last_name);
323 if (class != last_class)
325 msg (SE, _("When using the TO keyword to specify several "
326 "variables, both variables must be from "
327 "the same variable dictionaries, of either "
328 "ordinary, scratch, or system variables. "
329 "%s is a %s variable, whereas %s is %s."),
330 var_get_name (first_var), dict_class_to_name (class),
331 var_get_name (last_var),
332 dict_class_to_name (last_class));
336 add_variables (v, nv, &mv, included, pv_opts,
337 vs, first_idx, last_idx, class);
341 if (pv_opts & PV_SINGLE)
343 lex_match (lexer, T_COMMA);
345 while (lex_token (lexer) == T_ALL
346 || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokcstr (lexer)) != NULL));
362 /* Attempts to break UTF-8 encoded NAME into a root (whose contents are
363 arbitrary except that it does not end in a digit) followed by an integer
364 numeric suffix. On success, stores the value of the suffix into *NUMBERP,
365 the number of digits in the suffix into *N_DIGITSP, and returns the number
366 of bytes in the root. On failure, returns 0. */
368 extract_numeric_suffix (const char *name,
369 unsigned long int *numberp, int *n_digitsp)
371 size_t root_len, n_digits;
374 /* Count length of root. */
375 root_len = 1; /* Valid identifier never starts with digit. */
376 for (i = 1; name[i] != '\0'; i++)
377 if (!c_isdigit (name[i]))
379 n_digits = i - root_len;
383 msg (SE, _("`%s' cannot be used with TO because it does not end in "
388 *numberp = strtoull (name + root_len, NULL, 10);
389 if (*numberp == ULONG_MAX)
391 msg (SE, _("Numeric suffix on `%s' is larger than supported with TO."),
395 *n_digitsp = n_digits;
400 add_var_name (char *name,
401 char ***names, size_t *n_vars, size_t *allocated_vars,
402 struct stringi_set *set, int pv_opts)
404 if (pv_opts & PV_NO_DUPLICATE && !stringi_set_insert (set, name))
406 msg (SE, _("Variable %s appears twice in variable list."),
411 if (*n_vars >= *allocated_vars)
412 *names = x2nrealloc (*names, allocated_vars, sizeof **names);
413 (*names)[(*n_vars)++] = name;
417 /* Parses a list of variable names according to the DATA LIST version
418 of the TO convention. */
420 parse_DATA_LIST_vars (struct lexer *lexer, const struct dictionary *dict,
421 char ***namesp, size_t *n_varsp, int pv_opts)
425 size_t allocated_vars;
427 struct stringi_set set;
433 assert ((pv_opts & ~(PV_APPEND | PV_SINGLE
434 | PV_NO_SCRATCH | PV_NO_DUPLICATE)) == 0);
435 stringi_set_init (&set);
437 if (pv_opts & PV_APPEND)
439 n_vars = allocated_vars = *n_varsp;
442 if (pv_opts & PV_NO_DUPLICATE)
446 for (i = 0; i < n_vars; i++)
447 stringi_set_insert (&set, names[i]);
452 n_vars = allocated_vars = 0;
458 if (lex_token (lexer) != T_ID
459 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
461 lex_error (lexer, "expecting variable name");
464 if (dict_class_from_id (lex_tokcstr (lexer)) == DC_SCRATCH
465 && (pv_opts & PV_NO_SCRATCH))
467 msg (SE, _("Scratch variables not allowed here."));
470 name1 = xstrdup (lex_tokcstr (lexer));
472 if (lex_token (lexer) == T_TO)
474 unsigned long int num1, num2;
475 int n_digits1, n_digits2;
476 int root_len1, root_len2;
477 unsigned long int number;
480 if (lex_token (lexer) != T_ID
481 || !dict_id_is_valid (dict, lex_tokcstr (lexer), true))
483 lex_error (lexer, "expecting variable name");
486 name2 = xstrdup (lex_tokcstr (lexer));
489 root_len1 = extract_numeric_suffix (name1, &num1, &n_digits1);
493 root_len2 = extract_numeric_suffix (name2, &num2, &n_digits2);
497 if (root_len1 != root_len2 || memcasecmp (name1, name2, root_len1))
499 msg (SE, _("Prefixes don't match in use of TO convention."));
504 msg (SE, _("Bad bounds in use of TO convention."));
508 for (number = num1; number <= num2; number++)
510 char *name = xasprintf ("%.*s%0*lu",
513 if (!add_var_name (name, &names, &n_vars, &allocated_vars,
528 if (!add_var_name (name1, &names, &n_vars, &allocated_vars,
534 lex_match (lexer, T_COMMA);
536 if (pv_opts & PV_SINGLE)
539 while (lex_token (lexer) == T_ID);
543 stringi_set_destroy (&set);
552 for (i = 0; i < n_vars; i++)
564 /* Registers each of the NAMES[0...NNAMES - 1] in POOL, as well
567 register_vars_pool (struct pool *pool, char **names, size_t nnames)
571 for (i = 0; i < nnames; i++)
572 pool_register (pool, free, names[i]);
573 pool_register (pool, free, names);
576 /* Parses a list of variable names according to the DATA LIST
577 version of the TO convention. Same args as
578 parse_DATA_LIST_vars(), except that all allocations are taken
579 from the given POOL. */
581 parse_DATA_LIST_vars_pool (struct lexer *lexer, const struct dictionary *dict,
583 char ***names, size_t *nnames, int pv_opts)
587 /* PV_APPEND is unsafe because parse_DATA_LIST_vars would free
588 the existing names on failure, but those names are
589 presumably already in the pool, which would attempt to
591 assert (!(pv_opts & PV_APPEND));
593 retval = parse_DATA_LIST_vars (lexer, dict, names, nnames, pv_opts);
595 register_vars_pool (pool, *names, *nnames);
599 /* Parses a list of variables where some of the variables may be
600 existing and the rest are to be created. Same args as
601 parse_DATA_LIST_vars(). */
603 parse_mixed_vars (struct lexer *lexer, const struct dictionary *dict,
604 char ***names, size_t *nnames, int pv_opts)
608 assert (names != NULL);
609 assert (nnames != NULL);
610 assert ((pv_opts & ~PV_APPEND) == 0);
612 if (!(pv_opts & PV_APPEND))
617 while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
619 if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
624 if (!parse_variables (lexer, dict, &v, &nv, PV_NONE))
626 *names = xnrealloc (*names, *nnames + nv, sizeof **names);
627 for (i = 0; i < nv; i++)
628 (*names)[*nnames + i] = xstrdup (var_get_name (v[i]));
632 else if (!parse_DATA_LIST_vars (lexer, dict, names, nnames, PV_APPEND))
638 for (i = 0; i < *nnames; i++)
646 /* Parses a list of variables where some of the variables may be
647 existing and the rest are to be created. Same args as
648 parse_mixed_vars(), except that all allocations are taken
649 from the given POOL. */
651 parse_mixed_vars_pool (struct lexer *lexer, const struct dictionary *dict, struct pool *pool,
652 char ***names, size_t *nnames, int pv_opts)
656 /* PV_APPEND is unsafe because parse_mixed_vars_pool would free
657 the existing names on failure, but those names are
658 presumably already in the pool, which would attempt to
660 assert (!(pv_opts & PV_APPEND));
662 retval = parse_mixed_vars (lexer, dict, names, nnames, pv_opts);
664 register_vars_pool (pool, *names, *nnames);
668 /* A set of variables. */
671 size_t (*get_cnt) (const struct var_set *);
672 struct variable *(*get_var) (const struct var_set *, size_t idx);
673 bool (*lookup_var_idx) (const struct var_set *, const char *, size_t *);
674 void (*destroy) (struct var_set *);
678 /* Returns the number of variables in VS. */
680 var_set_get_cnt (const struct var_set *vs)
684 return vs->get_cnt (vs);
687 /* Return variable with index IDX in VS.
688 IDX must be less than the number of variables in VS. */
689 static struct variable *
690 var_set_get_var (const struct var_set *vs, size_t idx)
693 assert (idx < var_set_get_cnt (vs));
695 return vs->get_var (vs, idx);
698 /* Returns the variable in VS named NAME, or a null pointer if VS
699 contains no variable with that name. */
701 var_set_lookup_var (const struct var_set *vs, const char *name)
704 return (var_set_lookup_var_idx (vs, name, &idx)
705 ? var_set_get_var (vs, idx)
709 /* If VS contains a variable named NAME, sets *IDX to its index
710 and returns true. Otherwise, returns false. */
712 var_set_lookup_var_idx (const struct var_set *vs, const char *name,
716 assert (name != NULL);
718 return vs->lookup_var_idx (vs, name, idx);
723 var_set_destroy (struct var_set *vs)
729 /* Returns the number of variables in VS. */
731 dict_var_set_get_cnt (const struct var_set *vs)
733 struct dictionary *d = vs->aux;
735 return dict_get_var_cnt (d);
738 /* Return variable with index IDX in VS.
739 IDX must be less than the number of variables in VS. */
740 static struct variable *
741 dict_var_set_get_var (const struct var_set *vs, size_t idx)
743 struct dictionary *d = vs->aux;
745 return dict_get_var (d, idx);
748 /* If VS contains a variable named NAME, sets *IDX to its index
749 and returns true. Otherwise, returns false. */
751 dict_var_set_lookup_var_idx (const struct var_set *vs, const char *name,
754 struct dictionary *d = vs->aux;
755 struct variable *v = dict_lookup_var (d, name);
758 *idx = var_get_dict_index (v);
767 dict_var_set_destroy (struct var_set *vs)
772 /* Returns a variable set based on D. */
774 var_set_create_from_dict (const struct dictionary *d)
776 struct var_set *vs = xmalloc (sizeof *vs);
777 vs->get_cnt = dict_var_set_get_cnt;
778 vs->get_var = dict_var_set_get_var;
779 vs->lookup_var_idx = dict_var_set_lookup_var_idx;
780 vs->destroy = dict_var_set_destroy;
781 vs->aux = (void *) d;
785 /* A variable set based on an array. */
788 struct variable *const *var;/* Array of variables. */
789 size_t var_cnt; /* Number of elements in var. */
790 struct hmapx vars_by_name; /* Variables hashed by name. */
793 /* Returns the number of variables in VS. */
795 array_var_set_get_cnt (const struct var_set *vs)
797 struct array_var_set *avs = vs->aux;
802 /* Return variable with index IDX in VS.
803 IDX must be less than the number of variables in VS. */
804 static struct variable *
805 array_var_set_get_var (const struct var_set *vs, size_t idx)
807 struct array_var_set *avs = vs->aux;
809 return CONST_CAST (struct variable *, avs->var[idx]);
812 /* If VS contains a variable named NAME, sets *IDX to its index
813 and returns true. Otherwise, returns false. */
815 array_var_set_lookup_var_idx (const struct var_set *vs, const char *name,
818 struct array_var_set *avs = vs->aux;
819 struct hmapx_node *node;
820 struct variable **varp;
822 HMAPX_FOR_EACH_WITH_HASH (varp, node, hash_case_string (name, 0),
824 if (!strcasecmp (name, var_get_name (*varp)))
826 *idx = varp - avs->var;
835 array_var_set_destroy (struct var_set *vs)
837 struct array_var_set *avs = vs->aux;
839 hmapx_destroy (&avs->vars_by_name);
844 /* Returns a variable set based on the VAR_CNT variables in VAR. */
846 var_set_create_from_array (struct variable *const *var, size_t var_cnt)
849 struct array_var_set *avs;
852 vs = xmalloc (sizeof *vs);
853 vs->get_cnt = array_var_set_get_cnt;
854 vs->get_var = array_var_set_get_var;
855 vs->lookup_var_idx = array_var_set_lookup_var_idx;
856 vs->destroy = array_var_set_destroy;
857 vs->aux = avs = xmalloc (sizeof *avs);
859 avs->var_cnt = var_cnt;
860 hmapx_init (&avs->vars_by_name);
861 for (i = 0; i < var_cnt; i++)
863 const char *name = var_get_name (var[i]);
866 if (array_var_set_lookup_var_idx (vs, name, &idx))
868 var_set_destroy (vs);
871 hmapx_insert (&avs->vars_by_name, CONST_CAST (void *, &avs->var[i]),
872 hash_case_string (name, 0));
880 If the match succeeds, the variable will be placed in VAR.
881 Returns true if successful */
883 lex_match_variable (struct lexer *lexer, const struct dictionary *dict, const struct variable **var)
885 if (lex_token (lexer) != T_ID)
888 *var = parse_variable_const (lexer, dict);
895 /* An interaction is a variable followed by {*, BY} followed by an interaction */
897 parse_internal_interaction (struct lexer *lexer, const struct dictionary *dict, struct interaction **iact, struct interaction **it)
899 const struct variable *v = NULL;
902 switch (lex_next_token (lexer, 1))
916 if (! lex_match_variable (lexer, dict, &v))
919 interaction_destroy (*it);
927 *iact = interaction_create (v);
929 interaction_add_variable (*iact, v);
931 if ( lex_match (lexer, T_ASTERISK) || lex_match (lexer, T_BY))
933 return parse_internal_interaction (lexer, dict, iact, iact);
940 parse_design_interaction (struct lexer *lexer, const struct dictionary *dict, struct interaction **iact)
942 return parse_internal_interaction (lexer, dict, iact, NULL);