1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
59 struct vardict_info *var; /* Variables. */
60 size_t var_cnt, var_cap; /* Number of variables, capacity. */
61 struct caseproto *proto; /* Prototype for dictionary cases
63 struct hmap name_map; /* Variable index by name. */
64 int next_value_idx; /* Index of next `union value' to allocate. */
65 const struct variable **split; /* SPLIT FILE vars. */
66 size_t split_cnt; /* SPLIT FILE count. */
67 struct variable *weight; /* WEIGHT variable. */
68 struct variable *filter; /* FILTER variable. */
69 casenumber case_limit; /* Current case limit (N command). */
70 char *label; /* File label. */
71 struct string_array documents; /* Documents. */
72 struct vector **vector; /* Vectors of variables. */
73 size_t vector_cnt; /* Number of vectors. */
74 struct attrset attributes; /* Custom attributes. */
75 struct mrset **mrsets; /* Multiple response sets. */
76 size_t n_mrsets; /* Number of multiple response sets. */
78 /* Whether variable names must be valid identifiers. Normally, this is
79 true, but sometimes a dictionary is prepared for external use
80 (e.g. output to a CSV file) where names don't have to be valid. */
81 bool names_must_be_ids;
83 char *encoding; /* Character encoding of string data */
85 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
87 void *cb_data ; /* Data passed to callbacks */
89 void (*changed) (struct dictionary *, void *); /* Generic change callback */
93 static void dict_unset_split_var (struct dictionary *, struct variable *);
94 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
96 /* Returns the encoding for data in dictionary D. The return value is a
97 nonnull string that contains an IANA character set name. */
99 dict_get_encoding (const struct dictionary *d)
104 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
105 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
106 error message on failure. */
108 dict_id_is_valid (const struct dictionary *dict, const char *id,
111 return (!dict->names_must_be_ids
112 || id_is_valid (id, dict->encoding, issue_error));
116 dict_set_change_callback (struct dictionary *d,
117 void (*changed) (struct dictionary *, void*),
120 d->changed = changed;
121 d->changed_data = data;
124 /* Discards dictionary D's caseproto. (It will be regenerated
125 lazily, on demand.) */
127 invalidate_proto (struct dictionary *d)
129 caseproto_unref (d->proto);
133 /* Print a representation of dictionary D to stdout, for
134 debugging purposes. */
136 dict_dump (const struct dictionary *d)
139 for (i = 0 ; i < d->var_cnt ; ++i )
141 const struct variable *v = d->var[i].var;
142 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
144 var_get_dict_index (v),
145 var_get_case_index (v));
150 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
151 the dictionary or any of the variables it contains are modified.
152 Each callback will get passed CALLBACK_DATA.
153 Any callback may be NULL, in which case it'll be ignored.
156 dict_set_callbacks (struct dictionary *dict,
157 const struct dict_callbacks *callbacks,
160 dict->callbacks = callbacks;
161 dict->cb_data = callback_data;
164 /* Shallow copy the callbacks from SRC to DEST */
166 dict_copy_callbacks (struct dictionary *dest,
167 const struct dictionary *src)
169 dest->callbacks = src->callbacks;
170 dest->cb_data = src->cb_data;
173 /* Creates and returns a new dictionary with the specified ENCODING. */
175 dict_create (const char *encoding)
177 struct dictionary *d = xzalloc (sizeof *d);
179 d->encoding = xstrdup (encoding);
180 d->names_must_be_ids = true;
181 hmap_init (&d->name_map);
182 attrset_init (&d->attributes);
189 dict_ref (struct dictionary *s)
195 /* Creates and returns a (deep) copy of an existing
198 The new dictionary's case indexes are copied from the old
199 dictionary. If the new dictionary won't be used to access
200 cases produced with the old dictionary, then the new
201 dictionary's case indexes should be compacted with
202 dict_compact_values to save space.
204 Callbacks are not cloned. */
206 dict_clone (const struct dictionary *s)
208 struct dictionary *d;
211 d = dict_create (s->encoding);
212 dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));
214 for (i = 0; i < s->var_cnt; i++)
216 struct variable *sv = s->var[i].var;
217 struct variable *dv = dict_clone_var_assert (d, sv);
220 for (i = 0; i < var_get_short_name_cnt (sv); i++)
221 var_set_short_name (dv, i, var_get_short_name (sv, i));
223 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
226 d->next_value_idx = s->next_value_idx;
228 d->split_cnt = s->split_cnt;
229 if (d->split_cnt > 0)
231 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
232 for (i = 0; i < d->split_cnt; i++)
233 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
236 if (s->weight != NULL)
237 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
239 if (s->filter != NULL)
240 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
242 d->case_limit = s->case_limit;
243 dict_set_label (d, dict_get_label (s));
244 dict_set_documents (d, dict_get_documents (s));
246 d->vector_cnt = s->vector_cnt;
247 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
248 for (i = 0; i < s->vector_cnt; i++)
249 d->vector[i] = vector_clone (s->vector[i], s, d);
251 dict_set_attributes (d, dict_get_attributes (s));
253 for (i = 0; i < s->n_mrsets; i++)
255 const struct mrset *old = s->mrsets[i];
259 /* Clone old mrset, then replace vars from D by vars from S. */
260 new = mrset_clone (old);
261 for (j = 0; j < new->n_vars; j++)
262 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
264 dict_add_mrset (d, new);
270 /* Clears the contents from a dictionary without destroying the
271 dictionary itself. */
273 dict_clear (struct dictionary *d)
275 /* FIXME? Should we really clear case_limit, label, documents?
276 Others are necessarily cleared by deleting all the variables.*/
277 while (d->var_cnt > 0 )
279 dict_delete_var (d, d->var[d->var_cnt - 1].var);
284 d->var_cnt = d->var_cap = 0;
285 invalidate_proto (d);
286 hmap_clear (&d->name_map);
287 d->next_value_idx = 0;
288 dict_set_split_vars (d, NULL, 0);
289 dict_set_weight (d, NULL);
290 dict_set_filter (d, NULL);
294 string_array_clear (&d->documents);
295 dict_clear_vectors (d);
296 attrset_clear (&d->attributes);
299 /* Clears a dictionary and destroys it. */
301 _dict_destroy (struct dictionary *d)
303 /* In general, we don't want callbacks occurring, if the dictionary
304 is being destroyed */
305 d->callbacks = NULL ;
308 string_array_destroy (&d->documents);
309 hmap_destroy (&d->name_map);
310 attrset_destroy (&d->attributes);
311 dict_clear_mrsets (d);
317 dict_unref (struct dictionary *d)
322 assert (d->ref_cnt >= 0);
327 /* Returns the number of variables in D. */
329 dict_get_var_cnt (const struct dictionary *d)
334 /* Returns the variable in D with dictionary index IDX, which
335 must be between 0 and the count returned by
336 dict_get_var_cnt(), exclusive. */
338 dict_get_var (const struct dictionary *d, size_t idx)
340 assert (idx < d->var_cnt);
342 return d->var[idx].var;
345 /* Sets *VARS to an array of pointers to variables in D and *CNT
346 to the number of variables in *D. All variables are returned
347 except for those, if any, in the classes indicated by EXCLUDE.
348 (There is no point in putting DC_SYSTEM in EXCLUDE as
349 dictionaries never include system variables.) */
351 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
352 size_t *cnt, enum dict_class exclude)
354 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
357 /* Sets *VARS to an array of pointers to variables in D and *CNT
358 to the number of variables in *D. All variables are returned
359 except for those, if any, in the classes indicated by EXCLUDE.
360 (There is no point in putting DC_SYSTEM in EXCLUDE as
361 dictionaries never include system variables.) */
363 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
364 size_t *cnt, enum dict_class exclude)
369 assert (exclude == (exclude & DC_ALL));
372 for (i = 0; i < d->var_cnt; i++)
374 enum dict_class class = var_get_dict_class (d->var[i].var);
375 if (!(class & exclude))
379 *vars = xnmalloc (count, sizeof **vars);
381 for (i = 0; i < d->var_cnt; i++)
383 enum dict_class class = var_get_dict_class (d->var[i].var);
384 if (!(class & exclude))
385 (*vars)[(*cnt)++] = d->var[i].var;
387 assert (*cnt == count);
390 static struct variable *
391 add_var_with_case_index (struct dictionary *d, struct variable *v,
394 struct vardict_info *vardict;
396 assert (case_index >= d->next_value_idx);
398 /* Update dictionary. */
399 if (d->var_cnt >= d->var_cap)
403 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
404 hmap_clear (&d->name_map);
405 for (i = 0; i < d->var_cnt; i++)
407 var_set_vardict (d->var[i].var, &d->var[i]);
408 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
409 d->var[i].name_node.hash);
413 vardict = &d->var[d->var_cnt++];
416 hmap_insert (&d->name_map, &vardict->name_node,
417 utf8_hash_case_string (var_get_name (v), 0));
418 vardict->case_index = case_index;
419 var_set_vardict (v, vardict);
421 if ( d->changed ) d->changed (d, d->changed_data);
422 if ( d->callbacks && d->callbacks->var_added )
423 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
425 invalidate_proto (d);
426 d->next_value_idx = case_index + 1;
431 static struct variable *
432 add_var (struct dictionary *d, struct variable *v)
434 return add_var_with_case_index (d, v, d->next_value_idx);
437 /* Creates and returns a new variable in D with the given NAME
438 and WIDTH. Returns a null pointer if the given NAME would
439 duplicate that of an existing variable in the dictionary. */
441 dict_create_var (struct dictionary *d, const char *name, int width)
443 return (dict_lookup_var (d, name) == NULL
444 ? dict_create_var_assert (d, name, width)
448 /* Creates and returns a new variable in D with the given NAME
449 and WIDTH. Assert-fails if the given NAME would duplicate
450 that of an existing variable in the dictionary. */
452 dict_create_var_assert (struct dictionary *d, const char *name, int width)
454 assert (dict_lookup_var (d, name) == NULL);
455 return add_var (d, var_create (name, width));
458 /* Creates and returns a new variable in D, as a copy of existing variable
459 OLD_VAR, which need not be in D or in any dictionary. Returns a null
460 pointer if OLD_VAR's name would duplicate that of an existing variable in
463 dict_clone_var (struct dictionary *d, const struct variable *old_var)
465 return dict_clone_var_as (d, old_var, var_get_name (old_var));
468 /* Creates and returns a new variable in D, as a copy of existing variable
469 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
470 OLD_VAR's name would duplicate that of an existing variable in the
473 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
475 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
478 /* Creates and returns a new variable in D with name NAME, as a copy of
479 existing variable OLD_VAR, which need not be in D or in any dictionary.
480 Returns a null pointer if the given NAME would duplicate that of an existing
481 variable in the dictionary. */
483 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
486 return (dict_lookup_var (d, name) == NULL
487 ? dict_clone_var_as_assert (d, old_var, name)
491 /* Creates and returns a new variable in D with name NAME, as a copy of
492 existing variable OLD_VAR, which need not be in D or in any dictionary.
493 Assert-fails if the given NAME would duplicate that of an existing variable
494 in the dictionary. */
496 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
499 struct variable *new_var = var_clone (old_var);
500 assert (dict_lookup_var (d, name) == NULL);
501 var_set_name (new_var, name);
502 return add_var (d, new_var);
506 dict_clone_var_in_place_assert (struct dictionary *d,
507 const struct variable *old_var)
509 assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
510 return add_var_with_case_index (d, var_clone (old_var),
511 var_get_case_index (old_var));
514 /* Returns the variable named NAME in D, or a null pointer if no
515 variable has that name. */
517 dict_lookup_var (const struct dictionary *d, const char *name)
519 struct vardict_info *vardict;
521 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
522 utf8_hash_case_string (name, 0), &d->name_map)
524 struct variable *var = vardict->var;
525 if (!utf8_strcasecmp (var_get_name (var), name))
532 /* Returns the variable named NAME in D. Assert-fails if no
533 variable has that name. */
535 dict_lookup_var_assert (const struct dictionary *d, const char *name)
537 struct variable *v = dict_lookup_var (d, name);
542 /* Returns true if variable V is in dictionary D,
545 dict_contains_var (const struct dictionary *d, const struct variable *v)
547 return (var_has_vardict (v)
548 && vardict_get_dictionary (var_get_vardict (v)) == d);
551 /* Compares two double pointers to variables, which should point
552 to elements of a struct dictionary's `var' member array. */
554 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
556 struct variable *const *a = a_;
557 struct variable *const *b = b_;
559 return *a < *b ? -1 : *a > *b;
563 unindex_var (struct dictionary *d, struct vardict_info *vardict)
565 hmap_delete (&d->name_map, &vardict->name_node);
568 /* This function assumes that vardict->name_node.hash is valid, that is, that
569 its name has not changed since it was hashed (rename_var() updates this
570 hash along with the name itself). */
572 reindex_var (struct dictionary *d, struct vardict_info *vardict)
574 struct variable *old = (d->callbacks && d->callbacks->var_changed
575 ? var_clone (vardict->var)
578 struct variable *var = vardict->var;
579 var_set_vardict (var, vardict);
580 hmap_insert_fast (&d->name_map, &vardict->name_node,
581 vardict->name_node.hash);
583 if ( d->changed ) d->changed (d, d->changed_data);
586 d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
591 /* Sets the case_index in V's vardict to CASE_INDEX. */
593 set_var_case_index (struct variable *v, int case_index)
595 var_get_vardict (v)->case_index = case_index;
598 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
601 unindex_vars (struct dictionary *d, size_t from, size_t to)
605 for (i = from; i < to; i++)
606 unindex_var (d, &d->var[i]);
609 /* Re-sets the dict_index in the dictionary variables with
610 indexes from FROM to TO (exclusive). */
612 reindex_vars (struct dictionary *d, size_t from, size_t to)
616 for (i = from; i < to; i++)
617 reindex_var (d, &d->var[i]);
620 /* Deletes variable V from dictionary D and frees V.
622 This is a very bad idea if there might be any pointers to V
623 from outside D. In general, no variable in the active dataset's
624 dictionary should be deleted when any transformations are
625 active on the dictionary's dataset, because those
626 transformations might reference the deleted variable. The
627 safest time to delete a variable is just after a procedure has
628 been executed, as done by DELETE VARIABLES.
630 Pointers to V within D are not a problem, because
631 dict_delete_var() knows to remove V from split variables,
632 weights, filters, etc. */
634 dict_delete_var (struct dictionary *d, struct variable *v)
636 int dict_index = var_get_dict_index (v);
637 const int case_index = var_get_case_index (v);
639 assert (dict_contains_var (d, v));
641 dict_unset_split_var (d, v);
642 dict_unset_mrset_var (d, v);
645 dict_set_weight (d, NULL);
648 dict_set_filter (d, NULL);
650 dict_clear_vectors (d);
652 /* Remove V from var array. */
653 unindex_vars (d, dict_index, d->var_cnt);
654 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
657 /* Update dict_index for each affected variable. */
658 reindex_vars (d, dict_index, d->var_cnt);
661 var_clear_vardict (v);
663 if ( d->changed ) d->changed (d, d->changed_data);
665 invalidate_proto (d);
666 if (d->callbacks && d->callbacks->var_deleted )
667 d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
672 /* Deletes the COUNT variables listed in VARS from D. This is
673 unsafe; see the comment on dict_delete_var() for details. */
675 dict_delete_vars (struct dictionary *d,
676 struct variable *const *vars, size_t count)
678 /* FIXME: this can be done in O(count) time, but this algorithm
680 assert (count == 0 || vars != NULL);
683 dict_delete_var (d, *vars++);
686 /* Deletes the COUNT variables in D starting at index IDX. This
687 is unsafe; see the comment on dict_delete_var() for
690 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
692 /* FIXME: this can be done in O(count) time, but this algorithm
694 assert (idx + count <= d->var_cnt);
697 dict_delete_var (d, d->var[idx].var);
700 /* Deletes scratch variables from dictionary D. */
702 dict_delete_scratch_vars (struct dictionary *d)
706 /* FIXME: this can be done in O(count) time, but this algorithm
708 for (i = 0; i < d->var_cnt; )
709 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
710 dict_delete_var (d, d->var[i].var);
715 /* Moves V to 0-based position IDX in D. Other variables in D,
716 if any, retain their relative positions. Runs in time linear
717 in the distance moved. */
719 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
721 size_t old_index = var_get_dict_index (v);
723 assert (new_index < d->var_cnt);
725 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
726 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
727 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
730 /* Reorders the variables in D, placing the COUNT variables
731 listed in ORDER in that order at the beginning of D. The
732 other variables in D, if any, retain their relative
735 dict_reorder_vars (struct dictionary *d,
736 struct variable *const *order, size_t count)
738 struct vardict_info *new_var;
741 assert (count == 0 || order != NULL);
742 assert (count <= d->var_cnt);
744 new_var = xnmalloc (d->var_cap, sizeof *new_var);
746 /* Add variables in ORDER to new_var. */
747 for (i = 0; i < count; i++)
749 struct vardict_info *old_var;
751 assert (dict_contains_var (d, order[i]));
753 old_var = var_get_vardict (order[i]);
754 new_var[i] = *old_var;
755 old_var->dict = NULL;
758 /* Add remaining variables to new_var. */
759 for (i = 0; i < d->var_cnt; i++)
760 if (d->var[i].dict != NULL)
761 new_var[count++] = d->var[i];
762 assert (count == d->var_cnt);
764 /* Replace old vardicts by new ones. */
768 hmap_clear (&d->name_map);
769 reindex_vars (d, 0, d->var_cnt);
772 /* Changes the name of variable V that is currently in a dictionary to
775 rename_var (struct variable *v, const char *new_name)
777 struct vardict_info *vardict = var_get_vardict (v);
778 var_clear_vardict (v);
779 var_set_name (v, new_name);
780 vardict->name_node.hash = utf8_hash_case_string (new_name, 0);
781 var_set_vardict (v, vardict);
784 /* Tries to changes the name of V in D to name NEW_NAME. Returns true if
785 successful, false if a variable (other than V) with the given name already
788 dict_try_rename_var (struct dictionary *d, struct variable *v,
789 const char *new_name)
791 struct variable *conflict = dict_lookup_var (d, new_name);
792 if (conflict && v != conflict)
795 struct variable *old = var_clone (v);
796 unindex_var (d, var_get_vardict (v));
797 rename_var (v, new_name);
798 reindex_var (d, var_get_vardict (v));
800 if (settings_get_algorithm () == ENHANCED)
801 var_clear_short_names (v);
803 if ( d->changed ) d->changed (d, d->changed_data);
804 if ( d->callbacks && d->callbacks->var_changed )
805 d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
812 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
813 a variable named NEW_NAME is already in D, except that
814 NEW_NAME may be the same as V's existing name. */
816 dict_rename_var (struct dictionary *d, struct variable *v,
817 const char *new_name)
819 bool ok UNUSED = dict_try_rename_var (d, v, new_name);
823 /* Renames COUNT variables specified in VARS to the names given
824 in NEW_NAMES within dictionary D. If the renaming would
825 result in a duplicate variable name, returns false and stores a
826 name that would be duplicated into *ERR_NAME (if ERR_NAME is
827 non-null). Otherwise, the renaming is successful, and true
830 dict_rename_vars (struct dictionary *d,
831 struct variable **vars, char **new_names, size_t count,
838 assert (count == 0 || vars != NULL);
839 assert (count == 0 || new_names != NULL);
841 /* Save the names of the variables to be renamed. */
842 pool = pool_create ();
843 old_names = pool_nalloc (pool, count, sizeof *old_names);
844 for (i = 0; i < count; i++)
845 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
847 /* Remove the variables to be renamed from the name hash,
849 for (i = 0; i < count; i++)
851 unindex_var (d, var_get_vardict (vars[i]));
852 rename_var (vars[i], new_names[i]);
855 /* Add the renamed variables back into the name hash,
856 checking for conflicts. */
857 for (i = 0; i < count; i++)
859 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
861 /* There is a name conflict.
862 Back out all the name changes that have already
863 taken place, and indicate failure. */
865 if (err_name != NULL)
866 *err_name = new_names[i];
868 for (i = 0; i < fail_idx; i++)
869 unindex_var (d, var_get_vardict (vars[i]));
871 for (i = 0; i < count; i++)
873 rename_var (vars[i], old_names[i]);
874 reindex_var (d, var_get_vardict (vars[i]));
880 reindex_var (d, var_get_vardict (vars[i]));
883 /* Clear short names. */
884 if (settings_get_algorithm () == ENHANCED)
885 for (i = 0; i < count; i++)
886 var_clear_short_names (vars[i]);
892 /* Returns true if a variable named NAME may be inserted in DICT;
893 that is, if there is not already a variable with that name in
894 DICT and if NAME is not a reserved word. (The caller's checks
895 have already verified that NAME is otherwise acceptable as a
898 var_name_is_insertable (const struct dictionary *dict, const char *name)
900 return (dict_lookup_var (dict, name) == NULL
901 && lex_id_to_token (ss_cstr (name)) == T_ID);
905 make_hinted_name (const struct dictionary *dict, const char *hint)
907 size_t hint_len = strlen (hint);
908 bool dropped = false;
913 /* The allocation size here is OK: characters that are copied directly fit
914 OK, and characters that are not copied directly are replaced by a single
915 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
916 replaced by '_' too. */
917 root = rp = xmalloc (hint_len + 1);
918 for (ofs = 0; ofs < hint_len; ofs += mblen)
922 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
925 ? lex_uc_is_id1 (uc) && uc != '$'
926 : lex_uc_is_idn (uc))
933 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
944 if (var_name_is_insertable (dict, root))
947 for (i = 0; i < ULONG_MAX; i++)
949 char suffix[INT_BUFSIZE_BOUND (i) + 1];
953 if (!str_format_26adic (i + 1, true, &suffix[1], sizeof suffix - 1))
956 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
957 if (var_name_is_insertable (dict, name))
972 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
974 unsigned long int number;
976 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
980 char name[3 + INT_STRLEN_BOUND (number) + 1];
982 sprintf (name, "VAR%03lu", number);
983 if (dict_lookup_var (dict, name) == NULL)
985 if (num_start != NULL)
986 *num_start = number + 1;
987 return xstrdup (name);
995 /* Devises and returns a variable name unique within DICT. The variable name
996 is owned by the caller, which must free it with free() when it is no longer
999 HINT, if it is non-null, is used as a suggestion that will be
1000 modified for suitability as a variable name and for
1003 If HINT is null or entirely unsuitable, a name in the form
1004 "VAR%03d" will be generated, where the smallest unused integer
1005 value is used. If NUM_START is non-null, then its value is
1006 used as the minimum numeric value to check, and it is updated
1007 to the next value to be checked.
1010 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
1011 unsigned long int *num_start)
1015 char *hinted_name = make_hinted_name (dict, hint);
1016 if (hinted_name != NULL)
1019 return make_numeric_name (dict, num_start);
1022 /* Returns whether variable names must be valid identifiers. Normally, this is
1023 true, but sometimes a dictionary is prepared for external use (e.g. output
1024 to a CSV file) where names don't have to be valid. */
1026 dict_get_names_must_be_ids (const struct dictionary *d)
1028 return d->names_must_be_ids;
1031 /* Sets whether variable names must be valid identifiers. Normally, this is
1032 true, but sometimes a dictionary is prepared for external use (e.g. output
1033 to a CSV file) where names don't have to be valid.
1035 Changing this setting from false to true doesn't make the dictionary check
1036 all the existing variable names, so it can cause an invariant violation. */
1038 dict_set_names_must_be_ids (struct dictionary *d, bool names_must_be_ids)
1040 d->names_must_be_ids = names_must_be_ids;
1043 /* Returns the weighting variable in dictionary D, or a null
1044 pointer if the dictionary is unweighted. */
1046 dict_get_weight (const struct dictionary *d)
1048 assert (d->weight == NULL || dict_contains_var (d, d->weight));
1053 /* Returns the value of D's weighting variable in case C, except
1054 that a negative weight is returned as 0. Returns 1 if the
1055 dictionary is unweighted. Will warn about missing, negative,
1056 or zero values if *WARN_ON_INVALID is true. The function will
1057 set *WARN_ON_INVALID to false if an invalid weight is
1060 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
1061 bool *warn_on_invalid)
1065 if (d->weight == NULL)
1069 double w = case_num (c, d->weight);
1071 return var_force_valid_weight (d->weight, w, warn_on_invalid);
1075 /* Returns the format to use for weights. */
1076 const struct fmt_spec *
1077 dict_get_weight_format (const struct dictionary *d)
1079 return d->weight ? var_get_print_format (d->weight) : &F_8_0;
1082 /* Sets the weighting variable of D to V, or turning off
1083 weighting if V is a null pointer. */
1085 dict_set_weight (struct dictionary *d, struct variable *v)
1087 assert (v == NULL || dict_contains_var (d, v));
1088 assert (v == NULL || var_is_numeric (v));
1092 if (d->changed) d->changed (d, d->changed_data);
1093 if ( d->callbacks && d->callbacks->weight_changed )
1094 d->callbacks->weight_changed (d,
1095 v ? var_get_dict_index (v) : -1,
1099 /* Returns the filter variable in dictionary D (see cmd_filter())
1100 or a null pointer if the dictionary is unfiltered. */
1102 dict_get_filter (const struct dictionary *d)
1104 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1109 /* Sets V as the filter variable for dictionary D. Passing a
1110 null pointer for V turn off filtering. */
1112 dict_set_filter (struct dictionary *d, struct variable *v)
1114 assert (v == NULL || dict_contains_var (d, v));
1115 assert (v == NULL || var_is_numeric (v));
1117 /* When a filter is set, we ref the dictionary.
1118 This is because the GUI maintains a pointer
1119 to the dict's variables, and the variables'
1120 addresses change in the callback. */
1121 if (d->filter == NULL && v != NULL)
1126 /* Deref the dict when a filter is removed. */
1127 if (d->filter != NULL && v == NULL)
1129 assert (d->ref_cnt > 0);
1135 if (d->changed) d->changed (d, d->changed_data);
1136 if ( d->callbacks && d->callbacks->filter_changed )
1137 d->callbacks->filter_changed (d,
1138 v ? var_get_dict_index (v) : -1,
1142 /* Returns the case limit for dictionary D, or zero if the number
1143 of cases is unlimited. */
1145 dict_get_case_limit (const struct dictionary *d)
1147 return d->case_limit;
1150 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1151 0 for CASE_LIMIT to indicate no limit. */
1153 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1155 d->case_limit = case_limit;
1158 /* Returns the prototype used for cases created by dictionary D. */
1159 const struct caseproto *
1160 dict_get_proto (const struct dictionary *d_)
1162 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1163 if (d->proto == NULL)
1167 d->proto = caseproto_create ();
1168 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1169 for (i = 0; i < d->var_cnt; i++)
1170 d->proto = caseproto_set_width (d->proto,
1171 var_get_case_index (d->var[i].var),
1172 var_get_width (d->var[i].var));
1177 /* Returns the case index of the next value to be added to D.
1178 This value is the number of `union value's that need to be
1179 allocated to store a case for dictionary D. */
1181 dict_get_next_value_idx (const struct dictionary *d)
1183 return d->next_value_idx;
1186 /* Returns the number of bytes needed to store a case for
1189 dict_get_case_size (const struct dictionary *d)
1191 return sizeof (union value) * dict_get_next_value_idx (d);
1194 /* Reassigns values in dictionary D so that fragmentation is
1197 dict_compact_values (struct dictionary *d)
1201 d->next_value_idx = 0;
1202 for (i = 0; i < d->var_cnt; i++)
1204 struct variable *v = d->var[i].var;
1205 set_var_case_index (v, d->next_value_idx++);
1207 invalidate_proto (d);
1210 /* Returns the number of values occupied by the variables in
1211 dictionary D. All variables are considered if EXCLUDE_CLASSES
1212 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1213 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1214 corresponding type of variable.
1216 The return value may be less than the number of values in one
1217 of dictionary D's cases (as returned by
1218 dict_get_next_value_idx) even if E is 0, because there may be
1219 gaps in D's cases due to deleted variables. */
1221 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1226 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1228 | (1u << DC_SCRATCH))) == 0);
1231 for (i = 0; i < d->var_cnt; i++)
1233 enum dict_class class = var_get_dict_class (d->var[i].var);
1234 if (!(exclude_classes & (1u << class)))
1240 /* Returns the case prototype that would result after deleting
1241 all variables from D that are not in one of the
1242 EXCLUDE_CLASSES and compacting the dictionary with
1245 The caller must unref the returned caseproto when it is no
1248 dict_get_compacted_proto (const struct dictionary *d,
1249 unsigned int exclude_classes)
1251 struct caseproto *proto;
1254 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1256 | (1u << DC_SCRATCH))) == 0);
1258 proto = caseproto_create ();
1259 for (i = 0; i < d->var_cnt; i++)
1261 struct variable *v = d->var[i].var;
1262 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1263 proto = caseproto_add_width (proto, var_get_width (v));
1268 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1269 dict_get_split_cnt() to determine how many SPLIT FILE vars
1270 there are. Returns a null pointer if and only if there are no
1272 const struct variable *const *
1273 dict_get_split_vars (const struct dictionary *d)
1278 /* Returns the number of SPLIT FILE vars. */
1280 dict_get_split_cnt (const struct dictionary *d)
1282 return d->split_cnt;
1285 /* Removes variable V, which must be in D, from D's set of split
1288 dict_unset_split_var (struct dictionary *d, struct variable *v)
1292 assert (dict_contains_var (d, v));
1294 orig_count = d->split_cnt;
1295 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1296 &v, compare_var_ptrs, NULL);
1297 if (orig_count != d->split_cnt)
1299 if (d->changed) d->changed (d, d->changed_data);
1300 /* We changed the set of split variables so invoke the
1302 if (d->callbacks && d->callbacks->split_changed)
1303 d->callbacks->split_changed (d, d->cb_data);
1307 /* Sets CNT split vars SPLIT in dictionary D. */
1309 dict_set_split_vars (struct dictionary *d,
1310 struct variable *const *split, size_t cnt)
1312 assert (cnt == 0 || split != NULL);
1317 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1318 memcpy (d->split, split, cnt * sizeof *d->split);
1326 if (d->changed) d->changed (d, d->changed_data);
1327 if ( d->callbacks && d->callbacks->split_changed )
1328 d->callbacks->split_changed (d, d->cb_data);
1331 /* Returns the file label for D, or a null pointer if D is
1332 unlabeled (see cmd_file_label()). */
1334 dict_get_label (const struct dictionary *d)
1339 /* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
1342 Removes D's label if LABEL is null or the empty string. */
1344 dict_set_label (struct dictionary *d, const char *label)
1347 if (label == NULL || label[0] == '\0')
1350 d->label = utf8_encoding_trunc (label, d->encoding, 60);
1353 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1354 return value is always nonnull; if there are no documents then the
1355 string_arary is empty.*/
1356 const struct string_array *
1357 dict_get_documents (const struct dictionary *d)
1359 return &d->documents;
1362 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1364 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1368 dict_clear_documents (d);
1370 for (i = 0; i < new_docs->n; i++)
1371 dict_add_document_line (d, new_docs->strings[i], false);
1374 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1375 into individual lines at new-line characters. Each line is truncated to at
1376 most DOC_LINE_LENGTH bytes in D's encoding. */
1378 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1382 dict_clear_documents (d);
1383 for (s = new_docs; *s != '\0'; )
1385 size_t len = strcspn (s, "\n");
1386 char *line = xmemdup0 (s, len);
1387 dict_add_document_line (d, line, false);
1396 /* Drops the documents from dictionary D. */
1398 dict_clear_documents (struct dictionary *d)
1400 string_array_clear (&d->documents);
1403 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1404 truncated so that it is no more than 80 bytes in the dictionary's
1405 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1406 then a warning will be issued. */
1408 dict_add_document_line (struct dictionary *d, const char *line,
1414 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1415 truncated = line[trunc_len] != '\0';
1416 if (truncated && issue_warning)
1418 /* Note to translators: "bytes" is correct, not characters */
1419 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1422 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1427 /* Returns the number of document lines in dictionary D. */
1429 dict_get_document_line_cnt (const struct dictionary *d)
1431 return d->documents.n;
1434 /* Returns document line number IDX in dictionary D. The caller must not
1435 modify or free the returned string. */
1437 dict_get_document_line (const struct dictionary *d, size_t idx)
1439 assert (idx < d->documents.n);
1440 return d->documents.strings[idx];
1443 /* Creates in D a vector named NAME that contains the CNT
1444 variables in VAR. Returns true if successful, or false if a
1445 vector named NAME already exists in D. */
1447 dict_create_vector (struct dictionary *d,
1449 struct variable **var, size_t cnt)
1454 for (i = 0; i < cnt; i++)
1455 assert (dict_contains_var (d, var[i]));
1457 if (dict_lookup_vector (d, name) == NULL)
1459 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1460 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1467 /* Creates in D a vector named NAME that contains the CNT
1468 variables in VAR. A vector named NAME must not already exist
1471 dict_create_vector_assert (struct dictionary *d,
1473 struct variable **var, size_t cnt)
1475 assert (dict_lookup_vector (d, name) == NULL);
1476 dict_create_vector (d, name, var, cnt);
1479 /* Returns the vector in D with index IDX, which must be less
1480 than dict_get_vector_cnt (D). */
1481 const struct vector *
1482 dict_get_vector (const struct dictionary *d, size_t idx)
1484 assert (idx < d->vector_cnt);
1486 return d->vector[idx];
1489 /* Returns the number of vectors in D. */
1491 dict_get_vector_cnt (const struct dictionary *d)
1493 return d->vector_cnt;
1496 /* Looks up and returns the vector within D with the given
1498 const struct vector *
1499 dict_lookup_vector (const struct dictionary *d, const char *name)
1502 for (i = 0; i < d->vector_cnt; i++)
1503 if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
1504 return d->vector[i];
1508 /* Deletes all vectors from D. */
1510 dict_clear_vectors (struct dictionary *d)
1514 for (i = 0; i < d->vector_cnt; i++)
1515 vector_destroy (d->vector[i]);
1522 /* Multiple response sets. */
1524 /* Returns the multiple response set in DICT with index IDX, which must be
1525 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1526 const struct mrset *
1527 dict_get_mrset (const struct dictionary *dict, size_t idx)
1529 assert (idx < dict->n_mrsets);
1530 return dict->mrsets[idx];
1533 /* Returns the number of multiple response sets in DICT. */
1535 dict_get_n_mrsets (const struct dictionary *dict)
1537 return dict->n_mrsets;
1540 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1541 returns its index; otherwise, returns SIZE_MAX. */
1543 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1547 for (i = 0; i < dict->n_mrsets; i++)
1548 if (!utf8_strcasecmp (name, dict->mrsets[i]->name))
1554 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1555 returns it; otherwise, returns NULL. */
1556 const struct mrset *
1557 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1559 size_t idx = dict_lookup_mrset_idx (dict, name);
1560 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1563 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1564 true if a set was replaced, false if none existed with the specified name.
1566 Ownership of MRSET is transferred to DICT. */
1568 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1572 assert (mrset_ok (mrset, dict));
1574 idx = dict_lookup_mrset_idx (dict, mrset->name);
1575 if (idx == SIZE_MAX)
1577 dict->mrsets = xrealloc (dict->mrsets,
1578 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1579 dict->mrsets[dict->n_mrsets++] = mrset;
1584 mrset_destroy (dict->mrsets[idx]);
1585 dict->mrsets[idx] = mrset;
1590 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1591 from DICT and returns true. If none is found, returns false without
1594 Deleting one multiple response set causes the indexes of other sets within
1597 dict_delete_mrset (struct dictionary *dict, const char *name)
1599 size_t idx = dict_lookup_mrset_idx (dict, name);
1600 if (idx != SIZE_MAX)
1602 mrset_destroy (dict->mrsets[idx]);
1603 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1610 /* Deletes all multiple response sets from DICT. */
1612 dict_clear_mrsets (struct dictionary *dict)
1616 for (i = 0; i < dict->n_mrsets; i++)
1617 mrset_destroy (dict->mrsets[i]);
1618 free (dict->mrsets);
1619 dict->mrsets = NULL;
1623 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1625 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1629 assert (dict_contains_var (dict, var));
1631 for (i = 0; i < dict->n_mrsets; )
1633 struct mrset *mrset = dict->mrsets[i];
1636 for (j = 0; j < mrset->n_vars; )
1637 if (mrset->vars[j] == var)
1638 remove_element (mrset->vars, mrset->n_vars--,
1639 sizeof *mrset->vars, j);
1643 if (mrset->n_vars < 2)
1645 mrset_destroy (mrset);
1646 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1653 /* Returns D's attribute set. The caller may examine or modify
1654 the attribute set, but must not destroy it. Destroying D or
1655 calling dict_set_attributes for D will also destroy D's
1658 dict_get_attributes (const struct dictionary *d)
1660 return CONST_CAST (struct attrset *, &d->attributes);
1663 /* Replaces D's attributes set by a copy of ATTRS. */
1665 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1667 attrset_destroy (&d->attributes);
1668 attrset_clone (&d->attributes, attrs);
1671 /* Returns true if D has at least one attribute in its attribute
1672 set, false if D's attribute set is empty. */
1674 dict_has_attributes (const struct dictionary *d)
1676 return attrset_count (&d->attributes) > 0;
1679 /* Called from variable.c to notify the dictionary that some property (indicated
1680 by WHAT) of the variable has changed. OLDVAR is a copy of V as it existed
1681 prior to the change. OLDVAR is destroyed by this function.
1684 dict_var_changed (const struct variable *v, unsigned int what, struct variable *oldvar)
1686 if ( var_has_vardict (v))
1688 const struct vardict_info *vardict = var_get_vardict (v);
1689 struct dictionary *d = vardict->dict;
1694 if (d->changed ) d->changed (d, d->changed_data);
1695 if ( d->callbacks && d->callbacks->var_changed )
1696 d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
1698 var_destroy (oldvar);
1703 /* Dictionary used to contain "internal variables". */
1704 static struct dictionary *internal_dict;
1706 /* Create a variable of the specified WIDTH to be used for internal
1707 calculations only. The variable is assigned case index CASE_IDX. */
1709 dict_create_internal_var (int case_idx, int width)
1711 if (internal_dict == NULL)
1712 internal_dict = dict_create ("UTF-8");
1716 static int counter = INT_MAX / 2;
1717 struct variable *var;
1720 if (++counter == INT_MAX)
1721 counter = INT_MAX / 2;
1723 sprintf (name, "$internal%d", counter);
1724 var = dict_create_var (internal_dict, name, width);
1727 set_var_case_index (var, case_idx);
1733 /* Destroys VAR, which must have been created with
1734 dict_create_internal_var(). */
1736 dict_destroy_internal_var (struct variable *var)
1740 dict_delete_var (internal_dict, var);
1742 /* Destroy internal_dict if it has no variables left, just so that
1743 valgrind --leak-check --show-reachable won't show internal_dict. */
1744 if (dict_get_var_cnt (internal_dict) == 0)
1746 dict_unref (internal_dict);
1747 internal_dict = NULL;
1753 vardict_get_dict_index (const struct vardict_info *vardict)
1755 return vardict - vardict->dict->var;