1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
46 #include "libpspp/ll.h"
48 #include "gl/intprops.h"
49 #include "gl/minmax.h"
50 #include "gl/xalloc.h"
51 #include "gl/xmemdup0.h"
54 #define _(msgid) gettext (msgid)
60 struct vardict_info *var; /* Variables. */
61 size_t var_cnt, var_cap; /* Number of variables, capacity. */
62 struct caseproto *proto; /* Prototype for dictionary cases
64 struct hmap name_map; /* Variable index by name. */
65 int next_value_idx; /* Index of next `union value' to allocate. */
66 const struct variable **split; /* SPLIT FILE vars. */
67 size_t split_cnt; /* SPLIT FILE count. */
68 struct variable *weight; /* WEIGHT variable. */
69 struct variable *filter; /* FILTER variable. */
70 casenumber case_limit; /* Current case limit (N command). */
71 char *label; /* File label. */
72 struct string_array documents; /* Documents. */
73 struct vector **vector; /* Vectors of variables. */
74 size_t vector_cnt; /* Number of vectors. */
75 struct attrset attributes; /* Custom attributes. */
76 struct mrset **mrsets; /* Multiple response sets. */
77 size_t n_mrsets; /* Number of multiple response sets. */
79 /* Whether variable names must be valid identifiers. Normally, this is
80 true, but sometimes a dictionary is prepared for external use
81 (e.g. output to a CSV file) where names don't have to be valid. */
82 bool names_must_be_ids;
84 char *encoding; /* Character encoding of string data */
86 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
88 void *cb_data ; /* Data passed to callbacks */
90 void (*changed) (struct dictionary *, void *); /* Generic change callback */
94 static void dict_unset_split_var (struct dictionary *, struct variable *);
95 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
97 /* Returns the encoding for data in dictionary D. The return value is a
98 nonnull string that contains an IANA character set name. */
100 dict_get_encoding (const struct dictionary *d)
105 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
106 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
107 error message on failure. */
109 dict_id_is_valid (const struct dictionary *dict, const char *id,
112 return (!dict->names_must_be_ids
113 || id_is_valid (id, dict->encoding, issue_error));
117 dict_set_change_callback (struct dictionary *d,
118 void (*changed) (struct dictionary *, void*),
121 d->changed = changed;
122 d->changed_data = data;
125 /* Discards dictionary D's caseproto. (It will be regenerated
126 lazily, on demand.) */
128 invalidate_proto (struct dictionary *d)
130 caseproto_unref (d->proto);
134 /* Print a representation of dictionary D to stdout, for
135 debugging purposes. */
137 dict_dump (const struct dictionary *d)
140 for (i = 0 ; i < d->var_cnt ; ++i)
142 const struct variable *v = d->var[i].var;
143 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
145 var_get_dict_index (v),
146 var_get_case_index (v));
151 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
152 the dictionary or any of the variables it contains are modified.
153 Each callback will get passed CALLBACK_DATA.
154 Any callback may be NULL, in which case it'll be ignored.
157 dict_set_callbacks (struct dictionary *dict,
158 const struct dict_callbacks *callbacks,
161 dict->callbacks = callbacks;
162 dict->cb_data = callback_data;
165 /* Shallow copy the callbacks from SRC to DEST */
167 dict_copy_callbacks (struct dictionary *dest,
168 const struct dictionary *src)
170 dest->callbacks = src->callbacks;
171 dest->cb_data = src->cb_data;
174 /* Creates and returns a new dictionary with the specified ENCODING. */
176 dict_create (const char *encoding)
178 struct dictionary *d = xzalloc (sizeof *d);
180 d->encoding = xstrdup (encoding);
181 d->names_must_be_ids = true;
182 hmap_init (&d->name_map);
183 attrset_init (&d->attributes);
189 /* Creates and returns a (deep) copy of an existing
192 The new dictionary's case indexes are copied from the old
193 dictionary. If the new dictionary won't be used to access
194 cases produced with the old dictionary, then the new
195 dictionary's case indexes should be compacted with
196 dict_compact_values to save space.
198 Callbacks are not cloned. */
200 dict_clone (const struct dictionary *s)
202 struct dictionary *d;
205 d = dict_create (s->encoding);
206 dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));
208 for (i = 0; i < s->var_cnt; i++)
210 struct variable *sv = s->var[i].var;
211 struct variable *dv = dict_clone_var_assert (d, sv);
214 for (i = 0; i < var_get_short_name_cnt (sv); i++)
215 var_set_short_name (dv, i, var_get_short_name (sv, i));
217 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
220 d->next_value_idx = s->next_value_idx;
222 d->split_cnt = s->split_cnt;
223 if (d->split_cnt > 0)
225 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
226 for (i = 0; i < d->split_cnt; i++)
227 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
230 if (s->weight != NULL)
231 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
233 if (s->filter != NULL)
234 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
236 d->case_limit = s->case_limit;
237 dict_set_label (d, dict_get_label (s));
238 dict_set_documents (d, dict_get_documents (s));
240 d->vector_cnt = s->vector_cnt;
241 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
242 for (i = 0; i < s->vector_cnt; i++)
243 d->vector[i] = vector_clone (s->vector[i], s, d);
245 dict_set_attributes (d, dict_get_attributes (s));
247 for (i = 0; i < s->n_mrsets; i++)
249 const struct mrset *old = s->mrsets[i];
253 /* Clone old mrset, then replace vars from D by vars from S. */
254 new = mrset_clone (old);
255 for (j = 0; j < new->n_vars; j++)
256 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
258 dict_add_mrset (d, new);
264 /* Clears the contents from a dictionary without destroying the
265 dictionary itself. */
267 dict_clear (struct dictionary *d)
269 /* FIXME? Should we really clear case_limit, label, documents?
270 Others are necessarily cleared by deleting all the variables.*/
271 while (d->var_cnt > 0)
273 dict_delete_var (d, d->var[d->var_cnt - 1].var);
278 d->var_cnt = d->var_cap = 0;
279 invalidate_proto (d);
280 hmap_clear (&d->name_map);
281 d->next_value_idx = 0;
282 dict_set_split_vars (d, NULL, 0);
283 dict_set_weight (d, NULL);
284 dict_set_filter (d, NULL);
288 string_array_clear (&d->documents);
289 dict_clear_vectors (d);
290 attrset_clear (&d->attributes);
293 /* Clears a dictionary and destroys it. */
295 _dict_destroy (struct dictionary *d)
297 /* In general, we don't want callbacks occurring, if the dictionary
298 is being destroyed */
299 d->callbacks = NULL ;
302 string_array_destroy (&d->documents);
303 hmap_destroy (&d->name_map);
304 attrset_destroy (&d->attributes);
305 dict_clear_mrsets (d);
311 dict_ref (struct dictionary *d)
318 dict_unref (struct dictionary *d)
323 assert (d->ref_cnt >= 0);
328 /* Returns the number of variables in D. */
330 dict_get_var_cnt (const struct dictionary *d)
335 /* Returns the variable in D with dictionary index IDX, which
336 must be between 0 and the count returned by
337 dict_get_var_cnt(), exclusive. */
339 dict_get_var (const struct dictionary *d, size_t idx)
341 assert (idx < d->var_cnt);
343 return d->var[idx].var;
346 /* Sets *VARS to an array of pointers to variables in D and *CNT
347 to the number of variables in *D. All variables are returned
348 except for those, if any, in the classes indicated by EXCLUDE.
349 (There is no point in putting DC_SYSTEM in EXCLUDE as
350 dictionaries never include system variables.) */
352 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
353 size_t *cnt, enum dict_class exclude)
355 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
358 /* Sets *VARS to an array of pointers to variables in D and *CNT
359 to the number of variables in *D. All variables are returned
360 except for those, if any, in the classes indicated by EXCLUDE.
361 (There is no point in putting DC_SYSTEM in EXCLUDE as
362 dictionaries never include system variables.) */
364 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
365 size_t *cnt, enum dict_class exclude)
370 assert (exclude == (exclude & DC_ALL));
373 for (i = 0; i < d->var_cnt; i++)
375 enum dict_class class = var_get_dict_class (d->var[i].var);
376 if (!(class & exclude))
380 *vars = xnmalloc (count, sizeof **vars);
382 for (i = 0; i < d->var_cnt; i++)
384 enum dict_class class = var_get_dict_class (d->var[i].var);
385 if (!(class & exclude))
386 (*vars)[(*cnt)++] = d->var[i].var;
388 assert (*cnt == count);
391 static struct variable *
392 add_var_with_case_index (struct dictionary *d, struct variable *v,
395 struct vardict_info *vardict;
397 assert (case_index >= d->next_value_idx);
399 /* Update dictionary. */
400 if (d->var_cnt >= d->var_cap)
404 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
405 hmap_clear (&d->name_map);
406 for (i = 0; i < d->var_cnt; i++)
408 var_set_vardict (d->var[i].var, &d->var[i]);
409 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
410 d->var[i].name_node.hash);
414 vardict = &d->var[d->var_cnt++];
417 hmap_insert (&d->name_map, &vardict->name_node,
418 utf8_hash_case_string (var_get_name (v), 0));
419 vardict->case_index = case_index;
420 var_set_vardict (v, vardict);
422 if (d->changed) d->changed (d, d->changed_data);
423 if (d->callbacks && d->callbacks->var_added)
424 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
426 invalidate_proto (d);
427 d->next_value_idx = case_index + 1;
432 static struct variable *
433 add_var (struct dictionary *d, struct variable *v)
435 return add_var_with_case_index (d, v, d->next_value_idx);
438 /* Creates and returns a new variable in D with the given NAME
439 and WIDTH. Returns a null pointer if the given NAME would
440 duplicate that of an existing variable in the dictionary. */
442 dict_create_var (struct dictionary *d, const char *name, int width)
444 return (dict_lookup_var (d, name) == NULL
445 ? dict_create_var_assert (d, name, width)
449 /* Creates and returns a new variable in D with the given NAME
450 and WIDTH. Assert-fails if the given NAME would duplicate
451 that of an existing variable in the dictionary. */
453 dict_create_var_assert (struct dictionary *d, const char *name, int width)
455 assert (dict_lookup_var (d, name) == NULL);
456 return add_var (d, var_create (name, width));
459 /* Creates and returns a new variable in D, as a copy of existing variable
460 OLD_VAR, which need not be in D or in any dictionary. Returns a null
461 pointer if OLD_VAR's name would duplicate that of an existing variable in
464 dict_clone_var (struct dictionary *d, const struct variable *old_var)
466 return dict_clone_var_as (d, old_var, var_get_name (old_var));
469 /* Creates and returns a new variable in D, as a copy of existing variable
470 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
471 OLD_VAR's name would duplicate that of an existing variable in the
474 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
476 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
479 /* Creates and returns a new variable in D with name NAME, as a copy of
480 existing variable OLD_VAR, which need not be in D or in any dictionary.
481 Returns a null pointer if the given NAME would duplicate that of an existing
482 variable in the dictionary. */
484 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
487 return (dict_lookup_var (d, name) == NULL
488 ? dict_clone_var_as_assert (d, old_var, name)
492 /* Creates and returns a new variable in D with name NAME, as a copy of
493 existing variable OLD_VAR, which need not be in D or in any dictionary.
494 Assert-fails if the given NAME would duplicate that of an existing variable
495 in the dictionary. */
497 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
500 struct variable *new_var = var_clone (old_var);
501 assert (dict_lookup_var (d, name) == NULL);
502 var_set_name (new_var, name);
503 return add_var (d, new_var);
507 dict_clone_var_in_place_assert (struct dictionary *d,
508 const struct variable *old_var)
510 assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
511 return add_var_with_case_index (d, var_clone (old_var),
512 var_get_case_index (old_var));
515 /* Returns the variable named NAME in D, or a null pointer if no
516 variable has that name. */
518 dict_lookup_var (const struct dictionary *d, const char *name)
520 struct vardict_info *vardict;
522 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
523 utf8_hash_case_string (name, 0), &d->name_map)
525 struct variable *var = vardict->var;
526 if (!utf8_strcasecmp (var_get_name (var), name))
533 /* Returns the variable named NAME in D. Assert-fails if no
534 variable has that name. */
536 dict_lookup_var_assert (const struct dictionary *d, const char *name)
538 struct variable *v = dict_lookup_var (d, name);
543 /* Returns true if variable V is in dictionary D,
546 dict_contains_var (const struct dictionary *d, const struct variable *v)
548 return (var_has_vardict (v)
549 && vardict_get_dictionary (var_get_vardict (v)) == d);
552 /* Compares two double pointers to variables, which should point
553 to elements of a struct dictionary's `var' member array. */
555 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
557 struct variable *const *a = a_;
558 struct variable *const *b = b_;
560 return *a < *b ? -1 : *a > *b;
564 unindex_var (struct dictionary *d, struct vardict_info *vardict)
566 hmap_delete (&d->name_map, &vardict->name_node);
569 /* This function assumes that vardict->name_node.hash is valid, that is, that
570 its name has not changed since it was hashed (rename_var() updates this
571 hash along with the name itself). */
573 reindex_var (struct dictionary *d, struct vardict_info *vardict)
575 struct variable *old = (d->callbacks && d->callbacks->var_changed
576 ? var_clone (vardict->var)
579 struct variable *var = vardict->var;
580 var_set_vardict (var, vardict);
581 hmap_insert_fast (&d->name_map, &vardict->name_node,
582 vardict->name_node.hash);
584 if (d->changed) d->changed (d, d->changed_data);
587 d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
592 /* Sets the case_index in V's vardict to CASE_INDEX. */
594 set_var_case_index (struct variable *v, int case_index)
596 var_get_vardict (v)->case_index = case_index;
599 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
602 unindex_vars (struct dictionary *d, size_t from, size_t to)
606 for (i = from; i < to; i++)
607 unindex_var (d, &d->var[i]);
610 /* Re-sets the dict_index in the dictionary variables with
611 indexes from FROM to TO (exclusive). */
613 reindex_vars (struct dictionary *d, size_t from, size_t to)
617 for (i = from; i < to; i++)
618 reindex_var (d, &d->var[i]);
621 /* Deletes variable V from dictionary D and frees V.
623 This is a very bad idea if there might be any pointers to V
624 from outside D. In general, no variable in the active dataset's
625 dictionary should be deleted when any transformations are
626 active on the dictionary's dataset, because those
627 transformations might reference the deleted variable. The
628 safest time to delete a variable is just after a procedure has
629 been executed, as done by DELETE VARIABLES.
631 Pointers to V within D are not a problem, because
632 dict_delete_var() knows to remove V from split variables,
633 weights, filters, etc. */
635 dict_delete_var (struct dictionary *d, struct variable *v)
637 int dict_index = var_get_dict_index (v);
638 const int case_index = var_get_case_index (v);
640 assert (dict_contains_var (d, v));
642 dict_unset_split_var (d, v);
643 dict_unset_mrset_var (d, v);
646 dict_set_weight (d, NULL);
649 dict_set_filter (d, NULL);
651 dict_clear_vectors (d);
653 /* Remove V from var array. */
654 unindex_vars (d, dict_index, d->var_cnt);
655 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
658 /* Update dict_index for each affected variable. */
659 reindex_vars (d, dict_index, d->var_cnt);
662 var_clear_vardict (v);
664 if (d->changed) d->changed (d, d->changed_data);
666 invalidate_proto (d);
667 if (d->callbacks && d->callbacks->var_deleted)
668 d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
673 /* Deletes the COUNT variables listed in VARS from D. This is
674 unsafe; see the comment on dict_delete_var() for details. */
676 dict_delete_vars (struct dictionary *d,
677 struct variable *const *vars, size_t count)
679 /* FIXME: this can be done in O(count) time, but this algorithm
681 assert (count == 0 || vars != NULL);
684 dict_delete_var (d, *vars++);
687 /* Deletes the COUNT variables in D starting at index IDX. This
688 is unsafe; see the comment on dict_delete_var() for
689 details. Deleting consecutive vars will result in less callbacks
690 compared to iterating over dict_delete_var.
691 A simple while loop over dict_delete_var will
692 produce (d->var_cnt - IDX) * COUNT variable changed callbacks
693 plus COUNT variable delete callbacks.
694 This here produces d->var_cnt - IDX variable changed callbacks
695 plus COUNT variable delete callbacks. */
697 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
699 assert (idx + count <= d->var_cnt);
701 /* We need to store the variable and the corresponding case_index
702 for the delete callbacks later. We store them in a linked list.*/
705 struct variable *var;
708 struct ll_list list = LL_INITIALIZER (list);
710 for (size_t i = idx; i < idx + count; i++)
712 struct delvar *dv = xmalloc (sizeof (struct delvar));
714 struct variable *v = d->var[i].var;
716 dict_unset_split_var (d, v);
717 dict_unset_mrset_var (d, v);
720 dict_set_weight (d, NULL);
723 dict_set_filter (d, NULL);
726 dv->case_index = var_get_case_index (v);
727 ll_push_tail (&list, (struct ll *)dv);
730 dict_clear_vectors (d);
732 /* Remove variables from var array. */
733 unindex_vars (d, idx, d->var_cnt);
734 remove_range (d->var, d->var_cnt, sizeof *d->var, idx, count);
737 /* Reindexing will result variable-changed callback */
738 reindex_vars (d, idx, d->var_cnt);
740 invalidate_proto (d);
741 if (d->changed) d->changed (d, d->changed_data);
743 /* Now issue the variable delete callbacks and delete
744 the variables. The vardict is not valid at this point
745 anymore. That is the reason why we stored the
746 caseindex before reindexing. */
747 for (size_t vi = idx; vi < idx + count; vi++)
749 struct delvar *dv = (struct delvar *) ll_pop_head (&list);
750 var_clear_vardict (dv->var);
751 if (d->callbacks && d->callbacks->var_deleted)
752 d->callbacks->var_deleted (d, dv->var, vi, dv->case_index, d->cb_data);
753 var_destroy (dv->var);
758 /* Deletes scratch variables from dictionary D. */
760 dict_delete_scratch_vars (struct dictionary *d)
764 /* FIXME: this can be done in O(count) time, but this algorithm
766 for (i = 0; i < d->var_cnt;)
767 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
768 dict_delete_var (d, d->var[i].var);
773 /* Moves V to 0-based position IDX in D. Other variables in D,
774 if any, retain their relative positions. Runs in time linear
775 in the distance moved. */
777 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
779 size_t old_index = var_get_dict_index (v);
781 assert (new_index < d->var_cnt);
783 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
784 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
785 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
788 /* Reorders the variables in D, placing the COUNT variables
789 listed in ORDER in that order at the beginning of D. The
790 other variables in D, if any, retain their relative
793 dict_reorder_vars (struct dictionary *d,
794 struct variable *const *order, size_t count)
796 struct vardict_info *new_var;
799 assert (count == 0 || order != NULL);
800 assert (count <= d->var_cnt);
802 new_var = xnmalloc (d->var_cap, sizeof *new_var);
804 /* Add variables in ORDER to new_var. */
805 for (i = 0; i < count; i++)
807 struct vardict_info *old_var;
809 assert (dict_contains_var (d, order[i]));
811 old_var = var_get_vardict (order[i]);
812 new_var[i] = *old_var;
813 old_var->dict = NULL;
816 /* Add remaining variables to new_var. */
817 for (i = 0; i < d->var_cnt; i++)
818 if (d->var[i].dict != NULL)
819 new_var[count++] = d->var[i];
820 assert (count == d->var_cnt);
822 /* Replace old vardicts by new ones. */
826 hmap_clear (&d->name_map);
827 reindex_vars (d, 0, d->var_cnt);
830 /* Changes the name of variable V that is currently in a dictionary to
833 rename_var (struct variable *v, const char *new_name)
835 struct vardict_info *vardict = var_get_vardict (v);
836 var_clear_vardict (v);
837 var_set_name (v, new_name);
838 vardict->name_node.hash = utf8_hash_case_string (new_name, 0);
839 var_set_vardict (v, vardict);
842 /* Tries to changes the name of V in D to name NEW_NAME. Returns true if
843 successful, false if a variable (other than V) with the given name already
846 dict_try_rename_var (struct dictionary *d, struct variable *v,
847 const char *new_name)
849 struct variable *conflict = dict_lookup_var (d, new_name);
850 if (conflict && v != conflict)
853 struct variable *old = var_clone (v);
854 unindex_var (d, var_get_vardict (v));
855 rename_var (v, new_name);
856 reindex_var (d, var_get_vardict (v));
858 if (settings_get_algorithm () == ENHANCED)
859 var_clear_short_names (v);
861 if (d->changed) d->changed (d, d->changed_data);
862 if (d->callbacks && d->callbacks->var_changed)
863 d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
870 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
871 a variable named NEW_NAME is already in D, except that
872 NEW_NAME may be the same as V's existing name. */
874 dict_rename_var (struct dictionary *d, struct variable *v,
875 const char *new_name)
877 bool ok UNUSED = dict_try_rename_var (d, v, new_name);
881 /* Renames COUNT variables specified in VARS to the names given
882 in NEW_NAMES within dictionary D. If the renaming would
883 result in a duplicate variable name, returns false and stores a
884 name that would be duplicated into *ERR_NAME (if ERR_NAME is
885 non-null). Otherwise, the renaming is successful, and true
888 dict_rename_vars (struct dictionary *d,
889 struct variable **vars, char **new_names, size_t count,
896 assert (count == 0 || vars != NULL);
897 assert (count == 0 || new_names != NULL);
899 /* Save the names of the variables to be renamed. */
900 pool = pool_create ();
901 old_names = pool_nalloc (pool, count, sizeof *old_names);
902 for (i = 0; i < count; i++)
903 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
905 /* Remove the variables to be renamed from the name hash,
907 for (i = 0; i < count; i++)
909 unindex_var (d, var_get_vardict (vars[i]));
910 rename_var (vars[i], new_names[i]);
913 /* Add the renamed variables back into the name hash,
914 checking for conflicts. */
915 for (i = 0; i < count; i++)
917 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
919 /* There is a name conflict.
920 Back out all the name changes that have already
921 taken place, and indicate failure. */
923 if (err_name != NULL)
924 *err_name = new_names[i];
926 for (i = 0; i < fail_idx; i++)
927 unindex_var (d, var_get_vardict (vars[i]));
929 for (i = 0; i < count; i++)
931 rename_var (vars[i], old_names[i]);
932 reindex_var (d, var_get_vardict (vars[i]));
938 reindex_var (d, var_get_vardict (vars[i]));
941 /* Clear short names. */
942 if (settings_get_algorithm () == ENHANCED)
943 for (i = 0; i < count; i++)
944 var_clear_short_names (vars[i]);
950 /* Returns true if a variable named NAME may be inserted in DICT;
951 that is, if there is not already a variable with that name in
952 DICT and if NAME is not a reserved word. (The caller's checks
953 have already verified that NAME is otherwise acceptable as a
956 var_name_is_insertable (const struct dictionary *dict, const char *name)
958 return (dict_lookup_var (dict, name) == NULL
959 && lex_id_to_token (ss_cstr (name)) == T_ID);
963 make_hinted_name (const struct dictionary *dict, const char *hint)
965 size_t hint_len = strlen (hint);
966 bool dropped = false;
971 /* The allocation size here is OK: characters that are copied directly fit
972 OK, and characters that are not copied directly are replaced by a single
973 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
974 replaced by '_' too. */
975 root = rp = xmalloc (hint_len + 1);
976 for (ofs = 0; ofs < hint_len; ofs += mblen)
980 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
983 ? lex_uc_is_id1 (uc) && uc != '$'
984 : lex_uc_is_idn (uc))
991 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
1000 unsigned long int i;
1002 if (var_name_is_insertable (dict, root))
1005 for (i = 0; i < ULONG_MAX; i++)
1007 char suffix[INT_BUFSIZE_BOUND (i) + 1];
1011 if (!str_format_26adic (i + 1, true, &suffix[1], sizeof suffix - 1))
1014 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
1015 if (var_name_is_insertable (dict, name))
1030 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
1032 unsigned long int number;
1034 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
1038 char name[3 + INT_STRLEN_BOUND (number) + 1];
1040 sprintf (name, "VAR%03lu", number);
1041 if (dict_lookup_var (dict, name) == NULL)
1043 if (num_start != NULL)
1044 *num_start = number + 1;
1045 return xstrdup (name);
1053 /* Devises and returns a variable name unique within DICT. The variable name
1054 is owned by the caller, which must free it with free() when it is no longer
1057 HINT, if it is non-null, is used as a suggestion that will be
1058 modified for suitability as a variable name and for
1061 If HINT is null or entirely unsuitable, a name in the form
1062 "VAR%03d" will be generated, where the smallest unused integer
1063 value is used. If NUM_START is non-null, then its value is
1064 used as the minimum numeric value to check, and it is updated
1065 to the next value to be checked.
1068 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
1069 unsigned long int *num_start)
1073 char *hinted_name = make_hinted_name (dict, hint);
1074 if (hinted_name != NULL)
1077 return make_numeric_name (dict, num_start);
1080 /* Returns whether variable names must be valid identifiers. Normally, this is
1081 true, but sometimes a dictionary is prepared for external use (e.g. output
1082 to a CSV file) where names don't have to be valid. */
1084 dict_get_names_must_be_ids (const struct dictionary *d)
1086 return d->names_must_be_ids;
1089 /* Sets whether variable names must be valid identifiers. Normally, this is
1090 true, but sometimes a dictionary is prepared for external use (e.g. output
1091 to a CSV file) where names don't have to be valid.
1093 Changing this setting from false to true doesn't make the dictionary check
1094 all the existing variable names, so it can cause an invariant violation. */
1096 dict_set_names_must_be_ids (struct dictionary *d, bool names_must_be_ids)
1098 d->names_must_be_ids = names_must_be_ids;
1101 /* Returns the weighting variable in dictionary D, or a null
1102 pointer if the dictionary is unweighted. */
1104 dict_get_weight (const struct dictionary *d)
1106 assert (d->weight == NULL || dict_contains_var (d, d->weight));
1111 /* Returns the value of D's weighting variable in case C, except
1112 that a negative weight is returned as 0. Returns 1 if the
1113 dictionary is unweighted. Will warn about missing, negative,
1114 or zero values if *WARN_ON_INVALID is true. The function will
1115 set *WARN_ON_INVALID to false if an invalid weight is
1118 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
1119 bool *warn_on_invalid)
1123 if (d->weight == NULL)
1127 double w = case_num (c, d->weight);
1129 return var_force_valid_weight (d->weight, w, warn_on_invalid);
1133 /* Returns the format to use for weights. */
1134 const struct fmt_spec *
1135 dict_get_weight_format (const struct dictionary *d)
1137 return d->weight ? var_get_print_format (d->weight) : &F_8_0;
1140 /* Sets the weighting variable of D to V, or turning off
1141 weighting if V is a null pointer. */
1143 dict_set_weight (struct dictionary *d, struct variable *v)
1145 assert (v == NULL || dict_contains_var (d, v));
1146 assert (v == NULL || var_is_numeric (v));
1150 if (d->changed) d->changed (d, d->changed_data);
1151 if (d->callbacks && d->callbacks->weight_changed)
1152 d->callbacks->weight_changed (d,
1153 v ? var_get_dict_index (v) : -1,
1157 /* Returns the filter variable in dictionary D (see cmd_filter())
1158 or a null pointer if the dictionary is unfiltered. */
1160 dict_get_filter (const struct dictionary *d)
1162 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1167 /* Sets V as the filter variable for dictionary D. Passing a
1168 null pointer for V turn off filtering. */
1170 dict_set_filter (struct dictionary *d, struct variable *v)
1172 assert (v == NULL || dict_contains_var (d, v));
1173 assert (v == NULL || var_is_numeric (v));
1177 if (d->changed) d->changed (d, d->changed_data);
1178 if (d->callbacks && d->callbacks->filter_changed)
1179 d->callbacks->filter_changed (d,
1180 v ? var_get_dict_index (v) : -1,
1184 /* Returns the case limit for dictionary D, or zero if the number
1185 of cases is unlimited. */
1187 dict_get_case_limit (const struct dictionary *d)
1189 return d->case_limit;
1192 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1193 0 for CASE_LIMIT to indicate no limit. */
1195 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1197 d->case_limit = case_limit;
1200 /* Returns the prototype used for cases created by dictionary D. */
1201 const struct caseproto *
1202 dict_get_proto (const struct dictionary *d_)
1204 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1205 if (d->proto == NULL)
1209 d->proto = caseproto_create ();
1210 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1211 for (i = 0; i < d->var_cnt; i++)
1212 d->proto = caseproto_set_width (d->proto,
1213 var_get_case_index (d->var[i].var),
1214 var_get_width (d->var[i].var));
1219 /* Returns the case index of the next value to be added to D.
1220 This value is the number of `union value's that need to be
1221 allocated to store a case for dictionary D. */
1223 dict_get_next_value_idx (const struct dictionary *d)
1225 return d->next_value_idx;
1228 /* Returns the number of bytes needed to store a case for
1231 dict_get_case_size (const struct dictionary *d)
1233 return sizeof (union value) * dict_get_next_value_idx (d);
1236 /* Reassigns values in dictionary D so that fragmentation is
1239 dict_compact_values (struct dictionary *d)
1243 d->next_value_idx = 0;
1244 for (i = 0; i < d->var_cnt; i++)
1246 struct variable *v = d->var[i].var;
1247 set_var_case_index (v, d->next_value_idx++);
1249 invalidate_proto (d);
1252 /* Returns the number of values occupied by the variables in
1253 dictionary D. All variables are considered if EXCLUDE_CLASSES
1254 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1255 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1256 corresponding type of variable.
1258 The return value may be less than the number of values in one
1259 of dictionary D's cases (as returned by
1260 dict_get_next_value_idx) even if E is 0, because there may be
1261 gaps in D's cases due to deleted variables. */
1263 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1268 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1270 | (1u << DC_SCRATCH))) == 0);
1273 for (i = 0; i < d->var_cnt; i++)
1275 enum dict_class class = var_get_dict_class (d->var[i].var);
1276 if (!(exclude_classes & (1u << class)))
1282 /* Returns the case prototype that would result after deleting
1283 all variables from D that are not in one of the
1284 EXCLUDE_CLASSES and compacting the dictionary with
1287 The caller must unref the returned caseproto when it is no
1290 dict_get_compacted_proto (const struct dictionary *d,
1291 unsigned int exclude_classes)
1293 struct caseproto *proto;
1296 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1298 | (1u << DC_SCRATCH))) == 0);
1300 proto = caseproto_create ();
1301 for (i = 0; i < d->var_cnt; i++)
1303 struct variable *v = d->var[i].var;
1304 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1305 proto = caseproto_add_width (proto, var_get_width (v));
1310 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1311 dict_get_split_cnt() to determine how many SPLIT FILE vars
1312 there are. Returns a null pointer if and only if there are no
1314 const struct variable *const *
1315 dict_get_split_vars (const struct dictionary *d)
1320 /* Returns the number of SPLIT FILE vars. */
1322 dict_get_split_cnt (const struct dictionary *d)
1324 return d->split_cnt;
1327 /* Removes variable V, which must be in D, from D's set of split
1330 dict_unset_split_var (struct dictionary *d, struct variable *v)
1334 assert (dict_contains_var (d, v));
1336 orig_count = d->split_cnt;
1337 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1338 &v, compare_var_ptrs, NULL);
1339 if (orig_count != d->split_cnt)
1341 if (d->changed) d->changed (d, d->changed_data);
1342 /* We changed the set of split variables so invoke the
1344 if (d->callbacks && d->callbacks->split_changed)
1345 d->callbacks->split_changed (d, d->cb_data);
1349 /* Sets CNT split vars SPLIT in dictionary D. */
1351 dict_set_split_vars (struct dictionary *d,
1352 struct variable *const *split, size_t cnt)
1354 assert (cnt == 0 || split != NULL);
1359 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1360 memcpy (d->split, split, cnt * sizeof *d->split);
1368 if (d->changed) d->changed (d, d->changed_data);
1369 if (d->callbacks && d->callbacks->split_changed)
1370 d->callbacks->split_changed (d, d->cb_data);
1373 /* Returns the file label for D, or a null pointer if D is
1374 unlabeled (see cmd_file_label()). */
1376 dict_get_label (const struct dictionary *d)
1381 /* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
1384 Removes D's label if LABEL is null or the empty string. */
1386 dict_set_label (struct dictionary *d, const char *label)
1389 if (label == NULL || label[0] == '\0')
1392 d->label = utf8_encoding_trunc (label, d->encoding, 60);
1395 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1396 return value is always nonnull; if there are no documents then the
1397 string_arary is empty.*/
1398 const struct string_array *
1399 dict_get_documents (const struct dictionary *d)
1401 return &d->documents;
1404 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1406 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1410 dict_clear_documents (d);
1412 for (i = 0; i < new_docs->n; i++)
1413 dict_add_document_line (d, new_docs->strings[i], false);
1416 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1417 into individual lines at new-line characters. Each line is truncated to at
1418 most DOC_LINE_LENGTH bytes in D's encoding. */
1420 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1424 dict_clear_documents (d);
1425 for (s = new_docs; *s != '\0';)
1427 size_t len = strcspn (s, "\n");
1428 char *line = xmemdup0 (s, len);
1429 dict_add_document_line (d, line, false);
1438 /* Drops the documents from dictionary D. */
1440 dict_clear_documents (struct dictionary *d)
1442 string_array_clear (&d->documents);
1445 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1446 truncated so that it is no more than 80 bytes in the dictionary's
1447 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1448 then a warning will be issued. */
1450 dict_add_document_line (struct dictionary *d, const char *line,
1456 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1457 truncated = line[trunc_len] != '\0';
1458 if (truncated && issue_warning)
1460 /* Note to translators: "bytes" is correct, not characters */
1461 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1464 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1469 /* Returns the number of document lines in dictionary D. */
1471 dict_get_document_line_cnt (const struct dictionary *d)
1473 return d->documents.n;
1476 /* Returns document line number IDX in dictionary D. The caller must not
1477 modify or free the returned string. */
1479 dict_get_document_line (const struct dictionary *d, size_t idx)
1481 assert (idx < d->documents.n);
1482 return d->documents.strings[idx];
1485 /* Creates in D a vector named NAME that contains the CNT
1486 variables in VAR. Returns true if successful, or false if a
1487 vector named NAME already exists in D. */
1489 dict_create_vector (struct dictionary *d,
1491 struct variable **var, size_t cnt)
1496 for (i = 0; i < cnt; i++)
1497 assert (dict_contains_var (d, var[i]));
1499 if (dict_lookup_vector (d, name) == NULL)
1501 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1502 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1509 /* Creates in D a vector named NAME that contains the CNT
1510 variables in VAR. A vector named NAME must not already exist
1513 dict_create_vector_assert (struct dictionary *d,
1515 struct variable **var, size_t cnt)
1517 assert (dict_lookup_vector (d, name) == NULL);
1518 dict_create_vector (d, name, var, cnt);
1521 /* Returns the vector in D with index IDX, which must be less
1522 than dict_get_vector_cnt (D). */
1523 const struct vector *
1524 dict_get_vector (const struct dictionary *d, size_t idx)
1526 assert (idx < d->vector_cnt);
1528 return d->vector[idx];
1531 /* Returns the number of vectors in D. */
1533 dict_get_vector_cnt (const struct dictionary *d)
1535 return d->vector_cnt;
1538 /* Looks up and returns the vector within D with the given
1540 const struct vector *
1541 dict_lookup_vector (const struct dictionary *d, const char *name)
1544 for (i = 0; i < d->vector_cnt; i++)
1545 if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
1546 return d->vector[i];
1550 /* Deletes all vectors from D. */
1552 dict_clear_vectors (struct dictionary *d)
1556 for (i = 0; i < d->vector_cnt; i++)
1557 vector_destroy (d->vector[i]);
1564 /* Multiple response sets. */
1566 /* Returns the multiple response set in DICT with index IDX, which must be
1567 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1568 const struct mrset *
1569 dict_get_mrset (const struct dictionary *dict, size_t idx)
1571 assert (idx < dict->n_mrsets);
1572 return dict->mrsets[idx];
1575 /* Returns the number of multiple response sets in DICT. */
1577 dict_get_n_mrsets (const struct dictionary *dict)
1579 return dict->n_mrsets;
1582 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1583 returns its index; otherwise, returns SIZE_MAX. */
1585 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1589 for (i = 0; i < dict->n_mrsets; i++)
1590 if (!utf8_strcasecmp (name, dict->mrsets[i]->name))
1596 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1597 returns it; otherwise, returns NULL. */
1598 const struct mrset *
1599 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1601 size_t idx = dict_lookup_mrset_idx (dict, name);
1602 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1605 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1606 true if a set was replaced, false if none existed with the specified name.
1608 Ownership of MRSET is transferred to DICT. */
1610 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1614 assert (mrset_ok (mrset, dict));
1616 idx = dict_lookup_mrset_idx (dict, mrset->name);
1617 if (idx == SIZE_MAX)
1619 dict->mrsets = xrealloc (dict->mrsets,
1620 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1621 dict->mrsets[dict->n_mrsets++] = mrset;
1626 mrset_destroy (dict->mrsets[idx]);
1627 dict->mrsets[idx] = mrset;
1632 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1633 from DICT and returns true. If none is found, returns false without
1636 Deleting one multiple response set causes the indexes of other sets within
1639 dict_delete_mrset (struct dictionary *dict, const char *name)
1641 size_t idx = dict_lookup_mrset_idx (dict, name);
1642 if (idx != SIZE_MAX)
1644 mrset_destroy (dict->mrsets[idx]);
1645 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1652 /* Deletes all multiple response sets from DICT. */
1654 dict_clear_mrsets (struct dictionary *dict)
1658 for (i = 0; i < dict->n_mrsets; i++)
1659 mrset_destroy (dict->mrsets[i]);
1660 free (dict->mrsets);
1661 dict->mrsets = NULL;
1665 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1667 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1671 assert (dict_contains_var (dict, var));
1673 for (i = 0; i < dict->n_mrsets;)
1675 struct mrset *mrset = dict->mrsets[i];
1678 for (j = 0; j < mrset->n_vars;)
1679 if (mrset->vars[j] == var)
1680 remove_element (mrset->vars, mrset->n_vars--,
1681 sizeof *mrset->vars, j);
1685 if (mrset->n_vars < 2)
1687 mrset_destroy (mrset);
1688 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1695 /* Returns D's attribute set. The caller may examine or modify
1696 the attribute set, but must not destroy it. Destroying D or
1697 calling dict_set_attributes for D will also destroy D's
1700 dict_get_attributes (const struct dictionary *d)
1702 return CONST_CAST (struct attrset *, &d->attributes);
1705 /* Replaces D's attributes set by a copy of ATTRS. */
1707 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1709 attrset_destroy (&d->attributes);
1710 attrset_clone (&d->attributes, attrs);
1713 /* Returns true if D has at least one attribute in its attribute
1714 set, false if D's attribute set is empty. */
1716 dict_has_attributes (const struct dictionary *d)
1718 return attrset_count (&d->attributes) > 0;
1721 /* Called from variable.c to notify the dictionary that some property (indicated
1722 by WHAT) of the variable has changed. OLDVAR is a copy of V as it existed
1723 prior to the change. OLDVAR is destroyed by this function.
1726 dict_var_changed (const struct variable *v, unsigned int what, struct variable *oldvar)
1728 if (var_has_vardict (v))
1730 const struct vardict_info *vardict = var_get_vardict (v);
1731 struct dictionary *d = vardict->dict;
1736 if (d->changed) d->changed (d, d->changed_data);
1737 if (d->callbacks && d->callbacks->var_changed)
1738 d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
1740 var_destroy (oldvar);
1745 /* Dictionary used to contain "internal variables". */
1746 static struct dictionary *internal_dict;
1748 /* Create a variable of the specified WIDTH to be used for internal
1749 calculations only. The variable is assigned case index CASE_IDX. */
1751 dict_create_internal_var (int case_idx, int width)
1753 if (internal_dict == NULL)
1754 internal_dict = dict_create ("UTF-8");
1758 static int counter = INT_MAX / 2;
1759 struct variable *var;
1762 if (++counter == INT_MAX)
1763 counter = INT_MAX / 2;
1765 sprintf (name, "$internal%d", counter);
1766 var = dict_create_var (internal_dict, name, width);
1769 set_var_case_index (var, case_idx);
1775 /* Destroys VAR, which must have been created with
1776 dict_create_internal_var(). */
1778 dict_destroy_internal_var (struct variable *var)
1782 dict_delete_var (internal_dict, var);
1784 /* Destroy internal_dict if it has no variables left, just so that
1785 valgrind --leak-check --show-reachable won't show internal_dict. */
1786 if (dict_get_var_cnt (internal_dict) == 0)
1788 dict_unref (internal_dict);
1789 internal_dict = NULL;
1795 vardict_get_dict_index (const struct vardict_info *vardict)
1797 return vardict - vardict->dict->var;