1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
58 struct vardict_info *var; /* Variables. */
59 size_t var_cnt, var_cap; /* Number of variables, capacity. */
60 struct caseproto *proto; /* Prototype for dictionary cases
62 struct hmap name_map; /* Variable index by name. */
63 int next_value_idx; /* Index of next `union value' to allocate. */
64 const struct variable **split; /* SPLIT FILE vars. */
65 size_t split_cnt; /* SPLIT FILE count. */
66 struct variable *weight; /* WEIGHT variable. */
67 struct variable *filter; /* FILTER variable. */
68 casenumber case_limit; /* Current case limit (N command). */
69 char *label; /* File label. */
70 struct string_array documents; /* Documents. */
71 struct vector **vector; /* Vectors of variables. */
72 size_t vector_cnt; /* Number of vectors. */
73 struct attrset attributes; /* Custom attributes. */
74 struct mrset **mrsets; /* Multiple response sets. */
75 size_t n_mrsets; /* Number of multiple response sets. */
77 /* Whether variable names must be valid identifiers. Normally, this is
78 true, but sometimes a dictionary is prepared for external use
79 (e.g. output to a CSV file) where names don't have to be valid. */
80 bool names_must_be_ids;
82 char *encoding; /* Character encoding of string data */
84 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
86 void *cb_data ; /* Data passed to callbacks */
88 void (*changed) (struct dictionary *, void *); /* Generic change callback */
92 static void dict_unset_split_var (struct dictionary *, struct variable *);
93 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
95 /* Returns the encoding for data in dictionary D. The return value is a
96 nonnull string that contains an IANA character set name. */
98 dict_get_encoding (const struct dictionary *d)
103 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
104 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
105 error message on failure. */
107 dict_id_is_valid (const struct dictionary *dict, const char *id,
110 return (!dict->names_must_be_ids
111 || id_is_valid (id, dict->encoding, issue_error));
115 dict_set_change_callback (struct dictionary *d,
116 void (*changed) (struct dictionary *, void*),
119 d->changed = changed;
120 d->changed_data = data;
123 /* Discards dictionary D's caseproto. (It will be regenerated
124 lazily, on demand.) */
126 invalidate_proto (struct dictionary *d)
128 caseproto_unref (d->proto);
132 /* Print a representation of dictionary D to stdout, for
133 debugging purposes. */
135 dict_dump (const struct dictionary *d)
138 for (i = 0 ; i < d->var_cnt ; ++i )
140 const struct variable *v = d->var[i].var;
141 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
143 var_get_dict_index (v),
144 var_get_case_index (v));
149 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
150 the dictionary or any of the variables it contains are modified.
151 Each callback will get passed CALLBACK_DATA.
152 Any callback may be NULL, in which case it'll be ignored.
155 dict_set_callbacks (struct dictionary *dict,
156 const struct dict_callbacks *callbacks,
159 dict->callbacks = callbacks;
160 dict->cb_data = callback_data;
163 /* Shallow copy the callbacks from SRC to DEST */
165 dict_copy_callbacks (struct dictionary *dest,
166 const struct dictionary *src)
168 dest->callbacks = src->callbacks;
169 dest->cb_data = src->cb_data;
172 /* Creates and returns a new dictionary with the specified ENCODING. */
174 dict_create (const char *encoding)
176 struct dictionary *d = xzalloc (sizeof *d);
178 d->encoding = xstrdup (encoding);
179 d->names_must_be_ids = true;
180 hmap_init (&d->name_map);
181 attrset_init (&d->attributes);
186 /* Creates and returns a (deep) copy of an existing
189 The new dictionary's case indexes are copied from the old
190 dictionary. If the new dictionary won't be used to access
191 cases produced with the old dictionary, then the new
192 dictionary's case indexes should be compacted with
193 dict_compact_values to save space.
195 Callbacks are not cloned. */
197 dict_clone (const struct dictionary *s)
199 struct dictionary *d;
202 d = dict_create (s->encoding);
203 dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));
205 for (i = 0; i < s->var_cnt; i++)
207 struct variable *sv = s->var[i].var;
208 struct variable *dv = dict_clone_var_assert (d, sv);
211 for (i = 0; i < var_get_short_name_cnt (sv); i++)
212 var_set_short_name (dv, i, var_get_short_name (sv, i));
214 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
217 d->next_value_idx = s->next_value_idx;
219 d->split_cnt = s->split_cnt;
220 if (d->split_cnt > 0)
222 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
223 for (i = 0; i < d->split_cnt; i++)
224 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
227 if (s->weight != NULL)
228 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
230 if (s->filter != NULL)
231 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
233 d->case_limit = s->case_limit;
234 dict_set_label (d, dict_get_label (s));
235 dict_set_documents (d, dict_get_documents (s));
237 d->vector_cnt = s->vector_cnt;
238 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
239 for (i = 0; i < s->vector_cnt; i++)
240 d->vector[i] = vector_clone (s->vector[i], s, d);
242 dict_set_attributes (d, dict_get_attributes (s));
244 for (i = 0; i < s->n_mrsets; i++)
246 const struct mrset *old = s->mrsets[i];
250 /* Clone old mrset, then replace vars from D by vars from S. */
251 new = mrset_clone (old);
252 for (j = 0; j < new->n_vars; j++)
253 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
255 dict_add_mrset (d, new);
261 /* Clears the contents from a dictionary without destroying the
262 dictionary itself. */
264 dict_clear (struct dictionary *d)
266 /* FIXME? Should we really clear case_limit, label, documents?
267 Others are necessarily cleared by deleting all the variables.*/
268 while (d->var_cnt > 0 )
270 dict_delete_var (d, d->var[d->var_cnt - 1].var);
275 d->var_cnt = d->var_cap = 0;
276 invalidate_proto (d);
277 hmap_clear (&d->name_map);
278 d->next_value_idx = 0;
279 dict_set_split_vars (d, NULL, 0);
280 dict_set_weight (d, NULL);
281 dict_set_filter (d, NULL);
285 string_array_clear (&d->documents);
286 dict_clear_vectors (d);
287 attrset_clear (&d->attributes);
290 /* Clears a dictionary and destroys it. */
292 dict_destroy (struct dictionary *d)
296 /* In general, we don't want callbacks occurring, if the dictionary
297 is being destroyed */
298 d->callbacks = NULL ;
301 string_array_destroy (&d->documents);
302 hmap_destroy (&d->name_map);
303 attrset_destroy (&d->attributes);
304 dict_clear_mrsets (d);
310 /* Returns the number of variables in D. */
312 dict_get_var_cnt (const struct dictionary *d)
317 /* Returns the variable in D with dictionary index IDX, which
318 must be between 0 and the count returned by
319 dict_get_var_cnt(), exclusive. */
321 dict_get_var (const struct dictionary *d, size_t idx)
323 assert (idx < d->var_cnt);
325 return d->var[idx].var;
328 /* Sets *VARS to an array of pointers to variables in D and *CNT
329 to the number of variables in *D. All variables are returned
330 except for those, if any, in the classes indicated by EXCLUDE.
331 (There is no point in putting DC_SYSTEM in EXCLUDE as
332 dictionaries never include system variables.) */
334 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
335 size_t *cnt, enum dict_class exclude)
337 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
340 /* Sets *VARS to an array of pointers to variables in D and *CNT
341 to the number of variables in *D. All variables are returned
342 except for those, if any, in the classes indicated by EXCLUDE.
343 (There is no point in putting DC_SYSTEM in EXCLUDE as
344 dictionaries never include system variables.) */
346 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
347 size_t *cnt, enum dict_class exclude)
352 assert (exclude == (exclude & DC_ALL));
355 for (i = 0; i < d->var_cnt; i++)
357 enum dict_class class = var_get_dict_class (d->var[i].var);
358 if (!(class & exclude))
362 *vars = xnmalloc (count, sizeof **vars);
364 for (i = 0; i < d->var_cnt; i++)
366 enum dict_class class = var_get_dict_class (d->var[i].var);
367 if (!(class & exclude))
368 (*vars)[(*cnt)++] = d->var[i].var;
370 assert (*cnt == count);
373 static struct variable *
374 add_var_with_case_index (struct dictionary *d, struct variable *v,
377 struct vardict_info *vardict;
379 assert (case_index >= d->next_value_idx);
381 /* Update dictionary. */
382 if (d->var_cnt >= d->var_cap)
386 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
387 hmap_clear (&d->name_map);
388 for (i = 0; i < d->var_cnt; i++)
390 var_set_vardict (d->var[i].var, &d->var[i]);
391 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
392 d->var[i].name_node.hash);
396 vardict = &d->var[d->var_cnt++];
399 hmap_insert (&d->name_map, &vardict->name_node,
400 utf8_hash_case_string (var_get_name (v), 0));
401 vardict->case_index = case_index;
402 var_set_vardict (v, vardict);
404 if ( d->changed ) d->changed (d, d->changed_data);
405 if ( d->callbacks && d->callbacks->var_added )
406 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
408 invalidate_proto (d);
409 d->next_value_idx = case_index + 1;
414 static struct variable *
415 add_var (struct dictionary *d, struct variable *v)
417 return add_var_with_case_index (d, v, d->next_value_idx);
420 /* Creates and returns a new variable in D with the given NAME
421 and WIDTH. Returns a null pointer if the given NAME would
422 duplicate that of an existing variable in the dictionary. */
424 dict_create_var (struct dictionary *d, const char *name, int width)
426 return (dict_lookup_var (d, name) == NULL
427 ? dict_create_var_assert (d, name, width)
431 /* Creates and returns a new variable in D with the given NAME
432 and WIDTH. Assert-fails if the given NAME would duplicate
433 that of an existing variable in the dictionary. */
435 dict_create_var_assert (struct dictionary *d, const char *name, int width)
437 assert (dict_lookup_var (d, name) == NULL);
438 return add_var (d, var_create (name, width));
441 /* Creates and returns a new variable in D, as a copy of existing variable
442 OLD_VAR, which need not be in D or in any dictionary. Returns a null
443 pointer if OLD_VAR's name would duplicate that of an existing variable in
446 dict_clone_var (struct dictionary *d, const struct variable *old_var)
448 return dict_clone_var_as (d, old_var, var_get_name (old_var));
451 /* Creates and returns a new variable in D, as a copy of existing variable
452 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
453 OLD_VAR's name would duplicate that of an existing variable in the
456 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
458 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
461 /* Creates and returns a new variable in D with name NAME, as a copy of
462 existing variable OLD_VAR, which need not be in D or in any dictionary.
463 Returns a null pointer if the given NAME would duplicate that of an existing
464 variable in the dictionary. */
466 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
469 return (dict_lookup_var (d, name) == NULL
470 ? dict_clone_var_as_assert (d, old_var, name)
474 /* Creates and returns a new variable in D with name NAME, as a copy of
475 existing variable OLD_VAR, which need not be in D or in any dictionary.
476 Assert-fails if the given NAME would duplicate that of an existing variable
477 in the dictionary. */
479 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
482 struct variable *new_var = var_clone (old_var);
483 assert (dict_lookup_var (d, name) == NULL);
484 var_set_name (new_var, name);
485 return add_var (d, new_var);
489 dict_clone_var_in_place_assert (struct dictionary *d,
490 const struct variable *old_var)
492 assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
493 return add_var_with_case_index (d, var_clone (old_var),
494 var_get_case_index (old_var));
497 /* Returns the variable named NAME in D, or a null pointer if no
498 variable has that name. */
500 dict_lookup_var (const struct dictionary *d, const char *name)
502 struct vardict_info *vardict;
504 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
505 utf8_hash_case_string (name, 0), &d->name_map)
507 struct variable *var = vardict->var;
508 if (!utf8_strcasecmp (var_get_name (var), name))
515 /* Returns the variable named NAME in D. Assert-fails if no
516 variable has that name. */
518 dict_lookup_var_assert (const struct dictionary *d, const char *name)
520 struct variable *v = dict_lookup_var (d, name);
525 /* Returns true if variable V is in dictionary D,
528 dict_contains_var (const struct dictionary *d, const struct variable *v)
530 return (var_has_vardict (v)
531 && vardict_get_dictionary (var_get_vardict (v)) == d);
534 /* Compares two double pointers to variables, which should point
535 to elements of a struct dictionary's `var' member array. */
537 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
539 struct variable *const *a = a_;
540 struct variable *const *b = b_;
542 return *a < *b ? -1 : *a > *b;
546 unindex_var (struct dictionary *d, struct vardict_info *vardict)
548 hmap_delete (&d->name_map, &vardict->name_node);
551 /* This function assumes that vardict->name_node.hash is valid, that is, that
552 its name has not changed since it was hashed (rename_var() updates this
553 hash along with the name itself). */
555 reindex_var (struct dictionary *d, struct vardict_info *vardict)
557 struct variable *old = (d->callbacks && d->callbacks->var_changed
558 ? var_clone (vardict->var)
561 struct variable *var = vardict->var;
562 var_set_vardict (var, vardict);
563 hmap_insert_fast (&d->name_map, &vardict->name_node,
564 vardict->name_node.hash);
566 if ( d->changed ) d->changed (d, d->changed_data);
569 d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
574 /* Sets the case_index in V's vardict to CASE_INDEX. */
576 set_var_case_index (struct variable *v, int case_index)
578 var_get_vardict (v)->case_index = case_index;
581 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
584 unindex_vars (struct dictionary *d, size_t from, size_t to)
588 for (i = from; i < to; i++)
589 unindex_var (d, &d->var[i]);
592 /* Re-sets the dict_index in the dictionary variables with
593 indexes from FROM to TO (exclusive). */
595 reindex_vars (struct dictionary *d, size_t from, size_t to)
599 for (i = from; i < to; i++)
600 reindex_var (d, &d->var[i]);
603 /* Deletes variable V from dictionary D and frees V.
605 This is a very bad idea if there might be any pointers to V
606 from outside D. In general, no variable in the active dataset's
607 dictionary should be deleted when any transformations are
608 active on the dictionary's dataset, because those
609 transformations might reference the deleted variable. The
610 safest time to delete a variable is just after a procedure has
611 been executed, as done by DELETE VARIABLES.
613 Pointers to V within D are not a problem, because
614 dict_delete_var() knows to remove V from split variables,
615 weights, filters, etc. */
617 dict_delete_var (struct dictionary *d, struct variable *v)
619 int dict_index = var_get_dict_index (v);
620 const int case_index = var_get_case_index (v);
622 assert (dict_contains_var (d, v));
624 dict_unset_split_var (d, v);
625 dict_unset_mrset_var (d, v);
628 dict_set_weight (d, NULL);
631 dict_set_filter (d, NULL);
633 dict_clear_vectors (d);
635 /* Remove V from var array. */
636 unindex_vars (d, dict_index, d->var_cnt);
637 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
640 /* Update dict_index for each affected variable. */
641 reindex_vars (d, dict_index, d->var_cnt);
644 var_clear_vardict (v);
646 if ( d->changed ) d->changed (d, d->changed_data);
648 invalidate_proto (d);
649 if (d->callbacks && d->callbacks->var_deleted )
650 d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
655 /* Deletes the COUNT variables listed in VARS from D. This is
656 unsafe; see the comment on dict_delete_var() for details. */
658 dict_delete_vars (struct dictionary *d,
659 struct variable *const *vars, size_t count)
661 /* FIXME: this can be done in O(count) time, but this algorithm
663 assert (count == 0 || vars != NULL);
666 dict_delete_var (d, *vars++);
669 /* Deletes the COUNT variables in D starting at index IDX. This
670 is unsafe; see the comment on dict_delete_var() for
673 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
675 /* FIXME: this can be done in O(count) time, but this algorithm
677 assert (idx + count <= d->var_cnt);
680 dict_delete_var (d, d->var[idx].var);
683 /* Deletes scratch variables from dictionary D. */
685 dict_delete_scratch_vars (struct dictionary *d)
689 /* FIXME: this can be done in O(count) time, but this algorithm
691 for (i = 0; i < d->var_cnt; )
692 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
693 dict_delete_var (d, d->var[i].var);
698 /* Moves V to 0-based position IDX in D. Other variables in D,
699 if any, retain their relative positions. Runs in time linear
700 in the distance moved. */
702 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
704 size_t old_index = var_get_dict_index (v);
706 assert (new_index < d->var_cnt);
708 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
709 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
710 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
713 /* Reorders the variables in D, placing the COUNT variables
714 listed in ORDER in that order at the beginning of D. The
715 other variables in D, if any, retain their relative
718 dict_reorder_vars (struct dictionary *d,
719 struct variable *const *order, size_t count)
721 struct vardict_info *new_var;
724 assert (count == 0 || order != NULL);
725 assert (count <= d->var_cnt);
727 new_var = xnmalloc (d->var_cap, sizeof *new_var);
729 /* Add variables in ORDER to new_var. */
730 for (i = 0; i < count; i++)
732 struct vardict_info *old_var;
734 assert (dict_contains_var (d, order[i]));
736 old_var = var_get_vardict (order[i]);
737 new_var[i] = *old_var;
738 old_var->dict = NULL;
741 /* Add remaining variables to new_var. */
742 for (i = 0; i < d->var_cnt; i++)
743 if (d->var[i].dict != NULL)
744 new_var[count++] = d->var[i];
745 assert (count == d->var_cnt);
747 /* Replace old vardicts by new ones. */
751 hmap_clear (&d->name_map);
752 reindex_vars (d, 0, d->var_cnt);
755 /* Changes the name of variable V that is currently in a dictionary to
758 rename_var (struct variable *v, const char *new_name)
760 struct vardict_info *vardict = var_get_vardict (v);
761 var_clear_vardict (v);
762 var_set_name (v, new_name);
763 vardict->name_node.hash = utf8_hash_case_string (new_name, 0);
764 var_set_vardict (v, vardict);
767 /* Tries to changes the name of V in D to name NEW_NAME. Returns true if
768 successful, false if a variable (other than V) with the given name already
771 dict_try_rename_var (struct dictionary *d, struct variable *v,
772 const char *new_name)
774 struct variable *conflict = dict_lookup_var (d, new_name);
775 if (conflict && v != conflict)
778 struct variable *old = var_clone (v);
779 unindex_var (d, var_get_vardict (v));
780 rename_var (v, new_name);
781 reindex_var (d, var_get_vardict (v));
783 if (settings_get_algorithm () == ENHANCED)
784 var_clear_short_names (v);
786 if ( d->changed ) d->changed (d, d->changed_data);
787 if ( d->callbacks && d->callbacks->var_changed )
788 d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
795 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
796 a variable named NEW_NAME is already in D, except that
797 NEW_NAME may be the same as V's existing name. */
799 dict_rename_var (struct dictionary *d, struct variable *v,
800 const char *new_name)
802 bool ok UNUSED = dict_try_rename_var (d, v, new_name);
806 /* Renames COUNT variables specified in VARS to the names given
807 in NEW_NAMES within dictionary D. If the renaming would
808 result in a duplicate variable name, returns false and stores a
809 name that would be duplicated into *ERR_NAME (if ERR_NAME is
810 non-null). Otherwise, the renaming is successful, and true
813 dict_rename_vars (struct dictionary *d,
814 struct variable **vars, char **new_names, size_t count,
821 assert (count == 0 || vars != NULL);
822 assert (count == 0 || new_names != NULL);
824 /* Save the names of the variables to be renamed. */
825 pool = pool_create ();
826 old_names = pool_nalloc (pool, count, sizeof *old_names);
827 for (i = 0; i < count; i++)
828 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
830 /* Remove the variables to be renamed from the name hash,
832 for (i = 0; i < count; i++)
834 unindex_var (d, var_get_vardict (vars[i]));
835 rename_var (vars[i], new_names[i]);
838 /* Add the renamed variables back into the name hash,
839 checking for conflicts. */
840 for (i = 0; i < count; i++)
842 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
844 /* There is a name conflict.
845 Back out all the name changes that have already
846 taken place, and indicate failure. */
848 if (err_name != NULL)
849 *err_name = new_names[i];
851 for (i = 0; i < fail_idx; i++)
852 unindex_var (d, var_get_vardict (vars[i]));
854 for (i = 0; i < count; i++)
856 rename_var (vars[i], old_names[i]);
857 reindex_var (d, var_get_vardict (vars[i]));
863 reindex_var (d, var_get_vardict (vars[i]));
866 /* Clear short names. */
867 if (settings_get_algorithm () == ENHANCED)
868 for (i = 0; i < count; i++)
869 var_clear_short_names (vars[i]);
875 /* Returns true if a variable named NAME may be inserted in DICT;
876 that is, if there is not already a variable with that name in
877 DICT and if NAME is not a reserved word. (The caller's checks
878 have already verified that NAME is otherwise acceptable as a
881 var_name_is_insertable (const struct dictionary *dict, const char *name)
883 return (dict_lookup_var (dict, name) == NULL
884 && lex_id_to_token (ss_cstr (name)) == T_ID);
888 make_hinted_name (const struct dictionary *dict, const char *hint)
890 size_t hint_len = strlen (hint);
891 bool dropped = false;
896 /* The allocation size here is OK: characters that are copied directly fit
897 OK, and characters that are not copied directly are replaced by a single
898 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
899 replaced by '_' too. */
900 root = rp = xmalloc (hint_len + 1);
901 for (ofs = 0; ofs < hint_len; ofs += mblen)
905 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
908 ? lex_uc_is_id1 (uc) && uc != '$'
909 : lex_uc_is_idn (uc))
916 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
927 if (var_name_is_insertable (dict, root))
930 for (i = 0; i < ULONG_MAX; i++)
932 char suffix[INT_BUFSIZE_BOUND (i) + 1];
936 if (!str_format_26adic (i + 1, true, &suffix[1], sizeof suffix - 1))
939 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
940 if (var_name_is_insertable (dict, name))
955 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
957 unsigned long int number;
959 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
963 char name[3 + INT_STRLEN_BOUND (number) + 1];
965 sprintf (name, "VAR%03lu", number);
966 if (dict_lookup_var (dict, name) == NULL)
968 if (num_start != NULL)
969 *num_start = number + 1;
970 return xstrdup (name);
978 /* Devises and returns a variable name unique within DICT. The variable name
979 is owned by the caller, which must free it with free() when it is no longer
982 HINT, if it is non-null, is used as a suggestion that will be
983 modified for suitability as a variable name and for
986 If HINT is null or entirely unsuitable, a name in the form
987 "VAR%03d" will be generated, where the smallest unused integer
988 value is used. If NUM_START is non-null, then its value is
989 used as the minimum numeric value to check, and it is updated
990 to the next value to be checked.
993 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
994 unsigned long int *num_start)
998 char *hinted_name = make_hinted_name (dict, hint);
999 if (hinted_name != NULL)
1002 return make_numeric_name (dict, num_start);
1005 /* Returns whether variable names must be valid identifiers. Normally, this is
1006 true, but sometimes a dictionary is prepared for external use (e.g. output
1007 to a CSV file) where names don't have to be valid. */
1009 dict_get_names_must_be_ids (const struct dictionary *d)
1011 return d->names_must_be_ids;
1014 /* Sets whether variable names must be valid identifiers. Normally, this is
1015 true, but sometimes a dictionary is prepared for external use (e.g. output
1016 to a CSV file) where names don't have to be valid.
1018 Changing this setting from false to true doesn't make the dictionary check
1019 all the existing variable names, so it can cause an invariant violation. */
1021 dict_set_names_must_be_ids (struct dictionary *d, bool names_must_be_ids)
1023 d->names_must_be_ids = names_must_be_ids;
1026 /* Returns the weighting variable in dictionary D, or a null
1027 pointer if the dictionary is unweighted. */
1029 dict_get_weight (const struct dictionary *d)
1031 assert (d->weight == NULL || dict_contains_var (d, d->weight));
1036 /* Returns the value of D's weighting variable in case C, except
1037 that a negative weight is returned as 0. Returns 1 if the
1038 dictionary is unweighted. Will warn about missing, negative,
1039 or zero values if *WARN_ON_INVALID is true. The function will
1040 set *WARN_ON_INVALID to false if an invalid weight is
1043 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
1044 bool *warn_on_invalid)
1048 if (d->weight == NULL)
1052 double w = case_num (c, d->weight);
1054 return var_force_valid_weight (d->weight, w, warn_on_invalid);
1058 /* Sets the weighting variable of D to V, or turning off
1059 weighting if V is a null pointer. */
1061 dict_set_weight (struct dictionary *d, struct variable *v)
1063 assert (v == NULL || dict_contains_var (d, v));
1064 assert (v == NULL || var_is_numeric (v));
1068 if (d->changed) d->changed (d, d->changed_data);
1069 if ( d->callbacks && d->callbacks->weight_changed )
1070 d->callbacks->weight_changed (d,
1071 v ? var_get_dict_index (v) : -1,
1075 /* Returns the filter variable in dictionary D (see cmd_filter())
1076 or a null pointer if the dictionary is unfiltered. */
1078 dict_get_filter (const struct dictionary *d)
1080 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1085 /* Sets V as the filter variable for dictionary D. Passing a
1086 null pointer for V turn off filtering. */
1088 dict_set_filter (struct dictionary *d, struct variable *v)
1090 assert (v == NULL || dict_contains_var (d, v));
1091 assert (v == NULL || var_is_numeric (v));
1095 if (d->changed) d->changed (d, d->changed_data);
1096 if ( d->callbacks && d->callbacks->filter_changed )
1097 d->callbacks->filter_changed (d,
1098 v ? var_get_dict_index (v) : -1,
1102 /* Returns the case limit for dictionary D, or zero if the number
1103 of cases is unlimited. */
1105 dict_get_case_limit (const struct dictionary *d)
1107 return d->case_limit;
1110 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1111 0 for CASE_LIMIT to indicate no limit. */
1113 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1115 d->case_limit = case_limit;
1118 /* Returns the prototype used for cases created by dictionary D. */
1119 const struct caseproto *
1120 dict_get_proto (const struct dictionary *d_)
1122 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1123 if (d->proto == NULL)
1127 d->proto = caseproto_create ();
1128 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1129 for (i = 0; i < d->var_cnt; i++)
1130 d->proto = caseproto_set_width (d->proto,
1131 var_get_case_index (d->var[i].var),
1132 var_get_width (d->var[i].var));
1137 /* Returns the case index of the next value to be added to D.
1138 This value is the number of `union value's that need to be
1139 allocated to store a case for dictionary D. */
1141 dict_get_next_value_idx (const struct dictionary *d)
1143 return d->next_value_idx;
1146 /* Returns the number of bytes needed to store a case for
1149 dict_get_case_size (const struct dictionary *d)
1151 return sizeof (union value) * dict_get_next_value_idx (d);
1154 /* Reassigns values in dictionary D so that fragmentation is
1157 dict_compact_values (struct dictionary *d)
1161 d->next_value_idx = 0;
1162 for (i = 0; i < d->var_cnt; i++)
1164 struct variable *v = d->var[i].var;
1165 set_var_case_index (v, d->next_value_idx++);
1167 invalidate_proto (d);
1170 /* Returns the number of values occupied by the variables in
1171 dictionary D. All variables are considered if EXCLUDE_CLASSES
1172 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1173 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1174 corresponding type of variable.
1176 The return value may be less than the number of values in one
1177 of dictionary D's cases (as returned by
1178 dict_get_next_value_idx) even if E is 0, because there may be
1179 gaps in D's cases due to deleted variables. */
1181 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1186 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1188 | (1u << DC_SCRATCH))) == 0);
1191 for (i = 0; i < d->var_cnt; i++)
1193 enum dict_class class = var_get_dict_class (d->var[i].var);
1194 if (!(exclude_classes & (1u << class)))
1200 /* Returns the case prototype that would result after deleting
1201 all variables from D that are not in one of the
1202 EXCLUDE_CLASSES and compacting the dictionary with
1205 The caller must unref the returned caseproto when it is no
1208 dict_get_compacted_proto (const struct dictionary *d,
1209 unsigned int exclude_classes)
1211 struct caseproto *proto;
1214 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1216 | (1u << DC_SCRATCH))) == 0);
1218 proto = caseproto_create ();
1219 for (i = 0; i < d->var_cnt; i++)
1221 struct variable *v = d->var[i].var;
1222 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1223 proto = caseproto_add_width (proto, var_get_width (v));
1228 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1229 dict_get_split_cnt() to determine how many SPLIT FILE vars
1230 there are. Returns a null pointer if and only if there are no
1232 const struct variable *const *
1233 dict_get_split_vars (const struct dictionary *d)
1238 /* Returns the number of SPLIT FILE vars. */
1240 dict_get_split_cnt (const struct dictionary *d)
1242 return d->split_cnt;
1245 /* Removes variable V, which must be in D, from D's set of split
1248 dict_unset_split_var (struct dictionary *d, struct variable *v)
1252 assert (dict_contains_var (d, v));
1254 orig_count = d->split_cnt;
1255 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1256 &v, compare_var_ptrs, NULL);
1257 if (orig_count != d->split_cnt)
1259 if (d->changed) d->changed (d, d->changed_data);
1260 /* We changed the set of split variables so invoke the
1262 if (d->callbacks && d->callbacks->split_changed)
1263 d->callbacks->split_changed (d, d->cb_data);
1267 /* Sets CNT split vars SPLIT in dictionary D. */
1269 dict_set_split_vars (struct dictionary *d,
1270 struct variable *const *split, size_t cnt)
1272 assert (cnt == 0 || split != NULL);
1277 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1278 memcpy (d->split, split, cnt * sizeof *d->split);
1286 if (d->changed) d->changed (d, d->changed_data);
1287 if ( d->callbacks && d->callbacks->split_changed )
1288 d->callbacks->split_changed (d, d->cb_data);
1291 /* Returns the file label for D, or a null pointer if D is
1292 unlabeled (see cmd_file_label()). */
1294 dict_get_label (const struct dictionary *d)
1299 /* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
1302 Removes D's label if LABEL is null or the empty string. */
1304 dict_set_label (struct dictionary *d, const char *label)
1307 if (label == NULL || label[0] == '\0')
1310 d->label = utf8_encoding_trunc (label, d->encoding, 60);
1313 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1314 return value is always nonnull; if there are no documents then the
1315 string_arary is empty.*/
1316 const struct string_array *
1317 dict_get_documents (const struct dictionary *d)
1319 return &d->documents;
1322 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1324 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1328 dict_clear_documents (d);
1330 for (i = 0; i < new_docs->n; i++)
1331 dict_add_document_line (d, new_docs->strings[i], false);
1334 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1335 into individual lines at new-line characters. Each line is truncated to at
1336 most DOC_LINE_LENGTH bytes in D's encoding. */
1338 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1342 dict_clear_documents (d);
1343 for (s = new_docs; *s != '\0'; )
1345 size_t len = strcspn (s, "\n");
1346 char *line = xmemdup0 (s, len);
1347 dict_add_document_line (d, line, false);
1356 /* Drops the documents from dictionary D. */
1358 dict_clear_documents (struct dictionary *d)
1360 string_array_clear (&d->documents);
1363 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1364 truncated so that it is no more than 80 bytes in the dictionary's
1365 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1366 then a warning will be issued. */
1368 dict_add_document_line (struct dictionary *d, const char *line,
1374 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1375 truncated = line[trunc_len] != '\0';
1376 if (truncated && issue_warning)
1378 /* Note to translators: "bytes" is correct, not characters */
1379 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1382 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1387 /* Returns the number of document lines in dictionary D. */
1389 dict_get_document_line_cnt (const struct dictionary *d)
1391 return d->documents.n;
1394 /* Returns document line number IDX in dictionary D. The caller must not
1395 modify or free the returned string. */
1397 dict_get_document_line (const struct dictionary *d, size_t idx)
1399 assert (idx < d->documents.n);
1400 return d->documents.strings[idx];
1403 /* Creates in D a vector named NAME that contains the CNT
1404 variables in VAR. Returns true if successful, or false if a
1405 vector named NAME already exists in D. */
1407 dict_create_vector (struct dictionary *d,
1409 struct variable **var, size_t cnt)
1414 for (i = 0; i < cnt; i++)
1415 assert (dict_contains_var (d, var[i]));
1417 if (dict_lookup_vector (d, name) == NULL)
1419 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1420 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1427 /* Creates in D a vector named NAME that contains the CNT
1428 variables in VAR. A vector named NAME must not already exist
1431 dict_create_vector_assert (struct dictionary *d,
1433 struct variable **var, size_t cnt)
1435 assert (dict_lookup_vector (d, name) == NULL);
1436 dict_create_vector (d, name, var, cnt);
1439 /* Returns the vector in D with index IDX, which must be less
1440 than dict_get_vector_cnt (D). */
1441 const struct vector *
1442 dict_get_vector (const struct dictionary *d, size_t idx)
1444 assert (idx < d->vector_cnt);
1446 return d->vector[idx];
1449 /* Returns the number of vectors in D. */
1451 dict_get_vector_cnt (const struct dictionary *d)
1453 return d->vector_cnt;
1456 /* Looks up and returns the vector within D with the given
1458 const struct vector *
1459 dict_lookup_vector (const struct dictionary *d, const char *name)
1462 for (i = 0; i < d->vector_cnt; i++)
1463 if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
1464 return d->vector[i];
1468 /* Deletes all vectors from D. */
1470 dict_clear_vectors (struct dictionary *d)
1474 for (i = 0; i < d->vector_cnt; i++)
1475 vector_destroy (d->vector[i]);
1482 /* Multiple response sets. */
1484 /* Returns the multiple response set in DICT with index IDX, which must be
1485 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1486 const struct mrset *
1487 dict_get_mrset (const struct dictionary *dict, size_t idx)
1489 assert (idx < dict->n_mrsets);
1490 return dict->mrsets[idx];
1493 /* Returns the number of multiple response sets in DICT. */
1495 dict_get_n_mrsets (const struct dictionary *dict)
1497 return dict->n_mrsets;
1500 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1501 returns its index; otherwise, returns SIZE_MAX. */
1503 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1507 for (i = 0; i < dict->n_mrsets; i++)
1508 if (!utf8_strcasecmp (name, dict->mrsets[i]->name))
1514 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1515 returns it; otherwise, returns NULL. */
1516 const struct mrset *
1517 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1519 size_t idx = dict_lookup_mrset_idx (dict, name);
1520 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1523 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1524 true if a set was replaced, false if none existed with the specified name.
1526 Ownership of MRSET is transferred to DICT. */
1528 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1532 assert (mrset_ok (mrset, dict));
1534 idx = dict_lookup_mrset_idx (dict, mrset->name);
1535 if (idx == SIZE_MAX)
1537 dict->mrsets = xrealloc (dict->mrsets,
1538 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1539 dict->mrsets[dict->n_mrsets++] = mrset;
1544 mrset_destroy (dict->mrsets[idx]);
1545 dict->mrsets[idx] = mrset;
1550 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1551 from DICT and returns true. If none is found, returns false without
1554 Deleting one multiple response set causes the indexes of other sets within
1557 dict_delete_mrset (struct dictionary *dict, const char *name)
1559 size_t idx = dict_lookup_mrset_idx (dict, name);
1560 if (idx != SIZE_MAX)
1562 mrset_destroy (dict->mrsets[idx]);
1563 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1570 /* Deletes all multiple response sets from DICT. */
1572 dict_clear_mrsets (struct dictionary *dict)
1576 for (i = 0; i < dict->n_mrsets; i++)
1577 mrset_destroy (dict->mrsets[i]);
1578 free (dict->mrsets);
1579 dict->mrsets = NULL;
1583 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1585 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1589 assert (dict_contains_var (dict, var));
1591 for (i = 0; i < dict->n_mrsets; )
1593 struct mrset *mrset = dict->mrsets[i];
1596 for (j = 0; j < mrset->n_vars; )
1597 if (mrset->vars[j] == var)
1598 remove_element (mrset->vars, mrset->n_vars--,
1599 sizeof *mrset->vars, j);
1603 if (mrset->n_vars < 2)
1605 mrset_destroy (mrset);
1606 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1613 /* Returns D's attribute set. The caller may examine or modify
1614 the attribute set, but must not destroy it. Destroying D or
1615 calling dict_set_attributes for D will also destroy D's
1618 dict_get_attributes (const struct dictionary *d)
1620 return CONST_CAST (struct attrset *, &d->attributes);
1623 /* Replaces D's attributes set by a copy of ATTRS. */
1625 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1627 attrset_destroy (&d->attributes);
1628 attrset_clone (&d->attributes, attrs);
1631 /* Returns true if D has at least one attribute in its attribute
1632 set, false if D's attribute set is empty. */
1634 dict_has_attributes (const struct dictionary *d)
1636 return attrset_count (&d->attributes) > 0;
1639 /* Called from variable.c to notify the dictionary that some property (indicated
1640 by WHAT) of the variable has changed. OLDVAR is a copy of V as it existed
1641 prior to the change. OLDVAR is destroyed by this function.
1644 dict_var_changed (const struct variable *v, unsigned int what, struct variable *oldvar)
1646 if ( var_has_vardict (v))
1648 const struct vardict_info *vardict = var_get_vardict (v);
1649 struct dictionary *d = vardict->dict;
1654 if (d->changed ) d->changed (d, d->changed_data);
1655 if ( d->callbacks && d->callbacks->var_changed )
1656 d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
1658 var_destroy (oldvar);
1663 /* Dictionary used to contain "internal variables". */
1664 static struct dictionary *internal_dict;
1666 /* Create a variable of the specified WIDTH to be used for internal
1667 calculations only. The variable is assigned case index CASE_IDX. */
1669 dict_create_internal_var (int case_idx, int width)
1671 if (internal_dict == NULL)
1672 internal_dict = dict_create ("UTF-8");
1676 static int counter = INT_MAX / 2;
1677 struct variable *var;
1680 if (++counter == INT_MAX)
1681 counter = INT_MAX / 2;
1683 sprintf (name, "$internal%d", counter);
1684 var = dict_create_var (internal_dict, name, width);
1687 set_var_case_index (var, case_idx);
1693 /* Destroys VAR, which must have been created with
1694 dict_create_internal_var(). */
1696 dict_destroy_internal_var (struct variable *var)
1700 dict_delete_var (internal_dict, var);
1702 /* Destroy internal_dict if it has no variables left, just so that
1703 valgrind --leak-check --show-reachable won't show internal_dict. */
1704 if (dict_get_var_cnt (internal_dict) == 0)
1706 dict_destroy (internal_dict);
1707 internal_dict = NULL;
1713 vardict_get_dict_index (const struct vardict_info *vardict)
1715 return vardict - vardict->dict->var;