1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
58 struct vardict_info *var; /* Variables. */
59 size_t var_cnt, var_cap; /* Number of variables, capacity. */
60 struct caseproto *proto; /* Prototype for dictionary cases
62 struct hmap name_map; /* Variable index by name. */
63 int next_value_idx; /* Index of next `union value' to allocate. */
64 const struct variable **split; /* SPLIT FILE vars. */
65 size_t split_cnt; /* SPLIT FILE count. */
66 struct variable *weight; /* WEIGHT variable. */
67 struct variable *filter; /* FILTER variable. */
68 casenumber case_limit; /* Current case limit (N command). */
69 char *label; /* File label. */
70 struct string_array documents; /* Documents. */
71 struct vector **vector; /* Vectors of variables. */
72 size_t vector_cnt; /* Number of vectors. */
73 struct attrset attributes; /* Custom attributes. */
74 struct mrset **mrsets; /* Multiple response sets. */
75 size_t n_mrsets; /* Number of multiple response sets. */
77 char *encoding; /* Character encoding of string data */
79 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
81 void *cb_data ; /* Data passed to callbacks */
83 void (*changed) (struct dictionary *, void *); /* Generic change callback */
87 static void dict_unset_split_var (struct dictionary *, struct variable *);
88 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
91 dict_get_encoding (const struct dictionary *d)
96 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
97 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
98 error message on failure. */
100 dict_id_is_valid (const struct dictionary *dict, const char *id,
103 return id_is_valid (id, dict->encoding, issue_error);
107 dict_set_change_callback (struct dictionary *d,
108 void (*changed) (struct dictionary *, void*),
111 d->changed = changed;
112 d->changed_data = data;
115 /* Discards dictionary D's caseproto. (It will be regenerated
116 lazily, on demand.) */
118 invalidate_proto (struct dictionary *d)
120 caseproto_unref (d->proto);
124 /* Print a representation of dictionary D to stdout, for
125 debugging purposes. */
127 dict_dump (const struct dictionary *d)
130 for (i = 0 ; i < d->var_cnt ; ++i )
132 const struct variable *v = d->var[i].var;
133 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
135 var_get_dict_index (v),
136 var_get_case_index (v));
141 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
142 the dictionary or any of the variables it contains are modified.
143 Each callback will get passed CALLBACK_DATA.
144 Any callback may be NULL, in which case it'll be ignored.
147 dict_set_callbacks (struct dictionary *dict,
148 const struct dict_callbacks *callbacks,
151 dict->callbacks = callbacks;
152 dict->cb_data = callback_data;
155 /* Shallow copy the callbacks from SRC to DEST */
157 dict_copy_callbacks (struct dictionary *dest,
158 const struct dictionary *src)
160 dest->callbacks = src->callbacks;
161 dest->cb_data = src->cb_data;
164 /* Creates and returns a new dictionary with the specified ENCODING. */
166 dict_create (const char *encoding)
168 struct dictionary *d = xzalloc (sizeof *d);
170 d->encoding = xstrdup (encoding);
171 hmap_init (&d->name_map);
172 attrset_init (&d->attributes);
177 /* Creates and returns a (deep) copy of an existing
180 The new dictionary's case indexes are copied from the old
181 dictionary. If the new dictionary won't be used to access
182 cases produced with the old dictionary, then the new
183 dictionary's case indexes should be compacted with
184 dict_compact_values to save space.
186 Callbacks are not cloned. */
188 dict_clone (const struct dictionary *s)
190 struct dictionary *d;
193 d = dict_create (s->encoding);
195 /* Set the new dictionary's encoding early so that string length limitations
196 are interpreted correctly. */
198 d->encoding = xstrdup (s->encoding);
200 for (i = 0; i < s->var_cnt; i++)
202 struct variable *sv = s->var[i].var;
203 struct variable *dv = dict_clone_var_assert (d, sv);
206 for (i = 0; i < var_get_short_name_cnt (sv); i++)
207 var_set_short_name (dv, i, var_get_short_name (sv, i));
209 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
212 d->next_value_idx = s->next_value_idx;
214 d->split_cnt = s->split_cnt;
215 if (d->split_cnt > 0)
217 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
218 for (i = 0; i < d->split_cnt; i++)
219 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
222 if (s->weight != NULL)
223 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
225 if (s->filter != NULL)
226 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
228 d->case_limit = s->case_limit;
229 dict_set_label (d, dict_get_label (s));
230 dict_set_documents (d, dict_get_documents (s));
232 d->vector_cnt = s->vector_cnt;
233 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
234 for (i = 0; i < s->vector_cnt; i++)
235 d->vector[i] = vector_clone (s->vector[i], s, d);
237 dict_set_attributes (d, dict_get_attributes (s));
239 for (i = 0; i < s->n_mrsets; i++)
241 const struct mrset *old = s->mrsets[i];
245 /* Clone old mrset, then replace vars from D by vars from S. */
246 new = mrset_clone (old);
247 for (j = 0; j < new->n_vars; j++)
248 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
250 dict_add_mrset (d, new);
256 /* Clears the contents from a dictionary without destroying the
257 dictionary itself. */
259 dict_clear (struct dictionary *d)
261 /* FIXME? Should we really clear case_limit, label, documents?
262 Others are necessarily cleared by deleting all the variables.*/
263 while (d->var_cnt > 0 )
265 dict_delete_var (d, d->var[d->var_cnt - 1].var);
270 d->var_cnt = d->var_cap = 0;
271 invalidate_proto (d);
272 hmap_clear (&d->name_map);
273 d->next_value_idx = 0;
274 dict_set_split_vars (d, NULL, 0);
275 dict_set_weight (d, NULL);
276 dict_set_filter (d, NULL);
280 string_array_clear (&d->documents);
281 dict_clear_vectors (d);
282 attrset_clear (&d->attributes);
285 /* Destroys the aux data for every variable in D, by calling
286 var_clear_aux() for each variable. */
288 dict_clear_aux (struct dictionary *d)
292 for (i = 0; i < d->var_cnt; i++)
293 var_clear_aux (d->var[i].var);
296 /* Clears a dictionary and destroys it. */
298 dict_destroy (struct dictionary *d)
302 /* In general, we don't want callbacks occuring, if the dictionary
303 is being destroyed */
304 d->callbacks = NULL ;
307 hmap_destroy (&d->name_map);
308 attrset_destroy (&d->attributes);
309 dict_clear_mrsets (d);
315 /* Returns the number of variables in D. */
317 dict_get_var_cnt (const struct dictionary *d)
322 /* Returns the variable in D with dictionary index IDX, which
323 must be between 0 and the count returned by
324 dict_get_var_cnt(), exclusive. */
326 dict_get_var (const struct dictionary *d, size_t idx)
328 assert (idx < d->var_cnt);
330 return d->var[idx].var;
333 /* Sets *VARS to an array of pointers to variables in D and *CNT
334 to the number of variables in *D. All variables are returned
335 except for those, if any, in the classes indicated by EXCLUDE.
336 (There is no point in putting DC_SYSTEM in EXCLUDE as
337 dictionaries never include system variables.) */
339 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
340 size_t *cnt, enum dict_class exclude)
342 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
345 /* Sets *VARS to an array of pointers to variables in D and *CNT
346 to the number of variables in *D. All variables are returned
347 except for those, if any, in the classes indicated by EXCLUDE.
348 (There is no point in putting DC_SYSTEM in EXCLUDE as
349 dictionaries never include system variables.) */
351 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
352 size_t *cnt, enum dict_class exclude)
357 assert (exclude == (exclude & DC_ALL));
360 for (i = 0; i < d->var_cnt; i++)
362 enum dict_class class = var_get_dict_class (d->var[i].var);
363 if (!(class & exclude))
367 *vars = xnmalloc (count, sizeof **vars);
369 for (i = 0; i < d->var_cnt; i++)
371 enum dict_class class = var_get_dict_class (d->var[i].var);
372 if (!(class & exclude))
373 (*vars)[(*cnt)++] = d->var[i].var;
375 assert (*cnt == count);
378 static struct variable *
379 add_var (struct dictionary *d, struct variable *v)
381 struct vardict_info *vardict;
383 /* Update dictionary. */
384 if (d->var_cnt >= d->var_cap)
388 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
389 hmap_clear (&d->name_map);
390 for (i = 0; i < d->var_cnt; i++)
392 var_set_vardict (d->var[i].var, &d->var[i]);
393 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
394 d->var[i].name_node.hash);
398 vardict = &d->var[d->var_cnt++];
401 hmap_insert (&d->name_map, &vardict->name_node,
402 hash_case_string (var_get_name (v), 0));
403 vardict->case_index = d->next_value_idx;
404 var_set_vardict (v, vardict);
406 if ( d->changed ) d->changed (d, d->changed_data);
407 if ( d->callbacks && d->callbacks->var_added )
408 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
411 invalidate_proto (d);
416 /* Creates and returns a new variable in D with the given NAME
417 and WIDTH. Returns a null pointer if the given NAME would
418 duplicate that of an existing variable in the dictionary. */
420 dict_create_var (struct dictionary *d, const char *name, int width)
422 return (dict_lookup_var (d, name) == NULL
423 ? dict_create_var_assert (d, name, width)
427 /* Creates and returns a new variable in D with the given NAME
428 and WIDTH. Assert-fails if the given NAME would duplicate
429 that of an existing variable in the dictionary. */
431 dict_create_var_assert (struct dictionary *d, const char *name, int width)
433 assert (dict_lookup_var (d, name) == NULL);
434 return add_var (d, var_create (name, width));
437 /* Creates and returns a new variable in D, as a copy of existing variable
438 OLD_VAR, which need not be in D or in any dictionary. Returns a null
439 pointer if OLD_VAR's name would duplicate that of an existing variable in
442 dict_clone_var (struct dictionary *d, const struct variable *old_var)
444 return dict_clone_var_as (d, old_var, var_get_name (old_var));
447 /* Creates and returns a new variable in D, as a copy of existing variable
448 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
449 OLD_VAR's name would duplicate that of an existing variable in the
452 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
454 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
457 /* Creates and returns a new variable in D with name NAME, as a copy of
458 existing variable OLD_VAR, which need not be in D or in any dictionary.
459 Returns a null pointer if the given NAME would duplicate that of an existing
460 variable in the dictionary. */
462 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
465 return (dict_lookup_var (d, name) == NULL
466 ? dict_clone_var_as_assert (d, old_var, name)
470 /* Creates and returns a new variable in D with name NAME, as a copy of
471 existing variable OLD_VAR, which need not be in D or in any dictionary.
472 Assert-fails if the given NAME would duplicate that of an existing variable
473 in the dictionary. */
475 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
478 struct variable *new_var = var_clone (old_var);
479 assert (dict_lookup_var (d, name) == NULL);
480 var_set_name (new_var, name);
481 return add_var (d, new_var);
484 /* Returns the variable named NAME in D, or a null pointer if no
485 variable has that name. */
487 dict_lookup_var (const struct dictionary *d, const char *name)
489 struct vardict_info *vardict;
491 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
492 hash_case_string (name, 0), &d->name_map)
494 struct variable *var = vardict->var;
495 if (!strcasecmp (var_get_name (var), name))
502 /* Returns the variable named NAME in D. Assert-fails if no
503 variable has that name. */
505 dict_lookup_var_assert (const struct dictionary *d, const char *name)
507 struct variable *v = dict_lookup_var (d, name);
512 /* Returns true if variable V is in dictionary D,
515 dict_contains_var (const struct dictionary *d, const struct variable *v)
517 return (var_has_vardict (v)
518 && vardict_get_dictionary (var_get_vardict (v)) == d);
521 /* Compares two double pointers to variables, which should point
522 to elements of a struct dictionary's `var' member array. */
524 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
526 struct variable *const *a = a_;
527 struct variable *const *b = b_;
529 return *a < *b ? -1 : *a > *b;
533 unindex_var (struct dictionary *d, struct vardict_info *vardict)
535 hmap_delete (&d->name_map, &vardict->name_node);
538 /* This function assumes that vardict->name_node.hash is valid, that is, that
539 its name has not changed since it was hashed (rename_var() updates this
540 hash along with the name itself). */
542 reindex_var (struct dictionary *d, struct vardict_info *vardict)
544 struct variable *var = vardict->var;
546 var_set_vardict (var, vardict);
547 hmap_insert_fast (&d->name_map, &vardict->name_node,
548 vardict->name_node.hash);
550 if ( d->changed ) d->changed (d, d->changed_data);
551 if ( d->callbacks && d->callbacks->var_changed )
552 d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
555 /* Sets the case_index in V's vardict to CASE_INDEX. */
557 set_var_case_index (struct variable *v, int case_index)
559 var_get_vardict (v)->case_index = case_index;
562 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
565 unindex_vars (struct dictionary *d, size_t from, size_t to)
569 for (i = from; i < to; i++)
570 unindex_var (d, &d->var[i]);
573 /* Re-sets the dict_index in the dictionary variables with
574 indexes from FROM to TO (exclusive). */
576 reindex_vars (struct dictionary *d, size_t from, size_t to)
580 for (i = from; i < to; i++)
581 reindex_var (d, &d->var[i]);
584 /* Deletes variable V from dictionary D and frees V.
586 This is a very bad idea if there might be any pointers to V
587 from outside D. In general, no variable in the active dataset's
588 dictionary should be deleted when any transformations are
589 active on the dictionary's dataset, because those
590 transformations might reference the deleted variable. The
591 safest time to delete a variable is just after a procedure has
592 been executed, as done by DELETE VARIABLES.
594 Pointers to V within D are not a problem, because
595 dict_delete_var() knows to remove V from split variables,
596 weights, filters, etc. */
598 dict_delete_var (struct dictionary *d, struct variable *v)
600 int dict_index = var_get_dict_index (v);
601 const int case_index = var_get_case_index (v);
602 const int width = var_get_width (v);
604 assert (dict_contains_var (d, v));
606 /* Delete aux data. */
609 dict_unset_split_var (d, v);
610 dict_unset_mrset_var (d, v);
613 dict_set_weight (d, NULL);
616 dict_set_filter (d, NULL);
618 dict_clear_vectors (d);
620 /* Remove V from var array. */
621 unindex_vars (d, dict_index, d->var_cnt);
622 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
625 /* Update dict_index for each affected variable. */
626 reindex_vars (d, dict_index, d->var_cnt);
629 var_clear_vardict (v);
632 if ( d->changed ) d->changed (d, d->changed_data);
634 invalidate_proto (d);
635 if (d->callbacks && d->callbacks->var_deleted )
636 d->callbacks->var_deleted (d, dict_index, case_index, width, d->cb_data);
639 /* Deletes the COUNT variables listed in VARS from D. This is
640 unsafe; see the comment on dict_delete_var() for details. */
642 dict_delete_vars (struct dictionary *d,
643 struct variable *const *vars, size_t count)
645 /* FIXME: this can be done in O(count) time, but this algorithm
647 assert (count == 0 || vars != NULL);
650 dict_delete_var (d, *vars++);
653 /* Deletes the COUNT variables in D starting at index IDX. This
654 is unsafe; see the comment on dict_delete_var() for
657 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
659 /* FIXME: this can be done in O(count) time, but this algorithm
661 assert (idx + count <= d->var_cnt);
664 dict_delete_var (d, d->var[idx].var);
667 /* Deletes scratch variables from dictionary D. */
669 dict_delete_scratch_vars (struct dictionary *d)
673 /* FIXME: this can be done in O(count) time, but this algorithm
675 for (i = 0; i < d->var_cnt; )
676 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
677 dict_delete_var (d, d->var[i].var);
682 /* Moves V to 0-based position IDX in D. Other variables in D,
683 if any, retain their relative positions. Runs in time linear
684 in the distance moved. */
686 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
688 size_t old_index = var_get_dict_index (v);
690 assert (new_index < d->var_cnt);
692 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
693 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
694 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
697 /* Reorders the variables in D, placing the COUNT variables
698 listed in ORDER in that order at the beginning of D. The
699 other variables in D, if any, retain their relative
702 dict_reorder_vars (struct dictionary *d,
703 struct variable *const *order, size_t count)
705 struct vardict_info *new_var;
708 assert (count == 0 || order != NULL);
709 assert (count <= d->var_cnt);
711 new_var = xnmalloc (d->var_cap, sizeof *new_var);
713 /* Add variables in ORDER to new_var. */
714 for (i = 0; i < count; i++)
716 struct vardict_info *old_var;
718 assert (dict_contains_var (d, order[i]));
720 old_var = var_get_vardict (order[i]);
721 new_var[i] = *old_var;
722 old_var->dict = NULL;
725 /* Add remaining variables to new_var. */
726 for (i = 0; i < d->var_cnt; i++)
727 if (d->var[i].dict != NULL)
728 new_var[count++] = d->var[i];
729 assert (count == d->var_cnt);
731 /* Replace old vardicts by new ones. */
735 hmap_clear (&d->name_map);
736 reindex_vars (d, 0, d->var_cnt);
739 /* Changes the name of variable V that is currently in a dictionary to
742 rename_var (struct variable *v, const char *new_name)
744 struct vardict_info *vardict = var_get_vardict (v);
745 var_clear_vardict (v);
746 var_set_name (v, new_name);
747 vardict->name_node.hash = hash_case_string (new_name, 0);
748 var_set_vardict (v, vardict);
751 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
752 a variable named NEW_NAME is already in D, except that
753 NEW_NAME may be the same as V's existing name. */
755 dict_rename_var (struct dictionary *d, struct variable *v,
756 const char *new_name)
758 assert (!strcasecmp (var_get_name (v), new_name)
759 || dict_lookup_var (d, new_name) == NULL);
761 unindex_var (d, var_get_vardict (v));
762 rename_var (v, new_name);
763 reindex_var (d, var_get_vardict (v));
765 if (settings_get_algorithm () == ENHANCED)
766 var_clear_short_names (v);
768 if ( d->changed ) d->changed (d, d->changed_data);
769 if ( d->callbacks && d->callbacks->var_changed )
770 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
773 /* Renames COUNT variables specified in VARS to the names given
774 in NEW_NAMES within dictionary D. If the renaming would
775 result in a duplicate variable name, returns false and stores a
776 name that would be duplicated into *ERR_NAME (if ERR_NAME is
777 non-null). Otherwise, the renaming is successful, and true
780 dict_rename_vars (struct dictionary *d,
781 struct variable **vars, char **new_names, size_t count,
788 assert (count == 0 || vars != NULL);
789 assert (count == 0 || new_names != NULL);
791 /* Save the names of the variables to be renamed. */
792 pool = pool_create ();
793 old_names = pool_nalloc (pool, count, sizeof *old_names);
794 for (i = 0; i < count; i++)
795 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
797 /* Remove the variables to be renamed from the name hash,
799 for (i = 0; i < count; i++)
801 unindex_var (d, var_get_vardict (vars[i]));
802 rename_var (vars[i], new_names[i]);
805 /* Add the renamed variables back into the name hash,
806 checking for conflicts. */
807 for (i = 0; i < count; i++)
809 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
811 /* There is a name conflict.
812 Back out all the name changes that have already
813 taken place, and indicate failure. */
815 if (err_name != NULL)
816 *err_name = new_names[i];
818 for (i = 0; i < fail_idx; i++)
819 unindex_var (d, var_get_vardict (vars[i]));
821 for (i = 0; i < count; i++)
823 rename_var (vars[i], old_names[i]);
824 reindex_var (d, var_get_vardict (vars[i]));
830 reindex_var (d, var_get_vardict (vars[i]));
833 /* Clear short names. */
834 if (settings_get_algorithm () == ENHANCED)
835 for (i = 0; i < count; i++)
836 var_clear_short_names (vars[i]);
842 /* Returns true if a variable named NAME may be inserted in DICT;
843 that is, if there is not already a variable with that name in
844 DICT and if NAME is not a reserved word. (The caller's checks
845 have already verified that NAME is otherwise acceptable as a
848 var_name_is_insertable (const struct dictionary *dict, const char *name)
850 return (dict_lookup_var (dict, name) == NULL
851 && lex_id_to_token (ss_cstr (name)) == T_ID);
855 make_hinted_name (const struct dictionary *dict, const char *hint)
857 size_t hint_len = strlen (hint);
858 bool dropped = false;
863 /* The allocation size here is OK: characters that are copied directly fit
864 OK, and characters that are not copied directly are replaced by a single
865 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
866 replaced by '_' too. */
867 root = rp = xmalloc (hint_len + 1);
868 for (ofs = 0; ofs < hint_len; ofs += mblen)
872 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
875 ? lex_uc_is_id1 (uc) && uc != '$'
876 : lex_uc_is_idn (uc))
883 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
894 if (var_name_is_insertable (dict, root))
897 for (i = 0; i < ULONG_MAX; i++)
899 char suffix[INT_BUFSIZE_BOUND (i) + 1];
903 if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
906 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
907 if (var_name_is_insertable (dict, name))
922 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
924 unsigned long int number;
926 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
930 char name[3 + INT_STRLEN_BOUND (number) + 1];
932 sprintf (name, "VAR%03lu", number);
933 if (dict_lookup_var (dict, name) == NULL)
935 if (num_start != NULL)
936 *num_start = number + 1;
937 return xstrdup (name);
945 /* Devises and returns a variable name unique within DICT. The variable name
946 is owned by the caller, which must free it with free() when it is no longer
949 HINT, if it is non-null, is used as a suggestion that will be
950 modified for suitability as a variable name and for
953 If HINT is null or entirely unsuitable, a name in the form
954 "VAR%03d" will be generated, where the smallest unused integer
955 value is used. If NUM_START is non-null, then its value is
956 used as the minimum numeric value to check, and it is updated
957 to the next value to be checked.
960 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
961 unsigned long int *num_start)
965 char *hinted_name = make_hinted_name (dict, hint);
966 if (hinted_name != NULL)
969 return make_numeric_name (dict, num_start);
972 /* Returns the weighting variable in dictionary D, or a null
973 pointer if the dictionary is unweighted. */
975 dict_get_weight (const struct dictionary *d)
977 assert (d->weight == NULL || dict_contains_var (d, d->weight));
982 /* Returns the value of D's weighting variable in case C, except
983 that a negative weight is returned as 0. Returns 1 if the
984 dictionary is unweighted. Will warn about missing, negative,
985 or zero values if *WARN_ON_INVALID is true. The function will
986 set *WARN_ON_INVALID to false if an invalid weight is
989 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
990 bool *warn_on_invalid)
994 if (d->weight == NULL)
998 double w = case_num (c, d->weight);
999 if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
1001 if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
1002 *warn_on_invalid = false;
1003 msg (SW, _("At least one case in the data file had a weight value "
1004 "that was user-missing, system-missing, zero, or "
1005 "negative. These case(s) were ignored."));
1011 /* Sets the weighting variable of D to V, or turning off
1012 weighting if V is a null pointer. */
1014 dict_set_weight (struct dictionary *d, struct variable *v)
1016 assert (v == NULL || dict_contains_var (d, v));
1017 assert (v == NULL || var_is_numeric (v));
1021 if (d->changed) d->changed (d, d->changed_data);
1022 if ( d->callbacks && d->callbacks->weight_changed )
1023 d->callbacks->weight_changed (d,
1024 v ? var_get_dict_index (v) : -1,
1028 /* Returns the filter variable in dictionary D (see cmd_filter())
1029 or a null pointer if the dictionary is unfiltered. */
1031 dict_get_filter (const struct dictionary *d)
1033 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1038 /* Sets V as the filter variable for dictionary D. Passing a
1039 null pointer for V turn off filtering. */
1041 dict_set_filter (struct dictionary *d, struct variable *v)
1043 assert (v == NULL || dict_contains_var (d, v));
1044 assert (v == NULL || var_is_numeric (v));
1048 if (d->changed) d->changed (d, d->changed_data);
1049 if ( d->callbacks && d->callbacks->filter_changed )
1050 d->callbacks->filter_changed (d,
1051 v ? var_get_dict_index (v) : -1,
1055 /* Returns the case limit for dictionary D, or zero if the number
1056 of cases is unlimited. */
1058 dict_get_case_limit (const struct dictionary *d)
1060 return d->case_limit;
1063 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1064 0 for CASE_LIMIT to indicate no limit. */
1066 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1068 d->case_limit = case_limit;
1071 /* Returns the prototype used for cases created by dictionary D. */
1072 const struct caseproto *
1073 dict_get_proto (const struct dictionary *d_)
1075 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1076 if (d->proto == NULL)
1080 d->proto = caseproto_create ();
1081 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1082 for (i = 0; i < d->var_cnt; i++)
1083 d->proto = caseproto_set_width (d->proto,
1084 var_get_case_index (d->var[i].var),
1085 var_get_width (d->var[i].var));
1090 /* Returns the case index of the next value to be added to D.
1091 This value is the number of `union value's that need to be
1092 allocated to store a case for dictionary D. */
1094 dict_get_next_value_idx (const struct dictionary *d)
1096 return d->next_value_idx;
1099 /* Returns the number of bytes needed to store a case for
1102 dict_get_case_size (const struct dictionary *d)
1104 return sizeof (union value) * dict_get_next_value_idx (d);
1107 /* Reassigns values in dictionary D so that fragmentation is
1110 dict_compact_values (struct dictionary *d)
1114 d->next_value_idx = 0;
1115 for (i = 0; i < d->var_cnt; i++)
1117 struct variable *v = d->var[i].var;
1118 set_var_case_index (v, d->next_value_idx++);
1120 invalidate_proto (d);
1123 /* Returns the number of values occupied by the variables in
1124 dictionary D. All variables are considered if EXCLUDE_CLASSES
1125 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1126 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1127 corresponding type of variable.
1129 The return value may be less than the number of values in one
1130 of dictionary D's cases (as returned by
1131 dict_get_next_value_idx) even if E is 0, because there may be
1132 gaps in D's cases due to deleted variables. */
1134 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1139 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1141 | (1u << DC_SCRATCH))) == 0);
1144 for (i = 0; i < d->var_cnt; i++)
1146 enum dict_class class = var_get_dict_class (d->var[i].var);
1147 if (!(exclude_classes & (1u << class)))
1153 /* Returns the case prototype that would result after deleting
1154 all variables from D that are not in one of the
1155 EXCLUDE_CLASSES and compacting the dictionary with
1158 The caller must unref the returned caseproto when it is no
1161 dict_get_compacted_proto (const struct dictionary *d,
1162 unsigned int exclude_classes)
1164 struct caseproto *proto;
1167 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1169 | (1u << DC_SCRATCH))) == 0);
1171 proto = caseproto_create ();
1172 for (i = 0; i < d->var_cnt; i++)
1174 struct variable *v = d->var[i].var;
1175 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1176 proto = caseproto_add_width (proto, var_get_width (v));
1181 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1182 dict_get_split_cnt() to determine how many SPLIT FILE vars
1183 there are. Returns a null pointer if and only if there are no
1185 const struct variable *const *
1186 dict_get_split_vars (const struct dictionary *d)
1191 /* Returns the number of SPLIT FILE vars. */
1193 dict_get_split_cnt (const struct dictionary *d)
1195 return d->split_cnt;
1198 /* Removes variable V, which must be in D, from D's set of split
1201 dict_unset_split_var (struct dictionary *d, struct variable *v)
1205 assert (dict_contains_var (d, v));
1207 orig_count = d->split_cnt;
1208 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1209 &v, compare_var_ptrs, NULL);
1210 if (orig_count != d->split_cnt)
1212 if (d->changed) d->changed (d, d->changed_data);
1213 /* We changed the set of split variables so invoke the
1215 if (d->callbacks && d->callbacks->split_changed)
1216 d->callbacks->split_changed (d, d->cb_data);
1220 /* Sets CNT split vars SPLIT in dictionary D. */
1222 dict_set_split_vars (struct dictionary *d,
1223 struct variable *const *split, size_t cnt)
1225 assert (cnt == 0 || split != NULL);
1230 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1231 memcpy (d->split, split, cnt * sizeof *d->split);
1239 if (d->changed) d->changed (d, d->changed_data);
1240 if ( d->callbacks && d->callbacks->split_changed )
1241 d->callbacks->split_changed (d, d->cb_data);
1244 /* Returns the file label for D, or a null pointer if D is
1245 unlabeled (see cmd_file_label()). */
1247 dict_get_label (const struct dictionary *d)
1252 /* Sets D's file label to LABEL, truncating it to a maximum of 60
1255 Removes D's label if LABEL is null or the empty string. */
1257 dict_set_label (struct dictionary *d, const char *label)
1260 d->label = label != NULL && label[0] != '\0' ? xstrndup (label, 60) : NULL;
1263 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1264 return value is always nonnull; if there are no documents then the
1265 string_arary is empty.*/
1266 const struct string_array *
1267 dict_get_documents (const struct dictionary *d)
1269 return &d->documents;
1272 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1274 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1278 dict_clear_documents (d);
1280 for (i = 0; i < new_docs->n; i++)
1281 dict_add_document_line (d, new_docs->strings[i], false);
1284 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1285 into individual lines at new-line characters. Each line is truncated to at
1286 most DOC_LINE_LENGTH bytes in D's encoding. */
1288 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1292 dict_clear_documents (d);
1293 for (s = new_docs; *s != '\0'; )
1295 size_t len = strcspn (s, "\n");
1296 char *line = xmemdup0 (s, len);
1297 dict_add_document_line (d, line, false);
1306 /* Drops the documents from dictionary D. */
1308 dict_clear_documents (struct dictionary *d)
1310 string_array_clear (&d->documents);
1313 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1314 truncated so that it is no more than 80 bytes in the dictionary's
1315 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1316 then a warning will be issued. */
1318 dict_add_document_line (struct dictionary *d, const char *line,
1324 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1325 truncated = line[trunc_len] != '\0';
1326 if (truncated && issue_warning)
1328 /* Note to translators: "bytes" is correct, not characters */
1329 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1332 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1337 /* Returns the number of document lines in dictionary D. */
1339 dict_get_document_line_cnt (const struct dictionary *d)
1341 return d->documents.n;
1344 /* Returns document line number IDX in dictionary D. The caller must not
1345 modify or free the returned string. */
1347 dict_get_document_line (const struct dictionary *d, size_t idx)
1349 assert (idx < d->documents.n);
1350 return d->documents.strings[idx];
1353 /* Creates in D a vector named NAME that contains the CNT
1354 variables in VAR. Returns true if successful, or false if a
1355 vector named NAME already exists in D. */
1357 dict_create_vector (struct dictionary *d,
1359 struct variable **var, size_t cnt)
1364 for (i = 0; i < cnt; i++)
1365 assert (dict_contains_var (d, var[i]));
1367 if (dict_lookup_vector (d, name) == NULL)
1369 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1370 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1377 /* Creates in D a vector named NAME that contains the CNT
1378 variables in VAR. A vector named NAME must not already exist
1381 dict_create_vector_assert (struct dictionary *d,
1383 struct variable **var, size_t cnt)
1385 assert (dict_lookup_vector (d, name) == NULL);
1386 dict_create_vector (d, name, var, cnt);
1389 /* Returns the vector in D with index IDX, which must be less
1390 than dict_get_vector_cnt (D). */
1391 const struct vector *
1392 dict_get_vector (const struct dictionary *d, size_t idx)
1394 assert (idx < d->vector_cnt);
1396 return d->vector[idx];
1399 /* Returns the number of vectors in D. */
1401 dict_get_vector_cnt (const struct dictionary *d)
1403 return d->vector_cnt;
1406 /* Looks up and returns the vector within D with the given
1408 const struct vector *
1409 dict_lookup_vector (const struct dictionary *d, const char *name)
1412 for (i = 0; i < d->vector_cnt; i++)
1413 if (!strcasecmp (vector_get_name (d->vector[i]), name))
1414 return d->vector[i];
1418 /* Deletes all vectors from D. */
1420 dict_clear_vectors (struct dictionary *d)
1424 for (i = 0; i < d->vector_cnt; i++)
1425 vector_destroy (d->vector[i]);
1432 /* Multiple response sets. */
1434 /* Returns the multiple response set in DICT with index IDX, which must be
1435 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1436 const struct mrset *
1437 dict_get_mrset (const struct dictionary *dict, size_t idx)
1439 assert (idx < dict->n_mrsets);
1440 return dict->mrsets[idx];
1443 /* Returns the number of multiple response sets in DICT. */
1445 dict_get_n_mrsets (const struct dictionary *dict)
1447 return dict->n_mrsets;
1450 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1451 returns its index; otherwise, returns SIZE_MAX. */
1453 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1457 for (i = 0; i < dict->n_mrsets; i++)
1458 if (!strcasecmp (name, dict->mrsets[i]->name))
1464 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1465 returns it; otherwise, returns NULL. */
1466 const struct mrset *
1467 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1469 size_t idx = dict_lookup_mrset_idx (dict, name);
1470 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1473 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1474 true if a set was replaced, false if none existed with the specified name.
1476 Ownership of MRSET is transferred to DICT. */
1478 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1482 assert (mrset_ok (mrset, dict));
1484 idx = dict_lookup_mrset_idx (dict, mrset->name);
1485 if (idx == SIZE_MAX)
1487 dict->mrsets = xrealloc (dict->mrsets,
1488 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1489 dict->mrsets[dict->n_mrsets++] = mrset;
1494 mrset_destroy (dict->mrsets[idx]);
1495 dict->mrsets[idx] = mrset;
1500 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1501 from DICT and returns true. If none is found, returns false without
1504 Deleting one multiple response set causes the indexes of other sets within
1507 dict_delete_mrset (struct dictionary *dict, const char *name)
1509 size_t idx = dict_lookup_mrset_idx (dict, name);
1510 if (idx != SIZE_MAX)
1512 mrset_destroy (dict->mrsets[idx]);
1513 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1520 /* Deletes all multiple response sets from DICT. */
1522 dict_clear_mrsets (struct dictionary *dict)
1526 for (i = 0; i < dict->n_mrsets; i++)
1527 mrset_destroy (dict->mrsets[i]);
1528 free (dict->mrsets);
1529 dict->mrsets = NULL;
1533 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1535 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1539 assert (dict_contains_var (dict, var));
1541 for (i = 0; i < dict->n_mrsets; )
1543 struct mrset *mrset = dict->mrsets[i];
1546 for (j = 0; j < mrset->n_vars; )
1547 if (mrset->vars[j] == var)
1548 remove_element (mrset->vars, mrset->n_vars--,
1549 sizeof *mrset->vars, j);
1553 if (mrset->n_vars < 2)
1555 mrset_destroy (mrset);
1556 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1563 /* Returns D's attribute set. The caller may examine or modify
1564 the attribute set, but must not destroy it. Destroying D or
1565 calling dict_set_attributes for D will also destroy D's
1568 dict_get_attributes (const struct dictionary *d)
1570 return CONST_CAST (struct attrset *, &d->attributes);
1573 /* Replaces D's attributes set by a copy of ATTRS. */
1575 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1577 attrset_destroy (&d->attributes);
1578 attrset_clone (&d->attributes, attrs);
1581 /* Returns true if D has at least one attribute in its attribute
1582 set, false if D's attribute set is empty. */
1584 dict_has_attributes (const struct dictionary *d)
1586 return attrset_count (&d->attributes) > 0;
1589 /* Called from variable.c to notify the dictionary that some property of
1590 the variable has changed */
1592 dict_var_changed (const struct variable *v)
1594 if ( var_has_vardict (v))
1596 const struct vardict_info *vardict = var_get_vardict (v);
1597 struct dictionary *d = vardict->dict;
1602 if (d->changed ) d->changed (d, d->changed_data);
1603 if ( d->callbacks && d->callbacks->var_changed )
1604 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
1609 /* Called from variable.c to notify the dictionary that the variable's width
1612 dict_var_resized (const struct variable *v, int old_width)
1614 if ( var_has_vardict (v))
1616 const struct vardict_info *vardict = var_get_vardict (v);
1617 struct dictionary *d;
1621 if (d->changed) d->changed (d, d->changed_data);
1623 invalidate_proto (d);
1624 if ( d->callbacks && d->callbacks->var_resized )
1625 d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
1630 /* Called from variable.c to notify the dictionary that the variable's display width
1633 dict_var_display_width_changed (const struct variable *v)
1635 if ( var_has_vardict (v))
1637 const struct vardict_info *vardict = var_get_vardict (v);
1638 struct dictionary *d;
1642 if (d->changed) d->changed (d, d->changed_data);
1643 if ( d->callbacks && d->callbacks->var_display_width_changed )
1644 d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
1648 /* Dictionary used to contain "internal variables". */
1649 static struct dictionary *internal_dict;
1651 /* Create a variable of the specified WIDTH to be used for internal
1652 calculations only. The variable is assigned case index CASE_IDX. */
1654 dict_create_internal_var (int case_idx, int width)
1656 if (internal_dict == NULL)
1657 internal_dict = dict_create ("UTF-8");
1661 static int counter = INT_MAX / 2;
1662 struct variable *var;
1665 if (++counter == INT_MAX)
1666 counter = INT_MAX / 2;
1668 sprintf (name, "$internal%d", counter);
1669 var = dict_create_var (internal_dict, name, width);
1672 set_var_case_index (var, case_idx);
1678 /* Destroys VAR, which must have been created with
1679 dict_create_internal_var(). */
1681 dict_destroy_internal_var (struct variable *var)
1685 dict_delete_var (internal_dict, var);
1687 /* Destroy internal_dict if it has no variables left, just so that
1688 valgrind --leak-check --show-reachable won't show internal_dict. */
1689 if (dict_get_var_cnt (internal_dict) == 0)
1691 dict_destroy (internal_dict);
1692 internal_dict = NULL;
1698 vardict_get_dict_index (const struct vardict_info *vardict)
1700 return vardict - vardict->dict->var;