1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
58 struct vardict_info *var; /* Variables. */
59 size_t var_cnt, var_cap; /* Number of variables, capacity. */
60 struct caseproto *proto; /* Prototype for dictionary cases
62 struct hmap name_map; /* Variable index by name. */
63 int next_value_idx; /* Index of next `union value' to allocate. */
64 const struct variable **split; /* SPLIT FILE vars. */
65 size_t split_cnt; /* SPLIT FILE count. */
66 struct variable *weight; /* WEIGHT variable. */
67 struct variable *filter; /* FILTER variable. */
68 casenumber case_limit; /* Current case limit (N command). */
69 char *label; /* File label. */
70 struct string_array documents; /* Documents. */
71 struct vector **vector; /* Vectors of variables. */
72 size_t vector_cnt; /* Number of vectors. */
73 struct attrset attributes; /* Custom attributes. */
74 struct mrset **mrsets; /* Multiple response sets. */
75 size_t n_mrsets; /* Number of multiple response sets. */
77 char *encoding; /* Character encoding of string data */
79 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
81 void *cb_data ; /* Data passed to callbacks */
83 void (*changed) (struct dictionary *, void *); /* Generic change callback */
87 static void dict_unset_split_var (struct dictionary *, struct variable *);
88 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
91 dict_set_encoding (struct dictionary *d, const char *enc)
96 d->encoding = xstrdup (enc);
101 dict_get_encoding (const struct dictionary *d)
106 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
107 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
108 error message on failure. */
110 dict_id_is_valid (const struct dictionary *dict, const char *id,
113 return id_is_valid (id, dict->encoding, issue_error);
117 dict_set_change_callback (struct dictionary *d,
118 void (*changed) (struct dictionary *, void*),
121 d->changed = changed;
122 d->changed_data = data;
125 /* Discards dictionary D's caseproto. (It will be regenerated
126 lazily, on demand.) */
128 invalidate_proto (struct dictionary *d)
130 caseproto_unref (d->proto);
134 /* Print a representation of dictionary D to stdout, for
135 debugging purposes. */
137 dict_dump (const struct dictionary *d)
140 for (i = 0 ; i < d->var_cnt ; ++i )
142 const struct variable *v = d->var[i].var;
143 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
145 var_get_dict_index (v),
146 var_get_case_index (v));
151 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
152 the dictionary or any of the variables it contains are modified.
153 Each callback will get passed CALLBACK_DATA.
154 Any callback may be NULL, in which case it'll be ignored.
157 dict_set_callbacks (struct dictionary *dict,
158 const struct dict_callbacks *callbacks,
161 dict->callbacks = callbacks;
162 dict->cb_data = callback_data;
165 /* Shallow copy the callbacks from SRC to DEST */
167 dict_copy_callbacks (struct dictionary *dest,
168 const struct dictionary *src)
170 dest->callbacks = src->callbacks;
171 dest->cb_data = src->cb_data;
174 /* Creates and returns a new dictionary. */
178 struct dictionary *d = xzalloc (sizeof *d);
180 hmap_init (&d->name_map);
181 attrset_init (&d->attributes);
185 /* Creates and returns a (deep) copy of an existing
188 The new dictionary's case indexes are copied from the old
189 dictionary. If the new dictionary won't be used to access
190 cases produced with the old dictionary, then the new
191 dictionary's case indexes should be compacted with
192 dict_compact_values to save space. */
194 dict_clone (const struct dictionary *s)
196 struct dictionary *d;
201 for (i = 0; i < s->var_cnt; i++)
203 struct variable *sv = s->var[i].var;
204 struct variable *dv = dict_clone_var_assert (d, sv);
207 for (i = 0; i < var_get_short_name_cnt (sv); i++)
208 var_set_short_name (dv, i, var_get_short_name (sv, i));
210 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
213 d->next_value_idx = s->next_value_idx;
215 d->split_cnt = s->split_cnt;
216 if (d->split_cnt > 0)
218 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
219 for (i = 0; i < d->split_cnt; i++)
220 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
223 if (s->weight != NULL)
224 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
226 if (s->filter != NULL)
227 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
229 d->case_limit = s->case_limit;
230 dict_set_label (d, dict_get_label (s));
231 dict_set_documents (d, dict_get_documents (s));
233 d->vector_cnt = s->vector_cnt;
234 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
235 for (i = 0; i < s->vector_cnt; i++)
236 d->vector[i] = vector_clone (s->vector[i], s, d);
239 d->encoding = xstrdup (s->encoding);
241 dict_set_attributes (d, dict_get_attributes (s));
243 for (i = 0; i < s->n_mrsets; i++)
245 const struct mrset *old = s->mrsets[i];
249 /* Clone old mrset, then replace vars from D by vars from S. */
250 new = mrset_clone (old);
251 for (j = 0; j < new->n_vars; j++)
252 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
254 dict_add_mrset (d, new);
260 /* Clears the contents from a dictionary without destroying the
261 dictionary itself. */
263 dict_clear (struct dictionary *d)
265 /* FIXME? Should we really clear case_limit, label, documents?
266 Others are necessarily cleared by deleting all the variables.*/
267 while (d->var_cnt > 0 )
269 dict_delete_var (d, d->var[d->var_cnt - 1].var);
274 d->var_cnt = d->var_cap = 0;
275 invalidate_proto (d);
276 hmap_clear (&d->name_map);
277 d->next_value_idx = 0;
278 dict_set_split_vars (d, NULL, 0);
279 dict_set_weight (d, NULL);
280 dict_set_filter (d, NULL);
284 string_array_clear (&d->documents);
285 dict_clear_vectors (d);
286 attrset_clear (&d->attributes);
289 /* Destroys the aux data for every variable in D, by calling
290 var_clear_aux() for each variable. */
292 dict_clear_aux (struct dictionary *d)
296 for (i = 0; i < d->var_cnt; i++)
297 var_clear_aux (d->var[i].var);
300 /* Clears a dictionary and destroys it. */
302 dict_destroy (struct dictionary *d)
306 /* In general, we don't want callbacks occuring, if the dictionary
307 is being destroyed */
308 d->callbacks = NULL ;
311 hmap_destroy (&d->name_map);
312 attrset_destroy (&d->attributes);
313 dict_clear_mrsets (d);
319 /* Returns the number of variables in D. */
321 dict_get_var_cnt (const struct dictionary *d)
326 /* Returns the variable in D with dictionary index IDX, which
327 must be between 0 and the count returned by
328 dict_get_var_cnt(), exclusive. */
330 dict_get_var (const struct dictionary *d, size_t idx)
332 assert (idx < d->var_cnt);
334 return d->var[idx].var;
337 /* Sets *VARS to an array of pointers to variables in D and *CNT
338 to the number of variables in *D. All variables are returned
339 except for those, if any, in the classes indicated by EXCLUDE.
340 (There is no point in putting DC_SYSTEM in EXCLUDE as
341 dictionaries never include system variables.) */
343 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
344 size_t *cnt, enum dict_class exclude)
346 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
349 /* Sets *VARS to an array of pointers to variables in D and *CNT
350 to the number of variables in *D. All variables are returned
351 except for those, if any, in the classes indicated by EXCLUDE.
352 (There is no point in putting DC_SYSTEM in EXCLUDE as
353 dictionaries never include system variables.) */
355 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
356 size_t *cnt, enum dict_class exclude)
361 assert (exclude == (exclude & DC_ALL));
364 for (i = 0; i < d->var_cnt; i++)
366 enum dict_class class = var_get_dict_class (d->var[i].var);
367 if (!(class & exclude))
371 *vars = xnmalloc (count, sizeof **vars);
373 for (i = 0; i < d->var_cnt; i++)
375 enum dict_class class = var_get_dict_class (d->var[i].var);
376 if (!(class & exclude))
377 (*vars)[(*cnt)++] = d->var[i].var;
379 assert (*cnt == count);
382 static struct variable *
383 add_var (struct dictionary *d, struct variable *v)
385 struct vardict_info *vardict;
387 /* Update dictionary. */
388 if (d->var_cnt >= d->var_cap)
392 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
393 hmap_clear (&d->name_map);
394 for (i = 0; i < d->var_cnt; i++)
396 var_set_vardict (d->var[i].var, &d->var[i]);
397 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
398 d->var[i].name_node.hash);
402 vardict = &d->var[d->var_cnt++];
405 hmap_insert (&d->name_map, &vardict->name_node,
406 hash_case_string (var_get_name (v), 0));
407 vardict->case_index = d->next_value_idx;
408 var_set_vardict (v, vardict);
410 if ( d->changed ) d->changed (d, d->changed_data);
411 if ( d->callbacks && d->callbacks->var_added )
412 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
415 invalidate_proto (d);
420 /* Creates and returns a new variable in D with the given NAME
421 and WIDTH. Returns a null pointer if the given NAME would
422 duplicate that of an existing variable in the dictionary. */
424 dict_create_var (struct dictionary *d, const char *name, int width)
426 return (dict_lookup_var (d, name) == NULL
427 ? dict_create_var_assert (d, name, width)
431 /* Creates and returns a new variable in D with the given NAME
432 and WIDTH. Assert-fails if the given NAME would duplicate
433 that of an existing variable in the dictionary. */
435 dict_create_var_assert (struct dictionary *d, const char *name, int width)
437 assert (dict_lookup_var (d, name) == NULL);
438 return add_var (d, var_create (name, width));
441 /* Creates and returns a new variable in D, as a copy of existing variable
442 OLD_VAR, which need not be in D or in any dictionary. Returns a null
443 pointer if OLD_VAR's name would duplicate that of an existing variable in
446 dict_clone_var (struct dictionary *d, const struct variable *old_var)
448 return dict_clone_var_as (d, old_var, var_get_name (old_var));
451 /* Creates and returns a new variable in D, as a copy of existing variable
452 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
453 OLD_VAR's name would duplicate that of an existing variable in the
456 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
458 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
461 /* Creates and returns a new variable in D with name NAME, as a copy of
462 existing variable OLD_VAR, which need not be in D or in any dictionary.
463 Returns a null pointer if the given NAME would duplicate that of an existing
464 variable in the dictionary. */
466 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
469 return (dict_lookup_var (d, name) == NULL
470 ? dict_clone_var_as_assert (d, old_var, name)
474 /* Creates and returns a new variable in D with name NAME, as a copy of
475 existing variable OLD_VAR, which need not be in D or in any dictionary.
476 Assert-fails if the given NAME would duplicate that of an existing variable
477 in the dictionary. */
479 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
482 struct variable *new_var = var_clone (old_var);
483 assert (dict_lookup_var (d, name) == NULL);
484 var_set_name (new_var, name);
485 return add_var (d, new_var);
488 /* Returns the variable named NAME in D, or a null pointer if no
489 variable has that name. */
491 dict_lookup_var (const struct dictionary *d, const char *name)
493 struct vardict_info *vardict;
495 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
496 hash_case_string (name, 0), &d->name_map)
498 struct variable *var = vardict->var;
499 if (!strcasecmp (var_get_name (var), name))
506 /* Returns the variable named NAME in D. Assert-fails if no
507 variable has that name. */
509 dict_lookup_var_assert (const struct dictionary *d, const char *name)
511 struct variable *v = dict_lookup_var (d, name);
516 /* Returns true if variable V is in dictionary D,
519 dict_contains_var (const struct dictionary *d, const struct variable *v)
521 return (var_has_vardict (v)
522 && vardict_get_dictionary (var_get_vardict (v)) == d);
525 /* Compares two double pointers to variables, which should point
526 to elements of a struct dictionary's `var' member array. */
528 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
530 struct variable *const *a = a_;
531 struct variable *const *b = b_;
533 return *a < *b ? -1 : *a > *b;
537 unindex_var (struct dictionary *d, struct vardict_info *vardict)
539 hmap_delete (&d->name_map, &vardict->name_node);
542 /* This function assumes that vardict->name_node.hash is valid, that is, that
543 its name has not changed since it was hashed (rename_var() updates this
544 hash along with the name itself). */
546 reindex_var (struct dictionary *d, struct vardict_info *vardict)
548 struct variable *var = vardict->var;
550 var_set_vardict (var, vardict);
551 hmap_insert_fast (&d->name_map, &vardict->name_node,
552 vardict->name_node.hash);
554 if ( d->changed ) d->changed (d, d->changed_data);
555 if ( d->callbacks && d->callbacks->var_changed )
556 d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
559 /* Sets the case_index in V's vardict to CASE_INDEX. */
561 set_var_case_index (struct variable *v, int case_index)
563 var_get_vardict (v)->case_index = case_index;
566 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
569 unindex_vars (struct dictionary *d, size_t from, size_t to)
573 for (i = from; i < to; i++)
574 unindex_var (d, &d->var[i]);
577 /* Re-sets the dict_index in the dictionary variables with
578 indexes from FROM to TO (exclusive). */
580 reindex_vars (struct dictionary *d, size_t from, size_t to)
584 for (i = from; i < to; i++)
585 reindex_var (d, &d->var[i]);
588 /* Deletes variable V from dictionary D and frees V.
590 This is a very bad idea if there might be any pointers to V
591 from outside D. In general, no variable in the active file's
592 dictionary should be deleted when any transformations are
593 active on the dictionary's dataset, because those
594 transformations might reference the deleted variable. The
595 safest time to delete a variable is just after a procedure has
596 been executed, as done by DELETE VARIABLES.
598 Pointers to V within D are not a problem, because
599 dict_delete_var() knows to remove V from split variables,
600 weights, filters, etc. */
602 dict_delete_var (struct dictionary *d, struct variable *v)
604 int dict_index = var_get_dict_index (v);
605 const int case_index = var_get_case_index (v);
606 const int width = var_get_width (v);
608 assert (dict_contains_var (d, v));
610 /* Delete aux data. */
613 dict_unset_split_var (d, v);
614 dict_unset_mrset_var (d, v);
617 dict_set_weight (d, NULL);
620 dict_set_filter (d, NULL);
622 dict_clear_vectors (d);
624 /* Remove V from var array. */
625 unindex_vars (d, dict_index, d->var_cnt);
626 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
629 /* Update dict_index for each affected variable. */
630 reindex_vars (d, dict_index, d->var_cnt);
633 var_clear_vardict (v);
636 if ( d->changed ) d->changed (d, d->changed_data);
638 invalidate_proto (d);
639 if (d->callbacks && d->callbacks->var_deleted )
640 d->callbacks->var_deleted (d, dict_index, case_index, width, d->cb_data);
643 /* Deletes the COUNT variables listed in VARS from D. This is
644 unsafe; see the comment on dict_delete_var() for details. */
646 dict_delete_vars (struct dictionary *d,
647 struct variable *const *vars, size_t count)
649 /* FIXME: this can be done in O(count) time, but this algorithm
651 assert (count == 0 || vars != NULL);
654 dict_delete_var (d, *vars++);
657 /* Deletes the COUNT variables in D starting at index IDX. This
658 is unsafe; see the comment on dict_delete_var() for
661 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
663 /* FIXME: this can be done in O(count) time, but this algorithm
665 assert (idx + count <= d->var_cnt);
668 dict_delete_var (d, d->var[idx].var);
671 /* Deletes scratch variables from dictionary D. */
673 dict_delete_scratch_vars (struct dictionary *d)
677 /* FIXME: this can be done in O(count) time, but this algorithm
679 for (i = 0; i < d->var_cnt; )
680 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
681 dict_delete_var (d, d->var[i].var);
686 /* Moves V to 0-based position IDX in D. Other variables in D,
687 if any, retain their relative positions. Runs in time linear
688 in the distance moved. */
690 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
692 size_t old_index = var_get_dict_index (v);
694 assert (new_index < d->var_cnt);
696 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
697 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
698 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
701 /* Reorders the variables in D, placing the COUNT variables
702 listed in ORDER in that order at the beginning of D. The
703 other variables in D, if any, retain their relative
706 dict_reorder_vars (struct dictionary *d,
707 struct variable *const *order, size_t count)
709 struct vardict_info *new_var;
712 assert (count == 0 || order != NULL);
713 assert (count <= d->var_cnt);
715 new_var = xnmalloc (d->var_cap, sizeof *new_var);
717 /* Add variables in ORDER to new_var. */
718 for (i = 0; i < count; i++)
720 struct vardict_info *old_var;
722 assert (dict_contains_var (d, order[i]));
724 old_var = var_get_vardict (order[i]);
725 new_var[i] = *old_var;
726 old_var->dict = NULL;
729 /* Add remaining variables to new_var. */
730 for (i = 0; i < d->var_cnt; i++)
731 if (d->var[i].dict != NULL)
732 new_var[count++] = d->var[i];
733 assert (count == d->var_cnt);
735 /* Replace old vardicts by new ones. */
739 hmap_clear (&d->name_map);
740 reindex_vars (d, 0, d->var_cnt);
743 /* Changes the name of variable V that is currently in a dictionary to
746 rename_var (struct variable *v, const char *new_name)
748 struct vardict_info *vardict = var_get_vardict (v);
749 var_clear_vardict (v);
750 var_set_name (v, new_name);
751 vardict->name_node.hash = hash_case_string (new_name, 0);
752 var_set_vardict (v, vardict);
755 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
756 a variable named NEW_NAME is already in D, except that
757 NEW_NAME may be the same as V's existing name. */
759 dict_rename_var (struct dictionary *d, struct variable *v,
760 const char *new_name)
762 assert (!strcasecmp (var_get_name (v), new_name)
763 || dict_lookup_var (d, new_name) == NULL);
765 unindex_var (d, var_get_vardict (v));
766 rename_var (v, new_name);
767 reindex_var (d, var_get_vardict (v));
769 if (settings_get_algorithm () == ENHANCED)
770 var_clear_short_names (v);
772 if ( d->changed ) d->changed (d, d->changed_data);
773 if ( d->callbacks && d->callbacks->var_changed )
774 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
777 /* Renames COUNT variables specified in VARS to the names given
778 in NEW_NAMES within dictionary D. If the renaming would
779 result in a duplicate variable name, returns false and stores a
780 name that would be duplicated into *ERR_NAME (if ERR_NAME is
781 non-null). Otherwise, the renaming is successful, and true
784 dict_rename_vars (struct dictionary *d,
785 struct variable **vars, char **new_names, size_t count,
792 assert (count == 0 || vars != NULL);
793 assert (count == 0 || new_names != NULL);
795 /* Save the names of the variables to be renamed. */
796 pool = pool_create ();
797 old_names = pool_nalloc (pool, count, sizeof *old_names);
798 for (i = 0; i < count; i++)
799 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
801 /* Remove the variables to be renamed from the name hash,
803 for (i = 0; i < count; i++)
805 unindex_var (d, var_get_vardict (vars[i]));
806 rename_var (vars[i], new_names[i]);
809 /* Add the renamed variables back into the name hash,
810 checking for conflicts. */
811 for (i = 0; i < count; i++)
813 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
815 /* There is a name conflict.
816 Back out all the name changes that have already
817 taken place, and indicate failure. */
819 if (err_name != NULL)
820 *err_name = new_names[i];
822 for (i = 0; i < fail_idx; i++)
823 unindex_var (d, var_get_vardict (vars[i]));
825 for (i = 0; i < count; i++)
827 rename_var (vars[i], old_names[i]);
828 reindex_var (d, var_get_vardict (vars[i]));
834 reindex_var (d, var_get_vardict (vars[i]));
837 /* Clear short names. */
838 if (settings_get_algorithm () == ENHANCED)
839 for (i = 0; i < count; i++)
840 var_clear_short_names (vars[i]);
846 /* Returns true if a variable named NAME may be inserted in DICT;
847 that is, if there is not already a variable with that name in
848 DICT and if NAME is not a reserved word. (The caller's checks
849 have already verified that NAME is otherwise acceptable as a
852 var_name_is_insertable (const struct dictionary *dict, const char *name)
854 return (dict_lookup_var (dict, name) == NULL
855 && lex_id_to_token (ss_cstr (name)) == T_ID);
859 make_hinted_name (const struct dictionary *dict, const char *hint)
861 size_t hint_len = strlen (hint);
862 bool dropped = false;
867 /* The allocation size here is OK: characters that are copied directly fit
868 OK, and characters that are not copied directly are replaced by a single
869 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
870 replaced by '_' too. */
871 root = rp = xmalloc (hint_len + 1);
872 for (ofs = 0; ofs < hint_len; ofs += mblen)
876 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
879 ? lex_uc_is_id1 (uc) && uc != '$'
880 : lex_uc_is_idn (uc))
887 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
898 if (var_name_is_insertable (dict, root))
901 for (i = 0; i < ULONG_MAX; i++)
903 char suffix[INT_BUFSIZE_BOUND (i) + 1];
907 if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
910 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
911 if (var_name_is_insertable (dict, name))
926 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
928 unsigned long int number;
930 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
934 char name[3 + INT_STRLEN_BOUND (number) + 1];
936 sprintf (name, "VAR%03lu", number);
937 if (dict_lookup_var (dict, name) == NULL)
939 if (num_start != NULL)
940 *num_start = number + 1;
941 return xstrdup (name);
949 /* Devises and returns a variable name unique within DICT. The variable name
950 is owned by the caller, which must free it with free() when it is no longer
953 HINT, if it is non-null, is used as a suggestion that will be
954 modified for suitability as a variable name and for
957 If HINT is null or entirely unsuitable, a name in the form
958 "VAR%03d" will be generated, where the smallest unused integer
959 value is used. If NUM_START is non-null, then its value is
960 used as the minimum numeric value to check, and it is updated
961 to the next value to be checked.
964 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
965 unsigned long int *num_start)
969 char *hinted_name = make_hinted_name (dict, hint);
970 if (hinted_name != NULL)
973 return make_numeric_name (dict, num_start);
976 /* Returns the weighting variable in dictionary D, or a null
977 pointer if the dictionary is unweighted. */
979 dict_get_weight (const struct dictionary *d)
981 assert (d->weight == NULL || dict_contains_var (d, d->weight));
986 /* Returns the value of D's weighting variable in case C, except
987 that a negative weight is returned as 0. Returns 1 if the
988 dictionary is unweighted. Will warn about missing, negative,
989 or zero values if *WARN_ON_INVALID is true. The function will
990 set *WARN_ON_INVALID to false if an invalid weight is
993 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
994 bool *warn_on_invalid)
998 if (d->weight == NULL)
1002 double w = case_num (c, d->weight);
1003 if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
1005 if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
1006 *warn_on_invalid = false;
1007 msg (SW, _("At least one case in the data file had a weight value "
1008 "that was user-missing, system-missing, zero, or "
1009 "negative. These case(s) were ignored."));
1015 /* Sets the weighting variable of D to V, or turning off
1016 weighting if V is a null pointer. */
1018 dict_set_weight (struct dictionary *d, struct variable *v)
1020 assert (v == NULL || dict_contains_var (d, v));
1021 assert (v == NULL || var_is_numeric (v));
1025 if (d->changed) d->changed (d, d->changed_data);
1026 if ( d->callbacks && d->callbacks->weight_changed )
1027 d->callbacks->weight_changed (d,
1028 v ? var_get_dict_index (v) : -1,
1032 /* Returns the filter variable in dictionary D (see cmd_filter())
1033 or a null pointer if the dictionary is unfiltered. */
1035 dict_get_filter (const struct dictionary *d)
1037 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1042 /* Sets V as the filter variable for dictionary D. Passing a
1043 null pointer for V turn off filtering. */
1045 dict_set_filter (struct dictionary *d, struct variable *v)
1047 assert (v == NULL || dict_contains_var (d, v));
1048 assert (v == NULL || var_is_numeric (v));
1052 if (d->changed) d->changed (d, d->changed_data);
1053 if ( d->callbacks && d->callbacks->filter_changed )
1054 d->callbacks->filter_changed (d,
1055 v ? var_get_dict_index (v) : -1,
1059 /* Returns the case limit for dictionary D, or zero if the number
1060 of cases is unlimited. */
1062 dict_get_case_limit (const struct dictionary *d)
1064 return d->case_limit;
1067 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1068 0 for CASE_LIMIT to indicate no limit. */
1070 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1072 d->case_limit = case_limit;
1075 /* Returns the prototype used for cases created by dictionary D. */
1076 const struct caseproto *
1077 dict_get_proto (const struct dictionary *d_)
1079 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1080 if (d->proto == NULL)
1084 d->proto = caseproto_create ();
1085 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1086 for (i = 0; i < d->var_cnt; i++)
1087 d->proto = caseproto_set_width (d->proto,
1088 var_get_case_index (d->var[i].var),
1089 var_get_width (d->var[i].var));
1094 /* Returns the case index of the next value to be added to D.
1095 This value is the number of `union value's that need to be
1096 allocated to store a case for dictionary D. */
1098 dict_get_next_value_idx (const struct dictionary *d)
1100 return d->next_value_idx;
1103 /* Returns the number of bytes needed to store a case for
1106 dict_get_case_size (const struct dictionary *d)
1108 return sizeof (union value) * dict_get_next_value_idx (d);
1111 /* Reassigns values in dictionary D so that fragmentation is
1114 dict_compact_values (struct dictionary *d)
1118 d->next_value_idx = 0;
1119 for (i = 0; i < d->var_cnt; i++)
1121 struct variable *v = d->var[i].var;
1122 set_var_case_index (v, d->next_value_idx++);
1124 invalidate_proto (d);
1127 /* Returns the number of values occupied by the variables in
1128 dictionary D. All variables are considered if EXCLUDE_CLASSES
1129 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1130 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1131 corresponding type of variable.
1133 The return value may be less than the number of values in one
1134 of dictionary D's cases (as returned by
1135 dict_get_next_value_idx) even if E is 0, because there may be
1136 gaps in D's cases due to deleted variables. */
1138 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1143 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1145 | (1u << DC_SCRATCH))) == 0);
1148 for (i = 0; i < d->var_cnt; i++)
1150 enum dict_class class = var_get_dict_class (d->var[i].var);
1151 if (!(exclude_classes & (1u << class)))
1157 /* Returns the case prototype that would result after deleting
1158 all variables from D that are not in one of the
1159 EXCLUDE_CLASSES and compacting the dictionary with
1162 The caller must unref the returned caseproto when it is no
1165 dict_get_compacted_proto (const struct dictionary *d,
1166 unsigned int exclude_classes)
1168 struct caseproto *proto;
1171 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1173 | (1u << DC_SCRATCH))) == 0);
1175 proto = caseproto_create ();
1176 for (i = 0; i < d->var_cnt; i++)
1178 struct variable *v = d->var[i].var;
1179 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1180 proto = caseproto_add_width (proto, var_get_width (v));
1185 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1186 dict_get_split_cnt() to determine how many SPLIT FILE vars
1187 there are. Returns a null pointer if and only if there are no
1189 const struct variable *const *
1190 dict_get_split_vars (const struct dictionary *d)
1195 /* Returns the number of SPLIT FILE vars. */
1197 dict_get_split_cnt (const struct dictionary *d)
1199 return d->split_cnt;
1202 /* Removes variable V, which must be in D, from D's set of split
1205 dict_unset_split_var (struct dictionary *d, struct variable *v)
1209 assert (dict_contains_var (d, v));
1211 orig_count = d->split_cnt;
1212 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1213 &v, compare_var_ptrs, NULL);
1214 if (orig_count != d->split_cnt)
1216 if (d->changed) d->changed (d, d->changed_data);
1217 /* We changed the set of split variables so invoke the
1219 if (d->callbacks && d->callbacks->split_changed)
1220 d->callbacks->split_changed (d, d->cb_data);
1224 /* Sets CNT split vars SPLIT in dictionary D. */
1226 dict_set_split_vars (struct dictionary *d,
1227 struct variable *const *split, size_t cnt)
1229 assert (cnt == 0 || split != NULL);
1234 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1235 memcpy (d->split, split, cnt * sizeof *d->split);
1243 if (d->changed) d->changed (d, d->changed_data);
1244 if ( d->callbacks && d->callbacks->split_changed )
1245 d->callbacks->split_changed (d, d->cb_data);
1248 /* Returns the file label for D, or a null pointer if D is
1249 unlabeled (see cmd_file_label()). */
1251 dict_get_label (const struct dictionary *d)
1256 /* Sets D's file label to LABEL, truncating it to a maximum of 60
1259 Removes D's label if LABEL is null or the empty string. */
1261 dict_set_label (struct dictionary *d, const char *label)
1264 d->label = label != NULL && label[0] != '\0' ? xstrndup (label, 60) : NULL;
1267 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1268 return value is always nonnull; if there are no documents then the
1269 string_arary is empty.*/
1270 const struct string_array *
1271 dict_get_documents (const struct dictionary *d)
1273 return &d->documents;
1276 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1278 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1282 dict_clear_documents (d);
1284 for (i = 0; i < new_docs->n; i++)
1285 dict_add_document_line (d, new_docs->strings[i], false);
1288 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1289 into individual lines at new-line characters. Each line is truncated to at
1290 most DOC_LINE_LENGTH bytes in D's encoding. */
1292 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1296 dict_clear_documents (d);
1297 for (s = new_docs; *s != '\0'; )
1299 size_t len = strcspn (s, "\n");
1300 char *line = xmemdup0 (s, len);
1301 dict_add_document_line (d, line, false);
1310 /* Drops the documents from dictionary D. */
1312 dict_clear_documents (struct dictionary *d)
1314 string_array_clear (&d->documents);
1317 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1318 truncated so that it is no more than 80 bytes in the dictionary's
1319 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1320 then a warning will be issued. */
1322 dict_add_document_line (struct dictionary *d, const char *line,
1328 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1329 truncated = line[trunc_len] != '\0';
1330 if (truncated && issue_warning)
1332 /* Note to translators: "bytes" is correct, not characters */
1333 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1336 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1341 /* Returns the number of document lines in dictionary D. */
1343 dict_get_document_line_cnt (const struct dictionary *d)
1345 return d->documents.n;
1348 /* Returns document line number IDX in dictionary D. The caller must not
1349 modify or free the returned string. */
1351 dict_get_document_line (const struct dictionary *d, size_t idx)
1353 assert (idx < d->documents.n);
1354 return d->documents.strings[idx];
1357 /* Creates in D a vector named NAME that contains the CNT
1358 variables in VAR. Returns true if successful, or false if a
1359 vector named NAME already exists in D. */
1361 dict_create_vector (struct dictionary *d,
1363 struct variable **var, size_t cnt)
1368 for (i = 0; i < cnt; i++)
1369 assert (dict_contains_var (d, var[i]));
1371 if (dict_lookup_vector (d, name) == NULL)
1373 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1374 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1381 /* Creates in D a vector named NAME that contains the CNT
1382 variables in VAR. A vector named NAME must not already exist
1385 dict_create_vector_assert (struct dictionary *d,
1387 struct variable **var, size_t cnt)
1389 assert (dict_lookup_vector (d, name) == NULL);
1390 dict_create_vector (d, name, var, cnt);
1393 /* Returns the vector in D with index IDX, which must be less
1394 than dict_get_vector_cnt (D). */
1395 const struct vector *
1396 dict_get_vector (const struct dictionary *d, size_t idx)
1398 assert (idx < d->vector_cnt);
1400 return d->vector[idx];
1403 /* Returns the number of vectors in D. */
1405 dict_get_vector_cnt (const struct dictionary *d)
1407 return d->vector_cnt;
1410 /* Looks up and returns the vector within D with the given
1412 const struct vector *
1413 dict_lookup_vector (const struct dictionary *d, const char *name)
1416 for (i = 0; i < d->vector_cnt; i++)
1417 if (!strcasecmp (vector_get_name (d->vector[i]), name))
1418 return d->vector[i];
1422 /* Deletes all vectors from D. */
1424 dict_clear_vectors (struct dictionary *d)
1428 for (i = 0; i < d->vector_cnt; i++)
1429 vector_destroy (d->vector[i]);
1436 /* Multiple response sets. */
1438 /* Returns the multiple response set in DICT with index IDX, which must be
1439 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1440 const struct mrset *
1441 dict_get_mrset (const struct dictionary *dict, size_t idx)
1443 assert (idx < dict->n_mrsets);
1444 return dict->mrsets[idx];
1447 /* Returns the number of multiple response sets in DICT. */
1449 dict_get_n_mrsets (const struct dictionary *dict)
1451 return dict->n_mrsets;
1454 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1455 returns its index; otherwise, returns SIZE_MAX. */
1457 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1461 for (i = 0; i < dict->n_mrsets; i++)
1462 if (!strcasecmp (name, dict->mrsets[i]->name))
1468 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1469 returns it; otherwise, returns NULL. */
1470 const struct mrset *
1471 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1473 size_t idx = dict_lookup_mrset_idx (dict, name);
1474 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1477 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1478 true if a set was replaced, false if none existed with the specified name.
1480 Ownership of MRSET is transferred to DICT. */
1482 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1486 assert (mrset_ok (mrset, dict));
1488 idx = dict_lookup_mrset_idx (dict, mrset->name);
1489 if (idx == SIZE_MAX)
1491 dict->mrsets = xrealloc (dict->mrsets,
1492 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1493 dict->mrsets[dict->n_mrsets++] = mrset;
1498 mrset_destroy (dict->mrsets[idx]);
1499 dict->mrsets[idx] = mrset;
1504 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1505 from DICT and returns true. If none is found, returns false without
1508 Deleting one multiple response set causes the indexes of other sets within
1511 dict_delete_mrset (struct dictionary *dict, const char *name)
1513 size_t idx = dict_lookup_mrset_idx (dict, name);
1514 if (idx != SIZE_MAX)
1516 mrset_destroy (dict->mrsets[idx]);
1517 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1524 /* Deletes all multiple response sets from DICT. */
1526 dict_clear_mrsets (struct dictionary *dict)
1530 for (i = 0; i < dict->n_mrsets; i++)
1531 mrset_destroy (dict->mrsets[i]);
1532 free (dict->mrsets);
1533 dict->mrsets = NULL;
1537 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1539 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1543 assert (dict_contains_var (dict, var));
1545 for (i = 0; i < dict->n_mrsets; )
1547 struct mrset *mrset = dict->mrsets[i];
1550 for (j = 0; j < mrset->n_vars; )
1551 if (mrset->vars[j] == var)
1552 remove_element (mrset->vars, mrset->n_vars--,
1553 sizeof *mrset->vars, j);
1557 if (mrset->n_vars < 2)
1559 mrset_destroy (mrset);
1560 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1567 /* Returns D's attribute set. The caller may examine or modify
1568 the attribute set, but must not destroy it. Destroying D or
1569 calling dict_set_attributes for D will also destroy D's
1572 dict_get_attributes (const struct dictionary *d)
1574 return CONST_CAST (struct attrset *, &d->attributes);
1577 /* Replaces D's attributes set by a copy of ATTRS. */
1579 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1581 attrset_destroy (&d->attributes);
1582 attrset_clone (&d->attributes, attrs);
1585 /* Returns true if D has at least one attribute in its attribute
1586 set, false if D's attribute set is empty. */
1588 dict_has_attributes (const struct dictionary *d)
1590 return attrset_count (&d->attributes) > 0;
1593 /* Called from variable.c to notify the dictionary that some property of
1594 the variable has changed */
1596 dict_var_changed (const struct variable *v)
1598 if ( var_has_vardict (v))
1600 const struct vardict_info *vardict = var_get_vardict (v);
1601 struct dictionary *d = vardict->dict;
1606 if (d->changed ) d->changed (d, d->changed_data);
1607 if ( d->callbacks && d->callbacks->var_changed )
1608 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
1613 /* Called from variable.c to notify the dictionary that the variable's width
1616 dict_var_resized (const struct variable *v, int old_width)
1618 if ( var_has_vardict (v))
1620 const struct vardict_info *vardict = var_get_vardict (v);
1621 struct dictionary *d;
1625 if (d->changed) d->changed (d, d->changed_data);
1627 invalidate_proto (d);
1628 if ( d->callbacks && d->callbacks->var_resized )
1629 d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
1634 /* Called from variable.c to notify the dictionary that the variable's display width
1637 dict_var_display_width_changed (const struct variable *v)
1639 if ( var_has_vardict (v))
1641 const struct vardict_info *vardict = var_get_vardict (v);
1642 struct dictionary *d;
1646 if (d->changed) d->changed (d, d->changed_data);
1647 if ( d->callbacks && d->callbacks->var_display_width_changed )
1648 d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
1652 /* Dictionary used to contain "internal variables". */
1653 static struct dictionary *internal_dict;
1655 /* Create a variable of the specified WIDTH to be used for internal
1656 calculations only. The variable is assigned case index CASE_IDX. */
1658 dict_create_internal_var (int case_idx, int width)
1660 if (internal_dict == NULL)
1661 internal_dict = dict_create ();
1665 static int counter = INT_MAX / 2;
1666 struct variable *var;
1669 if (++counter == INT_MAX)
1670 counter = INT_MAX / 2;
1672 sprintf (name, "$internal%d", counter);
1673 var = dict_create_var (internal_dict, name, width);
1676 set_var_case_index (var, case_idx);
1682 /* Destroys VAR, which must have been created with
1683 dict_create_internal_var(). */
1685 dict_destroy_internal_var (struct variable *var)
1689 dict_delete_var (internal_dict, var);
1691 /* Destroy internal_dict if it has no variables left, just so that
1692 valgrind --leak-check --show-reachable won't show internal_dict. */
1693 if (dict_get_var_cnt (internal_dict) == 0)
1695 dict_destroy (internal_dict);
1696 internal_dict = NULL;
1702 vardict_get_dict_index (const struct vardict_info *vardict)
1704 return vardict - vardict->dict->var;