1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
58 struct vardict_info *var; /* Variables. */
59 size_t var_cnt, var_cap; /* Number of variables, capacity. */
60 struct caseproto *proto; /* Prototype for dictionary cases
62 struct hmap name_map; /* Variable index by name. */
63 int next_value_idx; /* Index of next `union value' to allocate. */
64 const struct variable **split; /* SPLIT FILE vars. */
65 size_t split_cnt; /* SPLIT FILE count. */
66 struct variable *weight; /* WEIGHT variable. */
67 struct variable *filter; /* FILTER variable. */
68 casenumber case_limit; /* Current case limit (N command). */
69 char *label; /* File label. */
70 struct string_array documents; /* Documents. */
71 struct vector **vector; /* Vectors of variables. */
72 size_t vector_cnt; /* Number of vectors. */
73 struct attrset attributes; /* Custom attributes. */
74 struct mrset **mrsets; /* Multiple response sets. */
75 size_t n_mrsets; /* Number of multiple response sets. */
77 char *encoding; /* Character encoding of string data */
79 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
81 void *cb_data ; /* Data passed to callbacks */
83 void (*changed) (struct dictionary *, void *); /* Generic change callback */
87 static void dict_unset_split_var (struct dictionary *, struct variable *);
88 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
90 /* Returns the encoding for data in dictionary D. The return value is a
91 nonnull string that contains an IANA character set name. */
93 dict_get_encoding (const struct dictionary *d)
98 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
99 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
100 error message on failure. */
102 dict_id_is_valid (const struct dictionary *dict, const char *id,
105 return id_is_valid (id, dict->encoding, issue_error);
109 dict_set_change_callback (struct dictionary *d,
110 void (*changed) (struct dictionary *, void*),
113 d->changed = changed;
114 d->changed_data = data;
117 /* Discards dictionary D's caseproto. (It will be regenerated
118 lazily, on demand.) */
120 invalidate_proto (struct dictionary *d)
122 caseproto_unref (d->proto);
126 /* Print a representation of dictionary D to stdout, for
127 debugging purposes. */
129 dict_dump (const struct dictionary *d)
132 for (i = 0 ; i < d->var_cnt ; ++i )
134 const struct variable *v = d->var[i].var;
135 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
137 var_get_dict_index (v),
138 var_get_case_index (v));
143 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
144 the dictionary or any of the variables it contains are modified.
145 Each callback will get passed CALLBACK_DATA.
146 Any callback may be NULL, in which case it'll be ignored.
149 dict_set_callbacks (struct dictionary *dict,
150 const struct dict_callbacks *callbacks,
153 dict->callbacks = callbacks;
154 dict->cb_data = callback_data;
157 /* Shallow copy the callbacks from SRC to DEST */
159 dict_copy_callbacks (struct dictionary *dest,
160 const struct dictionary *src)
162 dest->callbacks = src->callbacks;
163 dest->cb_data = src->cb_data;
166 /* Creates and returns a new dictionary with the specified ENCODING. */
168 dict_create (const char *encoding)
170 struct dictionary *d = xzalloc (sizeof *d);
172 d->encoding = xstrdup (encoding);
173 hmap_init (&d->name_map);
174 attrset_init (&d->attributes);
179 /* Creates and returns a (deep) copy of an existing
182 The new dictionary's case indexes are copied from the old
183 dictionary. If the new dictionary won't be used to access
184 cases produced with the old dictionary, then the new
185 dictionary's case indexes should be compacted with
186 dict_compact_values to save space.
188 Callbacks are not cloned. */
190 dict_clone (const struct dictionary *s)
192 struct dictionary *d;
195 d = dict_create (s->encoding);
197 for (i = 0; i < s->var_cnt; i++)
199 struct variable *sv = s->var[i].var;
200 struct variable *dv = dict_clone_var_assert (d, sv);
203 for (i = 0; i < var_get_short_name_cnt (sv); i++)
204 var_set_short_name (dv, i, var_get_short_name (sv, i));
206 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
209 d->next_value_idx = s->next_value_idx;
211 d->split_cnt = s->split_cnt;
212 if (d->split_cnt > 0)
214 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
215 for (i = 0; i < d->split_cnt; i++)
216 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
219 if (s->weight != NULL)
220 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
222 if (s->filter != NULL)
223 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
225 d->case_limit = s->case_limit;
226 dict_set_label (d, dict_get_label (s));
227 dict_set_documents (d, dict_get_documents (s));
229 d->vector_cnt = s->vector_cnt;
230 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
231 for (i = 0; i < s->vector_cnt; i++)
232 d->vector[i] = vector_clone (s->vector[i], s, d);
234 dict_set_attributes (d, dict_get_attributes (s));
236 for (i = 0; i < s->n_mrsets; i++)
238 const struct mrset *old = s->mrsets[i];
242 /* Clone old mrset, then replace vars from D by vars from S. */
243 new = mrset_clone (old);
244 for (j = 0; j < new->n_vars; j++)
245 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
247 dict_add_mrset (d, new);
253 /* Clears the contents from a dictionary without destroying the
254 dictionary itself. */
256 dict_clear (struct dictionary *d)
258 /* FIXME? Should we really clear case_limit, label, documents?
259 Others are necessarily cleared by deleting all the variables.*/
260 while (d->var_cnt > 0 )
262 dict_delete_var (d, d->var[d->var_cnt - 1].var);
267 d->var_cnt = d->var_cap = 0;
268 invalidate_proto (d);
269 hmap_clear (&d->name_map);
270 d->next_value_idx = 0;
271 dict_set_split_vars (d, NULL, 0);
272 dict_set_weight (d, NULL);
273 dict_set_filter (d, NULL);
277 string_array_clear (&d->documents);
278 dict_clear_vectors (d);
279 attrset_clear (&d->attributes);
282 /* Clears a dictionary and destroys it. */
284 dict_destroy (struct dictionary *d)
288 /* In general, we don't want callbacks occuring, if the dictionary
289 is being destroyed */
290 d->callbacks = NULL ;
293 string_array_destroy (&d->documents);
294 hmap_destroy (&d->name_map);
295 attrset_destroy (&d->attributes);
296 dict_clear_mrsets (d);
302 /* Returns the number of variables in D. */
304 dict_get_var_cnt (const struct dictionary *d)
309 /* Returns the variable in D with dictionary index IDX, which
310 must be between 0 and the count returned by
311 dict_get_var_cnt(), exclusive. */
313 dict_get_var (const struct dictionary *d, size_t idx)
315 assert (idx < d->var_cnt);
317 return d->var[idx].var;
320 /* Sets *VARS to an array of pointers to variables in D and *CNT
321 to the number of variables in *D. All variables are returned
322 except for those, if any, in the classes indicated by EXCLUDE.
323 (There is no point in putting DC_SYSTEM in EXCLUDE as
324 dictionaries never include system variables.) */
326 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
327 size_t *cnt, enum dict_class exclude)
329 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
332 /* Sets *VARS to an array of pointers to variables in D and *CNT
333 to the number of variables in *D. All variables are returned
334 except for those, if any, in the classes indicated by EXCLUDE.
335 (There is no point in putting DC_SYSTEM in EXCLUDE as
336 dictionaries never include system variables.) */
338 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
339 size_t *cnt, enum dict_class exclude)
344 assert (exclude == (exclude & DC_ALL));
347 for (i = 0; i < d->var_cnt; i++)
349 enum dict_class class = var_get_dict_class (d->var[i].var);
350 if (!(class & exclude))
354 *vars = xnmalloc (count, sizeof **vars);
356 for (i = 0; i < d->var_cnt; i++)
358 enum dict_class class = var_get_dict_class (d->var[i].var);
359 if (!(class & exclude))
360 (*vars)[(*cnt)++] = d->var[i].var;
362 assert (*cnt == count);
365 static struct variable *
366 add_var (struct dictionary *d, struct variable *v)
368 struct vardict_info *vardict;
370 /* Update dictionary. */
371 if (d->var_cnt >= d->var_cap)
375 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
376 hmap_clear (&d->name_map);
377 for (i = 0; i < d->var_cnt; i++)
379 var_set_vardict (d->var[i].var, &d->var[i]);
380 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
381 d->var[i].name_node.hash);
385 vardict = &d->var[d->var_cnt++];
388 hmap_insert (&d->name_map, &vardict->name_node,
389 utf8_hash_case_string (var_get_name (v), 0));
390 vardict->case_index = d->next_value_idx;
391 var_set_vardict (v, vardict);
393 if ( d->changed ) d->changed (d, d->changed_data);
394 if ( d->callbacks && d->callbacks->var_added )
395 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
398 invalidate_proto (d);
403 /* Creates and returns a new variable in D with the given NAME
404 and WIDTH. Returns a null pointer if the given NAME would
405 duplicate that of an existing variable in the dictionary. */
407 dict_create_var (struct dictionary *d, const char *name, int width)
409 return (dict_lookup_var (d, name) == NULL
410 ? dict_create_var_assert (d, name, width)
414 /* Creates and returns a new variable in D with the given NAME
415 and WIDTH. Assert-fails if the given NAME would duplicate
416 that of an existing variable in the dictionary. */
418 dict_create_var_assert (struct dictionary *d, const char *name, int width)
420 assert (dict_lookup_var (d, name) == NULL);
421 return add_var (d, var_create (name, width));
424 /* Creates and returns a new variable in D, as a copy of existing variable
425 OLD_VAR, which need not be in D or in any dictionary. Returns a null
426 pointer if OLD_VAR's name would duplicate that of an existing variable in
429 dict_clone_var (struct dictionary *d, const struct variable *old_var)
431 return dict_clone_var_as (d, old_var, var_get_name (old_var));
434 /* Creates and returns a new variable in D, as a copy of existing variable
435 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
436 OLD_VAR's name would duplicate that of an existing variable in the
439 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
441 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
444 /* Creates and returns a new variable in D with name NAME, as a copy of
445 existing variable OLD_VAR, which need not be in D or in any dictionary.
446 Returns a null pointer if the given NAME would duplicate that of an existing
447 variable in the dictionary. */
449 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
452 return (dict_lookup_var (d, name) == NULL
453 ? dict_clone_var_as_assert (d, old_var, name)
457 /* Creates and returns a new variable in D with name NAME, as a copy of
458 existing variable OLD_VAR, which need not be in D or in any dictionary.
459 Assert-fails if the given NAME would duplicate that of an existing variable
460 in the dictionary. */
462 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
465 struct variable *new_var = var_clone (old_var);
466 assert (dict_lookup_var (d, name) == NULL);
467 var_set_name (new_var, name);
468 return add_var (d, new_var);
471 /* Returns the variable named NAME in D, or a null pointer if no
472 variable has that name. */
474 dict_lookup_var (const struct dictionary *d, const char *name)
476 struct vardict_info *vardict;
478 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
479 utf8_hash_case_string (name, 0), &d->name_map)
481 struct variable *var = vardict->var;
482 if (!utf8_strcasecmp (var_get_name (var), name))
489 /* Returns the variable named NAME in D. Assert-fails if no
490 variable has that name. */
492 dict_lookup_var_assert (const struct dictionary *d, const char *name)
494 struct variable *v = dict_lookup_var (d, name);
499 /* Returns true if variable V is in dictionary D,
502 dict_contains_var (const struct dictionary *d, const struct variable *v)
504 return (var_has_vardict (v)
505 && vardict_get_dictionary (var_get_vardict (v)) == d);
508 /* Compares two double pointers to variables, which should point
509 to elements of a struct dictionary's `var' member array. */
511 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
513 struct variable *const *a = a_;
514 struct variable *const *b = b_;
516 return *a < *b ? -1 : *a > *b;
520 unindex_var (struct dictionary *d, struct vardict_info *vardict)
522 hmap_delete (&d->name_map, &vardict->name_node);
525 /* This function assumes that vardict->name_node.hash is valid, that is, that
526 its name has not changed since it was hashed (rename_var() updates this
527 hash along with the name itself). */
529 reindex_var (struct dictionary *d, struct vardict_info *vardict)
531 struct variable *var = vardict->var;
533 var_set_vardict (var, vardict);
534 hmap_insert_fast (&d->name_map, &vardict->name_node,
535 vardict->name_node.hash);
537 if ( d->changed ) d->changed (d, d->changed_data);
538 if ( d->callbacks && d->callbacks->var_changed )
539 d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
542 /* Sets the case_index in V's vardict to CASE_INDEX. */
544 set_var_case_index (struct variable *v, int case_index)
546 var_get_vardict (v)->case_index = case_index;
549 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
552 unindex_vars (struct dictionary *d, size_t from, size_t to)
556 for (i = from; i < to; i++)
557 unindex_var (d, &d->var[i]);
560 /* Re-sets the dict_index in the dictionary variables with
561 indexes from FROM to TO (exclusive). */
563 reindex_vars (struct dictionary *d, size_t from, size_t to)
567 for (i = from; i < to; i++)
568 reindex_var (d, &d->var[i]);
571 /* Deletes variable V from dictionary D and frees V.
573 This is a very bad idea if there might be any pointers to V
574 from outside D. In general, no variable in the active dataset's
575 dictionary should be deleted when any transformations are
576 active on the dictionary's dataset, because those
577 transformations might reference the deleted variable. The
578 safest time to delete a variable is just after a procedure has
579 been executed, as done by DELETE VARIABLES.
581 Pointers to V within D are not a problem, because
582 dict_delete_var() knows to remove V from split variables,
583 weights, filters, etc. */
585 dict_delete_var (struct dictionary *d, struct variable *v)
587 int dict_index = var_get_dict_index (v);
588 const int case_index = var_get_case_index (v);
590 assert (dict_contains_var (d, v));
592 dict_unset_split_var (d, v);
593 dict_unset_mrset_var (d, v);
596 dict_set_weight (d, NULL);
599 dict_set_filter (d, NULL);
601 dict_clear_vectors (d);
603 /* Remove V from var array. */
604 unindex_vars (d, dict_index, d->var_cnt);
605 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
608 /* Update dict_index for each affected variable. */
609 reindex_vars (d, dict_index, d->var_cnt);
612 var_clear_vardict (v);
614 if ( d->changed ) d->changed (d, d->changed_data);
616 invalidate_proto (d);
617 if (d->callbacks && d->callbacks->var_deleted )
618 d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
623 /* Deletes the COUNT variables listed in VARS from D. This is
624 unsafe; see the comment on dict_delete_var() for details. */
626 dict_delete_vars (struct dictionary *d,
627 struct variable *const *vars, size_t count)
629 /* FIXME: this can be done in O(count) time, but this algorithm
631 assert (count == 0 || vars != NULL);
634 dict_delete_var (d, *vars++);
637 /* Deletes the COUNT variables in D starting at index IDX. This
638 is unsafe; see the comment on dict_delete_var() for
641 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
643 /* FIXME: this can be done in O(count) time, but this algorithm
645 assert (idx + count <= d->var_cnt);
648 dict_delete_var (d, d->var[idx].var);
651 /* Deletes scratch variables from dictionary D. */
653 dict_delete_scratch_vars (struct dictionary *d)
657 /* FIXME: this can be done in O(count) time, but this algorithm
659 for (i = 0; i < d->var_cnt; )
660 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
661 dict_delete_var (d, d->var[i].var);
666 /* Moves V to 0-based position IDX in D. Other variables in D,
667 if any, retain their relative positions. Runs in time linear
668 in the distance moved. */
670 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
672 size_t old_index = var_get_dict_index (v);
674 assert (new_index < d->var_cnt);
676 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
677 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
678 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
681 /* Reorders the variables in D, placing the COUNT variables
682 listed in ORDER in that order at the beginning of D. The
683 other variables in D, if any, retain their relative
686 dict_reorder_vars (struct dictionary *d,
687 struct variable *const *order, size_t count)
689 struct vardict_info *new_var;
692 assert (count == 0 || order != NULL);
693 assert (count <= d->var_cnt);
695 new_var = xnmalloc (d->var_cap, sizeof *new_var);
697 /* Add variables in ORDER to new_var. */
698 for (i = 0; i < count; i++)
700 struct vardict_info *old_var;
702 assert (dict_contains_var (d, order[i]));
704 old_var = var_get_vardict (order[i]);
705 new_var[i] = *old_var;
706 old_var->dict = NULL;
709 /* Add remaining variables to new_var. */
710 for (i = 0; i < d->var_cnt; i++)
711 if (d->var[i].dict != NULL)
712 new_var[count++] = d->var[i];
713 assert (count == d->var_cnt);
715 /* Replace old vardicts by new ones. */
719 hmap_clear (&d->name_map);
720 reindex_vars (d, 0, d->var_cnt);
723 /* Changes the name of variable V that is currently in a dictionary to
726 rename_var (struct variable *v, const char *new_name)
728 struct vardict_info *vardict = var_get_vardict (v);
729 var_clear_vardict (v);
730 var_set_name (v, new_name);
731 vardict->name_node.hash = utf8_hash_case_string (new_name, 0);
732 var_set_vardict (v, vardict);
735 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
736 a variable named NEW_NAME is already in D, except that
737 NEW_NAME may be the same as V's existing name. */
739 dict_rename_var (struct dictionary *d, struct variable *v,
740 const char *new_name)
742 assert (!utf8_strcasecmp (var_get_name (v), new_name)
743 || dict_lookup_var (d, new_name) == NULL);
745 unindex_var (d, var_get_vardict (v));
746 rename_var (v, new_name);
747 reindex_var (d, var_get_vardict (v));
749 if (settings_get_algorithm () == ENHANCED)
750 var_clear_short_names (v);
752 if ( d->changed ) d->changed (d, d->changed_data);
753 if ( d->callbacks && d->callbacks->var_changed )
754 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
757 /* Renames COUNT variables specified in VARS to the names given
758 in NEW_NAMES within dictionary D. If the renaming would
759 result in a duplicate variable name, returns false and stores a
760 name that would be duplicated into *ERR_NAME (if ERR_NAME is
761 non-null). Otherwise, the renaming is successful, and true
764 dict_rename_vars (struct dictionary *d,
765 struct variable **vars, char **new_names, size_t count,
772 assert (count == 0 || vars != NULL);
773 assert (count == 0 || new_names != NULL);
775 /* Save the names of the variables to be renamed. */
776 pool = pool_create ();
777 old_names = pool_nalloc (pool, count, sizeof *old_names);
778 for (i = 0; i < count; i++)
779 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
781 /* Remove the variables to be renamed from the name hash,
783 for (i = 0; i < count; i++)
785 unindex_var (d, var_get_vardict (vars[i]));
786 rename_var (vars[i], new_names[i]);
789 /* Add the renamed variables back into the name hash,
790 checking for conflicts. */
791 for (i = 0; i < count; i++)
793 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
795 /* There is a name conflict.
796 Back out all the name changes that have already
797 taken place, and indicate failure. */
799 if (err_name != NULL)
800 *err_name = new_names[i];
802 for (i = 0; i < fail_idx; i++)
803 unindex_var (d, var_get_vardict (vars[i]));
805 for (i = 0; i < count; i++)
807 rename_var (vars[i], old_names[i]);
808 reindex_var (d, var_get_vardict (vars[i]));
814 reindex_var (d, var_get_vardict (vars[i]));
817 /* Clear short names. */
818 if (settings_get_algorithm () == ENHANCED)
819 for (i = 0; i < count; i++)
820 var_clear_short_names (vars[i]);
826 /* Returns true if a variable named NAME may be inserted in DICT;
827 that is, if there is not already a variable with that name in
828 DICT and if NAME is not a reserved word. (The caller's checks
829 have already verified that NAME is otherwise acceptable as a
832 var_name_is_insertable (const struct dictionary *dict, const char *name)
834 return (dict_lookup_var (dict, name) == NULL
835 && lex_id_to_token (ss_cstr (name)) == T_ID);
839 make_hinted_name (const struct dictionary *dict, const char *hint)
841 size_t hint_len = strlen (hint);
842 bool dropped = false;
847 /* The allocation size here is OK: characters that are copied directly fit
848 OK, and characters that are not copied directly are replaced by a single
849 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
850 replaced by '_' too. */
851 root = rp = xmalloc (hint_len + 1);
852 for (ofs = 0; ofs < hint_len; ofs += mblen)
856 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
859 ? lex_uc_is_id1 (uc) && uc != '$'
860 : lex_uc_is_idn (uc))
867 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
878 if (var_name_is_insertable (dict, root))
881 for (i = 0; i < ULONG_MAX; i++)
883 char suffix[INT_BUFSIZE_BOUND (i) + 1];
887 if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
890 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
891 if (var_name_is_insertable (dict, name))
906 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
908 unsigned long int number;
910 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
914 char name[3 + INT_STRLEN_BOUND (number) + 1];
916 sprintf (name, "VAR%03lu", number);
917 if (dict_lookup_var (dict, name) == NULL)
919 if (num_start != NULL)
920 *num_start = number + 1;
921 return xstrdup (name);
929 /* Devises and returns a variable name unique within DICT. The variable name
930 is owned by the caller, which must free it with free() when it is no longer
933 HINT, if it is non-null, is used as a suggestion that will be
934 modified for suitability as a variable name and for
937 If HINT is null or entirely unsuitable, a name in the form
938 "VAR%03d" will be generated, where the smallest unused integer
939 value is used. If NUM_START is non-null, then its value is
940 used as the minimum numeric value to check, and it is updated
941 to the next value to be checked.
944 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
945 unsigned long int *num_start)
949 char *hinted_name = make_hinted_name (dict, hint);
950 if (hinted_name != NULL)
953 return make_numeric_name (dict, num_start);
956 /* Returns the weighting variable in dictionary D, or a null
957 pointer if the dictionary is unweighted. */
959 dict_get_weight (const struct dictionary *d)
961 assert (d->weight == NULL || dict_contains_var (d, d->weight));
966 /* Returns the value of D's weighting variable in case C, except
967 that a negative weight is returned as 0. Returns 1 if the
968 dictionary is unweighted. Will warn about missing, negative,
969 or zero values if *WARN_ON_INVALID is true. The function will
970 set *WARN_ON_INVALID to false if an invalid weight is
973 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
974 bool *warn_on_invalid)
978 if (d->weight == NULL)
982 double w = case_num (c, d->weight);
983 if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
985 if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
986 *warn_on_invalid = false;
987 msg (SW, _("At least one case in the data file had a weight value "
988 "that was user-missing, system-missing, zero, or "
989 "negative. These case(s) were ignored."));
995 /* Sets the weighting variable of D to V, or turning off
996 weighting if V is a null pointer. */
998 dict_set_weight (struct dictionary *d, struct variable *v)
1000 assert (v == NULL || dict_contains_var (d, v));
1001 assert (v == NULL || var_is_numeric (v));
1005 if (d->changed) d->changed (d, d->changed_data);
1006 if ( d->callbacks && d->callbacks->weight_changed )
1007 d->callbacks->weight_changed (d,
1008 v ? var_get_dict_index (v) : -1,
1012 /* Returns the filter variable in dictionary D (see cmd_filter())
1013 or a null pointer if the dictionary is unfiltered. */
1015 dict_get_filter (const struct dictionary *d)
1017 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1022 /* Sets V as the filter variable for dictionary D. Passing a
1023 null pointer for V turn off filtering. */
1025 dict_set_filter (struct dictionary *d, struct variable *v)
1027 assert (v == NULL || dict_contains_var (d, v));
1028 assert (v == NULL || var_is_numeric (v));
1032 if (d->changed) d->changed (d, d->changed_data);
1033 if ( d->callbacks && d->callbacks->filter_changed )
1034 d->callbacks->filter_changed (d,
1035 v ? var_get_dict_index (v) : -1,
1039 /* Returns the case limit for dictionary D, or zero if the number
1040 of cases is unlimited. */
1042 dict_get_case_limit (const struct dictionary *d)
1044 return d->case_limit;
1047 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1048 0 for CASE_LIMIT to indicate no limit. */
1050 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1052 d->case_limit = case_limit;
1055 /* Returns the prototype used for cases created by dictionary D. */
1056 const struct caseproto *
1057 dict_get_proto (const struct dictionary *d_)
1059 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1060 if (d->proto == NULL)
1064 d->proto = caseproto_create ();
1065 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1066 for (i = 0; i < d->var_cnt; i++)
1067 d->proto = caseproto_set_width (d->proto,
1068 var_get_case_index (d->var[i].var),
1069 var_get_width (d->var[i].var));
1074 /* Returns the case index of the next value to be added to D.
1075 This value is the number of `union value's that need to be
1076 allocated to store a case for dictionary D. */
1078 dict_get_next_value_idx (const struct dictionary *d)
1080 return d->next_value_idx;
1083 /* Returns the number of bytes needed to store a case for
1086 dict_get_case_size (const struct dictionary *d)
1088 return sizeof (union value) * dict_get_next_value_idx (d);
1091 /* Reassigns values in dictionary D so that fragmentation is
1094 dict_compact_values (struct dictionary *d)
1098 d->next_value_idx = 0;
1099 for (i = 0; i < d->var_cnt; i++)
1101 struct variable *v = d->var[i].var;
1102 set_var_case_index (v, d->next_value_idx++);
1104 invalidate_proto (d);
1107 /* Returns the number of values occupied by the variables in
1108 dictionary D. All variables are considered if EXCLUDE_CLASSES
1109 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1110 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1111 corresponding type of variable.
1113 The return value may be less than the number of values in one
1114 of dictionary D's cases (as returned by
1115 dict_get_next_value_idx) even if E is 0, because there may be
1116 gaps in D's cases due to deleted variables. */
1118 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1123 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1125 | (1u << DC_SCRATCH))) == 0);
1128 for (i = 0; i < d->var_cnt; i++)
1130 enum dict_class class = var_get_dict_class (d->var[i].var);
1131 if (!(exclude_classes & (1u << class)))
1137 /* Returns the case prototype that would result after deleting
1138 all variables from D that are not in one of the
1139 EXCLUDE_CLASSES and compacting the dictionary with
1142 The caller must unref the returned caseproto when it is no
1145 dict_get_compacted_proto (const struct dictionary *d,
1146 unsigned int exclude_classes)
1148 struct caseproto *proto;
1151 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1153 | (1u << DC_SCRATCH))) == 0);
1155 proto = caseproto_create ();
1156 for (i = 0; i < d->var_cnt; i++)
1158 struct variable *v = d->var[i].var;
1159 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1160 proto = caseproto_add_width (proto, var_get_width (v));
1165 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1166 dict_get_split_cnt() to determine how many SPLIT FILE vars
1167 there are. Returns a null pointer if and only if there are no
1169 const struct variable *const *
1170 dict_get_split_vars (const struct dictionary *d)
1175 /* Returns the number of SPLIT FILE vars. */
1177 dict_get_split_cnt (const struct dictionary *d)
1179 return d->split_cnt;
1182 /* Removes variable V, which must be in D, from D's set of split
1185 dict_unset_split_var (struct dictionary *d, struct variable *v)
1189 assert (dict_contains_var (d, v));
1191 orig_count = d->split_cnt;
1192 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1193 &v, compare_var_ptrs, NULL);
1194 if (orig_count != d->split_cnt)
1196 if (d->changed) d->changed (d, d->changed_data);
1197 /* We changed the set of split variables so invoke the
1199 if (d->callbacks && d->callbacks->split_changed)
1200 d->callbacks->split_changed (d, d->cb_data);
1204 /* Sets CNT split vars SPLIT in dictionary D. */
1206 dict_set_split_vars (struct dictionary *d,
1207 struct variable *const *split, size_t cnt)
1209 assert (cnt == 0 || split != NULL);
1214 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1215 memcpy (d->split, split, cnt * sizeof *d->split);
1223 if (d->changed) d->changed (d, d->changed_data);
1224 if ( d->callbacks && d->callbacks->split_changed )
1225 d->callbacks->split_changed (d, d->cb_data);
1228 /* Returns the file label for D, or a null pointer if D is
1229 unlabeled (see cmd_file_label()). */
1231 dict_get_label (const struct dictionary *d)
1236 /* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
1239 Removes D's label if LABEL is null or the empty string. */
1241 dict_set_label (struct dictionary *d, const char *label)
1244 if (label == NULL || label[0] == '\0')
1247 d->label = utf8_encoding_trunc (label, d->encoding, 60);
1250 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1251 return value is always nonnull; if there are no documents then the
1252 string_arary is empty.*/
1253 const struct string_array *
1254 dict_get_documents (const struct dictionary *d)
1256 return &d->documents;
1259 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1261 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1265 dict_clear_documents (d);
1267 for (i = 0; i < new_docs->n; i++)
1268 dict_add_document_line (d, new_docs->strings[i], false);
1271 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1272 into individual lines at new-line characters. Each line is truncated to at
1273 most DOC_LINE_LENGTH bytes in D's encoding. */
1275 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1279 dict_clear_documents (d);
1280 for (s = new_docs; *s != '\0'; )
1282 size_t len = strcspn (s, "\n");
1283 char *line = xmemdup0 (s, len);
1284 dict_add_document_line (d, line, false);
1293 /* Drops the documents from dictionary D. */
1295 dict_clear_documents (struct dictionary *d)
1297 string_array_clear (&d->documents);
1300 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1301 truncated so that it is no more than 80 bytes in the dictionary's
1302 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1303 then a warning will be issued. */
1305 dict_add_document_line (struct dictionary *d, const char *line,
1311 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1312 truncated = line[trunc_len] != '\0';
1313 if (truncated && issue_warning)
1315 /* Note to translators: "bytes" is correct, not characters */
1316 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1319 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1324 /* Returns the number of document lines in dictionary D. */
1326 dict_get_document_line_cnt (const struct dictionary *d)
1328 return d->documents.n;
1331 /* Returns document line number IDX in dictionary D. The caller must not
1332 modify or free the returned string. */
1334 dict_get_document_line (const struct dictionary *d, size_t idx)
1336 assert (idx < d->documents.n);
1337 return d->documents.strings[idx];
1340 /* Creates in D a vector named NAME that contains the CNT
1341 variables in VAR. Returns true if successful, or false if a
1342 vector named NAME already exists in D. */
1344 dict_create_vector (struct dictionary *d,
1346 struct variable **var, size_t cnt)
1351 for (i = 0; i < cnt; i++)
1352 assert (dict_contains_var (d, var[i]));
1354 if (dict_lookup_vector (d, name) == NULL)
1356 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1357 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1364 /* Creates in D a vector named NAME that contains the CNT
1365 variables in VAR. A vector named NAME must not already exist
1368 dict_create_vector_assert (struct dictionary *d,
1370 struct variable **var, size_t cnt)
1372 assert (dict_lookup_vector (d, name) == NULL);
1373 dict_create_vector (d, name, var, cnt);
1376 /* Returns the vector in D with index IDX, which must be less
1377 than dict_get_vector_cnt (D). */
1378 const struct vector *
1379 dict_get_vector (const struct dictionary *d, size_t idx)
1381 assert (idx < d->vector_cnt);
1383 return d->vector[idx];
1386 /* Returns the number of vectors in D. */
1388 dict_get_vector_cnt (const struct dictionary *d)
1390 return d->vector_cnt;
1393 /* Looks up and returns the vector within D with the given
1395 const struct vector *
1396 dict_lookup_vector (const struct dictionary *d, const char *name)
1399 for (i = 0; i < d->vector_cnt; i++)
1400 if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
1401 return d->vector[i];
1405 /* Deletes all vectors from D. */
1407 dict_clear_vectors (struct dictionary *d)
1411 for (i = 0; i < d->vector_cnt; i++)
1412 vector_destroy (d->vector[i]);
1419 /* Multiple response sets. */
1421 /* Returns the multiple response set in DICT with index IDX, which must be
1422 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1423 const struct mrset *
1424 dict_get_mrset (const struct dictionary *dict, size_t idx)
1426 assert (idx < dict->n_mrsets);
1427 return dict->mrsets[idx];
1430 /* Returns the number of multiple response sets in DICT. */
1432 dict_get_n_mrsets (const struct dictionary *dict)
1434 return dict->n_mrsets;
1437 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1438 returns its index; otherwise, returns SIZE_MAX. */
1440 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1444 for (i = 0; i < dict->n_mrsets; i++)
1445 if (!utf8_strcasecmp (name, dict->mrsets[i]->name))
1451 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1452 returns it; otherwise, returns NULL. */
1453 const struct mrset *
1454 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1456 size_t idx = dict_lookup_mrset_idx (dict, name);
1457 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1460 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1461 true if a set was replaced, false if none existed with the specified name.
1463 Ownership of MRSET is transferred to DICT. */
1465 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1469 assert (mrset_ok (mrset, dict));
1471 idx = dict_lookup_mrset_idx (dict, mrset->name);
1472 if (idx == SIZE_MAX)
1474 dict->mrsets = xrealloc (dict->mrsets,
1475 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1476 dict->mrsets[dict->n_mrsets++] = mrset;
1481 mrset_destroy (dict->mrsets[idx]);
1482 dict->mrsets[idx] = mrset;
1487 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1488 from DICT and returns true. If none is found, returns false without
1491 Deleting one multiple response set causes the indexes of other sets within
1494 dict_delete_mrset (struct dictionary *dict, const char *name)
1496 size_t idx = dict_lookup_mrset_idx (dict, name);
1497 if (idx != SIZE_MAX)
1499 mrset_destroy (dict->mrsets[idx]);
1500 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1507 /* Deletes all multiple response sets from DICT. */
1509 dict_clear_mrsets (struct dictionary *dict)
1513 for (i = 0; i < dict->n_mrsets; i++)
1514 mrset_destroy (dict->mrsets[i]);
1515 free (dict->mrsets);
1516 dict->mrsets = NULL;
1520 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1522 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1526 assert (dict_contains_var (dict, var));
1528 for (i = 0; i < dict->n_mrsets; )
1530 struct mrset *mrset = dict->mrsets[i];
1533 for (j = 0; j < mrset->n_vars; )
1534 if (mrset->vars[j] == var)
1535 remove_element (mrset->vars, mrset->n_vars--,
1536 sizeof *mrset->vars, j);
1540 if (mrset->n_vars < 2)
1542 mrset_destroy (mrset);
1543 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1550 /* Returns D's attribute set. The caller may examine or modify
1551 the attribute set, but must not destroy it. Destroying D or
1552 calling dict_set_attributes for D will also destroy D's
1555 dict_get_attributes (const struct dictionary *d)
1557 return CONST_CAST (struct attrset *, &d->attributes);
1560 /* Replaces D's attributes set by a copy of ATTRS. */
1562 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1564 attrset_destroy (&d->attributes);
1565 attrset_clone (&d->attributes, attrs);
1568 /* Returns true if D has at least one attribute in its attribute
1569 set, false if D's attribute set is empty. */
1571 dict_has_attributes (const struct dictionary *d)
1573 return attrset_count (&d->attributes) > 0;
1576 /* Called from variable.c to notify the dictionary that some property of
1577 the variable has changed */
1579 dict_var_changed (const struct variable *v)
1581 if ( var_has_vardict (v))
1583 const struct vardict_info *vardict = var_get_vardict (v);
1584 struct dictionary *d = vardict->dict;
1589 if (d->changed ) d->changed (d, d->changed_data);
1590 if ( d->callbacks && d->callbacks->var_changed )
1591 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
1596 /* Called from variable.c to notify the dictionary that the variable's width
1599 dict_var_resized (const struct variable *v, int old_width)
1601 if ( var_has_vardict (v))
1603 const struct vardict_info *vardict = var_get_vardict (v);
1604 struct dictionary *d;
1608 if (d->changed) d->changed (d, d->changed_data);
1610 invalidate_proto (d);
1611 if ( d->callbacks && d->callbacks->var_resized )
1612 d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
1617 /* Called from variable.c to notify the dictionary that the variable's display width
1620 dict_var_display_width_changed (const struct variable *v)
1622 if ( var_has_vardict (v))
1624 const struct vardict_info *vardict = var_get_vardict (v);
1625 struct dictionary *d;
1629 if (d->changed) d->changed (d, d->changed_data);
1630 if ( d->callbacks && d->callbacks->var_display_width_changed )
1631 d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
1635 /* Dictionary used to contain "internal variables". */
1636 static struct dictionary *internal_dict;
1638 /* Create a variable of the specified WIDTH to be used for internal
1639 calculations only. The variable is assigned case index CASE_IDX. */
1641 dict_create_internal_var (int case_idx, int width)
1643 if (internal_dict == NULL)
1644 internal_dict = dict_create ("UTF-8");
1648 static int counter = INT_MAX / 2;
1649 struct variable *var;
1652 if (++counter == INT_MAX)
1653 counter = INT_MAX / 2;
1655 sprintf (name, "$internal%d", counter);
1656 var = dict_create_var (internal_dict, name, width);
1659 set_var_case_index (var, case_idx);
1665 /* Destroys VAR, which must have been created with
1666 dict_create_internal_var(). */
1668 dict_destroy_internal_var (struct variable *var)
1672 dict_delete_var (internal_dict, var);
1674 /* Destroy internal_dict if it has no variables left, just so that
1675 valgrind --leak-check --show-reachable won't show internal_dict. */
1676 if (dict_get_var_cnt (internal_dict) == 0)
1678 dict_destroy (internal_dict);
1679 internal_dict = NULL;
1685 vardict_get_dict_index (const struct vardict_info *vardict)
1687 return vardict - vardict->dict->var;