1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "data/dictionary.h"
26 #include "data/attributes.h"
27 #include "data/case.h"
28 #include "data/identifier.h"
29 #include "data/mrset.h"
30 #include "data/settings.h"
31 #include "data/value-labels.h"
32 #include "data/vardict.h"
33 #include "data/variable.h"
34 #include "data/vector.h"
35 #include "libpspp/array.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/compiler.h"
38 #include "libpspp/hash-functions.h"
39 #include "libpspp/hmap.h"
40 #include "libpspp/i18n.h"
41 #include "libpspp/message.h"
42 #include "libpspp/misc.h"
43 #include "libpspp/pool.h"
44 #include "libpspp/str.h"
45 #include "libpspp/string-array.h"
47 #include "gl/intprops.h"
48 #include "gl/minmax.h"
49 #include "gl/xalloc.h"
50 #include "gl/xmemdup0.h"
53 #define _(msgid) gettext (msgid)
58 struct vardict_info *var; /* Variables. */
59 size_t var_cnt, var_cap; /* Number of variables, capacity. */
60 struct caseproto *proto; /* Prototype for dictionary cases
62 struct hmap name_map; /* Variable index by name. */
63 int next_value_idx; /* Index of next `union value' to allocate. */
64 const struct variable **split; /* SPLIT FILE vars. */
65 size_t split_cnt; /* SPLIT FILE count. */
66 struct variable *weight; /* WEIGHT variable. */
67 struct variable *filter; /* FILTER variable. */
68 casenumber case_limit; /* Current case limit (N command). */
69 char *label; /* File label. */
70 struct string_array documents; /* Documents. */
71 struct vector **vector; /* Vectors of variables. */
72 size_t vector_cnt; /* Number of vectors. */
73 struct attrset attributes; /* Custom attributes. */
74 struct mrset **mrsets; /* Multiple response sets. */
75 size_t n_mrsets; /* Number of multiple response sets. */
77 char *encoding; /* Character encoding of string data */
79 const struct dict_callbacks *callbacks; /* Callbacks on dictionary
81 void *cb_data ; /* Data passed to callbacks */
83 void (*changed) (struct dictionary *, void *); /* Generic change callback */
87 static void dict_unset_split_var (struct dictionary *, struct variable *);
88 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
91 dict_get_encoding (const struct dictionary *d)
96 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
97 encoding, false otherwise. If ISSUE_ERROR is true, issues an explanatory
98 error message on failure. */
100 dict_id_is_valid (const struct dictionary *dict, const char *id,
103 return id_is_valid (id, dict->encoding, issue_error);
107 dict_set_change_callback (struct dictionary *d,
108 void (*changed) (struct dictionary *, void*),
111 d->changed = changed;
112 d->changed_data = data;
115 /* Discards dictionary D's caseproto. (It will be regenerated
116 lazily, on demand.) */
118 invalidate_proto (struct dictionary *d)
120 caseproto_unref (d->proto);
124 /* Print a representation of dictionary D to stdout, for
125 debugging purposes. */
127 dict_dump (const struct dictionary *d)
130 for (i = 0 ; i < d->var_cnt ; ++i )
132 const struct variable *v = d->var[i].var;
133 printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
135 var_get_dict_index (v),
136 var_get_case_index (v));
141 /* Associate CALLBACKS with DICT. Callbacks will be invoked whenever
142 the dictionary or any of the variables it contains are modified.
143 Each callback will get passed CALLBACK_DATA.
144 Any callback may be NULL, in which case it'll be ignored.
147 dict_set_callbacks (struct dictionary *dict,
148 const struct dict_callbacks *callbacks,
151 dict->callbacks = callbacks;
152 dict->cb_data = callback_data;
155 /* Shallow copy the callbacks from SRC to DEST */
157 dict_copy_callbacks (struct dictionary *dest,
158 const struct dictionary *src)
160 dest->callbacks = src->callbacks;
161 dest->cb_data = src->cb_data;
164 /* Creates and returns a new dictionary with the specified ENCODING. */
166 dict_create (const char *encoding)
168 struct dictionary *d = xzalloc (sizeof *d);
170 d->encoding = xstrdup (encoding);
171 hmap_init (&d->name_map);
172 attrset_init (&d->attributes);
177 /* Creates and returns a (deep) copy of an existing
180 The new dictionary's case indexes are copied from the old
181 dictionary. If the new dictionary won't be used to access
182 cases produced with the old dictionary, then the new
183 dictionary's case indexes should be compacted with
184 dict_compact_values to save space. */
186 dict_clone (const struct dictionary *s)
188 struct dictionary *d;
191 d = dict_create (s->encoding);
193 /* Set the new dictionary's encoding early so that string length limitations
194 are interpreted correctly. */
196 d->encoding = xstrdup (s->encoding);
198 for (i = 0; i < s->var_cnt; i++)
200 struct variable *sv = s->var[i].var;
201 struct variable *dv = dict_clone_var_assert (d, sv);
204 for (i = 0; i < var_get_short_name_cnt (sv); i++)
205 var_set_short_name (dv, i, var_get_short_name (sv, i));
207 var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
210 d->next_value_idx = s->next_value_idx;
212 d->split_cnt = s->split_cnt;
213 if (d->split_cnt > 0)
215 d->split = xnmalloc (d->split_cnt, sizeof *d->split);
216 for (i = 0; i < d->split_cnt; i++)
217 d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
220 if (s->weight != NULL)
221 dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
223 if (s->filter != NULL)
224 dict_set_filter (d, dict_lookup_var_assert (d, var_get_name (s->filter)));
226 d->case_limit = s->case_limit;
227 dict_set_label (d, dict_get_label (s));
228 dict_set_documents (d, dict_get_documents (s));
230 d->vector_cnt = s->vector_cnt;
231 d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
232 for (i = 0; i < s->vector_cnt; i++)
233 d->vector[i] = vector_clone (s->vector[i], s, d);
235 dict_set_attributes (d, dict_get_attributes (s));
237 for (i = 0; i < s->n_mrsets; i++)
239 const struct mrset *old = s->mrsets[i];
243 /* Clone old mrset, then replace vars from D by vars from S. */
244 new = mrset_clone (old);
245 for (j = 0; j < new->n_vars; j++)
246 new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
248 dict_add_mrset (d, new);
254 /* Clears the contents from a dictionary without destroying the
255 dictionary itself. */
257 dict_clear (struct dictionary *d)
259 /* FIXME? Should we really clear case_limit, label, documents?
260 Others are necessarily cleared by deleting all the variables.*/
261 while (d->var_cnt > 0 )
263 dict_delete_var (d, d->var[d->var_cnt - 1].var);
268 d->var_cnt = d->var_cap = 0;
269 invalidate_proto (d);
270 hmap_clear (&d->name_map);
271 d->next_value_idx = 0;
272 dict_set_split_vars (d, NULL, 0);
273 dict_set_weight (d, NULL);
274 dict_set_filter (d, NULL);
278 string_array_clear (&d->documents);
279 dict_clear_vectors (d);
280 attrset_clear (&d->attributes);
283 /* Destroys the aux data for every variable in D, by calling
284 var_clear_aux() for each variable. */
286 dict_clear_aux (struct dictionary *d)
290 for (i = 0; i < d->var_cnt; i++)
291 var_clear_aux (d->var[i].var);
294 /* Clears a dictionary and destroys it. */
296 dict_destroy (struct dictionary *d)
300 /* In general, we don't want callbacks occuring, if the dictionary
301 is being destroyed */
302 d->callbacks = NULL ;
305 hmap_destroy (&d->name_map);
306 attrset_destroy (&d->attributes);
307 dict_clear_mrsets (d);
313 /* Returns the number of variables in D. */
315 dict_get_var_cnt (const struct dictionary *d)
320 /* Returns the variable in D with dictionary index IDX, which
321 must be between 0 and the count returned by
322 dict_get_var_cnt(), exclusive. */
324 dict_get_var (const struct dictionary *d, size_t idx)
326 assert (idx < d->var_cnt);
328 return d->var[idx].var;
331 /* Sets *VARS to an array of pointers to variables in D and *CNT
332 to the number of variables in *D. All variables are returned
333 except for those, if any, in the classes indicated by EXCLUDE.
334 (There is no point in putting DC_SYSTEM in EXCLUDE as
335 dictionaries never include system variables.) */
337 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
338 size_t *cnt, enum dict_class exclude)
340 dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
343 /* Sets *VARS to an array of pointers to variables in D and *CNT
344 to the number of variables in *D. All variables are returned
345 except for those, if any, in the classes indicated by EXCLUDE.
346 (There is no point in putting DC_SYSTEM in EXCLUDE as
347 dictionaries never include system variables.) */
349 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
350 size_t *cnt, enum dict_class exclude)
355 assert (exclude == (exclude & DC_ALL));
358 for (i = 0; i < d->var_cnt; i++)
360 enum dict_class class = var_get_dict_class (d->var[i].var);
361 if (!(class & exclude))
365 *vars = xnmalloc (count, sizeof **vars);
367 for (i = 0; i < d->var_cnt; i++)
369 enum dict_class class = var_get_dict_class (d->var[i].var);
370 if (!(class & exclude))
371 (*vars)[(*cnt)++] = d->var[i].var;
373 assert (*cnt == count);
376 static struct variable *
377 add_var (struct dictionary *d, struct variable *v)
379 struct vardict_info *vardict;
381 /* Update dictionary. */
382 if (d->var_cnt >= d->var_cap)
386 d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
387 hmap_clear (&d->name_map);
388 for (i = 0; i < d->var_cnt; i++)
390 var_set_vardict (d->var[i].var, &d->var[i]);
391 hmap_insert_fast (&d->name_map, &d->var[i].name_node,
392 d->var[i].name_node.hash);
396 vardict = &d->var[d->var_cnt++];
399 hmap_insert (&d->name_map, &vardict->name_node,
400 hash_case_string (var_get_name (v), 0));
401 vardict->case_index = d->next_value_idx;
402 var_set_vardict (v, vardict);
404 if ( d->changed ) d->changed (d, d->changed_data);
405 if ( d->callbacks && d->callbacks->var_added )
406 d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
409 invalidate_proto (d);
414 /* Creates and returns a new variable in D with the given NAME
415 and WIDTH. Returns a null pointer if the given NAME would
416 duplicate that of an existing variable in the dictionary. */
418 dict_create_var (struct dictionary *d, const char *name, int width)
420 return (dict_lookup_var (d, name) == NULL
421 ? dict_create_var_assert (d, name, width)
425 /* Creates and returns a new variable in D with the given NAME
426 and WIDTH. Assert-fails if the given NAME would duplicate
427 that of an existing variable in the dictionary. */
429 dict_create_var_assert (struct dictionary *d, const char *name, int width)
431 assert (dict_lookup_var (d, name) == NULL);
432 return add_var (d, var_create (name, width));
435 /* Creates and returns a new variable in D, as a copy of existing variable
436 OLD_VAR, which need not be in D or in any dictionary. Returns a null
437 pointer if OLD_VAR's name would duplicate that of an existing variable in
440 dict_clone_var (struct dictionary *d, const struct variable *old_var)
442 return dict_clone_var_as (d, old_var, var_get_name (old_var));
445 /* Creates and returns a new variable in D, as a copy of existing variable
446 OLD_VAR, which need not be in D or in any dictionary. Assert-fails if
447 OLD_VAR's name would duplicate that of an existing variable in the
450 dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
452 return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
455 /* Creates and returns a new variable in D with name NAME, as a copy of
456 existing variable OLD_VAR, which need not be in D or in any dictionary.
457 Returns a null pointer if the given NAME would duplicate that of an existing
458 variable in the dictionary. */
460 dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
463 return (dict_lookup_var (d, name) == NULL
464 ? dict_clone_var_as_assert (d, old_var, name)
468 /* Creates and returns a new variable in D with name NAME, as a copy of
469 existing variable OLD_VAR, which need not be in D or in any dictionary.
470 Assert-fails if the given NAME would duplicate that of an existing variable
471 in the dictionary. */
473 dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
476 struct variable *new_var = var_clone (old_var);
477 assert (dict_lookup_var (d, name) == NULL);
478 var_set_name (new_var, name);
479 return add_var (d, new_var);
482 /* Returns the variable named NAME in D, or a null pointer if no
483 variable has that name. */
485 dict_lookup_var (const struct dictionary *d, const char *name)
487 struct vardict_info *vardict;
489 HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
490 hash_case_string (name, 0), &d->name_map)
492 struct variable *var = vardict->var;
493 if (!strcasecmp (var_get_name (var), name))
500 /* Returns the variable named NAME in D. Assert-fails if no
501 variable has that name. */
503 dict_lookup_var_assert (const struct dictionary *d, const char *name)
505 struct variable *v = dict_lookup_var (d, name);
510 /* Returns true if variable V is in dictionary D,
513 dict_contains_var (const struct dictionary *d, const struct variable *v)
515 return (var_has_vardict (v)
516 && vardict_get_dictionary (var_get_vardict (v)) == d);
519 /* Compares two double pointers to variables, which should point
520 to elements of a struct dictionary's `var' member array. */
522 compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
524 struct variable *const *a = a_;
525 struct variable *const *b = b_;
527 return *a < *b ? -1 : *a > *b;
531 unindex_var (struct dictionary *d, struct vardict_info *vardict)
533 hmap_delete (&d->name_map, &vardict->name_node);
536 /* This function assumes that vardict->name_node.hash is valid, that is, that
537 its name has not changed since it was hashed (rename_var() updates this
538 hash along with the name itself). */
540 reindex_var (struct dictionary *d, struct vardict_info *vardict)
542 struct variable *var = vardict->var;
544 var_set_vardict (var, vardict);
545 hmap_insert_fast (&d->name_map, &vardict->name_node,
546 vardict->name_node.hash);
548 if ( d->changed ) d->changed (d, d->changed_data);
549 if ( d->callbacks && d->callbacks->var_changed )
550 d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
553 /* Sets the case_index in V's vardict to CASE_INDEX. */
555 set_var_case_index (struct variable *v, int case_index)
557 var_get_vardict (v)->case_index = case_index;
560 /* Removes the dictionary variables with indexes from FROM to TO (exclusive)
563 unindex_vars (struct dictionary *d, size_t from, size_t to)
567 for (i = from; i < to; i++)
568 unindex_var (d, &d->var[i]);
571 /* Re-sets the dict_index in the dictionary variables with
572 indexes from FROM to TO (exclusive). */
574 reindex_vars (struct dictionary *d, size_t from, size_t to)
578 for (i = from; i < to; i++)
579 reindex_var (d, &d->var[i]);
582 /* Deletes variable V from dictionary D and frees V.
584 This is a very bad idea if there might be any pointers to V
585 from outside D. In general, no variable in the active dataset's
586 dictionary should be deleted when any transformations are
587 active on the dictionary's dataset, because those
588 transformations might reference the deleted variable. The
589 safest time to delete a variable is just after a procedure has
590 been executed, as done by DELETE VARIABLES.
592 Pointers to V within D are not a problem, because
593 dict_delete_var() knows to remove V from split variables,
594 weights, filters, etc. */
596 dict_delete_var (struct dictionary *d, struct variable *v)
598 int dict_index = var_get_dict_index (v);
599 const int case_index = var_get_case_index (v);
600 const int width = var_get_width (v);
602 assert (dict_contains_var (d, v));
604 /* Delete aux data. */
607 dict_unset_split_var (d, v);
608 dict_unset_mrset_var (d, v);
611 dict_set_weight (d, NULL);
614 dict_set_filter (d, NULL);
616 dict_clear_vectors (d);
618 /* Remove V from var array. */
619 unindex_vars (d, dict_index, d->var_cnt);
620 remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
623 /* Update dict_index for each affected variable. */
624 reindex_vars (d, dict_index, d->var_cnt);
627 var_clear_vardict (v);
630 if ( d->changed ) d->changed (d, d->changed_data);
632 invalidate_proto (d);
633 if (d->callbacks && d->callbacks->var_deleted )
634 d->callbacks->var_deleted (d, dict_index, case_index, width, d->cb_data);
637 /* Deletes the COUNT variables listed in VARS from D. This is
638 unsafe; see the comment on dict_delete_var() for details. */
640 dict_delete_vars (struct dictionary *d,
641 struct variable *const *vars, size_t count)
643 /* FIXME: this can be done in O(count) time, but this algorithm
645 assert (count == 0 || vars != NULL);
648 dict_delete_var (d, *vars++);
651 /* Deletes the COUNT variables in D starting at index IDX. This
652 is unsafe; see the comment on dict_delete_var() for
655 dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
657 /* FIXME: this can be done in O(count) time, but this algorithm
659 assert (idx + count <= d->var_cnt);
662 dict_delete_var (d, d->var[idx].var);
665 /* Deletes scratch variables from dictionary D. */
667 dict_delete_scratch_vars (struct dictionary *d)
671 /* FIXME: this can be done in O(count) time, but this algorithm
673 for (i = 0; i < d->var_cnt; )
674 if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
675 dict_delete_var (d, d->var[i].var);
680 /* Moves V to 0-based position IDX in D. Other variables in D,
681 if any, retain their relative positions. Runs in time linear
682 in the distance moved. */
684 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
686 size_t old_index = var_get_dict_index (v);
688 assert (new_index < d->var_cnt);
690 unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
691 move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
692 reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
695 /* Reorders the variables in D, placing the COUNT variables
696 listed in ORDER in that order at the beginning of D. The
697 other variables in D, if any, retain their relative
700 dict_reorder_vars (struct dictionary *d,
701 struct variable *const *order, size_t count)
703 struct vardict_info *new_var;
706 assert (count == 0 || order != NULL);
707 assert (count <= d->var_cnt);
709 new_var = xnmalloc (d->var_cap, sizeof *new_var);
711 /* Add variables in ORDER to new_var. */
712 for (i = 0; i < count; i++)
714 struct vardict_info *old_var;
716 assert (dict_contains_var (d, order[i]));
718 old_var = var_get_vardict (order[i]);
719 new_var[i] = *old_var;
720 old_var->dict = NULL;
723 /* Add remaining variables to new_var. */
724 for (i = 0; i < d->var_cnt; i++)
725 if (d->var[i].dict != NULL)
726 new_var[count++] = d->var[i];
727 assert (count == d->var_cnt);
729 /* Replace old vardicts by new ones. */
733 hmap_clear (&d->name_map);
734 reindex_vars (d, 0, d->var_cnt);
737 /* Changes the name of variable V that is currently in a dictionary to
740 rename_var (struct variable *v, const char *new_name)
742 struct vardict_info *vardict = var_get_vardict (v);
743 var_clear_vardict (v);
744 var_set_name (v, new_name);
745 vardict->name_node.hash = hash_case_string (new_name, 0);
746 var_set_vardict (v, vardict);
749 /* Changes the name of V in D to name NEW_NAME. Assert-fails if
750 a variable named NEW_NAME is already in D, except that
751 NEW_NAME may be the same as V's existing name. */
753 dict_rename_var (struct dictionary *d, struct variable *v,
754 const char *new_name)
756 assert (!strcasecmp (var_get_name (v), new_name)
757 || dict_lookup_var (d, new_name) == NULL);
759 unindex_var (d, var_get_vardict (v));
760 rename_var (v, new_name);
761 reindex_var (d, var_get_vardict (v));
763 if (settings_get_algorithm () == ENHANCED)
764 var_clear_short_names (v);
766 if ( d->changed ) d->changed (d, d->changed_data);
767 if ( d->callbacks && d->callbacks->var_changed )
768 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
771 /* Renames COUNT variables specified in VARS to the names given
772 in NEW_NAMES within dictionary D. If the renaming would
773 result in a duplicate variable name, returns false and stores a
774 name that would be duplicated into *ERR_NAME (if ERR_NAME is
775 non-null). Otherwise, the renaming is successful, and true
778 dict_rename_vars (struct dictionary *d,
779 struct variable **vars, char **new_names, size_t count,
786 assert (count == 0 || vars != NULL);
787 assert (count == 0 || new_names != NULL);
789 /* Save the names of the variables to be renamed. */
790 pool = pool_create ();
791 old_names = pool_nalloc (pool, count, sizeof *old_names);
792 for (i = 0; i < count; i++)
793 old_names[i] = pool_strdup (pool, var_get_name (vars[i]));
795 /* Remove the variables to be renamed from the name hash,
797 for (i = 0; i < count; i++)
799 unindex_var (d, var_get_vardict (vars[i]));
800 rename_var (vars[i], new_names[i]);
803 /* Add the renamed variables back into the name hash,
804 checking for conflicts. */
805 for (i = 0; i < count; i++)
807 if (dict_lookup_var (d, var_get_name (vars[i])) != NULL)
809 /* There is a name conflict.
810 Back out all the name changes that have already
811 taken place, and indicate failure. */
813 if (err_name != NULL)
814 *err_name = new_names[i];
816 for (i = 0; i < fail_idx; i++)
817 unindex_var (d, var_get_vardict (vars[i]));
819 for (i = 0; i < count; i++)
821 rename_var (vars[i], old_names[i]);
822 reindex_var (d, var_get_vardict (vars[i]));
828 reindex_var (d, var_get_vardict (vars[i]));
831 /* Clear short names. */
832 if (settings_get_algorithm () == ENHANCED)
833 for (i = 0; i < count; i++)
834 var_clear_short_names (vars[i]);
840 /* Returns true if a variable named NAME may be inserted in DICT;
841 that is, if there is not already a variable with that name in
842 DICT and if NAME is not a reserved word. (The caller's checks
843 have already verified that NAME is otherwise acceptable as a
846 var_name_is_insertable (const struct dictionary *dict, const char *name)
848 return (dict_lookup_var (dict, name) == NULL
849 && lex_id_to_token (ss_cstr (name)) == T_ID);
853 make_hinted_name (const struct dictionary *dict, const char *hint)
855 size_t hint_len = strlen (hint);
856 bool dropped = false;
861 /* The allocation size here is OK: characters that are copied directly fit
862 OK, and characters that are not copied directly are replaced by a single
863 '_' byte. If u8_mbtouc() replaces bad input by 0xfffd, then that will get
864 replaced by '_' too. */
865 root = rp = xmalloc (hint_len + 1);
866 for (ofs = 0; ofs < hint_len; ofs += mblen)
870 mblen = u8_mbtouc (&uc, CHAR_CAST (const uint8_t *, hint + ofs),
873 ? lex_uc_is_id1 (uc) && uc != '$'
874 : lex_uc_is_idn (uc))
881 rp += u8_uctomb (CHAR_CAST (uint8_t *, rp), uc, 6);
892 if (var_name_is_insertable (dict, root))
895 for (i = 0; i < ULONG_MAX; i++)
897 char suffix[INT_BUFSIZE_BOUND (i) + 1];
901 if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
904 name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
905 if (var_name_is_insertable (dict, name))
920 make_numeric_name (const struct dictionary *dict, unsigned long int *num_start)
922 unsigned long int number;
924 for (number = num_start != NULL ? MAX (*num_start, 1) : 1;
928 char name[3 + INT_STRLEN_BOUND (number) + 1];
930 sprintf (name, "VAR%03lu", number);
931 if (dict_lookup_var (dict, name) == NULL)
933 if (num_start != NULL)
934 *num_start = number + 1;
935 return xstrdup (name);
943 /* Devises and returns a variable name unique within DICT. The variable name
944 is owned by the caller, which must free it with free() when it is no longer
947 HINT, if it is non-null, is used as a suggestion that will be
948 modified for suitability as a variable name and for
951 If HINT is null or entirely unsuitable, a name in the form
952 "VAR%03d" will be generated, where the smallest unused integer
953 value is used. If NUM_START is non-null, then its value is
954 used as the minimum numeric value to check, and it is updated
955 to the next value to be checked.
958 dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
959 unsigned long int *num_start)
963 char *hinted_name = make_hinted_name (dict, hint);
964 if (hinted_name != NULL)
967 return make_numeric_name (dict, num_start);
970 /* Returns the weighting variable in dictionary D, or a null
971 pointer if the dictionary is unweighted. */
973 dict_get_weight (const struct dictionary *d)
975 assert (d->weight == NULL || dict_contains_var (d, d->weight));
980 /* Returns the value of D's weighting variable in case C, except
981 that a negative weight is returned as 0. Returns 1 if the
982 dictionary is unweighted. Will warn about missing, negative,
983 or zero values if *WARN_ON_INVALID is true. The function will
984 set *WARN_ON_INVALID to false if an invalid weight is
987 dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
988 bool *warn_on_invalid)
992 if (d->weight == NULL)
996 double w = case_num (c, d->weight);
997 if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
999 if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
1000 *warn_on_invalid = false;
1001 msg (SW, _("At least one case in the data file had a weight value "
1002 "that was user-missing, system-missing, zero, or "
1003 "negative. These case(s) were ignored."));
1009 /* Sets the weighting variable of D to V, or turning off
1010 weighting if V is a null pointer. */
1012 dict_set_weight (struct dictionary *d, struct variable *v)
1014 assert (v == NULL || dict_contains_var (d, v));
1015 assert (v == NULL || var_is_numeric (v));
1019 if (d->changed) d->changed (d, d->changed_data);
1020 if ( d->callbacks && d->callbacks->weight_changed )
1021 d->callbacks->weight_changed (d,
1022 v ? var_get_dict_index (v) : -1,
1026 /* Returns the filter variable in dictionary D (see cmd_filter())
1027 or a null pointer if the dictionary is unfiltered. */
1029 dict_get_filter (const struct dictionary *d)
1031 assert (d->filter == NULL || dict_contains_var (d, d->filter));
1036 /* Sets V as the filter variable for dictionary D. Passing a
1037 null pointer for V turn off filtering. */
1039 dict_set_filter (struct dictionary *d, struct variable *v)
1041 assert (v == NULL || dict_contains_var (d, v));
1042 assert (v == NULL || var_is_numeric (v));
1046 if (d->changed) d->changed (d, d->changed_data);
1047 if ( d->callbacks && d->callbacks->filter_changed )
1048 d->callbacks->filter_changed (d,
1049 v ? var_get_dict_index (v) : -1,
1053 /* Returns the case limit for dictionary D, or zero if the number
1054 of cases is unlimited. */
1056 dict_get_case_limit (const struct dictionary *d)
1058 return d->case_limit;
1061 /* Sets CASE_LIMIT as the case limit for dictionary D. Use
1062 0 for CASE_LIMIT to indicate no limit. */
1064 dict_set_case_limit (struct dictionary *d, casenumber case_limit)
1066 d->case_limit = case_limit;
1069 /* Returns the prototype used for cases created by dictionary D. */
1070 const struct caseproto *
1071 dict_get_proto (const struct dictionary *d_)
1073 struct dictionary *d = CONST_CAST (struct dictionary *, d_);
1074 if (d->proto == NULL)
1078 d->proto = caseproto_create ();
1079 d->proto = caseproto_reserve (d->proto, d->var_cnt);
1080 for (i = 0; i < d->var_cnt; i++)
1081 d->proto = caseproto_set_width (d->proto,
1082 var_get_case_index (d->var[i].var),
1083 var_get_width (d->var[i].var));
1088 /* Returns the case index of the next value to be added to D.
1089 This value is the number of `union value's that need to be
1090 allocated to store a case for dictionary D. */
1092 dict_get_next_value_idx (const struct dictionary *d)
1094 return d->next_value_idx;
1097 /* Returns the number of bytes needed to store a case for
1100 dict_get_case_size (const struct dictionary *d)
1102 return sizeof (union value) * dict_get_next_value_idx (d);
1105 /* Reassigns values in dictionary D so that fragmentation is
1108 dict_compact_values (struct dictionary *d)
1112 d->next_value_idx = 0;
1113 for (i = 0; i < d->var_cnt; i++)
1115 struct variable *v = d->var[i].var;
1116 set_var_case_index (v, d->next_value_idx++);
1118 invalidate_proto (d);
1121 /* Returns the number of values occupied by the variables in
1122 dictionary D. All variables are considered if EXCLUDE_CLASSES
1123 is 0, or it may contain one or more of (1u << DC_ORDINARY),
1124 (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
1125 corresponding type of variable.
1127 The return value may be less than the number of values in one
1128 of dictionary D's cases (as returned by
1129 dict_get_next_value_idx) even if E is 0, because there may be
1130 gaps in D's cases due to deleted variables. */
1132 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
1137 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1139 | (1u << DC_SCRATCH))) == 0);
1142 for (i = 0; i < d->var_cnt; i++)
1144 enum dict_class class = var_get_dict_class (d->var[i].var);
1145 if (!(exclude_classes & (1u << class)))
1151 /* Returns the case prototype that would result after deleting
1152 all variables from D that are not in one of the
1153 EXCLUDE_CLASSES and compacting the dictionary with
1156 The caller must unref the returned caseproto when it is no
1159 dict_get_compacted_proto (const struct dictionary *d,
1160 unsigned int exclude_classes)
1162 struct caseproto *proto;
1165 assert ((exclude_classes & ~((1u << DC_ORDINARY)
1167 | (1u << DC_SCRATCH))) == 0);
1169 proto = caseproto_create ();
1170 for (i = 0; i < d->var_cnt; i++)
1172 struct variable *v = d->var[i].var;
1173 if (!(exclude_classes & (1u << var_get_dict_class (v))))
1174 proto = caseproto_add_width (proto, var_get_width (v));
1179 /* Returns the SPLIT FILE vars (see cmd_split_file()). Call
1180 dict_get_split_cnt() to determine how many SPLIT FILE vars
1181 there are. Returns a null pointer if and only if there are no
1183 const struct variable *const *
1184 dict_get_split_vars (const struct dictionary *d)
1189 /* Returns the number of SPLIT FILE vars. */
1191 dict_get_split_cnt (const struct dictionary *d)
1193 return d->split_cnt;
1196 /* Removes variable V, which must be in D, from D's set of split
1199 dict_unset_split_var (struct dictionary *d, struct variable *v)
1203 assert (dict_contains_var (d, v));
1205 orig_count = d->split_cnt;
1206 d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
1207 &v, compare_var_ptrs, NULL);
1208 if (orig_count != d->split_cnt)
1210 if (d->changed) d->changed (d, d->changed_data);
1211 /* We changed the set of split variables so invoke the
1213 if (d->callbacks && d->callbacks->split_changed)
1214 d->callbacks->split_changed (d, d->cb_data);
1218 /* Sets CNT split vars SPLIT in dictionary D. */
1220 dict_set_split_vars (struct dictionary *d,
1221 struct variable *const *split, size_t cnt)
1223 assert (cnt == 0 || split != NULL);
1228 d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
1229 memcpy (d->split, split, cnt * sizeof *d->split);
1237 if (d->changed) d->changed (d, d->changed_data);
1238 if ( d->callbacks && d->callbacks->split_changed )
1239 d->callbacks->split_changed (d, d->cb_data);
1242 /* Returns the file label for D, or a null pointer if D is
1243 unlabeled (see cmd_file_label()). */
1245 dict_get_label (const struct dictionary *d)
1250 /* Sets D's file label to LABEL, truncating it to a maximum of 60
1253 Removes D's label if LABEL is null or the empty string. */
1255 dict_set_label (struct dictionary *d, const char *label)
1258 d->label = label != NULL && label[0] != '\0' ? xstrndup (label, 60) : NULL;
1261 /* Returns the documents for D, as an UTF-8 encoded string_array. The
1262 return value is always nonnull; if there are no documents then the
1263 string_arary is empty.*/
1264 const struct string_array *
1265 dict_get_documents (const struct dictionary *d)
1267 return &d->documents;
1270 /* Replaces the documents for D by NEW_DOCS, a UTF-8 encoded string_array. */
1272 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
1276 dict_clear_documents (d);
1278 for (i = 0; i < new_docs->n; i++)
1279 dict_add_document_line (d, new_docs->strings[i], false);
1282 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
1283 into individual lines at new-line characters. Each line is truncated to at
1284 most DOC_LINE_LENGTH bytes in D's encoding. */
1286 dict_set_documents_string (struct dictionary *d, const char *new_docs)
1290 dict_clear_documents (d);
1291 for (s = new_docs; *s != '\0'; )
1293 size_t len = strcspn (s, "\n");
1294 char *line = xmemdup0 (s, len);
1295 dict_add_document_line (d, line, false);
1304 /* Drops the documents from dictionary D. */
1306 dict_clear_documents (struct dictionary *d)
1308 string_array_clear (&d->documents);
1311 /* Appends the UTF-8 encoded LINE to the documents in D. LINE will be
1312 truncated so that it is no more than 80 bytes in the dictionary's
1313 encoding. If this causes some text to be lost, and ISSUE_WARNING is true,
1314 then a warning will be issued. */
1316 dict_add_document_line (struct dictionary *d, const char *line,
1322 trunc_len = utf8_encoding_trunc_len (line, d->encoding, DOC_LINE_LENGTH);
1323 truncated = line[trunc_len] != '\0';
1324 if (truncated && issue_warning)
1326 /* Note to translators: "bytes" is correct, not characters */
1327 msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
1330 string_array_append_nocopy (&d->documents, xmemdup0 (line, trunc_len));
1335 /* Returns the number of document lines in dictionary D. */
1337 dict_get_document_line_cnt (const struct dictionary *d)
1339 return d->documents.n;
1342 /* Returns document line number IDX in dictionary D. The caller must not
1343 modify or free the returned string. */
1345 dict_get_document_line (const struct dictionary *d, size_t idx)
1347 assert (idx < d->documents.n);
1348 return d->documents.strings[idx];
1351 /* Creates in D a vector named NAME that contains the CNT
1352 variables in VAR. Returns true if successful, or false if a
1353 vector named NAME already exists in D. */
1355 dict_create_vector (struct dictionary *d,
1357 struct variable **var, size_t cnt)
1362 for (i = 0; i < cnt; i++)
1363 assert (dict_contains_var (d, var[i]));
1365 if (dict_lookup_vector (d, name) == NULL)
1367 d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
1368 d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
1375 /* Creates in D a vector named NAME that contains the CNT
1376 variables in VAR. A vector named NAME must not already exist
1379 dict_create_vector_assert (struct dictionary *d,
1381 struct variable **var, size_t cnt)
1383 assert (dict_lookup_vector (d, name) == NULL);
1384 dict_create_vector (d, name, var, cnt);
1387 /* Returns the vector in D with index IDX, which must be less
1388 than dict_get_vector_cnt (D). */
1389 const struct vector *
1390 dict_get_vector (const struct dictionary *d, size_t idx)
1392 assert (idx < d->vector_cnt);
1394 return d->vector[idx];
1397 /* Returns the number of vectors in D. */
1399 dict_get_vector_cnt (const struct dictionary *d)
1401 return d->vector_cnt;
1404 /* Looks up and returns the vector within D with the given
1406 const struct vector *
1407 dict_lookup_vector (const struct dictionary *d, const char *name)
1410 for (i = 0; i < d->vector_cnt; i++)
1411 if (!strcasecmp (vector_get_name (d->vector[i]), name))
1412 return d->vector[i];
1416 /* Deletes all vectors from D. */
1418 dict_clear_vectors (struct dictionary *d)
1422 for (i = 0; i < d->vector_cnt; i++)
1423 vector_destroy (d->vector[i]);
1430 /* Multiple response sets. */
1432 /* Returns the multiple response set in DICT with index IDX, which must be
1433 between 0 and the count returned by dict_get_n_mrsets(), exclusive. */
1434 const struct mrset *
1435 dict_get_mrset (const struct dictionary *dict, size_t idx)
1437 assert (idx < dict->n_mrsets);
1438 return dict->mrsets[idx];
1441 /* Returns the number of multiple response sets in DICT. */
1443 dict_get_n_mrsets (const struct dictionary *dict)
1445 return dict->n_mrsets;
1448 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1449 returns its index; otherwise, returns SIZE_MAX. */
1451 dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
1455 for (i = 0; i < dict->n_mrsets; i++)
1456 if (!strcasecmp (name, dict->mrsets[i]->name))
1462 /* Looks for a multiple response set named NAME in DICT. If it finds one,
1463 returns it; otherwise, returns NULL. */
1464 const struct mrset *
1465 dict_lookup_mrset (const struct dictionary *dict, const char *name)
1467 size_t idx = dict_lookup_mrset_idx (dict, name);
1468 return idx != SIZE_MAX ? dict->mrsets[idx] : NULL;
1471 /* Adds MRSET to DICT, replacing any existing set with the same name. Returns
1472 true if a set was replaced, false if none existed with the specified name.
1474 Ownership of MRSET is transferred to DICT. */
1476 dict_add_mrset (struct dictionary *dict, struct mrset *mrset)
1480 assert (mrset_ok (mrset, dict));
1482 idx = dict_lookup_mrset_idx (dict, mrset->name);
1483 if (idx == SIZE_MAX)
1485 dict->mrsets = xrealloc (dict->mrsets,
1486 (dict->n_mrsets + 1) * sizeof *dict->mrsets);
1487 dict->mrsets[dict->n_mrsets++] = mrset;
1492 mrset_destroy (dict->mrsets[idx]);
1493 dict->mrsets[idx] = mrset;
1498 /* Looks for a multiple response set in DICT named NAME. If found, removes it
1499 from DICT and returns true. If none is found, returns false without
1502 Deleting one multiple response set causes the indexes of other sets within
1505 dict_delete_mrset (struct dictionary *dict, const char *name)
1507 size_t idx = dict_lookup_mrset_idx (dict, name);
1508 if (idx != SIZE_MAX)
1510 mrset_destroy (dict->mrsets[idx]);
1511 dict->mrsets[idx] = dict->mrsets[--dict->n_mrsets];
1518 /* Deletes all multiple response sets from DICT. */
1520 dict_clear_mrsets (struct dictionary *dict)
1524 for (i = 0; i < dict->n_mrsets; i++)
1525 mrset_destroy (dict->mrsets[i]);
1526 free (dict->mrsets);
1527 dict->mrsets = NULL;
1531 /* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
1533 dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
1537 assert (dict_contains_var (dict, var));
1539 for (i = 0; i < dict->n_mrsets; )
1541 struct mrset *mrset = dict->mrsets[i];
1544 for (j = 0; j < mrset->n_vars; )
1545 if (mrset->vars[j] == var)
1546 remove_element (mrset->vars, mrset->n_vars--,
1547 sizeof *mrset->vars, j);
1551 if (mrset->n_vars < 2)
1553 mrset_destroy (mrset);
1554 dict->mrsets[i] = dict->mrsets[--dict->n_mrsets];
1561 /* Returns D's attribute set. The caller may examine or modify
1562 the attribute set, but must not destroy it. Destroying D or
1563 calling dict_set_attributes for D will also destroy D's
1566 dict_get_attributes (const struct dictionary *d)
1568 return CONST_CAST (struct attrset *, &d->attributes);
1571 /* Replaces D's attributes set by a copy of ATTRS. */
1573 dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
1575 attrset_destroy (&d->attributes);
1576 attrset_clone (&d->attributes, attrs);
1579 /* Returns true if D has at least one attribute in its attribute
1580 set, false if D's attribute set is empty. */
1582 dict_has_attributes (const struct dictionary *d)
1584 return attrset_count (&d->attributes) > 0;
1587 /* Called from variable.c to notify the dictionary that some property of
1588 the variable has changed */
1590 dict_var_changed (const struct variable *v)
1592 if ( var_has_vardict (v))
1594 const struct vardict_info *vardict = var_get_vardict (v);
1595 struct dictionary *d = vardict->dict;
1600 if (d->changed ) d->changed (d, d->changed_data);
1601 if ( d->callbacks && d->callbacks->var_changed )
1602 d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
1607 /* Called from variable.c to notify the dictionary that the variable's width
1610 dict_var_resized (const struct variable *v, int old_width)
1612 if ( var_has_vardict (v))
1614 const struct vardict_info *vardict = var_get_vardict (v);
1615 struct dictionary *d;
1619 if (d->changed) d->changed (d, d->changed_data);
1621 invalidate_proto (d);
1622 if ( d->callbacks && d->callbacks->var_resized )
1623 d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
1628 /* Called from variable.c to notify the dictionary that the variable's display width
1631 dict_var_display_width_changed (const struct variable *v)
1633 if ( var_has_vardict (v))
1635 const struct vardict_info *vardict = var_get_vardict (v);
1636 struct dictionary *d;
1640 if (d->changed) d->changed (d, d->changed_data);
1641 if ( d->callbacks && d->callbacks->var_display_width_changed )
1642 d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
1646 /* Dictionary used to contain "internal variables". */
1647 static struct dictionary *internal_dict;
1649 /* Create a variable of the specified WIDTH to be used for internal
1650 calculations only. The variable is assigned case index CASE_IDX. */
1652 dict_create_internal_var (int case_idx, int width)
1654 if (internal_dict == NULL)
1655 internal_dict = dict_create ("UTF-8");
1659 static int counter = INT_MAX / 2;
1660 struct variable *var;
1663 if (++counter == INT_MAX)
1664 counter = INT_MAX / 2;
1666 sprintf (name, "$internal%d", counter);
1667 var = dict_create_var (internal_dict, name, width);
1670 set_var_case_index (var, case_idx);
1676 /* Destroys VAR, which must have been created with
1677 dict_create_internal_var(). */
1679 dict_destroy_internal_var (struct variable *var)
1683 dict_delete_var (internal_dict, var);
1685 /* Destroy internal_dict if it has no variables left, just so that
1686 valgrind --leak-check --show-reachable won't show internal_dict. */
1687 if (dict_get_var_cnt (internal_dict) == 0)
1689 dict_destroy (internal_dict);
1690 internal_dict = NULL;
1696 vardict_get_dict_index (const struct vardict_info *vardict)
1698 return vardict - vardict->dict->var;