X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fdictionary.c;h=4fbcdf2a1e16f237b2988b10d80800a3752c7343;hb=b4062aaae1b2686f7d4ece15420550519e9a9823;hp=40352b0f2ba9e97708fb21c8aec422c75dddb4b7;hpb=96994a54e60e9c95b8bba54c2281acf7059b1203;p=pspp

diff --git a/src/data/dictionary.c b/src/data/dictionary.c
index 40352b0f2b..4fbcdf2a1e 100644
--- a/src/data/dictionary.c
+++ b/src/data/dictionary.c
@@ -1,5 +1,6 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014,
+   2015, 2020 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -91,9 +92,83 @@ struct dictionary
     void *changed_data;
   };
 
-static void dict_unset_split_var (struct dictionary *, struct variable *);
+static void dict_unset_split_var (struct dictionary *, struct variable *, bool);
 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
 
+/* Compares two double pointers to variables, which should point
+   to elements of a struct dictionary's `var' member array. */
+static int
+compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
+{
+  struct variable *const *a = a_;
+  struct variable *const *b = b_;
+
+  return *a < *b ? -1 : *a > *b;
+}
+
+static void
+unindex_var (struct dictionary *d, struct vardict_info *vardict)
+{
+  hmap_delete (&d->name_map, &vardict->name_node);
+}
+
+/* This function assumes that vardict->name_node.hash is valid, that is, that
+   its name has not changed since it was hashed (rename_var() updates this
+   hash along with the name itself). */
+static void
+reindex_var (struct dictionary *d, struct vardict_info *vardict, bool skip_callbacks)
+{
+  struct variable *old = (d->callbacks && d->callbacks->var_changed
+                          ? var_clone (vardict->var)
+                          : NULL);
+
+  struct variable *var = vardict->var;
+  var_set_vardict (var, vardict);
+  hmap_insert_fast (&d->name_map, &vardict->name_node,
+                    vardict->name_node.hash);
+
+  if (! skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (old)
+        {
+          d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
+          var_unref (old);
+        }
+    }
+}
+
+/* Sets the case_index in V's vardict to CASE_INDEX. */
+static void
+set_var_case_index (struct variable *v, int case_index)
+{
+  var_get_vardict (v)->case_index = case_index;
+}
+
+/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
+   from name_map. */
+static void
+unindex_vars (struct dictionary *d, size_t from, size_t to)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    unindex_var (d, &d->var[i]);
+}
+
+/* Re-sets the dict_index in the dictionary variables with
+   indexes from FROM to TO (exclusive). */
+static void
+reindex_vars (struct dictionary *d, size_t from, size_t to, bool skip_callbacks)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    reindex_var (d, &d->var[i], skip_callbacks);
+}
+
+
+
 /* Returns the encoding for data in dictionary D.  The return value is a
    nonnull string that contains an IANA character set name. */
 const char *
@@ -175,7 +250,7 @@ dict_copy_callbacks (struct dictionary *dest,
 struct dictionary *
 dict_create (const char *encoding)
 {
-  struct dictionary *d = xzalloc (sizeof *d);
+  struct dictionary *d = XZALLOC (struct dictionary);
 
   d->encoding = xstrdup (encoding);
   d->names_must_be_ids = true;
@@ -186,13 +261,6 @@ dict_create (const char *encoding)
   return d;
 }
 
-struct dictionary *
-dict_ref (struct dictionary *s)
-{
-  s->ref_cnt++;
-  return s;
-}
-
 /* Creates and returns a (deep) copy of an existing
    dictionary.
 
@@ -268,506 +336,564 @@ dict_clone (const struct dictionary *s)
   return d;
 }
 
-/* Clears the contents from a dictionary without destroying the
-   dictionary itself. */
-void
-dict_clear (struct dictionary *d)
+
+
+/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
+   dict_get_split_cnt() to determine how many SPLIT FILE vars
+   there are.  Returns a null pointer if and only if there are no
+   SPLIT FILE vars. */
+const struct variable *const *
+dict_get_split_vars (const struct dictionary *d)
 {
-  /* FIXME?  Should we really clear case_limit, label, documents?
-     Others are necessarily cleared by deleting all the variables.*/
-  while (d->var_cnt > 0)
-    {
-      dict_delete_var (d, d->var[d->var_cnt - 1].var);
-    }
+  return d->split;
+}
 
-  free (d->var);
-  d->var = NULL;
-  d->var_cnt = d->var_cap = 0;
-  invalidate_proto (d);
-  hmap_clear (&d->name_map);
-  d->next_value_idx = 0;
-  dict_set_split_vars (d, NULL, 0);
-  dict_set_weight (d, NULL);
-  dict_set_filter (d, NULL);
-  d->case_limit = 0;
-  free (d->label);
-  d->label = NULL;
-  string_array_clear (&d->documents);
-  dict_clear_vectors (d);
-  attrset_clear (&d->attributes);
+/* Returns the number of SPLIT FILE vars. */
+size_t
+dict_get_split_cnt (const struct dictionary *d)
+{
+  return d->split_cnt;
 }
 
-/* Clears a dictionary and destroys it. */
+/* Removes variable V, which must be in D, from D's set of split
+   variables. */
 static void
-_dict_destroy (struct dictionary *d)
+dict_unset_split_var (struct dictionary *d, struct variable *v, bool skip_callbacks)
 {
-  /* In general, we don't want callbacks occurring, if the dictionary
-     is being destroyed */
-  d->callbacks  = NULL ;
+  int orig_count;
 
-  dict_clear (d);
-  string_array_destroy (&d->documents);
-  hmap_destroy (&d->name_map);
-  attrset_destroy (&d->attributes);
-  dict_clear_mrsets (d);
-  free (d->encoding);
-  free (d);
-}
+  assert (dict_contains_var (d, v));
 
-void
-dict_unref (struct dictionary *d)
-{
-  if (d == NULL)
-    return;
-  d->ref_cnt--;
-  assert (d->ref_cnt >= 0);
-  if (d->ref_cnt == 0)
-    _dict_destroy (d);
+  orig_count = d->split_cnt;
+  d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
+                               &v, compare_var_ptrs, NULL);
+  if (orig_count != d->split_cnt && !skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      /* We changed the set of split variables so invoke the
+         callback. */
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
 }
 
-/* Returns the number of variables in D. */
-size_t
-dict_get_var_cnt (const struct dictionary *d)
-{
-  return d->var_cnt;
-}
 
-/* Returns the variable in D with dictionary index IDX, which
-   must be between 0 and the count returned by
-   dict_get_var_cnt(), exclusive. */
-struct variable *
-dict_get_var (const struct dictionary *d, size_t idx)
+/* Sets CNT split vars SPLIT in dictionary D. */
+static void
+dict_set_split_vars__ (struct dictionary *d,
+                       struct variable *const *split, size_t cnt, bool skip_callbacks)
 {
-  assert (idx < d->var_cnt);
+  assert (cnt == 0 || split != NULL);
 
-  return d->var[idx].var;
-}
+  d->split_cnt = cnt;
+  if (cnt > 0)
+   {
+    d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
+    memcpy (d->split, split, cnt * sizeof *d->split);
+   }
+  else
+   {
+    free (d->split);
+    d->split = NULL;
+   }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
-   to the number of variables in *D.  All variables are returned
-   except for those, if any, in the classes indicated by EXCLUDE.
-   (There is no point in putting DC_SYSTEM in EXCLUDE as
-   dictionaries never include system variables.) */
-void
-dict_get_vars (const struct dictionary *d, const struct variable ***vars,
-               size_t *cnt, enum dict_class exclude)
-{
-  dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
+ if (!skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
 }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
-   to the number of variables in *D.  All variables are returned
-   except for those, if any, in the classes indicated by EXCLUDE.
-   (There is no point in putting DC_SYSTEM in EXCLUDE as
-   dictionaries never include system variables.) */
+/* Sets CNT split vars SPLIT in dictionary D. */
 void
-dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
-                       size_t *cnt, enum dict_class exclude)
+dict_set_split_vars (struct dictionary *d,
+                     struct variable *const *split, size_t cnt)
 {
-  size_t count;
-  size_t i;
+  dict_set_split_vars__ (d, split, cnt, false);
+}
 
-  assert (exclude == (exclude & DC_ALL));
+
 
-  count = 0;
-  for (i = 0; i < d->var_cnt; i++)
-    {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
-      if (!(class & exclude))
-        count++;
-    }
+/* Deletes variable V from dictionary D and frees V.
 
-  *vars = xnmalloc (count, sizeof **vars);
-  *cnt = 0;
-  for (i = 0; i < d->var_cnt; i++)
-    {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
-      if (!(class & exclude))
-        (*vars)[(*cnt)++] = d->var[i].var;
-    }
-  assert (*cnt == count);
-}
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
 
-static struct variable *
-add_var_with_case_index (struct dictionary *d, struct variable *v,
-                         int case_index)
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+static void
+dict_delete_var__ (struct dictionary *d, struct variable *v, bool skip_callbacks)
 {
-  struct vardict_info *vardict;
+  int dict_index = var_get_dict_index (v);
+  const int case_index = var_get_case_index (v);
 
-  assert (case_index >= d->next_value_idx);
+  assert (dict_contains_var (d, v));
 
-  /* Update dictionary. */
-  if (d->var_cnt >= d->var_cap)
-    {
-      size_t i;
+  dict_unset_split_var (d, v, skip_callbacks);
+  dict_unset_mrset_var (d, v);
 
-      d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
-      hmap_clear (&d->name_map);
-      for (i = 0; i < d->var_cnt; i++)
-        {
-          var_set_vardict (d->var[i].var, &d->var[i]);
-          hmap_insert_fast (&d->name_map, &d->var[i].name_node,
-                            d->var[i].name_node.hash);
-        }
-    }
+  if (d->weight == v)
+    dict_set_weight (d, NULL);
 
-  vardict = &d->var[d->var_cnt++];
-  vardict->dict = d;
-  vardict->var = v;
-  hmap_insert (&d->name_map, &vardict->name_node,
-               utf8_hash_case_string (var_get_name (v), 0));
-  vardict->case_index = case_index;
-  var_set_vardict (v, vardict);
+  if (d->filter == v)
+    dict_set_filter (d, NULL);
 
-  if (d->changed) d->changed (d, d->changed_data);
-  if (d->callbacks &&  d->callbacks->var_added)
-    d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
+  dict_clear_vectors (d);
 
-  invalidate_proto (d);
-  d->next_value_idx = case_index + 1;
+  /* Remove V from var array. */
+  unindex_vars (d, dict_index, d->var_cnt);
+  remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
+  d->var_cnt--;
 
-  return v;
-}
+  /* Update dict_index for each affected variable. */
+  reindex_vars (d, dict_index, d->var_cnt, skip_callbacks);
 
-static struct variable *
-add_var (struct dictionary *d, struct variable *v)
-{
-  return add_var_with_case_index (d, v, d->next_value_idx);
-}
+  /* Free memory. */
+  var_clear_vardict (v);
 
-/* Creates and returns a new variable in D with the given NAME
-   and WIDTH.  Returns a null pointer if the given NAME would
-   duplicate that of an existing variable in the dictionary. */
-struct variable *
-dict_create_var (struct dictionary *d, const char *name, int width)
-{
-  return (dict_lookup_var (d, name) == NULL
-          ? dict_create_var_assert (d, name, width)
-          : NULL);
-}
+  if (! skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->var_deleted)
+        d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
+    }
 
-/* Creates and returns a new variable in D with the given NAME
-   and WIDTH.  Assert-fails if the given NAME would duplicate
-   that of an existing variable in the dictionary. */
-struct variable *
-dict_create_var_assert (struct dictionary *d, const char *name, int width)
-{
-  assert (dict_lookup_var (d, name) == NULL);
-  return add_var (d, var_create (name, width));
+  invalidate_proto (d);
+  var_unref (v);
 }
 
-/* Creates and returns a new variable in D, as a copy of existing variable
-   OLD_VAR, which need not be in D or in any dictionary.  Returns a null
-   pointer if OLD_VAR's name would duplicate that of an existing variable in
-   the dictionary. */
-struct variable *
-dict_clone_var (struct dictionary *d, const struct variable *old_var)
-{
-  return dict_clone_var_as (d, old_var, var_get_name (old_var));
-}
+/* Deletes variable V from dictionary D and frees V.
 
-/* Creates and returns a new variable in D, as a copy of existing variable
-   OLD_VAR, which need not be in D or in any dictionary.  Assert-fails if
-   OLD_VAR's name would duplicate that of an existing variable in the
-   dictionary. */
-struct variable *
-dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
-{
-  return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
-}
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
 
-/* Creates and returns a new variable in D with name NAME, as a copy of
-   existing variable OLD_VAR, which need not be in D or in any dictionary.
-   Returns a null pointer if the given NAME would duplicate that of an existing
-   variable in the dictionary. */
-struct variable *
-dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
-                   const char *name)
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+void
+dict_delete_var (struct dictionary *d, struct variable *v)
 {
-  return (dict_lookup_var (d, name) == NULL
-          ? dict_clone_var_as_assert (d, old_var, name)
-          : NULL);
+  dict_delete_var__ (d, v, false);
 }
 
-/* Creates and returns a new variable in D with name NAME, as a copy of
-   existing variable OLD_VAR, which need not be in D or in any dictionary.
-   Assert-fails if the given NAME would duplicate that of an existing variable
-   in the dictionary. */
-struct variable *
-dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
-                          const char *name)
-{
-  struct variable *new_var = var_clone (old_var);
-  assert (dict_lookup_var (d, name) == NULL);
-  var_set_name (new_var, name);
-  return add_var (d, new_var);
-}
 
-struct variable *
-dict_clone_var_in_place_assert (struct dictionary *d,
-                                const struct variable *old_var)
+/* Deletes the COUNT variables listed in VARS from D.  This is
+   unsafe; see the comment on dict_delete_var() for details. */
+void
+dict_delete_vars (struct dictionary *d,
+                  struct variable *const *vars, size_t count)
 {
-  assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
-  return add_var_with_case_index (d, var_clone (old_var),
-                                  var_get_case_index (old_var));
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  assert (count == 0 || vars != NULL);
+
+  while (count-- > 0)
+    dict_delete_var (d, *vars++);
 }
 
-/* Returns the variable named NAME in D, or a null pointer if no
-   variable has that name. */
-struct variable *
-dict_lookup_var (const struct dictionary *d, const char *name)
+/* Deletes the COUNT variables in D starting at index IDX.  This
+   is unsafe; see the comment on dict_delete_var() for
+   details. Deleting consecutive vars will result in less callbacks
+   compared to iterating over dict_delete_var.
+   A simple while loop over dict_delete_var will
+   produce (d->var_cnt - IDX) * COUNT variable changed callbacks
+   plus COUNT variable delete callbacks.
+   This here produces d->var_cnt - IDX variable changed callbacks
+   plus COUNT variable delete callbacks. */
+void
+dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
 {
-  struct vardict_info *vardict;
+  assert (idx + count <= d->var_cnt);
 
-  HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
-                           utf8_hash_case_string (name, 0), &d->name_map)
+  /* We need to store the variable and the corresponding case_index
+     for the delete callbacks later. We store them in a linked list.*/
+  struct delvar {
+    struct ll ll;
+    struct variable *var;
+    int case_index;
+  };
+  struct ll_list list = LL_INITIALIZER (list);
+
+  for (size_t i = idx; i < idx + count; i++)
     {
-      struct variable *var = vardict->var;
-      if (!utf8_strcasecmp (var_get_name (var), name))
-        return var;
+      struct delvar *dv = xmalloc (sizeof (struct delvar));
+      assert (dv);
+      struct variable *v = d->var[i].var;
+
+      dict_unset_split_var (d, v, false);
+      dict_unset_mrset_var (d, v);
+
+      if (d->weight == v)
+	dict_set_weight (d, NULL);
+
+      if (d->filter == v)
+	dict_set_filter (d, NULL);
+
+      dv->var = v;
+      dv->case_index = var_get_case_index (v);
+      ll_push_tail (&list, (struct ll *)dv);
     }
 
-  return NULL;
-}
+  dict_clear_vectors (d);
 
-/* Returns the variable named NAME in D.  Assert-fails if no
-   variable has that name. */
-struct variable *
-dict_lookup_var_assert (const struct dictionary *d, const char *name)
-{
-  struct variable *v = dict_lookup_var (d, name);
-  assert (v != NULL);
-  return v;
-}
+  /* Remove variables from var array. */
+  unindex_vars (d, idx, d->var_cnt);
+  remove_range (d->var, d->var_cnt, sizeof *d->var, idx, count);
+  d->var_cnt -= count;
 
-/* Returns true if variable V is in dictionary D,
-   false otherwise. */
-bool
-dict_contains_var (const struct dictionary *d, const struct variable *v)
-{
-  return (var_has_vardict (v)
-          && vardict_get_dictionary (var_get_vardict (v)) == d);
-}
+  /* Reindexing will result variable-changed callback */
+  reindex_vars (d, idx, d->var_cnt, false);
 
-/* Compares two double pointers to variables, which should point
-   to elements of a struct dictionary's `var' member array. */
-static int
-compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
-{
-  struct variable *const *a = a_;
-  struct variable *const *b = b_;
+  invalidate_proto (d);
+  if (d->changed) d->changed (d, d->changed_data);
 
-  return *a < *b ? -1 : *a > *b;
+  /* Now issue the variable delete callbacks and delete
+     the variables. The vardict is not valid at this point
+     anymore. That is the reason why we stored the
+     caseindex before reindexing. */
+  for (size_t vi = idx; vi < idx + count; vi++)
+    {
+      struct delvar *dv = (struct delvar *) ll_pop_head (&list);
+      var_clear_vardict (dv->var);
+      if (d->callbacks &&  d->callbacks->var_deleted)
+        d->callbacks->var_deleted (d, dv->var, vi, dv->case_index, d->cb_data);
+      var_unref (dv->var);
+      free (dv);
+    }
 }
 
-static void
-unindex_var (struct dictionary *d, struct vardict_info *vardict)
+/* Deletes scratch variables from dictionary D. */
+void
+dict_delete_scratch_vars (struct dictionary *d)
 {
-  hmap_delete (&d->name_map, &vardict->name_node);
+  int i;
+
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  for (i = 0; i < d->var_cnt;)
+    if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
+      dict_delete_var (d, d->var[i].var);
+    else
+      i++;
 }
 
-/* This function assumes that vardict->name_node.hash is valid, that is, that
-   its name has not changed since it was hashed (rename_var() updates this
-   hash along with the name itself). */
+
+
+/* Clears the contents from a dictionary without destroying the
+   dictionary itself. */
 static void
-reindex_var (struct dictionary *d, struct vardict_info *vardict)
+dict_clear__ (struct dictionary *d, bool skip_callbacks)
 {
-  struct variable *old = (d->callbacks && d->callbacks->var_changed
-                          ? var_clone (vardict->var)
-                          : NULL);
+  /* FIXME?  Should we really clear case_limit, label, documents?
+     Others are necessarily cleared by deleting all the variables.*/
+  while (d->var_cnt > 0)
+    {
+      dict_delete_var__ (d, d->var[d->var_cnt - 1].var, skip_callbacks);
+    }
 
-  struct variable *var = vardict->var;
-  var_set_vardict (var, vardict);
-  hmap_insert_fast (&d->name_map, &vardict->name_node,
-                    vardict->name_node.hash);
+  free (d->var);
+  d->var = NULL;
+  d->var_cnt = d->var_cap = 0;
+  invalidate_proto (d);
+  hmap_clear (&d->name_map);
+  d->next_value_idx = 0;
+  dict_set_split_vars__ (d, NULL, 0, skip_callbacks);
 
-  if (d->changed) d->changed (d, d->changed_data);
-  if (old)
+  if (skip_callbacks)
+    {
+      d->weight = NULL;
+      d->filter = NULL;
+    }
+  else
     {
-      d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
-      var_destroy (old);
+      dict_set_weight (d, NULL);
+      dict_set_filter (d, NULL);
     }
+  d->case_limit = 0;
+  free (d->label);
+  d->label = NULL;
+  string_array_clear (&d->documents);
+  dict_clear_vectors (d);
+  attrset_clear (&d->attributes);
 }
 
-/* Sets the case_index in V's vardict to CASE_INDEX. */
-static void
-set_var_case_index (struct variable *v, int case_index)
+/* Clears the contents from a dictionary without destroying the
+   dictionary itself. */
+void
+dict_clear (struct dictionary *d)
 {
-  var_get_vardict (v)->case_index = case_index;
+  dict_clear__ (d, false);
 }
 
-/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
-   from name_map. */
+/* Clears a dictionary and destroys it. */
 static void
-unindex_vars (struct dictionary *d, size_t from, size_t to)
+_dict_destroy (struct dictionary *d)
 {
-  size_t i;
+  /* In general, we don't want callbacks occurring, if the dictionary
+     is being destroyed */
+  d->callbacks  = NULL ;
 
-  for (i = from; i < to; i++)
-    unindex_var (d, &d->var[i]);
+  dict_clear__ (d, true);
+  string_array_destroy (&d->documents);
+  hmap_destroy (&d->name_map);
+  attrset_destroy (&d->attributes);
+  dict_clear_mrsets (d);
+  free (d->encoding);
+  free (d);
 }
 
-/* Re-sets the dict_index in the dictionary variables with
-   indexes from FROM to TO (exclusive). */
-static void
-reindex_vars (struct dictionary *d, size_t from, size_t to)
+struct dictionary *
+dict_ref (struct dictionary *d)
 {
-  size_t i;
+  d->ref_cnt++;
+  return d;
+}
 
-  for (i = from; i < to; i++)
-    reindex_var (d, &d->var[i]);
+void
+dict_unref (struct dictionary *d)
+{
+  if (d == NULL)
+    return;
+  d->ref_cnt--;
+  assert (d->ref_cnt >= 0);
+  if (d->ref_cnt == 0)
+    _dict_destroy (d);
 }
 
-/* Deletes variable V from dictionary D and frees V.
+/* Returns the number of variables in D. */
+size_t
+dict_get_var_cnt (const struct dictionary *d)
+{
+  return d->var_cnt;
+}
 
-   This is a very bad idea if there might be any pointers to V
-   from outside D.  In general, no variable in the active dataset's
-   dictionary should be deleted when any transformations are
-   active on the dictionary's dataset, because those
-   transformations might reference the deleted variable.  The
-   safest time to delete a variable is just after a procedure has
-   been executed, as done by DELETE VARIABLES.
+/* Returns the variable in D with dictionary index IDX, which
+   must be between 0 and the count returned by
+   dict_get_var_cnt(), exclusive. */
+struct variable *
+dict_get_var (const struct dictionary *d, size_t idx)
+{
+  assert (idx < d->var_cnt);
 
-   Pointers to V within D are not a problem, because
-   dict_delete_var() knows to remove V from split variables,
-   weights, filters, etc. */
+  return d->var[idx].var;
+}
+
+/* Sets *VARS to an array of pointers to variables in D and *CNT
+   to the number of variables in *D.  All variables are returned
+   except for those, if any, in the classes indicated by EXCLUDE.
+   (There is no point in putting DC_SYSTEM in EXCLUDE as
+   dictionaries never include system variables.) */
 void
-dict_delete_var (struct dictionary *d, struct variable *v)
+dict_get_vars (const struct dictionary *d, const struct variable ***vars,
+               size_t *cnt, enum dict_class exclude)
 {
-  int dict_index = var_get_dict_index (v);
-  const int case_index = var_get_case_index (v);
+  dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
+}
 
-  assert (dict_contains_var (d, v));
+/* Sets *VARS to an array of pointers to variables in D and *CNT
+   to the number of variables in *D.  All variables are returned
+   except for those, if any, in the classes indicated by EXCLUDE.
+   (There is no point in putting DC_SYSTEM in EXCLUDE as
+   dictionaries never include system variables.) */
+void
+dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
+                       size_t *cnt, enum dict_class exclude)
+{
+  size_t count;
+  size_t i;
 
-  dict_unset_split_var (d, v);
-  dict_unset_mrset_var (d, v);
+  assert (exclude == (exclude & DC_ALL));
 
-  if (d->weight == v)
-    dict_set_weight (d, NULL);
+  count = 0;
+  for (i = 0; i < d->var_cnt; i++)
+    {
+      enum dict_class class = var_get_dict_class (d->var[i].var);
+      if (!(class & exclude))
+        count++;
+    }
 
-  if (d->filter == v)
-    dict_set_filter (d, NULL);
+  *vars = xnmalloc (count, sizeof **vars);
+  *cnt = 0;
+  for (i = 0; i < d->var_cnt; i++)
+    {
+      enum dict_class class = var_get_dict_class (d->var[i].var);
+      if (!(class & exclude))
+        (*vars)[(*cnt)++] = d->var[i].var;
+    }
+  assert (*cnt == count);
+}
 
-  dict_clear_vectors (d);
+static struct variable *
+add_var_with_case_index (struct dictionary *d, struct variable *v,
+                         int case_index)
+{
+  struct vardict_info *vardict;
 
-  /* Remove V from var array. */
-  unindex_vars (d, dict_index, d->var_cnt);
-  remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
-  d->var_cnt--;
+  assert (case_index >= d->next_value_idx);
 
-  /* Update dict_index for each affected variable. */
-  reindex_vars (d, dict_index, d->var_cnt);
+  /* Update dictionary. */
+  if (d->var_cnt >= d->var_cap)
+    {
+      size_t i;
 
-  /* Free memory. */
-  var_clear_vardict (v);
+      d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
+      hmap_clear (&d->name_map);
+      for (i = 0; i < d->var_cnt; i++)
+        {
+          var_set_vardict (d->var[i].var, &d->var[i]);
+          hmap_insert_fast (&d->name_map, &d->var[i].name_node,
+                            d->var[i].name_node.hash);
+        }
+    }
+
+  vardict = &d->var[d->var_cnt++];
+  vardict->dict = d;
+  vardict->var = v;
+  hmap_insert (&d->name_map, &vardict->name_node,
+               utf8_hash_case_string (var_get_name (v), 0));
+  vardict->case_index = case_index;
+  var_set_vardict (v, vardict);
 
   if (d->changed) d->changed (d, d->changed_data);
+  if (d->callbacks &&  d->callbacks->var_added)
+    d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
 
   invalidate_proto (d);
-  if (d->callbacks &&  d->callbacks->var_deleted)
-    d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
+  d->next_value_idx = case_index + 1;
 
-  var_destroy (v);
+  return v;
 }
 
-/* Deletes the COUNT variables listed in VARS from D.  This is
-   unsafe; see the comment on dict_delete_var() for details. */
-void
-dict_delete_vars (struct dictionary *d,
-                  struct variable *const *vars, size_t count)
+static struct variable *
+add_var (struct dictionary *d, struct variable *v)
 {
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  assert (count == 0 || vars != NULL);
-
-  while (count-- > 0)
-    dict_delete_var (d, *vars++);
+  return add_var_with_case_index (d, v, d->next_value_idx);
 }
 
-/* Deletes the COUNT variables in D starting at index IDX.  This
-   is unsafe; see the comment on dict_delete_var() for
-   details. Deleting consecutive vars will result in less callbacks
-   compared to iterating over dict_delete_var.
-   A simple while loop over dict_delete_var will
-   produce (d->var_cnt - IDX) * COUNT variable changed callbacks
-   plus COUNT variable delete callbacks.
-   This here produces d->var_cnt - IDX variable changed callbacks
-   plus COUNT variable delete callbacks. */
-void
-dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
+/* Creates and returns a new variable in D with the given NAME
+   and WIDTH.  Returns a null pointer if the given NAME would
+   duplicate that of an existing variable in the dictionary. */
+struct variable *
+dict_create_var (struct dictionary *d, const char *name, int width)
 {
-  assert (idx + count <= d->var_cnt);
-
-  /* We need to store the variable and the corresponding case_index
-     for the delete callbacks later. We store them in a linked list.*/
-  struct delvar {
-    struct ll ll;
-    struct variable *var;
-    int case_index;
-  };
-  struct ll_list list = LL_INITIALIZER (list);
-
-  for (size_t i = idx; i < idx + count; i++)
-    {
-      struct delvar *dv = xmalloc (sizeof (struct delvar));
-      assert (dv);
-      struct variable *v = d->var[i].var;
-
-      dict_unset_split_var (d, v);
-      dict_unset_mrset_var (d, v);
+  return (dict_lookup_var (d, name) == NULL
+          ? dict_create_var_assert (d, name, width)
+          : NULL);
+}
 
-      if (d->weight == v)
-	dict_set_weight (d, NULL);
+/* Creates and returns a new variable in D with the given NAME
+   and WIDTH.  Assert-fails if the given NAME would duplicate
+   that of an existing variable in the dictionary. */
+struct variable *
+dict_create_var_assert (struct dictionary *d, const char *name, int width)
+{
+  assert (dict_lookup_var (d, name) == NULL);
+  return add_var (d, var_create (name, width));
+}
 
-      if (d->filter == v)
-	dict_set_filter (d, NULL);
+/* Creates and returns a new variable in D, as a copy of existing variable
+   OLD_VAR, which need not be in D or in any dictionary.  Returns a null
+   pointer if OLD_VAR's name would duplicate that of an existing variable in
+   the dictionary. */
+struct variable *
+dict_clone_var (struct dictionary *d, const struct variable *old_var)
+{
+  return dict_clone_var_as (d, old_var, var_get_name (old_var));
+}
 
-      dv->var = v;
-      dv->case_index = var_get_case_index (v);
-      ll_push_tail (&list, (struct ll *)dv);
-    }
+/* Creates and returns a new variable in D, as a copy of existing variable
+   OLD_VAR, which need not be in D or in any dictionary.  Assert-fails if
+   OLD_VAR's name would duplicate that of an existing variable in the
+   dictionary. */
+struct variable *
+dict_clone_var_assert (struct dictionary *d, const struct variable *old_var)
+{
+  return dict_clone_var_as_assert (d, old_var, var_get_name (old_var));
+}
 
-  dict_clear_vectors (d);
+/* Creates and returns a new variable in D with name NAME, as a copy of
+   existing variable OLD_VAR, which need not be in D or in any dictionary.
+   Returns a null pointer if the given NAME would duplicate that of an existing
+   variable in the dictionary. */
+struct variable *
+dict_clone_var_as (struct dictionary *d, const struct variable *old_var,
+                   const char *name)
+{
+  return (dict_lookup_var (d, name) == NULL
+          ? dict_clone_var_as_assert (d, old_var, name)
+          : NULL);
+}
 
-  /* Remove variables from var array. */
-  unindex_vars (d, idx, d->var_cnt);
-  remove_range (d->var, d->var_cnt, sizeof *d->var, idx, count);
-  d->var_cnt -= count;
+/* Creates and returns a new variable in D with name NAME, as a copy of
+   existing variable OLD_VAR, which need not be in D or in any dictionary.
+   Assert-fails if the given NAME would duplicate that of an existing variable
+   in the dictionary. */
+struct variable *
+dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
+                          const char *name)
+{
+  struct variable *new_var = var_clone (old_var);
+  assert (dict_lookup_var (d, name) == NULL);
+  var_set_name (new_var, name);
+  return add_var (d, new_var);
+}
 
-  /* Reindexing will result variable-changed callback */
-  reindex_vars (d, idx, d->var_cnt);
+struct variable *
+dict_clone_var_in_place_assert (struct dictionary *d,
+                                const struct variable *old_var)
+{
+  assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
+  return add_var_with_case_index (d, var_clone (old_var),
+                                  var_get_case_index (old_var));
+}
 
-  invalidate_proto (d);
-  if (d->changed) d->changed (d, d->changed_data);
+/* Returns the variable named NAME in D, or a null pointer if no
+   variable has that name. */
+struct variable *
+dict_lookup_var (const struct dictionary *d, const char *name)
+{
+  struct vardict_info *vardict;
 
-  /* Now issue the variable delete callbacks and delete
-     the variables. The vardict is not valid at this point
-     anymore. That is the reason why we stored the
-     caseindex before reindexing. */
-  for (size_t vi = idx; vi < idx + count; vi++)
+  HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
+                           utf8_hash_case_string (name, 0), &d->name_map)
     {
-      struct delvar *dv = (struct delvar *) ll_pop_head (&list);
-      var_clear_vardict (dv->var);
-      if (d->callbacks &&  d->callbacks->var_deleted)
-	d->callbacks->var_deleted (d, dv->var, vi, dv->case_index, d->cb_data);
-      var_destroy (dv->var);
-      free (dv);
+      struct variable *var = vardict->var;
+      if (!utf8_strcasecmp (var_get_name (var), name))
+        return var;
     }
+
+  return NULL;
 }
 
-/* Deletes scratch variables from dictionary D. */
-void
-dict_delete_scratch_vars (struct dictionary *d)
+/* Returns the variable named NAME in D.  Assert-fails if no
+   variable has that name. */
+struct variable *
+dict_lookup_var_assert (const struct dictionary *d, const char *name)
 {
-  int i;
+  struct variable *v = dict_lookup_var (d, name);
+  assert (v != NULL);
+  return v;
+}
 
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  for (i = 0; i < d->var_cnt;)
-    if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
-      dict_delete_var (d, d->var[i].var);
-    else
-      i++;
+/* Returns true if variable V is in dictionary D,
+   false otherwise. */
+bool
+dict_contains_var (const struct dictionary *d, const struct variable *v)
+{
+  return (var_has_vardict (v)
+          && vardict_get_dictionary (var_get_vardict (v)) == d);
 }
 
 /* Moves V to 0-based position IDX in D.  Other variables in D,
@@ -782,7 +908,7 @@ dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
 
   unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
   move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
-  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
+  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1, false);
 }
 
 /* Reorders the variables in D, placing the COUNT variables
@@ -824,7 +950,7 @@ dict_reorder_vars (struct dictionary *d,
   d->var = new_var;
 
   hmap_clear (&d->name_map);
-  reindex_vars (d, 0, d->var_cnt);
+  reindex_vars (d, 0, d->var_cnt, false);
 }
 
 /* Changes the name of variable V that is currently in a dictionary to
@@ -853,7 +979,7 @@ dict_try_rename_var (struct dictionary *d, struct variable *v,
   struct variable *old = var_clone (v);
   unindex_var (d, var_get_vardict (v));
   rename_var (v, new_name);
-  reindex_var (d, var_get_vardict (v));
+  reindex_var (d, var_get_vardict (v), false);
 
   if (settings_get_algorithm () == ENHANCED)
     var_clear_short_names (v);
@@ -862,7 +988,7 @@ dict_try_rename_var (struct dictionary *d, struct variable *v,
   if (d->callbacks &&  d->callbacks->var_changed)
     d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
 
-  var_destroy (old);
+  var_unref (old);
 
   return true;
 }
@@ -929,13 +1055,13 @@ dict_rename_vars (struct dictionary *d,
           for (i = 0; i < count; i++)
             {
               rename_var (vars[i], old_names[i]);
-              reindex_var (d, var_get_vardict (vars[i]));
+              reindex_var (d, var_get_vardict (vars[i]), false);
             }
 
           pool_destroy (pool);
           return false;
         }
-      reindex_var (d, var_get_vardict (vars[i]));
+      reindex_var (d, var_get_vardict (vars[i]), false);
     }
 
   /* Clear short names. */
@@ -968,6 +1094,9 @@ make_hinted_name (const struct dictionary *dict, const char *hint)
   size_t ofs;
   int mblen;
 
+  if (hint_len > ID_MAX_LEN)
+    hint_len = ID_MAX_LEN;
+
   /* The allocation size here is OK: characters that are copied directly fit
      OK, and characters that are not copied directly are replaced by a single
      '_' byte.  If u8_mbtouc() replaces bad input by 0xfffd, then that will get
@@ -1074,6 +1203,7 @@ dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
       if (hinted_name != NULL)
         return hinted_name;
     }
+
   return make_numeric_name (dict, num_start);
 }
 
@@ -1150,8 +1280,8 @@ dict_set_weight (struct dictionary *d, struct variable *v)
   if (d->changed) d->changed (d, d->changed_data);
   if (d->callbacks &&  d->callbacks->weight_changed)
     d->callbacks->weight_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                  d->cb_data);
 }
 
 /* Returns the filter variable in dictionary D (see cmd_filter())
@@ -1172,29 +1302,13 @@ dict_set_filter (struct dictionary *d, struct variable *v)
   assert (v == NULL || dict_contains_var (d, v));
   assert (v == NULL || var_is_numeric (v));
 
-  /* When a filter is set, we ref the dictionary.
-     This is because the GUI maintains a pointer
-     to the dict's variables, and the variables'
-     addresses change in the callback.  */
-  if (d->filter == NULL && v != NULL)
-    {
-      d = dict_ref (d);
-    }
-
-  /* Deref the dict when a filter is removed.  */
-  if (d->filter != NULL && v == NULL)
-    {
-      assert (d->ref_cnt > 0);
-      dict_unref (d);
-    }
-
   d->filter = v;
 
   if (d->changed) d->changed (d, d->changed_data);
   if (d->callbacks && d->callbacks->filter_changed)
     d->callbacks->filter_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                      d->cb_data);
 }
 
 /* Returns the case limit for dictionary D, or zero if the number
@@ -1322,70 +1436,6 @@ dict_get_compacted_proto (const struct dictionary *d,
     }
   return proto;
 }
-
-/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
-   dict_get_split_cnt() to determine how many SPLIT FILE vars
-   there are.  Returns a null pointer if and only if there are no
-   SPLIT FILE vars. */
-const struct variable *const *
-dict_get_split_vars (const struct dictionary *d)
-{
-  return d->split;
-}
-
-/* Returns the number of SPLIT FILE vars. */
-size_t
-dict_get_split_cnt (const struct dictionary *d)
-{
-  return d->split_cnt;
-}
-
-/* Removes variable V, which must be in D, from D's set of split
-   variables. */
-static void
-dict_unset_split_var (struct dictionary *d, struct variable *v)
-{
-  int orig_count;
-
-  assert (dict_contains_var (d, v));
-
-  orig_count = d->split_cnt;
-  d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
-                               &v, compare_var_ptrs, NULL);
-  if (orig_count != d->split_cnt)
-    {
-      if (d->changed) d->changed (d, d->changed_data);
-      /* We changed the set of split variables so invoke the
-         callback. */
-      if (d->callbacks &&  d->callbacks->split_changed)
-        d->callbacks->split_changed (d, d->cb_data);
-    }
-}
-
-/* Sets CNT split vars SPLIT in dictionary D. */
-void
-dict_set_split_vars (struct dictionary *d,
-                     struct variable *const *split, size_t cnt)
-{
-  assert (cnt == 0 || split != NULL);
-
-  d->split_cnt = cnt;
-  if (cnt > 0)
-   {
-    d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
-    memcpy (d->split, split, cnt * sizeof *d->split);
-   }
-  else
-   {
-    free (d->split);
-    d->split = NULL;
-   }
-
-  if (d->changed) d->changed (d, d->changed_data);
-  if (d->callbacks &&  d->callbacks->split_changed)
-    d->callbacks->split_changed (d, d->cb_data);
-}
-
 /* Returns the file label for D, or a null pointer if D is
    unlabeled (see cmd_file_label()). */
 const char *
@@ -1421,12 +1471,15 @@ dict_get_documents (const struct dictionary *d)
 void
 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
 {
-  size_t i;
+  /* Swap out the old documents, instead of destroying them immediately, to
+     allow the new documents to include pointers into the old ones. */
+  struct string_array old_docs = STRING_ARRAY_INITIALIZER;
+  string_array_swap (&d->documents, &old_docs);
 
-  dict_clear_documents (d);
-
-  for (i = 0; i < new_docs->n; i++)
+  for (size_t i = 0; i < new_docs->n; i++)
     dict_add_document_line (d, new_docs->strings[i], false);
+
+  string_array_destroy (&old_docs);
 }
 
 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
@@ -1749,11 +1802,14 @@ dict_var_changed (const struct variable *v, unsigned int what, struct variable *
       if (NULL == d)
 	return;
 
+      if (what & (VAR_TRAIT_WIDTH | VAR_TRAIT_POSITION))
+        invalidate_proto (d);
+
       if (d->changed) d->changed (d, d->changed_data);
       if (d->callbacks && d->callbacks->var_changed)
-	d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
+        d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
     }
-  var_destroy (oldvar);
+  var_unref (oldvar);
 }