X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fdictionary.c;h=c13321c7827bb2cd877c598d834f5bbc2a341c6e;hb=24cbabb1547682037cda854b46c2e4bdf87a2c8a;hp=2c35dcf4e6cc83424068867efa2fd4958c9b2044;hpb=a16ae567635169e8a0f6e7eae037728c88ed17fe;p=pspp

diff --git a/src/data/dictionary.c b/src/data/dictionary.c
index 2c35dcf4e6..c13321c782 100644
--- a/src/data/dictionary.c
+++ b/src/data/dictionary.c
@@ -1,5 +1,6 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014,
+   2015, 2020 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -57,21 +58,23 @@
 struct dictionary
   {
     int ref_cnt;
-    struct vardict_info *var;	/* Variables. */
-    size_t var_cnt, var_cap;    /* Number of variables, capacity. */
+    struct vardict_info *vars;	/* Variables. */
+    size_t n_vars;              /* Number of variables. */
+    size_t allocated_vars;      /* Allocated space in 'vars'. */
     struct caseproto *proto;    /* Prototype for dictionary cases
                                    (updated lazily). */
     struct hmap name_map;	/* Variable index by name. */
     int next_value_idx;         /* Index of next `union value' to allocate. */
     const struct variable **split;    /* SPLIT FILE vars. */
-    size_t split_cnt;           /* SPLIT FILE count. */
+    size_t n_splits;            /* SPLIT FILE count. */
+    enum split_type split_type;
     struct variable *weight;    /* WEIGHT variable. */
     struct variable *filter;    /* FILTER variable. */
     casenumber case_limit;      /* Current case limit (N command). */
     char *label;		/* File label. */
     struct string_array documents; /* Documents. */
     struct vector **vector;     /* Vectors of variables. */
-    size_t vector_cnt;          /* Number of vectors. */
+    size_t n_vectors;           /* Number of vectors. */
     struct attrset attributes;  /* Custom attributes. */
     struct mrset **mrsets;      /* Multiple response sets. */
     size_t n_mrsets;            /* Number of multiple response sets. */
@@ -91,9 +94,83 @@ struct dictionary
     void *changed_data;
   };
 
-static void dict_unset_split_var (struct dictionary *, struct variable *);
+static void dict_unset_split_var (struct dictionary *, struct variable *, bool);
 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
 
+/* Compares two double pointers to variables, which should point
+   to elements of a struct dictionary's `var' member array. */
+static int
+compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
+{
+  struct variable *const *a = a_;
+  struct variable *const *b = b_;
+
+  return *a < *b ? -1 : *a > *b;
+}
+
+static void
+unindex_var (struct dictionary *d, struct vardict_info *vardict)
+{
+  hmap_delete (&d->name_map, &vardict->name_node);
+}
+
+/* This function assumes that vardict->name_node.hash is valid, that is, that
+   its name has not changed since it was hashed (rename_var() updates this
+   hash along with the name itself). */
+static void
+reindex_var (struct dictionary *d, struct vardict_info *vardict, bool skip_callbacks)
+{
+  struct variable *old = (d->callbacks && d->callbacks->var_changed
+                          ? var_clone (vardict->var)
+                          : NULL);
+
+  struct variable *var = vardict->var;
+  var_set_vardict (var, vardict);
+  hmap_insert_fast (&d->name_map, &vardict->name_node,
+                    vardict->name_node.hash);
+
+  if (! skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (old)
+        {
+          d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
+          var_unref (old);
+        }
+    }
+}
+
+/* Sets the case_index in V's vardict to CASE_INDEX. */
+static void
+set_var_case_index (struct variable *v, int case_index)
+{
+  var_get_vardict (v)->case_index = case_index;
+}
+
+/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
+   from name_map. */
+static void
+unindex_vars (struct dictionary *d, size_t from, size_t to)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    unindex_var (d, &d->vars[i]);
+}
+
+/* Re-sets the dict_index in the dictionary variables with
+   indexes from FROM to TO (exclusive). */
+static void
+reindex_vars (struct dictionary *d, size_t from, size_t to, bool skip_callbacks)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    reindex_var (d, &d->vars[i], skip_callbacks);
+}
+
+
+
 /* Returns the encoding for data in dictionary D.  The return value is a
    nonnull string that contains an IANA character set name. */
 const char *
@@ -137,9 +214,9 @@ void
 dict_dump (const struct dictionary *d)
 {
   int i;
-  for (i = 0 ; i < d->var_cnt ; ++i)
+  for (i = 0 ; i < d->n_vars ; ++i)
     {
-      const struct variable *v = d->var[i].var;
+      const struct variable *v = d->vars[i].var;
       printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
 	      var_get_name (v),
 	      var_get_dict_index (v),
@@ -175,13 +252,16 @@ dict_copy_callbacks (struct dictionary *dest,
 struct dictionary *
 dict_create (const char *encoding)
 {
-  struct dictionary *d = xzalloc (sizeof *d);
+  struct dictionary *d = xmalloc (sizeof *d);
 
-  d->encoding = xstrdup (encoding);
-  d->names_must_be_ids = true;
-  hmap_init (&d->name_map);
-  attrset_init (&d->attributes);
-  d->ref_cnt = 1;
+  *d = (struct dictionary) {
+    .encoding = xstrdup (encoding),
+    .names_must_be_ids = true,
+    .name_map = HMAP_INITIALIZER (d->name_map),
+    .attributes = ATTRSET_INITIALIZER (d->attributes),
+    .split_type = SPLIT_NONE,
+    .ref_cnt = 1,
+  };
 
   return d;
 }
@@ -205,13 +285,13 @@ dict_clone (const struct dictionary *s)
   d = dict_create (s->encoding);
   dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));
 
-  for (i = 0; i < s->var_cnt; i++)
+  for (i = 0; i < s->n_vars; i++)
     {
-      struct variable *sv = s->var[i].var;
+      struct variable *sv = s->vars[i].var;
       struct variable *dv = dict_clone_var_assert (d, sv);
       size_t i;
 
-      for (i = 0; i < var_get_short_name_cnt (sv); i++)
+      for (i = 0; i < var_get_n_short_names (sv); i++)
         var_set_short_name (dv, i, var_get_short_name (sv, i));
 
       var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
@@ -219,13 +299,14 @@ dict_clone (const struct dictionary *s)
 
   d->next_value_idx = s->next_value_idx;
 
-  d->split_cnt = s->split_cnt;
-  if (d->split_cnt > 0)
+  d->n_splits = s->n_splits;
+  if (d->n_splits > 0)
     {
-       d->split = xnmalloc (d->split_cnt, sizeof *d->split);
-      for (i = 0; i < d->split_cnt; i++)
-        d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
+       d->split = xnmalloc (d->n_splits, sizeof *d->split);
+       for (i = 0; i < d->n_splits; i++)
+         d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
     }
+  d->split_type = s->split_type;
 
   if (s->weight != NULL)
     dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
@@ -237,9 +318,9 @@ dict_clone (const struct dictionary *s)
   dict_set_label (d, dict_get_label (s));
   dict_set_documents (d, dict_get_documents (s));
 
-  d->vector_cnt = s->vector_cnt;
-  d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
-  for (i = 0; i < s->vector_cnt; i++)
+  d->n_vectors = s->n_vectors;
+  d->vector = xnmalloc (d->n_vectors, sizeof *d->vector);
+  for (i = 0; i < s->n_vectors; i++)
     d->vector[i] = vector_clone (s->vector[i], s, d);
 
   dict_set_attributes (d, dict_get_attributes (s));
@@ -261,27 +342,302 @@ dict_clone (const struct dictionary *s)
   return d;
 }
 
+
+
+/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
+   dict_get_n_splits() to determine how many SPLIT FILE vars
+   there are.  Returns a null pointer if and only if there are no
+   SPLIT FILE vars. */
+const struct variable *const *
+dict_get_split_vars (const struct dictionary *d)
+{
+  return d->split;
+}
+
+/* Returns the number of SPLIT FILE vars. */
+size_t
+dict_get_n_splits (const struct dictionary *d)
+{
+  return d->n_splits;
+}
+
+/* Removes variable V, which must be in D, from D's set of split
+   variables. */
+static void
+dict_unset_split_var (struct dictionary *d, struct variable *v, bool skip_callbacks)
+{
+  int orig_count;
+
+  assert (dict_contains_var (d, v));
+
+  orig_count = d->n_splits;
+  d->n_splits = remove_equal (d->split, d->n_splits, sizeof *d->split,
+                               &v, compare_var_ptrs, NULL);
+  if (orig_count != d->n_splits && !skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      /* We changed the set of split variables so invoke the
+         callback. */
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
+}
+
+
+/* Sets N split vars SPLIT in dictionary D.  N is silently capped to a maximum
+   of MAX_SPLITS. */
+static void
+dict_set_split_vars__ (struct dictionary *d,
+                       struct variable *const *split, size_t n,
+                       enum split_type type, bool skip_callbacks)
+{
+  if (n > MAX_SPLITS)
+    n = MAX_SPLITS;
+  assert (n == 0 || split != NULL);
+
+  d->n_splits = n;
+  d->split_type = type == SPLIT_NONE ? SPLIT_LAYERED : type;
+  if (n > 0)
+   {
+    d->split = xnrealloc (d->split, n, sizeof *d->split) ;
+    memcpy (d->split, split, n * sizeof *d->split);
+   }
+  else
+   {
+    free (d->split);
+    d->split = NULL;
+   }
+
+ if (!skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
+}
+
+/* Sets N split vars SPLIT in dictionary D. */
+void
+dict_set_split_vars (struct dictionary *d,
+                     struct variable *const *split, size_t n,
+                     enum split_type type)
+{
+  dict_set_split_vars__ (d, split, n, type, false);
+}
+
+void
+dict_clear_split_vars (struct dictionary *d)
+{
+  dict_set_split_vars (d, NULL, 0, SPLIT_NONE);
+}
+
+
+/* Deletes variable V from dictionary D and frees V.
+
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
+
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+static void
+dict_delete_var__ (struct dictionary *d, struct variable *v, bool skip_callbacks)
+{
+  int dict_index = var_get_dict_index (v);
+  const int case_index = var_get_case_index (v);
+
+  assert (dict_contains_var (d, v));
+
+  dict_unset_split_var (d, v, skip_callbacks);
+  dict_unset_mrset_var (d, v);
+
+  if (d->weight == v)
+    dict_set_weight (d, NULL);
+
+  if (d->filter == v)
+    dict_set_filter (d, NULL);
+
+  dict_clear_vectors (d);
+
+  /* Remove V from var array. */
+  unindex_vars (d, dict_index, d->n_vars);
+  remove_element (d->vars, d->n_vars, sizeof *d->vars, dict_index);
+  d->n_vars--;
+
+  /* Update dict_index for each affected variable. */
+  reindex_vars (d, dict_index, d->n_vars, skip_callbacks);
+
+  /* Free memory. */
+  var_clear_vardict (v);
+
+  if (! skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->var_deleted)
+        d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
+    }
+
+  invalidate_proto (d);
+  var_unref (v);
+}
+
+/* Deletes variable V from dictionary D and frees V.
+
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
+
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+void
+dict_delete_var (struct dictionary *d, struct variable *v)
+{
+  dict_delete_var__ (d, v, false);
+}
+
+
+/* Deletes the COUNT variables listed in VARS from D.  This is
+   unsafe; see the comment on dict_delete_var() for details. */
+void
+dict_delete_vars (struct dictionary *d,
+                  struct variable *const *vars, size_t count)
+{
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  assert (count == 0 || vars != NULL);
+
+  while (count-- > 0)
+    dict_delete_var (d, *vars++);
+}
+
+/* Deletes the COUNT variables in D starting at index IDX.  This
+   is unsafe; see the comment on dict_delete_var() for
+   details. Deleting consecutive vars will result in less callbacks
+   compared to iterating over dict_delete_var.
+   A simple while loop over dict_delete_var will
+   produce (d->n_vars - IDX) * COUNT variable changed callbacks
+   plus COUNT variable delete callbacks.
+   This here produces d->n_vars - IDX variable changed callbacks
+   plus COUNT variable delete callbacks. */
+void
+dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
+{
+  assert (idx + count <= d->n_vars);
+
+  /* We need to store the variable and the corresponding case_index
+     for the delete callbacks later. We store them in a linked list.*/
+  struct delvar {
+    struct ll ll;
+    struct variable *var;
+    int case_index;
+  };
+  struct ll_list list = LL_INITIALIZER (list);
+
+  for (size_t i = idx; i < idx + count; i++)
+    {
+      struct delvar *dv = xmalloc (sizeof (struct delvar));
+      assert (dv);
+      struct variable *v = d->vars[i].var;
+
+      dict_unset_split_var (d, v, false);
+      dict_unset_mrset_var (d, v);
+
+      if (d->weight == v)
+	dict_set_weight (d, NULL);
+
+      if (d->filter == v)
+	dict_set_filter (d, NULL);
+
+      dv->var = v;
+      dv->case_index = var_get_case_index (v);
+      ll_push_tail (&list, (struct ll *)dv);
+    }
+
+  dict_clear_vectors (d);
+
+  /* Remove variables from var array. */
+  unindex_vars (d, idx, d->n_vars);
+  remove_range (d->vars, d->n_vars, sizeof *d->vars, idx, count);
+  d->n_vars -= count;
+
+  /* Reindexing will result variable-changed callback */
+  reindex_vars (d, idx, d->n_vars, false);
+
+  invalidate_proto (d);
+  if (d->changed) d->changed (d, d->changed_data);
+
+  /* Now issue the variable delete callbacks and delete
+     the variables. The vardict is not valid at this point
+     anymore. That is the reason why we stored the
+     caseindex before reindexing. */
+  for (size_t vi = idx; vi < idx + count; vi++)
+    {
+      struct delvar *dv = (struct delvar *) ll_pop_head (&list);
+      var_clear_vardict (dv->var);
+      if (d->callbacks &&  d->callbacks->var_deleted)
+        d->callbacks->var_deleted (d, dv->var, vi, dv->case_index, d->cb_data);
+      var_unref (dv->var);
+      free (dv);
+    }
+}
+
+/* Deletes scratch variables from dictionary D. */
+void
+dict_delete_scratch_vars (struct dictionary *d)
+{
+  int i;
+
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  for (i = 0; i < d->n_vars;)
+    if (var_get_dict_class (d->vars[i].var) == DC_SCRATCH)
+      dict_delete_var (d, d->vars[i].var);
+    else
+      i++;
+}
+
+
+
 /* Clears the contents from a dictionary without destroying the
    dictionary itself. */
-void
-dict_clear (struct dictionary *d)
+static void
+dict_clear__ (struct dictionary *d, bool skip_callbacks)
 {
   /* FIXME?  Should we really clear case_limit, label, documents?
      Others are necessarily cleared by deleting all the variables.*/
-  while (d->var_cnt > 0)
+  while (d->n_vars > 0)
     {
-      dict_delete_var (d, d->var[d->var_cnt - 1].var);
+      dict_delete_var__ (d, d->vars[d->n_vars - 1].var, skip_callbacks);
     }
 
-  free (d->var);
-  d->var = NULL;
-  d->var_cnt = d->var_cap = 0;
+  free (d->vars);
+  d->vars = NULL;
+  d->n_vars = d->allocated_vars = 0;
   invalidate_proto (d);
   hmap_clear (&d->name_map);
   d->next_value_idx = 0;
-  dict_set_split_vars (d, NULL, 0);
-  dict_set_weight (d, NULL);
-  dict_set_filter (d, NULL);
+  dict_set_split_vars__ (d, NULL, 0, SPLIT_NONE, skip_callbacks);
+
+  if (skip_callbacks)
+    {
+      d->weight = NULL;
+      d->filter = NULL;
+    }
+  else
+    {
+      dict_set_weight (d, NULL);
+      dict_set_filter (d, NULL);
+    }
   d->case_limit = 0;
   free (d->label);
   d->label = NULL;
@@ -290,6 +646,14 @@ dict_clear (struct dictionary *d)
   attrset_clear (&d->attributes);
 }
 
+/* Clears the contents from a dictionary without destroying the
+   dictionary itself. */
+void
+dict_clear (struct dictionary *d)
+{
+  dict_clear__ (d, false);
+}
+
 /* Clears a dictionary and destroys it. */
 static void
 _dict_destroy (struct dictionary *d)
@@ -298,7 +662,7 @@ _dict_destroy (struct dictionary *d)
      is being destroyed */
   d->callbacks  = NULL ;
 
-  dict_clear (d);
+  dict_clear__ (d, true);
   string_array_destroy (&d->documents);
   hmap_destroy (&d->name_map);
   attrset_destroy (&d->attributes);
@@ -327,42 +691,42 @@ dict_unref (struct dictionary *d)
 
 /* Returns the number of variables in D. */
 size_t
-dict_get_var_cnt (const struct dictionary *d)
+dict_get_n_vars (const struct dictionary *d)
 {
-  return d->var_cnt;
+  return d->n_vars;
 }
 
 /* Returns the variable in D with dictionary index IDX, which
    must be between 0 and the count returned by
-   dict_get_var_cnt(), exclusive. */
+   dict_get_n_vars(), exclusive. */
 struct variable *
 dict_get_var (const struct dictionary *d, size_t idx)
 {
-  assert (idx < d->var_cnt);
+  assert (idx < d->n_vars);
 
-  return d->var[idx].var;
+  return d->vars[idx].var;
 }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
+/* Sets *VARS to an array of pointers to variables in D and *N
    to the number of variables in *D.  All variables are returned
    except for those, if any, in the classes indicated by EXCLUDE.
    (There is no point in putting DC_SYSTEM in EXCLUDE as
    dictionaries never include system variables.) */
 void
 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
-               size_t *cnt, enum dict_class exclude)
+               size_t *n, enum dict_class exclude)
 {
-  dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
+  dict_get_vars_mutable (d, (struct variable ***) vars, n, exclude);
 }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
+/* Sets *VARS to an array of pointers to variables in D and *N
    to the number of variables in *D.  All variables are returned
    except for those, if any, in the classes indicated by EXCLUDE.
    (There is no point in putting DC_SYSTEM in EXCLUDE as
    dictionaries never include system variables.) */
 void
 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
-                       size_t *cnt, enum dict_class exclude)
+                       size_t *n, enum dict_class exclude)
 {
   size_t count;
   size_t i;
@@ -370,22 +734,22 @@ dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
   assert (exclude == (exclude & DC_ALL));
 
   count = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  for (i = 0; i < d->n_vars; i++)
     {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
       if (!(class & exclude))
         count++;
     }
 
   *vars = xnmalloc (count, sizeof **vars);
-  *cnt = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  *n = 0;
+  for (i = 0; i < d->n_vars; i++)
     {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
       if (!(class & exclude))
-        (*vars)[(*cnt)++] = d->var[i].var;
+        (*vars)[(*n)++] = d->vars[i].var;
     }
-  assert (*cnt == count);
+  assert (*n == count);
 }
 
 static struct variable *
@@ -397,21 +761,21 @@ add_var_with_case_index (struct dictionary *d, struct variable *v,
   assert (case_index >= d->next_value_idx);
 
   /* Update dictionary. */
-  if (d->var_cnt >= d->var_cap)
+  if (d->n_vars >= d->allocated_vars)
     {
       size_t i;
 
-      d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
+      d->vars = x2nrealloc (d->vars, &d->allocated_vars, sizeof *d->vars);
       hmap_clear (&d->name_map);
-      for (i = 0; i < d->var_cnt; i++)
+      for (i = 0; i < d->n_vars; i++)
         {
-          var_set_vardict (d->var[i].var, &d->var[i]);
-          hmap_insert_fast (&d->name_map, &d->var[i].name_node,
-                            d->var[i].name_node.hash);
+          var_set_vardict (d->vars[i].var, &d->vars[i]);
+          hmap_insert_fast (&d->name_map, &d->vars[i].name_node,
+                            d->vars[i].name_node.hash);
         }
     }
 
-  vardict = &d->var[d->var_cnt++];
+  vardict = &d->vars[d->n_vars++];
   vardict->dict = d;
   vardict->var = v;
   hmap_insert (&d->name_map, &vardict->name_node,
@@ -549,227 +913,6 @@ dict_contains_var (const struct dictionary *d, const struct variable *v)
           && vardict_get_dictionary (var_get_vardict (v)) == d);
 }
 
-/* Compares two double pointers to variables, which should point
-   to elements of a struct dictionary's `var' member array. */
-static int
-compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
-{
-  struct variable *const *a = a_;
-  struct variable *const *b = b_;
-
-  return *a < *b ? -1 : *a > *b;
-}
-
-static void
-unindex_var (struct dictionary *d, struct vardict_info *vardict)
-{
-  hmap_delete (&d->name_map, &vardict->name_node);
-}
-
-/* This function assumes that vardict->name_node.hash is valid, that is, that
-   its name has not changed since it was hashed (rename_var() updates this
-   hash along with the name itself). */
-static void
-reindex_var (struct dictionary *d, struct vardict_info *vardict)
-{
-  struct variable *old = (d->callbacks && d->callbacks->var_changed
-                          ? var_clone (vardict->var)
-                          : NULL);
-
-  struct variable *var = vardict->var;
-  var_set_vardict (var, vardict);
-  hmap_insert_fast (&d->name_map, &vardict->name_node,
-                    vardict->name_node.hash);
-
-  if (d->changed) d->changed (d, d->changed_data);
-  if (old)
-    {
-      d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
-      var_destroy (old);
-    }
-}
-
-/* Sets the case_index in V's vardict to CASE_INDEX. */
-static void
-set_var_case_index (struct variable *v, int case_index)
-{
-  var_get_vardict (v)->case_index = case_index;
-}
-
-/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
-   from name_map. */
-static void
-unindex_vars (struct dictionary *d, size_t from, size_t to)
-{
-  size_t i;
-
-  for (i = from; i < to; i++)
-    unindex_var (d, &d->var[i]);
-}
-
-/* Re-sets the dict_index in the dictionary variables with
-   indexes from FROM to TO (exclusive). */
-static void
-reindex_vars (struct dictionary *d, size_t from, size_t to)
-{
-  size_t i;
-
-  for (i = from; i < to; i++)
-    reindex_var (d, &d->var[i]);
-}
-
-/* Deletes variable V from dictionary D and frees V.
-
-   This is a very bad idea if there might be any pointers to V
-   from outside D.  In general, no variable in the active dataset's
-   dictionary should be deleted when any transformations are
-   active on the dictionary's dataset, because those
-   transformations might reference the deleted variable.  The
-   safest time to delete a variable is just after a procedure has
-   been executed, as done by DELETE VARIABLES.
-
-   Pointers to V within D are not a problem, because
-   dict_delete_var() knows to remove V from split variables,
-   weights, filters, etc. */
-void
-dict_delete_var (struct dictionary *d, struct variable *v)
-{
-  int dict_index = var_get_dict_index (v);
-  const int case_index = var_get_case_index (v);
-
-  assert (dict_contains_var (d, v));
-
-  dict_unset_split_var (d, v);
-  dict_unset_mrset_var (d, v);
-
-  if (d->weight == v)
-    dict_set_weight (d, NULL);
-
-  if (d->filter == v)
-    dict_set_filter (d, NULL);
-
-  dict_clear_vectors (d);
-
-  /* Remove V from var array. */
-  unindex_vars (d, dict_index, d->var_cnt);
-  remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
-  d->var_cnt--;
-
-  /* Update dict_index for each affected variable. */
-  reindex_vars (d, dict_index, d->var_cnt);
-
-  /* Free memory. */
-  var_clear_vardict (v);
-
-  if (d->changed) d->changed (d, d->changed_data);
-
-  invalidate_proto (d);
-  if (d->callbacks &&  d->callbacks->var_deleted)
-    d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
-
-  var_destroy (v);
-}
-
-/* Deletes the COUNT variables listed in VARS from D.  This is
-   unsafe; see the comment on dict_delete_var() for details. */
-void
-dict_delete_vars (struct dictionary *d,
-                  struct variable *const *vars, size_t count)
-{
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  assert (count == 0 || vars != NULL);
-
-  while (count-- > 0)
-    dict_delete_var (d, *vars++);
-}
-
-/* Deletes the COUNT variables in D starting at index IDX.  This
-   is unsafe; see the comment on dict_delete_var() for
-   details. Deleting consecutive vars will result in less callbacks
-   compared to iterating over dict_delete_var.
-   A simple while loop over dict_delete_var will
-   produce (d->var_cnt - IDX) * COUNT variable changed callbacks
-   plus COUNT variable delete callbacks.
-   This here produces d->var_cnt - IDX variable changed callbacks
-   plus COUNT variable delete callbacks. */
-void
-dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
-{
-  assert (idx + count <= d->var_cnt);
-
-  /* We need to store the variable and the corresponding case_index
-     for the delete callbacks later. We store them in a linked list.*/
-  struct delvar {
-    struct ll ll;
-    struct variable *var;
-    int case_index;
-  };
-  struct ll_list list = LL_INITIALIZER (list);
-
-  for (size_t i = idx; i < idx + count; i++)
-    {
-      struct delvar *dv = xmalloc (sizeof (struct delvar));
-      assert (dv);
-      struct variable *v = d->var[i].var;
-
-      dict_unset_split_var (d, v);
-      dict_unset_mrset_var (d, v);
-
-      if (d->weight == v)
-	dict_set_weight (d, NULL);
-
-      if (d->filter == v)
-	dict_set_filter (d, NULL);
-
-      dv->var = v;
-      dv->case_index = var_get_case_index (v);
-      ll_push_tail (&list, (struct ll *)dv);
-    }
-
-  dict_clear_vectors (d);
-
-  /* Remove variables from var array. */
-  unindex_vars (d, idx, d->var_cnt);
-  remove_range (d->var, d->var_cnt, sizeof *d->var, idx, count);
-  d->var_cnt -= count;
-
-  /* Reindexing will result variable-changed callback */
-  reindex_vars (d, idx, d->var_cnt);
-
-  invalidate_proto (d);
-  if (d->changed) d->changed (d, d->changed_data);
-
-  /* Now issue the variable delete callbacks and delete
-     the variables. The vardict is not valid at this point
-     anymore. That is the reason why we stored the
-     caseindex before reindexing. */
-  for (size_t vi = idx; vi < idx + count; vi++)
-    {
-      struct delvar *dv = (struct delvar *) ll_pop_head (&list);
-      var_clear_vardict (dv->var);
-      if (d->callbacks &&  d->callbacks->var_deleted)
-	d->callbacks->var_deleted (d, dv->var, vi, dv->case_index, d->cb_data);
-      var_destroy (dv->var);
-      free (dv);
-    }
-}
-
-/* Deletes scratch variables from dictionary D. */
-void
-dict_delete_scratch_vars (struct dictionary *d)
-{
-  int i;
-
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  for (i = 0; i < d->var_cnt;)
-    if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
-      dict_delete_var (d, d->var[i].var);
-    else
-      i++;
-}
-
 /* Moves V to 0-based position IDX in D.  Other variables in D,
    if any, retain their relative positions.  Runs in time linear
    in the distance moved. */
@@ -778,11 +921,11 @@ dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
 {
   size_t old_index = var_get_dict_index (v);
 
-  assert (new_index < d->var_cnt);
+  assert (new_index < d->n_vars);
 
   unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
-  move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
-  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
+  move_element (d->vars, d->n_vars, sizeof *d->vars, old_index, new_index);
+  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1, false);
 }
 
 /* Reorders the variables in D, placing the COUNT variables
@@ -797,9 +940,9 @@ dict_reorder_vars (struct dictionary *d,
   size_t i;
 
   assert (count == 0 || order != NULL);
-  assert (count <= d->var_cnt);
+  assert (count <= d->n_vars);
 
-  new_var = xnmalloc (d->var_cap, sizeof *new_var);
+  new_var = xnmalloc (d->allocated_vars, sizeof *new_var);
 
   /* Add variables in ORDER to new_var. */
   for (i = 0; i < count; i++)
@@ -814,17 +957,17 @@ dict_reorder_vars (struct dictionary *d,
     }
 
   /* Add remaining variables to new_var. */
-  for (i = 0; i < d->var_cnt; i++)
-    if (d->var[i].dict != NULL)
-      new_var[count++] = d->var[i];
-  assert (count == d->var_cnt);
+  for (i = 0; i < d->n_vars; i++)
+    if (d->vars[i].dict != NULL)
+      new_var[count++] = d->vars[i];
+  assert (count == d->n_vars);
 
   /* Replace old vardicts by new ones. */
-  free (d->var);
-  d->var = new_var;
+  free (d->vars);
+  d->vars = new_var;
 
   hmap_clear (&d->name_map);
-  reindex_vars (d, 0, d->var_cnt);
+  reindex_vars (d, 0, d->n_vars, false);
 }
 
 /* Changes the name of variable V that is currently in a dictionary to
@@ -853,7 +996,7 @@ dict_try_rename_var (struct dictionary *d, struct variable *v,
   struct variable *old = var_clone (v);
   unindex_var (d, var_get_vardict (v));
   rename_var (v, new_name);
-  reindex_var (d, var_get_vardict (v));
+  reindex_var (d, var_get_vardict (v), false);
 
   if (settings_get_algorithm () == ENHANCED)
     var_clear_short_names (v);
@@ -862,7 +1005,7 @@ dict_try_rename_var (struct dictionary *d, struct variable *v,
   if (d->callbacks &&  d->callbacks->var_changed)
     d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
 
-  var_destroy (old);
+  var_unref (old);
 
   return true;
 }
@@ -929,13 +1072,13 @@ dict_rename_vars (struct dictionary *d,
           for (i = 0; i < count; i++)
             {
               rename_var (vars[i], old_names[i]);
-              reindex_var (d, var_get_vardict (vars[i]));
+              reindex_var (d, var_get_vardict (vars[i]), false);
             }
 
           pool_destroy (pool);
           return false;
         }
-      reindex_var (d, var_get_vardict (vars[i]));
+      reindex_var (d, var_get_vardict (vars[i]), false);
     }
 
   /* Clear short names. */
@@ -968,6 +1111,9 @@ make_hinted_name (const struct dictionary *dict, const char *hint)
   size_t ofs;
   int mblen;
 
+  if (hint_len > ID_MAX_LEN)
+    hint_len = ID_MAX_LEN;
+
   /* The allocation size here is OK: characters that are copied directly fit
      OK, and characters that are not copied directly are replaced by a single
      '_' byte.  If u8_mbtouc() replaces bad input by 0xfffd, then that will get
@@ -1074,6 +1220,7 @@ dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
       if (hinted_name != NULL)
         return hinted_name;
     }
+
   return make_numeric_name (dict, num_start);
 }
 
@@ -1150,8 +1297,8 @@ dict_set_weight (struct dictionary *d, struct variable *v)
   if (d->changed) d->changed (d, d->changed_data);
   if (d->callbacks &&  d->callbacks->weight_changed)
     d->callbacks->weight_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                  d->cb_data);
 }
 
 /* Returns the filter variable in dictionary D (see cmd_filter())
@@ -1177,8 +1324,8 @@ dict_set_filter (struct dictionary *d, struct variable *v)
   if (d->changed) d->changed (d, d->changed_data);
   if (d->callbacks && d->callbacks->filter_changed)
     d->callbacks->filter_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                      d->cb_data);
 }
 
 /* Returns the case limit for dictionary D, or zero if the number
@@ -1207,11 +1354,11 @@ dict_get_proto (const struct dictionary *d_)
       size_t i;
 
       d->proto = caseproto_create ();
-      d->proto = caseproto_reserve (d->proto, d->var_cnt);
-      for (i = 0; i < d->var_cnt; i++)
+      d->proto = caseproto_reserve (d->proto, d->n_vars);
+      for (i = 0; i < d->n_vars; i++)
         d->proto = caseproto_set_width (d->proto,
-                                        var_get_case_index (d->var[i].var),
-                                        var_get_width (d->var[i].var));
+                                        var_get_case_index (d->vars[i].var),
+                                        var_get_width (d->vars[i].var));
     }
   return d->proto;
 }
@@ -1241,9 +1388,9 @@ dict_compact_values (struct dictionary *d)
   size_t i;
 
   d->next_value_idx = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  for (i = 0; i < d->n_vars; i++)
     {
-      struct variable *v = d->var[i].var;
+      struct variable *v = d->vars[i].var;
       set_var_case_index (v, d->next_value_idx++);
     }
   invalidate_proto (d);
@@ -1262,21 +1409,18 @@ dict_compact_values (struct dictionary *d)
 size_t
 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
 {
-  size_t i;
-  size_t cnt;
-
   assert ((exclude_classes & ~((1u << DC_ORDINARY)
                                | (1u << DC_SYSTEM)
                                | (1u << DC_SCRATCH))) == 0);
 
-  cnt = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  size_t n = 0;
+  for (size_t i = 0; i < d->n_vars; i++)
     {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
       if (!(exclude_classes & (1u << class)))
-        cnt++;
+        n++;
     }
-  return cnt;
+  return n;
 }
 
 /* Returns the case prototype that would result after deleting
@@ -1298,78 +1442,14 @@ dict_get_compacted_proto (const struct dictionary *d,
                                | (1u << DC_SCRATCH))) == 0);
 
   proto = caseproto_create ();
-  for (i = 0; i < d->var_cnt; i++)
+  for (i = 0; i < d->n_vars; i++)
     {
-      struct variable *v = d->var[i].var;
+      struct variable *v = d->vars[i].var;
       if (!(exclude_classes & (1u << var_get_dict_class (v))))
         proto = caseproto_add_width (proto, var_get_width (v));
     }
   return proto;
 }
-
-/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
-   dict_get_split_cnt() to determine how many SPLIT FILE vars
-   there are.  Returns a null pointer if and only if there are no
-   SPLIT FILE vars. */
-const struct variable *const *
-dict_get_split_vars (const struct dictionary *d)
-{
-  return d->split;
-}
-
-/* Returns the number of SPLIT FILE vars. */
-size_t
-dict_get_split_cnt (const struct dictionary *d)
-{
-  return d->split_cnt;
-}
-
-/* Removes variable V, which must be in D, from D's set of split
-   variables. */
-static void
-dict_unset_split_var (struct dictionary *d, struct variable *v)
-{
-  int orig_count;
-
-  assert (dict_contains_var (d, v));
-
-  orig_count = d->split_cnt;
-  d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
-                               &v, compare_var_ptrs, NULL);
-  if (orig_count != d->split_cnt)
-    {
-      if (d->changed) d->changed (d, d->changed_data);
-      /* We changed the set of split variables so invoke the
-         callback. */
-      if (d->callbacks &&  d->callbacks->split_changed)
-        d->callbacks->split_changed (d, d->cb_data);
-    }
-}
-
-/* Sets CNT split vars SPLIT in dictionary D. */
-void
-dict_set_split_vars (struct dictionary *d,
-                     struct variable *const *split, size_t cnt)
-{
-  assert (cnt == 0 || split != NULL);
-
-  d->split_cnt = cnt;
-  if (cnt > 0)
-   {
-    d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
-    memcpy (d->split, split, cnt * sizeof *d->split);
-   }
-  else
-   {
-    free (d->split);
-    d->split = NULL;
-   }
-
-  if (d->changed) d->changed (d, d->changed_data);
-  if (d->callbacks &&  d->callbacks->split_changed)
-    d->callbacks->split_changed (d, d->cb_data);
-}
-
 /* Returns the file label for D, or a null pointer if D is
    unlabeled (see cmd_file_label()). */
 const char *
@@ -1405,12 +1485,15 @@ dict_get_documents (const struct dictionary *d)
 void
 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
 {
-  size_t i;
-
-  dict_clear_documents (d);
+  /* Swap out the old documents, instead of destroying them immediately, to
+     allow the new documents to include pointers into the old ones. */
+  struct string_array old_docs = STRING_ARRAY_INITIALIZER;
+  string_array_swap (&d->documents, &old_docs);
 
-  for (i = 0; i < new_docs->n; i++)
+  for (size_t i = 0; i < new_docs->n; i++)
     dict_add_document_line (d, new_docs->strings[i], false);
+
+  string_array_destroy (&old_docs);
 }
 
 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
@@ -1457,7 +1540,7 @@ dict_add_document_line (struct dictionary *d, const char *line,
   truncated = line[trunc_len] != '\0';
   if (truncated && issue_warning)
     {
-      /* Note to translators: "bytes" is correct, not characters */
+      /* TRANSLATORS: "bytes" is correct, not characters due to UTF encoding */
       msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
     }
 
@@ -1468,7 +1551,7 @@ dict_add_document_line (struct dictionary *d, const char *line,
 
 /* Returns the number of document lines in dictionary D. */
 size_t
-dict_get_document_line_cnt (const struct dictionary *d)
+dict_get_document_n_lines (const struct dictionary *d)
 {
   return d->documents.n;
 }
@@ -1482,57 +1565,55 @@ dict_get_document_line (const struct dictionary *d, size_t idx)
   return d->documents.strings[idx];
 }
 
-/* Creates in D a vector named NAME that contains the CNT
+/* Creates in D a vector named NAME that contains the N
    variables in VAR.  Returns true if successful, or false if a
    vector named NAME already exists in D. */
 bool
 dict_create_vector (struct dictionary *d,
                     const char *name,
-                    struct variable **var, size_t cnt)
+                    struct variable **var, size_t n)
 {
-  size_t i;
-
-  assert (cnt > 0);
-  for (i = 0; i < cnt; i++)
+  assert (n > 0);
+  for (size_t i = 0; i < n; i++)
     assert (dict_contains_var (d, var[i]));
 
   if (dict_lookup_vector (d, name) == NULL)
     {
-      d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
-      d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
+      d->vector = xnrealloc (d->vector, d->n_vectors + 1, sizeof *d->vector);
+      d->vector[d->n_vectors++] = vector_create (name, var, n);
       return true;
     }
   else
     return false;
 }
 
-/* Creates in D a vector named NAME that contains the CNT
+/* Creates in D a vector named NAME that contains the N
    variables in VAR.  A vector named NAME must not already exist
    in D. */
 void
 dict_create_vector_assert (struct dictionary *d,
                            const char *name,
-                           struct variable **var, size_t cnt)
+                           struct variable **var, size_t n)
 {
   assert (dict_lookup_vector (d, name) == NULL);
-  dict_create_vector (d, name, var, cnt);
+  dict_create_vector (d, name, var, n);
 }
 
 /* Returns the vector in D with index IDX, which must be less
-   than dict_get_vector_cnt (D). */
+   than dict_get_n_vectors (D). */
 const struct vector *
 dict_get_vector (const struct dictionary *d, size_t idx)
 {
-  assert (idx < d->vector_cnt);
+  assert (idx < d->n_vectors);
 
   return d->vector[idx];
 }
 
 /* Returns the number of vectors in D. */
 size_t
-dict_get_vector_cnt (const struct dictionary *d)
+dict_get_n_vectors (const struct dictionary *d)
 {
-  return d->vector_cnt;
+  return d->n_vectors;
 }
 
 /* Looks up and returns the vector within D with the given
@@ -1541,7 +1622,7 @@ const struct vector *
 dict_lookup_vector (const struct dictionary *d, const char *name)
 {
   size_t i;
-  for (i = 0; i < d->vector_cnt; i++)
+  for (i = 0; i < d->n_vectors; i++)
     if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
       return d->vector[i];
   return NULL;
@@ -1553,12 +1634,12 @@ dict_clear_vectors (struct dictionary *d)
 {
   size_t i;
 
-  for (i = 0; i < d->vector_cnt; i++)
+  for (i = 0; i < d->n_vectors; i++)
     vector_destroy (d->vector[i]);
   free (d->vector);
 
   d->vector = NULL;
-  d->vector_cnt = 0;
+  d->n_vectors = 0;
 }
 
 /* Multiple response sets. */
@@ -1733,11 +1814,14 @@ dict_var_changed (const struct variable *v, unsigned int what, struct variable *
       if (NULL == d)
 	return;
 
+      if (what & (VAR_TRAIT_WIDTH | VAR_TRAIT_POSITION))
+        invalidate_proto (d);
+
       if (d->changed) d->changed (d, d->changed_data);
       if (d->callbacks && d->callbacks->var_changed)
-	d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
+        d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
     }
-  var_destroy (oldvar);
+  var_unref (oldvar);
 }
 
 
@@ -1783,7 +1867,7 @@ dict_destroy_internal_var (struct variable *var)
 
       /* Destroy internal_dict if it has no variables left, just so that
          valgrind --leak-check --show-reachable won't show internal_dict. */
-      if (dict_get_var_cnt (internal_dict) == 0)
+      if (dict_get_n_vars (internal_dict) == 0)
         {
           dict_unref (internal_dict);
           internal_dict = NULL;
@@ -1794,5 +1878,5 @@ dict_destroy_internal_var (struct variable *var)
 int
 vardict_get_dict_index (const struct vardict_info *vardict)
 {
-  return vardict - vardict->dict->var;
+  return vardict - vardict->dict->vars;
 }