X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Fdata%2Fdictionary.c;h=5c8d6e8dc313f156c664db1417b9b4cf9f7a398a;hb=refs%2Fheads%2Frust;hp=c00bd4face382c363990b6185741bdf1d62080c5;hpb=b401615e6db40bf74394839b96600afe3a868a95;p=pspp

diff --git a/src/data/dictionary.c b/src/data/dictionary.c
index c00bd4face..5c8d6e8dc3 100644
--- a/src/data/dictionary.c
+++ b/src/data/dictionary.c
@@ -1,5 +1,6 @@
 /* PSPP - a program for statistical analysis.
-   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014,
+   2015, 2020 Free Software Foundation, Inc.
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -31,6 +32,7 @@
 #include "data/value-labels.h"
 #include "data/vardict.h"
 #include "data/variable.h"
+#include "data/varset.h"
 #include "data/vector.h"
 #include "libpspp/array.h"
 #include "libpspp/assertion.h"
@@ -43,6 +45,7 @@
 #include "libpspp/pool.h"
 #include "libpspp/str.h"
 #include "libpspp/string-array.h"
+#include "libpspp/ll.h"
 
 #include "gl/intprops.h"
 #include "gl/minmax.h"
@@ -55,24 +58,33 @@
 /* A dictionary. */
 struct dictionary
   {
-    struct vardict_info *var;	/* Variables. */
-    size_t var_cnt, var_cap;    /* Number of variables, capacity. */
+    int ref_cnt;
+    struct vardict_info *vars;	/* Variables. */
+    size_t n_vars;              /* Number of variables. */
+    size_t allocated_vars;      /* Allocated space in 'vars'. */
     struct caseproto *proto;    /* Prototype for dictionary cases
                                    (updated lazily). */
     struct hmap name_map;	/* Variable index by name. */
-    int next_value_idx;         /* Index of next `union value' to allocate. */
     const struct variable **split;    /* SPLIT FILE vars. */
-    size_t split_cnt;           /* SPLIT FILE count. */
+    size_t n_splits;            /* SPLIT FILE count. */
+    enum split_type split_type;
     struct variable *weight;    /* WEIGHT variable. */
     struct variable *filter;    /* FILTER variable. */
     casenumber case_limit;      /* Current case limit (N command). */
     char *label;		/* File label. */
     struct string_array documents; /* Documents. */
     struct vector **vector;     /* Vectors of variables. */
-    size_t vector_cnt;          /* Number of vectors. */
+    size_t n_vectors;           /* Number of vectors. */
     struct attrset attributes;  /* Custom attributes. */
     struct mrset **mrsets;      /* Multiple response sets. */
     size_t n_mrsets;            /* Number of multiple response sets. */
+    struct varset **varsets;    /* Variable sets. */
+    size_t n_varsets;           /* Number of variable sets. */
+
+    /* Whether variable names must be valid identifiers.  Normally, this is
+       true, but sometimes a dictionary is prepared for external use
+       (e.g. output to a CSV file) where names don't have to be valid. */
+    bool names_must_be_ids;
 
     char *encoding;             /* Character encoding of string data */
 
@@ -84,33 +96,114 @@ struct dictionary
     void *changed_data;
   };
 
-static void dict_unset_split_var (struct dictionary *, struct variable *);
+static void dict_unset_split_var (struct dictionary *, struct variable *, bool);
 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
+static void dict_unset_varset_var (struct dictionary *, struct variable *);
 
-void
-dict_set_encoding (struct dictionary *d, const char *enc)
+/* Compares two double pointers to variables, which should point
+   to elements of a struct dictionary's `var' member array. */
+static int
+compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
+{
+  struct variable *const *a = a_;
+  struct variable *const *b = b_;
+
+  return *a < *b ? -1 : *a > *b;
+}
+
+static void
+unindex_var (struct dictionary *d, struct vardict_info *vardict)
+{
+  hmap_delete (&d->name_map, &vardict->name_node);
+}
+
+/* This function assumes that vardict->name_node.hash is valid, that is, that
+   its name has not changed since it was hashed (rename_var() updates this
+   hash along with the name itself). */
+static void
+reindex_var (struct dictionary *d, struct vardict_info *vardict, bool skip_callbacks)
 {
-  if (enc)
+  struct variable *old = (d->callbacks && d->callbacks->var_changed
+                          ? var_clone (vardict->var)
+                          : NULL);
+
+  struct variable *var = vardict->var;
+  var_set_vardict (var, vardict);
+  hmap_insert_fast (&d->name_map, &vardict->name_node,
+                    vardict->name_node.hash);
+
+  if (! skip_callbacks)
     {
-      free (d->encoding);
-      d->encoding = xstrdup (enc);
+      if (d->changed) d->changed (d, d->changed_data);
+      if (old)
+        {
+          d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
+          var_unref (old);
+        }
     }
 }
 
+/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
+   from name_map. */
+static void
+unindex_vars (struct dictionary *d, size_t from, size_t to)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    unindex_var (d, &d->vars[i]);
+}
+
+/* Re-sets the dict_index in the dictionary variables with
+   indexes from FROM to TO (exclusive). */
+static void
+reindex_vars (struct dictionary *d, size_t from, size_t to, bool skip_callbacks)
+{
+  size_t i;
+
+  for (i = from; i < to; i++)
+    reindex_var (d, &d->vars[i], skip_callbacks);
+}
+
+
+
+/* Returns the encoding for data in dictionary D.  The return value is a
+   nonnull string that contains an IANA character set name. */
 const char *
 dict_get_encoding (const struct dictionary *d)
 {
   return d->encoding ;
 }
 
+/* Checks whether UTF-8 string ID is an acceptable identifier in DICT's
+   encoding.  Returns true if it is, otherwise an error message that the caller
+   must free(). */
+char * WARN_UNUSED_RESULT
+dict_id_is_valid__ (const struct dictionary *dict, const char *id)
+{
+  if (!dict->names_must_be_ids)
+    return NULL;
+  return id_is_valid__ (id, dict->encoding);
+}
+
+static bool
+error_to_bool (char *error)
+{
+  if (error)
+    {
+      free (error);
+      return false;
+    }
+  else
+    return true;
+}
+
 /* Returns true if UTF-8 string ID is an acceptable identifier in DICT's
-   encoding, false otherwise.  If ISSUE_ERROR is true, issues an explanatory
-   error message on failure. */
+   encoding, false otherwise. */
 bool
-dict_id_is_valid (const struct dictionary *dict, const char *id,
-                  bool issue_error)
+dict_id_is_valid (const struct dictionary *dict, const char *id)
 {
-  return id_is_valid (id, dict->encoding, issue_error);
+  return error_to_bool (dict_id_is_valid__ (dict, id));
 }
 
 void
@@ -136,16 +229,8 @@ invalidate_proto (struct dictionary *d)
 void
 dict_dump (const struct dictionary *d)
 {
-  int i;
-  for (i = 0 ; i < d->var_cnt ; ++i )
-    {
-      const struct variable *v = d->var[i].var;
-      printf ("Name: %s;\tdict_idx: %zu; case_idx: %zu\n",
-	      var_get_name (v),
-	      var_get_dict_index (v),
-	      var_get_case_index (v));
-
-    }
+  for (size_t i = 0; i < d->n_vars; ++i)
+    printf ("%zu: %s\n", i, var_get_name (d->vars[i].var));
 }
 
 /* Associate CALLBACKS with DICT.  Callbacks will be invoked whenever
@@ -171,54 +256,51 @@ dict_copy_callbacks (struct dictionary *dest,
   dest->cb_data = src->cb_data;
 }
 
-/* Creates and returns a new dictionary. */
+/* Creates and returns a new dictionary with the specified ENCODING. */
 struct dictionary *
-dict_create (void)
+dict_create (const char *encoding)
 {
-  struct dictionary *d = xzalloc (sizeof *d);
+  struct dictionary *d = xmalloc (sizeof *d);
+
+  *d = (struct dictionary) {
+    .encoding = xstrdup (encoding),
+    .names_must_be_ids = true,
+    .name_map = HMAP_INITIALIZER (d->name_map),
+    .attributes = ATTRSET_INITIALIZER (d->attributes),
+    .split_type = SPLIT_NONE,
+    .ref_cnt = 1,
+  };
 
-  hmap_init (&d->name_map);
-  attrset_init (&d->attributes);
   return d;
 }
 
 /* Creates and returns a (deep) copy of an existing
    dictionary.
 
-   The new dictionary's case indexes are copied from the old
-   dictionary.  If the new dictionary won't be used to access
-   cases produced with the old dictionary, then the new
-   dictionary's case indexes should be compacted with
-   dict_compact_values to save space. */
+   Callbacks are not cloned. */
 struct dictionary *
 dict_clone (const struct dictionary *s)
 {
-  struct dictionary *d;
-  size_t i;
+  struct dictionary *d = dict_create (s->encoding);
+  dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));
 
-  d = dict_create ();
-
-  for (i = 0; i < s->var_cnt; i++)
+  for (size_t i = 0; i < s->n_vars; i++)
     {
-      struct variable *sv = s->var[i].var;
+      struct variable *sv = s->vars[i].var;
       struct variable *dv = dict_clone_var_assert (d, sv);
-      size_t i;
 
-      for (i = 0; i < var_get_short_name_cnt (sv); i++)
-        var_set_short_name (dv, i, var_get_short_name (sv, i));
-
-      var_get_vardict (dv)->case_index = var_get_vardict (sv)->case_index;
+      for (size_t j = 0; j < var_get_n_short_names (sv); j++)
+        var_set_short_name (dv, j, var_get_short_name (sv, j));
     }
 
-  d->next_value_idx = s->next_value_idx;
-
-  d->split_cnt = s->split_cnt;
-  if (d->split_cnt > 0)
+  d->n_splits = s->n_splits;
+  if (d->n_splits > 0)
     {
-      d->split = xnmalloc (d->split_cnt, sizeof *d->split);
-      for (i = 0; i < d->split_cnt; i++)
-        d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
+       d->split = xnmalloc (d->n_splits, sizeof *d->split);
+       for (size_t i = 0; i < d->n_splits; i++)
+         d->split[i] = dict_lookup_var_assert (d, var_get_name (s->split[i]));
     }
+  d->split_type = s->split_type;
 
   if (s->weight != NULL)
     dict_set_weight (d, dict_lookup_var_assert (d, var_get_name (s->weight)));
@@ -230,17 +312,14 @@ dict_clone (const struct dictionary *s)
   dict_set_label (d, dict_get_label (s));
   dict_set_documents (d, dict_get_documents (s));
 
-  d->vector_cnt = s->vector_cnt;
-  d->vector = xnmalloc (d->vector_cnt, sizeof *d->vector);
-  for (i = 0; i < s->vector_cnt; i++)
+  d->n_vectors = s->n_vectors;
+  d->vector = xnmalloc (d->n_vectors, sizeof *d->vector);
+  for (size_t i = 0; i < s->n_vectors; i++)
     d->vector[i] = vector_clone (s->vector[i], s, d);
 
-  if ( s->encoding)
-    d->encoding = xstrdup (s->encoding);
-
   dict_set_attributes (d, dict_get_attributes (s));
 
-  for (i = 0; i < s->n_mrsets; i++)
+  for (size_t i = 0; i < s->n_mrsets; i++)
     {
       const struct mrset *old = s->mrsets[i];
       struct mrset *new;
@@ -254,30 +333,315 @@ dict_clone (const struct dictionary *s)
       dict_add_mrset (d, new);
     }
 
+  for (size_t i = 0; i < s->n_varsets; i++)
+    {
+      const struct varset *old = s->varsets[i];
+
+      /* Clone old varset, then replace vars from D by vars from S. */
+      struct varset *new = varset_clone (old);
+      for (size_t j = 0; j < new->n_vars; j++)
+        new->vars[j] = dict_lookup_var_assert (d, var_get_name (new->vars[j]));
+
+      dict_add_varset (d, new);
+    }
+
   return d;
 }
+
+/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
+   dict_get_n_splits() to determine how many SPLIT FILE vars
+   there are.  Returns a null pointer if and only if there are no
+   SPLIT FILE vars. */
+const struct variable *const *
+dict_get_split_vars (const struct dictionary *d)
+{
+  return d->split;
+}
+
+/* Returns the number of SPLIT FILE vars. */
+size_t
+dict_get_n_splits (const struct dictionary *d)
+{
+  return d->n_splits;
+}
+
+/* Removes variable V, which must be in D, from D's set of split
+   variables. */
+static void
+dict_unset_split_var (struct dictionary *d, struct variable *v, bool skip_callbacks)
+{
+  int orig_count;
+
+  assert (dict_contains_var (d, v));
+
+  orig_count = d->n_splits;
+  d->n_splits = remove_equal (d->split, d->n_splits, sizeof *d->split,
+                               &v, compare_var_ptrs, NULL);
+  if (orig_count != d->n_splits && !skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      /* We changed the set of split variables so invoke the
+         callback. */
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
+}
+
+
+/* Sets N split vars SPLIT in dictionary D.  N is silently capped to a maximum
+   of MAX_SPLITS. */
+static void
+dict_set_split_vars__ (struct dictionary *d,
+                       struct variable *const *split, size_t n,
+                       enum split_type type, bool skip_callbacks)
+{
+  if (n > MAX_SPLITS)
+    n = MAX_SPLITS;
+  assert (n == 0 || split != NULL);
+
+  d->n_splits = n;
+  d->split_type = (n == 0 ? SPLIT_NONE
+                   : type == SPLIT_NONE ? SPLIT_LAYERED
+                   : type);
+  if (n > 0)
+    {
+      d->split = xnrealloc (d->split, n, sizeof *d->split) ;
+      memcpy (d->split, split, n * sizeof *d->split);
+    }
+  else
+    {
+      free (d->split);
+      d->split = NULL;
+    }
+
+  if (!skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->split_changed)
+        d->callbacks->split_changed (d, d->cb_data);
+    }
+}
+
+enum split_type
+dict_get_split_type (const struct dictionary *d)
+{
+  return d->split_type;
+}
+
+/* Sets N split vars SPLIT in dictionary D. */
+void
+dict_set_split_vars (struct dictionary *d,
+                     struct variable *const *split, size_t n,
+                     enum split_type type)
+{
+  dict_set_split_vars__ (d, split, n, type, false);
+}
+
+void
+dict_clear_split_vars (struct dictionary *d)
+{
+  dict_set_split_vars (d, NULL, 0, SPLIT_NONE);
+}
+
+
+/* Deletes variable V from dictionary D and frees V.
+
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
+
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+static void
+dict_delete_var__ (struct dictionary *d, struct variable *v, bool skip_callbacks)
+{
+  int dict_index = var_get_dict_index (v);
+
+  assert (dict_contains_var (d, v));
+
+  dict_unset_split_var (d, v, skip_callbacks);
+  dict_unset_mrset_var (d, v);
+  dict_unset_varset_var (d, v);
+
+  if (d->weight == v)
+    dict_set_weight (d, NULL);
+
+  if (d->filter == v)
+    dict_set_filter (d, NULL);
+
+  dict_clear_vectors (d);
+
+  /* Remove V from var array. */
+  unindex_vars (d, dict_index, d->n_vars);
+  remove_element (d->vars, d->n_vars, sizeof *d->vars, dict_index);
+  d->n_vars--;
+
+  /* Update dict_index for each affected variable. */
+  reindex_vars (d, dict_index, d->n_vars, skip_callbacks);
+
+  /* Free memory. */
+  var_clear_vardict (v);
+
+  if (! skip_callbacks)
+    {
+      if (d->changed) d->changed (d, d->changed_data);
+      if (d->callbacks &&  d->callbacks->vars_deleted)
+        d->callbacks->vars_deleted (d, dict_index, 1, d->cb_data);
+    }
+
+  invalidate_proto (d);
+  var_unref (v);
+}
+
+/* Deletes variable V from dictionary D and frees V.
+
+   This is a very bad idea if there might be any pointers to V
+   from outside D.  In general, no variable in the active dataset's
+   dictionary should be deleted when any transformations are
+   active on the dictionary's dataset, because those
+   transformations might reference the deleted variable.  The
+   safest time to delete a variable is just after a procedure has
+   been executed, as done by DELETE VARIABLES.
+
+   Pointers to V within D are not a problem, because
+   dict_delete_var() knows to remove V from split variables,
+   weights, filters, etc. */
+void
+dict_delete_var (struct dictionary *d, struct variable *v)
+{
+  dict_delete_var__ (d, v, false);
+  invalidate_proto (d);
+}
+
+
+/* Deletes the COUNT variables listed in VARS from D.  This is
+   unsafe; see the comment on dict_delete_var() for details. */
+void
+dict_delete_vars (struct dictionary *d,
+                  struct variable *const *vars, size_t count)
+{
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  assert (count == 0 || vars != NULL);
+
+  while (count-- > 0)
+    dict_delete_var (d, *vars++);
+  invalidate_proto (d);
+}
+
+/* Deletes the COUNT variables in D starting at index IDX.  This
+   is unsafe; see the comment on dict_delete_var() for
+   details. Deleting consecutive vars will result in less callbacks
+   compared to iterating over dict_delete_var.
+   A simple while loop over dict_delete_var will
+   produce (d->n_vars - IDX) * COUNT variable changed callbacks
+   plus COUNT variable delete callbacks.
+   This here produces d->n_vars - IDX variable changed callbacks
+   plus COUNT variable delete callbacks. */
+void
+dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
+{
+  assert (idx + count <= d->n_vars);
+
+  struct variable **vars = xnmalloc (count, sizeof *vars);
+
+  for (size_t i = 0; i < count; i++)
+    {
+      struct variable *v = d->vars[idx + i].var;
+      vars[i] = v;
+
+      dict_unset_split_var (d, v, false);
+      dict_unset_mrset_var (d, v);
+      dict_unset_varset_var (d, v);
+
+      if (d->weight == v)
+	dict_set_weight (d, NULL);
+
+      if (d->filter == v)
+	dict_set_filter (d, NULL);
+    }
+
+  dict_clear_vectors (d);
+
+  /* Remove variables from var array. */
+  unindex_vars (d, idx, d->n_vars);
+  remove_range (d->vars, d->n_vars, sizeof *d->vars, idx, count);
+  d->n_vars -= count;
+
+  /* Reindexing will result variable-changed callback */
+  reindex_vars (d, idx, d->n_vars, false);
+
+  invalidate_proto (d);
+  if (d->changed) d->changed (d, d->changed_data);
+
+  /* Now issue the variable delete callbacks and delete
+     the variables. The vardict is not valid at this point
+     anymore. */
+  if (d->callbacks &&  d->callbacks->vars_deleted)
+    d->callbacks->vars_deleted (d, idx, count, d->cb_data);
+  for (size_t i = 0; i < count; i++)
+    {
+      var_clear_vardict (vars[i]);
+      var_unref (vars[i]);
+    }
+  free (vars);
+
+  invalidate_proto (d);
+}
+
+/* Deletes scratch variables from dictionary D. */
+void
+dict_delete_scratch_vars (struct dictionary *d)
+{
+  int i;
+
+  /* FIXME: this can be done in O(count) time, but this algorithm
+     is O(count**2). */
+  for (i = 0; i < d->n_vars;)
+    if (var_get_dict_class (d->vars[i].var) == DC_SCRATCH)
+      dict_delete_var (d, d->vars[i].var);
+    else
+      i++;
+
+  invalidate_proto (d);
+}
+
+
 
 /* Clears the contents from a dictionary without destroying the
    dictionary itself. */
-void
-dict_clear (struct dictionary *d)
+static void
+dict_clear__ (struct dictionary *d, bool skip_callbacks)
 {
   /* FIXME?  Should we really clear case_limit, label, documents?
      Others are necessarily cleared by deleting all the variables.*/
-  while (d->var_cnt > 0 )
+  while (d->n_vars > 0)
     {
-      dict_delete_var (d, d->var[d->var_cnt - 1].var);
+      dict_delete_var__ (d, d->vars[d->n_vars - 1].var, skip_callbacks);
     }
 
-  free (d->var);
-  d->var = NULL;
-  d->var_cnt = d->var_cap = 0;
+  free (d->vars);
+  d->vars = NULL;
+  d->n_vars = d->allocated_vars = 0;
   invalidate_proto (d);
   hmap_clear (&d->name_map);
-  d->next_value_idx = 0;
-  dict_set_split_vars (d, NULL, 0);
-  dict_set_weight (d, NULL);
-  dict_set_filter (d, NULL);
+  dict_set_split_vars__ (d, NULL, 0, SPLIT_NONE, skip_callbacks);
+
+  if (skip_callbacks)
+    {
+      d->weight = NULL;
+      d->filter = NULL;
+    }
+  else
+    {
+      dict_set_weight (d, NULL);
+      dict_set_filter (d, NULL);
+    }
   d->case_limit = 0;
   free (d->label);
   d->label = NULL;
@@ -286,74 +650,88 @@ dict_clear (struct dictionary *d)
   attrset_clear (&d->attributes);
 }
 
-/* Destroys the aux data for every variable in D, by calling
-   var_clear_aux() for each variable. */
+/* Clears the contents from a dictionary without destroying the
+   dictionary itself. */
 void
-dict_clear_aux (struct dictionary *d)
+dict_clear (struct dictionary *d)
 {
-  int i;
-
-  for (i = 0; i < d->var_cnt; i++)
-    var_clear_aux (d->var[i].var);
+  dict_clear__ (d, false);
 }
 
 /* Clears a dictionary and destroys it. */
+static void
+_dict_destroy (struct dictionary *d)
+{
+  /* In general, we don't want callbacks occurring, if the dictionary
+     is being destroyed */
+  d->callbacks  = NULL ;
+
+  dict_clear__ (d, true);
+  string_array_destroy (&d->documents);
+  hmap_destroy (&d->name_map);
+  attrset_destroy (&d->attributes);
+  dict_clear_mrsets (d);
+  dict_clear_varsets (d);
+  free (d->encoding);
+  free (d);
+}
+
+struct dictionary *
+dict_ref (struct dictionary *d)
+{
+  d->ref_cnt++;
+  return d;
+}
+
 void
-dict_destroy (struct dictionary *d)
+dict_unref (struct dictionary *d)
 {
-  if (d != NULL)
-    {
-      /* In general, we don't want callbacks occuring, if the dictionary
-	 is being destroyed */
-      d->callbacks  = NULL ;
-
-      dict_clear (d);
-      hmap_destroy (&d->name_map);
-      attrset_destroy (&d->attributes);
-      dict_clear_mrsets (d);
-      free (d->encoding);
-      free (d);
-    }
+  if (d == NULL)
+    return;
+  d->ref_cnt--;
+  assert (d->ref_cnt >= 0);
+  if (d->ref_cnt == 0)
+    _dict_destroy (d);
 }
 
 /* Returns the number of variables in D. */
 size_t
-dict_get_var_cnt (const struct dictionary *d)
+dict_get_n_vars (const struct dictionary *d)
 {
-  return d->var_cnt;
+  return d->n_vars;
 }
 
 /* Returns the variable in D with dictionary index IDX, which
    must be between 0 and the count returned by
-   dict_get_var_cnt(), exclusive. */
+   dict_get_n_vars(), exclusive. */
 struct variable *
 dict_get_var (const struct dictionary *d, size_t idx)
 {
-  assert (idx < d->var_cnt);
+  assert (idx < d->n_vars);
 
-  return d->var[idx].var;
+  return d->vars[idx].var;
 }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
+/* Sets *VARS to an array of pointers to variables in D and *N
    to the number of variables in *D.  All variables are returned
    except for those, if any, in the classes indicated by EXCLUDE.
    (There is no point in putting DC_SYSTEM in EXCLUDE as
    dictionaries never include system variables.) */
 void
 dict_get_vars (const struct dictionary *d, const struct variable ***vars,
-               size_t *cnt, enum dict_class exclude)
+               size_t *n, enum dict_class exclude)
 {
-  dict_get_vars_mutable (d, (struct variable ***) vars, cnt, exclude);
+  dict_get_vars_mutable (d, (struct variable ***) vars, n, exclude);
 }
 
-/* Sets *VARS to an array of pointers to variables in D and *CNT
+/* Sets *VARS to an array of pointers to variables in D and *N
    to the number of variables in *D.  All variables are returned
    except for those, if any, in the classes indicated by EXCLUDE.
    (There is no point in putting DC_SYSTEM in EXCLUDE as
    dictionaries never include system variables.) */
 void
 dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
-                       size_t *cnt, enum dict_class exclude)
+                       size_t *n, enum dict_class exclude)
 {
   size_t count;
   size_t i;
@@ -361,57 +739,55 @@ dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
   assert (exclude == (exclude & DC_ALL));
 
   count = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  for (i = 0; i < d->n_vars; i++)
     {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
       if (!(class & exclude))
         count++;
     }
 
   *vars = xnmalloc (count, sizeof **vars);
-  *cnt = 0;
-  for (i = 0; i < d->var_cnt; i++)
+  *n = 0;
+  for (i = 0; i < d->n_vars; i++)
     {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
       if (!(class & exclude))
-        (*vars)[(*cnt)++] = d->var[i].var;
+        (*vars)[(*n)++] = d->vars[i].var;
     }
-  assert (*cnt == count);
+  assert (*n == count);
 }
 
 static struct variable *
 add_var (struct dictionary *d, struct variable *v)
 {
-  struct vardict_info *vardict;
-
   /* Update dictionary. */
-  if (d->var_cnt >= d->var_cap)
+  if (d->n_vars >= d->allocated_vars)
     {
       size_t i;
 
-      d->var = x2nrealloc (d->var, &d->var_cap, sizeof *d->var);
+      d->vars = x2nrealloc (d->vars, &d->allocated_vars, sizeof *d->vars);
       hmap_clear (&d->name_map);
-      for (i = 0; i < d->var_cnt; i++)
+      for (i = 0; i < d->n_vars; i++)
         {
-          var_set_vardict (d->var[i].var, &d->var[i]);
-          hmap_insert_fast (&d->name_map, &d->var[i].name_node,
-                            d->var[i].name_node.hash);
+          var_set_vardict (d->vars[i].var, &d->vars[i]);
+          hmap_insert_fast (&d->name_map, &d->vars[i].name_node,
+                            d->vars[i].name_node.hash);
         }
     }
 
-  vardict = &d->var[d->var_cnt++];
-  vardict->dict = d;
-  vardict->var = v;
+  struct vardict_info *vardict = &d->vars[d->n_vars++];
+  *vardict = (struct vardict_info) {
+    .dict = d,
+    .var = v,
+  };
   hmap_insert (&d->name_map, &vardict->name_node,
-               hash_case_string (var_get_name (v), 0));
-  vardict->case_index = d->next_value_idx;
+               utf8_hash_case_string (var_get_name (v), 0));
   var_set_vardict (v, vardict);
 
-  if ( d->changed ) d->changed (d, d->changed_data);
-  if ( d->callbacks &&  d->callbacks->var_added )
+  if (d->changed) d->changed (d, d->changed_data);
+  if (d->callbacks &&  d->callbacks->var_added)
     d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
 
-  d->next_value_idx++;
   invalidate_proto (d);
 
   return v;
@@ -480,207 +856,46 @@ dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
                           const char *name)
 {
   struct variable *new_var = var_clone (old_var);
-  assert (dict_lookup_var (d, name) == NULL);
-  var_set_name (new_var, name);
-  return add_var (d, new_var);
-}
-
-/* Returns the variable named NAME in D, or a null pointer if no
-   variable has that name. */
-struct variable *
-dict_lookup_var (const struct dictionary *d, const char *name)
-{
-  struct vardict_info *vardict;
-
-  HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
-                           hash_case_string (name, 0), &d->name_map)
-    {
-      struct variable *var = vardict->var;
-      if (!strcasecmp (var_get_name (var), name))
-        return var;
-    }
-
-  return NULL;
-}
-
-/* Returns the variable named NAME in D.  Assert-fails if no
-   variable has that name. */
-struct variable *
-dict_lookup_var_assert (const struct dictionary *d, const char *name)
-{
-  struct variable *v = dict_lookup_var (d, name);
-  assert (v != NULL);
-  return v;
-}
-
-/* Returns true if variable V is in dictionary D,
-   false otherwise. */
-bool
-dict_contains_var (const struct dictionary *d, const struct variable *v)
-{
-  return (var_has_vardict (v)
-          && vardict_get_dictionary (var_get_vardict (v)) == d);
-}
-
-/* Compares two double pointers to variables, which should point
-   to elements of a struct dictionary's `var' member array. */
-static int
-compare_var_ptrs (const void *a_, const void *b_, const void *aux UNUSED)
-{
-  struct variable *const *a = a_;
-  struct variable *const *b = b_;
-
-  return *a < *b ? -1 : *a > *b;
-}
-
-static void
-unindex_var (struct dictionary *d, struct vardict_info *vardict)
-{
-  hmap_delete (&d->name_map, &vardict->name_node);
-}
-
-/* This function assumes that vardict->name_node.hash is valid, that is, that
-   its name has not changed since it was hashed (rename_var() updates this
-   hash along with the name itself). */
-static void
-reindex_var (struct dictionary *d, struct vardict_info *vardict)
-{
-  struct variable *var = vardict->var;
-
-  var_set_vardict (var, vardict);
-  hmap_insert_fast (&d->name_map, &vardict->name_node,
-                    vardict->name_node.hash);
-
-  if ( d->changed ) d->changed (d, d->changed_data);
-  if ( d->callbacks &&  d->callbacks->var_changed )
-    d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
-}
-
-/* Sets the case_index in V's vardict to CASE_INDEX. */
-static void
-set_var_case_index (struct variable *v, int case_index)
-{
-  var_get_vardict (v)->case_index = case_index;
-}
-
-/* Removes the dictionary variables with indexes from FROM to TO (exclusive)
-   from name_map. */
-static void
-unindex_vars (struct dictionary *d, size_t from, size_t to)
-{
-  size_t i;
-
-  for (i = from; i < to; i++)
-    unindex_var (d, &d->var[i]);
-}
-
-/* Re-sets the dict_index in the dictionary variables with
-   indexes from FROM to TO (exclusive). */
-static void
-reindex_vars (struct dictionary *d, size_t from, size_t to)
-{
-  size_t i;
-
-  for (i = from; i < to; i++)
-    reindex_var (d, &d->var[i]);
-}
-
-/* Deletes variable V from dictionary D and frees V.
-
-   This is a very bad idea if there might be any pointers to V
-   from outside D.  In general, no variable in the active dataset's
-   dictionary should be deleted when any transformations are
-   active on the dictionary's dataset, because those
-   transformations might reference the deleted variable.  The
-   safest time to delete a variable is just after a procedure has
-   been executed, as done by DELETE VARIABLES.
-
-   Pointers to V within D are not a problem, because
-   dict_delete_var() knows to remove V from split variables,
-   weights, filters, etc. */
-void
-dict_delete_var (struct dictionary *d, struct variable *v)
-{
-  int dict_index = var_get_dict_index (v);
-  const int case_index = var_get_case_index (v);
-  const int width = var_get_width (v);
-
-  assert (dict_contains_var (d, v));
-
-  /* Delete aux data. */
-  var_clear_aux (v);
-
-  dict_unset_split_var (d, v);
-  dict_unset_mrset_var (d, v);
-
-  if (d->weight == v)
-    dict_set_weight (d, NULL);
-
-  if (d->filter == v)
-    dict_set_filter (d, NULL);
-
-  dict_clear_vectors (d);
-
-  /* Remove V from var array. */
-  unindex_vars (d, dict_index, d->var_cnt);
-  remove_element (d->var, d->var_cnt, sizeof *d->var, dict_index);
-  d->var_cnt--;
-
-  /* Update dict_index for each affected variable. */
-  reindex_vars (d, dict_index, d->var_cnt);
-
-  /* Free memory. */
-  var_clear_vardict (v);
-  var_destroy (v);
-
-  if ( d->changed ) d->changed (d, d->changed_data);
-
-  invalidate_proto (d);
-  if (d->callbacks &&  d->callbacks->var_deleted )
-    d->callbacks->var_deleted (d, dict_index, case_index, width, d->cb_data);
-}
-
-/* Deletes the COUNT variables listed in VARS from D.  This is
-   unsafe; see the comment on dict_delete_var() for details. */
-void
-dict_delete_vars (struct dictionary *d,
-                  struct variable *const *vars, size_t count)
-{
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  assert (count == 0 || vars != NULL);
-
-  while (count-- > 0)
-    dict_delete_var (d, *vars++);
+  assert (dict_lookup_var (d, name) == NULL);
+  var_set_name (new_var, name);
+  return add_var (d, new_var);
 }
 
-/* Deletes the COUNT variables in D starting at index IDX.  This
-   is unsafe; see the comment on dict_delete_var() for
-   details. */
-void
-dict_delete_consecutive_vars (struct dictionary *d, size_t idx, size_t count)
+/* Returns the variable named NAME in D, or a null pointer if no
+   variable has that name. */
+struct variable *
+dict_lookup_var (const struct dictionary *d, const char *name)
 {
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  assert (idx + count <= d->var_cnt);
+  struct vardict_info *vardict;
 
-  while (count-- > 0)
-    dict_delete_var (d, d->var[idx].var);
+  HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
+                           utf8_hash_case_string (name, 0), &d->name_map)
+    {
+      struct variable *var = vardict->var;
+      if (!utf8_strcasecmp (var_get_name (var), name))
+        return var;
+    }
+
+  return NULL;
 }
 
-/* Deletes scratch variables from dictionary D. */
-void
-dict_delete_scratch_vars (struct dictionary *d)
+/* Returns the variable named NAME in D.  Assert-fails if no
+   variable has that name. */
+struct variable *
+dict_lookup_var_assert (const struct dictionary *d, const char *name)
 {
-  int i;
+  struct variable *v = dict_lookup_var (d, name);
+  assert (v != NULL);
+  return v;
+}
 
-  /* FIXME: this can be done in O(count) time, but this algorithm
-     is O(count**2). */
-  for (i = 0; i < d->var_cnt; )
-    if (var_get_dict_class (d->var[i].var) == DC_SCRATCH)
-      dict_delete_var (d, d->var[i].var);
-    else
-      i++;
+/* Returns true if variable V is in dictionary D,
+   false otherwise. */
+bool
+dict_contains_var (const struct dictionary *d, const struct variable *v)
+{
+  return (var_has_vardict (v)
+          && vardict_get_dictionary (var_get_vardict (v)) == d);
 }
 
 /* Moves V to 0-based position IDX in D.  Other variables in D,
@@ -689,13 +904,18 @@ dict_delete_scratch_vars (struct dictionary *d)
 void
 dict_reorder_var (struct dictionary *d, struct variable *v, size_t new_index)
 {
-  size_t old_index = var_get_dict_index (v);
+  assert (new_index < d->n_vars);
 
-  assert (new_index < d->var_cnt);
+  size_t old_index = var_get_dict_index (v);
+  if (new_index == old_index)
+    return;
 
   unindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
-  move_element (d->var, d->var_cnt, sizeof *d->var, old_index, new_index);
-  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1);
+  move_element (d->vars, d->n_vars, sizeof *d->vars, old_index, new_index);
+  reindex_vars (d, MIN (old_index, new_index), MAX (old_index, new_index) + 1, false);
+
+  if (d->callbacks && d->callbacks->var_moved)
+    d->callbacks->var_moved (d, new_index, old_index, d->cb_data);
 }
 
 /* Reorders the variables in D, placing the COUNT variables
@@ -710,9 +930,9 @@ dict_reorder_vars (struct dictionary *d,
   size_t i;
 
   assert (count == 0 || order != NULL);
-  assert (count <= d->var_cnt);
+  assert (count <= d->n_vars);
 
-  new_var = xnmalloc (d->var_cap, sizeof *new_var);
+  new_var = xnmalloc (d->allocated_vars, sizeof *new_var);
 
   /* Add variables in ORDER to new_var. */
   for (i = 0; i < count; i++)
@@ -727,17 +947,17 @@ dict_reorder_vars (struct dictionary *d,
     }
 
   /* Add remaining variables to new_var. */
-  for (i = 0; i < d->var_cnt; i++)
-    if (d->var[i].dict != NULL)
-      new_var[count++] = d->var[i];
-  assert (count == d->var_cnt);
+  for (i = 0; i < d->n_vars; i++)
+    if (d->vars[i].dict != NULL)
+      new_var[count++] = d->vars[i];
+  assert (count == d->n_vars);
 
   /* Replace old vardicts by new ones. */
-  free (d->var);
-  d->var = new_var;
+  free (d->vars);
+  d->vars = new_var;
 
   hmap_clear (&d->name_map);
-  reindex_vars (d, 0, d->var_cnt);
+  reindex_vars (d, 0, d->n_vars, false);
 }
 
 /* Changes the name of variable V that is currently in a dictionary to
@@ -748,30 +968,47 @@ rename_var (struct variable *v, const char *new_name)
   struct vardict_info *vardict = var_get_vardict (v);
   var_clear_vardict (v);
   var_set_name (v, new_name);
-  vardict->name_node.hash = hash_case_string (new_name, 0);
+  vardict->name_node.hash = utf8_hash_case_string (new_name, 0);
   var_set_vardict (v, vardict);
 }
 
-/* Changes the name of V in D to name NEW_NAME.  Assert-fails if
-   a variable named NEW_NAME is already in D, except that
-   NEW_NAME may be the same as V's existing name. */
-void
-dict_rename_var (struct dictionary *d, struct variable *v,
-                 const char *new_name)
+/* Tries to changes the name of V in D to name NEW_NAME.  Returns true if
+   successful, false if a variable (other than V) with the given name already
+   exists in D. */
+bool
+dict_try_rename_var (struct dictionary *d, struct variable *v,
+                     const char *new_name)
 {
-  assert (!strcasecmp (var_get_name (v), new_name)
-          || dict_lookup_var (d, new_name) == NULL);
+  struct variable *conflict = dict_lookup_var (d, new_name);
+  if (conflict && v != conflict)
+    return false;
 
+  struct variable *old = var_clone (v);
   unindex_var (d, var_get_vardict (v));
   rename_var (v, new_name);
-  reindex_var (d, var_get_vardict (v));
+  reindex_var (d, var_get_vardict (v), false);
 
   if (settings_get_algorithm () == ENHANCED)
     var_clear_short_names (v);
 
-  if ( d->changed ) d->changed (d, d->changed_data);
-  if ( d->callbacks &&  d->callbacks->var_changed )
-    d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
+  if (d->changed) d->changed (d, d->changed_data);
+  if (d->callbacks &&  d->callbacks->var_changed)
+    d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
+
+  var_unref (old);
+
+  return true;
+}
+
+/* Changes the name of V in D to name NEW_NAME.  Assert-fails if
+   a variable named NEW_NAME is already in D, except that
+   NEW_NAME may be the same as V's existing name. */
+void
+dict_rename_var (struct dictionary *d, struct variable *v,
+                 const char *new_name)
+{
+  bool ok UNUSED = dict_try_rename_var (d, v, new_name);
+  assert (ok);
 }
 
 /* Renames COUNT variables specified in VARS to the names given
@@ -825,13 +1062,13 @@ dict_rename_vars (struct dictionary *d,
           for (i = 0; i < count; i++)
             {
               rename_var (vars[i], old_names[i]);
-              reindex_var (d, var_get_vardict (vars[i]));
+              reindex_var (d, var_get_vardict (vars[i]), false);
             }
 
           pool_destroy (pool);
           return false;
         }
-      reindex_var (d, var_get_vardict (vars[i]));
+      reindex_var (d, var_get_vardict (vars[i]), false);
     }
 
   /* Clear short names. */
@@ -864,6 +1101,9 @@ make_hinted_name (const struct dictionary *dict, const char *hint)
   size_t ofs;
   int mblen;
 
+  if (hint_len > ID_MAX_LEN)
+    hint_len = ID_MAX_LEN;
+
   /* The allocation size here is OK: characters that are copied directly fit
      OK, and characters that are not copied directly are replaced by a single
      '_' byte.  If u8_mbtouc() replaces bad input by 0xfffd, then that will get
@@ -904,7 +1144,7 @@ make_hinted_name (const struct dictionary *dict, const char *hint)
           char *name;
 
           suffix[0] = '_';
-          if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
+          if (!str_format_26adic (i + 1, true, &suffix[1], sizeof suffix - 1))
             NOT_REACHED ();
 
           name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
@@ -970,9 +1210,31 @@ dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
       if (hinted_name != NULL)
         return hinted_name;
     }
+
   return make_numeric_name (dict, num_start);
 }
 
+/* Returns whether variable names must be valid identifiers.  Normally, this is
+   true, but sometimes a dictionary is prepared for external use (e.g. output
+   to a CSV file) where names don't have to be valid. */
+bool
+dict_get_names_must_be_ids (const struct dictionary *d)
+{
+  return d->names_must_be_ids;
+}
+
+/* Sets whether variable names must be valid identifiers.  Normally, this is
+   true, but sometimes a dictionary is prepared for external use (e.g. output
+   to a CSV file) where names don't have to be valid.
+
+   Changing this setting from false to true doesn't make the dictionary check
+   all the existing variable names, so it can cause an invariant violation. */
+void
+dict_set_names_must_be_ids (struct dictionary *d, bool names_must_be_ids)
+{
+  d->names_must_be_ids = names_must_be_ids;
+}
+
 /* Returns the weighting variable in dictionary D, or a null
    pointer if the dictionary is unweighted. */
 struct variable *
@@ -984,7 +1246,7 @@ dict_get_weight (const struct dictionary *d)
 }
 
 /* Returns the value of D's weighting variable in case C, except
-   that a negative weight is returned as 0.  Returns 1 if the
+   that a negative or missing weight is returned as 0.  Returns 1 if the
    dictionary is unweighted.  Will warn about missing, negative,
    or zero values if *WARN_ON_INVALID is true.  The function will
    set *WARN_ON_INVALID to false if an invalid weight is
@@ -1000,18 +1262,27 @@ dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
   else
     {
       double w = case_num (c, d->weight);
-      if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
-        w = 0.0;
-      if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
-	  *warn_on_invalid = false;
-	  msg (SW, _("At least one case in the data file had a weight value "
-		     "that was user-missing, system-missing, zero, or "
-		     "negative.  These case(s) were ignored."));
-      }
-      return w;
+
+      return var_force_valid_weight (d->weight, w, warn_on_invalid);
     }
 }
 
+/* Like dict_get_case_weight(), but additionally rounds each weight to the
+   nearest integer.  */
+double
+dict_get_rounded_case_weight (const struct dictionary *d,
+                              const struct ccase *c, bool *warn_on_invalid)
+{
+  return floor (dict_get_case_weight (d, c, warn_on_invalid) + 0.5);
+}
+
+/* Returns the format to use for weights. */
+struct fmt_spec
+dict_get_weight_format (const struct dictionary *d)
+{
+  return d->weight ? var_get_print_format (d->weight) : F_8_0;
+}
+
 /* Sets the weighting variable of D to V, or turning off
    weighting if V is a null pointer. */
 void
@@ -1023,10 +1294,10 @@ dict_set_weight (struct dictionary *d, struct variable *v)
   d->weight = v;
 
   if (d->changed) d->changed (d, d->changed_data);
-  if ( d->callbacks &&  d->callbacks->weight_changed )
+  if (d->callbacks &&  d->callbacks->weight_changed)
     d->callbacks->weight_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                  d->cb_data);
 }
 
 /* Returns the filter variable in dictionary D (see cmd_filter())
@@ -1050,10 +1321,10 @@ dict_set_filter (struct dictionary *d, struct variable *v)
   d->filter = v;
 
   if (d->changed) d->changed (d, d->changed_data);
-  if ( d->callbacks && d->callbacks->filter_changed )
+  if (d->callbacks && d->callbacks->filter_changed)
     d->callbacks->filter_changed (d,
-				  v ? var_get_dict_index (v) : -1,
-				  d->cb_data);
+                                  v ? var_get_dict_index (v) : -1,
+                                      d->cb_data);
 }
 
 /* Returns the case limit for dictionary D, or zero if the number
@@ -1079,170 +1350,31 @@ dict_get_proto (const struct dictionary *d_)
   struct dictionary *d = CONST_CAST (struct dictionary *, d_);
   if (d->proto == NULL)
     {
-      size_t i;
-
-      d->proto = caseproto_create ();
-      d->proto = caseproto_reserve (d->proto, d->var_cnt);
-      for (i = 0; i < d->var_cnt; i++)
-        d->proto = caseproto_set_width (d->proto,
-                                        var_get_case_index (d->var[i].var),
-                                        var_get_width (d->var[i].var));
+      short int *widths = xnmalloc (d->n_vars, sizeof *widths);
+      for (size_t i = 0; i < d->n_vars; i++)
+        widths[i] = var_get_width (d->vars[i].var);
+      d->proto = caseproto_from_widths (widths, d->n_vars);
     }
   return d->proto;
 }
 
-/* Returns the case index of the next value to be added to D.
-   This value is the number of `union value's that need to be
-   allocated to store a case for dictionary D. */
-int
-dict_get_next_value_idx (const struct dictionary *d)
-{
-  return d->next_value_idx;
-}
-
-/* Returns the number of bytes needed to store a case for
-   dictionary D. */
-size_t
-dict_get_case_size (const struct dictionary *d)
-{
-  return sizeof (union value) * dict_get_next_value_idx (d);
-}
-
-/* Reassigns values in dictionary D so that fragmentation is
-   eliminated. */
-void
-dict_compact_values (struct dictionary *d)
-{
-  size_t i;
-
-  d->next_value_idx = 0;
-  for (i = 0; i < d->var_cnt; i++)
-    {
-      struct variable *v = d->var[i].var;
-      set_var_case_index (v, d->next_value_idx++);
-    }
-  invalidate_proto (d);
-}
-
 /* Returns the number of values occupied by the variables in
    dictionary D.  All variables are considered if EXCLUDE_CLASSES
-   is 0, or it may contain one or more of (1u << DC_ORDINARY),
-   (1u << DC_SYSTEM), or (1u << DC_SCRATCH) to exclude the
-   corresponding type of variable.
-
-   The return value may be less than the number of values in one
-   of dictionary D's cases (as returned by
-   dict_get_next_value_idx) even if E is 0, because there may be
-   gaps in D's cases due to deleted variables. */
+   is 0, or it may contain one or more of DC_ORDINARY, DC_SYSTEM,
+   or DC_SCRATCH to exclude the corresponding type of variable. */
 size_t
 dict_count_values (const struct dictionary *d, unsigned int exclude_classes)
 {
-  size_t i;
-  size_t cnt;
-
-  assert ((exclude_classes & ~((1u << DC_ORDINARY)
-                               | (1u << DC_SYSTEM)
-                               | (1u << DC_SCRATCH))) == 0);
-
-  cnt = 0;
-  for (i = 0; i < d->var_cnt; i++)
-    {
-      enum dict_class class = var_get_dict_class (d->var[i].var);
-      if (!(exclude_classes & (1u << class)))
-        cnt++;
-    }
-  return cnt;
-}
-
-/* Returns the case prototype that would result after deleting
-   all variables from D that are not in one of the
-   EXCLUDE_CLASSES and compacting the dictionary with
-   dict_compact().
-
-   The caller must unref the returned caseproto when it is no
-   longer needed. */
-struct caseproto *
-dict_get_compacted_proto (const struct dictionary *d,
-                          unsigned int exclude_classes)
-{
-  struct caseproto *proto;
-  size_t i;
-
-  assert ((exclude_classes & ~((1u << DC_ORDINARY)
-                               | (1u << DC_SYSTEM)
-                               | (1u << DC_SCRATCH))) == 0);
-
-  proto = caseproto_create ();
-  for (i = 0; i < d->var_cnt; i++)
-    {
-      struct variable *v = d->var[i].var;
-      if (!(exclude_classes & (1u << var_get_dict_class (v))))
-        proto = caseproto_add_width (proto, var_get_width (v));
-    }
-  return proto;
-}
-
-/* Returns the SPLIT FILE vars (see cmd_split_file()).  Call
-   dict_get_split_cnt() to determine how many SPLIT FILE vars
-   there are.  Returns a null pointer if and only if there are no
-   SPLIT FILE vars. */
-const struct variable *const *
-dict_get_split_vars (const struct dictionary *d)
-{
-  return d->split;
-}
-
-/* Returns the number of SPLIT FILE vars. */
-size_t
-dict_get_split_cnt (const struct dictionary *d)
-{
-  return d->split_cnt;
-}
-
-/* Removes variable V, which must be in D, from D's set of split
-   variables. */
-static void
-dict_unset_split_var (struct dictionary *d, struct variable *v)
-{
-  int orig_count;
-
-  assert (dict_contains_var (d, v));
+  assert (!(exclude_classes & ~DC_ALL));
 
-  orig_count = d->split_cnt;
-  d->split_cnt = remove_equal (d->split, d->split_cnt, sizeof *d->split,
-                               &v, compare_var_ptrs, NULL);
-  if (orig_count != d->split_cnt)
+  size_t n = 0;
+  for (size_t i = 0; i < d->n_vars; i++)
     {
-      if (d->changed) d->changed (d, d->changed_data);
-      /* We changed the set of split variables so invoke the
-         callback. */
-      if (d->callbacks &&  d->callbacks->split_changed)
-        d->callbacks->split_changed (d, d->cb_data);
+      enum dict_class class = var_get_dict_class (d->vars[i].var);
+      if (!(exclude_classes & class))
+        n++;
     }
-}
-
-/* Sets CNT split vars SPLIT in dictionary D. */
-void
-dict_set_split_vars (struct dictionary *d,
-                     struct variable *const *split, size_t cnt)
-{
-  assert (cnt == 0 || split != NULL);
-
-  d->split_cnt = cnt;
-  if ( cnt > 0 )
-   {
-    d->split = xnrealloc (d->split, cnt, sizeof *d->split) ;
-    memcpy (d->split, split, cnt * sizeof *d->split);
-   }
-  else
-   {
-    free (d->split);
-    d->split = NULL;
-   }
-
-  if (d->changed) d->changed (d, d->changed_data);
-  if ( d->callbacks &&  d->callbacks->split_changed )
-    d->callbacks->split_changed (d, d->cb_data);
+  return n;
 }
 
 /* Returns the file label for D, or a null pointer if D is
@@ -1253,15 +1385,18 @@ dict_get_label (const struct dictionary *d)
   return d->label;
 }
 
-/* Sets D's file label to LABEL, truncating it to a maximum of 60
-   characters.
+/* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
+   encoding.
 
    Removes D's label if LABEL is null or the empty string. */
 void
 dict_set_label (struct dictionary *d, const char *label)
 {
   free (d->label);
-  d->label = label != NULL && label[0] != '\0' ? xstrndup (label, 60) : NULL;
+  if (label == NULL || label[0] == '\0')
+    d->label = NULL;
+  else
+    d->label = utf8_encoding_trunc (label, d->encoding, 60);
 }
 
 /* Returns the documents for D, as an UTF-8 encoded string_array.  The
@@ -1277,12 +1412,15 @@ dict_get_documents (const struct dictionary *d)
 void
 dict_set_documents (struct dictionary *d, const struct string_array *new_docs)
 {
-  size_t i;
-
-  dict_clear_documents (d);
+  /* Swap out the old documents, instead of destroying them immediately, to
+     allow the new documents to include pointers into the old ones. */
+  struct string_array old_docs = STRING_ARRAY_INITIALIZER;
+  string_array_swap (&d->documents, &old_docs);
 
-  for (i = 0; i < new_docs->n; i++)
+  for (size_t i = 0; i < new_docs->n; i++)
     dict_add_document_line (d, new_docs->strings[i], false);
+
+  string_array_destroy (&old_docs);
 }
 
 /* Replaces the documents for D by UTF-8 encoded string NEW_DOCS, dividing it
@@ -1294,7 +1432,7 @@ dict_set_documents_string (struct dictionary *d, const char *new_docs)
   const char *s;
 
   dict_clear_documents (d);
-  for (s = new_docs; *s != '\0'; )
+  for (s = new_docs; *s != '\0';)
     {
       size_t len = strcspn (s, "\n");
       char *line = xmemdup0 (s, len);
@@ -1329,7 +1467,7 @@ dict_add_document_line (struct dictionary *d, const char *line,
   truncated = line[trunc_len] != '\0';
   if (truncated && issue_warning)
     {
-      /* Note to translators: "bytes" is correct, not characters */
+      /* TRANSLATORS: "bytes" is correct, not characters due to UTF encoding */
       msg (SW, _("Truncating document line to %d bytes."), DOC_LINE_LENGTH);
     }
 
@@ -1340,7 +1478,7 @@ dict_add_document_line (struct dictionary *d, const char *line,
 
 /* Returns the number of document lines in dictionary D. */
 size_t
-dict_get_document_line_cnt (const struct dictionary *d)
+dict_get_document_n_lines (const struct dictionary *d)
 {
   return d->documents.n;
 }
@@ -1354,57 +1492,55 @@ dict_get_document_line (const struct dictionary *d, size_t idx)
   return d->documents.strings[idx];
 }
 
-/* Creates in D a vector named NAME that contains the CNT
+/* Creates in D a vector named NAME that contains the N
    variables in VAR.  Returns true if successful, or false if a
    vector named NAME already exists in D. */
 bool
 dict_create_vector (struct dictionary *d,
                     const char *name,
-                    struct variable **var, size_t cnt)
+                    struct variable **var, size_t n)
 {
-  size_t i;
-
-  assert (cnt > 0);
-  for (i = 0; i < cnt; i++)
+  assert (n > 0);
+  for (size_t i = 0; i < n; i++)
     assert (dict_contains_var (d, var[i]));
 
   if (dict_lookup_vector (d, name) == NULL)
     {
-      d->vector = xnrealloc (d->vector, d->vector_cnt + 1, sizeof *d->vector);
-      d->vector[d->vector_cnt++] = vector_create (name, var, cnt);
+      d->vector = xnrealloc (d->vector, d->n_vectors + 1, sizeof *d->vector);
+      d->vector[d->n_vectors++] = vector_create (name, var, n);
       return true;
     }
   else
     return false;
 }
 
-/* Creates in D a vector named NAME that contains the CNT
+/* Creates in D a vector named NAME that contains the N
    variables in VAR.  A vector named NAME must not already exist
    in D. */
 void
 dict_create_vector_assert (struct dictionary *d,
                            const char *name,
-                           struct variable **var, size_t cnt)
+                           struct variable **var, size_t n)
 {
   assert (dict_lookup_vector (d, name) == NULL);
-  dict_create_vector (d, name, var, cnt);
+  dict_create_vector (d, name, var, n);
 }
 
 /* Returns the vector in D with index IDX, which must be less
-   than dict_get_vector_cnt (D). */
+   than dict_get_n_vectors (D). */
 const struct vector *
 dict_get_vector (const struct dictionary *d, size_t idx)
 {
-  assert (idx < d->vector_cnt);
+  assert (idx < d->n_vectors);
 
   return d->vector[idx];
 }
 
 /* Returns the number of vectors in D. */
 size_t
-dict_get_vector_cnt (const struct dictionary *d)
+dict_get_n_vectors (const struct dictionary *d)
 {
-  return d->vector_cnt;
+  return d->n_vectors;
 }
 
 /* Looks up and returns the vector within D with the given
@@ -1413,8 +1549,8 @@ const struct vector *
 dict_lookup_vector (const struct dictionary *d, const char *name)
 {
   size_t i;
-  for (i = 0; i < d->vector_cnt; i++)
-    if (!strcasecmp (vector_get_name (d->vector[i]), name))
+  for (i = 0; i < d->n_vectors; i++)
+    if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))
       return d->vector[i];
   return NULL;
 }
@@ -1425,12 +1561,12 @@ dict_clear_vectors (struct dictionary *d)
 {
   size_t i;
 
-  for (i = 0; i < d->vector_cnt; i++)
+  for (i = 0; i < d->n_vectors; i++)
     vector_destroy (d->vector[i]);
   free (d->vector);
 
   d->vector = NULL;
-  d->vector_cnt = 0;
+  d->n_vectors = 0;
 }
 
 /* Multiple response sets. */
@@ -1459,7 +1595,7 @@ dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
   size_t i;
 
   for (i = 0; i < dict->n_mrsets; i++)
-    if (!strcasecmp (name, dict->mrsets[i]->name))
+    if (!utf8_strcasecmp (name, dict->mrsets[i]->name))
       return i;
 
   return SIZE_MAX;
@@ -1542,12 +1678,12 @@ dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
 
   assert (dict_contains_var (dict, var));
 
-  for (i = 0; i < dict->n_mrsets; )
+  for (i = 0; i < dict->n_mrsets;)
     {
       struct mrset *mrset = dict->mrsets[i];
       size_t j;
 
-      for (j = 0; j < mrset->n_vars; )
+      for (j = 0; j < mrset->n_vars;)
         if (mrset->vars[j] == var)
           remove_element (mrset->vars, mrset->n_vars--,
                           sizeof *mrset->vars, j);
@@ -1564,12 +1700,103 @@ dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
     }
 }
 
+
+/* Returns the variable set in DICT with index IDX, which must be between 0 and
+   the count returned by dict_get_n_varsets(), exclusive. */
+const struct varset *
+dict_get_varset (const struct dictionary *dict, size_t idx)
+{
+  assert (idx < dict->n_varsets);
+  return dict->varsets[idx];
+}
+
+/* Returns the number of variable sets in DICT. */
+size_t
+dict_get_n_varsets (const struct dictionary *dict)
+{
+  return dict->n_varsets;
+}
+
+/* Looks for a variable set named NAME in DICT.  If it finds one, returns its
+   index; otherwise, returns SIZE_MAX. */
+static size_t
+dict_lookup_varset_idx (const struct dictionary *dict, const char *name)
+{
+  for (size_t i = 0; i < dict->n_varsets; i++)
+    if (!utf8_strcasecmp (name, dict->varsets[i]->name))
+      return i;
+
+  return SIZE_MAX;
+}
+
+/* Looks for a multiple response set named NAME in DICT.  If it finds one,
+   returns it; otherwise, returns NULL. */
+const struct varset *
+dict_lookup_varset (const struct dictionary *dict, const char *name)
+{
+  size_t idx = dict_lookup_varset_idx (dict, name);
+  return idx != SIZE_MAX ? dict->varsets[idx] : NULL;
+}
+
+/* Adds VARSET to DICT, replacing any existing set with the same name.  Returns
+   true if a set was replaced, false if none existed with the specified name.
+
+   Ownership of VARSET is transferred to DICT. */
+bool
+dict_add_varset (struct dictionary *dict, struct varset *varset)
+{
+  size_t idx = dict_lookup_varset_idx (dict, varset->name);
+  if (idx == SIZE_MAX)
+    {
+      dict->varsets = xrealloc (dict->varsets,
+                               (dict->n_varsets + 1) * sizeof *dict->varsets);
+      dict->varsets[dict->n_varsets++] = varset;
+      return true;
+    }
+  else
+    {
+      varset_destroy (dict->varsets[idx]);
+      dict->varsets[idx] = varset;
+      return false;
+    }
+}
+
+/* Deletes all variable sets from DICT. */
+void
+dict_clear_varsets (struct dictionary *dict)
+{
+  for (size_t i = 0; i < dict->n_varsets; i++)
+    varset_destroy (dict->varsets[i]);
+  free (dict->varsets);
+  dict->varsets = NULL;
+  dict->n_varsets = 0;
+}
+
+/* Removes VAR, which must be in DICT, from DICT's multiple response sets. */
+static void
+dict_unset_varset_var (struct dictionary *dict, struct variable *var)
+{
+  assert (dict_contains_var (dict, var));
+
+  for (size_t i = 0; i < dict->n_varsets; i++)
+    {
+      struct varset *varset = dict->varsets[i];
+
+      for (size_t j = 0; j < varset->n_vars;)
+        if (varset->vars[j] == var)
+          remove_element (varset->vars, varset->n_vars--,
+                          sizeof *varset->vars, j);
+        else
+          j++;
+    }
+}
+
 /* Returns D's attribute set.  The caller may examine or modify
    the attribute set, but must not destroy it.  Destroying D or
    calling dict_set_attributes for D will also destroy D's
    attribute set. */
 struct attrset *
-dict_get_attributes (const struct dictionary *d) 
+dict_get_attributes (const struct dictionary *d)
 {
   return CONST_CAST (struct attrset *, &d->attributes);
 }
@@ -1585,121 +1812,40 @@ dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
 /* Returns true if D has at least one attribute in its attribute
    set, false if D's attribute set is empty. */
 bool
-dict_has_attributes (const struct dictionary *d) 
+dict_has_attributes (const struct dictionary *d)
 {
   return attrset_count (&d->attributes) > 0;
 }
 
-/* Called from variable.c to notify the dictionary that some property of
-   the variable has changed */
+/* Called from variable.c to notify the dictionary that some property (indicated
+   by WHAT) of the variable has changed.  OLDVAR is a copy of V as it existed
+   prior to the change.  OLDVAR is destroyed by this function.
+*/
 void
-dict_var_changed (const struct variable *v)
+dict_var_changed (const struct variable *v, unsigned int what, struct variable *oldvar)
 {
-  if ( var_has_vardict (v))
+  if (var_has_vardict (v))
     {
       const struct vardict_info *vardict = var_get_vardict (v);
       struct dictionary *d = vardict->dict;
 
-      if ( NULL == d)
+      if (NULL == d)
 	return;
 
-      if (d->changed ) d->changed (d, d->changed_data);
-      if ( d->callbacks && d->callbacks->var_changed )
-	d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
-    }
-}
-
-
-/* Called from variable.c to notify the dictionary that the variable's width
-   has changed */
-void
-dict_var_resized (const struct variable *v, int old_width)
-{
-  if ( var_has_vardict (v))
-    {
-      const struct vardict_info *vardict = var_get_vardict (v);
-      struct dictionary *d;
-
-      d = vardict->dict;
-
-      if (d->changed) d->changed (d, d->changed_data);
-
-      invalidate_proto (d);
-      if ( d->callbacks && d->callbacks->var_resized )
-	d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
-                                   d->cb_data);
-    }
-}
-
-/* Called from variable.c to notify the dictionary that the variable's display width
-   has changed */
-void
-dict_var_display_width_changed (const struct variable *v)
-{
-  if ( var_has_vardict (v))
-    {
-      const struct vardict_info *vardict = var_get_vardict (v);
-      struct dictionary *d;
-
-      d = vardict->dict;
+      if (what & (VAR_TRAIT_WIDTH | VAR_TRAIT_POSITION))
+        invalidate_proto (d);
 
       if (d->changed) d->changed (d, d->changed_data);
-      if ( d->callbacks && d->callbacks->var_display_width_changed )
-	d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
-    }
-}
-
-/* Dictionary used to contain "internal variables". */
-static struct dictionary *internal_dict;
-
-/* Create a variable of the specified WIDTH to be used for internal
-   calculations only.  The variable is assigned case index CASE_IDX. */
-struct variable *
-dict_create_internal_var (int case_idx, int width)
-{
-  if (internal_dict == NULL)
-    internal_dict = dict_create ();
-
-  for (;;)
-    {
-      static int counter = INT_MAX / 2;
-      struct variable *var;
-      char name[64];
-
-      if (++counter == INT_MAX)
-        counter = INT_MAX / 2;
-
-      sprintf (name, "$internal%d", counter);
-      var = dict_create_var (internal_dict, name, width);
-      if (var != NULL)
-        {
-          set_var_case_index (var, case_idx);
-          return var;
-        }
+      if (d->callbacks && d->callbacks->var_changed)
+        d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);
     }
+  var_unref (oldvar);
 }
 
-/* Destroys VAR, which must have been created with
-   dict_create_internal_var(). */
-void
-dict_destroy_internal_var (struct variable *var)
-{
-  if (var != NULL)
-    {
-      dict_delete_var (internal_dict, var);
 
-      /* Destroy internal_dict if it has no variables left, just so that
-         valgrind --leak-check --show-reachable won't show internal_dict. */
-      if (dict_get_var_cnt (internal_dict) == 0)
-        {
-          dict_destroy (internal_dict);
-          internal_dict = NULL;
-        }
-    }
-}
 
 int
 vardict_get_dict_index (const struct vardict_info *vardict)
 {
-  return vardict - vardict->dict->var;
+  return vardict - vardict->dict->vars;
 }