SAVE TRANSLATE: Allow variable names with space, etc. in output.

[pspp] / src / data / dictionary.c

diff --git a/src/data/dictionary.c b/src/data/dictionary.c
index c8f58516ecc79d1d6a7bc78a29353f838745634c..ff5f8ec027f16cbe3907ac5ab47acecb1e338425 100644 (file)
--- a/src/data/dictionary.c
+++ b/src/data/dictionary.c
@@ -1,5 +1,5 @@
 /* PSPP - a program for statistical analysis.
 /* PSPP - a program for statistical analysis.

-   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011 Free Software Foundation, Inc.
+   Copyright (C) 1997-9, 2000, 2006, 2007, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc.

 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
 
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by


@@ -74,6 +74,11 @@ struct dictionary
     struct mrset **mrsets;      /* Multiple response sets. */
     size_t n_mrsets;            /* Number of multiple response sets. */
 
     struct mrset **mrsets;      /* Multiple response sets. */
     size_t n_mrsets;            /* Number of multiple response sets. */
 

+    /* Whether variable names must be valid identifiers.  Normally, this is
+       true, but sometimes a dictionary is prepared for external use
+       (e.g. output to a CSV file) where names don't have to be valid. */
+    bool names_must_be_ids;
+

     char *encoding;             /* Character encoding of string data */
 
     const struct dict_callbacks *callbacks; /* Callbacks on dictionary
     char *encoding;             /* Character encoding of string data */
 
     const struct dict_callbacks *callbacks; /* Callbacks on dictionary


@@ -87,6 +92,8 @@ struct dictionary
 static void dict_unset_split_var (struct dictionary *, struct variable *);
 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
 
 static void dict_unset_split_var (struct dictionary *, struct variable *);
 static void dict_unset_mrset_var (struct dictionary *, struct variable *);
 

+/* Returns the encoding for data in dictionary D.  The return value is a
+   nonnull string that contains an IANA character set name. */

 const char *
 dict_get_encoding (const struct dictionary *d)
 {
 const char *
 dict_get_encoding (const struct dictionary *d)
 {


@@ -100,7 +107,8 @@ bool
 dict_id_is_valid (const struct dictionary *dict, const char *id,
                   bool issue_error)
 {
 dict_id_is_valid (const struct dictionary *dict, const char *id,
                   bool issue_error)
 {

-  return id_is_valid (id, dict->encoding, issue_error);
+  return (!dict->names_must_be_ids
+          || id_is_valid (id, dict->encoding, issue_error));

 }
 
 void
 }
 
 void


@@ -168,6 +176,7 @@ dict_create (const char *encoding)
   struct dictionary *d = xzalloc (sizeof *d);
 
   d->encoding = xstrdup (encoding);
   struct dictionary *d = xzalloc (sizeof *d);
 
   d->encoding = xstrdup (encoding);

+  d->names_must_be_ids = true;

   hmap_init (&d->name_map);
   attrset_init (&d->attributes);
 
   hmap_init (&d->name_map);
   attrset_init (&d->attributes);
 


@@ -181,7 +190,9 @@ dict_create (const char *encoding)
    dictionary.  If the new dictionary won't be used to access
    cases produced with the old dictionary, then the new
    dictionary's case indexes should be compacted with
    dictionary.  If the new dictionary won't be used to access
    cases produced with the old dictionary, then the new
    dictionary's case indexes should be compacted with

-   dict_compact_values to save space. */
+   dict_compact_values to save space.
+
+   Callbacks are not cloned. */

 struct dictionary *
 dict_clone (const struct dictionary *s)
 {
 struct dictionary *
 dict_clone (const struct dictionary *s)
 {


@@ -189,11 +200,7 @@ dict_clone (const struct dictionary *s)
   size_t i;
 
   d = dict_create (s->encoding);
   size_t i;
 
   d = dict_create (s->encoding);

-
-  /* Set the new dictionary's encoding early so that string length limitations
-     are interpreted correctly. */
-  if ( s->encoding)
-    d->encoding = xstrdup (s->encoding);
+  dict_set_names_must_be_ids (d, dict_get_names_must_be_ids (s));

 
   for (i = 0; i < s->var_cnt; i++)
     {
 
   for (i = 0; i < s->var_cnt; i++)
     {


@@ -280,28 +287,18 @@ dict_clear (struct dictionary *d)
   attrset_clear (&d->attributes);
 }
 
   attrset_clear (&d->attributes);
 }
 

-/* Destroys the aux data for every variable in D, by calling
-   var_clear_aux() for each variable. */
-void
-dict_clear_aux (struct dictionary *d)
-{
-  int i;
-
-  for (i = 0; i < d->var_cnt; i++)
-    var_clear_aux (d->var[i].var);
-}
-

 /* Clears a dictionary and destroys it. */
 void
 dict_destroy (struct dictionary *d)
 {
   if (d != NULL)
     {
 /* Clears a dictionary and destroys it. */
 void
 dict_destroy (struct dictionary *d)
 {
   if (d != NULL)
     {

-      /* In general, we don't want callbacks occuring, if the dictionary
+      /* In general, we don't want callbacks occurring, if the dictionary

         is being destroyed */
       d->callbacks  = NULL ;
 
       dict_clear (d);
         is being destroyed */
       d->callbacks  = NULL ;
 
       dict_clear (d);

+      string_array_destroy (&d->documents);

       hmap_destroy (&d->name_map);
       attrset_destroy (&d->attributes);
       dict_clear_mrsets (d);
       hmap_destroy (&d->name_map);
       attrset_destroy (&d->attributes);
       dict_clear_mrsets (d);


@@ -374,10 +371,13 @@ dict_get_vars_mutable (const struct dictionary *d, struct variable ***vars,
 }
 
 static struct variable *
 }
 
 static struct variable *

-add_var (struct dictionary *d, struct variable *v)
+add_var_with_case_index (struct dictionary *d, struct variable *v,
+                         int case_index)

 {
   struct vardict_info *vardict;
 
 {
   struct vardict_info *vardict;
 

+  assert (case_index >= d->next_value_idx);
+

   /* Update dictionary. */
   if (d->var_cnt >= d->var_cap)
     {
   /* Update dictionary. */
   if (d->var_cnt >= d->var_cap)
     {


@@ -397,20 +397,26 @@ add_var (struct dictionary *d, struct variable *v)
   vardict->dict = d;
   vardict->var = v;
   hmap_insert (&d->name_map, &vardict->name_node,
   vardict->dict = d;
   vardict->var = v;
   hmap_insert (&d->name_map, &vardict->name_node,

-               hash_case_string (var_get_name (v), 0));
-  vardict->case_index = d->next_value_idx;
+               utf8_hash_case_string (var_get_name (v), 0));
+  vardict->case_index = case_index;

   var_set_vardict (v, vardict);
 
   if ( d->changed ) d->changed (d, d->changed_data);
   if ( d->callbacks &&  d->callbacks->var_added )
     d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
 
   var_set_vardict (v, vardict);
 
   if ( d->changed ) d->changed (d, d->changed_data);
   if ( d->callbacks &&  d->callbacks->var_added )
     d->callbacks->var_added (d, var_get_dict_index (v), d->cb_data);
 

-  d->next_value_idx++;

   invalidate_proto (d);
   invalidate_proto (d);

+  d->next_value_idx = case_index + 1;

 
   return v;
 }
 
 
   return v;
 }
 

+static struct variable *
+add_var (struct dictionary *d, struct variable *v)
+{
+  return add_var_with_case_index (d, v, d->next_value_idx);
+}
+

 /* Creates and returns a new variable in D with the given NAME
    and WIDTH.  Returns a null pointer if the given NAME would
    duplicate that of an existing variable in the dictionary. */
 /* Creates and returns a new variable in D with the given NAME
    and WIDTH.  Returns a null pointer if the given NAME would
    duplicate that of an existing variable in the dictionary. */


@@ -479,6 +485,15 @@ dict_clone_var_as_assert (struct dictionary *d, const struct variable *old_var,
   return add_var (d, new_var);
 }
 
   return add_var (d, new_var);
 }
 

+struct variable *
+dict_clone_var_in_place_assert (struct dictionary *d,
+                                const struct variable *old_var)
+{
+  assert (dict_lookup_var (d, var_get_name (old_var)) == NULL);
+  return add_var_with_case_index (d, var_clone (old_var),
+                                  var_get_case_index (old_var));
+}
+

 /* Returns the variable named NAME in D, or a null pointer if no
    variable has that name. */
 struct variable *
 /* Returns the variable named NAME in D, or a null pointer if no
    variable has that name. */
 struct variable *


@@ -487,10 +502,10 @@ dict_lookup_var (const struct dictionary *d, const char *name)
   struct vardict_info *vardict;
 
   HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,
   struct vardict_info *vardict;
 
   HMAP_FOR_EACH_WITH_HASH (vardict, struct vardict_info, name_node,

-                           hash_case_string (name, 0), &d->name_map)
+                           utf8_hash_case_string (name, 0), &d->name_map)

     {
       struct variable *var = vardict->var;
     {
       struct variable *var = vardict->var;

-      if (!strcasecmp (var_get_name (var), name))
+      if (!utf8_strcasecmp (var_get_name (var), name))

         return var;
     }
 
         return var;
     }
 


@@ -539,15 +554,21 @@ unindex_var (struct dictionary *d, struct vardict_info *vardict)
 static void
 reindex_var (struct dictionary *d, struct vardict_info *vardict)
 {
 static void
 reindex_var (struct dictionary *d, struct vardict_info *vardict)
 {

-  struct variable *var = vardict->var;
+  struct variable *old = (d->callbacks && d->callbacks->var_changed
+                          ? var_clone (vardict->var)
+                          : NULL);

 
 

+  struct variable *var = vardict->var;

   var_set_vardict (var, vardict);
   hmap_insert_fast (&d->name_map, &vardict->name_node,
                     vardict->name_node.hash);
 
   if ( d->changed ) d->changed (d, d->changed_data);
   var_set_vardict (var, vardict);
   hmap_insert_fast (&d->name_map, &vardict->name_node,
                     vardict->name_node.hash);
 
   if ( d->changed ) d->changed (d, d->changed_data);

-  if ( d->callbacks &&  d->callbacks->var_changed )
-    d->callbacks->var_changed (d, var_get_dict_index (var), d->cb_data);
+  if (old)
+    {
+      d->callbacks->var_changed (d, var_get_dict_index (var), VAR_TRAIT_POSITION, old, d->cb_data);
+      var_destroy (old);
+    }

 }
 
 /* Sets the case_index in V's vardict to CASE_INDEX. */
 }
 
 /* Sets the case_index in V's vardict to CASE_INDEX. */


@@ -597,13 +618,9 @@ dict_delete_var (struct dictionary *d, struct variable *v)
 {
   int dict_index = var_get_dict_index (v);
   const int case_index = var_get_case_index (v);
 {
   int dict_index = var_get_dict_index (v);
   const int case_index = var_get_case_index (v);

-  const int width = var_get_width (v);

 
   assert (dict_contains_var (d, v));
 
 
   assert (dict_contains_var (d, v));
 

-  /* Delete aux data. */
-  var_clear_aux (v);
-

   dict_unset_split_var (d, v);
   dict_unset_mrset_var (d, v);
 
   dict_unset_split_var (d, v);
   dict_unset_mrset_var (d, v);
 


@@ -625,13 +642,14 @@ dict_delete_var (struct dictionary *d, struct variable *v)
 
   /* Free memory. */
   var_clear_vardict (v);
 
   /* Free memory. */
   var_clear_vardict (v);

-  var_destroy (v);

 
   if ( d->changed ) d->changed (d, d->changed_data);
 
   invalidate_proto (d);
   if (d->callbacks &&  d->callbacks->var_deleted )
 
   if ( d->changed ) d->changed (d, d->changed_data);
 
   invalidate_proto (d);
   if (d->callbacks &&  d->callbacks->var_deleted )

-    d->callbacks->var_deleted (d, dict_index, case_index, width, d->cb_data);
+    d->callbacks->var_deleted (d, v, dict_index, case_index, d->cb_data);
+
+  var_destroy (v);

 }
 
 /* Deletes the COUNT variables listed in VARS from D.  This is
 }
 
 /* Deletes the COUNT variables listed in VARS from D.  This is


@@ -742,20 +760,22 @@ rename_var (struct variable *v, const char *new_name)
   struct vardict_info *vardict = var_get_vardict (v);
   var_clear_vardict (v);
   var_set_name (v, new_name);
   struct vardict_info *vardict = var_get_vardict (v);
   var_clear_vardict (v);
   var_set_name (v, new_name);

-  vardict->name_node.hash = hash_case_string (new_name, 0);
+  vardict->name_node.hash = utf8_hash_case_string (new_name, 0);

   var_set_vardict (v, vardict);
 }
 
   var_set_vardict (v, vardict);
 }
 

-/* Changes the name of V in D to name NEW_NAME.  Assert-fails if
-   a variable named NEW_NAME is already in D, except that
-   NEW_NAME may be the same as V's existing name. */
-void
-dict_rename_var (struct dictionary *d, struct variable *v,
-                 const char *new_name)
+/* Tries to changes the name of V in D to name NEW_NAME.  Returns true if
+   successful, false if a variable (other than V) with the given name already
+   exists in D. */
+bool
+dict_try_rename_var (struct dictionary *d, struct variable *v,
+                     const char *new_name)

 {
 {

-  assert (!strcasecmp (var_get_name (v), new_name)
-          || dict_lookup_var (d, new_name) == NULL);
+  struct variable *conflict = dict_lookup_var (d, new_name);
+  if (conflict && v != conflict)
+    return false;

 
 

+  struct variable *old = var_clone (v);

   unindex_var (d, var_get_vardict (v));
   rename_var (v, new_name);
   reindex_var (d, var_get_vardict (v));
   unindex_var (d, var_get_vardict (v));
   rename_var (v, new_name);
   reindex_var (d, var_get_vardict (v));


@@ -765,7 +785,22 @@ dict_rename_var (struct dictionary *d, struct variable *v,
 
   if ( d->changed ) d->changed (d, d->changed_data);
   if ( d->callbacks &&  d->callbacks->var_changed )
 
   if ( d->changed ) d->changed (d, d->changed_data);
   if ( d->callbacks &&  d->callbacks->var_changed )

-    d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
+    d->callbacks->var_changed (d, var_get_dict_index (v), VAR_TRAIT_NAME, old, d->cb_data);
+
+  var_destroy (old);
+
+  return true;
+}
+
+/* Changes the name of V in D to name NEW_NAME.  Assert-fails if
+   a variable named NEW_NAME is already in D, except that
+   NEW_NAME may be the same as V's existing name. */
+void
+dict_rename_var (struct dictionary *d, struct variable *v,
+                 const char *new_name)
+{
+  bool ok UNUSED = dict_try_rename_var (d, v, new_name);
+  assert (ok);

 }
 
 /* Renames COUNT variables specified in VARS to the names given
 }
 
 /* Renames COUNT variables specified in VARS to the names given


@@ -898,7 +933,7 @@ make_hinted_name (const struct dictionary *dict, const char *hint)
           char *name;
 
           suffix[0] = '_';
           char *name;
 
           suffix[0] = '_';

-          if (!str_format_26adic (i + 1, &suffix[1], sizeof suffix - 1))
+          if (!str_format_26adic (i + 1, true, &suffix[1], sizeof suffix - 1))

             NOT_REACHED ();
 
           name = utf8_encoding_concat (root, suffix, dict->encoding, 64);
             NOT_REACHED ();
 
           name = utf8_encoding_concat (root, suffix, dict->encoding, 64);


@@ -967,6 +1002,27 @@ dict_make_unique_var_name (const struct dictionary *dict, const char *hint,
   return make_numeric_name (dict, num_start);
 }
 
   return make_numeric_name (dict, num_start);
 }
 

+/* Returns whether variable names must be valid identifiers.  Normally, this is
+   true, but sometimes a dictionary is prepared for external use (e.g. output
+   to a CSV file) where names don't have to be valid. */
+bool
+dict_get_names_must_be_ids (const struct dictionary *d)
+{
+  return d->names_must_be_ids;
+}
+
+/* Sets whether variable names must be valid identifiers.  Normally, this is
+   true, but sometimes a dictionary is prepared for external use (e.g. output
+   to a CSV file) where names don't have to be valid.
+
+   Changing this setting from false to true doesn't make the dictionary check
+   all the existing variable names, so it can cause an invariant violation. */
+void
+dict_set_names_must_be_ids (struct dictionary *d, bool names_must_be_ids)
+{
+  d->names_must_be_ids = names_must_be_ids;
+}
+

 /* Returns the weighting variable in dictionary D, or a null
    pointer if the dictionary is unweighted. */
 struct variable *
 /* Returns the weighting variable in dictionary D, or a null
    pointer if the dictionary is unweighted. */
 struct variable *


@@ -994,15 +1050,8 @@ dict_get_case_weight (const struct dictionary *d, const struct ccase *c,
   else
     {
       double w = case_num (c, d->weight);
   else
     {
       double w = case_num (c, d->weight);

-      if (w < 0.0 || var_is_num_missing (d->weight, w, MV_ANY))
-        w = 0.0;
-      if ( w == 0.0 && warn_on_invalid != NULL && *warn_on_invalid ) {
-         *warn_on_invalid = false;
-         msg (SW, _("At least one case in the data file had a weight value "
-                    "that was user-missing, system-missing, zero, or "
-                    "negative.  These case(s) were ignored."));
-      }
-      return w;
+
+      return var_force_valid_weight (d->weight, w, warn_on_invalid);

     }
 }
 
     }
 }
 


@@ -1247,15 +1296,18 @@ dict_get_label (const struct dictionary *d)
   return d->label;
 }
 
   return d->label;
 }
 

-/* Sets D's file label to LABEL, truncating it to a maximum of 60
-   characters.
+/* Sets D's file label to LABEL, truncating it to at most 60 bytes in D's
+   encoding.

 
    Removes D's label if LABEL is null or the empty string. */
 void
 dict_set_label (struct dictionary *d, const char *label)
 {
   free (d->label);
 
    Removes D's label if LABEL is null or the empty string. */
 void
 dict_set_label (struct dictionary *d, const char *label)
 {
   free (d->label);

-  d->label = label != NULL && label[0] != '\0' ? xstrndup (label, 60) : NULL;
+  if (label == NULL || label[0] == '\0')
+    d->label = NULL;
+  else
+    d->label = utf8_encoding_trunc (label, d->encoding, 60);

 }
 
 /* Returns the documents for D, as an UTF-8 encoded string_array.  The
 }
 
 /* Returns the documents for D, as an UTF-8 encoded string_array.  The


@@ -1408,7 +1460,7 @@ dict_lookup_vector (const struct dictionary *d, const char *name)
 {
   size_t i;
   for (i = 0; i < d->vector_cnt; i++)
 {
   size_t i;
   for (i = 0; i < d->vector_cnt; i++)

-    if (!strcasecmp (vector_get_name (d->vector[i]), name))
+    if (!utf8_strcasecmp (vector_get_name (d->vector[i]), name))

       return d->vector[i];
   return NULL;
 }
       return d->vector[i];
   return NULL;
 }


@@ -1453,7 +1505,7 @@ dict_lookup_mrset_idx (const struct dictionary *dict, const char *name)
   size_t i;
 
   for (i = 0; i < dict->n_mrsets; i++)
   size_t i;
 
   for (i = 0; i < dict->n_mrsets; i++)

-    if (!strcasecmp (name, dict->mrsets[i]->name))
+    if (!utf8_strcasecmp (name, dict->mrsets[i]->name))

       return i;
 
   return SIZE_MAX;
       return i;
 
   return SIZE_MAX;


@@ -1563,7 +1615,7 @@ dict_unset_mrset_var (struct dictionary *dict, struct variable *var)
    calling dict_set_attributes for D will also destroy D's
    attribute set. */
 struct attrset *
    calling dict_set_attributes for D will also destroy D's
    attribute set. */
 struct attrset *

-dict_get_attributes (const struct dictionary *d) 
+dict_get_attributes (const struct dictionary *d)

 {
   return CONST_CAST (struct attrset *, &d->attributes);
 }
 {
   return CONST_CAST (struct attrset *, &d->attributes);
 }


@@ -1579,15 +1631,17 @@ dict_set_attributes (struct dictionary *d, const struct attrset *attrs)
 /* Returns true if D has at least one attribute in its attribute
    set, false if D's attribute set is empty. */
 bool
 /* Returns true if D has at least one attribute in its attribute
    set, false if D's attribute set is empty. */
 bool

-dict_has_attributes (const struct dictionary *d) 
+dict_has_attributes (const struct dictionary *d)

 {
   return attrset_count (&d->attributes) > 0;
 }
 
 {
   return attrset_count (&d->attributes) > 0;
 }
 

-/* Called from variable.c to notify the dictionary that some property of
-   the variable has changed */
+/* Called from variable.c to notify the dictionary that some property (indicated
+   by WHAT) of the variable has changed.  OLDVAR is a copy of V as it existed
+   prior to the change.  OLDVAR is destroyed by this function.
+*/

 void
 void

-dict_var_changed (const struct variable *v)
+dict_var_changed (const struct variable *v, unsigned int what, struct variable *oldvar)

 {
   if ( var_has_vardict (v))
     {
 {
   if ( var_has_vardict (v))
     {


@@ -1599,49 +1653,12 @@ dict_var_changed (const struct variable *v)
 
       if (d->changed ) d->changed (d, d->changed_data);
       if ( d->callbacks && d->callbacks->var_changed )
 
       if (d->changed ) d->changed (d, d->changed_data);
       if ( d->callbacks && d->callbacks->var_changed )

-       d->callbacks->var_changed (d, var_get_dict_index (v), d->cb_data);
-    }
-}
-
-
-/* Called from variable.c to notify the dictionary that the variable's width
-   has changed */
-void
-dict_var_resized (const struct variable *v, int old_width)
-{
-  if ( var_has_vardict (v))
-    {
-      const struct vardict_info *vardict = var_get_vardict (v);
-      struct dictionary *d;
-
-      d = vardict->dict;
-
-      if (d->changed) d->changed (d, d->changed_data);
-
-      invalidate_proto (d);
-      if ( d->callbacks && d->callbacks->var_resized )
-       d->callbacks->var_resized (d, var_get_dict_index (v), old_width,
-                                   d->cb_data);
+       d->callbacks->var_changed (d, var_get_dict_index (v), what, oldvar, d->cb_data);

     }
     }

+  var_destroy (oldvar);

 }
 
 }
 

-/* Called from variable.c to notify the dictionary that the variable's display width
-   has changed */
-void
-dict_var_display_width_changed (const struct variable *v)
-{
-  if ( var_has_vardict (v))
-    {
-      const struct vardict_info *vardict = var_get_vardict (v);
-      struct dictionary *d;
-
-      d = vardict->dict;

 
 

-      if (d->changed) d->changed (d, d->changed_data);
-      if ( d->callbacks && d->callbacks->var_display_width_changed )
-       d->callbacks->var_display_width_changed (d, var_get_dict_index (v), d->cb_data);
-    }
-}

 \f
 /* Dictionary used to contain "internal variables". */
 static struct dictionary *internal_dict;
 \f
 /* Dictionary used to contain "internal variables". */
 static struct dictionary *internal_dict;