From a44813e69593b8c3a8c1ee08c0f08a18a52af256 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Wed, 20 Jan 2016 20:47:22 -0800
Subject: [PATCH 01/16] Add graphviz file for structure of detail XML.

---
 detail-xml    |  3 +-
 spv-detail.gv | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+), 2 deletions(-)
 create mode 100644 spv-detail.gv

diff --git a/detail-xml b/detail-xml
index 1684915bf5..3735a657fa 100644
--- a/detail-xml
+++ b/detail-xml
@@ -119,9 +119,8 @@ style :=
       textAlignment
       size?)?
      visible?
-     color? color2?
+     color? color2?]
      
-
 layer := [id? value variable method? visible?]
 layerController := [id? source target]
 
diff --git a/spv-detail.gv b/spv-detail.gv
new file mode 100644
index 0000000000..e026a483e3
--- /dev/null
+++ b/spv-detail.gv
@@ -0,0 +1,92 @@
+digraph spv_detail {
+    "visualization" -> "extension" [label="?"];
+    "visualization" -> "userSource";
+    "visualization" -> "sourceVariable" [label="*"];
+    "visualization" -> "derivedVariable" [label="*"];
+    "visualization" -> "graph";
+    "visualization" -> "labelFrame" [label="+"];
+    "visualization" -> "container" [label="?"];
+    "visualization" -> "style" [label="+"];
+    "visualization" -> "layerController" [label="?"];
+
+    "sourceVariable" -> "extension" [label="*"];
+    "sourceVariable" -> "format" [label="?"];
+    "sourceVariable" -> "stringFormat" [label="?"];
+
+    "derivedVariable" -> "extension" [label="*"];
+    "derivedVariable" -> "format" [label="?"];
+    "derivedVariable" -> "stringFormat" [label="?"];
+    "derivedVariable" -> "valueMapEntry" [label="*"];
+
+    "format" -> "affix" [label="*"];
+    "format" -> "relabel" [label="?"];
+
+    "dateTimeFormat" -> "affix" [label="?"];
+
+    "stringFormat" -> "affix" [label="*"];
+    "stringFormat" -> "relabel" [label="?"];
+
+    "graph" -> "location" [label="+"];
+    "graph" -> "coordinates";
+    "graph" -> "faceting";
+    "graph" -> "facetLayout";
+    "graph" -> "interval";
+
+    "faceting" -> "cross";
+    "faceting" -> "layer" [label="*"];
+
+    "interval" -> "labeling";
+    "interval" -> "footnotes" [label="?"];
+
+    "labeling" -> "format" [label="*"];
+    "labeling" -> "formatting" [label="*"];
+    "labeling" -> "footnotes" [label="*"];
+
+    "formatting" -> "formatMapping" [label="*"];
+
+    "formatMapping" -> "format";
+
+    "footnotes" -> "footnoteMapping" [label="+"];
+
+    "cross" -> "nest" [label="+"];
+
+    "nest" -> "variableReference" [label="+"];
+
+    "facetLayout" -> "tableLayout";
+    "facetLayout" -> "facetLevel" [label="+"];
+    "facetLayout" -> "setCellProperties" [label="*"];
+    "facetLevel" -> "axis";
+
+    "axis" -> "majorTicks";
+
+    "label" -> "descriptionGroup" [label="*"];
+    "label" -> "text" [label="*"];
+
+    "majorTicks" -> "gridline" [label="?"];
+
+    "descriptionGroup" -> "description" [label="+"];
+    "descriptionGroup" -> "text";
+
+    "setCellProperties" -> "setMetadata";
+    "setCellProperties" -> "setStyle" [label="*"];
+    "setCellProperties" -> "setFormat" [label="+"];
+    "setCellProperties" -> "union" [label="?"];
+    
+    "setFormat" -> "dateTimeFormat" [label="?"];
+    "setFormat" -> "format" [label="?"];
+    "setFormat" -> "numberFormat" [label="?"];
+    "setFormat" -> "stringFormat" [label="*"];
+
+    "labelFrame" -> "location" [label="+"];
+    "labelFrame" -> "label";
+    "labelFrame" -> "paragraph" [label="?"];
+
+    "container" -> "extension" [label="?"];
+    "container" -> "location" [label="+"];
+    "container" -> "labelFrame" [label="+"];
+
+    "union" -> "intersect" [label="+"];
+
+    "intersect" -> "intersectWhere" [label="?"];
+    "intersect" -> "where" [label="*"];
+}
-- 
2.30.2


From 2fd5ae863380ca80272555ae606ac814dfd2e403 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Wed, 20 Jan 2016 22:47:05 -0800
Subject: [PATCH 02/16] Work on describing the detail XML format at a high
 level

(What's a "facet"?)
---
 parse-xml.c          | 20 ++++++++++++++++++++
 spv-file-format.texi | 39 +++++++++++++++++++++++++++++++++++----
 2 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/parse-xml.c b/parse-xml.c
index 2f6b33813e..3a32ee36e2 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -159,6 +159,24 @@ print_attribute (xmlNode *node, const char *attr)
     }
 }
 
+static void
+print_element (xmlDoc *doc, xmlNode *node, const char *element)
+{
+  for (; node; node = node->next)
+    {
+      if (!strcmp(element, (char *) node->name))
+        {
+          xmlBuffer *buf = xmlBufferCreate();
+          xmlNodeDump (buf, doc, node, 0, 1);
+          xmlBufferDump (stdout, buf);
+          xmlBufferFree (buf);
+          putchar ('\n');
+        }
+
+      print_element (doc, node->children, element);
+    }
+}
+
 static __attribute__((unused)) xmlNode *
 find_page_setup (xmlNode *node)
 {
@@ -237,6 +255,8 @@ main (int argc, char **argv)
     print_cdata (root);
   else if (!strcmp(argv[2], "text"))
     print_text (root);
+  else if (!strncmp(argv[2], "element:", 8))
+    print_element (doc, root, argv[2] + 8);
   else if (!strncmp(argv[2], "attr:", 5))
     print_attribute (root, argv[2] + 5);
   else if (!strcmp(argv[2], "labels"))
diff --git a/spv-file-format.texi b/spv-file-format.texi
index 789a77707c..6e73aef309 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1227,7 +1227,35 @@ label is the string @code{s}.  Each label also includes a
 
 This format is still under investigation.
 
-All elements have an optional @code{id} attribute.
+The design of the detail XML format is not what one would end up with
+for describing pivot tables.  This is because it is a specialization
+of a much more general format (``visualization XML'' or ``VizML'')
+that can describe a wide range of visualizations.  Most of this
+generality is overkill for tables, and so we end up with a funny
+subset of a general-purpose format.
+
+The important elements of the detail XML format are:
+
+@itemize @bullet
+@item
+Variables.  Variables in detail XML roughly correspond to the
+dimensions in a light detail member.  There is one variable for each
+dimension, plus one variable for each level of labeling along an axis.
+
+The bulk of variables are defined with @code{sourceVariable} elements.
+The data for these variables comes from the associated
+@code{tableData.bin} member.  Some variables are defined, with
+@code{derivedVariable} elements, as a constant or in terms of a
+mapping function from a source variable.
+
+@item
+Assignment of variables to axes.  A variable can appear as columns, or
+rows, or layers.  The @code{faceting} element and its sub-elements
+describe this assignment.
+@end itemize
+
+All elements have an optional @code{id} attribute.  In practice many
+elements are assigned @code{id} attributes that are never referenced.
 
 @node SPV Detail visualization Element
 @subsection The @code{visualization} Element
@@ -1309,7 +1337,8 @@ Contents: @code{extension}* (@code{format} @math{|} @code{stringFormat})?
 
 This element defines a variable whose values can be used elsewhere in
 the visualization.  It ties this element's @code{id} to a variable
-from the @file{.bin} member that corresponds to this @file{.xml}.
+from the @file{tableData.bin} member that corresponds to this
+@file{.xml}.
 
 This element has the following attributes.
 
@@ -1319,12 +1348,14 @@ Always set to @code{true}.
 
 @defvr {Required} source
 Always set to @code{tableData}, the @code{source-name} in the
-corresponding @file{.bin} member (@pxref{SPV Legacy Member Metadata}).
+corresponding @file{tableData.bin} member (@pxref{SPV Legacy Member
+Metadata}).
 @end defvr
 
 @defvr {Required} sourceName
 The name of a variable within the source, the @code{variable-name} in
-the corresponding @file{.bin} member (@pxref{SPV Legacy Member Data}).
+the corresponding @file{tableData.bin} member (@pxref{SPV Legacy
+Member Data}).
 @end defvr
 
 @defvr {Optional} dependsOn
-- 
2.30.2


From adc35059e3ae7e8968bc6c7a835abc412920e9b5 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 31 Jan 2016 09:48:45 -0800
Subject: [PATCH 03/16] work on figuring more stuff out--may want to revert
 this commit

---
 dump.c           |  2 +-
 dump2.c          |  3 ++-
 parse-all-heavy  | 11 +++++------
 parse-detail-xml |  4 +++-
 parse-xml.c      | 43 +++++++++++++++++++++++++++++++++++++------
 5 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/dump.c b/dump.c
index a52cafddf0..c57057f025 100644
--- a/dump.c
+++ b/dump.c
@@ -823,7 +823,7 @@ dump_title(void)
       dump_value(stdout, 0);
       /* Custom footnote marker string. */
       if (match_byte (0x31))
-        dump_value(stdout, 0);
+        dump_value(stderr, 0);
       else
         match_byte_assert (0x58);
       get_u32 ();
diff --git a/dump2.c b/dump2.c
index 15670ecf4b..740cbbedaa 100644
--- a/dump2.c
+++ b/dump2.c
@@ -217,7 +217,8 @@ dump_source(int end, int count, int n_series, const char *name)
   int n_sysmis = 0;
   for (int i = 0; i < n_series; i++)
     {
-      printf ("    series %d: \"%s\"\n        ", i, get_fixed_string(288));
+      printf ("    series %d: \"%s\", %d values:\n        ",
+              i, get_fixed_string(288), count);
       for (int i = 0; i < count; i++)
         {
           double d = get_double();
diff --git a/parse-all-heavy b/parse-all-heavy
index 757e46d200..20370b1fea 100755
--- a/parse-all-heavy
+++ b/parse-all-heavy
@@ -1,10 +1,9 @@
 #! /bin/sh
-heavy=`ls -1 unzipped/*/*.bin | grep -v light`
+heavy=`ls -1 unzipped*/*/*.bin | grep -v light`
 
 for d in $heavy; do
-    if ! ./dump2 < $d > /dev/null 2>&1; then
-	echo $d
-	./dump2 < $d
-	echo
-    fi
+    echo $d
+    ./dump2 < $d
+    echo
+    echo
 done
diff --git a/parse-detail-xml b/parse-detail-xml
index f0b175e7ab..6d7858835e 100755
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -2,8 +2,10 @@
 
 # Parse the detail XML members.
 legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
+for d in $legacyXML; do echo $d; done | wc
+#grep -h -o '<style id="visualizationStyle"[^>]*>' $legacyXML
 if test -n "$1"; then
     for d in $legacyXML; do
 	./parse-xml $d $1
-    done | sort | uniq
+    done
 fi
diff --git a/parse-xml.c b/parse-xml.c
index 3a32ee36e2..0c00abd908 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -36,15 +36,26 @@ print_containment (xmlNode *node)
     {
       if (node->type == XML_ELEMENT_NODE)
         {
-          const char *child_names[512];
-          int child_name_cnt[512];
+          const char *child_names[5000];
+          int child_name_cnt[5000];
           int n_names = 0;
           for (xmlNode *child = node->children; child; child = child->next)
             {
               const char *name;
 
               if (child->type == XML_ELEMENT_NODE)
-                name = (char *) child->name;
+                {
+                  name = (char *) child->name;
+                  if (!strcmp((char *) node->name, "derivedVariable")
+                      && !strcmp((char *) name, "extension"))
+                    {
+                      char *p;
+                      asprintf(&p, "%s/%s", 
+                               (char *) xmlGetProp (child, (xmlChar *) "from"),
+                               (char *) xmlGetProp (child, (xmlChar *) "helpId"));
+                      name = p;
+                    }
+                }
               else if (child->type == XML_TEXT_NODE)
                 name = "<text>";
               else if (child->type == XML_CDATA_SECTION_NODE)
@@ -58,17 +69,19 @@ print_containment (xmlNode *node)
               else
                 name = "<other>";
 
+#if 0
               for (int i = 0; i < n_names; i++)
                 if (!strcmp(name, child_names[i]))
                   {
                     child_name_cnt[i]++;
                     goto next;
                   }
+#endif
               child_names[n_names] = name;
               child_name_cnt[n_names] = 1;
               n_names++;
 
-            next:;
+              //next:;
             }
 
           printf ("%s", node->name);
@@ -104,6 +117,14 @@ print_labels (xmlNode *node)
     }
 }
 
+static int
+compare_strings(const void *a_, const void *b_)
+{
+  const void *const *ap = a_;
+  const void *const *bp = b_;
+  return strcmp (*ap, *bp);
+}
+
 static void
 print_attributes (xmlNode * a_node)
 {
@@ -111,9 +132,19 @@ print_attributes (xmlNode * a_node)
     {
       if (node->type == XML_ELEMENT_NODE)
         {
-          printf ("%s", node->name);
+          printf ("%s<-%s", node->name, node->parent->name);
+
+          char *attrs[500];
+          int n_attrs = 0;
           for (xmlAttr *attr = node->properties; attr; attr = attr->next)
-            printf (" %s", attr->name);
+            if (!strcmp((char *) attr->name, "baseFormat"))
+              printf(" baseFormat=%s", attr->children->content);
+            else
+              attrs[n_attrs++] = (char *) attr->name;
+          qsort(attrs, n_attrs, sizeof *attrs, compare_strings);
+
+          for (int i = 0; i < n_attrs; i++)
+            printf (" %s", attrs[i]);
           putchar ('\n');
         }
 
-- 
2.30.2


From f926879f4be2b8eca78860e22f00d1001b91c175 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 21 Feb 2016 17:49:48 -0800
Subject: [PATCH 04/16] Revert "work on figuring more stuff out--may want to
 revert this commit"

This reverts commit adc35059e3ae7e8968bc6c7a835abc412920e9b5.
---
 dump.c           |  2 +-
 dump2.c          |  3 +--
 parse-all-heavy  | 11 ++++++-----
 parse-detail-xml |  4 +---
 parse-xml.c      | 43 ++++++-------------------------------------
 5 files changed, 15 insertions(+), 48 deletions(-)

diff --git a/dump.c b/dump.c
index c57057f025..a52cafddf0 100644
--- a/dump.c
+++ b/dump.c
@@ -823,7 +823,7 @@ dump_title(void)
       dump_value(stdout, 0);
       /* Custom footnote marker string. */
       if (match_byte (0x31))
-        dump_value(stderr, 0);
+        dump_value(stdout, 0);
       else
         match_byte_assert (0x58);
       get_u32 ();
diff --git a/dump2.c b/dump2.c
index 740cbbedaa..15670ecf4b 100644
--- a/dump2.c
+++ b/dump2.c
@@ -217,8 +217,7 @@ dump_source(int end, int count, int n_series, const char *name)
   int n_sysmis = 0;
   for (int i = 0; i < n_series; i++)
     {
-      printf ("    series %d: \"%s\", %d values:\n        ",
-              i, get_fixed_string(288), count);
+      printf ("    series %d: \"%s\"\n        ", i, get_fixed_string(288));
       for (int i = 0; i < count; i++)
         {
           double d = get_double();
diff --git a/parse-all-heavy b/parse-all-heavy
index 20370b1fea..757e46d200 100755
--- a/parse-all-heavy
+++ b/parse-all-heavy
@@ -1,9 +1,10 @@
 #! /bin/sh
-heavy=`ls -1 unzipped*/*/*.bin | grep -v light`
+heavy=`ls -1 unzipped/*/*.bin | grep -v light`
 
 for d in $heavy; do
-    echo $d
-    ./dump2 < $d
-    echo
-    echo
+    if ! ./dump2 < $d > /dev/null 2>&1; then
+	echo $d
+	./dump2 < $d
+	echo
+    fi
 done
diff --git a/parse-detail-xml b/parse-detail-xml
index 6d7858835e..f0b175e7ab 100755
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -2,10 +2,8 @@
 
 # Parse the detail XML members.
 legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
-for d in $legacyXML; do echo $d; done | wc
-#grep -h -o '<style id="visualizationStyle"[^>]*>' $legacyXML
 if test -n "$1"; then
     for d in $legacyXML; do
 	./parse-xml $d $1
-    done
+    done | sort | uniq
 fi
diff --git a/parse-xml.c b/parse-xml.c
index 0c00abd908..3a32ee36e2 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -36,26 +36,15 @@ print_containment (xmlNode *node)
     {
       if (node->type == XML_ELEMENT_NODE)
         {
-          const char *child_names[5000];
-          int child_name_cnt[5000];
+          const char *child_names[512];
+          int child_name_cnt[512];
           int n_names = 0;
           for (xmlNode *child = node->children; child; child = child->next)
             {
               const char *name;
 
               if (child->type == XML_ELEMENT_NODE)
-                {
-                  name = (char *) child->name;
-                  if (!strcmp((char *) node->name, "derivedVariable")
-                      && !strcmp((char *) name, "extension"))
-                    {
-                      char *p;
-                      asprintf(&p, "%s/%s", 
-                               (char *) xmlGetProp (child, (xmlChar *) "from"),
-                               (char *) xmlGetProp (child, (xmlChar *) "helpId"));
-                      name = p;
-                    }
-                }
+                name = (char *) child->name;
               else if (child->type == XML_TEXT_NODE)
                 name = "<text>";
               else if (child->type == XML_CDATA_SECTION_NODE)
@@ -69,19 +58,17 @@ print_containment (xmlNode *node)
               else
                 name = "<other>";
 
-#if 0
               for (int i = 0; i < n_names; i++)
                 if (!strcmp(name, child_names[i]))
                   {
                     child_name_cnt[i]++;
                     goto next;
                   }
-#endif
               child_names[n_names] = name;
               child_name_cnt[n_names] = 1;
               n_names++;
 
-              //next:;
+            next:;
             }
 
           printf ("%s", node->name);
@@ -117,14 +104,6 @@ print_labels (xmlNode *node)
     }
 }
 
-static int
-compare_strings(const void *a_, const void *b_)
-{
-  const void *const *ap = a_;
-  const void *const *bp = b_;
-  return strcmp (*ap, *bp);
-}
-
 static void
 print_attributes (xmlNode * a_node)
 {
@@ -132,19 +111,9 @@ print_attributes (xmlNode * a_node)
     {
       if (node->type == XML_ELEMENT_NODE)
         {
-          printf ("%s<-%s", node->name, node->parent->name);
-
-          char *attrs[500];
-          int n_attrs = 0;
+          printf ("%s", node->name);
           for (xmlAttr *attr = node->properties; attr; attr = attr->next)
-            if (!strcmp((char *) attr->name, "baseFormat"))
-              printf(" baseFormat=%s", attr->children->content);
-            else
-              attrs[n_attrs++] = (char *) attr->name;
-          qsort(attrs, n_attrs, sizeof *attrs, compare_strings);
-
-          for (int i = 0; i < n_attrs; i++)
-            printf (" %s", attrs[i]);
+            printf (" %s", attr->name);
           putchar ('\n');
         }
 
-- 
2.30.2


From 1713de7bb9768a933a7974a3713586a63f53d638 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 21 Feb 2016 17:51:45 -0800
Subject: [PATCH 05/16] Document more detail XML.

---
 detail-xml           |   2 +-
 dump2.c              |   3 +-
 parse-xml.c          |   2 +-
 spv-detail.gv        |   2 +-
 spv-file-format.texi | 145 ++++++++++++++++++++++++++++++++++++-------
 5 files changed, 128 insertions(+), 26 deletions(-)

diff --git a/detail-xml b/detail-xml
index 3735a657fa..9fca90acf4 100644
--- a/detail-xml
+++ b/detail-xml
@@ -66,7 +66,7 @@ formatMapping := [id? from] format
 footnotes := [id? superscript variable] footnoteMapping+
 footnoteMapping := [id? from to definesReference]
 
-cross := [id?] nest+
+cross := [id?] nest nest
 nest := [id?] variableReference+
 variableReference := [ref]
 
diff --git a/dump2.c b/dump2.c
index 15670ecf4b..740cbbedaa 100644
--- a/dump2.c
+++ b/dump2.c
@@ -217,7 +217,8 @@ dump_source(int end, int count, int n_series, const char *name)
   int n_sysmis = 0;
   for (int i = 0; i < n_series; i++)
     {
-      printf ("    series %d: \"%s\"\n        ", i, get_fixed_string(288));
+      printf ("    series %d: \"%s\", %d values:\n        ",
+              i, get_fixed_string(288), count);
       for (int i = 0; i < count; i++)
         {
           double d = get_double();
diff --git a/parse-xml.c b/parse-xml.c
index 3a32ee36e2..9cf6694f91 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -164,7 +164,7 @@ print_element (xmlDoc *doc, xmlNode *node, const char *element)
 {
   for (; node; node = node->next)
     {
-      if (!strcmp(element, (char *) node->name))
+      if (node->name && !strcmp(element, (char *) node->name))
         {
           xmlBuffer *buf = xmlBufferCreate();
           xmlNodeDump (buf, doc, node, 0, 1);
diff --git a/spv-detail.gv b/spv-detail.gv
index e026a483e3..5704c6a49d 100644
--- a/spv-detail.gv
+++ b/spv-detail.gv
@@ -48,7 +48,7 @@ digraph spv_detail {
 
     "footnotes" -> "footnoteMapping" [label="+"];
 
-    "cross" -> "nest" [label="+"];
+    "cross" -> "nest" [label="2"];
 
     "nest" -> "variableReference" [label="+"];
 
diff --git a/spv-file-format.texi b/spv-file-format.texi
index 6e73aef309..6c6878b274 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -143,18 +143,18 @@ times.
 @end iftex
 
 @menu
-* SPV heading Element::
-* SPV label Element::
-* SPV container Element::
-* SPV text Element (Inside @code{container})::
-* SPV html Element::
-* SPV table Element::
-* SPV tableStructure Element::
-* SPV dataPath Element::
-* SPV pageSetup Element::
-* SPV pageHeader and pageFooter Elements::
-* SPV pageParagraph Element::
-* SPV @code{text} Element (Inside @code{pageParagraph})::
+* SPV Structure heading Element::
+* SPV Structure label Element::
+* SPV Structure container Element::
+* SPV Structure text Element (Inside @code{container})::
+* SPV Structure html Element::
+* SPV Structure table Element::
+* SPV Structure tableStructure Element::
+* SPV Structure dataPath Element::
+* SPV Structure pageSetup Element::
+* SPV Structure pageHeader and pageFooter Elements::
+* SPV Structure pageParagraph Element::
+* SPV Structure @code{text} Element (Inside @code{pageParagraph})::
 @end menu
 
 @node SPV Structure heading Element
@@ -440,7 +440,7 @@ This element has no attributes.
 @node SPV Structure @code{text} Element (Inside @code{pageParagraph})
 @subsection The @code{text} Element (Inside @code{pageParagraph})
 
-Parent: @code{pageParagraph} @*
+Parent: @code{pageParagraph}
 Contents: CDATA?
 
 This @code{text} element is nested inside a @code{pageParagraph}.  There
@@ -1228,7 +1228,7 @@ label is the string @code{s}.  Each label also includes a
 This format is still under investigation.
 
 The design of the detail XML format is not what one would end up with
-for describing pivot tables.  This is because it is a specialization
+for describing pivot tables.  This is because it is a special case
 of a much more general format (``visualization XML'' or ``VizML'')
 that can describe a wide range of visualizations.  Most of this
 generality is overkill for tables, and so we end up with a funny
@@ -1257,6 +1257,17 @@ describe this assignment.
 All elements have an optional @code{id} attribute.  In practice many
 elements are assigned @code{id} attributes that are never referenced.
 
+@menu
+* SPV Detail visualization Element::
+* SPV Detail userSource Element::
+* SPV Detail sourceVariable Element::
+* SPV Detail derivedVariable Element::
+* SPV Detail extension Element::
+* SPV Detail graph Element::
+* SPV Detail location Element::
+* SPV Detail coordinates Element::
+@end menu
+
 @node SPV Detail visualization Element
 @subsection The @code{visualization} Element
 
@@ -1295,6 +1306,7 @@ format used in Unix with the underscore replaced by a hyphen, e.g.@:
 
 @defvr {Required} name
 The title of the pivot table, localized to the output language.
+@end defvr
 
 @defvr {Required} style
 The @code{id} of a @code{style} element (@pxref{SPV Detail style
@@ -1316,10 +1328,8 @@ one of 2.4, 2.5, 2.7, and 2.8.
 @node SPV Detail userSource Element
 @subsection The @code{userSource} Element
 
-@format
 Parent: @code{visualization} @*
 Contents:
-@end format
 
 This element has the following attributes.
 
@@ -1330,10 +1340,8 @@ Always @code{listwise}.
 @node SPV Detail sourceVariable Element
 @subsection The @code{sourceVariable} Element
 
-@format
 Parent: @code{visualization} @*
 Contents: @code{extension}* (@code{format} @math{|} @code{stringFormat})?
-@end format
 
 This element defines a variable whose values can be used elsewhere in
 the visualization.  It ties this element's @code{id} to a variable
@@ -1377,10 +1385,8 @@ as value labels.
 @node SPV Detail derivedVariable Element
 @subsection The @code{derivedVariable} Element
 
-@format
 Parent: @code{visualization} @*
 Contents: @code{extension}* (@code{format} @math{|} @code{stringFormat} @code{valueMapEntry}*)
-@end format
 
 Like @code{sourceVariable}, this element defines a variable whose
 values can be used elsewhere in the visualization.  Instead of being
@@ -1417,13 +1423,15 @@ viewer can work with them together.  For a group variable, this is the
 name of the corresponding categorical variable.
 @end defvr
 
+@menu
+* SPV Detail valueMapEntry Element::
+@end menu
+
 @node SPV Detail valueMapEntry Element
 @subsubsection The @code{valueMapEntry} Element
 
-@format
 Parent: @code{derivedVariable} @*
 Contents: empty
-@end format
 
 A @code{valueMapEntry} element defines a mapping from one or more
 values of a source expression to a target value.  (In the corpus, the
@@ -1500,3 +1508,96 @@ An integer or a name like ``dimension0''.
 @defvr {Required} helpId
 An identifier.
 @end defvr
+
+@node SPV Detail graph Element
+@subsection The @code{graph} Element
+
+Parent: @code{visualization} @*
+Contents: @code{location}@math{+} @code{coordinates} @code{faceting} @code{facetLayout} @code{interval}
+
+@code{graph} has the following attributes.
+
+@defvr {Required} cellStyle
+@defvrx {Required} style
+Each of these is the @code{id} of a @code{style} element (@pxref{SPV
+Detail style element}).  The former is the default style for
+individual cells, the latter for the entire table.
+@end defvr
+
+@node SPV Detail location Element
+@subsection The @code{location} Element
+
+Parent: @code{graph} @*
+Contents: empty
+
+Each instance of this element specifies where some part of the table
+frame is located.  All the examples in the corpus have four instances
+of this element, one for each of the parts @code{height},
+@code{width}, @code{left}, and @code{top}.  Some examples in the
+corpus add a fifth for part @code{bottom}, even though it is not clear
+how all of @code{top}, @code{bottom}, and @code{heigth} can be honored
+at the same time.  In any case, @code{location} seems to have little
+importance in representing tables; a reader can safely ignore it.
+
+@defvr {Required} part
+One of @code{height}, @code{width}, @code{top}, @code{bottom}, or
+@code{left}.  Presumably @code{right} is acceptable as well but the
+corpus contains no examples.
+@end defvr
+
+@defvr {Required} method
+How the location is determined:
+
+@table @code
+@item sizeToContent
+Based on the natural size of the table.  Observed only for
+parts @code{height} and @code{width}.
+
+@item attach
+Based on the location specified in @code{target}.  Observed only for
+parts @code{top} and @code{bottom}.
+
+@item fixed
+Using the value in @code{value}.  Observed only for parts @code{top},
+@code{bottom}, and @code{left}.
+
+@item same
+Same as the specified @code{target}.  Observed only for part
+@code{left}.
+@end table
+@end defvr
+
+@defvr {Optional} min
+Minimum size.  Only observed with value @code{100pt}.  Only observed
+for part @code{width}.
+@end defvr
+
+@defvr {Dependent} target
+Required when @code{method} is @code{attach} or @code{same}, not
+observed otherwise.  This is the ID of an element to attach to.
+Observed with the ID of @code{title}, @code{footnote}, @code{graph},
+and other elements.
+@end defvr
+
+@defvr {Dependent} value
+Required when @code{method} is @code{fixed}, not observed otherwise.
+Observed values are @code{0%}, @code{0px}, @code{1px}, and @code{3px}
+on parts @code{top} and @code{left}, and @code{100%} on part
+@code{bottom}.
+@end defvr
+
+@node SPV Detail coordinates Element
+@subsection The @code{coordinates} Element
+
+Parent: @code{graph} @*
+Contents: empty
+
+This element is always present and always empty, with no attributes
+(except @code{id}).
+
+@node SPV Detail faceting Element
+@subsection The @code{faceting} Element
+
+Parent: @code{graph} @*
+Contents: @code{cross} @code{layer}@math{+}
+
-- 
2.30.2


From 5763c3751c67a228c6ab3e33934c66247551af52 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 21 Feb 2016 22:33:56 -0800
Subject: [PATCH 06/16] Document faceting and work on facetLayout.

---
 spv-file-format.texi | 202 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 201 insertions(+), 1 deletion(-)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index 6c6878b274..dd4b0b8f5a 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1266,6 +1266,7 @@ elements are assigned @code{id} attributes that are never referenced.
 * SPV Detail graph Element::
 * SPV Detail location Element::
 * SPV Detail coordinates Element::
+* SPV Detail faceting Element::
 @end menu
 
 @node SPV Detail visualization Element
@@ -1599,5 +1600,204 @@ This element is always present and always empty, with no attributes
 @subsection The @code{faceting} Element
 
 Parent: @code{graph} @*
-Contents: @code{cross} @code{layer}@math{+}
+Contents: @code{cross} @code{layer}*
 
+The @code{faceting} element describes the row, column, and layer
+structure of the table.  Its @code{cross} child determines the row and
+column structure, and each @code{layer} child (if any) represents a
+layer.
+
+@code{faceting} has no attributes (other than @code{id}).
+
+@subsubheading The @code{cross} Element
+
+Parent: @code{faceting} @*
+Contents: @code{nest} @code{nest}
+
+The @code{cross} element describes the row and column structure of the
+table.  It has exactly two @code{nest} children, the first of which
+describes the table's rows and the second the table's columns.
+
+@code{cross} has no attributes (other than @code{id}).
+
+@subsubheading The @code{nest} Element
+
+Parent: @code{cross} @*
+Contents: @code{variableReference}@math{+}
+
+A given @code{nest} usually consists of one or more dimensions, each
+of which is represented by @code{variableReference} child elements.
+Minimally, a dimension has two @code{variableReference} children, one
+for the categories, one for the data, e.g.:
+
+@example
+<nest>
+  <variableReference ref="dimension0categories"/>
+  <variableReference ref="dimension0"/>
+</nest>
+@end example
+
+@noindent
+Groups of categories introduce additional variable references, e.g.@:
+
+@example
+<nest>
+  <variableReference ref="dimension0categories"/>
+  <variableReference ref="dimension0group0"/>
+  <variableReference ref="dimension0"/>
+</nest>
+@end example
+
+@noindent
+Grouping can be hierarchical, e.g.@:
+
+@example
+<nest>
+  <variableReference ref="dimension0categories"/>
+  <variableReference ref="dimension0group1"/>
+  <variableReference ref="dimension0group0"/>
+  <variableReference ref="dimension0"/>
+</nest>
+@end example
+
+@noindent
+XXX what are group maps?
+
+@example
+<nest id="nest_1973">
+  <variableReference ref="dimension1categories"/>
+  <variableReference ref="dimension1group1map"/>
+  <variableReference ref="dimension1group0map"/>
+  <variableReference ref="dimension1"/>
+</nest>
+<nest>
+  <variableReference ref="dimension0categories"/>
+  <variableReference ref="dimension0group0map"/>
+  <variableReference ref="dimension0"/>
+</nest>
+@end example
+
+@noindent
+A @code{nest} can contain multiple dimensions:
+
+@example
+<nest>
+  <variableReference ref="dimension1categories"/>
+  <variableReference ref="dimension1group0"/>
+  <variableReference ref="dimension1"/>
+  <variableReference ref="dimension0categories"/>
+  <variableReference ref="dimension0"/>
+</nest>
+@end example
+
+One @code{nest} within a given @code{cross} may have no dimensions, in
+which case it still has one @code{variableReference} child, which
+references a @code{derivedVariable} whose @code{value} attribute is
+@code{constant(0)}.  In the corpus, such a @code{derivedVariable} has
+@code{row} or @code{column}, respectively, as its @code{id}.
+
+@code{nest} has no attributes (other than @code{id}).
+
+@subsubheading The @code{variableReference} Element
+
+Parent: @code{nest} @*
+Contents: empty
+
+@code{variableReference} has one attribute.
+
+@defvr {Required} ref
+The @code{id} of a @code{sourceVariable} or @code{derivedVariable}
+element.
+@end defvr
+
+@subsubheading The @code{layer} Element
+
+Parent: @code{faceting} @*
+Contents: empty
+
+Each layer is represented by a pair of @code{layer} elements.  The
+first of this pair is for a category variable, the second for the data
+variable, e.g.:
+
+@example
+<layer value="0" variable="dimension0categories" visible="true"/>
+<layer value="dimension0" variable="dimension0" visible="false"/>
+@end example
+
+@noindent
+@code{layer} has the following attributes.
+
+@defvr {Required} variable
+The @code{id} of a @code{sourceVariable} or @code{derivedVariable}
+element.
+@end defvr
+
+@defvr {Required} value
+The value to select.  For a category variable, this is always
+@code{0}; for a data variable, it is the same as the @code{variable}
+attribute.
+@end defvr
+
+@defvr {Optional} visible
+Whether the layer is visible.  Generally, category layers are visible
+and data layers are not, but sometimes this attribute is omitted.
+@end defvr
+
+@defvr {Optional} method
+When present, this is always @code{nest}.
+@end defvr
+
+@node SPV Detail facetLayout Element
+@subsection The @code{facetLayout} Element
+
+Parent: @code{graph} @*
+Contents: @code{tableLayout} @code{facetLevel}@math{+} @code{setCellProperties}*
+@subsubheading The @code{tableLayout} Element
+
+Parent: @code{facetLayout} @*
+Contents: empty
+
+@defvr {Required} verticalTitlesInCorner
+Always set to @code{true}.
+@end defvr
+
+@defvr {Optional} style
+The @code{id} of a @code{style} element.
+@end defvr
+
+@defvr {Optional} fitCells
+Always set to @code{ticks}.
+@end defvr
+
+@subsubheading The @code{facetLevel} Element
+
+Parent: @code{facetLayout}
+Contents: @code{axis}
+
+Each @code{facetLevel} describes a @code{variableReference} or
+@code{layer}, and thus a table has one @code{facetLevel} element for
+each such element.  For example, an SPV detail member that contains
+four @code{variableReference} elements and two @code{layer} elements
+will contain six @code{facetLevel} elements.
+
+In the corpus, @code{facetLevel} elements and the elements that they
+describe are always in the same order.  The correspondence may also be
+observed in two other ways.  First, one may use the @code{level}
+attribute, described below.  Second, in the corpus, a
+@code{facetLevel} always has an @code{id} that is the same as the
+@code{id} of the element it describes with @code{_facetLevel}
+appended.  One should not formally rely on this, of course, but it is
+usefully indicative.
+
+@defvr {Required} level
+A 1-based index into the @code{variableReference} and @code{layer}
+elements, e.g.@: a @code{facetLayout} with a @code{level} of 1
+describes the first @code{variableReference} in the SPV detail member,
+and in a member with four @code{variableReference} elements, a
+@code{facetLayout} with a @code{level} of 5 describes the first
+@code{layer} in the member.
+@end defvr
+
+@defvr {Required} gap
+Always observed as @code{0pt}.
+@end defvr
-- 
2.30.2


From d756f0a8abc31ab634f9aff532021e3eb88374ab Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Tue, 23 Feb 2016 22:22:04 -0800
Subject: [PATCH 07/16] Document a lot of setCellProperties.

---
 detail-xml           |  4 +--
 parse-detail-xml     |  2 +-
 parse-xml.c          | 37 ++++++++++++++++----
 spv-file-format.texi | 80 +++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 113 insertions(+), 10 deletions(-)

diff --git a/detail-xml b/detail-xml
index 9fca90acf4..cae148b688 100644
--- a/detail-xml
+++ b/detail-xml
@@ -88,8 +88,8 @@ paragraph := [id? hangingIndent]
 
 setCellProperties :=
     [id? applyToConverse]
-    setMetadata setStyle* setFormat+ union?
-setMetadata := [id? key target value]
+    setMetaData setStyle* setFormat+ union?
+setMetaData := [id? key target value]
 setStyle := [id? style target]
 setFormat :=
     [id? target reset?]
diff --git a/parse-detail-xml b/parse-detail-xml
index f0b175e7ab..0eb3555a74 100755
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -5,5 +5,5 @@ legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
 if test -n "$1"; then
     for d in $legacyXML; do
 	./parse-xml $d $1
-    done | sort | uniq
+    done
 fi
diff --git a/parse-xml.c b/parse-xml.c
index 9cf6694f91..70a7a89928 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -159,21 +159,44 @@ print_attribute (xmlNode *node, const char *attr)
     }
 }
 
+static void
+print_xml (xmlDoc *doc, xmlNode *node)
+{
+  xmlBuffer *buf = xmlBufferCreate();
+  xmlNodeDump (buf, doc, node, 0, 1);
+  xmlBufferDump (stdout, buf);
+  xmlBufferFree (buf);
+  putchar ('\n');
+}
+
 static void
 print_element (xmlDoc *doc, xmlNode *node, const char *element)
 {
   for (; node; node = node->next)
     {
       if (node->name && !strcmp(element, (char *) node->name))
+        print_xml (doc, node);
+
+      print_element (doc, node->children, element);
+    }
+}
+
+static void
+print_id (xmlDoc *doc, xmlNode *node, const char *id)
+{
+  for (; node; node = node->next)
+    {
+      if (node->type == XML_ELEMENT_NODE)
         {
-          xmlBuffer *buf = xmlBufferCreate();
-          xmlNodeDump (buf, doc, node, 0, 1);
-          xmlBufferDump (stdout, buf);
-          xmlBufferFree (buf);
-          putchar ('\n');
+          const char *node_id = (char *) xmlGetProp (node, (xmlChar *) "id");
+          if (node_id && !strcmp (node_id, id))
+            {
+              print_xml (doc, node);
+              break;
+            }
         }
 
-      print_element (doc, node->children, element);
+      print_id (doc, node->children, id);
     }
 }
 
@@ -259,6 +282,8 @@ main (int argc, char **argv)
     print_element (doc, root, argv[2] + 8);
   else if (!strncmp(argv[2], "attr:", 5))
     print_attribute (root, argv[2] + 5);
+  else if (!strncmp(argv[2], "id:", 3))
+    print_id (doc, root, argv[2] + 3);
   else if (!strcmp(argv[2], "labels"))
     print_labels (root);
   else
diff --git a/spv-file-format.texi b/spv-file-format.texi
index dd4b0b8f5a..1eb9a3ef52 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1267,6 +1267,7 @@ elements are assigned @code{id} attributes that are never referenced.
 * SPV Detail location Element::
 * SPV Detail coordinates Element::
 * SPV Detail faceting Element::
+* SPV Detail facetLayout Element::
 @end menu
 
 @node SPV Detail visualization Element
@@ -1752,6 +1753,7 @@ When present, this is always @code{nest}.
 
 Parent: @code{graph} @*
 Contents: @code{tableLayout} @code{facetLevel}@math{+} @code{setCellProperties}*
+
 @subsubheading The @code{tableLayout} Element
 
 Parent: @code{facetLayout} @*
@@ -1775,7 +1777,7 @@ Parent: @code{facetLayout}
 Contents: @code{axis}
 
 Each @code{facetLevel} describes a @code{variableReference} or
-@code{layer}, and thus a table has one @code{facetLevel} element for
+@code{layer}, and a table has one @code{facetLevel} element for
 each such element.  For example, an SPV detail member that contains
 four @code{variableReference} elements and two @code{layer} elements
 will contain six @code{facetLevel} elements.
@@ -1801,3 +1803,79 @@ and in a member with four @code{variableReference} elements, a
 @defvr {Required} gap
 Always observed as @code{0pt}.
 @end defvr
+
+@subsubheading The @code{setCellProperties} Element
+
+Parent: @code{facetLayout} @*
+Contents: @code{setMetaData} @code{setStyle}* @code{setFormat}@math{+} @code{union}?
+
+This element sets style properties of cells designated by the
+@code{target} attribute of its child elements.
+
+@defvr {Optional} applyToConverse
+If present, always @code{true}.  This appears to invert the meaning of
+the @code{target} of sub-elements: the selected cells are the ones
+@emph{not} designated by @code{target}.
+@end defvr
+
+@subsubheading The @code{setMetaData} Element
+
+Parent: @code{setCellProperties} @*
+Contents: empty
+
+It's really not clear what visible effect this element has, if any.
+
+@defvr {Required} target
+The @code{id} of an element whose metadata is to be set.  In the
+corpus, this is always @code{graph}, the @code{id} used for the
+@code{graph} element.
+@end defvr
+
+@defvr {Required} key
+@defvrx {Required} value
+A key-value pair to set for the target.
+
+In the corpus, @code{key} is @code{cellPropId} or, rarely,
+@code{diagProps}, and @code{value} is always the @code{id} of the
+parent @code{setCellProperties}.
+@end defvr
+
+@subsubheading The @code{setStyle} Element
+
+Parent: @code{setCellProperties} @*
+Contents: empty
+
+This element associates a style with the target.
+
+@defvr {Required} target
+The @code{id} of an element whose style is to be set.  In the corpus,
+this is always the @code{id} of an @code{interval}, @code{labeling},
+or, rarely, @code{graph} element.
+@end defvr
+
+@defvr {Required} style
+The @code{id} of a @code{style} element that identifies the style to
+set on the target.
+@end defvr
+
+@subsubheading The @code{setFormat} Element
+
+@format
+Parent: @code{setCellProperties}
+Contents:
+    @code{format}
+  @math{|} @code{numberFormat}
+  @math{|} @code{stringFormat}@math{+}
+  @math{|} @code{dateTimeFormat}
+@end format
+
+This element sets the format of the target, ``format'' in this case
+meaning an the SPSS print format for a variable.
+
+The details of this element vary depending on the schema version, as
+declared in the root @code{visualization} element's @code{version}
+attribute (@pxref{SPV Detail visualization Element}).  In version 2.5
+and earlier, @code{setFormat} contains one of a number of child
+elements that correspond to the different varieties of print formats.
+In version 2.7 and later, @code{setFormat} instead always contains a
+@code{format} element.
-- 
2.30.2


From e2c8f78fb3d4434e4d71aedd9b96c03b17835bf3 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 13 Mar 2016 13:52:22 -0700
Subject: [PATCH 08/16] parse-xml: Add ability to limit versions dumped.

---
 parse-detail-xml |  2 +-
 parse-xml.c      | 62 ++++++++++++++++++++++++++++++++----------------
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/parse-detail-xml b/parse-detail-xml
index 0eb3555a74..15a19341e5 100755
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -4,6 +4,6 @@
 legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
 if test -n "$1"; then
     for d in $legacyXML; do
-	./parse-xml $d $1
+	./parse-xml $d "$@"
     done
 fi
diff --git a/parse-xml.c b/parse-xml.c
index 70a7a89928..4d352ce8d8 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -236,6 +236,21 @@ print_text (xmlNode *node)
     }
 }
 
+static void
+check_version (xmlNode *root, const char *version)
+{
+  const char *s = (char *) xmlGetProp (root, (xmlChar *) "version");
+  if (!s)
+    {
+      fprintf (stderr, "missing version\n");
+      exit (1);
+    }
+  else if (strcmp(version, s))
+    {
+      exit (0);
+    }
+}
+
 static void
 usage (void)
 {
@@ -246,7 +261,7 @@ usage (void)
 int
 main (int argc, char **argv)
 {
-  if (argc != 3)
+  if (argc < 3)
     usage ();
 
   LIBXML_TEST_VERSION;
@@ -268,26 +283,31 @@ main (int argc, char **argv)
   root->next = NULL;
 #endif
 
-  if (!strcmp(argv[2], "parents"))
-    print_parents (root);
-  else if (!strcmp(argv[2], "containment"))
-    print_containment (root);
-  else if (!strcmp(argv[2], "attributes"))
-    print_attributes (root);
-  else if (!strcmp(argv[2], "cdata"))
-    print_cdata (root);
-  else if (!strcmp(argv[2], "text"))
-    print_text (root);
-  else if (!strncmp(argv[2], "element:", 8))
-    print_element (doc, root, argv[2] + 8);
-  else if (!strncmp(argv[2], "attr:", 5))
-    print_attribute (root, argv[2] + 5);
-  else if (!strncmp(argv[2], "id:", 3))
-    print_id (doc, root, argv[2] + 3);
-  else if (!strcmp(argv[2], "labels"))
-    print_labels (root);
-  else
-    usage ();
+  for (int i = 2; i < argc; i++)
+    {
+      if (!strcmp(argv[i], "parents"))
+        print_parents (root);
+      else if (!strcmp(argv[i], "containment"))
+        print_containment (root);
+      else if (!strcmp(argv[i], "attributes"))
+        print_attributes (root);
+      else if (!strcmp(argv[i], "cdata"))
+        print_cdata (root);
+      else if (!strcmp(argv[i], "text"))
+        print_text (root);
+      else if (!strncmp(argv[i], "element:", 8))
+        print_element (doc, root, argv[i] + 8);
+      else if (!strncmp(argv[i], "attr:", 5))
+        print_attribute (root, argv[i] + 5);
+      else if (!strncmp(argv[i], "id:", 3))
+        print_id (doc, root, argv[i] + 3);
+      else if (!strcmp(argv[i], "labels"))
+        print_labels (root);
+      else if (!strncmp(argv[i], "version=", 8))
+        check_version (root, argv[i] + 8);
+      else
+        usage ();
+    }
 
   xmlFreeDoc(doc);
   xmlCleanupParser();
-- 
2.30.2


From b0ea6e3d2705778be5e6ff8d079c72278d06b7b7 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 13 Mar 2016 17:39:49 -0700
Subject: [PATCH 09/16] Work on "format" element.

---
 spv-file-format.texi | 61 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index 1eb9a3ef52..15eef0eedf 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1879,3 +1879,64 @@ and earlier, @code{setFormat} contains one of a number of child
 elements that correspond to the different varieties of print formats.
 In version 2.7 and later, @code{setFormat} instead always contains a
 @code{format} element.
+
+@subsubheading The @code{format} Element
+
+Parent: @code{sourceVariable}, @code{derivedVariable}, @code{formatMapping}, @code{labeling}, @code{formatMapping}, @code{setFormat} @*
+Contents: (@code{affix}@math{+} @math{|} @code{relabel})?
+
+This element determines a format, equivalent to an SPSS print format.
+The following attribute determines the high-level kind of formatting
+in use:
+
+@defvr {Optional} baseFormat
+One of @code{date}, @code{time}, @code{dateTime}, or
+@code{elapsedTime}.  When this attribute is omitted, this element is a
+numeric or string format.
+@end defvr
+
+@noindent
+Whether other attributes are required (``yes''), prohibited (``no''),
+or optional (``opt'') depends on @code{baseFormat}:
+
+@multitable {maximumFractionDigits} {@code{dateTime}} {@code{elapsedTime}} {number} {string}
+@headitem Attribute @tab @code{dateTime} @tab @code{elapsedTime} @tab number @tab string
+@item errorCharacter        @tab yes @tab yes @tab yes @tab opt
+@item maximumFractionDigits @tab  no @tab yes @tab yes @tab no
+@item minimumFractionDigits @tab  no @tab yes @tab yes @tab no
+@item separatorChars        @tab yes @tab  no @tab  no @tab no
+@item
+@item useGrouping           @tab  no @tab opt @tab  no @tab no
+@item mdyOrder              @tab yes @tab  no @tab  no @tab no
+@item
+@item showYear              @tab yes @tab  no @tab  no @tab no
+@item yearAbbreviation      @tab yes @tab  no @tab  no @tab no
+@item
+@item showMonth             @tab yes @tab  no @tab  no @tab no
+@item monthFormat           @tab yes @tab  no @tab  no @tab no
+@item
+@item showDay               @tab yes @tab opt @tab  no @tab no
+@item dayPadding            @tab yes @tab opt @tab  no @tab no
+@item dayOfMonthPadding     @tab yes @tab  no @tab  no @tab no
+@item dayType               @tab yes @tab  no @tab  no @tab no
+@item
+@item showHour              @tab yes @tab opt @tab  no @tab no
+@item hourFormat            @tab yes @tab opt @tab  no @tab no
+@item hourPadding           @tab yes @tab yes @tab  no @tab no
+@item
+@item showMinute            @tab yes @tab  no @tab  no @tab no
+@item minutePadding         @tab yes @tab yes @tab  no @tab no
+@item
+@item showSecond            @tab yes @tab yes @tab  no @tab no
+@item secondPadding         @tab  no @tab yes @tab  no @tab no
+@item
+@item showMillis            @tab  no @tab yes @tab  no @tab no
+@item
+@item tryStringsAsNumbers   @tab  no @tab  no @tab  no @tab no
+@item
+@item minimumIntegerDigits  @tab  no @tab  no @tab yes @tab no
+@item scientific            @tab  no @tab  no @tab yes @tab no
+@item small                 @tab  no @tab  no @tab opt @tab no
+@item
+@item suffix                @tab  no @tab  no @tab opt @tab no
+@end multitable
-- 
2.30.2


From ce608320e0b30b4bfa7155acf500852f8783e4a4 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 10 Apr 2016 22:22:27 -0700
Subject: [PATCH 10/16] spv-file-format.texi: Work on more of the "format"
 element attributes.

---
 spv-file-format.texi | 145 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 128 insertions(+), 17 deletions(-)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index 15eef0eedf..f4cceb4b4c 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1896,47 +1896,158 @@ numeric or string format.
 @end defvr
 
 @noindent
-Whether other attributes are required (``yes''), prohibited (``no''),
-or optional (``opt'') depends on @code{baseFormat}:
+Whether, in the corpus, other attributes are always present (``yes''),
+never present (``no''), or sometimes present (``opt'') depends on
+@code{baseFormat}:
 
 @multitable {maximumFractionDigits} {@code{dateTime}} {@code{elapsedTime}} {number} {string}
 @headitem Attribute @tab @code{dateTime} @tab @code{elapsedTime} @tab number @tab string
 @item errorCharacter        @tab yes @tab yes @tab yes @tab opt
-@item maximumFractionDigits @tab  no @tab yes @tab yes @tab no
-@item minimumFractionDigits @tab  no @tab yes @tab yes @tab no
 @item separatorChars        @tab yes @tab  no @tab  no @tab no
-@item
-@item useGrouping           @tab  no @tab opt @tab  no @tab no
+@item @w{ }
 @item mdyOrder              @tab yes @tab  no @tab  no @tab no
-@item
+@item @w{ }
 @item showYear              @tab yes @tab  no @tab  no @tab no
 @item yearAbbreviation      @tab yes @tab  no @tab  no @tab no
-@item
+@item @w{ }
 @item showMonth             @tab yes @tab  no @tab  no @tab no
 @item monthFormat           @tab yes @tab  no @tab  no @tab no
-@item
+@item @w{ }
 @item showDay               @tab yes @tab opt @tab  no @tab no
 @item dayPadding            @tab yes @tab opt @tab  no @tab no
 @item dayOfMonthPadding     @tab yes @tab  no @tab  no @tab no
 @item dayType               @tab yes @tab  no @tab  no @tab no
-@item
+@item @w{ }
 @item showHour              @tab yes @tab opt @tab  no @tab no
 @item hourFormat            @tab yes @tab opt @tab  no @tab no
 @item hourPadding           @tab yes @tab yes @tab  no @tab no
-@item
+@item @w{ }
 @item showMinute            @tab yes @tab  no @tab  no @tab no
 @item minutePadding         @tab yes @tab yes @tab  no @tab no
-@item
+@item @w{ }
 @item showSecond            @tab yes @tab yes @tab  no @tab no
 @item secondPadding         @tab  no @tab yes @tab  no @tab no
-@item
+@item @w{ }
 @item showMillis            @tab  no @tab yes @tab  no @tab no
-@item
-@item tryStringsAsNumbers   @tab  no @tab  no @tab  no @tab no
-@item
+@item @w{ }
 @item minimumIntegerDigits  @tab  no @tab  no @tab yes @tab no
+@item maximumFractionDigits @tab  no @tab yes @tab yes @tab no
+@item minimumFractionDigits @tab  no @tab yes @tab yes @tab no
+@item useGrouping           @tab  no @tab opt @tab  no @tab no
+@item @w{ }
+@item tryStringsAsNumbers   @tab  no @tab  no @tab  no @tab no
+@item @w{ }
 @item scientific            @tab  no @tab  no @tab yes @tab no
 @item small                 @tab  no @tab  no @tab opt @tab no
-@item
+@item @w{ }
 @item suffix                @tab  no @tab  no @tab opt @tab no
 @end multitable
+
+Each attribute is described below.
+
+@defvr {Attribute} errorCharacter
+A character that replaces the formatted value when it cannot otherwise
+be represented in the given format.  Always @samp{*}.
+@end defvr
+
+@defvr {Attribute} separatorChars
+Exactly four characters.  In order, these are used for: decimal point,
+grouping, date separator, time separator.  Always @samp{.,-:}.
+@end defvr
+
+@defvr {Attribute} mdyOrder
+Within a date, the order of the days, months, and years.
+@code{dayMonthYear} is the only observed value, but one would expect
+that @code{monthDayYear} and @code{yearMonthDay} to be reasonable as
+well.
+@end defvr
+
+@defvr {Attribute} showYear
+@defvrx {Attribute} yearAbbreviation
+Whether to include the year and, if so, whether the year should be
+shown abbreviated, that is, with only 2 digits.  Each is @code{true}
+or @code{false}; only values of @code{true} and @code{false},
+respectively, have been observed.
+@end defvr
+
+@defvr {Attribute} showMonth
+@defvrx {Attribute} monthFormat
+Whether to include the month (@code{true} or @code{false}) and, if so,
+how to format it.  @code{monthFormat} is one of the following:
+
+@table @code
+@item long
+The full name of the month, e.g.@: in an English locale,
+@code{September}.
+
+@item short
+The abbreviated name of the month, e.g.@: in an English locale,
+@code{Sep}.
+
+@item number
+The number representing the month, e.g.@: 9 for September.
+
+@item paddedNumber
+A two-digit number representing the month, e.g.@: 09 for September.
+@end table
+
+Only values of @code{true} and @code{short}, respectively, have been
+observed.
+@end defvr
+
+@defvr {Attribute} showDay
+@defvrx {Attribute} dayPadding
+@defvrx {Attribute} dayOfMonthPadding
+@defvrx {Attribute} dayType
+When @code{baseFormat} is @code{elapsedTime}, @code{showDay} controls
+whether and how the ``day'' component (e.g.@: the number of elapsed
+24-hour periods) of the elapsed time is displayed.  @code{dayPadding}
+presumably controls whether the day is padded with spaces, but the
+details are not understood.  The only observed values are @code{true}
+and @code{true}, respectively.  @code{dayOfMonthPadding} and
+@code{dayType} are not used.
+
+When @code{baseFormat} is @code{dateTime}, @code{showDay}, which is
+always @code{true} in the corpus, controls whether a day is displayed.
+@code{dayType} is always @code{month} in the corpus, specifying that
+the day of the month is to be displayed; a value of @code{year} is
+supposed to indicate that the day of the year, where 1 is January 1,
+is to be displayed instead.  @code{dayOfMonthPadding} or possibly
+@code{dayPadding} presumably controls whether a 1-digit day of the
+month is padded with a space to occupy two positions; in the corpus
+both are always present and always @code{true}.  (A day of the year
+would presumably be padded to 3 positions.)
+@end defvr
+
+@defvr {Attribute} showHour
+@defvrx {Attribute} hourFormat
+@defvrx {Attribute} hourPadding
+@code{showHour} controls whether and how the hour component of a date
+or an elapsed time is displayed.  @code{hourPadding} presumably
+controls whether the hour is padded with spaces, but the details are
+not understood.  The only observed values are @code{true} and
+@code{true}, respectively.
+
+@code{hourFormat}, if present, is one of:
+
+@table @code
+@item AMPM
+The time is displayed with an @code{am} or @code{pm} suffix, e.g.@:
+@code{10:15pm}.
+
+@item AS_24
+The time is displayed in a 24-hour format, e.g.@: @code{22:15}.
+
+This is the only value observed in the corpus.
+
+@item AS_12
+The time is displayed in a 12-hour format, without distinguishing
+morning or evening, e.g.@: @code{10;15}.
+@end table
+
+@code{hourFormat} is sometimes present for @code{elapsedTime} formats,
+which is confusing since a time duration does not have a concept of AM
+or PM.  This might indicate a bug in the code that generated the XML
+in the corpus, or it might indicate that @code{elapsedTime} is
+sometimes used to format a time of day.
+@end defvr
-- 
2.30.2


From 317044f9d3cd7abf6dff515574538e59657f98af Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sat, 23 Jul 2016 11:02:06 -0700
Subject: [PATCH 11/16] Some progress on format documentation.

---
 Makefile             |  4 +++-
 parse-xml.c          |  8 +++++--
 spv-file-format.texi | 54 ++++++++++++++++++++------------------------
 3 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/Makefile b/Makefile
index cf5aa787de..348323a9a6 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,9 @@ parse-xml.o: CFLAGS := $(shell pkg-config --cflags libxml-2.0) $(base_cflags)
 parse-xml: LDFLAGS := $(shell pkg-config --libs libxml-2.0) $(LDFLAGS)
 dump2.o: CFLAGS := $(base_cflags) -Wno-unused
 
-all: dump dump2 parse-xml
+all: dump dump2 parse-xml spv-file-format.text
 dump: dump.o u8-mbtouc.o
 dump2: dump2.o u8-mbtouc.o
 parse-xml: parse-xml.o
+spv-file-format.text: spv-file-format.texi
+	makeinfo --force --plaintext -o $@ $<
diff --git a/parse-xml.c b/parse-xml.c
index 4d352ce8d8..d8726abcce 100644
--- a/parse-xml.c
+++ b/parse-xml.c
@@ -113,7 +113,11 @@ print_attributes (xmlNode * a_node)
         {
           printf ("%s", node->name);
           for (xmlAttr *attr = node->properties; attr; attr = attr->next)
-            printf (" %s", attr->name);
+            {
+              printf (" %s", attr->name);
+              if (!strcmp ((char *) attr->name, "baseFormat"))
+                printf ("=%s", (char *) xmlGetProp (node, attr->name));
+            }
           putchar ('\n');
         }
 
@@ -163,7 +167,7 @@ static void
 print_xml (xmlDoc *doc, xmlNode *node)
 {
   xmlBuffer *buf = xmlBufferCreate();
-  xmlNodeDump (buf, doc, node, 0, 1);
+  xmlNodeDump (buf, doc, node, 0, 0);
   xmlBufferDump (stdout, buf);
   xmlBufferFree (buf);
   putchar ('\n');
diff --git a/spv-file-format.texi b/spv-file-format.texi
index f4cceb4b4c..ce094484da 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1922,7 +1922,7 @@ never present (``no''), or sometimes present (``opt'') depends on
 @item hourFormat            @tab yes @tab opt @tab  no @tab no
 @item hourPadding           @tab yes @tab yes @tab  no @tab no
 @item @w{ }
-@item showMinute            @tab yes @tab  no @tab  no @tab no
+@item showMinute            @tab yes @tab yes @tab  no @tab no
 @item minutePadding         @tab yes @tab yes @tab  no @tab no
 @item @w{ }
 @item showSecond            @tab yes @tab yes @tab  no @tab no
@@ -1995,39 +1995,35 @@ Only values of @code{true} and @code{short}, respectively, have been
 observed.
 @end defvr
 
-@defvr {Attribute} showDay
-@defvrx {Attribute} dayPadding
+@defvr {Attribute} dayPadding
 @defvrx {Attribute} dayOfMonthPadding
-@defvrx {Attribute} dayType
-When @code{baseFormat} is @code{elapsedTime}, @code{showDay} controls
-whether and how the ``day'' component (e.g.@: the number of elapsed
-24-hour periods) of the elapsed time is displayed.  @code{dayPadding}
-presumably controls whether the day is padded with spaces, but the
-details are not understood.  The only observed values are @code{true}
-and @code{true}, respectively.  @code{dayOfMonthPadding} and
-@code{dayType} are not used.
-
-When @code{baseFormat} is @code{dateTime}, @code{showDay}, which is
-always @code{true} in the corpus, controls whether a day is displayed.
-@code{dayType} is always @code{month} in the corpus, specifying that
+@defvrx {Attribute} hourPadding
+@defvrx {Attribute} minutePadding
+@defvrx {Attribute} secondPadding
+These attributes presumably control whether each field in the output
+is padded with spaces to its maximum width, but the details are not
+understood.  The only observed value for any of these attributes is
+@code{true}.
+@end defvr
+
+@defvr {Attribute} showDay
+@defvrx {Attribute} showHour
+@defvrx {Attribute} showMinute
+@defvrx {Attribute} showSecond
+@defvrx {Attribute} showMillis
+These attributes presumably control whether each field is displayed
+in the output, but the details are not understood.  The only
+observed value for any of these attributes is @code{true}.
+@end defvr
+
+@defvr {Attribute} dayType
+This attribute is always @code{month} in the corpus, specifying that
 the day of the month is to be displayed; a value of @code{year} is
 supposed to indicate that the day of the year, where 1 is January 1,
-is to be displayed instead.  @code{dayOfMonthPadding} or possibly
-@code{dayPadding} presumably controls whether a 1-digit day of the
-month is padded with a space to occupy two positions; in the corpus
-both are always present and always @code{true}.  (A day of the year
-would presumably be padded to 3 positions.)
+is to be displayed instead.
 @end defvr
 
-@defvr {Attribute} showHour
-@defvrx {Attribute} hourFormat
-@defvrx {Attribute} hourPadding
-@code{showHour} controls whether and how the hour component of a date
-or an elapsed time is displayed.  @code{hourPadding} presumably
-controls whether the hour is padded with spaces, but the details are
-not understood.  The only observed values are @code{true} and
-@code{true}, respectively.
-
+@defvr {Attribute} hourFormat
 @code{hourFormat}, if present, is one of:
 
 @table @code
-- 
2.30.2


From 60e0b4f8a7f3210c94d9d488d7faac3b0d1d16f9 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 24 Jul 2016 19:26:18 -0700
Subject: [PATCH 12/16] spv-file-format: Finish documenting "format".

---
 spv-file-format.texi | 111 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 98 insertions(+), 13 deletions(-)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index ce094484da..e96ce17cb9 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1773,7 +1773,7 @@ Always set to @code{ticks}.
 
 @subsubheading The @code{facetLevel} Element
 
-Parent: @code{facetLayout}
+Parent: @code{facetLayout} @*
 Contents: @code{axis}
 
 Each @code{facetLevel} describes a @code{variableReference} or
@@ -1823,7 +1823,7 @@ the @code{target} of sub-elements: the selected cells are the ones
 Parent: @code{setCellProperties} @*
 Contents: empty
 
-It's really not clear what visible effect this element has, if any.
+This element is not known to have any visible effect.
 
 @defvr {Required} target
 The @code{id} of an element whose metadata is to be set.  In the
@@ -1870,7 +1870,7 @@ Contents:
 @end format
 
 This element sets the format of the target, ``format'' in this case
-meaning an the SPSS print format for a variable.
+meaning the SPSS print format for a variable.
 
 The details of this element vary depending on the schema version, as
 declared in the root @code{visualization} element's @code{version}
@@ -1880,14 +1880,17 @@ elements that correspond to the different varieties of print formats.
 In version 2.7 and later, @code{setFormat} instead always contains a
 @code{format} element.
 
-@subsubheading The @code{format} Element
+@subsubsection The @code{format} Element
 
 Parent: @code{sourceVariable}, @code{derivedVariable}, @code{formatMapping}, @code{labeling}, @code{formatMapping}, @code{setFormat} @*
 Contents: (@code{affix}@math{+} @math{|} @code{relabel})?
 
 This element determines a format, equivalent to an SPSS print format.
-The following attribute determines the high-level kind of formatting
-in use:
+
+@subsubheading Attributes for All Formats
+
+These attributes apply to all kinds of formats.  The most important of
+these attributes determines the high-level kind of formatting in use:
 
 @defvr {Optional} baseFormat
 One of @code{date}, @code{time}, @code{dateTime}, or
@@ -1903,6 +1906,7 @@ never present (``no''), or sometimes present (``opt'') depends on
 @multitable {maximumFractionDigits} {@code{dateTime}} {@code{elapsedTime}} {number} {string}
 @headitem Attribute @tab @code{dateTime} @tab @code{elapsedTime} @tab number @tab string
 @item errorCharacter        @tab yes @tab yes @tab yes @tab opt
+@item @w{ }
 @item separatorChars        @tab yes @tab  no @tab  no @tab no
 @item @w{ }
 @item mdyOrder              @tab yes @tab  no @tab  no @tab no
@@ -1933,23 +1937,25 @@ never present (``no''), or sometimes present (``opt'') depends on
 @item minimumIntegerDigits  @tab  no @tab  no @tab yes @tab no
 @item maximumFractionDigits @tab  no @tab yes @tab yes @tab no
 @item minimumFractionDigits @tab  no @tab yes @tab yes @tab no
-@item useGrouping           @tab  no @tab opt @tab  no @tab no
-@item @w{ }
-@item tryStringsAsNumbers   @tab  no @tab  no @tab  no @tab no
-@item @w{ }
+@item useGrouping           @tab  no @tab opt @tab yes @tab no
 @item scientific            @tab  no @tab  no @tab yes @tab no
 @item small                 @tab  no @tab  no @tab opt @tab no
-@item @w{ }
 @item suffix                @tab  no @tab  no @tab opt @tab no
+@item @w{ }
+@item tryStringsAsNumbers   @tab  no @tab  no @tab  no @tab yes
+@item @w{ }
 @end multitable
 
-Each attribute is described below.
-
 @defvr {Attribute} errorCharacter
 A character that replaces the formatted value when it cannot otherwise
 be represented in the given format.  Always @samp{*}.
 @end defvr
 
+@subsubheading Date and Time Attributes
+
+These attributes are used with @code{dateTime} and @code{elapsedTime}
+formats or both.
+
 @defvr {Attribute} separatorChars
 Exactly four characters.  In order, these are used for: decimal point,
 grouping, date separator, time separator.  Always @samp{.,-:}.
@@ -2047,3 +2053,82 @@ or PM.  This might indicate a bug in the code that generated the XML
 in the corpus, or it might indicate that @code{elapsedTime} is
 sometimes used to format a time of day.
 @end defvr
+
+@subsubheading Numeric Attributes
+
+These attributes are used for formats when @code{baseFormat} is
+@code{number}.  Attributes @code{maximumFractionDigits}, and
+@code{minimumFractionDigits}, and @code{useGrouping} are also used
+when @code{baseFormat} is @code{elapsedTime}.
+
+@defvr {Attribute} minimumIntegerDigits
+Minimum number of digits to display before the decimal point.  Always
+observed as @code{0}.
+@end defvr
+
+@defvr {Attribute} maximumFractionDigits
+@defvrx {Attribute} maximumFractionDigits
+Maximum or minimum, respectively, number of digits to display after
+the decimal point.  The observed values of each attribute range from 0
+to 9.
+@end defvr
+
+@defvr {Attribute} useGrouping
+Whether to use the grouping character to group digits in large
+numbers.  It would make sense for the grouping character to come from
+the @code{separatorChars} attribute, but that attribute is only
+present when @code{baseFormat} is @code{dateTime} or
+@code{elapsedTime}, in the corpus at least.  Perhaps that is because
+this attribute has only been observed as @code{false}.
+@end defvr
+
+@defvr {Attribute} scientific
+This attribute controls when and whether the number is formatted in
+scientific notation.  It takes the following values:
+
+@table @code
+@item onlyForSmall
+Use scientific notation only when the number's magnitude is smaller
+than the value of the @code{small} attribute.
+
+@item whenNeeded
+Use scientific notation when the number will not otherwise fit in the
+available space.
+
+@item true
+Always use scientific notation.  Not observed in the corpus.
+
+@item false
+Never use scientific notation.  A number that won't otherwise fit will
+be replaced by an error indication (see the @code{errorCharacter}
+attribute).  Not observed in the corpus.
+@end table
+@end defvr
+
+@defvr {Optional} small
+Only present when the @code{scientific} attribute is
+@code{onlyForSmall}, this is a numeric magnitude below which the
+number will be formatted in scientific notation.  The values @code{0}
+and @code{0.0001} have been observed.  The value @code{0} seems like a
+pathological choice, since no real number has a magnitude less than 0;
+perhaps in practice such a choice is equivalent to setting
+@code{scientific} to @code{false}.
+@end defvr
+
+@defvr {Optional} prefix
+@defvrx {Optional} suffix
+Specifies a prefix or a suffix to apply to the formatted number.  Only
+@code{suffix} has been observed, with value @samp{%}.
+@end defvr
+
+@subsubheading String Attributes
+
+These attributes are used for formats when @code{baseFormat} is
+@code{string}.
+
+@defvr {Attribute} tryStringsAsNumbers
+When this is @code{true}, it is supposed to indicate that string
+values should be parsed as numbers and then displayed according to
+numeric formatting rules.  However, in the corpus it is always
+@code{false}.
+@end defvr
-- 
2.30.2


From acbc23b06be71e6341fe9434c2f2e55e76ddb282 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 24 Jul 2016 20:41:52 -0700
Subject: [PATCH 13/16] Document other format-related elements.

---
 spv-file-format.texi | 177 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 173 insertions(+), 4 deletions(-)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index e96ce17cb9..a530ec9add 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1880,10 +1880,26 @@ elements that correspond to the different varieties of print formats.
 In version 2.7 and later, @code{setFormat} instead always contains a
 @code{format} element.
 
+XXX reinvestigate the above claim about versions: it appears to be
+incorrect.
+
+@menu
+* SPV Detail format Element::
+* SPV Detail numberFormat Element::
+* SPV Detail stringFormat Element::
+* SPV Detail dateTimeFormat Element::
+* SPV Detail affix Element::
+* SPV Detail relabel Element::
+@end menu
+
+@node SPV Detail format Element
 @subsubsection The @code{format} Element
 
 Parent: @code{sourceVariable}, @code{derivedVariable}, @code{formatMapping}, @code{labeling}, @code{formatMapping}, @code{setFormat} @*
-Contents: (@code{affix}@math{+} @math{|} @code{relabel})?
+Contents: (@code{affix}@math{+} @math{|} @code{relabel}@math{+})?
+
+This element appears only in schema version 2.7 (@pxref{SPV Detail
+visualization Element}).
 
 This element determines a format, equivalent to an SPSS print format.
 
@@ -1893,9 +1909,8 @@ These attributes apply to all kinds of formats.  The most important of
 these attributes determines the high-level kind of formatting in use:
 
 @defvr {Optional} baseFormat
-One of @code{date}, @code{time}, @code{dateTime}, or
-@code{elapsedTime}.  When this attribute is omitted, this element is a
-numeric or string format.
+Either @code{dateTime} or @code{elapsedTime}.  When this attribute is
+omitted, this element is a numeric or string format.
 @end defvr
 
 @noindent
@@ -2132,3 +2147,157 @@ values should be parsed as numbers and then displayed according to
 numeric formatting rules.  However, in the corpus it is always
 @code{false}.
 @end defvr
+
+@node SPV Detail numberFormat Element
+@subsubsection The @code{numberFormat} Element
+
+Parent: @code{setFormat} @*
+Contents: @code{affix}@math{+}
+
+This element appears only in schema version 2.5 and earlier
+(@pxref{SPV Detail visualization Element}).  Possibly this element
+could also contain @code{relabel} elements in a more diverse corpus.
+
+This element has the following attributes.
+
+@defvr {Attribute} maximumFractionDigits
+@defvrx {Attribute} minimumFractionDigits
+@defvrx {Attribute} minimumIntegerDigits
+@defvrx {Optional} scientific
+@defvrx {Optional} small
+@defvrx {Optional} suffix
+@defvrx {Optional} useGroupging
+The syntax and meaning of these attributes is the same as on the
+@code{format} element for a numeric format.  @pxref{SPV Detail format
+element}.
+@end defvr
+
+@node SPV Detail stringFormat Element
+@subsubsection The @code{stringFormat} Element
+
+Parent: @code{setFormat} @*
+Contents: (@code{affix}@math{+} @math{|} @code{relabel}@math{+})?
+
+This element appears only in schema version 2.5 and earlier
+(@pxref{SPV Detail visualization Element}).
+
+This element has no attributes.
+
+@node SPV Detail dateTimeFormat Element
+@subsubsection The @code{dateTimeFormat} Element
+
+Parent: @code{setFormat} @*
+Contents: empty
+
+This element appears only in schema version 2.5 and earlier
+(@pxref{SPV Detail visualization Element}).  Possibly this element
+could also contain @code{affix} and @code{relabel} elements in a more
+diverse corpus.
+
+The following attribute is required.
+
+@defvr {Attribute} baseFormat
+Either @code{dateTime} or @code{time}.
+@end defvr
+
+When @code{baseFormat} is @code{dateTime}, the following attributes
+are available.
+
+@defvr {Attribute} dayOfMonthPadding
+@defvrx {Attribute} dayPadding
+@defvrx {Attribute} dayType
+@defvrx {Attribute} hourFormat
+@defvrx {Attribute} hourPadding
+@defvrx {Attribute} mdyOrder
+@defvrx {Attribute} minutePadding
+@defvrx {Attribute} monthFormat
+@defvrx {Attribute} separatorChars
+@defvrx {Attribute} showDay
+@defvrx {Attribute} showHour
+@defvrx {Attribute} showMinute
+@defvrx {Attribute} showMonth
+@defvrx {Attribute} showSecond
+@defvrx {Attribute} showYear
+@defvrx {Attribute} yearAbbreviation
+The syntax and meaning of these attributes is the same as on the
+@code{format} element when that element's @code{baseFormat} is
+@code{dateTime}.  @pxref{SPV Detail format Element}.
+@end defvr
+
+When @code{baseFormat} is @code{time}, the following attributes are
+available.
+
+@defvr {Attribute} hourFormat
+@defvrx {Attribute} hourPadding
+@defvrx {Attribute} minutePadding
+@defvrx {Attribute} monthFormat
+@defvrx {Attribute} separatorChars
+@defvrx {Attribute} showDay
+@defvrx {Attribute} showHour
+@defvrx {Attribute} showMinute
+@defvrx {Attribute} showMonth
+@defvrx {Attribute} showSecond
+@defvrx {Attribute} showYear
+@defvrx {Attribute} yearAbbreviation
+The syntax and meaning of these attributes is the same as on the
+@code{format} element when that element's @code{baseFormat} is
+@code{elapsedTime}.  @pxref{SPV Detail format Element}.
+@end defvr
+
+@node SPV Detail affix Element
+@subsubsection The @code{affix} Element
+
+Parent: @code{format} or @code{numberFormat} or @code{stringFormat} @*
+Contents: empty
+
+Possibly this element could have @code{dateTimeFormat} as a parent in
+a more diverse corpus.
+
+This defines a suffix (or, theoretically, a prefix) for a formatted
+value.  It is used to insert a reference to a footnote.  It has the
+following attributes:
+
+@defvr {Attribute} definesReference
+This specifies the footnote number as a natural number: 1 for the
+first footnote, 2 for the second, and so on.
+@end defvr
+
+@defvr {Attribute} position
+Position for the footnote label.  Always @code{superscript}.
+@end defvr
+
+@defvr {Attribute} suffix
+Whether the affix is a suffix (@code{true}) or a prefix
+(@code{false}).  Always @code{true}.
+@end defvr
+
+@defvr {Attribute} value
+The text of the suffix or prefix.  Typically a letter, e.g.@: @code{a}
+for footnote 1, @code{b} for footnote 2, @enddots{}  The corpus
+contains other values: @code{*}, @code{**}, and a few that begin with
+at least one comma: @code{,b}, @code{,c}, @code{,,b}, and @code{,,c}.
+@end defvr
+
+@node SPV Detail relabel Element
+@subsubsection The @code{relabel} Element
+
+Parent: @code{format} or @code{stringFormat} @*
+Contents: empty
+
+Possibly this element could have @code{numberFormat} or
+@code{dateTimeFormat} as a parent in a more diverse corpus.
+
+This specifies how to display a given value.  It is used to implement
+value labels and to display the system-missing value in a
+human-readable way.  It has the following attributes:
+
+@defvr {Attribute} from
+The value to map.  In the corpus this is an integer or the
+system-missing value @code{-1.797693134862316E300}.
+@end defvr
+
+@defvr {Attribute} to
+The string to display in place of the value of @code{from}.  In the
+corpus this is a wide variety of value labels; the system-missing
+value is mapped to @samp{.}.
+@end defvr
-- 
2.30.2


From ebc66e3005143a6feb347e99e2d6e7e900623bf0 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 24 Jul 2016 22:22:41 -0700
Subject: [PATCH 14/16] Describe union element and its children, and some
 missed details earlier.

Next: axis.
---
 spv-file-format.texi | 80 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 78 insertions(+), 2 deletions(-)

diff --git a/spv-file-format.texi b/spv-file-format.texi
index a530ec9add..82afa6de72 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1810,12 +1810,18 @@ Parent: @code{facetLayout} @*
 Contents: @code{setMetaData} @code{setStyle}* @code{setFormat}@math{+} @code{union}?
 
 This element sets style properties of cells designated by the
-@code{target} attribute of its child elements.
+@code{target} attribute of its child elements, as further restricted
+by the optional @code{union} element if present.  The @code{target}
+values often used, e.g.@: @code{graph} or @code{labeling}, actually
+affect every cell, so the @code{union} element is a useful
+restriction.
 
 @defvr {Optional} applyToConverse
 If present, always @code{true}.  This appears to invert the meaning of
 the @code{target} of sub-elements: the selected cells are the ones
-@emph{not} designated by @code{target}.
+@emph{not} designated by @code{target}.  This is confusing, given the
+additional restrictions of @code{union}, but in the corpus
+@code{applyToConverse} is never present along with @code{union}.
 @end defvr
 
 @subsubheading The @code{setMetaData} Element
@@ -1883,6 +1889,21 @@ In version 2.7 and later, @code{setFormat} instead always contains a
 XXX reinvestigate the above claim about versions: it appears to be
 incorrect.
 
+The @code{setFormat} element itself has the following attributes.
+
+@defvr {Required} target
+The @code{id} of an element whose style is to be set.  In the corpus,
+this is always the @code{id} of an @code{majorTicks} or
+@code{labeling} element.
+@end defvr
+
+@defvr {Optional} reset
+If this is @code{true}, this format overrides the target's previous
+format.  If it is @code{false}, the adds to the previous format.  In
+the corpus this is always @code{true}.  The default behavior is
+unknown.
+@end defvr
+
 @menu
 * SPV Detail format Element::
 * SPV Detail numberFormat Element::
@@ -2301,3 +2322,58 @@ The string to display in place of the value of @code{from}.  In the
 corpus this is a wide variety of value labels; the system-missing
 value is mapped to @samp{.}.
 @end defvr
+
+@node SPV Detail union Element
+@subsubsection The @code{union} Element
+
+Parent: @code{setCellProperties} @*
+Contents: @code{intersect}@math{+}
+
+This element represents a set of cells, computed as the union of the
+sets represented by each of its children.
+
+@subsubheading The @code{intersect} Element
+
+Parent: @code{union} @*
+Contents: @code{where}@math{+} @math{|} @code{intersectWhere}?
+
+This element represents a set of cells, computed as the intersection
+of the sets represented by each of its children.
+
+Of the two possible children, in the corpus @code{where} is far more
+common, appearing thousands of times, whereas @code{intersectWhere}
+only appears 4 times.
+
+Most @code{intersect} elements have two or more children.
+
+@subsubheading The @code{where} Element
+
+Parent: @code{intersect} @*
+Contents: empty
+
+This element represents the set of cells in which the value of a
+specified variable falls within a specified set.
+
+@defvr {Attribute} variable
+The @code{id} of a variable, e.g.@: @code{dimension0categories} or
+@code{dimension0group0map}.
+@end defvr
+
+@defvr {Attribute} include
+A value, or multiple values separated by semicolons,
+e.g.@: @code{0} or @code{13;14;15;16}.
+@end defvr
+
+@subsubheading The @code{intersectWhere}
+
+Parent: @code{intersect} @*
+Contents: empty
+
+The meaning of this element is unknown.
+
+@defvr {Attribute} variable
+@defvrx {Attribute} variable2
+The meaning of these attributes is unknown.  In the four examples in
+the corpus they always take the values @code{dimension2categories} and
+@code{dimension0categories}, respectively.
+@end defvr
-- 
2.30.2


From c3f177db0bedf5629cc27faae5ed5de7f4c78e87 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Mon, 25 Jul 2016 23:05:24 -0700
Subject: [PATCH 15/16] Document labels and axes.

---
 Makefile             |   4 +-
 spv-detail.gv        |   7 ++-
 spv-file-format.texi | 110 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 119 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 348323a9a6..25f48159a5 100644
--- a/Makefile
+++ b/Makefile
@@ -5,9 +5,11 @@ parse-xml.o: CFLAGS := $(shell pkg-config --cflags libxml-2.0) $(base_cflags)
 parse-xml: LDFLAGS := $(shell pkg-config --libs libxml-2.0) $(LDFLAGS)
 dump2.o: CFLAGS := $(base_cflags) -Wno-unused
 
-all: dump dump2 parse-xml spv-file-format.text
+all: dump dump2 parse-xml spv-file-format.text spv-detail.pdf
 dump: dump.o u8-mbtouc.o
 dump2: dump2.o u8-mbtouc.o
 parse-xml: parse-xml.o
 spv-file-format.text: spv-file-format.texi
 	makeinfo --force --plaintext -o $@ $<
+spv-detail.pdf: spv-detail.gv
+	dot -T pdf -o$@ $<
diff --git a/spv-detail.gv b/spv-detail.gv
index 5704c6a49d..936ccd25b9 100644
--- a/spv-detail.gv
+++ b/spv-detail.gv
@@ -57,9 +57,10 @@ digraph spv_detail {
     "facetLayout" -> "setCellProperties" [label="*"];
     "facetLevel" -> "axis";
 
+    "axis" -> "label" [label="?"];
     "axis" -> "majorTicks";
 
-    "label" -> "descriptionGroup" [label="*"];
+    "label" -> "descriptionGroup" [label="?"];
     "label" -> "text" [label="*"];
 
     "majorTicks" -> "gridline" [label="?"];
@@ -89,4 +90,8 @@ digraph spv_detail {
 
     "intersect" -> "intersectWhere" [label="?"];
     "intersect" -> "where" [label="*"];
+
+    "text" -> "<text>";
+
+    "<text>" [label="text", shape=plaintext];
 }
diff --git a/spv-file-format.texi b/spv-file-format.texi
index 82afa6de72..3d0ffd725c 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -1804,6 +1804,115 @@ and in a member with four @code{variableReference} elements, a
 Always observed as @code{0pt}.
 @end defvr
 
+@subsubheading The @code{axis} Element
+
+Parent: @code{facetLevel} @*
+Contents: @code{label}? @code{majorTicks}
+
+@defvr {Attribute} style
+The @code{id} of a @code{style} element.
+@end defvr
+
+@subsubheading The @code{label} Element
+
+Parent: @code{axis} or @code{labelFrame} @*
+Contents: @code{text}@math{+} @math{|} @code{descriptionGroup}
+
+This element represents a label on some aspect of the table.  For example,
+the table's title is a @code{label}.
+
+The contents of the label can be one or more @code{text} elements or a
+@code{descriptionGroup}.
+
+@defvr {Attribute} style
+@defvrx {Optional} textFrameStyle
+Each of these is the @code{id} of a @code{style} element.
+@code{style} is the style of the label text, @code{textFrameStyle} the
+style for the frame around the label.
+@end defvr
+
+@defvr {Optional} purpose
+The kind of entity being labeled, one of @code{title},
+@code{subTitle}, @code{layer}, or @code{footnote}.
+@end defvr
+
+@subsubheading The @code{descriptionGroup} Element
+
+Parent: @code{label} @*
+Contents: (@code{description} @math{|} @code{text})@math{+}
+
+A @code{descriptionGroup} concatenates one or more elements to form a
+label.  Each element can be a @code{text} element, which contains
+literal text, or a @code{description} element that substitutes a value
+or a variable name.
+
+@defvr {Attribute} target
+The @code{id} of an element being described.  In the corpus, this is
+always @code{faceting}.
+@end defvr
+
+@defvr {Attribute} separator
+A string to separate the description of multiple groups, if the
+@code{target} has more than one.  In the corpus, this is always a
+new-line.
+@end defvr
+
+Typical contents for a @code{descriptionGroup} are a value by itself:
+@example
+<description name="value"/>
+@end example
+@noindent or a variable and its value, separated by a colon:
+@example
+<description name="variable"/><text>:</text><description name="value"/>
+@end example
+
+@subsubheading The @code{description} Element
+
+Parent: @code{descriptionGroup} @*
+Contents: empty
+
+A @code{description} is like a macro that expands to some property of
+the target of its parent @code{descriptionGroup}.
+
+@defvr {Attribute} name
+The name of the property.  Only @code{variable} and @code{value}
+appear in the corpus.
+@end defvr
+
+@subsubheading The @code{majorTicks} Element
+
+Parent: @code{axis} @*
+Contents: @code{gridline}?
+
+@defvr {Attribute} labelAngle
+@defvrx {Attribute} length
+Both always defined to @code{0}.
+@end defvr
+
+@defvr {Attribute} style
+@defvrx {Attribute} tickFrameStyle
+Each of these is the @code{id} of a @code{style} element.
+@code{style} is the style of the tick labels, @code{tickFrameStyle}
+the style for the frames around the labels.
+@end defvr
+
+@subsubheading The @code{gridline} Element
+
+Parent: @code{majorTicks} @*
+Contents: empty
+
+Represents ``gridlines,'' which for a table represents the lines
+between the rows or columns of a table (XXX?).
+
+@defvr {Attribute} style
+The style for the gridline.
+@end defvr
+
+@defvr {Attribute} zOrder
+Observed as a number between 28 and 31.  Does not seem to be
+important.
+@end defvr
+
 @subsubheading The @code{setCellProperties} Element
 
 Parent: @code{facetLayout} @*
@@ -1911,6 +2020,7 @@ unknown.
 * SPV Detail dateTimeFormat Element::
 * SPV Detail affix Element::
 * SPV Detail relabel Element::
+* SPV Detail union Element::
 @end menu
 
 @node SPV Detail format Element
-- 
2.30.2


From 91e1d822895e84a15e17b15762c6d33629f83342 Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sun, 21 May 2017 16:30:24 -0700
Subject: [PATCH 16/16] dump: Make it work with updated corpus.

---
 dump.c               | 167 +++++++++++++++++++++++--------------------
 parse-all-light      |   7 +-
 parse-detail-xml     |   2 +-
 spv-file-format.texi |   2 +-
 4 files changed, 96 insertions(+), 82 deletions(-)

diff --git a/dump.c b/dump.c
index a52cafddf0..61ee61ac0c 100644
--- a/dump.c
+++ b/dump.c
@@ -57,6 +57,15 @@ get_u32(void)
   return x;
 }
 
+static unsigned int
+get_u16(void)
+{
+  uint16_t x;
+  memcpy(&x, &data[pos], 2);
+  pos += 2;
+  return x;
+}
+
 static double
 get_double(void)
 {
@@ -225,6 +234,24 @@ get_end(void)
   return pos + len;
 }
 
+static void __attribute__((unused))
+hex_dump(int ofs, int n)
+{
+  for (int i = 0; i < n; i++)
+    {
+      int c = data[ofs + i];
+#if 1
+      if (i && !(i % 16))
+        fprintf(stderr, "-");
+      else
+        fprintf(stderr, " ");
+#endif
+      fprintf(stderr, "%02x", c);
+      //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
+    }
+  fprintf(stderr, "\n");
+}
+
 static char *
 dump_counted_string(void)
 {
@@ -232,7 +259,13 @@ dump_counted_string(void)
   int inner_end = get_end();
   if (pos != inner_end)
     {
-      match_u32_assert(0);
+      if (match_u32(5))
+        {
+          match_u32_assert(0);
+          match_byte_assert(0x58);
+        }
+      else
+        match_u32_assert(0);
       if (match_byte(0x31))
         s = get_string();
       else
@@ -246,24 +279,6 @@ dump_counted_string(void)
   return s;
 }
 
-static void __attribute__((unused))
-hex_dump(int ofs, int n)
-{
-  for (int i = 0; i < n; i++)
-    {
-      int c = data[ofs + i];
-#if 1
-      if (i && !(i % 16))
-        fprintf(stderr, "-");
-      else
-        fprintf(stderr, " ");
-#endif
-      fprintf(stderr, "%02x", c);
-      //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
-    }
-  fprintf(stderr, "\n");
-}
-
 static void
 dump_style(FILE *stream)
 {
@@ -385,33 +400,20 @@ dump_value_modifier(FILE *stream)
             }
           fprintf(stream, "/>\n");
         }
-      else if (match_u32 (1))
-        {
-          fprintf(stream, "<footnote-ref index=%d", get_u32());
-          dump_nested_string(stream);
-          fprintf(stream, "/>\n");
-        }
-      else if (match_u32 (2))
-        {
-          fprintf(stream, "<special2 %d %d", data[pos], data[pos + 2]);
-          if (!match_byte(0) && !match_byte(1))
-            match_byte_assert(2);
-          match_byte_assert(0);
-          if (!match_u32 (2) && !match_u32(1))
-            match_u32_assert(3);
-          dump_nested_string(stream);
-          fprintf(stream, "/>\n");
-        }
       else
         {
-          fprintf(stream, "<special3");
-          match_u32_assert(3);
-          match_byte_assert(0);
+          int count = get_u32();
+          fprintf(stream, "<footnote-ref indexes=\"");
+          for (int i = 0; i < count; i++)
+            {
+              if (i)
+                putc(' ', stream);
+              fprintf(stream, "%d", get_u16());
+            }
+          putc('"', stream);
           match_byte_assert(0);
-          match_byte_assert(1);
           match_byte_assert(0);
-          match_u32_assert(2);
-          dump_nested_string(stream); /* Our corpus doesn't contain any examples with strings though. */
+          dump_nested_string(stream);
           fprintf(stream, "/>\n");
         }
     }
@@ -612,7 +614,8 @@ check_permutation(int *a, int n, const char *name)
 }
 
 static void
-dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
+dump_category(FILE *stream, int level, int **indexes, int *allocated_indexes,
+              int *n_indexes)
 {
   for (int i = 0; i <= level; i++)
     fprintf (stream, "    ");
@@ -662,12 +665,12 @@ dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_ind
           fprintf(stderr, "index not -1 but subcategories\n");
           exit(1);
         }
-      if (*n_indexes >= max_indexes)
+      if (*n_indexes >= *allocated_indexes)
         {
-          fprintf(stderr, "too many categories (increase max_indexes)\n");
-          exit(1);
+          *allocated_indexes = *allocated_indexes ? 2 * *allocated_indexes : 16;
+          *indexes = realloc(*indexes, *allocated_indexes * sizeof **indexes);
         }
-      indexes[(*n_indexes)++] = indx;
+      (*indexes)[(*n_indexes)++] = indx;
     }
 
   int expected_unindexed = indx == -1;
@@ -685,7 +688,7 @@ dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_ind
       fprintf (stream, "<category-index>%d</category-index>\n", indx);
     }
   for (int i = 0; i < n_categories; i++)
-    dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
+    dump_category (stream, level + 1, indexes, allocated_indexes, n_indexes);
   for (int i = 0; i <= level; i++)
     fprintf (stream, "    ");
   printf ("</category>\n");
@@ -715,10 +718,11 @@ dump_dim(int indx)
     match_u32_assert(indx);
   n_categories = get_u32();
 
-  int indexes[2048];
+  int *indexes = NULL;
   int n_indexes = 0;
+  int allocated_indexes = 0;
   for (int i = 0; i < n_categories; i++)
-    dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
+    dump_category (stdout, 0, &indexes, &allocated_indexes, &n_indexes);
   check_permutation(indexes, n_indexes, "categories");
 
   fprintf (stdout, "</dimension>\n");
@@ -805,8 +809,14 @@ dump_title(void)
   match_byte(1);
   printf ("</title-c>\n");
 
-  match_byte(0);
-  match_byte_assert(0x58);
+  if (match_byte(0x31))
+    {
+      printf ("<user-caption>\n");
+      dump_value(stdout, 0);
+      printf ("</user-caption>\n");
+    }
+  else
+    match_byte_assert(0x58);
   if (match_byte(0x31))
     {
       printf ("<caption>\n");
@@ -851,7 +861,7 @@ dump_fonts(void)
       match_byte_assert(0);
 
       /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
-      if (!match_u32(0) && !match_u32(2))
+      if (!match_u32(0) && !match_u32(2) && !match_u32(4))
         {
           if (i == 7)
             match_u32_assert(0xfaad);
@@ -863,20 +873,18 @@ dump_fonts(void)
         match_u32_assert(3);
       printf (" fgcolor=\"%s\"", get_string());
       printf (" bgcolor=\"%s\"", get_string());
+      if (!match_byte(0))
+        match_byte_assert(1);
       match_u32_assert(0);
-      match_u32_assert(0);
-      match_byte_assert(0);
+      char *othercolor = get_string();
+      if (othercolor[0])
+        printf(" othercolor=\"%s\"", othercolor);
 
       if (version > 1)
         {
           if (i != 3)
             {
-              if (!match_u32(8))
-                match_u32_assert(5);
-              if (!match_u32(10) && !match_u32(11) && !match_u32(5))
-                match_u32_assert(9);
-              if (!match_u32(0) && !match_u32(1))
-                match_u32_assert(2);
+              pos += 12;
             }
           else
             {
@@ -897,8 +905,9 @@ dump_fonts(void)
   match_u32_assert(240);
   pos += 240;
 
-  match_u32_assert(18);
-  pos += 18;
+  int skip = get_u32();
+  assert(skip == 18 || skip == 25);
+  pos += skip;
 
   int x3 = get_u32();
   if (version == 3)
@@ -916,8 +925,7 @@ dump_fonts(void)
   const char *locale = get_string();
   printf ("<locale>%s</locale>\n", locale);
 
-  if (!match_u32(0))
-    match_u32_assert(UINT32_MAX);
+  get_u32();            /* Seen: 0, UINT32_MAX, 2, 3, 4, 5, 6, 8, 9, 21, 24. */
   if (!match_byte(0))
     match_byte_assert(1);
   match_byte_assert(0);
@@ -926,14 +934,16 @@ dump_fonts(void)
   if (version > 1)
     {
       if (!match_byte(0x97) && !match_byte(0x98)
-          && !match_byte(0x99) && !match_byte(0x9a))
-        match_byte_assert(0x9b);
+          && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b))
+        match_byte_assert(0x9c);
       match_byte_assert(7);
       match_byte_assert(0);
       match_byte_assert(0);
     }
   else
-    match_u32_assert(UINT32_MAX);
+    {
+      printf("%x\n", get_u32());
+    }
 
   int decimal = data[pos];
   int grouping = data[pos + 1];
@@ -945,7 +955,7 @@ dump_fonts(void)
   else
     {
       match_byte_assert(',');
-      if (!match_byte('.') && !match_byte(' '))
+      if (!match_byte('.') && !match_byte(' ') && !match_byte(','))
         match_byte_assert(0);
     }
   printf("<format decimal=\"%c\" grouping=\"", decimal);
@@ -996,8 +1006,8 @@ dump_fonts(void)
         match_byte_assert(1);
 
       if (!match_byte(0x97) && !match_byte(0x98)
-          && !match_byte(0x99) && !match_byte(0x9a))
-        match_byte_assert(0x9b);
+          && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b))
+        match_byte_assert(0x9c);
       match_byte_assert(7);
       match_byte_assert(0);
       match_byte_assert(0);
@@ -1010,7 +1020,7 @@ dump_fonts(void)
       else
         {
           match_byte_assert(',');
-          if (!match_byte('.') && !match_byte(' '))
+          if (!match_byte('.') && !match_byte(' ') && !match_byte(','))
             match_byte_assert(0);
         }
 
@@ -1055,7 +1065,7 @@ dump_fonts(void)
 
       if (pos < outer_end)
         {
-          match_u32_assert(2000000);
+          printf("<seed>%d</seed>\n", get_u32());
           match_u32_assert(0);
         }
       assert(pos == outer_end);
@@ -1063,7 +1073,9 @@ dump_fonts(void)
       pos = outer_end;
     }
   else
-    match_u32_assert(0);
+    {
+      pos = get_end();
+    }
 }
 
 int
@@ -1157,7 +1169,8 @@ main(int argc, char *argv[])
 
       /* Offset 8. */
       match_byte_assert(0);
-      match_byte_assert(0);
+      if (!match_byte(0))
+        match_byte_assert(1);
       if (!match_byte(0))
         match_byte_assert(1);
 
@@ -1191,8 +1204,8 @@ main(int argc, char *argv[])
       /* Offset 27. */
       pos++;
       pos++;
-      match_byte_assert(0);
-      match_byte_assert(0);
+      pos++;
+      pos++;
 
       /* Offset 31.
 
diff --git a/parse-all-light b/parse-all-light
index 1facaec43d..e63811cd9f 100755
--- a/parse-all-light
+++ b/parse-all-light
@@ -1,8 +1,9 @@
 #! /bin/bash
-for d in {williams,germano,smekens,unzipped,unzipped2}/*/*light*.bin; do
-    if ! ./dump all < $d >/dev/null; then
+make || exit 1
+for d in {unzipped3,williams,germano,smekens,unzipped,unzipped2}/*/*light*.bin; do
+    if ! ./dump all < $d > /dev/null; then
 	echo $d
-	./dump all < $d
+	#./dump all < $d
 	echo
     fi
 done
diff --git a/parse-detail-xml b/parse-detail-xml
index 15a19341e5..28fa551dc3 100755
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -1,7 +1,7 @@
 #! /bin/sh
 
 # Parse the detail XML members.
-legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
+legacyXML=`ls -1 unzipped*/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
 if test -n "$1"; then
     for d in $legacyXML; do
 	./parse-xml $d "$@"
diff --git a/spv-file-format.texi b/spv-file-format.texi
index 3d0ffd725c..ecc5f8e1b5 100644
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -194,7 +194,7 @@ file.
 
 @defvr {Optional} @code{creation-date-time}
 The date and time at which the SPV file was written, in a
-locale-specific format, e.g. @code{Friday, May 16, 2014 6:47:37 PM
+locale-specific format, e.g.@: @code{Friday, May 16, 2014 6:47:37 PM
 PDT} or @code{lunedì 17 marzo 2014 3.15.48 CET} or even @code{Friday,
 December 5, 2014 5:00:19 o'clock PM EST}.
 @end defvr
-- 
2.30.2