dump: Make it work with updated corpus.

diff --git a/dump.c b/dump.c
index a52cafddf0d414cb06ddd8464c9795949b91e2d9..61ee61ac0c72639825ca6e258a4386606b40020e 100644 (file)
--- a/dump.c
+++ b/dump.c
@@ -57,6 +57,15 @@ get_u32(void)
   return x;
 }
 
+static unsigned int
+get_u16(void)
+{
+  uint16_t x;
+  memcpy(&x, &data[pos], 2);
+  pos += 2;
+  return x;
+}
+
 static double
 get_double(void)
 {
@@ -225,6 +234,24 @@ get_end(void)
   return pos + len;
 }
 
+static void __attribute__((unused))
+hex_dump(int ofs, int n)
+{
+  for (int i = 0; i < n; i++)
+    {
+      int c = data[ofs + i];
+#if 1
+      if (i && !(i % 16))
+        fprintf(stderr, "-");
+      else
+        fprintf(stderr, " ");
+#endif
+      fprintf(stderr, "%02x", c);
+      //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
+    }
+  fprintf(stderr, "\n");
+}
+
 static char *
 dump_counted_string(void)
 {
@@ -232,7 +259,13 @@ dump_counted_string(void)
   int inner_end = get_end();
   if (pos != inner_end)
     {
-      match_u32_assert(0);
+      if (match_u32(5))
+        {
+          match_u32_assert(0);
+          match_byte_assert(0x58);
+        }
+      else
+        match_u32_assert(0);
       if (match_byte(0x31))
         s = get_string();
       else
@@ -246,24 +279,6 @@ dump_counted_string(void)
   return s;
 }
 
-static void __attribute__((unused))
-hex_dump(int ofs, int n)
-{
-  for (int i = 0; i < n; i++)
-    {
-      int c = data[ofs + i];
-#if 1
-      if (i && !(i % 16))
-        fprintf(stderr, "-");
-      else
-        fprintf(stderr, " ");
-#endif
-      fprintf(stderr, "%02x", c);
-      //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
-    }
-  fprintf(stderr, "\n");
-}
-
 static void
 dump_style(FILE *stream)
 {
@@ -385,33 +400,20 @@ dump_value_modifier(FILE *stream)
             }
           fprintf(stream, "/>\n");
         }
-      else if (match_u32 (1))
-        {
-          fprintf(stream, "<footnote-ref index=%d", get_u32());
-          dump_nested_string(stream);
-          fprintf(stream, "/>\n");
-        }
-      else if (match_u32 (2))
-        {
-          fprintf(stream, "<special2 %d %d", data[pos], data[pos + 2]);
-          if (!match_byte(0) && !match_byte(1))
-            match_byte_assert(2);
-          match_byte_assert(0);
-          if (!match_u32 (2) && !match_u32(1))
-            match_u32_assert(3);
-          dump_nested_string(stream);
-          fprintf(stream, "/>\n");
-        }
       else
         {
-          fprintf(stream, "<special3");
-          match_u32_assert(3);
-          match_byte_assert(0);
+          int count = get_u32();
+          fprintf(stream, "<footnote-ref indexes=\"");
+          for (int i = 0; i < count; i++)
+            {
+              if (i)
+                putc(' ', stream);
+              fprintf(stream, "%d", get_u16());
+            }
+          putc('"', stream);
           match_byte_assert(0);
-          match_byte_assert(1);
           match_byte_assert(0);
-          match_u32_assert(2);
-          dump_nested_string(stream); /* Our corpus doesn't contain any examples with strings though. */
+          dump_nested_string(stream);
           fprintf(stream, "/>\n");
         }
     }
@@ -612,7 +614,8 @@ check_permutation(int *a, int n, const char *name)
 }
 
 static void
-dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
+dump_category(FILE *stream, int level, int **indexes, int *allocated_indexes,
+              int *n_indexes)
 {
   for (int i = 0; i <= level; i++)
     fprintf (stream, "    ");
@@ -662,12 +665,12 @@ dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_ind
           fprintf(stderr, "index not -1 but subcategories\n");
           exit(1);
         }
-      if (*n_indexes >= max_indexes)
+      if (*n_indexes >= *allocated_indexes)
         {
-          fprintf(stderr, "too many categories (increase max_indexes)\n");
-          exit(1);
+          *allocated_indexes = *allocated_indexes ? 2 * *allocated_indexes : 16;
+          *indexes = realloc(*indexes, *allocated_indexes * sizeof **indexes);
         }
-      indexes[(*n_indexes)++] = indx;
+      (*indexes)[(*n_indexes)++] = indx;
     }
 
   int expected_unindexed = indx == -1;
@@ -685,7 +688,7 @@ dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_ind
       fprintf (stream, "<category-index>%d</category-index>\n", indx);
     }
   for (int i = 0; i < n_categories; i++)
-    dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
+    dump_category (stream, level + 1, indexes, allocated_indexes, n_indexes);
   for (int i = 0; i <= level; i++)
     fprintf (stream, "    ");
   printf ("</category>\n");
@@ -715,10 +718,11 @@ dump_dim(int indx)
     match_u32_assert(indx);
   n_categories = get_u32();
 
-  int indexes[2048];
+  int *indexes = NULL;
   int n_indexes = 0;
+  int allocated_indexes = 0;
   for (int i = 0; i < n_categories; i++)
-    dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
+    dump_category (stdout, 0, &indexes, &allocated_indexes, &n_indexes);
   check_permutation(indexes, n_indexes, "categories");
 
   fprintf (stdout, "</dimension>\n");
@@ -805,8 +809,14 @@ dump_title(void)
   match_byte(1);
   printf ("</title-c>\n");
 
-  match_byte(0);
-  match_byte_assert(0x58);
+  if (match_byte(0x31))
+    {
+      printf ("<user-caption>\n");
+      dump_value(stdout, 0);
+      printf ("</user-caption>\n");
+    }
+  else
+    match_byte_assert(0x58);
   if (match_byte(0x31))
     {
       printf ("<caption>\n");
@@ -851,7 +861,7 @@ dump_fonts(void)
       match_byte_assert(0);
 
       /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
-      if (!match_u32(0) && !match_u32(2))
+      if (!match_u32(0) && !match_u32(2) && !match_u32(4))
         {
           if (i == 7)
             match_u32_assert(0xfaad);
@@ -863,20 +873,18 @@ dump_fonts(void)
         match_u32_assert(3);
       printf (" fgcolor=\"%s\"", get_string());
       printf (" bgcolor=\"%s\"", get_string());
+      if (!match_byte(0))
+        match_byte_assert(1);
       match_u32_assert(0);
-      match_u32_assert(0);
-      match_byte_assert(0);
+      char *othercolor = get_string();
+      if (othercolor[0])
+        printf(" othercolor=\"%s\"", othercolor);
 
       if (version > 1)
         {
           if (i != 3)
             {
-              if (!match_u32(8))
-                match_u32_assert(5);
-              if (!match_u32(10) && !match_u32(11) && !match_u32(5))
-                match_u32_assert(9);
-              if (!match_u32(0) && !match_u32(1))
-                match_u32_assert(2);
+              pos += 12;
             }
           else
             {
@@ -897,8 +905,9 @@ dump_fonts(void)
   match_u32_assert(240);
   pos += 240;
 
-  match_u32_assert(18);
-  pos += 18;
+  int skip = get_u32();
+  assert(skip == 18 || skip == 25);
+  pos += skip;
 
   int x3 = get_u32();
   if (version == 3)
@@ -916,8 +925,7 @@ dump_fonts(void)
   const char *locale = get_string();
   printf ("<locale>%s</locale>\n", locale);
 
-  if (!match_u32(0))
-    match_u32_assert(UINT32_MAX);
+  get_u32();            /* Seen: 0, UINT32_MAX, 2, 3, 4, 5, 6, 8, 9, 21, 24. */
   if (!match_byte(0))
     match_byte_assert(1);
   match_byte_assert(0);
@@ -926,14 +934,16 @@ dump_fonts(void)
   if (version > 1)
     {
       if (!match_byte(0x97) && !match_byte(0x98)
-          && !match_byte(0x99) && !match_byte(0x9a))
-        match_byte_assert(0x9b);
+          && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b))
+        match_byte_assert(0x9c);
       match_byte_assert(7);
       match_byte_assert(0);
       match_byte_assert(0);
     }
   else
-    match_u32_assert(UINT32_MAX);
+    {
+      printf("%x\n", get_u32());
+    }
 
   int decimal = data[pos];
   int grouping = data[pos + 1];
@@ -945,7 +955,7 @@ dump_fonts(void)
   else
     {
       match_byte_assert(',');
-      if (!match_byte('.') && !match_byte(' '))
+      if (!match_byte('.') && !match_byte(' ') && !match_byte(','))
         match_byte_assert(0);
     }
   printf("<format decimal=\"%c\" grouping=\"", decimal);
@@ -996,8 +1006,8 @@ dump_fonts(void)
         match_byte_assert(1);
 
       if (!match_byte(0x97) && !match_byte(0x98)
-          && !match_byte(0x99) && !match_byte(0x9a))
-        match_byte_assert(0x9b);
+          && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b))
+        match_byte_assert(0x9c);
       match_byte_assert(7);
       match_byte_assert(0);
       match_byte_assert(0);
@@ -1010,7 +1020,7 @@ dump_fonts(void)
       else
         {
           match_byte_assert(',');
-          if (!match_byte('.') && !match_byte(' '))
+          if (!match_byte('.') && !match_byte(' ') && !match_byte(','))
             match_byte_assert(0);
         }
 
@@ -1055,7 +1065,7 @@ dump_fonts(void)
 
       if (pos < outer_end)
         {
-          match_u32_assert(2000000);
+          printf("<seed>%d</seed>\n", get_u32());
           match_u32_assert(0);
         }
       assert(pos == outer_end);
@@ -1063,7 +1073,9 @@ dump_fonts(void)
       pos = outer_end;
     }
   else
-    match_u32_assert(0);
+    {
+      pos = get_end();
+    }
 }
 
 int
@@ -1157,7 +1169,8 @@ main(int argc, char *argv[])
 
       /* Offset 8. */
       match_byte_assert(0);
-      match_byte_assert(0);
+      if (!match_byte(0))
+        match_byte_assert(1);
       if (!match_byte(0))
         match_byte_assert(1);
 
@@ -1191,8 +1204,8 @@ main(int argc, char *argv[])
       /* Offset 27. */
       pos++;
       pos++;
-      match_byte_assert(0);
-      match_byte_assert(0);
+      pos++;
+      pos++;
 
       /* Offset 31.
 

diff --git a/parse-all-light b/parse-all-light
index 1facaec43d2bf858272a30f356862464e2d724af..e63811cd9fb973d2405544b905ea5229098ff5f4 100755 (executable)
--- a/parse-all-light
+++ b/parse-all-light
@@ -1,8 +1,9 @@
 #! /bin/bash
-for d in {williams,germano,smekens,unzipped,unzipped2}/*/*light*.bin; do
-    if ! ./dump all < $d >/dev/null; then
+make || exit 1
+for d in {unzipped3,williams,germano,smekens,unzipped,unzipped2}/*/*light*.bin; do
+    if ! ./dump all < $d > /dev/null; then
        echo $d
-       ./dump all < $d
+       #./dump all < $d
        echo
     fi
 done

diff --git a/parse-detail-xml b/parse-detail-xml
index 15a19341e596ce497eda47901a8b52deb8b5b633..28fa551dc3b210d4a1ea863eb857e2a2f4ce7107 100755 (executable)
--- a/parse-detail-xml
+++ b/parse-detail-xml
@@ -1,7 +1,7 @@
 #! /bin/sh
 
 # Parse the detail XML members.
-legacyXML=`ls -1 unzipped/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
+legacyXML=`ls -1 unzipped*/*/*.xml |grep -vE 'outputViewer|stats|chart|model'`
 if test -n "$1"; then
     for d in $legacyXML; do
        ./parse-xml $d "$@"

diff --git a/spv-file-format.texi b/spv-file-format.texi
index 3d0ffd725c7d067b7e9b980d4182e3f68d5e2001..ecc5f8e1b5df4fb3ebe5a9cf25ddbe08bc30b2a2 100644 (file)
--- a/spv-file-format.texi
+++ b/spv-file-format.texi
@@ -194,7 +194,7 @@ file.
 
 @defvr {Optional} @code{creation-date-time}
 The date and time at which the SPV file was written, in a
-locale-specific format, e.g. @code{Friday, May 16, 2014 6:47:37 PM
+locale-specific format, e.g.@: @code{Friday, May 16, 2014 6:47:37 PM
 PDT} or @code{lunedì 17 marzo 2014 3.15.48 CET} or even @code{Friday,
 December 5, 2014 5:00:19 o'clock PM EST}.
 @end defvr