+#include <assert.h>
#include <float.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
+#include "u8-mbtouc.h"
static uint8_t *data;
static size_t n;
}
+static bool __attribute__((unused))
+all_utf8(const char *p_)
+{
+ const uint8_t *p = (const uint8_t *) p_;
+ size_t len = strlen ((char *) p);
+ for (size_t ofs = 0, mblen; ofs < len; ofs += mblen)
+ {
+ ucs4_t uc;
+
+ mblen = u8_mbtouc (&uc, p + ofs, len - ofs);
+ if ((uc < 32 && uc != '\n') || uc == 127 || uc == 0xfffd)
+ return false;
+ }
+ return true;
+}
+
static char *
get_string(const char *where)
{
}
#define get_string() get_string(WHERE)
+static int
+get_end(void)
+{
+ int len = get_u32();
+ return pos + len;
+}
+
static char *
-dump_nested_string(void)
+dump_counted_string(void)
{
char *s = NULL;
-
- match_byte_assert (0);
- match_byte_assert (0);
- int outer_end = pos + get_u32();
- int inner_end = pos + get_u32();
+ int inner_end = get_end();
if (pos != inner_end)
{
match_u32_assert(0);
exit(1);
}
}
- match_byte_assert(0x58);
+ return s;
+}
+
+static void __attribute__((unused))
+hex_dump(int ofs, int n)
+{
+ for (int i = 0; i < n; i++)
+ {
+ int c = data[ofs + i];
+#if 1
+ if (i && !(i % 16))
+ fprintf(stderr, "-");
+ else
+ fprintf(stderr, " ");
+#endif
+ fprintf(stderr, "%02x", c);
+ //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
+ }
+ fprintf(stderr, "\n");
+}
+
+static void
+dump_style(FILE *stream)
+{
+ match_byte(1);
+ match_byte(0);
+ match_byte(0);
+ match_byte(0);
+ match_byte_assert(1);
+ char *fg = get_string(); /* foreground */
+ char *bg = get_string(); /* background */
+ char *font = get_string(); /* font */
+ int size = data[pos];
+ if (!match_byte(14))
+ match_byte_assert(12); /* size? */
+ fprintf(stream, " fgcolor=\"%s\" bgcolor=\"%s\" font=\"%s\" size=\"%d\"",
+ fg, bg, font, size);
+}
+
+static char *
+dump_nested_string(FILE *stream)
+{
+ char *s = NULL;
+
+ match_byte_assert (0);
+ match_byte_assert (0);
+ int outer_end = get_end();
+ s = dump_counted_string();
+ if (s)
+ fprintf(stream, " \"%s\"", s);
+ if (match_byte(0x31))
+ dump_style(stream);
+ else
+ match_byte_assert(0x58);
match_byte_assert(0x58);
if (pos != outer_end)
{
}
static void
-dump_value_31(FILE *stream)
+dump_value_modifier(FILE *stream)
{
if (match_byte (0x31))
{
if (match_u32 (0))
{
+ fprintf(stream, "<special0");
if (match_u32 (1))
{
- /* Only "a" observed as a sample value (although it appears 44 times in the corpus). */
- get_string();
+ /* Corpus frequencies:
+ 124 "a"
+ 12 "b"
+ 8 "a, b"
+
+ The given text is appended to the cell in a subscript font.
+ */
+ fprintf(stream, " subscript=\"%s\"", get_string());
}
else
match_u32_assert (0);
match_u32_assert(10);
match_byte(0);
match_byte(0);
+ fprintf(stream, "/>\n");
return;
}
- int outer_end = pos + get_u32();
- int inner_end = pos + get_u32();
- if (pos != inner_end)
- {
- match_u32_assert(0);
- if (match_byte(0x31))
- {
- /* Appears to be a template string, e.g. '^1 cells (^2) expf < 5. Min exp = ^3...'.
- Probably doesn't actually appear in output because many examples look unpolished,
- e.g. 'partial list cases value ^1 shown upper...' */
- get_string();
- }
- else
- match_byte_assert(0x58);
- if (pos != inner_end)
- {
- fprintf(stderr, "inner end discrepancy\n");
- exit(1);
- }
- }
+ int outer_end = get_end();
+
+ /* This counted-string appears to be a template string,
+ e.g. "Design\: [:^1:]1 Within Subjects Design\: [:^1:]2". */
+ char *template = dump_counted_string();
+ if (template)
+ fprintf(stream, " template=\"%s\"", template);
if (match_byte(0x31))
- {
- /* Only one example in the corpus. */
- match_byte(1);
- match_byte(0);
- match_byte(0);
- match_byte(0);
- match_byte_assert(1);
- get_string(); /* foreground */
- get_string(); /* background */
- get_string(); /* font */
- if (!match_byte(14))
- match_byte_assert(12); /* size? */
- }
+ dump_style(stream);
else
match_byte_assert(0x58);
if (match_byte(0x31))
fprintf(stderr, "outer end discrepancy\n");
exit(1);
}
+ fprintf(stream, "/>\n");
}
else if (match_u32 (1))
{
- fprintf(stream, "(footnote %d) ", get_u32());
- dump_nested_string();
+ fprintf(stream, "<footnote-ref index=%d", get_u32());
+ dump_nested_string(stream);
+ fprintf(stream, "/>\n");
}
else if (match_u32 (2))
{
- fprintf(stream, "(special 2)");
- if (!match_byte(0))
+ fprintf(stream, "<special2 %d %d", data[pos], data[pos + 2]);
+ if (!match_byte(0) && !match_byte(1))
match_byte_assert(2);
match_byte_assert(0);
if (!match_u32 (2) && !match_u32(1))
match_u32_assert(3);
- dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */
+ dump_nested_string(stream);
+ fprintf(stream, "/>\n");
}
else
{
+ fprintf(stream, "<special3");
match_u32_assert(3);
- fprintf(stream, "(special 3)");
match_byte_assert(0);
match_byte_assert(0);
match_byte_assert(1);
match_byte_assert(0);
match_u32_assert(2);
- dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */
+ dump_nested_string(stream); /* Our corpus doesn't contain any examples with strings though. */
+ fprintf(stream, "/>\n");
}
}
else
for (int i = 0; i <= level; i++)
fprintf (stream, " ");
- if (match_byte (3))
+ if (match_byte (1))
{
- char *text = get_string();
- dump_value_31(stream);
- char *identifier = get_string();
- char *text_eng = get_string();
- fprintf (stream, "<string c=\"%s\"", text_eng);
- if (identifier[0])
- fprintf (stream, " identifier=\"%s\"", identifier);
- if (strcmp(text_eng, text))
- fprintf (stream, " local=\"%s\"", text);
- fprintf (stream, "/>\n");
- if (!match_byte (0))
- match_byte_assert(1);
+ unsigned int format;
+ double value;
- }
- else if (match_byte (5))
- {
- dump_value_31(stream);
- char *name = get_string ();
- char *label = get_string ();
- fprintf (stream, "<variable name=\"%s\"", name);
- if (label[0])
- fprintf (stream, " label=\"%s\"", label);
- fprintf (stream, "/>\n");
- if (!match_byte(1) && !match_byte(2))
- match_byte_assert(3);
+ dump_value_modifier(stream);
+ format = get_u32 ();
+ value = get_double ();
+ fprintf (stream, "<number value=\"%.*g\" format=\"%s%d.%d\"/>\n",
+ DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
}
else if (match_byte (2))
{
char *var, *vallab;
double value;
- dump_value_31 (stream);
+ dump_value_modifier (stream);
format = get_u32 ();
value = get_double ();
var = get_string ();
if (var[0])
fprintf (stream, " variable=\"%s\"", var);
if (vallab[0])
- fprintf (stream, " label=\"%s\"/>\n", vallab);
+ fprintf (stream, " label=\"%s\"", vallab);
fprintf (stream, "/>\n");
if (!match_byte (1) && !match_byte(2))
match_byte_assert (3);
}
+ else if (match_byte (3))
+ {
+ char *text = get_string();
+ dump_value_modifier(stream);
+ char *identifier = get_string();
+ char *text_eng = get_string();
+ fprintf (stream, "<string c=\"%s\"", text_eng);
+ if (identifier[0])
+ fprintf (stream, " identifier=\"%s\"", identifier);
+ if (strcmp(text_eng, text))
+ fprintf (stream, " local=\"%s\"", text);
+ fprintf (stream, "/>\n");
+ if (!match_byte (0))
+ match_byte_assert(1);
+ }
else if (match_byte (4))
{
unsigned int format;
char *var, *vallab, *value;
- match_byte_assert (0x58);
+ dump_value_modifier(stream);
format = get_u32 ();
vallab = get_string ();
var = get_string ();
fprintf (stream, " label=\"%s\"/>\n", vallab);
fprintf (stream, "/>\n");
}
- else if (match_byte (1))
+ else if (match_byte (5))
{
- unsigned int format;
- double value;
-
- dump_value_31(stream);
- format = get_u32 ();
- value = get_double ();
- fprintf (stream, "<number value=\"%.*g\" format=\"%s%d.%d\"/>\n",
- DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
+ dump_value_modifier(stream);
+ char *name = get_string ();
+ char *label = get_string ();
+ fprintf (stream, "<variable name=\"%s\"", name);
+ if (label[0])
+ fprintf (stream, " label=\"%s\"", label);
+ fprintf (stream, "/>\n");
+ if (!match_byte(1) && !match_byte(2))
+ match_byte_assert(3);
}
else
{
- dump_value_31(stream);
+ dump_value_modifier(stream);
char *base = get_string();
int x = get_u32();
}
static void
-dump_category(int level, int *indexes, int *n_indexes, int max_indexes)
+dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
{
for (int i = 0; i <= level; i++)
- fprintf (stdout, " ");
+ fprintf (stream, " ");
printf ("<category>\n");
- dump_value (stdout, level + 1);
- match_u32(1);
+ dump_value (stream, level + 1);
- if (match_u32 (1))
- match_byte (0);
- else if (match_byte (1))
- {
- match_byte (0);
- if (!match_u32 (2))
- match_u32_assert (1);
- match_byte (0);
- }
- else if (!match_u32(2))
- match_u32_assert (0);
+ int merge = data[pos];
+ if (!match_byte(0))
+ match_byte_assert (1);
+
+ match_byte_assert (0);
+
+ int unindexed = data[pos];
+ if (!match_byte(0))
+ match_byte_assert (1);
+
+ int x = get_u32 ();
+ pos -= 4;
+ if (!match_u32 (0))
+ match_u32_assert (2);
int indx = get_u32();
int n_categories = get_u32();
- if (indx != -1)
+ if (indx == -1)
{
+ if (merge)
+ {
+ for (int i = 0; i <= level + 1; i++)
+ fprintf (stream, " ");
+ fprintf (stream, "<merge/>\n");
+ }
+ }
+ else
+ {
+ if (merge)
+ {
+ fprintf(stderr, "index not -1 but merged\n");
+ exit(1);
+ }
+ if (x != 2)
+ {
+ fprintf(stderr, "index not -1 but x != 2\n");
+ exit(1);
+ }
if (n_categories != 0)
{
fprintf(stderr, "index not -1 but subcategories\n");
}
indexes[(*n_indexes)++] = indx;
}
+
+ int expected_unindexed = indx == -1;
+ if (unindexed != expected_unindexed)
+ {
+ fprintf(stderr, "unindexed (%d) mismatch with indx (%d)\n",
+ unindexed, indx);
+ exit(1);
+ }
+
if (n_categories == 0)
{
for (int i = 0; i <= level + 1; i++)
- fprintf (stdout, " ");
- fprintf (stdout, "<category-index>%d</category-index>\n", indx);
+ fprintf (stream, " ");
+ fprintf (stream, "<category-index>%d</category-index>\n", indx);
}
for (int i = 0; i < n_categories; i++)
- dump_category (level + 1, indexes, n_indexes, max_indexes);
+ dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
for (int i = 0; i <= level; i++)
- fprintf (stdout, " ");
+ fprintf (stream, " ");
printf ("</category>\n");
}
-static void
+static int
dump_dim(int indx)
{
int n_categories;
printf ("<dimension index=\"%d\">\n", indx);
dump_value (stdout, 0);
- /* This byte is usually 0x02 but many other values have been spotted. */
+ /* This byte is usually 0 but many other values have been spotted. */
pos++;
if (!match_byte(0) && !match_byte(1))
int indexes[2048];
int n_indexes = 0;
for (int i = 0; i < n_categories; i++)
- dump_category (0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
+ dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
check_permutation(indexes, n_indexes, "categories");
fprintf (stdout, "</dimension>\n");
+ return n_indexes;
}
int n_dims;
+static int dim_n_cats[64];
+#define MAX_DIMS (sizeof dim_n_cats / sizeof *dim_n_cats)
+
static void
dump_dims(void)
{
n_dims = get_u32();
+ assert(n_dims < MAX_DIMS);
for (int i = 0; i < n_dims; i++)
- dump_dim (i);
+ dim_n_cats[i] = dump_dim (i);
}
static void
dump_data(void)
{
/* The first three numbers add to the number of dimensions. */
- int t = get_u32();
- t += get_u32();
- match_u32_assert(n_dims - t);
+ int l = get_u32();
+ int r = get_u32();
+ int c = n_dims - l - r;
+ match_u32_assert(c);
/* The next n_dims numbers are a permutation of the dimension numbers. */
int a[n_dims];
for (int i = 0; i < n_dims; i++)
- a[i] = get_u32();
+ {
+ int dim = get_u32();
+ a[i] = dim;
+
+ const char *name = i < l ? "layer" : i < l + r ? "row" : "column";
+ printf ("<%s dimension=\"%d\"/>\n", name, dim);
+ }
check_permutation(a, n_dims, "dimensions");
int x = get_u32();
printf ("<data>\n");
for (int i = 0; i < x; i++)
{
- printf (" <datum index=\"%d\">\n", get_u32());
+ unsigned int indx = get_u32();
+ printf (" <datum index=\"%d\" coords=", indx);
+
+ int coords[MAX_DIMS];
+ for (int i = n_dims; i-- > 0; )
+ {
+ coords[i] = indx % dim_n_cats[i];
+ indx /= dim_n_cats[i];
+ }
+ for (int i = 0; i < n_dims; i++)
+ printf("%c%d", i ? ',' : '"', coords[i]);
+
+ printf ("\">\n");
match_u32_assert(0);
if (version == 1)
match_byte(0);
static void
dump_title(void)
{
- pos = 0x27;
printf ("<title-local>\n");
dump_value(stdout, 0);
match_byte(1);
else
match_byte_assert(0x58);
-
int n_footnotes = get_u32();
for (int i = 0; i < n_footnotes; i++)
{
printf ("<footnote index=\"%d\">\n", i);
dump_value(stdout, 0);
+ /* Custom footnote marker string. */
if (match_byte (0x31))
- {
- /* Custom footnote marker string. */
- match_byte_assert(3);
- get_string();
- match_byte_assert(0x58);
- match_u32_assert(0);
- get_string();
- }
+ dump_value(stdout, 0);
else
match_byte_assert (0x58);
- printf("(%d)\n", get_u32());
+ get_u32 ();
printf ("</footnote>\n");
}
}
match_byte_assert(0);
if (!match_byte(0x40) && !match_byte(0x20) && !match_byte(0x80) && !match_byte(0x10) && !match_byte(0x70))
match_byte_assert(0x50);
- if (!match_byte(0x41))
- match_byte_assert(0x51);
+ match_byte_assert(0x41);
if (!match_u32(0) && !match_u32(1))
match_u32_assert(2);
match_byte_assert(0);
/* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
if (!match_u32(0) && !match_u32(2))
- match_u32_assert(0xfaad);
+ {
+ if (i == 7)
+ match_u32_assert(0xfaad);
+ else
+ match_u32_assert(0);
+ }
if (!match_u32(0) && !match_u32(1) && !match_u32(2))
match_u32_assert(3);
if (version > 1)
{
- /* These seem unlikely to be correct too. */
if (i != 3)
{
if (!match_u32(8))
match_u32_assert(5);
if (!match_u32(10) && !match_u32(11) && !match_u32(5))
match_u32_assert(9);
- if (!match_u32(0))
- match_u32_assert(1);
+ if (!match_u32(0) && !match_u32(1))
+ match_u32_assert(2);
}
else
{
match_u32_assert(18);
pos += 18;
- if (match_u32(117))
- pos += 117;
- else if (match_u32(142))
- pos += 142;
- else if (match_u32(143))
- pos += 143;
- else if (match_u32(150))
- pos += 150;
- else
+ int x3 = get_u32();
+ if (version == 3)
{
- match_u32_assert(16);
- pos += 16;
+ assert(x3 >= 117);
+ int len = data[pos + 0x34];
+ if (len)
+ printf("<tablelook>%.*s</tablelook>\n", len, &data[pos + 0x35]);
}
+ pos += x3;
int count = get_u32();
pos += 4 * count;
- printf ("<encoding>%s</encoding>\n", get_string ());
+ const char *locale = get_string();
+ printf ("<locale>%s</locale>\n", locale);
if (!match_u32(0))
match_u32_assert(UINT32_MAX);
match_byte_assert(1);
if (version > 1)
{
- if (!match_byte(0x97) && !match_byte(0x98) && !match_byte(0x99))
- match_byte_assert(0x9a);
+ if (!match_byte(0x97) && !match_byte(0x98)
+ && !match_byte(0x99) && !match_byte(0x9a))
+ match_byte_assert(0x9b);
match_byte_assert(7);
match_byte_assert(0);
match_byte_assert(0);
}
else
match_u32_assert(UINT32_MAX);
+
+ int decimal = data[pos];
+ int grouping = data[pos + 1];
if (match_byte('.'))
{
- if (!match_byte(','))
+ if (!match_byte(',') && !match_byte('\''))
match_byte_assert(' ');
}
else
if (!match_byte('.') && !match_byte(' '))
match_byte_assert(0);
}
+ printf("<format decimal=\"%c\" grouping=\"", decimal);
+ if (grouping)
+ putchar(grouping);
+ printf("\"/>\n");
if (match_u32(5))
{
for (int i = 0; i < 5; i++)
- get_string();
+ printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
+ }
+ else
+ match_u32_assert(0);
+
+ /* The last chunk is an outer envelope that contains two inner envelopes.
+ The second inner envelope has some interesting data like the encoding and
+ the locale. */
+ if (version == 3)
+ {
+ int outer_end = get_end();
+
+ /* First inner envelope: byte*33 int[n] int*[n]. */
+ pos = get_end();
+
+ /* Second inner envelope. */
+ assert(get_end() == outer_end);
+
+ match_byte_assert(1);
+ match_byte_assert(0);
+ if (!match_byte(3) && !match_byte(4))
+ match_byte_assert(5);
+ match_byte_assert(0);
+ match_byte_assert(0);
+ match_byte_assert(0);
+
+ printf("<command>%s</command>\n", get_string());
+ printf("<subcommand>%s</subcommand>\n", get_string());
+ printf("<language>%s</language>\n", get_string());
+ printf("<charset>%s</charset>\n", get_string());
+ printf("<locale>%s</locale>\n", get_string());
+
+ if (!match_byte(0))
+ match_byte_assert(1);
+ match_byte_assert(0);
+ if (!match_byte(0))
+ match_byte_assert(1);
+ if (!match_byte(0))
+ match_byte_assert(1);
+
+ if (!match_byte(0x97) && !match_byte(0x98)
+ && !match_byte(0x99) && !match_byte(0x9a))
+ match_byte_assert(0x9b);
+ match_byte_assert(7);
+ match_byte_assert(0);
+ match_byte_assert(0);
+
+ if (match_byte('.'))
+ {
+ if (!match_byte(',') && !match_byte('\''))
+ match_byte_assert(' ');
+ }
+ else
+ {
+ match_byte_assert(',');
+ if (!match_byte('.') && !match_byte(' '))
+ match_byte_assert(0);
+ }
+
+ pos += 8;
+ match_byte_assert(1);
+
+ if (outer_end - pos > 6)
+ {
+ /* There might be a pair of strings representing a dataset and
+ datafile name, or there might be a set of custom currency strings.
+ The custom currency strings start with a pair of integers, so we
+ can distinguish these from a string by checking for a null byte; a
+ small 32-bit integer will always contain a null and a text string
+ never will. */
+ int save_pos = pos;
+ int len = get_u32();
+ bool has_dataset = !memchr(&data[pos], '\0', len);
+ pos = save_pos;
+
+ if (has_dataset)
+ {
+ printf("<dataset>%s</dataset>\n", get_string());
+ printf("<datafile>%s</datafile>\n", get_string());
+
+ match_u32_assert(0);
+ get_u32();
+ match_u32_assert(0);
+ }
+ }
+
+ if (match_u32(5))
+ {
+ for (int i = 0; i < 5; i++)
+ printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
+ }
+ else
+ match_u32_assert(0);
+
+ match_byte_assert(0x2e);
+ if (!match_byte(0))
+ match_byte_assert(1);
+
+ if (pos < outer_end)
+ {
+ match_u32_assert(2000000);
+ match_u32_assert(0);
+ }
+ assert(pos == outer_end);
+
+ pos = outer_end;
}
else
match_u32_assert(0);
- int skip = get_u32();
- pos += skip;
}
int
}
else if (!strcmp(argv[1], "title"))
{
+ pos = 0x27;
dump_title();
exit(0);
}
/* This might be a version number of some kind, because value 1 seems
to only appear in an SPV file that also required its own weird
- special cases in dump_value_31(). */
+ special cases in dump_value_modifier(). */
version = get_u32();
pos -= 4;
if (!match_u32(1))
match_u32_assert(3);
match_byte_assert(1);
+ if (!match_byte(0))
+ match_byte_assert(1);
+
+ /* Offset 8. */
+ match_byte_assert(0);
+ match_byte_assert(0);
+ if (!match_byte(0))
+ match_byte_assert(1);
+
+ /* Offset 11. */
+ pos++;
+ match_byte_assert(0);
+ match_byte_assert(0);
+ match_byte_assert(0);
+
+ /* Offset 15. */
+ pos++;
if (!match_byte(0))
match_byte_assert(1);
match_byte_assert(0);
match_byte_assert(0);
+
+ /* Offset 19. */
+ pos++;
if (!match_byte(0))
match_byte_assert(1);
+ match_byte_assert(0);
+ match_byte_assert(0);
+
+ /* Offset 23. */
pos++;
+ if (!match_byte(0))
+ match_byte_assert(1);
match_byte_assert(0);
match_byte_assert(0);
+
+ /* Offset 27. */
+ pos++;
+ pos++;
match_byte_assert(0);
+ match_byte_assert(0);
+
+ /* Offset 31.
+
+ This is the tableId, e.g. -4154297861994971133 would be 0xdca00003.
+ We don't have enough context to validate it. */
+ pos += 4;
+
+ /* Offset 35. */
+ pos += 4;
+
dump_title ();
dump_fonts();
dump_dims ();
projects / pspp / blobdiff