X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=dump.c;h=61ee61ac0c72639825ca6e258a4386606b40020e;hb=91e1d822895e84a15e17b15762c6d33629f83342;hp=9b90fe651d62db98b82be547cb8618e99abeb384;hpb=6f432d8f62b7298cc4d6897a42e471774bd55938;p=pspp diff --git a/dump.c b/dump.c index 9b90fe651d..61ee61ac0c 100644 --- a/dump.c +++ b/dump.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -6,6 +7,7 @@ #include #include #include +#include "u8-mbtouc.h" static uint8_t *data; static size_t n; @@ -55,6 +57,15 @@ get_u32(void) return x; } +static unsigned int +get_u16(void) +{ + uint16_t x; + memcpy(&x, &data[pos], 2); + pos += 2; + return x; +} + static double get_double(void) { @@ -177,6 +188,22 @@ dump_raw(FILE *stream, int start, int end) } +static bool __attribute__((unused)) +all_utf8(const char *p_) +{ + const uint8_t *p = (const uint8_t *) p_; + size_t len = strlen ((char *) p); + for (size_t ofs = 0, mblen; ofs < len; ofs += mblen) + { + ucs4_t uc; + + mblen = u8_mbtouc (&uc, p + ofs, len - ofs); + if ((uc < 32 && uc != '\n') || uc == 127 || uc == 0xfffd) + return false; + } + return true; +} + static char * get_string(const char *where) { @@ -200,18 +227,45 @@ get_string(const char *where) } #define get_string() get_string(WHERE) +static int +get_end(void) +{ + int len = get_u32(); + return pos + len; +} + +static void __attribute__((unused)) +hex_dump(int ofs, int n) +{ + for (int i = 0; i < n; i++) + { + int c = data[ofs + i]; +#if 1 + if (i && !(i % 16)) + fprintf(stderr, "-"); + else + fprintf(stderr, " "); +#endif + fprintf(stderr, "%02x", c); + //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.'); + } + fprintf(stderr, "\n"); +} + static char * -dump_nested_string(void) +dump_counted_string(void) { char *s = NULL; - - match_byte_assert (0); - match_byte_assert (0); - int outer_end = pos + get_u32(); - int inner_end = pos + get_u32(); + int inner_end = get_end(); if (pos != inner_end) { - match_u32_assert(0); + if (match_u32(5)) + { + match_u32_assert(0); + match_byte_assert(0x58); + } + else + match_u32_assert(0); if (match_byte(0x31)) s = get_string(); else @@ -222,7 +276,42 @@ dump_nested_string(void) exit(1); } } - match_byte_assert(0x58); + return s; +} + +static void +dump_style(FILE *stream) +{ + match_byte(1); + match_byte(0); + match_byte(0); + match_byte(0); + match_byte_assert(1); + char *fg = get_string(); /* foreground */ + char *bg = get_string(); /* background */ + char *font = get_string(); /* font */ + int size = data[pos]; + if (!match_byte(14)) + match_byte_assert(12); /* size? */ + fprintf(stream, " fgcolor=\"%s\" bgcolor=\"%s\" font=\"%s\" size=\"%d\"", + fg, bg, font, size); +} + +static char * +dump_nested_string(FILE *stream) +{ + char *s = NULL; + + match_byte_assert (0); + match_byte_assert (0); + int outer_end = get_end(); + s = dump_counted_string(); + if (s) + fprintf(stream, " \"%s\"", s); + if (match_byte(0x31)) + dump_style(stream); + else + match_byte_assert(0x58); match_byte_assert(0x58); if (pos != outer_end) { @@ -234,16 +323,23 @@ dump_nested_string(void) } static void -dump_value_31(FILE *stream) +dump_value_modifier(FILE *stream) { if (match_byte (0x31)) { if (match_u32 (0)) { + fprintf(stream, "\n"); return; } - int outer_end = pos + get_u32(); - int inner_end = pos + get_u32(); - if (pos != inner_end) - { - match_u32_assert(0); - if (match_byte(0x31)) - { - /* Appears to be a template string, e.g. '^1 cells (^2) expf < 5. Min exp = ^3...'. - Probably doesn't actually appear in output because many examples look unpolished, - e.g. 'partial list cases value ^1 shown upper...' */ - get_string(); - } - else - match_byte_assert(0x58); - if (pos != inner_end) - { - fprintf(stderr, "inner end discrepancy\n"); - exit(1); - } - } + int outer_end = get_end(); + + /* This counted-string appears to be a template string, + e.g. "Design\: [:^1:]1 Within Subjects Design\: [:^1:]2". */ + char *template = dump_counted_string(); + if (template) + fprintf(stream, " template=\"%s\"", template); if (match_byte(0x31)) - { - /* Only one example in the corpus. */ - match_byte(1); - match_byte(0); - match_byte(0); - match_byte(0); - match_byte_assert(1); - get_string(); /* foreground */ - get_string(); /* background */ - get_string(); /* font */ - if (!match_byte(14)) - match_byte_assert(12); /* size? */ - } + dump_style(stream); else match_byte_assert(0x58); if (match_byte(0x31)) @@ -326,32 +398,23 @@ dump_value_31(FILE *stream) fprintf(stderr, "outer end discrepancy\n"); exit(1); } - } - else if (match_u32 (1)) - { - fprintf(stream, "(footnote %d) ", get_u32()); - dump_nested_string(); - } - else if (match_u32 (2)) - { - fprintf(stream, "(special 2)"); - if (!match_byte(0)) - match_byte_assert(2); - match_byte_assert(0); - if (!match_u32 (2) && !match_u32(1)) - match_u32_assert(3); - dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */ + fprintf(stream, "/>\n"); } else { - match_u32_assert(3); - fprintf(stream, "(special 3)"); - match_byte_assert(0); + int count = get_u32(); + fprintf(stream, "\n"); } } else @@ -417,32 +480,16 @@ dump_value(FILE *stream, int level) for (int i = 0; i <= level; i++) fprintf (stream, " "); - if (match_byte (3)) + if (match_byte (1)) { - char *text = get_string(); - dump_value_31(stream); - char *identifier = get_string(); - char *text_eng = get_string(); - fprintf (stream, "\n"); - if (!match_byte (0)) - match_byte_assert(1); - } - else if (match_byte (5)) - { - dump_value_31(stream); - char *name = get_string (); - char *label = get_string (); - fprintf (stream, "\n"); - if (!match_byte(1) && !match_byte(2)) - match_byte_assert(3); + unsigned int format; + double value; + + dump_value_modifier(stream); + format = get_u32 (); + value = get_double (); + fprintf (stream, "\n", + DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff); } else if (match_byte (2)) { @@ -450,7 +497,7 @@ dump_value(FILE *stream, int level) char *var, *vallab; double value; - dump_value_31 (stream); + dump_value_modifier (stream); format = get_u32 (); value = get_double (); var = get_string (); @@ -460,17 +507,32 @@ dump_value(FILE *stream, int level) if (var[0]) fprintf (stream, " variable=\"%s\"", var); if (vallab[0]) - fprintf (stream, " label=\"%s\"/>\n", vallab); + fprintf (stream, " label=\"%s\"", vallab); fprintf (stream, "/>\n"); if (!match_byte (1) && !match_byte(2)) match_byte_assert (3); } + else if (match_byte (3)) + { + char *text = get_string(); + dump_value_modifier(stream); + char *identifier = get_string(); + char *text_eng = get_string(); + fprintf (stream, "\n"); + if (!match_byte (0)) + match_byte_assert(1); + } else if (match_byte (4)) { unsigned int format; char *var, *vallab, *value; - match_byte_assert (0x58); + dump_value_modifier(stream); format = get_u32 (); vallab = get_string (); var = get_string (); @@ -485,20 +547,21 @@ dump_value(FILE *stream, int level) fprintf (stream, " label=\"%s\"/>\n", vallab); fprintf (stream, "/>\n"); } - else if (match_byte (1)) + else if (match_byte (5)) { - unsigned int format; - double value; - - dump_value_31(stream); - format = get_u32 (); - value = get_double (); - fprintf (stream, "\n", - DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff); + dump_value_modifier(stream); + char *name = get_string (); + char *label = get_string (); + fprintf (stream, "\n"); + if (!match_byte(1) && !match_byte(2)) + match_byte_assert(3); } else { - dump_value_31(stream); + dump_value_modifier(stream); char *base = get_string(); int x = get_u32(); @@ -551,59 +614,87 @@ check_permutation(int *a, int n, const char *name) } static void -dump_category(int level, int *indexes, int *n_indexes, int max_indexes) +dump_category(FILE *stream, int level, int **indexes, int *allocated_indexes, + int *n_indexes) { for (int i = 0; i <= level; i++) - fprintf (stdout, " "); + fprintf (stream, " "); printf ("\n"); - dump_value (stdout, level + 1); - match_byte(1); - match_byte(0); - match_byte(0); - match_byte(0); + dump_value (stream, level + 1); - if (match_u32 (1)) - match_byte (0); - else if (match_byte (1)) - { - match_byte (0); - if (!match_u32 (2)) - match_u32_assert (1); - match_byte (0); - } - else if (!match_u32(2)) - match_u32_assert (0); + int merge = data[pos]; + if (!match_byte(0)) + match_byte_assert (1); + + match_byte_assert (0); + + int unindexed = data[pos]; + if (!match_byte(0)) + match_byte_assert (1); + + int x = get_u32 (); + pos -= 4; + if (!match_u32 (0)) + match_u32_assert (2); int indx = get_u32(); int n_categories = get_u32(); - if (indx != -1) + if (indx == -1) + { + if (merge) + { + for (int i = 0; i <= level + 1; i++) + fprintf (stream, " "); + fprintf (stream, "\n"); + } + } + else { + if (merge) + { + fprintf(stderr, "index not -1 but merged\n"); + exit(1); + } + if (x != 2) + { + fprintf(stderr, "index not -1 but x != 2\n"); + exit(1); + } if (n_categories != 0) { fprintf(stderr, "index not -1 but subcategories\n"); exit(1); } - if (*n_indexes >= max_indexes) + if (*n_indexes >= *allocated_indexes) { - fprintf(stderr, "too many categories (increase max_indexes)\n"); - exit(1); + *allocated_indexes = *allocated_indexes ? 2 * *allocated_indexes : 16; + *indexes = realloc(*indexes, *allocated_indexes * sizeof **indexes); } - indexes[(*n_indexes)++] = indx; + (*indexes)[(*n_indexes)++] = indx; } + + int expected_unindexed = indx == -1; + if (unindexed != expected_unindexed) + { + fprintf(stderr, "unindexed (%d) mismatch with indx (%d)\n", + unindexed, indx); + exit(1); + } + if (n_categories == 0) { for (int i = 0; i <= level + 1; i++) - fprintf (stdout, " "); - fprintf (stdout, "%d\n", indx); + fprintf (stream, " "); + fprintf (stream, "%d\n", indx); } for (int i = 0; i < n_categories; i++) - dump_category (level + 1, indexes, n_indexes, max_indexes); + dump_category (stream, level + 1, indexes, allocated_indexes, n_indexes); for (int i = 0; i <= level; i++) - fprintf (stdout, " "); + fprintf (stream, " "); printf ("\n"); } -static void +static int dump_dim(int indx) { int n_categories; @@ -611,7 +702,7 @@ dump_dim(int indx) printf ("\n", indx); dump_value (stdout, 0); - /* This byte is usually 0x02 but many other values have been spotted. */ + /* This byte is usually 0 but many other values have been spotted. */ pos++; if (!match_byte(0) && !match_byte(1)) @@ -627,43 +718,68 @@ dump_dim(int indx) match_u32_assert(indx); n_categories = get_u32(); - int indexes[2048]; + int *indexes = NULL; int n_indexes = 0; + int allocated_indexes = 0; for (int i = 0; i < n_categories; i++) - dump_category (0, indexes, &n_indexes, sizeof indexes / sizeof *indexes); + dump_category (stdout, 0, &indexes, &allocated_indexes, &n_indexes); check_permutation(indexes, n_indexes, "categories"); fprintf (stdout, "\n"); + return n_indexes; } int n_dims; +static int dim_n_cats[64]; +#define MAX_DIMS (sizeof dim_n_cats / sizeof *dim_n_cats) + static void dump_dims(void) { n_dims = get_u32(); + assert(n_dims < MAX_DIMS); for (int i = 0; i < n_dims; i++) - dump_dim (i); + dim_n_cats[i] = dump_dim (i); } static void dump_data(void) { /* The first three numbers add to the number of dimensions. */ - int t = get_u32(); - t += get_u32(); - match_u32_assert(n_dims - t); + int l = get_u32(); + int r = get_u32(); + int c = n_dims - l - r; + match_u32_assert(c); /* The next n_dims numbers are a permutation of the dimension numbers. */ int a[n_dims]; for (int i = 0; i < n_dims; i++) - a[i] = get_u32(); + { + int dim = get_u32(); + a[i] = dim; + + const char *name = i < l ? "layer" : i < l + r ? "row" : "column"; + printf ("<%s dimension=\"%d\"/>\n", name, dim); + } check_permutation(a, n_dims, "dimensions"); int x = get_u32(); printf ("\n"); for (int i = 0; i < x; i++) { - printf (" \n", get_u32()); + unsigned int indx = get_u32(); + printf (" \n"); match_u32_assert(0); if (version == 1) match_byte(0); @@ -676,7 +792,6 @@ dump_data(void) static void dump_title(void) { - pos = 0x27; printf ("\n"); dump_value(stdout, 0); match_byte(1); @@ -694,8 +809,14 @@ dump_title(void) match_byte(1); printf ("\n"); - match_byte(0); - match_byte_assert(0x58); + if (match_byte(0x31)) + { + printf ("\n"); + dump_value(stdout, 0); + printf ("\n"); + } + else + match_byte_assert(0x58); if (match_byte(0x31)) { printf ("\n"); @@ -705,24 +826,17 @@ dump_title(void) else match_byte_assert(0x58); - int n_footnotes = get_u32(); for (int i = 0; i < n_footnotes; i++) { printf ("\n", i); dump_value(stdout, 0); + /* Custom footnote marker string. */ if (match_byte (0x31)) - { - /* Custom footnote marker string. */ - match_byte_assert(3); - get_string(); - match_byte_assert(0x58); - match_u32_assert(0); - get_string(); - } + dump_value(stdout, 0); else match_byte_assert (0x58); - printf("(%d)\n", get_u32()); + get_u32 (); printf ("\n"); } } @@ -741,35 +855,36 @@ dump_fonts(void) match_byte_assert(0); if (!match_byte(0x40) && !match_byte(0x20) && !match_byte(0x80) && !match_byte(0x10) && !match_byte(0x70)) match_byte_assert(0x50); - if (!match_byte(0x41)) - match_byte_assert(0x51); + match_byte_assert(0x41); if (!match_u32(0) && !match_u32(1)) match_u32_assert(2); match_byte_assert(0); /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */ - if (!match_u32(0) && !match_u32(2)) - match_u32_assert(0xfaad); + if (!match_u32(0) && !match_u32(2) && !match_u32(4)) + { + if (i == 7) + match_u32_assert(0xfaad); + else + match_u32_assert(0); + } if (!match_u32(0) && !match_u32(1) && !match_u32(2)) match_u32_assert(3); printf (" fgcolor=\"%s\"", get_string()); printf (" bgcolor=\"%s\"", get_string()); + if (!match_byte(0)) + match_byte_assert(1); match_u32_assert(0); - match_u32_assert(0); - match_byte_assert(0); + char *othercolor = get_string(); + if (othercolor[0]) + printf(" othercolor=\"%s\"", othercolor); if (version > 1) { - /* These seem unlikely to be correct too. */ if (i != 3) { - if (!match_u32(8)) - match_u32_assert(5); - if (!match_u32(10) && !match_u32(11) && !match_u32(5)) - match_u32_assert(9); - if (!match_u32(0)) - match_u32_assert(1); + pos += 12; } else { @@ -790,30 +905,27 @@ dump_fonts(void) match_u32_assert(240); pos += 240; - match_u32_assert(18); - pos += 18; - - if (match_u32(117)) - pos += 117; - else if (match_u32(142)) - pos += 142; - else if (match_u32(143)) - pos += 143; - else if (match_u32(150)) - pos += 150; - else + int skip = get_u32(); + assert(skip == 18 || skip == 25); + pos += skip; + + int x3 = get_u32(); + if (version == 3) { - match_u32_assert(16); - pos += 16; + assert(x3 >= 117); + int len = data[pos + 0x34]; + if (len) + printf("%.*s\n", len, &data[pos + 0x35]); } + pos += x3; int count = get_u32(); pos += 4 * count; - printf ("%s\n", get_string ()); + const char *locale = get_string(); + printf ("%s\n", locale); - if (!match_u32(0)) - match_u32_assert(UINT32_MAX); + get_u32(); /* Seen: 0, UINT32_MAX, 2, 3, 4, 5, 6, 8, 9, 21, 24. */ if (!match_byte(0)) match_byte_assert(1); match_byte_assert(0); @@ -821,14 +933,20 @@ dump_fonts(void) match_byte_assert(1); if (version > 1) { - if (!match_byte(0x97) && !match_byte(0x98) && !match_byte(0x99)) - match_byte_assert(0x9a); + if (!match_byte(0x97) && !match_byte(0x98) + && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b)) + match_byte_assert(0x9c); match_byte_assert(7); match_byte_assert(0); match_byte_assert(0); } else - match_u32_assert(UINT32_MAX); + { + printf("%x\n", get_u32()); + } + + int decimal = data[pos]; + int grouping = data[pos + 1]; if (match_byte('.')) { if (!match_byte(',') && !match_byte('\'')) @@ -837,18 +955,127 @@ dump_fonts(void) else { match_byte_assert(','); - if (!match_byte('.') && !match_byte(' ')) + if (!match_byte('.') && !match_byte(' ') && !match_byte(',')) match_byte_assert(0); } + printf("\n"); if (match_u32(5)) { for (int i = 0; i < 5; i++) - get_string(); + printf("%s\n", 'A' + i, get_string(), 'A' + i); } else match_u32_assert(0); - int skip = get_u32(); - pos += skip; + + /* The last chunk is an outer envelope that contains two inner envelopes. + The second inner envelope has some interesting data like the encoding and + the locale. */ + if (version == 3) + { + int outer_end = get_end(); + + /* First inner envelope: byte*33 int[n] int*[n]. */ + pos = get_end(); + + /* Second inner envelope. */ + assert(get_end() == outer_end); + + match_byte_assert(1); + match_byte_assert(0); + if (!match_byte(3) && !match_byte(4)) + match_byte_assert(5); + match_byte_assert(0); + match_byte_assert(0); + match_byte_assert(0); + + printf("%s\n", get_string()); + printf("%s\n", get_string()); + printf("%s\n", get_string()); + printf("%s\n", get_string()); + printf("%s\n", get_string()); + + if (!match_byte(0)) + match_byte_assert(1); + match_byte_assert(0); + if (!match_byte(0)) + match_byte_assert(1); + if (!match_byte(0)) + match_byte_assert(1); + + if (!match_byte(0x97) && !match_byte(0x98) + && !match_byte(0x99) && !match_byte(0x9a) && !match_byte(0x9b)) + match_byte_assert(0x9c); + match_byte_assert(7); + match_byte_assert(0); + match_byte_assert(0); + + if (match_byte('.')) + { + if (!match_byte(',') && !match_byte('\'')) + match_byte_assert(' '); + } + else + { + match_byte_assert(','); + if (!match_byte('.') && !match_byte(' ') && !match_byte(',')) + match_byte_assert(0); + } + + pos += 8; + match_byte_assert(1); + + if (outer_end - pos > 6) + { + /* There might be a pair of strings representing a dataset and + datafile name, or there might be a set of custom currency strings. + The custom currency strings start with a pair of integers, so we + can distinguish these from a string by checking for a null byte; a + small 32-bit integer will always contain a null and a text string + never will. */ + int save_pos = pos; + int len = get_u32(); + bool has_dataset = !memchr(&data[pos], '\0', len); + pos = save_pos; + + if (has_dataset) + { + printf("%s\n", get_string()); + printf("%s\n", get_string()); + + match_u32_assert(0); + get_u32(); + match_u32_assert(0); + } + } + + if (match_u32(5)) + { + for (int i = 0; i < 5; i++) + printf("%s\n", 'A' + i, get_string(), 'A' + i); + } + else + match_u32_assert(0); + + match_byte_assert(0x2e); + if (!match_byte(0)) + match_byte_assert(1); + + if (pos < outer_end) + { + printf("%d\n", get_u32()); + match_u32_assert(0); + } + assert(pos == outer_end); + + pos = outer_end; + } + else + { + pos = get_end(); + } } int @@ -898,6 +1125,7 @@ main(int argc, char *argv[]) } else if (!strcmp(argv[1], "title")) { + pos = 0x27; dump_title(); exit(0); } @@ -929,7 +1157,7 @@ main(int argc, char *argv[]) /* This might be a version number of some kind, because value 1 seems to only appear in an SPV file that also required its own weird - special cases in dump_value_31(). */ + special cases in dump_value_modifier(). */ version = get_u32(); pos -= 4; if (!match_u32(1)) @@ -938,14 +1166,56 @@ main(int argc, char *argv[]) match_byte_assert(1); if (!match_byte(0)) match_byte_assert(1); + + /* Offset 8. */ + match_byte_assert(0); + if (!match_byte(0)) + match_byte_assert(1); + if (!match_byte(0)) + match_byte_assert(1); + + /* Offset 11. */ + pos++; match_byte_assert(0); match_byte_assert(0); + match_byte_assert(0); + + /* Offset 15. */ + pos++; if (!match_byte(0)) match_byte_assert(1); + match_byte_assert(0); + match_byte_assert(0); + + /* Offset 19. */ pos++; + if (!match_byte(0)) + match_byte_assert(1); + match_byte_assert(0); match_byte_assert(0); + + /* Offset 23. */ + pos++; + if (!match_byte(0)) + match_byte_assert(1); match_byte_assert(0); match_byte_assert(0); + + /* Offset 27. */ + pos++; + pos++; + pos++; + pos++; + + /* Offset 31. + + This is the tableId, e.g. -4154297861994971133 would be 0xdca00003. + We don't have enough context to validate it. */ + pos += 4; + + /* Offset 35. */ + pos += 4; + dump_title (); dump_fonts(); dump_dims ();