10 #include "u8-mbtouc.h"
18 all_ascii(const uint8_t *p, size_t n)
20 for (size_t i = 0; i < n; i++)
21 if (p[i] < 32 || p[i] > 126)
27 try_find(const char *target, size_t target_len)
29 const uint8_t *pos = (const uint8_t *) memmem (data, n, target, target_len);
30 return pos ? pos - data : 0;
34 find(const char *target, size_t target_len)
36 size_t pos = try_find(target, target_len);
39 fprintf (stderr, "not found\n");
48 #define STR(x) XSTR(x)
49 #define WHERE __FILE__":" STR(__LINE__)
55 memcpy(&x, &data[pos], 4);
64 memcpy(&x, &data[pos], 8);
79 match_u32_assert(uint32_t x, const char *where)
81 unsigned int y = get_u32();
84 fprintf(stderr, "%s: 0x%x: expected i%u, got i%u\n", where, pos - 4, x, y);
88 #define match_u32_assert(x) match_u32_assert(x, WHERE)
93 if (pos < n && data[pos] == b)
103 match_byte_assert(uint8_t b, const char *where)
107 fprintf(stderr, "%s: 0x%x: expected %02x, got %02x\n", where, pos, b, data[pos]);
111 #define match_byte_assert(b) match_byte_assert(b, WHERE)
114 newline(FILE *stream, int pos)
116 fprintf(stream, "\n%08x: ", pos);
120 dump_raw(FILE *stream, int start, int end)
122 for (size_t i = start; i < end; )
129 && i + 4 + data[i] + data[i + 1] * 256 <= end
130 && all_ascii(&data[i + 4], data[i] + data[i + 1] * 256))
133 fprintf(stream, "\"");
134 fwrite(&data[i + 4], 1, data[i] + data[i + 1] * 256, stream);
135 fputs("\" ", stream);
137 i += 4 + data[i] + data[i + 1] * 256;
139 else if (i + 12 <= end
146 memcpy (&d, &data[i + 4], 8);
147 fprintf (stream, "F40.%d(%.*f)", data[i], data[i], d);
151 else if (i + 12 <= end
158 memcpy (&d, &data[i + 4], 8);
159 fprintf (stream, "PCT40.%d(%.*f)", data[i], data[i], d);
163 else if (i + 4 <= end
164 && (data[i] && data[i] != 88 && data[i] != 0x41)
169 fprintf (stream, "i%d ", data[i]);
174 fprintf(stream, "%02x ", data[i]);
182 static bool __attribute__((unused))
183 all_utf8(const char *p_)
185 const uint8_t *p = (const uint8_t *) p_;
186 size_t len = strlen ((char *) p);
187 for (size_t ofs = 0, mblen; ofs < len; ofs += mblen)
191 mblen = u8_mbtouc (&uc, p + ofs, len - ofs);
192 if ((uc < 32 && uc != '\n') || uc == 127 || uc == 0xfffd)
199 get_string(const char *where)
202 /*data[pos + 1] == 0 && data[pos + 2] == 0 && data[pos + 3] == 0*/
203 /*&& all_ascii(&data[pos + 4], data[pos])*/)
205 int len = data[pos] + data[pos + 1] * 256;
206 char *s = malloc(len + 1);
208 memcpy(s, &data[pos + 4], len);
215 fprintf(stderr, "%s: 0x%x: expected string\n", where, pos);
219 #define get_string() get_string(WHERE)
222 dump_nested_string(void)
226 match_byte_assert (0);
227 match_byte_assert (0);
228 int outer_end = pos + get_u32();
229 int inner_end = pos + get_u32();
230 if (pos != inner_end)
233 if (match_byte(0x31))
236 match_byte_assert(0x58);
237 if (pos != inner_end)
239 fprintf(stderr, "inner end discrepancy\n");
243 match_byte_assert(0x58);
244 match_byte_assert(0x58);
245 if (pos != outer_end)
247 fprintf(stderr, "outer end discrepancy\n");
255 dump_optional_value(FILE *stream)
257 if (match_byte (0x31))
263 /* Only "a" observed as a sample value (although it appears 44 times in the corpus). */
267 match_u32_assert (0);
271 /* We only have one SPV file for this version (with many
278 if (!match_u32(0) && !match_u32(1) && !match_u32(2) && !match_u32(3) && !match_u32(4) && !match_u32(5) && !match_u32(6) && !match_u32(7) && !match_u32(8) && !match_u32(9))
279 match_u32_assert(10);
285 int outer_end = pos + get_u32();
286 int inner_end = pos + get_u32();
287 if (pos != inner_end)
290 if (match_byte(0x31))
292 /* Appears to be a template string, e.g. '^1 cells (^2) expf < 5. Min exp = ^3...'.
293 Probably doesn't actually appear in output because many examples look unpolished,
294 e.g. 'partial list cases value ^1 shown upper...' */
298 match_byte_assert(0x58);
299 if (pos != inner_end)
301 fprintf(stderr, "inner end discrepancy\n");
306 if (match_byte(0x31))
308 /* Only one example in the corpus. */
313 match_byte_assert(1);
314 get_string(); /* foreground */
315 get_string(); /* background */
316 get_string(); /* font */
318 match_byte_assert(12); /* size? */
321 match_byte_assert(0x58);
322 if (match_byte(0x31))
324 /* Only two SPV files have anything like this, so it's hard to
330 match_byte_assert(1);
331 match_byte_assert(0);
332 if (!match_byte(8) && !match_byte(1))
333 match_byte_assert(2);
334 match_byte_assert(0);
335 match_byte_assert(8);
336 match_byte_assert(0);
337 match_byte_assert(10);
338 match_byte_assert(0);
341 match_byte_assert(0x58);
342 if (pos != outer_end)
344 fprintf(stderr, "outer end discrepancy\n");
348 else if (match_u32 (1))
350 fprintf(stream, "(footnote %d) ", get_u32());
351 dump_nested_string();
353 else if (match_u32 (2))
355 fprintf(stream, "(special 2)");
357 match_byte_assert(2);
358 match_byte_assert(0);
359 if (!match_u32 (2) && !match_u32(1))
361 dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */
366 fprintf(stream, "(special 3)");
367 match_byte_assert(0);
368 match_byte_assert(0);
369 match_byte_assert(1);
370 match_byte_assert(0);
372 dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */
376 match_byte_assert (0x58);
380 format_to_string (int type)
386 case 2: return "AHEX";
387 case 3: return "COMMA";
388 case 4: return "DOLLAR";
389 case 5: case 40: return "F";
391 case 7: return "PIBHEX";
393 case 9: return "PIB";
394 case 10: return "PK";
395 case 11: return "RB";
396 case 12: return "RBHEX";
400 case 20: return "DATE";
401 case 21: return "TIME";
402 case 22: return "DATETIME";
403 case 23: return "ADATE";
404 case 24: return "JDATE";
405 case 25: return "DTIME";
406 case 26: return "WKDAY";
407 case 27: return "MONTH";
408 case 28: return "MOYR";
409 case 29: return "QYR";
410 case 30: return "WKYR";
411 case 31: return "PCT";
412 case 32: return "DOT";
413 case 33: return "CCA";
414 case 34: return "CCB";
415 case 35: return "CCC";
416 case 36: return "CCD";
417 case 37: return "CCE";
418 case 38: return "EDATE";
419 case 39: return "SDATE";
422 sprintf(tmp, "<%d>", type);
428 dump_value(FILE *stream, int level)
435 for (int i = 0; i <= level; i++)
436 fprintf (stream, " ");
440 char *text = get_string();
441 dump_optional_value(stream);
442 char *identifier = get_string();
443 char *text_eng = get_string();
444 fprintf (stream, "<string c=\"%s\"", text_eng);
446 fprintf (stream, " identifier=\"%s\"", identifier);
447 if (strcmp(text_eng, text))
448 fprintf (stream, " local=\"%s\"", text);
449 fprintf (stream, "/>\n");
451 match_byte_assert(1);
453 else if (match_byte (5))
455 dump_optional_value(stream);
456 char *name = get_string ();
457 char *label = get_string ();
458 fprintf (stream, "<variable name=\"%s\"", name);
460 fprintf (stream, " label=\"%s\"", label);
461 fprintf (stream, "/>\n");
462 if (!match_byte(1) && !match_byte(2))
463 match_byte_assert(3);
465 else if (match_byte (2))
471 dump_optional_value (stream);
473 value = get_double ();
475 vallab = get_string ();
476 fprintf (stream, "<numeric-datum value=\"%.*g\" format=\"%s%d.%d\"",
477 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
479 fprintf (stream, " variable=\"%s\"", var);
481 fprintf (stream, " label=\"%s\"/>\n", vallab);
482 fprintf (stream, "/>\n");
483 if (!match_byte (1) && !match_byte(2))
484 match_byte_assert (3);
486 else if (match_byte (4))
489 char *var, *vallab, *value;
491 match_byte_assert (0x58);
493 vallab = get_string ();
495 if (!match_byte(1) && !match_byte(2))
496 match_byte_assert (3);
497 value = get_string ();
498 fprintf (stream, "<string-datum value=\"%s\" format=\"%s%d.%d\"",
499 value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
501 fprintf (stream, " variable=\"%s\"", var);
503 fprintf (stream, " label=\"%s\"/>\n", vallab);
504 fprintf (stream, "/>\n");
506 else if (match_byte (1))
511 dump_optional_value(stream);
513 value = get_double ();
514 fprintf (stream, "<number value=\"%.*g\" format=\"%s%d.%d\"/>\n",
515 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
519 dump_optional_value(stream);
521 char *base = get_string();
523 fprintf (stream, "<template format=\"%s\">\n", base);
524 for (int i = 0; i < x; i++)
531 for (int j = 0; j <= level + 1; j++)
532 fprintf (stream, " ");
533 fprintf (stream, "<substitution index=\"%d\">\n", i + 1);
534 for (int j = 0; j < y; j++)
535 dump_value (stream, level + 2);
536 for (int j = 0; j <= level + 1; j++)
537 fprintf (stream, " ");
538 fprintf (stream, "</substitution>\n");
540 for (int j = 0; j <= level; j++)
541 fprintf (stream, " ");
542 fprintf (stream, "</template>\n");
547 compare_int(const void *a_, const void *b_)
551 return *a < *b ? -1 : *a > *b;
555 check_permutation(int *a, int n, const char *name)
558 memcpy(b, a, n * sizeof *a);
559 qsort(b, n, sizeof *b, compare_int);
560 for (int i = 0; i < n; i++)
563 fprintf(stderr, "bad %s permutation:", name);
564 for (int i = 0; i < n; i++)
565 fprintf(stderr, " %d", a[i]);
572 dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
574 for (int i = 0; i <= level; i++)
575 fprintf (stream, " ");
576 printf ("<category>\n");
577 dump_value (stream, level + 1);
579 int merge = data[pos];
581 match_byte_assert (1);
583 match_byte_assert (0);
585 int unindexed = data[pos];
587 match_byte_assert (1);
592 match_u32_assert (2);
594 int indx = get_u32();
595 int n_categories = get_u32();
600 for (int i = 0; i <= level + 1; i++)
601 fprintf (stream, " ");
602 fprintf (stream, "<merge/>\n");
609 fprintf(stderr, "index not -1 but merged\n");
614 fprintf(stderr, "index not -1 but x != 2\n");
617 if (n_categories != 0)
619 fprintf(stderr, "index not -1 but subcategories\n");
622 if (*n_indexes >= max_indexes)
624 fprintf(stderr, "too many categories (increase max_indexes)\n");
627 indexes[(*n_indexes)++] = indx;
630 int expected_unindexed = indx == -1;
631 if (unindexed != expected_unindexed)
633 fprintf(stderr, "unindexed (%d) mismatch with indx (%d)\n",
638 if (n_categories == 0)
640 for (int i = 0; i <= level + 1; i++)
641 fprintf (stream, " ");
642 fprintf (stream, "<category-index>%d</category-index>\n", indx);
644 for (int i = 0; i < n_categories; i++)
645 dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
646 for (int i = 0; i <= level; i++)
647 fprintf (stream, " ");
648 printf ("</category>\n");
656 printf ("<dimension index=\"%d\">\n", indx);
657 dump_value (stdout, 0);
659 /* This byte is usually 0 but many other values have been spotted. */
662 if (!match_byte(0) && !match_byte(1))
663 match_byte_assert(2);
667 match_byte_assert(1);
669 match_byte_assert(1);
670 match_byte_assert(1);
671 if (!match_u32(UINT32_MAX))
672 match_u32_assert(indx);
673 n_categories = get_u32();
677 for (int i = 0; i < n_categories; i++)
678 dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
679 check_permutation(indexes, n_indexes, "categories");
681 fprintf (stdout, "</dimension>\n");
686 static int dim_n_cats[64];
687 #define MAX_DIMS (sizeof dim_n_cats / sizeof *dim_n_cats)
693 assert(n_dims < MAX_DIMS);
694 for (int i = 0; i < n_dims; i++)
695 dim_n_cats[i] = dump_dim (i);
701 /* The first three numbers add to the number of dimensions. */
704 int c = n_dims - l - r;
707 /* The next n_dims numbers are a permutation of the dimension numbers. */
709 for (int i = 0; i < n_dims; i++)
714 const char *name = i < l ? "layer" : i < l + r ? "row" : "column";
715 printf ("<%s dimension=\"%d\"/>\n", name, dim);
717 check_permutation(a, n_dims, "dimensions");
721 for (int i = 0; i < x; i++)
723 unsigned int indx = get_u32();
724 printf (" <datum index=\"%d\" coords=", indx);
726 int coords[MAX_DIMS];
727 for (int i = n_dims; i-- > 0; )
729 coords[i] = indx % dim_n_cats[i];
730 indx /= dim_n_cats[i];
732 for (int i = 0; i < n_dims; i++)
733 printf("%c%d", i ? ',' : '"', coords[i]);
739 dump_value(stdout, 1);
740 fprintf (stdout, " </datum>\n");
742 printf ("</data>\n");
748 printf ("<title-local>\n");
749 dump_value(stdout, 0);
751 printf ("</title-local>\n");
753 printf ("<subtype>\n");
754 dump_value(stdout, 0);
756 printf ("</subtype>\n");
758 match_byte_assert(0x31);
760 printf ("<title-c>\n");
761 dump_value(stdout, 0);
763 printf ("</title-c>\n");
766 match_byte_assert(0x58);
767 if (match_byte(0x31))
769 printf ("<caption>\n");
770 dump_value(stdout, 0);
771 printf ("</caption>\n");
774 match_byte_assert(0x58);
776 int n_footnotes = get_u32();
777 for (int i = 0; i < n_footnotes; i++)
779 printf ("<footnote index=\"%d\">\n", i);
780 dump_value(stdout, 0);
781 /* Custom footnote marker string. */
782 if (match_byte (0x31))
783 dump_value(stdout, 0);
785 match_byte_assert (0x58);
787 printf ("</footnote>\n");
795 for (int i = 1; i <= 8; i++)
797 printf ("<style index=\"%d\"", i);
798 match_byte_assert(i);
799 match_byte_assert(0x31);
800 printf(" font=\"%s\"", get_string());
801 match_byte_assert(0);
802 match_byte_assert(0);
803 if (!match_byte(0x40) && !match_byte(0x20) && !match_byte(0x80) && !match_byte(0x10) && !match_byte(0x70))
804 match_byte_assert(0x50);
805 match_byte_assert(0x41);
806 if (!match_u32(0) && !match_u32(1))
808 match_byte_assert(0);
810 /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
811 if (!match_u32(0) && !match_u32(2))
814 match_u32_assert(0xfaad);
819 if (!match_u32(0) && !match_u32(1) && !match_u32(2))
821 printf (" fgcolor=\"%s\"", get_string());
822 printf (" bgcolor=\"%s\"", get_string());
825 match_byte_assert(0);
833 if (!match_u32(10) && !match_u32(11) && !match_u32(5))
841 if (!match_u32(-1) && !match_u32(8))
842 match_u32_assert(24);
843 if (!match_u32(-1) && !match_u32(2))
847 /* Who knows? Ranges from -1 to 8 with no obvious pattern. */
854 match_u32_assert(240);
857 match_u32_assert(18);
862 else if (match_u32(142))
864 else if (match_u32(143))
866 else if (match_u32(150))
870 match_u32_assert(16);
874 int count = get_u32();
877 const char *encoding = get_string();
878 printf ("<encoding>%s</encoding>\n", encoding);
881 match_u32_assert(UINT32_MAX);
883 match_byte_assert(1);
884 match_byte_assert(0);
886 match_byte_assert(1);
889 if (!match_byte(0x97) && !match_byte(0x98) && !match_byte(0x99))
890 match_byte_assert(0x9a);
891 match_byte_assert(7);
892 match_byte_assert(0);
893 match_byte_assert(0);
896 match_u32_assert(UINT32_MAX);
898 int decimal = data[pos];
899 int grouping = data[pos + 1];
902 if (!match_byte(',') && !match_byte('\''))
903 match_byte_assert(' ');
907 match_byte_assert(',');
908 if (!match_byte('.') && !match_byte(' '))
909 match_byte_assert(0);
911 printf("<format decimal=\"%c\" grouping=\"", decimal);
917 for (int i = 0; i < 5; i++)
918 printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
922 int skip = get_u32();
927 main(int argc, char *argv[])
932 if (isatty(STDIN_FILENO))
934 fprintf(stderr, "redirect stdin from a .bin file\n");
937 if (fstat(STDIN_FILENO, &s))
949 if (read(STDIN_FILENO, data, n) != n)
957 fprintf (stderr, "usage: %s TYPE < .bin", argv[0]);
961 if (!strcmp(argv[1], "title0"))
964 if (match_byte (0x03)
965 || (match_byte (0x05) && match_byte (0x58)))
966 printf ("%s\n", get_string());
968 printf ("<unknown>\n");
971 else if (!strcmp(argv[1], "title"))
977 else if (!strcmp(argv[1], "titleraw"))
979 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
981 n = find(fonts, sizeof fonts - 1);
983 else if (!strcmp(argv[1], "fonts"))
985 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
986 const char styles[] = "\xf0\0\0\0";
987 start = find(fonts, sizeof fonts - 1);
988 n = find(styles, sizeof styles - 1);
990 else if (!strcmp(argv[1], "styles"))
992 const char styles[] = "\xf0\0\0\0";
993 const char dimensions[] = "-,,,.\0";
994 start = find(styles, sizeof styles - 1);
995 n = find(dimensions, sizeof dimensions - 1) + sizeof dimensions - 1;
997 else if (!strcmp(argv[1], "dimensions") || !strcmp(argv[1], "all"))
1000 match_byte_assert(1);
1001 match_byte_assert(0);
1003 /* This might be a version number of some kind, because value 1 seems
1004 to only appear in an SPV file that also required its own weird
1005 special cases in dump_optional_value(). */
1006 version = get_u32();
1009 match_u32_assert(3);
1011 match_byte_assert(1);
1013 match_byte_assert(1);
1016 match_byte_assert(0);
1017 match_byte_assert(0);
1019 match_byte_assert(1);
1023 match_byte_assert(0);
1024 match_byte_assert(0);
1025 match_byte_assert(0);
1030 match_byte_assert(1);
1031 match_byte_assert(0);
1032 match_byte_assert(0);
1037 match_byte_assert(1);
1038 match_byte_assert(0);
1039 match_byte_assert(0);
1044 match_byte_assert(1);
1045 match_byte_assert(0);
1046 match_byte_assert(0);
1051 match_byte_assert(0);
1052 match_byte_assert(0);
1056 This is the tableId, e.g. -4154297861994971133 would be 0xdca00003.
1057 We don't have enough context to validate it. */
1070 fprintf (stderr, "%x / %x\n", pos, n);
1075 else if (!strcmp(argv[1], "raw"))
1079 dump_raw(stdout, start, n);
1083 fprintf (stderr, "unknown section %s\n", argv[1]);