10 #include "u8-mbtouc.h"
18 all_ascii(const uint8_t *p, size_t n)
20 for (size_t i = 0; i < n; i++)
21 if (p[i] < 32 || p[i] > 126)
27 try_find(const char *target, size_t target_len)
29 const uint8_t *pos = (const uint8_t *) memmem (data, n, target, target_len);
30 return pos ? pos - data : 0;
34 find(const char *target, size_t target_len)
36 size_t pos = try_find(target, target_len);
39 fprintf (stderr, "not found\n");
48 #define STR(x) XSTR(x)
49 #define WHERE __FILE__":" STR(__LINE__)
55 memcpy(&x, &data[pos], 4);
64 memcpy(&x, &data[pos], 8);
79 match_u32_assert(uint32_t x, const char *where)
81 unsigned int y = get_u32();
84 fprintf(stderr, "%s: 0x%x: expected i%u, got i%u\n", where, pos - 4, x, y);
88 #define match_u32_assert(x) match_u32_assert(x, WHERE)
93 if (pos < n && data[pos] == b)
103 match_byte_assert(uint8_t b, const char *where)
107 fprintf(stderr, "%s: 0x%x: expected %02x, got %02x\n", where, pos, b, data[pos]);
111 #define match_byte_assert(b) match_byte_assert(b, WHERE)
114 newline(FILE *stream, int pos)
116 fprintf(stream, "\n%08x: ", pos);
120 dump_raw(FILE *stream, int start, int end)
122 for (size_t i = start; i < end; )
129 && i + 4 + data[i] + data[i + 1] * 256 <= end
130 && all_ascii(&data[i + 4], data[i] + data[i + 1] * 256))
133 fprintf(stream, "\"");
134 fwrite(&data[i + 4], 1, data[i] + data[i + 1] * 256, stream);
135 fputs("\" ", stream);
137 i += 4 + data[i] + data[i + 1] * 256;
139 else if (i + 12 <= end
146 memcpy (&d, &data[i + 4], 8);
147 fprintf (stream, "F40.%d(%.*f)", data[i], data[i], d);
151 else if (i + 12 <= end
158 memcpy (&d, &data[i + 4], 8);
159 fprintf (stream, "PCT40.%d(%.*f)", data[i], data[i], d);
163 else if (i + 4 <= end
164 && (data[i] && data[i] != 88 && data[i] != 0x41)
169 fprintf (stream, "i%d ", data[i]);
174 fprintf(stream, "%02x ", data[i]);
182 static bool __attribute__((unused))
183 all_utf8(const char *p_)
185 const uint8_t *p = (const uint8_t *) p_;
186 size_t len = strlen ((char *) p);
187 for (size_t ofs = 0, mblen; ofs < len; ofs += mblen)
191 mblen = u8_mbtouc (&uc, p + ofs, len - ofs);
192 if ((uc < 32 && uc != '\n') || uc == 127 || uc == 0xfffd)
199 get_string(const char *where)
202 /*data[pos + 1] == 0 && data[pos + 2] == 0 && data[pos + 3] == 0*/
203 /*&& all_ascii(&data[pos + 4], data[pos])*/)
205 int len = data[pos] + data[pos + 1] * 256;
206 char *s = malloc(len + 1);
208 memcpy(s, &data[pos + 4], len);
215 fprintf(stderr, "%s: 0x%x: expected string\n", where, pos);
219 #define get_string() get_string(WHERE)
222 dump_counted_string(void)
225 int inner_end = pos + get_u32();
226 if (pos != inner_end)
229 if (match_byte(0x31))
232 match_byte_assert(0x58);
233 if (pos != inner_end)
235 fprintf(stderr, "inner end discrepancy\n");
243 dump_nested_string(void)
247 match_byte_assert (0);
248 match_byte_assert (0);
249 int outer_end = pos + get_u32();
250 s = dump_counted_string();
251 match_byte_assert(0x58);
252 match_byte_assert(0x58);
253 if (pos != outer_end)
255 fprintf(stderr, "outer end discrepancy\n");
263 dump_optional_value(FILE *stream)
265 if (match_byte (0x31))
271 /* Corpus frequencies:
279 match_u32_assert (0);
283 /* We only have one SPV file for this version (with many
290 if (!match_u32(0) && !match_u32(1) && !match_u32(2) && !match_u32(3) && !match_u32(4) && !match_u32(5) && !match_u32(6) && !match_u32(7) && !match_u32(8) && !match_u32(9))
291 match_u32_assert(10);
297 int outer_end = pos + get_u32();
299 /* This counted-string appears to be a template string,
300 e.g. "Design\: [:^1:]1 Within Subjects Design\: [:^1:]2". */
301 dump_counted_string();
303 if (match_byte(0x31))
305 /* Only one example in the corpus. */
310 match_byte_assert(1);
311 get_string(); /* foreground */
312 get_string(); /* background */
313 get_string(); /* font */
315 match_byte_assert(12); /* size? */
318 match_byte_assert(0x58);
319 if (match_byte(0x31))
321 /* Only two SPV files have anything like this, so it's hard to
327 match_byte_assert(1);
328 match_byte_assert(0);
329 if (!match_byte(8) && !match_byte(1))
330 match_byte_assert(2);
331 match_byte_assert(0);
332 match_byte_assert(8);
333 match_byte_assert(0);
334 match_byte_assert(10);
335 match_byte_assert(0);
338 match_byte_assert(0x58);
339 if (pos != outer_end)
341 fprintf(stderr, "outer end discrepancy\n");
345 else if (match_u32 (1))
347 fprintf(stream, "(footnote %d) ", get_u32());
348 dump_nested_string();
350 else if (match_u32 (2))
352 fprintf(stream, "(special 2)");
354 match_byte_assert(2);
355 match_byte_assert(0);
356 if (!match_u32 (2) && !match_u32(1))
358 dump_nested_string();
363 fprintf(stream, "(special 3)");
364 match_byte_assert(0);
365 match_byte_assert(0);
366 match_byte_assert(1);
367 match_byte_assert(0);
369 dump_nested_string(); /* Our corpus doesn't contain any examples with strings though. */
373 match_byte_assert (0x58);
377 format_to_string (int type)
383 case 2: return "AHEX";
384 case 3: return "COMMA";
385 case 4: return "DOLLAR";
386 case 5: case 40: return "F";
388 case 7: return "PIBHEX";
390 case 9: return "PIB";
391 case 10: return "PK";
392 case 11: return "RB";
393 case 12: return "RBHEX";
397 case 20: return "DATE";
398 case 21: return "TIME";
399 case 22: return "DATETIME";
400 case 23: return "ADATE";
401 case 24: return "JDATE";
402 case 25: return "DTIME";
403 case 26: return "WKDAY";
404 case 27: return "MONTH";
405 case 28: return "MOYR";
406 case 29: return "QYR";
407 case 30: return "WKYR";
408 case 31: return "PCT";
409 case 32: return "DOT";
410 case 33: return "CCA";
411 case 34: return "CCB";
412 case 35: return "CCC";
413 case 36: return "CCD";
414 case 37: return "CCE";
415 case 38: return "EDATE";
416 case 39: return "SDATE";
419 sprintf(tmp, "<%d>", type);
425 dump_value(FILE *stream, int level)
432 for (int i = 0; i <= level; i++)
433 fprintf (stream, " ");
440 dump_optional_value(stream);
442 value = get_double ();
443 fprintf (stream, "<number value=\"%.*g\" format=\"%s%d.%d\"/>\n",
444 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
446 else if (match_byte (2))
452 dump_optional_value (stream);
454 value = get_double ();
456 vallab = get_string ();
457 fprintf (stream, "<numeric-datum value=\"%.*g\" format=\"%s%d.%d\"",
458 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
460 fprintf (stream, " variable=\"%s\"", var);
462 fprintf (stream, " label=\"%s\"/>\n", vallab);
463 fprintf (stream, "/>\n");
464 if (!match_byte (1) && !match_byte(2))
465 match_byte_assert (3);
467 else if (match_byte (3))
469 char *text = get_string();
470 dump_optional_value(stream);
471 char *identifier = get_string();
472 char *text_eng = get_string();
473 fprintf (stream, "<string c=\"%s\"", text_eng);
475 fprintf (stream, " identifier=\"%s\"", identifier);
476 if (strcmp(text_eng, text))
477 fprintf (stream, " local=\"%s\"", text);
478 fprintf (stream, "/>\n");
480 match_byte_assert(1);
482 else if (match_byte (4))
485 char *var, *vallab, *value;
487 dump_optional_value(stream);
489 vallab = get_string ();
491 if (!match_byte(1) && !match_byte(2))
492 match_byte_assert (3);
493 value = get_string ();
494 fprintf (stream, "<string-datum value=\"%s\" format=\"%s%d.%d\"",
495 value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
497 fprintf (stream, " variable=\"%s\"", var);
499 fprintf (stream, " label=\"%s\"/>\n", vallab);
500 fprintf (stream, "/>\n");
502 else if (match_byte (5))
504 dump_optional_value(stream);
505 char *name = get_string ();
506 char *label = get_string ();
507 fprintf (stream, "<variable name=\"%s\"", name);
509 fprintf (stream, " label=\"%s\"", label);
510 fprintf (stream, "/>\n");
511 if (!match_byte(1) && !match_byte(2))
512 match_byte_assert(3);
516 dump_optional_value(stream);
518 char *base = get_string();
520 fprintf (stream, "<template format=\"%s\">\n", base);
521 for (int i = 0; i < x; i++)
528 for (int j = 0; j <= level + 1; j++)
529 fprintf (stream, " ");
530 fprintf (stream, "<substitution index=\"%d\">\n", i + 1);
531 for (int j = 0; j < y; j++)
532 dump_value (stream, level + 2);
533 for (int j = 0; j <= level + 1; j++)
534 fprintf (stream, " ");
535 fprintf (stream, "</substitution>\n");
537 for (int j = 0; j <= level; j++)
538 fprintf (stream, " ");
539 fprintf (stream, "</template>\n");
544 compare_int(const void *a_, const void *b_)
548 return *a < *b ? -1 : *a > *b;
552 check_permutation(int *a, int n, const char *name)
555 memcpy(b, a, n * sizeof *a);
556 qsort(b, n, sizeof *b, compare_int);
557 for (int i = 0; i < n; i++)
560 fprintf(stderr, "bad %s permutation:", name);
561 for (int i = 0; i < n; i++)
562 fprintf(stderr, " %d", a[i]);
569 dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
571 for (int i = 0; i <= level; i++)
572 fprintf (stream, " ");
573 printf ("<category>\n");
574 dump_value (stream, level + 1);
576 int merge = data[pos];
578 match_byte_assert (1);
580 match_byte_assert (0);
582 int unindexed = data[pos];
584 match_byte_assert (1);
589 match_u32_assert (2);
591 int indx = get_u32();
592 int n_categories = get_u32();
597 for (int i = 0; i <= level + 1; i++)
598 fprintf (stream, " ");
599 fprintf (stream, "<merge/>\n");
606 fprintf(stderr, "index not -1 but merged\n");
611 fprintf(stderr, "index not -1 but x != 2\n");
614 if (n_categories != 0)
616 fprintf(stderr, "index not -1 but subcategories\n");
619 if (*n_indexes >= max_indexes)
621 fprintf(stderr, "too many categories (increase max_indexes)\n");
624 indexes[(*n_indexes)++] = indx;
627 int expected_unindexed = indx == -1;
628 if (unindexed != expected_unindexed)
630 fprintf(stderr, "unindexed (%d) mismatch with indx (%d)\n",
635 if (n_categories == 0)
637 for (int i = 0; i <= level + 1; i++)
638 fprintf (stream, " ");
639 fprintf (stream, "<category-index>%d</category-index>\n", indx);
641 for (int i = 0; i < n_categories; i++)
642 dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
643 for (int i = 0; i <= level; i++)
644 fprintf (stream, " ");
645 printf ("</category>\n");
653 printf ("<dimension index=\"%d\">\n", indx);
654 dump_value (stdout, 0);
656 /* This byte is usually 0 but many other values have been spotted. */
659 if (!match_byte(0) && !match_byte(1))
660 match_byte_assert(2);
664 match_byte_assert(1);
666 match_byte_assert(1);
667 match_byte_assert(1);
668 if (!match_u32(UINT32_MAX))
669 match_u32_assert(indx);
670 n_categories = get_u32();
674 for (int i = 0; i < n_categories; i++)
675 dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
676 check_permutation(indexes, n_indexes, "categories");
678 fprintf (stdout, "</dimension>\n");
683 static int dim_n_cats[64];
684 #define MAX_DIMS (sizeof dim_n_cats / sizeof *dim_n_cats)
690 assert(n_dims < MAX_DIMS);
691 for (int i = 0; i < n_dims; i++)
692 dim_n_cats[i] = dump_dim (i);
698 /* The first three numbers add to the number of dimensions. */
701 int c = n_dims - l - r;
704 /* The next n_dims numbers are a permutation of the dimension numbers. */
706 for (int i = 0; i < n_dims; i++)
711 const char *name = i < l ? "layer" : i < l + r ? "row" : "column";
712 printf ("<%s dimension=\"%d\"/>\n", name, dim);
714 check_permutation(a, n_dims, "dimensions");
718 for (int i = 0; i < x; i++)
720 unsigned int indx = get_u32();
721 printf (" <datum index=\"%d\" coords=", indx);
723 int coords[MAX_DIMS];
724 for (int i = n_dims; i-- > 0; )
726 coords[i] = indx % dim_n_cats[i];
727 indx /= dim_n_cats[i];
729 for (int i = 0; i < n_dims; i++)
730 printf("%c%d", i ? ',' : '"', coords[i]);
736 dump_value(stdout, 1);
737 fprintf (stdout, " </datum>\n");
739 printf ("</data>\n");
745 printf ("<title-local>\n");
746 dump_value(stdout, 0);
748 printf ("</title-local>\n");
750 printf ("<subtype>\n");
751 dump_value(stdout, 0);
753 printf ("</subtype>\n");
755 match_byte_assert(0x31);
757 printf ("<title-c>\n");
758 dump_value(stdout, 0);
760 printf ("</title-c>\n");
763 match_byte_assert(0x58);
764 if (match_byte(0x31))
766 printf ("<caption>\n");
767 dump_value(stdout, 0);
768 printf ("</caption>\n");
771 match_byte_assert(0x58);
773 int n_footnotes = get_u32();
774 for (int i = 0; i < n_footnotes; i++)
776 printf ("<footnote index=\"%d\">\n", i);
777 dump_value(stdout, 0);
778 /* Custom footnote marker string. */
779 if (match_byte (0x31))
780 dump_value(stdout, 0);
782 match_byte_assert (0x58);
784 printf ("</footnote>\n");
792 for (int i = 1; i <= 8; i++)
794 printf ("<style index=\"%d\"", i);
795 match_byte_assert(i);
796 match_byte_assert(0x31);
797 printf(" font=\"%s\"", get_string());
798 match_byte_assert(0);
799 match_byte_assert(0);
800 if (!match_byte(0x40) && !match_byte(0x20) && !match_byte(0x80) && !match_byte(0x10) && !match_byte(0x70))
801 match_byte_assert(0x50);
802 match_byte_assert(0x41);
803 if (!match_u32(0) && !match_u32(1))
805 match_byte_assert(0);
807 /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
808 if (!match_u32(0) && !match_u32(2))
811 match_u32_assert(0xfaad);
816 if (!match_u32(0) && !match_u32(1) && !match_u32(2))
818 printf (" fgcolor=\"%s\"", get_string());
819 printf (" bgcolor=\"%s\"", get_string());
822 match_byte_assert(0);
830 if (!match_u32(10) && !match_u32(11) && !match_u32(5))
838 if (!match_u32(-1) && !match_u32(8))
839 match_u32_assert(24);
840 if (!match_u32(-1) && !match_u32(2))
844 /* Who knows? Ranges from -1 to 8 with no obvious pattern. */
851 match_u32_assert(240);
854 match_u32_assert(18);
859 else if (match_u32(142))
861 else if (match_u32(143))
863 else if (match_u32(150))
867 match_u32_assert(16);
871 int count = get_u32();
874 const char *encoding = get_string();
875 printf ("<encoding>%s</encoding>\n", encoding);
878 match_u32_assert(UINT32_MAX);
880 match_byte_assert(1);
881 match_byte_assert(0);
883 match_byte_assert(1);
886 if (!match_byte(0x97) && !match_byte(0x98) && !match_byte(0x99))
887 match_byte_assert(0x9a);
888 match_byte_assert(7);
889 match_byte_assert(0);
890 match_byte_assert(0);
893 match_u32_assert(UINT32_MAX);
895 int decimal = data[pos];
896 int grouping = data[pos + 1];
899 if (!match_byte(',') && !match_byte('\''))
900 match_byte_assert(' ');
904 match_byte_assert(',');
905 if (!match_byte('.') && !match_byte(' '))
906 match_byte_assert(0);
908 printf("<format decimal=\"%c\" grouping=\"", decimal);
914 for (int i = 0; i < 5; i++)
915 printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
919 int skip = get_u32();
924 main(int argc, char *argv[])
929 if (isatty(STDIN_FILENO))
931 fprintf(stderr, "redirect stdin from a .bin file\n");
934 if (fstat(STDIN_FILENO, &s))
946 if (read(STDIN_FILENO, data, n) != n)
954 fprintf (stderr, "usage: %s TYPE < .bin", argv[0]);
958 if (!strcmp(argv[1], "title0"))
961 if (match_byte (0x03)
962 || (match_byte (0x05) && match_byte (0x58)))
963 printf ("%s\n", get_string());
965 printf ("<unknown>\n");
968 else if (!strcmp(argv[1], "title"))
974 else if (!strcmp(argv[1], "titleraw"))
976 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
978 n = find(fonts, sizeof fonts - 1);
980 else if (!strcmp(argv[1], "fonts"))
982 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
983 const char styles[] = "\xf0\0\0\0";
984 start = find(fonts, sizeof fonts - 1);
985 n = find(styles, sizeof styles - 1);
987 else if (!strcmp(argv[1], "styles"))
989 const char styles[] = "\xf0\0\0\0";
990 const char dimensions[] = "-,,,.\0";
991 start = find(styles, sizeof styles - 1);
992 n = find(dimensions, sizeof dimensions - 1) + sizeof dimensions - 1;
994 else if (!strcmp(argv[1], "dimensions") || !strcmp(argv[1], "all"))
997 match_byte_assert(1);
998 match_byte_assert(0);
1000 /* This might be a version number of some kind, because value 1 seems
1001 to only appear in an SPV file that also required its own weird
1002 special cases in dump_optional_value(). */
1003 version = get_u32();
1006 match_u32_assert(3);
1008 match_byte_assert(1);
1010 match_byte_assert(1);
1013 match_byte_assert(0);
1014 match_byte_assert(0);
1016 match_byte_assert(1);
1020 match_byte_assert(0);
1021 match_byte_assert(0);
1022 match_byte_assert(0);
1027 match_byte_assert(1);
1028 match_byte_assert(0);
1029 match_byte_assert(0);
1034 match_byte_assert(1);
1035 match_byte_assert(0);
1036 match_byte_assert(0);
1041 match_byte_assert(1);
1042 match_byte_assert(0);
1043 match_byte_assert(0);
1048 match_byte_assert(0);
1049 match_byte_assert(0);
1053 This is the tableId, e.g. -4154297861994971133 would be 0xdca00003.
1054 We don't have enough context to validate it. */
1067 fprintf (stderr, "%x / %x\n", pos, n);
1072 else if (!strcmp(argv[1], "raw"))
1076 dump_raw(stdout, start, n);
1080 fprintf (stderr, "unknown section %s\n", argv[1]);