10 #include "u8-mbtouc.h"
18 all_ascii(const uint8_t *p, size_t n)
20 for (size_t i = 0; i < n; i++)
21 if (p[i] < 32 || p[i] > 126)
27 try_find(const char *target, size_t target_len)
29 const uint8_t *pos = (const uint8_t *) memmem (data, n, target, target_len);
30 return pos ? pos - data : 0;
34 find(const char *target, size_t target_len)
36 size_t pos = try_find(target, target_len);
39 fprintf (stderr, "not found\n");
48 #define STR(x) XSTR(x)
49 #define WHERE __FILE__":" STR(__LINE__)
55 memcpy(&x, &data[pos], 4);
64 memcpy(&x, &data[pos], 8);
79 match_u32_assert(uint32_t x, const char *where)
81 unsigned int y = get_u32();
84 fprintf(stderr, "%s: 0x%x: expected i%u, got i%u\n", where, pos - 4, x, y);
88 #define match_u32_assert(x) match_u32_assert(x, WHERE)
93 if (pos < n && data[pos] == b)
103 match_byte_assert(uint8_t b, const char *where)
107 fprintf(stderr, "%s: 0x%x: expected %02x, got %02x\n", where, pos, b, data[pos]);
111 #define match_byte_assert(b) match_byte_assert(b, WHERE)
114 newline(FILE *stream, int pos)
116 fprintf(stream, "\n%08x: ", pos);
120 dump_raw(FILE *stream, int start, int end)
122 for (size_t i = start; i < end; )
129 && i + 4 + data[i] + data[i + 1] * 256 <= end
130 && all_ascii(&data[i + 4], data[i] + data[i + 1] * 256))
133 fprintf(stream, "\"");
134 fwrite(&data[i + 4], 1, data[i] + data[i + 1] * 256, stream);
135 fputs("\" ", stream);
137 i += 4 + data[i] + data[i + 1] * 256;
139 else if (i + 12 <= end
146 memcpy (&d, &data[i + 4], 8);
147 fprintf (stream, "F40.%d(%.*f)", data[i], data[i], d);
151 else if (i + 12 <= end
158 memcpy (&d, &data[i + 4], 8);
159 fprintf (stream, "PCT40.%d(%.*f)", data[i], data[i], d);
163 else if (i + 4 <= end
164 && (data[i] && data[i] != 88 && data[i] != 0x41)
169 fprintf (stream, "i%d ", data[i]);
174 fprintf(stream, "%02x ", data[i]);
182 static bool __attribute__((unused))
183 all_utf8(const char *p_)
185 const uint8_t *p = (const uint8_t *) p_;
186 size_t len = strlen ((char *) p);
187 for (size_t ofs = 0, mblen; ofs < len; ofs += mblen)
191 mblen = u8_mbtouc (&uc, p + ofs, len - ofs);
192 if ((uc < 32 && uc != '\n') || uc == 127 || uc == 0xfffd)
199 get_string(const char *where)
202 /*data[pos + 1] == 0 && data[pos + 2] == 0 && data[pos + 3] == 0*/
203 /*&& all_ascii(&data[pos + 4], data[pos])*/)
205 int len = data[pos] + data[pos + 1] * 256;
206 char *s = malloc(len + 1);
208 memcpy(s, &data[pos + 4], len);
215 fprintf(stderr, "%s: 0x%x: expected string\n", where, pos);
219 #define get_string() get_string(WHERE)
229 dump_counted_string(void)
232 int inner_end = get_end();
233 if (pos != inner_end)
236 if (match_byte(0x31))
239 match_byte_assert(0x58);
240 if (pos != inner_end)
242 fprintf(stderr, "inner end discrepancy\n");
249 static void __attribute__((unused))
250 hex_dump(int ofs, int n)
252 for (int i = 0; i < n; i++)
254 int c = data[ofs + i];
257 fprintf(stderr, "-");
259 fprintf(stderr, " ");
261 fprintf(stderr, "%02x", c);
262 //fprintf(stderr, "%c", c >= 32 && c < 127 ? c : '.');
264 fprintf(stderr, "\n");
268 dump_style(FILE *stream)
274 match_byte_assert(1);
275 char *fg = get_string(); /* foreground */
276 char *bg = get_string(); /* background */
277 char *font = get_string(); /* font */
278 int size = data[pos];
280 match_byte_assert(12); /* size? */
281 fprintf(stream, " fgcolor=\"%s\" bgcolor=\"%s\" font=\"%s\" size=\"%d\"",
286 dump_nested_string(FILE *stream)
290 match_byte_assert (0);
291 match_byte_assert (0);
292 int outer_end = get_end();
293 s = dump_counted_string();
295 fprintf(stream, " \"%s\"", s);
296 if (match_byte(0x31))
299 match_byte_assert(0x58);
300 match_byte_assert(0x58);
301 if (pos != outer_end)
303 fprintf(stderr, "outer end discrepancy\n");
311 dump_value_modifier(FILE *stream)
313 if (match_byte (0x31))
317 fprintf(stream, "<special0");
320 /* Corpus frequencies:
325 The given text is appended to the cell in a subscript font.
327 fprintf(stream, " subscript=\"%s\"", get_string());
330 match_u32_assert (0);
334 /* We only have one SPV file for this version (with many
341 if (!match_u32(0) && !match_u32(1) && !match_u32(2) && !match_u32(3) && !match_u32(4) && !match_u32(5) && !match_u32(6) && !match_u32(7) && !match_u32(8) && !match_u32(9))
342 match_u32_assert(10);
345 fprintf(stream, "/>\n");
349 int outer_end = get_end();
351 /* This counted-string appears to be a template string,
352 e.g. "Design\: [:^1:]1 Within Subjects Design\: [:^1:]2". */
353 char *template = dump_counted_string();
355 fprintf(stream, " template=\"%s\"", template);
357 if (match_byte(0x31))
360 match_byte_assert(0x58);
361 if (match_byte(0x31))
363 /* Only two SPV files have anything like this, so it's hard to
369 match_byte_assert(1);
370 match_byte_assert(0);
371 if (!match_byte(8) && !match_byte(1))
372 match_byte_assert(2);
373 match_byte_assert(0);
374 match_byte_assert(8);
375 match_byte_assert(0);
376 match_byte_assert(10);
377 match_byte_assert(0);
380 match_byte_assert(0x58);
381 if (pos != outer_end)
383 fprintf(stderr, "outer end discrepancy\n");
386 fprintf(stream, "/>\n");
388 else if (match_u32 (1))
390 fprintf(stream, "<footnote-ref index=%d", get_u32());
391 dump_nested_string(stream);
392 fprintf(stream, "/>\n");
394 else if (match_u32 (2))
396 fprintf(stream, "<special2 %d %d", data[pos], data[pos + 2]);
397 if (!match_byte(0) && !match_byte(1))
398 match_byte_assert(2);
399 match_byte_assert(0);
400 if (!match_u32 (2) && !match_u32(1))
402 dump_nested_string(stream);
403 fprintf(stream, "/>\n");
407 fprintf(stream, "<special3");
409 match_byte_assert(0);
410 match_byte_assert(0);
411 match_byte_assert(1);
412 match_byte_assert(0);
414 dump_nested_string(stream); /* Our corpus doesn't contain any examples with strings though. */
415 fprintf(stream, "/>\n");
419 match_byte_assert (0x58);
423 format_to_string (int type)
429 case 2: return "AHEX";
430 case 3: return "COMMA";
431 case 4: return "DOLLAR";
432 case 5: case 40: return "F";
434 case 7: return "PIBHEX";
436 case 9: return "PIB";
437 case 10: return "PK";
438 case 11: return "RB";
439 case 12: return "RBHEX";
443 case 20: return "DATE";
444 case 21: return "TIME";
445 case 22: return "DATETIME";
446 case 23: return "ADATE";
447 case 24: return "JDATE";
448 case 25: return "DTIME";
449 case 26: return "WKDAY";
450 case 27: return "MONTH";
451 case 28: return "MOYR";
452 case 29: return "QYR";
453 case 30: return "WKYR";
454 case 31: return "PCT";
455 case 32: return "DOT";
456 case 33: return "CCA";
457 case 34: return "CCB";
458 case 35: return "CCC";
459 case 36: return "CCD";
460 case 37: return "CCE";
461 case 38: return "EDATE";
462 case 39: return "SDATE";
465 sprintf(tmp, "<%d>", type);
471 dump_value(FILE *stream, int level)
478 for (int i = 0; i <= level; i++)
479 fprintf (stream, " ");
486 dump_value_modifier(stream);
488 value = get_double ();
489 fprintf (stream, "<number value=\"%.*g\" format=\"%s%d.%d\"/>\n",
490 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
492 else if (match_byte (2))
498 dump_value_modifier (stream);
500 value = get_double ();
502 vallab = get_string ();
503 fprintf (stream, "<numeric-datum value=\"%.*g\" format=\"%s%d.%d\"",
504 DBL_DIG, value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
506 fprintf (stream, " variable=\"%s\"", var);
508 fprintf (stream, " label=\"%s\"", vallab);
509 fprintf (stream, "/>\n");
510 if (!match_byte (1) && !match_byte(2))
511 match_byte_assert (3);
513 else if (match_byte (3))
515 char *text = get_string();
516 dump_value_modifier(stream);
517 char *identifier = get_string();
518 char *text_eng = get_string();
519 fprintf (stream, "<string c=\"%s\"", text_eng);
521 fprintf (stream, " identifier=\"%s\"", identifier);
522 if (strcmp(text_eng, text))
523 fprintf (stream, " local=\"%s\"", text);
524 fprintf (stream, "/>\n");
526 match_byte_assert(1);
528 else if (match_byte (4))
531 char *var, *vallab, *value;
533 dump_value_modifier(stream);
535 vallab = get_string ();
537 if (!match_byte(1) && !match_byte(2))
538 match_byte_assert (3);
539 value = get_string ();
540 fprintf (stream, "<string-datum value=\"%s\" format=\"%s%d.%d\"",
541 value, format_to_string(format >> 16), (format >> 8) & 0xff, format & 0xff);
543 fprintf (stream, " variable=\"%s\"", var);
545 fprintf (stream, " label=\"%s\"/>\n", vallab);
546 fprintf (stream, "/>\n");
548 else if (match_byte (5))
550 dump_value_modifier(stream);
551 char *name = get_string ();
552 char *label = get_string ();
553 fprintf (stream, "<variable name=\"%s\"", name);
555 fprintf (stream, " label=\"%s\"", label);
556 fprintf (stream, "/>\n");
557 if (!match_byte(1) && !match_byte(2))
558 match_byte_assert(3);
562 dump_value_modifier(stream);
564 char *base = get_string();
566 fprintf (stream, "<template format=\"%s\">\n", base);
567 for (int i = 0; i < x; i++)
574 for (int j = 0; j <= level + 1; j++)
575 fprintf (stream, " ");
576 fprintf (stream, "<substitution index=\"%d\">\n", i + 1);
577 for (int j = 0; j < y; j++)
578 dump_value (stream, level + 2);
579 for (int j = 0; j <= level + 1; j++)
580 fprintf (stream, " ");
581 fprintf (stream, "</substitution>\n");
583 for (int j = 0; j <= level; j++)
584 fprintf (stream, " ");
585 fprintf (stream, "</template>\n");
590 compare_int(const void *a_, const void *b_)
594 return *a < *b ? -1 : *a > *b;
598 check_permutation(int *a, int n, const char *name)
601 memcpy(b, a, n * sizeof *a);
602 qsort(b, n, sizeof *b, compare_int);
603 for (int i = 0; i < n; i++)
606 fprintf(stderr, "bad %s permutation:", name);
607 for (int i = 0; i < n; i++)
608 fprintf(stderr, " %d", a[i]);
615 dump_category(FILE *stream, int level, int *indexes, int *n_indexes, int max_indexes)
617 for (int i = 0; i <= level; i++)
618 fprintf (stream, " ");
619 printf ("<category>\n");
620 dump_value (stream, level + 1);
622 int merge = data[pos];
624 match_byte_assert (1);
626 match_byte_assert (0);
628 int unindexed = data[pos];
630 match_byte_assert (1);
635 match_u32_assert (2);
637 int indx = get_u32();
638 int n_categories = get_u32();
643 for (int i = 0; i <= level + 1; i++)
644 fprintf (stream, " ");
645 fprintf (stream, "<merge/>\n");
652 fprintf(stderr, "index not -1 but merged\n");
657 fprintf(stderr, "index not -1 but x != 2\n");
660 if (n_categories != 0)
662 fprintf(stderr, "index not -1 but subcategories\n");
665 if (*n_indexes >= max_indexes)
667 fprintf(stderr, "too many categories (increase max_indexes)\n");
670 indexes[(*n_indexes)++] = indx;
673 int expected_unindexed = indx == -1;
674 if (unindexed != expected_unindexed)
676 fprintf(stderr, "unindexed (%d) mismatch with indx (%d)\n",
681 if (n_categories == 0)
683 for (int i = 0; i <= level + 1; i++)
684 fprintf (stream, " ");
685 fprintf (stream, "<category-index>%d</category-index>\n", indx);
687 for (int i = 0; i < n_categories; i++)
688 dump_category (stream, level + 1, indexes, n_indexes, max_indexes);
689 for (int i = 0; i <= level; i++)
690 fprintf (stream, " ");
691 printf ("</category>\n");
699 printf ("<dimension index=\"%d\">\n", indx);
700 dump_value (stdout, 0);
702 /* This byte is usually 0 but many other values have been spotted. */
705 if (!match_byte(0) && !match_byte(1))
706 match_byte_assert(2);
710 match_byte_assert(1);
712 match_byte_assert(1);
713 match_byte_assert(1);
714 if (!match_u32(UINT32_MAX))
715 match_u32_assert(indx);
716 n_categories = get_u32();
720 for (int i = 0; i < n_categories; i++)
721 dump_category (stdout, 0, indexes, &n_indexes, sizeof indexes / sizeof *indexes);
722 check_permutation(indexes, n_indexes, "categories");
724 fprintf (stdout, "</dimension>\n");
729 static int dim_n_cats[64];
730 #define MAX_DIMS (sizeof dim_n_cats / sizeof *dim_n_cats)
736 assert(n_dims < MAX_DIMS);
737 for (int i = 0; i < n_dims; i++)
738 dim_n_cats[i] = dump_dim (i);
744 /* The first three numbers add to the number of dimensions. */
747 int c = n_dims - l - r;
750 /* The next n_dims numbers are a permutation of the dimension numbers. */
752 for (int i = 0; i < n_dims; i++)
757 const char *name = i < l ? "layer" : i < l + r ? "row" : "column";
758 printf ("<%s dimension=\"%d\"/>\n", name, dim);
760 check_permutation(a, n_dims, "dimensions");
764 for (int i = 0; i < x; i++)
766 unsigned int indx = get_u32();
767 printf (" <datum index=\"%d\" coords=", indx);
769 int coords[MAX_DIMS];
770 for (int i = n_dims; i-- > 0; )
772 coords[i] = indx % dim_n_cats[i];
773 indx /= dim_n_cats[i];
775 for (int i = 0; i < n_dims; i++)
776 printf("%c%d", i ? ',' : '"', coords[i]);
782 dump_value(stdout, 1);
783 fprintf (stdout, " </datum>\n");
785 printf ("</data>\n");
791 printf ("<title-local>\n");
792 dump_value(stdout, 0);
794 printf ("</title-local>\n");
796 printf ("<subtype>\n");
797 dump_value(stdout, 0);
799 printf ("</subtype>\n");
801 match_byte_assert(0x31);
803 printf ("<title-c>\n");
804 dump_value(stdout, 0);
806 printf ("</title-c>\n");
809 match_byte_assert(0x58);
810 if (match_byte(0x31))
812 printf ("<caption>\n");
813 dump_value(stdout, 0);
814 printf ("</caption>\n");
817 match_byte_assert(0x58);
819 int n_footnotes = get_u32();
820 for (int i = 0; i < n_footnotes; i++)
822 printf ("<footnote index=\"%d\">\n", i);
823 dump_value(stdout, 0);
824 /* Custom footnote marker string. */
825 if (match_byte (0x31))
826 dump_value(stdout, 0);
828 match_byte_assert (0x58);
830 printf ("</footnote>\n");
838 for (int i = 1; i <= 8; i++)
840 printf ("<style index=\"%d\"", i);
841 match_byte_assert(i);
842 match_byte_assert(0x31);
843 printf(" font=\"%s\"", get_string());
844 match_byte_assert(0);
845 match_byte_assert(0);
846 if (!match_byte(0x40) && !match_byte(0x20) && !match_byte(0x80) && !match_byte(0x10) && !match_byte(0x70))
847 match_byte_assert(0x50);
848 match_byte_assert(0x41);
849 if (!match_u32(0) && !match_u32(1))
851 match_byte_assert(0);
853 /* OK, this seems really unlikely to be totally correct, but it matches my corpus... */
854 if (!match_u32(0) && !match_u32(2))
857 match_u32_assert(0xfaad);
862 if (!match_u32(0) && !match_u32(1) && !match_u32(2))
864 printf (" fgcolor=\"%s\"", get_string());
865 printf (" bgcolor=\"%s\"", get_string());
868 match_byte_assert(0);
876 if (!match_u32(10) && !match_u32(11) && !match_u32(5))
878 if (!match_u32(0) && !match_u32(1))
884 if (!match_u32(-1) && !match_u32(8))
885 match_u32_assert(24);
886 if (!match_u32(-1) && !match_u32(2))
890 /* Who knows? Ranges from -1 to 8 with no obvious pattern. */
897 match_u32_assert(240);
900 match_u32_assert(18);
907 int len = data[pos + 0x34];
909 printf("<tablelook>%.*s</tablelook>\n", len, &data[pos + 0x35]);
913 int count = get_u32();
916 const char *locale = get_string();
917 printf ("<locale>%s</locale>\n", locale);
920 match_u32_assert(UINT32_MAX);
922 match_byte_assert(1);
923 match_byte_assert(0);
925 match_byte_assert(1);
928 if (!match_byte(0x97) && !match_byte(0x98)
929 && !match_byte(0x99) && !match_byte(0x9a))
930 match_byte_assert(0x9b);
931 match_byte_assert(7);
932 match_byte_assert(0);
933 match_byte_assert(0);
936 match_u32_assert(UINT32_MAX);
938 int decimal = data[pos];
939 int grouping = data[pos + 1];
942 if (!match_byte(',') && !match_byte('\''))
943 match_byte_assert(' ');
947 match_byte_assert(',');
948 if (!match_byte('.') && !match_byte(' '))
949 match_byte_assert(0);
951 printf("<format decimal=\"%c\" grouping=\"", decimal);
957 for (int i = 0; i < 5; i++)
958 printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
963 /* The last chunk is an outer envelope that contains two inner envelopes.
964 The second inner envelope has some interesting data like the encoding and
968 int outer_end = get_end();
970 /* First inner envelope: byte*33 int[n] int*[n]. */
973 /* Second inner envelope. */
974 assert(get_end() == outer_end);
976 match_byte_assert(1);
977 match_byte_assert(0);
978 if (!match_byte(3) && !match_byte(4))
979 match_byte_assert(5);
980 match_byte_assert(0);
981 match_byte_assert(0);
982 match_byte_assert(0);
984 printf("<command>%s</command>\n", get_string());
985 printf("<subcommand>%s</subcommand>\n", get_string());
986 printf("<language>%s</language>\n", get_string());
987 printf("<charset>%s</charset>\n", get_string());
988 printf("<locale>%s</locale>\n", get_string());
991 match_byte_assert(1);
992 match_byte_assert(0);
994 match_byte_assert(1);
996 match_byte_assert(1);
998 if (!match_byte(0x97) && !match_byte(0x98)
999 && !match_byte(0x99) && !match_byte(0x9a))
1000 match_byte_assert(0x9b);
1001 match_byte_assert(7);
1002 match_byte_assert(0);
1003 match_byte_assert(0);
1005 if (match_byte('.'))
1007 if (!match_byte(',') && !match_byte('\''))
1008 match_byte_assert(' ');
1012 match_byte_assert(',');
1013 if (!match_byte('.') && !match_byte(' '))
1014 match_byte_assert(0);
1018 match_byte_assert(1);
1020 if (outer_end - pos > 6)
1022 /* There might be a pair of strings representing a dataset and
1023 datafile name, or there might be a set of custom currency strings.
1024 The custom currency strings start with a pair of integers, so we
1025 can distinguish these from a string by checking for a null byte; a
1026 small 32-bit integer will always contain a null and a text string
1029 int len = get_u32();
1030 bool has_dataset = !memchr(&data[pos], '\0', len);
1035 printf("<dataset>%s</dataset>\n", get_string());
1036 printf("<datafile>%s</datafile>\n", get_string());
1038 match_u32_assert(0);
1040 match_u32_assert(0);
1046 for (int i = 0; i < 5; i++)
1047 printf("<CC%c>%s</CC%c>\n", 'A' + i, get_string(), 'A' + i);
1050 match_u32_assert(0);
1052 match_byte_assert(0x2e);
1054 match_byte_assert(1);
1056 if (pos < outer_end)
1058 match_u32_assert(2000000);
1059 match_u32_assert(0);
1061 assert(pos == outer_end);
1066 match_u32_assert(0);
1070 main(int argc, char *argv[])
1075 if (isatty(STDIN_FILENO))
1077 fprintf(stderr, "redirect stdin from a .bin file\n");
1080 if (fstat(STDIN_FILENO, &s))
1092 if (read(STDIN_FILENO, data, n) != n)
1100 fprintf (stderr, "usage: %s TYPE < .bin", argv[0]);
1104 if (!strcmp(argv[1], "title0"))
1107 if (match_byte (0x03)
1108 || (match_byte (0x05) && match_byte (0x58)))
1109 printf ("%s\n", get_string());
1111 printf ("<unknown>\n");
1114 else if (!strcmp(argv[1], "title"))
1120 else if (!strcmp(argv[1], "titleraw"))
1122 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
1124 n = find(fonts, sizeof fonts - 1);
1126 else if (!strcmp(argv[1], "fonts"))
1128 const char fonts[] = "\x01\x31\x09\0\0\0SansSerif";
1129 const char styles[] = "\xf0\0\0\0";
1130 start = find(fonts, sizeof fonts - 1);
1131 n = find(styles, sizeof styles - 1);
1133 else if (!strcmp(argv[1], "styles"))
1135 const char styles[] = "\xf0\0\0\0";
1136 const char dimensions[] = "-,,,.\0";
1137 start = find(styles, sizeof styles - 1);
1138 n = find(dimensions, sizeof dimensions - 1) + sizeof dimensions - 1;
1140 else if (!strcmp(argv[1], "dimensions") || !strcmp(argv[1], "all"))
1143 match_byte_assert(1);
1144 match_byte_assert(0);
1146 /* This might be a version number of some kind, because value 1 seems
1147 to only appear in an SPV file that also required its own weird
1148 special cases in dump_value_modifier(). */
1149 version = get_u32();
1152 match_u32_assert(3);
1154 match_byte_assert(1);
1156 match_byte_assert(1);
1159 match_byte_assert(0);
1160 match_byte_assert(0);
1162 match_byte_assert(1);
1166 match_byte_assert(0);
1167 match_byte_assert(0);
1168 match_byte_assert(0);
1173 match_byte_assert(1);
1174 match_byte_assert(0);
1175 match_byte_assert(0);
1180 match_byte_assert(1);
1181 match_byte_assert(0);
1182 match_byte_assert(0);
1187 match_byte_assert(1);
1188 match_byte_assert(0);
1189 match_byte_assert(0);
1194 match_byte_assert(0);
1195 match_byte_assert(0);
1199 This is the tableId, e.g. -4154297861994971133 would be 0xdca00003.
1200 We don't have enough context to validate it. */
1213 fprintf (stderr, "%x / %x\n", pos, n);
1218 else if (!strcmp(argv[1], "raw"))
1222 dump_raw(stdout, start, n);
1226 fprintf (stderr, "unknown section %s\n", argv[1]);