12 sys.stderr.write("%s:%d: %s\n" % (file_name, line_number, msg))
24 line = input_file.readline()
29 return s.isdigit() or (s[0] == '-' and s[1].isdigit())
32 xdigits = "0123456789abcdefABCDEF"
42 fatal("syntax error expecting %s" % type)
63 if token == ('eof', ):
64 fatal("unexpected end of input")
72 elif not line[0].isspace():
82 elif line[0] in '[]()?|*':
85 elif line.startswith('=>'):
88 elif line.startswith('...'):
91 elif line[0].isalnum() or line[0] == '-':
93 while n < len(line) and (line[n].isalnum() or line[n] == '-'):
98 if prev[0] == '*' and is_num(s):
99 token = ('number', int(s, 10))
100 elif len(s) == 2 and is_xdigits(s):
101 token = ('bytes', struct.pack('B', int(s, 16)))
102 elif s[0] == 'i' and is_num(s[1:]):
103 token = ('bytes', struct.pack('<i', int(s[1:])))
104 elif s[:2] == 'ib' and is_num(s[2:]):
105 token = ('bytes', struct.pack('>i', int(s[2:])))
107 token = ('nonterminal', s)
108 elif s in ('bool', 'int16', 'int32', 'int64', 'be16', 'be32', 'be64',
109 'string', 'bestring', 'byte', 'float', 'double',
110 'count', 'becount', 'v1', 'v3', 'vAF', 'vB0',
116 fatal("unknown character %c" % line[0])
120 argv0 = os.path.basename(sys.argv[0])
122 %(argv0)s, parser generator for SPV binary members
123 usage: %(argv0)s GRAMMAR header
124 %(argv0)s GRAMMAR code HEADER_NAME
125 where GRAMMAR contains grammar definitions\
126 ''' % {"argv0": argv0})
131 def __init__(self, type_, name, n, content):
135 self.content = content
137 if self.type_ == 'constant':
138 return ' '.join(['%02x' % ord(x) for x in self.content])
140 return "%s(%s)" % (self.type_, self.content)
151 elif t[0] in ('bool', 'byte',
152 'int16', 'int32', 'int64',
153 'be16', 'be32', 'be64',
154 'string', 'bestring',
156 'nonterminal', '...'):
160 if t[0] == 'nonterminal':
161 name = name_to_id(content[1])
162 elif t[0] in ('v1', 'v3', 'vAF', 'vB0', 'count', 'becount'):
166 content = parse_choice()
172 content = parse_choice()
176 fatal('syntax error expecting item')
181 if token[0] == 'number':
189 if n.startswith('n-'):
192 fatal('expecting quantity')
198 if type_ == 'constant' and not optional:
199 fatal("%s: cannot name a constant" % token[1])
205 if type_ == 'constant':
209 item = Item(type_, name, n, content)
211 item = Item('|', None, 1, [[item], []])
215 def parse_concatenation():
217 while token[0] not in (')', ';', '|', 'eof'):
219 if (item.type_ == 'constant'
221 and items[-1].type_ == 'constant'):
222 items[-1].content += item.content
229 sub = parse_concatenation()
235 choices.append(parse_concatenation())
237 return [Item('|', None, 1, choices)]
248 items = parse_concatenation()
251 or items[0].type_ != 'constant'
252 or len(items[0].content) != 1):
253 fatal("choice must begin with xx (or 'else')")
254 choice = '%02x' % ord(items[0].content)
256 if choice in choices:
257 fatal("duplicate choice %s" % choice)
258 choices[choice] = items
271 return Item('case', case_name, 1,
272 { '%s_%s' % (case_name, k) : v for k, v in choices.items() })
275 def parse_production():
276 expect('nonterminal')
280 return name, parse_choice()
283 def print_members(p, indent):
285 if item.type_ == 'variable' and item.name:
286 if item.content[0] == 'nonterminal':
287 typename = 'struct %s%s' % (prefix,
288 name_to_id(item.content[1]))
291 c_types = {'bool': ('bool', 0),
292 'byte': ('uint8_t', 0),
293 'int16': ('uint16_t', 0),
294 'int32': ('uint32_t', 0),
295 'int64': ('uint64_t', 0),
296 'be16': ('uint16_t', 0),
297 'be32': ('uint32_t', 0),
298 'be64': ('uint64_t', 0),
299 'string': ('char', 1),
300 'bestring': ('char', 1),
301 'float': ('double', 0),
302 'double': ('double', 0),
303 '...': ('uint8_t', 1)}
304 typename, n_stars = c_types[item.content[0]]
308 if isinstance(item.n, int):
310 array_suffix = '[%d]' % item.n
314 print "%s%s %s%s%s;" % (indent, typename, '*' * n_stars,
315 name_to_id(item.name),
317 elif item.type_ in ('v1', 'v3', 'vAF', 'vB0',
318 'count', 'becount', '()'):
319 print_members(item.content, indent)
320 elif item.type_ == '|':
321 for choice in item.content:
322 print_members(choice, indent)
323 elif item.type_ == 'case':
324 print "%sint %s;" % (indent, item.name)
325 print "%sunion {" % indent
326 for name, choice in sorted(item.content.items()):
327 print "%s struct {" % indent
328 print_members(choice, indent + ' ' * 8)
329 print "%s } %s;" % (indent, name)
330 print "%s};" % indent
331 elif item.type_ == 'constant':
333 print "%sbool %s;" % (indent, item.name)
334 elif item.type_ not in ("constant", "variable"):
335 fatal("unhandled type %s" % item.type_)
339 return ''.join(['"'] + ["\\x%02x" % ord(x) for x in s] + ['"'])
342 class Parser_Context(object):
346 self.need_error_handler = False
347 def gen_name(self, prefix):
348 n = self.suffixes.get(prefix, 0) + 1
349 self.suffixes[prefix] = n
350 return '%s%d' % (prefix, n) if n > 1 else prefix
351 def save_pos(self, indent):
352 pos = self.gen_name('pos')
353 print "%sstruct spvbin_position %s = spvbin_position_save (input);" % (indent, pos)
355 def save_error(self, indent):
356 error = self.gen_name('save_n_errors')
357 print "%ssize_t %s = input->n_errors;" % (indent, error)
359 def parse_limit(self, endian, indent):
360 limit = self.gen_name('saved_limit')
362 %sstruct spvbin_limit %s;
363 %sif (!spvbin_limit_parse%s (&%s, input))
366 indent, '_be' if endian == 'big' else '', limit,
371 def print_parser_items(name, production, indent, accessor, ctx):
372 for item_idx in range(len(production)):
376 item = production[item_idx]
377 if item.type_ == 'constant':
378 print """%sif (!spvbin_match_bytes (input, %s, %d))
380 indent, bytes_to_hex(item.content), len(item.content),
382 ctx.need_error_handler = True
384 print "%sp->%s = true;" % (indent, item.name)
385 elif item.type_ == 'variable':
386 if item.content[0] == 'nonterminal':
387 func = '%sparse_%s' % (prefix, name_to_id(item.content[1]))
389 func = 'spvbin_parse_%s' % item.content[0]
392 dst = "&p->%s%s" % (accessor, name_to_id(item.name))
396 print """%sif (!%s (input, %s))
397 %s goto %s;""" % (indent, func, dst,
400 if item.content[0] != 'nonterminal' and item.name == 'version':
401 print "%sinput->version = p->%s%s;" % (
402 indent, accessor, name_to_id(item.name))
404 if isinstance(item.n, int):
407 count = 'p->%s%s' % (accessor, name_to_id(item.n))
409 i_name = ctx.gen_name('i')
411 if not isinstance(item.n, int):
412 print "%sp->%s%s = xcalloc (%s, sizeof *p->%s%s);" % (
414 accessor, name_to_id(item.name), count,
415 accessor, name_to_id(item.name))
416 dst += '[%s]' % i_name
417 print "%sfor (int %s = 0; %s < %s; %s++)" % (
418 indent, i_name, i_name, count, i_name)
419 print """%s if (!%s (input, %s))
420 %s goto %s;""" % (indent, func, dst,
423 ctx.need_error_handler = True
424 elif item.type_ == '()':
426 # Not yet implemented
429 print_parser_items(name, item.content, indent, accessor, ctx)
430 elif item.type_ in ('v1', 'v3', 'vAF', 'vB0'):
432 # Not yet implemented
435 print "%sif (input->version == 0x%s) {" % (indent, item.type_[1:])
436 print_parser_items(name, item.content, indent + ' ', accessor, ctx)
438 elif item.type_ in ('count', 'becount'):
440 # Not yet implemented
443 pos = ctx.save_pos(indent)
444 endian = 'big' if item.type_ == 'becount' else 'little'
445 limit = ctx.parse_limit(endian, indent)
448 ctx.bail = ctx.gen_name('backtrack')
450 print "%sdo {" % indent
453 and item.content[-1].type_ == 'variable'
454 and item.content[-1].content[0] == '...'):
455 content = item.content[:-1]
458 content = item.content
460 print_parser_items(name, content, indent, accessor, ctx)
463 print "%sinput->ofs = input->size;" % indent
465 print """%sif (!spvbin_input_at_end (input))
466 %s goto %s;""" % (indent,
468 print '%sspvbin_limit_pop (&%s, input);' % (indent, limit)
469 print '%sbreak;' % indent
471 print '%s%s:' % (indent[4:], ctx.bail)
472 # In theory, we should emit code to clear whatever we're
473 # backtracking from. In practice, it's not important to
475 print "%sspvbin_position_restore (&%s, input);" % (indent, pos)
476 print '%sspvbin_limit_pop (&%s, input);' % (indent, limit)
477 print '%sgoto %s;' % (indent, save_bail)
479 print "%s} while (0);" % indent
482 elif item.type_ == '|':
485 print "%sdo {" % indent
487 pos = ctx.save_pos(indent)
488 error = ctx.save_error(indent)
490 for choice in item.content:
492 print "%sspvbin_position_restore (&%s, input);" % (indent, pos)
493 print "%sinput->n_errors = %s;" % (indent, error)
496 if i != len(item.content):
497 ctx.bail = ctx.gen_name('backtrack')
500 print_parser_items(name, choice, indent, accessor, ctx)
501 print "%sbreak;" % indent
502 if i != len(item.content):
504 print '%s%s:' % (indent[4:], ctx.bail)
505 # In theory, we should emit code to clear whatever we're
506 # backtracking from. In practice, it's not important to
509 print "%s} while (0);" % indent
510 elif item.type_ == 'case':
512 for choice_name, choice in sorted(item.content.items()):
513 if choice_name.endswith('else'):
514 print "%s} else {" % indent
515 print "%s p->%s%s = -1;" % (indent, accessor, item.name)
518 print "%s%sif (spvbin_match_byte (input, 0x%s)) {" % (
519 indent, '} else ' if i else '', choice_name[-2:])
520 print "%s p->%s%s = 0x%s;" % (
521 indent, accessor, item.name, choice_name[-2:])
525 print_parser_items(name, choice, indent + ' ',
526 accessor + choice_name + '.', ctx)
534 def print_parser(name, production, indent):
537 %(prefix)sparse_%(name)s (struct spvbin_input *input, struct %(prefix)s%(name)s **p_)
540 struct %(prefix)s%(name)s *p = xzalloc (sizeof *p);
541 p->start = input->ofs;
542 ''' % {'prefix': prefix,
543 'name': name_to_id(name)}
545 ctx = Parser_Context()
546 print_parser_items(name, production, indent, '', ctx)
549 p->len = input->ofs - p->start;
553 if ctx.need_error_handler:
556 spvbin_error (input, "%s", p->start);
558 return false;""" % (name, prefix, name_to_id(name))
562 def print_free_items(name, production, indent, accessor, ctx):
563 for item in production:
564 if item.type_ == 'constant':
566 elif item.type_ == 'variable':
570 if item.content[0] == 'nonterminal':
571 free_func = '%sfree_%s' % (prefix, name_to_id(item.content[1]))
572 elif item.content[0] in ('string', 'bestring', '...'):
577 dst = "p->%s%s" % (accessor, name_to_id(item.name))
581 print "%s%s (%s);" % (indent, free_func, dst)
583 if isinstance(item.n, int):
586 count = 'p->%s%s' % (accessor, name_to_id(item.n))
588 i_name = ctx.gen_name('i')
590 print "%sfor (int %s = 0; %s < %s; %s++)" % (
591 indent, i_name, i_name, count, i_name)
592 print "%s %s (%s[%s]);" % (
593 indent, free_func, dst, i_name)
594 if not isinstance(item.n, int):
595 print "%sfree (p->%s%s);" % (
596 indent, accessor, name_to_id(item.name))
597 elif item.type_ in ('()', 'v1', 'v3', 'vAF', 'vB0',
600 # Not yet implemented
603 print_free_items(name, item.content, indent, accessor, ctx)
604 elif item.type_ == '|':
605 for choice in item.content:
606 print_free_items(name, choice, indent, accessor, ctx)
607 elif item.type_ == 'case':
609 for choice_name, choice in sorted(item.content.items()):
610 if choice_name.endswith('else'):
613 value_name = '0x%s' % choice_name[-2:]
615 print '%s%sif (p->%s%s == %s) {' % (
616 indent, '} else ' if i else '', accessor, item.name,
619 print_free_items(name, choice, indent + ' ',
620 accessor + choice_name + '.', ctx)
627 def print_free(name, production, indent):
630 %(prefix)sfree_%(name)s (struct %(prefix)s%(name)s *p)
634 ''' % {'prefix': prefix,
635 'name': name_to_id(name)}
637 print_free_items(name, production, indent, '', Parser_Context())
642 def print_print_items(name, production, indent, accessor, ctx):
643 for item_idx in range(len(production)):
647 item = production[item_idx]
648 if item.type_ == 'constant':
650 print '%sspvbin_print_presence ("%s", indent + 1, p->%s);' % (
651 indent, item.name, item.name)
652 elif item.type_ == 'variable':
656 if item.content[0] == 'nonterminal':
657 func = '%sprint_%s' % (prefix, name_to_id(item.content[1]))
659 c_types = {'bool': 'bool',
668 'bestring': 'string',
671 '...': ('uint8_t', 1)}
672 func = 'spvbin_print_%s' % c_types[item.content[0]]
674 dst = "p->%s%s" % (accessor, name_to_id(item.name))
676 print '%s%s ("%s", indent + 1, %s);' % (indent, func,
679 if isinstance(item.n, int):
682 count = 'p->%s%s' % (accessor, name_to_id(item.n))
684 i_name = ctx.gen_name('i')
685 elem_name = ctx.gen_name('elem_name')
686 dst += '[%s]' % i_name
688 %(indent)sfor (int %(index)s = 0; %(index)s < %(count)s; %(index)s++) {
689 %(indent)s char *%(elem_name)s = xasprintf ("%(item.name)s[%%d]", %(index)s);
690 %(indent)s %(func)s (%(elem_name)s, indent + 1, %(dst)s);
691 %(indent)s free (%(elem_name)s);
692 %(indent)s}""" % {'indent': indent,
695 'elem_name' : elem_name,
696 'item.name': item.name,
699 elif item.type_ == '()':
701 # Not yet implemented
704 print_print_items(name, item.content, indent, accessor, ctx)
705 elif item.type_ in ('v1', 'v3', 'vAF', 'vB0'):
707 # Not yet implemented
710 print_print_items(name, item.content, indent, accessor, ctx)
711 elif item.type_ in ('count', 'becount'):
713 # Not yet implemented
718 and item.content[-1].type_ == 'variable'
719 and item.content[-1].content[0] == '...'):
720 content = item.content[:-1]
722 content = item.content
723 print_print_items(name, content, indent, accessor, ctx)
724 elif item.type_ == '|':
725 for choice in item.content:
726 print_print_items(name, choice, indent, accessor, ctx)
727 elif item.type_ == 'case':
730 %sspvbin_print_case ("%s", indent + 1, p->%s%s);""" % (
731 indent, item.name, accessor, name_to_id(item.name))
732 for choice_name, choice in sorted(item.content.items()):
733 if choice_name.endswith('else'):
736 value_name = '0x%s' % choice_name[-2:]
738 print '%s%sif (p->%s%s == %s) {' % (
739 indent, '} else ' if i else '', accessor, item.name,
742 print_print_items(name, choice, indent + ' ',
743 accessor + choice_name + '.', ctx)
751 def print_print(name, production, indent):
754 %(prefix)sprint_%(name)s (const char *title, int indent, const struct %(prefix)s%(name)s *p)
756 spvbin_print_header (title, p ? p->start : -1, p ? p->len : -1, indent);
762 ''' % {'prefix': prefix,
764 'name': name_to_id(name)}
766 ctx = Parser_Context()
767 print_print_items(name, production, indent, '', ctx)
772 return s[0].lower() + ''.join(['_%c' % x.lower() if x.isupper() else x
773 for x in s[1:]]).replace('-', '_')
776 if __name__ == "__main__":
779 options, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['help'])
780 except getopt.GetoptError as e:
781 sys.stderr.write("%s: %s\n" % (argv0, e.msg))
784 for key, value in options:
785 if key in ['-h', '--help']:
791 sys.stderr.write("%s: bad usage (use --help for help)\n" % argv0)
795 file_name, output_type, prefix = args[:3]
796 input_file = open(file_name)
798 prefix = '%s_' % prefix
813 if token[0] == 'eof':
816 name, production = parse_production()
817 if name in productions:
818 fatal("%s: duplicate production" % name)
819 productions[name] = production
821 print '/* Generated automatically -- do not modify! -*- buffer-read-only: t -*- */'
822 if output_type == 'code' and len(args) == 4:
823 header_name = args[3]
830 #include "libpspp/str.h"
831 #include "gl/xalloc.h"\
833 for name, production in productions.items():
834 print_parser(name, production, ' ' * 4)
835 print_free(name, production, ' ' * 4)
836 print_print(name, production, ' ' * 4)
837 elif output_type == 'header' and len(args) == 3:
839 #ifndef %(PREFIX)sPARSER_H
840 #define %(PREFIX)sPARSER_H
845 #include "output/spv/spvbin-helpers.h"\
846 """ % {'PREFIX': prefix.upper()}
847 for name, production in productions.items():
848 print '\nstruct %s%s {' % (prefix, name_to_id(name))
849 print " size_t start, len;"
850 print_members(production, ' ' * 4)
852 bool %(prefix)sparse_%(name)s (struct spvbin_input *, struct %(prefix)s%(name)s **);
853 void %(prefix)sfree_%(name)s (struct %(prefix)s%(name)s *);
854 void %(prefix)sprint_%(name)s (const char *title, int indent, const struct %(prefix)s%(name)s *);\
855 ''' % {'prefix': prefix,
856 'name': name_to_id(name)}
859 #endif /* %(PREFIX)sPARSER_H */""" % {'PREFIX': prefix.upper()}
861 sys.stderr.write("%s: bad usage (use --help for help)" % argv0)