1#!/usr/local/bin/python3.8 2 3license = '''/** 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, software 15 * distributed under the License is distributed on an "AS IS" BASIS, 16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 * See the License for the specific language governing permissions and 18 * limitations under the License. 19 */ 20''' 21 22headers = ''' 23#include <stdint.h> 24#include <string> 25#include <vector> 26#include <map> 27#include "Boost.hh" 28#include "Exception.hh" 29#include "AvroSerialize.hh" 30#include "AvroParse.hh" 31#include "Layout.hh" 32''' 33 34done = False 35 36typeToC = { 37 'int': 'int32_t', 38 'long': 'int64_t', 39 'float': 'float', 40 'double': 'double', 41 42 'boolean': 'bool', 'null': 'internal_avro::Null', 'string': 'std::string', 'bytes': 'std::vector<uint8_t>'} 43 44structList = [] 45structNames = {} 46forwardDeclareList = [] 47 48 49def addStruct(name, declaration): 50 if name not in structNames: 51 structNames[name] = True 52 structList.append(declaration) 53 54 55def addForwardDeclare(declaration): 56 code = 'struct ' + declaration + ';' 57 forwardDeclareList.append(code) 58 59 60def doPrimitive(type): 61 return (typeToC[type], type) 62 63 64def doSymbolic(args): 65 addForwardDeclare(args[1]) 66 return (args[1], args[1]) 67 68 69def addLayout(name, type, var): 70 result = ' add(new $offsetType$(offset + offsetof($name$, $var$)));\n' 71 result = result.replace('$name$', name) 72 if type in typeToC: 73 offsetType = 'internal_avro::PrimitiveLayout' 74 else: 75 offsetType = type + '_Layout' 76 result = result.replace('$offsetType$', offsetType) 77 result = result.replace('$var$', var) 78 return result 79 80 81def addSimpleLayout(type): 82 result = ' add(new $offsetType$);\n' 83 if type in typeToC: 84 offsetType = 'internal_avro::PrimitiveLayout' 85 else: 86 offsetType = type + '_Layout' 87 return result.replace('$offsetType$', offsetType) 88 89recordfieldTemplate = '$type$ $name$\n' 90recordTemplate = '''struct $name$ { 91 92 $name$ () : 93$initializers$ 94 { } 95 96$recordfields$}; 97 98template <typename Serializer> 99inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 100 s.writeRecord(); 101$serializefields$ s.writeRecordEnd(); 102} 103 104template <typename Parser> 105inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 106 p.readRecord(); 107$parsefields$ p.readRecordEnd(); 108} 109 110class $name$_Layout : public internal_avro::CompoundLayout { 111 public: 112 $name$_Layout(size_t offset = 0) : 113 CompoundLayout(offset) 114 { 115$offsetlist$ } 116}; 117''' 118 119 120def doRecord(args): 121 structDef = recordTemplate 122 typename = args[1] 123 structDef = structDef.replace('$name$', typename) 124 fields = '' 125 serializefields = '' 126 parsefields = '' 127 initlist = '' 128 offsetlist = '' 129 end = False 130 while not end: 131 line = getNextLine() 132 if line[0] == 'end': 133 end = True 134 initlist = initlist.rstrip(',\n') 135 elif line[0] == 'name': 136 fieldname = line[1] 137 fieldline = getNextLine() 138 fieldtypename, fieldtype = processType(fieldline) 139 fields += ' ' + fieldtypename + ' ' + fieldname + ';\n' 140 serializefields += ' serialize(s, val.' + fieldname + ');\n' 141 initlist += ' ' + fieldname + '(),\n' 142 parsefields += ' parse(p, val.' + fieldname + ');\n' 143 offsetlist += addLayout(typename, fieldtype, fieldname) 144 structDef = structDef.replace('$initializers$', initlist) 145 structDef = structDef.replace('$recordfields$', fields) 146 structDef = structDef.replace('$serializefields$', serializefields) 147 structDef = structDef.replace('$parsefields$', parsefields) 148 structDef = structDef.replace('$offsetlist$', offsetlist) 149 addStruct(typename, structDef) 150 return (typename, typename) 151 152uniontypestemplate = 'typedef $type$ Choice$N$Type' 153unionTemplate = '''struct $name$ { 154 155$typedeflist$ 156 typedef void* (*GenericSetter)($name$ *, int64_t); 157 158 $name$() : 159 choice(0), 160 value(T0()), 161 genericSetter(&$name$::genericSet) 162 { } 163 164$setfuncs$ 165#ifdef AVRO_BOOST_NO_ANYREF 166 template<typename T> 167 const T &getValue() const { 168 const T *ptr = boost::any_cast<T>(&value); 169 return *ptr; 170 } 171#else 172 template<typename T> 173 const T &getValue() const { 174 return boost::any_cast<const T&>(value); 175 } 176#endif 177 178 static void *genericSet($name$ *u, int64_t choice) { 179 boost::any *val = &(u->value); 180 void *data = NULL; 181 switch (choice) {$switch$ 182 } 183 return data; 184 } 185 186 int64_t choice; 187 boost::any value; 188 GenericSetter genericSetter; 189}; 190 191template <typename Serializer> 192inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 193 s.writeUnion(val.choice); 194 switch(val.choice) { 195$switchserialize$ default : 196 throw internal_avro::Exception("Unrecognized union choice"); 197 } 198} 199 200template <typename Parser> 201inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 202 val.choice = p.readUnion(); 203 switch(val.choice) { 204$switchparse$ default : 205 throw internal_avro::Exception("Unrecognized union choice"); 206 } 207} 208 209class $name$_Layout : public internal_avro::CompoundLayout { 210 public: 211 $name$_Layout(size_t offset = 0) : 212 CompoundLayout(offset) 213 { 214 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, choice))); 215 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, genericSetter))); 216$offsetlist$ } 217}; 218''' 219 220unionser = ' case $choice$:\n serialize(s, val.getValue< $type$ >());\n break;\n' 221unionpar = ' case $choice$:\n { $type$ chosenVal; parse(p, chosenVal); val.value = chosenVal; }\n break;\n' 222 223setfunc = ''' void set_$name$(const $type$ &val) { 224 choice = $N$; 225 value = val; 226 };\n''' 227 228switcher = '''\n case $N$: 229 *val = T$N$(); 230 data = boost::any_cast<T$N$>(val); 231 break;''' 232 233 234def doUnion(args): 235 structDef = unionTemplate 236 uniontypes = '' 237 switchserialize = '' 238 switchparse = '' 239 typename = 'Union_of' 240 setters = '' 241 switches = '' 242 offsetlist = '' 243 i = 0 244 end = False 245 while not end: 246 line = getNextLine() 247 if line[0] == 'end': 248 end = True 249 else: 250 uniontype, name = processType(line) 251 typename += '_' + name 252 uniontypes += ' ' + 'typedef ' + \ 253 uniontype + ' T' + str(i) + ';\n' 254 switch = unionser 255 switch = switch.replace('$choice$', str(i)) 256 switch = switch.replace('$type$', uniontype) 257 switchserialize += switch 258 switch = unionpar 259 switch = switch.replace('$choice$', str(i)) 260 switch = switch.replace('$type$', uniontype) 261 switchparse += switch 262 setter = setfunc 263 setter = setter.replace('$name$', name) 264 setter = setter.replace('$type$', uniontype) 265 setter = setter.replace('$N$', str(i)) 266 setters += setter 267 switch = switcher 268 switches += switch.replace('$N$', str(i)) 269 offsetlist += addSimpleLayout(name) 270 i += 1 271 structDef = structDef.replace('$name$', typename) 272 structDef = structDef.replace('$typedeflist$', uniontypes) 273 structDef = structDef.replace('$switchserialize$', switchserialize) 274 structDef = structDef.replace('$switchparse$', switchparse) 275 structDef = structDef.replace('$setfuncs$', setters) 276 structDef = structDef.replace('$switch$', switches) 277 structDef = structDef.replace('$offsetlist$', offsetlist) 278 addStruct(typename, structDef) 279 return (typename, typename) 280 281enumTemplate = '''struct $name$ { 282 283 enum EnumSymbols { 284 $enumsymbols$ 285 }; 286 287 $name$() : 288 value($firstsymbol$) 289 { } 290 291 EnumSymbols value; 292}; 293 294template <typename Serializer> 295inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 296 s.writeEnum(val.value); 297} 298 299template <typename Parser> 300inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 301 val.value = static_cast<$name$::EnumSymbols>(p.readEnum()); 302} 303 304class $name$_Layout : public internal_avro::CompoundLayout { 305 public: 306 $name$_Layout(size_t offset = 0) : 307 CompoundLayout(offset) 308 { 309 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, value))); 310 } 311}; 312''' 313 314 315def doEnum(args): 316 structDef = enumTemplate 317 typename = args[1] 318 structDef = structDef.replace('$name$', typename) 319 end = False 320 symbols = '' 321 firstsymbol = '' 322 while not end: 323 line = getNextLine() 324 if line[0] == 'end': 325 end = True 326 elif line[0] == 'name': 327 if symbols == '': 328 firstsymbol = line[1] 329 else: 330 symbols += ', ' 331 symbols += line[1] 332 else: 333 print "error" 334 structDef = structDef.replace('$enumsymbols$', symbols) 335 structDef = structDef.replace('$firstsymbol$', firstsymbol) 336 addStruct(typename, structDef) 337 return (typename, typename) 338 339arrayTemplate = '''struct $name$ { 340 typedef $valuetype$ ValueType; 341 typedef std::vector<ValueType> ArrayType; 342 typedef ValueType* (*GenericSetter)($name$ *); 343 344 $name$() : 345 value(), 346 genericSetter(&$name$::genericSet) 347 { } 348 349 static ValueType *genericSet($name$ *array) { 350 array->value.push_back(ValueType()); 351 return &array->value.back(); 352 } 353 354 void addValue(const ValueType &val) { 355 value.push_back(val); 356 } 357 358 ArrayType value; 359 GenericSetter genericSetter; 360 361}; 362 363template <typename Serializer> 364inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 365 const size_t size = val.value.size(); 366 if(size) { 367 s.writeArrayBlock(size); 368 for(size_t i = 0; i < size; ++i) { 369 serialize(s, val.value[i]); 370 } 371 } 372 s.writeArrayEnd(); 373} 374 375template <typename Parser> 376inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 377 val.value.clear(); 378 while(1) { 379 int size = p.readArrayBlockSize(); 380 if(size > 0) { 381 val.value.reserve(val.value.size() + size); 382 while (size-- > 0) { 383 val.value.push_back($name$::ValueType()); 384 parse(p, val.value.back()); 385 } 386 } 387 else { 388 break; 389 } 390 } 391} 392 393class $name$_Layout : public internal_avro::CompoundLayout { 394 public: 395 $name$_Layout(size_t offset = 0) : 396 CompoundLayout(offset) 397 { 398 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, genericSetter))); 399$offsetlist$ } 400}; 401''' 402 403 404def doArray(args): 405 structDef = arrayTemplate 406 line = getNextLine() 407 arraytype, typename = processType(line) 408 offsetlist = addSimpleLayout(typename) 409 typename = 'Array_of_' + typename 410 411 structDef = structDef.replace('$name$', typename) 412 structDef = structDef.replace('$valuetype$', arraytype) 413 structDef = structDef.replace('$offsetlist$', offsetlist) 414 415 line = getNextLine() 416 if line[0] != 'end': 417 print 'error' 418 419 addStruct(typename, structDef) 420 return (typename, typename) 421 422mapTemplate = '''struct $name$ { 423 typedef $valuetype$ ValueType; 424 typedef std::map<std::string, ValueType> MapType; 425 typedef ValueType* (*GenericSetter)($name$ *, const std::string &); 426 427 $name$() : 428 value(), 429 genericSetter(&$name$::genericSet) 430 { } 431 432 void addValue(const std::string &key, const ValueType &val) { 433 value.insert(MapType::value_type(key, val)); 434 } 435 436 static ValueType *genericSet($name$ *map, const std::string &key) { 437 map->value[key] = ValueType(); 438 return &(map->value[key]); 439 } 440 441 MapType value; 442 GenericSetter genericSetter; 443 444}; 445 446template <typename Serializer> 447inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 448 if(val.value.size()) { 449 s.writeMapBlock(val.value.size()); 450 $name$::MapType::const_iterator iter = val.value.begin(); 451 $name$::MapType::const_iterator end = val.value.end(); 452 while(iter!=end) { 453 serialize(s, iter->first); 454 serialize(s, iter->second); 455 ++iter; 456 } 457 } 458 s.writeMapEnd(); 459} 460 461template <typename Parser> 462inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 463 val.value.clear(); 464 while(1) { 465 int size = p.readMapBlockSize(); 466 if(size > 0) { 467 while (size-- > 0) { 468 std::string key; 469 parse(p, key); 470 $name$::ValueType m; 471 parse(p, m); 472 val.value.insert($name$::MapType::value_type(key, m)); 473 } 474 } 475 else { 476 break; 477 } 478 } 479} 480 481class $name$_Layout : public internal_avro::CompoundLayout { 482 public: 483 $name$_Layout(size_t offset = 0) : 484 CompoundLayout(offset) 485 { 486 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, genericSetter))); 487$offsetlist$ } 488}; 489''' 490 491 492def doMap(args): 493 structDef = mapTemplate 494 line = getNextLine() # must be string 495 line = getNextLine() 496 maptype, typename = processType(line) 497 498 offsetlist = addSimpleLayout(typename) 499 typename = 'Map_of_' + typename 500 501 structDef = structDef.replace('$name$', typename) 502 structDef = structDef.replace('$valuetype$', maptype) 503 structDef = structDef.replace('$offsetlist$', offsetlist) 504 505 line = getNextLine() 506 if line[0] != 'end': 507 print 'error' 508 addStruct(typename, structDef) 509 return (typename, typename) 510 511fixedTemplate = '''struct $name$ { 512 enum { 513 fixedSize = $N$ 514 }; 515 516 $name$() { 517 memset(value, 0, sizeof(value)); 518 } 519 520 uint8_t value[fixedSize]; 521}; 522 523template <typename Serializer> 524inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 525 s.writeFixed(val.value); 526} 527 528template <typename Parser> 529inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 530 p.readFixed(val.value); 531} 532 533class $name$_Layout : public internal_avro::CompoundLayout { 534 public: 535 $name$_Layout(size_t offset = 0) : 536 CompoundLayout(offset) 537 { 538 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, value))); 539 } 540}; 541''' 542 543 544def doFixed(args): 545 structDef = fixedTemplate 546 typename = args[1] 547 size = args[2] 548 549 line = getNextLine() 550 if line[0] != 'end': 551 print 'error' 552 553 structDef = structDef.replace('$name$', typename) 554 structDef = structDef.replace('$N$', size) 555 addStruct(typename, structDef) 556 return (typename, typename) 557 558primitiveTemplate = '''struct $name$ { 559 $type$ value; 560}; 561 562template <typename Serializer> 563inline void serialize(Serializer &s, const $name$ &val, const boost::true_type &) { 564 s.writeValue(val.value); 565} 566 567template <typename Parser> 568inline void parse(Parser &p, $name$ &val, const boost::true_type &) { 569 p.readValue(val.value); 570} 571 572class $name$_Layout : public internal_avro::CompoundLayout { 573 public: 574 $name$_Layout(size_t offset = 0) : 575 CompoundLayout(offset) 576 { 577 add(new internal_avro::PrimitiveLayout(offset + offsetof($name$, value))); 578 } 579}; 580''' 581 582 583def doPrimitiveStruct(type): 584 structDef = primitiveTemplate 585 name = type.capitalize() 586 structDef = structDef.replace('$name$', name) 587 structDef = structDef.replace('$type$', typeToC[type]) 588 addStruct(name, structDef) 589 590compoundBuilder = {'record': doRecord, 'union': doUnion, 'enum': doEnum, 591 'map': doMap, 'array': doArray, 'fixed': doFixed, 'symbolic': doSymbolic} 592 593 594def processType(inputs): 595 type = inputs[0] 596 if type in typeToC: 597 result = doPrimitive(type) 598 else: 599 func = compoundBuilder[type] 600 result = func(inputs) 601 return result 602 603 604def generateCode(): 605 inputs = getNextLine() 606 type = inputs[0] 607 if type in typeToC: 608 doPrimitiveStruct(type) 609 else: 610 func = compoundBuilder[type] 611 func(inputs) 612 613 614def getNextLine(): 615 try: 616 line = raw_input() 617 except: 618 line = '' 619 globals()["done"] = True 620 621 if line == '': 622 globals()["done"] = True 623 return line.split(' ') 624 625 626def writeHeader(filebase, namespace): 627 headerstring = "%s_%s_hh__" % (namespace, filebase) 628 629 print license 630 print "#ifndef %s" % headerstring 631 print "#define %s" % headerstring 632 print headers 633 print "namespace %s {\n" % namespace 634 635 for x in forwardDeclareList: 636 print "%s\n" % x 637 638 for x in structList: 639 print "/*----------------------------------------------------------------------------------*/\n" 640 print "%s\n" % x 641 642 print "\n} // namespace %s\n" % namespace 643 644 print "namespace internal_avro {\n" 645 for x in structNames: 646 print 'template <> struct is_serializable<%s::%s> : public boost::true_type{};' % (namespace, x) 647 648 print "\n} // namespace internal_avro\n" 649 650 print "#endif // %s" % headerstring 651 652 653def usage(): 654 print "-h, --help print this helpful message" 655 print "-i, --input=FILE input file to read (default is stdin)" 656 print "-o, --output=PATH output file to generate (default is stdout)" 657 print "-n, --namespace=LABEL namespace for schema (default is avrouser)" 658 659if __name__ == "__main__": 660 from sys import argv 661 import getopt 662 import sys 663 664 try: 665 opts, args = getopt.getopt( 666 argv[1:], "hi:o:n:", ["help", "input=", "output=", "namespace="]) 667 668 except getopt.GetoptError as err: 669 print str(err) 670 usage() 671 sys.exit(2) 672 673 namespace = 'avrouser' 674 675 savein = sys.stdin 676 saveout = sys.stdout 677 inputFile = False 678 outputFile = False 679 outputFileBase = 'AvroGenerated' 680 681 for o, a in opts: 682 if o in ("-i", "--input"): 683 try: 684 inputFile = open(a, 'r') 685 sys.stdin = inputFile 686 except: 687 print "Could not open file " + a 688 sys.exit() 689 elif o in ("-o", "--output"): 690 try: 691 outputFile = open(a, 'w') 692 sys.stdout = outputFile 693 except: 694 print "Could not open file " + a 695 outputFileBase = a.rstrip('.hp') # strip for .h, .hh, .hpp 696 elif o in ("-n", "--namespace"): 697 namespace = a 698 elif o in ("-h", "--help"): 699 usage() 700 sys.exit() 701 else: 702 print "Unhandled option: " + o 703 usage() 704 sys.exit() 705 706 generateCode() 707 writeHeader(outputFileBase, namespace) 708 709 sys.stdin = savein 710 sys.stdout = saveout 711 if inputFile: 712 inputFile.close() 713 if outputFile: 714 outputFile.close() 715