1#! /usr/bin/env python 2"""Generate C code from an ASDL description.""" 3 4# TO DO 5# handle fields that have a type but no name 6 7import os, sys 8 9import asdl 10 11TABSIZE = 8 12MAX_COL = 100 13 14def get_c_type(name): 15 """Return a string for the C name of the type. 16 17 This function special cases the default types provided by asdl: 18 identifier, string, int, bool. 19 """ 20 # XXX ack! need to figure out where Id is useful and where string 21 if isinstance(name, asdl.Id): 22 name = name.value 23 if name in asdl.builtin_types: 24 return name 25 else: 26 return "%s_ty" % name 27 28def reflow_lines(s, depth): 29 """Reflow the line s indented depth tabs. 30 31 Return a sequence of lines where no line extends beyond MAX_COL 32 when properly indented. The first line is properly indented based 33 exclusively on depth * TABSIZE. All following lines -- these are 34 the reflowed lines generated by this function -- start at the same 35 column as the first character beyond the opening { in the first 36 line. 37 """ 38 size = MAX_COL - depth * TABSIZE 39 if len(s) < size: 40 return [s] 41 42 lines = [] 43 cur = s 44 padding = "" 45 while len(cur) > size: 46 i = cur.rfind(' ', 0, size) 47 # XXX this should be fixed for real 48 if i == -1 and 'GeneratorExp' in cur: 49 i = size + 3 50 assert i != -1, "Impossible line %d to reflow: %r" % (size, s) 51 lines.append(padding + cur[:i]) 52 if len(lines) == 1: 53 # find new size based on brace 54 j = cur.find('{', 0, i) 55 if j >= 0: 56 j += 2 # account for the brace and the space after it 57 size -= j 58 padding = " " * j 59 else: 60 j = cur.find('(', 0, i) 61 if j >= 0: 62 j += 1 # account for the paren (no space after it) 63 size -= j 64 padding = " " * j 65 cur = cur[i+1:] 66 else: 67 lines.append(padding + cur) 68 return lines 69 70def is_simple(sum): 71 """Return True if a sum is a simple. 72 73 A sum is simple if its types have no fields, e.g. 74 unaryop = Invert | Not | UAdd | USub 75 """ 76 for t in sum.types: 77 if t.fields: 78 return False 79 return True 80 81 82class EmitVisitor(asdl.VisitorBase): 83 """Visit that emits lines""" 84 85 def __init__(self, file): 86 self.file = file 87 super(EmitVisitor, self).__init__() 88 89 def emit(self, s, depth, reflow=True): 90 # XXX reflow long lines? 91 if reflow: 92 lines = reflow_lines(s, depth) 93 else: 94 lines = [s] 95 for line in lines: 96 line = (" " * TABSIZE * depth) + line + "\n" 97 self.file.write(line) 98 99 100class TypeDefVisitor(EmitVisitor): 101 def visitModule(self, mod): 102 for dfn in mod.dfns: 103 self.visit(dfn) 104 105 def visitType(self, type, depth=0): 106 self.visit(type.value, type.name, depth) 107 108 def visitSum(self, sum, name, depth): 109 if is_simple(sum): 110 self.simple_sum(sum, name, depth) 111 else: 112 self.sum_with_constructors(sum, name, depth) 113 114 def simple_sum(self, sum, name, depth): 115 enum = [] 116 for i in range(len(sum.types)): 117 type = sum.types[i] 118 enum.append("%s=%d" % (type.name, i + 1)) 119 enums = ", ".join(enum) 120 ctype = get_c_type(name) 121 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype) 122 self.emit(s, depth) 123 self.emit("", depth) 124 125 def sum_with_constructors(self, sum, name, depth): 126 ctype = get_c_type(name) 127 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 128 self.emit(s, depth) 129 self.emit("", depth) 130 131 def visitProduct(self, product, name, depth): 132 ctype = get_c_type(name) 133 s = "typedef struct _%(name)s *%(ctype)s;" % locals() 134 self.emit(s, depth) 135 self.emit("", depth) 136 137 138class StructVisitor(EmitVisitor): 139 """Visitor to generate typdefs for AST.""" 140 141 def visitModule(self, mod): 142 for dfn in mod.dfns: 143 self.visit(dfn) 144 145 def visitType(self, type, depth=0): 146 self.visit(type.value, type.name, depth) 147 148 def visitSum(self, sum, name, depth): 149 if not is_simple(sum): 150 self.sum_with_constructors(sum, name, depth) 151 152 def sum_with_constructors(self, sum, name, depth): 153 def emit(s, depth=depth): 154 self.emit(s % sys._getframe(1).f_locals, depth) 155 enum = [] 156 for i in range(len(sum.types)): 157 type = sum.types[i] 158 enum.append("%s_kind=%d" % (type.name, i + 1)) 159 160 emit("enum _%(name)s_kind {" + ", ".join(enum) + "};") 161 162 emit("struct _%(name)s {") 163 emit("enum _%(name)s_kind kind;", depth + 1) 164 emit("union {", depth + 1) 165 for t in sum.types: 166 self.visit(t, depth + 2) 167 emit("} v;", depth + 1) 168 for field in sum.attributes: 169 # rudimentary attribute handling 170 type = str(field.type) 171 assert type in asdl.builtin_types, type 172 emit("%s %s;" % (type, field.name), depth + 1); 173 emit("};") 174 emit("") 175 176 def visitConstructor(self, cons, depth): 177 if cons.fields: 178 self.emit("struct {", depth) 179 for f in cons.fields: 180 self.visit(f, depth + 1) 181 self.emit("} %s;" % cons.name, depth) 182 self.emit("", depth) 183 else: 184 # XXX not sure what I want here, nothing is probably fine 185 pass 186 187 def visitField(self, field, depth): 188 # XXX need to lookup field.type, because it might be something 189 # like a builtin... 190 ctype = get_c_type(field.type) 191 name = field.name 192 if field.seq: 193 if field.type.value in ('cmpop',): 194 self.emit("asdl_int_seq *%(name)s;" % locals(), depth) 195 else: 196 self.emit("asdl_seq *%(name)s;" % locals(), depth) 197 else: 198 self.emit("%(ctype)s %(name)s;" % locals(), depth) 199 200 def visitProduct(self, product, name, depth): 201 self.emit("struct _%(name)s {" % locals(), depth) 202 for f in product.fields: 203 self.visit(f, depth + 1) 204 self.emit("};", depth) 205 self.emit("", depth) 206 207 208class PrototypeVisitor(EmitVisitor): 209 """Generate function prototypes for the .h file""" 210 211 def visitModule(self, mod): 212 for dfn in mod.dfns: 213 self.visit(dfn) 214 215 def visitType(self, type): 216 self.visit(type.value, type.name) 217 218 def visitSum(self, sum, name): 219 if is_simple(sum): 220 pass # XXX 221 else: 222 for t in sum.types: 223 self.visit(t, name, sum.attributes) 224 225 def get_args(self, fields): 226 """Return list of C argument into, one for each field. 227 228 Argument info is 3-tuple of a C type, variable name, and flag 229 that is true if type can be NULL. 230 """ 231 args = [] 232 unnamed = {} 233 for f in fields: 234 if f.name is None: 235 name = f.type 236 c = unnamed[name] = unnamed.get(name, 0) + 1 237 if c > 1: 238 name = "name%d" % (c - 1) 239 else: 240 name = f.name 241 # XXX should extend get_c_type() to handle this 242 if f.seq: 243 if f.type.value in ('cmpop',): 244 ctype = "asdl_int_seq *" 245 else: 246 ctype = "asdl_seq *" 247 else: 248 ctype = get_c_type(f.type) 249 args.append((ctype, name, f.opt or f.seq)) 250 return args 251 252 def visitConstructor(self, cons, type, attrs): 253 args = self.get_args(cons.fields) 254 attrs = self.get_args(attrs) 255 ctype = get_c_type(type) 256 self.emit_function(cons.name, ctype, args, attrs) 257 258 def emit_function(self, name, ctype, args, attrs, union=True): 259 args = args + attrs 260 if args: 261 argstr = ", ".join(["%s %s" % (atype, aname) 262 for atype, aname, opt in args]) 263 argstr += ", PyArena *arena" 264 else: 265 argstr = "PyArena *arena" 266 margs = "a0" 267 for i in range(1, len(args)+1): 268 margs += ", a%d" % i 269 self.emit("#define %s(%s) _Ta27_%s(%s)" % (name, margs, name, margs), 0, 270 reflow=False) 271 self.emit("%s _Ta27_%s(%s);" % (ctype, name, argstr), False) 272 273 def visitProduct(self, prod, name): 274 self.emit_function(name, get_c_type(name), 275 self.get_args(prod.fields), [], union=False) 276 277 278class FunctionVisitor(PrototypeVisitor): 279 """Visitor to generate constructor functions for AST.""" 280 281 def emit_function(self, name, ctype, args, attrs, union=True): 282 def emit(s, depth=0, reflow=True): 283 self.emit(s, depth, reflow) 284 argstr = ", ".join(["%s %s" % (atype, aname) 285 for atype, aname, opt in args + attrs]) 286 if argstr: 287 argstr += ", PyArena *arena" 288 else: 289 argstr = "PyArena *arena" 290 self.emit("%s" % ctype, 0) 291 emit("%s(%s)" % (name, argstr)) 292 emit("{") 293 emit("%s p;" % ctype, 1) 294 for argtype, argname, opt in args: 295 # XXX hack alert: false is allowed for a bool 296 if not opt and not (argtype == "bool" or argtype == "int"): 297 emit("if (!%s) {" % argname, 1) 298 emit("PyErr_SetString(PyExc_ValueError,", 2) 299 msg = "field %s is required for %s" % (argname, name) 300 emit(' "%s");' % msg, 301 2, reflow=False) 302 emit('return NULL;', 2) 303 emit('}', 1) 304 305 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1); 306 emit("if (!p)", 1) 307 emit("return NULL;", 2) 308 if union: 309 self.emit_body_union(name, args, attrs) 310 else: 311 self.emit_body_struct(name, args, attrs) 312 emit("return p;", 1) 313 emit("}") 314 emit("") 315 316 def emit_body_union(self, name, args, attrs): 317 def emit(s, depth=0, reflow=True): 318 self.emit(s, depth, reflow) 319 emit("p->kind = %s_kind;" % name, 1) 320 for argtype, argname, opt in args: 321 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1) 322 for argtype, argname, opt in attrs: 323 emit("p->%s = %s;" % (argname, argname), 1) 324 325 def emit_body_struct(self, name, args, attrs): 326 def emit(s, depth=0, reflow=True): 327 self.emit(s, depth, reflow) 328 for argtype, argname, opt in args: 329 emit("p->%s = %s;" % (argname, argname), 1) 330 assert not attrs 331 332 333class PickleVisitor(EmitVisitor): 334 335 def visitModule(self, mod): 336 for dfn in mod.dfns: 337 self.visit(dfn) 338 339 def visitType(self, type): 340 self.visit(type.value, type.name) 341 342 def visitSum(self, sum, name): 343 pass 344 345 def visitProduct(self, sum, name): 346 pass 347 348 def visitConstructor(self, cons, name): 349 pass 350 351 def visitField(self, sum): 352 pass 353 354 355class Obj2ModPrototypeVisitor(PickleVisitor): 356 def visitProduct(self, prod, name): 357 code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);" 358 self.emit(code % (name, get_c_type(name)), 0) 359 360 visitSum = visitProduct 361 362 363class Obj2ModVisitor(PickleVisitor): 364 def funcHeader(self, name): 365 ctype = get_c_type(name) 366 self.emit("int", 0) 367 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 368 self.emit("{", 0) 369 self.emit("PyObject* tmp = NULL;", 1) 370 self.emit("int isinstance;", 1) 371 self.emit("", 0) 372 373 def sumTrailer(self, name): 374 self.emit("", 0) 375 self.emit("tmp = PyObject_Repr(obj);", 1) 376 # there's really nothing more we can do if this fails ... 377 self.emit("if (tmp == NULL) goto failed;", 1) 378 error = "expected some sort of %s, but got %%.400s" % name 379 format = "PyErr_Format(PyExc_TypeError, \"%s\", _PyUnicode_AsString(tmp));" 380 self.emit(format % error, 1, reflow=False) 381 self.emit("failed:", 0) 382 self.emit("Py_XDECREF(tmp);", 1) 383 self.emit("return 1;", 1) 384 self.emit("}", 0) 385 self.emit("", 0) 386 387 def simpleSum(self, sum, name): 388 self.funcHeader(name) 389 for t in sum.types: 390 line = ("isinstance = PyObject_IsInstance(obj, " 391 "(PyObject *)%s_type);") 392 self.emit(line % (t.name,), 1) 393 self.emit("if (isinstance == -1) {", 1) 394 self.emit("return 1;", 2) 395 self.emit("}", 1) 396 self.emit("if (isinstance) {", 1) 397 self.emit("*out = %s;" % t.name, 2) 398 self.emit("return 0;", 2) 399 self.emit("}", 1) 400 self.sumTrailer(name) 401 402 def buildArgs(self, fields): 403 return ", ".join(fields + ["arena"]) 404 405 def complexSum(self, sum, name): 406 self.funcHeader(name) 407 for a in sum.attributes: 408 self.visitAttributeDeclaration(a, name, sum=sum) 409 self.emit("", 0) 410 # XXX: should we only do this for 'expr'? 411 self.emit("if (obj == Py_None) {", 1) 412 self.emit("*out = NULL;", 2) 413 self.emit("return 0;", 2) 414 self.emit("}", 1) 415 for a in sum.attributes: 416 self.visitField(a, name, sum=sum, depth=1) 417 for t in sum.types: 418 line = "isinstance = PyObject_IsInstance(obj, (PyObject*)%s_type);" 419 self.emit(line % (t.name,), 1) 420 self.emit("if (isinstance == -1) {", 1) 421 self.emit("return 1;", 2) 422 self.emit("}", 1) 423 self.emit("if (isinstance) {", 1) 424 for f in t.fields: 425 self.visitFieldDeclaration(f, t.name, sum=sum, depth=2) 426 self.emit("", 0) 427 for f in t.fields: 428 self.visitField(f, t.name, sum=sum, depth=2) 429 args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes] 430 self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2) 431 self.emit("if (*out == NULL) goto failed;", 2) 432 self.emit("return 0;", 2) 433 self.emit("}", 1) 434 self.sumTrailer(name) 435 436 def visitAttributeDeclaration(self, a, name, sum=sum): 437 ctype = get_c_type(a.type) 438 self.emit("%s %s;" % (ctype, a.name), 1) 439 440 def visitSum(self, sum, name): 441 if is_simple(sum): 442 self.simpleSum(sum, name) 443 else: 444 self.complexSum(sum, name) 445 446 def visitProduct(self, prod, name): 447 ctype = get_c_type(name) 448 self.emit("int", 0) 449 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) 450 self.emit("{", 0) 451 self.emit("PyObject* tmp = NULL;", 1) 452 for f in prod.fields: 453 self.visitFieldDeclaration(f, name, prod=prod, depth=1) 454 self.emit("", 0) 455 for f in prod.fields: 456 self.visitField(f, name, prod=prod, depth=1) 457 args = [f.name.value for f in prod.fields] 458 self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) 459 self.emit("return 0;", 1) 460 self.emit("failed:", 0) 461 self.emit("Py_XDECREF(tmp);", 1) 462 self.emit("return 1;", 1) 463 self.emit("}", 0) 464 self.emit("", 0) 465 466 def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0): 467 ctype = get_c_type(field.type) 468 if field.seq: 469 if self.isSimpleType(field): 470 self.emit("asdl_int_seq* %s;" % field.name, depth) 471 else: 472 self.emit("asdl_seq* %s;" % field.name, depth) 473 else: 474 ctype = get_c_type(field.type) 475 self.emit("%s %s;" % (ctype, field.name), depth) 476 477 def isSimpleSum(self, field): 478 # XXX can the members of this list be determined automatically? 479 return field.type.value in ('expr_context', 'boolop', 'operator', 480 'unaryop', 'cmpop') 481 482 def isNumeric(self, field): 483 return get_c_type(field.type) in ("int", "bool") 484 485 def isSimpleType(self, field): 486 return self.isSimpleSum(field) or self.isNumeric(field) 487 488 def visitField(self, field, name, sum=None, prod=None, depth=0): 489 ctype = get_c_type(field.type) 490 self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth) 491 self.emit("int res;", depth+1) 492 if field.seq: 493 self.emit("Py_ssize_t len;", depth+1) 494 self.emit("Py_ssize_t i;", depth+1) 495 self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1) 496 self.emit("if (tmp == NULL) goto failed;", depth+1) 497 if field.seq: 498 self.emit("if (!PyList_Check(tmp)) {", depth+1) 499 self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must " 500 "be a list, not a %%.200s\", tmp->ob_type->tp_name);" % 501 (name, field.name), 502 depth+2, reflow=False) 503 self.emit("goto failed;", depth+2) 504 self.emit("}", depth+1) 505 self.emit("len = PyList_GET_SIZE(tmp);", depth+1) 506 if self.isSimpleType(field): 507 self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) 508 else: 509 self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) 510 self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) 511 self.emit("for (i = 0; i < len; i++) {", depth+1) 512 self.emit("%s value;" % ctype, depth+2) 513 self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % 514 field.type, depth+2, reflow=False) 515 self.emit("if (res != 0) goto failed;", depth+2) 516 self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) 517 self.emit("}", depth+1) 518 else: 519 self.emit("res = obj2ast_%s(tmp, &%s, arena);" % 520 (field.type, field.name), depth+1) 521 self.emit("if (res != 0) goto failed;", depth+1) 522 523 self.emit("Py_XDECREF(tmp);", depth+1) 524 self.emit("tmp = NULL;", depth+1) 525 self.emit("} else {", depth) 526 if not field.opt: 527 message = "required field \\\"%s\\\" missing from %s" % (field.name, name) 528 format = "PyErr_SetString(PyExc_TypeError, \"%s\");" 529 self.emit(format % message, depth+1, reflow=False) 530 self.emit("return 1;", depth+1) 531 else: 532 if self.isNumeric(field): 533 self.emit("%s = 0;" % field.name, depth+1) 534 elif not self.isSimpleType(field): 535 self.emit("%s = NULL;" % field.name, depth+1) 536 else: 537 raise TypeError("could not determine the default value for %s" % field.name) 538 self.emit("}", depth) 539 540 541class MarshalPrototypeVisitor(PickleVisitor): 542 543 def prototype(self, sum, name): 544 ctype = get_c_type(name) 545 self.emit("static int marshal_write_%s(PyObject **, int *, %s);" 546 % (name, ctype), 0) 547 548 visitProduct = visitSum = prototype 549 550 551class PyTypesDeclareVisitor(PickleVisitor): 552 553 def visitProduct(self, prod, name): 554 self.emit("static PyTypeObject *%s_type;" % name, 0) 555 self.emit("static PyObject* ast2obj_%s(void*);" % name, 0) 556 if prod.fields: 557 self.emit("static char *%s_fields[]={" % name,0) 558 for f in prod.fields: 559 self.emit('"%s",' % f.name, 1) 560 self.emit("};", 0) 561 562 def visitSum(self, sum, name): 563 self.emit("static PyTypeObject *%s_type;" % name, 0) 564 if sum.attributes: 565 self.emit("static char *%s_attributes[] = {" % name, 0) 566 for a in sum.attributes: 567 self.emit('"%s",' % a.name, 1) 568 self.emit("};", 0) 569 ptype = "void*" 570 if is_simple(sum): 571 ptype = get_c_type(name) 572 tnames = [] 573 for t in sum.types: 574 tnames.append(str(t.name)+"_singleton") 575 tnames = ", *".join(tnames) 576 self.emit("static PyObject *%s;" % tnames, 0) 577 self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0) 578 for t in sum.types: 579 self.visitConstructor(t, name) 580 581 def visitConstructor(self, cons, name): 582 self.emit("static PyTypeObject *%s_type;" % cons.name, 0) 583 if cons.fields: 584 self.emit("static char *%s_fields[]={" % cons.name, 0) 585 for t in cons.fields: 586 self.emit('"%s",' % t.name, 1) 587 self.emit("};",0) 588 589class PyTypesVisitor(PickleVisitor): 590 591 def visitModule(self, mod): 592 self.emit(""" 593static int 594ast_type_init(PyObject *self, PyObject *args, PyObject *kw) 595{ 596 Py_ssize_t i, numfields = 0; 597 int res = -1; 598 PyObject *key, *value, *fields; 599 fields = PyObject_GetAttrString((PyObject*)Py_TYPE(self), "_fields"); 600 if (!fields) 601 PyErr_Clear(); 602 if (fields) { 603 numfields = PySequence_Size(fields); 604 if (numfields == -1) 605 goto cleanup; 606 } 607 res = 0; /* if no error occurs, this stays 0 to the end */ 608 if (PyTuple_GET_SIZE(args) > 0) { 609 if (numfields != PyTuple_GET_SIZE(args)) { 610 PyErr_Format(PyExc_TypeError, "%.400s constructor takes %s" 611 "%zd positional argument%s", 612 Py_TYPE(self)->tp_name, 613 numfields == 0 ? "" : "either 0 or ", 614 numfields, numfields == 1 ? "" : "s"); 615 res = -1; 616 goto cleanup; 617 } 618 for (i = 0; i < PyTuple_GET_SIZE(args); i++) { 619 /* cannot be reached when fields is NULL */ 620 PyObject *name = PySequence_GetItem(fields, i); 621 if (!name) { 622 res = -1; 623 goto cleanup; 624 } 625 res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i)); 626 Py_DECREF(name); 627 if (res < 0) 628 goto cleanup; 629 } 630 } 631 if (kw) { 632 i = 0; /* needed by PyDict_Next */ 633 while (PyDict_Next(kw, &i, &key, &value)) { 634 res = PyObject_SetAttr(self, key, value); 635 if (res < 0) 636 goto cleanup; 637 } 638 } 639 cleanup: 640 Py_XDECREF(fields); 641 return res; 642} 643 644/* Pickling support */ 645static PyObject * 646ast_type_reduce(PyObject *self, PyObject *unused) 647{ 648 PyObject *res; 649 PyObject *dict = PyObject_GetAttrString(self, "__dict__"); 650 if (dict == NULL) { 651 if (PyErr_ExceptionMatches(PyExc_AttributeError)) 652 PyErr_Clear(); 653 else 654 return NULL; 655 } 656 if (dict) { 657 res = Py_BuildValue("O()O", Py_TYPE(self), dict); 658 Py_DECREF(dict); 659 return res; 660 } 661 return Py_BuildValue("O()", Py_TYPE(self)); 662} 663 664static PyMethodDef ast_type_methods[] = { 665 {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, 666 {NULL} 667}; 668 669static PyTypeObject AST_type = { 670 PyVarObject_HEAD_INIT(NULL, 0) 671 "typed_ast._ast27.AST", 672 sizeof(PyObject), 673 0, 674 0, /* tp_dealloc */ 675 0, /* tp_print */ 676 0, /* tp_getattr */ 677 0, /* tp_setattr */ 678 0, /* tp_compare */ 679 0, /* tp_repr */ 680 0, /* tp_as_number */ 681 0, /* tp_as_sequence */ 682 0, /* tp_as_mapping */ 683 0, /* tp_hash */ 684 0, /* tp_call */ 685 0, /* tp_str */ 686 PyObject_GenericGetAttr, /* tp_getattro */ 687 PyObject_GenericSetAttr, /* tp_setattro */ 688 0, /* tp_as_buffer */ 689 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ 690 0, /* tp_doc */ 691 0, /* tp_traverse */ 692 0, /* tp_clear */ 693 0, /* tp_richcompare */ 694 0, /* tp_weaklistoffset */ 695 0, /* tp_iter */ 696 0, /* tp_iternext */ 697 ast_type_methods, /* tp_methods */ 698 0, /* tp_members */ 699 0, /* tp_getset */ 700 0, /* tp_base */ 701 0, /* tp_dict */ 702 0, /* tp_descr_get */ 703 0, /* tp_descr_set */ 704 0, /* tp_dictoffset */ 705 (initproc)ast_type_init, /* tp_init */ 706 PyType_GenericAlloc, /* tp_alloc */ 707 PyType_GenericNew, /* tp_new */ 708 PyObject_Del, /* tp_free */ 709}; 710 711 712static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int num_fields) 713{ 714 PyObject *fnames, *result; 715 int i; 716 fnames = PyTuple_New(num_fields); 717 if (!fnames) return NULL; 718 for (i = 0; i < num_fields; i++) { 719 PyObject *field = PyUnicode_FromString(fields[i]); 720 if (!field) { 721 Py_DECREF(fnames); 722 return NULL; 723 } 724 PyTuple_SET_ITEM(fnames, i, field); 725 } 726 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}", 727 type, base, "_fields", fnames, "__module__", "typed_ast._ast27"); 728 Py_DECREF(fnames); 729 return (PyTypeObject*)result; 730} 731 732static int add_attributes(PyTypeObject* type, char**attrs, int num_fields) 733{ 734 int i, result; 735 PyObject *s, *l = PyTuple_New(num_fields); 736 if (!l) 737 return 0; 738 for (i = 0; i < num_fields; i++) { 739 s = PyUnicode_FromString(attrs[i]); 740 if (!s) { 741 Py_DECREF(l); 742 return 0; 743 } 744 PyTuple_SET_ITEM(l, i, s); 745 } 746 result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0; 747 Py_DECREF(l); 748 return result; 749} 750 751/* Conversion AST -> Python */ 752 753static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*)) 754{ 755 int i, n = asdl_seq_LEN(seq); 756 PyObject *result = PyList_New(n); 757 PyObject *value; 758 if (!result) 759 return NULL; 760 for (i = 0; i < n; i++) { 761 value = func(asdl_seq_GET(seq, i)); 762 if (!value) { 763 Py_DECREF(result); 764 return NULL; 765 } 766 PyList_SET_ITEM(result, i, value); 767 } 768 return result; 769} 770 771static PyObject* ast2obj_object(void *o) 772{ 773 if (!o) 774 o = Py_None; 775 Py_INCREF((PyObject*)o); 776 return (PyObject*)o; 777} 778#define ast2obj_identifier ast2obj_object 779#define ast2obj_string ast2obj_object 780static PyObject* ast2obj_bool(bool b) 781{ 782 return PyBool_FromLong(b); 783} 784 785static PyObject* ast2obj_int(long b) 786{ 787 return PyLong_FromLong(b); 788} 789 790/* Conversion Python -> AST */ 791 792static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) 793{ 794 if (obj == Py_None) 795 obj = NULL; 796 if (obj) 797 PyArena_AddPyObject(arena, obj); 798 Py_XINCREF(obj); 799 *out = obj; 800 return 0; 801} 802 803static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) 804{ 805 if (!PyUnicode_CheckExact(obj) && obj != Py_None) { 806 PyErr_Format(PyExc_TypeError, 807 "AST identifier must be of type str"); 808 return 1; 809 } 810 return obj2ast_object(obj, out, arena); 811} 812 813static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena) 814{ 815 if (!PyUnicode_CheckExact(obj) && !PyUnicode_CheckExact(obj)) { 816 PyErr_SetString(PyExc_TypeError, 817 "AST string must be of type str or unicode"); 818 return 1; 819 } 820 return obj2ast_object(obj, out, arena); 821} 822 823static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) 824{ 825 int i; 826 if (!PyLong_Check(obj) && !PyLong_Check(obj)) { 827 PyObject *s = PyObject_Repr(obj); 828 if (s == NULL) return 1; 829 PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s", 830 _PyUnicode_AsString(s)); 831 Py_DECREF(s); 832 return 1; 833 } 834 835 i = (int)PyLong_AsLong(obj); 836 if (i == -1 && PyErr_Occurred()) 837 return 1; 838 *out = i; 839 return 0; 840} 841 842static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena) 843{ 844 if (!PyBool_Check(obj)) { 845 PyObject *s = PyObject_Repr(obj); 846 if (s == NULL) return 1; 847 PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s", 848 _PyUnicode_AsString(s)); 849 Py_DECREF(s); 850 return 1; 851 } 852 853 *out = (obj == Py_True); 854 return 0; 855} 856 857static int add_ast_fields(void) 858{ 859 PyObject *empty_tuple, *d; 860 if (PyType_Ready(&AST_type) < 0) 861 return -1; 862 d = AST_type.tp_dict; 863 empty_tuple = PyTuple_New(0); 864 if (!empty_tuple || 865 PyDict_SetItemString(d, "_fields", empty_tuple) < 0 || 866 PyDict_SetItemString(d, "_attributes", empty_tuple) < 0) { 867 Py_XDECREF(empty_tuple); 868 return -1; 869 } 870 Py_DECREF(empty_tuple); 871 return 0; 872} 873 874""", 0, reflow=False) 875 876 self.emit("static int init_types(void)",0) 877 self.emit("{", 0) 878 self.emit("static int initialized;", 1) 879 self.emit("if (initialized) return 1;", 1) 880 self.emit("if (add_ast_fields() < 0) return 0;", 1) 881 for dfn in mod.dfns: 882 self.visit(dfn) 883 self.emit("initialized = 1;", 1) 884 self.emit("return 1;", 1); 885 self.emit("}", 0) 886 887 def visitProduct(self, prod, name): 888 if prod.fields: 889 fields = name.value+"_fields" 890 else: 891 fields = "NULL" 892 self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' % 893 (name, name, fields, len(prod.fields)), 1) 894 self.emit("if (!%s_type) return 0;" % name, 1) 895 896 def visitSum(self, sum, name): 897 self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' % 898 (name, name), 1) 899 self.emit("if (!%s_type) return 0;" % name, 1) 900 if sum.attributes: 901 self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" % 902 (name, name, len(sum.attributes)), 1) 903 else: 904 self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1) 905 simple = is_simple(sum) 906 for t in sum.types: 907 self.visitConstructor(t, name, simple) 908 909 def visitConstructor(self, cons, name, simple): 910 if cons.fields: 911 fields = cons.name.value+"_fields" 912 else: 913 fields = "NULL" 914 self.emit('%s_type = make_type("%s", %s_type, %s, %d);' % 915 (cons.name, cons.name, name, fields, len(cons.fields)), 1) 916 self.emit("if (!%s_type) return 0;" % cons.name, 1) 917 if simple: 918 self.emit("%s_singleton = PyType_GenericNew(%s_type, NULL, NULL);" % 919 (cons.name, cons.name), 1) 920 self.emit("if (!%s_singleton) return 0;" % cons.name, 1) 921 922 923class ASTModuleVisitor(PickleVisitor): 924 925 def visitModule(self, mod): 926 self.emit('PyObject *ast27_parse(PyObject *self, PyObject *args);', 0) 927 self.emit('static PyMethodDef ast27_methods[] = {', 0) 928 self.emit('{"parse", ast27_parse, METH_VARARGS, "Parse string into typed AST."},', 1) 929 self.emit('{NULL, NULL, 0, NULL}', 1) 930 self.emit('};', 0) 931 932 self.emit("static struct PyModuleDef _astmodule27 = {", 0) 933 self.emit(' PyModuleDef_HEAD_INIT, "_ast27", NULL, 0, ast27_methods', 0) 934 self.emit("};", 0) 935 self.emit("PyMODINIT_FUNC", 0) 936 self.emit("PyInit__ast27(void)", 0) 937 self.emit("{", 0) 938 self.emit("PyObject *m, *d;", 1) 939 self.emit("if (!init_types()) return NULL;", 1) 940 self.emit('m = PyModule_Create(&_astmodule27);', 1) 941 self.emit("if (!m) return NULL;", 1) 942 self.emit("d = PyModule_GetDict(m);", 1) 943 self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1) 944 self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) 945 self.emit("return NULL;", 2) 946 for dfn in mod.dfns: 947 self.visit(dfn) 948 self.emit("return m;", 1) 949 self.emit("}", 0) 950 951 def visitProduct(self, prod, name): 952 self.addObj(name) 953 954 def visitSum(self, sum, name): 955 self.addObj(name) 956 for t in sum.types: 957 self.visitConstructor(t, name) 958 959 def visitConstructor(self, cons, name): 960 self.addObj(cons.name) 961 962 def addObj(self, name): 963 self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return NULL;' % (name, name), 1) 964 965 966_SPECIALIZED_SEQUENCES = ('stmt', 'expr') 967 968def find_sequence(fields, doing_specialization): 969 """Return True if any field uses a sequence.""" 970 for f in fields: 971 if f.seq: 972 if not doing_specialization: 973 return True 974 if str(f.type) not in _SPECIALIZED_SEQUENCES: 975 return True 976 return False 977 978def has_sequence(types, doing_specialization): 979 for t in types: 980 if find_sequence(t.fields, doing_specialization): 981 return True 982 return False 983 984 985class StaticVisitor(PickleVisitor): 986 CODE = '''Very simple, always emit this static code. Override CODE''' 987 988 def visit(self, object): 989 self.emit(self.CODE, 0, reflow=False) 990 991 992class ObjVisitor(PickleVisitor): 993 994 def func_begin(self, name): 995 ctype = get_c_type(name) 996 self.emit("PyObject*", 0) 997 self.emit("ast2obj_%s(void* _o)" % (name), 0) 998 self.emit("{", 0) 999 self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) 1000 self.emit("PyObject *result = NULL, *value = NULL;", 1) 1001 self.emit('if (!o) {', 1) 1002 self.emit("Py_INCREF(Py_None);", 2) 1003 self.emit('return Py_None;', 2) 1004 self.emit("}", 1) 1005 self.emit('', 0) 1006 1007 def func_end(self): 1008 self.emit("return result;", 1) 1009 self.emit("failed:", 0) 1010 self.emit("Py_XDECREF(value);", 1) 1011 self.emit("Py_XDECREF(result);", 1) 1012 self.emit("return NULL;", 1) 1013 self.emit("}", 0) 1014 self.emit("", 0) 1015 1016 def visitSum(self, sum, name): 1017 if is_simple(sum): 1018 self.simpleSum(sum, name) 1019 return 1020 self.func_begin(name) 1021 self.emit("switch (o->kind) {", 1) 1022 for i in range(len(sum.types)): 1023 t = sum.types[i] 1024 self.visitConstructor(t, i + 1, name) 1025 self.emit("}", 1) 1026 for a in sum.attributes: 1027 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1) 1028 self.emit("if (!value) goto failed;", 1) 1029 self.emit('if (PyObject_SetAttrString(result, "%s", value) < 0)' % a.name, 1) 1030 self.emit('goto failed;', 2) 1031 self.emit('Py_DECREF(value);', 1) 1032 self.func_end() 1033 1034 def simpleSum(self, sum, name): 1035 self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0) 1036 self.emit("{", 0) 1037 self.emit("switch(o) {", 1) 1038 for t in sum.types: 1039 self.emit("case %s:" % t.name, 2) 1040 self.emit("Py_INCREF(%s_singleton);" % t.name, 3) 1041 self.emit("return %s_singleton;" % t.name, 3) 1042 self.emit("default:", 2) 1043 self.emit('/* should never happen, but just in case ... */', 3) 1044 code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name 1045 self.emit(code, 3, reflow=False) 1046 self.emit("return NULL;", 3) 1047 self.emit("}", 1) 1048 self.emit("}", 0) 1049 1050 def visitProduct(self, prod, name): 1051 self.func_begin(name) 1052 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % name, 1); 1053 self.emit("if (!result) return NULL;", 1) 1054 for field in prod.fields: 1055 self.visitField(field, name, 1, True) 1056 self.func_end() 1057 1058 def visitConstructor(self, cons, enum, name): 1059 self.emit("case %s_kind:" % cons.name, 1) 1060 self.emit("result = PyType_GenericNew(%s_type, NULL, NULL);" % cons.name, 2); 1061 self.emit("if (!result) goto failed;", 2) 1062 for f in cons.fields: 1063 self.visitField(f, cons.name, 2, False) 1064 self.emit("break;", 2) 1065 1066 def visitField(self, field, name, depth, product): 1067 def emit(s, d): 1068 self.emit(s, depth + d) 1069 if product: 1070 value = "o->%s" % field.name 1071 else: 1072 value = "o->v.%s.%s" % (name, field.name) 1073 self.set(field, value, depth) 1074 emit("if (!value) goto failed;", 0) 1075 emit('if (PyObject_SetAttrString(result, "%s", value) == -1)' % field.name, 0) 1076 emit("goto failed;", 1) 1077 emit("Py_DECREF(value);", 0) 1078 1079 def emitSeq(self, field, value, depth, emit): 1080 emit("seq = %s;" % value, 0) 1081 emit("n = asdl_seq_LEN(seq);", 0) 1082 emit("value = PyList_New(n);", 0) 1083 emit("if (!value) goto failed;", 0) 1084 emit("for (i = 0; i < n; i++) {", 0) 1085 self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1) 1086 emit("if (!value1) goto failed;", 1) 1087 emit("PyList_SET_ITEM(value, i, value1);", 1) 1088 emit("value1 = NULL;", 1) 1089 emit("}", 0) 1090 1091 def set(self, field, value, depth): 1092 if field.seq: 1093 # XXX should really check for is_simple, but that requires a symbol table 1094 if field.type.value == "cmpop": 1095 # While the sequence elements are stored as void*, 1096 # ast2obj_cmpop expects an enum 1097 self.emit("{", depth) 1098 self.emit("int i, n = asdl_seq_LEN(%s);" % value, depth+1) 1099 self.emit("value = PyList_New(n);", depth+1) 1100 self.emit("if (!value) goto failed;", depth+1) 1101 self.emit("for(i = 0; i < n; i++)", depth+1) 1102 # This cannot fail, so no need for error handling 1103 self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value, 1104 depth+2, reflow=False) 1105 self.emit("}", depth) 1106 else: 1107 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth) 1108 else: 1109 ctype = get_c_type(field.type) 1110 self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False) 1111 1112 1113class PartingShots(StaticVisitor): 1114 1115 CODE = """ 1116PyObject* Ta27AST_mod2obj(mod_ty t) 1117{ 1118 init_types(); 1119 return ast2obj_mod(t); 1120} 1121 1122/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */ 1123mod_ty Ta27AST_obj2mod(PyObject* ast, PyArena* arena, int mode) 1124{ 1125 mod_ty res; 1126 PyObject *req_type[3]; 1127 char *req_name[3]; 1128 int isinstance; 1129 1130 req_type[0] = (PyObject*)Module_type; 1131 req_type[1] = (PyObject*)Expression_type; 1132 req_type[2] = (PyObject*)Interactive_type; 1133 1134 req_name[0] = "Module"; 1135 req_name[1] = "Expression"; 1136 req_name[2] = "Interactive"; 1137 1138 assert(0 <= mode && mode <= 2); 1139 1140 init_types(); 1141 1142 isinstance = PyObject_IsInstance(ast, req_type[mode]); 1143 if (isinstance == -1) 1144 return NULL; 1145 if (!isinstance) { 1146 PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s", 1147 req_name[mode], Py_TYPE(ast)->tp_name); 1148 return NULL; 1149 } 1150 if (obj2ast_mod(ast, &res, arena) != 0) 1151 return NULL; 1152 else 1153 return res; 1154} 1155 1156int Ta27AST_Check(PyObject* obj) 1157{ 1158 init_types(); 1159 return PyObject_IsInstance(obj, (PyObject*)&AST_type); 1160} 1161""" 1162 1163class ChainOfVisitors: 1164 def __init__(self, *visitors): 1165 self.visitors = visitors 1166 1167 def visit(self, object): 1168 for v in self.visitors: 1169 v.visit(object) 1170 v.emit("", 0) 1171 1172common_msg = "/* File automatically generated by %s. */\n\n" 1173 1174c_file_msg = """ 1175/* 1176 __version__ %s. 1177 1178 This module must be committed separately after each AST grammar change; 1179 The __version__ number is set to the revision number of the commit 1180 containing the grammar change. 1181*/ 1182 1183""" 1184 1185def main(srcfile): 1186 argv0 = sys.argv[0] 1187 components = argv0.split(os.sep) 1188 argv0 = os.sep.join(components[-2:]) 1189 auto_gen_msg = common_msg % argv0 1190 mod = asdl.parse(srcfile) 1191 mod.version = "82160" 1192 if not asdl.check(mod): 1193 sys.exit(1) 1194 if INC_DIR: 1195 p = "%s/%s-ast.h" % (INC_DIR, mod.name) 1196 f = open(p, "wb") 1197 f.write(auto_gen_msg) 1198 f.write('#include "asdl.h"\n\n') 1199 c = ChainOfVisitors(TypeDefVisitor(f), 1200 StructVisitor(f), 1201 PrototypeVisitor(f), 1202 ) 1203 c.visit(mod) 1204 f.write("PyObject* Ta27AST_mod2obj(mod_ty t);\n") 1205 f.write("mod_ty Ta27AST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n") 1206 f.write("int Ta27AST_Check(PyObject* obj);\n") 1207 f.close() 1208 1209 if SRC_DIR: 1210 p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c") 1211 f = open(p, "wb") 1212 f.write(auto_gen_msg) 1213 f.write(c_file_msg % mod.version) 1214 f.write('#include "Python.h"\n') 1215 f.write('#include "%s-ast.h"\n' % mod.name) 1216 f.write('\n') 1217 f.write("static PyTypeObject AST_type;\n") 1218 v = ChainOfVisitors( 1219 PyTypesDeclareVisitor(f), 1220 PyTypesVisitor(f), 1221 Obj2ModPrototypeVisitor(f), 1222 FunctionVisitor(f), 1223 ObjVisitor(f), 1224 Obj2ModVisitor(f), 1225 ASTModuleVisitor(f), 1226 PartingShots(f), 1227 ) 1228 v.visit(mod) 1229 f.close() 1230 1231if __name__ == "__main__": 1232 import sys 1233 import getopt 1234 1235 INC_DIR = '' 1236 SRC_DIR = '' 1237 opts, args = getopt.getopt(sys.argv[1:], "h:c:") 1238 if len(opts) != 1: 1239 print "Must specify exactly one output file" 1240 sys.exit(1) 1241 for o, v in opts: 1242 if o == '-h': 1243 INC_DIR = v 1244 if o == '-c': 1245 SRC_DIR = v 1246 if len(args) != 1: 1247 print "Must specify single input file" 1248 sys.exit(1) 1249 main(args[0]) 1250