1 /*
2 * This file compiles an abstract syntax tree (AST) into Python bytecode.
3 *
4 * The primary entry point is PyAST_Compile(), which returns a
5 * PyCodeObject. The compiler makes several passes to build the code
6 * object:
7 * 1. Checks for future statements. See future.c
8 * 2. Builds a symbol table. See symtable.c.
9 * 3. Generate code for basic blocks. See compiler_mod() in this file.
10 * 4. Assemble the basic blocks into final code. See assemble() in
11 * this file.
12 * 5. Optimize the byte code (peephole optimizations). See peephole.c
13 *
14 * Note that compiler_mod() suggests module, but the module ast type
15 * (mod_ty) has cases for expressions and interactive statements.
16 *
17 * CAUTION: The VISIT_* macros abort the current function when they
18 * encounter a problem. So don't invoke them when there is memory
19 * which needs to be released. Code blocks are OK, as the compiler
20 * structure takes care of releasing those. Use the arena to manage
21 * objects.
22 */
23
24 #include "Python.h"
25
26 #include "Python-ast.h"
27 #include "node.h"
28 #include "ast.h"
29 #include "code.h"
30 #include "symtable.h"
31 #include "opcode.h"
32 #include "wordcode_helpers.h"
33
34 #define DEFAULT_BLOCK_SIZE 16
35 #define DEFAULT_BLOCKS 8
36 #define DEFAULT_CODE_SIZE 128
37 #define DEFAULT_LNOTAB_SIZE 16
38
39 #define COMP_GENEXP 0
40 #define COMP_LISTCOMP 1
41 #define COMP_SETCOMP 2
42 #define COMP_DICTCOMP 3
43
44 struct instr {
45 unsigned i_jabs : 1;
46 unsigned i_jrel : 1;
47 unsigned char i_opcode;
48 int i_oparg;
49 struct basicblock_ *i_target; /* target block (if jump instruction) */
50 int i_lineno;
51 };
52
53 typedef struct basicblock_ {
54 /* Each basicblock in a compilation unit is linked via b_list in the
55 reverse order that the block are allocated. b_list points to the next
56 block, not to be confused with b_next, which is next by control flow. */
57 struct basicblock_ *b_list;
58 /* number of instructions used */
59 int b_iused;
60 /* length of instruction array (b_instr) */
61 int b_ialloc;
62 /* pointer to an array of instructions, initially NULL */
63 struct instr *b_instr;
64 /* If b_next is non-NULL, it is a pointer to the next
65 block reached by normal control flow. */
66 struct basicblock_ *b_next;
67 /* b_seen is used to perform a DFS of basicblocks. */
68 unsigned b_seen : 1;
69 /* b_return is true if a RETURN_VALUE opcode is inserted. */
70 unsigned b_return : 1;
71 /* depth of stack upon entry of block, computed by stackdepth() */
72 int b_startdepth;
73 /* instruction offset for block, computed by assemble_jump_offsets() */
74 int b_offset;
75 } basicblock;
76
77 /* fblockinfo tracks the current frame block.
78
79 A frame block is used to handle loops, try/except, and try/finally.
80 It's called a frame block to distinguish it from a basic block in the
81 compiler IR.
82 */
83
84 enum fblocktype { LOOP, EXCEPT, FINALLY_TRY, FINALLY_END };
85
86 struct fblockinfo {
87 enum fblocktype fb_type;
88 basicblock *fb_block;
89 };
90
91 enum {
92 COMPILER_SCOPE_MODULE,
93 COMPILER_SCOPE_CLASS,
94 COMPILER_SCOPE_FUNCTION,
95 COMPILER_SCOPE_ASYNC_FUNCTION,
96 COMPILER_SCOPE_LAMBDA,
97 COMPILER_SCOPE_COMPREHENSION,
98 };
99
100 /* The following items change on entry and exit of code blocks.
101 They must be saved and restored when returning to a block.
102 */
103 struct compiler_unit {
104 PySTEntryObject *u_ste;
105
106 PyObject *u_name;
107 PyObject *u_qualname; /* dot-separated qualified name (lazy) */
108 int u_scope_type;
109
110 /* The following fields are dicts that map objects to
111 the index of them in co_XXX. The index is used as
112 the argument for opcodes that refer to those collections.
113 */
114 PyObject *u_consts; /* all constants */
115 PyObject *u_names; /* all names */
116 PyObject *u_varnames; /* local variables */
117 PyObject *u_cellvars; /* cell variables */
118 PyObject *u_freevars; /* free variables */
119
120 PyObject *u_private; /* for private name mangling */
121
122 Py_ssize_t u_argcount; /* number of arguments for block */
123 Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */
124 /* Pointer to the most recently allocated block. By following b_list
125 members, you can reach all early allocated blocks. */
126 basicblock *u_blocks;
127 basicblock *u_curblock; /* pointer to current block */
128
129 int u_nfblocks;
130 struct fblockinfo u_fblock[CO_MAXBLOCKS];
131
132 int u_firstlineno; /* the first lineno of the block */
133 int u_lineno; /* the lineno for the current stmt */
134 int u_col_offset; /* the offset of the current stmt */
135 int u_lineno_set; /* boolean to indicate whether instr
136 has been generated with current lineno */
137 };
138
139 /* This struct captures the global state of a compilation.
140
141 The u pointer points to the current compilation unit, while units
142 for enclosing blocks are stored in c_stack. The u and c_stack are
143 managed by compiler_enter_scope() and compiler_exit_scope().
144
145 Note that we don't track recursion levels during compilation - the
146 task of detecting and rejecting excessive levels of nesting is
147 handled by the symbol analysis pass.
148
149 */
150
151 struct compiler {
152 PyObject *c_filename;
153 struct symtable *c_st;
154 PyFutureFeatures *c_future; /* pointer to module's __future__ */
155 PyCompilerFlags *c_flags;
156
157 int c_optimize; /* optimization level */
158 int c_interactive; /* true if in interactive mode */
159 int c_nestlevel;
160
161 struct compiler_unit *u; /* compiler state for current block */
162 PyObject *c_stack; /* Python list holding compiler_unit ptrs */
163 PyArena *c_arena; /* pointer to memory allocation arena */
164 };
165
166 static int compiler_enter_scope(struct compiler *, identifier, int, void *, int);
167 static void compiler_free(struct compiler *);
168 static basicblock *compiler_new_block(struct compiler *);
169 static int compiler_next_instr(struct compiler *, basicblock *);
170 static int compiler_addop(struct compiler *, int);
171 static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *);
172 static int compiler_addop_i(struct compiler *, int, Py_ssize_t);
173 static int compiler_addop_j(struct compiler *, int, basicblock *, int);
174 static int compiler_error(struct compiler *, const char *);
175 static int compiler_nameop(struct compiler *, identifier, expr_context_ty);
176
177 static PyCodeObject *compiler_mod(struct compiler *, mod_ty);
178 static int compiler_visit_stmt(struct compiler *, stmt_ty);
179 static int compiler_visit_keyword(struct compiler *, keyword_ty);
180 static int compiler_visit_expr(struct compiler *, expr_ty);
181 static int compiler_augassign(struct compiler *, stmt_ty);
182 static int compiler_annassign(struct compiler *, stmt_ty);
183 static int compiler_visit_slice(struct compiler *, slice_ty,
184 expr_context_ty);
185
186 static int compiler_push_fblock(struct compiler *, enum fblocktype,
187 basicblock *);
188 static void compiler_pop_fblock(struct compiler *, enum fblocktype,
189 basicblock *);
190 /* Returns true if there is a loop on the fblock stack. */
191 static int compiler_in_loop(struct compiler *);
192
193 static int inplace_binop(struct compiler *, operator_ty);
194 static int expr_constant(expr_ty);
195
196 static int compiler_with(struct compiler *, stmt_ty, int);
197 static int compiler_async_with(struct compiler *, stmt_ty, int);
198 static int compiler_async_for(struct compiler *, stmt_ty);
199 static int compiler_call_helper(struct compiler *c, int n,
200 asdl_seq *args,
201 asdl_seq *keywords);
202 static int compiler_try_except(struct compiler *, stmt_ty);
203 static int compiler_set_qualname(struct compiler *);
204
205 static int compiler_sync_comprehension_generator(
206 struct compiler *c,
207 asdl_seq *generators, int gen_index,
208 expr_ty elt, expr_ty val, int type);
209
210 static int compiler_async_comprehension_generator(
211 struct compiler *c,
212 asdl_seq *generators, int gen_index,
213 expr_ty elt, expr_ty val, int type);
214
215 static PyCodeObject *assemble(struct compiler *, int addNone);
216 static PyObject *__doc__, *__annotations__;
217
218 #define CAPSULE_NAME "compile.c compiler unit"
219
220 PyObject *
_Py_Mangle(PyObject * privateobj,PyObject * ident)221 _Py_Mangle(PyObject *privateobj, PyObject *ident)
222 {
223 /* Name mangling: __private becomes _classname__private.
224 This is independent from how the name is used. */
225 PyObject *result;
226 size_t nlen, plen, ipriv;
227 Py_UCS4 maxchar;
228 if (privateobj == NULL || !PyUnicode_Check(privateobj) ||
229 PyUnicode_READ_CHAR(ident, 0) != '_' ||
230 PyUnicode_READ_CHAR(ident, 1) != '_') {
231 Py_INCREF(ident);
232 return ident;
233 }
234 nlen = PyUnicode_GET_LENGTH(ident);
235 plen = PyUnicode_GET_LENGTH(privateobj);
236 /* Don't mangle __id__ or names with dots.
237
238 The only time a name with a dot can occur is when
239 we are compiling an import statement that has a
240 package name.
241
242 TODO(jhylton): Decide whether we want to support
243 mangling of the module name, e.g. __M.X.
244 */
245 if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
246 PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||
247 PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {
248 Py_INCREF(ident);
249 return ident; /* Don't mangle __whatever__ */
250 }
251 /* Strip leading underscores from class name */
252 ipriv = 0;
253 while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_')
254 ipriv++;
255 if (ipriv == plen) {
256 Py_INCREF(ident);
257 return ident; /* Don't mangle if class is just underscores */
258 }
259 plen -= ipriv;
260
261 if (plen + nlen >= PY_SSIZE_T_MAX - 1) {
262 PyErr_SetString(PyExc_OverflowError,
263 "private identifier too large to be mangled");
264 return NULL;
265 }
266
267 maxchar = PyUnicode_MAX_CHAR_VALUE(ident);
268 if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar)
269 maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);
270
271 result = PyUnicode_New(1 + nlen + plen, maxchar);
272 if (!result)
273 return 0;
274 /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */
275 PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');
276 if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) {
277 Py_DECREF(result);
278 return NULL;
279 }
280 if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) {
281 Py_DECREF(result);
282 return NULL;
283 }
284 assert(_PyUnicode_CheckConsistency(result, 1));
285 return result;
286 }
287
288 static int
compiler_init(struct compiler * c)289 compiler_init(struct compiler *c)
290 {
291 memset(c, 0, sizeof(struct compiler));
292
293 c->c_stack = PyList_New(0);
294 if (!c->c_stack)
295 return 0;
296
297 return 1;
298 }
299
300 PyCodeObject *
PyAST_CompileObject(mod_ty mod,PyObject * filename,PyCompilerFlags * flags,int optimize,PyArena * arena)301 PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
302 int optimize, PyArena *arena)
303 {
304 struct compiler c;
305 PyCodeObject *co = NULL;
306 PyCompilerFlags local_flags;
307 int merged;
308
309 if (!__doc__) {
310 __doc__ = PyUnicode_InternFromString("__doc__");
311 if (!__doc__)
312 return NULL;
313 }
314 if (!__annotations__) {
315 __annotations__ = PyUnicode_InternFromString("__annotations__");
316 if (!__annotations__)
317 return NULL;
318 }
319 if (!compiler_init(&c))
320 return NULL;
321 Py_INCREF(filename);
322 c.c_filename = filename;
323 c.c_arena = arena;
324 c.c_future = PyFuture_FromASTObject(mod, filename);
325 if (c.c_future == NULL)
326 goto finally;
327 if (!flags) {
328 local_flags.cf_flags = 0;
329 flags = &local_flags;
330 }
331 merged = c.c_future->ff_features | flags->cf_flags;
332 c.c_future->ff_features = merged;
333 flags->cf_flags = merged;
334 c.c_flags = flags;
335 c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize;
336 c.c_nestlevel = 0;
337
338 if (!_PyAST_Optimize(mod, arena, c.c_optimize)) {
339 goto finally;
340 }
341
342 c.c_st = PySymtable_BuildObject(mod, filename, c.c_future);
343 if (c.c_st == NULL) {
344 if (!PyErr_Occurred())
345 PyErr_SetString(PyExc_SystemError, "no symtable");
346 goto finally;
347 }
348
349 co = compiler_mod(&c, mod);
350
351 finally:
352 compiler_free(&c);
353 assert(co || PyErr_Occurred());
354 return co;
355 }
356
357 PyCodeObject *
PyAST_CompileEx(mod_ty mod,const char * filename_str,PyCompilerFlags * flags,int optimize,PyArena * arena)358 PyAST_CompileEx(mod_ty mod, const char *filename_str, PyCompilerFlags *flags,
359 int optimize, PyArena *arena)
360 {
361 PyObject *filename;
362 PyCodeObject *co;
363 filename = PyUnicode_DecodeFSDefault(filename_str);
364 if (filename == NULL)
365 return NULL;
366 co = PyAST_CompileObject(mod, filename, flags, optimize, arena);
367 Py_DECREF(filename);
368 return co;
369
370 }
371
372 PyCodeObject *
PyNode_Compile(struct _node * n,const char * filename)373 PyNode_Compile(struct _node *n, const char *filename)
374 {
375 PyCodeObject *co = NULL;
376 mod_ty mod;
377 PyArena *arena = PyArena_New();
378 if (!arena)
379 return NULL;
380 mod = PyAST_FromNode(n, NULL, filename, arena);
381 if (mod)
382 co = PyAST_Compile(mod, filename, NULL, arena);
383 PyArena_Free(arena);
384 return co;
385 }
386
387 static void
compiler_free(struct compiler * c)388 compiler_free(struct compiler *c)
389 {
390 if (c->c_st)
391 PySymtable_Free(c->c_st);
392 if (c->c_future)
393 PyObject_Free(c->c_future);
394 Py_XDECREF(c->c_filename);
395 Py_DECREF(c->c_stack);
396 }
397
398 static PyObject *
list2dict(PyObject * list)399 list2dict(PyObject *list)
400 {
401 Py_ssize_t i, n;
402 PyObject *v, *k;
403 PyObject *dict = PyDict_New();
404 if (!dict) return NULL;
405
406 n = PyList_Size(list);
407 for (i = 0; i < n; i++) {
408 v = PyLong_FromSsize_t(i);
409 if (!v) {
410 Py_DECREF(dict);
411 return NULL;
412 }
413 k = PyList_GET_ITEM(list, i);
414 k = _PyCode_ConstantKey(k);
415 if (k == NULL || PyDict_SetItem(dict, k, v) < 0) {
416 Py_XDECREF(k);
417 Py_DECREF(v);
418 Py_DECREF(dict);
419 return NULL;
420 }
421 Py_DECREF(k);
422 Py_DECREF(v);
423 }
424 return dict;
425 }
426
427 /* Return new dict containing names from src that match scope(s).
428
429 src is a symbol table dictionary. If the scope of a name matches
430 either scope_type or flag is set, insert it into the new dict. The
431 values are integers, starting at offset and increasing by one for
432 each key.
433 */
434
435 static PyObject *
dictbytype(PyObject * src,int scope_type,int flag,Py_ssize_t offset)436 dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset)
437 {
438 Py_ssize_t i = offset, scope, num_keys, key_i;
439 PyObject *k, *v, *dest = PyDict_New();
440 PyObject *sorted_keys;
441
442 assert(offset >= 0);
443 if (dest == NULL)
444 return NULL;
445
446 /* Sort the keys so that we have a deterministic order on the indexes
447 saved in the returned dictionary. These indexes are used as indexes
448 into the free and cell var storage. Therefore if they aren't
449 deterministic, then the generated bytecode is not deterministic.
450 */
451 sorted_keys = PyDict_Keys(src);
452 if (sorted_keys == NULL)
453 return NULL;
454 if (PyList_Sort(sorted_keys) != 0) {
455 Py_DECREF(sorted_keys);
456 return NULL;
457 }
458 num_keys = PyList_GET_SIZE(sorted_keys);
459
460 for (key_i = 0; key_i < num_keys; key_i++) {
461 /* XXX this should probably be a macro in symtable.h */
462 long vi;
463 k = PyList_GET_ITEM(sorted_keys, key_i);
464 v = PyDict_GetItem(src, k);
465 assert(PyLong_Check(v));
466 vi = PyLong_AS_LONG(v);
467 scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK;
468
469 if (scope == scope_type || vi & flag) {
470 PyObject *tuple, *item = PyLong_FromSsize_t(i);
471 if (item == NULL) {
472 Py_DECREF(sorted_keys);
473 Py_DECREF(dest);
474 return NULL;
475 }
476 i++;
477 tuple = _PyCode_ConstantKey(k);
478 if (!tuple || PyDict_SetItem(dest, tuple, item) < 0) {
479 Py_DECREF(sorted_keys);
480 Py_DECREF(item);
481 Py_DECREF(dest);
482 Py_XDECREF(tuple);
483 return NULL;
484 }
485 Py_DECREF(item);
486 Py_DECREF(tuple);
487 }
488 }
489 Py_DECREF(sorted_keys);
490 return dest;
491 }
492
493 static void
compiler_unit_check(struct compiler_unit * u)494 compiler_unit_check(struct compiler_unit *u)
495 {
496 basicblock *block;
497 for (block = u->u_blocks; block != NULL; block = block->b_list) {
498 assert((uintptr_t)block != 0xcbcbcbcbU);
499 assert((uintptr_t)block != 0xfbfbfbfbU);
500 assert((uintptr_t)block != 0xdbdbdbdbU);
501 if (block->b_instr != NULL) {
502 assert(block->b_ialloc > 0);
503 assert(block->b_iused > 0);
504 assert(block->b_ialloc >= block->b_iused);
505 }
506 else {
507 assert (block->b_iused == 0);
508 assert (block->b_ialloc == 0);
509 }
510 }
511 }
512
513 static void
compiler_unit_free(struct compiler_unit * u)514 compiler_unit_free(struct compiler_unit *u)
515 {
516 basicblock *b, *next;
517
518 compiler_unit_check(u);
519 b = u->u_blocks;
520 while (b != NULL) {
521 if (b->b_instr)
522 PyObject_Free((void *)b->b_instr);
523 next = b->b_list;
524 PyObject_Free((void *)b);
525 b = next;
526 }
527 Py_CLEAR(u->u_ste);
528 Py_CLEAR(u->u_name);
529 Py_CLEAR(u->u_qualname);
530 Py_CLEAR(u->u_consts);
531 Py_CLEAR(u->u_names);
532 Py_CLEAR(u->u_varnames);
533 Py_CLEAR(u->u_freevars);
534 Py_CLEAR(u->u_cellvars);
535 Py_CLEAR(u->u_private);
536 PyObject_Free(u);
537 }
538
539 static int
compiler_enter_scope(struct compiler * c,identifier name,int scope_type,void * key,int lineno)540 compiler_enter_scope(struct compiler *c, identifier name,
541 int scope_type, void *key, int lineno)
542 {
543 struct compiler_unit *u;
544 basicblock *block;
545
546 u = (struct compiler_unit *)PyObject_Malloc(sizeof(
547 struct compiler_unit));
548 if (!u) {
549 PyErr_NoMemory();
550 return 0;
551 }
552 memset(u, 0, sizeof(struct compiler_unit));
553 u->u_scope_type = scope_type;
554 u->u_argcount = 0;
555 u->u_kwonlyargcount = 0;
556 u->u_ste = PySymtable_Lookup(c->c_st, key);
557 if (!u->u_ste) {
558 compiler_unit_free(u);
559 return 0;
560 }
561 Py_INCREF(name);
562 u->u_name = name;
563 u->u_varnames = list2dict(u->u_ste->ste_varnames);
564 u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0);
565 if (!u->u_varnames || !u->u_cellvars) {
566 compiler_unit_free(u);
567 return 0;
568 }
569 if (u->u_ste->ste_needs_class_closure) {
570 /* Cook up an implicit __class__ cell. */
571 _Py_IDENTIFIER(__class__);
572 PyObject *tuple, *name;
573 int res;
574 assert(u->u_scope_type == COMPILER_SCOPE_CLASS);
575 assert(PyDict_GET_SIZE(u->u_cellvars) == 0);
576 name = _PyUnicode_FromId(&PyId___class__);
577 if (!name) {
578 compiler_unit_free(u);
579 return 0;
580 }
581 tuple = _PyCode_ConstantKey(name);
582 if (!tuple) {
583 compiler_unit_free(u);
584 return 0;
585 }
586 res = PyDict_SetItem(u->u_cellvars, tuple, _PyLong_Zero);
587 Py_DECREF(tuple);
588 if (res < 0) {
589 compiler_unit_free(u);
590 return 0;
591 }
592 }
593
594 u->u_freevars = dictbytype(u->u_ste->ste_symbols, FREE, DEF_FREE_CLASS,
595 PyDict_GET_SIZE(u->u_cellvars));
596 if (!u->u_freevars) {
597 compiler_unit_free(u);
598 return 0;
599 }
600
601 u->u_blocks = NULL;
602 u->u_nfblocks = 0;
603 u->u_firstlineno = lineno;
604 u->u_lineno = 0;
605 u->u_col_offset = 0;
606 u->u_lineno_set = 0;
607 u->u_consts = PyDict_New();
608 if (!u->u_consts) {
609 compiler_unit_free(u);
610 return 0;
611 }
612 u->u_names = PyDict_New();
613 if (!u->u_names) {
614 compiler_unit_free(u);
615 return 0;
616 }
617
618 u->u_private = NULL;
619
620 /* Push the old compiler_unit on the stack. */
621 if (c->u) {
622 PyObject *capsule = PyCapsule_New(c->u, CAPSULE_NAME, NULL);
623 if (!capsule || PyList_Append(c->c_stack, capsule) < 0) {
624 Py_XDECREF(capsule);
625 compiler_unit_free(u);
626 return 0;
627 }
628 Py_DECREF(capsule);
629 u->u_private = c->u->u_private;
630 Py_XINCREF(u->u_private);
631 }
632 c->u = u;
633
634 c->c_nestlevel++;
635
636 block = compiler_new_block(c);
637 if (block == NULL)
638 return 0;
639 c->u->u_curblock = block;
640
641 if (u->u_scope_type != COMPILER_SCOPE_MODULE) {
642 if (!compiler_set_qualname(c))
643 return 0;
644 }
645
646 return 1;
647 }
648
649 static void
compiler_exit_scope(struct compiler * c)650 compiler_exit_scope(struct compiler *c)
651 {
652 Py_ssize_t n;
653 PyObject *capsule;
654
655 c->c_nestlevel--;
656 compiler_unit_free(c->u);
657 /* Restore c->u to the parent unit. */
658 n = PyList_GET_SIZE(c->c_stack) - 1;
659 if (n >= 0) {
660 capsule = PyList_GET_ITEM(c->c_stack, n);
661 c->u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
662 assert(c->u);
663 /* we are deleting from a list so this really shouldn't fail */
664 if (PySequence_DelItem(c->c_stack, n) < 0)
665 Py_FatalError("compiler_exit_scope()");
666 compiler_unit_check(c->u);
667 }
668 else
669 c->u = NULL;
670
671 }
672
673 static int
compiler_set_qualname(struct compiler * c)674 compiler_set_qualname(struct compiler *c)
675 {
676 _Py_static_string(dot, ".");
677 _Py_static_string(dot_locals, ".<locals>");
678 Py_ssize_t stack_size;
679 struct compiler_unit *u = c->u;
680 PyObject *name, *base, *dot_str, *dot_locals_str;
681
682 base = NULL;
683 stack_size = PyList_GET_SIZE(c->c_stack);
684 assert(stack_size >= 1);
685 if (stack_size > 1) {
686 int scope, force_global = 0;
687 struct compiler_unit *parent;
688 PyObject *mangled, *capsule;
689
690 capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
691 parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
692 assert(parent);
693
694 if (u->u_scope_type == COMPILER_SCOPE_FUNCTION
695 || u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
696 || u->u_scope_type == COMPILER_SCOPE_CLASS) {
697 assert(u->u_name);
698 mangled = _Py_Mangle(parent->u_private, u->u_name);
699 if (!mangled)
700 return 0;
701 scope = PyST_GetScope(parent->u_ste, mangled);
702 Py_DECREF(mangled);
703 assert(scope != GLOBAL_IMPLICIT);
704 if (scope == GLOBAL_EXPLICIT)
705 force_global = 1;
706 }
707
708 if (!force_global) {
709 if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION
710 || parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
711 || parent->u_scope_type == COMPILER_SCOPE_LAMBDA) {
712 dot_locals_str = _PyUnicode_FromId(&dot_locals);
713 if (dot_locals_str == NULL)
714 return 0;
715 base = PyUnicode_Concat(parent->u_qualname, dot_locals_str);
716 if (base == NULL)
717 return 0;
718 }
719 else {
720 Py_INCREF(parent->u_qualname);
721 base = parent->u_qualname;
722 }
723 }
724 }
725
726 if (base != NULL) {
727 dot_str = _PyUnicode_FromId(&dot);
728 if (dot_str == NULL) {
729 Py_DECREF(base);
730 return 0;
731 }
732 name = PyUnicode_Concat(base, dot_str);
733 Py_DECREF(base);
734 if (name == NULL)
735 return 0;
736 PyUnicode_Append(&name, u->u_name);
737 if (name == NULL)
738 return 0;
739 }
740 else {
741 Py_INCREF(u->u_name);
742 name = u->u_name;
743 }
744 u->u_qualname = name;
745
746 return 1;
747 }
748
749
750 /* Allocate a new block and return a pointer to it.
751 Returns NULL on error.
752 */
753
754 static basicblock *
compiler_new_block(struct compiler * c)755 compiler_new_block(struct compiler *c)
756 {
757 basicblock *b;
758 struct compiler_unit *u;
759
760 u = c->u;
761 b = (basicblock *)PyObject_Malloc(sizeof(basicblock));
762 if (b == NULL) {
763 PyErr_NoMemory();
764 return NULL;
765 }
766 memset((void *)b, 0, sizeof(basicblock));
767 /* Extend the singly linked list of blocks with new block. */
768 b->b_list = u->u_blocks;
769 u->u_blocks = b;
770 return b;
771 }
772
773 static basicblock *
compiler_next_block(struct compiler * c)774 compiler_next_block(struct compiler *c)
775 {
776 basicblock *block = compiler_new_block(c);
777 if (block == NULL)
778 return NULL;
779 c->u->u_curblock->b_next = block;
780 c->u->u_curblock = block;
781 return block;
782 }
783
784 static basicblock *
compiler_use_next_block(struct compiler * c,basicblock * block)785 compiler_use_next_block(struct compiler *c, basicblock *block)
786 {
787 assert(block != NULL);
788 c->u->u_curblock->b_next = block;
789 c->u->u_curblock = block;
790 return block;
791 }
792
793 /* Returns the offset of the next instruction in the current block's
794 b_instr array. Resizes the b_instr as necessary.
795 Returns -1 on failure.
796 */
797
798 static int
compiler_next_instr(struct compiler * c,basicblock * b)799 compiler_next_instr(struct compiler *c, basicblock *b)
800 {
801 assert(b != NULL);
802 if (b->b_instr == NULL) {
803 b->b_instr = (struct instr *)PyObject_Malloc(
804 sizeof(struct instr) * DEFAULT_BLOCK_SIZE);
805 if (b->b_instr == NULL) {
806 PyErr_NoMemory();
807 return -1;
808 }
809 b->b_ialloc = DEFAULT_BLOCK_SIZE;
810 memset((char *)b->b_instr, 0,
811 sizeof(struct instr) * DEFAULT_BLOCK_SIZE);
812 }
813 else if (b->b_iused == b->b_ialloc) {
814 struct instr *tmp;
815 size_t oldsize, newsize;
816 oldsize = b->b_ialloc * sizeof(struct instr);
817 newsize = oldsize << 1;
818
819 if (oldsize > (SIZE_MAX >> 1)) {
820 PyErr_NoMemory();
821 return -1;
822 }
823
824 if (newsize == 0) {
825 PyErr_NoMemory();
826 return -1;
827 }
828 b->b_ialloc <<= 1;
829 tmp = (struct instr *)PyObject_Realloc(
830 (void *)b->b_instr, newsize);
831 if (tmp == NULL) {
832 PyErr_NoMemory();
833 return -1;
834 }
835 b->b_instr = tmp;
836 memset((char *)b->b_instr + oldsize, 0, newsize - oldsize);
837 }
838 return b->b_iused++;
839 }
840
841 /* Set the i_lineno member of the instruction at offset off if the
842 line number for the current expression/statement has not
843 already been set. If it has been set, the call has no effect.
844
845 The line number is reset in the following cases:
846 - when entering a new scope
847 - on each statement
848 - on each expression that start a new line
849 - before the "except" clause
850 - before the "for" and "while" expressions
851 */
852
853 static void
compiler_set_lineno(struct compiler * c,int off)854 compiler_set_lineno(struct compiler *c, int off)
855 {
856 basicblock *b;
857 if (c->u->u_lineno_set)
858 return;
859 c->u->u_lineno_set = 1;
860 b = c->u->u_curblock;
861 b->b_instr[off].i_lineno = c->u->u_lineno;
862 }
863
864 /* Return the stack effect of opcode with argument oparg.
865
866 Some opcodes have different stack effect when jump to the target and
867 when not jump. The 'jump' parameter specifies the case:
868
869 * 0 -- when not jump
870 * 1 -- when jump
871 * -1 -- maximal
872 */
873 /* XXX Make the stack effect of WITH_CLEANUP_START and
874 WITH_CLEANUP_FINISH deterministic. */
875 static int
stack_effect(int opcode,int oparg,int jump)876 stack_effect(int opcode, int oparg, int jump)
877 {
878 switch (opcode) {
879 /* Stack manipulation */
880 case POP_TOP:
881 return -1;
882 case ROT_TWO:
883 case ROT_THREE:
884 return 0;
885 case DUP_TOP:
886 return 1;
887 case DUP_TOP_TWO:
888 return 2;
889
890 /* Unary operators */
891 case UNARY_POSITIVE:
892 case UNARY_NEGATIVE:
893 case UNARY_NOT:
894 case UNARY_INVERT:
895 return 0;
896
897 case SET_ADD:
898 case LIST_APPEND:
899 return -1;
900 case MAP_ADD:
901 return -2;
902
903 /* Binary operators */
904 case BINARY_POWER:
905 case BINARY_MULTIPLY:
906 case BINARY_MATRIX_MULTIPLY:
907 case BINARY_MODULO:
908 case BINARY_ADD:
909 case BINARY_SUBTRACT:
910 case BINARY_SUBSCR:
911 case BINARY_FLOOR_DIVIDE:
912 case BINARY_TRUE_DIVIDE:
913 return -1;
914 case INPLACE_FLOOR_DIVIDE:
915 case INPLACE_TRUE_DIVIDE:
916 return -1;
917
918 case INPLACE_ADD:
919 case INPLACE_SUBTRACT:
920 case INPLACE_MULTIPLY:
921 case INPLACE_MATRIX_MULTIPLY:
922 case INPLACE_MODULO:
923 return -1;
924 case STORE_SUBSCR:
925 return -3;
926 case DELETE_SUBSCR:
927 return -2;
928
929 case BINARY_LSHIFT:
930 case BINARY_RSHIFT:
931 case BINARY_AND:
932 case BINARY_XOR:
933 case BINARY_OR:
934 return -1;
935 case INPLACE_POWER:
936 return -1;
937 case GET_ITER:
938 return 0;
939
940 case PRINT_EXPR:
941 return -1;
942 case LOAD_BUILD_CLASS:
943 return 1;
944 case INPLACE_LSHIFT:
945 case INPLACE_RSHIFT:
946 case INPLACE_AND:
947 case INPLACE_XOR:
948 case INPLACE_OR:
949 return -1;
950 case BREAK_LOOP:
951 return 0;
952 case SETUP_WITH:
953 /* 1 in the normal flow.
954 * Restore the stack position and push 6 values before jumping to
955 * the handler if an exception be raised. */
956 return jump ? 6 : 1;
957 case WITH_CLEANUP_START:
958 return 2; /* or 1, depending on TOS */
959 case WITH_CLEANUP_FINISH:
960 /* Pop a variable number of values pushed by WITH_CLEANUP_START
961 * + __exit__ or __aexit__. */
962 return -3;
963 case RETURN_VALUE:
964 return -1;
965 case IMPORT_STAR:
966 return -1;
967 case SETUP_ANNOTATIONS:
968 return 0;
969 case YIELD_VALUE:
970 return 0;
971 case YIELD_FROM:
972 return -1;
973 case POP_BLOCK:
974 return 0;
975 case POP_EXCEPT:
976 return -3;
977 case END_FINALLY:
978 /* Pop 6 values when an exception was raised. */
979 return -6;
980
981 case STORE_NAME:
982 return -1;
983 case DELETE_NAME:
984 return 0;
985 case UNPACK_SEQUENCE:
986 return oparg-1;
987 case UNPACK_EX:
988 return (oparg&0xFF) + (oparg>>8);
989 case FOR_ITER:
990 /* -1 at end of iterator, 1 if continue iterating. */
991 return jump > 0 ? -1 : 1;
992
993 case STORE_ATTR:
994 return -2;
995 case DELETE_ATTR:
996 return -1;
997 case STORE_GLOBAL:
998 return -1;
999 case DELETE_GLOBAL:
1000 return 0;
1001 case LOAD_CONST:
1002 return 1;
1003 case LOAD_NAME:
1004 return 1;
1005 case BUILD_TUPLE:
1006 case BUILD_LIST:
1007 case BUILD_SET:
1008 case BUILD_STRING:
1009 return 1-oparg;
1010 case BUILD_LIST_UNPACK:
1011 case BUILD_TUPLE_UNPACK:
1012 case BUILD_TUPLE_UNPACK_WITH_CALL:
1013 case BUILD_SET_UNPACK:
1014 case BUILD_MAP_UNPACK:
1015 case BUILD_MAP_UNPACK_WITH_CALL:
1016 return 1 - oparg;
1017 case BUILD_MAP:
1018 return 1 - 2*oparg;
1019 case BUILD_CONST_KEY_MAP:
1020 return -oparg;
1021 case LOAD_ATTR:
1022 return 0;
1023 case COMPARE_OP:
1024 return -1;
1025 case IMPORT_NAME:
1026 return -1;
1027 case IMPORT_FROM:
1028 return 1;
1029
1030 /* Jumps */
1031 case JUMP_FORWARD:
1032 case JUMP_ABSOLUTE:
1033 return 0;
1034
1035 case JUMP_IF_TRUE_OR_POP:
1036 case JUMP_IF_FALSE_OR_POP:
1037 return jump ? 0 : -1;
1038
1039 case POP_JUMP_IF_FALSE:
1040 case POP_JUMP_IF_TRUE:
1041 return -1;
1042
1043 case LOAD_GLOBAL:
1044 return 1;
1045
1046 case CONTINUE_LOOP:
1047 return 0;
1048 case SETUP_LOOP:
1049 return 0;
1050 case SETUP_EXCEPT:
1051 case SETUP_FINALLY:
1052 /* 0 in the normal flow.
1053 * Restore the stack position and push 6 values before jumping to
1054 * the handler if an exception be raised. */
1055 return jump ? 6 : 0;
1056
1057 case LOAD_FAST:
1058 return 1;
1059 case STORE_FAST:
1060 return -1;
1061 case DELETE_FAST:
1062 return 0;
1063
1064 case RAISE_VARARGS:
1065 return -oparg;
1066
1067 /* Functions and calls */
1068 case CALL_FUNCTION:
1069 return -oparg;
1070 case CALL_METHOD:
1071 return -oparg-1;
1072 case CALL_FUNCTION_KW:
1073 return -oparg-1;
1074 case CALL_FUNCTION_EX:
1075 return -1 - ((oparg & 0x01) != 0);
1076 case MAKE_FUNCTION:
1077 return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) -
1078 ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0);
1079 case BUILD_SLICE:
1080 if (oparg == 3)
1081 return -2;
1082 else
1083 return -1;
1084
1085 /* Closures */
1086 case LOAD_CLOSURE:
1087 return 1;
1088 case LOAD_DEREF:
1089 case LOAD_CLASSDEREF:
1090 return 1;
1091 case STORE_DEREF:
1092 return -1;
1093 case DELETE_DEREF:
1094 return 0;
1095
1096 /* Iterators and generators */
1097 case GET_AWAITABLE:
1098 return 0;
1099 case SETUP_ASYNC_WITH:
1100 /* 0 in the normal flow.
1101 * Restore the stack position to the position before the result
1102 * of __aenter__ and push 6 values before jumping to the handler
1103 * if an exception be raised. */
1104 return jump ? -1 + 6 : 0;
1105 case BEFORE_ASYNC_WITH:
1106 return 1;
1107 case GET_AITER:
1108 return 0;
1109 case GET_ANEXT:
1110 return 1;
1111 case GET_YIELD_FROM_ITER:
1112 return 0;
1113 case FORMAT_VALUE:
1114 /* If there's a fmt_spec on the stack, we go from 2->1,
1115 else 1->1. */
1116 return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0;
1117 case LOAD_METHOD:
1118 return 1;
1119 default:
1120 return PY_INVALID_STACK_EFFECT;
1121 }
1122 return PY_INVALID_STACK_EFFECT; /* not reachable */
1123 }
1124
1125 int
PyCompile_OpcodeStackEffect(int opcode,int oparg)1126 PyCompile_OpcodeStackEffect(int opcode, int oparg)
1127 {
1128 return stack_effect(opcode, oparg, -1);
1129 }
1130
1131 /* Add an opcode with no argument.
1132 Returns 0 on failure, 1 on success.
1133 */
1134
1135 static int
compiler_addop(struct compiler * c,int opcode)1136 compiler_addop(struct compiler *c, int opcode)
1137 {
1138 basicblock *b;
1139 struct instr *i;
1140 int off;
1141 assert(!HAS_ARG(opcode));
1142 off = compiler_next_instr(c, c->u->u_curblock);
1143 if (off < 0)
1144 return 0;
1145 b = c->u->u_curblock;
1146 i = &b->b_instr[off];
1147 i->i_opcode = opcode;
1148 i->i_oparg = 0;
1149 if (opcode == RETURN_VALUE)
1150 b->b_return = 1;
1151 compiler_set_lineno(c, off);
1152 return 1;
1153 }
1154
1155 static Py_ssize_t
compiler_add_o(struct compiler * c,PyObject * dict,PyObject * o)1156 compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o)
1157 {
1158 PyObject *t, *v;
1159 Py_ssize_t arg;
1160
1161 t = _PyCode_ConstantKey(o);
1162 if (t == NULL)
1163 return -1;
1164
1165 v = PyDict_GetItem(dict, t);
1166 if (!v) {
1167 if (PyErr_Occurred()) {
1168 Py_DECREF(t);
1169 return -1;
1170 }
1171 arg = PyDict_GET_SIZE(dict);
1172 v = PyLong_FromSsize_t(arg);
1173 if (!v) {
1174 Py_DECREF(t);
1175 return -1;
1176 }
1177 if (PyDict_SetItem(dict, t, v) < 0) {
1178 Py_DECREF(t);
1179 Py_DECREF(v);
1180 return -1;
1181 }
1182 Py_DECREF(v);
1183 }
1184 else
1185 arg = PyLong_AsLong(v);
1186 Py_DECREF(t);
1187 return arg;
1188 }
1189
1190 static int
compiler_addop_o(struct compiler * c,int opcode,PyObject * dict,PyObject * o)1191 compiler_addop_o(struct compiler *c, int opcode, PyObject *dict,
1192 PyObject *o)
1193 {
1194 Py_ssize_t arg = compiler_add_o(c, dict, o);
1195 if (arg < 0)
1196 return 0;
1197 return compiler_addop_i(c, opcode, arg);
1198 }
1199
1200 static int
compiler_addop_name(struct compiler * c,int opcode,PyObject * dict,PyObject * o)1201 compiler_addop_name(struct compiler *c, int opcode, PyObject *dict,
1202 PyObject *o)
1203 {
1204 Py_ssize_t arg;
1205 PyObject *mangled = _Py_Mangle(c->u->u_private, o);
1206 if (!mangled)
1207 return 0;
1208 arg = compiler_add_o(c, dict, mangled);
1209 Py_DECREF(mangled);
1210 if (arg < 0)
1211 return 0;
1212 return compiler_addop_i(c, opcode, arg);
1213 }
1214
1215 /* Add an opcode with an integer argument.
1216 Returns 0 on failure, 1 on success.
1217 */
1218
1219 static int
compiler_addop_i(struct compiler * c,int opcode,Py_ssize_t oparg)1220 compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
1221 {
1222 struct instr *i;
1223 int off;
1224
1225 /* oparg value is unsigned, but a signed C int is usually used to store
1226 it in the C code (like Python/ceval.c).
1227
1228 Limit to 32-bit signed C int (rather than INT_MAX) for portability.
1229
1230 The argument of a concrete bytecode instruction is limited to 8-bit.
1231 EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */
1232 assert(HAS_ARG(opcode));
1233 assert(0 <= oparg && oparg <= 2147483647);
1234
1235 off = compiler_next_instr(c, c->u->u_curblock);
1236 if (off < 0)
1237 return 0;
1238 i = &c->u->u_curblock->b_instr[off];
1239 i->i_opcode = opcode;
1240 i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
1241 compiler_set_lineno(c, off);
1242 return 1;
1243 }
1244
1245 static int
compiler_addop_j(struct compiler * c,int opcode,basicblock * b,int absolute)1246 compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
1247 {
1248 struct instr *i;
1249 int off;
1250
1251 assert(HAS_ARG(opcode));
1252 assert(b != NULL);
1253 off = compiler_next_instr(c, c->u->u_curblock);
1254 if (off < 0)
1255 return 0;
1256 i = &c->u->u_curblock->b_instr[off];
1257 i->i_opcode = opcode;
1258 i->i_target = b;
1259 if (absolute)
1260 i->i_jabs = 1;
1261 else
1262 i->i_jrel = 1;
1263 compiler_set_lineno(c, off);
1264 return 1;
1265 }
1266
1267 /* NEXT_BLOCK() creates an implicit jump from the current block
1268 to the new block.
1269
1270 The returns inside this macro make it impossible to decref objects
1271 created in the local function. Local objects should use the arena.
1272 */
1273 #define NEXT_BLOCK(C) { \
1274 if (compiler_next_block((C)) == NULL) \
1275 return 0; \
1276 }
1277
1278 #define ADDOP(C, OP) { \
1279 if (!compiler_addop((C), (OP))) \
1280 return 0; \
1281 }
1282
1283 #define ADDOP_IN_SCOPE(C, OP) { \
1284 if (!compiler_addop((C), (OP))) { \
1285 compiler_exit_scope(c); \
1286 return 0; \
1287 } \
1288 }
1289
1290 #define ADDOP_O(C, OP, O, TYPE) { \
1291 if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) \
1292 return 0; \
1293 }
1294
1295 /* Same as ADDOP_O, but steals a reference. */
1296 #define ADDOP_N(C, OP, O, TYPE) { \
1297 if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \
1298 Py_DECREF((O)); \
1299 return 0; \
1300 } \
1301 Py_DECREF((O)); \
1302 }
1303
1304 #define ADDOP_NAME(C, OP, O, TYPE) { \
1305 if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \
1306 return 0; \
1307 }
1308
1309 #define ADDOP_I(C, OP, O) { \
1310 if (!compiler_addop_i((C), (OP), (O))) \
1311 return 0; \
1312 }
1313
1314 #define ADDOP_JABS(C, OP, O) { \
1315 if (!compiler_addop_j((C), (OP), (O), 1)) \
1316 return 0; \
1317 }
1318
1319 #define ADDOP_JREL(C, OP, O) { \
1320 if (!compiler_addop_j((C), (OP), (O), 0)) \
1321 return 0; \
1322 }
1323
1324 /* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use
1325 the ASDL name to synthesize the name of the C type and the visit function.
1326 */
1327
1328 #define VISIT(C, TYPE, V) {\
1329 if (!compiler_visit_ ## TYPE((C), (V))) \
1330 return 0; \
1331 }
1332
1333 #define VISIT_IN_SCOPE(C, TYPE, V) {\
1334 if (!compiler_visit_ ## TYPE((C), (V))) { \
1335 compiler_exit_scope(c); \
1336 return 0; \
1337 } \
1338 }
1339
1340 #define VISIT_SLICE(C, V, CTX) {\
1341 if (!compiler_visit_slice((C), (V), (CTX))) \
1342 return 0; \
1343 }
1344
1345 #define VISIT_SEQ(C, TYPE, SEQ) { \
1346 int _i; \
1347 asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1348 for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
1349 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
1350 if (!compiler_visit_ ## TYPE((C), elt)) \
1351 return 0; \
1352 } \
1353 }
1354
1355 #define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) { \
1356 int _i; \
1357 asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1358 for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
1359 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
1360 if (!compiler_visit_ ## TYPE((C), elt)) { \
1361 compiler_exit_scope(c); \
1362 return 0; \
1363 } \
1364 } \
1365 }
1366
1367 static int
compiler_isdocstring(stmt_ty s)1368 compiler_isdocstring(stmt_ty s)
1369 {
1370 if (s->kind != Expr_kind)
1371 return 0;
1372 if (s->v.Expr.value->kind == Str_kind)
1373 return 1;
1374 if (s->v.Expr.value->kind == Constant_kind)
1375 return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
1376 return 0;
1377 }
1378
1379 static int
is_const(expr_ty e)1380 is_const(expr_ty e)
1381 {
1382 switch (e->kind) {
1383 case Constant_kind:
1384 case Num_kind:
1385 case Str_kind:
1386 case Bytes_kind:
1387 case Ellipsis_kind:
1388 case NameConstant_kind:
1389 return 1;
1390 default:
1391 return 0;
1392 }
1393 }
1394
1395 static PyObject *
get_const_value(expr_ty e)1396 get_const_value(expr_ty e)
1397 {
1398 switch (e->kind) {
1399 case Constant_kind:
1400 return e->v.Constant.value;
1401 case Num_kind:
1402 return e->v.Num.n;
1403 case Str_kind:
1404 return e->v.Str.s;
1405 case Bytes_kind:
1406 return e->v.Bytes.s;
1407 case Ellipsis_kind:
1408 return Py_Ellipsis;
1409 case NameConstant_kind:
1410 return e->v.NameConstant.value;
1411 default:
1412 Py_UNREACHABLE();
1413 }
1414 }
1415
1416 /* Search if variable annotations are present statically in a block. */
1417
1418 static int
find_ann(asdl_seq * stmts)1419 find_ann(asdl_seq *stmts)
1420 {
1421 int i, j, res = 0;
1422 stmt_ty st;
1423
1424 for (i = 0; i < asdl_seq_LEN(stmts); i++) {
1425 st = (stmt_ty)asdl_seq_GET(stmts, i);
1426 switch (st->kind) {
1427 case AnnAssign_kind:
1428 return 1;
1429 case For_kind:
1430 res = find_ann(st->v.For.body) ||
1431 find_ann(st->v.For.orelse);
1432 break;
1433 case AsyncFor_kind:
1434 res = find_ann(st->v.AsyncFor.body) ||
1435 find_ann(st->v.AsyncFor.orelse);
1436 break;
1437 case While_kind:
1438 res = find_ann(st->v.While.body) ||
1439 find_ann(st->v.While.orelse);
1440 break;
1441 case If_kind:
1442 res = find_ann(st->v.If.body) ||
1443 find_ann(st->v.If.orelse);
1444 break;
1445 case With_kind:
1446 res = find_ann(st->v.With.body);
1447 break;
1448 case AsyncWith_kind:
1449 res = find_ann(st->v.AsyncWith.body);
1450 break;
1451 case Try_kind:
1452 for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) {
1453 excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
1454 st->v.Try.handlers, j);
1455 if (find_ann(handler->v.ExceptHandler.body)) {
1456 return 1;
1457 }
1458 }
1459 res = find_ann(st->v.Try.body) ||
1460 find_ann(st->v.Try.finalbody) ||
1461 find_ann(st->v.Try.orelse);
1462 break;
1463 default:
1464 res = 0;
1465 }
1466 if (res) {
1467 break;
1468 }
1469 }
1470 return res;
1471 }
1472
1473 /* Compile a sequence of statements, checking for a docstring
1474 and for annotations. */
1475
1476 static int
compiler_body(struct compiler * c,asdl_seq * stmts)1477 compiler_body(struct compiler *c, asdl_seq *stmts)
1478 {
1479 int i = 0;
1480 stmt_ty st;
1481
1482 /* Set current line number to the line number of first statement.
1483 This way line number for SETUP_ANNOTATIONS will always
1484 coincide with the line number of first "real" statement in module.
1485 If body is empty, then lineno will be set later in assemble. */
1486 if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
1487 !c->u->u_lineno && asdl_seq_LEN(stmts)) {
1488 st = (stmt_ty)asdl_seq_GET(stmts, 0);
1489 c->u->u_lineno = st->lineno;
1490 }
1491 /* Every annotated class and module should have __annotations__. */
1492 if (find_ann(stmts)) {
1493 ADDOP(c, SETUP_ANNOTATIONS);
1494 }
1495 if (!asdl_seq_LEN(stmts))
1496 return 1;
1497 st = (stmt_ty)asdl_seq_GET(stmts, 0);
1498 /* if not -OO mode, set docstring */
1499 if (compiler_isdocstring(st) && c->c_optimize < 2) {
1500 /* don't generate docstrings if -OO */
1501 i = 1;
1502 VISIT(c, expr, st->v.Expr.value);
1503 if (!compiler_nameop(c, __doc__, Store))
1504 return 0;
1505 }
1506 for (; i < asdl_seq_LEN(stmts); i++)
1507 VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
1508 return 1;
1509 }
1510
1511 static PyCodeObject *
compiler_mod(struct compiler * c,mod_ty mod)1512 compiler_mod(struct compiler *c, mod_ty mod)
1513 {
1514 PyCodeObject *co;
1515 int addNone = 1;
1516 static PyObject *module;
1517 if (!module) {
1518 module = PyUnicode_InternFromString("<module>");
1519 if (!module)
1520 return NULL;
1521 }
1522 /* Use 0 for firstlineno initially, will fixup in assemble(). */
1523 if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 0))
1524 return NULL;
1525 switch (mod->kind) {
1526 case Module_kind:
1527 if (!compiler_body(c, mod->v.Module.body)) {
1528 compiler_exit_scope(c);
1529 return 0;
1530 }
1531 break;
1532 case Interactive_kind:
1533 if (find_ann(mod->v.Interactive.body)) {
1534 ADDOP(c, SETUP_ANNOTATIONS);
1535 }
1536 c->c_interactive = 1;
1537 VISIT_SEQ_IN_SCOPE(c, stmt,
1538 mod->v.Interactive.body);
1539 break;
1540 case Expression_kind:
1541 VISIT_IN_SCOPE(c, expr, mod->v.Expression.body);
1542 addNone = 0;
1543 break;
1544 case Suite_kind:
1545 PyErr_SetString(PyExc_SystemError,
1546 "suite should not be possible");
1547 return 0;
1548 default:
1549 PyErr_Format(PyExc_SystemError,
1550 "module kind %d should not be possible",
1551 mod->kind);
1552 return 0;
1553 }
1554 co = assemble(c, addNone);
1555 compiler_exit_scope(c);
1556 return co;
1557 }
1558
1559 /* The test for LOCAL must come before the test for FREE in order to
1560 handle classes where name is both local and free. The local var is
1561 a method and the free var is a free var referenced within a method.
1562 */
1563
1564 static int
get_ref_type(struct compiler * c,PyObject * name)1565 get_ref_type(struct compiler *c, PyObject *name)
1566 {
1567 int scope;
1568 if (c->u->u_scope_type == COMPILER_SCOPE_CLASS &&
1569 _PyUnicode_EqualToASCIIString(name, "__class__"))
1570 return CELL;
1571 scope = PyST_GetScope(c->u->u_ste, name);
1572 if (scope == 0) {
1573 char buf[350];
1574 PyOS_snprintf(buf, sizeof(buf),
1575 "unknown scope for %.100s in %.100s(%s)\n"
1576 "symbols: %s\nlocals: %s\nglobals: %s",
1577 PyUnicode_AsUTF8(name),
1578 PyUnicode_AsUTF8(c->u->u_name),
1579 PyUnicode_AsUTF8(PyObject_Repr(c->u->u_ste->ste_id)),
1580 PyUnicode_AsUTF8(PyObject_Repr(c->u->u_ste->ste_symbols)),
1581 PyUnicode_AsUTF8(PyObject_Repr(c->u->u_varnames)),
1582 PyUnicode_AsUTF8(PyObject_Repr(c->u->u_names))
1583 );
1584 Py_FatalError(buf);
1585 }
1586
1587 return scope;
1588 }
1589
1590 static int
compiler_lookup_arg(PyObject * dict,PyObject * name)1591 compiler_lookup_arg(PyObject *dict, PyObject *name)
1592 {
1593 PyObject *k, *v;
1594 k = _PyCode_ConstantKey(name);
1595 if (k == NULL)
1596 return -1;
1597 v = PyDict_GetItem(dict, k);
1598 Py_DECREF(k);
1599 if (v == NULL)
1600 return -1;
1601 return PyLong_AS_LONG(v);
1602 }
1603
1604 static int
compiler_make_closure(struct compiler * c,PyCodeObject * co,Py_ssize_t flags,PyObject * qualname)1605 compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname)
1606 {
1607 Py_ssize_t i, free = PyCode_GetNumFree(co);
1608 if (qualname == NULL)
1609 qualname = co->co_name;
1610
1611 if (free) {
1612 for (i = 0; i < free; ++i) {
1613 /* Bypass com_addop_varname because it will generate
1614 LOAD_DEREF but LOAD_CLOSURE is needed.
1615 */
1616 PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
1617 int arg, reftype;
1618
1619 /* Special case: If a class contains a method with a
1620 free variable that has the same name as a method,
1621 the name will be considered free *and* local in the
1622 class. It should be handled by the closure, as
1623 well as by the normal name loookup logic.
1624 */
1625 reftype = get_ref_type(c, name);
1626 if (reftype == CELL)
1627 arg = compiler_lookup_arg(c->u->u_cellvars, name);
1628 else /* (reftype == FREE) */
1629 arg = compiler_lookup_arg(c->u->u_freevars, name);
1630 if (arg == -1) {
1631 fprintf(stderr,
1632 "lookup %s in %s %d %d\n"
1633 "freevars of %s: %s\n",
1634 PyUnicode_AsUTF8(PyObject_Repr(name)),
1635 PyUnicode_AsUTF8(c->u->u_name),
1636 reftype, arg,
1637 PyUnicode_AsUTF8(co->co_name),
1638 PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars)));
1639 Py_FatalError("compiler_make_closure()");
1640 }
1641 ADDOP_I(c, LOAD_CLOSURE, arg);
1642 }
1643 flags |= 0x08;
1644 ADDOP_I(c, BUILD_TUPLE, free);
1645 }
1646 ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts);
1647 ADDOP_O(c, LOAD_CONST, qualname, consts);
1648 ADDOP_I(c, MAKE_FUNCTION, flags);
1649 return 1;
1650 }
1651
1652 static int
compiler_decorators(struct compiler * c,asdl_seq * decos)1653 compiler_decorators(struct compiler *c, asdl_seq* decos)
1654 {
1655 int i;
1656
1657 if (!decos)
1658 return 1;
1659
1660 for (i = 0; i < asdl_seq_LEN(decos); i++) {
1661 VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i));
1662 }
1663 return 1;
1664 }
1665
1666 static int
compiler_visit_kwonlydefaults(struct compiler * c,asdl_seq * kwonlyargs,asdl_seq * kw_defaults)1667 compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs,
1668 asdl_seq *kw_defaults)
1669 {
1670 /* Push a dict of keyword-only default values.
1671
1672 Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed.
1673 */
1674 int i;
1675 PyObject *keys = NULL;
1676
1677 for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
1678 arg_ty arg = asdl_seq_GET(kwonlyargs, i);
1679 expr_ty default_ = asdl_seq_GET(kw_defaults, i);
1680 if (default_) {
1681 PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
1682 if (!mangled) {
1683 goto error;
1684 }
1685 if (keys == NULL) {
1686 keys = PyList_New(1);
1687 if (keys == NULL) {
1688 Py_DECREF(mangled);
1689 return 0;
1690 }
1691 PyList_SET_ITEM(keys, 0, mangled);
1692 }
1693 else {
1694 int res = PyList_Append(keys, mangled);
1695 Py_DECREF(mangled);
1696 if (res == -1) {
1697 goto error;
1698 }
1699 }
1700 if (!compiler_visit_expr(c, default_)) {
1701 goto error;
1702 }
1703 }
1704 }
1705 if (keys != NULL) {
1706 Py_ssize_t default_count = PyList_GET_SIZE(keys);
1707 PyObject *keys_tuple = PyList_AsTuple(keys);
1708 Py_DECREF(keys);
1709 if (keys_tuple == NULL) {
1710 return 0;
1711 }
1712 ADDOP_N(c, LOAD_CONST, keys_tuple, consts);
1713 ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count);
1714 assert(default_count > 0);
1715 return 1;
1716 }
1717 else {
1718 return -1;
1719 }
1720
1721 error:
1722 Py_XDECREF(keys);
1723 return 0;
1724 }
1725
1726 static int
compiler_visit_annexpr(struct compiler * c,expr_ty annotation)1727 compiler_visit_annexpr(struct compiler *c, expr_ty annotation)
1728 {
1729 PyObject *ann_as_str;
1730 ann_as_str = _PyAST_ExprAsUnicode(annotation);
1731 if (!ann_as_str) {
1732 return 0;
1733 }
1734 ADDOP_N(c, LOAD_CONST, ann_as_str, consts);
1735 return 1;
1736 }
1737
1738 static int
compiler_visit_argannotation(struct compiler * c,identifier id,expr_ty annotation,PyObject * names)1739 compiler_visit_argannotation(struct compiler *c, identifier id,
1740 expr_ty annotation, PyObject *names)
1741 {
1742 if (annotation) {
1743 PyObject *mangled;
1744 if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) {
1745 VISIT(c, annexpr, annotation)
1746 }
1747 else {
1748 VISIT(c, expr, annotation);
1749 }
1750 mangled = _Py_Mangle(c->u->u_private, id);
1751 if (!mangled)
1752 return 0;
1753 if (PyList_Append(names, mangled) < 0) {
1754 Py_DECREF(mangled);
1755 return 0;
1756 }
1757 Py_DECREF(mangled);
1758 }
1759 return 1;
1760 }
1761
1762 static int
compiler_visit_argannotations(struct compiler * c,asdl_seq * args,PyObject * names)1763 compiler_visit_argannotations(struct compiler *c, asdl_seq* args,
1764 PyObject *names)
1765 {
1766 int i;
1767 for (i = 0; i < asdl_seq_LEN(args); i++) {
1768 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1769 if (!compiler_visit_argannotation(
1770 c,
1771 arg->arg,
1772 arg->annotation,
1773 names))
1774 return 0;
1775 }
1776 return 1;
1777 }
1778
1779 static int
compiler_visit_annotations(struct compiler * c,arguments_ty args,expr_ty returns)1780 compiler_visit_annotations(struct compiler *c, arguments_ty args,
1781 expr_ty returns)
1782 {
1783 /* Push arg annotation dict.
1784 The expressions are evaluated out-of-order wrt the source code.
1785
1786 Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed.
1787 */
1788 static identifier return_str;
1789 PyObject *names;
1790 Py_ssize_t len;
1791 names = PyList_New(0);
1792 if (!names)
1793 return 0;
1794
1795 if (!compiler_visit_argannotations(c, args->args, names))
1796 goto error;
1797 if (args->vararg && args->vararg->annotation &&
1798 !compiler_visit_argannotation(c, args->vararg->arg,
1799 args->vararg->annotation, names))
1800 goto error;
1801 if (!compiler_visit_argannotations(c, args->kwonlyargs, names))
1802 goto error;
1803 if (args->kwarg && args->kwarg->annotation &&
1804 !compiler_visit_argannotation(c, args->kwarg->arg,
1805 args->kwarg->annotation, names))
1806 goto error;
1807
1808 if (!return_str) {
1809 return_str = PyUnicode_InternFromString("return");
1810 if (!return_str)
1811 goto error;
1812 }
1813 if (!compiler_visit_argannotation(c, return_str, returns, names)) {
1814 goto error;
1815 }
1816
1817 len = PyList_GET_SIZE(names);
1818 if (len) {
1819 PyObject *keytuple = PyList_AsTuple(names);
1820 Py_DECREF(names);
1821 if (keytuple == NULL) {
1822 return 0;
1823 }
1824 ADDOP_N(c, LOAD_CONST, keytuple, consts);
1825 ADDOP_I(c, BUILD_CONST_KEY_MAP, len);
1826 return 1;
1827 }
1828 else {
1829 Py_DECREF(names);
1830 return -1;
1831 }
1832
1833 error:
1834 Py_DECREF(names);
1835 return 0;
1836 }
1837
1838 static int
compiler_visit_defaults(struct compiler * c,arguments_ty args)1839 compiler_visit_defaults(struct compiler *c, arguments_ty args)
1840 {
1841 VISIT_SEQ(c, expr, args->defaults);
1842 ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults));
1843 return 1;
1844 }
1845
1846 static Py_ssize_t
compiler_default_arguments(struct compiler * c,arguments_ty args)1847 compiler_default_arguments(struct compiler *c, arguments_ty args)
1848 {
1849 Py_ssize_t funcflags = 0;
1850 if (args->defaults && asdl_seq_LEN(args->defaults) > 0) {
1851 if (!compiler_visit_defaults(c, args))
1852 return -1;
1853 funcflags |= 0x01;
1854 }
1855 if (args->kwonlyargs) {
1856 int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs,
1857 args->kw_defaults);
1858 if (res == 0) {
1859 return -1;
1860 }
1861 else if (res > 0) {
1862 funcflags |= 0x02;
1863 }
1864 }
1865 return funcflags;
1866 }
1867
1868 static int
compiler_function(struct compiler * c,stmt_ty s,int is_async)1869 compiler_function(struct compiler *c, stmt_ty s, int is_async)
1870 {
1871 PyCodeObject *co;
1872 PyObject *qualname, *first_const = Py_None;
1873 arguments_ty args;
1874 expr_ty returns;
1875 identifier name;
1876 asdl_seq* decos;
1877 asdl_seq *body;
1878 stmt_ty st;
1879 Py_ssize_t i, funcflags;
1880 int docstring;
1881 int annotations;
1882 int scope_type;
1883
1884 if (is_async) {
1885 assert(s->kind == AsyncFunctionDef_kind);
1886
1887 args = s->v.AsyncFunctionDef.args;
1888 returns = s->v.AsyncFunctionDef.returns;
1889 decos = s->v.AsyncFunctionDef.decorator_list;
1890 name = s->v.AsyncFunctionDef.name;
1891 body = s->v.AsyncFunctionDef.body;
1892
1893 scope_type = COMPILER_SCOPE_ASYNC_FUNCTION;
1894 } else {
1895 assert(s->kind == FunctionDef_kind);
1896
1897 args = s->v.FunctionDef.args;
1898 returns = s->v.FunctionDef.returns;
1899 decos = s->v.FunctionDef.decorator_list;
1900 name = s->v.FunctionDef.name;
1901 body = s->v.FunctionDef.body;
1902
1903 scope_type = COMPILER_SCOPE_FUNCTION;
1904 }
1905
1906 if (!compiler_decorators(c, decos))
1907 return 0;
1908
1909 funcflags = compiler_default_arguments(c, args);
1910 if (funcflags == -1) {
1911 return 0;
1912 }
1913
1914 annotations = compiler_visit_annotations(c, args, returns);
1915 if (annotations == 0) {
1916 return 0;
1917 }
1918 else if (annotations > 0) {
1919 funcflags |= 0x04;
1920 }
1921
1922 if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) {
1923 return 0;
1924 }
1925
1926 st = (stmt_ty)asdl_seq_GET(body, 0);
1927 docstring = compiler_isdocstring(st);
1928 if (docstring && c->c_optimize < 2) {
1929 if (st->v.Expr.value->kind == Constant_kind)
1930 first_const = st->v.Expr.value->v.Constant.value;
1931 else
1932 first_const = st->v.Expr.value->v.Str.s;
1933 }
1934 if (compiler_add_o(c, c->u->u_consts, first_const) < 0) {
1935 compiler_exit_scope(c);
1936 return 0;
1937 }
1938
1939 c->u->u_argcount = asdl_seq_LEN(args->args);
1940 c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
1941 VISIT_SEQ_IN_SCOPE(c, stmt, body);
1942 co = assemble(c, 1);
1943 qualname = c->u->u_qualname;
1944 Py_INCREF(qualname);
1945 compiler_exit_scope(c);
1946 if (co == NULL) {
1947 Py_XDECREF(qualname);
1948 Py_XDECREF(co);
1949 return 0;
1950 }
1951
1952 compiler_make_closure(c, co, funcflags, qualname);
1953 Py_DECREF(qualname);
1954 Py_DECREF(co);
1955
1956 /* decorators */
1957 for (i = 0; i < asdl_seq_LEN(decos); i++) {
1958 ADDOP_I(c, CALL_FUNCTION, 1);
1959 }
1960
1961 return compiler_nameop(c, name, Store);
1962 }
1963
1964 static int
compiler_class(struct compiler * c,stmt_ty s)1965 compiler_class(struct compiler *c, stmt_ty s)
1966 {
1967 PyCodeObject *co;
1968 PyObject *str;
1969 int i;
1970 asdl_seq* decos = s->v.ClassDef.decorator_list;
1971
1972 if (!compiler_decorators(c, decos))
1973 return 0;
1974
1975 /* ultimately generate code for:
1976 <name> = __build_class__(<func>, <name>, *<bases>, **<keywords>)
1977 where:
1978 <func> is a function/closure created from the class body;
1979 it has a single argument (__locals__) where the dict
1980 (or MutableSequence) representing the locals is passed
1981 <name> is the class name
1982 <bases> is the positional arguments and *varargs argument
1983 <keywords> is the keyword arguments and **kwds argument
1984 This borrows from compiler_call.
1985 */
1986
1987 /* 1. compile the class body into a code object */
1988 if (!compiler_enter_scope(c, s->v.ClassDef.name,
1989 COMPILER_SCOPE_CLASS, (void *)s, s->lineno))
1990 return 0;
1991 /* this block represents what we do in the new scope */
1992 {
1993 /* use the class name for name mangling */
1994 Py_INCREF(s->v.ClassDef.name);
1995 Py_XSETREF(c->u->u_private, s->v.ClassDef.name);
1996 /* load (global) __name__ ... */
1997 str = PyUnicode_InternFromString("__name__");
1998 if (!str || !compiler_nameop(c, str, Load)) {
1999 Py_XDECREF(str);
2000 compiler_exit_scope(c);
2001 return 0;
2002 }
2003 Py_DECREF(str);
2004 /* ... and store it as __module__ */
2005 str = PyUnicode_InternFromString("__module__");
2006 if (!str || !compiler_nameop(c, str, Store)) {
2007 Py_XDECREF(str);
2008 compiler_exit_scope(c);
2009 return 0;
2010 }
2011 Py_DECREF(str);
2012 assert(c->u->u_qualname);
2013 ADDOP_O(c, LOAD_CONST, c->u->u_qualname, consts);
2014 str = PyUnicode_InternFromString("__qualname__");
2015 if (!str || !compiler_nameop(c, str, Store)) {
2016 Py_XDECREF(str);
2017 compiler_exit_scope(c);
2018 return 0;
2019 }
2020 Py_DECREF(str);
2021 /* compile the body proper */
2022 if (!compiler_body(c, s->v.ClassDef.body)) {
2023 compiler_exit_scope(c);
2024 return 0;
2025 }
2026 /* Return __classcell__ if it is referenced, otherwise return None */
2027 if (c->u->u_ste->ste_needs_class_closure) {
2028 /* Store __classcell__ into class namespace & return it */
2029 str = PyUnicode_InternFromString("__class__");
2030 if (str == NULL) {
2031 compiler_exit_scope(c);
2032 return 0;
2033 }
2034 i = compiler_lookup_arg(c->u->u_cellvars, str);
2035 Py_DECREF(str);
2036 if (i < 0) {
2037 compiler_exit_scope(c);
2038 return 0;
2039 }
2040 assert(i == 0);
2041
2042 ADDOP_I(c, LOAD_CLOSURE, i);
2043 ADDOP(c, DUP_TOP);
2044 str = PyUnicode_InternFromString("__classcell__");
2045 if (!str || !compiler_nameop(c, str, Store)) {
2046 Py_XDECREF(str);
2047 compiler_exit_scope(c);
2048 return 0;
2049 }
2050 Py_DECREF(str);
2051 }
2052 else {
2053 /* No methods referenced __class__, so just return None */
2054 assert(PyDict_GET_SIZE(c->u->u_cellvars) == 0);
2055 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2056 }
2057 ADDOP_IN_SCOPE(c, RETURN_VALUE);
2058 /* create the code object */
2059 co = assemble(c, 1);
2060 }
2061 /* leave the new scope */
2062 compiler_exit_scope(c);
2063 if (co == NULL)
2064 return 0;
2065
2066 /* 2. load the 'build_class' function */
2067 ADDOP(c, LOAD_BUILD_CLASS);
2068
2069 /* 3. load a function (or closure) made from the code object */
2070 compiler_make_closure(c, co, 0, NULL);
2071 Py_DECREF(co);
2072
2073 /* 4. load class name */
2074 ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts);
2075
2076 /* 5. generate the rest of the code for the call */
2077 if (!compiler_call_helper(c, 2,
2078 s->v.ClassDef.bases,
2079 s->v.ClassDef.keywords))
2080 return 0;
2081
2082 /* 6. apply decorators */
2083 for (i = 0; i < asdl_seq_LEN(decos); i++) {
2084 ADDOP_I(c, CALL_FUNCTION, 1);
2085 }
2086
2087 /* 7. store into <name> */
2088 if (!compiler_nameop(c, s->v.ClassDef.name, Store))
2089 return 0;
2090 return 1;
2091 }
2092
2093 static int
cmpop(cmpop_ty op)2094 cmpop(cmpop_ty op)
2095 {
2096 switch (op) {
2097 case Eq:
2098 return PyCmp_EQ;
2099 case NotEq:
2100 return PyCmp_NE;
2101 case Lt:
2102 return PyCmp_LT;
2103 case LtE:
2104 return PyCmp_LE;
2105 case Gt:
2106 return PyCmp_GT;
2107 case GtE:
2108 return PyCmp_GE;
2109 case Is:
2110 return PyCmp_IS;
2111 case IsNot:
2112 return PyCmp_IS_NOT;
2113 case In:
2114 return PyCmp_IN;
2115 case NotIn:
2116 return PyCmp_NOT_IN;
2117 default:
2118 return PyCmp_BAD;
2119 }
2120 }
2121
2122 static int
compiler_jump_if(struct compiler * c,expr_ty e,basicblock * next,int cond)2123 compiler_jump_if(struct compiler *c, expr_ty e, basicblock *next, int cond)
2124 {
2125 switch (e->kind) {
2126 case UnaryOp_kind:
2127 if (e->v.UnaryOp.op == Not)
2128 return compiler_jump_if(c, e->v.UnaryOp.operand, next, !cond);
2129 /* fallback to general implementation */
2130 break;
2131 case BoolOp_kind: {
2132 asdl_seq *s = e->v.BoolOp.values;
2133 Py_ssize_t i, n = asdl_seq_LEN(s) - 1;
2134 assert(n >= 0);
2135 int cond2 = e->v.BoolOp.op == Or;
2136 basicblock *next2 = next;
2137 if (!cond2 != !cond) {
2138 next2 = compiler_new_block(c);
2139 if (next2 == NULL)
2140 return 0;
2141 }
2142 for (i = 0; i < n; ++i) {
2143 if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, i), next2, cond2))
2144 return 0;
2145 }
2146 if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, n), next, cond))
2147 return 0;
2148 if (next2 != next)
2149 compiler_use_next_block(c, next2);
2150 return 1;
2151 }
2152 case IfExp_kind: {
2153 basicblock *end, *next2;
2154 end = compiler_new_block(c);
2155 if (end == NULL)
2156 return 0;
2157 next2 = compiler_new_block(c);
2158 if (next2 == NULL)
2159 return 0;
2160 if (!compiler_jump_if(c, e->v.IfExp.test, next2, 0))
2161 return 0;
2162 if (!compiler_jump_if(c, e->v.IfExp.body, next, cond))
2163 return 0;
2164 ADDOP_JREL(c, JUMP_FORWARD, end);
2165 compiler_use_next_block(c, next2);
2166 if (!compiler_jump_if(c, e->v.IfExp.orelse, next, cond))
2167 return 0;
2168 compiler_use_next_block(c, end);
2169 return 1;
2170 }
2171 case Compare_kind: {
2172 Py_ssize_t i, n = asdl_seq_LEN(e->v.Compare.ops) - 1;
2173 if (n > 0) {
2174 basicblock *cleanup = compiler_new_block(c);
2175 if (cleanup == NULL)
2176 return 0;
2177 VISIT(c, expr, e->v.Compare.left);
2178 for (i = 0; i < n; i++) {
2179 VISIT(c, expr,
2180 (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i));
2181 ADDOP(c, DUP_TOP);
2182 ADDOP(c, ROT_THREE);
2183 ADDOP_I(c, COMPARE_OP,
2184 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, i))));
2185 ADDOP_JABS(c, POP_JUMP_IF_FALSE, cleanup);
2186 NEXT_BLOCK(c);
2187 }
2188 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n));
2189 ADDOP_I(c, COMPARE_OP,
2190 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n))));
2191 ADDOP_JABS(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next);
2192 basicblock *end = compiler_new_block(c);
2193 if (end == NULL)
2194 return 0;
2195 ADDOP_JREL(c, JUMP_FORWARD, end);
2196 compiler_use_next_block(c, cleanup);
2197 ADDOP(c, POP_TOP);
2198 if (!cond) {
2199 ADDOP_JREL(c, JUMP_FORWARD, next);
2200 }
2201 compiler_use_next_block(c, end);
2202 return 1;
2203 }
2204 /* fallback to general implementation */
2205 break;
2206 }
2207 default:
2208 /* fallback to general implementation */
2209 break;
2210 }
2211
2212 /* general implementation */
2213 VISIT(c, expr, e);
2214 ADDOP_JABS(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next);
2215 return 1;
2216 }
2217
2218 static int
compiler_ifexp(struct compiler * c,expr_ty e)2219 compiler_ifexp(struct compiler *c, expr_ty e)
2220 {
2221 basicblock *end, *next;
2222
2223 assert(e->kind == IfExp_kind);
2224 end = compiler_new_block(c);
2225 if (end == NULL)
2226 return 0;
2227 next = compiler_new_block(c);
2228 if (next == NULL)
2229 return 0;
2230 if (!compiler_jump_if(c, e->v.IfExp.test, next, 0))
2231 return 0;
2232 VISIT(c, expr, e->v.IfExp.body);
2233 ADDOP_JREL(c, JUMP_FORWARD, end);
2234 compiler_use_next_block(c, next);
2235 VISIT(c, expr, e->v.IfExp.orelse);
2236 compiler_use_next_block(c, end);
2237 return 1;
2238 }
2239
2240 static int
compiler_lambda(struct compiler * c,expr_ty e)2241 compiler_lambda(struct compiler *c, expr_ty e)
2242 {
2243 PyCodeObject *co;
2244 PyObject *qualname;
2245 static identifier name;
2246 Py_ssize_t funcflags;
2247 arguments_ty args = e->v.Lambda.args;
2248 assert(e->kind == Lambda_kind);
2249
2250 if (!name) {
2251 name = PyUnicode_InternFromString("<lambda>");
2252 if (!name)
2253 return 0;
2254 }
2255
2256 funcflags = compiler_default_arguments(c, args);
2257 if (funcflags == -1) {
2258 return 0;
2259 }
2260
2261 if (!compiler_enter_scope(c, name, COMPILER_SCOPE_LAMBDA,
2262 (void *)e, e->lineno))
2263 return 0;
2264
2265 /* Make None the first constant, so the lambda can't have a
2266 docstring. */
2267 if (compiler_add_o(c, c->u->u_consts, Py_None) < 0)
2268 return 0;
2269
2270 c->u->u_argcount = asdl_seq_LEN(args->args);
2271 c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
2272 VISIT_IN_SCOPE(c, expr, e->v.Lambda.body);
2273 if (c->u->u_ste->ste_generator) {
2274 co = assemble(c, 0);
2275 }
2276 else {
2277 ADDOP_IN_SCOPE(c, RETURN_VALUE);
2278 co = assemble(c, 1);
2279 }
2280 qualname = c->u->u_qualname;
2281 Py_INCREF(qualname);
2282 compiler_exit_scope(c);
2283 if (co == NULL)
2284 return 0;
2285
2286 compiler_make_closure(c, co, funcflags, qualname);
2287 Py_DECREF(qualname);
2288 Py_DECREF(co);
2289
2290 return 1;
2291 }
2292
2293 static int
compiler_if(struct compiler * c,stmt_ty s)2294 compiler_if(struct compiler *c, stmt_ty s)
2295 {
2296 basicblock *end, *next;
2297 int constant;
2298 assert(s->kind == If_kind);
2299 end = compiler_new_block(c);
2300 if (end == NULL)
2301 return 0;
2302
2303 constant = expr_constant(s->v.If.test);
2304 /* constant = 0: "if 0"
2305 * constant = 1: "if 1", "if 2", ...
2306 * constant = -1: rest */
2307 if (constant == 0) {
2308 if (s->v.If.orelse)
2309 VISIT_SEQ(c, stmt, s->v.If.orelse);
2310 } else if (constant == 1) {
2311 VISIT_SEQ(c, stmt, s->v.If.body);
2312 } else {
2313 if (asdl_seq_LEN(s->v.If.orelse)) {
2314 next = compiler_new_block(c);
2315 if (next == NULL)
2316 return 0;
2317 }
2318 else
2319 next = end;
2320 if (!compiler_jump_if(c, s->v.If.test, next, 0))
2321 return 0;
2322 VISIT_SEQ(c, stmt, s->v.If.body);
2323 if (asdl_seq_LEN(s->v.If.orelse)) {
2324 ADDOP_JREL(c, JUMP_FORWARD, end);
2325 compiler_use_next_block(c, next);
2326 VISIT_SEQ(c, stmt, s->v.If.orelse);
2327 }
2328 }
2329 compiler_use_next_block(c, end);
2330 return 1;
2331 }
2332
2333 static int
compiler_for(struct compiler * c,stmt_ty s)2334 compiler_for(struct compiler *c, stmt_ty s)
2335 {
2336 basicblock *start, *cleanup, *end;
2337
2338 start = compiler_new_block(c);
2339 cleanup = compiler_new_block(c);
2340 end = compiler_new_block(c);
2341 if (start == NULL || end == NULL || cleanup == NULL)
2342 return 0;
2343 ADDOP_JREL(c, SETUP_LOOP, end);
2344 if (!compiler_push_fblock(c, LOOP, start))
2345 return 0;
2346 VISIT(c, expr, s->v.For.iter);
2347 ADDOP(c, GET_ITER);
2348 compiler_use_next_block(c, start);
2349 ADDOP_JREL(c, FOR_ITER, cleanup);
2350 VISIT(c, expr, s->v.For.target);
2351 VISIT_SEQ(c, stmt, s->v.For.body);
2352 ADDOP_JABS(c, JUMP_ABSOLUTE, start);
2353 compiler_use_next_block(c, cleanup);
2354 ADDOP(c, POP_BLOCK);
2355 compiler_pop_fblock(c, LOOP, start);
2356 VISIT_SEQ(c, stmt, s->v.For.orelse);
2357 compiler_use_next_block(c, end);
2358 return 1;
2359 }
2360
2361
2362 static int
compiler_async_for(struct compiler * c,stmt_ty s)2363 compiler_async_for(struct compiler *c, stmt_ty s)
2364 {
2365 _Py_IDENTIFIER(StopAsyncIteration);
2366
2367 basicblock *try, *except, *end, *after_try, *try_cleanup,
2368 *after_loop_else;
2369
2370 if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
2371 return compiler_error(c, "'async for' outside async function");
2372 }
2373
2374 PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration);
2375 if (stop_aiter_error == NULL) {
2376 return 0;
2377 }
2378
2379 try = compiler_new_block(c);
2380 except = compiler_new_block(c);
2381 end = compiler_new_block(c);
2382 after_try = compiler_new_block(c);
2383 try_cleanup = compiler_new_block(c);
2384 after_loop_else = compiler_new_block(c);
2385
2386 if (try == NULL || except == NULL || end == NULL
2387 || after_try == NULL || try_cleanup == NULL
2388 || after_loop_else == NULL)
2389 return 0;
2390
2391 ADDOP_JREL(c, SETUP_LOOP, end);
2392 if (!compiler_push_fblock(c, LOOP, try))
2393 return 0;
2394
2395 VISIT(c, expr, s->v.AsyncFor.iter);
2396 ADDOP(c, GET_AITER);
2397
2398 compiler_use_next_block(c, try);
2399
2400
2401 ADDOP_JREL(c, SETUP_EXCEPT, except);
2402 if (!compiler_push_fblock(c, EXCEPT, try))
2403 return 0;
2404
2405 ADDOP(c, GET_ANEXT);
2406 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2407 ADDOP(c, YIELD_FROM);
2408 VISIT(c, expr, s->v.AsyncFor.target);
2409 ADDOP(c, POP_BLOCK);
2410 compiler_pop_fblock(c, EXCEPT, try);
2411 ADDOP_JREL(c, JUMP_FORWARD, after_try);
2412
2413
2414 compiler_use_next_block(c, except);
2415 ADDOP(c, DUP_TOP);
2416 ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names);
2417 ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
2418 ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup);
2419 ADDOP(c, END_FINALLY);
2420
2421 compiler_use_next_block(c, after_try);
2422 VISIT_SEQ(c, stmt, s->v.AsyncFor.body);
2423 ADDOP_JABS(c, JUMP_ABSOLUTE, try);
2424
2425 compiler_use_next_block(c, try_cleanup);
2426 ADDOP(c, POP_TOP);
2427 ADDOP(c, POP_TOP);
2428 ADDOP(c, POP_TOP);
2429 ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
2430 ADDOP(c, POP_TOP); /* for correct calculation of stack effect */
2431 ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
2432 compiler_pop_fblock(c, LOOP, try);
2433
2434 compiler_use_next_block(c, after_loop_else);
2435 VISIT_SEQ(c, stmt, s->v.For.orelse);
2436
2437 compiler_use_next_block(c, end);
2438
2439 return 1;
2440 }
2441
2442 static int
compiler_while(struct compiler * c,stmt_ty s)2443 compiler_while(struct compiler *c, stmt_ty s)
2444 {
2445 basicblock *loop, *orelse, *end, *anchor = NULL;
2446 int constant = expr_constant(s->v.While.test);
2447
2448 if (constant == 0) {
2449 if (s->v.While.orelse)
2450 VISIT_SEQ(c, stmt, s->v.While.orelse);
2451 return 1;
2452 }
2453 loop = compiler_new_block(c);
2454 end = compiler_new_block(c);
2455 if (constant == -1) {
2456 anchor = compiler_new_block(c);
2457 if (anchor == NULL)
2458 return 0;
2459 }
2460 if (loop == NULL || end == NULL)
2461 return 0;
2462 if (s->v.While.orelse) {
2463 orelse = compiler_new_block(c);
2464 if (orelse == NULL)
2465 return 0;
2466 }
2467 else
2468 orelse = NULL;
2469
2470 ADDOP_JREL(c, SETUP_LOOP, end);
2471 compiler_use_next_block(c, loop);
2472 if (!compiler_push_fblock(c, LOOP, loop))
2473 return 0;
2474 if (constant == -1) {
2475 if (!compiler_jump_if(c, s->v.While.test, anchor, 0))
2476 return 0;
2477 }
2478 VISIT_SEQ(c, stmt, s->v.While.body);
2479 ADDOP_JABS(c, JUMP_ABSOLUTE, loop);
2480
2481 /* XXX should the two POP instructions be in a separate block
2482 if there is no else clause ?
2483 */
2484
2485 if (constant == -1)
2486 compiler_use_next_block(c, anchor);
2487 ADDOP(c, POP_BLOCK);
2488 compiler_pop_fblock(c, LOOP, loop);
2489 if (orelse != NULL) /* what if orelse is just pass? */
2490 VISIT_SEQ(c, stmt, s->v.While.orelse);
2491 compiler_use_next_block(c, end);
2492
2493 return 1;
2494 }
2495
2496 static int
compiler_continue(struct compiler * c)2497 compiler_continue(struct compiler *c)
2498 {
2499 static const char LOOP_ERROR_MSG[] = "'continue' not properly in loop";
2500 static const char IN_FINALLY_ERROR_MSG[] =
2501 "'continue' not supported inside 'finally' clause";
2502 int i;
2503
2504 if (!c->u->u_nfblocks)
2505 return compiler_error(c, LOOP_ERROR_MSG);
2506 i = c->u->u_nfblocks - 1;
2507 switch (c->u->u_fblock[i].fb_type) {
2508 case LOOP:
2509 ADDOP_JABS(c, JUMP_ABSOLUTE, c->u->u_fblock[i].fb_block);
2510 break;
2511 case EXCEPT:
2512 case FINALLY_TRY:
2513 while (--i >= 0 && c->u->u_fblock[i].fb_type != LOOP) {
2514 /* Prevent continue anywhere under a finally
2515 even if hidden in a sub-try or except. */
2516 if (c->u->u_fblock[i].fb_type == FINALLY_END)
2517 return compiler_error(c, IN_FINALLY_ERROR_MSG);
2518 }
2519 if (i == -1)
2520 return compiler_error(c, LOOP_ERROR_MSG);
2521 ADDOP_JABS(c, CONTINUE_LOOP, c->u->u_fblock[i].fb_block);
2522 break;
2523 case FINALLY_END:
2524 return compiler_error(c, IN_FINALLY_ERROR_MSG);
2525 }
2526
2527 return 1;
2528 }
2529
2530 /* Code generated for "try: <body> finally: <finalbody>" is as follows:
2531
2532 SETUP_FINALLY L
2533 <code for body>
2534 POP_BLOCK
2535 LOAD_CONST <None>
2536 L: <code for finalbody>
2537 END_FINALLY
2538
2539 The special instructions use the block stack. Each block
2540 stack entry contains the instruction that created it (here
2541 SETUP_FINALLY), the level of the value stack at the time the
2542 block stack entry was created, and a label (here L).
2543
2544 SETUP_FINALLY:
2545 Pushes the current value stack level and the label
2546 onto the block stack.
2547 POP_BLOCK:
2548 Pops en entry from the block stack, and pops the value
2549 stack until its level is the same as indicated on the
2550 block stack. (The label is ignored.)
2551 END_FINALLY:
2552 Pops a variable number of entries from the *value* stack
2553 and re-raises the exception they specify. The number of
2554 entries popped depends on the (pseudo) exception type.
2555
2556 The block stack is unwound when an exception is raised:
2557 when a SETUP_FINALLY entry is found, the exception is pushed
2558 onto the value stack (and the exception condition is cleared),
2559 and the interpreter jumps to the label gotten from the block
2560 stack.
2561 */
2562
2563 static int
compiler_try_finally(struct compiler * c,stmt_ty s)2564 compiler_try_finally(struct compiler *c, stmt_ty s)
2565 {
2566 basicblock *body, *end;
2567 body = compiler_new_block(c);
2568 end = compiler_new_block(c);
2569 if (body == NULL || end == NULL)
2570 return 0;
2571
2572 ADDOP_JREL(c, SETUP_FINALLY, end);
2573 compiler_use_next_block(c, body);
2574 if (!compiler_push_fblock(c, FINALLY_TRY, body))
2575 return 0;
2576 if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) {
2577 if (!compiler_try_except(c, s))
2578 return 0;
2579 }
2580 else {
2581 VISIT_SEQ(c, stmt, s->v.Try.body);
2582 }
2583 ADDOP(c, POP_BLOCK);
2584 compiler_pop_fblock(c, FINALLY_TRY, body);
2585
2586 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2587 compiler_use_next_block(c, end);
2588 if (!compiler_push_fblock(c, FINALLY_END, end))
2589 return 0;
2590 VISIT_SEQ(c, stmt, s->v.Try.finalbody);
2591 ADDOP(c, END_FINALLY);
2592 compiler_pop_fblock(c, FINALLY_END, end);
2593
2594 return 1;
2595 }
2596
2597 /*
2598 Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...":
2599 (The contents of the value stack is shown in [], with the top
2600 at the right; 'tb' is trace-back info, 'val' the exception's
2601 associated value, and 'exc' the exception.)
2602
2603 Value stack Label Instruction Argument
2604 [] SETUP_EXCEPT L1
2605 [] <code for S>
2606 [] POP_BLOCK
2607 [] JUMP_FORWARD L0
2608
2609 [tb, val, exc] L1: DUP )
2610 [tb, val, exc, exc] <evaluate E1> )
2611 [tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1
2612 [tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 )
2613 [tb, val, exc] POP
2614 [tb, val] <assign to V1> (or POP if no V1)
2615 [tb] POP
2616 [] <code for S1>
2617 JUMP_FORWARD L0
2618
2619 [tb, val, exc] L2: DUP
2620 .............................etc.......................
2621
2622 [tb, val, exc] Ln+1: END_FINALLY # re-raise exception
2623
2624 [] L0: <next statement>
2625
2626 Of course, parts are not generated if Vi or Ei is not present.
2627 */
2628 static int
compiler_try_except(struct compiler * c,stmt_ty s)2629 compiler_try_except(struct compiler *c, stmt_ty s)
2630 {
2631 basicblock *body, *orelse, *except, *end;
2632 Py_ssize_t i, n;
2633
2634 body = compiler_new_block(c);
2635 except = compiler_new_block(c);
2636 orelse = compiler_new_block(c);
2637 end = compiler_new_block(c);
2638 if (body == NULL || except == NULL || orelse == NULL || end == NULL)
2639 return 0;
2640 ADDOP_JREL(c, SETUP_EXCEPT, except);
2641 compiler_use_next_block(c, body);
2642 if (!compiler_push_fblock(c, EXCEPT, body))
2643 return 0;
2644 VISIT_SEQ(c, stmt, s->v.Try.body);
2645 ADDOP(c, POP_BLOCK);
2646 compiler_pop_fblock(c, EXCEPT, body);
2647 ADDOP_JREL(c, JUMP_FORWARD, orelse);
2648 n = asdl_seq_LEN(s->v.Try.handlers);
2649 compiler_use_next_block(c, except);
2650 for (i = 0; i < n; i++) {
2651 excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
2652 s->v.Try.handlers, i);
2653 if (!handler->v.ExceptHandler.type && i < n-1)
2654 return compiler_error(c, "default 'except:' must be last");
2655 c->u->u_lineno_set = 0;
2656 c->u->u_lineno = handler->lineno;
2657 c->u->u_col_offset = handler->col_offset;
2658 except = compiler_new_block(c);
2659 if (except == NULL)
2660 return 0;
2661 if (handler->v.ExceptHandler.type) {
2662 ADDOP(c, DUP_TOP);
2663 VISIT(c, expr, handler->v.ExceptHandler.type);
2664 ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
2665 ADDOP_JABS(c, POP_JUMP_IF_FALSE, except);
2666 }
2667 ADDOP(c, POP_TOP);
2668 if (handler->v.ExceptHandler.name) {
2669 basicblock *cleanup_end, *cleanup_body;
2670
2671 cleanup_end = compiler_new_block(c);
2672 cleanup_body = compiler_new_block(c);
2673 if (cleanup_end == NULL || cleanup_body == NULL) {
2674 return 0;
2675 }
2676
2677 compiler_nameop(c, handler->v.ExceptHandler.name, Store);
2678 ADDOP(c, POP_TOP);
2679
2680 /*
2681 try:
2682 # body
2683 except type as name:
2684 try:
2685 # body
2686 finally:
2687 name = None
2688 del name
2689 */
2690
2691 /* second try: */
2692 ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
2693 compiler_use_next_block(c, cleanup_body);
2694 if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
2695 return 0;
2696
2697 /* second # body */
2698 VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
2699 ADDOP(c, POP_BLOCK);
2700 compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
2701
2702 /* finally: */
2703 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2704 compiler_use_next_block(c, cleanup_end);
2705 if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
2706 return 0;
2707
2708 /* name = None */
2709 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2710 compiler_nameop(c, handler->v.ExceptHandler.name, Store);
2711
2712 /* del name */
2713 compiler_nameop(c, handler->v.ExceptHandler.name, Del);
2714
2715 ADDOP(c, END_FINALLY);
2716 ADDOP(c, POP_EXCEPT);
2717 compiler_pop_fblock(c, FINALLY_END, cleanup_end);
2718 }
2719 else {
2720 basicblock *cleanup_body;
2721
2722 cleanup_body = compiler_new_block(c);
2723 if (!cleanup_body)
2724 return 0;
2725
2726 ADDOP(c, POP_TOP);
2727 ADDOP(c, POP_TOP);
2728 compiler_use_next_block(c, cleanup_body);
2729 if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
2730 return 0;
2731 VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
2732 ADDOP(c, POP_EXCEPT);
2733 compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
2734 }
2735 ADDOP_JREL(c, JUMP_FORWARD, end);
2736 compiler_use_next_block(c, except);
2737 }
2738 ADDOP(c, END_FINALLY);
2739 compiler_use_next_block(c, orelse);
2740 VISIT_SEQ(c, stmt, s->v.Try.orelse);
2741 compiler_use_next_block(c, end);
2742 return 1;
2743 }
2744
2745 static int
compiler_try(struct compiler * c,stmt_ty s)2746 compiler_try(struct compiler *c, stmt_ty s) {
2747 if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody))
2748 return compiler_try_finally(c, s);
2749 else
2750 return compiler_try_except(c, s);
2751 }
2752
2753
2754 static int
compiler_import_as(struct compiler * c,identifier name,identifier asname)2755 compiler_import_as(struct compiler *c, identifier name, identifier asname)
2756 {
2757 /* The IMPORT_NAME opcode was already generated. This function
2758 merely needs to bind the result to a name.
2759
2760 If there is a dot in name, we need to split it and emit a
2761 IMPORT_FROM for each name.
2762 */
2763 Py_ssize_t len = PyUnicode_GET_LENGTH(name);
2764 Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1);
2765 if (dot == -2)
2766 return 0;
2767 if (dot != -1) {
2768 /* Consume the base module name to get the first attribute */
2769 while (1) {
2770 Py_ssize_t pos = dot + 1;
2771 PyObject *attr;
2772 dot = PyUnicode_FindChar(name, '.', pos, len, 1);
2773 if (dot == -2)
2774 return 0;
2775 attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len);
2776 if (!attr)
2777 return 0;
2778 ADDOP_N(c, IMPORT_FROM, attr, names);
2779 if (dot == -1) {
2780 break;
2781 }
2782 ADDOP(c, ROT_TWO);
2783 ADDOP(c, POP_TOP);
2784 }
2785 if (!compiler_nameop(c, asname, Store)) {
2786 return 0;
2787 }
2788 ADDOP(c, POP_TOP);
2789 return 1;
2790 }
2791 return compiler_nameop(c, asname, Store);
2792 }
2793
2794 static int
compiler_import(struct compiler * c,stmt_ty s)2795 compiler_import(struct compiler *c, stmt_ty s)
2796 {
2797 /* The Import node stores a module name like a.b.c as a single
2798 string. This is convenient for all cases except
2799 import a.b.c as d
2800 where we need to parse that string to extract the individual
2801 module names.
2802 XXX Perhaps change the representation to make this case simpler?
2803 */
2804 Py_ssize_t i, n = asdl_seq_LEN(s->v.Import.names);
2805
2806 for (i = 0; i < n; i++) {
2807 alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i);
2808 int r;
2809
2810 ADDOP_O(c, LOAD_CONST, _PyLong_Zero, consts);
2811 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2812 ADDOP_NAME(c, IMPORT_NAME, alias->name, names);
2813
2814 if (alias->asname) {
2815 r = compiler_import_as(c, alias->name, alias->asname);
2816 if (!r)
2817 return r;
2818 }
2819 else {
2820 identifier tmp = alias->name;
2821 Py_ssize_t dot = PyUnicode_FindChar(
2822 alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1);
2823 if (dot != -1) {
2824 tmp = PyUnicode_Substring(alias->name, 0, dot);
2825 if (tmp == NULL)
2826 return 0;
2827 }
2828 r = compiler_nameop(c, tmp, Store);
2829 if (dot != -1) {
2830 Py_DECREF(tmp);
2831 }
2832 if (!r)
2833 return r;
2834 }
2835 }
2836 return 1;
2837 }
2838
2839 static int
compiler_from_import(struct compiler * c,stmt_ty s)2840 compiler_from_import(struct compiler *c, stmt_ty s)
2841 {
2842 Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names);
2843 PyObject *level, *names;
2844 static PyObject *empty_string;
2845
2846 if (!empty_string) {
2847 empty_string = PyUnicode_FromString("");
2848 if (!empty_string)
2849 return 0;
2850 }
2851
2852 level = PyLong_FromLong(s->v.ImportFrom.level);
2853 if (!level) {
2854 return 0;
2855 }
2856 ADDOP_N(c, LOAD_CONST, level, consts);
2857
2858 names = PyTuple_New(n);
2859 if (!names)
2860 return 0;
2861
2862 /* build up the names */
2863 for (i = 0; i < n; i++) {
2864 alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
2865 Py_INCREF(alias->name);
2866 PyTuple_SET_ITEM(names, i, alias->name);
2867 }
2868
2869 if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module &&
2870 _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) {
2871 Py_DECREF(names);
2872 return compiler_error(c, "from __future__ imports must occur "
2873 "at the beginning of the file");
2874 }
2875 ADDOP_N(c, LOAD_CONST, names, consts);
2876
2877 if (s->v.ImportFrom.module) {
2878 ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names);
2879 }
2880 else {
2881 ADDOP_NAME(c, IMPORT_NAME, empty_string, names);
2882 }
2883 for (i = 0; i < n; i++) {
2884 alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
2885 identifier store_name;
2886
2887 if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') {
2888 assert(n == 1);
2889 ADDOP(c, IMPORT_STAR);
2890 return 1;
2891 }
2892
2893 ADDOP_NAME(c, IMPORT_FROM, alias->name, names);
2894 store_name = alias->name;
2895 if (alias->asname)
2896 store_name = alias->asname;
2897
2898 if (!compiler_nameop(c, store_name, Store)) {
2899 return 0;
2900 }
2901 }
2902 /* remove imported module */
2903 ADDOP(c, POP_TOP);
2904 return 1;
2905 }
2906
2907 static int
compiler_assert(struct compiler * c,stmt_ty s)2908 compiler_assert(struct compiler *c, stmt_ty s)
2909 {
2910 static PyObject *assertion_error = NULL;
2911 basicblock *end;
2912 PyObject* msg;
2913
2914 if (c->c_optimize)
2915 return 1;
2916 if (assertion_error == NULL) {
2917 assertion_error = PyUnicode_InternFromString("AssertionError");
2918 if (assertion_error == NULL)
2919 return 0;
2920 }
2921 if (s->v.Assert.test->kind == Tuple_kind &&
2922 asdl_seq_LEN(s->v.Assert.test->v.Tuple.elts) > 0) {
2923 msg = PyUnicode_FromString("assertion is always true, "
2924 "perhaps remove parentheses?");
2925 if (msg == NULL)
2926 return 0;
2927 if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg,
2928 c->c_filename, c->u->u_lineno,
2929 NULL, NULL) == -1) {
2930 Py_DECREF(msg);
2931 return 0;
2932 }
2933 Py_DECREF(msg);
2934 }
2935 end = compiler_new_block(c);
2936 if (end == NULL)
2937 return 0;
2938 if (!compiler_jump_if(c, s->v.Assert.test, end, 1))
2939 return 0;
2940 ADDOP_O(c, LOAD_GLOBAL, assertion_error, names);
2941 if (s->v.Assert.msg) {
2942 VISIT(c, expr, s->v.Assert.msg);
2943 ADDOP_I(c, CALL_FUNCTION, 1);
2944 }
2945 ADDOP_I(c, RAISE_VARARGS, 1);
2946 compiler_use_next_block(c, end);
2947 return 1;
2948 }
2949
2950 static int
compiler_visit_stmt_expr(struct compiler * c,expr_ty value)2951 compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
2952 {
2953 if (c->c_interactive && c->c_nestlevel <= 1) {
2954 VISIT(c, expr, value);
2955 ADDOP(c, PRINT_EXPR);
2956 return 1;
2957 }
2958
2959 if (is_const(value)) {
2960 /* ignore constant statement */
2961 return 1;
2962 }
2963
2964 VISIT(c, expr, value);
2965 ADDOP(c, POP_TOP);
2966 return 1;
2967 }
2968
2969 static int
compiler_visit_stmt(struct compiler * c,stmt_ty s)2970 compiler_visit_stmt(struct compiler *c, stmt_ty s)
2971 {
2972 Py_ssize_t i, n;
2973
2974 /* Always assign a lineno to the next instruction for a stmt. */
2975 c->u->u_lineno = s->lineno;
2976 c->u->u_col_offset = s->col_offset;
2977 c->u->u_lineno_set = 0;
2978
2979 switch (s->kind) {
2980 case FunctionDef_kind:
2981 return compiler_function(c, s, 0);
2982 case ClassDef_kind:
2983 return compiler_class(c, s);
2984 case Return_kind:
2985 if (c->u->u_ste->ste_type != FunctionBlock)
2986 return compiler_error(c, "'return' outside function");
2987 if (s->v.Return.value) {
2988 if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator)
2989 return compiler_error(
2990 c, "'return' with value in async generator");
2991 VISIT(c, expr, s->v.Return.value);
2992 }
2993 else
2994 ADDOP_O(c, LOAD_CONST, Py_None, consts);
2995 ADDOP(c, RETURN_VALUE);
2996 break;
2997 case Delete_kind:
2998 VISIT_SEQ(c, expr, s->v.Delete.targets)
2999 break;
3000 case Assign_kind:
3001 n = asdl_seq_LEN(s->v.Assign.targets);
3002 VISIT(c, expr, s->v.Assign.value);
3003 for (i = 0; i < n; i++) {
3004 if (i < n - 1)
3005 ADDOP(c, DUP_TOP);
3006 VISIT(c, expr,
3007 (expr_ty)asdl_seq_GET(s->v.Assign.targets, i));
3008 }
3009 break;
3010 case AugAssign_kind:
3011 return compiler_augassign(c, s);
3012 case AnnAssign_kind:
3013 return compiler_annassign(c, s);
3014 case For_kind:
3015 return compiler_for(c, s);
3016 case While_kind:
3017 return compiler_while(c, s);
3018 case If_kind:
3019 return compiler_if(c, s);
3020 case Raise_kind:
3021 n = 0;
3022 if (s->v.Raise.exc) {
3023 VISIT(c, expr, s->v.Raise.exc);
3024 n++;
3025 if (s->v.Raise.cause) {
3026 VISIT(c, expr, s->v.Raise.cause);
3027 n++;
3028 }
3029 }
3030 ADDOP_I(c, RAISE_VARARGS, (int)n);
3031 break;
3032 case Try_kind:
3033 return compiler_try(c, s);
3034 case Assert_kind:
3035 return compiler_assert(c, s);
3036 case Import_kind:
3037 return compiler_import(c, s);
3038 case ImportFrom_kind:
3039 return compiler_from_import(c, s);
3040 case Global_kind:
3041 case Nonlocal_kind:
3042 break;
3043 case Expr_kind:
3044 return compiler_visit_stmt_expr(c, s->v.Expr.value);
3045 case Pass_kind:
3046 break;
3047 case Break_kind:
3048 if (!compiler_in_loop(c))
3049 return compiler_error(c, "'break' outside loop");
3050 ADDOP(c, BREAK_LOOP);
3051 break;
3052 case Continue_kind:
3053 return compiler_continue(c);
3054 case With_kind:
3055 return compiler_with(c, s, 0);
3056 case AsyncFunctionDef_kind:
3057 return compiler_function(c, s, 1);
3058 case AsyncWith_kind:
3059 return compiler_async_with(c, s, 0);
3060 case AsyncFor_kind:
3061 return compiler_async_for(c, s);
3062 }
3063
3064 return 1;
3065 }
3066
3067 static int
unaryop(unaryop_ty op)3068 unaryop(unaryop_ty op)
3069 {
3070 switch (op) {
3071 case Invert:
3072 return UNARY_INVERT;
3073 case Not:
3074 return UNARY_NOT;
3075 case UAdd:
3076 return UNARY_POSITIVE;
3077 case USub:
3078 return UNARY_NEGATIVE;
3079 default:
3080 PyErr_Format(PyExc_SystemError,
3081 "unary op %d should not be possible", op);
3082 return 0;
3083 }
3084 }
3085
3086 static int
binop(struct compiler * c,operator_ty op)3087 binop(struct compiler *c, operator_ty op)
3088 {
3089 switch (op) {
3090 case Add:
3091 return BINARY_ADD;
3092 case Sub:
3093 return BINARY_SUBTRACT;
3094 case Mult:
3095 return BINARY_MULTIPLY;
3096 case MatMult:
3097 return BINARY_MATRIX_MULTIPLY;
3098 case Div:
3099 return BINARY_TRUE_DIVIDE;
3100 case Mod:
3101 return BINARY_MODULO;
3102 case Pow:
3103 return BINARY_POWER;
3104 case LShift:
3105 return BINARY_LSHIFT;
3106 case RShift:
3107 return BINARY_RSHIFT;
3108 case BitOr:
3109 return BINARY_OR;
3110 case BitXor:
3111 return BINARY_XOR;
3112 case BitAnd:
3113 return BINARY_AND;
3114 case FloorDiv:
3115 return BINARY_FLOOR_DIVIDE;
3116 default:
3117 PyErr_Format(PyExc_SystemError,
3118 "binary op %d should not be possible", op);
3119 return 0;
3120 }
3121 }
3122
3123 static int
inplace_binop(struct compiler * c,operator_ty op)3124 inplace_binop(struct compiler *c, operator_ty op)
3125 {
3126 switch (op) {
3127 case Add:
3128 return INPLACE_ADD;
3129 case Sub:
3130 return INPLACE_SUBTRACT;
3131 case Mult:
3132 return INPLACE_MULTIPLY;
3133 case MatMult:
3134 return INPLACE_MATRIX_MULTIPLY;
3135 case Div:
3136 return INPLACE_TRUE_DIVIDE;
3137 case Mod:
3138 return INPLACE_MODULO;
3139 case Pow:
3140 return INPLACE_POWER;
3141 case LShift:
3142 return INPLACE_LSHIFT;
3143 case RShift:
3144 return INPLACE_RSHIFT;
3145 case BitOr:
3146 return INPLACE_OR;
3147 case BitXor:
3148 return INPLACE_XOR;
3149 case BitAnd:
3150 return INPLACE_AND;
3151 case FloorDiv:
3152 return INPLACE_FLOOR_DIVIDE;
3153 default:
3154 PyErr_Format(PyExc_SystemError,
3155 "inplace binary op %d should not be possible", op);
3156 return 0;
3157 }
3158 }
3159
3160 static int
compiler_nameop(struct compiler * c,identifier name,expr_context_ty ctx)3161 compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
3162 {
3163 int op, scope;
3164 Py_ssize_t arg;
3165 enum { OP_FAST, OP_GLOBAL, OP_DEREF, OP_NAME } optype;
3166
3167 PyObject *dict = c->u->u_names;
3168 PyObject *mangled;
3169 /* XXX AugStore isn't used anywhere! */
3170
3171 assert(!_PyUnicode_EqualToASCIIString(name, "None") &&
3172 !_PyUnicode_EqualToASCIIString(name, "True") &&
3173 !_PyUnicode_EqualToASCIIString(name, "False"));
3174
3175 mangled = _Py_Mangle(c->u->u_private, name);
3176 if (!mangled)
3177 return 0;
3178
3179 op = 0;
3180 optype = OP_NAME;
3181 scope = PyST_GetScope(c->u->u_ste, mangled);
3182 switch (scope) {
3183 case FREE:
3184 dict = c->u->u_freevars;
3185 optype = OP_DEREF;
3186 break;
3187 case CELL:
3188 dict = c->u->u_cellvars;
3189 optype = OP_DEREF;
3190 break;
3191 case LOCAL:
3192 if (c->u->u_ste->ste_type == FunctionBlock)
3193 optype = OP_FAST;
3194 break;
3195 case GLOBAL_IMPLICIT:
3196 if (c->u->u_ste->ste_type == FunctionBlock)
3197 optype = OP_GLOBAL;
3198 break;
3199 case GLOBAL_EXPLICIT:
3200 optype = OP_GLOBAL;
3201 break;
3202 default:
3203 /* scope can be 0 */
3204 break;
3205 }
3206
3207 /* XXX Leave assert here, but handle __doc__ and the like better */
3208 assert(scope || PyUnicode_READ_CHAR(name, 0) == '_');
3209
3210 switch (optype) {
3211 case OP_DEREF:
3212 switch (ctx) {
3213 case Load:
3214 op = (c->u->u_ste->ste_type == ClassBlock) ? LOAD_CLASSDEREF : LOAD_DEREF;
3215 break;
3216 case Store: op = STORE_DEREF; break;
3217 case AugLoad:
3218 case AugStore:
3219 break;
3220 case Del: op = DELETE_DEREF; break;
3221 case Param:
3222 default:
3223 PyErr_SetString(PyExc_SystemError,
3224 "param invalid for deref variable");
3225 return 0;
3226 }
3227 break;
3228 case OP_FAST:
3229 switch (ctx) {
3230 case Load: op = LOAD_FAST; break;
3231 case Store: op = STORE_FAST; break;
3232 case Del: op = DELETE_FAST; break;
3233 case AugLoad:
3234 case AugStore:
3235 break;
3236 case Param:
3237 default:
3238 PyErr_SetString(PyExc_SystemError,
3239 "param invalid for local variable");
3240 return 0;
3241 }
3242 ADDOP_N(c, op, mangled, varnames);
3243 return 1;
3244 case OP_GLOBAL:
3245 switch (ctx) {
3246 case Load: op = LOAD_GLOBAL; break;
3247 case Store: op = STORE_GLOBAL; break;
3248 case Del: op = DELETE_GLOBAL; break;
3249 case AugLoad:
3250 case AugStore:
3251 break;
3252 case Param:
3253 default:
3254 PyErr_SetString(PyExc_SystemError,
3255 "param invalid for global variable");
3256 return 0;
3257 }
3258 break;
3259 case OP_NAME:
3260 switch (ctx) {
3261 case Load: op = LOAD_NAME; break;
3262 case Store: op = STORE_NAME; break;
3263 case Del: op = DELETE_NAME; break;
3264 case AugLoad:
3265 case AugStore:
3266 break;
3267 case Param:
3268 default:
3269 PyErr_SetString(PyExc_SystemError,
3270 "param invalid for name variable");
3271 return 0;
3272 }
3273 break;
3274 }
3275
3276 assert(op);
3277 arg = compiler_add_o(c, dict, mangled);
3278 Py_DECREF(mangled);
3279 if (arg < 0)
3280 return 0;
3281 return compiler_addop_i(c, op, arg);
3282 }
3283
3284 static int
compiler_boolop(struct compiler * c,expr_ty e)3285 compiler_boolop(struct compiler *c, expr_ty e)
3286 {
3287 basicblock *end;
3288 int jumpi;
3289 Py_ssize_t i, n;
3290 asdl_seq *s;
3291
3292 assert(e->kind == BoolOp_kind);
3293 if (e->v.BoolOp.op == And)
3294 jumpi = JUMP_IF_FALSE_OR_POP;
3295 else
3296 jumpi = JUMP_IF_TRUE_OR_POP;
3297 end = compiler_new_block(c);
3298 if (end == NULL)
3299 return 0;
3300 s = e->v.BoolOp.values;
3301 n = asdl_seq_LEN(s) - 1;
3302 assert(n >= 0);
3303 for (i = 0; i < n; ++i) {
3304 VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i));
3305 ADDOP_JABS(c, jumpi, end);
3306 }
3307 VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n));
3308 compiler_use_next_block(c, end);
3309 return 1;
3310 }
3311
3312 static int
starunpack_helper(struct compiler * c,asdl_seq * elts,int single_op,int inner_op,int outer_op)3313 starunpack_helper(struct compiler *c, asdl_seq *elts,
3314 int single_op, int inner_op, int outer_op)
3315 {
3316 Py_ssize_t n = asdl_seq_LEN(elts);
3317 Py_ssize_t i, nsubitems = 0, nseen = 0;
3318 for (i = 0; i < n; i++) {
3319 expr_ty elt = asdl_seq_GET(elts, i);
3320 if (elt->kind == Starred_kind) {
3321 if (nseen) {
3322 ADDOP_I(c, inner_op, nseen);
3323 nseen = 0;
3324 nsubitems++;
3325 }
3326 VISIT(c, expr, elt->v.Starred.value);
3327 nsubitems++;
3328 }
3329 else {
3330 VISIT(c, expr, elt);
3331 nseen++;
3332 }
3333 }
3334 if (nsubitems) {
3335 if (nseen) {
3336 ADDOP_I(c, inner_op, nseen);
3337 nsubitems++;
3338 }
3339 ADDOP_I(c, outer_op, nsubitems);
3340 }
3341 else
3342 ADDOP_I(c, single_op, nseen);
3343 return 1;
3344 }
3345
3346 static int
assignment_helper(struct compiler * c,asdl_seq * elts)3347 assignment_helper(struct compiler *c, asdl_seq *elts)
3348 {
3349 Py_ssize_t n = asdl_seq_LEN(elts);
3350 Py_ssize_t i;
3351 int seen_star = 0;
3352 for (i = 0; i < n; i++) {
3353 expr_ty elt = asdl_seq_GET(elts, i);
3354 if (elt->kind == Starred_kind && !seen_star) {
3355 if ((i >= (1 << 8)) ||
3356 (n-i-1 >= (INT_MAX >> 8)))
3357 return compiler_error(c,
3358 "too many expressions in "
3359 "star-unpacking assignment");
3360 ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8)));
3361 seen_star = 1;
3362 asdl_seq_SET(elts, i, elt->v.Starred.value);
3363 }
3364 else if (elt->kind == Starred_kind) {
3365 return compiler_error(c,
3366 "two starred expressions in assignment");
3367 }
3368 }
3369 if (!seen_star) {
3370 ADDOP_I(c, UNPACK_SEQUENCE, n);
3371 }
3372 VISIT_SEQ(c, expr, elts);
3373 return 1;
3374 }
3375
3376 static int
compiler_list(struct compiler * c,expr_ty e)3377 compiler_list(struct compiler *c, expr_ty e)
3378 {
3379 asdl_seq *elts = e->v.List.elts;
3380 if (e->v.List.ctx == Store) {
3381 return assignment_helper(c, elts);
3382 }
3383 else if (e->v.List.ctx == Load) {
3384 return starunpack_helper(c, elts,
3385 BUILD_LIST, BUILD_TUPLE, BUILD_LIST_UNPACK);
3386 }
3387 else
3388 VISIT_SEQ(c, expr, elts);
3389 return 1;
3390 }
3391
3392 static int
compiler_tuple(struct compiler * c,expr_ty e)3393 compiler_tuple(struct compiler *c, expr_ty e)
3394 {
3395 asdl_seq *elts = e->v.Tuple.elts;
3396 if (e->v.Tuple.ctx == Store) {
3397 return assignment_helper(c, elts);
3398 }
3399 else if (e->v.Tuple.ctx == Load) {
3400 return starunpack_helper(c, elts,
3401 BUILD_TUPLE, BUILD_TUPLE, BUILD_TUPLE_UNPACK);
3402 }
3403 else
3404 VISIT_SEQ(c, expr, elts);
3405 return 1;
3406 }
3407
3408 static int
compiler_set(struct compiler * c,expr_ty e)3409 compiler_set(struct compiler *c, expr_ty e)
3410 {
3411 return starunpack_helper(c, e->v.Set.elts, BUILD_SET,
3412 BUILD_SET, BUILD_SET_UNPACK);
3413 }
3414
3415 static int
are_all_items_const(asdl_seq * seq,Py_ssize_t begin,Py_ssize_t end)3416 are_all_items_const(asdl_seq *seq, Py_ssize_t begin, Py_ssize_t end)
3417 {
3418 Py_ssize_t i;
3419 for (i = begin; i < end; i++) {
3420 expr_ty key = (expr_ty)asdl_seq_GET(seq, i);
3421 if (key == NULL || !is_const(key))
3422 return 0;
3423 }
3424 return 1;
3425 }
3426
3427 static int
compiler_subdict(struct compiler * c,expr_ty e,Py_ssize_t begin,Py_ssize_t end)3428 compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end)
3429 {
3430 Py_ssize_t i, n = end - begin;
3431 PyObject *keys, *key;
3432 if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) {
3433 for (i = begin; i < end; i++) {
3434 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3435 }
3436 keys = PyTuple_New(n);
3437 if (keys == NULL) {
3438 return 0;
3439 }
3440 for (i = begin; i < end; i++) {
3441 key = get_const_value((expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3442 Py_INCREF(key);
3443 PyTuple_SET_ITEM(keys, i - begin, key);
3444 }
3445 ADDOP_N(c, LOAD_CONST, keys, consts);
3446 ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
3447 }
3448 else {
3449 for (i = begin; i < end; i++) {
3450 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3451 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3452 }
3453 ADDOP_I(c, BUILD_MAP, n);
3454 }
3455 return 1;
3456 }
3457
3458 static int
compiler_dict(struct compiler * c,expr_ty e)3459 compiler_dict(struct compiler *c, expr_ty e)
3460 {
3461 Py_ssize_t i, n, elements;
3462 int containers;
3463 int is_unpacking = 0;
3464 n = asdl_seq_LEN(e->v.Dict.values);
3465 containers = 0;
3466 elements = 0;
3467 for (i = 0; i < n; i++) {
3468 is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL;
3469 if (elements == 0xFFFF || (elements && is_unpacking)) {
3470 if (!compiler_subdict(c, e, i - elements, i))
3471 return 0;
3472 containers++;
3473 elements = 0;
3474 }
3475 if (is_unpacking) {
3476 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3477 containers++;
3478 }
3479 else {
3480 elements++;
3481 }
3482 }
3483 if (elements || containers == 0) {
3484 if (!compiler_subdict(c, e, n - elements, n))
3485 return 0;
3486 containers++;
3487 }
3488 /* If there is more than one dict, they need to be merged into a new
3489 * dict. If there is one dict and it's an unpacking, then it needs
3490 * to be copied into a new dict." */
3491 if (containers > 1 || is_unpacking) {
3492 ADDOP_I(c, BUILD_MAP_UNPACK, containers);
3493 }
3494 return 1;
3495 }
3496
3497 static int
compiler_compare(struct compiler * c,expr_ty e)3498 compiler_compare(struct compiler *c, expr_ty e)
3499 {
3500 Py_ssize_t i, n;
3501
3502 VISIT(c, expr, e->v.Compare.left);
3503 assert(asdl_seq_LEN(e->v.Compare.ops) > 0);
3504 n = asdl_seq_LEN(e->v.Compare.ops) - 1;
3505 if (n == 0) {
3506 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0));
3507 ADDOP_I(c, COMPARE_OP,
3508 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, 0))));
3509 }
3510 else {
3511 basicblock *cleanup = compiler_new_block(c);
3512 if (cleanup == NULL)
3513 return 0;
3514 for (i = 0; i < n; i++) {
3515 VISIT(c, expr,
3516 (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i));
3517 ADDOP(c, DUP_TOP);
3518 ADDOP(c, ROT_THREE);
3519 ADDOP_I(c, COMPARE_OP,
3520 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, i))));
3521 ADDOP_JABS(c, JUMP_IF_FALSE_OR_POP, cleanup);
3522 NEXT_BLOCK(c);
3523 }
3524 VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n));
3525 ADDOP_I(c, COMPARE_OP,
3526 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n))));
3527 basicblock *end = compiler_new_block(c);
3528 if (end == NULL)
3529 return 0;
3530 ADDOP_JREL(c, JUMP_FORWARD, end);
3531 compiler_use_next_block(c, cleanup);
3532 ADDOP(c, ROT_TWO);
3533 ADDOP(c, POP_TOP);
3534 compiler_use_next_block(c, end);
3535 }
3536 return 1;
3537 }
3538
3539 // Return 1 if the method call was optimized, -1 if not, and 0 on error.
3540 static int
maybe_optimize_method_call(struct compiler * c,expr_ty e)3541 maybe_optimize_method_call(struct compiler *c, expr_ty e)
3542 {
3543 Py_ssize_t argsl, i;
3544 expr_ty meth = e->v.Call.func;
3545 asdl_seq *args = e->v.Call.args;
3546
3547 /* Check that the call node is an attribute access, and that
3548 the call doesn't have keyword parameters. */
3549 if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load ||
3550 asdl_seq_LEN(e->v.Call.keywords))
3551 return -1;
3552
3553 /* Check that there are no *varargs types of arguments. */
3554 argsl = asdl_seq_LEN(args);
3555 for (i = 0; i < argsl; i++) {
3556 expr_ty elt = asdl_seq_GET(args, i);
3557 if (elt->kind == Starred_kind) {
3558 return -1;
3559 }
3560 }
3561
3562 /* Alright, we can optimize the code. */
3563 VISIT(c, expr, meth->v.Attribute.value);
3564 ADDOP_NAME(c, LOAD_METHOD, meth->v.Attribute.attr, names);
3565 VISIT_SEQ(c, expr, e->v.Call.args);
3566 ADDOP_I(c, CALL_METHOD, asdl_seq_LEN(e->v.Call.args));
3567 return 1;
3568 }
3569
3570 static int
compiler_call(struct compiler * c,expr_ty e)3571 compiler_call(struct compiler *c, expr_ty e)
3572 {
3573 int ret = maybe_optimize_method_call(c, e);
3574 if (ret >= 0) {
3575 return ret;
3576 }
3577 VISIT(c, expr, e->v.Call.func);
3578 return compiler_call_helper(c, 0,
3579 e->v.Call.args,
3580 e->v.Call.keywords);
3581 }
3582
3583 static int
compiler_joined_str(struct compiler * c,expr_ty e)3584 compiler_joined_str(struct compiler *c, expr_ty e)
3585 {
3586 VISIT_SEQ(c, expr, e->v.JoinedStr.values);
3587 if (asdl_seq_LEN(e->v.JoinedStr.values) != 1)
3588 ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));
3589 return 1;
3590 }
3591
3592 /* Used to implement f-strings. Format a single value. */
3593 static int
compiler_formatted_value(struct compiler * c,expr_ty e)3594 compiler_formatted_value(struct compiler *c, expr_ty e)
3595 {
3596 /* Our oparg encodes 2 pieces of information: the conversion
3597 character, and whether or not a format_spec was provided.
3598
3599 Convert the conversion char to 2 bits:
3600 None: 000 0x0 FVC_NONE
3601 !s : 001 0x1 FVC_STR
3602 !r : 010 0x2 FVC_REPR
3603 !a : 011 0x3 FVC_ASCII
3604
3605 next bit is whether or not we have a format spec:
3606 yes : 100 0x4
3607 no : 000 0x0
3608 */
3609
3610 int oparg;
3611
3612 /* Evaluate the expression to be formatted. */
3613 VISIT(c, expr, e->v.FormattedValue.value);
3614
3615 switch (e->v.FormattedValue.conversion) {
3616 case 's': oparg = FVC_STR; break;
3617 case 'r': oparg = FVC_REPR; break;
3618 case 'a': oparg = FVC_ASCII; break;
3619 case -1: oparg = FVC_NONE; break;
3620 default:
3621 PyErr_SetString(PyExc_SystemError,
3622 "Unrecognized conversion character");
3623 return 0;
3624 }
3625 if (e->v.FormattedValue.format_spec) {
3626 /* Evaluate the format spec, and update our opcode arg. */
3627 VISIT(c, expr, e->v.FormattedValue.format_spec);
3628 oparg |= FVS_HAVE_SPEC;
3629 }
3630
3631 /* And push our opcode and oparg */
3632 ADDOP_I(c, FORMAT_VALUE, oparg);
3633 return 1;
3634 }
3635
3636 static int
compiler_subkwargs(struct compiler * c,asdl_seq * keywords,Py_ssize_t begin,Py_ssize_t end)3637 compiler_subkwargs(struct compiler *c, asdl_seq *keywords, Py_ssize_t begin, Py_ssize_t end)
3638 {
3639 Py_ssize_t i, n = end - begin;
3640 keyword_ty kw;
3641 PyObject *keys, *key;
3642 assert(n > 0);
3643 if (n > 1) {
3644 for (i = begin; i < end; i++) {
3645 kw = asdl_seq_GET(keywords, i);
3646 VISIT(c, expr, kw->value);
3647 }
3648 keys = PyTuple_New(n);
3649 if (keys == NULL) {
3650 return 0;
3651 }
3652 for (i = begin; i < end; i++) {
3653 key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg;
3654 Py_INCREF(key);
3655 PyTuple_SET_ITEM(keys, i - begin, key);
3656 }
3657 ADDOP_N(c, LOAD_CONST, keys, consts);
3658 ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
3659 }
3660 else {
3661 /* a for loop only executes once */
3662 for (i = begin; i < end; i++) {
3663 kw = asdl_seq_GET(keywords, i);
3664 ADDOP_O(c, LOAD_CONST, kw->arg, consts);
3665 VISIT(c, expr, kw->value);
3666 }
3667 ADDOP_I(c, BUILD_MAP, n);
3668 }
3669 return 1;
3670 }
3671
3672 /* shared code between compiler_call and compiler_class */
3673 static int
compiler_call_helper(struct compiler * c,int n,asdl_seq * args,asdl_seq * keywords)3674 compiler_call_helper(struct compiler *c,
3675 int n, /* Args already pushed */
3676 asdl_seq *args,
3677 asdl_seq *keywords)
3678 {
3679 Py_ssize_t i, nseen, nelts, nkwelts;
3680 int mustdictunpack = 0;
3681
3682 /* the number of tuples and dictionaries on the stack */
3683 Py_ssize_t nsubargs = 0, nsubkwargs = 0;
3684
3685 nelts = asdl_seq_LEN(args);
3686 nkwelts = asdl_seq_LEN(keywords);
3687
3688 for (i = 0; i < nkwelts; i++) {
3689 keyword_ty kw = asdl_seq_GET(keywords, i);
3690 if (kw->arg == NULL) {
3691 mustdictunpack = 1;
3692 break;
3693 }
3694 }
3695
3696 nseen = n; /* the number of positional arguments on the stack */
3697 for (i = 0; i < nelts; i++) {
3698 expr_ty elt = asdl_seq_GET(args, i);
3699 if (elt->kind == Starred_kind) {
3700 /* A star-arg. If we've seen positional arguments,
3701 pack the positional arguments into a tuple. */
3702 if (nseen) {
3703 ADDOP_I(c, BUILD_TUPLE, nseen);
3704 nseen = 0;
3705 nsubargs++;
3706 }
3707 VISIT(c, expr, elt->v.Starred.value);
3708 nsubargs++;
3709 }
3710 else {
3711 VISIT(c, expr, elt);
3712 nseen++;
3713 }
3714 }
3715
3716 /* Same dance again for keyword arguments */
3717 if (nsubargs || mustdictunpack) {
3718 if (nseen) {
3719 /* Pack up any trailing positional arguments. */
3720 ADDOP_I(c, BUILD_TUPLE, nseen);
3721 nsubargs++;
3722 }
3723 if (nsubargs > 1) {
3724 /* If we ended up with more than one stararg, we need
3725 to concatenate them into a single sequence. */
3726 ADDOP_I(c, BUILD_TUPLE_UNPACK_WITH_CALL, nsubargs);
3727 }
3728 else if (nsubargs == 0) {
3729 ADDOP_I(c, BUILD_TUPLE, 0);
3730 }
3731 nseen = 0; /* the number of keyword arguments on the stack following */
3732 for (i = 0; i < nkwelts; i++) {
3733 keyword_ty kw = asdl_seq_GET(keywords, i);
3734 if (kw->arg == NULL) {
3735 /* A keyword argument unpacking. */
3736 if (nseen) {
3737 if (!compiler_subkwargs(c, keywords, i - nseen, i))
3738 return 0;
3739 nsubkwargs++;
3740 nseen = 0;
3741 }
3742 VISIT(c, expr, kw->value);
3743 nsubkwargs++;
3744 }
3745 else {
3746 nseen++;
3747 }
3748 }
3749 if (nseen) {
3750 /* Pack up any trailing keyword arguments. */
3751 if (!compiler_subkwargs(c, keywords, nkwelts - nseen, nkwelts))
3752 return 0;
3753 nsubkwargs++;
3754 }
3755 if (nsubkwargs > 1) {
3756 /* Pack it all up */
3757 ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs);
3758 }
3759 ADDOP_I(c, CALL_FUNCTION_EX, nsubkwargs > 0);
3760 return 1;
3761 }
3762 else if (nkwelts) {
3763 PyObject *names;
3764 VISIT_SEQ(c, keyword, keywords);
3765 names = PyTuple_New(nkwelts);
3766 if (names == NULL) {
3767 return 0;
3768 }
3769 for (i = 0; i < nkwelts; i++) {
3770 keyword_ty kw = asdl_seq_GET(keywords, i);
3771 Py_INCREF(kw->arg);
3772 PyTuple_SET_ITEM(names, i, kw->arg);
3773 }
3774 ADDOP_N(c, LOAD_CONST, names, consts);
3775 ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts);
3776 return 1;
3777 }
3778 else {
3779 ADDOP_I(c, CALL_FUNCTION, n + nelts);
3780 return 1;
3781 }
3782 }
3783
3784
3785 /* List and set comprehensions and generator expressions work by creating a
3786 nested function to perform the actual iteration. This means that the
3787 iteration variables don't leak into the current scope.
3788 The defined function is called immediately following its definition, with the
3789 result of that call being the result of the expression.
3790 The LC/SC version returns the populated container, while the GE version is
3791 flagged in symtable.c as a generator, so it returns the generator object
3792 when the function is called.
3793 This code *knows* that the loop cannot contain break, continue, or return,
3794 so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
3795
3796 Possible cleanups:
3797 - iterate over the generator sequence instead of using recursion
3798 */
3799
3800
3801 static int
compiler_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3802 compiler_comprehension_generator(struct compiler *c,
3803 asdl_seq *generators, int gen_index,
3804 expr_ty elt, expr_ty val, int type)
3805 {
3806 comprehension_ty gen;
3807 gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3808 if (gen->is_async) {
3809 return compiler_async_comprehension_generator(
3810 c, generators, gen_index, elt, val, type);
3811 } else {
3812 return compiler_sync_comprehension_generator(
3813 c, generators, gen_index, elt, val, type);
3814 }
3815 }
3816
3817 static int
compiler_sync_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3818 compiler_sync_comprehension_generator(struct compiler *c,
3819 asdl_seq *generators, int gen_index,
3820 expr_ty elt, expr_ty val, int type)
3821 {
3822 /* generate code for the iterator, then each of the ifs,
3823 and then write to the element */
3824
3825 comprehension_ty gen;
3826 basicblock *start, *anchor, *skip, *if_cleanup;
3827 Py_ssize_t i, n;
3828
3829 start = compiler_new_block(c);
3830 skip = compiler_new_block(c);
3831 if_cleanup = compiler_new_block(c);
3832 anchor = compiler_new_block(c);
3833
3834 if (start == NULL || skip == NULL || if_cleanup == NULL ||
3835 anchor == NULL)
3836 return 0;
3837
3838 gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3839
3840 if (gen_index == 0) {
3841 /* Receive outermost iter as an implicit argument */
3842 c->u->u_argcount = 1;
3843 ADDOP_I(c, LOAD_FAST, 0);
3844 }
3845 else {
3846 /* Sub-iter - calculate on the fly */
3847 VISIT(c, expr, gen->iter);
3848 ADDOP(c, GET_ITER);
3849 }
3850 compiler_use_next_block(c, start);
3851 ADDOP_JREL(c, FOR_ITER, anchor);
3852 NEXT_BLOCK(c);
3853 VISIT(c, expr, gen->target);
3854
3855 /* XXX this needs to be cleaned up...a lot! */
3856 n = asdl_seq_LEN(gen->ifs);
3857 for (i = 0; i < n; i++) {
3858 expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
3859 if (!compiler_jump_if(c, e, if_cleanup, 0))
3860 return 0;
3861 NEXT_BLOCK(c);
3862 }
3863
3864 if (++gen_index < asdl_seq_LEN(generators))
3865 if (!compiler_comprehension_generator(c,
3866 generators, gen_index,
3867 elt, val, type))
3868 return 0;
3869
3870 /* only append after the last for generator */
3871 if (gen_index >= asdl_seq_LEN(generators)) {
3872 /* comprehension specific code */
3873 switch (type) {
3874 case COMP_GENEXP:
3875 VISIT(c, expr, elt);
3876 ADDOP(c, YIELD_VALUE);
3877 ADDOP(c, POP_TOP);
3878 break;
3879 case COMP_LISTCOMP:
3880 VISIT(c, expr, elt);
3881 ADDOP_I(c, LIST_APPEND, gen_index + 1);
3882 break;
3883 case COMP_SETCOMP:
3884 VISIT(c, expr, elt);
3885 ADDOP_I(c, SET_ADD, gen_index + 1);
3886 break;
3887 case COMP_DICTCOMP:
3888 /* With 'd[k] = v', v is evaluated before k, so we do
3889 the same. */
3890 VISIT(c, expr, val);
3891 VISIT(c, expr, elt);
3892 ADDOP_I(c, MAP_ADD, gen_index + 1);
3893 break;
3894 default:
3895 return 0;
3896 }
3897
3898 compiler_use_next_block(c, skip);
3899 }
3900 compiler_use_next_block(c, if_cleanup);
3901 ADDOP_JABS(c, JUMP_ABSOLUTE, start);
3902 compiler_use_next_block(c, anchor);
3903
3904 return 1;
3905 }
3906
3907 static int
compiler_async_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3908 compiler_async_comprehension_generator(struct compiler *c,
3909 asdl_seq *generators, int gen_index,
3910 expr_ty elt, expr_ty val, int type)
3911 {
3912 _Py_IDENTIFIER(StopAsyncIteration);
3913
3914 comprehension_ty gen;
3915 basicblock *if_cleanup, *try,
3916 *after_try, *except, *try_cleanup;
3917 Py_ssize_t i, n;
3918
3919 PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration);
3920 if (stop_aiter_error == NULL) {
3921 return 0;
3922 }
3923
3924 try = compiler_new_block(c);
3925 after_try = compiler_new_block(c);
3926 except = compiler_new_block(c);
3927 if_cleanup = compiler_new_block(c);
3928 try_cleanup = compiler_new_block(c);
3929
3930 if (if_cleanup == NULL ||
3931 try == NULL || after_try == NULL ||
3932 except == NULL || try_cleanup == NULL) {
3933 return 0;
3934 }
3935
3936 gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3937
3938 if (gen_index == 0) {
3939 /* Receive outermost iter as an implicit argument */
3940 c->u->u_argcount = 1;
3941 ADDOP_I(c, LOAD_FAST, 0);
3942 }
3943 else {
3944 /* Sub-iter - calculate on the fly */
3945 VISIT(c, expr, gen->iter);
3946 ADDOP(c, GET_AITER);
3947 }
3948
3949 compiler_use_next_block(c, try);
3950
3951
3952 ADDOP_JREL(c, SETUP_EXCEPT, except);
3953 if (!compiler_push_fblock(c, EXCEPT, try))
3954 return 0;
3955
3956 ADDOP(c, GET_ANEXT);
3957 ADDOP_O(c, LOAD_CONST, Py_None, consts);
3958 ADDOP(c, YIELD_FROM);
3959 VISIT(c, expr, gen->target);
3960 ADDOP(c, POP_BLOCK);
3961 compiler_pop_fblock(c, EXCEPT, try);
3962 ADDOP_JREL(c, JUMP_FORWARD, after_try);
3963
3964
3965 compiler_use_next_block(c, except);
3966 ADDOP(c, DUP_TOP);
3967 ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names);
3968 ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
3969 ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup);
3970 ADDOP(c, END_FINALLY);
3971
3972 compiler_use_next_block(c, after_try);
3973
3974 n = asdl_seq_LEN(gen->ifs);
3975 for (i = 0; i < n; i++) {
3976 expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
3977 if (!compiler_jump_if(c, e, if_cleanup, 0))
3978 return 0;
3979 NEXT_BLOCK(c);
3980 }
3981
3982 if (++gen_index < asdl_seq_LEN(generators))
3983 if (!compiler_comprehension_generator(c,
3984 generators, gen_index,
3985 elt, val, type))
3986 return 0;
3987
3988 /* only append after the last for generator */
3989 if (gen_index >= asdl_seq_LEN(generators)) {
3990 /* comprehension specific code */
3991 switch (type) {
3992 case COMP_GENEXP:
3993 VISIT(c, expr, elt);
3994 ADDOP(c, YIELD_VALUE);
3995 ADDOP(c, POP_TOP);
3996 break;
3997 case COMP_LISTCOMP:
3998 VISIT(c, expr, elt);
3999 ADDOP_I(c, LIST_APPEND, gen_index + 1);
4000 break;
4001 case COMP_SETCOMP:
4002 VISIT(c, expr, elt);
4003 ADDOP_I(c, SET_ADD, gen_index + 1);
4004 break;
4005 case COMP_DICTCOMP:
4006 /* With 'd[k] = v', v is evaluated before k, so we do
4007 the same. */
4008 VISIT(c, expr, val);
4009 VISIT(c, expr, elt);
4010 ADDOP_I(c, MAP_ADD, gen_index + 1);
4011 break;
4012 default:
4013 return 0;
4014 }
4015 }
4016 compiler_use_next_block(c, if_cleanup);
4017 ADDOP_JABS(c, JUMP_ABSOLUTE, try);
4018
4019 compiler_use_next_block(c, try_cleanup);
4020 ADDOP(c, POP_TOP);
4021 ADDOP(c, POP_TOP);
4022 ADDOP(c, POP_TOP);
4023 ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
4024 ADDOP(c, POP_TOP);
4025
4026 return 1;
4027 }
4028
4029 static int
compiler_comprehension(struct compiler * c,expr_ty e,int type,identifier name,asdl_seq * generators,expr_ty elt,expr_ty val)4030 compiler_comprehension(struct compiler *c, expr_ty e, int type,
4031 identifier name, asdl_seq *generators, expr_ty elt,
4032 expr_ty val)
4033 {
4034 PyCodeObject *co = NULL;
4035 comprehension_ty outermost;
4036 PyObject *qualname = NULL;
4037 int is_async_function = c->u->u_ste->ste_coroutine;
4038 int is_async_generator = 0;
4039
4040 outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
4041
4042 if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
4043 (void *)e, e->lineno))
4044 {
4045 goto error;
4046 }
4047
4048 is_async_generator = c->u->u_ste->ste_coroutine;
4049
4050 if (is_async_generator && !is_async_function && type != COMP_GENEXP) {
4051 if (e->lineno > c->u->u_lineno) {
4052 c->u->u_lineno = e->lineno;
4053 c->u->u_lineno_set = 0;
4054 }
4055 compiler_error(c, "asynchronous comprehension outside of "
4056 "an asynchronous function");
4057 goto error_in_scope;
4058 }
4059
4060 if (type != COMP_GENEXP) {
4061 int op;
4062 switch (type) {
4063 case COMP_LISTCOMP:
4064 op = BUILD_LIST;
4065 break;
4066 case COMP_SETCOMP:
4067 op = BUILD_SET;
4068 break;
4069 case COMP_DICTCOMP:
4070 op = BUILD_MAP;
4071 break;
4072 default:
4073 PyErr_Format(PyExc_SystemError,
4074 "unknown comprehension type %d", type);
4075 goto error_in_scope;
4076 }
4077
4078 ADDOP_I(c, op, 0);
4079 }
4080
4081 if (!compiler_comprehension_generator(c, generators, 0, elt,
4082 val, type))
4083 goto error_in_scope;
4084
4085 if (type != COMP_GENEXP) {
4086 ADDOP(c, RETURN_VALUE);
4087 }
4088
4089 co = assemble(c, 1);
4090 qualname = c->u->u_qualname;
4091 Py_INCREF(qualname);
4092 compiler_exit_scope(c);
4093 if (co == NULL)
4094 goto error;
4095
4096 if (!compiler_make_closure(c, co, 0, qualname))
4097 goto error;
4098 Py_DECREF(qualname);
4099 Py_DECREF(co);
4100
4101 VISIT(c, expr, outermost->iter);
4102
4103 if (outermost->is_async) {
4104 ADDOP(c, GET_AITER);
4105 } else {
4106 ADDOP(c, GET_ITER);
4107 }
4108
4109 ADDOP_I(c, CALL_FUNCTION, 1);
4110
4111 if (is_async_generator && type != COMP_GENEXP) {
4112 ADDOP(c, GET_AWAITABLE);
4113 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4114 ADDOP(c, YIELD_FROM);
4115 }
4116
4117 return 1;
4118 error_in_scope:
4119 compiler_exit_scope(c);
4120 error:
4121 Py_XDECREF(qualname);
4122 Py_XDECREF(co);
4123 return 0;
4124 }
4125
4126 static int
compiler_genexp(struct compiler * c,expr_ty e)4127 compiler_genexp(struct compiler *c, expr_ty e)
4128 {
4129 static identifier name;
4130 if (!name) {
4131 name = PyUnicode_FromString("<genexpr>");
4132 if (!name)
4133 return 0;
4134 }
4135 assert(e->kind == GeneratorExp_kind);
4136 return compiler_comprehension(c, e, COMP_GENEXP, name,
4137 e->v.GeneratorExp.generators,
4138 e->v.GeneratorExp.elt, NULL);
4139 }
4140
4141 static int
compiler_listcomp(struct compiler * c,expr_ty e)4142 compiler_listcomp(struct compiler *c, expr_ty e)
4143 {
4144 static identifier name;
4145 if (!name) {
4146 name = PyUnicode_FromString("<listcomp>");
4147 if (!name)
4148 return 0;
4149 }
4150 assert(e->kind == ListComp_kind);
4151 return compiler_comprehension(c, e, COMP_LISTCOMP, name,
4152 e->v.ListComp.generators,
4153 e->v.ListComp.elt, NULL);
4154 }
4155
4156 static int
compiler_setcomp(struct compiler * c,expr_ty e)4157 compiler_setcomp(struct compiler *c, expr_ty e)
4158 {
4159 static identifier name;
4160 if (!name) {
4161 name = PyUnicode_FromString("<setcomp>");
4162 if (!name)
4163 return 0;
4164 }
4165 assert(e->kind == SetComp_kind);
4166 return compiler_comprehension(c, e, COMP_SETCOMP, name,
4167 e->v.SetComp.generators,
4168 e->v.SetComp.elt, NULL);
4169 }
4170
4171
4172 static int
compiler_dictcomp(struct compiler * c,expr_ty e)4173 compiler_dictcomp(struct compiler *c, expr_ty e)
4174 {
4175 static identifier name;
4176 if (!name) {
4177 name = PyUnicode_FromString("<dictcomp>");
4178 if (!name)
4179 return 0;
4180 }
4181 assert(e->kind == DictComp_kind);
4182 return compiler_comprehension(c, e, COMP_DICTCOMP, name,
4183 e->v.DictComp.generators,
4184 e->v.DictComp.key, e->v.DictComp.value);
4185 }
4186
4187
4188 static int
compiler_visit_keyword(struct compiler * c,keyword_ty k)4189 compiler_visit_keyword(struct compiler *c, keyword_ty k)
4190 {
4191 VISIT(c, expr, k->value);
4192 return 1;
4193 }
4194
4195 /* Test whether expression is constant. For constants, report
4196 whether they are true or false.
4197
4198 Return values: 1 for true, 0 for false, -1 for non-constant.
4199 */
4200
4201 static int
expr_constant(expr_ty e)4202 expr_constant(expr_ty e)
4203 {
4204 if (is_const(e)) {
4205 return PyObject_IsTrue(get_const_value(e));
4206 }
4207 return -1;
4208 }
4209
4210
4211 /*
4212 Implements the async with statement.
4213
4214 The semantics outlined in that PEP are as follows:
4215
4216 async with EXPR as VAR:
4217 BLOCK
4218
4219 It is implemented roughly as:
4220
4221 context = EXPR
4222 exit = context.__aexit__ # not calling it
4223 value = await context.__aenter__()
4224 try:
4225 VAR = value # if VAR present in the syntax
4226 BLOCK
4227 finally:
4228 if an exception was raised:
4229 exc = copy of (exception, instance, traceback)
4230 else:
4231 exc = (None, None, None)
4232 if not (await exit(*exc)):
4233 raise
4234 */
4235 static int
compiler_async_with(struct compiler * c,stmt_ty s,int pos)4236 compiler_async_with(struct compiler *c, stmt_ty s, int pos)
4237 {
4238 basicblock *block, *finally;
4239 withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
4240
4241 assert(s->kind == AsyncWith_kind);
4242 if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
4243 return compiler_error(c, "'async with' outside async function");
4244 }
4245
4246 block = compiler_new_block(c);
4247 finally = compiler_new_block(c);
4248 if (!block || !finally)
4249 return 0;
4250
4251 /* Evaluate EXPR */
4252 VISIT(c, expr, item->context_expr);
4253
4254 ADDOP(c, BEFORE_ASYNC_WITH);
4255 ADDOP(c, GET_AWAITABLE);
4256 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4257 ADDOP(c, YIELD_FROM);
4258
4259 ADDOP_JREL(c, SETUP_ASYNC_WITH, finally);
4260
4261 /* SETUP_ASYNC_WITH pushes a finally block. */
4262 compiler_use_next_block(c, block);
4263 if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
4264 return 0;
4265 }
4266
4267 if (item->optional_vars) {
4268 VISIT(c, expr, item->optional_vars);
4269 }
4270 else {
4271 /* Discard result from context.__aenter__() */
4272 ADDOP(c, POP_TOP);
4273 }
4274
4275 pos++;
4276 if (pos == asdl_seq_LEN(s->v.AsyncWith.items))
4277 /* BLOCK code */
4278 VISIT_SEQ(c, stmt, s->v.AsyncWith.body)
4279 else if (!compiler_async_with(c, s, pos))
4280 return 0;
4281
4282 /* End of try block; start the finally block */
4283 ADDOP(c, POP_BLOCK);
4284 compiler_pop_fblock(c, FINALLY_TRY, block);
4285
4286 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4287 compiler_use_next_block(c, finally);
4288 if (!compiler_push_fblock(c, FINALLY_END, finally))
4289 return 0;
4290
4291 /* Finally block starts; context.__exit__ is on the stack under
4292 the exception or return information. Just issue our magic
4293 opcode. */
4294 ADDOP(c, WITH_CLEANUP_START);
4295
4296 ADDOP(c, GET_AWAITABLE);
4297 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4298 ADDOP(c, YIELD_FROM);
4299
4300 ADDOP(c, WITH_CLEANUP_FINISH);
4301
4302 /* Finally block ends. */
4303 ADDOP(c, END_FINALLY);
4304 compiler_pop_fblock(c, FINALLY_END, finally);
4305 return 1;
4306 }
4307
4308
4309 /*
4310 Implements the with statement from PEP 343.
4311
4312 The semantics outlined in that PEP are as follows:
4313
4314 with EXPR as VAR:
4315 BLOCK
4316
4317 It is implemented roughly as:
4318
4319 context = EXPR
4320 exit = context.__exit__ # not calling it
4321 value = context.__enter__()
4322 try:
4323 VAR = value # if VAR present in the syntax
4324 BLOCK
4325 finally:
4326 if an exception was raised:
4327 exc = copy of (exception, instance, traceback)
4328 else:
4329 exc = (None, None, None)
4330 exit(*exc)
4331 */
4332 static int
compiler_with(struct compiler * c,stmt_ty s,int pos)4333 compiler_with(struct compiler *c, stmt_ty s, int pos)
4334 {
4335 basicblock *block, *finally;
4336 withitem_ty item = asdl_seq_GET(s->v.With.items, pos);
4337
4338 assert(s->kind == With_kind);
4339
4340 block = compiler_new_block(c);
4341 finally = compiler_new_block(c);
4342 if (!block || !finally)
4343 return 0;
4344
4345 /* Evaluate EXPR */
4346 VISIT(c, expr, item->context_expr);
4347 ADDOP_JREL(c, SETUP_WITH, finally);
4348
4349 /* SETUP_WITH pushes a finally block. */
4350 compiler_use_next_block(c, block);
4351 if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
4352 return 0;
4353 }
4354
4355 if (item->optional_vars) {
4356 VISIT(c, expr, item->optional_vars);
4357 }
4358 else {
4359 /* Discard result from context.__enter__() */
4360 ADDOP(c, POP_TOP);
4361 }
4362
4363 pos++;
4364 if (pos == asdl_seq_LEN(s->v.With.items))
4365 /* BLOCK code */
4366 VISIT_SEQ(c, stmt, s->v.With.body)
4367 else if (!compiler_with(c, s, pos))
4368 return 0;
4369
4370 /* End of try block; start the finally block */
4371 ADDOP(c, POP_BLOCK);
4372 compiler_pop_fblock(c, FINALLY_TRY, block);
4373
4374 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4375 compiler_use_next_block(c, finally);
4376 if (!compiler_push_fblock(c, FINALLY_END, finally))
4377 return 0;
4378
4379 /* Finally block starts; context.__exit__ is on the stack under
4380 the exception or return information. Just issue our magic
4381 opcode. */
4382 ADDOP(c, WITH_CLEANUP_START);
4383 ADDOP(c, WITH_CLEANUP_FINISH);
4384
4385 /* Finally block ends. */
4386 ADDOP(c, END_FINALLY);
4387 compiler_pop_fblock(c, FINALLY_END, finally);
4388 return 1;
4389 }
4390
4391 static int
compiler_visit_expr(struct compiler * c,expr_ty e)4392 compiler_visit_expr(struct compiler *c, expr_ty e)
4393 {
4394 /* If expr e has a different line number than the last expr/stmt,
4395 set a new line number for the next instruction.
4396 */
4397 if (e->lineno > c->u->u_lineno) {
4398 c->u->u_lineno = e->lineno;
4399 c->u->u_lineno_set = 0;
4400 }
4401 /* Updating the column offset is always harmless. */
4402 c->u->u_col_offset = e->col_offset;
4403 switch (e->kind) {
4404 case BoolOp_kind:
4405 return compiler_boolop(c, e);
4406 case BinOp_kind:
4407 VISIT(c, expr, e->v.BinOp.left);
4408 VISIT(c, expr, e->v.BinOp.right);
4409 ADDOP(c, binop(c, e->v.BinOp.op));
4410 break;
4411 case UnaryOp_kind:
4412 VISIT(c, expr, e->v.UnaryOp.operand);
4413 ADDOP(c, unaryop(e->v.UnaryOp.op));
4414 break;
4415 case Lambda_kind:
4416 return compiler_lambda(c, e);
4417 case IfExp_kind:
4418 return compiler_ifexp(c, e);
4419 case Dict_kind:
4420 return compiler_dict(c, e);
4421 case Set_kind:
4422 return compiler_set(c, e);
4423 case GeneratorExp_kind:
4424 return compiler_genexp(c, e);
4425 case ListComp_kind:
4426 return compiler_listcomp(c, e);
4427 case SetComp_kind:
4428 return compiler_setcomp(c, e);
4429 case DictComp_kind:
4430 return compiler_dictcomp(c, e);
4431 case Yield_kind:
4432 if (c->u->u_ste->ste_type != FunctionBlock)
4433 return compiler_error(c, "'yield' outside function");
4434 if (e->v.Yield.value) {
4435 VISIT(c, expr, e->v.Yield.value);
4436 }
4437 else {
4438 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4439 }
4440 ADDOP(c, YIELD_VALUE);
4441 break;
4442 case YieldFrom_kind:
4443 if (c->u->u_ste->ste_type != FunctionBlock)
4444 return compiler_error(c, "'yield' outside function");
4445
4446 if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
4447 return compiler_error(c, "'yield from' inside async function");
4448
4449 VISIT(c, expr, e->v.YieldFrom.value);
4450 ADDOP(c, GET_YIELD_FROM_ITER);
4451 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4452 ADDOP(c, YIELD_FROM);
4453 break;
4454 case Await_kind:
4455 if (c->u->u_ste->ste_type != FunctionBlock)
4456 return compiler_error(c, "'await' outside function");
4457
4458 if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
4459 c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
4460 return compiler_error(c, "'await' outside async function");
4461
4462 VISIT(c, expr, e->v.Await.value);
4463 ADDOP(c, GET_AWAITABLE);
4464 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4465 ADDOP(c, YIELD_FROM);
4466 break;
4467 case Compare_kind:
4468 return compiler_compare(c, e);
4469 case Call_kind:
4470 return compiler_call(c, e);
4471 case Constant_kind:
4472 ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts);
4473 break;
4474 case Num_kind:
4475 ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts);
4476 break;
4477 case Str_kind:
4478 ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts);
4479 break;
4480 case JoinedStr_kind:
4481 return compiler_joined_str(c, e);
4482 case FormattedValue_kind:
4483 return compiler_formatted_value(c, e);
4484 case Bytes_kind:
4485 ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts);
4486 break;
4487 case Ellipsis_kind:
4488 ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts);
4489 break;
4490 case NameConstant_kind:
4491 ADDOP_O(c, LOAD_CONST, e->v.NameConstant.value, consts);
4492 break;
4493 /* The following exprs can be assignment targets. */
4494 case Attribute_kind:
4495 if (e->v.Attribute.ctx != AugStore)
4496 VISIT(c, expr, e->v.Attribute.value);
4497 switch (e->v.Attribute.ctx) {
4498 case AugLoad:
4499 ADDOP(c, DUP_TOP);
4500 /* Fall through */
4501 case Load:
4502 ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names);
4503 break;
4504 case AugStore:
4505 ADDOP(c, ROT_TWO);
4506 /* Fall through */
4507 case Store:
4508 ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names);
4509 break;
4510 case Del:
4511 ADDOP_NAME(c, DELETE_ATTR, e->v.Attribute.attr, names);
4512 break;
4513 case Param:
4514 default:
4515 PyErr_SetString(PyExc_SystemError,
4516 "param invalid in attribute expression");
4517 return 0;
4518 }
4519 break;
4520 case Subscript_kind:
4521 switch (e->v.Subscript.ctx) {
4522 case AugLoad:
4523 VISIT(c, expr, e->v.Subscript.value);
4524 VISIT_SLICE(c, e->v.Subscript.slice, AugLoad);
4525 break;
4526 case Load:
4527 VISIT(c, expr, e->v.Subscript.value);
4528 VISIT_SLICE(c, e->v.Subscript.slice, Load);
4529 break;
4530 case AugStore:
4531 VISIT_SLICE(c, e->v.Subscript.slice, AugStore);
4532 break;
4533 case Store:
4534 VISIT(c, expr, e->v.Subscript.value);
4535 VISIT_SLICE(c, e->v.Subscript.slice, Store);
4536 break;
4537 case Del:
4538 VISIT(c, expr, e->v.Subscript.value);
4539 VISIT_SLICE(c, e->v.Subscript.slice, Del);
4540 break;
4541 case Param:
4542 default:
4543 PyErr_SetString(PyExc_SystemError,
4544 "param invalid in subscript expression");
4545 return 0;
4546 }
4547 break;
4548 case Starred_kind:
4549 switch (e->v.Starred.ctx) {
4550 case Store:
4551 /* In all legitimate cases, the Starred node was already replaced
4552 * by compiler_list/compiler_tuple. XXX: is that okay? */
4553 return compiler_error(c,
4554 "starred assignment target must be in a list or tuple");
4555 default:
4556 return compiler_error(c,
4557 "can't use starred expression here");
4558 }
4559 break;
4560 case Name_kind:
4561 return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx);
4562 /* child nodes of List and Tuple will have expr_context set */
4563 case List_kind:
4564 return compiler_list(c, e);
4565 case Tuple_kind:
4566 return compiler_tuple(c, e);
4567 }
4568 return 1;
4569 }
4570
4571 static int
compiler_augassign(struct compiler * c,stmt_ty s)4572 compiler_augassign(struct compiler *c, stmt_ty s)
4573 {
4574 expr_ty e = s->v.AugAssign.target;
4575 expr_ty auge;
4576
4577 assert(s->kind == AugAssign_kind);
4578
4579 switch (e->kind) {
4580 case Attribute_kind:
4581 auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
4582 AugLoad, e->lineno, e->col_offset, c->c_arena);
4583 if (auge == NULL)
4584 return 0;
4585 VISIT(c, expr, auge);
4586 VISIT(c, expr, s->v.AugAssign.value);
4587 ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4588 auge->v.Attribute.ctx = AugStore;
4589 VISIT(c, expr, auge);
4590 break;
4591 case Subscript_kind:
4592 auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
4593 AugLoad, e->lineno, e->col_offset, c->c_arena);
4594 if (auge == NULL)
4595 return 0;
4596 VISIT(c, expr, auge);
4597 VISIT(c, expr, s->v.AugAssign.value);
4598 ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4599 auge->v.Subscript.ctx = AugStore;
4600 VISIT(c, expr, auge);
4601 break;
4602 case Name_kind:
4603 if (!compiler_nameop(c, e->v.Name.id, Load))
4604 return 0;
4605 VISIT(c, expr, s->v.AugAssign.value);
4606 ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4607 return compiler_nameop(c, e->v.Name.id, Store);
4608 default:
4609 PyErr_Format(PyExc_SystemError,
4610 "invalid node type (%d) for augmented assignment",
4611 e->kind);
4612 return 0;
4613 }
4614 return 1;
4615 }
4616
4617 static int
check_ann_expr(struct compiler * c,expr_ty e)4618 check_ann_expr(struct compiler *c, expr_ty e)
4619 {
4620 VISIT(c, expr, e);
4621 ADDOP(c, POP_TOP);
4622 return 1;
4623 }
4624
4625 static int
check_annotation(struct compiler * c,stmt_ty s)4626 check_annotation(struct compiler *c, stmt_ty s)
4627 {
4628 /* Annotations are only evaluated in a module or class. */
4629 if (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
4630 c->u->u_scope_type == COMPILER_SCOPE_CLASS) {
4631 return check_ann_expr(c, s->v.AnnAssign.annotation);
4632 }
4633 return 1;
4634 }
4635
4636 static int
check_ann_slice(struct compiler * c,slice_ty sl)4637 check_ann_slice(struct compiler *c, slice_ty sl)
4638 {
4639 switch(sl->kind) {
4640 case Index_kind:
4641 return check_ann_expr(c, sl->v.Index.value);
4642 case Slice_kind:
4643 if (sl->v.Slice.lower && !check_ann_expr(c, sl->v.Slice.lower)) {
4644 return 0;
4645 }
4646 if (sl->v.Slice.upper && !check_ann_expr(c, sl->v.Slice.upper)) {
4647 return 0;
4648 }
4649 if (sl->v.Slice.step && !check_ann_expr(c, sl->v.Slice.step)) {
4650 return 0;
4651 }
4652 break;
4653 default:
4654 PyErr_SetString(PyExc_SystemError,
4655 "unexpected slice kind");
4656 return 0;
4657 }
4658 return 1;
4659 }
4660
4661 static int
check_ann_subscr(struct compiler * c,slice_ty sl)4662 check_ann_subscr(struct compiler *c, slice_ty sl)
4663 {
4664 /* We check that everything in a subscript is defined at runtime. */
4665 Py_ssize_t i, n;
4666
4667 switch (sl->kind) {
4668 case Index_kind:
4669 case Slice_kind:
4670 if (!check_ann_slice(c, sl)) {
4671 return 0;
4672 }
4673 break;
4674 case ExtSlice_kind:
4675 n = asdl_seq_LEN(sl->v.ExtSlice.dims);
4676 for (i = 0; i < n; i++) {
4677 slice_ty subsl = (slice_ty)asdl_seq_GET(sl->v.ExtSlice.dims, i);
4678 switch (subsl->kind) {
4679 case Index_kind:
4680 case Slice_kind:
4681 if (!check_ann_slice(c, subsl)) {
4682 return 0;
4683 }
4684 break;
4685 case ExtSlice_kind:
4686 default:
4687 PyErr_SetString(PyExc_SystemError,
4688 "extended slice invalid in nested slice");
4689 return 0;
4690 }
4691 }
4692 break;
4693 default:
4694 PyErr_Format(PyExc_SystemError,
4695 "invalid subscript kind %d", sl->kind);
4696 return 0;
4697 }
4698 return 1;
4699 }
4700
4701 static int
compiler_annassign(struct compiler * c,stmt_ty s)4702 compiler_annassign(struct compiler *c, stmt_ty s)
4703 {
4704 expr_ty targ = s->v.AnnAssign.target;
4705 PyObject* mangled;
4706
4707 assert(s->kind == AnnAssign_kind);
4708
4709 /* We perform the actual assignment first. */
4710 if (s->v.AnnAssign.value) {
4711 VISIT(c, expr, s->v.AnnAssign.value);
4712 VISIT(c, expr, targ);
4713 }
4714 switch (targ->kind) {
4715 case Name_kind:
4716 /* If we have a simple name in a module or class, store annotation. */
4717 if (s->v.AnnAssign.simple &&
4718 (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
4719 c->u->u_scope_type == COMPILER_SCOPE_CLASS)) {
4720 if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) {
4721 VISIT(c, annexpr, s->v.AnnAssign.annotation)
4722 }
4723 else {
4724 VISIT(c, expr, s->v.AnnAssign.annotation);
4725 }
4726 ADDOP_NAME(c, LOAD_NAME, __annotations__, names);
4727 mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
4728 if (!mangled) {
4729 return 0;
4730 }
4731 ADDOP_N(c, LOAD_CONST, mangled, consts);
4732 ADDOP(c, STORE_SUBSCR);
4733 }
4734 break;
4735 case Attribute_kind:
4736 if (!s->v.AnnAssign.value &&
4737 !check_ann_expr(c, targ->v.Attribute.value)) {
4738 return 0;
4739 }
4740 break;
4741 case Subscript_kind:
4742 if (!s->v.AnnAssign.value &&
4743 (!check_ann_expr(c, targ->v.Subscript.value) ||
4744 !check_ann_subscr(c, targ->v.Subscript.slice))) {
4745 return 0;
4746 }
4747 break;
4748 default:
4749 PyErr_Format(PyExc_SystemError,
4750 "invalid node type (%d) for annotated assignment",
4751 targ->kind);
4752 return 0;
4753 }
4754 /* Annotation is evaluated last. */
4755 if (!s->v.AnnAssign.simple && !check_annotation(c, s)) {
4756 return 0;
4757 }
4758 return 1;
4759 }
4760
4761 static int
compiler_push_fblock(struct compiler * c,enum fblocktype t,basicblock * b)4762 compiler_push_fblock(struct compiler *c, enum fblocktype t, basicblock *b)
4763 {
4764 struct fblockinfo *f;
4765 if (c->u->u_nfblocks >= CO_MAXBLOCKS) {
4766 PyErr_SetString(PyExc_SyntaxError,
4767 "too many statically nested blocks");
4768 return 0;
4769 }
4770 f = &c->u->u_fblock[c->u->u_nfblocks++];
4771 f->fb_type = t;
4772 f->fb_block = b;
4773 return 1;
4774 }
4775
4776 static void
compiler_pop_fblock(struct compiler * c,enum fblocktype t,basicblock * b)4777 compiler_pop_fblock(struct compiler *c, enum fblocktype t, basicblock *b)
4778 {
4779 struct compiler_unit *u = c->u;
4780 assert(u->u_nfblocks > 0);
4781 u->u_nfblocks--;
4782 assert(u->u_fblock[u->u_nfblocks].fb_type == t);
4783 assert(u->u_fblock[u->u_nfblocks].fb_block == b);
4784 }
4785
4786 static int
compiler_in_loop(struct compiler * c)4787 compiler_in_loop(struct compiler *c) {
4788 int i;
4789 struct compiler_unit *u = c->u;
4790 for (i = 0; i < u->u_nfblocks; ++i) {
4791 if (u->u_fblock[i].fb_type == LOOP)
4792 return 1;
4793 }
4794 return 0;
4795 }
4796 /* Raises a SyntaxError and returns 0.
4797 If something goes wrong, a different exception may be raised.
4798 */
4799
4800 static int
compiler_error(struct compiler * c,const char * errstr)4801 compiler_error(struct compiler *c, const char *errstr)
4802 {
4803 PyObject *loc;
4804 PyObject *u = NULL, *v = NULL;
4805
4806 loc = PyErr_ProgramTextObject(c->c_filename, c->u->u_lineno);
4807 if (!loc) {
4808 Py_INCREF(Py_None);
4809 loc = Py_None;
4810 }
4811 u = Py_BuildValue("(OiiO)", c->c_filename, c->u->u_lineno,
4812 c->u->u_col_offset, loc);
4813 if (!u)
4814 goto exit;
4815 v = Py_BuildValue("(zO)", errstr, u);
4816 if (!v)
4817 goto exit;
4818 PyErr_SetObject(PyExc_SyntaxError, v);
4819 exit:
4820 Py_DECREF(loc);
4821 Py_XDECREF(u);
4822 Py_XDECREF(v);
4823 return 0;
4824 }
4825
4826 static int
compiler_handle_subscr(struct compiler * c,const char * kind,expr_context_ty ctx)4827 compiler_handle_subscr(struct compiler *c, const char *kind,
4828 expr_context_ty ctx)
4829 {
4830 int op = 0;
4831
4832 /* XXX this code is duplicated */
4833 switch (ctx) {
4834 case AugLoad: /* fall through to Load */
4835 case Load: op = BINARY_SUBSCR; break;
4836 case AugStore:/* fall through to Store */
4837 case Store: op = STORE_SUBSCR; break;
4838 case Del: op = DELETE_SUBSCR; break;
4839 case Param:
4840 PyErr_Format(PyExc_SystemError,
4841 "invalid %s kind %d in subscript\n",
4842 kind, ctx);
4843 return 0;
4844 }
4845 if (ctx == AugLoad) {
4846 ADDOP(c, DUP_TOP_TWO);
4847 }
4848 else if (ctx == AugStore) {
4849 ADDOP(c, ROT_THREE);
4850 }
4851 ADDOP(c, op);
4852 return 1;
4853 }
4854
4855 static int
compiler_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4856 compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
4857 {
4858 int n = 2;
4859 assert(s->kind == Slice_kind);
4860
4861 /* only handles the cases where BUILD_SLICE is emitted */
4862 if (s->v.Slice.lower) {
4863 VISIT(c, expr, s->v.Slice.lower);
4864 }
4865 else {
4866 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4867 }
4868
4869 if (s->v.Slice.upper) {
4870 VISIT(c, expr, s->v.Slice.upper);
4871 }
4872 else {
4873 ADDOP_O(c, LOAD_CONST, Py_None, consts);
4874 }
4875
4876 if (s->v.Slice.step) {
4877 n++;
4878 VISIT(c, expr, s->v.Slice.step);
4879 }
4880 ADDOP_I(c, BUILD_SLICE, n);
4881 return 1;
4882 }
4883
4884 static int
compiler_visit_nested_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4885 compiler_visit_nested_slice(struct compiler *c, slice_ty s,
4886 expr_context_ty ctx)
4887 {
4888 switch (s->kind) {
4889 case Slice_kind:
4890 return compiler_slice(c, s, ctx);
4891 case Index_kind:
4892 VISIT(c, expr, s->v.Index.value);
4893 break;
4894 case ExtSlice_kind:
4895 default:
4896 PyErr_SetString(PyExc_SystemError,
4897 "extended slice invalid in nested slice");
4898 return 0;
4899 }
4900 return 1;
4901 }
4902
4903 static int
compiler_visit_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4904 compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
4905 {
4906 const char * kindname = NULL;
4907 switch (s->kind) {
4908 case Index_kind:
4909 kindname = "index";
4910 if (ctx != AugStore) {
4911 VISIT(c, expr, s->v.Index.value);
4912 }
4913 break;
4914 case Slice_kind:
4915 kindname = "slice";
4916 if (ctx != AugStore) {
4917 if (!compiler_slice(c, s, ctx))
4918 return 0;
4919 }
4920 break;
4921 case ExtSlice_kind:
4922 kindname = "extended slice";
4923 if (ctx != AugStore) {
4924 Py_ssize_t i, n = asdl_seq_LEN(s->v.ExtSlice.dims);
4925 for (i = 0; i < n; i++) {
4926 slice_ty sub = (slice_ty)asdl_seq_GET(
4927 s->v.ExtSlice.dims, i);
4928 if (!compiler_visit_nested_slice(c, sub, ctx))
4929 return 0;
4930 }
4931 ADDOP_I(c, BUILD_TUPLE, n);
4932 }
4933 break;
4934 default:
4935 PyErr_Format(PyExc_SystemError,
4936 "invalid subscript kind %d", s->kind);
4937 return 0;
4938 }
4939 return compiler_handle_subscr(c, kindname, ctx);
4940 }
4941
4942 /* End of the compiler section, beginning of the assembler section */
4943
4944 /* do depth-first search of basic block graph, starting with block.
4945 post records the block indices in post-order.
4946
4947 XXX must handle implicit jumps from one block to next
4948 */
4949
4950 struct assembler {
4951 PyObject *a_bytecode; /* string containing bytecode */
4952 int a_offset; /* offset into bytecode */
4953 int a_nblocks; /* number of reachable blocks */
4954 basicblock **a_postorder; /* list of blocks in dfs postorder */
4955 PyObject *a_lnotab; /* string containing lnotab */
4956 int a_lnotab_off; /* offset into lnotab */
4957 int a_lineno; /* last lineno of emitted instruction */
4958 int a_lineno_off; /* bytecode offset of last lineno */
4959 };
4960
4961 static void
dfs(struct compiler * c,basicblock * b,struct assembler * a,int end)4962 dfs(struct compiler *c, basicblock *b, struct assembler *a, int end)
4963 {
4964 int i, j;
4965
4966 /* Get rid of recursion for normal control flow.
4967 Since the number of blocks is limited, unused space in a_postorder
4968 (from a_nblocks to end) can be used as a stack for still not ordered
4969 blocks. */
4970 for (j = end; b && !b->b_seen; b = b->b_next) {
4971 b->b_seen = 1;
4972 assert(a->a_nblocks < j);
4973 a->a_postorder[--j] = b;
4974 }
4975 while (j < end) {
4976 b = a->a_postorder[j++];
4977 for (i = 0; i < b->b_iused; i++) {
4978 struct instr *instr = &b->b_instr[i];
4979 if (instr->i_jrel || instr->i_jabs)
4980 dfs(c, instr->i_target, a, j);
4981 }
4982 assert(a->a_nblocks < j);
4983 a->a_postorder[a->a_nblocks++] = b;
4984 }
4985 }
4986
4987 Py_LOCAL_INLINE(void)
stackdepth_push(basicblock *** sp,basicblock * b,int depth)4988 stackdepth_push(basicblock ***sp, basicblock *b, int depth)
4989 {
4990 /* XXX b->b_startdepth > depth only for the target of SETUP_FINALLY,
4991 * SETUP_WITH and SETUP_ASYNC_WITH. */
4992 assert(b->b_startdepth < 0 || b->b_startdepth >= depth);
4993 if (b->b_startdepth < depth) {
4994 assert(b->b_startdepth < 0);
4995 b->b_startdepth = depth;
4996 *(*sp)++ = b;
4997 }
4998 }
4999
5000 /* Find the flow path that needs the largest stack. We assume that
5001 * cycles in the flow graph have no net effect on the stack depth.
5002 */
5003 static int
stackdepth(struct compiler * c)5004 stackdepth(struct compiler *c)
5005 {
5006 basicblock *b, *entryblock = NULL;
5007 basicblock **stack, **sp;
5008 int nblocks = 0, maxdepth = 0;
5009 for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5010 b->b_startdepth = INT_MIN;
5011 entryblock = b;
5012 nblocks++;
5013 }
5014 if (!entryblock)
5015 return 0;
5016 stack = (basicblock **)PyObject_Malloc(sizeof(basicblock *) * nblocks);
5017 if (!stack) {
5018 PyErr_NoMemory();
5019 return -1;
5020 }
5021
5022 sp = stack;
5023 stackdepth_push(&sp, entryblock, 0);
5024 while (sp != stack) {
5025 b = *--sp;
5026 int depth = b->b_startdepth;
5027 assert(depth >= 0);
5028 basicblock *next = b->b_next;
5029 for (int i = 0; i < b->b_iused; i++) {
5030 struct instr *instr = &b->b_instr[i];
5031 int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0);
5032 if (effect == PY_INVALID_STACK_EFFECT) {
5033 fprintf(stderr, "opcode = %d\n", instr->i_opcode);
5034 Py_FatalError("PyCompile_OpcodeStackEffect()");
5035 }
5036 int new_depth = depth + effect;
5037 if (new_depth > maxdepth) {
5038 maxdepth = new_depth;
5039 }
5040 assert(depth >= 0); /* invalid code or bug in stackdepth() */
5041 if (instr->i_jrel || instr->i_jabs) {
5042 effect = stack_effect(instr->i_opcode, instr->i_oparg, 1);
5043 assert(effect != PY_INVALID_STACK_EFFECT);
5044 int target_depth = depth + effect;
5045 if (target_depth > maxdepth) {
5046 maxdepth = target_depth;
5047 }
5048 assert(target_depth >= 0); /* invalid code or bug in stackdepth() */
5049 if (instr->i_opcode == CONTINUE_LOOP) {
5050 /* Pops a variable number of values from the stack,
5051 * but the target should be already proceeding.
5052 */
5053 assert(instr->i_target->b_startdepth >= 0);
5054 assert(instr->i_target->b_startdepth <= depth);
5055 /* remaining code is dead */
5056 next = NULL;
5057 break;
5058 }
5059 stackdepth_push(&sp, instr->i_target, target_depth);
5060 }
5061 depth = new_depth;
5062 if (instr->i_opcode == JUMP_ABSOLUTE ||
5063 instr->i_opcode == JUMP_FORWARD ||
5064 instr->i_opcode == RETURN_VALUE ||
5065 instr->i_opcode == RAISE_VARARGS ||
5066 instr->i_opcode == BREAK_LOOP)
5067 {
5068 /* remaining code is dead */
5069 next = NULL;
5070 break;
5071 }
5072 }
5073 if (next != NULL) {
5074 stackdepth_push(&sp, next, depth);
5075 }
5076 }
5077 PyObject_Free(stack);
5078 return maxdepth;
5079 }
5080
5081 static int
assemble_init(struct assembler * a,int nblocks,int firstlineno)5082 assemble_init(struct assembler *a, int nblocks, int firstlineno)
5083 {
5084 memset(a, 0, sizeof(struct assembler));
5085 a->a_lineno = firstlineno;
5086 a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE);
5087 if (!a->a_bytecode)
5088 return 0;
5089 a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE);
5090 if (!a->a_lnotab)
5091 return 0;
5092 if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) {
5093 PyErr_NoMemory();
5094 return 0;
5095 }
5096 a->a_postorder = (basicblock **)PyObject_Malloc(
5097 sizeof(basicblock *) * nblocks);
5098 if (!a->a_postorder) {
5099 PyErr_NoMemory();
5100 return 0;
5101 }
5102 return 1;
5103 }
5104
5105 static void
assemble_free(struct assembler * a)5106 assemble_free(struct assembler *a)
5107 {
5108 Py_XDECREF(a->a_bytecode);
5109 Py_XDECREF(a->a_lnotab);
5110 if (a->a_postorder)
5111 PyObject_Free(a->a_postorder);
5112 }
5113
5114 static int
blocksize(basicblock * b)5115 blocksize(basicblock *b)
5116 {
5117 int i;
5118 int size = 0;
5119
5120 for (i = 0; i < b->b_iused; i++)
5121 size += instrsize(b->b_instr[i].i_oparg);
5122 return size;
5123 }
5124
5125 /* Appends a pair to the end of the line number table, a_lnotab, representing
5126 the instruction's bytecode offset and line number. See
5127 Objects/lnotab_notes.txt for the description of the line number table. */
5128
5129 static int
assemble_lnotab(struct assembler * a,struct instr * i)5130 assemble_lnotab(struct assembler *a, struct instr *i)
5131 {
5132 int d_bytecode, d_lineno;
5133 Py_ssize_t len;
5134 unsigned char *lnotab;
5135
5136 d_bytecode = (a->a_offset - a->a_lineno_off) * sizeof(_Py_CODEUNIT);
5137 d_lineno = i->i_lineno - a->a_lineno;
5138
5139 assert(d_bytecode >= 0);
5140
5141 if(d_bytecode == 0 && d_lineno == 0)
5142 return 1;
5143
5144 if (d_bytecode > 255) {
5145 int j, nbytes, ncodes = d_bytecode / 255;
5146 nbytes = a->a_lnotab_off + 2 * ncodes;
5147 len = PyBytes_GET_SIZE(a->a_lnotab);
5148 if (nbytes >= len) {
5149 if ((len <= INT_MAX / 2) && (len * 2 < nbytes))
5150 len = nbytes;
5151 else if (len <= INT_MAX / 2)
5152 len *= 2;
5153 else {
5154 PyErr_NoMemory();
5155 return 0;
5156 }
5157 if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
5158 return 0;
5159 }
5160 lnotab = (unsigned char *)
5161 PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5162 for (j = 0; j < ncodes; j++) {
5163 *lnotab++ = 255;
5164 *lnotab++ = 0;
5165 }
5166 d_bytecode -= ncodes * 255;
5167 a->a_lnotab_off += ncodes * 2;
5168 }
5169 assert(0 <= d_bytecode && d_bytecode <= 255);
5170
5171 if (d_lineno < -128 || 127 < d_lineno) {
5172 int j, nbytes, ncodes, k;
5173 if (d_lineno < 0) {
5174 k = -128;
5175 /* use division on positive numbers */
5176 ncodes = (-d_lineno) / 128;
5177 }
5178 else {
5179 k = 127;
5180 ncodes = d_lineno / 127;
5181 }
5182 d_lineno -= ncodes * k;
5183 assert(ncodes >= 1);
5184 nbytes = a->a_lnotab_off + 2 * ncodes;
5185 len = PyBytes_GET_SIZE(a->a_lnotab);
5186 if (nbytes >= len) {
5187 if ((len <= INT_MAX / 2) && len * 2 < nbytes)
5188 len = nbytes;
5189 else if (len <= INT_MAX / 2)
5190 len *= 2;
5191 else {
5192 PyErr_NoMemory();
5193 return 0;
5194 }
5195 if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
5196 return 0;
5197 }
5198 lnotab = (unsigned char *)
5199 PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5200 *lnotab++ = d_bytecode;
5201 *lnotab++ = k;
5202 d_bytecode = 0;
5203 for (j = 1; j < ncodes; j++) {
5204 *lnotab++ = 0;
5205 *lnotab++ = k;
5206 }
5207 a->a_lnotab_off += ncodes * 2;
5208 }
5209 assert(-128 <= d_lineno && d_lineno <= 127);
5210
5211 len = PyBytes_GET_SIZE(a->a_lnotab);
5212 if (a->a_lnotab_off + 2 >= len) {
5213 if (_PyBytes_Resize(&a->a_lnotab, len * 2) < 0)
5214 return 0;
5215 }
5216 lnotab = (unsigned char *)
5217 PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5218
5219 a->a_lnotab_off += 2;
5220 if (d_bytecode) {
5221 *lnotab++ = d_bytecode;
5222 *lnotab++ = d_lineno;
5223 }
5224 else { /* First line of a block; def stmt, etc. */
5225 *lnotab++ = 0;
5226 *lnotab++ = d_lineno;
5227 }
5228 a->a_lineno = i->i_lineno;
5229 a->a_lineno_off = a->a_offset;
5230 return 1;
5231 }
5232
5233 /* assemble_emit()
5234 Extend the bytecode with a new instruction.
5235 Update lnotab if necessary.
5236 */
5237
5238 static int
assemble_emit(struct assembler * a,struct instr * i)5239 assemble_emit(struct assembler *a, struct instr *i)
5240 {
5241 int size, arg = 0;
5242 Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode);
5243 _Py_CODEUNIT *code;
5244
5245 arg = i->i_oparg;
5246 size = instrsize(arg);
5247 if (i->i_lineno && !assemble_lnotab(a, i))
5248 return 0;
5249 if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) {
5250 if (len > PY_SSIZE_T_MAX / 2)
5251 return 0;
5252 if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0)
5253 return 0;
5254 }
5255 code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset;
5256 a->a_offset += size;
5257 write_op_arg(code, i->i_opcode, arg, size);
5258 return 1;
5259 }
5260
5261 static void
assemble_jump_offsets(struct assembler * a,struct compiler * c)5262 assemble_jump_offsets(struct assembler *a, struct compiler *c)
5263 {
5264 basicblock *b;
5265 int bsize, totsize, extended_arg_recompile;
5266 int i;
5267
5268 /* Compute the size of each block and fixup jump args.
5269 Replace block pointer with position in bytecode. */
5270 do {
5271 totsize = 0;
5272 for (i = a->a_nblocks - 1; i >= 0; i--) {
5273 b = a->a_postorder[i];
5274 bsize = blocksize(b);
5275 b->b_offset = totsize;
5276 totsize += bsize;
5277 }
5278 extended_arg_recompile = 0;
5279 for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5280 bsize = b->b_offset;
5281 for (i = 0; i < b->b_iused; i++) {
5282 struct instr *instr = &b->b_instr[i];
5283 int isize = instrsize(instr->i_oparg);
5284 /* Relative jumps are computed relative to
5285 the instruction pointer after fetching
5286 the jump instruction.
5287 */
5288 bsize += isize;
5289 if (instr->i_jabs || instr->i_jrel) {
5290 instr->i_oparg = instr->i_target->b_offset;
5291 if (instr->i_jrel) {
5292 instr->i_oparg -= bsize;
5293 }
5294 instr->i_oparg *= sizeof(_Py_CODEUNIT);
5295 if (instrsize(instr->i_oparg) != isize) {
5296 extended_arg_recompile = 1;
5297 }
5298 }
5299 }
5300 }
5301
5302 /* XXX: This is an awful hack that could hurt performance, but
5303 on the bright side it should work until we come up
5304 with a better solution.
5305
5306 The issue is that in the first loop blocksize() is called
5307 which calls instrsize() which requires i_oparg be set
5308 appropriately. There is a bootstrap problem because
5309 i_oparg is calculated in the second loop above.
5310
5311 So we loop until we stop seeing new EXTENDED_ARGs.
5312 The only EXTENDED_ARGs that could be popping up are
5313 ones in jump instructions. So this should converge
5314 fairly quickly.
5315 */
5316 } while (extended_arg_recompile);
5317 }
5318
5319 static PyObject *
dict_keys_inorder(PyObject * dict,Py_ssize_t offset)5320 dict_keys_inorder(PyObject *dict, Py_ssize_t offset)
5321 {
5322 PyObject *tuple, *k, *v;
5323 Py_ssize_t i, pos = 0, size = PyDict_GET_SIZE(dict);
5324
5325 tuple = PyTuple_New(size);
5326 if (tuple == NULL)
5327 return NULL;
5328 while (PyDict_Next(dict, &pos, &k, &v)) {
5329 i = PyLong_AS_LONG(v);
5330 /* The keys of the dictionary are tuples. (see compiler_add_o
5331 * and _PyCode_ConstantKey). The object we want is always second,
5332 * though. */
5333 k = PyTuple_GET_ITEM(k, 1);
5334 Py_INCREF(k);
5335 assert((i - offset) < size);
5336 assert((i - offset) >= 0);
5337 PyTuple_SET_ITEM(tuple, i - offset, k);
5338 }
5339 return tuple;
5340 }
5341
5342 static int
compute_code_flags(struct compiler * c)5343 compute_code_flags(struct compiler *c)
5344 {
5345 PySTEntryObject *ste = c->u->u_ste;
5346 int flags = 0;
5347 if (ste->ste_type == FunctionBlock) {
5348 flags |= CO_NEWLOCALS | CO_OPTIMIZED;
5349 if (ste->ste_nested)
5350 flags |= CO_NESTED;
5351 if (ste->ste_generator && !ste->ste_coroutine)
5352 flags |= CO_GENERATOR;
5353 if (!ste->ste_generator && ste->ste_coroutine)
5354 flags |= CO_COROUTINE;
5355 if (ste->ste_generator && ste->ste_coroutine)
5356 flags |= CO_ASYNC_GENERATOR;
5357 if (ste->ste_varargs)
5358 flags |= CO_VARARGS;
5359 if (ste->ste_varkeywords)
5360 flags |= CO_VARKEYWORDS;
5361 }
5362
5363 /* (Only) inherit compilerflags in PyCF_MASK */
5364 flags |= (c->c_flags->cf_flags & PyCF_MASK);
5365
5366 return flags;
5367 }
5368
5369 static PyCodeObject *
makecode(struct compiler * c,struct assembler * a)5370 makecode(struct compiler *c, struct assembler *a)
5371 {
5372 PyObject *tmp;
5373 PyCodeObject *co = NULL;
5374 PyObject *consts = NULL;
5375 PyObject *names = NULL;
5376 PyObject *varnames = NULL;
5377 PyObject *name = NULL;
5378 PyObject *freevars = NULL;
5379 PyObject *cellvars = NULL;
5380 PyObject *bytecode = NULL;
5381 Py_ssize_t nlocals;
5382 int nlocals_int;
5383 int flags;
5384 int argcount, kwonlyargcount, maxdepth;
5385
5386 tmp = dict_keys_inorder(c->u->u_consts, 0);
5387 if (!tmp)
5388 goto error;
5389 consts = PySequence_List(tmp); /* optimize_code requires a list */
5390 Py_DECREF(tmp);
5391
5392 names = dict_keys_inorder(c->u->u_names, 0);
5393 varnames = dict_keys_inorder(c->u->u_varnames, 0);
5394 if (!consts || !names || !varnames)
5395 goto error;
5396
5397 cellvars = dict_keys_inorder(c->u->u_cellvars, 0);
5398 if (!cellvars)
5399 goto error;
5400 freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
5401 if (!freevars)
5402 goto error;
5403
5404 nlocals = PyDict_GET_SIZE(c->u->u_varnames);
5405 assert(nlocals < INT_MAX);
5406 nlocals_int = Py_SAFE_DOWNCAST(nlocals, Py_ssize_t, int);
5407
5408 flags = compute_code_flags(c);
5409 if (flags < 0)
5410 goto error;
5411
5412 bytecode = PyCode_Optimize(a->a_bytecode, consts, names, a->a_lnotab);
5413 if (!bytecode)
5414 goto error;
5415
5416 tmp = PyList_AsTuple(consts); /* PyCode_New requires a tuple */
5417 if (!tmp)
5418 goto error;
5419 Py_DECREF(consts);
5420 consts = tmp;
5421
5422 argcount = Py_SAFE_DOWNCAST(c->u->u_argcount, Py_ssize_t, int);
5423 kwonlyargcount = Py_SAFE_DOWNCAST(c->u->u_kwonlyargcount, Py_ssize_t, int);
5424 maxdepth = stackdepth(c);
5425 if (maxdepth < 0) {
5426 goto error;
5427 }
5428 co = PyCode_New(argcount, kwonlyargcount,
5429 nlocals_int, maxdepth, flags,
5430 bytecode, consts, names, varnames,
5431 freevars, cellvars,
5432 c->c_filename, c->u->u_name,
5433 c->u->u_firstlineno,
5434 a->a_lnotab);
5435 error:
5436 Py_XDECREF(consts);
5437 Py_XDECREF(names);
5438 Py_XDECREF(varnames);
5439 Py_XDECREF(name);
5440 Py_XDECREF(freevars);
5441 Py_XDECREF(cellvars);
5442 Py_XDECREF(bytecode);
5443 return co;
5444 }
5445
5446
5447 /* For debugging purposes only */
5448 #if 0
5449 static void
5450 dump_instr(const struct instr *i)
5451 {
5452 const char *jrel = i->i_jrel ? "jrel " : "";
5453 const char *jabs = i->i_jabs ? "jabs " : "";
5454 char arg[128];
5455
5456 *arg = '\0';
5457 if (HAS_ARG(i->i_opcode)) {
5458 sprintf(arg, "arg: %d ", i->i_oparg);
5459 }
5460 fprintf(stderr, "line: %d, opcode: %d %s%s%s\n",
5461 i->i_lineno, i->i_opcode, arg, jabs, jrel);
5462 }
5463
5464 static void
5465 dump_basicblock(const basicblock *b)
5466 {
5467 const char *seen = b->b_seen ? "seen " : "";
5468 const char *b_return = b->b_return ? "return " : "";
5469 fprintf(stderr, "used: %d, depth: %d, offset: %d %s%s\n",
5470 b->b_iused, b->b_startdepth, b->b_offset, seen, b_return);
5471 if (b->b_instr) {
5472 int i;
5473 for (i = 0; i < b->b_iused; i++) {
5474 fprintf(stderr, " [%02d] ", i);
5475 dump_instr(b->b_instr + i);
5476 }
5477 }
5478 }
5479 #endif
5480
5481 static PyCodeObject *
assemble(struct compiler * c,int addNone)5482 assemble(struct compiler *c, int addNone)
5483 {
5484 basicblock *b, *entryblock;
5485 struct assembler a;
5486 int i, j, nblocks;
5487 PyCodeObject *co = NULL;
5488
5489 /* Make sure every block that falls off the end returns None.
5490 XXX NEXT_BLOCK() isn't quite right, because if the last
5491 block ends with a jump or return b_next shouldn't set.
5492 */
5493 if (!c->u->u_curblock->b_return) {
5494 NEXT_BLOCK(c);
5495 if (addNone)
5496 ADDOP_O(c, LOAD_CONST, Py_None, consts);
5497 ADDOP(c, RETURN_VALUE);
5498 }
5499
5500 nblocks = 0;
5501 entryblock = NULL;
5502 for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5503 nblocks++;
5504 entryblock = b;
5505 }
5506
5507 /* Set firstlineno if it wasn't explicitly set. */
5508 if (!c->u->u_firstlineno) {
5509 if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno)
5510 c->u->u_firstlineno = entryblock->b_instr->i_lineno;
5511 else
5512 c->u->u_firstlineno = 1;
5513 }
5514 if (!assemble_init(&a, nblocks, c->u->u_firstlineno))
5515 goto error;
5516 dfs(c, entryblock, &a, nblocks);
5517
5518 /* Can't modify the bytecode after computing jump offsets. */
5519 assemble_jump_offsets(&a, c);
5520
5521 /* Emit code in reverse postorder from dfs. */
5522 for (i = a.a_nblocks - 1; i >= 0; i--) {
5523 b = a.a_postorder[i];
5524 for (j = 0; j < b->b_iused; j++)
5525 if (!assemble_emit(&a, &b->b_instr[j]))
5526 goto error;
5527 }
5528
5529 if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0)
5530 goto error;
5531 if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0)
5532 goto error;
5533
5534 co = makecode(c, &a);
5535 error:
5536 assemble_free(&a);
5537 return co;
5538 }
5539
5540 #undef PyAST_Compile
5541 PyAPI_FUNC(PyCodeObject *)
PyAST_Compile(mod_ty mod,const char * filename,PyCompilerFlags * flags,PyArena * arena)5542 PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
5543 PyArena *arena)
5544 {
5545 return PyAST_CompileEx(mod, filename, flags, -1, arena);
5546 }
5547