1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #if ZEND_JIT_TARGET_X86
43 # include "jit/zend_jit_x86.h"
44 #elif ZEND_JIT_TARGET_ARM64
45 # include "jit/zend_jit_arm64.h"
46 #endif
47
48 #include "jit/zend_jit_internal.h"
49
50 #ifdef ZTS
51 int jit_globals_id;
52 #else
53 zend_jit_globals jit_globals;
54 #endif
55
56 //#define CONTEXT_THREADED_JIT
57 #define ZEND_JIT_USE_RC_INFERENCE
58
59 #ifdef ZEND_JIT_USE_RC_INFERENCE
60 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
61 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
62 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
63 #else
64 # define ZEND_SSA_RC_INFERENCE_FLAG 0
65 # define RC_MAY_BE_1(info) 1
66 # define RC_MAY_BE_N(info) 1
67 #endif
68
69 #define JIT_PREFIX "JIT$"
70 #define JIT_STUB_PREFIX "JIT$$"
71 #define TRACE_PREFIX "TRACE-"
72
73 #define DASM_M_GROW(ctx, t, p, sz, need) \
74 do { \
75 size_t _sz = (sz), _need = (need); \
76 if (_sz < _need) { \
77 if (_sz < 16) _sz = 16; \
78 while (_sz < _need) _sz += _sz; \
79 (p) = (t *)erealloc((p), _sz); \
80 (sz) = _sz; \
81 } \
82 } while(0)
83
84 #define DASM_M_FREE(ctx, p, sz) efree(p)
85
86 #if ZEND_DEBUG
87 # define DASM_CHECKS 1
88 #endif
89
90 #include "dynasm/dasm_proto.h"
91
92 typedef struct _zend_jit_stub {
93 const char *name;
94 int (*stub)(dasm_State **Dst);
95 uint32_t offset;
96 uint32_t adjustment;
97 } zend_jit_stub;
98
99 #define JIT_STUB(name, offset, adjustment) \
100 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment}
101
102 zend_ulong zend_jit_profile_counter = 0;
103 int zend_jit_profile_counter_rid = -1;
104
105 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
106
107 const zend_op *zend_jit_halt_op = NULL;
108 static int zend_jit_vm_kind = 0;
109
110 static void *dasm_buf = NULL;
111 static void *dasm_end = NULL;
112 static void **dasm_ptr = NULL;
113
114 static size_t dasm_size = 0;
115
116 static zend_long jit_bisect_pos = 0;
117
118 static const void *zend_jit_runtime_jit_handler = NULL;
119 static const void *zend_jit_profile_jit_handler = NULL;
120 static const void *zend_jit_func_hot_counter_handler = NULL;
121 static const void *zend_jit_loop_hot_counter_handler = NULL;
122 static const void *zend_jit_func_trace_counter_handler = NULL;
123 static const void *zend_jit_ret_trace_counter_handler = NULL;
124 static const void *zend_jit_loop_trace_counter_handler = NULL;
125
126 static int ZEND_FASTCALL zend_runtime_jit(void);
127
128 static int zend_jit_trace_op_len(const zend_op *opline);
129 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
130 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
131 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
132 static void zend_jit_trace_add_code(const void *start, uint32_t size);
133 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
134
135 #if ZEND_JIT_TARGET_ARM64
136 static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
137 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
138 #endif
139
140 static int zend_jit_assign_to_variable(dasm_State **Dst,
141 const zend_op *opline,
142 zend_jit_addr var_use_addr,
143 zend_jit_addr var_addr,
144 uint32_t var_info,
145 uint32_t var_def_info,
146 zend_uchar val_type,
147 zend_jit_addr val_addr,
148 uint32_t val_info,
149 zend_jit_addr res_addr,
150 bool check_exception);
151
dominates(const zend_basic_block * blocks,int a,int b)152 static bool dominates(const zend_basic_block *blocks, int a, int b) {
153 while (blocks[b].level > blocks[a].level) {
154 b = blocks[b].idom;
155 }
156 return a == b;
157 }
158
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)159 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
160 {
161 int next_use;
162
163 if (ssa->vars[var].phi_use_chain) {
164 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
165 do {
166 if (!ssa->vars[phi->ssa_var].no_val) {
167 return 0;
168 }
169 phi = zend_ssa_next_use_phi(ssa, var, phi);
170 } while (phi);
171 }
172
173 next_use = zend_ssa_next_use(ssa->ops, var, use);
174 if (next_use < 0) {
175 int b = ssa->cfg.map[use];
176 int prev_use = ssa->vars[var].use_chain;
177
178 while (prev_use >= 0 && prev_use != use) {
179 if (b != ssa->cfg.map[prev_use]
180 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
181 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
182 return 0;
183 }
184 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
185 }
186 return 1;
187 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
188 return 1;
189 }
190 return 0;
191 }
192
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)193 static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
194 {
195 if (ival->flags & ZREG_LAST_USE) {
196 const zend_life_range *range = &ival->range;
197
198 while (range->next) {
199 range = range->next;
200 }
201 return range->end == use;
202 }
203 return 0;
204 }
205
zend_is_commutative(zend_uchar opcode)206 static bool zend_is_commutative(zend_uchar opcode)
207 {
208 return
209 opcode == ZEND_ADD ||
210 opcode == ZEND_MUL ||
211 opcode == ZEND_BW_OR ||
212 opcode == ZEND_BW_AND ||
213 opcode == ZEND_BW_XOR;
214 }
215
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)216 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
217 zend_ssa_range *op1_range,
218 zend_jit_addr op1_addr,
219 zend_ssa_range *op2_range,
220 zend_jit_addr op2_addr,
221 bool *result)
222 {
223 zend_long op1_min;
224 zend_long op1_max;
225 zend_long op2_min;
226 zend_long op2_max;
227
228 if (op1_range) {
229 op1_min = op1_range->min;
230 op1_max = op1_range->max;
231 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
232 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
233 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
234 } else {
235 return 0;
236 }
237
238 if (op2_range) {
239 op2_min = op2_range->min;
240 op2_max = op2_range->max;
241 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
242 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
243 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
244 } else {
245 return 0;
246 }
247
248 switch (opline->opcode) {
249 case ZEND_IS_EQUAL:
250 case ZEND_IS_IDENTICAL:
251 case ZEND_CASE:
252 case ZEND_CASE_STRICT:
253 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
254 *result = 1;
255 return 1;
256 } else if (op1_max < op2_min || op1_min > op2_max) {
257 *result = 0;
258 return 1;
259 }
260 return 0;
261 case ZEND_IS_NOT_EQUAL:
262 case ZEND_IS_NOT_IDENTICAL:
263 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
264 *result = 0;
265 return 1;
266 } else if (op1_max < op2_min || op1_min > op2_max) {
267 *result = 1;
268 return 1;
269 }
270 return 0;
271 case ZEND_IS_SMALLER:
272 if (op1_max < op2_min) {
273 *result = 1;
274 return 1;
275 } else if (op1_min >= op2_max) {
276 *result = 0;
277 return 1;
278 }
279 return 0;
280 case ZEND_IS_SMALLER_OR_EQUAL:
281 if (op1_max <= op2_min) {
282 *result = 1;
283 return 1;
284 } else if (op1_min > op2_max) {
285 *result = 0;
286 return 1;
287 }
288 return 0;
289 default:
290 ZEND_UNREACHABLE();
291 }
292 return 0;
293 }
294
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)295 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
296 {
297 int skip;
298
299 if (trace) {
300 zend_jit_trace_rec *p = trace;
301
302 ssa_op++;
303 while (1) {
304 if (p->op == ZEND_JIT_TRACE_VM) {
305 switch (p->opline->opcode) {
306 case ZEND_SEND_ARRAY:
307 case ZEND_SEND_USER:
308 case ZEND_SEND_UNPACK:
309 case ZEND_INIT_FCALL:
310 case ZEND_INIT_METHOD_CALL:
311 case ZEND_INIT_STATIC_METHOD_CALL:
312 case ZEND_INIT_FCALL_BY_NAME:
313 case ZEND_INIT_NS_FCALL_BY_NAME:
314 case ZEND_INIT_DYNAMIC_CALL:
315 case ZEND_NEW:
316 case ZEND_INIT_USER_CALL:
317 case ZEND_FAST_CALL:
318 case ZEND_JMP:
319 case ZEND_JMPZNZ:
320 case ZEND_JMPZ:
321 case ZEND_JMPNZ:
322 case ZEND_JMPZ_EX:
323 case ZEND_JMPNZ_EX:
324 case ZEND_FE_RESET_R:
325 case ZEND_FE_RESET_RW:
326 case ZEND_JMP_SET:
327 case ZEND_COALESCE:
328 case ZEND_JMP_NULL:
329 case ZEND_ASSERT_CHECK:
330 case ZEND_CATCH:
331 case ZEND_DECLARE_ANON_CLASS:
332 case ZEND_FE_FETCH_R:
333 case ZEND_FE_FETCH_RW:
334 return 1;
335 case ZEND_DO_ICALL:
336 case ZEND_DO_UCALL:
337 case ZEND_DO_FCALL_BY_NAME:
338 case ZEND_DO_FCALL:
339 case ZEND_CALLABLE_CONVERT:
340 return 0;
341 case ZEND_SEND_VAL:
342 case ZEND_SEND_VAR:
343 case ZEND_SEND_VAL_EX:
344 case ZEND_SEND_VAR_EX:
345 case ZEND_SEND_FUNC_ARG:
346 case ZEND_SEND_REF:
347 case ZEND_SEND_VAR_NO_REF:
348 case ZEND_SEND_VAR_NO_REF_EX:
349 /* skip */
350 break;
351 default:
352 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
353 return 1;
354 }
355 }
356 ssa_op += zend_jit_trace_op_len(opline);
357 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
358 p->op == ZEND_JIT_TRACE_BACK ||
359 p->op == ZEND_JIT_TRACE_END) {
360 return 1;
361 }
362 p++;
363 }
364 }
365
366 if (!call_info) {
367 const zend_op *end = op_array->opcodes + op_array->last;
368
369 opline++;
370 ssa_op++;
371 skip = (call_level == 1);
372 while (opline != end) {
373 if (!skip) {
374 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
375 return 1;
376 }
377 }
378 switch (opline->opcode) {
379 case ZEND_SEND_VAL:
380 case ZEND_SEND_VAR:
381 case ZEND_SEND_VAL_EX:
382 case ZEND_SEND_VAR_EX:
383 case ZEND_SEND_FUNC_ARG:
384 case ZEND_SEND_REF:
385 case ZEND_SEND_VAR_NO_REF:
386 case ZEND_SEND_VAR_NO_REF_EX:
387 skip = 0;
388 break;
389 case ZEND_SEND_ARRAY:
390 case ZEND_SEND_USER:
391 case ZEND_SEND_UNPACK:
392 case ZEND_INIT_FCALL:
393 case ZEND_INIT_METHOD_CALL:
394 case ZEND_INIT_STATIC_METHOD_CALL:
395 case ZEND_INIT_FCALL_BY_NAME:
396 case ZEND_INIT_NS_FCALL_BY_NAME:
397 case ZEND_INIT_DYNAMIC_CALL:
398 case ZEND_NEW:
399 case ZEND_INIT_USER_CALL:
400 case ZEND_FAST_CALL:
401 case ZEND_JMP:
402 case ZEND_JMPZNZ:
403 case ZEND_JMPZ:
404 case ZEND_JMPNZ:
405 case ZEND_JMPZ_EX:
406 case ZEND_JMPNZ_EX:
407 case ZEND_FE_RESET_R:
408 case ZEND_FE_RESET_RW:
409 case ZEND_JMP_SET:
410 case ZEND_COALESCE:
411 case ZEND_JMP_NULL:
412 case ZEND_ASSERT_CHECK:
413 case ZEND_CATCH:
414 case ZEND_DECLARE_ANON_CLASS:
415 case ZEND_FE_FETCH_R:
416 case ZEND_FE_FETCH_RW:
417 return 1;
418 case ZEND_DO_ICALL:
419 case ZEND_DO_UCALL:
420 case ZEND_DO_FCALL_BY_NAME:
421 case ZEND_DO_FCALL:
422 case ZEND_CALLABLE_CONVERT:
423 end = opline;
424 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
425 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
426 return 1;
427 }
428 return 0;
429 }
430 opline++;
431 ssa_op++;
432 }
433
434 return 1;
435 } else {
436 const zend_op *end = call_info->caller_call_opline;
437
438 /* end may be null if an opcode like EXIT is part of the argument list. */
439 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
440 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
441 return 1;
442 }
443
444 opline++;
445 ssa_op++;
446 skip = (call_level == 1);
447 while (opline != end) {
448 if (skip) {
449 switch (opline->opcode) {
450 case ZEND_SEND_VAL:
451 case ZEND_SEND_VAR:
452 case ZEND_SEND_VAL_EX:
453 case ZEND_SEND_VAR_EX:
454 case ZEND_SEND_FUNC_ARG:
455 case ZEND_SEND_REF:
456 case ZEND_SEND_VAR_NO_REF:
457 case ZEND_SEND_VAR_NO_REF_EX:
458 skip = 0;
459 break;
460 case ZEND_SEND_ARRAY:
461 case ZEND_SEND_USER:
462 case ZEND_SEND_UNPACK:
463 return 1;
464 }
465 } else {
466 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
467 return 1;
468 }
469 }
470 opline++;
471 ssa_op++;
472 }
473
474 return 0;
475 }
476 }
477
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)478 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
479 {
480 uint32_t num_args = 0;
481 zend_function *func = call_info->callee_func;
482
483 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
484 * Anything legal for the parent method is also legal for the parent method. */
485 while (num_args < call_info->num_args) {
486 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
487
488 if (ZEND_TYPE_IS_SET(arg_info->type)) {
489 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
490 zend_op *opline = call_info->arg_info[num_args].opline;
491 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
492 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
493 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
494 break;
495 }
496 } else {
497 break;
498 }
499 }
500 num_args++;
501 }
502 return num_args;
503 }
504
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)505 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
506 {
507 uint32_t j, info;
508
509 if (ssa->vars && ssa->var_info) {
510 info = ssa->var_info[var].type;
511 for (j = op_array->last_var; j < ssa->vars_count; j++) {
512 if (ssa->vars[j].var == var) {
513 info |= ssa->var_info[j].type;
514 }
515 }
516 } else {
517 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
518 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
519 }
520
521 #ifdef ZEND_JIT_USE_RC_INFERENCE
522 /* Refcount may be increased by RETURN opcode */
523 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
524 for (j = 0; j < ssa->cfg.blocks_count; j++) {
525 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
526 ssa->cfg.blocks[j].len > 0) {
527 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
528
529 if (opline->opcode == ZEND_RETURN) {
530 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
531 info |= MAY_BE_RCN;
532 break;
533 }
534 }
535 }
536 }
537 }
538 #endif
539
540 return info;
541 }
542
zend_jit_may_avoid_refcounting(const zend_op * opline)543 static bool zend_jit_may_avoid_refcounting(const zend_op *opline)
544 {
545 switch (opline->opcode) {
546 case ZEND_FETCH_OBJ_FUNC_ARG:
547 if (!JIT_G(current_frame) ||
548 !JIT_G(current_frame)->call->func ||
549 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
550 return 0;
551 }
552 /* break missing intentionally */
553 case ZEND_FETCH_OBJ_R:
554 case ZEND_FETCH_OBJ_IS:
555 if (opline->op2_type == IS_CONST
556 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
557 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
558 return 1;
559 }
560 break;
561 case ZEND_FETCH_DIM_FUNC_ARG:
562 if (!JIT_G(current_frame) ||
563 !JIT_G(current_frame)->call->func ||
564 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
565 return 0;
566 }
567 /* break missing intentionally */
568 case ZEND_FETCH_DIM_R:
569 case ZEND_FETCH_DIM_IS:
570 return 1;
571 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
572 if (!(opline->extended_value & ZEND_ISEMPTY)) {
573 return 1;
574 }
575 break;
576 }
577 return 0;
578 }
579
zend_jit_is_persistent_constant(zval * key,uint32_t flags)580 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
581 {
582 zval *zv;
583 zend_constant *c = NULL;
584
585 /* null/true/false are resolved during compilation, so don't check for them here. */
586 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
587 if (zv) {
588 c = (zend_constant*)Z_PTR_P(zv);
589 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
590 key++;
591 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
592 if (zv) {
593 c = (zend_constant*)Z_PTR_P(zv);
594 }
595 }
596 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
597 }
598
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)599 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
600 {
601 zend_property_info *info = NULL;
602
603 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
604 !ce ||
605 !(ce->ce_flags & ZEND_ACC_LINKED) ||
606 (ce->ce_flags & ZEND_ACC_TRAIT) ||
607 ce->create_object) {
608 return NULL;
609 }
610
611 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
612 if (ce->info.user.filename != filename) {
613 /* class declaration might be changed independently */
614 return NULL;
615 }
616
617 if (ce->parent) {
618 zend_class_entry *parent = ce->parent;
619
620 do {
621 if (parent->type == ZEND_INTERNAL_CLASS) {
622 break;
623 } else if (parent->info.user.filename != filename) {
624 /* some of parents class declarations might be changed independently */
625 /* TODO: this check may be not enough, because even
626 * in the same it's possible to conditionally define
627 * few classes with the same name, and "parent" may
628 * change from request to request.
629 */
630 return NULL;
631 }
632 parent = parent->parent;
633 } while (parent);
634 }
635 }
636
637 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
638 if (info == NULL ||
639 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
640 (info->flags & ZEND_ACC_STATIC)) {
641 return NULL;
642 }
643
644 if (info->flags & ZEND_ACC_PUBLIC) {
645 return info;
646 } else if (on_this) {
647 if (ce == info->ce) {
648 return info;
649 } else if ((info->flags & ZEND_ACC_PROTECTED)
650 && instanceof_function_slow(ce, info->ce)) {
651 return info;
652 }
653 }
654
655 return NULL;
656 }
657
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)658 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
659 {
660 zend_property_info *info;
661
662 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
663 return 1;
664 }
665
666 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
667 if (ce->info.user.filename != filename) {
668 /* class declaration might be changed independently */
669 return 1;
670 }
671 }
672
673 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
674 if (info == NULL ||
675 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
676 (info->flags & ZEND_ACC_STATIC)) {
677 return 1;
678 }
679
680 if (!(info->flags & ZEND_ACC_PUBLIC) &&
681 (!on_this || info->ce != ce)) {
682 return 1;
683 }
684
685 return 0;
686 }
687
688 #define OP_RANGE(ssa_op, opN) \
689 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
690 ssa->var_info && \
691 (ssa_op)->opN##_use >= 0 && \
692 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
693 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
694
695 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
696 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
697 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
698
699 #if ZEND_JIT_TARGET_X86
700 # include "dynasm/dasm_x86.h"
701 #elif ZEND_JIT_TARGET_ARM64
702 static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
703 # define DASM_ADD_VENEER zend_jit_add_veneer
704 # include "dynasm/dasm_arm64.h"
705 #endif
706
707 #include "jit/zend_jit_helpers.c"
708 #include "jit/zend_jit_disasm.c"
709 #ifndef _WIN32
710 # include "jit/zend_jit_gdb.h"
711 # include "jit/zend_jit_perf_dump.c"
712 #endif
713 #ifdef HAVE_OPROFILE
714 # include "jit/zend_jit_oprofile.c"
715 #endif
716
717 #include "Zend/zend_cpuinfo.h"
718
719 #ifdef HAVE_VALGRIND
720 # include <valgrind/valgrind.h>
721 #endif
722
723 #ifdef HAVE_GCC_GLOBAL_REGS
724 # define GCC_GLOBAL_REGS 1
725 #else
726 # define GCC_GLOBAL_REGS 0
727 #endif
728
729 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
730 * type information. Disabling this option allows testing some JIT handlers in the
731 * presence of try/catch blocks, which prevent SSA construction. */
732 #ifndef PROFITABILITY_CHECKS
733 # define PROFITABILITY_CHECKS 1
734 #endif
735
736 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
737
738 typedef enum _sp_adj_kind {
739 SP_ADJ_NONE,
740 SP_ADJ_RET,
741 SP_ADJ_VM,
742 SP_ADJ_JIT,
743 SP_ADJ_ASSIGN,
744 SP_ADJ_LAST
745 } sp_adj_kind;
746
747 static int sp_adj[SP_ADJ_LAST];
748
749 /* The generated code may contain tautological comparisons, ignore them. */
750 #if defined(__clang__)
751 # pragma clang diagnostic push
752 # pragma clang diagnostic ignored "-Wtautological-compare"
753 # pragma clang diagnostic ignored "-Wstring-compare"
754 #endif
755
756 #if ZEND_JIT_TARGET_X86
757 # include "jit/zend_jit_vtune.c"
758 # include "jit/zend_jit_x86.c"
759 #elif ZEND_JIT_TARGET_ARM64
760 # include "jit/zend_jit_arm64.c"
761 #endif
762
763 #if defined(__clang__)
764 # pragma clang diagnostic pop
765 #endif
766
767 #if _WIN32
768 # include <Windows.h>
769 #else
770 # include <sys/mman.h>
771 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
772 # define MAP_ANONYMOUS MAP_ANON
773 # endif
774 #endif
775
zend_jit_status(zval * ret)776 ZEND_EXT_API void zend_jit_status(zval *ret)
777 {
778 zval stats;
779 array_init(&stats);
780 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
781 add_assoc_bool(&stats, "on", JIT_G(on));
782 add_assoc_long(&stats, "kind", JIT_G(trigger));
783 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
784 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
785 if (dasm_buf) {
786 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
787 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
788 } else {
789 add_assoc_long(&stats, "buffer_size", 0);
790 add_assoc_long(&stats, "buffer_free", 0);
791 }
792 add_assoc_zval(ret, "jit", &stats);
793 }
794
zend_jit_func_name(const zend_op_array * op_array)795 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
796 {
797 smart_str buf = {0};
798
799 if (op_array->function_name) {
800 if (op_array->scope) {
801 smart_str_appends(&buf, JIT_PREFIX);
802 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
803 smart_str_appends(&buf, "::");
804 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
805 smart_str_0(&buf);
806 return buf.s;
807 } else {
808 smart_str_appends(&buf, JIT_PREFIX);
809 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
810 smart_str_0(&buf);
811 return buf.s;
812 }
813 } else if (op_array->filename) {
814 smart_str_appends(&buf, JIT_PREFIX);
815 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
816 smart_str_0(&buf);
817 return buf.s;
818 } else {
819 return NULL;
820 }
821 }
822
823 #if ZEND_DEBUG
handle_dasm_error(int ret)824 static void handle_dasm_error(int ret) {
825 switch (ret & 0xff000000u) {
826 case DASM_S_NOMEM:
827 fprintf(stderr, "DASM_S_NOMEM\n");
828 break;
829 case DASM_S_PHASE:
830 fprintf(stderr, "DASM_S_PHASE\n");
831 break;
832 case DASM_S_MATCH_SEC:
833 fprintf(stderr, "DASM_S_MATCH_SEC\n");
834 break;
835 case DASM_S_RANGE_I:
836 fprintf(stderr, "DASM_S_RANGE_I\n");
837 break;
838 case DASM_S_RANGE_SEC:
839 fprintf(stderr, "DASM_S_RANGE_SEC\n");
840 break;
841 case DASM_S_RANGE_LG:
842 fprintf(stderr, "DASM_S_RANGE_LG\n");
843 break;
844 case DASM_S_RANGE_PC:
845 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
846 break;
847 #ifdef DASM_S_RANGE_VREG
848 case DASM_S_RANGE_VREG:
849 fprintf(stderr, "DASM_S_RANGE_VREG\n");
850 break;
851 #endif
852 #ifdef DASM_S_UNDEF_L
853 case DASM_S_UNDEF_L:
854 fprintf(stderr, "DASM_S_UNDEF_L\n");
855 break;
856 #endif
857 #ifdef DASM_S_UNDEF_LG
858 case DASM_S_UNDEF_LG:
859 fprintf(stderr, "DASM_S_UNDEF_LG\n");
860 break;
861 #endif
862 #ifdef DASM_S_RANGE_REL
863 case DASM_S_RANGE_REL:
864 fprintf(stderr, "DASM_S_RANGE_REL\n");
865 break;
866 #endif
867 case DASM_S_UNDEF_PC:
868 fprintf(stderr, "DASM_S_UNDEF_PC\n");
869 break;
870 default:
871 fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
872 break;
873 }
874 ZEND_UNREACHABLE();
875 }
876 #endif
877
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num,uint32_t sp_offset,uint32_t sp_adjustment)878 static void *dasm_link_and_encode(dasm_State **dasm_state,
879 const zend_op_array *op_array,
880 zend_ssa *ssa,
881 const zend_op *rt_opline,
882 zend_lifetime_interval **ra,
883 const char *name,
884 uint32_t trace_num,
885 uint32_t sp_offset,
886 uint32_t sp_adjustment)
887 {
888 size_t size;
889 int ret;
890 void *entry;
891 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
892 zend_string *str = NULL;
893 #endif
894
895 if (rt_opline && ssa && ssa->cfg.map) {
896 /* Create additional entry point, to switch from interpreter to JIT-ed
897 * code at run-time.
898 */
899 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
900
901 //#ifdef CONTEXT_THREADED_JIT
902 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
903 //#else
904 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
905 //#endif
906 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
907 zend_jit_prologue(dasm_state);
908 if (ra) {
909 int i;
910 zend_lifetime_interval *ival;
911 zend_life_range *range;
912 uint32_t pos = rt_opline - op_array->opcodes;
913
914 for (i = 0; i < ssa->vars_count; i++) {
915 ival = ra[i];
916
917 if (ival && ival->reg != ZREG_NONE) {
918 range = &ival->range;
919
920 if (pos >= range->start && pos <= range->end) {
921 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
922 return NULL;
923 }
924 break;
925 }
926 range = range->next;
927 }
928 }
929 }
930 zend_jit_jmp(dasm_state, b);
931 }
932 }
933
934 ret = dasm_link(dasm_state, &size);
935 if (ret != DASM_S_OK) {
936 #if ZEND_DEBUG
937 handle_dasm_error(ret);
938 #endif
939 return NULL;
940 }
941
942 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
943 *dasm_ptr = dasm_end; //prevent further try
944 // TODO: jit_buffer_size overflow ???
945 return NULL;
946 }
947
948 #if ZEND_JIT_TARGET_ARM64
949 dasm_venners_size = 0;
950 #endif
951
952 ret = dasm_encode(dasm_state, *dasm_ptr);
953 if (ret != DASM_S_OK) {
954 #if ZEND_DEBUG
955 handle_dasm_error(ret);
956 #endif
957 return NULL;
958 }
959
960 #if ZEND_JIT_TARGET_ARM64
961 size += dasm_venners_size;
962 #endif
963
964 entry = *dasm_ptr;
965 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
966
967 /* flush the hardware I-cache */
968 JIT_CACHE_FLUSH(entry, entry + size);
969
970 if (trace_num) {
971 zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
972 }
973
974 if (op_array && ssa) {
975 int b;
976
977 for (b = 0; b < ssa->cfg.blocks_count; b++) {
978 //#ifdef CONTEXT_THREADED_JIT
979 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
980 //#else
981 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
982 //#endif
983 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
984 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
985
986 if (offset >= 0) {
987 opline->handler = (void*)(((char*)entry) + offset);
988 }
989 }
990 }
991 if (rt_opline && ssa && ssa->cfg.map) {
992 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
993 zend_op *opline = (zend_op*)rt_opline;
994 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
995
996 if (offset >= 0) {
997 opline->handler = (void*)(((char*)entry) + offset);
998 }
999 }
1000 }
1001
1002 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1003 if (!name) {
1004 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
1005 str = zend_jit_func_name(op_array);
1006 if (str) {
1007 name = ZSTR_VAL(str);
1008 }
1009 }
1010 #ifdef HAVE_DISASM
1011 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
1012 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1013 zend_jit_disasm(
1014 name,
1015 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1016 op_array,
1017 &ssa->cfg,
1018 entry,
1019 size);
1020 }
1021 } else {
1022 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
1023 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1024 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
1025 zend_jit_disasm(
1026 name,
1027 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1028 op_array,
1029 ssa ? &ssa->cfg : NULL,
1030 entry,
1031 size);
1032 }
1033 }
1034 # endif
1035 }
1036 #endif
1037
1038 #ifdef HAVE_GDB
1039 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
1040 if (name) {
1041 zend_jit_gdb_register(
1042 name,
1043 op_array,
1044 entry,
1045 size,
1046 sp_adj[sp_offset],
1047 sp_adj[sp_adjustment]);
1048 }
1049 }
1050 #endif
1051
1052 #ifdef HAVE_OPROFILE
1053 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
1054 zend_jit_oprofile_register(
1055 name,
1056 entry,
1057 size);
1058 }
1059 #endif
1060
1061 #ifdef HAVE_PERFTOOLS
1062 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
1063 if (name) {
1064 zend_jit_perf_map_register(
1065 name,
1066 entry,
1067 size);
1068 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
1069 zend_jit_perf_jitdump_register(
1070 name,
1071 entry,
1072 size);
1073 }
1074 }
1075 }
1076 #endif
1077
1078 #ifdef HAVE_VTUNE
1079 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
1080 if (name) {
1081 zend_jit_vtune_register(
1082 name,
1083 entry,
1084 size);
1085 }
1086 }
1087 #endif
1088
1089 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1090 if (str) {
1091 zend_string_release(str);
1092 }
1093 #endif
1094
1095 return entry;
1096 }
1097
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)1098 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
1099 {
1100 int res;
1101 zend_long op1_min, op1_max, op2_min, op2_max;
1102
1103 if (!ssa->ops || !ssa->var_info) {
1104 return 1;
1105 }
1106 switch (opline->opcode) {
1107 case ZEND_PRE_INC:
1108 case ZEND_POST_INC:
1109 res = ssa_op->op1_def;
1110 if (res < 0
1111 || !ssa->var_info[res].has_range
1112 || ssa->var_info[res].range.overflow) {
1113 if (!OP1_HAS_RANGE()) {
1114 return 1;
1115 }
1116 op1_max = OP1_MAX_RANGE();
1117 if (op1_max == ZEND_LONG_MAX) {
1118 return 1;
1119 }
1120 }
1121 return 0;
1122 case ZEND_PRE_DEC:
1123 case ZEND_POST_DEC:
1124 res = ssa_op->op1_def;
1125 if (res < 0
1126 || !ssa->var_info[res].has_range
1127 || ssa->var_info[res].range.underflow) {
1128 if (!OP1_HAS_RANGE()) {
1129 return 1;
1130 }
1131 op1_min = OP1_MIN_RANGE();
1132 if (op1_min == ZEND_LONG_MIN) {
1133 return 1;
1134 }
1135 }
1136 return 0;
1137 case ZEND_ADD:
1138 res = ssa_op->result_def;
1139 if (res < 0
1140 || !ssa->var_info[res].has_range
1141 || ssa->var_info[res].range.underflow) {
1142 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1143 return 1;
1144 }
1145 op1_min = OP1_MIN_RANGE();
1146 op2_min = OP2_MIN_RANGE();
1147 if (zend_add_will_overflow(op1_min, op2_min)) {
1148 return 1;
1149 }
1150 }
1151 if (res < 0
1152 || !ssa->var_info[res].has_range
1153 || ssa->var_info[res].range.overflow) {
1154 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1155 return 1;
1156 }
1157 op1_max = OP1_MAX_RANGE();
1158 op2_max = OP2_MAX_RANGE();
1159 if (zend_add_will_overflow(op1_max, op2_max)) {
1160 return 1;
1161 }
1162 }
1163 return 0;
1164 case ZEND_SUB:
1165 res = ssa_op->result_def;
1166 if (res < 0
1167 || !ssa->var_info[res].has_range
1168 || ssa->var_info[res].range.underflow) {
1169 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1170 return 1;
1171 }
1172 op1_min = OP1_MIN_RANGE();
1173 op2_max = OP2_MAX_RANGE();
1174 if (zend_sub_will_overflow(op1_min, op2_max)) {
1175 return 1;
1176 }
1177 }
1178 if (res < 0
1179 || !ssa->var_info[res].has_range
1180 || ssa->var_info[res].range.overflow) {
1181 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1182 return 1;
1183 }
1184 op1_max = OP1_MAX_RANGE();
1185 op2_min = OP2_MIN_RANGE();
1186 if (zend_sub_will_overflow(op1_max, op2_min)) {
1187 return 1;
1188 }
1189 }
1190 return 0;
1191 case ZEND_MUL:
1192 res = ssa_op->result_def;
1193 return (res < 0 ||
1194 !ssa->var_info[res].has_range ||
1195 ssa->var_info[res].range.underflow ||
1196 ssa->var_info[res].range.overflow);
1197 case ZEND_ASSIGN_OP:
1198 if (opline->extended_value == ZEND_ADD) {
1199 res = ssa_op->op1_def;
1200 if (res < 0
1201 || !ssa->var_info[res].has_range
1202 || ssa->var_info[res].range.underflow) {
1203 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1204 return 1;
1205 }
1206 op1_min = OP1_MIN_RANGE();
1207 op2_min = OP2_MIN_RANGE();
1208 if (zend_add_will_overflow(op1_min, op2_min)) {
1209 return 1;
1210 }
1211 }
1212 if (res < 0
1213 || !ssa->var_info[res].has_range
1214 || ssa->var_info[res].range.overflow) {
1215 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1216 return 1;
1217 }
1218 op1_max = OP1_MAX_RANGE();
1219 op2_max = OP2_MAX_RANGE();
1220 if (zend_add_will_overflow(op1_max, op2_max)) {
1221 return 1;
1222 }
1223 }
1224 return 0;
1225 } else if (opline->extended_value == ZEND_SUB) {
1226 res = ssa_op->op1_def;
1227 if (res < 0
1228 || !ssa->var_info[res].has_range
1229 || ssa->var_info[res].range.underflow) {
1230 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1231 return 1;
1232 }
1233 op1_min = OP1_MIN_RANGE();
1234 op2_max = OP2_MAX_RANGE();
1235 if (zend_sub_will_overflow(op1_min, op2_max)) {
1236 return 1;
1237 }
1238 }
1239 if (res < 0
1240 || !ssa->var_info[res].has_range
1241 || ssa->var_info[res].range.overflow) {
1242 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1243 return 1;
1244 }
1245 op1_max = OP1_MAX_RANGE();
1246 op2_min = OP2_MIN_RANGE();
1247 if (zend_sub_will_overflow(op1_max, op2_min)) {
1248 return 1;
1249 }
1250 }
1251 return 0;
1252 } else if (opline->extended_value == ZEND_MUL) {
1253 res = ssa_op->op1_def;
1254 return (res < 0 ||
1255 !ssa->var_info[res].has_range ||
1256 ssa->var_info[res].range.underflow ||
1257 ssa->var_info[res].range.overflow);
1258 }
1259 ZEND_FALLTHROUGH;
1260 default:
1261 return 1;
1262 }
1263 }
1264
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1265 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1266 {
1267 uint32_t flags;
1268
1269 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1270
1271 if (zend_build_cfg(&CG(arena), op_array, flags, cfg) != SUCCESS) {
1272 return FAILURE;
1273 }
1274
1275 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1276 * generated code, some of our analysis is recursive and will stack overflow with many
1277 * blocks. */
1278 if (cfg->blocks_count > 100000) {
1279 return FAILURE;
1280 }
1281
1282 if (zend_cfg_build_predecessors(&CG(arena), cfg) != SUCCESS) {
1283 return FAILURE;
1284 }
1285
1286 /* Compute Dominators Tree */
1287 if (zend_cfg_compute_dominators_tree(op_array, cfg) != SUCCESS) {
1288 return FAILURE;
1289 }
1290
1291 /* Identify reducible and irreducible loops */
1292 if (zend_cfg_identify_loops(op_array, cfg) != SUCCESS) {
1293 return FAILURE;
1294 }
1295
1296 return SUCCESS;
1297 }
1298
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1299 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1300 {
1301 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1302 return FAILURE;
1303 }
1304
1305 #if 0
1306 /* TODO: debugger and profiler supports? */
1307 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1308 return FAILURE;
1309 }
1310 #endif
1311
1312 /* TODO: move this to zend_cfg.c ? */
1313 if (!op_array->function_name) {
1314 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1315 }
1316
1317 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1318 && ssa->cfg.blocks
1319 && op_array->last_try_catch == 0
1320 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1321 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1322 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1323 return FAILURE;
1324 }
1325
1326 if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa) != SUCCESS) {
1327 return FAILURE;
1328 }
1329
1330 if (zend_ssa_find_false_dependencies(op_array, ssa) != SUCCESS) {
1331 return FAILURE;
1332 }
1333
1334 if (zend_ssa_find_sccs(op_array, ssa) != SUCCESS){
1335 return FAILURE;
1336 }
1337 }
1338
1339 return SUCCESS;
1340 }
1341
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1342 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1343 {
1344 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1345 && ssa->cfg.blocks
1346 && op_array->last_try_catch == 0
1347 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1348 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1349 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1350 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1351 return FAILURE;
1352 }
1353 }
1354
1355 return SUCCESS;
1356 }
1357
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)1358 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
1359 {
1360 zend_lifetime_interval *ival = intervals[var];
1361
1362 if (!ival) {
1363 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1364 if (!ival) {
1365 return FAILURE;
1366 }
1367 ival->ssa_var = var;
1368 ival->reg = ZREG_NONE;
1369 ival->flags = 0;
1370 ival->range.start = from;
1371 ival->range.end = to;
1372 ival->range.next = NULL;
1373 ival->hint = NULL;
1374 ival->used_as_hint = NULL;
1375 intervals[var] = ival;
1376 } else if (ival->range.start > to + 1) {
1377 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1378
1379 if (!range) {
1380 return FAILURE;
1381 }
1382 range->start = ival->range.start;
1383 range->end = ival->range.end;
1384 range->next = ival->range.next;
1385 ival->range.start = from;
1386 ival->range.end = to;
1387 ival->range.next = range;
1388 } else if (ival->range.start == to + 1) {
1389 ival->range.start = from;
1390 } else {
1391 zend_life_range *range = &ival->range;
1392 zend_life_range *last = NULL;
1393
1394 do {
1395 if (range->start > to + 1) {
1396 break;
1397 } else if (range->end + 1 >= from) {
1398 if (range->start > from) {
1399 range->start = from;
1400 }
1401 last = range;
1402 range = range->next;
1403 while (range) {
1404 if (range->start > to + 1) {
1405 break;
1406 }
1407 last->end = range->end;
1408 range = range->next;
1409 last->next = range;
1410 }
1411 if (to > last->end) {
1412 last->end = to;
1413 }
1414 return SUCCESS;
1415 }
1416 last = range;
1417 range = range->next;
1418 } while (range);
1419
1420 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1421 if (!range) {
1422 return FAILURE;
1423 }
1424 range->start = from;
1425 range->end = to;
1426 range->next = last->next;
1427 last->next = range;
1428 }
1429
1430 return SUCCESS;
1431 }
1432
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)1433 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
1434 {
1435 if (block_start != from && intervals[var]) {
1436 zend_life_range *range = &intervals[var]->range;
1437
1438 do {
1439 if (from >= range->start && from <= range->end) {
1440 if (range->start == block_start) {
1441 range->start = from;
1442 } else {
1443 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1444 if (!r) {
1445 return FAILURE;
1446 }
1447 r->start = from;
1448 r->end = range->end;
1449 r->next = range->next;
1450 range->end = block_start - 1;
1451 range->next = r;
1452 }
1453 return SUCCESS;
1454 }
1455 range = range->next;
1456 } while (range);
1457 }
1458
1459 // dead store
1460 return zend_jit_add_range(intervals, var, from, from);
1461 }
1462
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)1463 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
1464 {
1465 while (1) {
1466 if (*list == NULL) {
1467 *list = ival;
1468 ival->list_next = NULL;
1469 return;
1470 } else if (ival->range.start < (*list)->range.start) {
1471 ival->list_next = *list;
1472 *list = ival;
1473 return;
1474 }
1475 list = &(*list)->list_next;
1476 }
1477 }
1478
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1479 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
1480 {
1481 zend_lifetime_interval *ival;
1482 zend_life_range *range = ¤t->range;
1483 zend_life_range *prev = NULL;
1484
1485 if (*free) {
1486 ival = *free;
1487 *free = ival->list_next;
1488 } else {
1489 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1490
1491 if (!ival) {
1492 return FAILURE;
1493 }
1494 }
1495
1496 current->flags |= ZREG_STORE;
1497
1498 ival->ssa_var = current->ssa_var;
1499 ival->reg = ZREG_NONE;
1500 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
1501 ival->flags &= ~ZREG_STORE;
1502 ival->hint = NULL;
1503
1504 do {
1505 if (pos >= range->start && pos <= range->end) {
1506 break;
1507 }
1508 prev = range;
1509 range = range->next;
1510 } while(range);
1511
1512 ZEND_ASSERT(range != NULL);
1513
1514 ival->range.start = pos;
1515 ival->range.end = range->end;
1516 ival->range.next = range->next;
1517
1518 if (pos == range->start) {
1519 ZEND_ASSERT(prev != NULL);
1520 prev->next = NULL;
1521 } else {
1522 range->end = pos - 1;
1523 }
1524
1525 zend_jit_insert_interval(list, ival);
1526
1527 return SUCCESS;
1528 }
1529
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)1530 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
1531 {
1532 zend_lifetime_interval *list, *last;
1533 int i;
1534
1535 list = NULL;
1536 i = 0;
1537 while (i < count) {
1538 list = intervals[i];
1539 i++;
1540 if (list) {
1541 last = list;
1542 last->list_next = NULL;
1543 break;
1544 }
1545 }
1546
1547 while (i < count) {
1548 zend_lifetime_interval *ival = intervals[i];
1549
1550 i++;
1551 if (ival) {
1552 if ((ival->range.start > last->range.start) ||
1553 (ival->range.start == last->range.start &&
1554 ((!ival->hint && last->hint && last->hint != ival) ||
1555 ival->range.end > last->range.end))) {
1556 last->list_next = ival;
1557 last = ival;
1558 ival->list_next = NULL;
1559 } else {
1560 zend_lifetime_interval **p = &list;
1561
1562 while (1) {
1563 if (*p == NULL) {
1564 *p = last = ival;
1565 ival->list_next = NULL;
1566 break;
1567 } else if ((ival->range.start < (*p)->range.start) ||
1568 (ival->range.start == (*p)->range.start &&
1569 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1570 ival->range.end < (*p)->range.end))) {
1571 ival->list_next = *p;
1572 *p = ival;
1573 break;
1574 }
1575 p = &(*p)->list_next;
1576 }
1577 }
1578 }
1579 }
1580
1581 return list;
1582 }
1583
zend_jit_print_regset(zend_regset regset)1584 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1585 {
1586 zend_reg reg;
1587 int first = 1;
1588
1589 ZEND_REGSET_FOREACH(regset, reg) {
1590 if (first) {
1591 first = 0;
1592 fprintf(stderr, "%s", zend_reg_name[reg]);
1593 } else {
1594 fprintf(stderr, ", %s", zend_reg_name[reg]);
1595 }
1596 } ZEND_REGSET_FOREACH_END();
1597 }
1598
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1599 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1600 {
1601 zend_basic_block *b = ssa->cfg.blocks + n;
1602
1603 tail_call:
1604 *block_order = n;
1605 block_order++;
1606
1607 n = b->children;
1608 while (n >= 0) {
1609 b = ssa->cfg.blocks + n;
1610 if (b->next_child < 0) {
1611 goto tail_call;
1612 }
1613 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1614 n = b->next_child;
1615 }
1616
1617 return block_order;
1618 }
1619
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1620 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1621 {
1622 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1623
1624 return end - block_order;
1625 }
1626
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1627 static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1628 {
1629 while (b->loop_header >= 0) {
1630 if (b->loop_header == header) {
1631 return 1;
1632 }
1633 b = ssa->cfg.blocks + b->loop_header;
1634 }
1635 return 0;
1636 }
1637
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1638 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1639 {
1640 zend_basic_block *b = ssa->cfg.blocks + n;
1641 uint32_t i;
1642
1643 tail_call:
1644 if (b->len) {
1645 for (i = b->start; i < b->start + b->len; i++) {
1646 zend_bitset_incl(loop_body, i);
1647 }
1648 }
1649
1650 n = b->children;
1651 while (n >= 0) {
1652 b = ssa->cfg.blocks + n;
1653 if (zend_jit_in_loop(ssa, header, b)) {
1654 if (b->next_child < 0) {
1655 goto tail_call;
1656 }
1657 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1658 }
1659 n = b->next_child;
1660 }
1661 }
1662
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1663 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1664 {
1665 if (intervals[dst]->range.start < intervals[src]->range.start) {
1666 int tmp = src;
1667 src = dst;
1668 dst = tmp;
1669 }
1670 while (1) {
1671 if (intervals[dst]->hint) {
1672 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1673 int tmp = src;
1674 src = intervals[dst]->hint->ssa_var;
1675 dst = tmp;
1676 } else {
1677 dst = intervals[dst]->hint->ssa_var;
1678 }
1679 } else {
1680 if (dst != src) {
1681 intervals[dst]->hint = intervals[src];
1682 }
1683 return;
1684 }
1685 }
1686 }
1687
1688 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1689 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1690 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1691 {
1692 int set_size, i, j, k, l;
1693 uint32_t n;
1694 zend_bitset live, live_in, pi_vars, loop_body;
1695 int *block_order;
1696 zend_ssa_phi *phi;
1697 zend_lifetime_interval **intervals;
1698 size_t mem_size;
1699 ALLOCA_FLAG(use_heap);
1700
1701 set_size = zend_bitset_len(ssa->vars_count);
1702 mem_size =
1703 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1704 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1705 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1706 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1707 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1708 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1709 intervals = do_alloca(mem_size, use_heap);
1710 if (!intervals) {
1711 *list = NULL;
1712 return FAILURE;
1713 }
1714
1715 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1716 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1717 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1718 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1719 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1720
1721 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1722 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1723
1724 /* TODO: Provide a linear block order where all dominators of a block
1725 * are before this block, and where all blocks belonging to the same loop
1726 * are contiguous ???
1727 */
1728 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1729 zend_basic_block *b;
1730
1731 i = block_order[l];
1732 b = ssa->cfg.blocks + i;
1733
1734 /* live = UNION of successor.liveIn for each successor of b */
1735 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1736 zend_bitset_clear(live, set_size);
1737 for (j = 0; j < b->successors_count; j++) {
1738 int succ = b->successors[j];
1739
1740 zend_bitset_union(live, live_in + set_size * succ, set_size);
1741 zend_bitset_clear(pi_vars, set_size);
1742 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1743 if (ssa->vars[phi->ssa_var].no_val) {
1744 /* skip */
1745 } else if (phi->pi >= 0) {
1746 if (phi->pi == i && phi->sources[0] >= 0) {
1747 if (zend_bitset_in(candidates, phi->sources[0])) {
1748 zend_bitset_incl(live, phi->sources[0]);
1749 }
1750 zend_bitset_incl(pi_vars, phi->var);
1751 }
1752 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1753 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1754 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1755 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1756 zend_bitset_incl(live, phi->sources[k]);
1757 }
1758 break;
1759 }
1760 }
1761 }
1762 }
1763 }
1764
1765 /* addRange(var, b.from, b.to) for each var in live */
1766 ZEND_BITSET_FOREACH(live, set_size, j) {
1767 if (zend_bitset_in(candidates, j)) {
1768 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1769 goto failure;
1770 }
1771 }
1772 } ZEND_BITSET_FOREACH_END();
1773
1774 /* for each operation op of b in reverse order */
1775 for (n = b->start + b->len; n > b->start;) {
1776 zend_ssa_op *op;
1777 const zend_op *opline;
1778 uint32_t num;
1779
1780 n--;
1781 op = ssa->ops + n;
1782 opline = op_array->opcodes + n;
1783
1784 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1785 num = n - 1;
1786 } else {
1787 num = n;
1788 }
1789
1790 /* for each output operand opd of op do */
1791 /* setFrom(opd, op) */
1792 /* live.remove(opd) */
1793 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1794 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1795 goto failure;
1796 }
1797 zend_bitset_excl(live, op->op1_def);
1798 }
1799 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1800 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1801 goto failure;
1802 }
1803 zend_bitset_excl(live, op->op2_def);
1804 }
1805 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1806 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1807 goto failure;
1808 }
1809 zend_bitset_excl(live, op->result_def);
1810 }
1811
1812 /* for each input operand opd of op do */
1813 /* live.add(opd) */
1814 /* addRange(opd, b.from, op) */
1815 if (op->op1_use >= 0
1816 && zend_bitset_in(candidates, op->op1_use)
1817 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1818 zend_bitset_incl(live, op->op1_use);
1819 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1820 goto failure;
1821 }
1822 }
1823 if (op->op2_use >= 0
1824 && zend_bitset_in(candidates, op->op2_use)
1825 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1826 zend_bitset_incl(live, op->op2_use);
1827 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1828 goto failure;
1829 }
1830 }
1831 if (op->result_use >= 0
1832 && zend_bitset_in(candidates, op->result_use)
1833 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1834 zend_bitset_incl(live, op->result_use);
1835 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1836 goto failure;
1837 }
1838 }
1839 }
1840
1841 /* live.remove(phi.output) for each phi of b */
1842 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1843 zend_bitset_excl(live, phi->ssa_var);
1844 }
1845
1846 /* b.liveIn = live */
1847 zend_bitset_copy(live_in + set_size * i, live, set_size);
1848 }
1849
1850 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1851 zend_basic_block *b = ssa->cfg.blocks + i;
1852
1853 /* if b is loop header */
1854 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1855 live = live_in + set_size * i;
1856
1857 if (!zend_bitset_empty(live, set_size)) {
1858 uint32_t set_size2 = zend_bitset_len(op_array->last);
1859
1860 zend_bitset_clear(loop_body, set_size2);
1861 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1862 while (!zend_bitset_empty(loop_body, set_size2)) {
1863 uint32_t from = zend_bitset_first(loop_body, set_size2);
1864 uint32_t to = from;
1865
1866 do {
1867 zend_bitset_excl(loop_body, to);
1868 to++;
1869 } while (zend_bitset_in(loop_body, to));
1870 to--;
1871
1872 ZEND_BITSET_FOREACH(live, set_size, j) {
1873 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1874 goto failure;
1875 }
1876 } ZEND_BITSET_FOREACH_END();
1877 }
1878 }
1879 }
1880
1881 }
1882
1883 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1884 /* Register hinting (a cheap way for register coalescing) */
1885 for (i = 0; i < ssa->vars_count; i++) {
1886 if (intervals[i]) {
1887 int src;
1888
1889 if (ssa->vars[i].definition_phi) {
1890 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1891
1892 if (phi->pi >= 0) {
1893 src = phi->sources[0];
1894 if (intervals[src]) {
1895 zend_jit_add_hint(intervals, i, src);
1896 }
1897 } else {
1898 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1899 src = phi->sources[k];
1900 if (src >= 0) {
1901 if (ssa->vars[src].definition_phi
1902 && ssa->vars[src].definition_phi->pi >= 0
1903 && phi->block == ssa->vars[src].definition_phi->block) {
1904 /* Skip zero-length interval for Pi variable */
1905 src = ssa->vars[src].definition_phi->sources[0];
1906 }
1907 if (intervals[src]) {
1908 zend_jit_add_hint(intervals, i, src);
1909 }
1910 }
1911 }
1912 }
1913 }
1914 }
1915 }
1916 for (i = 0; i < ssa->vars_count; i++) {
1917 if (intervals[i] && !intervals[i]->hint) {
1918
1919 if (ssa->vars[i].definition >= 0) {
1920 uint32_t line = ssa->vars[i].definition;
1921 const zend_op *opline = op_array->opcodes + line;
1922
1923 switch (opline->opcode) {
1924 case ZEND_QM_ASSIGN:
1925 case ZEND_POST_INC:
1926 case ZEND_POST_DEC:
1927 if (ssa->ops[line].op1_use >= 0 &&
1928 intervals[ssa->ops[line].op1_use] &&
1929 (i == ssa->ops[line].op1_def ||
1930 (i == ssa->ops[line].result_def &&
1931 (ssa->ops[line].op1_def < 0 ||
1932 !intervals[ssa->ops[line].op1_def])))) {
1933 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1934 }
1935 break;
1936 case ZEND_SEND_VAR:
1937 case ZEND_PRE_INC:
1938 case ZEND_PRE_DEC:
1939 if (i == ssa->ops[line].op1_def &&
1940 ssa->ops[line].op1_use >= 0 &&
1941 intervals[ssa->ops[line].op1_use]) {
1942 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1943 }
1944 break;
1945 case ZEND_ASSIGN:
1946 if (ssa->ops[line].op2_use >= 0 &&
1947 intervals[ssa->ops[line].op2_use] &&
1948 (i == ssa->ops[line].op2_def ||
1949 (i == ssa->ops[line].op1_def &&
1950 (ssa->ops[line].op2_def < 0 ||
1951 !intervals[ssa->ops[line].op2_def])) ||
1952 (i == ssa->ops[line].result_def &&
1953 (ssa->ops[line].op2_def < 0 ||
1954 !intervals[ssa->ops[line].op2_def]) &&
1955 (ssa->ops[line].op1_def < 0 ||
1956 !intervals[ssa->ops[line].op1_def])))) {
1957 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1958 }
1959 break;
1960 case ZEND_SUB:
1961 case ZEND_ADD:
1962 case ZEND_MUL:
1963 case ZEND_BW_OR:
1964 case ZEND_BW_AND:
1965 case ZEND_BW_XOR:
1966 if (i == ssa->ops[line].result_def) {
1967 if (ssa->ops[line].op1_use >= 0 &&
1968 intervals[ssa->ops[line].op1_use] &&
1969 ssa->ops[line].op1_use_chain < 0 &&
1970 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain) {
1971 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1972 } else if (opline->opcode != ZEND_SUB &&
1973 ssa->ops[line].op2_use >= 0 &&
1974 intervals[ssa->ops[line].op2_use] &&
1975 ssa->ops[line].op2_use_chain < 0 &&
1976 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain) {
1977 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1978 }
1979 }
1980 break;
1981 }
1982 }
1983 }
1984 }
1985 }
1986
1987 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
1988
1989 if (*list) {
1990 zend_lifetime_interval *ival = *list;
1991 while (ival) {
1992 if (ival->hint) {
1993 ival->hint->used_as_hint = ival;
1994 }
1995 ival = ival->list_next;
1996 }
1997 }
1998
1999 free_alloca(intervals, use_heap);
2000 return SUCCESS;
2001
2002 failure:
2003 *list = NULL;
2004 free_alloca(intervals, use_heap);
2005 return FAILURE;
2006 }
2007
zend_interval_end(zend_lifetime_interval * ival)2008 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
2009 {
2010 zend_life_range *range = &ival->range;
2011
2012 while (range->next) {
2013 range = range->next;
2014 }
2015 return range->end;
2016 }
2017
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)2018 static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
2019 {
2020 zend_life_range *range = &ival->range;
2021
2022 do {
2023 if (position >= range->start && position <= range->end) {
2024 return 1;
2025 }
2026 range = range->next;
2027 } while (range);
2028
2029 return 0;
2030 }
2031
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)2032 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
2033 {
2034 zend_life_range *r1 = &ival1->range;
2035 zend_life_range *r2 = &ival2->range;
2036
2037 do {
2038 if (r1->start <= r2->end) {
2039 if (r2->start <= r1->end) {
2040 return MAX(r1->start, r2->start);
2041 } else {
2042 r2 = r2->next;
2043 }
2044 } else {
2045 r1 = r1->next;
2046 }
2047 } while (r1 && r2);
2048
2049 return 0xffffffff;
2050 }
2051
2052 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2053 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)2054 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
2055 {
2056 zend_lifetime_interval *it;
2057 uint32_t freeUntilPos[ZREG_NUM];
2058 uint32_t pos, pos2;
2059 zend_reg i, reg, reg2;
2060 zend_reg hint = ZREG_NONE;
2061 zend_regset low_priority_regs;
2062 zend_life_range *range;
2063
2064 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
2065 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
2066 } else {
2067 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
2068 }
2069
2070 /* TODO: Allow usage of preserved registers ???
2071 * Their values have to be stored in prologue and restored in epilogue
2072 */
2073 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
2074
2075 /* Set freeUntilPos of all physical registers to maxInt */
2076 for (i = 0; i < ZREG_NUM; i++) {
2077 freeUntilPos[i] = 0xffffffff;
2078 }
2079
2080 /* for each interval it in active do */
2081 /* freeUntilPos[it.reg] = 0 */
2082 it = active;
2083 if (ssa->vars[current->ssa_var].definition == current->range.start) {
2084 while (it) {
2085 if (current->range.start != zend_interval_end(it)) {
2086 freeUntilPos[it->reg] = 0;
2087 } else if (zend_jit_may_reuse_reg(
2088 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
2089 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
2090 if (!ZEND_REGSET_IN(*hints, it->reg) &&
2091 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2092 (!current->used_as_hint ||
2093 !ZEND_REGSET_IN(ZEND_REGSET_LOW_PRIORITY, it->reg))) {
2094 hint = it->reg;
2095 }
2096 } else {
2097 freeUntilPos[it->reg] = 0;
2098 }
2099 it = it->list_next;
2100 }
2101 } else {
2102 while (it) {
2103 freeUntilPos[it->reg] = 0;
2104 it = it->list_next;
2105 }
2106 }
2107 if (current->hint) {
2108 hint = current->hint->reg;
2109 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
2110 ZEND_REGSET_EXCL(*hints, hint);
2111 }
2112 }
2113
2114 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
2115 return 0;
2116 }
2117
2118 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2119 Michael Franz, CGO'10 (2010), Figure 6. */
2120 if (current->flags & ZREG_SPLIT) {
2121 /* for each interval it in inactive intersecting with current do */
2122 /* freeUntilPos[it.reg] = next intersection of it with current */
2123 it = inactive;
2124 while (it) {
2125 uint32_t next = zend_interval_intersection(current, it);
2126
2127 //ZEND_ASSERT(next != 0xffffffff && !current->split);
2128 if (next < freeUntilPos[it->reg]) {
2129 freeUntilPos[it->reg] = next;
2130 }
2131 it = it->list_next;
2132 }
2133 }
2134
2135 /* Handle Scratch Registers */
2136 /* TODO: Optimize ??? */
2137 range = ¤t->range;
2138 do {
2139 uint32_t line = range->start;
2140 uint32_t last_use_line = (uint32_t)-1;
2141 zend_regset regset;
2142 zend_reg reg;
2143
2144 if ((current->flags & ZREG_LAST_USE) && !range->next) {
2145 last_use_line = range->end;
2146 }
2147 if (ssa->ops[line].op1_def == current->ssa_var ||
2148 ssa->ops[line].op2_def == current->ssa_var ||
2149 ssa->ops[line].result_def == current->ssa_var) {
2150 regset = zend_jit_get_def_scratch_regset(
2151 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2152 ssa->ops + line,
2153 op_array, ssa, current->ssa_var, line == last_use_line);
2154 ZEND_REGSET_FOREACH(regset, reg) {
2155 if (line < freeUntilPos[reg]) {
2156 freeUntilPos[reg] = line;
2157 }
2158 } ZEND_REGSET_FOREACH_END();
2159 line++;
2160 }
2161 while (line <= range->end) {
2162 regset = zend_jit_get_scratch_regset(
2163 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2164 ssa->ops + line,
2165 op_array, ssa, current->ssa_var, line == last_use_line);
2166 ZEND_REGSET_FOREACH(regset, reg) {
2167 if (line < freeUntilPos[reg]) {
2168 freeUntilPos[reg] = line;
2169 }
2170 } ZEND_REGSET_FOREACH_END();
2171 line++;
2172 }
2173 range = range->next;
2174 } while (range);
2175
2176 #if 0
2177 /* Coalescing */
2178 if (ssa->vars[current->ssa_var].definition == current->start) {
2179 zend_op *opline = op_array->opcodes + current->start;
2180 int hint = -1;
2181
2182 switch (opline->opcode) {
2183 case ZEND_ASSIGN:
2184 hint = ssa->ops[current->start].op2_use;
2185 case ZEND_QM_ASSIGN:
2186 hint = ssa->ops[current->start].op1_use;
2187 break;
2188 case ZEND_ADD:
2189 case ZEND_SUB:
2190 case ZEND_MUL:
2191 hint = ssa->ops[current->start].op1_use;
2192 break;
2193 case ZEND_ASSIGN_OP:
2194 if (opline->extended_value == ZEND_ADD
2195 || opline->extended_value == ZEND_SUB
2196 || opline->extended_value == ZEND_MUL) {
2197 hint = ssa->ops[current->start].op1_use;
2198 }
2199 break;
2200 }
2201 if (hint >= 0) {
2202 }
2203 }
2204 #endif
2205
2206 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
2207 current->reg = hint;
2208 if (current->used_as_hint) {
2209 ZEND_REGSET_INCL(*hints, hint);
2210 }
2211 return 1;
2212 }
2213
2214 if (ZEND_REGSET_IS_EMPTY(available)) {
2215 return 0;
2216 }
2217
2218 pos = 0; reg = ZREG_NONE;
2219 pos2 = 0; reg2 = ZREG_NONE;
2220 low_priority_regs = *hints;
2221 if (current->used_as_hint) {
2222 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2223 low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY);
2224 }
2225
2226 ZEND_REGSET_FOREACH(available, i) {
2227 if (ZEND_REGSET_IN(low_priority_regs, i)) {
2228 if (freeUntilPos[i] > pos2) {
2229 reg2 = i;
2230 pos2 = freeUntilPos[i];
2231 }
2232 } else if (freeUntilPos[i] > pos) {
2233 reg = i;
2234 pos = freeUntilPos[i];
2235 }
2236 } ZEND_REGSET_FOREACH_END();
2237
2238 if (reg == ZREG_NONE) {
2239 if (reg2 != ZREG_NONE) {
2240 reg = reg2;
2241 pos = pos2;
2242 reg2 = ZREG_NONE;
2243 }
2244 }
2245
2246 if (reg == ZREG_NONE) {
2247 /* no register available without spilling */
2248 return 0;
2249 } else if (zend_interval_end(current) < pos) {
2250 /* register available for the whole interval */
2251 current->reg = reg;
2252 if (current->used_as_hint) {
2253 ZEND_REGSET_INCL(*hints, reg);
2254 }
2255 return 1;
2256 #if 0
2257 // TODO: allow low priority register usage
2258 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
2259 /* register available for the whole interval */
2260 current->reg = reg2;
2261 if (current->used_as_hint) {
2262 ZEND_REGSET_INCL(*hints, reg2);
2263 }
2264 return 1;
2265 #endif
2266 } else {
2267 /* TODO: enable interval splitting ??? */
2268 /* register available for the first part of the interval */
2269 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
2270 return 0;
2271 }
2272 current->reg = reg;
2273 if (current->used_as_hint) {
2274 ZEND_REGSET_INCL(*hints, reg);
2275 }
2276 return 1;
2277 }
2278 }
2279
2280 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2281 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
2282 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2283 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)2284 static int zend_jit_allocate_blocked_reg(void)
2285 {
2286 /* TODO: ??? */
2287 return 0;
2288 }
2289
2290 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2291 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)2292 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
2293 {
2294 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
2295 zend_lifetime_interval *current, **p, *q;
2296 uint32_t position;
2297 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
2298 zend_regset hints = ZEND_REGSET_EMPTY;
2299
2300 unhandled = list;
2301 /* active = inactive = handled = free = {} */
2302 active = inactive = handled = free = NULL;
2303 while (unhandled != NULL) {
2304 current = unhandled;
2305 unhandled = unhandled->list_next;
2306 position = current->range.start;
2307
2308 p = &active;
2309 while (*p) {
2310 uint32_t end = zend_interval_end(*p);
2311
2312 q = *p;
2313 if (end < position) {
2314 /* move ival from active to handled */
2315 ZEND_REGSET_INCL(available, q->reg);
2316 *p = q->list_next;
2317 q->list_next = handled;
2318 handled = q;
2319 } else if (!zend_interval_covers(q, position)) {
2320 /* move ival from active to inactive */
2321 ZEND_REGSET_INCL(available, q->reg);
2322 *p = q->list_next;
2323 q->list_next = inactive;
2324 inactive = q;
2325 } else {
2326 p = &q->list_next;
2327 }
2328 }
2329
2330 p = &inactive;
2331 while (*p) {
2332 uint32_t end = zend_interval_end(*p);
2333
2334 q = *p;
2335 if (end < position) {
2336 /* move ival from inactive to handled */
2337 *p = q->list_next;
2338 q->list_next = handled;
2339 handled = q;
2340 } else if (zend_interval_covers(q, position)) {
2341 /* move ival from inactive to active */
2342 ZEND_REGSET_EXCL(available, q->reg);
2343 *p = q->list_next;
2344 q->list_next = active;
2345 active = q;
2346 } else {
2347 p = &q->list_next;
2348 }
2349 }
2350
2351 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
2352 zend_jit_allocate_blocked_reg()) {
2353 ZEND_REGSET_EXCL(available, current->reg);
2354 current->list_next = active;
2355 active = current;
2356 } else {
2357 current->list_next = free;
2358 free = current;
2359 }
2360 }
2361
2362 /* move active to handled */
2363 while (active) {
2364 current = active;
2365 active = active->list_next;
2366 current->list_next = handled;
2367 handled = current;
2368 }
2369
2370 /* move inactive to handled */
2371 while (inactive) {
2372 current = inactive;
2373 inactive = inactive->list_next;
2374 current->list_next = handled;
2375 handled = current;
2376 }
2377
2378 return handled;
2379 }
2380
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)2381 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
2382 {
2383 zend_life_range *range;
2384 int var_num = ssa->vars[ival->ssa_var].var;
2385
2386 fprintf(stderr, "#%d.", ival->ssa_var);
2387 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2388 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
2389 range = ival->range.next;
2390 while (range) {
2391 fprintf(stderr, ", %u-%u", range->start, range->end);
2392 range = range->next;
2393 }
2394 if (ival->reg != ZREG_NONE) {
2395 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
2396 }
2397 if (ival->flags & ZREG_LAST_USE) {
2398 fprintf(stderr, " last_use");
2399 }
2400 if (ival->flags & ZREG_LOAD) {
2401 fprintf(stderr, " load");
2402 }
2403 if (ival->flags & ZREG_STORE) {
2404 fprintf(stderr, " store");
2405 }
2406 if (ival->hint) {
2407 fprintf(stderr, " hint");
2408 if (ival->hint->ssa_var >= 0) {
2409 var_num = ssa->vars[ival->hint->ssa_var].var;
2410 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
2411 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2412 }
2413 if (ival->hint->reg != ZREG_NONE) {
2414 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
2415 }
2416 }
2417 fprintf(stderr, "\n");
2418 }
2419
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)2420 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
2421 {
2422 void *checkpoint;
2423 int set_size, candidates_count, i;
2424 zend_bitset candidates = NULL;
2425 zend_lifetime_interval *list, *ival;
2426 zend_lifetime_interval **intervals;
2427 ALLOCA_FLAG(use_heap);
2428
2429 if (!ssa->var_info) {
2430 return NULL;
2431 }
2432
2433 /* Identify SSA variables suitable for register allocation */
2434 set_size = zend_bitset_len(ssa->vars_count);
2435 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
2436 if (!candidates) {
2437 return NULL;
2438 }
2439 candidates_count = 0;
2440 zend_bitset_clear(candidates, set_size);
2441 for (i = 0; i < ssa->vars_count; i++) {
2442 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
2443 zend_bitset_incl(candidates, i);
2444 candidates_count++;
2445 }
2446 }
2447 if (!candidates_count) {
2448 free_alloca(candidates, use_heap);
2449 return NULL;
2450 }
2451
2452 checkpoint = zend_arena_checkpoint(CG(arena));
2453
2454 /* Find life-time intervals */
2455 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
2456 goto failure;
2457 }
2458
2459 if (list) {
2460 /* Set ZREG_LAST_USE flags */
2461 ival = list;
2462 while (ival) {
2463 zend_life_range *range = &ival->range;
2464
2465 while (range->next) {
2466 range = range->next;
2467 }
2468 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
2469 ival->flags |= ZREG_LAST_USE;
2470 }
2471 ival = ival->list_next;
2472 }
2473 }
2474
2475 if (list) {
2476 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2477 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2478 ival = list;
2479 while (ival) {
2480 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2481 ival = ival->list_next;
2482 }
2483 fprintf(stderr, "\n");
2484 }
2485
2486 /* Linear Scan Register Allocation */
2487 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
2488
2489 if (list) {
2490 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
2491 if (!intervals) {
2492 goto failure;
2493 }
2494
2495 ival = list;
2496 while (ival != NULL) {
2497 zend_lifetime_interval *next = ival->list_next;
2498
2499 ival->list_next = intervals[ival->ssa_var];
2500 intervals[ival->ssa_var] = ival;
2501 ival = next;
2502 }
2503
2504 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
2505 /* Naive SSA resolution */
2506 for (i = 0; i < ssa->vars_count; i++) {
2507 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
2508 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
2509 int k, src;
2510
2511 if (phi->pi >= 0) {
2512 if (!ssa->vars[i].phi_use_chain
2513 || ssa->vars[i].phi_use_chain->block != phi->block) {
2514 src = phi->sources[0];
2515 if (intervals[i]) {
2516 if (!intervals[src]) {
2517 intervals[i]->flags |= ZREG_LOAD;
2518 } else if (intervals[i]->reg != intervals[src]->reg) {
2519 intervals[i]->flags |= ZREG_LOAD;
2520 intervals[src]->flags |= ZREG_STORE;
2521 }
2522 } else if (intervals[src]) {
2523 intervals[src]->flags |= ZREG_STORE;
2524 }
2525 }
2526 } else {
2527 int need_move = 0;
2528
2529 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2530 src = phi->sources[k];
2531 if (src >= 0) {
2532 if (ssa->vars[src].definition_phi
2533 && ssa->vars[src].definition_phi->pi >= 0
2534 && phi->block == ssa->vars[src].definition_phi->block) {
2535 /* Skip zero-length interval for Pi variable */
2536 src = ssa->vars[src].definition_phi->sources[0];
2537 }
2538 if (intervals[i]) {
2539 if (!intervals[src]) {
2540 need_move = 1;
2541 } else if (intervals[i]->reg != intervals[src]->reg) {
2542 need_move = 1;
2543 }
2544 } else if (intervals[src]) {
2545 need_move = 1;
2546 }
2547 }
2548 }
2549 if (need_move) {
2550 if (intervals[i]) {
2551 intervals[i]->flags |= ZREG_LOAD;
2552 }
2553 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2554 src = phi->sources[k];
2555 if (src >= 0) {
2556 if (ssa->vars[src].definition_phi
2557 && ssa->vars[src].definition_phi->pi >= 0
2558 && phi->block == ssa->vars[src].definition_phi->block) {
2559 /* Skip zero-length interval for Pi variable */
2560 src = ssa->vars[src].definition_phi->sources[0];
2561 }
2562 if (intervals[src]) {
2563 intervals[src]->flags |= ZREG_STORE;
2564 }
2565 }
2566 }
2567 }
2568 }
2569 }
2570 }
2571 /* Remove useless register allocation */
2572 for (i = 0; i < ssa->vars_count; i++) {
2573 if (intervals[i] &&
2574 ((intervals[i]->flags & ZREG_LOAD) ||
2575 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2576 ssa->vars[i].use_chain < 0) {
2577 bool may_remove = 1;
2578 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2579
2580 while (phi) {
2581 if (intervals[phi->ssa_var] &&
2582 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2583 may_remove = 0;
2584 break;
2585 }
2586 phi = zend_ssa_next_use_phi(ssa, i, phi);
2587 }
2588 if (may_remove) {
2589 intervals[i] = NULL;
2590 }
2591 }
2592 }
2593 /* Remove intervals used once */
2594 for (i = 0; i < ssa->vars_count; i++) {
2595 if (intervals[i] &&
2596 (intervals[i]->flags & ZREG_LOAD) &&
2597 (intervals[i]->flags & ZREG_STORE) &&
2598 (ssa->vars[i].use_chain < 0 ||
2599 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2600 bool may_remove = 1;
2601 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2602
2603 while (phi) {
2604 if (intervals[phi->ssa_var] &&
2605 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2606 may_remove = 0;
2607 break;
2608 }
2609 phi = zend_ssa_next_use_phi(ssa, i, phi);
2610 }
2611 if (may_remove) {
2612 intervals[i] = NULL;
2613 }
2614 }
2615 }
2616 }
2617
2618 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2619 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2620 for (i = 0; i < ssa->vars_count; i++) {
2621 ival = intervals[i];
2622 while (ival) {
2623 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2624 ival = ival->list_next;
2625 }
2626 }
2627 fprintf(stderr, "\n");
2628 }
2629
2630 free_alloca(candidates, use_heap);
2631 return intervals;
2632 }
2633 }
2634
2635 failure:
2636 zend_arena_release(&CG(arena), checkpoint);
2637 free_alloca(candidates, use_heap);
2638 return NULL;
2639 }
2640
zend_jit_next_is_send_result(const zend_op * opline)2641 static bool zend_jit_next_is_send_result(const zend_op *opline)
2642 {
2643 if (opline->result_type == IS_TMP_VAR
2644 && (opline+1)->opcode == ZEND_SEND_VAL
2645 && (opline+1)->op1_type == IS_TMP_VAR
2646 && (opline+1)->op2_type != IS_CONST
2647 && (opline+1)->op1.var == opline->result.var) {
2648 return 1;
2649 }
2650 return 0;
2651 }
2652
zend_jit_supported_binary_op(zend_uchar op,uint32_t op1_info,uint32_t op2_info)2653 static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info)
2654 {
2655 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2656 return false;
2657 }
2658 switch (op) {
2659 case ZEND_POW:
2660 case ZEND_DIV:
2661 // TODO: check for division by zero ???
2662 return false;
2663 case ZEND_ADD:
2664 case ZEND_SUB:
2665 case ZEND_MUL:
2666 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2667 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2668 case ZEND_BW_OR:
2669 case ZEND_BW_AND:
2670 case ZEND_BW_XOR:
2671 case ZEND_SL:
2672 case ZEND_SR:
2673 case ZEND_MOD:
2674 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2675 case ZEND_CONCAT:
2676 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2677 EMPTY_SWITCH_DEFAULT_CASE()
2678 }
2679 }
2680
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2681 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2682 {
2683 int b, i, end;
2684 zend_op *opline;
2685 dasm_State* dasm_state = NULL;
2686 void *handler;
2687 int call_level = 0;
2688 void *checkpoint = NULL;
2689 zend_lifetime_interval **ra = NULL;
2690 bool is_terminated = 1; /* previous basic block is terminated by jump */
2691 bool recv_emitted = 0; /* emitted at least one RECV opcode */
2692 zend_uchar smart_branch_opcode;
2693 uint32_t target_label, target_label2;
2694 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2695 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2696 zend_class_entry *ce;
2697 bool ce_is_instanceof;
2698 bool on_this;
2699
2700 if (JIT_G(bisect_limit)) {
2701 jit_bisect_pos++;
2702 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2703 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2704 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2705 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2706 op_array->scope ? "::" : "",
2707 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2708 ZSTR_VAL(op_array->filename), op_array->line_start);
2709 }
2710 return FAILURE;
2711 }
2712 }
2713
2714 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2715 checkpoint = zend_arena_checkpoint(CG(arena));
2716 ra = zend_jit_allocate_registers(op_array, ssa);
2717 }
2718
2719 /* mark hidden branch targets */
2720 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2721 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2722 ssa->cfg.blocks[b].len > 1) {
2723
2724 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2725 if (opline->opcode == ZEND_DO_FCALL &&
2726 (opline-1)->opcode == ZEND_NEW) {
2727 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2728 }
2729 }
2730 }
2731
2732 dasm_init(&dasm_state, DASM_MAXSECTION);
2733 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2734 dasm_setup(&dasm_state, dasm_actions);
2735
2736 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2737
2738 zend_jit_align_func(&dasm_state);
2739 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2740 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2741 continue;
2742 }
2743 //#ifndef CONTEXT_THREADED_JIT
2744 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2745 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2746 /* pass */
2747 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2748 ssa->cfg.blocks[b].len == 1 &&
2749 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2750 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2751 /* don't generate code for BB with single opcode */
2752 continue;
2753 }
2754 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2755 if (!is_terminated) {
2756 zend_jit_jmp(&dasm_state, b);
2757 }
2758 }
2759 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2760 zend_jit_prologue(&dasm_state);
2761 } else
2762 //#endif
2763 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2764 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2765 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2766 if (opline->opcode == ZEND_RECV_INIT) {
2767 if (opline == op_array->opcodes ||
2768 (opline-1)->opcode != ZEND_RECV_INIT) {
2769 if (recv_emitted) {
2770 zend_jit_jmp(&dasm_state, b);
2771 }
2772 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2773 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2774 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2775 }
2776 zend_jit_prologue(&dasm_state);
2777 }
2778 recv_emitted = 1;
2779 } else if (opline->opcode == ZEND_RECV) {
2780 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2781 /* skip */
2782 continue;
2783 } else if (recv_emitted) {
2784 zend_jit_jmp(&dasm_state, b);
2785 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2786 zend_jit_prologue(&dasm_state);
2787 } else {
2788 zend_arg_info *arg_info;
2789
2790 if (opline->op1.num <= op_array->num_args) {
2791 arg_info = &op_array->arg_info[opline->op1.num - 1];
2792 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2793 arg_info = &op_array->arg_info[op_array->num_args];
2794 } else {
2795 /* skip */
2796 continue;
2797 }
2798 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2799 /* skip */
2800 continue;
2801 }
2802 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2803 zend_jit_prologue(&dasm_state);
2804 recv_emitted = 1;
2805 }
2806 } else {
2807 if (recv_emitted) {
2808 zend_jit_jmp(&dasm_state, b);
2809 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2810 ssa->cfg.blocks[b].len == 1 &&
2811 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2812 /* don't generate code for BB with single opcode */
2813 dasm_free(&dasm_state);
2814
2815 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2816 zend_arena_release(&CG(arena), checkpoint);
2817 }
2818 return SUCCESS;
2819 }
2820 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2821 zend_jit_prologue(&dasm_state);
2822 recv_emitted = 1;
2823 }
2824 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2825 ssa->cfg.blocks[b].len == 1 &&
2826 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2827 /* don't generate code for BB with single opcode */
2828 dasm_free(&dasm_state);
2829
2830 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2831 zend_arena_release(&CG(arena), checkpoint);
2832 }
2833 return SUCCESS;
2834 } else {
2835 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2836 zend_jit_prologue(&dasm_state);
2837 }
2838 }
2839
2840 is_terminated = 0;
2841
2842 zend_jit_label(&dasm_state, b);
2843 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2844 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2845 && ssa->cfg.blocks[b].start != 0
2846 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2847 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2848 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2849 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2850 zend_jit_reset_last_valid_opline();
2851 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2852 goto jit_failure;
2853 }
2854 } else {
2855 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2856 }
2857 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2858 zend_jit_reset_last_valid_opline();
2859 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2860 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2861 }
2862 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2863 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2864 goto jit_failure;
2865 }
2866 }
2867 if (!ssa->cfg.blocks[b].len) {
2868 continue;
2869 }
2870 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2871 zend_ssa_phi *phi = ssa->blocks[b].phis;
2872
2873 while (phi) {
2874 zend_lifetime_interval *ival = ra[phi->ssa_var];
2875
2876 if (ival) {
2877 if (ival->flags & ZREG_LOAD) {
2878 ZEND_ASSERT(ival->reg != ZREG_NONE);
2879
2880 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2881 goto jit_failure;
2882 }
2883 } else if (ival->flags & ZREG_STORE) {
2884 ZEND_ASSERT(ival->reg != ZREG_NONE);
2885
2886 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2887 goto jit_failure;
2888 }
2889 }
2890 }
2891 phi = phi->next;
2892 }
2893 }
2894 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2895 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2896 zend_ssa_op *ssa_op = &ssa->ops[i];
2897 opline = op_array->opcodes + i;
2898 switch (opline->opcode) {
2899 case ZEND_INIT_FCALL:
2900 case ZEND_INIT_FCALL_BY_NAME:
2901 case ZEND_INIT_NS_FCALL_BY_NAME:
2902 case ZEND_INIT_METHOD_CALL:
2903 case ZEND_INIT_DYNAMIC_CALL:
2904 case ZEND_INIT_STATIC_METHOD_CALL:
2905 case ZEND_INIT_USER_CALL:
2906 case ZEND_NEW:
2907 call_level++;
2908 }
2909
2910 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2911 switch (opline->opcode) {
2912 case ZEND_PRE_INC:
2913 case ZEND_PRE_DEC:
2914 case ZEND_POST_INC:
2915 case ZEND_POST_DEC:
2916 if (opline->op1_type != IS_CV) {
2917 break;
2918 }
2919 op1_info = OP1_INFO();
2920 if (!(op1_info & MAY_BE_LONG)) {
2921 break;
2922 }
2923 if (opline->result_type != IS_UNUSED) {
2924 res_use_info = -1;
2925
2926 if (opline->result_type == IS_CV) {
2927 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2928
2929 if (Z_MODE(res_use_addr) != IS_REG
2930 || Z_LOAD(res_use_addr)
2931 || Z_STORE(res_use_addr)) {
2932 res_use_info = RES_USE_INFO();
2933 }
2934 }
2935 res_info = RES_INFO();
2936 res_addr = RES_REG_ADDR();
2937 } else {
2938 res_use_info = -1;
2939 res_info = -1;
2940 res_addr = 0;
2941 }
2942 op1_def_info = OP1_DEF_INFO();
2943 if (!zend_jit_inc_dec(&dasm_state, opline,
2944 op1_info, OP1_REG_ADDR(),
2945 op1_def_info, OP1_DEF_REG_ADDR(),
2946 res_use_info, res_info,
2947 res_addr,
2948 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2949 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2950 goto jit_failure;
2951 }
2952 goto done;
2953 case ZEND_BW_OR:
2954 case ZEND_BW_AND:
2955 case ZEND_BW_XOR:
2956 case ZEND_SL:
2957 case ZEND_SR:
2958 case ZEND_MOD:
2959 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2960 break;
2961 }
2962 op1_info = OP1_INFO();
2963 op2_info = OP2_INFO();
2964 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2965 break;
2966 }
2967 if (!(op1_info & MAY_BE_LONG)
2968 || !(op2_info & MAY_BE_LONG)) {
2969 break;
2970 }
2971 res_addr = RES_REG_ADDR();
2972 if (Z_MODE(res_addr) != IS_REG
2973 && (i + 1) <= end
2974 && zend_jit_next_is_send_result(opline)) {
2975 i++;
2976 res_use_info = -1;
2977 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2978 if (!zend_jit_reuse_ip(&dasm_state)) {
2979 goto jit_failure;
2980 }
2981 } else {
2982 res_use_info = -1;
2983
2984 if (opline->result_type == IS_CV) {
2985 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2986
2987 if (Z_MODE(res_use_addr) != IS_REG
2988 || Z_LOAD(res_use_addr)
2989 || Z_STORE(res_use_addr)) {
2990 res_use_info = RES_USE_INFO();
2991 }
2992 }
2993 }
2994 if (!zend_jit_long_math(&dasm_state, opline,
2995 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
2996 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
2997 res_use_info, RES_INFO(), res_addr,
2998 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2999 goto jit_failure;
3000 }
3001 goto done;
3002 case ZEND_ADD:
3003 case ZEND_SUB:
3004 case ZEND_MUL:
3005 // case ZEND_DIV: // TODO: check for division by zero ???
3006 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3007 break;
3008 }
3009 op1_info = OP1_INFO();
3010 op2_info = OP2_INFO();
3011 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3012 break;
3013 }
3014 if (opline->opcode == ZEND_ADD &&
3015 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3016 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3017 /* pass */
3018 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
3019 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
3020 break;
3021 }
3022 res_addr = RES_REG_ADDR();
3023 if (Z_MODE(res_addr) != IS_REG
3024 && (i + 1) <= end
3025 && zend_jit_next_is_send_result(opline)) {
3026 i++;
3027 res_use_info = -1;
3028 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3029 if (!zend_jit_reuse_ip(&dasm_state)) {
3030 goto jit_failure;
3031 }
3032 } else {
3033 res_use_info = -1;
3034
3035 if (opline->result_type == IS_CV) {
3036 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3037
3038 if (Z_MODE(res_use_addr) != IS_REG
3039 || Z_LOAD(res_use_addr)
3040 || Z_STORE(res_use_addr)) {
3041 res_use_info = RES_USE_INFO();
3042 }
3043 }
3044 }
3045 res_info = RES_INFO();
3046 if (opline->opcode == ZEND_ADD &&
3047 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3048 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3049 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
3050 goto jit_failure;
3051 }
3052 } else {
3053 if (!zend_jit_math(&dasm_state, opline,
3054 op1_info, OP1_REG_ADDR(),
3055 op2_info, OP2_REG_ADDR(),
3056 res_use_info, res_info, res_addr,
3057 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3058 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3059 goto jit_failure;
3060 }
3061 }
3062 goto done;
3063 case ZEND_CONCAT:
3064 case ZEND_FAST_CONCAT:
3065 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3066 break;
3067 }
3068 op1_info = OP1_INFO();
3069 op2_info = OP2_INFO();
3070 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3071 break;
3072 }
3073 if (!(op1_info & MAY_BE_STRING) ||
3074 !(op2_info & MAY_BE_STRING)) {
3075 break;
3076 }
3077 res_addr = RES_REG_ADDR();
3078 if ((i + 1) <= end
3079 && zend_jit_next_is_send_result(opline)) {
3080 i++;
3081 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3082 if (!zend_jit_reuse_ip(&dasm_state)) {
3083 goto jit_failure;
3084 }
3085 }
3086 if (!zend_jit_concat(&dasm_state, opline,
3087 op1_info, op2_info, res_addr,
3088 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3089 goto jit_failure;
3090 }
3091 goto done;
3092 case ZEND_ASSIGN_OP:
3093 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3094 break;
3095 }
3096 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3097 break;
3098 }
3099 op1_info = OP1_INFO();
3100 op2_info = OP2_INFO();
3101 if (!zend_jit_supported_binary_op(
3102 opline->extended_value, op1_info, op2_info)) {
3103 break;
3104 }
3105 op1_def_info = OP1_DEF_INFO();
3106 if (!zend_jit_assign_op(&dasm_state, opline,
3107 op1_info, op1_def_info, OP1_RANGE(),
3108 op2_info, OP2_RANGE(),
3109 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3110 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3111 goto jit_failure;
3112 }
3113 goto done;
3114 case ZEND_ASSIGN_DIM_OP:
3115 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3116 break;
3117 }
3118 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3119 break;
3120 }
3121 if (!zend_jit_supported_binary_op(
3122 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3123 break;
3124 }
3125 if (!zend_jit_assign_dim_op(&dasm_state, opline,
3126 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
3127 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
3128 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3129 goto jit_failure;
3130 }
3131 goto done;
3132 case ZEND_ASSIGN_DIM:
3133 if (opline->op1_type != IS_CV) {
3134 break;
3135 }
3136 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3137 break;
3138 }
3139 if (!zend_jit_assign_dim(&dasm_state, opline,
3140 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
3141 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3142 goto jit_failure;
3143 }
3144 goto done;
3145 case ZEND_PRE_INC_OBJ:
3146 case ZEND_PRE_DEC_OBJ:
3147 case ZEND_POST_INC_OBJ:
3148 case ZEND_POST_DEC_OBJ:
3149 if (opline->op2_type != IS_CONST
3150 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3151 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3152 break;
3153 }
3154 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3155 break;
3156 }
3157 ce = NULL;
3158 ce_is_instanceof = 0;
3159 on_this = 0;
3160 if (opline->op1_type == IS_UNUSED) {
3161 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3162 ce = op_array->scope;
3163 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3164 op1_addr = 0;
3165 on_this = 1;
3166 } else {
3167 op1_info = OP1_INFO();
3168 if (!(op1_info & MAY_BE_OBJECT)) {
3169 break;
3170 }
3171 op1_addr = OP1_REG_ADDR();
3172 if (ssa->var_info && ssa->ops) {
3173 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3174 if (ssa_op->op1_use >= 0) {
3175 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3176 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3177 ce = op1_ssa->ce;
3178 ce_is_instanceof = op1_ssa->is_instanceof;
3179 }
3180 }
3181 }
3182 }
3183 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3184 op1_info, op1_addr,
3185 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3186 goto jit_failure;
3187 }
3188 goto done;
3189 case ZEND_ASSIGN_OBJ_OP:
3190 if (opline->result_type != IS_UNUSED) {
3191 break;
3192 }
3193 if (opline->op2_type != IS_CONST
3194 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3195 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3196 break;
3197 }
3198 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3199 break;
3200 }
3201 if (!zend_jit_supported_binary_op(
3202 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3203 break;
3204 }
3205 ce = NULL;
3206 ce_is_instanceof = 0;
3207 on_this = 0;
3208 if (opline->op1_type == IS_UNUSED) {
3209 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3210 ce = op_array->scope;
3211 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3212 op1_addr = 0;
3213 on_this = 1;
3214 } else {
3215 op1_info = OP1_INFO();
3216 if (!(op1_info & MAY_BE_OBJECT)) {
3217 break;
3218 }
3219 op1_addr = OP1_REG_ADDR();
3220 if (ssa->var_info && ssa->ops) {
3221 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3222 if (ssa_op->op1_use >= 0) {
3223 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3224 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3225 ce = op1_ssa->ce;
3226 ce_is_instanceof = op1_ssa->is_instanceof;
3227 }
3228 }
3229 }
3230 }
3231 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
3232 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
3233 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3234 goto jit_failure;
3235 }
3236 goto done;
3237 case ZEND_ASSIGN_OBJ:
3238 if (opline->op2_type != IS_CONST
3239 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3240 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3241 break;
3242 }
3243 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3244 break;
3245 }
3246 ce = NULL;
3247 ce_is_instanceof = 0;
3248 on_this = 0;
3249 if (opline->op1_type == IS_UNUSED) {
3250 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3251 ce = op_array->scope;
3252 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3253 op1_addr = 0;
3254 on_this = 1;
3255 } else {
3256 op1_info = OP1_INFO();
3257 if (!(op1_info & MAY_BE_OBJECT)) {
3258 break;
3259 }
3260 op1_addr = OP1_REG_ADDR();
3261 if (ssa->var_info && ssa->ops) {
3262 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3263 if (ssa_op->op1_use >= 0) {
3264 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3265 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3266 ce = op1_ssa->ce;
3267 ce_is_instanceof = op1_ssa->is_instanceof;
3268 }
3269 }
3270 }
3271 }
3272 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3273 op1_info, op1_addr, OP1_DATA_INFO(),
3274 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
3275 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3276 goto jit_failure;
3277 }
3278 goto done;
3279 case ZEND_ASSIGN:
3280 if (opline->op1_type != IS_CV) {
3281 break;
3282 }
3283 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3284 break;
3285 }
3286 op2_addr = OP2_REG_ADDR();
3287 if (ra
3288 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
3289 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
3290 op2_def_addr = OP2_DEF_REG_ADDR();
3291 } else {
3292 op2_def_addr = op2_addr;
3293 }
3294 op1_info = OP1_INFO();
3295 if (opline->result_type == IS_UNUSED) {
3296 res_addr = 0;
3297 res_info = -1;
3298 } else {
3299 res_addr = RES_REG_ADDR();
3300 res_info = RES_INFO();
3301 if (Z_MODE(res_addr) != IS_REG
3302 && (i + 1) <= end
3303 && zend_jit_next_is_send_result(opline)
3304 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
3305 i++;
3306 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3307 if (!zend_jit_reuse_ip(&dasm_state)) {
3308 goto jit_failure;
3309 }
3310 }
3311 }
3312 if (!zend_jit_assign(&dasm_state, opline,
3313 op1_info, OP1_REG_ADDR(),
3314 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
3315 OP2_INFO(), op2_addr, op2_def_addr,
3316 res_info, res_addr,
3317 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3318 goto jit_failure;
3319 }
3320 goto done;
3321 case ZEND_QM_ASSIGN:
3322 op1_addr = OP1_REG_ADDR();
3323 if (ra
3324 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3325 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3326 op1_def_addr = OP1_DEF_REG_ADDR();
3327 } else {
3328 op1_def_addr = op1_addr;
3329 }
3330 if (!zend_jit_qm_assign(&dasm_state, opline,
3331 OP1_INFO(), op1_addr, op1_def_addr,
3332 -1, RES_INFO(), RES_REG_ADDR())) {
3333 goto jit_failure;
3334 }
3335 goto done;
3336 case ZEND_INIT_FCALL:
3337 case ZEND_INIT_FCALL_BY_NAME:
3338 case ZEND_INIT_NS_FCALL_BY_NAME:
3339 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
3340 goto jit_failure;
3341 }
3342 goto done;
3343 case ZEND_SEND_VAL:
3344 case ZEND_SEND_VAL_EX:
3345 if (opline->op2_type == IS_CONST) {
3346 /* Named parameters not supported in JIT (yet) */
3347 break;
3348 }
3349 if (opline->opcode == ZEND_SEND_VAL_EX
3350 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3351 break;
3352 }
3353 if (!zend_jit_send_val(&dasm_state, opline,
3354 OP1_INFO(), OP1_REG_ADDR())) {
3355 goto jit_failure;
3356 }
3357 goto done;
3358 case ZEND_SEND_REF:
3359 if (opline->op2_type == IS_CONST) {
3360 /* Named parameters not supported in JIT (yet) */
3361 break;
3362 }
3363 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
3364 OP1_INFO(), 0)) {
3365 goto jit_failure;
3366 }
3367 goto done;
3368 case ZEND_SEND_VAR:
3369 case ZEND_SEND_VAR_EX:
3370 case ZEND_SEND_VAR_NO_REF:
3371 case ZEND_SEND_VAR_NO_REF_EX:
3372 case ZEND_SEND_FUNC_ARG:
3373 if (opline->op2_type == IS_CONST) {
3374 /* Named parameters not supported in JIT (yet) */
3375 break;
3376 }
3377 if ((opline->opcode == ZEND_SEND_VAR_EX
3378 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
3379 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3380 break;
3381 }
3382 op1_addr = OP1_REG_ADDR();
3383 if (ra
3384 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3385 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3386 op1_def_addr = OP1_DEF_REG_ADDR();
3387 } else {
3388 op1_def_addr = op1_addr;
3389 }
3390 if (!zend_jit_send_var(&dasm_state, opline, op_array,
3391 OP1_INFO(), op1_addr, op1_def_addr)) {
3392 goto jit_failure;
3393 }
3394 goto done;
3395 case ZEND_CHECK_FUNC_ARG:
3396 if (opline->op2_type == IS_CONST) {
3397 /* Named parameters not supported in JIT (yet) */
3398 break;
3399 }
3400 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
3401 break;
3402 }
3403 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
3404 goto jit_failure;
3405 }
3406 goto done;
3407 case ZEND_CHECK_UNDEF_ARGS:
3408 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
3409 goto jit_failure;
3410 }
3411 goto done;
3412 case ZEND_DO_UCALL:
3413 is_terminated = 1;
3414 ZEND_FALLTHROUGH;
3415 case ZEND_DO_ICALL:
3416 case ZEND_DO_FCALL_BY_NAME:
3417 case ZEND_DO_FCALL:
3418 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
3419 goto jit_failure;
3420 }
3421 goto done;
3422 case ZEND_IS_EQUAL:
3423 case ZEND_IS_NOT_EQUAL:
3424 case ZEND_IS_SMALLER:
3425 case ZEND_IS_SMALLER_OR_EQUAL:
3426 case ZEND_CASE: {
3427 res_addr = RES_REG_ADDR();
3428 if ((opline->result_type & IS_TMP_VAR)
3429 && (i + 1) <= end
3430 && ((opline+1)->opcode == ZEND_JMPZ
3431 || (opline+1)->opcode == ZEND_JMPNZ
3432 || (opline+1)->opcode == ZEND_JMPZ_EX
3433 || (opline+1)->opcode == ZEND_JMPNZ_EX
3434 || (opline+1)->opcode == ZEND_JMPZNZ)
3435 && (opline+1)->op1_type == IS_TMP_VAR
3436 && (opline+1)->op1.var == opline->result.var) {
3437 i++;
3438 smart_branch_opcode = (opline+1)->opcode;
3439 target_label = ssa->cfg.blocks[b].successors[0];
3440 target_label2 = ssa->cfg.blocks[b].successors[1];
3441 /* For EX variant write into the result of EX opcode. */
3442 if ((opline+1)->opcode == ZEND_JMPZ_EX
3443 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
3444 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
3445 }
3446 } else {
3447 smart_branch_opcode = 0;
3448 target_label = target_label2 = (uint32_t)-1;
3449 }
3450 if (!zend_jit_cmp(&dasm_state, opline,
3451 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3452 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3453 res_addr,
3454 zend_may_throw(opline, ssa_op, op_array, ssa),
3455 smart_branch_opcode, target_label, target_label2,
3456 NULL, 0)) {
3457 goto jit_failure;
3458 }
3459 goto done;
3460 }
3461 case ZEND_IS_IDENTICAL:
3462 case ZEND_IS_NOT_IDENTICAL:
3463 case ZEND_CASE_STRICT:
3464 if ((opline->result_type & IS_TMP_VAR)
3465 && (i + 1) <= end
3466 && ((opline+1)->opcode == ZEND_JMPZ
3467 || (opline+1)->opcode == ZEND_JMPNZ
3468 || (opline+1)->opcode == ZEND_JMPZNZ)
3469 && (opline+1)->op1_type == IS_TMP_VAR
3470 && (opline+1)->op1.var == opline->result.var) {
3471 i++;
3472 smart_branch_opcode = (opline+1)->opcode;
3473 target_label = ssa->cfg.blocks[b].successors[0];
3474 target_label2 = ssa->cfg.blocks[b].successors[1];
3475 } else {
3476 smart_branch_opcode = 0;
3477 target_label = target_label2 = (uint32_t)-1;
3478 }
3479 if (!zend_jit_identical(&dasm_state, opline,
3480 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3481 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3482 RES_REG_ADDR(),
3483 zend_may_throw(opline, ssa_op, op_array, ssa),
3484 smart_branch_opcode, target_label, target_label2,
3485 NULL, 0)) {
3486 goto jit_failure;
3487 }
3488 goto done;
3489 case ZEND_DEFINED:
3490 if ((opline->result_type & IS_TMP_VAR)
3491 && (i + 1) <= end
3492 && ((opline+1)->opcode == ZEND_JMPZ
3493 || (opline+1)->opcode == ZEND_JMPNZ
3494 || (opline+1)->opcode == ZEND_JMPZNZ)
3495 && (opline+1)->op1_type == IS_TMP_VAR
3496 && (opline+1)->op1.var == opline->result.var) {
3497 i++;
3498 smart_branch_opcode = (opline+1)->opcode;
3499 target_label = ssa->cfg.blocks[b].successors[0];
3500 target_label2 = ssa->cfg.blocks[b].successors[1];
3501 } else {
3502 smart_branch_opcode = 0;
3503 target_label = target_label2 = (uint32_t)-1;
3504 }
3505 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
3506 goto jit_failure;
3507 }
3508 goto done;
3509 case ZEND_TYPE_CHECK:
3510 if (opline->extended_value == MAY_BE_RESOURCE) {
3511 // TODO: support for is_resource() ???
3512 break;
3513 }
3514 if ((opline->result_type & IS_TMP_VAR)
3515 && (i + 1) <= end
3516 && ((opline+1)->opcode == ZEND_JMPZ
3517 || (opline+1)->opcode == ZEND_JMPNZ
3518 || (opline+1)->opcode == ZEND_JMPZNZ)
3519 && (opline+1)->op1_type == IS_TMP_VAR
3520 && (opline+1)->op1.var == opline->result.var) {
3521 i++;
3522 smart_branch_opcode = (opline+1)->opcode;
3523 target_label = ssa->cfg.blocks[b].successors[0];
3524 target_label2 = ssa->cfg.blocks[b].successors[1];
3525 } else {
3526 smart_branch_opcode = 0;
3527 target_label = target_label2 = (uint32_t)-1;
3528 }
3529 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
3530 goto jit_failure;
3531 }
3532 goto done;
3533 case ZEND_RETURN:
3534 op1_info = OP1_INFO();
3535 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
3536 || op_array->type == ZEND_EVAL_CODE
3537 // TODO: support for top-level code
3538 || !op_array->function_name
3539 // TODO: support for IS_UNDEF ???
3540 || (op1_info & MAY_BE_UNDEF)) {
3541 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3542 goto jit_failure;
3543 }
3544 } else {
3545 int j;
3546 bool left_frame = 0;
3547
3548 if (!zend_jit_return(&dasm_state, opline, op_array,
3549 op1_info, OP1_REG_ADDR())) {
3550 goto jit_failure;
3551 }
3552 if (jit_return_label >= 0) {
3553 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3554 goto jit_failure;
3555 }
3556 goto done;
3557 }
3558 jit_return_label = ssa->cfg.blocks_count * 2;
3559 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3560 goto jit_failure;
3561 }
3562 if (op_array->last_var > 100) {
3563 /* To many CVs to unroll */
3564 if (!zend_jit_free_cvs(&dasm_state)) {
3565 goto jit_failure;
3566 }
3567 left_frame = 1;
3568 }
3569 if (!left_frame) {
3570 for (j = 0 ; j < op_array->last_var; j++) {
3571 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3572
3573 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3574 if (!left_frame) {
3575 left_frame = 1;
3576 if (!zend_jit_leave_frame(&dasm_state)) {
3577 goto jit_failure;
3578 }
3579 }
3580 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3581 goto jit_failure;
3582 }
3583 }
3584 }
3585 }
3586 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3587 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3588 goto jit_failure;
3589 }
3590 }
3591 goto done;
3592 case ZEND_BOOL:
3593 case ZEND_BOOL_NOT:
3594 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3595 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3596 -1, -1,
3597 zend_may_throw(opline, ssa_op, op_array, ssa),
3598 opline->opcode, NULL)) {
3599 goto jit_failure;
3600 }
3601 goto done;
3602 case ZEND_JMPZ:
3603 case ZEND_JMPNZ:
3604 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3605 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3606 /* smart branch */
3607 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3608 goto jit_failure;
3609 }
3610 goto done;
3611 }
3612 ZEND_FALLTHROUGH;
3613 case ZEND_JMPZNZ:
3614 case ZEND_JMPZ_EX:
3615 case ZEND_JMPNZ_EX:
3616 if (opline->result_type == IS_UNDEF) {
3617 res_addr = 0;
3618 } else {
3619 res_addr = RES_REG_ADDR();
3620 }
3621 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3622 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3623 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3624 zend_may_throw(opline, ssa_op, op_array, ssa),
3625 opline->opcode, NULL)) {
3626 goto jit_failure;
3627 }
3628 goto done;
3629 case ZEND_ISSET_ISEMPTY_CV:
3630 if ((opline->extended_value & ZEND_ISEMPTY)) {
3631 // TODO: support for empty() ???
3632 break;
3633 }
3634 if ((opline->result_type & IS_TMP_VAR)
3635 && (i + 1) <= end
3636 && ((opline+1)->opcode == ZEND_JMPZ
3637 || (opline+1)->opcode == ZEND_JMPNZ
3638 || (opline+1)->opcode == ZEND_JMPZNZ)
3639 && (opline+1)->op1_type == IS_TMP_VAR
3640 && (opline+1)->op1.var == opline->result.var) {
3641 i++;
3642 smart_branch_opcode = (opline+1)->opcode;
3643 target_label = ssa->cfg.blocks[b].successors[0];
3644 target_label2 = ssa->cfg.blocks[b].successors[1];
3645 } else {
3646 smart_branch_opcode = 0;
3647 target_label = target_label2 = (uint32_t)-1;
3648 }
3649 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3650 OP1_INFO(), OP1_REG_ADDR(),
3651 smart_branch_opcode, target_label, target_label2,
3652 NULL)) {
3653 goto jit_failure;
3654 }
3655 goto done;
3656 case ZEND_IN_ARRAY:
3657 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3658 break;
3659 }
3660 op1_info = OP1_INFO();
3661 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3662 break;
3663 }
3664 if ((opline->result_type & IS_TMP_VAR)
3665 && (i + 1) <= end
3666 && ((opline+1)->opcode == ZEND_JMPZ
3667 || (opline+1)->opcode == ZEND_JMPNZ
3668 || (opline+1)->opcode == ZEND_JMPZNZ)
3669 && (opline+1)->op1_type == IS_TMP_VAR
3670 && (opline+1)->op1.var == opline->result.var) {
3671 i++;
3672 smart_branch_opcode = (opline+1)->opcode;
3673 target_label = ssa->cfg.blocks[b].successors[0];
3674 target_label2 = ssa->cfg.blocks[b].successors[1];
3675 } else {
3676 smart_branch_opcode = 0;
3677 target_label = target_label2 = (uint32_t)-1;
3678 }
3679 if (!zend_jit_in_array(&dasm_state, opline,
3680 op1_info, OP1_REG_ADDR(),
3681 smart_branch_opcode, target_label, target_label2,
3682 NULL)) {
3683 goto jit_failure;
3684 }
3685 goto done;
3686 case ZEND_FETCH_DIM_R:
3687 case ZEND_FETCH_DIM_IS:
3688 case ZEND_FETCH_LIST_R:
3689 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3690 break;
3691 }
3692 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3693 OP1_INFO(), OP1_REG_ADDR(), 0,
3694 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3695 goto jit_failure;
3696 }
3697 goto done;
3698 case ZEND_FETCH_DIM_W:
3699 case ZEND_FETCH_DIM_RW:
3700 // case ZEND_FETCH_DIM_UNSET:
3701 case ZEND_FETCH_LIST_W:
3702 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3703 break;
3704 }
3705 if (opline->op1_type != IS_CV) {
3706 break;
3707 }
3708 if (!zend_jit_fetch_dim(&dasm_state, opline,
3709 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3710 goto jit_failure;
3711 }
3712 goto done;
3713 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3714 if ((opline->extended_value & ZEND_ISEMPTY)) {
3715 // TODO: support for empty() ???
3716 break;
3717 }
3718 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3719 break;
3720 }
3721 if ((opline->result_type & IS_TMP_VAR)
3722 && (i + 1) <= end
3723 && ((opline+1)->opcode == ZEND_JMPZ
3724 || (opline+1)->opcode == ZEND_JMPNZ
3725 || (opline+1)->opcode == ZEND_JMPZNZ)
3726 && (opline+1)->op1_type == IS_TMP_VAR
3727 && (opline+1)->op1.var == opline->result.var) {
3728 i++;
3729 smart_branch_opcode = (opline+1)->opcode;
3730 target_label = ssa->cfg.blocks[b].successors[0];
3731 target_label2 = ssa->cfg.blocks[b].successors[1];
3732 } else {
3733 smart_branch_opcode = 0;
3734 target_label = target_label2 = (uint32_t)-1;
3735 }
3736 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3737 OP1_INFO(), OP1_REG_ADDR(), 0,
3738 OP2_INFO(), IS_UNKNOWN,
3739 zend_may_throw(opline, ssa_op, op_array, ssa),
3740 smart_branch_opcode, target_label, target_label2,
3741 NULL)) {
3742 goto jit_failure;
3743 }
3744 goto done;
3745 case ZEND_FETCH_OBJ_R:
3746 case ZEND_FETCH_OBJ_IS:
3747 case ZEND_FETCH_OBJ_W:
3748 if (opline->op2_type != IS_CONST
3749 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3750 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3751 break;
3752 }
3753 ce = NULL;
3754 ce_is_instanceof = 0;
3755 on_this = 0;
3756 if (opline->op1_type == IS_UNUSED) {
3757 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3758 op1_addr = 0;
3759 ce = op_array->scope;
3760 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3761 on_this = 1;
3762 } else {
3763 op1_info = OP1_INFO();
3764 if (!(op1_info & MAY_BE_OBJECT)) {
3765 break;
3766 }
3767 op1_addr = OP1_REG_ADDR();
3768 if (ssa->var_info && ssa->ops) {
3769 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3770 if (ssa_op->op1_use >= 0) {
3771 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3772 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3773 ce = op1_ssa->ce;
3774 ce_is_instanceof = op1_ssa->is_instanceof;
3775 }
3776 }
3777 }
3778 }
3779 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3780 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
3781 IS_UNKNOWN,
3782 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3783 goto jit_failure;
3784 }
3785 goto done;
3786 case ZEND_BIND_GLOBAL:
3787 if (!ssa->ops || !ssa->var_info) {
3788 op1_info = MAY_BE_ANY|MAY_BE_REF;
3789 } else {
3790 op1_info = OP1_INFO();
3791 }
3792 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3793 goto jit_failure;
3794 }
3795 goto done;
3796 case ZEND_RECV:
3797 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3798 goto jit_failure;
3799 }
3800 goto done;
3801 case ZEND_RECV_INIT:
3802 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3803 (opline + 1)->opcode != ZEND_RECV_INIT,
3804 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3805 goto jit_failure;
3806 }
3807 goto done;
3808 case ZEND_FREE:
3809 case ZEND_FE_FREE:
3810 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3811 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3812 goto jit_failure;
3813 }
3814 goto done;
3815 case ZEND_ECHO:
3816 op1_info = OP1_INFO();
3817 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3818 break;
3819 }
3820 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3821 goto jit_failure;
3822 }
3823 goto done;
3824 case ZEND_STRLEN:
3825 op1_info = OP1_INFO();
3826 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3827 break;
3828 }
3829 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
3830 goto jit_failure;
3831 }
3832 goto done;
3833 case ZEND_COUNT:
3834 op1_info = OP1_INFO();
3835 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
3836 break;
3837 }
3838 if (!zend_jit_count(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
3839 goto jit_failure;
3840 }
3841 goto done;
3842 case ZEND_FETCH_THIS:
3843 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3844 goto jit_failure;
3845 }
3846 goto done;
3847 case ZEND_SWITCH_LONG:
3848 case ZEND_SWITCH_STRING:
3849 case ZEND_MATCH:
3850 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3851 goto jit_failure;
3852 }
3853 goto done;
3854 case ZEND_VERIFY_RETURN_TYPE:
3855 if (opline->op1_type == IS_UNUSED) {
3856 /* Always throws */
3857 break;
3858 }
3859 if (opline->op1_type == IS_CONST) {
3860 /* TODO Different instruction format, has return value */
3861 break;
3862 }
3863 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3864 /* Not worth bothering with */
3865 break;
3866 }
3867 if (OP1_INFO() & MAY_BE_REF) {
3868 /* TODO May need reference unwrapping. */
3869 break;
3870 }
3871 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3872 goto jit_failure;
3873 }
3874 goto done;
3875 case ZEND_FE_RESET_R:
3876 op1_info = OP1_INFO();
3877 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3878 break;
3879 }
3880 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3881 goto jit_failure;
3882 }
3883 goto done;
3884 case ZEND_FE_FETCH_R:
3885 op1_info = OP1_INFO();
3886 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3887 break;
3888 }
3889 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3890 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3891 goto jit_failure;
3892 }
3893 goto done;
3894 case ZEND_FETCH_CONSTANT:
3895 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
3896 goto jit_failure;
3897 }
3898 goto done;
3899 case ZEND_INIT_METHOD_CALL:
3900 if (opline->op2_type != IS_CONST
3901 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3902 break;
3903 }
3904 ce = NULL;
3905 ce_is_instanceof = 0;
3906 on_this = 0;
3907 if (opline->op1_type == IS_UNUSED) {
3908 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3909 op1_addr = 0;
3910 ce = op_array->scope;
3911 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3912 on_this = 1;
3913 } else {
3914 op1_info = OP1_INFO();
3915 if (!(op1_info & MAY_BE_OBJECT)) {
3916 break;
3917 }
3918 op1_addr = OP1_REG_ADDR();
3919 if (ssa->var_info && ssa->ops) {
3920 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3921 if (ssa_op->op1_use >= 0) {
3922 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3923 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3924 ce = op1_ssa->ce;
3925 ce_is_instanceof = op1_ssa->is_instanceof;
3926 }
3927 }
3928 }
3929 }
3930 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3931 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
3932 NULL, 0, 0)) {
3933 goto jit_failure;
3934 }
3935 goto done;
3936 case ZEND_ROPE_INIT:
3937 case ZEND_ROPE_ADD:
3938 case ZEND_ROPE_END:
3939 op2_info = OP2_INFO();
3940 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3941 break;
3942 }
3943 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
3944 goto jit_failure;
3945 }
3946 goto done;
3947 default:
3948 break;
3949 }
3950 }
3951
3952 switch (opline->opcode) {
3953 case ZEND_RECV_INIT:
3954 case ZEND_BIND_GLOBAL:
3955 if (opline == op_array->opcodes ||
3956 opline->opcode != op_array->opcodes[i-1].opcode) {
3957 /* repeatable opcodes */
3958 if (!zend_jit_handler(&dasm_state, opline,
3959 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3960 goto jit_failure;
3961 }
3962 }
3963 zend_jit_set_last_valid_opline(opline+1);
3964 break;
3965 case ZEND_NOP:
3966 case ZEND_OP_DATA:
3967 case ZEND_SWITCH_LONG:
3968 case ZEND_SWITCH_STRING:
3969 case ZEND_MATCH:
3970 break;
3971 case ZEND_JMP:
3972 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
3973 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
3974
3975 if (!zend_jit_set_ip(&dasm_state, target)) {
3976 goto jit_failure;
3977 }
3978 }
3979 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
3980 goto jit_failure;
3981 }
3982 is_terminated = 1;
3983 break;
3984 case ZEND_CATCH:
3985 case ZEND_FAST_CALL:
3986 case ZEND_FAST_RET:
3987 case ZEND_GENERATOR_CREATE:
3988 case ZEND_GENERATOR_RETURN:
3989 case ZEND_RETURN_BY_REF:
3990 case ZEND_RETURN:
3991 case ZEND_EXIT:
3992 case ZEND_MATCH_ERROR:
3993 /* switch through trampoline */
3994 case ZEND_YIELD:
3995 case ZEND_YIELD_FROM:
3996 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3997 goto jit_failure;
3998 }
3999 is_terminated = 1;
4000 break;
4001 /* stackless execution */
4002 case ZEND_INCLUDE_OR_EVAL:
4003 case ZEND_DO_FCALL:
4004 case ZEND_DO_UCALL:
4005 case ZEND_DO_FCALL_BY_NAME:
4006 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
4007 goto jit_failure;
4008 }
4009 is_terminated = 1;
4010 break;
4011 case ZEND_JMPZNZ:
4012 if (!zend_jit_handler(&dasm_state, opline,
4013 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4014 !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) ||
4015 !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
4016 goto jit_failure;
4017 }
4018 is_terminated = 1;
4019 break;
4020 case ZEND_JMPZ:
4021 case ZEND_JMPNZ:
4022 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
4023 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4024 /* smart branch */
4025 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4026 goto jit_failure;
4027 }
4028 goto done;
4029 }
4030 ZEND_FALLTHROUGH;
4031 case ZEND_JMPZ_EX:
4032 case ZEND_JMPNZ_EX:
4033 case ZEND_JMP_SET:
4034 case ZEND_COALESCE:
4035 case ZEND_JMP_NULL:
4036 case ZEND_FE_RESET_R:
4037 case ZEND_FE_RESET_RW:
4038 case ZEND_ASSERT_CHECK:
4039 case ZEND_FE_FETCH_R:
4040 case ZEND_FE_FETCH_RW:
4041 if (!zend_jit_handler(&dasm_state, opline,
4042 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4043 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4044 goto jit_failure;
4045 }
4046 break;
4047 case ZEND_NEW:
4048 if (!zend_jit_handler(&dasm_state, opline, 1)) {
4049 return 0;
4050 }
4051 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
4052 zend_class_entry *ce = NULL;
4053
4054 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
4055 if (ssa->ops && ssa->var_info) {
4056 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
4057 if (res_ssa->ce && !res_ssa->is_instanceof) {
4058 ce = res_ssa->ce;
4059 }
4060 }
4061 } else {
4062 if (opline->op1_type == IS_CONST) {
4063 zval *zv = RT_CONSTANT(opline, opline->op1);
4064 if (Z_TYPE_P(zv) == IS_STRING) {
4065 zval *lc = zv + 1;
4066 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
4067 }
4068 }
4069 }
4070
4071 i++;
4072
4073 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
4074 const zend_op *next_opline = opline + 1;
4075
4076 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
4077 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4078 zend_jit_call(&dasm_state, next_opline, b + 1);
4079 is_terminated = 1;
4080 } else {
4081 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
4082 }
4083 }
4084
4085 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
4086 call_level--;
4087 }
4088 break;
4089 default:
4090 if (!zend_jit_handler(&dasm_state, opline,
4091 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4092 goto jit_failure;
4093 }
4094 if (i == end
4095 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4096 /* smart branch split across basic blocks */
4097 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
4098 goto jit_failure;
4099 }
4100 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
4101 goto jit_failure;
4102 }
4103 is_terminated = 1;
4104 }
4105 }
4106 done:
4107 switch (opline->opcode) {
4108 case ZEND_DO_FCALL:
4109 case ZEND_DO_ICALL:
4110 case ZEND_DO_UCALL:
4111 case ZEND_DO_FCALL_BY_NAME:
4112 case ZEND_CALLABLE_CONVERT:
4113 call_level--;
4114 }
4115 }
4116 }
4117
4118 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0,
4119 (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT);
4120 if (!handler) {
4121 goto jit_failure;
4122 }
4123 dasm_free(&dasm_state);
4124
4125 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4126 zend_arena_release(&CG(arena), checkpoint);
4127 }
4128 return SUCCESS;
4129
4130 jit_failure:
4131 if (dasm_state) {
4132 dasm_free(&dasm_state);
4133 }
4134 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4135 zend_arena_release(&CG(arena), checkpoint);
4136 }
4137 return FAILURE;
4138 }
4139
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)4140 static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
4141 {
4142 zend_func_info *func_info;
4143
4144 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4145 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4146 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4147 func_info = ZEND_FUNC_INFO(op_array);
4148 } else {
4149 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
4150 ZEND_SET_FUNC_INFO(op_array, func_info);
4151 }
4152 return zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
4153 }
4154
zend_jit_cleanup_func_info(zend_op_array * op_array)4155 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
4156 {
4157 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4158 zend_call_info *caller_info, *callee_info;
4159
4160 if (func_info) {
4161 caller_info = func_info->caller_info;
4162 callee_info = func_info->callee_info;
4163
4164 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4165 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4166 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4167 func_info->num = 0;
4168 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
4169 | ZEND_FUNC_JIT_ON_PROF_REQUEST
4170 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
4171 | ZEND_FUNC_JIT_ON_HOT_TRACE;
4172 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
4173 } else {
4174 ZEND_SET_FUNC_INFO(op_array, NULL);
4175 }
4176
4177 while (caller_info) {
4178 if (caller_info->caller_op_array) {
4179 zend_jit_cleanup_func_info(caller_info->caller_op_array);
4180 }
4181 caller_info = caller_info->next_caller;
4182 }
4183 while (callee_info) {
4184 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
4185 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
4186 }
4187 callee_info = callee_info->next_callee;
4188 }
4189 }
4190 }
4191
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)4192 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
4193 {
4194 zend_ssa ssa;
4195 void *checkpoint;
4196 zend_func_info *func_info;
4197
4198 if (*dasm_ptr == dasm_end) {
4199 return FAILURE;
4200 }
4201
4202 checkpoint = zend_arena_checkpoint(CG(arena));
4203
4204 /* Build SSA */
4205 memset(&ssa, 0, sizeof(zend_ssa));
4206
4207 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
4208 goto jit_failure;
4209 }
4210
4211 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
4212 if (zend_jit_collect_calls(op_array, script) != SUCCESS) {
4213 goto jit_failure;
4214 }
4215 func_info = ZEND_FUNC_INFO(op_array);
4216 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
4217 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4218 zend_init_func_return_info(op_array, script, &func_info->return_info);
4219 }
4220 }
4221
4222 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4223 goto jit_failure;
4224 }
4225
4226 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4227 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
4228 }
4229
4230 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
4231 goto jit_failure;
4232 }
4233
4234 zend_jit_cleanup_func_info(op_array);
4235 zend_arena_release(&CG(arena), checkpoint);
4236 return SUCCESS;
4237
4238 jit_failure:
4239 zend_jit_cleanup_func_info(op_array);
4240 zend_arena_release(&CG(arena), checkpoint);
4241 return FAILURE;
4242 }
4243
4244 /* Run-time JIT handler */
zend_runtime_jit(void)4245 static int ZEND_FASTCALL zend_runtime_jit(void)
4246 {
4247 zend_execute_data *execute_data = EG(current_execute_data);
4248 zend_op_array *op_array = &EX(func)->op_array;
4249 zend_op *opline = op_array->opcodes;
4250 zend_jit_op_array_extension *jit_extension;
4251
4252 zend_shared_alloc_lock();
4253
4254 if (ZEND_FUNC_INFO(op_array)) {
4255 SHM_UNPROTECT();
4256 zend_jit_unprotect();
4257
4258 /* restore original opcode handlers */
4259 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4260 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4261 opline++;
4262 }
4263 }
4264 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4265 opline->handler = jit_extension->orig_handler;
4266
4267 /* perform real JIT for this function */
4268 zend_real_jit_func(op_array, NULL, NULL);
4269
4270 zend_jit_protect();
4271 SHM_PROTECT();
4272 }
4273
4274 zend_shared_alloc_unlock();
4275
4276 /* JIT-ed code is going to be called by VM */
4277 return 0;
4278 }
4279
zend_jit_check_funcs(HashTable * function_table,bool is_method)4280 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
4281 zend_op *opline;
4282 zend_function *func;
4283 zend_op_array *op_array;
4284 uintptr_t counter;
4285 zend_jit_op_array_extension *jit_extension;
4286
4287 ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) {
4288 if (func->type == ZEND_INTERNAL_FUNCTION) {
4289 break;
4290 }
4291 op_array = &func->op_array;
4292 opline = op_array->opcodes;
4293 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4294 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4295 opline++;
4296 }
4297 }
4298 if (opline->handler == zend_jit_profile_jit_handler) {
4299 if (!RUN_TIME_CACHE(op_array)) {
4300 continue;
4301 }
4302 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
4303 ZEND_COUNTER_INFO(op_array) = 0;
4304 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4305 opline->handler = jit_extension->orig_handler;
4306 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
4307 zend_real_jit_func(op_array, NULL, NULL);
4308 }
4309 }
4310 } ZEND_HASH_FOREACH_END();
4311 }
4312
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)4313 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
4314 {
4315 zend_op_array *op_array = &EX(func)->op_array;
4316 zend_jit_op_array_hot_extension *jit_extension;
4317 uint32_t i;
4318
4319 zend_shared_alloc_lock();
4320 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4321
4322 if (jit_extension) {
4323 SHM_UNPROTECT();
4324 zend_jit_unprotect();
4325
4326 for (i = 0; i < op_array->last; i++) {
4327 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4328 }
4329
4330 /* perform real JIT for this function */
4331 zend_real_jit_func(op_array, NULL, opline);
4332
4333 zend_jit_protect();
4334 SHM_PROTECT();
4335 }
4336
4337 zend_shared_alloc_unlock();
4338
4339 /* JIT-ed code is going to be called by VM */
4340 }
4341
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)4342 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
4343 {
4344 if (JIT_G(hot_func)) {
4345 zend_op *opline = op_array->opcodes;
4346
4347 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4348 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4349 opline++;
4350 }
4351 }
4352
4353 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
4354 }
4355
4356 if (JIT_G(hot_loop)) {
4357 uint32_t i;
4358
4359 for (i = 0; i < cfg->blocks_count; i++) {
4360 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
4361 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
4362 op_array->opcodes[cfg->blocks[i].start].handler =
4363 (const void*)zend_jit_loop_hot_counter_handler;
4364 }
4365 }
4366 }
4367 }
4368
zend_jit_restart_hot_counters(zend_op_array * op_array)4369 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
4370 {
4371 zend_jit_op_array_hot_extension *jit_extension;
4372 zend_cfg cfg;
4373 uint32_t i;
4374
4375 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4376 for (i = 0; i < op_array->last; i++) {
4377 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4378 }
4379
4380 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4381 return FAILURE;
4382 }
4383
4384 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4385
4386 return SUCCESS;
4387 }
4388
zend_jit_setup_hot_counters(zend_op_array * op_array)4389 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
4390 {
4391 zend_jit_op_array_hot_extension *jit_extension;
4392 zend_cfg cfg;
4393 uint32_t i;
4394
4395 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
4396 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
4397
4398 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4399 return FAILURE;
4400 }
4401
4402 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
4403 if (!jit_extension) {
4404 return FAILURE;
4405 }
4406 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4407 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
4408 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
4409 for (i = 0; i < op_array->last; i++) {
4410 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
4411 }
4412 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4413
4414 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4415
4416 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4417
4418 return SUCCESS;
4419 }
4420
4421 #include "jit/zend_jit_trace.c"
4422
zend_jit_op_array(zend_op_array * op_array,zend_script * script)4423 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
4424 {
4425 if (dasm_ptr == NULL) {
4426 return FAILURE;
4427 }
4428
4429 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
4430 zend_jit_op_array_extension *jit_extension;
4431 zend_op *opline = op_array->opcodes;
4432
4433 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4434 ZEND_SET_FUNC_INFO(op_array, NULL);
4435 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4436 return SUCCESS;
4437 }
4438
4439 /* Set run-time JIT handler */
4440 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
4441 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4442 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4443 opline++;
4444 }
4445 }
4446 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4447 if (!jit_extension) {
4448 return FAILURE;
4449 }
4450 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4451 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
4452 jit_extension->orig_handler = (void*)opline->handler;
4453 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4454 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4455 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4456
4457 return SUCCESS;
4458 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
4459 zend_jit_op_array_extension *jit_extension;
4460 zend_op *opline = op_array->opcodes;
4461
4462 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4463 ZEND_SET_FUNC_INFO(op_array, NULL);
4464 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4465 return SUCCESS;
4466 }
4467
4468 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
4469 if (op_array->function_name) {
4470 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4471 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4472 opline++;
4473 }
4474 }
4475 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4476 if (!jit_extension) {
4477 return FAILURE;
4478 }
4479 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4480 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
4481 jit_extension->orig_handler = (void*)opline->handler;
4482 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4483 opline->handler = (const void*)zend_jit_profile_jit_handler;
4484 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4485 }
4486
4487 return SUCCESS;
4488 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4489 return zend_jit_setup_hot_counters(op_array);
4490 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4491 return zend_jit_setup_hot_trace_counters(op_array);
4492 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4493 return zend_real_jit_func(op_array, script, NULL);
4494 } else {
4495 ZEND_UNREACHABLE();
4496 }
4497 }
4498
zend_jit_script(zend_script * script)4499 ZEND_EXT_API int zend_jit_script(zend_script *script)
4500 {
4501 void *checkpoint;
4502 zend_call_graph call_graph;
4503 zend_func_info *info;
4504 int i;
4505
4506 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
4507 return FAILURE;
4508 }
4509
4510 checkpoint = zend_arena_checkpoint(CG(arena));
4511
4512 call_graph.op_arrays_count = 0;
4513 if (zend_build_call_graph(&CG(arena), script, &call_graph) != SUCCESS) {
4514 goto jit_failure;
4515 }
4516
4517 zend_analyze_call_graph(&CG(arena), script, &call_graph);
4518
4519 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4520 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4521 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
4522 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4523 for (i = 0; i < call_graph.op_arrays_count; i++) {
4524 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
4525 goto jit_failure;
4526 }
4527 }
4528 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4529 for (i = 0; i < call_graph.op_arrays_count; i++) {
4530 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4531 if (info) {
4532 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
4533 goto jit_failure;
4534 }
4535 info->flags = info->ssa.cfg.flags;
4536 }
4537 }
4538
4539 for (i = 0; i < call_graph.op_arrays_count; i++) {
4540 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4541 if (info) {
4542 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
4543 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4544 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
4545 }
4546 }
4547 }
4548
4549 for (i = 0; i < call_graph.op_arrays_count; i++) {
4550 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4551 if (info) {
4552 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4553 goto jit_failure;
4554 }
4555 info->flags = info->ssa.cfg.flags;
4556 }
4557 }
4558
4559 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4560 for (i = 0; i < call_graph.op_arrays_count; i++) {
4561 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4562 if (info) {
4563 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
4564 }
4565 }
4566 }
4567
4568 for (i = 0; i < call_graph.op_arrays_count; i++) {
4569 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4570 if (info) {
4571 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
4572 goto jit_failure;
4573 }
4574 }
4575 }
4576
4577 for (i = 0; i < call_graph.op_arrays_count; i++) {
4578 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4579 }
4580 } else {
4581 ZEND_UNREACHABLE();
4582 }
4583
4584 zend_arena_release(&CG(arena), checkpoint);
4585
4586 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
4587 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
4588 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4589 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4590 zend_class_entry *ce;
4591 zend_op_array *op_array;
4592
4593 ZEND_HASH_FOREACH_PTR(&script->class_table, ce) {
4594 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
4595 if (!ZEND_FUNC_INFO(op_array)) {
4596 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4597
4598 if (jit_extension) {
4599 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4600 }
4601 }
4602 } ZEND_HASH_FOREACH_END();
4603 } ZEND_HASH_FOREACH_END();
4604 }
4605
4606 return SUCCESS;
4607
4608 jit_failure:
4609 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4610 for (i = 0; i < call_graph.op_arrays_count; i++) {
4611 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4612 }
4613 }
4614 zend_arena_release(&CG(arena), checkpoint);
4615 return FAILURE;
4616 }
4617
zend_jit_unprotect(void)4618 ZEND_EXT_API void zend_jit_unprotect(void)
4619 {
4620 #ifdef HAVE_MPROTECT
4621 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4622 int opts = PROT_READ | PROT_WRITE;
4623 #ifdef ZTS
4624 /* TODO: EXEC+WRITE is not supported in macOS. Removing EXEC is still buggy as
4625 * other threads, which are executing the JITed code, would crash anyway. */
4626 # ifndef __APPLE__
4627 /* Another thread may be executing JITed code. */
4628 opts |= PROT_EXEC;
4629 # endif
4630 #endif
4631 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4632 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4633 }
4634 }
4635 #elif _WIN32
4636 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4637 DWORD old, new;
4638 #ifdef ZTS
4639 new = PAGE_EXECUTE_READWRITE;
4640 #else
4641 new = PAGE_READWRITE;
4642 #endif
4643 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4644 DWORD err = GetLastError();
4645 char *msg = php_win32_error_to_msg(err);
4646 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4647 php_win32_error_msg_free(msg);
4648 }
4649 }
4650 #endif
4651 }
4652
zend_jit_protect(void)4653 ZEND_EXT_API void zend_jit_protect(void)
4654 {
4655 #ifdef HAVE_MPROTECT
4656 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4657 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4658 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4659 }
4660 }
4661 #elif _WIN32
4662 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4663 DWORD old;
4664
4665 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4666 DWORD err = GetLastError();
4667 char *msg = php_win32_error_to_msg(err);
4668 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4669 php_win32_error_msg_free(msg);
4670 }
4671 }
4672 #endif
4673 }
4674
zend_jit_init_handlers(void)4675 static void zend_jit_init_handlers(void)
4676 {
4677 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4678 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4679 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4680 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4681 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4682 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4683 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4684 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4685 } else {
4686 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4687 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4688 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4689 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4690 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4691 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4692 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4693 }
4694 }
4695
zend_jit_make_stubs(void)4696 static int zend_jit_make_stubs(void)
4697 {
4698 dasm_State* dasm_state = NULL;
4699 uint32_t i;
4700
4701 dasm_init(&dasm_state, DASM_MAXSECTION);
4702 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4703
4704 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4705 dasm_setup(&dasm_state, dasm_actions);
4706 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4707 return 0;
4708 }
4709 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0,
4710 zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) {
4711 return 0;
4712 }
4713 }
4714
4715 zend_jit_init_handlers();
4716
4717 dasm_free(&dasm_state);
4718 return 1;
4719 }
4720
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4721 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4722 {
4723 memset(jit_globals, 0, sizeof(zend_jit_globals));
4724 zend_jit_trace_init_caches();
4725 }
4726
zend_jit_parse_config_num(zend_long jit)4727 static int zend_jit_parse_config_num(zend_long jit)
4728 {
4729 if (jit == 0) {
4730 JIT_G(on) = 0;
4731 return SUCCESS;
4732 }
4733
4734 if (jit < 0) return FAILURE;
4735
4736 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4737 JIT_G(opt_level) = jit % 10;
4738
4739 jit /= 10;
4740 if (jit % 10 > 5) return FAILURE;
4741 JIT_G(trigger) = jit % 10;
4742
4743 jit /= 10;
4744 if (jit % 10 > 2) return FAILURE;
4745 JIT_G(opt_flags) = jit % 10;
4746
4747 jit /= 10;
4748 if (jit % 10 > 1) return FAILURE;
4749 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4750
4751 if (jit / 10 != 0) return FAILURE;
4752
4753 JIT_G(on) = 1;
4754
4755 return SUCCESS;
4756 }
4757
zend_jit_config(zend_string * jit,int stage)4758 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4759 {
4760 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4761 if (stage == ZEND_INI_STAGE_RUNTIME) {
4762 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4763 }
4764 return FAILURE;
4765 }
4766
4767 if (ZSTR_LEN(jit) == 0
4768 || zend_string_equals_literal_ci(jit, "disable")) {
4769 JIT_G(enabled) = 0;
4770 JIT_G(on) = 0;
4771 return SUCCESS;
4772 } else if (zend_string_equals_literal_ci(jit, "0")
4773 || zend_string_equals_literal_ci(jit, "off")
4774 || zend_string_equals_literal_ci(jit, "no")
4775 || zend_string_equals_literal_ci(jit, "false")) {
4776 JIT_G(enabled) = 1;
4777 JIT_G(on) = 0;
4778 return SUCCESS;
4779 } else if (zend_string_equals_literal_ci(jit, "1")
4780 || zend_string_equals_literal_ci(jit, "on")
4781 || zend_string_equals_literal_ci(jit, "yes")
4782 || zend_string_equals_literal_ci(jit, "true")
4783 || zend_string_equals_literal_ci(jit, "tracing")) {
4784 JIT_G(enabled) = 1;
4785 JIT_G(on) = 1;
4786 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4787 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4788 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4789 return SUCCESS;
4790 } else if (zend_string_equals_literal_ci(jit, "function")) {
4791 JIT_G(enabled) = 1;
4792 JIT_G(on) = 1;
4793 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4794 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4795 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4796 return SUCCESS;
4797 } else {
4798 char *end;
4799 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4800 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4801 goto failure;
4802 }
4803 JIT_G(enabled) = 1;
4804 return SUCCESS;
4805 }
4806
4807 failure:
4808 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4809 JIT_G(enabled) = 0;
4810 JIT_G(on) = 0;
4811 return FAILURE;
4812 }
4813
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4814 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4815 {
4816 if (stage != ZEND_INI_STAGE_STARTUP) {
4817 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4818 if (stage == ZEND_INI_STAGE_RUNTIME) {
4819 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4820 }
4821 return FAILURE;
4822 }
4823 #ifdef HAVE_DISASM
4824 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4825 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4826 // TODO: error reporting and cleanup ???
4827 return FAILURE;
4828 }
4829 // TODO: symbols for JIT-ed code compiled before are missing ???
4830 }
4831 #endif
4832 }
4833 return SUCCESS;
4834 }
4835
zend_jit_init(void)4836 ZEND_EXT_API void zend_jit_init(void)
4837 {
4838 #ifdef ZTS
4839 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL);
4840 #else
4841 zend_jit_globals_ctor(&jit_globals);
4842 #endif
4843 }
4844
zend_jit_check_support(void)4845 ZEND_EXT_API int zend_jit_check_support(void)
4846 {
4847 int i;
4848
4849 zend_jit_vm_kind = zend_vm_kind();
4850 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4851 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4852 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4853 JIT_G(enabled) = 0;
4854 JIT_G(on) = 0;
4855 return FAILURE;
4856 }
4857
4858 if (zend_execute_ex != execute_ex) {
4859 if (strcmp(sapi_module.name, "phpdbg") != 0) {
4860 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4861 }
4862 JIT_G(enabled) = 0;
4863 JIT_G(on) = 0;
4864 return FAILURE;
4865 }
4866
4867 for (i = 0; i <= 256; i++) {
4868 switch (i) {
4869 /* JIT has no effect on these opcodes */
4870 case ZEND_BEGIN_SILENCE:
4871 case ZEND_END_SILENCE:
4872 case ZEND_EXIT:
4873 break;
4874 default:
4875 if (zend_get_user_opcode_handler(i) != NULL) {
4876 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4877 JIT_G(enabled) = 0;
4878 JIT_G(on) = 0;
4879 return FAILURE;
4880 }
4881 }
4882 }
4883
4884 return SUCCESS;
4885 }
4886
zend_jit_startup(void * buf,size_t size,bool reattached)4887 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
4888 {
4889 int ret;
4890
4891 zend_jit_halt_op = zend_get_halt_op();
4892
4893 if (zend_jit_setup() != SUCCESS) {
4894 // TODO: error reporting and cleanup ???
4895 return FAILURE;
4896 }
4897
4898 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4899
4900 #ifdef HAVE_GDB
4901 zend_jit_gdb_init();
4902 #endif
4903
4904 #ifdef HAVE_OPROFILE
4905 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
4906 if (!zend_jit_oprofile_startup()) {
4907 // TODO: error reporting and cleanup ???
4908 return FAILURE;
4909 }
4910 }
4911 #endif
4912
4913 dasm_buf = buf;
4914 dasm_size = size;
4915
4916 #ifdef HAVE_MPROTECT
4917 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4918 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4919 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4920 }
4921 } else {
4922 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4923 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4924 }
4925 }
4926 #elif _WIN32
4927 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4928 DWORD old;
4929
4930 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4931 DWORD err = GetLastError();
4932 char *msg = php_win32_error_to_msg(err);
4933 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4934 php_win32_error_msg_free(msg);
4935 }
4936 } else {
4937 DWORD old;
4938
4939 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4940 DWORD err = GetLastError();
4941 char *msg = php_win32_error_to_msg(err);
4942 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4943 php_win32_error_msg_free(msg);
4944 }
4945 }
4946 #endif
4947
4948 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
4949 if (!reattached) {
4950 zend_jit_unprotect();
4951 *dasm_ptr = dasm_buf;
4952 #if _WIN32
4953 /* reserve space for global labels */
4954 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
4955 #endif
4956 zend_jit_protect();
4957 }
4958
4959 #ifdef HAVE_DISASM
4960 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4961 if (!zend_jit_disasm_init()) {
4962 // TODO: error reporting and cleanup ???
4963 return FAILURE;
4964 }
4965 }
4966 #endif
4967
4968 #ifdef HAVE_PERFTOOLS
4969 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
4970 zend_jit_perf_jitdump_open();
4971 }
4972 #endif
4973
4974 if (!reattached) {
4975 zend_jit_unprotect();
4976 ret = zend_jit_make_stubs();
4977 #if _WIN32
4978 /* save global labels */
4979 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
4980 #endif
4981 zend_jit_protect();
4982 if (!ret) {
4983 // TODO: error reporting and cleanup ???
4984 return FAILURE;
4985 }
4986 } else {
4987 #if _WIN32
4988 /* restore global labels */
4989 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
4990 zend_jit_init_handlers();
4991 #endif
4992 }
4993
4994 if (zend_jit_trace_startup() != SUCCESS) {
4995 return FAILURE;
4996 }
4997
4998 zend_jit_unprotect();
4999 #if ZEND_JIT_TARGET_ARM64
5000 /* reserve space for global labels veneers */
5001 dasm_labels_veneers = *dasm_ptr;
5002 *dasm_ptr = (void**)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT);
5003 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5004 #endif
5005 /* save JIT buffer pos */
5006 dasm_ptr[1] = dasm_ptr[0];
5007 zend_jit_protect();
5008
5009 return SUCCESS;
5010 }
5011
zend_jit_shutdown(void)5012 ZEND_EXT_API void zend_jit_shutdown(void)
5013 {
5014 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) {
5015 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
5016 }
5017
5018 #ifdef HAVE_OPROFILE
5019 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
5020 zend_jit_oprofile_shutdown();
5021 }
5022 #endif
5023
5024 #ifdef HAVE_GDB
5025 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
5026 zend_jit_gdb_unregister();
5027 }
5028 #endif
5029
5030 #ifdef HAVE_DISASM
5031 zend_jit_disasm_shutdown();
5032 #endif
5033
5034 #ifdef HAVE_PERFTOOLS
5035 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5036 zend_jit_perf_jitdump_close();
5037 }
5038 #endif
5039 if (JIT_G(exit_counters)) {
5040 free(JIT_G(exit_counters));
5041 }
5042 }
5043
zend_jit_reset_counters(void)5044 static void zend_jit_reset_counters(void)
5045 {
5046 int i;
5047
5048 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
5049 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
5050 }
5051 }
5052
zend_jit_activate(void)5053 ZEND_EXT_API void zend_jit_activate(void)
5054 {
5055 zend_jit_profile_counter = 0;
5056 if (JIT_G(on)) {
5057 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
5058 zend_jit_reset_counters();
5059 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
5060 zend_jit_reset_counters();
5061 zend_jit_trace_reset_caches();
5062 }
5063 }
5064 }
5065
zend_jit_deactivate(void)5066 ZEND_EXT_API void zend_jit_deactivate(void)
5067 {
5068 if (zend_jit_profile_counter) {
5069 zend_class_entry *ce;
5070
5071 zend_shared_alloc_lock();
5072 SHM_UNPROTECT();
5073 zend_jit_unprotect();
5074
5075 zend_jit_check_funcs(EG(function_table), 0);
5076 ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) {
5077 if (ce->type == ZEND_INTERNAL_CLASS) {
5078 break;
5079 }
5080 zend_jit_check_funcs(&ce->function_table, 1);
5081 } ZEND_HASH_FOREACH_END();
5082
5083 zend_jit_protect();
5084 SHM_PROTECT();
5085 zend_shared_alloc_unlock();
5086
5087 zend_jit_profile_counter = 0;
5088 }
5089 }
5090
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)5091 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
5092 {
5093 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
5094
5095 if (!func_info) {
5096 return;
5097 }
5098
5099 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
5100 zend_jit_restart_hot_trace_counters(op_array);
5101 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
5102 zend_jit_restart_hot_counters(op_array);
5103 #if 0
5104 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
5105 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
5106 zend_op *opline = op_array->opcodes;
5107 zend_jit_op_array_extension *jit_extension =
5108 (zend_jit_op_array_extension*)func_info;
5109
5110 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
5111 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
5112 opline++;
5113 }
5114 }
5115 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
5116 opline->handler = (const void*)zend_jit_runtime_jit_handler;
5117 } else {
5118 opline->handler = (const void*)zend_jit_profile_jit_handler;
5119 }
5120 #endif
5121 }
5122 }
5123
zend_jit_restart_preloaded_script(zend_persistent_script * script)5124 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
5125 {
5126 zend_class_entry *ce;
5127 zend_op_array *op_array;
5128
5129 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
5130
5131 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
5132 zend_jit_restart_preloaded_op_array(op_array);
5133 } ZEND_HASH_FOREACH_END();
5134
5135 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
5136 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
5137 if (op_array->type == ZEND_USER_FUNCTION) {
5138 zend_jit_restart_preloaded_op_array(op_array);
5139 }
5140 } ZEND_HASH_FOREACH_END();
5141 } ZEND_HASH_FOREACH_END();
5142 }
5143
zend_jit_restart(void)5144 ZEND_EXT_API void zend_jit_restart(void)
5145 {
5146 if (dasm_buf) {
5147 zend_jit_unprotect();
5148
5149 /* restore JIT buffer pos */
5150 dasm_ptr[0] = dasm_ptr[1];
5151
5152 zend_jit_trace_restart();
5153
5154 if (ZCSG(preload_script)) {
5155 zend_jit_restart_preloaded_script(ZCSG(preload_script));
5156 if (ZCSG(saved_scripts)) {
5157 zend_persistent_script **p = ZCSG(saved_scripts);
5158
5159 while (*p) {
5160 zend_jit_restart_preloaded_script(*p);
5161 p++;
5162 }
5163 }
5164 }
5165
5166 zend_jit_protect();
5167 }
5168 }
5169
5170 #endif /* HAVE_JIT */
5171