1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * | Hao Sun <hao.sun@arm.com> | 18 * +----------------------------------------------------------------------+ 19 */ 20 21|.arch arm64 22 23|.define FP, x27 24|.define IP, x28 25|.define IPl, w28 26|.define RX, x28 // the same as VM IP reused as a general purpose reg 27|.define LR, x30 28|.define CARG1, x0 29|.define CARG2, x1 30|.define CARG3, x2 31|.define CARG4, x3 32|.define CARG5, x4 33|.define CARG6, x5 34|.define CARG1w, w0 35|.define CARG2w, w1 36|.define CARG3w, w2 37|.define CARG4w, w3 38|.define CARG5w, w4 39|.define CARG6w, w5 40|.define RETVALx, x0 41|.define RETVALw, w0 42|.define FCARG1x, x0 43|.define FCARG1w, w0 44|.define FCARG2x, x1 45|.define FCARG2w, w1 46|.define SPAD, 0x20 // padding for CPU stack alignment 47|.define NR_SPAD, 0x30 // padding for CPU stack alignment 48|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) 49|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) 50|.define T1, [sp, #0x10] 51 52// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. 53// Scratch registers 54|.define REG0, x8 55|.define REG0w, w8 56|.define REG1, x9 57|.define REG1w, w9 58|.define REG2, x10 59|.define REG2w, w10 60|.define FPR0, d0 61|.define FPR1, d1 62 63|.define ZREG_REG0, ZREG_X8 64|.define ZREG_REG1, ZREG_X9 65|.define ZREG_REG2, ZREG_X10 66|.define ZREG_FPR0, ZREG_V0 67|.define ZREG_FPR1, ZREG_V1 68 69// Temporaries, not preserved across calls 70|.define TMP1, x15 71|.define TMP1w, w15 72|.define TMP2, x16 73|.define TMP2w, w16 74|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 75|.define TMP3w, w17 76|.define FPTMP, d16 77 78|.define ZREG_TMP1, ZREG_X15 79|.define ZREG_TMP2, ZREG_X16 80|.define ZREG_TMP3, ZREG_X17 81|.define ZREG_FPTMP, ZREG_V16 82 83|.define HYBRID_SPAD, 32 // padding for stack alignment 84 85#define TMP_ZVAL_OFFSET 16 86#define DASM_ALIGNMENT 16 87 88const char* zend_reg_name[] = { 89 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", 90 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", 91 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", 92 "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", 93 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", 94 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15", 95 "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", 96 "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" 97}; 98 99#define ZREG_FCARG1 ZREG_X0 100#define ZREG_FCARG2 ZREG_X1 101 102|.type EX, zend_execute_data, FP 103|.type OP, zend_op 104|.type ZVAL, zval 105|.actionlist dasm_actions 106|.globals zend_lb 107|.section code, cold_code, jmp_table 108 109static void* dasm_labels[zend_lb_MAX]; 110 111#if ZTS 112static size_t tsrm_ls_cache_tcb_offset = 0; 113# ifdef __APPLE__ 114struct TLVDescriptor { 115 void* (*thunk)(struct TLVDescriptor*); 116 uint64_t key; 117 uint64_t offset; 118}; 119typedef struct TLVDescriptor TLVDescriptor; 120# endif 121#endif 122 123#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 124 125/* Encoding of immediate. */ 126#define MAX_IMM12 0xfff // maximum value for imm12 127#define MAX_IMM16 0xffff // maximum value for imm16 128#define MOVZ_IMM MAX_IMM16 // movz insn 129#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 130#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 131#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn 132 133#define B_IMM (1<<27) // signed imm26 * 4 134#define ADR_IMM (1<<20) // signed imm21 135#define ADRP_IMM (1LL<<32) // signed imm21 * 4096 136 137static bool arm64_may_use_b(const void *addr) 138{ 139 if (addr >= dasm_buf && addr < dasm_end) { 140 return (((char*)dasm_end - (char*)dasm_buf) < B_IMM); 141 } else if (addr >= dasm_end) { 142 return (((char*)addr - (char*)dasm_buf) < B_IMM); 143 } else if (addr < dasm_buf) { 144 return (((char*)dasm_end - (char*)addr) < B_IMM); 145 } 146 return 0; 147} 148 149static bool arm64_may_use_adr(const void *addr) 150{ 151 if (addr >= dasm_buf && addr < dasm_end) { 152 return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM); 153 } else if (addr >= dasm_end) { 154 return (((char*)addr - (char*)dasm_buf) < ADR_IMM); 155 } else if (addr < dasm_buf) { 156 return (((char*)dasm_end - (char*)addr) < ADR_IMM); 157 } 158 return 0; 159} 160 161static bool arm64_may_use_adrp(const void *addr) 162{ 163 if (addr >= dasm_buf && addr < dasm_end) { 164 return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM); 165 } else if (addr >= dasm_end) { 166 return (((char*)addr - (char*)dasm_buf) < ADRP_IMM); 167 } else if (addr < dasm_buf) { 168 return (((char*)dasm_end - (char*)addr) < ADRP_IMM); 169 } 170 return 0; 171} 172 173/* Determine whether "val" falls into two allowed ranges: 174 * Range 1: [0, 0xfff] 175 * Range 2: LSL #12 to Range 1 176 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */ 177static bool arm64_may_encode_imm12(const int64_t val) 178{ 179 return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff))); 180} 181 182/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */ 183static bool logical_immediate_p(uint64_t value, uint32_t reg_size) 184{ 185 /* fast path: power of two */ 186 if (value > 0 && !(value & (value - 1))) { 187 return true; 188 } 189 190 if (reg_size == 32) { 191 if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) { 192 return true; 193 } 194 } else if (reg_size == 64) { 195 if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) { 196 return true; 197 } 198 } else { 199 ZEND_UNREACHABLE(); 200 } 201 202 return false; 203} 204 205/* Not Implemented Yet */ 206|.macro NIY 207|| //ZEND_ASSERT(0); 208| brk #0 209|.endmacro 210 211|.macro NIY_STUB 212|| //ZEND_ASSERT(0); 213| brk #0 214|.endmacro 215 216|.macro ADD_HYBRID_SPAD 217||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 218| add sp, sp, # HYBRID_SPAD 219||#endif 220|.endmacro 221 222|.macro SUB_HYBRID_SPAD 223||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 224| sub sp, sp, # HYBRID_SPAD 225||#endif 226|.endmacro 227 228/* Move address into register. TODO: Support 52-bit address */ 229|.macro LOAD_ADDR, reg, addr 230| // 48-bit virtual address 231|| if (((uintptr_t)(addr)) == 0) { 232| mov reg, xzr 233|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { 234| movz reg, #((uint64_t)(addr)) 235|| } else if (arm64_may_use_adr((void*)(addr))) { 236| adr reg, &addr 237|| } else if (arm64_may_use_adrp((void*)(addr))) { 238| adrp reg, &(((uintptr_t)(addr))) 239|| if (((uintptr_t)(addr)) & 0xfff) { 240| add reg, reg, #(((uintptr_t)(addr)) & 0xfff) 241|| } 242|| } else if ((uintptr_t)(addr) & 0xffff) { 243| movz reg, #((uintptr_t)(addr) & 0xffff) 244|| if (((uintptr_t)(addr) >> 16) & 0xffff) { 245| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 246|| } 247|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 248| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 249|| } 250|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { 251| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 252|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 253| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 254|| } 255|| } else { 256| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 257|| } 258|.endmacro 259 260/* Move 32-bit immediate value into register. */ 261|.macro LOAD_32BIT_VAL, reg, val 262|| if (((uint32_t)(val)) <= MOVZ_IMM) { 263| movz reg, #((uint32_t)(val)) 264|| } else if (((uint32_t)(val) & 0xffff)) { 265| movz reg, #((uint32_t)(val) & 0xffff) 266|| if ((((uint32_t)(val) >> 16) & 0xffff)) { 267| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 268|| } 269|| } else { 270| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 271|| } 272|.endmacro 273 274/* Move 64-bit immediate value into register. */ 275|.macro LOAD_64BIT_VAL, reg, val 276|| if (((uint64_t)(val)) == 0) { 277| mov reg, xzr 278|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { 279| movz reg, #((uint64_t)(val)) 280|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { 281| movn reg, #(~((uint64_t)(val))) 282|| } else if ((uint64_t)(val) & 0xffff) { 283| movz reg, #((uint64_t)(val) & 0xffff) 284|| if (((uint64_t)(val) >> 16) & 0xffff) { 285| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 286|| } 287|| if (((uint64_t)(val) >> 32) & 0xffff) { 288| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 289|| } 290|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 291| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 292|| } 293|| } else if (((uint64_t)(val) >> 16) & 0xffff) { 294| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 295|| if (((uint64_t)(val) >> 32) & 0xffff) { 296| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 297|| } 298|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 299| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 300|| } 301|| } else if (((uint64_t)(val) >> 32) & 0xffff) { 302| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 303|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 304| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 305|| } 306|| } else { 307| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 308|| } 309|.endmacro 310 311/* Extract the low 8 bits from 'src_reg' into 'dst_reg'. 312 * Note: 0xff can be encoded as imm for 'and' instruction. */ 313|.macro GET_LOW_8BITS, dst_reg, src_reg 314| and dst_reg, src_reg, #0xff 315|.endmacro 316 317/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands. 318 * 32-bit and 64-bit registers are distinguished. */ 319|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 320|| if (val == 0) { 321| bw_ins dst_reg, src_reg1, wzr 322|| } else if (logical_immediate_p((uint32_t)val, 32)) { 323| bw_ins dst_reg, src_reg1, #val 324|| } else { 325| LOAD_32BIT_VAL tmp_reg, val 326| bw_ins dst_reg, src_reg1, tmp_reg 327|| } 328|.endmacro 329 330|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 331|| if (val == 0) { 332| bw_ins dst_reg, src_reg1, xzr 333|| } else if (logical_immediate_p(val, 64)) { 334| bw_ins dst_reg, src_reg1, #val 335|| } else { 336| LOAD_64BIT_VAL tmp_reg, val 337| bw_ins dst_reg, src_reg1, tmp_reg 338|| } 339|.endmacro 340 341/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */ 342|.macro TST_32_WITH_CONST, reg, val, tmp_reg 343|| if (val == 0) { 344| tst reg, wzr 345|| } else if (logical_immediate_p((uint32_t)val, 32)) { 346| tst reg, #val 347|| } else { 348| LOAD_32BIT_VAL tmp_reg, val 349| tst reg, tmp_reg 350|| } 351|.endmacro 352 353|.macro TST_64_WITH_CONST, reg, val, tmp_reg 354|| if (val == 0) { 355| tst reg, xzr 356|| } else if (logical_immediate_p(val, 64)) { 357| tst reg, #val 358|| } else { 359| LOAD_64BIT_VAL tmp_reg, val 360| tst reg, tmp_reg 361|| } 362|.endmacro 363 364/* Test bits between 64-bit register with constant 1. */ 365|.macro TST_64_WITH_ONE, reg 366| tst reg, #1 367|.endmacro 368 369/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished. 370 * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */ 371|.macro CMP_32_WITH_CONST, reg, val, tmp_reg 372|| if (val == 0) { 373| cmp reg, wzr 374|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 375| cmp reg, #val 376|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 377| cmn reg, #-val 378|| } else { 379| LOAD_32BIT_VAL tmp_reg, val 380| cmp reg, tmp_reg 381|| } 382|.endmacro 383 384|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg 385|| if (val == 0) { 386| cmp reg, xzr 387|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 388| cmp reg, #val 389|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 390| cmn reg, #-val 391|| } else { 392| LOAD_32BIT_VAL tmp_reg, val 393| cmp reg, tmp_reg 394|| } 395|.endmacro 396 397|.macro CMP_64_WITH_CONST, reg, val, tmp_reg 398|| if (val == 0) { 399| cmp reg, xzr 400|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 401| cmp reg, #val 402|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 403| cmn reg, #-val 404|| } else { 405| LOAD_64BIT_VAL tmp_reg, val 406| cmp reg, tmp_reg 407|| } 408|.endmacro 409 410/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs. 411 * 32-bit and 64-bit registers are distinguished. 412 * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */ 413|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 414|| if (val == 0) { 415| add_sub_ins dst_reg, src_reg1, wzr 416|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 417| add_sub_ins dst_reg, src_reg1, #val 418|| } else { 419| LOAD_32BIT_VAL tmp_reg, val 420| add_sub_ins dst_reg, src_reg1, tmp_reg 421|| } 422|.endmacro 423 424|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 425|| if (val == 0) { 426| add_sub_ins dst_reg, src_reg1, xzr 427|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 428| add_sub_ins dst_reg, src_reg1, #val 429|| } else { 430| LOAD_32BIT_VAL tmp_reg, val 431| add_sub_ins dst_reg, src_reg1, tmp_reg 432|| } 433|.endmacro 434 435|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 436|| if (val == 0) { 437| add_sub_ins dst_reg, src_reg1, xzr 438|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 439| add_sub_ins dst_reg, src_reg1, #val 440|| } else { 441| LOAD_64BIT_VAL tmp_reg, val 442| add_sub_ins dst_reg, src_reg1, tmp_reg 443|| } 444|.endmacro 445 446/* Memory access(load/store) with 32-bit 'offset'. 447 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. 448 * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished. 449 * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */ 450|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 451|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 452| LOAD_32BIT_VAL tmp_reg, offset 453| ldr_str_ins reg, [base_reg, tmp_reg] 454|| } else { 455| ldr_str_ins reg, [base_reg, #(offset)] 456|| } 457|.endmacro 458 459|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 460|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { 461| LOAD_32BIT_VAL tmp_reg, offset 462| ldr_str_ins reg, [base_reg, tmp_reg] 463|| } else { 464| ldr_str_ins reg, [base_reg, #(offset)] 465|| } 466|.endmacro 467 468|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg 469|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { 470| LOAD_32BIT_VAL tmp_reg, offset 471| ldrb_strb_ins reg, [base_reg, tmp_reg] 472|| } else { 473| ldrb_strb_ins reg, [base_reg, #(offset)] 474|| } 475|.endmacro 476 477/* Memory access(load/store) with 64-bit 'offset'. 478 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */ 479|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg 480|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 481| LOAD_64BIT_VAL tmp_reg, offset 482| ldr_str_ins reg, [base_reg, tmp_reg] 483|| } else { 484| ldr_str_ins reg, [base_reg, #(offset)] 485|| } 486|.endmacro 487 488/* ZTS: get thread local variable "_tsrm_ls_cache" */ 489|.macro LOAD_TSRM_CACHE, reg 490||#ifdef __APPLE__ 491| .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0 492| and TMP3, TMP3, #0xfffffffffffffff8 493| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1 494| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1 495||#else 496| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 497|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); 498| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] 499||#endif 500|.endmacro 501 502|.macro LOAD_ADDR_ZTS, reg, struct, field 503| .if ZTS 504| LOAD_TSRM_CACHE TMP3 505| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg 506| .else 507| LOAD_ADDR reg, &struct.field 508| .endif 509|.endmacro 510 511/* Store address 'addr' into memory 'mem'. */ 512|.macro ADDR_STORE, mem, addr, tmp_reg 513| LOAD_ADDR tmp_reg, addr 514| str tmp_reg, mem 515|.endmacro 516 517/* Store a register value 'reg' into memory 'addr'. 518 * For ZTS mode, base register with unsigned offset variant is used, 519 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 520|.macro MEM_STORE, str_ins, reg, addr, tmp_reg 521|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 522| adr tmp_reg, &addr 523| str_ins reg, [tmp_reg] 524|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 525| adrp tmp_reg, &(((uintptr_t)(addr))) 526| str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 527|| } else { 528| LOAD_ADDR tmp_reg, addr 529| str_ins reg, [tmp_reg] 530|| } 531|.endmacro 532 533|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg 534| .if ZTS 535| LOAD_TSRM_CACHE TMP3 536| MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 537| .else 538| MEM_STORE str_ins, reg, &struct.field, tmp_reg 539| .endif 540|.endmacro 541 542|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg 543| .if ZTS 544| LOAD_TSRM_CACHE TMP3 545| MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 546| .else 547| MEM_STORE str_ins, reg, &struct.field, tmp_reg 548| .endif 549|.endmacro 550 551|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg 552| .if ZTS 553| LOAD_TSRM_CACHE TMP3 554| MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 555| .else 556| MEM_STORE strb_ins, reg, &struct.field, tmp_reg 557| .endif 558|.endmacro 559 560/* Load value from memory 'addr' and write it into register 'reg'. 561 * For ZTS mode, base register with unsigned offset variant is used, 562 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 563|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg 564|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 565| adr tmp_reg, &addr 566| ldr_ins reg, [tmp_reg] 567|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 568| adrp tmp_reg, &(((uintptr_t)(addr))) 569| ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 570|| } else { 571| LOAD_ADDR tmp_reg, addr 572| ldr_ins reg, [tmp_reg] 573|| } 574|.endmacro 575 576|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg 577| .if ZTS 578| LOAD_TSRM_CACHE TMP3 579| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 580| .else 581| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 582| .endif 583|.endmacro 584 585|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg 586| .if ZTS 587| LOAD_TSRM_CACHE TMP3 588| MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 589| .else 590| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 591| .endif 592|.endmacro 593 594|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg 595| .if ZTS 596| LOAD_TSRM_CACHE TMP3 597| MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 598| .else 599| MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg 600| .endif 601|.endmacro 602 603/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg', 604 * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */ 605|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2 606| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 607| op_ins reg, reg, tmp_reg1 608|.endmacro 609 610|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2 611| .if ZTS 612| LOAD_TSRM_CACHE TMP3 613| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 614| op_ins reg, reg, tmp_reg2 615| .else 616| MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2 617| .endif 618|.endmacro 619 620/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation 621 * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value. 622 * Currently, only add instruction is used as 'op_ins'. 623 * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */ 624|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 625| LOAD_ADDR tmp_reg2, addr 626| ldr_ins, tmp_reg1, [tmp_reg2] 627| op_ins tmp_reg1, tmp_reg1, op 628| str_ins tmp_reg1, [tmp_reg2] 629|.endmacro 630 631|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 632| .if ZTS 633| LOAD_TSRM_CACHE TMP3 634|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { 635| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) 636| ldr_ins tmp_reg2, [TMP3, tmp_reg1] 637| op_ins tmp_reg2, tmp_reg2, op 638| str_ins tmp_reg2, [TMP3, tmp_reg1] 639|| } else { 640| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 641| op_ins tmp_reg2, tmp_reg2, op 642| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 643|| } 644| .else 645| MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 646| .endif 647|.endmacro 648 649|.macro EXT_CALL, func, tmp_reg 650|| if (arm64_may_use_b(func)) { 651| bl &func 652|| } else { 653| LOAD_ADDR tmp_reg, func 654| blr tmp_reg 655|| } 656|.endmacro 657 658|.macro EXT_JMP, func, tmp_reg 659|| if (arm64_may_use_b(func)) { 660| b &func 661|| } else { 662| LOAD_ADDR tmp_reg, func 663| br tmp_reg 664|| } 665|.endmacro 666 667|.macro SAVE_IP 668|| if (GCC_GLOBAL_REGS) { 669| str IP, EX->opline 670|| } 671|.endmacro 672 673|.macro LOAD_IP 674|| if (GCC_GLOBAL_REGS) { 675| ldr IP, EX->opline 676|| } 677|.endmacro 678 679|.macro LOAD_IP_ADDR, addr 680|| if (GCC_GLOBAL_REGS) { 681| LOAD_ADDR IP, addr 682|| } else { 683| ADDR_STORE EX->opline, addr, RX 684|| } 685|.endmacro 686 687|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg 688| .if ZTS 689|| if (GCC_GLOBAL_REGS) { 690| LOAD_TSRM_CACHE IP 691| MEM_ACCESS_64_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 692|| } else { 693| LOAD_TSRM_CACHE RX 694| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 695| str RX, EX->opline 696|| } 697| .else 698| LOAD_IP_ADDR &struct.field 699| .endif 700|.endmacro 701 702|.macro GET_IP, reg 703|| if (GCC_GLOBAL_REGS) { 704| mov reg, IP 705|| } else { 706| ldr reg, EX->opline 707|| } 708|.endmacro 709 710/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */ 711|.macro ADD_IP, reg, tmp_reg 712|| if (GCC_GLOBAL_REGS) { 713| add IP, IP, reg 714|| } else { 715| ldr tmp_reg, EX->opline 716| add tmp_reg, tmp_reg, reg 717| str tmp_reg, EX->opline 718|| } 719|.endmacro 720 721|.macro ADD_IP_SHIFT, reg, shift, tmp_reg 722|| if (GCC_GLOBAL_REGS) { 723| add IP, IP, reg, shift 724|| } else { 725| ldr tmp_reg, EX->opline 726| add tmp_reg, tmp_reg, reg, shift 727| str tmp_reg, EX->opline 728|| } 729|.endmacro 730 731/* Update IP with 32-bit immediate 'val'. */ 732|.macro ADD_IP_WITH_CONST, val, tmp_reg 733|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 734|| if (GCC_GLOBAL_REGS) { 735| add IP, IP, #val 736|| } else { 737| ldr tmp_reg, EX->opline 738| add tmp_reg, tmp_reg, #val 739| str tmp_reg, EX->opline 740|| } 741|.endmacro 742 743|.macro JMP_IP, tmp_reg 744|| if (GCC_GLOBAL_REGS) { 745| ldr tmp_reg, [IP] 746| br tmp_reg 747|| } else { 748| ldr tmp_reg, EX:CARG1->opline 749| br tmp_reg 750|| } 751|.endmacro 752 753|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 754| LOAD_ADDR tmp_reg1, addr 755|| if (GCC_GLOBAL_REGS) { 756| cmp IP, tmp_reg1 757|| } else { 758| ldr tmp_reg2, EX->opline 759| cmp tmp_reg2, tmp_reg1 760|| } 761|.endmacro 762 763|.macro LOAD_ZVAL_ADDR, reg, addr 764|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 765| LOAD_ADDR reg, Z_ZV(addr) 766|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 767|| if (Z_OFFSET(addr)) { 768| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg 769|| } else { 770|| if (Z_REG(addr) == ZREG_RSP) { 771| mov reg, sp 772|| } else { 773| mov reg, Rx(Z_REG(addr)) 774|| } 775|| } 776|| } else { 777|| ZEND_UNREACHABLE(); 778|| } 779|.endmacro 780 781|.macro GET_Z_TYPE_INFO, reg, zv 782| ldr reg, [zv, #offsetof(zval,u1.type_info)] 783|.endmacro 784 785|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg 786| LOAD_32BIT_VAL tmp_reg, type 787| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] 788|.endmacro 789 790|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg 791|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 792| MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg 793|.endmacro 794 795|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg 796|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 797| MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 798|.endmacro 799 800|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 801|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 802| LOAD_32BIT_VAL tmp_reg1, type 803| MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 804|.endmacro 805 806|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg 807|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 808| MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 809|.endmacro 810 811|.macro GET_Z_PTR, reg, zv 812| ldr reg, [zv] 813|.endmacro 814 815|.macro GET_ZVAL_PTR, reg, addr, tmp_reg 816|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 817| MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 818|.endmacro 819 820|.macro SET_ZVAL_PTR, addr, reg, tmp_reg 821|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 822| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 823|.endmacro 824 825|.macro UNDEF_OPLINE_RESULT, tmp_reg 826| ldr REG0, EX->opline 827| ldr REG0w, OP:REG0->result.var 828| add REG0, FP, REG0 829| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg 830|.endmacro 831 832|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2 833| ldrb tmp_reg1, OP:RX->result_type 834| TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2 835| beq >1 836| ldr REG0w, OP:RX->result.var 837| add REG0, FP, REG0 838| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1 839|1: 840|.endmacro 841 842/* Floating-point comparison between register 'reg' and value from memory 'addr'. 843 * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */ 844|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg 845|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 846| MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 847| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 848|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 849| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 850| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 851|| } else if (Z_MODE(addr) == IS_REG) { 852| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 853|| } else { 854|| ZEND_UNREACHABLE(); 855|| } 856|.endmacro 857 858/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'. 859 * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */ 860|.macro DOUBLE_GET_LONG, reg, val, tmp_reg 861|| if (val == 0) { 862| fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 863|| } else { 864| LOAD_64BIT_VAL Rx(tmp_reg), val 865| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) 866|| } 867|.endmacro 868 869/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'. 870 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */ 871|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 872|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 873| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 874|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 875| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) 876| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) 877|| } else if (Z_MODE(addr) == IS_REG) { 878| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) 879|| } else { 880|| ZEND_UNREACHABLE(); 881|| } 882|.endmacro 883 884/* Floating-point arithmetic operation between two FP registers. 885 * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */ 886|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg 887|| switch (opcode) { 888|| case ZEND_ADD: 889| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 890|| break; 891|| case ZEND_SUB: 892| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 893|| break; 894|| case ZEND_MUL: 895| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 896|| break; 897|| case ZEND_DIV: 898| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 899|| break; 900|| } 901|.endmacro 902 903/* Conduct binary operation between register 'reg' and value from memory 'addr', 904 * and the computation result is stored in 'reg'. 905 * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor. 906 * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected. 907 * Note: the equivalent macro in JIT/x86 is LONG_OP. */ 908|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg 909|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 910| ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 911|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 912| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 913| add_sub_ins Rx(reg), Rx(reg), tmp_reg 914|| } else if (Z_MODE(addr) == IS_REG) { 915| add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 916|| } else { 917|| ZEND_UNREACHABLE(); 918|| } 919|.endmacro 920 921|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg 922|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 923| BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 924|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 925| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 926| bw_ins Rx(reg), Rx(reg), tmp_reg 927|| } else if (Z_MODE(addr) == IS_REG) { 928| bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 929|| } else { 930|| ZEND_UNREACHABLE(); 931|| } 932|.endmacro 933 934|.macro LONG_CMP, reg, addr, tmp_reg 935|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 936| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 937|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 938| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 939| cmp Rx(reg), tmp_reg 940|| } else if (Z_MODE(addr) == IS_REG) { 941| cmp Rx(reg), Rx(Z_REG(addr)) 942|| } else { 943|| ZEND_UNREACHABLE(); 944|| } 945|.endmacro 946 947/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and 948 * the computation result is stored back into 'addr'. 949 * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */ 950|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2 951|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 952|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 953|| if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) { 954| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr) 955| ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 956| add_sub_ins tmp_reg1, tmp_reg1, #val 957| str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 958|| } else { 959| ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 960| add_sub_ins tmp_reg1, tmp_reg1, #val 961| str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 962|| } 963|| } else if (Z_MODE(addr) == IS_REG) { 964| add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val 965|| } else { 966|| ZEND_UNREACHABLE(); 967|| } 968|.endmacro 969 970/* Compare value from memory 'addr' with immediate value 'val'. 971 * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */ 972|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2 973|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 974| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 975| CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2 976|| } else if (Z_MODE(addr) == IS_REG) { 977| CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1 978|| } else { 979|| ZEND_UNREACHABLE(); 980|| } 981|.endmacro 982 983|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg 984|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 985|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 986| mov Rx(reg), xzr 987|| } else { 988| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) 989|| } 990|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 991| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 992|| } else if (Z_MODE(addr) == IS_REG) { 993|| if (reg != Z_REG(addr)) { 994| mov Rx(reg), Rx(Z_REG(addr)) 995|| } 996|| } else { 997|| ZEND_UNREACHABLE(); 998|| } 999|.endmacro 1000 1001|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 1002|| switch (opcode) { 1003|| case ZEND_ADD: 1004| LONG_ADD_SUB adds, reg, addr, tmp_reg1 1005|| break; 1006|| case ZEND_SUB: 1007| LONG_ADD_SUB subs, reg, addr, tmp_reg1 1008|| break; 1009|| case ZEND_BW_OR: 1010| LONG_BW_OP orr, reg, addr, tmp_reg1 1011|| break; 1012|| case ZEND_BW_AND: 1013| LONG_BW_OP and, reg, addr, tmp_reg1 1014|| break; 1015|| case ZEND_BW_XOR: 1016| LONG_BW_OP eor, reg, addr, tmp_reg1 1017|| break; 1018|| default: 1019|| ZEND_UNREACHABLE(); 1020|| } 1021|.endmacro 1022 1023|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 1024|| switch (opcode) { 1025|| case ZEND_ADD: 1026| adds dst_reg, src_reg1, src_reg2 1027|| break; 1028|| case ZEND_SUB: 1029| subs dst_reg, src_reg1, src_reg2 1030|| break; 1031|| case ZEND_BW_OR: 1032| orr dst_reg, src_reg1, src_reg2 1033|| break; 1034|| case ZEND_BW_AND: 1035| and dst_reg, src_reg1, src_reg2 1036|| break; 1037|| case ZEND_BW_XOR: 1038| eor dst_reg, src_reg1, src_reg2 1039|| break; 1040|| default: 1041|| ZEND_UNREACHABLE(); 1042|| } 1043|.endmacro 1044 1045/* Store LONG value into memory 'addr'. 1046 * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or 1047 * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */ 1048|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg 1049|| if (Z_MODE(addr) == IS_REG) { 1050| mov Rx(Z_REG(addr)), reg 1051|| } else { 1052|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1053| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1054|| } 1055|.endmacro 1056 1057|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2 1058|| if (val == 0) { 1059| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 1060|| } else { 1061| LOAD_64BIT_VAL tmp_reg1, val 1062| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 1063|| } 1064|.endmacro 1065 1066/* Store DOUBLE value from FP register 'reg' into memory 'addr'. 1067 * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */ 1068|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg 1069|| if (Z_MODE(addr) == IS_REG) { 1070|| if (reg != Z_REG(addr)) { 1071| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) 1072|| } 1073|| } else { 1074|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1075| MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1076|| } 1077|.endmacro 1078 1079/* Load DOUBLE value from memory 'addr' into FP register 'reg'. 1080 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */ 1081|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg 1082|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 1083|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1084| MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 1085|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1086| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1087|| } else if (Z_MODE(addr) == IS_REG) { 1088| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 1089|| } else { 1090|| ZEND_UNREACHABLE(); 1091|| } 1092|| } 1093|.endmacro 1094 1095|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1096|| if (Z_TYPE_P(zv) > IS_TRUE) { 1097|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1098|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1099| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1100| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1101|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1102|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1103| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 1104| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1105|| } else { 1106| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. 1107| // Note that imm32 is signed extended to 64 bits during mov. 1108| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are 1109| // needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue. 1110| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1111|| } 1112|| } 1113|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1114|| if (dst_def_info == MAY_BE_DOUBLE) { 1115|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1116| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1117|| } 1118|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1119| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1120|| } 1121|| } 1122|.endmacro 1123 1124|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1125|| if (Z_TYPE_P(zv) > IS_TRUE) { 1126|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1127|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 1128|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg); 1129| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1130| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1131| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 1132|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1133|| if (Z_MODE(dst_addr) == IS_REG) { 1134| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 1135| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 1136|| } else if (Z_MODE(res_addr) == IS_REG) { 1137| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 1138| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 1139|| } else { 1140| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 1141| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 1142| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 1143|| } 1144|| } else { 1145|| if (Z_MODE(dst_addr) == IS_REG) { 1146| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1147| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) 1148|| } else if (Z_MODE(res_addr) == IS_REG) { 1149| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1150| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) 1151|| } else { 1152| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1153| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1154|| } 1155|| } 1156|| } 1157|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1158|| if (dst_def_info == MAY_BE_DOUBLE) { 1159|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1160| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1161|| } 1162|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1163| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1164|| } 1165|| } 1166|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1167|| if (dst_def_info == MAY_BE_DOUBLE) { 1168| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1169|| } else { 1170| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1171|| } 1172|| } 1173|.endmacro 1174 1175// the same as above, but "src" may overlap with "reg1" 1176// Useful info would be stored into reg1 and reg2, and they might be used afterward. 1177|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg 1178| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg 1179|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1180|| !(src_info & MAY_BE_GUARD) && 1181|| has_concrete_type(src_info & MAY_BE_ANY)) { 1182|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1183|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1184|| uint32_t type = concrete_type(src_info); 1185| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) 1186|| } 1187|| } 1188|| } else { 1189| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1) 1190| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1) 1191|| } 1192|.endmacro 1193 1194|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg 1195|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1196|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1197|| if (Z_MODE(src_addr) == IS_REG) { 1198|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1199| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1200|| } 1201|| } else if (Z_MODE(dst_addr) == IS_REG) { 1202| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1203|| } else { 1204| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1205| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1206|| } 1207|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1208|| if (Z_MODE(src_addr) == IS_REG) { 1209| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1210|| } else if (Z_MODE(dst_addr) == IS_REG) { 1211| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1212|| } else { 1213| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1214| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1215|| } 1216|| // Combine the following two branches. 1217|| // } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1218|| } else { 1219| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1220| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1221|| } 1222|| } 1223|.endmacro 1224 1225|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg 1226|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1227|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1228|| if (Z_MODE(src_addr) == IS_REG) { 1229|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1230| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1231|| } 1232|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1233| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1234|| } 1235|| } else if (Z_MODE(dst_addr) == IS_REG) { 1236| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1237|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1238| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) 1239|| } 1240|| } else if (Z_MODE(res_addr) == IS_REG) { 1241| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) 1242| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) 1243|| } else { 1244| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1245| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1246| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) 1247|| } 1248|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1249|| if (Z_MODE(src_addr) == IS_REG) { 1250| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1251| SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg 1252|| } else if (Z_MODE(dst_addr) == IS_REG) { 1253| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1254| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg 1255|| } else if (Z_MODE(res_addr) == IS_REG) { 1256| GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg 1257| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg 1258|| } else { 1259| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1260| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1261| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg 1262|| } 1263|| } else { 1264| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1265| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1266| SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg) 1267|| } 1268|| } 1269|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1270|| has_concrete_type(src_info & MAY_BE_ANY)) { 1271|| uint32_t type = concrete_type(src_info); 1272|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1273|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1274| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1275|| } 1276|| } 1277|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1278| SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1279|| } 1280|| } else { 1281| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg) 1282| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg) 1283| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg) 1284|| } 1285|.endmacro 1286 1287|.macro IF_UNDEF, type_reg, label 1288| cbz type_reg, label 1289|.endmacro 1290 1291|.macro IF_TYPE, type, val, label 1292|| if (val == 0) { 1293| cbz type, label 1294|| } else { 1295| cmp type, #val 1296| beq label 1297|| } 1298|.endmacro 1299 1300|.macro IF_NOT_TYPE, type, val, label 1301|| if (val == 0) { 1302| cbnz type, label 1303|| } else { 1304| cmp type, #val 1305| bne label 1306|| } 1307|.endmacro 1308 1309|.macro IF_Z_TYPE, zv, val, label, tmp_reg 1310| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1311| IF_TYPE tmp_reg, val, label 1312|.endmacro 1313 1314|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg 1315| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1316| IF_NOT_TYPE tmp_reg, val, label 1317|.endmacro 1318 1319|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg 1320|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1321| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1322| cmp Rw(tmp_reg), #val 1323|.endmacro 1324 1325|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg 1326|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1327| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1328| IF_TYPE Rw(tmp_reg), val, label 1329|.endmacro 1330 1331|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg 1332|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1333| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1334| IF_NOT_TYPE Rw(tmp_reg), val, label 1335|.endmacro 1336 1337|.macro IF_FLAGS, type_flags, mask, label, tmp_reg 1338| TST_32_WITH_CONST type_flags, mask, tmp_reg 1339| bne label 1340|.endmacro 1341 1342|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg 1343| TST_32_WITH_CONST type_flags, mask, tmp_reg 1344| beq label 1345|.endmacro 1346 1347|.macro IF_REFCOUNTED, type_flags, label, tmp_reg 1348| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1349| bne label 1350|.endmacro 1351 1352|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg 1353| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1354| beq label 1355|.endmacro 1356 1357|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1358|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1359| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1360| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1361|.endmacro 1362 1363|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1364|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1365| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1366| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1367|.endmacro 1368 1369|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1370| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1371|.endmacro 1372 1373|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1374| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1375|.endmacro 1376 1377|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2 1378| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2 1379|.endmacro 1380 1381|.macro GC_ADDREF, zv, tmp_reg 1382| ldr tmp_reg, [zv] 1383| add tmp_reg, tmp_reg, #1 1384| str tmp_reg, [zv] 1385|.endmacro 1386 1387|.macro GC_ADDREF_2, zv, tmp_reg 1388| ldr tmp_reg, [zv] 1389| add tmp_reg, tmp_reg, #2 1390| str tmp_reg, [zv] 1391|.endmacro 1392 1393|.macro GC_DELREF, zv, tmp_reg 1394| ldr tmp_reg, [zv] 1395| subs tmp_reg, tmp_reg, #1 1396| str tmp_reg, [zv] 1397|.endmacro 1398 1399|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 1400| ldr tmp_reg1, [ptr, #4] 1401| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2 1402| bne label 1403|.endmacro 1404 1405|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2 1406| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1407| ldr tmp_reg2, [tmp_reg1] 1408| add tmp_reg2, tmp_reg2, #1 1409| str tmp_reg2, [tmp_reg1] 1410|.endmacro 1411 1412|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2 1413| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1414| ldr tmp_reg2, [tmp_reg1] 1415| add tmp_reg2, tmp_reg2, #2 1416| str tmp_reg2, [tmp_reg1] 1417|.endmacro 1418 1419|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1420|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1421|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1422| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1423|| } 1424| GC_ADDREF value_ptr_reg, tmp_reg 1425|1: 1426|| } 1427|.endmacro 1428 1429|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1430|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1431|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1432| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1433|| } 1434| ldr tmp_reg, [value_ptr_reg] 1435| add tmp_reg, tmp_reg, #2 1436| str tmp_reg, [value_ptr_reg] 1437|1: 1438|| } 1439|.endmacro 1440 1441|.macro ZVAL_DEREF, reg, info, tmp_reg 1442|| if (info & MAY_BE_REF) { 1443| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg 1444| GET_Z_PTR reg, reg 1445| add reg, reg, #offsetof(zend_reference, val) 1446|1: 1447|| } 1448|.endmacro 1449 1450|.macro SET_EX_OPLINE, op, tmp_reg 1451|| if (op == last_valid_opline) { 1452|| zend_jit_use_last_valid_opline(); 1453| SAVE_IP 1454|| } else { 1455| ADDR_STORE EX->opline, op, tmp_reg 1456|| if (!GCC_GLOBAL_REGS) { 1457|| zend_jit_reset_last_valid_opline(); 1458|| } 1459|| } 1460|.endmacro 1461 1462// arg1 "zval" should be in FCARG1x 1463|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg 1464|| do { 1465|| if (!((var_info) & MAY_BE_GUARD) 1466|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1467|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1468|| if (type == IS_STRING && !ZEND_DEBUG) { 1469| EXT_CALL _efree, tmp_reg 1470|| break; 1471|| } else if (type == IS_ARRAY) { 1472|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1473|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1474| SET_EX_OPLINE opline, tmp_reg 1475|| } 1476| EXT_CALL zend_array_destroy, tmp_reg 1477|| } else { 1478| EXT_CALL zend_jit_array_free, tmp_reg 1479|| } 1480|| break; 1481|| } else if (type == IS_OBJECT) { 1482|| if (opline) { 1483| SET_EX_OPLINE opline, REG0 1484|| } 1485| EXT_CALL zend_objects_store_del, tmp_reg 1486|| break; 1487|| } 1488|| } 1489|| if (opline) { 1490| SET_EX_OPLINE opline, tmp_reg 1491|| } 1492| EXT_CALL rc_dtor_func, tmp_reg 1493|| } while(0); 1494|.endmacro 1495 1496|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 1497|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1498|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1499| // if (Z_REFCOUNTED_P(cv)) { 1500|| if (cold) { 1501| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1502|.cold_code 1503|1: 1504|| } else { 1505| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 1506|| } 1507|| } 1508| // if (!Z_DELREF_P(cv)) { 1509| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) 1510| GC_DELREF FCARG1x, Rw(tmp_reg1) 1511|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1512|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1513|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1514| bne >3 1515|| } else { 1516| bne >4 1517|| } 1518|| } 1519| // zval_dtor_func(r); 1520| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) 1521|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1522| b >4 1523|| } 1524|3: 1525|| } 1526|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1527|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1528|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1529| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 1530| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 1531| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) 1532|1: 1533|| } 1534| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) 1535| // gc_possible_root(Z_COUNTED_P(z)) 1536| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1537|| } 1538|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1539| b >4 1540|.code 1541|| } 1542|4: 1543|| } 1544|.endmacro 1545 1546|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 1547|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1548|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 1549| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 1550|| } 1551|.endmacro 1552 1553|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 1554|| if (RC_MAY_BE_N(op_info)) { 1555|| if (Z_REG(addr) != ZREG_FP) { 1556| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) 1557|| if (RC_MAY_BE_1(op_info)) { 1558| // if (GC_REFCOUNT() > 1) 1559| ldr Rw(tmp_reg1), [REG0] 1560| cmp Rw(tmp_reg1), #1 1561| bls >2 1562|| } 1563|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1564| LOAD_ZVAL_ADDR FCARG1x, addr 1565|| } 1566| EXT_CALL zend_jit_zval_array_dup, REG0 1567| mov REG0, RETVALx 1568|2: 1569| mov FCARG1x, REG0 1570|| } else { 1571| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1572|| if (RC_MAY_BE_1(op_info)) { 1573| // if (GC_REFCOUNT() > 1) 1574| ldr Rw(tmp_reg1), [FCARG1x] 1575| cmp Rw(tmp_reg1), #1 1576|| if (cold) { 1577| bhi >1 1578|.cold_code 1579|1: 1580|| } else { 1581| bls >2 1582|| } 1583|| } 1584| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1585| GC_DELREF FCARG1x, Rw(tmp_reg1) 1586|1: 1587| EXT_CALL zend_array_dup, REG0 1588| mov REG0, RETVALx 1589| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) 1590| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) 1591| mov FCARG1x, REG0 1592|| if (RC_MAY_BE_1(op_info)) { 1593|| if (cold) { 1594| b >2 1595|.code 1596|| } 1597|| } 1598|2: 1599|| } 1600|| } else { 1601| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1602|| } 1603|.endmacro 1604 1605/* argument is passed in FCARG1x */ 1606|.macro EFREE_REFERENCE 1607||#if ZEND_DEBUG 1608| mov FCARG2x, xzr // filename 1609| mov CARG3w, wzr // lineno 1610| mov CARG4, xzr 1611| mov CARG5, xzr 1612| EXT_CALL _efree, REG0 1613||#else 1614||#ifdef HAVE_BUILTIN_CONSTANT_P 1615| EXT_CALL _efree_32, REG0 1616||#else 1617| EXT_CALL _efree, REG0 1618||#endif 1619||#endif 1620|.endmacro 1621 1622|.macro EMALLOC, size, op_array, opline 1623||#if ZEND_DEBUG 1624|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1625| mov FCARG1x, #size 1626| LOAD_ADDR FCARG2x, filename 1627| LOAD_32BIT_VAL CARG3w, opline->lineno 1628| mov CARG4, xzr 1629| mov CARG5, xzr 1630| EXT_CALL _emalloc, REG0 1631| mov REG0, RETVALx 1632||#else 1633||#ifdef HAVE_BUILTIN_CONSTANT_P 1634|| if (size > 24 && size <= 32) { 1635| EXT_CALL _emalloc_32, REG0 1636| mov REG0, RETVALx 1637|| } else { 1638| mov FCARG1x, #size 1639| EXT_CALL _emalloc, REG0 1640| mov REG0, RETVALx 1641|| } 1642||#else 1643| mov FCARG1x, #size 1644| EXT_CALL _emalloc, REG0 1645| mov REG0, RETVALx 1646||#endif 1647||#endif 1648|.endmacro 1649 1650|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 1651| GC_DELREF Rx(reg), Rw(tmp_reg1) 1652| bne >1 1653| // zend_objects_store_del(obj); 1654|| if (reg != ZREG_FCARG1) { 1655| mov FCARG1x, Rx(reg) 1656|| } 1657| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) 1658| b exit_label 1659|1: 1660| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) 1661| // gc_possible_root(obj) 1662|| if (reg != ZREG_FCARG1) { 1663| mov FCARG1x, Rx(reg) 1664|| } 1665| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1666|1: 1667|.endmacro 1668 1669|.macro UNDEFINED_OFFSET, opline 1670|| if (opline == last_valid_opline) { 1671|| zend_jit_use_last_valid_opline(); 1672| bl ->undefined_offset_ex 1673|| } else { 1674| SET_EX_OPLINE opline, REG0 1675| bl ->undefined_offset 1676|| } 1677|.endmacro 1678 1679|.macro UNDEFINED_INDEX, opline 1680|| if (opline == last_valid_opline) { 1681|| zend_jit_use_last_valid_opline(); 1682| bl ->undefined_index_ex 1683|| } else { 1684| SET_EX_OPLINE opline, REG0 1685| bl ->undefined_index 1686|| } 1687|.endmacro 1688 1689|.macro CANNOT_ADD_ELEMENT, opline 1690|| if (opline == last_valid_opline) { 1691|| zend_jit_use_last_valid_opline(); 1692| bl ->cannot_add_element_ex 1693|| } else { 1694| SET_EX_OPLINE opline, REG0 1695| bl ->cannot_add_element 1696|| } 1697|.endmacro 1698 1699static bool reuse_ip = 0; 1700static bool delayed_call_chain = 0; 1701static uint32_t delayed_call_level = 0; 1702static const zend_op *last_valid_opline = NULL; 1703static bool use_last_vald_opline = 0; 1704static bool track_last_valid_opline = 0; 1705static int jit_return_label = -1; 1706static uint32_t current_trace_num = 0; 1707static uint32_t allowed_opt_flags = 0; 1708 1709static void zend_jit_track_last_valid_opline(void) 1710{ 1711 use_last_vald_opline = 0; 1712 track_last_valid_opline = 1; 1713} 1714 1715static void zend_jit_use_last_valid_opline(void) 1716{ 1717 if (track_last_valid_opline) { 1718 use_last_vald_opline = 1; 1719 track_last_valid_opline = 0; 1720 } 1721} 1722 1723static bool zend_jit_trace_uses_initial_ip(void) 1724{ 1725 return use_last_vald_opline; 1726} 1727 1728static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1729{ 1730 if (!reuse_ip) { 1731 track_last_valid_opline = 0; 1732 last_valid_opline = target_opline; 1733 } 1734} 1735 1736static void zend_jit_reset_last_valid_opline(void) 1737{ 1738 track_last_valid_opline = 0; 1739 last_valid_opline = NULL; 1740} 1741 1742static void zend_jit_start_reuse_ip(void) 1743{ 1744 zend_jit_reset_last_valid_opline(); 1745 reuse_ip = 1; 1746} 1747 1748static int zend_jit_reuse_ip(dasm_State **Dst) 1749{ 1750 if (!reuse_ip) { 1751 zend_jit_start_reuse_ip(); 1752 | // call = EX(call); 1753 | ldr RX, EX->call 1754 } 1755 return 1; 1756} 1757 1758static void zend_jit_stop_reuse_ip(void) 1759{ 1760 reuse_ip = 0; 1761} 1762 1763static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1764{ 1765 |->interrupt_handler: 1766 | SAVE_IP 1767 | //EG(vm_interrupt) = 0; 1768 | MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 1769 | //if (EG(timed_out)) { 1770 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1 1771 | cbz TMP1w, >1 1772 | //zend_timeout(); 1773 | EXT_CALL zend_timeout, TMP1 1774 |1: 1775 | //} else if (zend_interrupt_function) { 1776 if (zend_interrupt_function) { 1777 | //zend_interrupt_function(execute_data); 1778 | mov CARG1, FP 1779 | EXT_CALL zend_interrupt_function, TMP1 1780 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 1781 | cbz REG0, >1 1782 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1 1783 |1: 1784 | //ZEND_VM_ENTER(); 1785 | //execute_data = EG(current_execute_data); 1786 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 1787 | LOAD_IP 1788 } 1789 | //ZEND_VM_CONTINUE() 1790 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1791 | ADD_HYBRID_SPAD 1792 | JMP_IP TMP1 1793 } else if (GCC_GLOBAL_REGS) { 1794 | ldp x29, x30, [sp], # SPAD // stack alignment 1795 | JMP_IP TMP1 1796 } else { 1797 | ldp FP, RX, T2 // retore FP and IP 1798 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1799 | mov RETVALx, #1 // ZEND_VM_ENTER 1800 | ret 1801 } 1802 1803 return 1; 1804} 1805 1806static int zend_jit_exception_handler_stub(dasm_State **Dst) 1807{ 1808 |->exception_handler: 1809 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1810 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1811 1812 | ADD_HYBRID_SPAD 1813 | EXT_CALL handler, REG0 1814 | JMP_IP TMP1 1815 } else { 1816 const void *handler = EG(exception_op)->handler; 1817 1818 if (GCC_GLOBAL_REGS) { 1819 | ldp x29, x30, [sp], # SPAD // stack alignment 1820 | EXT_JMP handler, REG0 1821 } else { 1822 | mov FCARG1x, FP 1823 | EXT_CALL handler, REG0 1824 | ldp FP, RX, T2 // retore FP and IP 1825 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1826 | tbnz RETVALw, #31, >1 1827 | mov RETVALw, #1 // ZEND_VM_ENTER 1828 |1: 1829 | ret 1830 } 1831 } 1832 1833 return 1; 1834} 1835 1836static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1837{ 1838 |->exception_handler_undef: 1839 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 1840 | ldrb TMP1w, OP:REG0->result_type 1841 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1842 | beq >1 1843 | ldr REG0w, OP:REG0->result.var 1844 | add REG0, REG0, FP 1845 | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w 1846 |1: 1847 | b ->exception_handler 1848 1849 return 1; 1850} 1851 1852static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1853{ 1854 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1855 1856 |->exception_handler_free_op1_op2: 1857 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1858 | ldrb TMP1w, OP:RX->op1_type 1859 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1860 | beq >9 1861 | ldr REG0w, OP:RX->op1.var 1862 | add REG0, REG0, FP 1863 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1864 |9: 1865 | ldrb TMP1w, OP:RX->op2_type 1866 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1867 | beq >9 1868 | ldr REG0w, OP:RX->op2.var 1869 | add REG0, REG0, FP 1870 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1871 |9: 1872 | b ->exception_handler 1873 return 1; 1874} 1875 1876static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1877{ 1878 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1879 1880 |->exception_handler_free_op2: 1881 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0 1882 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1883 | ldrb TMP1w, OP:RX->op2_type 1884 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1885 | beq >9 1886 | ldr REG0w, OP:RX->op2.var 1887 | add REG0, REG0, FP 1888 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1889 |9: 1890 | b ->exception_handler 1891 return 1; 1892} 1893 1894static int zend_jit_leave_function_stub(dasm_State **Dst) 1895{ 1896 |->leave_function_handler: 1897 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1898 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1899 | bne >1 1900 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 1901 | ADD_HYBRID_SPAD 1902 | JMP_IP TMP1 1903 |1: 1904 | EXT_CALL zend_jit_leave_top_func_helper, REG0 1905 | ADD_HYBRID_SPAD 1906 | JMP_IP TMP1 1907 } else { 1908 if (GCC_GLOBAL_REGS) { 1909 | ldp x29, x30, [sp], # SPAD // stack alignment 1910 } else { 1911 | mov FCARG2x, FP 1912 | ldp FP, RX, T2 // retore FP and IP 1913 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1914 } 1915 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1916 | bne >1 1917 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 1918 |1: 1919 | EXT_JMP zend_jit_leave_top_func_helper, REG0 1920 } 1921 1922 return 1; 1923} 1924 1925static int zend_jit_leave_throw_stub(dasm_State **Dst) 1926{ 1927 |->leave_throw_handler: 1928 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1929 if (GCC_GLOBAL_REGS) { 1930 | ldrb TMP1w, OP:IP->opcode 1931 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1932 | beq >5 1933 | // EG(opline_before_exception) = opline; 1934 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1935 |5: 1936 | // opline = EG(exception_op); 1937 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1938 | // HANDLE_EXCEPTION() 1939 | b ->exception_handler 1940 } else { 1941 | GET_IP TMP1 1942 | ldrb TMP2w, OP:TMP1->opcode 1943 | cmp TMP2w, #ZEND_HANDLE_EXCEPTION 1944 | beq >5 1945 | // EG(opline_before_exception) = opline; 1946 | MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 1947 |5: 1948 | // opline = EG(exception_op); 1949 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1950 | ldp FP, RX, T2 // retore FP and IP 1951 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1952 | mov RETVALx, #2 // ZEND_VM_LEAVE 1953 | ret 1954 } 1955 1956 return 1; 1957} 1958 1959static int zend_jit_icall_throw_stub(dasm_State **Dst) 1960{ 1961 |->icall_throw_handler: 1962 | // zend_rethrow_exception(zend_execute_data *execute_data) 1963 | ldr IP, EX->opline 1964 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1965 | ldrb TMP1w, OP:IP->opcode 1966 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1967 | beq >1 1968 | // EG(opline_before_exception) = opline; 1969 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1970 |1: 1971 | // opline = EG(exception_op); 1972 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1973 || if (GCC_GLOBAL_REGS) { 1974 | str IP, EX->opline 1975 || } 1976 | // HANDLE_EXCEPTION() 1977 | b ->exception_handler 1978 1979 return 1; 1980} 1981 1982static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1983{ 1984 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1985 1986 |->throw_cannot_pass_by_ref: 1987 | ldr REG0, EX->opline 1988 | ldr REG1w, OP:REG0->result.var 1989 | add REG1, REG1, RX 1990 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 1991 | // last EX(call) frame may be delayed 1992 | ldr TMP1, EX->call 1993 | cmp RX, TMP1 1994 | beq >1 1995 | ldr REG1, EX->call 1996 | str REG1, EX:RX->prev_execute_data 1997 | str RX, EX->call 1998 |1: 1999 | mov RX, REG0 2000 | ldr FCARG1w, OP:REG0->op2.num 2001 | EXT_CALL zend_cannot_pass_by_reference, REG0 2002 | ldrb TMP1w, OP:RX->op1_type 2003 | cmp TMP1w, #IS_TMP_VAR 2004 | bne >9 2005 | ldr REG0w, OP:RX->op1.var 2006 | add REG0, REG0, FP 2007 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 2008 |9: 2009 | b ->exception_handler 2010 2011 return 1; 2012} 2013 2014static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 2015{ 2016 |->undefined_offset_ex: 2017 | SAVE_IP 2018 | b ->undefined_offset 2019 2020 return 1; 2021} 2022 2023static int zend_jit_undefined_offset_stub(dasm_State **Dst) 2024{ 2025 |->undefined_offset: 2026#ifdef __APPLE__ 2027 | stp x29, x30, [sp, # -16]! 2028 | mov x29, sp 2029#endif 2030 | //sub r4, 8 2031 | ldr REG0, EX->opline 2032 | ldr REG1w, OP:REG0->result.var 2033 | add REG1, REG1, FP 2034 | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w 2035 | ldrb REG1w, OP:REG0->op2_type 2036 | cmp REG1w, #IS_CONST 2037 | bne >2 2038 | ldrsw REG1, OP:REG0->op2.constant 2039 | add REG0, REG0, REG1 2040 | b >3 2041 |2: 2042 | ldr REG0w, OP:REG0->op2.var 2043 | add REG0, REG0, FP 2044 |3: 2045 | mov CARG1, #E_WARNING 2046 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2047 | ldr CARG3, [REG0] 2048#ifdef __APPLE__ 2049 | str CARG3, [sp, #-16]! 2050 | EXT_CALL zend_error, REG0 2051 | add sp, sp, #16 2052 | ldp x29, x30, [sp], #16 2053 | ret 2054#else 2055 | EXT_JMP zend_error, REG0 // tail call 2056 | //add r4, 8 // stack alignment 2057 | //ret 2058#endif 2059 2060 return 1; 2061} 2062 2063static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2064{ 2065 |->undefined_index_ex: 2066 | SAVE_IP 2067 | b ->undefined_index 2068 2069 return 1; 2070} 2071 2072static int zend_jit_undefined_index_stub(dasm_State **Dst) 2073{ 2074 |->undefined_index: 2075#ifdef __APPLE__ 2076 | stp x29, x30, [sp, # -16]! 2077 | mov x29, sp 2078#endif 2079 | //sub r4, 8 2080 | ldr REG0, EX->opline 2081 | ldr REG1w, OP:REG0->result.var 2082 | add REG1, REG1, FP 2083 | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w 2084 | ldrb REG1w, OP:REG0->op2_type 2085 | cmp REG1w, #IS_CONST 2086 | bne >2 2087 | ldrsw REG1, OP:REG0->op2.constant 2088 | add REG0, REG0, REG1 2089 | b >3 2090 |2: 2091 | ldr REG0w, OP:REG0->op2.var 2092 | add REG0, REG0, FP 2093 |3: 2094 | mov CARG1, #E_WARNING 2095 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2096 | ldr CARG3, [REG0] 2097 | add CARG3, CARG3, #offsetof(zend_string, val) 2098#ifdef __APPLE__ 2099 | str CARG3, [sp, #-16]! 2100 | EXT_CALL zend_error, REG0 2101 | add sp, sp, #16 2102 | ldp x29, x30, [sp], #16 2103 | ret 2104#else 2105 | EXT_JMP zend_error, REG0 // tail call 2106 | //add r4, 8 2107 | //ret 2108#endif 2109 2110 return 1; 2111} 2112 2113static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2114{ 2115 |->cannot_add_element_ex: 2116 | SAVE_IP 2117 | b ->cannot_add_element 2118 2119 return 1; 2120} 2121 2122static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2123{ 2124 |->cannot_add_element: 2125 | // sub r4, 8 2126 | ldr REG0, EX->opline 2127 | ldrb TMP1w, OP:REG0->result_type 2128 | cmp TMP1w, #IS_UNUSED 2129 | beq >1 2130 | ldr REG0w, OP:REG0->result.var 2131 | add REG0, REG0, FP 2132 | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w 2133 |1: 2134 | mov CARG1, xzr 2135 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2136 | EXT_JMP zend_throw_error, REG0 // tail call 2137 | // add r4, 8 2138 | //ret 2139 2140 return 1; 2141} 2142 2143static int zend_jit_undefined_function_stub(dasm_State **Dst) 2144{ 2145 |->undefined_function: 2146 | ldr REG0, EX->opline 2147 | mov CARG1, xzr 2148 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2149 | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] 2150 | ldr CARG3, [REG0, CARG3] 2151 | add CARG3, CARG3, #offsetof(zend_string, val) 2152#ifdef __APPLE__ 2153 | str CARG3, [sp, #-16]! 2154#endif 2155 | EXT_CALL zend_throw_error, REG0 2156#ifdef __APPLE__ 2157 | add sp, sp, #16 2158#endif 2159 | b ->exception_handler 2160 return 1; 2161} 2162 2163static int zend_jit_negative_shift_stub(dasm_State **Dst) 2164{ 2165 |->negative_shift: 2166 | ldr RX, EX->opline 2167 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2168 | LOAD_ADDR CARG2, "Bit shift by negative number" 2169 | EXT_CALL zend_throw_error, REG0 2170 | b ->exception_handler_free_op1_op2 2171 return 1; 2172} 2173 2174static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2175{ 2176 |->mod_by_zero: 2177 | ldr RX, EX->opline 2178 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2179 | LOAD_ADDR CARG2, "Modulo by zero" 2180 | EXT_CALL zend_throw_error, REG0 2181 | b ->exception_handler_free_op1_op2 2182 return 1; 2183} 2184 2185static int zend_jit_invalid_this_stub(dasm_State **Dst) 2186{ 2187 |->invalid_this: 2188 | UNDEF_OPLINE_RESULT TMP1w 2189 | mov CARG1, xzr 2190 | LOAD_ADDR CARG2, "Using $this when not in object context" 2191 | EXT_CALL zend_throw_error, REG0 2192 | b ->exception_handler 2193 2194 return 1; 2195} 2196 2197static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2198{ 2199 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2200 return 1; 2201 } 2202 2203 |->hybrid_runtime_jit: 2204 | EXT_CALL zend_runtime_jit, REG0 2205 | JMP_IP TMP1 2206 return 1; 2207} 2208 2209static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2210{ 2211 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2212 return 1; 2213 } 2214 2215 |->hybrid_profile_jit: 2216 | // ++zend_jit_profile_counter; 2217 | LOAD_ADDR REG0, &zend_jit_profile_counter 2218 | ldr TMP1, [REG0] 2219 | add TMP1, TMP1, #1 2220 | str TMP1, [REG0] 2221 | // op_array = (zend_op_array*)EX(func); 2222 | ldr REG0, EX->func 2223 | // run_time_cache = EX(run_time_cache); 2224 | ldr REG2, EX->run_time_cache 2225 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2226 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2227 | // ++ZEND_COUNTER_INFO(op_array) 2228 || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { 2229 | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) 2230 | ldr TMP2, [REG2, TMP1] 2231 | add TMP2, TMP2, #1 2232 | str TMP2, [REG2, TMP1] 2233 || } else { 2234 | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2235 | add TMP2, TMP2, #1 2236 | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2237 || } 2238 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2239 | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] 2240 | br TMP1 2241 return 1; 2242} 2243 2244static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2245{ 2246 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2247 return 1; 2248 } 2249 2250 |->hybrid_hot_code: 2251 || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); 2252 | movz TMP1w, #ZEND_JIT_COUNTER_INIT 2253 | strh TMP1w, [REG2] 2254 | mov FCARG1x, FP 2255 | GET_IP FCARG2x 2256 | EXT_CALL zend_jit_hot_func, REG0 2257 | JMP_IP TMP1 2258 return 1; 2259} 2260 2261/* 2262 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2263 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2264 * property disclosure and research opportunities" email 2265 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2266 * 2267 * In addition we use a variation of Knuth's multiplicative hash function 2268 * described at https://code.i-harness.com/en/q/a21ce 2269 * 2270 * uint64_t hash(uint64_t x) { 2271 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2272 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2273 * x = x ^ (x >> 31); 2274 * return x; 2275 * } 2276 * 2277 * uint_32_t hash(uint32_t x) { 2278 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2279 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2280 * x = (x >> 16) ^ x; 2281 * return x; 2282 * } 2283 * 2284 */ 2285static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2286{ 2287 | ldr REG0, EX->func 2288 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2289 | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] 2290 | ldrh TMP2w, [REG2] 2291 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2292 | strh TMP2w, [REG2] 2293 | ble ->hybrid_hot_code 2294 | GET_IP REG2 2295 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 2296 | sub REG2, REG2, TMP1 2297 | // divide by sizeof(zend_op) 2298 || ZEND_ASSERT(sizeof(zend_op) == 32); 2299 | add TMP1, REG1, REG2, asr #2 2300 | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2301 | br TMP1 2302 return 1; 2303} 2304 2305static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2306{ 2307 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2308 return 1; 2309 } 2310 2311 |->hybrid_func_hot_counter: 2312 2313 return zend_jit_hybrid_hot_counter_stub(Dst, 2314 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2315} 2316 2317static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2318{ 2319 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2320 return 1; 2321 } 2322 2323 |->hybrid_loop_hot_counter: 2324 2325 return zend_jit_hybrid_hot_counter_stub(Dst, 2326 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2327} 2328 2329static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2330{ 2331 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2332 return 1; 2333 } 2334 2335 // On entry from counter stub: 2336 // REG2 -> zend_op_trace_info.counter 2337 2338 |->hybrid_hot_trace: 2339 | mov TMP1w, #ZEND_JIT_COUNTER_INIT 2340 | strh TMP1w, [REG2] 2341 | mov FCARG1x, FP 2342 | GET_IP FCARG2x 2343 | EXT_CALL zend_jit_trace_hot_root, REG0 2344 | tbnz RETVALw, #31, >1 // Result is < 0 on failure. 2345 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2346 | LOAD_IP 2347 | JMP_IP TMP1 2348 |1: 2349 | EXT_JMP zend_jit_halt_op->handler, REG0 2350 2351 return 1; 2352} 2353 2354static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2355{ 2356 | ldr REG0, EX->func 2357 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2358 | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] 2359 | add TMP1, REG1, IP 2360 | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] 2361 | ldrh TMP2w, [REG2] 2362 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2363 | strh TMP2w, [REG2] 2364 | ble ->hybrid_hot_trace 2365 // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior 2366 // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. 2367 | add TMP1, REG1, IP 2368 | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] 2369 | br TMP2 2370 2371 return 1; 2372} 2373 2374static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2375{ 2376 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2377 return 1; 2378 } 2379 2380 |->hybrid_func_trace_counter: 2381 2382 return zend_jit_hybrid_trace_counter_stub(Dst, 2383 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2384} 2385 2386static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2387{ 2388 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2389 return 1; 2390 } 2391 2392 |->hybrid_ret_trace_counter: 2393 2394 return zend_jit_hybrid_trace_counter_stub(Dst, 2395 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2396} 2397 2398static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2399{ 2400 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2401 return 1; 2402 } 2403 2404 |->hybrid_loop_trace_counter: 2405 2406 return zend_jit_hybrid_trace_counter_stub(Dst, 2407 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2408} 2409 2410static int zend_jit_trace_halt_stub(dasm_State **Dst) 2411{ 2412 |->trace_halt: 2413 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2414 | ADD_HYBRID_SPAD 2415 | EXT_JMP zend_jit_halt_op->handler, REG0 2416 } else if (GCC_GLOBAL_REGS) { 2417 | ldp x29, x30, [sp], # SPAD // stack alignment 2418 | mov IP, xzr // PC must be zero 2419 | ret 2420 } else { 2421 | ldp FP, RX, T2 // retore FP and IP 2422 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2423 | movn RETVALx, #0 // ZEND_VM_RETURN (-1) 2424 | ret 2425 } 2426 return 1; 2427} 2428 2429static int zend_jit_trace_exit_stub(dasm_State **Dst) 2430{ 2431 |->trace_exit: 2432 | 2433 | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 2434 | 2435 | stp d30, d31, [sp, #-16]! 2436 | stp d28, d29, [sp, #-16]! 2437 | stp d26, d27, [sp, #-16]! 2438 | stp d24, d25, [sp, #-16]! 2439 | stp d22, d23, [sp, #-16]! 2440 | stp d20, d21, [sp, #-16]! 2441 | stp d18, d19, [sp, #-16]! 2442 | stp d16, d17, [sp, #-16]! 2443 | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet 2444 | //stp d12, d13, [sp, #-16]! 2445 | //stp d10, d11, [sp, #-16]! 2446 | //stp d8, d9, [sp, #-16]! 2447 | stp d6, d7, [sp, #(-16*5)]! 2448 | stp d4, d5, [sp, #-16]! 2449 | stp d2, d3, [sp, #-16]! 2450 | stp d0, d1, [sp, #-16]! 2451 | 2452 | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) 2453 | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) 2454 | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet 2455 | //stp x24, x25, [sp, #-16]! 2456 | //stp x22, x23, [sp, #-16]! 2457 | //stp x20, x21, [sp, #-16]! 2458 | //stp x18, x19, [sp, #-16]! 2459 | //stp x16, x17, [sp, #-16]! // we don't need temporary registers 2460 | stp x14, x15, [sp, #-(16*7)]! 2461 | stp x12, x13, [sp, #-16]! 2462 | stp x10, x11, [sp, #-16]! 2463 | stp x8, x9, [sp, #-16]! 2464 | stp x6, x7, [sp, #-16]! 2465 | stp x4, x5, [sp, #-16]! 2466 | stp x2, x3, [sp, #-16]! 2467 | stp x0, x1, [sp, #-16]! 2468 | 2469 | mov FCARG1w, TMP1w // exit_num 2470 | mov FCARG2x, sp 2471 | 2472 | // EX(opline) = opline 2473 | SAVE_IP 2474 | // zend_jit_trace_exit(trace_num, exit_num) 2475 | EXT_CALL zend_jit_trace_exit, REG0 2476 | 2477 | add sp, sp, #(32 * 16) // restore sp 2478 | 2479 2480 | tst RETVALw, RETVALw 2481 | bne >1 // not zero 2482 2483 | // execute_data = EG(current_execute_data) 2484 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2485 | // opline = EX(opline) 2486 | LOAD_IP 2487 2488 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2489 | ADD_HYBRID_SPAD 2490 | JMP_IP TMP1 2491 } else if (GCC_GLOBAL_REGS) { 2492 | ldp x29, x30, [sp], # SPAD // stack alignment 2493 | JMP_IP TMP1 2494 } else { 2495 | ldp FP, RX, T2 // retore FP and IP 2496 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2497 | mov RETVALx, #1 // ZEND_VM_ENTER 2498 | ret 2499 } 2500 2501 |1: 2502 | blt ->trace_halt 2503 2504 | // execute_data = EG(current_execute_data) 2505 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2506 | // opline = EX(opline) 2507 | LOAD_IP 2508 2509 | // check for interrupt (try to avoid this ???) 2510 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2511 | cbnz TMP1w, ->interrupt_handler 2512 2513 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2514 | ADD_HYBRID_SPAD 2515 | ldr REG0, EX->func 2516 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2517 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2518 | ldr REG0, [IP, REG0] 2519 | br REG0 2520 } else if (GCC_GLOBAL_REGS) { 2521 | ldp x29, x30, [sp], # SPAD // stack alignment 2522 | ldr REG0, EX->func 2523 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2524 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2525 | ldr REG0, [IP, REG0] 2526 | br REG0 2527 } else { 2528 | ldr IP, EX->opline 2529 | mov FCARG1x, FP 2530 | ldr REG0, EX->func 2531 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2532 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2533 | ldr REG0, [IP, REG0] 2534 | blr REG0 2535 | 2536 | tst RETVALw, RETVALw 2537 | blt ->trace_halt 2538 | 2539 | ldp FP, RX, T2 // retore FP and IP 2540 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2541 | mov RETVALx, #1 // ZEND_VM_ENTER 2542 | ret 2543 } 2544 2545 return 1; 2546} 2547 2548static int zend_jit_trace_escape_stub(dasm_State **Dst) 2549{ 2550 |->trace_escape: 2551 | 2552 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2553 | ADD_HYBRID_SPAD 2554 | JMP_IP, TMP1 2555 } else if (GCC_GLOBAL_REGS) { 2556 | ldp x29, x30, [sp], # SPAD // stack alignment 2557 | JMP_IP, TMP1 2558 } else { 2559 | ldp FP, RX, T2 // retore FP and IP 2560 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2561 | mov RETVALx, #1 // ZEND_VM_ENTER 2562 | ret 2563 } 2564 2565 return 1; 2566} 2567 2568/* Keep 32 exit points in a single code block */ 2569#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes 2570#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2571 2572static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2573{ 2574 uint32_t i; 2575 2576 | bl >2 2577 |1: 2578 for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { 2579 | bl >2 2580 } 2581 |2: 2582 | adr TMP1, <1 2583 | sub TMP1, lr, TMP1 2584 | lsr TMP1, TMP1, #2 2585 if (n) { 2586 | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w 2587 } 2588 | b ->trace_exit // pass exit_num in TMP1w 2589 2590 return 1; 2591} 2592 2593#ifdef CONTEXT_THREADED_JIT 2594static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2595{ 2596 |->context_threaded_call: 2597 | NIY_STUB // TODO 2598 return 1; 2599} 2600#endif 2601 2602static int zend_jit_assign_const_stub(dasm_State **Dst) 2603{ 2604 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2605 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2606 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2607 2608 |->assign_const: 2609 | stp x29, x30, [sp, #-32]! 2610 | mov x29, sp 2611 if (!zend_jit_assign_to_variable( 2612 Dst, NULL, 2613 var_addr, var_addr, -1, -1, 2614 IS_CONST, val_addr, val_info, 2615 0, 0)) { 2616 return 0; 2617 } 2618 | ldp x29, x30, [sp], #32 2619 | ret 2620 return 1; 2621} 2622 2623static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2624{ 2625 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2626 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2627 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2628 2629 |->assign_tmp: 2630 | stp x29, x30, [sp, #-32]! 2631 | mov x29, sp 2632 if (!zend_jit_assign_to_variable( 2633 Dst, NULL, 2634 var_addr, var_addr, -1, -1, 2635 IS_TMP_VAR, val_addr, val_info, 2636 0, 0)) { 2637 return 0; 2638 } 2639 | ldp x29, x30, [sp], #32 2640 | ret 2641 return 1; 2642} 2643 2644static int zend_jit_assign_var_stub(dasm_State **Dst) 2645{ 2646 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2647 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2648 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2649 2650 |->assign_var: 2651 | stp x29, x30, [sp, #-32]! 2652 | mov x29, sp 2653 if (!zend_jit_assign_to_variable( 2654 Dst, NULL, 2655 var_addr, var_addr, -1, -1, 2656 IS_VAR, val_addr, val_info, 2657 0, 0)) { 2658 return 0; 2659 } 2660 | ldp x29, x30, [sp], #32 2661 | ret 2662 return 1; 2663} 2664 2665static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2666{ 2667 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2668 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2669 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2670 2671 |->assign_cv_noref: 2672 | stp x29, x30, [sp, #-32]! 2673 | mov x29, sp 2674 if (!zend_jit_assign_to_variable( 2675 Dst, NULL, 2676 var_addr, var_addr, -1, -1, 2677 IS_CV, val_addr, val_info, 2678 0, 0)) { 2679 return 0; 2680 } 2681 | ldp x29, x30, [sp], #32 2682 | ret 2683 return 1; 2684} 2685 2686static int zend_jit_assign_cv_stub(dasm_State **Dst) 2687{ 2688 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2689 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2690 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2691 2692 |->assign_cv: 2693 | stp x29, x30, [sp, #-32]! 2694 | mov x29, sp 2695 if (!zend_jit_assign_to_variable( 2696 Dst, NULL, 2697 var_addr, var_addr, -1, -1, 2698 IS_CV, val_addr, val_info, 2699 0, 0)) { 2700 return 0; 2701 } 2702 | ldp x29, x30, [sp], #32 2703 | ret 2704 return 1; 2705} 2706 2707static const zend_jit_stub zend_jit_stubs[] = { 2708 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2709 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2710 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2711 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2712 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2713 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2714 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2715 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2716 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2717 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2718 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2719 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2720 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2721 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2722 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2723 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2724 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2725 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2726 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2727 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2728 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2729 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2730 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2731 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2732 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2733 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2734 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2735 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2736 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2737 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2738 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2739 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2740 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2741 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2742 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2743 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2744#ifdef CONTEXT_THREADED_JIT 2745 JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), 2746#endif 2747}; 2748 2749#ifdef HAVE_GDB 2750# if 0 2751typedef struct _Unwind_Context _Unwind_Context; 2752typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2753extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2754extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2755 2756typedef struct _zend_jit_unwind_arg { 2757 int cnt; 2758 uintptr_t cfa[3]; 2759} zend_jit_unwind_arg; 2760 2761static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2762{ 2763 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2764 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2765 arg->cnt++; 2766 if (arg->cnt == 3) { 2767 return 5; // _URC_END_OF_STACK 2768 } 2769 return 0; // _URC_NO_REASON; 2770} 2771 2772static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2773{ 2774 zend_jit_unwind_arg arg; 2775 2776 memset(&arg, 0, sizeof(arg)); 2777 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2778 if (arg.cnt == 3) { 2779 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2780 } 2781} 2782# else 2783static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2784{ 2785 uintptr_t ret; 2786 2787 __asm__ ( 2788 "ldr %0, [x29]\n\t" 2789 "sub %0 ,%0, x29" 2790 : "=r"(ret)); 2791 2792 sp_adj[SP_ADJ_VM] = ret; 2793} 2794# endif 2795 2796extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2797 2798static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2799{ 2800 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2801 2802 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2803 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2804 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2805 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2806} 2807#endif 2808 2809static int zend_jit_setup(void) 2810{ 2811 allowed_opt_flags = 0; 2812 2813#if ZTS 2814 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2815 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); 2816#endif 2817 2818 memset(sp_adj, 0, sizeof(sp_adj)); 2819#ifdef HAVE_GDB 2820 sp_adj[SP_ADJ_RET] = 0; 2821 sp_adj[SP_ADJ_ASSIGN] = 32; 2822 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2823 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2824#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2825 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2826#else 2827 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2828#endif 2829 } else if (GCC_GLOBAL_REGS) { 2830 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2831 } else { 2832 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2833 } 2834#endif 2835 2836 return SUCCESS; 2837} 2838 2839static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2840{ 2841 | brk #0 2842 return 1; 2843} 2844 2845static int zend_jit_align_func(dasm_State **Dst) 2846{ 2847 reuse_ip = 0; 2848 delayed_call_chain = 0; 2849 last_valid_opline = NULL; 2850 use_last_vald_opline = 0; 2851 track_last_valid_opline = 0; 2852 jit_return_label = -1; 2853 |.align 16 2854 return 1; 2855} 2856 2857static int zend_jit_prologue(dasm_State **Dst) 2858{ 2859 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2860 | SUB_HYBRID_SPAD 2861 } else if (GCC_GLOBAL_REGS) { 2862 | stp x29, x30, [sp, # -SPAD]! // stack alignment 2863 |// mov x29, sp 2864 } else { 2865 | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 2866 |// mov x29, sp 2867 | stp FP, RX, T2 // save FP and IP 2868 | mov FP, FCARG1x 2869 } 2870 return 1; 2871} 2872 2873static int zend_jit_label(dasm_State **Dst, unsigned int label) 2874{ 2875 |=>label: 2876 return 1; 2877} 2878 2879static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 2880{ 2881 | // call->prev_execute_data = EX(call); 2882 if (call_level == 1) { 2883 | str xzr, EX:RX->prev_execute_data 2884 } else { 2885 | ldr REG0, EX->call 2886 | str REG0, EX:RX->prev_execute_data 2887 } 2888 | // EX(call) = call; 2889 | str RX, EX->call 2890 2891 delayed_call_chain = 0; 2892 2893 return 1; 2894} 2895 2896static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 2897{ 2898 if (last_valid_opline == opline) { 2899 zend_jit_use_last_valid_opline(); 2900 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 2901 zend_jit_use_last_valid_opline(); 2902 | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) 2903 | ADD_IP TMP1, TMP2 2904 } else { 2905 | LOAD_IP_ADDR opline 2906 } 2907 zend_jit_set_last_valid_opline(opline); 2908 2909 return 1; 2910} 2911 2912static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 2913{ 2914 return zend_jit_set_ip(Dst, opline); 2915} 2916 2917static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 2918{ 2919 if (delayed_call_chain) { 2920 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 2921 return 0; 2922 } 2923 } 2924 if (!zend_jit_set_ip(Dst, opline)) { 2925 return 0; 2926 } 2927 reuse_ip = 0; 2928 return 1; 2929} 2930 2931static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 2932{ 2933 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2934 if (exit_addr) { 2935 | cbnz TMP1w, &exit_addr 2936 } else if (last_valid_opline == opline) { 2937 || zend_jit_use_last_valid_opline(); 2938 | cbnz TMP1w, ->interrupt_handler 2939 } else { 2940 | cbnz TMP1w, >1 2941 |.cold_code 2942 |1: 2943 | LOAD_IP_ADDR opline 2944 | b ->interrupt_handler 2945 |.code 2946 } 2947 return 1; 2948} 2949 2950static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 2951{ 2952 if (timeout_exit_addr) { 2953 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2954 | cbz TMP1w, =>loop_label 2955 | b &timeout_exit_addr 2956 } else { 2957 | b =>loop_label 2958 } 2959 return 1; 2960} 2961 2962static int zend_jit_check_exception(dasm_State **Dst) 2963{ 2964 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2965 | cbnz TMP2, ->exception_handler 2966 return 1; 2967} 2968 2969static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 2970{ 2971 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 2972 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2973 | cbnz TMP2, ->exception_handler_undef 2974 return 1; 2975 } 2976 return zend_jit_check_exception(Dst); 2977} 2978 2979static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 2980{ 2981 2982 current_trace_num = trace_num; 2983 2984 | // EG(jit_trace_num) = trace_num; 2985 | LOAD_32BIT_VAL TMP1w, trace_num 2986 | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 2987 2988 return 1; 2989} 2990 2991static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 2992{ 2993 uint32_t i; 2994 const void *exit_addr; 2995 2996 /* Emit veneers table for exit points (B instruction for each exit number) */ 2997 |.cold_code 2998 for (i = 0; i < t->exit_count; i++) { 2999 exit_addr = zend_jit_trace_get_exit_addr(i); 3000 | b &exit_addr 3001 } 3002 |=>1: // end of the code 3003 |.code 3004 return 1; 3005} 3006 3007static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3008{ 3009 int ret = 0; 3010 uint8_t *p, *end; 3011 const void *veneer = NULL; 3012 ptrdiff_t delta; 3013 3014 if (jmp_table_size) { 3015 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3016 3017 do { 3018 if (*jmp_slot == from_addr) { 3019 *jmp_slot = to_addr; 3020 ret++; 3021 } 3022 jmp_slot++; 3023 } while (--jmp_table_size); 3024 } 3025 3026 end = (uint8_t*)code; 3027 p = end + size; 3028 while (p > end) { 3029 uint32_t *ins_ptr; 3030 uint32_t ins; 3031 3032 p -= 4; 3033 ins_ptr = (uint32_t*)p; 3034 ins = *ins_ptr; 3035 if ((ins & 0xfc000000u) == 0x14000000u) { 3036 // B (imm26:0..25) 3037 delta = (uint32_t*)from_addr - ins_ptr; 3038 if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { 3039 delta = (uint32_t*)to_addr - ins_ptr; 3040 if (((delta + 0x02000000) >> 26) != 0) { 3041 abort(); // brnach target out of range 3042 } 3043 *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); 3044 ret++; 3045 if (!veneer) { 3046 veneer = p; 3047 } 3048 } 3049 } else if ((ins & 0xff000000u) == 0x54000000u || 3050 (ins & 0x7e000000u) == 0x34000000u) { 3051 // B.cond, CBZ, CBNZ (imm19:5..23) 3052 delta = (uint32_t*)from_addr - ins_ptr; 3053 if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { 3054 delta = (uint32_t*)to_addr - ins_ptr; 3055 if (((delta + 0x40000) >> 19) != 0) { 3056 if (veneer) { 3057 delta = (uint32_t*)veneer - ins_ptr; 3058 if (((delta + 0x40000) >> 19) != 0) { 3059 abort(); // brnach target out of range 3060 } 3061 } else { 3062 abort(); // brnach target out of range 3063 } 3064 } 3065 *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); 3066 ret++; 3067 } 3068 } else if ((ins & 0x7e000000u) == 0x36000000u) { 3069 // TBZ, TBNZ (imm14:5..18) 3070 delta = (uint32_t*)from_addr - ins_ptr; 3071 if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { 3072 delta = (uint32_t*)to_addr - ins_ptr; 3073 if (((delta + 0x2000) >> 14) != 0) { 3074 if (veneer) { 3075 delta = (uint32_t*)veneer - ins_ptr; 3076 if (((delta + 0x2000) >> 14) != 0) { 3077 abort(); // brnach target out of range 3078 } 3079 } else { 3080 abort(); // brnach target out of range 3081 } 3082 } 3083 *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); 3084 ret++; 3085 } 3086 } 3087 } 3088 3089 JIT_CACHE_FLUSH(code, (char*)code + size); 3090 3091#ifdef HAVE_VALGRIND 3092 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3093#endif 3094 3095 return ret; 3096} 3097 3098static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3099{ 3100 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3101} 3102 3103static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3104{ 3105 const void *link_addr; 3106 size_t prologue_size; 3107 3108 /* Skip prologue. */ 3109 // TODO: don't hardcode this ??? 3110 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3111#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3112 prologue_size = 0; 3113#else 3114 // sub sp, sp, #0x20 3115 prologue_size = 4; 3116#endif 3117 } else if (GCC_GLOBAL_REGS) { 3118 // stp x29, x30, [sp, # -SPAD]! 3119 prologue_size = 4; 3120 } else { 3121 // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 3122 // stp FP, RX, T2 3123 // mov FP, FCARG1x 3124 prologue_size = 12; 3125 } 3126 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3127 3128 if (timeout_exit_addr) { 3129 /* Check timeout for links to LOOP */ 3130 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 3131 | cbz TMP1w, &link_addr 3132 | b &timeout_exit_addr 3133 } else { 3134 | b &link_addr 3135 } 3136 return 1; 3137} 3138 3139static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3140{ 3141 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3142 | ADD_HYBRID_SPAD 3143 if (!original_handler) { 3144 | JMP_IP TMP1 3145 } else { 3146 | ldr REG0, EX->func 3147 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3148 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3149 | ldr REG0, [IP, REG0] 3150 | br REG0 3151 } 3152 } else if (GCC_GLOBAL_REGS) { 3153 | ldp x29, x30, [sp], # SPAD // stack alignment 3154 if (!original_handler) { 3155 | JMP_IP TMP1 3156 } else { 3157 | ldr REG0, EX->func 3158 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3159 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3160 | ldr REG0, [IP, REG0] 3161 | br REG0 3162 } 3163 } else { 3164 if (original_handler) { 3165 | mov FCARG1x, FP 3166 | ldr REG0, EX->func 3167 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3168 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3169 | ldr REG0, [IP, REG0] 3170 | blr REG0 3171 } 3172 | ldp FP, RX, T2 // retore FP and IP 3173 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3174 if (!original_handler || !opline || 3175 (opline->opcode != ZEND_RETURN 3176 && opline->opcode != ZEND_RETURN_BY_REF 3177 && opline->opcode != ZEND_GENERATOR_RETURN 3178 && opline->opcode != ZEND_GENERATOR_CREATE 3179 && opline->opcode != ZEND_YIELD 3180 && opline->opcode != ZEND_YIELD_FROM)) { 3181 | mov RETVALx, #2 // ZEND_VM_LEAVE 3182 } 3183 | ret 3184 } 3185 return 1; 3186} 3187 3188static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3189{ 3190 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3191 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3192 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3193 3194 if (!exit_addr) { 3195 return 0; 3196 } 3197 3198 | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 3199 3200 return 1; 3201} 3202 3203static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3204{ 3205 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3206 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3207 3208 if (!exit_addr) { 3209 return 0; 3210 } 3211 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1 3212 | cmp TMP1w, #IS_STRING 3213 | bhs &exit_addr 3214 3215 return 1; 3216} 3217static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3218{ 3219 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3220 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3221 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3222 3223 if (!exit_addr) { 3224 return 0; 3225 } 3226 3227 | GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1 3228 if (op_info & MAY_BE_ARRAY_PACKED) { 3229 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3230 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3231 | beq &exit_addr 3232 } else { 3233 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3234 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3235 | bne &exit_addr 3236 } 3237 3238 return 1; 3239} 3240 3241static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3242{ 3243 zend_jit_op_array_trace_extension *jit_extension = 3244 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3245 size_t offset = jit_extension->offset; 3246 const void *handler = 3247 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3248 3249 if (!zend_jit_set_valid_ip(Dst, opline)) { 3250 return 0; 3251 } 3252 if (!GCC_GLOBAL_REGS) { 3253 | mov FCARG1x, FP 3254 } 3255 | EXT_CALL handler, REG0 3256 if (may_throw 3257 && opline->opcode != ZEND_RETURN 3258 && opline->opcode != ZEND_RETURN_BY_REF) { 3259 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 3260 | cbnz REG0, ->exception_handler 3261 } 3262 3263 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3264 trace++; 3265 } 3266 3267 if (!GCC_GLOBAL_REGS 3268 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3269 if (opline->opcode == ZEND_RETURN || 3270 opline->opcode == ZEND_RETURN_BY_REF || 3271 opline->opcode == ZEND_DO_UCALL || 3272 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3273 opline->opcode == ZEND_DO_FCALL || 3274 opline->opcode == ZEND_GENERATOR_CREATE) { 3275 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 3276 } 3277 } 3278 3279 if (zend_jit_trace_may_exit(op_array, opline)) { 3280 if (opline->opcode == ZEND_RETURN || 3281 opline->opcode == ZEND_RETURN_BY_REF || 3282 opline->opcode == ZEND_GENERATOR_CREATE) { 3283 3284 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3285#if 0 3286 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3287 | LOAD_ADDR TMP1, zend_jit_halt_op 3288 | cmp IP, TMP1 3289 | beq ->trace_halt 3290#endif 3291 } else if (GCC_GLOBAL_REGS) { 3292 | cbz IP, ->trace_halt 3293 } else { 3294 | tst RETVALw, RETVALw 3295 | blt ->trace_halt 3296 } 3297 } else if (opline->opcode == ZEND_EXIT || 3298 opline->opcode == ZEND_GENERATOR_RETURN || 3299 opline->opcode == ZEND_YIELD || 3300 opline->opcode == ZEND_YIELD_FROM) { 3301 | b ->trace_halt 3302 } 3303 if (trace->op != ZEND_JIT_TRACE_END || 3304 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3305 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3306 3307 const zend_op *next_opline = trace->opline; 3308 const zend_op *exit_opline = NULL; 3309 uint32_t exit_point; 3310 const void *exit_addr; 3311 uint32_t old_info = 0; 3312 uint32_t old_res_info = 0; 3313 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3314 3315 if (zend_is_smart_branch(opline)) { 3316 bool exit_if_true = 0; 3317 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3318 } else { 3319 switch (opline->opcode) { 3320 case ZEND_JMPZ: 3321 case ZEND_JMPNZ: 3322 case ZEND_JMPZ_EX: 3323 case ZEND_JMPNZ_EX: 3324 case ZEND_JMP_SET: 3325 case ZEND_COALESCE: 3326 case ZEND_JMP_NULL: 3327 case ZEND_FE_RESET_R: 3328 case ZEND_FE_RESET_RW: 3329 exit_opline = (trace->opline == opline + 1) ? 3330 OP_JMP_ADDR(opline, opline->op2) : 3331 opline + 1; 3332 break; 3333 case ZEND_JMPZNZ: 3334 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3335 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3336 OP_JMP_ADDR(opline, opline->op2); 3337 break; 3338 case ZEND_FE_FETCH_R: 3339 case ZEND_FE_FETCH_RW: 3340 if (opline->op2_type == IS_CV) { 3341 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3342 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3343 } 3344 exit_opline = (trace->opline == opline + 1) ? 3345 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3346 opline + 1; 3347 break; 3348 3349 } 3350 } 3351 3352 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3353 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3354 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3355 } 3356 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3357 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3358 3359 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3360 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3361 } 3362 switch (opline->opcode) { 3363 case ZEND_FE_FETCH_R: 3364 case ZEND_FE_FETCH_RW: 3365 if (opline->op2_type == IS_CV) { 3366 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3367 } 3368 break; 3369 } 3370 3371 if (!exit_addr) { 3372 return 0; 3373 } 3374 | CMP_IP next_opline, TMP1, TMP2 3375 | bne &exit_addr 3376 } 3377 } 3378 3379 zend_jit_set_last_valid_opline(trace->opline); 3380 3381 return 1; 3382} 3383 3384static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3385{ 3386 const void *handler; 3387 3388 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3389 handler = zend_get_opcode_handler_func(opline); 3390 } else { 3391 handler = opline->handler; 3392 } 3393 3394 if (!zend_jit_set_valid_ip(Dst, opline)) { 3395 return 0; 3396 } 3397 if (!GCC_GLOBAL_REGS) { 3398 | mov FCARG1x, FP 3399 } 3400 | EXT_CALL handler, REG0 3401 if (may_throw) { 3402 zend_jit_check_exception(Dst); 3403 } 3404 3405 /* Skip the following OP_DATA */ 3406 switch (opline->opcode) { 3407 case ZEND_ASSIGN_DIM: 3408 case ZEND_ASSIGN_OBJ: 3409 case ZEND_ASSIGN_STATIC_PROP: 3410 case ZEND_ASSIGN_DIM_OP: 3411 case ZEND_ASSIGN_OBJ_OP: 3412 case ZEND_ASSIGN_STATIC_PROP_OP: 3413 case ZEND_ASSIGN_STATIC_PROP_REF: 3414 case ZEND_ASSIGN_OBJ_REF: 3415 zend_jit_set_last_valid_opline(opline + 2); 3416 break; 3417 default: 3418 zend_jit_set_last_valid_opline(opline + 1); 3419 break; 3420 } 3421 3422 return 1; 3423} 3424 3425static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3426{ 3427 if (!zend_jit_set_valid_ip(Dst, opline)) { 3428 return 0; 3429 } 3430 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3431 if (opline->opcode == ZEND_DO_UCALL || 3432 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3433 opline->opcode == ZEND_DO_FCALL || 3434 opline->opcode == ZEND_RETURN) { 3435 3436 /* Use inlined HYBRID VM handler */ 3437 const void *handler = opline->handler; 3438 3439 | ADD_HYBRID_SPAD 3440 | EXT_JMP handler, REG0 3441 } else { 3442 const void *handler = zend_get_opcode_handler_func(opline); 3443 3444 | EXT_CALL handler, REG0 3445 | ADD_HYBRID_SPAD 3446 | JMP_IP TMP1 3447 } 3448 } else { 3449 const void *handler = opline->handler; 3450 3451 if (GCC_GLOBAL_REGS) { 3452 | ldp x29, x30, [sp], # SPAD // stack alignment 3453 } else { 3454 | mov FCARG1x, FP 3455 | ldp FP, RX, T2 // retore FP and IP 3456 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3457 } 3458 | EXT_JMP handler, REG0 3459 } 3460 zend_jit_reset_last_valid_opline(); 3461 return 1; 3462} 3463 3464static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3465{ 3466 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3467 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3468 3469 if (!exit_addr) { 3470 return 0; 3471 } 3472 | CMP_IP opline, TMP1, TMP2 3473 | bne &exit_addr 3474 3475 zend_jit_set_last_valid_opline(opline); 3476 3477 return 1; 3478} 3479 3480static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3481{ 3482 | b =>target_label 3483 return 1; 3484} 3485 3486static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3487{ 3488 | CMP_IP next_opline, TMP1, TMP2 3489 | bne =>target_label 3490 3491 zend_jit_set_last_valid_opline(next_opline); 3492 3493 return 1; 3494} 3495 3496#ifdef CONTEXT_THREADED_JIT 3497static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3498{ 3499 | NIY // TODO 3500 return 1; 3501} 3502#endif 3503 3504static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3505{ 3506#ifdef CONTEXT_THREADED_JIT 3507 return zend_jit_context_threaded_call(Dst, opline, next_block); 3508#else 3509 return zend_jit_tail_handler(Dst, opline); 3510#endif 3511} 3512 3513static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3514{ 3515 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3516 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3517 3518 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3519 | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 3520 if (set_type && 3521 (Z_REG(dst) != ZREG_FP || 3522 !JIT_G(current_frame) || 3523 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3524 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3525 } 3526 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3527 | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 3528 if (set_type && 3529 (Z_REG(dst) != ZREG_FP || 3530 !JIT_G(current_frame) || 3531 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3532 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3533 } 3534 } else { 3535 ZEND_UNREACHABLE(); 3536 } 3537 return 1; 3538} 3539 3540static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3541{ 3542 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3543 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3544 3545 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3546 | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 3547 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3548 | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 3549 } else { 3550 ZEND_UNREACHABLE(); 3551 } 3552 return 1; 3553} 3554 3555static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3556{ 3557 zend_jit_addr src = ZEND_ADDR_REG(reg); 3558 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3559 3560 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3561} 3562 3563static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3564{ 3565 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3566 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3567 return zend_jit_spill_store(Dst, src, dst, info, 1); 3568 } 3569 return 1; 3570} 3571 3572static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3573{ 3574 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3575 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3576 bool set_type = 1; 3577 3578 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3579 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3580 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3581 set_type = 0; 3582 } 3583 } 3584 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3585 } 3586 return 1; 3587} 3588 3589static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3590{ 3591 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3592 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3593 3594 return zend_jit_load_reg(Dst, src, dst, info); 3595} 3596 3597static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3598{ 3599 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3600 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3601 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2 3602 } 3603 return 1; 3604} 3605 3606static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3607{ 3608 if (!zend_jit_same_addr(src, dst)) { 3609 if (Z_MODE(src) == IS_REG) { 3610 if (Z_MODE(dst) == IS_REG) { 3611 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3612 | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) 3613 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3614 | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) 3615 } else { 3616 ZEND_UNREACHABLE(); 3617 } 3618 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3619 if (!Z_LOAD(src) && !Z_STORE(src)) { 3620 if (!zend_jit_spill_store(Dst, src, dst, info, 3621 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3622 JIT_G(current_frame) == NULL || 3623 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3624 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3625 )) { 3626 return 0; 3627 } 3628 } 3629 } else { 3630 ZEND_UNREACHABLE(); 3631 } 3632 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3633 if (Z_MODE(dst) == IS_REG) { 3634 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3635 return 0; 3636 } 3637 } else { 3638 ZEND_UNREACHABLE(); 3639 } 3640 } else { 3641 ZEND_UNREACHABLE(); 3642 } 3643 } 3644 return 1; 3645} 3646 3647static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3648{ 3649 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3650 3651 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 3652 3653 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3654 if (!zend_jit_save_call_chain(Dst, -1)) { 3655 return 0; 3656 } 3657 } 3658 3659 ZEND_ASSERT(opline); 3660 3661 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3662 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3663 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3664 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3665 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3666 3667 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2 3668 | GET_ZVAL_PTR TMP1, val_addr, TMP2 3669 | GC_ADDREF TMP1, TMP2w 3670 |2: 3671 } 3672 3673 | LOAD_IP_ADDR (opline - 1) 3674 | b ->trace_escape 3675 |1: 3676 3677 return 1; 3678} 3679 3680static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 3681{ 3682 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3683 3684 if (reg == ZREG_LONG_MIN_MINUS_1) { 3685 uint64_t val = 0xc3e0000000000000; 3686 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3687 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3688 } else if (reg == ZREG_LONG_MIN) { 3689 uint64_t val = 0x8000000000000000; 3690 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3691 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3692 } else if (reg == ZREG_LONG_MAX) { 3693 uint64_t val = 0x7fffffffffffffff; 3694 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3695 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3696 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 3697 uint64_t val = 0x43e0000000000000; 3698 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3699 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3700 } else if (reg == ZREG_NULL) { 3701 | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 3702 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 3703 | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 3704 | GET_ZVAL_PTR TMP1, dst, TMP2 3705 | GC_ADDREF TMP1, TMP2w 3706 |1: 3707 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 3708 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3709 3710 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3711 | TRY_ADDREF -1, REG1w, REG2, TMP1w 3712 } else { 3713 ZEND_UNREACHABLE(); 3714 } 3715 return 1; 3716} 3717 3718static int zend_jit_free_trampoline(dasm_State **Dst) 3719{ 3720 | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 3721 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 3722 | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w 3723 | beq >1 3724 | mov FCARG1x, REG0 3725 | EXT_CALL zend_jit_free_trampoline_helper, REG0 3726 |1: 3727 return 1; 3728} 3729 3730static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 3731{ 3732 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 3733 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 3734 } 3735 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3736 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3737 } 3738 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 3739 return 0; 3740 } 3741 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3742 | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 3743 } else { 3744 | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 3745 } 3746 3747 if (may_overflow && 3748 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 3749 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 3750 int32_t exit_point; 3751 const void *exit_addr; 3752 zend_jit_trace_stack *stack; 3753 uint32_t old_op1_info, old_res_info = 0; 3754 3755 stack = JIT_G(current_frame)->stack; 3756 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3757 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 3758 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3759 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 3760 } else { 3761 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 3762 } 3763 if (opline->result_type != IS_UNUSED) { 3764 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3765 if (opline->opcode == ZEND_PRE_INC) { 3766 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3767 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 3768 } else if (opline->opcode == ZEND_PRE_DEC) { 3769 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3770 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 3771 } else if (opline->opcode == ZEND_POST_INC) { 3772 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3773 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 3774 } else if (opline->opcode == ZEND_POST_DEC) { 3775 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3776 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 3777 } 3778 } 3779 3780 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 3781 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3782 | bvs &exit_addr 3783 3784 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3785 opline->result_type != IS_UNUSED) { 3786 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3787 } 3788 3789 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 3790 if (opline->result_type != IS_UNUSED) { 3791 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3792 } 3793 } else if (may_overflow) { 3794 | bvs >1 3795 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3796 opline->result_type != IS_UNUSED) { 3797 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3798 } 3799 |.cold_code 3800 |1: 3801 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3802 uint64_t val = 0x43e0000000000000; 3803 if (Z_MODE(op1_def_addr) == IS_REG) { 3804 | LOAD_64BIT_VAL TMP1, val 3805 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3806 } else { 3807 | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 3808 } 3809 } else { 3810 uint64_t val = 0xc3e0000000000000; 3811 if (Z_MODE(op1_def_addr) == IS_REG) { 3812 | LOAD_64BIT_VAL TMP1, val 3813 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3814 } else { 3815 | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 3816 } 3817 } 3818 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 3819 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 3820 } 3821 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3822 opline->result_type != IS_UNUSED) { 3823 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3824 } 3825 | b >3 3826 |.code 3827 } else { 3828 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3829 opline->result_type != IS_UNUSED) { 3830 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3831 } 3832 } 3833 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 3834 |.cold_code 3835 |2: 3836 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 3837 | SET_EX_OPLINE opline, REG0 3838 if (op1_info & MAY_BE_UNDEF) { 3839 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 3840 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 3841 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 3842 | EXT_CALL zend_jit_undefined_op_helper, REG0 3843 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 3844 op1_info |= MAY_BE_NULL; 3845 } 3846 |2: 3847 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 3848 3849 | // ZVAL_DEREF(var_ptr); 3850 if (op1_info & MAY_BE_REF) { 3851 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w 3852 | GET_Z_PTR FCARG1x, FCARG1x 3853 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 3854 | cbz TMP1, >1 3855 if (RETURN_VALUE_USED(opline)) { 3856 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3857 } else { 3858 | mov FCARG2x, xzr 3859 } 3860 if (opline->opcode == ZEND_PRE_INC) { 3861 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 3862 } else if (opline->opcode == ZEND_PRE_DEC) { 3863 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 3864 } else if (opline->opcode == ZEND_POST_INC) { 3865 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 3866 } else if (opline->opcode == ZEND_POST_DEC) { 3867 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 3868 } else { 3869 ZEND_UNREACHABLE(); 3870 } 3871 zend_jit_check_exception(Dst); 3872 | b >3 3873 |1: 3874 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 3875 |2: 3876 } 3877 3878 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3879 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 3880 3881 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3882 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 3883 } 3884 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3885 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 3886 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3887 | EXT_CALL zend_jit_pre_inc, REG0 3888 } else { 3889 | EXT_CALL increment_function, REG0 3890 } 3891 } else { 3892 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 3893 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3894 | EXT_CALL zend_jit_pre_dec, REG0 3895 } else { 3896 | EXT_CALL decrement_function, REG0 3897 } 3898 } 3899 if (may_throw) { 3900 zend_jit_check_exception(Dst); 3901 } 3902 } else { 3903 zend_reg tmp_reg; 3904 3905 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3906 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3907 } 3908 if (Z_MODE(op1_def_addr) == IS_REG) { 3909 tmp_reg = Z_REG(op1_def_addr); 3910 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 3911 tmp_reg = Z_REG(op1_addr); 3912 } else { 3913 tmp_reg = ZREG_FPR0; 3914 } 3915 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 3916 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3917 uint64_t val = 0x3ff0000000000000; // 1.0 3918 | LOAD_64BIT_VAL TMP1, val 3919 | fmov FPTMP, TMP1 3920 | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3921 } else { 3922 uint64_t val = 0x3ff0000000000000; // 1.0 3923 | LOAD_64BIT_VAL TMP1, val 3924 | fmov FPTMP, TMP1 3925 | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3926 } 3927 | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 3928 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3929 opline->result_type != IS_UNUSED) { 3930 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3931 | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w 3932 } 3933 } 3934 | b >3 3935 |.code 3936 } 3937 |3: 3938 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 3939 return 0; 3940 } 3941 if (opline->result_type != IS_UNUSED) { 3942 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 3943 return 0; 3944 } 3945 } 3946 return 1; 3947} 3948 3949static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 3950{ 3951 if ((opline+1)->opcode == ZEND_OP_DATA 3952 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 3953 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 3954 return 1; 3955 } 3956 return 3957 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3958 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 3959 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3960 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 3961 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3962 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 3963} 3964 3965static int zend_jit_math_long_long(dasm_State **Dst, 3966 const zend_op *opline, 3967 zend_uchar opcode, 3968 zend_jit_addr op1_addr, 3969 zend_jit_addr op2_addr, 3970 zend_jit_addr res_addr, 3971 uint32_t res_info, 3972 uint32_t res_use_info, 3973 int may_overflow) 3974{ 3975 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 3976 zend_reg result_reg; 3977 zend_reg tmp_reg = ZREG_REG0; 3978 bool use_ovf_flag = 1; 3979 3980 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 3981 if (may_overflow && (res_info & MAY_BE_GUARD) 3982 && JIT_G(current_frame) 3983 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 3984 result_reg = ZREG_REG0; 3985 } else { 3986 result_reg = Z_REG(res_addr); 3987 } 3988 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 3989 result_reg = Z_REG(op1_addr); 3990 } else if (Z_REG(res_addr) != ZREG_REG0) { 3991 result_reg = ZREG_REG0; 3992 } else { 3993 /* ASSIGN_DIM_OP */ 3994 result_reg = ZREG_FCARG1; 3995 tmp_reg = ZREG_FCARG1; 3996 } 3997 3998 if (opcode == ZEND_MUL && 3999 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4000 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4001 if (Z_MODE(op1_addr) == IS_REG) { 4002 | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4003 } else { 4004 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4005 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4006 } 4007 } else if (opcode == ZEND_MUL && 4008 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4009 !may_overflow && 4010 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4011 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4012 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4013 | lsl Rx(result_reg), Rx(result_reg), TMP1 4014 } else if (opcode == ZEND_MUL && 4015 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4016 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4017 if (Z_MODE(op2_addr) == IS_REG) { 4018 | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) 4019 } else { 4020 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4021 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4022 } 4023 } else if (opcode == ZEND_MUL && 4024 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4025 !may_overflow && 4026 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4027 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4028 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4029 | lsl Rx(result_reg), Rx(result_reg), TMP1 4030 } else if (opcode == ZEND_DIV && 4031 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4032 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4033 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4034 | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4035#if 0 4036 /* x86 specific optimizations through LEA instraction are not supported on ARM */ 4037 } else if (opcode == ZEND_ADD && 4038 !may_overflow && 4039 Z_MODE(op1_addr) == IS_REG && 4040 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4041 | NIY // TODO: test 4042 } else if (opcode == ZEND_ADD && 4043 !may_overflow && 4044 Z_MODE(op2_addr) == IS_REG && 4045 Z_MODE(op1_addr) == IS_CONST_ZVAL) { 4046 | NIY // TODO: test 4047 } else if (opcode == ZEND_SUB && 4048 !may_overflow && 4049 Z_MODE(op1_addr) == IS_REG && 4050 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4051 | NIY // TODO: test 4052#endif 4053 } else if (opcode == ZEND_MUL) { 4054 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4055 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4056 | mul Rx(result_reg), TMP1, TMP2 4057 if(may_overflow) { 4058 /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. 4059 * For signed multiplication, the top 65 bits of the result will contain 4060 * either all zeros or all ones if no overflow occurred. 4061 * Flag: bne -> overflow. beq -> no overflow. 4062 */ 4063 use_ovf_flag = 0; 4064 | smulh TMP1, TMP1, TMP2 4065 | cmp TMP1, Rx(result_reg), asr #63 4066 } 4067 } else { 4068 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4069 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4070 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4071 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4072 /* +/- 0 */ 4073 may_overflow = 0; 4074 } else if (same_ops && opcode != ZEND_DIV) { 4075 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4076 } else { 4077 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4078 } 4079 } 4080 if (may_overflow) { 4081 if (res_info & MAY_BE_GUARD) { 4082 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4083 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4084 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4085 if (use_ovf_flag) { 4086 | bvs &exit_addr 4087 } else { 4088 | bne &exit_addr 4089 } 4090 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4091 | mov Rx(Z_REG(res_addr)), Rx(result_reg) 4092 } 4093 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4094 if (use_ovf_flag) { 4095 | bvc &exit_addr 4096 } else { 4097 | beq &exit_addr 4098 } 4099 } else { 4100 ZEND_UNREACHABLE(); 4101 } 4102 } else { 4103 if (res_info & MAY_BE_LONG) { 4104 if (use_ovf_flag) { 4105 | bvs >1 4106 } else { 4107 | bne >1 4108 } 4109 } else { 4110 if (use_ovf_flag) { 4111 | bvc >1 4112 } else { 4113 | beq >1 4114 } 4115 } 4116 } 4117 } 4118 4119 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4120 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4121 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4122 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4123 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4124 } 4125 } 4126 } 4127 4128 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4129 zend_reg tmp_reg1 = ZREG_FPR0; 4130 zend_reg tmp_reg2 = ZREG_FPR1; 4131 4132 if (res_info & MAY_BE_LONG) { 4133 |.cold_code 4134 |1: 4135 } 4136 4137 do { 4138 if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4139 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { 4140 if (opcode == ZEND_ADD) { 4141 uint64_t val = 0x43e0000000000000; 4142 if (Z_MODE(res_addr) == IS_REG) { 4143 | LOAD_64BIT_VAL TMP1, val 4144 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4145 } else { 4146 | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 4147 } 4148 break; 4149 } else if (opcode == ZEND_SUB) { 4150 uint64_t val = 0xc3e0000000000000; 4151 if (Z_MODE(res_addr) == IS_REG) { 4152 | LOAD_64BIT_VAL TMP1, val 4153 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4154 } else { 4155 | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 4156 } 4157 break; 4158 } 4159 } 4160 4161 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 4162 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 4163 | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4164 | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 4165 } while (0); 4166 4167 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4168 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4169 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4170 } 4171 if (res_info & MAY_BE_LONG) { 4172 | b >2 4173 |.code 4174 } 4175 |2: 4176 } 4177 4178 return 1; 4179} 4180 4181static int zend_jit_math_long_double(dasm_State **Dst, 4182 zend_uchar opcode, 4183 zend_jit_addr op1_addr, 4184 zend_jit_addr op2_addr, 4185 zend_jit_addr res_addr, 4186 uint32_t res_use_info) 4187{ 4188 zend_reg result_reg = 4189 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; 4190 zend_reg op2_reg; 4191 4192 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 4193 4194 if (Z_MODE(op2_addr) == IS_REG) { 4195 op2_reg = Z_REG(op2_addr); 4196 } else { 4197 op2_reg = ZREG_FPTMP; 4198 | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 4199 } 4200 4201 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg 4202 4203 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4204 4205 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4206 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4207 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4208 } 4209 } 4210 4211 return 1; 4212} 4213 4214static int zend_jit_math_double_long(dasm_State **Dst, 4215 zend_uchar opcode, 4216 zend_jit_addr op1_addr, 4217 zend_jit_addr op2_addr, 4218 zend_jit_addr res_addr, 4219 uint32_t res_use_info) 4220{ 4221 zend_reg result_reg, op1_reg, op2_reg; 4222 4223 if (zend_is_commutative(opcode) 4224 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4225 if (Z_MODE(res_addr) == IS_REG) { 4226 result_reg = Z_REG(res_addr); 4227 } else { 4228 result_reg = ZREG_FPR0; 4229 } 4230 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4231 if (Z_MODE(op1_addr) == IS_REG) { 4232 op1_reg = Z_REG(op1_addr); 4233 } else { 4234 op1_reg = ZREG_FPTMP; 4235 | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 4236 } 4237 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg 4238 } else { 4239 if (Z_MODE(res_addr) == IS_REG) { 4240 result_reg = Z_REG(res_addr); 4241 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4242 result_reg = Z_REG(op1_addr); 4243 } else { 4244 result_reg = ZREG_FPR0; 4245 } 4246 4247 if (Z_MODE(op1_addr) == IS_REG) { 4248 op1_reg = Z_REG(op1_addr); 4249 } else { 4250 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4251 op1_reg = result_reg; 4252 } 4253 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4254 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4255 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4256 /* +/- 0 */ 4257 } else { 4258 op2_reg = ZREG_FPTMP; 4259 | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4260 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4261 } 4262 } 4263 4264 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4265 4266 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4267 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4268 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4269 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4270 } 4271 } 4272 } 4273 4274 return 1; 4275} 4276 4277static int zend_jit_math_double_double(dasm_State **Dst, 4278 zend_uchar opcode, 4279 zend_jit_addr op1_addr, 4280 zend_jit_addr op2_addr, 4281 zend_jit_addr res_addr, 4282 uint32_t res_use_info) 4283{ 4284 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4285 zend_reg result_reg, op1_reg, op2_reg; 4286 zend_jit_addr val_addr; 4287 4288 if (Z_MODE(res_addr) == IS_REG) { 4289 result_reg = Z_REG(res_addr); 4290 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4291 result_reg = Z_REG(op1_addr); 4292 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4293 result_reg = Z_REG(op2_addr); 4294 } else { 4295 result_reg = ZREG_FPR0; 4296 } 4297 4298 if (Z_MODE(op1_addr) == IS_REG) { 4299 op1_reg = Z_REG(op1_addr); 4300 val_addr = op2_addr; 4301 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4302 op1_reg = Z_REG(op2_addr); 4303 val_addr = op1_addr; 4304 } else { 4305 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4306 op1_reg = result_reg; 4307 val_addr = op2_addr; 4308 } 4309 4310 if ((opcode == ZEND_MUL) && 4311 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4312 | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4313 } else { 4314 if (same_ops) { 4315 op2_reg = op1_reg; 4316 } else if (Z_MODE(val_addr) == IS_REG) { 4317 op2_reg = Z_REG(val_addr); 4318 } else { 4319 op2_reg = ZREG_FPTMP; 4320 | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 4321 } 4322 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4323 } 4324 4325 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4326 4327 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4328 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4329 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4330 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4331 } 4332 } 4333 } 4334 return 1; 4335} 4336 4337static int zend_jit_math_helper(dasm_State **Dst, 4338 const zend_op *opline, 4339 zend_uchar opcode, 4340 zend_uchar op1_type, 4341 znode_op op1, 4342 zend_jit_addr op1_addr, 4343 uint32_t op1_info, 4344 zend_uchar op2_type, 4345 znode_op op2, 4346 zend_jit_addr op2_addr, 4347 uint32_t op2_info, 4348 uint32_t res_var, 4349 zend_jit_addr res_addr, 4350 uint32_t res_info, 4351 uint32_t res_use_info, 4352 int may_overflow, 4353 int may_throw) 4354/* Labels: 1,2,3,4,5,6 */ 4355{ 4356 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4357 4358 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4359 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4360 if (op1_info & MAY_BE_DOUBLE) { 4361 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 4362 } else { 4363 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4364 } 4365 } 4366 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4367 if (op2_info & MAY_BE_DOUBLE) { 4368 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 4369 |.cold_code 4370 |1: 4371 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4372 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4373 } 4374 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4375 return 0; 4376 } 4377 | b >5 4378 |.code 4379 } else { 4380 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4381 } 4382 } 4383 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4384 return 0; 4385 } 4386 if (op1_info & MAY_BE_DOUBLE) { 4387 |.cold_code 4388 |3: 4389 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4390 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4391 } 4392 if (op2_info & MAY_BE_DOUBLE) { 4393 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4394 if (!same_ops) { 4395 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4396 } else { 4397 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4398 } 4399 } 4400 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4401 return 0; 4402 } 4403 | b >5 4404 } 4405 if (!same_ops) { 4406 |1: 4407 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4408 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4409 } 4410 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4411 return 0; 4412 } 4413 | b >5 4414 } 4415 |.code 4416 } 4417 } else if ((op1_info & MAY_BE_DOUBLE) && 4418 !(op1_info & MAY_BE_LONG) && 4419 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4420 (res_info & MAY_BE_DOUBLE)) { 4421 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4422 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4423 } 4424 if (op2_info & MAY_BE_DOUBLE) { 4425 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4426 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4427 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4428 } else { 4429 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4430 } 4431 } 4432 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4433 return 0; 4434 } 4435 } 4436 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4437 if (op2_info & MAY_BE_DOUBLE) { 4438 |.cold_code 4439 } 4440 |1: 4441 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4442 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4443 } 4444 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4445 return 0; 4446 } 4447 if (op2_info & MAY_BE_DOUBLE) { 4448 | b >5 4449 |.code 4450 } 4451 } 4452 } else if ((op2_info & MAY_BE_DOUBLE) && 4453 !(op2_info & MAY_BE_LONG) && 4454 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4455 (res_info & MAY_BE_DOUBLE)) { 4456 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4457 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4458 } 4459 if (op1_info & MAY_BE_DOUBLE) { 4460 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4461 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4462 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 4463 } else { 4464 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4465 } 4466 } 4467 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4468 return 0; 4469 } 4470 } 4471 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4472 if (op1_info & MAY_BE_DOUBLE) { 4473 |.cold_code 4474 } 4475 |1: 4476 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4477 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4478 } 4479 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4480 return 0; 4481 } 4482 if (op1_info & MAY_BE_DOUBLE) { 4483 | b >5 4484 |.code 4485 } 4486 } 4487 } 4488 4489 |5: 4490 4491 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4492 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4493 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4494 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4495 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4496 |.cold_code 4497 } 4498 |6: 4499 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4500 if (Z_MODE(res_addr) == IS_REG) { 4501 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4502 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4503 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4504 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4505 } 4506 if (Z_MODE(op1_addr) == IS_REG) { 4507 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4508 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4509 return 0; 4510 } 4511 op1_addr = real_addr; 4512 } 4513 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4514 } else { 4515 if (Z_MODE(op1_addr) == IS_REG) { 4516 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4517 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4518 return 0; 4519 } 4520 op1_addr = real_addr; 4521 } 4522 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4523 if (Z_MODE(res_addr) == IS_REG) { 4524 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4525 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4526 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4527 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4528 } 4529 } 4530 if (Z_MODE(op2_addr) == IS_REG) { 4531 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4532 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4533 return 0; 4534 } 4535 op2_addr = real_addr; 4536 } 4537 | LOAD_ZVAL_ADDR CARG3, op2_addr 4538 | SET_EX_OPLINE opline, REG0 4539 if (opcode == ZEND_ADD) { 4540 | EXT_CALL add_function, REG0 4541 } else if (opcode == ZEND_SUB) { 4542 | EXT_CALL sub_function, REG0 4543 } else if (opcode == ZEND_MUL) { 4544 | EXT_CALL mul_function, REG0 4545 } else if (opcode == ZEND_DIV) { 4546 | EXT_CALL div_function, REG0 4547 } else { 4548 ZEND_UNREACHABLE(); 4549 } 4550 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4551 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4552 if (may_throw) { 4553 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4554 zend_jit_check_exception_undef_result(Dst, opline); 4555 } else { 4556 zend_jit_check_exception(Dst); 4557 } 4558 } 4559 if (Z_MODE(res_addr) == IS_REG) { 4560 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4561 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4562 return 0; 4563 } 4564 } 4565 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4566 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4567 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4568 | b <5 4569 |.code 4570 } 4571 } 4572 4573 return 1; 4574} 4575 4576static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4577{ 4578 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4579 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4580 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4581 4582 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4583 return 0; 4584 } 4585 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4586 return 0; 4587 } 4588 return 1; 4589} 4590 4591static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 4592{ 4593 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 4594 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4595 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4596 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4597 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4598 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4599 } else { 4600 | GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1 4601 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4602 | mov FCARG2x, REG0 4603 } 4604 | EXT_CALL zend_jit_add_arrays_helper, REG0 4605 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 4606 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 4607 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4608 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4609 return 1; 4610} 4611 4612static int zend_jit_long_math_helper(dasm_State **Dst, 4613 const zend_op *opline, 4614 zend_uchar opcode, 4615 zend_uchar op1_type, 4616 znode_op op1, 4617 zend_jit_addr op1_addr, 4618 uint32_t op1_info, 4619 zend_ssa_range *op1_range, 4620 zend_uchar op2_type, 4621 znode_op op2, 4622 zend_jit_addr op2_addr, 4623 uint32_t op2_info, 4624 zend_ssa_range *op2_range, 4625 uint32_t res_var, 4626 zend_jit_addr res_addr, 4627 uint32_t res_info, 4628 uint32_t res_use_info, 4629 int may_throw) 4630/* Labels: 6 */ 4631{ 4632 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4633 zend_reg result_reg; 4634 4635 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4636 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4637 } 4638 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4639 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4640 } 4641 4642 if (Z_MODE(res_addr) == IS_REG) { 4643 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 4644 && opline->op2_type != IS_CONST) { 4645 result_reg = ZREG_REG0; 4646 } else { 4647 result_reg = Z_REG(res_addr); 4648 } 4649 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4650 result_reg = Z_REG(op1_addr); 4651 } else if (Z_REG(res_addr) != ZREG_REG0) { 4652 result_reg = ZREG_REG0; 4653 } else { 4654 /* ASSIGN_DIM_OP */ 4655 result_reg = ZREG_FCARG1; 4656 } 4657 4658 if (opcode == ZEND_SL) { 4659 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4660 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4661 4662 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4663 if (EXPECTED(op2_lval > 0)) { 4664 | mov Rx(result_reg), xzr 4665 } else { 4666 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4667 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4668 | SET_EX_OPLINE opline, REG0 4669 | b ->negative_shift 4670 } 4671 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 4672 | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4673 } else { 4674 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4675 | lsl Rx(result_reg), Rx(result_reg), #op2_lval 4676 } 4677 } else { 4678 zend_reg op2_reg; 4679 4680 if (Z_MODE(op2_addr) == IS_REG) { 4681 op2_reg = Z_REG(op2_addr); 4682 } else { 4683 op2_reg = ZREG_TMP2; 4684 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4685 } 4686 if (!op2_range || 4687 op2_range->min < 0 || 4688 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4689 4690 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4691 | bhs >1 4692 |.cold_code 4693 |1: 4694 | mov Rx(result_reg), xzr 4695 | cmp Rx(op2_reg), xzr 4696 | bgt >1 4697 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4698 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4699 | SET_EX_OPLINE opline, REG0 4700 | b ->negative_shift 4701 |.code 4702 } 4703 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4704 | lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4705 |1: 4706 } 4707 } else if (opcode == ZEND_SR) { 4708 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4709 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4710 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4711 4712 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4713 if (EXPECTED(op2_lval > 0)) { 4714 | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4715 } else { 4716 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4717 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4718 | SET_EX_OPLINE opline, REG0 4719 | b ->negative_shift 4720 } 4721 } else { 4722 | asr Rx(result_reg), Rx(result_reg), #op2_lval 4723 } 4724 } else { 4725 zend_reg op2_reg; 4726 4727 if (Z_MODE(op2_addr) == IS_REG) { 4728 op2_reg = Z_REG(op2_addr); 4729 } else { 4730 op2_reg = ZREG_TMP2; 4731 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4732 } 4733 if (!op2_range || 4734 op2_range->min < 0 || 4735 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4736 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4737 | bhs >1 4738 |.cold_code 4739 |1: 4740 | cmp Rx(op2_reg), xzr 4741 | mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4742 | bgt >1 4743 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4744 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4745 | SET_EX_OPLINE opline, REG0 4746 | b ->negative_shift 4747 |.code 4748 } 4749 |1: 4750 | asr Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4751 } 4752 } else if (opcode == ZEND_MOD) { 4753 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4754 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4755 4756 if (op2_lval == 0) { 4757 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4758 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4759 | SET_EX_OPLINE opline, REG0 4760 | b ->mod_by_zero 4761 } else if (op2_lval == -1) { 4762 | mov Rx(result_reg), xzr 4763 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 4764 zval tmp; 4765 zend_jit_addr tmp_addr; 4766 4767 /* Optimisation for mod of power of 2 */ 4768 ZVAL_LONG(&tmp, op2_lval - 1); 4769 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 4770 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4771 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1 4772 } else { 4773 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4774 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4775 | sdiv Rx(result_reg), TMP1, TMP2 4776 | msub Rx(result_reg), Rx(result_reg), TMP2, TMP1 4777 } 4778 } else { 4779 zend_reg op2_reg; 4780 4781 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 4782 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 4783 op2_reg = ZREG_TMP2; 4784 } else { 4785 ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG); 4786 op2_reg = Z_REG(op2_addr); 4787 } 4788 4789 if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 4790 | cbz Rx(op2_reg), >1 4791 |.cold_code 4792 |1: 4793 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4794 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4795 | SET_EX_OPLINE opline, REG0 4796 | b ->mod_by_zero 4797 |.code 4798 } 4799 4800 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 4801 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 4802 | cmn Rx(op2_reg), #1 4803 | beq >1 4804 |.cold_code 4805 |1: 4806 | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 4807 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4808 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4809 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4810 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4811 } 4812 } 4813 } 4814 | b >5 4815 |.code 4816 } 4817 4818 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4819 | sdiv Rx(result_reg), TMP1, Rx(op2_reg) 4820 | msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1 4821 } 4822 } else if (same_ops) { 4823 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4824 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4825 } else { 4826 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4827 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4828 } 4829 4830 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 4831 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4832 } 4833 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4834 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4835 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4836 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4837 } 4838 } 4839 } 4840 4841 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 4842 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4843 if ((op1_info & MAY_BE_LONG) && 4844 (op2_info & MAY_BE_LONG)) { 4845 |.cold_code 4846 } 4847 |6: 4848 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4849 if (Z_MODE(res_addr) == IS_REG) { 4850 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4851 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4852 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4853 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4854 } 4855 if (Z_MODE(op1_addr) == IS_REG) { 4856 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4857 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4858 return 0; 4859 } 4860 op1_addr = real_addr; 4861 } 4862 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4863 } else { 4864 if (Z_MODE(op1_addr) == IS_REG) { 4865 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4866 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4867 return 0; 4868 } 4869 op1_addr = real_addr; 4870 } 4871 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4872 if (Z_MODE(res_addr) == IS_REG) { 4873 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4874 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4875 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4876 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4877 } 4878 } 4879 if (Z_MODE(op2_addr) == IS_REG) { 4880 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4881 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4882 return 0; 4883 } 4884 op2_addr = real_addr; 4885 } 4886 | LOAD_ZVAL_ADDR CARG3, op2_addr 4887 | SET_EX_OPLINE opline, REG0 4888 if (opcode == ZEND_BW_OR) { 4889 | EXT_CALL bitwise_or_function, REG0 4890 } else if (opcode == ZEND_BW_AND) { 4891 | EXT_CALL bitwise_and_function, REG0 4892 } else if (opcode == ZEND_BW_XOR) { 4893 | EXT_CALL bitwise_xor_function, REG0 4894 } else if (opcode == ZEND_SL) { 4895 | EXT_CALL shift_left_function, REG0 4896 } else if (opcode == ZEND_SR) { 4897 | EXT_CALL shift_right_function, REG0 4898 } else if (opcode == ZEND_MOD) { 4899 | EXT_CALL mod_function, REG0 4900 } else { 4901 ZEND_UNREACHABLE(); 4902 } 4903 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4904 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4905 if (may_throw) { 4906 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4907 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4908 | cbnz TMP2, ->exception_handler_free_op2 4909 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4910 zend_jit_check_exception_undef_result(Dst, opline); 4911 } else { 4912 zend_jit_check_exception(Dst); 4913 } 4914 } 4915 if (Z_MODE(res_addr) == IS_REG) { 4916 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4917 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4918 return 0; 4919 } 4920 } 4921 if ((op1_info & MAY_BE_LONG) && 4922 (op2_info & MAY_BE_LONG)) { 4923 | b >5 4924 |.code 4925 } 4926 } 4927 |5: 4928 4929 return 1; 4930} 4931 4932static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 4933{ 4934 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4935 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 4936 4937 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 4938 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 4939 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 4940 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 4941 return 0; 4942 } 4943 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4944 return 0; 4945 } 4946 return 1; 4947} 4948 4949static int zend_jit_concat_helper(dasm_State **Dst, 4950 const zend_op *opline, 4951 zend_uchar op1_type, 4952 znode_op op1, 4953 zend_jit_addr op1_addr, 4954 uint32_t op1_info, 4955 zend_uchar op2_type, 4956 znode_op op2, 4957 zend_jit_addr op2_addr, 4958 uint32_t op2_info, 4959 zend_jit_addr res_addr, 4960 int may_throw) 4961{ 4962 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 4963 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 4964 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 4965 } 4966 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 4967 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 4968 } 4969 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 4970 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4971 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4972 } 4973 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 4974 | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 4975 /* concatination with itself may reduce refcount */ 4976 op2_info |= MAY_BE_RC1; 4977 } else { 4978 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4979 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4980 } 4981 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4982 | LOAD_ZVAL_ADDR CARG3, op2_addr 4983 if (op1_type == IS_CV || op1_type == IS_CONST) { 4984 | EXT_CALL zend_jit_fast_concat_helper, REG0 4985 } else { 4986 | EXT_CALL zend_jit_fast_concat_tmp_helper, REG0 4987 } 4988 } 4989 /* concatination with empty string may increase refcount */ 4990 op2_info |= MAY_BE_RCN; 4991 | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4992 |5: 4993 } 4994 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 4995 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 4996 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 4997 |.cold_code 4998 |6: 4999 } 5000 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5001 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5002 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5003 } 5004 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5005 } else { 5006 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5007 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5008 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5009 } 5010 } 5011 | LOAD_ZVAL_ADDR CARG3, op2_addr 5012 | SET_EX_OPLINE opline, REG0 5013 | EXT_CALL concat_function, REG0 5014 /* concatination with empty string may increase refcount */ 5015 op1_info |= MAY_BE_RCN; 5016 op2_info |= MAY_BE_RCN; 5017 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5018 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5019 if (may_throw) { 5020 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5021 zend_jit_check_exception_undef_result(Dst, opline); 5022 } else { 5023 zend_jit_check_exception(Dst); 5024 } 5025 } 5026 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5027 | b <5 5028 |.code 5029 } 5030 } 5031 5032 return 1; 5033} 5034 5035static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5036{ 5037 zend_jit_addr op1_addr, op2_addr; 5038 5039 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5040 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5041 5042 op1_addr = OP1_ADDR(); 5043 op2_addr = OP2_ADDR(); 5044 5045 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5046} 5047 5048static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5049/* Labels: 1,2,3,4,5 */ 5050{ 5051 zend_jit_addr op2_addr = OP2_ADDR(); 5052 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5053 5054 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5055 && type == BP_VAR_R 5056 && !exit_addr) { 5057 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5058 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5059 if (!exit_addr) { 5060 return 0; 5061 } 5062 } 5063 5064 if (op2_info & MAY_BE_LONG) { 5065 bool op2_loaded = 0; 5066 bool packed_loaded = 0; 5067 bool bad_packed_key = 0; 5068 5069 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5070 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5071 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 5072 } 5073 if (op1_info & MAY_BE_PACKED_GUARD) { 5074 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5075 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5076 5077 if (!exit_addr) { 5078 return 0; 5079 } 5080 if (op1_info & MAY_BE_ARRAY_PACKED) { 5081 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5082 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5083 | beq &exit_addr 5084 } else { 5085 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5086 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5087 | bne &exit_addr 5088 } 5089 } 5090 if (type == BP_VAR_W) { 5091 | // hval = Z_LVAL_P(dim); 5092 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5093 op2_loaded = 1; 5094 } 5095 if (op1_info & MAY_BE_ARRAY_PACKED) { 5096 zend_long val = -1; 5097 5098 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5099 val = Z_LVAL_P(Z_ZV(op2_addr)); 5100 if (val >= 0 && val < HT_MAX_SIZE) { 5101 packed_loaded = 1; 5102 } else { 5103 bad_packed_key = 1; 5104 } 5105 } else { 5106 if (!op2_loaded) { 5107 | // hval = Z_LVAL_P(dim); 5108 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5109 op2_loaded = 1; 5110 } 5111 packed_loaded = 1; 5112 } 5113 5114 if (dim_type == IS_UNDEF && type == BP_VAR_W) { 5115 /* don't generate "fast" code for packed array */ 5116 packed_loaded = 0; 5117 } 5118 5119 if (packed_loaded) { 5120 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5121 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5122 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5123 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5124 | beq >4 // HASH_FIND 5125 } 5126 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5127 5128 | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 5129 if (val == 0) { 5130 | cmp REG0, xzr 5131 } else if (val > 0 && !op2_loaded) { 5132 | CMP_64_WITH_CONST REG0, val, TMP1 5133 } else { 5134 | cmp REG0, FCARG2x 5135 } 5136 5137 if (type == BP_JIT_IS) { 5138 if (not_found_exit_addr) { 5139 | bls ¬_found_exit_addr 5140 } else { 5141 | bls >9 // NOT_FOUND 5142 } 5143 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5144 | bls &exit_addr 5145 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5146 | bls ¬_found_exit_addr 5147 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5148 | bls ¬_found_exit_addr 5149 } else if (type == BP_VAR_IS && found_exit_addr) { 5150 | bls >7 // NOT_FOUND 5151 } else { 5152 | bls >2 // NOT_FOUND 5153 } 5154 | // _ret = &_ht->arData[_h].val; 5155 if (val >= 0) { 5156 | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] 5157 if (val != 0) { 5158 | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1 5159 } 5160 } else { 5161 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 5162 | add REG0, TMP1, FCARG2x, lsl #5 5163 } 5164 } 5165 } 5166 switch (type) { 5167 case BP_JIT_IS: 5168 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5169 if (packed_loaded) { 5170 | b >5 5171 } 5172 |4: 5173 if (!op2_loaded) { 5174 | // hval = Z_LVAL_P(dim); 5175 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5176 } 5177 if (packed_loaded) { 5178 | EXT_CALL _zend_hash_index_find, REG0 5179 } else { 5180 | EXT_CALL zend_hash_index_find, REG0 5181 } 5182 | mov REG0, RETVALx 5183 if (not_found_exit_addr) { 5184 | cbz REG0, ¬_found_exit_addr 5185 } else { 5186 | cbz REG0, >9 // NOT_FOUND 5187 } 5188 if (op2_info & MAY_BE_STRING) { 5189 | b >5 5190 } 5191 } else if (packed_loaded) { 5192 if (op2_info & MAY_BE_STRING) { 5193 | b >5 5194 } 5195 } else if (not_found_exit_addr) { 5196 | b ¬_found_exit_addr 5197 } else { 5198 | b >9 // NOT_FOUND 5199 } 5200 break; 5201 case BP_VAR_R: 5202 case BP_VAR_IS: 5203 case BP_VAR_UNSET: 5204 if (packed_loaded) { 5205 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5206 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5207 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5208 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5209 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5210 | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w 5211 } 5212 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5213 | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w 5214 } else if (type == BP_VAR_IS && found_exit_addr) { 5215 | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND 5216 } else { 5217 | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND 5218 } 5219 } 5220 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5221 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5222 | b &exit_addr 5223 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5224 | b ¬_found_exit_addr 5225 } else if (type == BP_VAR_IS && found_exit_addr) { 5226 | b >7 // NOT_FOUND 5227 } else { 5228 | b >2 // NOT_FOUND 5229 } 5230 } 5231 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5232 |4: 5233 if (!op2_loaded) { 5234 | // hval = Z_LVAL_P(dim); 5235 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5236 } 5237 if (packed_loaded) { 5238 | EXT_CALL _zend_hash_index_find, REG0 5239 } else { 5240 | EXT_CALL zend_hash_index_find, REG0 5241 } 5242 | mov REG0, RETVALx 5243 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5244 | cbz REG0, &exit_addr 5245 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5246 | cbz REG0, ¬_found_exit_addr 5247 } else if (type == BP_VAR_IS && found_exit_addr) { 5248 | cbz REG0, >7 // NOT_FOUND 5249 } else { 5250 | cbz REG0, >2 // NOT_FOUND 5251 } 5252 } 5253 |.cold_code 5254 |2: 5255 switch (type) { 5256 case BP_VAR_R: 5257 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5258 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5259 | // retval = &EG(uninitialized_zval); 5260 | UNDEFINED_OFFSET opline 5261 | b >9 5262 } 5263 break; 5264 case BP_VAR_IS: 5265 case BP_VAR_UNSET: 5266 if (!not_found_exit_addr && !found_exit_addr) { 5267 | // retval = &EG(uninitialized_zval); 5268 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5269 | b >9 5270 } 5271 break; 5272 default: 5273 ZEND_UNREACHABLE(); 5274 } 5275 |.code 5276 break; 5277 case BP_VAR_RW: 5278 if (packed_loaded && !not_found_exit_addr) { 5279 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5280 } 5281 if (!packed_loaded || 5282 !not_found_exit_addr || 5283 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5284 if (packed_loaded && not_found_exit_addr) { 5285 |.cold_code 5286 } 5287 |2: 5288 |4: 5289 if (!op2_loaded) { 5290 | // hval = Z_LVAL_P(dim); 5291 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5292 } 5293 | SET_EX_OPLINE opline, REG0 5294 if (packed_loaded) { 5295 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0 5296 } else { 5297 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 5298 } 5299 | mov REG0, RETVALx 5300 if (not_found_exit_addr) { 5301 if (packed_loaded) { 5302 | cbnz REG0, >8 5303 | b ¬_found_exit_addr 5304 |.code 5305 } else { 5306 | cbz REG0, ¬_found_exit_addr 5307 } 5308 } else { 5309 | cbz REG0, >9 5310 } 5311 } 5312 break; 5313 case BP_VAR_W: 5314 if (packed_loaded) { 5315 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5316 } 5317 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5318 |2: 5319 |4: 5320 if (!op2_loaded) { 5321 | // hval = Z_LVAL_P(dim); 5322 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5323 } 5324 | EXT_CALL zend_hash_index_lookup, REG0 5325 | mov REG0, RETVALx 5326 } 5327 break; 5328 default: 5329 ZEND_UNREACHABLE(); 5330 } 5331 5332 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5333 | b >8 5334 } 5335 } 5336 5337 if (op2_info & MAY_BE_STRING) { 5338 |3: 5339 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5340 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5341 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 5342 } 5343 | // offset_key = Z_STR_P(dim); 5344 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5345 | // retval = zend_hash_find(ht, offset_key); 5346 switch (type) { 5347 case BP_JIT_IS: 5348 if (opline->op2_type != IS_CONST) { 5349 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5350 | cmp TMP1w, #((uint8_t) ('9')) 5351 | ble >1 5352 |.cold_code 5353 |1: 5354 | EXT_CALL zend_jit_symtable_find, REG0 5355 | b >1 5356 |.code 5357 | EXT_CALL zend_hash_find, REG0 5358 |1: 5359 } else { 5360 | EXT_CALL zend_hash_find_known_hash, REG0 5361 } 5362 | mov REG0, RETVALx 5363 if (not_found_exit_addr) { 5364 | cbz REG0, ¬_found_exit_addr 5365 } else { 5366 | cbz REG0, >9 // NOT_FOUND 5367 } 5368 break; 5369 case BP_VAR_R: 5370 case BP_VAR_IS: 5371 case BP_VAR_UNSET: 5372 if (opline->op2_type != IS_CONST) { 5373 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5374 | cmp TMP1w, #((uint8_t) ('9')) 5375 | ble >1 5376 |.cold_code 5377 |1: 5378 | EXT_CALL zend_jit_symtable_find, REG0 5379 | b >1 5380 |.code 5381 | EXT_CALL zend_hash_find, REG0 5382 |1: 5383 } else { 5384 | EXT_CALL zend_hash_find_known_hash, REG0 5385 } 5386 | mov REG0, RETVALx 5387 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5388 | cbz REG0, &exit_addr 5389 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5390 | cbz REG0, ¬_found_exit_addr 5391 } else if (type == BP_VAR_IS && found_exit_addr) { 5392 | cbz REG0, >7 5393 } else { 5394 | cbz REG0, >2 // NOT_FOUND 5395 |.cold_code 5396 |2: 5397 switch (type) { 5398 case BP_VAR_R: 5399 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5400 | UNDEFINED_INDEX opline 5401 | b >9 5402 break; 5403 case BP_VAR_IS: 5404 case BP_VAR_UNSET: 5405 | // retval = &EG(uninitialized_zval); 5406 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5407 | b >9 5408 break; 5409 default: 5410 ZEND_UNREACHABLE(); 5411 } 5412 |.code 5413 } 5414 break; 5415 case BP_VAR_RW: 5416 | SET_EX_OPLINE opline, REG0 5417 if (opline->op2_type != IS_CONST) { 5418 | EXT_CALL zend_jit_symtable_lookup_rw, REG0 5419 } else { 5420 | EXT_CALL zend_jit_hash_lookup_rw, REG0 5421 } 5422 | mov REG0, RETVALx 5423 if (not_found_exit_addr) { 5424 | cbz REG0, ¬_found_exit_addr 5425 } else { 5426 | cbz REG0, >9 5427 } 5428 break; 5429 case BP_VAR_W: 5430 if (opline->op2_type != IS_CONST) { 5431 | EXT_CALL zend_jit_symtable_lookup_w, REG0 5432 } else { 5433 | EXT_CALL zend_hash_lookup, REG0 5434 } 5435 | mov REG0, RETVALx 5436 break; 5437 default: 5438 ZEND_UNREACHABLE(); 5439 } 5440 } 5441 5442 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5443 |5: 5444 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5445 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 5446 } 5447 | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] 5448 | cmp TMP1w, #IS_NULL 5449 if (not_found_exit_addr) { 5450 | ble ¬_found_exit_addr 5451 } else if (found_exit_addr) { 5452 | bgt &found_exit_addr 5453 } else { 5454 | ble >9 // NOT FOUND 5455 } 5456 } 5457 5458 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5459 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5460 |.cold_code 5461 |3: 5462 } 5463 | SET_EX_OPLINE opline, REG0 5464 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5465 switch (type) { 5466 case BP_VAR_R: 5467 | LOAD_ZVAL_ADDR CARG3, res_addr 5468 | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 5469 | mov REG0, RETVALx 5470 | b >9 5471 break; 5472 case BP_JIT_IS: 5473 | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 5474 | mov REG0, RETVALx 5475 if (not_found_exit_addr) { 5476 | cbz REG0, ¬_found_exit_addr 5477 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5478 | b >8 5479 } 5480 } else if (found_exit_addr) { 5481 | cbnz REG0, &found_exit_addr 5482 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5483 | b >9 5484 } 5485 } else { 5486 | cbnz REG0, >8 5487 | b >9 5488 } 5489 break; 5490 case BP_VAR_IS: 5491 case BP_VAR_UNSET: 5492 | LOAD_ZVAL_ADDR CARG3, res_addr 5493 | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 5494 | mov REG0, RETVALx 5495 | b >9 5496 break; 5497 case BP_VAR_RW: 5498 | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 5499 | mov REG0, RETVALx 5500 | cbnz REG0, >8 5501 | b >9 5502 break; 5503 case BP_VAR_W: 5504 | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 5505 | mov REG0, RETVALx 5506 | cbnz REG0, >8 5507 | b >9 5508 break; 5509 default: 5510 ZEND_UNREACHABLE(); 5511 } 5512 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5513 |.code 5514 } 5515 } 5516 5517 return 1; 5518} 5519 5520static int zend_jit_simple_assign(dasm_State **Dst, 5521 const zend_op *opline, 5522 zend_jit_addr var_addr, 5523 uint32_t var_info, 5524 uint32_t var_def_info, 5525 zend_uchar val_type, 5526 zend_jit_addr val_addr, 5527 uint32_t val_info, 5528 zend_jit_addr res_addr, 5529 int in_cold, 5530 int save_r1) 5531/* Labels: 1,2,3 */ 5532{ 5533 zend_reg tmp_reg; 5534 5535 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { 5536 tmp_reg = ZREG_REG0; 5537 } else { 5538 /* ASSIGN_DIM */ 5539 tmp_reg = ZREG_FCARG1; 5540 } 5541 5542 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5543 zval *zv = Z_ZV(val_addr); 5544 5545 if (!res_addr) { 5546 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5547 } else { 5548 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5549 } 5550 if (Z_REFCOUNTED_P(zv)) { 5551 if (!res_addr) { 5552 | ADDREF_CONST zv, TMP1, TMP2 5553 } else { 5554 | ADDREF_CONST_2 zv, TMP1, TMP2 5555 } 5556 } 5557 } else { 5558 if (val_info & MAY_BE_UNDEF) { 5559 if (in_cold) { 5560 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 5561 } else { 5562 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5563 |.cold_code 5564 |1: 5565 } 5566 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 5567 if (save_r1) { 5568 | str FCARG1x, T1 // save 5569 } 5570 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 5571 if (res_addr) { 5572 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5573 } 5574 if (opline) { 5575 | SET_EX_OPLINE opline, Rx(tmp_reg) 5576 } 5577 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 5578 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5579 | EXT_CALL zend_jit_undefined_op_helper, REG0 5580 | cbz RETVALx, ->exception_handler_undef 5581 if (save_r1) { 5582 | ldr FCARG1x, T1 // restore 5583 } 5584 | b >3 5585 if (in_cold) { 5586 |2: 5587 } else { 5588 |.code 5589 } 5590 } 5591 if (val_info & MAY_BE_REF) { 5592 if (val_type == IS_CV) { 5593 ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); 5594 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { 5595 | LOAD_ZVAL_ADDR REG2, val_addr 5596 } 5597 | ZVAL_DEREF REG2, val_info, TMP1w 5598 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 5599 } else { 5600 zend_jit_addr ref_addr; 5601 5602 if (in_cold) { 5603 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5604 } else { 5605 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5606 |.cold_code 5607 |1: 5608 } 5609 if (Z_REG(val_addr) == ZREG_REG2) { 5610 | str REG2, T1 // save 5611 } 5612 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 5613 | GET_ZVAL_PTR REG2, val_addr, TMP1 5614 | GC_DELREF REG2, TMP1w 5615 | // ZVAL_COPY_VALUE(return_value, &ref->val); 5616 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); 5617 if (!res_addr) { 5618 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5619 } else { 5620 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5621 } 5622 | beq >2 // GC_DELREF() reached zero 5623 | IF_NOT_REFCOUNTED REG2w, >3, TMP1w 5624 if (!res_addr) { 5625 | GC_ADDREF Rx(tmp_reg), TMP1w 5626 } else { 5627 | GC_ADDREF_2 Rx(tmp_reg), TMP1w 5628 } 5629 | b >3 5630 |2: 5631 if (res_addr) { 5632 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 5633 | GC_ADDREF Rx(tmp_reg), TMP1w 5634 |2: 5635 } 5636 if (Z_REG(val_addr) == ZREG_REG2) { 5637 | ldr REG2, T1 // restore 5638 } 5639 if (save_r1) { 5640 | str FCARG1x, T1 // save 5641 } 5642 | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 5643 | EFREE_REFERENCE 5644 if (save_r1) { 5645 | ldr FCARG1x, T1 // restore 5646 } 5647 | b >3 5648 if (in_cold) { 5649 |1: 5650 } else { 5651 |.code 5652 } 5653 } 5654 } 5655 5656 if (!res_addr) { 5657 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5658 } else { 5659 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5660 } 5661 5662 if (val_type == IS_CV) { 5663 if (!res_addr) { 5664 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5665 } else { 5666 | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w 5667 } 5668 } else { 5669 if (res_addr) { 5670 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5671 } 5672 } 5673 |3: 5674 } 5675 return 1; 5676} 5677 5678static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 5679 const zend_op *opline, 5680 zend_uchar val_type, 5681 zend_jit_addr val_addr, 5682 zend_jit_addr res_addr, 5683 bool check_exception) 5684{ 5685 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5686 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 5687 | cbnz TMP1, >2 5688 |.cold_code 5689 |2: 5690 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5691 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5692 } 5693 if (opline) { 5694 | SET_EX_OPLINE opline, REG0 5695 } 5696 if (val_type == IS_CONST) { 5697 | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 5698 } else if (val_type == IS_TMP_VAR) { 5699 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 5700 } else if (val_type == IS_VAR) { 5701 | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 5702 } else if (val_type == IS_CV) { 5703 | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 5704 } else { 5705 ZEND_UNREACHABLE(); 5706 } 5707 if (res_addr) { 5708 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // RETVAL 5709 5710 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5711 | TRY_ADDREF -1, REG1w, REG2, TMP1w 5712 } 5713 if (check_exception) { 5714 | // if (UNEXPECTED(EG(exception) != NULL)) { 5715 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5716 | cbz REG0, >8 // END OF zend_jit_assign_to_variable() 5717 | b ->exception_handler_undef 5718 } else { 5719 | b >8 5720 } 5721 |.code 5722 5723 return 1; 5724} 5725 5726static int zend_jit_assign_to_variable_call(dasm_State **Dst, 5727 const zend_op *opline, 5728 zend_jit_addr __var_use_addr, 5729 zend_jit_addr var_addr, 5730 uint32_t __var_info, 5731 uint32_t __var_def_info, 5732 zend_uchar val_type, 5733 zend_jit_addr val_addr, 5734 uint32_t val_info, 5735 zend_jit_addr __res_addr, 5736 bool __check_exception) 5737{ 5738 if (val_info & MAY_BE_UNDEF) { 5739 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 5740 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5741 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5742 5743 if (!exit_addr) { 5744 return 0; 5745 } 5746 5747 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 5748 } else { 5749 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5750 |.cold_code 5751 |1: 5752 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 5753 if (Z_REG(var_addr) != ZREG_FP) { 5754 | str Rx(Z_REG(var_addr)), T1 // save 5755 } 5756 | SET_EX_OPLINE opline, REG0 5757 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5758 | EXT_CALL zend_jit_undefined_op_helper, REG0 5759 if (Z_REG(var_addr) != ZREG_FP) { 5760 | ldr Rx(Z_REG(var_addr)), T1 // restore 5761 } 5762 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5763 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5764 } 5765 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 5766 | bl ->assign_const 5767 | b >9 5768 |.code 5769 |1: 5770 } 5771 } 5772 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5773 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5774 } 5775 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5776 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5777 } 5778 if (opline) { 5779 | SET_EX_OPLINE opline, REG0 5780 } 5781 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 5782 | bl ->assign_tmp 5783 } else if (val_type == IS_CONST) { 5784 | bl ->assign_const 5785 } else if (val_type == IS_TMP_VAR) { 5786 | bl ->assign_tmp 5787 } else if (val_type == IS_VAR) { 5788 if (!(val_info & MAY_BE_REF)) { 5789 | bl ->assign_tmp 5790 } else { 5791 | bl ->assign_var 5792 } 5793 } else if (val_type == IS_CV) { 5794 if (!(val_info & MAY_BE_REF)) { 5795 | bl ->assign_cv_noref 5796 } else { 5797 | bl ->assign_cv 5798 } 5799 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5800 |9: 5801 } 5802 } else { 5803 ZEND_UNREACHABLE(); 5804 } 5805 5806 return 1; 5807} 5808 5809static int zend_jit_assign_to_variable(dasm_State **Dst, 5810 const zend_op *opline, 5811 zend_jit_addr var_use_addr, 5812 zend_jit_addr var_addr, 5813 uint32_t var_info, 5814 uint32_t var_def_info, 5815 zend_uchar val_type, 5816 zend_jit_addr val_addr, 5817 uint32_t val_info, 5818 zend_jit_addr res_addr, 5819 bool check_exception) 5820/* Labels: 1,2,3,4,5,8 */ 5821{ 5822 int done = 0; 5823 zend_reg ref_reg, tmp_reg; 5824 5825 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { 5826 ref_reg = ZREG_FCARG1; 5827 tmp_reg = ZREG_REG0; 5828 } else { 5829 /* ASSIGN_DIM */ 5830 ref_reg = ZREG_REG0; 5831 tmp_reg = ZREG_FCARG1; 5832 } 5833 5834 if (var_info & MAY_BE_REF) { 5835 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 5836 | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr 5837 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 5838 } 5839 | // if (Z_ISREF_P(variable_ptr)) { 5840 | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w 5841 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5842 | GET_Z_PTR FCARG1x, Rx(ref_reg) 5843 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 5844 return 0; 5845 } 5846 | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) 5847 |3: 5848 } 5849 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 5850 if (RC_MAY_BE_1(var_info)) { 5851 int in_cold = 0; 5852 5853 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5854 | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 5855 |.cold_code 5856 |1: 5857 in_cold = 1; 5858 } 5859 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) { 5860 bool keep_gc = 0; 5861 5862 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5863#if 0 5864 // TODO: This optiization doesn't work on ARM 5865 if (tmp_reg == ZREG_FCARG1) { 5866 if (Z_MODE(val_addr) == IS_REG) { 5867 keep_gc = 1; 5868 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 5869 keep_gc = 1; 5870 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5871 zval *zv = Z_ZV(val_addr); 5872 if (Z_TYPE_P(zv) == IS_DOUBLE) { 5873 if (Z_DVAL_P(zv) == 0) { 5874 keep_gc = 1; 5875 } 5876 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 5877 keep_gc = 1; 5878 } 5879 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 5880 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 5881 keep_gc = 1; 5882 } 5883 } 5884 } 5885#endif 5886 if (!keep_gc) { 5887 | str Rx(tmp_reg), T1 // save 5888 } 5889 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { 5890 return 0; 5891 } 5892 if (!keep_gc) { 5893 | ldr FCARG1x, T1 // restore 5894 } 5895 } else { 5896 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5897 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { 5898 return 0; 5899 } 5900 } 5901 | GC_DELREF FCARG1x, TMP1w 5902 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5903 | bne >4 5904 } else { 5905 | bne >8 5906 } 5907 | ZVAL_DTOR_FUNC var_info, opline, TMP1 5908 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 5909 if (check_exception) { 5910 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5911 | cbz REG0, >8 5912 | b ->exception_handler 5913 } else { 5914 | b >8 5915 } 5916 } 5917 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5918 |4: 5919 | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w 5920 | EXT_CALL gc_possible_root, REG0 5921 if (in_cold) { 5922 | b >8 5923 } 5924 } 5925 if (in_cold) { 5926 |.code 5927 } else { 5928 done = 1; 5929 } 5930 } else /* if (RC_MAY_BE_N(var_info)) */ { 5931 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5932 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 5933 } 5934 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 5935 if (Z_REG(var_use_addr) != ZREG_FP) { 5936 | str Rx(Z_REG(var_use_addr)), T1 // save 5937 } 5938 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5939 | GC_DELREF FCARG1x, TMP1w 5940 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 5941 | EXT_CALL gc_possible_root, TMP1 5942 if (Z_REG(var_use_addr) != ZREG_FP) { 5943 | ldr Rx(Z_REG(var_use_addr)), T1 // restore 5944 } 5945 } else { 5946 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5947 | GC_DELREF Rx(tmp_reg), TMP1w 5948 } 5949 |5: 5950 } 5951 } 5952 5953 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { 5954 return 0; 5955 } 5956 5957 |8: 5958 5959 return 1; 5960} 5961 5962static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 5963{ 5964 zend_jit_addr op2_addr, op3_addr, res_addr; 5965 5966 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 5967 op3_addr = OP1_DATA_ADDR(); 5968 if (opline->result_type == IS_UNUSED) { 5969 res_addr = 0; 5970 } else { 5971 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5972 } 5973 5974 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 5975 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5976 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5977 5978 if (!exit_addr) { 5979 return 0; 5980 } 5981 5982 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 5983 5984 val_info &= ~MAY_BE_UNDEF; 5985 } 5986 5987 if (op1_info & MAY_BE_REF) { 5988 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 5989 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 5990 | GET_Z_PTR FCARG2x, FCARG1x 5991 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 5992 | cmp TMP1w, #IS_ARRAY 5993 | bne >2 5994 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 5995 | b >3 5996 |.cold_code 5997 |2: 5998 | SET_EX_OPLINE opline, REG0 5999 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6000 | mov FCARG1x, RETVALx 6001 | cbnz FCARG1x, >1 6002 | b ->exception_handler_undef 6003 |.code 6004 |1: 6005 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6006 } 6007 6008 if (op1_info & MAY_BE_ARRAY) { 6009 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6010 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6011 } 6012 |3: 6013 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6014 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6015 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6016 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6017 | bgt >7 6018 } 6019 | // ZVAL_ARR(container, zend_new_array(8)); 6020 if (Z_REG(op1_addr) != ZREG_FP) { 6021 | str Rx(Z_REG(op1_addr)), T1 // save 6022 } 6023 | EXT_CALL _zend_new_array_0, REG0 6024 | mov REG0, RETVALx 6025 if (Z_REG(op1_addr) != ZREG_FP) { 6026 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6027 } 6028 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6029 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6030 | mov FCARG1x, REG0 6031 } 6032 6033 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6034 |6: 6035 if (opline->op2_type == IS_UNUSED) { 6036 uint32_t var_info = MAY_BE_NULL; 6037 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6038 6039 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6040 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6041 | EXT_CALL zend_hash_next_index_insert, REG0 6042 | // if (UNEXPECTED(!var_ptr)) { 6043 | mov REG0, RETVALx 6044 | cbz REG0, >1 6045 |.cold_code 6046 |1: 6047 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6048 | CANNOT_ADD_ELEMENT opline 6049 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6050 | b >9 6051 |.code 6052 6053 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { 6054 return 0; 6055 } 6056 } else { 6057 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6058 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6059 6060 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6061 return 0; 6062 } 6063 6064 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6065 var_info |= MAY_BE_REF; 6066 } 6067 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6068 var_info |= MAY_BE_RC1; 6069 } 6070 6071 |8: 6072 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6073 if (opline->op1_type == IS_VAR) { 6074 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6075 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6076 return 0; 6077 } 6078 } else { 6079 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6080 return 0; 6081 } 6082 } 6083 } 6084 } 6085 6086 if (((op1_info & MAY_BE_ARRAY) && 6087 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6088 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6089 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6090 |.cold_code 6091 |7: 6092 } 6093 6094 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6095 (op1_info & MAY_BE_ARRAY)) { 6096 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6097 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6098 | bgt >2 6099 } 6100 | // ZVAL_ARR(container, zend_new_array(8)); 6101 if (Z_REG(op1_addr) != ZREG_FP) { 6102 | str Rx(Z_REG(op1_addr)), T1 // save 6103 } 6104 | EXT_CALL _zend_new_array_0, REG0 6105 | mov REG0, RETVALx 6106 if (Z_REG(op1_addr) != ZREG_FP) { 6107 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6108 } 6109 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6110 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6111 | mov FCARG1x, REG0 6112 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6113 | b <6 6114 |2: 6115 } 6116 6117 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6118 | SET_EX_OPLINE opline, REG0 6119 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6120 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6121 } 6122 if (opline->op2_type == IS_UNUSED) { 6123 | mov FCARG2x, xzr 6124 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6125 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6126 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6127 } else { 6128 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6129 } 6130 if (opline->result_type == IS_UNUSED) { 6131 | mov CARG4, xzr 6132 } else { 6133 | LOAD_ZVAL_ADDR CARG4, res_addr 6134 } 6135 | LOAD_ZVAL_ADDR CARG3, op3_addr 6136 | EXT_CALL zend_jit_assign_dim_helper, REG0 6137 6138#ifdef ZEND_JIT_USE_RC_INFERENCE 6139 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6140 /* ASSIGN_DIM may increase refcount of the value */ 6141 val_info |= MAY_BE_RCN; 6142 } 6143#endif 6144 6145 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6146 } 6147 6148 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6149 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6150 | b >9 // END 6151 } 6152 |.code 6153 } 6154 } 6155 6156#ifdef ZEND_JIT_USE_RC_INFERENCE 6157 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6158 /* ASSIGN_DIM may increase refcount of the key */ 6159 op2_info |= MAY_BE_RCN; 6160 } 6161#endif 6162 6163 |9: 6164 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6165 6166 if (may_throw) { 6167 zend_jit_check_exception(Dst); 6168 } 6169 6170 return 1; 6171} 6172 6173static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6174{ 6175 zend_jit_addr op2_addr, op3_addr, var_addr; 6176 const void *not_found_exit_addr = NULL; 6177 uint32_t var_info = MAY_BE_NULL; 6178 6179 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6180 6181 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6182 op3_addr = OP1_DATA_ADDR(); 6183 6184 if (op1_info & MAY_BE_REF) { 6185 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6186 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6187 | GET_Z_PTR FCARG2x, FCARG1x 6188 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6189 | cmp TMP1w, #IS_ARRAY 6190 | bne >2 6191 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6192 | b >3 6193 |.cold_code 6194 |2: 6195 | SET_EX_OPLINE opline, REG0 6196 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6197 | mov FCARG1x, RETVALx 6198 | cbnz RETVALx, >1 6199 | b ->exception_handler_undef 6200 |.code 6201 |1: 6202 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6203 } 6204 6205 if (op1_info & MAY_BE_ARRAY) { 6206 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6207 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6208 } 6209 |3: 6210 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6211 } 6212 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6213 if (op1_info & MAY_BE_ARRAY) { 6214 |.cold_code 6215 |7: 6216 } 6217 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6218 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6219 | bgt >7 6220 } 6221 if (Z_REG(op1_addr) != ZREG_FP) { 6222 | str Rx(Z_REG(op1_addr)), T1 // save 6223 } 6224 if (op1_info & MAY_BE_UNDEF) { 6225 if (op1_info & MAY_BE_NULL) { 6226 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 6227 } 6228 | SET_EX_OPLINE opline, REG0 6229 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 6230 | EXT_CALL zend_jit_undefined_op_helper, REG0 6231 |1: 6232 } 6233 | // ZVAL_ARR(container, zend_new_array(8)); 6234 | EXT_CALL _zend_new_array_0, REG0 6235 | mov REG0, RETVALx 6236 if (Z_REG(op1_addr) != ZREG_FP) { 6237 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6238 } 6239 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6240 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6241 | mov FCARG1x, REG0 6242 if (op1_info & MAY_BE_ARRAY) { 6243 | b >1 6244 |.code 6245 |1: 6246 } 6247 } 6248 6249 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6250 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6251 6252 |6: 6253 if (opline->op2_type == IS_UNUSED) { 6254 var_info = MAY_BE_NULL; 6255 6256 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6257 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6258 | EXT_CALL zend_hash_next_index_insert, REG0 6259 | mov REG0, RETVALx 6260 | // if (UNEXPECTED(!var_ptr)) { 6261 | cbz REG0, >1 6262 |.cold_code 6263 |1: 6264 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6265 | CANNOT_ADD_ELEMENT opline 6266 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6267 | b >9 6268 |.code 6269 } else { 6270 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6271 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6272 var_info |= MAY_BE_REF; 6273 } 6274 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6275 var_info |= MAY_BE_RC1; 6276 } 6277 6278 if (dim_type != IS_UNKNOWN 6279 && dim_type != IS_UNDEF 6280 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6281 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6282 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6283 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6284 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6285 } 6286 6287 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6288 return 0; 6289 } 6290 6291 |8: 6292 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6293 | IF_NOT_Z_TYPE, REG0, dim_type, ¬_found_exit_addr, TMP1w 6294 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6295 } 6296 if (var_info & MAY_BE_REF) { 6297 binary_op_type binary_op = get_binary_op(opline->extended_value); 6298 | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w 6299 | GET_Z_PTR FCARG1x, REG0 6300 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6301 | cbnz TMP1, >2 6302 | add REG0, FCARG1x, #offsetof(zend_reference, val) 6303 |.cold_code 6304 |2: 6305 | LOAD_ZVAL_ADDR FCARG2x, op3_addr 6306 | LOAD_ADDR CARG3, binary_op 6307 | SET_EX_OPLINE opline, REG0 6308 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6309 | b >9 6310 |.code 6311 |1: 6312 } 6313 } 6314 6315 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6316 switch (opline->extended_value) { 6317 case ZEND_ADD: 6318 case ZEND_SUB: 6319 case ZEND_MUL: 6320 case ZEND_DIV: 6321 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6322 1 /* may overflow */, may_throw)) { 6323 return 0; 6324 } 6325 break; 6326 case ZEND_BW_OR: 6327 case ZEND_BW_AND: 6328 case ZEND_BW_XOR: 6329 case ZEND_SL: 6330 case ZEND_SR: 6331 case ZEND_MOD: 6332 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6333 IS_CV, opline->op1, var_addr, var_info, NULL, 6334 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6335 op1_data_range, 6336 0, var_addr, var_def_info, var_info, may_throw)) { 6337 return 0; 6338 } 6339 break; 6340 case ZEND_CONCAT: 6341 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6342 may_throw)) { 6343 return 0; 6344 } 6345 break; 6346 default: 6347 ZEND_UNREACHABLE(); 6348 } 6349 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6350 } 6351 6352 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6353 binary_op_type binary_op; 6354 6355 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6356 |.cold_code 6357 |7: 6358 } 6359 6360 | SET_EX_OPLINE opline, REG0 6361 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6362 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6363 } 6364 if (opline->op2_type == IS_UNUSED) { 6365 | mov FCARG2x, xzr 6366 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6367 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6368 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6369 } else { 6370 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6371 } 6372 binary_op = get_binary_op(opline->extended_value); 6373 | LOAD_ZVAL_ADDR CARG3, op3_addr 6374 | LOAD_ADDR CARG4, binary_op 6375 | EXT_CALL zend_jit_assign_dim_op_helper, REG0 6376 6377 |9: 6378 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6379 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6380 if (may_throw) { 6381 zend_jit_check_exception(Dst); 6382 } 6383 6384 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6385 | b >9 // END 6386 |.code 6387 |9: 6388 } 6389 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 6390 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 6391 |.cold_code 6392 |9: 6393 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6394 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6395 if (may_throw) { 6396 zend_jit_check_exception(Dst); 6397 } 6398 | b >9 6399 |.code 6400 |9: 6401 } 6402 6403 return 1; 6404} 6405 6406static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6407{ 6408 zend_jit_addr op1_addr, op2_addr; 6409 6410 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6411 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6412 6413 op1_addr = OP1_ADDR(); 6414 op2_addr = OP2_ADDR(); 6415 6416 if (op1_info & MAY_BE_REF) { 6417 binary_op_type binary_op = get_binary_op(opline->extended_value); 6418 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6419 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w 6420 | GET_Z_PTR FCARG1x, FCARG1x 6421 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6422 | cbnz TMP1, >2 6423 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 6424 |.cold_code 6425 |2: 6426 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6427 | LOAD_ADDR CARG3, binary_op 6428 | SET_EX_OPLINE opline, REG0 6429 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6430 zend_jit_check_exception(Dst); 6431 | b >9 6432 |.code 6433 |1: 6434 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6435 } 6436 6437 int result; 6438 switch (opline->extended_value) { 6439 case ZEND_ADD: 6440 case ZEND_SUB: 6441 case ZEND_MUL: 6442 case ZEND_DIV: 6443 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6444 break; 6445 case ZEND_BW_OR: 6446 case ZEND_BW_AND: 6447 case ZEND_BW_XOR: 6448 case ZEND_SL: 6449 case ZEND_SR: 6450 case ZEND_MOD: 6451 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6452 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6453 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6454 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6455 break; 6456 case ZEND_CONCAT: 6457 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6458 break; 6459 default: 6460 ZEND_UNREACHABLE(); 6461 } 6462 |9: 6463 return result; 6464} 6465 6466static int zend_jit_cmp_long_long(dasm_State **Dst, 6467 const zend_op *opline, 6468 zend_ssa_range *op1_range, 6469 zend_jit_addr op1_addr, 6470 zend_ssa_range *op2_range, 6471 zend_jit_addr op2_addr, 6472 zend_jit_addr res_addr, 6473 zend_uchar smart_branch_opcode, 6474 uint32_t target_label, 6475 uint32_t target_label2, 6476 const void *exit_addr, 6477 bool skip_comparison) 6478{ 6479 bool swap = 0; 6480 bool result; 6481 6482 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 6483 if (!smart_branch_opcode || 6484 smart_branch_opcode == ZEND_JMPZ_EX || 6485 smart_branch_opcode == ZEND_JMPNZ_EX) { 6486 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 6487 } 6488 if (smart_branch_opcode && !exit_addr) { 6489 if (smart_branch_opcode == ZEND_JMPZ || 6490 smart_branch_opcode == ZEND_JMPZ_EX) { 6491 if (!result) { 6492 | b => target_label 6493 } 6494 } else if (smart_branch_opcode == ZEND_JMPNZ || 6495 smart_branch_opcode == ZEND_JMPNZ_EX) { 6496 if (result) { 6497 | b => target_label 6498 } 6499 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6500 if (!result) { 6501 | b => target_label 6502 } else { 6503 | b => target_label2 6504 } 6505 } else { 6506 ZEND_UNREACHABLE(); 6507 } 6508 } 6509 return 1; 6510 } 6511 6512 if (skip_comparison) { 6513 if (Z_MODE(op1_addr) != IS_REG && 6514 (Z_MODE(op2_addr) == IS_REG || 6515 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 6516 swap = 1; 6517 } 6518 } else if (Z_MODE(op1_addr) == IS_REG) { 6519 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6520 | cmp Rx(Z_REG(op1_addr)), xzr 6521 } else { 6522 | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 6523 } 6524 } else if (Z_MODE(op2_addr) == IS_REG) { 6525 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 6526 | cmp Rx(Z_REG(op2_addr)), xzr 6527 } else { 6528 | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 6529 } 6530 swap = 1; 6531 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 6532 | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 6533 swap = 1; 6534 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 6535 | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 6536 } else { 6537 | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 6538 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6539 | cmp Rx(ZREG_REG0), xzr 6540 } else { 6541 | LONG_CMP ZREG_REG0, op2_addr, TMP1 6542 } 6543 } 6544 6545 if (smart_branch_opcode) { 6546 if (smart_branch_opcode == ZEND_JMPZ_EX || 6547 smart_branch_opcode == ZEND_JMPNZ_EX) { 6548 6549 switch (opline->opcode) { 6550 case ZEND_IS_EQUAL: 6551 case ZEND_IS_IDENTICAL: 6552 case ZEND_CASE: 6553 case ZEND_CASE_STRICT: 6554 | cset REG0w, eq 6555 break; 6556 case ZEND_IS_NOT_EQUAL: 6557 case ZEND_IS_NOT_IDENTICAL: 6558 | cset REG0w, ne 6559 break; 6560 case ZEND_IS_SMALLER: 6561 if (swap) { 6562 | cset REG0w, gt 6563 } else { 6564 | cset REG0w, lt 6565 } 6566 break; 6567 case ZEND_IS_SMALLER_OR_EQUAL: 6568 if (swap) { 6569 | cset REG0w, ge 6570 } else { 6571 | cset REG0w, le 6572 } 6573 break; 6574 default: 6575 ZEND_UNREACHABLE(); 6576 } 6577 | add REG0w, REG0w, #2 6578 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6579 } 6580 if (smart_branch_opcode == ZEND_JMPZ || 6581 smart_branch_opcode == ZEND_JMPZ_EX) { 6582 switch (opline->opcode) { 6583 case ZEND_IS_EQUAL: 6584 case ZEND_IS_IDENTICAL: 6585 case ZEND_CASE: 6586 case ZEND_CASE_STRICT: 6587 if (exit_addr) { 6588 | bne &exit_addr 6589 } else { 6590 | bne => target_label 6591 } 6592 break; 6593 case ZEND_IS_NOT_EQUAL: 6594 if (exit_addr) { 6595 | beq &exit_addr 6596 } else { 6597 | beq => target_label 6598 } 6599 break; 6600 case ZEND_IS_NOT_IDENTICAL: 6601 if (exit_addr) { 6602 | bne &exit_addr 6603 } else { 6604 | beq => target_label 6605 } 6606 break; 6607 case ZEND_IS_SMALLER: 6608 if (swap) { 6609 if (exit_addr) { 6610 | ble &exit_addr 6611 } else { 6612 | ble => target_label 6613 } 6614 } else { 6615 if (exit_addr) { 6616 | bge &exit_addr 6617 } else { 6618 | bge => target_label 6619 } 6620 } 6621 break; 6622 case ZEND_IS_SMALLER_OR_EQUAL: 6623 if (swap) { 6624 if (exit_addr) { 6625 | blt &exit_addr 6626 } else { 6627 | blt => target_label 6628 } 6629 } else { 6630 if (exit_addr) { 6631 | bgt &exit_addr 6632 } else { 6633 | bgt => target_label 6634 } 6635 } 6636 break; 6637 default: 6638 ZEND_UNREACHABLE(); 6639 } 6640 } else if (smart_branch_opcode == ZEND_JMPNZ || 6641 smart_branch_opcode == ZEND_JMPNZ_EX) { 6642 switch (opline->opcode) { 6643 case ZEND_IS_EQUAL: 6644 case ZEND_IS_IDENTICAL: 6645 case ZEND_CASE: 6646 case ZEND_CASE_STRICT: 6647 if (exit_addr) { 6648 | beq &exit_addr 6649 } else { 6650 | beq => target_label 6651 } 6652 break; 6653 case ZEND_IS_NOT_EQUAL: 6654 if (exit_addr) { 6655 | bne &exit_addr 6656 } else { 6657 | bne => target_label 6658 } 6659 break; 6660 case ZEND_IS_NOT_IDENTICAL: 6661 if (exit_addr) { 6662 | beq &exit_addr 6663 } else { 6664 | bne => target_label 6665 } 6666 break; 6667 case ZEND_IS_SMALLER: 6668 if (swap) { 6669 if (exit_addr) { 6670 | bgt &exit_addr 6671 } else { 6672 | bgt => target_label 6673 } 6674 } else { 6675 if (exit_addr) { 6676 | blt &exit_addr 6677 } else { 6678 | blt => target_label 6679 } 6680 } 6681 break; 6682 case ZEND_IS_SMALLER_OR_EQUAL: 6683 if (swap) { 6684 if (exit_addr) { 6685 | bge &exit_addr 6686 } else { 6687 | bge => target_label 6688 } 6689 } else { 6690 if (exit_addr) { 6691 | ble &exit_addr 6692 } else { 6693 | ble => target_label 6694 } 6695 } 6696 break; 6697 default: 6698 ZEND_UNREACHABLE(); 6699 } 6700 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6701 switch (opline->opcode) { 6702 case ZEND_IS_EQUAL: 6703 case ZEND_IS_IDENTICAL: 6704 case ZEND_CASE: 6705 case ZEND_CASE_STRICT: 6706 | bne => target_label 6707 break; 6708 case ZEND_IS_NOT_EQUAL: 6709 case ZEND_IS_NOT_IDENTICAL: 6710 | beq => target_label 6711 break; 6712 case ZEND_IS_SMALLER: 6713 if (swap) { 6714 | ble => target_label 6715 } else { 6716 | bge => target_label 6717 } 6718 break; 6719 case ZEND_IS_SMALLER_OR_EQUAL: 6720 if (swap) { 6721 | blt => target_label 6722 } else { 6723 | bgt => target_label 6724 } 6725 break; 6726 default: 6727 ZEND_UNREACHABLE(); 6728 } 6729 | b => target_label2 6730 } else { 6731 ZEND_UNREACHABLE(); 6732 } 6733 } else { 6734 switch (opline->opcode) { 6735 case ZEND_IS_EQUAL: 6736 case ZEND_IS_IDENTICAL: 6737 case ZEND_CASE: 6738 case ZEND_CASE_STRICT: 6739 | cset REG0w, eq 6740 break; 6741 case ZEND_IS_NOT_EQUAL: 6742 case ZEND_IS_NOT_IDENTICAL: 6743 | cset REG0w, ne 6744 break; 6745 case ZEND_IS_SMALLER: 6746 if (swap) { 6747 | cset REG0w, gt 6748 } else { 6749 | cset REG0w, lt 6750 } 6751 break; 6752 case ZEND_IS_SMALLER_OR_EQUAL: 6753 if (swap) { 6754 | cset REG0w, ge 6755 } else { 6756 | cset REG0w, le 6757 } 6758 break; 6759 default: 6760 ZEND_UNREACHABLE(); 6761 } 6762 | add REG0w, REG0w, #2 6763 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6764 } 6765 6766 return 1; 6767} 6768 6769static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 6770{ 6771 if (smart_branch_opcode) { 6772 if (smart_branch_opcode == ZEND_JMPZ) { 6773 switch (opline->opcode) { 6774 case ZEND_IS_EQUAL: 6775 case ZEND_IS_IDENTICAL: 6776 case ZEND_CASE: 6777 case ZEND_CASE_STRICT: 6778 if (exit_addr) { 6779 | bne &exit_addr 6780 } else { 6781 | bne => target_label 6782 } 6783 break; 6784 case ZEND_IS_NOT_EQUAL: 6785 | bvs >1 6786 if (exit_addr) { 6787 | beq &exit_addr 6788 } else { 6789 | beq => target_label 6790 } 6791 |1: 6792 break; 6793 case ZEND_IS_NOT_IDENTICAL: 6794 if (exit_addr) { 6795 | bvs &exit_addr 6796 | bne &exit_addr 6797 } else { 6798 | bvs >1 6799 | beq => target_label 6800 |1: 6801 } 6802 break; 6803 case ZEND_IS_SMALLER: 6804 if (swap) { 6805 if (exit_addr) { 6806 | bvs &exit_addr 6807 | bls &exit_addr 6808 } else { 6809 | bvs => target_label 6810 | bls => target_label 6811 } 6812 } else { 6813 if (exit_addr) { 6814 | bhs &exit_addr 6815 } else { 6816 | bhs => target_label 6817 } 6818 } 6819 break; 6820 case ZEND_IS_SMALLER_OR_EQUAL: 6821 if (swap) { 6822 if (exit_addr) { 6823 | bvs &exit_addr 6824 | blo &exit_addr 6825 } else { 6826 | bvs => target_label 6827 | blo => target_label 6828 } 6829 } else { 6830 if (exit_addr) { 6831 | bhi &exit_addr 6832 } else { 6833 | bhi => target_label 6834 } 6835 } 6836 break; 6837 default: 6838 ZEND_UNREACHABLE(); 6839 } 6840 } else if (smart_branch_opcode == ZEND_JMPNZ) { 6841 switch (opline->opcode) { 6842 case ZEND_IS_EQUAL: 6843 case ZEND_IS_IDENTICAL: 6844 case ZEND_CASE: 6845 case ZEND_CASE_STRICT: 6846 | bvs >1 6847 if (exit_addr) { 6848 | beq &exit_addr 6849 } else { 6850 | beq => target_label 6851 } 6852 |1: 6853 break; 6854 case ZEND_IS_NOT_EQUAL: 6855 if (exit_addr) { 6856 | bne &exit_addr 6857 } else { 6858 | bne => target_label 6859 } 6860 break; 6861 case ZEND_IS_NOT_IDENTICAL: 6862 if (exit_addr) { 6863 | bvs >1 6864 | beq &exit_addr 6865 |1: 6866 } else { 6867 | bne => target_label 6868 } 6869 break; 6870 case ZEND_IS_SMALLER: 6871 if (swap) { 6872 | bvs >1 // Always False if involving NaN 6873 if (exit_addr) { 6874 | bhi &exit_addr 6875 } else { 6876 | bhi => target_label 6877 } 6878 |1: 6879 } else { 6880 | bvs >1 6881 if (exit_addr) { 6882 | blo &exit_addr 6883 } else { 6884 | blo => target_label 6885 } 6886 |1: 6887 } 6888 break; 6889 case ZEND_IS_SMALLER_OR_EQUAL: 6890 if (swap) { 6891 | bvs >1 // Always False if involving NaN 6892 if (exit_addr) { 6893 | bhs &exit_addr 6894 } else { 6895 | bhs => target_label 6896 } 6897 |1: 6898 } else { 6899 | bvs >1 6900 if (exit_addr) { 6901 | bls &exit_addr 6902 } else { 6903 | bls => target_label 6904 } 6905 |1: 6906 } 6907 break; 6908 default: 6909 ZEND_UNREACHABLE(); 6910 } 6911 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6912 switch (opline->opcode) { 6913 case ZEND_IS_EQUAL: 6914 case ZEND_IS_IDENTICAL: 6915 case ZEND_CASE: 6916 case ZEND_CASE_STRICT: 6917 | bne => target_label 6918 break; 6919 case ZEND_IS_NOT_EQUAL: 6920 case ZEND_IS_NOT_IDENTICAL: 6921 | bvs => target_label2 6922 | beq => target_label 6923 break; 6924 case ZEND_IS_SMALLER: 6925 if (swap) { 6926 | bvs => target_label 6927 | bls => target_label 6928 } else { 6929 | bhs => target_label 6930 } 6931 break; 6932 case ZEND_IS_SMALLER_OR_EQUAL: 6933 if (swap) { 6934 | bvs => target_label 6935 | blo => target_label 6936 } else { 6937 | bhi => target_label 6938 } 6939 break; 6940 default: 6941 ZEND_UNREACHABLE(); 6942 } 6943 | b => target_label2 6944 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 6945 switch (opline->opcode) { 6946 case ZEND_IS_EQUAL: 6947 case ZEND_IS_IDENTICAL: 6948 case ZEND_CASE: 6949 case ZEND_CASE_STRICT: 6950 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6951 | bne => target_label 6952 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6953 break; 6954 case ZEND_IS_NOT_EQUAL: 6955 case ZEND_IS_NOT_IDENTICAL: 6956 | bvs >1 6957 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6958 | beq => target_label 6959 |1: 6960 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6961 break; 6962 case ZEND_IS_SMALLER: 6963 if (swap) { 6964 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6965 | bvs => target_label 6966 | bls => target_label 6967 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6968 } else { 6969 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6970 | bhs => target_label 6971 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6972 } 6973 break; 6974 case ZEND_IS_SMALLER_OR_EQUAL: 6975 if (swap) { 6976 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6977 | bvs => target_label 6978 | blo => target_label 6979 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6980 } else { 6981 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6982 | bhi => target_label 6983 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6984 } 6985 break; 6986 default: 6987 ZEND_UNREACHABLE(); 6988 } 6989 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 6990 switch (opline->opcode) { 6991 case ZEND_IS_EQUAL: 6992 case ZEND_IS_IDENTICAL: 6993 case ZEND_CASE: 6994 case ZEND_CASE_STRICT: 6995 | bvs >1 6996 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6997 | beq => target_label 6998 |1: 6999 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7000 break; 7001 case ZEND_IS_NOT_EQUAL: 7002 case ZEND_IS_NOT_IDENTICAL: 7003 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7004 | bvs => target_label 7005 | bne => target_label 7006 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7007 break; 7008 case ZEND_IS_SMALLER: 7009 if (swap) { 7010 | cset REG0w, hi 7011 | add REG0w, REG0w, #2 7012 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7013 | bvs >1 // Always False if involving NaN 7014 | bhi => target_label 7015 |1: 7016 } else { 7017 | bvs >1 7018 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7019 | blo => target_label 7020 |1: 7021 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7022 } 7023 break; 7024 case ZEND_IS_SMALLER_OR_EQUAL: 7025 if (swap) { 7026 | cset REG0w, hs 7027 | add REG0w, REG0w, #2 7028 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7029 | bvs >1 // Always False if involving NaN 7030 | bhs => target_label 7031 |1: 7032 } else { 7033 | bvs >1 7034 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7035 | bls => target_label 7036 |1: 7037 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7038 } 7039 break; 7040 default: 7041 ZEND_UNREACHABLE(); 7042 } 7043 } else { 7044 ZEND_UNREACHABLE(); 7045 } 7046 } else { 7047 switch (opline->opcode) { 7048 case ZEND_IS_EQUAL: 7049 case ZEND_IS_IDENTICAL: 7050 case ZEND_CASE: 7051 case ZEND_CASE_STRICT: 7052 | bvs >1 7053 | mov REG0, #IS_TRUE 7054 | beq >2 7055 |1: 7056 | mov REG0, #IS_FALSE 7057 |2: 7058 break; 7059 case ZEND_IS_NOT_EQUAL: 7060 case ZEND_IS_NOT_IDENTICAL: 7061 | bvs >1 7062 | mov REG0, #IS_FALSE 7063 | beq >2 7064 |1: 7065 | mov REG0, #IS_TRUE 7066 |2: 7067 break; 7068 case ZEND_IS_SMALLER: 7069 | bvs >1 7070 | mov REG0, #IS_TRUE 7071 || if (swap) { 7072 | bhi >2 7073 || } else { 7074 | blo >2 7075 || } 7076 |1: 7077 | mov REG0, #IS_FALSE 7078 |2: 7079 break; 7080 case ZEND_IS_SMALLER_OR_EQUAL: 7081 | bvs >1 7082 | mov REG0, #IS_TRUE 7083 || if (swap) { 7084 | bhs >2 7085 || } else { 7086 | bls >2 7087 || } 7088 |1: 7089 | mov REG0, #IS_FALSE 7090 |2: 7091 break; 7092 default: 7093 ZEND_UNREACHABLE(); 7094 } 7095 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7096 } 7097 7098 return 1; 7099} 7100 7101static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7102{ 7103 zend_reg tmp_reg = ZREG_FPR0; 7104 7105 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 7106 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7107 7108 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7109} 7110 7111static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7112{ 7113 zend_reg tmp_reg = ZREG_FPR0; 7114 7115 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 7116 | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP 7117 7118 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7119} 7120 7121static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7122{ 7123 bool swap = 0; 7124 7125 if (Z_MODE(op1_addr) == IS_REG) { 7126 | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP 7127 } else if (Z_MODE(op2_addr) == IS_REG) { 7128 | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP 7129 swap = 1; 7130 } else { 7131 zend_reg tmp_reg = ZREG_FPR0; 7132 7133 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 7134 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7135 } 7136 7137 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7138} 7139 7140static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7141{ 7142 | tst RETVALw, RETVALw 7143 if (smart_branch_opcode) { 7144 if (smart_branch_opcode == ZEND_JMPZ_EX || 7145 smart_branch_opcode == ZEND_JMPNZ_EX) { 7146 switch (opline->opcode) { 7147 case ZEND_IS_EQUAL: 7148 case ZEND_CASE: 7149 | cset REG0w, eq 7150 break; 7151 case ZEND_IS_NOT_EQUAL: 7152 | cset REG0w, ne 7153 break; 7154 case ZEND_IS_SMALLER: 7155 | cset REG0w, lt 7156 break; 7157 case ZEND_IS_SMALLER_OR_EQUAL: 7158 | cset REG0w, le 7159 break; 7160 default: 7161 ZEND_UNREACHABLE(); 7162 } 7163 | add REG0w, REG0w, #2 7164 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7165 } 7166 if (smart_branch_opcode == ZEND_JMPZ || 7167 smart_branch_opcode == ZEND_JMPZ_EX) { 7168 switch (opline->opcode) { 7169 case ZEND_IS_EQUAL: 7170 case ZEND_CASE: 7171 if (exit_addr) { 7172 | bne &exit_addr 7173 } else { 7174 | bne => target_label 7175 } 7176 break; 7177 case ZEND_IS_NOT_EQUAL: 7178 if (exit_addr) { 7179 | beq &exit_addr 7180 } else { 7181 | beq => target_label 7182 } 7183 break; 7184 case ZEND_IS_SMALLER: 7185 if (exit_addr) { 7186 | bge &exit_addr 7187 } else { 7188 | bge => target_label 7189 } 7190 break; 7191 case ZEND_IS_SMALLER_OR_EQUAL: 7192 if (exit_addr) { 7193 | bgt &exit_addr 7194 } else { 7195 | bgt => target_label 7196 } 7197 break; 7198 default: 7199 ZEND_UNREACHABLE(); 7200 } 7201 } else if (smart_branch_opcode == ZEND_JMPNZ || 7202 smart_branch_opcode == ZEND_JMPNZ_EX) { 7203 switch (opline->opcode) { 7204 case ZEND_IS_EQUAL: 7205 case ZEND_CASE: 7206 if (exit_addr) { 7207 | beq &exit_addr 7208 } else { 7209 | beq => target_label 7210 } 7211 break; 7212 case ZEND_IS_NOT_EQUAL: 7213 if (exit_addr) { 7214 | bne &exit_addr 7215 } else { 7216 | bne => target_label 7217 } 7218 break; 7219 case ZEND_IS_SMALLER: 7220 if (exit_addr) { 7221 | blt &exit_addr 7222 } else { 7223 | blt => target_label 7224 } 7225 break; 7226 case ZEND_IS_SMALLER_OR_EQUAL: 7227 if (exit_addr) { 7228 | ble &exit_addr 7229 } else { 7230 | ble => target_label 7231 } 7232 break; 7233 default: 7234 ZEND_UNREACHABLE(); 7235 } 7236 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7237 switch (opline->opcode) { 7238 case ZEND_IS_EQUAL: 7239 case ZEND_CASE: 7240 | bne => target_label 7241 break; 7242 case ZEND_IS_NOT_EQUAL: 7243 | beq => target_label 7244 break; 7245 case ZEND_IS_SMALLER: 7246 | bge => target_label 7247 break; 7248 case ZEND_IS_SMALLER_OR_EQUAL: 7249 | bgt => target_label 7250 break; 7251 default: 7252 ZEND_UNREACHABLE(); 7253 } 7254 | b => target_label2 7255 } else { 7256 ZEND_UNREACHABLE(); 7257 } 7258 } else { 7259 switch (opline->opcode) { 7260 case ZEND_IS_EQUAL: 7261 case ZEND_CASE: 7262 | cset REG0w, eq 7263 break; 7264 case ZEND_IS_NOT_EQUAL: 7265 | cset REG0w, ne 7266 break; 7267 case ZEND_IS_SMALLER: 7268 | cset REG0w, lt 7269 break; 7270 case ZEND_IS_SMALLER_OR_EQUAL: 7271 | cset REG0w, le 7272 break; 7273 default: 7274 ZEND_UNREACHABLE(); 7275 } 7276 | add REG0w, REG0w, #2 7277 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7278 } 7279 7280 return 1; 7281} 7282 7283static int zend_jit_cmp(dasm_State **Dst, 7284 const zend_op *opline, 7285 uint32_t op1_info, 7286 zend_ssa_range *op1_range, 7287 zend_jit_addr op1_addr, 7288 uint32_t op2_info, 7289 zend_ssa_range *op2_range, 7290 zend_jit_addr op2_addr, 7291 zend_jit_addr res_addr, 7292 int may_throw, 7293 zend_uchar smart_branch_opcode, 7294 uint32_t target_label, 7295 uint32_t target_label2, 7296 const void *exit_addr, 7297 bool skip_comparison) 7298{ 7299 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7300 bool has_slow; 7301 7302 has_slow = 7303 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7304 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7305 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7306 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7307 7308 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7309 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7310 if (op1_info & MAY_BE_DOUBLE) { 7311 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 7312 } else { 7313 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7314 } 7315 } 7316 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7317 if (op2_info & MAY_BE_DOUBLE) { 7318 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 7319 |.cold_code 7320 |3: 7321 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7322 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7323 } 7324 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7325 return 0; 7326 } 7327 | b >6 7328 |.code 7329 } else { 7330 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7331 } 7332 } 7333 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7334 return 0; 7335 } 7336 if (op1_info & MAY_BE_DOUBLE) { 7337 |.cold_code 7338 |4: 7339 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7340 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7341 } 7342 if (op2_info & MAY_BE_DOUBLE) { 7343 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7344 if (!same_ops) { 7345 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 7346 } else { 7347 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7348 } 7349 } 7350 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7351 return 0; 7352 } 7353 | b >6 7354 } 7355 if (!same_ops) { 7356 |5: 7357 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7358 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7359 } 7360 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7361 return 0; 7362 } 7363 | b >6 7364 } 7365 |.code 7366 } 7367 } else if ((op1_info & MAY_BE_DOUBLE) && 7368 !(op1_info & MAY_BE_LONG) && 7369 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7370 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7371 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7372 } 7373 if (op2_info & MAY_BE_DOUBLE) { 7374 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7375 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7376 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 7377 } else { 7378 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7379 } 7380 } 7381 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7382 return 0; 7383 } 7384 } 7385 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7386 if (op2_info & MAY_BE_DOUBLE) { 7387 |.cold_code 7388 } 7389 |3: 7390 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7391 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7392 } 7393 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7394 return 0; 7395 } 7396 if (op2_info & MAY_BE_DOUBLE) { 7397 | b >6 7398 |.code 7399 } 7400 } 7401 } else if ((op2_info & MAY_BE_DOUBLE) && 7402 !(op2_info & MAY_BE_LONG) && 7403 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7404 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7405 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7406 } 7407 if (op1_info & MAY_BE_DOUBLE) { 7408 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7409 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7410 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 7411 } else { 7412 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7413 } 7414 } 7415 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7416 return 0; 7417 } 7418 } 7419 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7420 if (op1_info & MAY_BE_DOUBLE) { 7421 |.cold_code 7422 } 7423 |3: 7424 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7425 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7426 } 7427 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7428 return 0; 7429 } 7430 if (op1_info & MAY_BE_DOUBLE) { 7431 | b >6 7432 |.code 7433 } 7434 } 7435 } 7436 7437 if (has_slow || 7438 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7439 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7440 if (has_slow) { 7441 |.cold_code 7442 |9: 7443 } 7444 | SET_EX_OPLINE opline, REG0 7445 if (Z_MODE(op1_addr) == IS_REG) { 7446 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7447 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7448 return 0; 7449 } 7450 op1_addr = real_addr; 7451 } 7452 if (Z_MODE(op2_addr) == IS_REG) { 7453 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7454 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7455 return 0; 7456 } 7457 op2_addr = real_addr; 7458 } 7459 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7460 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 7461 | IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7462 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 7463 | EXT_CALL zend_jit_undefined_op_helper, REG0 7464 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7465 |1: 7466 } 7467 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 7468 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 7469 | str FCARG1x, T1 // save 7470 | LOAD_32BIT_VAL FCARG1x, opline->op2.var 7471 | EXT_CALL zend_jit_undefined_op_helper, REG0 7472 | ldr FCARG1x, T1 // restore 7473 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7474 | b >2 7475 |1: 7476 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7477 |2: 7478 } else { 7479 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7480 } 7481 | EXT_CALL zend_compare, REG0 7482 if ((opline->opcode != ZEND_CASE && 7483 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7484 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7485 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7486 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7487 | str RETVALw, T1 // save 7488 if (opline->opcode != ZEND_CASE) { 7489 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7490 } 7491 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7492 | ldr RETVALw, T1 // restore 7493 } 7494 if (may_throw) { 7495 zend_jit_check_exception_undef_result(Dst, opline); 7496 } 7497 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7498 return 0; 7499 } 7500 if (has_slow) { 7501 | b >6 7502 |.code 7503 } 7504 } 7505 7506 |6: 7507 7508 return 1; 7509} 7510 7511static int zend_jit_identical(dasm_State **Dst, 7512 const zend_op *opline, 7513 uint32_t op1_info, 7514 zend_ssa_range *op1_range, 7515 zend_jit_addr op1_addr, 7516 uint32_t op2_info, 7517 zend_ssa_range *op2_range, 7518 zend_jit_addr op2_addr, 7519 zend_jit_addr res_addr, 7520 int may_throw, 7521 zend_uchar smart_branch_opcode, 7522 uint32_t target_label, 7523 uint32_t target_label2, 7524 const void *exit_addr, 7525 bool skip_comparison) 7526{ 7527 uint32_t identical_label = (uint32_t)-1; 7528 uint32_t not_identical_label = (uint32_t)-1; 7529 7530 if (smart_branch_opcode && !exit_addr) { 7531 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7532 if (smart_branch_opcode == ZEND_JMPZ) { 7533 not_identical_label = target_label; 7534 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7535 identical_label = target_label; 7536 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7537 not_identical_label = target_label; 7538 identical_label = target_label2; 7539 } else { 7540 ZEND_UNREACHABLE(); 7541 } 7542 } else { 7543 if (smart_branch_opcode == ZEND_JMPZ) { 7544 identical_label = target_label; 7545 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7546 not_identical_label = target_label; 7547 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7548 identical_label = target_label; 7549 not_identical_label = target_label2; 7550 } else { 7551 ZEND_UNREACHABLE(); 7552 } 7553 } 7554 } 7555 7556 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 7557 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 7558 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7559 return 0; 7560 } 7561 return 1; 7562 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 7563 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 7564 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7565 return 0; 7566 } 7567 return 1; 7568 } 7569 7570 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 7571 op1_info |= MAY_BE_NULL; 7572 op2_info |= MAY_BE_NULL; 7573 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7574 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7575 |.cold_code 7576 |1: 7577 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7578 | SET_EX_OPLINE opline, REG0 7579 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7580 | EXT_CALL zend_jit_undefined_op_helper, REG0 7581 if (may_throw) { 7582 zend_jit_check_exception_undef_result(Dst, opline); 7583 } 7584 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7585 | b >1 7586 |.code 7587 |1: 7588 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7589 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7590 |.cold_code 7591 |1: 7592 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7593 | SET_EX_OPLINE opline, REG0 7594 | str FCARG1x, T1 // save 7595 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7596 | EXT_CALL zend_jit_undefined_op_helper, REG0 7597 if (may_throw) { 7598 zend_jit_check_exception_undef_result(Dst, opline); 7599 } 7600 | ldr FCARG1x, T1 // restore 7601 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7602 | b >1 7603 |.code 7604 |1: 7605 } else if (op1_info & MAY_BE_UNDEF) { 7606 op1_info |= MAY_BE_NULL; 7607 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7608 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7609 |.cold_code 7610 |1: 7611 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7612 | SET_EX_OPLINE opline, REG0 7613 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7614 | EXT_CALL zend_jit_undefined_op_helper, REG0 7615 if (may_throw) { 7616 zend_jit_check_exception_undef_result(Dst, opline); 7617 } 7618 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7619 | b >1 7620 |.code 7621 |1: 7622 if (opline->op2_type != IS_CONST) { 7623 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7624 } 7625 } else if (op2_info & MAY_BE_UNDEF) { 7626 op2_info |= MAY_BE_NULL; 7627 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7628 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7629 |.cold_code 7630 |1: 7631 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7632 | SET_EX_OPLINE opline, REG0 7633 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7634 | EXT_CALL zend_jit_undefined_op_helper, REG0 7635 if (may_throw) { 7636 zend_jit_check_exception_undef_result(Dst, opline); 7637 } 7638 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7639 | b >1 7640 |.code 7641 |1: 7642 if (opline->op1_type != IS_CONST) { 7643 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7644 } 7645 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 7646 if (opline->op1_type != IS_CONST) { 7647 if (Z_MODE(op1_addr) == IS_REG) { 7648 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7649 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7650 return 0; 7651 } 7652 op1_addr = real_addr; 7653 } 7654 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7655 } 7656 if (opline->op2_type != IS_CONST) { 7657 if (Z_MODE(op2_addr) == IS_REG) { 7658 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7659 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7660 return 0; 7661 } 7662 op2_addr = real_addr; 7663 } 7664 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7665 } 7666 } 7667 7668 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 7669 if ((opline->opcode != ZEND_CASE_STRICT && 7670 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7671 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7672 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7673 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7674 if (opline->opcode != ZEND_CASE_STRICT) { 7675 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7676 } 7677 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7678 } 7679 if (smart_branch_opcode) { 7680 if (may_throw) { 7681 zend_jit_check_exception_undef_result(Dst, opline); 7682 } 7683 if (exit_addr) { 7684 if (smart_branch_opcode == ZEND_JMPZ) { 7685 | b &exit_addr 7686 } 7687 } else if (not_identical_label != (uint32_t)-1) { 7688 | b =>not_identical_label 7689 } 7690 } else { 7691 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7692 if (may_throw) { 7693 zend_jit_check_exception(Dst); 7694 } 7695 } 7696 return 1; 7697 } 7698 7699 if (opline->op1_type & (IS_CV|IS_VAR)) { 7700 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7701 } 7702 if (opline->op2_type & (IS_CV|IS_VAR)) { 7703 | ZVAL_DEREF FCARG2x, op2_info, TMP1w 7704 } 7705 7706 if (has_concrete_type(op1_info) 7707 && has_concrete_type(op2_info) 7708 && concrete_type(op1_info) == concrete_type(op2_info) 7709 && concrete_type(op1_info) <= IS_TRUE) { 7710 if (smart_branch_opcode) { 7711 if (exit_addr) { 7712 if (smart_branch_opcode == ZEND_JMPNZ) { 7713 | b &exit_addr 7714 } 7715 } else if (identical_label != (uint32_t)-1) { 7716 | b =>identical_label 7717 } 7718 } else { 7719 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7720 } 7721 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 7722 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 7723 if (smart_branch_opcode) { 7724 if (exit_addr) { 7725 if (smart_branch_opcode == ZEND_JMPNZ) { 7726 | b &exit_addr 7727 } 7728 } else if (identical_label != (uint32_t)-1) { 7729 | b =>identical_label 7730 } 7731 } else { 7732 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7733 } 7734 } else { 7735 if (smart_branch_opcode) { 7736 if (exit_addr) { 7737 if (smart_branch_opcode == ZEND_JMPZ) { 7738 | b &exit_addr 7739 } 7740 } else if (not_identical_label != (uint32_t)-1) { 7741 | b =>not_identical_label 7742 } 7743 } else { 7744 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7745 } 7746 } 7747 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 7748 zval *val = Z_ZV(op1_addr); 7749 7750 | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] 7751 | cmp TMP1w, #Z_TYPE_P(val) 7752 if (smart_branch_opcode) { 7753 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 7754 | bne >8 7755 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7756 if (may_throw) { 7757 zend_jit_check_exception_undef_result(Dst, opline); 7758 } 7759 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7760 | b &exit_addr 7761 } else if (identical_label != (uint32_t)-1) { 7762 | b =>identical_label 7763 } else { 7764 | b >9 7765 } 7766 |8: 7767 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7768 | beq &exit_addr 7769 } else if (identical_label != (uint32_t)-1) { 7770 | beq =>identical_label 7771 } else { 7772 | beq >9 7773 } 7774 } else { 7775 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7776 | cset REG0w, eq 7777 } else { 7778 | cset REG0w, ne 7779 } 7780 | add REG0w, REG0w, #2 7781 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7782 } 7783 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7784 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7785 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7786 if (may_throw) { 7787 zend_jit_check_exception_undef_result(Dst, opline); 7788 } 7789 } 7790 if (exit_addr) { 7791 if (smart_branch_opcode == ZEND_JMPZ) { 7792 | b &exit_addr 7793 } 7794 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 7795 | b =>not_identical_label 7796 } 7797 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 7798 zval *val = Z_ZV(op2_addr); 7799 7800 | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] 7801 | cmp TMP1w, #Z_TYPE_P(val) 7802 if (smart_branch_opcode) { 7803 if (opline->opcode != ZEND_CASE_STRICT 7804 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 7805 | bne >8 7806 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7807 if (may_throw) { 7808 zend_jit_check_exception_undef_result(Dst, opline); 7809 } 7810 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7811 | b &exit_addr 7812 } else if (identical_label != (uint32_t)-1) { 7813 | b =>identical_label 7814 } else { 7815 | b >9 7816 } 7817 |8: 7818 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7819 | beq &exit_addr 7820 } else if (identical_label != (uint32_t)-1) { 7821 | beq =>identical_label 7822 } else { 7823 | beq >9 7824 } 7825 } else { 7826 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7827 | cset REG0w, eq 7828 } else { 7829 | cset REG0w, ne 7830 } 7831 | add REG0w, REG0w, #2 7832 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7833 } 7834 if (opline->opcode != ZEND_CASE_STRICT 7835 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7836 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7837 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7838 if (may_throw) { 7839 zend_jit_check_exception_undef_result(Dst, opline); 7840 } 7841 } 7842 if (smart_branch_opcode) { 7843 if (exit_addr) { 7844 if (smart_branch_opcode == ZEND_JMPZ) { 7845 | b &exit_addr 7846 } 7847 } else if (not_identical_label != (uint32_t)-1) { 7848 | b =>not_identical_label 7849 } 7850 } 7851 } else { 7852 if (opline->op1_type == IS_CONST) { 7853 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7854 } 7855 if (opline->op2_type == IS_CONST) { 7856 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7857 } 7858 | EXT_CALL zend_is_identical, REG0 7859 if ((opline->opcode != ZEND_CASE_STRICT && 7860 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7861 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7862 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7863 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7864 | str RETVALw, T1 // save 7865 if (opline->opcode != ZEND_CASE_STRICT) { 7866 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7867 } 7868 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7869 if (may_throw) { 7870 zend_jit_check_exception_undef_result(Dst, opline); 7871 } 7872 | ldr RETVALw, T1 // restore 7873 } 7874 if (smart_branch_opcode) { 7875 if (exit_addr) { 7876 if (smart_branch_opcode == ZEND_JMPNZ) { 7877 | cbnz RETVALw, &exit_addr 7878 } else { 7879 | cbz RETVALw, &exit_addr 7880 } 7881 } else if (not_identical_label != (uint32_t)-1) { 7882 | cbz RETVALw, =>not_identical_label 7883 if (identical_label != (uint32_t)-1) { 7884 | b =>identical_label 7885 } 7886 } else if (identical_label != (uint32_t)-1) { 7887 | cbnz RETVALw, =>identical_label 7888 } 7889 } else { 7890 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7891 | add RETVALw, RETVALw, #2 7892 } else { 7893 | neg RETVALw, RETVALw 7894 | add RETVALw, RETVALw, #3 7895 } 7896 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 7897 } 7898 } 7899 7900 |9: 7901 if (may_throw) { 7902 zend_jit_check_exception(Dst); 7903 } 7904 return 1; 7905} 7906 7907static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 7908{ 7909 uint32_t true_label = -1; 7910 uint32_t false_label = -1; 7911 bool set_bool = 0; 7912 bool set_bool_not = 0; 7913 bool set_delayed = 0; 7914 bool jmp_done = 0; 7915 7916 if (branch_opcode == ZEND_BOOL) { 7917 set_bool = 1; 7918 } else if (branch_opcode == ZEND_BOOL_NOT) { 7919 set_bool = 1; 7920 set_bool_not = 1; 7921 } else if (branch_opcode == ZEND_JMPZ) { 7922 false_label = target_label; 7923 } else if (branch_opcode == ZEND_JMPNZ) { 7924 true_label = target_label; 7925 } else if (branch_opcode == ZEND_JMPZNZ) { 7926 true_label = target_label2; 7927 false_label = target_label; 7928 } else if (branch_opcode == ZEND_JMPZ_EX) { 7929 set_bool = 1; 7930 false_label = target_label; 7931 } else if (branch_opcode == ZEND_JMPNZ_EX) { 7932 set_bool = 1; 7933 true_label = target_label; 7934 } else { 7935 ZEND_UNREACHABLE(); 7936 } 7937 7938 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 7939 if (zend_is_true(Z_ZV(op1_addr))) { 7940 /* Always TRUE */ 7941 if (set_bool) { 7942 if (set_bool_not) { 7943 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7944 } else { 7945 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7946 } 7947 } 7948 if (true_label != (uint32_t)-1) { 7949 | b =>true_label 7950 } 7951 } else { 7952 /* Always FALSE */ 7953 if (set_bool) { 7954 if (set_bool_not) { 7955 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7956 } else { 7957 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7958 } 7959 } 7960 if (false_label != (uint32_t)-1) { 7961 | b =>false_label 7962 } 7963 } 7964 return 1; 7965 } 7966 7967 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 7968 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7969 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7970 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7971 } 7972 7973 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 7974 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 7975 /* Always TRUE */ 7976 if (set_bool) { 7977 if (set_bool_not) { 7978 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7979 } else { 7980 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7981 } 7982 } 7983 if (true_label != (uint32_t)-1) { 7984 | b =>true_label 7985 } 7986 } else { 7987 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 7988 /* Always FALSE */ 7989 if (set_bool) { 7990 if (set_bool_not) { 7991 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7992 } else { 7993 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7994 } 7995 } 7996 } else { 7997 | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 7998 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 7999 if ((op1_info & MAY_BE_LONG) && 8000 !(op1_info & MAY_BE_UNDEF) && 8001 !set_bool) { 8002 if (exit_addr) { 8003 if (branch_opcode == ZEND_JMPNZ) { 8004 | blt >9 8005 } else { 8006 | blt &exit_addr 8007 } 8008 } else if (false_label != (uint32_t)-1) { 8009 | blt =>false_label 8010 } else { 8011 | blt >9 8012 } 8013 jmp_done = 1; 8014 } else { 8015 | bgt >2 8016 } 8017 } 8018 if (!(op1_info & MAY_BE_TRUE)) { 8019 /* It's FALSE */ 8020 if (set_bool) { 8021 if (set_bool_not) { 8022 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8023 } else { 8024 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8025 } 8026 } 8027 } else { 8028 if (exit_addr) { 8029 if (set_bool) { 8030 | bne >1 8031 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8032 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8033 | b &exit_addr 8034 } else { 8035 | b >9 8036 } 8037 |1: 8038 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8039 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8040 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8041 | bne &exit_addr 8042 } 8043 } 8044 } else { 8045 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8046 | beq &exit_addr 8047 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8048 | bne &exit_addr 8049 } else { 8050 | beq >9 8051 } 8052 } 8053 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8054 if (set_bool) { 8055 | bne >1 8056 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8057 if (true_label != (uint32_t)-1) { 8058 | b =>true_label 8059 } else { 8060 | b >9 8061 } 8062 |1: 8063 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8064 } else { 8065 if (true_label != (uint32_t)-1) { 8066 | beq =>true_label 8067 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8068 | bne =>false_label 8069 jmp_done = 1; 8070 } else { 8071 | beq >9 8072 } 8073 } 8074 } else if (set_bool) { 8075 | cset REG0w, eq 8076 if (set_bool_not) { 8077 | neg REG0w, REG0w 8078 | add REG0w, REG0w, #3 8079 } else { 8080 | add REG0w, REG0w, #2 8081 } 8082 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8083 set_delayed = 1; 8084 } else { 8085 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8086 } 8087 } 8088 } 8089 } 8090 8091 /* It's FALSE, but may be UNDEF */ 8092 if (op1_info & MAY_BE_UNDEF) { 8093 if (op1_info & MAY_BE_ANY) { 8094 if (set_delayed) { 8095 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 8096 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8097 | beq >1 8098 } else { 8099 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 8100 } 8101 |.cold_code 8102 |1: 8103 } 8104 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 8105 | SET_EX_OPLINE opline, REG0 8106 | EXT_CALL zend_jit_undefined_op_helper, REG0 8107 8108 if (may_throw) { 8109 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8110 return 0; 8111 } 8112 } 8113 8114 if (exit_addr) { 8115 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8116 | b &exit_addr 8117 } 8118 } else if (false_label != (uint32_t)-1) { 8119 | b =>false_label 8120 } 8121 if (op1_info & MAY_BE_ANY) { 8122 if (exit_addr) { 8123 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8124 | b >9 8125 } 8126 } else if (false_label == (uint32_t)-1) { 8127 | b >9 8128 } 8129 |.code 8130 } 8131 } 8132 8133 if (!jmp_done) { 8134 if (exit_addr) { 8135 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8136 if (op1_info & MAY_BE_LONG) { 8137 | b >9 8138 } 8139 } else if (op1_info & MAY_BE_LONG) { 8140 | b &exit_addr 8141 } 8142 } else if (false_label != (uint32_t)-1) { 8143 | b =>false_label 8144 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8145 | b >9 8146 } 8147 } 8148 } 8149 } 8150 8151 if (op1_info & MAY_BE_LONG) { 8152 |2: 8153 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8154 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 8155 } 8156 if (Z_MODE(op1_addr) == IS_REG) { 8157 | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 8158 } else { 8159 | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 8160 } 8161 if (set_bool) { 8162 | cset REG0w, ne 8163 if (set_bool_not) { 8164 | neg REG0w, REG0w 8165 | add REG0w, REG0w, #3 8166 } else { 8167 | add REG0w, REG0w, #2 8168 } 8169 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8170 } 8171 if (exit_addr) { 8172 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8173 | bne &exit_addr 8174 } else { 8175 | beq &exit_addr 8176 } 8177 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8178 if (true_label != (uint32_t)-1) { 8179 | bne =>true_label 8180 if (false_label != (uint32_t)-1) { 8181 | b =>false_label 8182 } 8183 } else { 8184 | beq =>false_label 8185 } 8186 } 8187 } 8188 8189 if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8190 |2: 8191 | fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 8192 | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP 8193 8194 if (set_bool) { 8195 if (exit_addr) { 8196 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8197 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8198 | bvs &exit_addr 8199 | bne &exit_addr 8200 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8201 } else { 8202 | bvs >1 8203 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8204 | beq &exit_addr 8205 |1: 8206 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8207 } 8208 } else if (false_label != (uint32_t)-1) { // JMPZ_EX 8209 | bvs >1 8210 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8211 | beq => false_label 8212 |1: 8213 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8214 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX 8215 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8216 | bvs => true_label 8217 | bne => true_label 8218 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8219 } else if (set_bool_not) { // BOOL_NOT 8220 | mov REG0w, #IS_FALSE 8221 | bvs >1 8222 | bne >1 8223 | mov REG0w, #IS_TRUE 8224 |1: 8225 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8226 } else { // BOOL 8227 | mov REG0w, #IS_TRUE 8228 | bvs >1 8229 | bne >1 8230 | mov REG0w, #IS_FALSE 8231 |1: 8232 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8233 } 8234 } else { 8235 if (exit_addr) { 8236 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8237 | bvs &exit_addr 8238 | bne &exit_addr 8239 |1: 8240 } else { 8241 | bvs >1 8242 | beq &exit_addr 8243 |1: 8244 } 8245 } else { 8246 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8247 if (false_label != (uint32_t)-1 ) { 8248 | bvs >1 8249 | beq => false_label 8250 |1: 8251 if (true_label != (uint32_t)-1) { 8252 | b =>true_label 8253 } 8254 } else { 8255 | bvs => true_label 8256 | bne => true_label 8257 } 8258 } 8259 } 8260 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8261 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8262 |.cold_code 8263 |2: 8264 } 8265 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8266 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8267 } 8268 | SET_EX_OPLINE opline, REG0 8269 | EXT_CALL zend_is_true, REG0 8270 | mov REG0, RETVALx 8271 8272 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8273 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8274 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8275 8276 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8277 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 8278 } 8279 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8280 | GC_DELREF FCARG1x, TMP1w 8281 | bne >3 8282 // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored 8283 // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, 8284 // because it's clobbered by function call. 8285 | str REG0, T1 // save 8286 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 8287 | ldr REG0, T1 // restore 8288 |3: 8289 } 8290 if (may_throw) { 8291 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1 8292 | cbnz REG1, ->exception_handler_undef 8293 } 8294 8295 if (set_bool) { 8296 if (set_bool_not) { 8297 | neg REG0w, REG0w 8298 | add REG0w, REG0w, #3 8299 } else { 8300 | add REG0w, REG0w, #2 8301 } 8302 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8303 if (exit_addr) { 8304 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8305 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8306 | bne &exit_addr 8307 } else { 8308 | beq &exit_addr 8309 } 8310 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8311 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8312 if (true_label != (uint32_t)-1) { 8313 | bne =>true_label 8314 if (false_label != (uint32_t)-1) { 8315 | b =>false_label 8316 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8317 | b >9 8318 } 8319 } else { 8320 | beq =>false_label 8321 } 8322 } 8323 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8324 | b >9 8325 |.code 8326 } 8327 } else { 8328 if (exit_addr) { 8329 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8330 | cbnz REG0w, &exit_addr 8331 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8332 | b >9 8333 } 8334 } else { 8335 | cbz REG0w, &exit_addr 8336 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8337 | b >9 8338 } 8339 } 8340 } else if (true_label != (uint32_t)-1) { 8341 | cbnz REG0w, =>true_label 8342 if (false_label != (uint32_t)-1) { 8343 | b =>false_label 8344 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8345 | b >9 8346 } 8347 } else { 8348 | cbz REG0w, =>false_label 8349 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8350 | b >9 8351 } 8352 } 8353 8354 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8355 |.code 8356 } 8357 } 8358 } 8359 8360 |9: 8361 8362 return 1; 8363} 8364 8365static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8366{ 8367 if (op1_addr != op1_def_addr) { 8368 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8369 return 0; 8370 } 8371 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8372 op1_addr = op1_def_addr; 8373 } 8374 } 8375 8376 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { 8377 return 0; 8378 } 8379 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8380 return 0; 8381 } 8382 if (op1_info & MAY_BE_UNDEF) { 8383 zend_jit_check_exception(Dst); 8384 } 8385 return 1; 8386} 8387 8388static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8389{ 8390 ZEND_ASSERT(opline->op1_type == IS_CV); 8391 8392 if (op2_addr != op2_def_addr) { 8393 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8394 return 0; 8395 } 8396 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8397 op2_addr = op2_def_addr; 8398 } 8399 } 8400 8401 if (Z_MODE(op1_addr) != IS_REG 8402 && Z_MODE(op1_use_addr) == IS_REG 8403 && !Z_LOAD(op1_use_addr) 8404 && !Z_STORE(op1_use_addr)) { 8405 /* Force type update */ 8406 op1_info |= MAY_BE_UNDEF; 8407 } 8408 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8409 may_throw)) { 8410 return 0; 8411 } 8412 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8413 return 0; 8414 } 8415 if (opline->result_type != IS_UNUSED) { 8416 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8417 return 0; 8418 } 8419 } 8420 8421 return 1; 8422} 8423 8424/* copy of hidden zend_closure */ 8425typedef struct _zend_closure { 8426 zend_object std; 8427 zend_function func; 8428 zval this_ptr; 8429 zend_class_entry *called_scope; 8430 zif_handler orig_internal_handler; 8431} zend_closure; 8432 8433static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8434{ 8435 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8436 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8437 8438 if (!exit_addr) { 8439 return 0; 8440 } 8441 8442 | // Check Stack Overflow 8443 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 8444 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 8445 | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 8446 | blo &exit_addr 8447 8448 return 1; 8449} 8450 8451static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 8452{ 8453 uint32_t used_stack; 8454 bool stack_check = 1; 8455 8456 // REG0 -> zend_function 8457 // FCARG1 -> used_stack 8458 8459 if (func) { 8460 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 8461 if ((int)used_stack <= checked_stack) { 8462 stack_check = 0; 8463 } 8464 } else { 8465 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 8466 8467 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 8468 if (!is_closure) { 8469 | LOAD_32BIT_VAL FCARG1w, used_stack 8470 | // Check whether REG0 is an internal function. 8471 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 8472 | TST_32_WITH_CONST TMP1w, 1, TMP2w 8473 | bne >1 8474 } else { 8475 | LOAD_32BIT_VAL FCARG1w, used_stack 8476 } 8477 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 8478 | LOAD_32BIT_VAL REG2w, opline->extended_value 8479 if (!is_closure) { 8480 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] 8481 | cmp REG2w, TMP1w 8482 | csel REG2w, REG2w, TMP1w, le 8483 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] 8484 | sub REG2w, REG2w, TMP1w 8485 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] 8486 | sub REG2w, REG2w, TMP1w 8487 } else { 8488 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] 8489 | cmp REG2w, TMP1w 8490 | csel REG2w, REG2w, TMP1w, le 8491 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] 8492 | sub REG2w, REG2w, TMP1w 8493 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] 8494 | sub REG2w, REG2w, TMP1w 8495 } 8496 | sxtw REG2, REG2w 8497 | sub FCARG1x, FCARG1x, REG2, lsl #4 8498 |1: 8499 } 8500 8501 zend_jit_start_reuse_ip(); 8502 8503 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 8504 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 8505 8506 if (stack_check) { 8507 | // Check Stack Overflow 8508 | MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 8509 | sub REG2, REG2, RX 8510 if (func) { 8511 | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 8512 } else { 8513 | cmp REG2, FCARG1x 8514 } 8515 8516 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8517 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8518 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8519 8520 if (!exit_addr) { 8521 return 0; 8522 } 8523 8524 | blo &exit_addr 8525 } else { 8526 | blo >1 8527 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 8528 |.cold_code 8529 |1: 8530 if (func) { 8531 | LOAD_32BIT_VAL FCARG1w, used_stack 8532 } 8533 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8534 | SET_EX_OPLINE opline, REG0 8535 | EXT_CALL zend_jit_int_extend_stack_helper, REG0 8536 } else { 8537 if (!is_closure) { 8538 | mov FCARG2x, REG0 8539 } else { 8540 | add FCARG2x, REG0, #offsetof(zend_closure, func) 8541 } 8542 | SET_EX_OPLINE opline, REG0 8543 | EXT_CALL zend_jit_extend_stack_helper, REG0 8544 } 8545 | mov RX, RETVALx 8546 | b >1 8547 |.code 8548 } 8549 } 8550 8551 if (func) { 8552 || if (arm64_may_encode_imm12((int64_t)used_stack)) { 8553 | MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 8554 || } else { 8555 | LOAD_32BIT_VAL TMP1w, used_stack 8556 | MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 8557 || } 8558 } else { 8559 | MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 8560 } 8561 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 8562 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 8563 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8564 | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 8565 | str TMP1w, EX:RX->This.u1.type_info 8566 } 8567 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8568 | // call->func = func; 8569 |1: 8570 | ADDR_STORE EX:RX->func, func, REG1 8571 } else { 8572 if (!is_closure) { 8573 | // call->func = func; 8574 | str REG0, EX:RX->func 8575 } else { 8576 | // call->func = &closure->func; 8577 | add REG1, REG0, #offsetof(zend_closure, func) 8578 | str REG1, EX:RX->func 8579 } 8580 |1: 8581 } 8582 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 8583 | // Z_PTR(call->This) = obj; 8584 | ldr REG1, T1 8585 | str REG1, EX:RX->This.value.ptr 8586 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 8587 | // call->call_info |= ZEND_CALL_HAS_THIS; 8588 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8589 | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS 8590 | str TMP1w, EX:RX->This.u1.type_info 8591 } else { 8592 | ldr TMP1w, EX:RX->This.u1.type_info 8593 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w 8594 | str TMP1w, EX:RX->This.u1.type_info 8595 } 8596 } else { 8597 if (opline->op1_type == IS_CV) { 8598 | // GC_ADDREF(obj); 8599 | GC_ADDREF REG1, TMP1w 8600 } 8601 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 8602 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8603 | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 8604 | str TMP1w, EX:RX->This.u1.type_info 8605 } else { 8606 | ldr TMP1w, EX:RX->This.u1.type_info 8607 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w 8608 | str TMP1w, EX:RX->This.u1.type_info 8609 } 8610 } 8611 } else if (!is_closure) { 8612 | // Z_CE(call->This) = called_scope; 8613 | str xzr, EX:RX->This.value.ptr 8614 } else { 8615 if (opline->op2_type == IS_CV) { 8616 | // GC_ADDREF(closure); 8617 | GC_ADDREF REG0, TMP1w 8618 } 8619 | // object_or_called_scope = closure->called_scope; 8620 | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] 8621 | str REG1, EX:RX->This.value.ptr 8622 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 8623 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 8624 | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] 8625 | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w 8626 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w 8627 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 8628 | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] 8629 | cmp TMP1w, #IS_UNDEF 8630 | beq >1 8631 | // call_info |= ZEND_CALL_HAS_THIS; 8632 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w 8633 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 8634 | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] 8635 |1: 8636 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8637 | ldr TMP1w, EX:RX->This.u1.type_info 8638 | orr TMP1w, TMP1w, REG2w 8639 | str TMP1w, EX:RX->This.u1.type_info 8640 | // Z_PTR(call->This) = object_or_called_scope; 8641 | str REG1, EX:RX->This.value.ptr 8642 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] 8643 | cbnz TMP1, >1 8644 | add FCARG1x, REG0, #offsetof(zend_closure, func) 8645 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8646 |1: 8647 } 8648 | // ZEND_CALL_NUM_ARGS(call) = num_args; 8649 | LOAD_32BIT_VAL TMP1w, opline->extended_value 8650 | str TMP1w, EX:RX->This.u2.num_args 8651 8652 return 1; 8653} 8654 8655static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 8656{ 8657 int32_t exit_point; 8658 const void *exit_addr; 8659 8660 if (func->type == ZEND_INTERNAL_FUNCTION) { 8661#ifdef ZEND_WIN32 8662 // TODO: ASLR may cause different addresses in different workers ??? 8663 return 0; 8664#endif 8665 } else if (func->type == ZEND_USER_FUNCTION) { 8666 if (!zend_accel_in_shm(func->op_array.opcodes)) { 8667 /* op_array and op_array->opcodes are not persistent. We can't link. */ 8668 return 0; 8669 } 8670 } else { 8671 ZEND_UNREACHABLE(); 8672 return 0; 8673 } 8674 8675 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 8676 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8677 if (!exit_addr) { 8678 return 0; 8679 } 8680 8681 | // call = EX(call); 8682 | ldr REG1, EX->call 8683 while (level > 0) { 8684 | ldr REG1, EX:REG1->prev_execute_data 8685 level--; 8686 } 8687 8688 if (func->type == ZEND_USER_FUNCTION && 8689 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8690 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8691 !func->common.function_name)) { 8692 const zend_op *opcodes = func->op_array.opcodes; 8693 8694 | ldr REG1, EX:REG1->func 8695 | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) 8696 | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] 8697 | cmp TMP1, REG2 8698 | bne &exit_addr 8699 } else { 8700 | LOAD_ADDR REG2, ((ptrdiff_t)func) 8701 | ldr TMP1, EX:REG1->func 8702 | cmp TMP1, REG2 8703 | bne &exit_addr 8704 } 8705 8706 return 1; 8707} 8708 8709static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 8710{ 8711 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8712 zend_call_info *call_info = NULL; 8713 zend_function *func = NULL; 8714 8715 if (delayed_call_chain) { 8716 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8717 return 0; 8718 } 8719 } 8720 8721 if (info) { 8722 call_info = info->callee_info; 8723 while (call_info && call_info->caller_init_opline != opline) { 8724 call_info = call_info->next_callee; 8725 } 8726 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8727 func = call_info->callee_func; 8728 } 8729 } 8730 8731 if (!func 8732 && trace 8733 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 8734 func = (zend_function*)trace->func; 8735 } 8736 8737 if (opline->opcode == ZEND_INIT_FCALL 8738 && func 8739 && func->type == ZEND_INTERNAL_FUNCTION) { 8740 /* load constant address later */ 8741 } else if (func && op_array == &func->op_array) { 8742 /* recursive call */ 8743 | ldr REG0, EX->func 8744 } else { 8745 | // if (CACHED_PTR(opline->result.num)) 8746 | ldr REG2, EX->run_time_cache 8747 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1 8748 | cbz REG0, >1 8749 |.cold_code 8750 |1: 8751 if (opline->opcode == ZEND_INIT_FCALL 8752 && func 8753 && func->type == ZEND_USER_FUNCTION 8754 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 8755 | LOAD_ADDR FCARG1x, func 8756 | MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1 8757 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8758 | mov REG0, RETVALx 8759 | b >3 8760 } else { 8761 zval *zv = RT_CONSTANT(opline, opline->op2); 8762 8763 if (opline->opcode == ZEND_INIT_FCALL) { 8764 | LOAD_ADDR FCARG1x, Z_STR_P(zv); 8765 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8766 | EXT_CALL zend_jit_find_func_helper, REG0 8767 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 8768 | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); 8769 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8770 | EXT_CALL zend_jit_find_func_helper, REG0 8771 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 8772 | LOAD_ADDR FCARG1x, zv; 8773 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8774 | EXT_CALL zend_jit_find_ns_func_helper, REG0 8775 } else { 8776 ZEND_UNREACHABLE(); 8777 } 8778 | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper 8779 | mov REG0, RETVALx 8780 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8781 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 8782 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8783 8784 if (!exit_addr) { 8785 return 0; 8786 } 8787 8788 if (!func || opline->opcode == ZEND_INIT_FCALL) { 8789 | cbnz REG0, >3 8790 } else if (func->type == ZEND_USER_FUNCTION 8791 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 8792 const zend_op *opcodes = func->op_array.opcodes; 8793 8794 | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) 8795 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 8796 | cmp TMP1, REG1 8797 | beq >3 8798 } else { 8799 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8800 | cmp REG0, REG1 8801 | beq >3 8802 } 8803 | b &exit_addr 8804 } else { 8805 | cbnz REG0, >3 8806 | // SAVE_OPLINE(); 8807 | SET_EX_OPLINE opline, REG0 8808 | b ->undefined_function 8809 } 8810 } 8811 |.code 8812 |3: 8813 } 8814 8815 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 8816 return 0; 8817 } 8818 8819 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 8820 if (!zend_jit_save_call_chain(Dst, call_level)) { 8821 return 0; 8822 } 8823 } else { 8824 delayed_call_chain = 1; 8825 delayed_call_level = call_level; 8826 } 8827 8828 return 1; 8829} 8830 8831static int zend_jit_init_method_call(dasm_State **Dst, 8832 const zend_op *opline, 8833 uint32_t b, 8834 const zend_op_array *op_array, 8835 zend_ssa *ssa, 8836 const zend_ssa_op *ssa_op, 8837 int call_level, 8838 uint32_t op1_info, 8839 zend_jit_addr op1_addr, 8840 zend_class_entry *ce, 8841 bool ce_is_instanceof, 8842 bool on_this, 8843 bool delayed_fetch_this, 8844 zend_class_entry *trace_ce, 8845 zend_jit_trace_rec *trace, 8846 int checked_stack, 8847 bool polymorphic_side_trace) 8848{ 8849 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8850 zend_call_info *call_info = NULL; 8851 zend_function *func = NULL; 8852 zval *function_name; 8853 8854 ZEND_ASSERT(opline->op2_type == IS_CONST); 8855 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 8856 8857 function_name = RT_CONSTANT(opline, opline->op2); 8858 8859 if (info) { 8860 call_info = info->callee_info; 8861 while (call_info && call_info->caller_init_opline != opline) { 8862 call_info = call_info->next_callee; 8863 } 8864 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8865 func = call_info->callee_func; 8866 } 8867 } 8868 8869 if (polymorphic_side_trace) { 8870 /* function is passed in r0 from parent_trace */ 8871 } else { 8872 if (on_this) { 8873 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 8874 8875 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 8876 } else { 8877 if (op1_info & MAY_BE_REF) { 8878 if (opline->op1_type == IS_CV) { 8879 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8880 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8881 } 8882 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 8883 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8884 } else { 8885 /* Hack: Convert reference to regular value to simplify JIT code */ 8886 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 8887 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 8888 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8889 | EXT_CALL zend_jit_unref_helper, REG0 8890 |1: 8891 } 8892 } 8893 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 8894 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8895 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8896 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8897 8898 if (!exit_addr) { 8899 return 0; 8900 } 8901 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 8902 } else { 8903 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 8904 |.cold_code 8905 |1: 8906 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8907 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8908 } 8909 | SET_EX_OPLINE opline, REG0 8910 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8911 | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 8912 } else { 8913 | EXT_CALL zend_jit_invalid_method_call, REG0 8914 } 8915 | b ->exception_handler 8916 |.code 8917 } 8918 } 8919 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8920 } 8921 8922 if (delayed_call_chain) { 8923 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8924 return 0; 8925 } 8926 } 8927 8928 | str FCARG1x, T1 // save 8929 8930 if (func) { 8931 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8932 | ldr REG0, EX->run_time_cache 8933 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8934 | cbz REG0, >1 8935 } else { 8936 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 8937 | ldr REG0, EX->run_time_cache 8938 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 8939 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 8940 | cmp REG2, TMP1 8941 | bne >1 8942 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8943 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8944 } 8945 8946 |.cold_code 8947 |1: 8948 | LOAD_ADDR FCARG2x, function_name 8949 if (TMP_ZVAL_OFFSET == 0) { 8950 | mov CARG3, sp 8951 } else { 8952 | add CARG3, sp, #TMP_ZVAL_OFFSET 8953 } 8954 | SET_EX_OPLINE opline, REG0 8955 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8956 | EXT_CALL zend_jit_find_method_tmp_helper, REG0 8957 } else { 8958 | EXT_CALL zend_jit_find_method_helper, REG0 8959 } 8960 | mov REG0, RETVALx 8961 | cbnz REG0, >2 8962 | b ->exception_handler 8963 |.code 8964 |2: 8965 } 8966 8967 if (!func 8968 && trace 8969 && trace->op == ZEND_JIT_TRACE_INIT_CALL 8970 && trace->func 8971 ) { 8972 int32_t exit_point; 8973 const void *exit_addr; 8974 8975 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); 8976 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8977 if (!exit_addr) { 8978 return 0; 8979 } 8980 8981 func = (zend_function*)trace->func; 8982 8983 if (func->type == ZEND_USER_FUNCTION && 8984 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8985 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8986 !func->common.function_name)) { 8987 const zend_op *opcodes = func->op_array.opcodes; 8988 8989 | LOAD_ADDR TMP1, opcodes 8990 | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] 8991 | cmp TMP2, TMP1 8992 | bne &exit_addr 8993 } else { 8994 | LOAD_ADDR TMP1, func 8995 | cmp REG0, TMP1 8996 | bne &exit_addr 8997 } 8998 } 8999 9000 if (!func) { 9001 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9002 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 9003 | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w 9004 | bne >1 9005 |.cold_code 9006 |1: 9007 } 9008 9009 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9010 | ldr FCARG1x, T1 // restore 9011 | mov FCARG2x, REG0 9012 | LOAD_32BIT_VAL CARG3w, opline->extended_value 9013 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9014 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 9015 } else { 9016 | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 9017 } 9018 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9019 | cbz RETVALx, ->exception_handler 9020 } 9021 | mov RX, RETVALx 9022 } 9023 9024 if (!func) { 9025 | b >9 9026 |.code 9027 } 9028 9029 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9030 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9031 return 0; 9032 } 9033 } 9034 9035 if (!func) { 9036 |9: 9037 } 9038 zend_jit_start_reuse_ip(); 9039 9040 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9041 if (!zend_jit_save_call_chain(Dst, call_level)) { 9042 return 0; 9043 } 9044 } else { 9045 delayed_call_chain = 1; 9046 delayed_call_level = call_level; 9047 } 9048 9049 return 1; 9050} 9051 9052static int zend_jit_init_closure_call(dasm_State **Dst, 9053 const zend_op *opline, 9054 uint32_t b, 9055 const zend_op_array *op_array, 9056 zend_ssa *ssa, 9057 const zend_ssa_op *ssa_op, 9058 int call_level, 9059 zend_jit_trace_rec *trace, 9060 int checked_stack) 9061{ 9062 zend_function *func = NULL; 9063 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9064 9065 | GET_ZVAL_PTR REG0, op2_addr, TMP1 9066 9067 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9068 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9069 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9070 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9071 9072 if (!exit_addr) { 9073 return 0; 9074 } 9075 9076 | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) 9077 | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] 9078 | cmp TMP1, FCARG1x 9079 | bne &exit_addr 9080 if (ssa->var_info && ssa_op->op2_use >= 0) { 9081 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9082 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9083 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9084 } 9085 } 9086 9087 if (trace 9088 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9089 && trace->func 9090 && trace->func->type == ZEND_USER_FUNCTION) { 9091 const zend_op *opcodes; 9092 int32_t exit_point; 9093 const void *exit_addr; 9094 9095 func = (zend_function*)trace->func; 9096 opcodes = func->op_array.opcodes; 9097 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9098 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9099 if (!exit_addr) { 9100 return 0; 9101 } 9102 9103 | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) 9104 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] 9105 | cmp TMP1, FCARG1x 9106 | bne &exit_addr 9107 } 9108 9109 if (delayed_call_chain) { 9110 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9111 return 0; 9112 } 9113 } 9114 9115 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9116 return 0; 9117 } 9118 9119 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9120 if (!zend_jit_save_call_chain(Dst, call_level)) { 9121 return 0; 9122 } 9123 } else { 9124 delayed_call_chain = 1; 9125 delayed_call_level = call_level; 9126 } 9127 9128 if (trace 9129 && trace->op == ZEND_JIT_TRACE_END 9130 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9131 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9132 return 0; 9133 } 9134 } 9135 9136 return 1; 9137} 9138 9139static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9140{ 9141 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9142 zend_call_info *call_info = NULL; 9143 const zend_function *func = NULL; 9144 uint32_t i; 9145 zend_jit_addr res_addr; 9146 uint32_t call_num_args = 0; 9147 bool unknown_num_args = 0; 9148 const void *exit_addr = NULL; 9149 const zend_op *prev_opline; 9150 9151 if (RETURN_VALUE_USED(opline)) { 9152 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9153 } else { 9154 /* CPU stack allocated temporary zval */ 9155 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); 9156 } 9157 9158 prev_opline = opline - 1; 9159 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9160 prev_opline--; 9161 } 9162 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9163 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9164 unknown_num_args = 1; 9165 } 9166 9167 if (info) { 9168 call_info = info->callee_info; 9169 while (call_info && call_info->caller_call_opline != opline) { 9170 call_info = call_info->next_callee; 9171 } 9172 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9173 func = call_info->callee_func; 9174 } 9175 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9176 && JIT_G(current_frame) 9177 && JIT_G(current_frame)->call 9178 && !JIT_G(current_frame)->call->func) { 9179 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9180 } 9181 } 9182 if (!func) { 9183 /* resolve function at run time */ 9184 } else if (func->type == ZEND_USER_FUNCTION) { 9185 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9186 call_num_args = call_info->num_args; 9187 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9188 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9189 call_num_args = call_info->num_args; 9190 } else { 9191 ZEND_UNREACHABLE(); 9192 } 9193 9194 if (trace && !func) { 9195 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9196 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9197#ifndef ZEND_WIN32 9198 // TODO: ASLR may cause different addresses in different workers ??? 9199 func = trace->func; 9200 if (JIT_G(current_frame) && 9201 JIT_G(current_frame)->call && 9202 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9203 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9204 } else { 9205 unknown_num_args = 1; 9206 } 9207#endif 9208 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9209 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9210 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9211 func = trace->func; 9212 if (JIT_G(current_frame) && 9213 JIT_G(current_frame)->call && 9214 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9215 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9216 } else { 9217 unknown_num_args = 1; 9218 } 9219 } 9220 } 9221 } 9222 9223 bool may_have_extra_named_params = 9224 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9225 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9226 9227 if (!reuse_ip) { 9228 zend_jit_start_reuse_ip(); 9229 | // call = EX(call); 9230 | ldr RX, EX->call 9231 } 9232 zend_jit_stop_reuse_ip(); 9233 9234 | // fbc = call->func; 9235 | // mov r2, EX:RX->func ??? 9236 | // SAVE_OPLINE(); 9237 | SET_EX_OPLINE opline, REG0 9238 9239 if (opline->opcode == ZEND_DO_FCALL) { 9240 if (!func) { 9241 if (trace) { 9242 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9243 9244 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9245 if (!exit_addr) { 9246 return 0; 9247 } 9248 | ldr REG0, EX:RX->func 9249 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9250 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9251 | bne &exit_addr 9252 } 9253 } 9254 } 9255 9256 if (!delayed_call_chain) { 9257 if (call_level == 1) { 9258 | str xzr, EX->call 9259 } else { 9260 | //EX(call) = call->prev_execute_data; 9261 | ldr REG0, EX:RX->prev_execute_data 9262 | str REG0, EX->call 9263 } 9264 } 9265 delayed_call_chain = 0; 9266 9267 | //call->prev_execute_data = execute_data; 9268 | str EX, EX:RX->prev_execute_data 9269 9270 if (!func) { 9271 | ldr REG0, EX:RX->func 9272 } 9273 9274 if (opline->opcode == ZEND_DO_FCALL) { 9275 if (!func) { 9276 if (!trace) { 9277 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9278 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9279 | bne >1 9280 |.cold_code 9281 |1: 9282 if (!GCC_GLOBAL_REGS) { 9283 | mov FCARG1x, RX 9284 } 9285 | EXT_CALL zend_jit_deprecated_helper, REG0 9286 | GET_LOW_8BITS RETVALw, RETVALw 9287 | ldr REG0, EX:RX->func // reload 9288 | cbnz RETVALw, >1 // Result is 0 on exception 9289 | b ->exception_handler 9290 |.code 9291 |1: 9292 } 9293 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9294 if (!GCC_GLOBAL_REGS) { 9295 | mov FCARG1x, RX 9296 } 9297 | EXT_CALL zend_jit_deprecated_helper, REG0 9298 | cbz RETVALw, ->exception_handler 9299 } 9300 } 9301 9302 if (!func 9303 && opline->opcode != ZEND_DO_UCALL 9304 && opline->opcode != ZEND_DO_ICALL) { 9305 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 9306 | cmp TMP1w, #ZEND_USER_FUNCTION 9307 | bne >8 9308 } 9309 9310 if ((!func || func->type == ZEND_USER_FUNCTION) 9311 && opline->opcode != ZEND_DO_ICALL) { 9312 | // EX(call) = NULL; 9313 | str xzr, EX:RX->call 9314 9315 if (RETURN_VALUE_USED(opline)) { 9316 | // EX(return_value) = EX_VAR(opline->result.var); 9317 | LOAD_ZVAL_ADDR REG2, res_addr 9318 | str REG2, EX:RX->return_value 9319 } else { 9320 | // EX(return_value) = 0; 9321 | str xzr, EX:RX->return_value 9322 } 9323 9324 //EX_LOAD_RUN_TIME_CACHE(op_array); 9325 if (!func || func->op_array.cache_size) { 9326 if (func && op_array == &func->op_array) { 9327 /* recursive call */ 9328 if (trace || func->op_array.cache_size > sizeof(void*)) { 9329 | ldr REG2, EX->run_time_cache 9330 | str REG2, EX:RX->run_time_cache 9331 } 9332 } else { 9333// Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. 9334#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 9335 if (func) { 9336 | ldr REG0, EX:RX->func 9337 } 9338 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9339 | ldr REG2, [REG2] 9340#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 9341 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 9342 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 9343 | MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1 9344 | ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1 9345 | ldr REG2, [REG2] 9346 } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9347 && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { 9348 if (func) { 9349 | ldr REG0, EX:RX->func 9350 } 9351 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9352 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9353 | ldr REG2, [REG2] 9354 } else { 9355 /* the called op_array may be not persisted yet */ 9356 if (func) { 9357 | ldr REG0, EX:RX->func 9358 } 9359 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9360 | TST_64_WITH_ONE REG2 9361 | beq >1 9362 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9363 |1: 9364 | ldr REG2, [REG2] 9365 } 9366 } else { 9367 if (func) { 9368 | ldr REG0, EX:RX->func 9369 } 9370 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9371 | TST_64_WITH_ONE REG2 9372 | beq >1 9373 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9374 |1: 9375 | ldr REG2, [REG2] 9376 } 9377#else 9378# error "Unknown ZEND_MAP_PTR_KIND" 9379#endif 9380 | str REG2, EX:RX->run_time_cache 9381 } 9382 } 9383 9384 | // EG(current_execute_data) = execute_data; 9385 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9386 | mov FP, RX 9387 9388 | // opline = op_array->opcodes; 9389 if (func && !unknown_num_args) { 9390 | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable 9391 for (i = call_num_args; i < func->op_array.last_var; i++) { 9392 | // ZVAL_UNDEF(EX_VAR(n)) 9393 | str wzr, [TMP1], #16 9394 } 9395 9396 if (call_num_args <= func->op_array.num_args) { 9397 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9398 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9399 uint32_t num_args; 9400 9401 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 9402 if (trace) { 9403 num_args = 0; 9404 } else if (call_info) { 9405 num_args = skip_valid_arguments(op_array, ssa, call_info); 9406 } else { 9407 num_args = call_num_args; 9408 } 9409 } else { 9410 num_args = call_num_args; 9411 } 9412 if (zend_accel_in_shm(func->op_array.opcodes)) { 9413 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 9414 } else { 9415 | ldr REG0, EX->func 9416 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op)))); 9417 if (GCC_GLOBAL_REGS) { 9418 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9419 if (num_args) { 9420 | add IP, IP, #(num_args * sizeof(zend_op)) 9421 } 9422 } else { 9423 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9424 if (num_args) { 9425 | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) 9426 } 9427 | str FCARG1x, EX->opline 9428 } 9429 } 9430 9431 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 9432 && num_args >= op_array->required_num_args) { 9433 /* recursive call */ 9434 if (ZEND_OBSERVER_ENABLED) { 9435 | SAVE_IP 9436 | mov FCARG1x, FP 9437 | EXT_CALL zend_observer_fcall_begin, REG0 9438 } 9439#ifdef CONTEXT_THREADED_JIT 9440 | NIY // TODO 9441#else 9442 | b =>num_args 9443#endif 9444 return 1; 9445 } 9446 } 9447 } else { 9448 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9449 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9450 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9451 | LOAD_IP_ADDR (func->op_array.opcodes) 9452 } else if (GCC_GLOBAL_REGS) { 9453 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9454 } else { 9455 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9456 | str FCARG1x, EX->opline 9457 } 9458 } 9459 if (!GCC_GLOBAL_REGS) { 9460 | mov FCARG1x, FP 9461 } 9462 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9463 } 9464 } else { 9465 | // opline = op_array->opcodes 9466 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9467 | LOAD_IP_ADDR (func->op_array.opcodes) 9468 } else if (GCC_GLOBAL_REGS) { 9469 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9470 } else { 9471 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9472 | str FCARG1x, EX->opline 9473 } 9474 if (func) { 9475 | // num_args = EX_NUM_ARGS(); 9476 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9477 | // if (UNEXPECTED(num_args > first_extra_arg)) 9478 | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w 9479 } else { 9480 | // first_extra_arg = op_array->num_args; 9481 | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] 9482 | // num_args = EX_NUM_ARGS(); 9483 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9484 | // if (UNEXPECTED(num_args > first_extra_arg)) 9485 | cmp REG1w, REG2w 9486 } 9487 | bgt >1 9488 |.cold_code 9489 |1: 9490 if (!GCC_GLOBAL_REGS) { 9491 | mov FCARG1x, FP 9492 } 9493 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9494 if (!func) { 9495 | ldr REG0, EX->func // reload 9496 } 9497 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload 9498 | b >1 9499 |.code 9500 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 9501 if (!func) { 9502 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 9503 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9504 | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w 9505 | bne >1 9506 } 9507 | // opline += num_args; 9508 || ZEND_ASSERT(sizeof(zend_op) == 32); 9509 | mov REG2w, REG1w 9510 | ADD_IP_SHIFT REG2, lsl #5, TMP1 9511 } 9512 |1: 9513 | // if (EXPECTED((int)num_args < op_array->last_var)) { 9514 if (func) { 9515 | LOAD_32BIT_VAL REG2w, func->op_array.last_var 9516 } else { 9517 | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] 9518 } 9519 | subs REG2w, REG2w, REG1w 9520 | ble >3 9521 | // zval *var = EX_VAR_NUM(num_args); 9522 | add REG1, FP, REG1, lsl #4 9523 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval)))); 9524 | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) 9525 |2: 9526 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 9527 | add REG1, REG1, #16 9528 | subs REG2w, REG2w, #1 9529 | bne <2 9530 |3: 9531 } 9532 9533 if (ZEND_OBSERVER_ENABLED) { 9534 | SAVE_IP 9535 | mov FCARG1x, FP 9536 | EXT_CALL zend_observer_fcall_begin, REG0 9537 } 9538 9539 if (trace) { 9540 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 9541 | b >9 9542 } 9543 } else { 9544#ifdef CONTEXT_THREADED_JIT 9545 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. 9546#else 9547 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 9548 | ADD_HYBRID_SPAD 9549 | JMP_IP TMP1 9550 } else if (GCC_GLOBAL_REGS) { 9551 | ldp x29, x30, [sp], # SPAD // stack alignment 9552 | JMP_IP TMP1 9553 } else { 9554 | ldp FP, RX, T2 // retore FP and IP 9555 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 9556 | mov RETVALx, #1 // ZEND_VM_ENTER 9557 | ret 9558 } 9559 } 9560#endif 9561 } 9562 9563 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 9564 && (opline->opcode != ZEND_DO_UCALL)) { 9565 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 9566 |8: 9567 } 9568 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 9569 if (!func) { 9570 if (trace) { 9571 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9572 9573 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9574 if (!exit_addr) { 9575 return 0; 9576 } 9577 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9578 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9579 | bne &exit_addr 9580 } else { 9581 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9582 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9583 | bne >1 9584 |.cold_code 9585 |1: 9586 if (!GCC_GLOBAL_REGS) { 9587 | mov FCARG1x, RX 9588 } 9589 | EXT_CALL zend_jit_deprecated_helper, REG0 9590 | GET_LOW_8BITS RETVALw, RETVALw 9591 | ldr REG0, EX:RX->func // reload 9592 | cbnz RETVALw, >1 // Result is 0 on exception 9593 | b ->exception_handler 9594 |.code 9595 |1: 9596 } 9597 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9598 if (!GCC_GLOBAL_REGS) { 9599 | mov FCARG1x, RX 9600 } 9601 | EXT_CALL zend_jit_deprecated_helper, REG0 9602 | cbz RETVALw, ->exception_handler 9603 | ldr REG0, EX:RX->func // reload 9604 } 9605 } 9606 9607 | // ZVAL_NULL(EX_VAR(opline->result.var)); 9608 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9609 | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w 9610 9611 | // EG(current_execute_data) = execute_data; 9612 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9613 9614 zend_jit_reset_last_valid_opline(); 9615 9616 | // fbc->internal_function.handler(call, ret); 9617 | mov FCARG1x, RX 9618 if (func) { 9619 | EXT_CALL func->internal_function.handler, REG0 9620 } else { 9621 | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] 9622 | blr TMP1 9623 } 9624 9625 | // EG(current_execute_data) = execute_data; 9626 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 9627 9628 | // zend_vm_stack_free_args(call); 9629 if (func && !unknown_num_args) { 9630 for (i = 0; i < call_num_args; i++ ) { 9631 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 9632 uint32_t offset = EX_NUM_TO_VAR(i); 9633 zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); 9634 | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 9635 } 9636 } 9637 } else { 9638 | mov FCARG1x, RX 9639 | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 9640 } 9641 if (may_have_extra_named_params) { 9642 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9643 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w 9644 | bne >1 9645 |.cold_code 9646 |1: 9647 | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] 9648 | EXT_CALL zend_free_extra_named_params, REG0 9649 | b >2 9650 |.code 9651 |2: 9652 } 9653 9654 |8: 9655 if (opline->opcode == ZEND_DO_FCALL) { 9656 // TODO: optimize ??? 9657 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 9658 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9659 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w 9660 | bne >1 9661 |.cold_code 9662 |1: 9663 | add TMP1, RX, #offsetof(zend_execute_data, This) 9664 | GET_Z_PTR FCARG1x, TMP1 9665 | // OBJ_RELEASE(object); 9666 | OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2 9667 | b >2 9668 |.code 9669 |2: 9670 } 9671 9672 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 9673 !JIT_G(current_frame) || 9674 !JIT_G(current_frame)->call || 9675 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 9676 prev_opline->opcode == ZEND_SEND_UNPACK || 9677 prev_opline->opcode == ZEND_SEND_ARRAY || 9678 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9679 9680 | // zend_vm_stack_free_call_frame(call); 9681 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9682 | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w 9683 | bne >1 9684 |.cold_code 9685 |1: 9686 | mov FCARG1x, RX 9687 | EXT_CALL zend_jit_free_call_frame, REG0 9688 | b >1 9689 |.code 9690 } 9691 | MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0 9692 |1: 9693 9694 if (!RETURN_VALUE_USED(opline)) { 9695 zend_class_entry *ce; 9696 bool ce_is_instanceof; 9697 uint32_t func_info = call_info ? 9698 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 9699 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 9700 9701 /* If an exception is thrown, the return_value may stay at the 9702 * original value of null. */ 9703 func_info |= MAY_BE_NULL; 9704 9705 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 9706 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 9707 } 9708 } 9709 9710 | // if (UNEXPECTED(EG(exception) != NULL)) { 9711 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 9712 | cbnz REG0, ->icall_throw_handler 9713 9714 // TODO: Can we avoid checking for interrupts after each call ??? 9715 if (trace && last_valid_opline != opline) { 9716 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 9717 9718 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9719 if (!exit_addr) { 9720 return 0; 9721 } 9722 } else { 9723 exit_addr = NULL; 9724 } 9725 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 9726 return 0; 9727 } 9728 9729 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 9730 | LOAD_IP_ADDR (opline + 1) 9731 } else if (trace 9732 && trace->op == ZEND_JIT_TRACE_END 9733 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9734 | LOAD_IP_ADDR (opline + 1) 9735 } 9736 } 9737 9738 if (!func) { 9739 |9: 9740 } 9741 9742 return 1; 9743} 9744 9745static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 9746{ 9747 uint32_t arg_num = opline->op2.num; 9748 zend_jit_addr arg_addr; 9749 9750 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 9751 9752 if (!zend_jit_reuse_ip(Dst)) { 9753 return 0; 9754 } 9755 9756 if (opline->opcode == ZEND_SEND_VAL_EX) { 9757 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 9758 9759 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 9760 9761 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9762 && JIT_G(current_frame) 9763 && JIT_G(current_frame)->call 9764 && JIT_G(current_frame)->call->func) { 9765 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9766 /* Don't generate code that always throws exception */ 9767 return 0; 9768 } 9769 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9770 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9771 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9772 if (!exit_addr) { 9773 return 0; 9774 } 9775 | ldr REG0, EX:RX->func 9776 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9777 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9778 | bne &exit_addr 9779 } else { 9780 | ldr REG0, EX:RX->func 9781 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9782 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9783 | bne >1 9784 |.cold_code 9785 |1: 9786 if (Z_MODE(op1_addr) == IS_REG) { 9787 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 9788 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 9789 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2 9790 } 9791 | SET_EX_OPLINE opline, REG0 9792 | b ->throw_cannot_pass_by_ref 9793 |.code 9794 } 9795 } 9796 9797 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9798 9799 if (opline->op1_type == IS_CONST) { 9800 zval *zv = RT_CONSTANT(opline, opline->op1); 9801 9802 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 9803 if (Z_REFCOUNTED_P(zv)) { 9804 | ADDREF_CONST zv, REG0, TMP1 9805 } 9806 } else { 9807 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9808 } 9809 9810 return 1; 9811} 9812 9813static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 9814{ 9815 | ldr FCARG1x, EX->call 9816 | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9817 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w 9818 | bne >1 9819 |.cold_code 9820 |1: 9821 | SET_EX_OPLINE opline, REG0 9822 | EXT_CALL zend_handle_undef_args, REG0 9823 | cbz RETVALw, >2 9824 | b ->exception_handler 9825 |.code 9826 |2: 9827 9828 return 1; 9829} 9830 9831static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 9832{ 9833 zend_jit_addr op1_addr, arg_addr, ref_addr; 9834 9835 op1_addr = OP1_ADDR(); 9836 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9837 9838 if (!zend_jit_reuse_ip(Dst)) { 9839 return 0; 9840 } 9841 9842 if (opline->op1_type == IS_VAR) { 9843 if (op1_info & MAY_BE_INDIRECT) { 9844 | LOAD_ZVAL_ADDR REG0, op1_addr 9845 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 9846 | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w 9847 | // ret = Z_INDIRECT_P(ret); 9848 | GET_Z_PTR REG0, REG0 9849 |1: 9850 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 9851 } 9852 } else if (opline->op1_type == IS_CV) { 9853 if (op1_info & MAY_BE_UNDEF) { 9854 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 9855 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 9856 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 9857 | b >2 9858 |1: 9859 } 9860 op1_info &= ~MAY_BE_UNDEF; 9861 op1_info |= MAY_BE_NULL; 9862 } 9863 } else { 9864 ZEND_UNREACHABLE(); 9865 } 9866 9867 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 9868 if (op1_info & MAY_BE_REF) { 9869 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 9870 | GET_ZVAL_PTR REG1, op1_addr, TMP1 9871 | GC_ADDREF REG1, TMP1w 9872 | SET_ZVAL_PTR arg_addr, REG1, TMP1 9873 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9874 | b >6 9875 } 9876 |2: 9877 | // ZVAL_NEW_REF(arg, varptr); 9878 if (opline->op1_type == IS_VAR) { 9879 if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { 9880 | LOAD_ZVAL_ADDR REG0, op1_addr 9881 } 9882 | str REG0, T1 // save 9883 } 9884 | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 9885 | mov TMP1w, #2 9886 | str TMP1w, [REG0] 9887 || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); 9888 | movz TMP1w, #GC_REFERENCE 9889 | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] 9890 | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] 9891 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 9892 if (opline->op1_type == IS_VAR) { 9893 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 9894 9895 | ldr REG1, T1 // restore 9896 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9897 | SET_ZVAL_PTR val_addr, REG0, TMP1 9898 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 9899 } else { 9900 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9901 | SET_ZVAL_PTR op1_addr, REG0, TMP1 9902 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 9903 } 9904 | SET_ZVAL_PTR arg_addr, REG0, TMP1 9905 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9906 } 9907 9908 |6: 9909 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 9910 |7: 9911 9912 return 1; 9913} 9914 9915static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 9916{ 9917 uint32_t arg_num = opline->op2.num; 9918 zend_jit_addr arg_addr; 9919 9920 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 9921 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 9922 arg_num <= MAX_ARG_FLAG_NUM); 9923 9924 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9925 9926 if (!zend_jit_reuse_ip(Dst)) { 9927 return 0; 9928 } 9929 9930 if (opline->opcode == ZEND_SEND_VAR_EX) { 9931 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9932 && JIT_G(current_frame) 9933 && JIT_G(current_frame)->call 9934 && JIT_G(current_frame)->call->func) { 9935 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9936 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 9937 return 0; 9938 } 9939 return 1; 9940 } 9941 } else { 9942 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9943 9944 | ldr REG0, EX:RX->func 9945 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9946 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9947 | bne >1 9948 |.cold_code 9949 |1: 9950 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 9951 return 0; 9952 } 9953 | b >7 9954 |.code 9955 } 9956 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 9957 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9958 && JIT_G(current_frame) 9959 && JIT_G(current_frame)->call 9960 && JIT_G(current_frame)->call->func) { 9961 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9962 9963 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9964 9965 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9966 if (!(op1_info & MAY_BE_REF)) { 9967 /* Don't generate code that always throws exception */ 9968 return 0; 9969 } else { 9970 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9971 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9972 if (!exit_addr) { 9973 return 0; 9974 } 9975 | GET_LOW_8BITS TMP1w, REG1w 9976 | cmp TMP1w, #IS_REFERENCE 9977 | bne &exit_addr 9978 } 9979 } 9980 return 1; 9981 } 9982 } else { 9983 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9984 9985 | ldr REG0, EX:RX->func 9986 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9987 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9988 | bne >1 9989 |.cold_code 9990 |1: 9991 9992 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 9993 9994 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9995 if (op1_info & MAY_BE_REF) { 9996 | GET_LOW_8BITS TMP1w, REG1w 9997 | cmp TMP1w, #IS_REFERENCE 9998 | beq >7 9999 } 10000 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10001 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10002 | bne >7 10003 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10004 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10005 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10006 if (!exit_addr) { 10007 return 0; 10008 } 10009 | b &exit_addr 10010 } else { 10011 | SET_EX_OPLINE opline, REG0 10012 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10013 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10014 if (!zend_jit_check_exception(Dst)) { 10015 return 0; 10016 } 10017 | b >7 10018 } 10019 10020 |.code 10021 } 10022 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10023 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10024 && JIT_G(current_frame) 10025 && JIT_G(current_frame)->call 10026 && JIT_G(current_frame)->call->func) { 10027 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10028 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10029 return 0; 10030 } 10031 return 1; 10032 } 10033 } else { 10034 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10035 | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10036 | bne >1 10037 |.cold_code 10038 |1: 10039 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10040 return 0; 10041 } 10042 | b >7 10043 |.code 10044 } 10045 } 10046 10047 if (op1_info & MAY_BE_UNDEF) { 10048 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10049 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10050 |.cold_code 10051 |1: 10052 } 10053 10054 | SET_EX_OPLINE opline, REG0 10055 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10056 | EXT_CALL zend_jit_undefined_op_helper, REG0 10057 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 10058 | cbz RETVALx, ->exception_handler 10059 10060 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10061 | b >7 10062 |.code 10063 } else { 10064 |7: 10065 return 1; 10066 } 10067 } 10068 10069 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10070 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10071 if (op1_info & MAY_BE_REF) { 10072 | GET_LOW_8BITS TMP1w, REG1w 10073 | cmp TMP1w, #IS_REFERENCE 10074 | beq >7 10075 } 10076 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10077 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10078 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10079 if (!exit_addr) { 10080 return 0; 10081 } 10082 | b &exit_addr 10083 } else { 10084 | SET_EX_OPLINE opline, REG0 10085 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10086 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10087 if (!zend_jit_check_exception(Dst)) { 10088 return 0; 10089 } 10090 } 10091 } else { 10092 if (op1_info & MAY_BE_REF) { 10093 if (opline->op1_type == IS_CV) { 10094 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10095 10096 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 10097 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 10098 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10099 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10100 } else { 10101 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10102 10103 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 10104 |.cold_code 10105 |1: 10106 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10107 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10108 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10109 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10110 | GC_DELREF FCARG1x, TMP1w 10111 | beq >1 10112 | IF_NOT_REFCOUNTED REG0w, >2, TMP1w 10113 | GC_ADDREF REG2, TMP1w 10114 | b >2 10115 |1: 10116 | EFREE_REFERENCE 10117 | b >2 10118 |.code 10119 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10120 |2: 10121 } 10122 } else { 10123 if (op1_addr != op1_def_addr) { 10124 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10125 return 0; 10126 } 10127 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10128 op1_addr= op1_def_addr; 10129 } 10130 } 10131 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10132 if (opline->op1_type == IS_CV) { 10133 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10134 } 10135 } 10136 } 10137 |7: 10138 10139 return 1; 10140} 10141 10142static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10143{ 10144 uint32_t arg_num = opline->op2.num; 10145 10146 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10147 && JIT_G(current_frame) 10148 && JIT_G(current_frame)->call 10149 && JIT_G(current_frame)->call->func) { 10150 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10151 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10152 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10153 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10154 || if (reuse_ip) { 10155 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10156 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10157 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10158 || } else { 10159 | ldr REG0, EX->call 10160 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10161 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10162 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10163 || } 10164 } 10165 } else { 10166 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10167 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10168 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10169 || if (reuse_ip) { 10170 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10171 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10172 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10173 || } else { 10174 | ldr REG0, EX->call 10175 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10176 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10177 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10178 || } 10179 } 10180 } 10181 } else { 10182 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10183 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10184 10185 if (!zend_jit_reuse_ip(Dst)) { 10186 return 0; 10187 } 10188 10189 | ldr REG0, EX:RX->func 10190 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10191 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10192 | bne >1 10193 |.cold_code 10194 |1: 10195 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10196 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10197 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10198 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10199 | b >1 10200 |.code 10201 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10202 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10203 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w 10204 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10205 |1: 10206 } 10207 10208 return 1; 10209} 10210 10211static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10212{ 10213 if (smart_branch_opcode) { 10214 if (smart_branch_opcode == ZEND_JMPZ) { 10215 if (jmp) { 10216 | b >7 10217 } 10218 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10219 | b =>target_label 10220 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10221 | b =>target_label2 10222 } else { 10223 ZEND_UNREACHABLE(); 10224 } 10225 } else { 10226 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10227 10228 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10229 if (jmp) { 10230 | b >7 10231 } 10232 } 10233 10234 return 1; 10235} 10236 10237static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 10238{ 10239 if (smart_branch_opcode) { 10240 if (smart_branch_opcode == ZEND_JMPZ) { 10241 | b =>target_label 10242 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10243 if (jmp) { 10244 | b >7 10245 } 10246 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10247 | b =>target_label 10248 } else { 10249 ZEND_UNREACHABLE(); 10250 } 10251 } else { 10252 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10253 10254 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10255 if (jmp) { 10256 | b >7 10257 } 10258 } 10259 10260 return 1; 10261} 10262 10263static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10264{ 10265 uint32_t defined_label = (uint32_t)-1; 10266 uint32_t undefined_label = (uint32_t)-1; 10267 zval *zv = RT_CONSTANT(opline, opline->op1); 10268 zend_jit_addr res_addr = 0; 10269 10270 if (smart_branch_opcode && !exit_addr) { 10271 if (smart_branch_opcode == ZEND_JMPZ) { 10272 undefined_label = target_label; 10273 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10274 defined_label = target_label; 10275 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10276 undefined_label = target_label; 10277 defined_label = target_label2; 10278 } else { 10279 ZEND_UNREACHABLE(); 10280 } 10281 } 10282 10283 | // if (CACHED_PTR(opline->extended_value)) { 10284 | ldr REG0, EX->run_time_cache 10285 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 10286 | cbz REG0, >1 10287 | TST_64_WITH_ONE REG0 10288 | bne >4 10289 |.cold_code 10290 |4: 10291 | MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x 10292 | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] 10293 | cmp TMP1, REG0, lsr #1 10294 10295 if (smart_branch_opcode) { 10296 if (exit_addr) { 10297 if (smart_branch_opcode == ZEND_JMPZ) { 10298 | beq &exit_addr 10299 } else { 10300 | beq >3 10301 } 10302 } else if (undefined_label != (uint32_t)-1) { 10303 | beq =>undefined_label 10304 } else { 10305 | beq >3 10306 } 10307 } else { 10308 | beq >2 10309 } 10310 |1: 10311 | SET_EX_OPLINE opline, REG0 10312 | LOAD_ADDR FCARG1x, zv 10313 | EXT_CALL zend_jit_check_constant, REG0 10314 if (exit_addr) { 10315 if (smart_branch_opcode == ZEND_JMPNZ) { 10316 | cbz RETVALx, >3 10317 } else { 10318 | cbnz RETVALx, >3 10319 } 10320 | b &exit_addr 10321 } else if (smart_branch_opcode) { 10322 if (undefined_label != (uint32_t)-1) { 10323 | cbz RETVALx, =>undefined_label 10324 } else { 10325 | cbz RETVALx, >3 10326 } 10327 if (defined_label != (uint32_t)-1) { 10328 | b =>defined_label 10329 } else { 10330 | b >3 10331 } 10332 } else { 10333 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10334 | cbnz RETVALx, >1 10335 |2: 10336 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10337 | b >3 10338 } 10339 |.code 10340 if (smart_branch_opcode) { 10341 if (exit_addr) { 10342 if (smart_branch_opcode == ZEND_JMPNZ) { 10343 | b &exit_addr 10344 } 10345 } else if (defined_label != (uint32_t)-1) { 10346 | b =>defined_label 10347 } 10348 } else { 10349 |1: 10350 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10351 } 10352 |3: 10353 10354 return 1; 10355} 10356 10357static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10358{ 10359 uint32_t mask; 10360 zend_jit_addr op1_addr = OP1_ADDR(); 10361 10362 // TODO: support for is_resource() ??? 10363 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 10364 10365 if (op1_info & MAY_BE_UNDEF) { 10366 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10367 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10368 |.cold_code 10369 |1: 10370 } 10371 | SET_EX_OPLINE opline, REG0 10372 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10373 | EXT_CALL zend_jit_undefined_op_helper, REG0 10374 zend_jit_check_exception_undef_result(Dst, opline); 10375 if (opline->extended_value & MAY_BE_NULL) { 10376 if (exit_addr) { 10377 if (smart_branch_opcode == ZEND_JMPNZ) { 10378 | b &exit_addr 10379 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10380 | b >7 10381 } 10382 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 10383 return 0; 10384 } 10385 } else { 10386 if (exit_addr) { 10387 if (smart_branch_opcode == ZEND_JMPZ) { 10388 | b &exit_addr 10389 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10390 | b >7 10391 } 10392 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 10393 return 0; 10394 } 10395 } 10396 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10397 |.code 10398 } 10399 } 10400 10401 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10402 mask = opline->extended_value; 10403 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 10404 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10405 if (exit_addr) { 10406 if (smart_branch_opcode == ZEND_JMPNZ) { 10407 | b &exit_addr 10408 } 10409 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 10410 return 0; 10411 } 10412 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 10413 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10414 if (exit_addr) { 10415 if (smart_branch_opcode == ZEND_JMPZ) { 10416 | b &exit_addr 10417 } 10418 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 10419 return 0; 10420 } 10421 } else { 10422 bool invert = 0; 10423 zend_uchar type; 10424 10425 switch (mask) { 10426 case MAY_BE_NULL: type = IS_NULL; break; 10427 case MAY_BE_FALSE: type = IS_FALSE; break; 10428 case MAY_BE_TRUE: type = IS_TRUE; break; 10429 case MAY_BE_LONG: type = IS_LONG; break; 10430 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 10431 case MAY_BE_STRING: type = IS_STRING; break; 10432 case MAY_BE_ARRAY: type = IS_ARRAY; break; 10433 case MAY_BE_OBJECT: type = IS_OBJECT; break; 10434 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 10435 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 10436 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 10437 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 10438 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 10439 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 10440 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 10441 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 10442 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 10443 default: 10444 type = 0; 10445 } 10446 10447 if (op1_info & MAY_BE_REF) { 10448 | LOAD_ZVAL_ADDR REG0, op1_addr 10449 | ZVAL_DEREF REG0, op1_info, TMP1w 10450 } 10451 if (type == 0) { 10452 if (smart_branch_opcode && 10453 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10454 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10455 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10456 | // if (Z_REFCOUNTED_P(cv)) { 10457 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10458 |.cold_code 10459 |1: 10460 } 10461 | // if (!Z_DELREF_P(cv)) { 10462 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10463 | GC_DELREF FCARG1x, TMP1w 10464 if (RC_MAY_BE_1(op1_info)) { 10465 if (RC_MAY_BE_N(op1_info)) { 10466 | bne >3 10467 } 10468 if (op1_info & MAY_BE_REF) { 10469 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10470 } else { 10471 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10472 } 10473 | str REG0w, T1 // save 10474 | // zval_dtor_func(r); 10475 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10476 | ldr REG1w, T1 // restore 10477 | b >2 10478 } 10479 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10480 if (!RC_MAY_BE_1(op1_info)) { 10481 | b >3 10482 } 10483 |.code 10484 } 10485 |3: 10486 if (op1_info & MAY_BE_REF) { 10487 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10488 } else { 10489 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10490 } 10491 |2: 10492 } else { 10493 if (op1_info & MAY_BE_REF) { 10494 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10495 } else { 10496 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10497 } 10498 } 10499 | mov REG0w, #1 10500 | lsl REG0w, REG0w, REG1w 10501 | TST_32_WITH_CONST REG0w, mask, TMP1w 10502 if (exit_addr) { 10503 if (smart_branch_opcode == ZEND_JMPNZ) { 10504 | bne &exit_addr 10505 } else { 10506 | beq &exit_addr 10507 } 10508 } else if (smart_branch_opcode) { 10509 if (smart_branch_opcode == ZEND_JMPZ) { 10510 | beq =>target_label 10511 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10512 | bne =>target_label 10513 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10514 | beq =>target_label 10515 | b =>target_label2 10516 } else { 10517 ZEND_UNREACHABLE(); 10518 } 10519 } else { 10520 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10521 10522 | cset REG0w, ne 10523 | add REG0w, REG0w, #2 10524 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10525 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10526 } 10527 } else { 10528 if (smart_branch_opcode && 10529 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10530 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10531 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10532 | // if (Z_REFCOUNTED_P(cv)) { 10533 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10534 |.cold_code 10535 |1: 10536 } 10537 | // if (!Z_DELREF_P(cv)) { 10538 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10539 | GC_DELREF FCARG1x, TMP1w 10540 if (RC_MAY_BE_1(op1_info)) { 10541 if (RC_MAY_BE_N(op1_info)) { 10542 | bne >3 10543 } 10544 if (op1_info & MAY_BE_REF) { 10545 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10546 } else { 10547 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10548 } 10549 | str REG0w, T1 // save 10550 | // zval_dtor_func(r); 10551 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10552 | ldr REG1w, T1 // restore 10553 | b >2 10554 } 10555 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10556 if (!RC_MAY_BE_1(op1_info)) { 10557 | b >3 10558 } 10559 |.code 10560 } 10561 |3: 10562 if (op1_info & MAY_BE_REF) { 10563 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10564 } else { 10565 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10566 } 10567 |2: 10568 // Note: 'type' is of uchar type and holds a positive value, 10569 // hence it's safe to directly encode it as the imm field of 'cmp' instruction. 10570 | cmp REG1w, #type 10571 } else { 10572 if (op1_info & MAY_BE_REF) { 10573 | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] 10574 | cmp TMP1w, #type 10575 } else { 10576 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10577 | cmp TMP1w, #type 10578 } 10579 } 10580 if (exit_addr) { 10581 if (invert) { 10582 if (smart_branch_opcode == ZEND_JMPNZ) { 10583 | bne &exit_addr 10584 } else { 10585 | beq &exit_addr 10586 } 10587 } else { 10588 if (smart_branch_opcode == ZEND_JMPNZ) { 10589 | beq &exit_addr 10590 } else { 10591 | bne &exit_addr 10592 } 10593 } 10594 } else if (smart_branch_opcode) { 10595 if (invert) { 10596 if (smart_branch_opcode == ZEND_JMPZ) { 10597 | beq =>target_label 10598 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10599 | bne =>target_label 10600 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10601 | beq =>target_label 10602 | b =>target_label2 10603 } else { 10604 ZEND_UNREACHABLE(); 10605 } 10606 } else { 10607 if (smart_branch_opcode == ZEND_JMPZ) { 10608 | bne =>target_label 10609 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10610 | beq =>target_label 10611 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10612 | bne =>target_label 10613 | b =>target_label2 10614 } else { 10615 ZEND_UNREACHABLE(); 10616 } 10617 } 10618 } else { 10619 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10620 10621 if (invert) { 10622 | cset REG0w, ne 10623 } else { 10624 | cset REG0w, eq 10625 } 10626 | add REG0w, REG0w, #2 10627 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10628 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10629 } 10630 } 10631 } 10632 } 10633 10634 |7: 10635 10636 return 1; 10637} 10638 10639static int zend_jit_leave_frame(dasm_State **Dst) 10640{ 10641 | // EG(current_execute_data) = EX(prev_execute_data); 10642 | ldr REG0, EX->prev_execute_data 10643 | MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2 10644 return 1; 10645} 10646 10647static int zend_jit_free_cvs(dasm_State **Dst) 10648{ 10649 | // EG(current_execute_data) = EX(prev_execute_data); 10650 | ldr FCARG1x, EX->prev_execute_data 10651 | MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0 10652 | // zend_free_compiled_variables(execute_data); 10653 | mov FCARG1x, FP 10654 | EXT_CALL zend_free_compiled_variables, REG0 10655 return 1; 10656} 10657 10658static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 10659{ 10660 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10661 uint32_t offset = EX_NUM_TO_VAR(var); 10662 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); 10663 | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 10664 } 10665 return 1; 10666} 10667 10668static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 10669{ 10670 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10671 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); 10672 | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 10673 } 10674 return 1; 10675} 10676 10677static int zend_jit_leave_func(dasm_State **Dst, 10678 const zend_op_array *op_array, 10679 const zend_op *opline, 10680 uint32_t op1_info, 10681 bool left_frame, 10682 zend_jit_trace_rec *trace, 10683 zend_jit_trace_info *trace_info, 10684 int indirect_var_access, 10685 int may_throw) 10686{ 10687 bool may_be_top_frame = 10688 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10689 !JIT_G(current_frame) || 10690 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 10691 bool may_need_call_helper = 10692 indirect_var_access || /* may have symbol table */ 10693 !op_array->function_name || /* may have symbol table */ 10694 may_be_top_frame || 10695 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 10696 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10697 !JIT_G(current_frame) || 10698 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 10699 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 10700 bool may_need_release_this = 10701 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 10702 op_array->scope && 10703 !(op_array->fn_flags & ZEND_ACC_STATIC) && 10704 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10705 !JIT_G(current_frame) || 10706 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 10707 10708 if (may_need_call_helper || may_need_release_this) { 10709 | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] 10710 } 10711 if (may_need_call_helper) { 10712 if (!left_frame) { 10713 left_frame = 1; 10714 if (!zend_jit_leave_frame(Dst)) { 10715 return 0; 10716 } 10717 } 10718 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 10719 10720 | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w 10721 if (trace && trace->op != ZEND_JIT_TRACE_END) { 10722 | bne >1 10723 |.cold_code 10724 |1: 10725 if (!GCC_GLOBAL_REGS) { 10726 | mov FCARG1x, FP 10727 } 10728 | EXT_CALL zend_jit_leave_func_helper, REG0 10729 10730 if (may_be_top_frame) { 10731 // TODO: try to avoid this check ??? 10732 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10733#if 0 10734 /* this check should be handled by the following OPLINE guard */ 10735 | LOAD_ADDR TMP1, zend_jit_halt_op 10736 | cmp IP, TMP1 10737 | beq ->trace_halt 10738#endif 10739 } else if (GCC_GLOBAL_REGS) { 10740 | cbz IP, ->trace_halt 10741 } else { 10742 | tst RETVALw, RETVALw 10743 | blt ->trace_halt 10744 } 10745 } 10746 10747 if (!GCC_GLOBAL_REGS) { 10748 | // execute_data = EG(current_execute_data) 10749 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 10750 } 10751 | b >8 10752 |.code 10753 } else { 10754 | bne ->leave_function_handler 10755 } 10756 } 10757 10758 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 10759 if (!left_frame) { 10760 left_frame = 1; 10761 if (!zend_jit_leave_frame(Dst)) { 10762 return 0; 10763 } 10764 } 10765 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 10766 | ldr FCARG1x, EX->func 10767 | sub FCARG1x, FCARG1x, #sizeof(zend_object) 10768 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10769 |4: 10770 } else if (may_need_release_this) { 10771 if (!left_frame) { 10772 left_frame = 1; 10773 if (!zend_jit_leave_frame(Dst)) { 10774 return 0; 10775 } 10776 } 10777 | // if (call_info & ZEND_CALL_RELEASE_THIS) 10778 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w 10779 | beq >4 10780 | // zend_object *object = Z_OBJ(execute_data->This); 10781 | ldr FCARG1x, EX->This.value.obj 10782 | // OBJ_RELEASE(object); 10783 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10784 |4: 10785 // TODO: avoid EG(excption) check for $this->foo() calls 10786 may_throw = 1; 10787 } 10788 10789 | // EG(vm_stack_top) = (zval*)execute_data; 10790 | MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0 10791 | // execute_data = EX(prev_execute_data); 10792 | ldr FP, EX->prev_execute_data 10793 10794 if (!left_frame) { 10795 | // EG(current_execute_data) = execute_data; 10796 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 10797 } 10798 10799 |9: 10800 if (trace) { 10801 if (trace->op != ZEND_JIT_TRACE_END 10802 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10803 zend_jit_reset_last_valid_opline(); 10804 } else { 10805 | LOAD_IP 10806 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10807 } 10808 10809 |8: 10810 10811 if (trace->op == ZEND_JIT_TRACE_BACK 10812 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10813 const zend_op *next_opline = trace->opline; 10814 10815 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10816 && (op1_info & MAY_BE_RC1) 10817 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 10818 /* exception might be thrown during destruction of unused return value */ 10819 | // if (EG(exception)) 10820 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10821 | cbnz REG0, ->leave_throw_handler 10822 } 10823 do { 10824 trace++; 10825 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 10826 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 10827 next_opline = trace->opline; 10828 ZEND_ASSERT(next_opline != NULL); 10829 10830 if (trace->op == ZEND_JIT_TRACE_END 10831 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 10832 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 10833 | CMP_IP next_opline, TMP1, TMP2 10834 | beq =>0 // LOOP 10835#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 10836 | JMP_IP TMP1 10837#else 10838 | b ->trace_escape 10839#endif 10840 } else { 10841 | CMP_IP next_opline, TMP1, TMP2 10842 | bne ->trace_escape 10843 } 10844 10845 zend_jit_set_last_valid_opline(trace->opline); 10846 10847 return 1; 10848 } else if (may_throw || 10849 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10850 && (op1_info & MAY_BE_RC1) 10851 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 10852 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 10853 | // if (EG(exception)) 10854 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10855 | cbnz REG0, ->leave_throw_handler 10856 } 10857 10858 return 1; 10859 } else { 10860 | // if (EG(exception)) 10861 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10862 | LOAD_IP 10863 | cbnz REG0, ->leave_throw_handler 10864 | // opline = EX(opline) + 1 10865 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10866 } 10867 10868 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10869 | ADD_HYBRID_SPAD 10870#ifdef CONTEXT_THREADED_JIT 10871 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined 10872#else 10873 | JMP_IP TMP1 10874#endif 10875 } else if (GCC_GLOBAL_REGS) { 10876 | ldp x29, x30, [sp], # SPAD // stack alignment 10877#ifdef CONTEXT_THREADED_JIT 10878 | NIY // TODO 10879#else 10880 | JMP_IP TMP1 10881#endif 10882 } else { 10883#ifdef CONTEXT_THREADED_JIT 10884 ZEND_UNREACHABLE(); 10885 // TODO: context threading can't work without GLOBAL REGS because we have to change 10886 // the value of execute_data in execute_ex() 10887 | NIY // TODO 10888#else 10889 | ldp FP, RX, T2 // retore FP and IP 10890 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 10891 | mov RETVALx, #2 // ZEND_VM_LEAVE ???? 10892 | ret 10893#endif 10894 } 10895 10896 return 1; 10897} 10898 10899static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 10900{ 10901 zend_jit_addr ret_addr; 10902 int8_t return_value_used; 10903 10904 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 10905 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 10906 10907 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 10908 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 10909 return_value_used = 1; 10910 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 10911 return_value_used = 0; 10912 } else { 10913 return_value_used = -1; 10914 } 10915 } else { 10916 return_value_used = -1; 10917 } 10918 10919 if (ZEND_OBSERVER_ENABLED) { 10920 if (Z_MODE(op1_addr) == IS_REG) { 10921 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10922 10923 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 10924 return 0; 10925 } 10926 op1_addr = dst; 10927 } 10928 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 10929 | mov FCARG1x, FP 10930 | SET_EX_OPLINE opline, REG0 10931 | EXT_CALL zend_observer_fcall_end, REG0 10932 } 10933 10934 // if (!EX(return_value)) 10935 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { 10936 if (return_value_used != 0) { 10937 | ldr REG2, EX->return_value 10938 } 10939 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 10940 } else { 10941 if (return_value_used != 0) { 10942 | ldr REG1, EX->return_value 10943 } 10944 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 10945 } 10946 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10947 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10948 if (return_value_used == -1) { 10949 | cbz Rx(Z_REG(ret_addr)), >1 10950 |.cold_code 10951 |1: 10952 } 10953 if (return_value_used != 1) { 10954 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10955 if (jit_return_label >= 0) { 10956 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 10957 } else { 10958 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 10959 } 10960 } 10961 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10962 | GC_DELREF FCARG1x, TMP1w 10963 if (RC_MAY_BE_1(op1_info)) { 10964 if (RC_MAY_BE_N(op1_info)) { 10965 if (jit_return_label >= 0) { 10966 | bne =>jit_return_label 10967 } else { 10968 | bne >9 10969 } 10970 } 10971 | //SAVE_OPLINE() 10972 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10973 | //????ldr REG1, EX->return_value // reload ??? 10974 } 10975 if (return_value_used == -1) { 10976 if (jit_return_label >= 0) { 10977 | b =>jit_return_label 10978 } else { 10979 | b >9 10980 } 10981 |.code 10982 } 10983 } 10984 } else if (return_value_used == -1) { 10985 if (jit_return_label >= 0) { 10986 | cbz Rx(Z_REG(ret_addr)), =>jit_return_label 10987 } else { 10988 | cbz Rx(Z_REG(ret_addr)), >9 10989 } 10990 } 10991 10992 if (return_value_used == 0) { 10993 |9: 10994 return 1; 10995 } 10996 10997 if (opline->op1_type == IS_CONST) { 10998 zval *zv = RT_CONSTANT(opline, opline->op1); 10999 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 11000 if (Z_REFCOUNTED_P(zv)) { 11001 | ADDREF_CONST zv, REG0, TMP1 11002 } 11003 } else if (opline->op1_type == IS_TMP_VAR) { 11004 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11005 } else if (opline->op1_type == IS_CV) { 11006 if (op1_info & MAY_BE_REF) { 11007 | LOAD_ZVAL_ADDR REG0, op1_addr 11008 | ZVAL_DEREF REG0, op1_info, TMP1w 11009 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11010 } 11011 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11012 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11013 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11014 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11015 !op_array->function_name) { 11016 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 11017 } else if (return_value_used != 1) { 11018 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11019 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 11020 } 11021 } 11022 } else { 11023 if (op1_info & MAY_BE_REF) { 11024 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 11025 11026 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 11027 |.cold_code 11028 |1: 11029 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11030 | GET_ZVAL_PTR REG0, op1_addr, TMP1 11031 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11032 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11033 | GC_DELREF REG0, TMP1w 11034 | beq >2 11035 | // if (IS_REFCOUNTED()) 11036 if (jit_return_label >= 0) { 11037 | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w 11038 } else { 11039 | IF_NOT_REFCOUNTED REG2w, >9, TMP1w 11040 } 11041 | // ADDREF 11042 | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload 11043 | GC_ADDREF REG2, TMP1w 11044 if (jit_return_label >= 0) { 11045 | b =>jit_return_label 11046 } else { 11047 | b >9 11048 } 11049 |2: 11050 | mov FCARG1x, REG0 11051 | EFREE_REFERENCE 11052 if (jit_return_label >= 0) { 11053 | b =>jit_return_label 11054 } else { 11055 | b >9 11056 } 11057 |.code 11058 } 11059 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11060 } 11061 11062 |9: 11063 return 1; 11064} 11065 11066static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11067{ 11068 ZEND_ASSERT(type_reg == ZREG_REG2); 11069 11070 | GET_ZVAL_PTR REG1, val_addr, TMP1 11071 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11072 | GET_LOW_8BITS TMP2w, REG2w 11073 | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 11074 | add REG1, REG1, #offsetof(zend_reference, val) 11075 | GET_Z_TYPE_INFO REG2w, REG1 11076 | GET_Z_PTR REG1, REG1 11077 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11078 |1: 11079 | GC_ADDREF REG1, TMP2w 11080 |2: 11081 | SET_ZVAL_PTR res_addr, REG1, TMP1 11082 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11083 11084 return 1; 11085} 11086 11087static int zend_jit_fetch_dim_read(dasm_State **Dst, 11088 const zend_op *opline, 11089 zend_ssa *ssa, 11090 const zend_ssa_op *ssa_op, 11091 uint32_t op1_info, 11092 zend_jit_addr op1_addr, 11093 bool op1_avoid_refcounting, 11094 uint32_t op2_info, 11095 uint32_t res_info, 11096 zend_jit_addr res_addr, 11097 uint8_t dim_type) 11098{ 11099 zend_jit_addr orig_op1_addr, op2_addr; 11100 const void *exit_addr = NULL; 11101 const void *not_found_exit_addr = NULL; 11102 const void *res_exit_addr = NULL; 11103 bool result_avoid_refcounting = 0; 11104 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11105 int may_throw = 0; 11106 11107 orig_op1_addr = OP1_ADDR(); 11108 op2_addr = OP2_ADDR(); 11109 11110 if (opline->opcode != ZEND_FETCH_DIM_IS 11111 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11112 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11113 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11114 if (!exit_addr) { 11115 return 0; 11116 } 11117 } 11118 11119 if ((res_info & MAY_BE_GUARD) 11120 && JIT_G(current_frame) 11121 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11122 uint32_t flags = 0; 11123 uint32_t old_op1_info = 0; 11124 uint32_t old_info; 11125 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11126 int32_t exit_point; 11127 11128 if (opline->opcode != ZEND_FETCH_LIST_R 11129 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11130 && !op1_avoid_refcounting) { 11131 flags |= ZEND_JIT_EXIT_FREE_OP1; 11132 } 11133 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11134 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11135 flags |= ZEND_JIT_EXIT_FREE_OP2; 11136 } 11137 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11138 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11139 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11140 && (ssa_op+1)->op1_use == ssa_op->result_def 11141 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11142 && zend_jit_may_avoid_refcounting(opline+1)) { 11143 result_avoid_refcounting = 1; 11144 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11145 } 11146 11147 if (op1_avoid_refcounting) { 11148 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11149 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11150 } 11151 11152 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11153 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11154 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11155 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11156 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11157 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11158 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11159 if (!res_exit_addr) { 11160 return 0; 11161 } 11162 res_info &= ~MAY_BE_GUARD; 11163 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11164 } 11165 11166 if (opline->opcode == ZEND_FETCH_DIM_IS 11167 && !(res_info & MAY_BE_NULL)) { 11168 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11169 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11170 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11171 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11172 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11173 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11174 if (!not_found_exit_addr) { 11175 return 0; 11176 } 11177 } 11178 11179 if (op1_avoid_refcounting) { 11180 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11181 } 11182 } 11183 11184 if (op1_info & MAY_BE_REF) { 11185 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11186 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11187 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11188 } 11189 11190 if (op1_info & MAY_BE_ARRAY) { 11191 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11192 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11193 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 11194 } else { 11195 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11196 } 11197 } 11198 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11199 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11200 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11201 may_throw = 1; 11202 } 11203 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11204 return 0; 11205 } 11206 } 11207 11208 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11209 if (op1_info & MAY_BE_ARRAY) { 11210 |.cold_code 11211 |7: 11212 } 11213 11214 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11215 may_throw = 1; 11216 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11217 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11218 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 11219 } else { 11220 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 11221 } 11222 } 11223 | SET_EX_OPLINE opline, REG0 11224 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11225 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11226 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11227 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 11228 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 11229 } else { 11230 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11231 | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 11232 } 11233 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11234 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 11235 } else { 11236 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11237 | LOAD_ZVAL_ADDR CARG3, res_addr 11238 | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 11239 } 11240 if ((op1_info & MAY_BE_ARRAY) || 11241 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11242 | b >9 // END 11243 } 11244 |6: 11245 } 11246 11247 if (op1_info & MAY_BE_OBJECT) { 11248 may_throw = 1; 11249 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11250 if (exit_addr) { 11251 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 11252 } else { 11253 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 11254 } 11255 } 11256 | SET_EX_OPLINE opline, REG0 11257 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11258 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11259 } 11260 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11261 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11262 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11263 } else { 11264 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11265 } 11266 | LOAD_ZVAL_ADDR CARG3, res_addr 11267 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11268 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 11269 } else { 11270 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 11271 } 11272 if ((op1_info & MAY_BE_ARRAY) || 11273 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11274 | b >9 // END 11275 } 11276 |6: 11277 } 11278 11279 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11280 | SET_EX_OPLINE opline, REG0 11281 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11282 may_throw = 1; 11283 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11284 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11285 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11286 | EXT_CALL zend_jit_undefined_op_helper, REG0 11287 |1: 11288 } 11289 11290 if (op2_info & MAY_BE_UNDEF) { 11291 may_throw = 1; 11292 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11293 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11294 | EXT_CALL zend_jit_undefined_op_helper, REG0 11295 |1: 11296 } 11297 } 11298 11299 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11300 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11301 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 11302 may_throw = 1; 11303 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 11304 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 11305 } else { 11306 | SET_EX_OPLINE opline, REG0 11307 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 11308 Z_REG(op1_addr) != ZREG_FCARG1 || 11309 Z_OFFSET(op1_addr) != 0) { 11310 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11311 } 11312 } 11313 | EXT_CALL zend_jit_invalid_array_access, REG0 11314 } 11315 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11316 if (op1_info & MAY_BE_ARRAY) { 11317 | b >9 // END 11318 } 11319 } 11320 11321 if (op1_info & MAY_BE_ARRAY) { 11322 |.code 11323 } 11324 } 11325 11326 if (op1_info & MAY_BE_ARRAY) { 11327 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11328 11329 |8: 11330 if (res_exit_addr) { 11331 uint32_t type = concrete_type(res_info); 11332 if ((op1_info & MAY_BE_ARRAY_OF_REF) 11333 && dim_type != IS_UNKNOWN 11334 && dim_type != IS_REFERENCE) { 11335 if (type < IS_STRING) { 11336 | IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11337 |.cold_code 11338 |1: 11339 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1 11340 | GET_Z_PTR REG0, REG0 11341 | add REG0, REG0, #offsetof(zend_reference, val) 11342 | IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11343 | b &res_exit_addr 11344 |.code 11345 |1: 11346 } else { 11347 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11348 | GET_LOW_8BITS TMP1w, REG2w 11349 | IF_NOT_TYPE TMP1w, type, >1 11350 |.cold_code 11351 |1: 11352 | IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr 11353 | GET_Z_PTR REG0, REG0 11354 | add REG0, REG0, #offsetof(zend_reference, val) 11355 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11356 | GET_LOW_8BITS TMP1w, REG2w 11357 | IF_TYPE TMP1w, type, >1 11358 | b &res_exit_addr 11359 |.code 11360 |1: 11361 } 11362 } else { 11363 if (op1_info & MAY_BE_ARRAY_OF_REF) { 11364 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 11365 } 11366 if (type < IS_STRING) { 11367 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 11368 } else { 11369 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11370 | GET_LOW_8BITS TMP1w, REG2w 11371 | IF_NOT_TYPE TMP1w, type, &res_exit_addr 11372 } 11373 } 11374 | // ZVAL_COPY 11375 |7: 11376 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 11377 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 11378 if (type < IS_STRING) { 11379 if (Z_REG(res_addr) != ZREG_FP || 11380 JIT_G(current_frame) == NULL || 11381 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 11382 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 11383 } 11384 } else { 11385 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11386 if (!result_avoid_refcounting) { 11387 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 11388 } 11389 } 11390 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 11391 return 0; 11392 } 11393 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 11394 | // ZVAL_COPY_DEREF 11395 | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 11396 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { 11397 return 0; 11398 } 11399 } else { 11400 | // ZVAL_COPY 11401 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11402 | TRY_ADDREF res_info, REG1w, REG2, TMP1w 11403 } 11404 } 11405 |9: // END 11406 11407#ifdef ZEND_JIT_USE_RC_INFERENCE 11408 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11409 /* Magic offsetGet() may increase refcount of the key */ 11410 op2_info |= MAY_BE_RCN; 11411 } 11412#endif 11413 11414 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 11415 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 11416 may_throw = 1; 11417 } 11418 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11419 } 11420 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 11421 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 11422 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 11423 may_throw = 1; 11424 } 11425 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11426 } 11427 } 11428 11429 if (may_throw) { 11430 if (!zend_jit_check_exception(Dst)) { 11431 return 0; 11432 } 11433 } 11434 11435 return 1; 11436} 11437 11438static int zend_jit_fetch_dim(dasm_State **Dst, 11439 const zend_op *opline, 11440 uint32_t op1_info, 11441 zend_jit_addr op1_addr, 11442 uint32_t op2_info, 11443 zend_jit_addr res_addr, 11444 uint8_t dim_type) 11445{ 11446 zend_jit_addr op2_addr; 11447 int may_throw = 0; 11448 11449 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 11450 11451 if (op1_info & MAY_BE_REF) { 11452 may_throw = 1; 11453 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11454 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 11455 | GET_Z_PTR FCARG2x, FCARG1x 11456 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 11457 | cmp TMP1w, #IS_ARRAY 11458 | bne >2 11459 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 11460 | b >3 11461 |.cold_code 11462 |2: 11463 | SET_EX_OPLINE opline, REG0 11464 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 11465 | mov FCARG1x, RETVALx 11466 | cbnz FCARG1x, >1 11467 | b ->exception_handler_undef 11468 |.code 11469 |1: 11470 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11471 } 11472 11473 if (op1_info & MAY_BE_ARRAY) { 11474 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11475 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11476 } 11477 |3: 11478 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 11479 } 11480 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 11481 if (op1_info & MAY_BE_ARRAY) { 11482 |.cold_code 11483 |7: 11484 } 11485 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11486 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 11487 | bgt >7 11488 } 11489 if (Z_REG(op1_addr) != ZREG_FP) { 11490 | str Rx(Z_REG(op1_addr)), T1 // save 11491 } 11492 if ((op1_info & MAY_BE_UNDEF) 11493 && opline->opcode == ZEND_FETCH_DIM_RW) { 11494 may_throw = 1; 11495 if (op1_info & MAY_BE_NULL) { 11496 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11497 } 11498 | SET_EX_OPLINE opline, REG0 11499 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11500 | EXT_CALL zend_jit_undefined_op_helper, REG0 11501 |1: 11502 } 11503 | // ZVAL_ARR(container, zend_new_array(8)); 11504 | EXT_CALL _zend_new_array_0, REG0 11505 | mov REG0, RETVALx 11506 if (Z_REG(op1_addr) != ZREG_FP) { 11507 | ldr Rx(Z_REG(op1_addr)), T1 // restore 11508 } 11509 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 11510 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 11511 | mov FCARG1x, REG0 11512 if (op1_info & MAY_BE_ARRAY) { 11513 | b >1 11514 |.code 11515 |1: 11516 } 11517 } 11518 11519 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11520 |6: 11521 if (opline->op2_type == IS_UNUSED) { 11522 may_throw = 1; 11523 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 11524 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 11525 | EXT_CALL zend_hash_next_index_insert, REG0 11526 | // if (UNEXPECTED(!var_ptr)) { 11527 | cbz RETVALx, >1 11528 |.cold_code 11529 |1: 11530 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 11531 | CANNOT_ADD_ELEMENT opline 11532 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 11533 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 11534 | b >8 11535 |.code 11536 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11537 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11538 } else { 11539 uint32_t type; 11540 11541 switch (opline->opcode) { 11542 case ZEND_FETCH_DIM_W: 11543 case ZEND_FETCH_LIST_W: 11544 type = BP_VAR_W; 11545 break; 11546 case ZEND_FETCH_DIM_RW: 11547 may_throw = 1; 11548 type = BP_VAR_RW; 11549 break; 11550 case ZEND_FETCH_DIM_UNSET: 11551 type = BP_VAR_UNSET; 11552 break; 11553 default: 11554 ZEND_UNREACHABLE(); 11555 } 11556 11557 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 11558 may_throw = 1; 11559 } 11560 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 11561 return 0; 11562 } 11563 11564 |8: 11565 | SET_ZVAL_PTR res_addr, REG0, TMP1 11566 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11567 11568 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 11569 |.cold_code 11570 |9: 11571 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11572 | b >8 11573 |.code 11574 } 11575 } 11576 } 11577 11578 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11579 may_throw = 1; 11580 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11581 |.cold_code 11582 |7: 11583 } 11584 11585 | SET_EX_OPLINE opline, REG0 11586 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11587 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11588 } 11589 if (opline->op2_type == IS_UNUSED) { 11590 | mov FCARG2x, xzr 11591 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11592 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11593 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11594 } else { 11595 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11596 } 11597 | LOAD_ZVAL_ADDR CARG3, res_addr 11598 switch (opline->opcode) { 11599 case ZEND_FETCH_DIM_W: 11600 case ZEND_FETCH_LIST_W: 11601 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 11602 break; 11603 case ZEND_FETCH_DIM_RW: 11604 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 11605 break; 11606// case ZEND_FETCH_DIM_UNSET: 11607// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 11608// break; 11609 default: 11610 ZEND_UNREACHABLE(); 11611 } 11612 11613 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11614 | b >8 // END 11615 |.code 11616 } 11617 } 11618 11619#ifdef ZEND_JIT_USE_RC_INFERENCE 11620 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 11621 /* ASSIGN_DIM may increase refcount of the key */ 11622 op2_info |= MAY_BE_RCN; 11623 } 11624#endif 11625 11626 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 11627 && (op2_info & MAY_HAVE_DTOR) 11628 && (op2_info & MAY_BE_RC1)) { 11629 may_throw = 1; 11630 } 11631 |8: 11632 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11633 11634 if (may_throw) { 11635 if (!zend_jit_check_exception(Dst)) { 11636 return 0; 11637 } 11638 } 11639 return 1; 11640} 11641 11642static int zend_jit_isset_isempty_dim(dasm_State **Dst, 11643 const zend_op *opline, 11644 uint32_t op1_info, 11645 zend_jit_addr op1_addr, 11646 bool op1_avoid_refcounting, 11647 uint32_t op2_info, 11648 uint8_t dim_type, 11649 int may_throw, 11650 zend_uchar smart_branch_opcode, 11651 uint32_t target_label, 11652 uint32_t target_label2, 11653 const void *exit_addr) 11654{ 11655 zend_jit_addr op2_addr, res_addr; 11656 11657 // TODO: support for empty() ??? 11658 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 11659 11660 op2_addr = OP2_ADDR(); 11661 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11662 11663 if (op1_info & MAY_BE_REF) { 11664 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11665 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11666 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11667 } 11668 11669 if (op1_info & MAY_BE_ARRAY) { 11670 const void *found_exit_addr = NULL; 11671 const void *not_found_exit_addr = NULL; 11672 11673 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11674 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11675 } 11676 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11677 if (exit_addr 11678 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 11679 && !may_throw 11680 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 11681 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 11682 if (smart_branch_opcode == ZEND_JMPNZ) { 11683 found_exit_addr = exit_addr; 11684 } else { 11685 not_found_exit_addr = exit_addr; 11686 } 11687 } 11688 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 11689 return 0; 11690 } 11691 11692 if (found_exit_addr) { 11693 |9: 11694 return 1; 11695 } else if (not_found_exit_addr) { 11696 |8: 11697 return 1; 11698 } 11699 } 11700 11701 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11702 if (op1_info & MAY_BE_ARRAY) { 11703 |.cold_code 11704 |7: 11705 } 11706 11707 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 11708 | SET_EX_OPLINE opline, REG0 11709 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11710 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11711 } 11712 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11713 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11714 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11715 } else { 11716 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11717 } 11718 | EXT_CALL zend_jit_isset_dim_helper, REG0 11719 | cbz RETVALw, >9 11720 if (op1_info & MAY_BE_ARRAY) { 11721 | b >8 11722 |.code 11723 } 11724 } else { 11725 if (op2_info & MAY_BE_UNDEF) { 11726 if (op2_info & MAY_BE_ANY) { 11727 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11728 } 11729 | SET_EX_OPLINE opline, REG0 11730 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11731 | EXT_CALL zend_jit_undefined_op_helper, REG0 11732 |1: 11733 } 11734 if (op1_info & MAY_BE_ARRAY) { 11735 | b >9 11736 |.code 11737 } 11738 } 11739 } 11740 11741#ifdef ZEND_JIT_USE_RC_INFERENCE 11742 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11743 /* Magic offsetExists() may increase refcount of the key */ 11744 op2_info |= MAY_BE_RCN; 11745 } 11746#endif 11747 11748 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 11749 |8: 11750 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11751 if (!op1_avoid_refcounting) { 11752 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11753 } 11754 if (may_throw) { 11755 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11756 return 0; 11757 } 11758 } 11759 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11760 if (exit_addr) { 11761 if (smart_branch_opcode == ZEND_JMPNZ) { 11762 | b &exit_addr 11763 } else { 11764 | b >8 11765 } 11766 } else if (smart_branch_opcode) { 11767 if (smart_branch_opcode == ZEND_JMPZ) { 11768 | b =>target_label2 11769 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11770 | b =>target_label 11771 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11772 | b =>target_label2 11773 } else { 11774 ZEND_UNREACHABLE(); 11775 } 11776 } else { 11777 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 11778 | b >8 11779 } 11780 } else { 11781 | NIY // TODO: support for empty() 11782 } 11783 } 11784 11785 |9: // not found 11786 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11787 if (!op1_avoid_refcounting) { 11788 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11789 } 11790 if (may_throw) { 11791 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11792 return 0; 11793 } 11794 } 11795 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11796 if (exit_addr) { 11797 if (smart_branch_opcode == ZEND_JMPZ) { 11798 | b &exit_addr 11799 } 11800 } else if (smart_branch_opcode) { 11801 if (smart_branch_opcode == ZEND_JMPZ) { 11802 | b =>target_label 11803 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11804 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11805 | b =>target_label 11806 } else { 11807 ZEND_UNREACHABLE(); 11808 } 11809 } else { 11810 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 11811 } 11812 } else { 11813 | NIY // TODO: support for empty() 11814 } 11815 11816 |8: 11817 11818 return 1; 11819} 11820 11821static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 11822{ 11823 zend_jit_addr op1_addr = OP1_ADDR(); 11824 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 11825 11826 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 11827 | ldr FCARG2x, EX->run_time_cache 11828 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1 11829 | sub REG0, REG0, #1 11830 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 11831 | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 11832 | cmp REG0, REG1, lsl #5 11833 | bhs >9 11834 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 11835 | MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 11836 | add REG0, REG0, TMP1 11837 | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w 11838 | // (EXPECTED(p->key == varname)) 11839 | ldr TMP1, [REG0, #offsetof(Bucket, key)] 11840 | LOAD_ADDR TMP2, varname 11841 | cmp TMP1, TMP2 11842 | bne >9 11843 | GET_Z_PTR REG0, REG0 11844 | GC_ADDREF REG0, TMP1w 11845 |1: 11846 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11847 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11848 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 11849 | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 11850 |.cold_code 11851 |2: 11852 } 11853 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 11854 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 11855 | // ZVAL_REF(variable_ptr, ref) 11856 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11857 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11858 | // if (GC_DELREF(garbage) == 0) 11859 | GC_DELREF FCARG1x, TMP1w 11860 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11861 | bne >3 11862 } else { 11863 | bne >5 11864 } 11865 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 11866 | b >5 11867 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11868 |3: 11869 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 11870 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 11871 | EXT_CALL gc_possible_root, REG0 11872 | b >5 11873 } 11874 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11875 |.code 11876 } 11877 } 11878 11879 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11880 | // ZVAL_REF(variable_ptr, ref) 11881 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11882 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11883 } 11884 |5: 11885 //END of handler 11886 11887 |.cold_code 11888 |9: 11889 | LOAD_ADDR FCARG1x, (ptrdiff_t)varname 11890 if (opline->extended_value) { 11891 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 11892 } 11893 | EXT_CALL zend_jit_fetch_global_helper, REG0 11894 | mov REG0, RETVALx 11895 | b <1 11896 |.code 11897 11898 return 1; 11899} 11900 11901static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 11902{ 11903 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11904 bool in_cold = 0; 11905 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 11906 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0; 11907 11908 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 11909 && JIT_G(current_frame) 11910 && JIT_G(current_frame)->prev) { 11911 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11912 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 11913 11914 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 11915 return 1; 11916 } 11917 } 11918 11919 if (ZEND_ARG_SEND_MODE(arg_info)) { 11920 if (opline->opcode == ZEND_RECV_INIT) { 11921 | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr 11922 | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w 11923 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 11924 } else { 11925 | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 11926 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 11927 } 11928 } 11929 11930 if (type_mask != 0) { 11931 if (is_power_of_two(type_mask)) { 11932 uint32_t type_code = concrete_type(type_mask); 11933 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 11934 } else { 11935 | mov REG2w, #1 11936 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 11937 | lsl REG2w, REG2w, REG1w 11938 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 11939 | beq >1 11940 } 11941 11942 |.cold_code 11943 |1: 11944 11945 in_cold = 1; 11946 } 11947 11948 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 11949 | LOAD_ZVAL_ADDR FCARG1x, res_addr 11950 } 11951 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11952 | SET_EX_OPLINE opline, REG0 11953 } else { 11954 | ADDR_STORE EX->opline, opline, REG0 11955 } 11956 | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info 11957 | EXT_CALL zend_jit_verify_arg_slow, REG0 11958 11959 if (check_exception) { 11960 | GET_LOW_8BITS REG0w, RETVALw 11961 if (in_cold) { 11962 | cbnz REG0w, >1 11963 | b ->exception_handler 11964 |.code 11965 |1: 11966 } else { 11967 | cbz REG0w, ->exception_handler 11968 } 11969 } else if (in_cold) { 11970 | b >1 11971 |.code 11972 |1: 11973 } 11974 11975 return 1; 11976} 11977 11978static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 11979{ 11980 uint32_t arg_num = opline->op1.num; 11981 zend_arg_info *arg_info = NULL; 11982 11983 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 11984 if (EXPECTED(arg_num <= op_array->num_args)) { 11985 arg_info = &op_array->arg_info[arg_num-1]; 11986 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 11987 arg_info = &op_array->arg_info[op_array->num_args]; 11988 } 11989 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 11990 arg_info = NULL; 11991 } 11992 } 11993 11994 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 11995 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11996 if (!JIT_G(current_frame) || 11997 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 11998 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 11999 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12000 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12001 12002 if (!exit_addr) { 12003 return 0; 12004 } 12005 | ldr TMP1w, EX->This.u2.num_args 12006 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12007 | blo &exit_addr 12008 } 12009 } else { 12010 | ldr TMP1w, EX->This.u2.num_args 12011 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12012 | blo >1 12013 |.cold_code 12014 |1: 12015 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12016 | SET_EX_OPLINE opline, REG0 12017 } else { 12018 | ADDR_STORE EX->opline, opline, REG0 12019 } 12020 | mov FCARG1x, FP 12021 | EXT_CALL zend_missing_arg_error, REG0 12022 | b ->exception_handler 12023 |.code 12024 } 12025 } 12026 12027 if (arg_info) { 12028 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12029 return 0; 12030 } 12031 } 12032 12033 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12034 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12035 | LOAD_IP_ADDR (opline + 1) 12036 zend_jit_set_last_valid_opline(opline + 1); 12037 } 12038 } 12039 12040 return 1; 12041} 12042 12043static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12044{ 12045 uint32_t arg_num = opline->op1.num; 12046 zval *zv = RT_CONSTANT(opline, opline->op2); 12047 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12048 12049 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12050 && JIT_G(current_frame) 12051 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12052 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12053 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12054 if (Z_REFCOUNTED_P(zv)) { 12055 | ADDREF_CONST zv, REG0, TMP1 12056 } 12057 } 12058 } else { 12059 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12060 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12061 | ldr TMP1w, EX->This.u2.num_args 12062 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12063 | bhs >5 12064 } 12065 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12066 if (Z_REFCOUNTED_P(zv)) { 12067 | ADDREF_CONST zv, REG0, TMP1 12068 } 12069 } 12070 12071 if (Z_CONSTANT_P(zv)) { 12072 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12073 | SET_EX_OPLINE opline, REG0 12074 } else { 12075 | ADDR_STORE EX->opline, opline, REG0 12076 } 12077 | LOAD_ZVAL_ADDR FCARG1x, res_addr 12078 | ldr REG0, EX->func 12079 | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] 12080 | EXT_CALL zval_update_constant_ex, REG0 12081 | cbnz RETVALw, >1 12082 |.cold_code 12083 |1: 12084 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 12085 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 12086 | b ->exception_handler 12087 |.code 12088 } 12089 12090 |5: 12091 12092 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12093 do { 12094 zend_arg_info *arg_info; 12095 12096 if (arg_num <= op_array->num_args) { 12097 arg_info = &op_array->arg_info[arg_num-1]; 12098 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12099 arg_info = &op_array->arg_info[op_array->num_args]; 12100 } else { 12101 break; 12102 } 12103 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12104 break; 12105 } 12106 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12107 return 0; 12108 } 12109 } while (0); 12110 } 12111 12112 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12113 if (is_last) { 12114 | LOAD_IP_ADDR (opline + 1) 12115 zend_jit_set_last_valid_opline(opline + 1); 12116 } 12117 } 12118 return 1; 12119} 12120 12121static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12122{ 12123 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12124 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12125 12126 if (!exit_addr) { 12127 return 0; 12128 } 12129 12130 | LOAD_ADDR TMP1, ((ptrdiff_t)ce) 12131 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 12132 | cmp TMP2, TMP1 12133 | bne &exit_addr 12134 12135 return 1; 12136} 12137 12138static int zend_jit_fetch_obj(dasm_State **Dst, 12139 const zend_op *opline, 12140 const zend_op_array *op_array, 12141 zend_ssa *ssa, 12142 const zend_ssa_op *ssa_op, 12143 uint32_t op1_info, 12144 zend_jit_addr op1_addr, 12145 bool op1_indirect, 12146 zend_class_entry *ce, 12147 bool ce_is_instanceof, 12148 bool on_this, 12149 bool delayed_fetch_this, 12150 bool op1_avoid_refcounting, 12151 zend_class_entry *trace_ce, 12152 uint8_t prop_type, 12153 int may_throw) 12154{ 12155 zval *member; 12156 zend_property_info *prop_info; 12157 bool may_be_dynamic = 1; 12158 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12159 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12160 zend_jit_addr prop_addr; 12161 uint32_t res_info = RES_INFO(); 12162 bool type_loaded = 0; 12163 12164 ZEND_ASSERT(opline->op2_type == IS_CONST); 12165 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12166 12167 member = RT_CONSTANT(opline, opline->op2); 12168 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12169 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12170 12171 if (on_this) { 12172 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12173 } else { 12174 if (opline->op1_type == IS_VAR 12175 && opline->opcode == ZEND_FETCH_OBJ_W 12176 && (op1_info & MAY_BE_INDIRECT) 12177 && Z_REG(op1_addr) == ZREG_FP) { 12178 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12179 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12180 | GET_Z_PTR FCARG1x, FCARG1x 12181 |1: 12182 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12183 } 12184 if (op1_info & MAY_BE_REF) { 12185 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12186 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12187 } 12188 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12189 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12190 } 12191 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12192 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12193 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12194 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12195 12196 if (!exit_addr) { 12197 return 0; 12198 } 12199 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12200 } else { 12201 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 12202 } 12203 } 12204 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12205 } 12206 12207 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12208 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12209 if (prop_info) { 12210 ce = trace_ce; 12211 ce_is_instanceof = 0; 12212 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12213 if (on_this && JIT_G(current_frame) 12214 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12215 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12216 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12217 if (on_this && JIT_G(current_frame)) { 12218 JIT_G(current_frame)->ce = ce; 12219 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12220 } 12221 } else { 12222 return 0; 12223 } 12224 if (ssa->var_info && ssa_op->op1_use >= 0) { 12225 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12226 ssa->var_info[ssa_op->op1_use].ce = ce; 12227 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12228 } 12229 } 12230 } 12231 } 12232 12233 if (!prop_info) { 12234 | ldr REG0, EX->run_time_cache 12235 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 12236 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12237 | cmp REG2, TMP1 12238 | bne >5 12239 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 12240 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 12241 if (may_be_dynamic) { 12242 | tst REG0, REG0 12243 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12244 | blt >5 12245 } else { 12246 | blt >8 // dynamic property 12247 } 12248 } 12249 | add TMP1, FCARG1x, REG0 12250 | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] 12251 | IF_UNDEF REG2w, >5 12252 | mov FCARG1x, TMP1 12253 type_loaded = 1; 12254 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12255 if (opline->opcode == ZEND_FETCH_OBJ_W 12256 && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { 12257 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12258 12259 | ldr REG0, EX->run_time_cache 12260 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 12261 | cbnz FCARG2x, >1 12262 |.cold_code 12263 |1: 12264 | ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)] 12265 | tst TMP1w, #ZEND_ACC_READONLY 12266 if (flags) { 12267 | beq >3 12268 } else { 12269 | beq >4 12270 } 12271 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2 12272 | GET_Z_PTR REG2, FCARG1x 12273 | GC_ADDREF REG2, TMP1w 12274 | SET_ZVAL_PTR res_addr, REG2, TMP1 12275 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12276 | b >9 12277 |2: 12278 | mov FCARG1x, FCARG2x 12279 | SET_EX_OPLINE opline, REG0 12280 | EXT_CALL zend_readonly_property_modification_error, REG0 12281 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12282 | b >9 12283 |3: 12284 if (flags == ZEND_FETCH_DIM_WRITE) { 12285 | SET_EX_OPLINE opline, REG0 12286 | EXT_CALL zend_jit_check_array_promotion, REG0 12287 | b >9 12288 } else if (flags == ZEND_FETCH_REF) { 12289 | LOAD_ZVAL_ADDR CARG3, res_addr 12290 | EXT_CALL zend_jit_create_typed_ref, REG0 12291 | b >9 12292 } else { 12293 ZEND_ASSERT(flags == 0); 12294 } 12295 |.code 12296 |4: 12297 } 12298 } else { 12299 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12300 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12301 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 12302 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 12303 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12304 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12305 12306 if (!exit_addr) { 12307 return 0; 12308 } 12309 type_loaded = 1; 12310 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12311 | IF_UNDEF REG2w, &exit_addr 12312 } 12313 } else { 12314 type_loaded = 1; 12315 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12316 | IF_UNDEF REG2w, >5 12317 } 12318 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 12319 if (!type_loaded) { 12320 type_loaded = 1; 12321 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12322 } 12323 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4 12324 | GET_ZVAL_PTR REG2, prop_addr, TMP1 12325 | GC_ADDREF REG2, TMP1w 12326 | SET_ZVAL_PTR res_addr, REG2, TMP1 12327 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12328 | b >9 12329 |.cold_code 12330 |4: 12331 | LOAD_ADDR FCARG1x, prop_info 12332 | SET_EX_OPLINE opline, REG0 12333 | EXT_CALL zend_readonly_property_modification_error, REG0 12334 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12335 | b >9 12336 |.code 12337 } 12338 if (opline->opcode == ZEND_FETCH_OBJ_W 12339 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 12340 && ZEND_TYPE_IS_SET(prop_info->type)) { 12341 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12342 12343 if (flags == ZEND_FETCH_DIM_WRITE) { 12344 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 12345 if (!type_loaded) { 12346 type_loaded = 1; 12347 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12348 } 12349 | cmp REG2w, #IS_FALSE 12350 | ble >1 12351 |.cold_code 12352 |1: 12353 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12354 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12355 } 12356 | LOAD_ADDR FCARG2x, prop_info 12357 | SET_EX_OPLINE opline, REG0 12358 | EXT_CALL zend_jit_check_array_promotion, REG0 12359 | b >9 12360 |.code 12361 } 12362 } else if (flags == ZEND_FETCH_REF) { 12363 if (!type_loaded) { 12364 type_loaded = 1; 12365 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12366 } 12367 | GET_LOW_8BITS TMP1w, REG2w 12368 | IF_TYPE TMP1w, IS_REFERENCE, >1 12369 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12370 | LOAD_ADDR FCARG2x, prop_info 12371 } else { 12372 int prop_info_offset = 12373 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12374 12375 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12376 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12377 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12378 } 12379 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12380 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12381 } 12382 | LOAD_ZVAL_ADDR CARG3, res_addr 12383 | EXT_CALL zend_jit_create_typed_ref, REG0 12384 | b >9 12385 |1: 12386 } else { 12387 ZEND_UNREACHABLE(); 12388 } 12389 } 12390 } 12391 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12392 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12393 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12394 } 12395 | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 12396 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 12397 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 12398 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 12399 } 12400 } else { 12401 bool result_avoid_refcounting = 0; 12402 12403 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 12404 uint32_t flags = 0; 12405 uint32_t old_info; 12406 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12407 int32_t exit_point; 12408 const void *exit_addr; 12409 uint32_t type; 12410 zend_jit_addr val_addr = prop_addr; 12411 12412 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 12413 && !delayed_fetch_this 12414 && !op1_avoid_refcounting) { 12415 flags = ZEND_JIT_EXIT_FREE_OP1; 12416 } 12417 12418 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 12419 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 12420 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 12421 && (ssa_op+1)->op1_use == ssa_op->result_def 12422 && zend_jit_may_avoid_refcounting(opline+1)) { 12423 result_avoid_refcounting = 1; 12424 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 12425 } 12426 12427 type = concrete_type(res_info); 12428 12429 if (prop_type != IS_UNKNOWN 12430 && prop_type != IS_UNDEF 12431 && prop_type != IS_REFERENCE 12432 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 12433 exit_point = zend_jit_trace_get_exit_point(opline, 0); 12434 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12435 } else { 12436 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 12437 | LOAD_ZVAL_ADDR REG0, prop_addr 12438 if (op1_avoid_refcounting) { 12439 SET_STACK_REG(JIT_G(current_frame)->stack, 12440 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12441 } 12442 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12443 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 12444 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 12445 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12446 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12447 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12448 if (!exit_addr) { 12449 return 0; 12450 } 12451 12452 if (!type_loaded) { 12453 type_loaded = 1; 12454 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12455 } 12456 | // ZVAL_DEREF() 12457 | GET_LOW_8BITS TMP1w, REG2w 12458 | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 12459 | GET_Z_PTR REG0, REG0 12460 | add REG0, REG0, #offsetof(zend_reference, val) 12461 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12462 } 12463 res_info &= ~MAY_BE_GUARD; 12464 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12465 if (type < IS_STRING) { 12466 |1: 12467 if (type_loaded) { 12468 | IF_NOT_TYPE REG2w, type, &exit_addr 12469 } else { 12470 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 12471 } 12472 } else { 12473 if (!type_loaded) { 12474 type_loaded = 1; 12475 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12476 } 12477 |1: 12478 | GET_LOW_8BITS TMP1w, REG2w 12479 | IF_NOT_TYPE TMP1w, type, &exit_addr 12480 } 12481 | // ZVAL_COPY 12482 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 12483 if (type < IS_STRING) { 12484 if (Z_REG(res_addr) != ZREG_FP || 12485 JIT_G(current_frame) == NULL || 12486 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12487 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 12488 } 12489 } else { 12490 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 12491 if (!result_avoid_refcounting) { 12492 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 12493 } 12494 } 12495 } else { 12496 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { 12497 return 0; 12498 } 12499 } 12500 } 12501 12502 if (op1_avoid_refcounting) { 12503 SET_STACK_REG(JIT_G(current_frame)->stack, 12504 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12505 } 12506 12507 |.cold_code 12508 12509 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 12510 |5: 12511 | SET_EX_OPLINE opline, REG0 12512 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12513 | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 12514 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12515 | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 12516 } else { 12517 | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 12518 } 12519 | b >9 12520 } 12521 12522 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12523 |7: 12524 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12525 | SET_EX_OPLINE opline, REG0 12526 if (opline->opcode != ZEND_FETCH_OBJ_W 12527 && (op1_info & MAY_BE_UNDEF)) { 12528 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12529 12530 if (op1_info & MAY_BE_ANY) { 12531 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 12532 } 12533 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 12534 | EXT_CALL zend_jit_undefined_op_helper, REG0 12535 |1: 12536 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 12537 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12538 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12539 } 12540 | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) 12541 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12542 | EXT_CALL zend_jit_invalid_property_write, REG0 12543 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12544 } else { 12545 | EXT_CALL zend_jit_invalid_property_read, REG0 12546 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12547 } 12548 | b >9 12549 } else { 12550 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12551 | b >9 12552 } 12553 } 12554 12555 if (!prop_info 12556 && may_be_dynamic 12557 && opline->opcode != ZEND_FETCH_OBJ_W) { 12558 |8: 12559 | mov FCARG2x, REG0 12560 | SET_EX_OPLINE opline, REG0 12561 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12562 | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 12563 } else { 12564 | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 12565 } 12566 | b >9 12567 } 12568 12569 |.code; 12570 |9: // END 12571 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 12572 if (opline->op1_type == IS_VAR 12573 && opline->opcode == ZEND_FETCH_OBJ_W 12574 && (op1_info & MAY_BE_RC1)) { 12575 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12576 12577 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 12578 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 12579 | GC_DELREF FCARG1x, TMP1w 12580 | bne >1 12581 | SET_EX_OPLINE opline, REG0 12582 | EXT_CALL zend_jit_extract_helper, REG0 12583 |1: 12584 } else if (!op1_avoid_refcounting) { 12585 if (on_this) { 12586 op1_info &= ~MAY_BE_RC1; 12587 } 12588 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 12589 } 12590 } 12591 12592 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12593 && prop_info 12594 && (opline->opcode != ZEND_FETCH_OBJ_W || 12595 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 12596 !ZEND_TYPE_IS_SET(prop_info->type)) 12597 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 12598 may_throw = 0; 12599 } 12600 12601 if (may_throw) { 12602 if (!zend_jit_check_exception(Dst)) { 12603 return 0; 12604 } 12605 } 12606 12607 return 1; 12608} 12609 12610static int zend_jit_incdec_obj(dasm_State **Dst, 12611 const zend_op *opline, 12612 const zend_op_array *op_array, 12613 zend_ssa *ssa, 12614 const zend_ssa_op *ssa_op, 12615 uint32_t op1_info, 12616 zend_jit_addr op1_addr, 12617 bool op1_indirect, 12618 zend_class_entry *ce, 12619 bool ce_is_instanceof, 12620 bool on_this, 12621 bool delayed_fetch_this, 12622 zend_class_entry *trace_ce, 12623 uint8_t prop_type) 12624{ 12625 zval *member; 12626 zend_string *name; 12627 zend_property_info *prop_info; 12628 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12629 zend_jit_addr res_addr = 0; 12630 zend_jit_addr prop_addr; 12631 bool needs_slow_path = 0; 12632 bool use_prop_guard = 0; 12633 bool may_throw = 0; 12634 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 12635 12636 ZEND_ASSERT(opline->op2_type == IS_CONST); 12637 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12638 12639 if (opline->result_type != IS_UNUSED) { 12640 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12641 } 12642 12643 member = RT_CONSTANT(opline, opline->op2); 12644 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12645 name = Z_STR_P(member); 12646 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 12647 12648 if (on_this) { 12649 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12650 } else { 12651 if (opline->op1_type == IS_VAR 12652 && (op1_info & MAY_BE_INDIRECT) 12653 && Z_REG(op1_addr) == ZREG_FP) { 12654 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12655 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12656 | GET_Z_PTR FCARG1x, FCARG1x 12657 |1: 12658 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12659 } 12660 if (op1_info & MAY_BE_REF) { 12661 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12662 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12663 } 12664 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12665 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12666 } 12667 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12668 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12669 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12670 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12671 12672 if (!exit_addr) { 12673 return 0; 12674 } 12675 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12676 } else { 12677 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 12678 |.cold_code 12679 |1: 12680 | SET_EX_OPLINE opline, REG0 12681 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12682 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12683 } 12684 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 12685 | EXT_CALL zend_jit_invalid_property_incdec, REG0 12686 | b ->exception_handler 12687 |.code 12688 } 12689 } 12690 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12691 } 12692 12693 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12694 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 12695 if (prop_info) { 12696 ce = trace_ce; 12697 ce_is_instanceof = 0; 12698 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12699 if (on_this && JIT_G(current_frame) 12700 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12701 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12702 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12703 if (on_this && JIT_G(current_frame)) { 12704 JIT_G(current_frame)->ce = ce; 12705 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12706 } 12707 } else { 12708 return 0; 12709 } 12710 if (ssa->var_info && ssa_op->op1_use >= 0) { 12711 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12712 ssa->var_info[ssa_op->op1_use].ce = ce; 12713 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12714 } 12715 if (ssa->var_info && ssa_op->op1_def >= 0) { 12716 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 12717 ssa->var_info[ssa_op->op1_def].ce = ce; 12718 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 12719 } 12720 } 12721 } 12722 } 12723 12724 use_prop_guard = (prop_type != IS_UNKNOWN 12725 && prop_type != IS_UNDEF 12726 && prop_type != IS_REFERENCE 12727 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 12728 12729 if (!prop_info) { 12730 needs_slow_path = 1; 12731 12732 | ldr REG0, EX->run_time_cache 12733 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 12734 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12735 | cmp REG2, TMP1 12736 | bne >7 12737 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12738 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 12739 | cbnz TMP1, >7 12740 } 12741 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 12742 | tst REG0, REG0 12743 | blt >7 12744 | add TMP1, FCARG1x, REG0 12745 if (!use_prop_guard) { 12746 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 12747 | IF_TYPE TMP2w , IS_UNDEF, >7 12748 } 12749 | mov FCARG1x, TMP1 12750 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12751 } else { 12752 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12753 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 12754 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12755 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12756 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12757 12758 if (!exit_addr) { 12759 return 0; 12760 } 12761 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12762 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 12763 } else { 12764 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12765 | IF_TYPE TMP2w, IS_UNDEF, >7 12766 needs_slow_path = 1; 12767 } 12768 } 12769 if (ZEND_TYPE_IS_SET(prop_info->type)) { 12770 may_throw = 1; 12771 | SET_EX_OPLINE opline, REG0 12772 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12773 | LOAD_ADDR FCARG2x, prop_info 12774 } else { 12775 int prop_info_offset = 12776 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12777 12778 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12779 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12780 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12781 } 12782 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12783 if (opline->result_type == IS_UNUSED) { 12784 switch (opline->opcode) { 12785 case ZEND_PRE_INC_OBJ: 12786 case ZEND_POST_INC_OBJ: 12787 | EXT_CALL zend_jit_inc_typed_prop, REG0 12788 break; 12789 case ZEND_PRE_DEC_OBJ: 12790 case ZEND_POST_DEC_OBJ: 12791 | EXT_CALL zend_jit_dec_typed_prop, REG0 12792 break; 12793 default: 12794 ZEND_UNREACHABLE(); 12795 } 12796 } else { 12797 | LOAD_ZVAL_ADDR CARG3, res_addr 12798 switch (opline->opcode) { 12799 case ZEND_PRE_INC_OBJ: 12800 | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 12801 break; 12802 case ZEND_PRE_DEC_OBJ: 12803 | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 12804 break; 12805 case ZEND_POST_INC_OBJ: 12806 | EXT_CALL zend_jit_post_inc_typed_prop, REG0 12807 break; 12808 case ZEND_POST_DEC_OBJ: 12809 | EXT_CALL zend_jit_post_dec_typed_prop, REG0 12810 break; 12811 default: 12812 ZEND_UNREACHABLE(); 12813 } 12814 } 12815 } 12816 } 12817 12818 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 12819 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 12820 zend_jit_addr var_addr = prop_addr; 12821 12822 if (use_prop_guard) { 12823 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12824 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12825 12826 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 12827 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 12828 } 12829 12830 if (var_info & MAY_BE_REF) { 12831 may_throw = 1; 12832 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12833 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12834 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12835 } 12836 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 12837 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 12838 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 12839 | cbnz TMP1, >1 12840 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 12841 |.cold_code 12842 |1: 12843 if (opline) { 12844 | SET_EX_OPLINE opline, REG0 12845 } 12846 if (opline->result_type == IS_UNUSED) { 12847 | mov FCARG2x, xzr 12848 } else { 12849 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12850 } 12851 switch (opline->opcode) { 12852 case ZEND_PRE_INC_OBJ: 12853 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 12854 break; 12855 case ZEND_PRE_DEC_OBJ: 12856 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 12857 break; 12858 case ZEND_POST_INC_OBJ: 12859 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 12860 break; 12861 case ZEND_POST_DEC_OBJ: 12862 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 12863 break; 12864 default: 12865 ZEND_UNREACHABLE(); 12866 } 12867 | b >9 12868 |.code 12869 |2: 12870 } 12871 12872 if (var_info & MAY_BE_LONG) { 12873 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12874 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 12875 } 12876 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12877 if (opline->result_type != IS_UNUSED) { 12878 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12879 } 12880 } 12881 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12882 | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 12883 } else { 12884 | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 12885 } 12886 | bvs >3 12887 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 12888 if (opline->result_type != IS_UNUSED) { 12889 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12890 } 12891 } 12892 |.cold_code 12893 } 12894 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12895 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 12896 may_throw = 1; 12897 } 12898 if (var_info & MAY_BE_LONG) { 12899 |2: 12900 } 12901 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 12902 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12903 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12904 } 12905 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12906 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12907 | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w 12908 } 12909 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12910 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12911 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12912 | EXT_CALL zend_jit_pre_inc, REG0 12913 } else { 12914 | EXT_CALL increment_function, REG0 12915 } 12916 } else { 12917 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12918 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12919 | EXT_CALL zend_jit_pre_dec, REG0 12920 } else { 12921 | EXT_CALL decrement_function, REG0 12922 } 12923 } 12924 if (var_info & MAY_BE_LONG) { 12925 | b >4 12926 } 12927 } 12928 12929 if (var_info & MAY_BE_LONG) { 12930 |3: 12931 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12932 uint64_t val = 0x43e0000000000000; 12933 | LOAD_64BIT_VAL REG0, val 12934 | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 12935 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12936 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12937 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 12938 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12939 } 12940 } else { 12941 uint64_t val = 0xc3e0000000000000; 12942 | LOAD_64BIT_VAL REG0, val 12943 | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 12944 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12945 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12946 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 12947 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12948 } 12949 } 12950 if (opline->result_type != IS_UNUSED 12951 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 12952 && prop_info 12953 && !ZEND_TYPE_IS_SET(prop_info->type) 12954 && (res_info & MAY_BE_GUARD) 12955 && (res_info & MAY_BE_LONG)) { 12956 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12957 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12958 int32_t exit_point; 12959 const void *exit_addr; 12960 12961 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 12962 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 12963 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12964 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 12965 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 12966 | b &exit_addr 12967 |.code 12968 } else { 12969 | b >4 12970 |.code 12971 |4: 12972 } 12973 } 12974 } 12975 12976 if (needs_slow_path) { 12977 may_throw = 1; 12978 |.cold_code 12979 |7: 12980 | SET_EX_OPLINE opline, REG0 12981 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 12982 | LOAD_ADDR FCARG2x, name 12983 | ldr CARG3, EX->run_time_cache 12984 | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 12985 if (opline->result_type == IS_UNUSED) { 12986 | mov CARG4, xzr 12987 } else { 12988 | LOAD_ZVAL_ADDR CARG4, res_addr 12989 } 12990 12991 switch (opline->opcode) { 12992 case ZEND_PRE_INC_OBJ: 12993 | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 12994 break; 12995 case ZEND_PRE_DEC_OBJ: 12996 | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 12997 break; 12998 case ZEND_POST_INC_OBJ: 12999 | EXT_CALL zend_jit_post_inc_obj_helper, REG0 13000 break; 13001 case ZEND_POST_DEC_OBJ: 13002 | EXT_CALL zend_jit_post_dec_obj_helper, REG0 13003 break; 13004 default: 13005 ZEND_UNREACHABLE(); 13006 } 13007 13008 | b >9 13009 |.code 13010 } 13011 13012 |9: 13013 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13014 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13015 may_throw = 1; 13016 } 13017 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13018 } 13019 13020 if (may_throw) { 13021 if (!zend_jit_check_exception(Dst)) { 13022 return 0; 13023 } 13024 } 13025 13026 return 1; 13027} 13028 13029static int zend_jit_assign_obj_op(dasm_State **Dst, 13030 const zend_op *opline, 13031 const zend_op_array *op_array, 13032 zend_ssa *ssa, 13033 const zend_ssa_op *ssa_op, 13034 uint32_t op1_info, 13035 zend_jit_addr op1_addr, 13036 uint32_t val_info, 13037 zend_ssa_range *val_range, 13038 bool op1_indirect, 13039 zend_class_entry *ce, 13040 bool ce_is_instanceof, 13041 bool on_this, 13042 bool delayed_fetch_this, 13043 zend_class_entry *trace_ce, 13044 uint8_t prop_type) 13045{ 13046 zval *member; 13047 zend_string *name; 13048 zend_property_info *prop_info; 13049 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13050 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13051 zend_jit_addr prop_addr; 13052 bool needs_slow_path = 0; 13053 bool use_prop_guard = 0; 13054 bool may_throw = 0; 13055 binary_op_type binary_op = get_binary_op(opline->extended_value); 13056 13057 ZEND_ASSERT(opline->op2_type == IS_CONST); 13058 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13059 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13060 13061 member = RT_CONSTANT(opline, opline->op2); 13062 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13063 name = Z_STR_P(member); 13064 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13065 13066 if (on_this) { 13067 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13068 } else { 13069 if (opline->op1_type == IS_VAR 13070 && (op1_info & MAY_BE_INDIRECT) 13071 && Z_REG(op1_addr) == ZREG_FP) { 13072 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13073 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13074 | GET_Z_PTR FCARG1x, FCARG1x 13075 |1: 13076 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13077 } 13078 if (op1_info & MAY_BE_REF) { 13079 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13080 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13081 } 13082 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13083 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13084 } 13085 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13086 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13087 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13088 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13089 13090 if (!exit_addr) { 13091 return 0; 13092 } 13093 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13094 } else { 13095 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13096 |.cold_code 13097 |1: 13098 | SET_EX_OPLINE opline, REG0 13099 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13100 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13101 } 13102 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13103 if (op1_info & MAY_BE_UNDEF) { 13104 | EXT_CALL zend_jit_invalid_property_assign_op, REG0 13105 } else { 13106 | EXT_CALL zend_jit_invalid_property_assign, REG0 13107 } 13108 may_throw = 1; 13109 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13110 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13111 | b >8 13112 } else { 13113 | b >9 13114 } 13115 |.code 13116 } 13117 } 13118 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13119 } 13120 13121 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13122 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13123 if (prop_info) { 13124 ce = trace_ce; 13125 ce_is_instanceof = 0; 13126 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13127 if (on_this && JIT_G(current_frame) 13128 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13129 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13130 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13131 if (on_this && JIT_G(current_frame)) { 13132 JIT_G(current_frame)->ce = ce; 13133 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13134 } 13135 } else { 13136 return 0; 13137 } 13138 if (ssa->var_info && ssa_op->op1_use >= 0) { 13139 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13140 ssa->var_info[ssa_op->op1_use].ce = ce; 13141 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13142 } 13143 if (ssa->var_info && ssa_op->op1_def >= 0) { 13144 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13145 ssa->var_info[ssa_op->op1_def].ce = ce; 13146 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13147 } 13148 } 13149 } 13150 } 13151 13152 use_prop_guard = (prop_type != IS_UNKNOWN 13153 && prop_type != IS_UNDEF 13154 && prop_type != IS_REFERENCE 13155 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13156 13157 if (!prop_info) { 13158 needs_slow_path = 1; 13159 13160 | ldr REG0, EX->run_time_cache 13161 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 13162 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 13163 | cmp REG2, TMP2 13164 | bne >7 13165 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13166 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 13167 | cbnz TMP1, >7 13168 } 13169 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 13170 | tst REG0, REG0 13171 | blt >7 13172 | add TMP1, FCARG1x, REG0 13173 if (!use_prop_guard) { 13174 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 13175 | IF_TYPE TMP2w, IS_UNDEF, >7 13176 } 13177 | mov FCARG1x, TMP1 13178 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13179 } else { 13180 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13181 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13182 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13183 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13184 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13185 13186 if (!exit_addr) { 13187 return 0; 13188 } 13189 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13190 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13191 } else { 13192 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13193 | IF_TYPE TMP2w, IS_UNDEF, >7 13194 needs_slow_path = 1; 13195 } 13196 } 13197 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13198 uint32_t info = val_info; 13199 13200 may_throw = 1; 13201 13202 if (opline) { 13203 | SET_EX_OPLINE opline, REG0 13204 } 13205 13206 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 13207 |.cold_code 13208 |1: 13209 | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 13210 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13211 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13212 } 13213 | LOAD_ADDR CARG3, binary_op 13214 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13215 | b >9 13216 |.code 13217 13218 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13219 13220 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13221 | LOAD_ADDR FCARG2x, prop_info 13222 } else { 13223 int prop_info_offset = 13224 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13225 13226 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13227 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13228 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13229 } 13230 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13231 | LOAD_ZVAL_ADDR CARG3, val_addr 13232 | LOAD_ADDR CARG4, binary_op 13233 13234 | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 13235 13236 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13237 info |= MAY_BE_RC1|MAY_BE_RCN; 13238 } 13239 13240 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13241 } 13242 } 13243 13244 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13245 zend_jit_addr var_addr = prop_addr; 13246 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13247 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13248 13249 if (use_prop_guard) { 13250 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13251 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13252 13253 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 13254 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13255 } 13256 13257 if (var_info & MAY_BE_REF) { 13258 may_throw = 1; 13259 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 13260 | LOAD_ZVAL_ADDR REG0, prop_addr 13261 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 13262 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 13263 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 13264 | cbnz TMP1, >1 13265 | add REG0, FCARG1x, #offsetof(zend_reference, val) 13266 |.cold_code 13267 |1: 13268 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13269 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13270 } 13271 if (opline) { 13272 | SET_EX_OPLINE opline, REG0 13273 } 13274 | LOAD_ADDR CARG3, binary_op 13275 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13276 | b >9 13277 |.code 13278 |2: 13279 } 13280 13281 switch (opline->extended_value) { 13282 case ZEND_ADD: 13283 case ZEND_SUB: 13284 case ZEND_MUL: 13285 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13286 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13287 if (opline->extended_value != ZEND_ADD || 13288 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13289 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13290 may_throw = 1; 13291 } 13292 } 13293 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 13294 1 /* may overflow */, 0)) { 13295 return 0; 13296 } 13297 break; 13298 case ZEND_BW_OR: 13299 case ZEND_BW_AND: 13300 case ZEND_BW_XOR: 13301 may_throw = 1; 13302 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13303 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13304 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 13305 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 13306 may_throw = 1; 13307 } 13308 } 13309 goto long_math; 13310 case ZEND_SL: 13311 case ZEND_SR: 13312 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13313 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13314 may_throw = 1; 13315 } 13316 if ((opline+1)->op1_type != IS_CONST || 13317 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13318 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 13319 may_throw = 1; 13320 } 13321 goto long_math; 13322 case ZEND_MOD: 13323 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13324 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13325 if (opline->extended_value != ZEND_ADD || 13326 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13327 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13328 may_throw = 1; 13329 } 13330 } 13331 if ((opline+1)->op1_type != IS_CONST || 13332 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13333 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 13334 may_throw = 1; 13335 } 13336long_math: 13337 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 13338 IS_CV, opline->op1, var_addr, var_info, NULL, 13339 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 13340 val_range, 13341 0, var_addr, var_def_info, var_info, 0)) { 13342 return 0; 13343 } 13344 break; 13345 case ZEND_CONCAT: 13346 may_throw = 1; 13347 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 13348 0)) { 13349 return 0; 13350 } 13351 break; 13352 default: 13353 ZEND_UNREACHABLE(); 13354 } 13355 } 13356 13357 if (needs_slow_path) { 13358 may_throw = 1; 13359 |.cold_code 13360 |7: 13361 | SET_EX_OPLINE opline, REG0 13362 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13363 | LOAD_ADDR FCARG2x, name 13364 | LOAD_ZVAL_ADDR CARG3, val_addr 13365 | ldr CARG4, EX->run_time_cache 13366 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 13367 | LOAD_ADDR CARG5, binary_op 13368 | EXT_CALL zend_jit_assign_obj_op_helper, REG0 13369 13370 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13371 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13372 } 13373 13374 |8: 13375 | // FREE_OP_DATA(); 13376 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13377 | b >9 13378 |.code 13379 } 13380 13381 |9: 13382 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13383 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13384 may_throw = 1; 13385 } 13386 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13387 } 13388 13389 if (may_throw) { 13390 if (!zend_jit_check_exception(Dst)) { 13391 return 0; 13392 } 13393 } 13394 13395 return 1; 13396} 13397 13398static int zend_jit_assign_obj(dasm_State **Dst, 13399 const zend_op *opline, 13400 const zend_op_array *op_array, 13401 zend_ssa *ssa, 13402 const zend_ssa_op *ssa_op, 13403 uint32_t op1_info, 13404 zend_jit_addr op1_addr, 13405 uint32_t val_info, 13406 bool op1_indirect, 13407 zend_class_entry *ce, 13408 bool ce_is_instanceof, 13409 bool on_this, 13410 bool delayed_fetch_this, 13411 zend_class_entry *trace_ce, 13412 uint8_t prop_type, 13413 int may_throw) 13414{ 13415 zval *member; 13416 zend_string *name; 13417 zend_property_info *prop_info; 13418 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13419 zend_jit_addr res_addr = 0; 13420 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13421 zend_jit_addr prop_addr; 13422 bool needs_slow_path = 0; 13423 13424 if (RETURN_VALUE_USED(opline)) { 13425 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13426 } 13427 13428 ZEND_ASSERT(opline->op2_type == IS_CONST); 13429 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13430 13431 member = RT_CONSTANT(opline, opline->op2); 13432 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13433 name = Z_STR_P(member); 13434 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13435 13436 if (on_this) { 13437 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13438 } else { 13439 if (opline->op1_type == IS_VAR 13440 && (op1_info & MAY_BE_INDIRECT) 13441 && Z_REG(op1_addr) == ZREG_FP) { 13442 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13443 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13444 | GET_Z_PTR FCARG1x, FCARG1x 13445 |1: 13446 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13447 } 13448 if (op1_info & MAY_BE_REF) { 13449 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13450 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13451 } 13452 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13453 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13454 } 13455 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13456 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13457 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13458 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13459 13460 if (!exit_addr) { 13461 return 0; 13462 } 13463 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13464 } else { 13465 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13466 |.cold_code 13467 |1: 13468 | SET_EX_OPLINE opline, REG0 13469 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13470 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13471 } 13472 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13473 | EXT_CALL zend_jit_invalid_property_assign, REG0 13474 if (RETURN_VALUE_USED(opline)) { 13475 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 13476 } 13477 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13478 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13479 | b >7 13480 } else { 13481 | b >9 13482 } 13483 |.code 13484 } 13485 } 13486 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13487 } 13488 13489 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13490 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13491 if (prop_info) { 13492 ce = trace_ce; 13493 ce_is_instanceof = 0; 13494 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13495 if (on_this && JIT_G(current_frame) 13496 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13497 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13498 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13499 if (on_this && JIT_G(current_frame)) { 13500 JIT_G(current_frame)->ce = ce; 13501 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13502 } 13503 } else { 13504 return 0; 13505 } 13506 if (ssa->var_info && ssa_op->op1_use >= 0) { 13507 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13508 ssa->var_info[ssa_op->op1_use].ce = ce; 13509 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13510 } 13511 if (ssa->var_info && ssa_op->op1_def >= 0) { 13512 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13513 ssa->var_info[ssa_op->op1_def].ce = ce; 13514 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13515 } 13516 } 13517 } 13518 } 13519 13520 if (!prop_info) { 13521 needs_slow_path = 1; 13522 13523 | ldr REG0, EX->run_time_cache 13524 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 13525 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 13526 | cmp REG2, TMP1 13527 | bne >5 13528 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13529 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 13530 } 13531 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 13532 | tst REG0, REG0 13533 | blt >5 13534 | add TMP2, FCARG1x, REG0 13535 | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] 13536 | IF_TYPE TMP1w, IS_UNDEF, >5 13537 | mov FCARG1x, TMP2 13538 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13539 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13540 | cbnz FCARG2x, >1 13541 |.cold_code 13542 |1: 13543 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13544 | SET_EX_OPLINE opline, REG0 13545 | LOAD_ZVAL_ADDR CARG3, val_addr 13546 if (RETURN_VALUE_USED(opline)) { 13547 | LOAD_ZVAL_ADDR CARG4, res_addr 13548 } else { 13549 | mov CARG4, xzr 13550 } 13551 13552 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13553 13554 if ((opline+1)->op1_type == IS_CONST) { 13555 | // TODO: ??? 13556 | // if (Z_TYPE_P(value) == orig_type) { 13557 | // CACHE_PTR_EX(cache_slot + 2, NULL); 13558 } 13559 13560 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13561 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13562 | b >7 13563 } else { 13564 | b >9 13565 } 13566 |.code 13567 } 13568 } else { 13569 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13570 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 13571 // Undefined property with magic __get()/__set() 13572 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13573 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13574 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13575 13576 if (!exit_addr) { 13577 return 0; 13578 } 13579 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13580 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13581 } else { 13582 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13583 | IF_TYPE TMP2w, IS_UNDEF, >5 13584 needs_slow_path = 1; 13585 } 13586 } 13587 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13588 uint32_t info = val_info; 13589 13590 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13591 | SET_EX_OPLINE opline, REG0 13592 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13593 | LOAD_ADDR FCARG2x, prop_info 13594 } else { 13595 int prop_info_offset = 13596 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13597 13598 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13599 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13600 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13601 } 13602 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13603 | LOAD_ZVAL_ADDR CARG3, val_addr 13604 if (RETURN_VALUE_USED(opline)) { 13605 | LOAD_ZVAL_ADDR CARG4, res_addr 13606 } else { 13607 | mov CARG4, xzr 13608 } 13609 13610 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13611 13612 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13613 info |= MAY_BE_RC1|MAY_BE_RCN; 13614 } 13615 13616 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13617 } 13618 } 13619 13620 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13621 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 13622 if (opline->result_type == IS_UNUSED) { 13623 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13624 return 0; 13625 } 13626 } else { 13627 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13628 return 0; 13629 } 13630 } 13631 } 13632 13633 if (needs_slow_path) { 13634 |.cold_code 13635 |5: 13636 | SET_EX_OPLINE opline, REG0 13637 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13638 | LOAD_ADDR FCARG2x, name 13639 13640 | LOAD_ZVAL_ADDR CARG3, val_addr 13641 | ldr CARG4, EX->run_time_cache 13642 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 13643 if (RETURN_VALUE_USED(opline)) { 13644 | LOAD_ZVAL_ADDR CARG5, res_addr 13645 } else { 13646 | mov CARG5, xzr 13647 } 13648 13649 | EXT_CALL zend_jit_assign_obj_helper, REG0 13650 13651 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13652 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13653 } 13654 13655 |7: 13656 | // FREE_OP_DATA(); 13657 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13658 | b >9 13659 |.code 13660 } 13661 13662 |9: 13663 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13664 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13665 } 13666 13667 if (may_throw) { 13668 if (!zend_jit_check_exception(Dst)) { 13669 return 0; 13670 } 13671 } 13672 13673 return 1; 13674} 13675 13676static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 13677{ 13678 zend_jit_addr op1_addr = OP1_ADDR(); 13679 13680 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 13681 if (may_throw) { 13682 | SET_EX_OPLINE opline, REG0 13683 } 13684 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 13685 if (op1_info & MAY_BE_ARRAY) { 13686 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 13687 } 13688 | MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 13689 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 13690 | cmp FCARG1w, TMP1w 13691 | beq >7 13692 | EXT_CALL zend_hash_iterator_del, REG0 13693 |7: 13694 } 13695 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13696 if (may_throw) { 13697 if (!zend_jit_check_exception(Dst)) { 13698 return 0; 13699 } 13700 } 13701 } 13702 return 1; 13703} 13704 13705static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 13706{ 13707 if (opline->op1_type == IS_CONST) { 13708 zval *zv; 13709 size_t len; 13710 13711 zv = RT_CONSTANT(opline, opline->op1); 13712 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13713 len = Z_STRLEN_P(zv); 13714 13715 if (len > 0) { 13716 const char *str = Z_STRVAL_P(zv); 13717 13718 | SET_EX_OPLINE opline, REG0 13719 | LOAD_ADDR CARG1, str 13720 | LOAD_64BIT_VAL CARG2, len 13721 | EXT_CALL zend_write, REG0 13722 if (!zend_jit_check_exception(Dst)) { 13723 return 0; 13724 } 13725 } 13726 } else { 13727 zend_jit_addr op1_addr = OP1_ADDR(); 13728 13729 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13730 13731 | SET_EX_OPLINE opline, REG0 13732 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13733 | add CARG1, REG0, #offsetof(zend_string, val) 13734 | ldr CARG2, [REG0, #offsetof(zend_string, len)] 13735 | EXT_CALL zend_write, REG0 13736 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 13737 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13738 } 13739 if (!zend_jit_check_exception(Dst)) { 13740 return 0; 13741 } 13742 } 13743 return 1; 13744} 13745 13746static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 13747{ 13748 if (opline->op1_type == IS_CONST) { 13749 zval *zv; 13750 size_t len; 13751 13752 zv = RT_CONSTANT(opline, opline->op1); 13753 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13754 len = Z_STRLEN_P(zv); 13755 13756 | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 13757 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13758 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13759 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13760 return 0; 13761 } 13762 } else { 13763 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13764 13765 if (Z_MODE(res_addr) == IS_REG) { 13766 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13767 | ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)] 13768 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13769 return 0; 13770 } 13771 } else { 13772 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13773 | ldr REG0, [REG0, #offsetof(zend_string, len)] 13774 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13775 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13776 } 13777 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13778 } 13779 return 1; 13780} 13781 13782static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 13783{ 13784 if (opline->op1_type == IS_CONST) { 13785 zval *zv; 13786 zend_long count; 13787 13788 zv = RT_CONSTANT(opline, opline->op1); 13789 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 13790 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 13791 13792 | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 13793 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13794 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13795 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13796 return 0; 13797 } 13798 } else { 13799 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 13800 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 13801 13802 if (Z_MODE(res_addr) == IS_REG) { 13803 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13804 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13805 | ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)] 13806 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13807 return 0; 13808 } 13809 } else { 13810 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13811 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13812 | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] 13813 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13814 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13815 } 13816 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13817 } 13818 13819 if (may_throw) { 13820 return zend_jit_check_exception(Dst); 13821 } 13822 return 1; 13823} 13824 13825static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 13826{ 13827 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 13828 13829 | ldr FCARG1x, EX->This.value.ptr 13830 | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 13831 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 13832 | GC_ADDREF FCARG1x, TMP1w 13833 return 1; 13834} 13835 13836static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 13837{ 13838 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 13839 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13840 if (!JIT_G(current_frame) || 13841 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 13842 13843 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13844 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13845 13846 if (!exit_addr) { 13847 return 0; 13848 } 13849 13850 | ldrb TMP1w, EX->This.u1.v.type 13851 | cmp TMP1w, #IS_OBJECT 13852 | bne &exit_addr 13853 13854 if (JIT_G(current_frame)) { 13855 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 13856 } 13857 } 13858 } else { 13859 13860 | ldrb TMP1w, EX->This.u1.v.type 13861 | cmp TMP1w, #IS_OBJECT 13862 | bne >1 13863 |.cold_code 13864 |1: 13865 | SET_EX_OPLINE opline, REG0 13866 | b ->invalid_this 13867 |.code 13868 } 13869 } 13870 13871 if (!check_only) { 13872 if (!zend_jit_load_this(Dst, opline->result.var)) { 13873 return 0; 13874 } 13875 } 13876 return 1; 13877} 13878 13879static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 13880{ 13881 uint32_t count; 13882 Bucket *p; 13883 const zend_op *target; 13884 int b; 13885 int32_t exit_point; 13886 const void *exit_addr; 13887 13888 if (default_label) { 13889 | cbz REG0, &default_label 13890 } else if (next_opline) { 13891 | cbz REG0, >3 13892 } else { 13893 | cbz REG0, =>default_b 13894 } 13895 | LOAD_ADDR FCARG1x, jumptable 13896 | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] 13897 | sub REG0, REG0, TMP1 13898 | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) 13899 | sdiv REG0, REG0, FCARG1x 13900 | adr FCARG1x, >4 13901 | ldr TMP1, [FCARG1x, REG0] 13902 | br TMP1 13903 13904 |.jmp_table 13905 |.align 8 13906 |4: 13907 if (trace_info) { 13908 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 13909 } 13910 13911 count = jumptable->nNumUsed; 13912 p = jumptable->arData; 13913 do { 13914 if (Z_TYPE(p->val) == IS_UNDEF) { 13915 if (default_label) { 13916 | .addr &default_label 13917 } else if (next_opline) { 13918 | .addr >3 13919 } else { 13920 | .addr =>default_b 13921 } 13922 } else { 13923 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 13924 if (!next_opline) { 13925 b = ssa->cfg.map[target - op_array->opcodes]; 13926 | .addr =>b 13927 } else if (next_opline == target) { 13928 | .addr >3 13929 } else { 13930 exit_point = zend_jit_trace_get_exit_point(target, 0); 13931 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13932 | .addr &exit_addr 13933 } 13934 } 13935 p++; 13936 count--; 13937 } while (count); 13938 |.code 13939 13940 return 1; 13941} 13942 13943static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 13944{ 13945 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 13946 const zend_op *next_opline = NULL; 13947 13948 if (trace) { 13949 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 13950 ZEND_ASSERT(trace->opline != NULL); 13951 next_opline = trace->opline; 13952 } 13953 13954 if (opline->op1_type == IS_CONST) { 13955 zval *zv = RT_CONSTANT(opline, opline->op1); 13956 zval *jump_zv = NULL; 13957 int b; 13958 13959 if (opline->opcode == ZEND_SWITCH_LONG) { 13960 if (Z_TYPE_P(zv) == IS_LONG) { 13961 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 13962 } 13963 } else if (opline->opcode == ZEND_SWITCH_STRING) { 13964 if (Z_TYPE_P(zv) == IS_STRING) { 13965 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 13966 } 13967 } else if (opline->opcode == ZEND_MATCH) { 13968 if (Z_TYPE_P(zv) == IS_LONG) { 13969 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 13970 } else if (Z_TYPE_P(zv) == IS_STRING) { 13971 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 13972 } 13973 } else { 13974 ZEND_UNREACHABLE(); 13975 } 13976 if (next_opline) { 13977 const zend_op *target; 13978 13979 if (jump_zv != NULL) { 13980 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 13981 } else { 13982 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 13983 } 13984 ZEND_ASSERT(target == next_opline); 13985 } else { 13986 if (jump_zv != NULL) { 13987 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 13988 } else { 13989 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 13990 } 13991 | b =>b 13992 } 13993 } else { 13994 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 13995 uint32_t op1_info = OP1_INFO(); 13996 zend_jit_addr op1_addr = OP1_ADDR(); 13997 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 13998 const zend_op *target; 13999 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14000 int b; 14001 int32_t exit_point; 14002 const void *fallback_label = NULL; 14003 const void *default_label = NULL; 14004 const void *exit_addr; 14005 14006 if (next_opline) { 14007 if (next_opline != opline + 1) { 14008 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14009 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14010 } 14011 if (next_opline != default_opline) { 14012 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14013 default_label = zend_jit_trace_get_exit_addr(exit_point); 14014 } 14015 } 14016 14017 if (opline->opcode == ZEND_SWITCH_LONG) { 14018 if (op1_info & MAY_BE_LONG) { 14019 if (op1_info & MAY_BE_REF) { 14020 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 14021 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14022 |.cold_code 14023 |1: 14024 | // ZVAL_DEREF(op) 14025 if (fallback_label) { 14026 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14027 } else { 14028 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14029 } 14030 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14031 if (fallback_label) { 14032 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14033 | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w 14034 } else { 14035 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14036 | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w 14037 } 14038 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] 14039 | b >2 14040 |.code 14041 |2: 14042 } else { 14043 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14044 if (fallback_label) { 14045 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 14046 } else { 14047 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14048 } 14049 } 14050 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14051 } 14052 if (HT_IS_PACKED(jumptable)) { 14053 uint32_t count = jumptable->nNumUsed; 14054 Bucket *p = jumptable->arData; 14055 14056 | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 14057 if (default_label) { 14058 | bhs &default_label 14059 } else if (next_opline) { 14060 | bhs >3 14061 } else { 14062 | bhs =>default_b 14063 } 14064 | adr REG0, >4 14065 | ldr TMP1, [REG0, FCARG2x, lsl #3] 14066 | br TMP1 14067 14068 |.jmp_table 14069 |.align 8 14070 |4: 14071 if (trace_info) { 14072 trace_info->jmp_table_size += count; 14073 } 14074 p = jumptable->arData; 14075 do { 14076 if (Z_TYPE(p->val) == IS_UNDEF) { 14077 if (default_label) { 14078 | .addr &default_label 14079 } else if (next_opline) { 14080 | .addr >3 14081 } else { 14082 | .addr =>default_b 14083 } 14084 } else { 14085 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14086 if (!next_opline) { 14087 b = ssa->cfg.map[target - op_array->opcodes]; 14088 | .addr =>b 14089 } else if (next_opline == target) { 14090 | .addr >3 14091 } else { 14092 exit_point = zend_jit_trace_get_exit_point(target, 0); 14093 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14094 | .addr &exit_addr 14095 } 14096 } 14097 p++; 14098 count--; 14099 } while (count); 14100 |.code 14101 |3: 14102 } else { 14103 | LOAD_ADDR FCARG1x, jumptable 14104 | EXT_CALL zend_hash_index_find, REG0 14105 | mov REG0, RETVALx 14106 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14107 return 0; 14108 } 14109 |3: 14110 } 14111 } 14112 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14113 if (op1_info & MAY_BE_STRING) { 14114 if (op1_info & MAY_BE_REF) { 14115 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 14116 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14117 |.cold_code 14118 |1: 14119 | // ZVAL_DEREF(op) 14120 if (fallback_label) { 14121 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14122 } else { 14123 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14124 } 14125 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14126 if (fallback_label) { 14127 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14128 | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w 14129 } else { 14130 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14131 | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w 14132 } 14133 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] 14134 | b >2 14135 |.code 14136 |2: 14137 } else { 14138 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14139 if (fallback_label) { 14140 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 14141 } else { 14142 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14143 } 14144 } 14145 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14146 } 14147 | LOAD_ADDR FCARG1x, jumptable 14148 | EXT_CALL zend_hash_find, REG0 14149 | mov REG0, RETVALx 14150 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14151 return 0; 14152 } 14153 |3: 14154 } 14155 } else if (opline->opcode == ZEND_MATCH) { 14156 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14157 if (op1_info & MAY_BE_REF) { 14158 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 14159 | ZVAL_DEREF FCARG2x, op1_info, TMP1w 14160 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14161 } 14162 | LOAD_ADDR FCARG1x, jumptable 14163 if (op1_info & MAY_BE_LONG) { 14164 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14165 if (op1_info & MAY_BE_STRING) { 14166 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 14167 } else if (op1_info & MAY_BE_UNDEF) { 14168 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 14169 } else if (default_label) { 14170 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 14171 } else if (next_opline) { 14172 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14173 } else { 14174 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 14175 } 14176 } 14177 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14178 | EXT_CALL zend_hash_index_find, REG0 14179 | mov REG0, RETVALx 14180 if (op1_info & MAY_BE_STRING) { 14181 | b >2 14182 } 14183 } 14184 if (op1_info & MAY_BE_STRING) { 14185 |5: 14186 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 14187 if (op1_info & MAY_BE_UNDEF) { 14188 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 14189 } else if (default_label) { 14190 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 14191 } else if (next_opline) { 14192 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14193 } else { 14194 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 14195 } 14196 } 14197 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14198 | EXT_CALL zend_hash_find, REG0 14199 | mov REG0, RETVALx 14200 } 14201 |2: 14202 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14203 return 0; 14204 } 14205 } 14206 if (op1_info & MAY_BE_UNDEF) { 14207 |6: 14208 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 14209 if (default_label) { 14210 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 14211 } else if (next_opline) { 14212 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 14213 } else { 14214 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 14215 } 14216 } 14217 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 14218 | SET_EX_OPLINE opline, REG0 14219 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 14220 | EXT_CALL zend_jit_undefined_op_helper, REG0 14221 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 14222 return 0; 14223 } 14224 } 14225 if (default_label) { 14226 | b &default_label 14227 } else if (next_opline) { 14228 | b >3 14229 } else { 14230 | b =>default_b 14231 } 14232 |3: 14233 } else { 14234 ZEND_UNREACHABLE(); 14235 } 14236 } 14237 return 1; 14238} 14239 14240static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 14241{ 14242 zend_arg_info *arg_info = &op_array->arg_info[-1]; 14243 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 14244 zend_jit_addr op1_addr = OP1_ADDR(); 14245 bool needs_slow_check = 1; 14246 bool slow_check_in_cold = 1; 14247 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 14248 14249 if (type_mask == 0) { 14250 slow_check_in_cold = 0; 14251 } else { 14252 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 14253 slow_check_in_cold = 0; 14254 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 14255 needs_slow_check = 0; 14256 } else if (is_power_of_two(type_mask)) { 14257 uint32_t type_code = concrete_type(type_mask); 14258 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1 14259 } else { 14260 | mov REG2w, #1 14261 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 14262 | lsl REG2w, REG2w, REG1w 14263 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 14264 | beq >6 14265 } 14266 } 14267 if (needs_slow_check) { 14268 if (slow_check_in_cold) { 14269 |.cold_code 14270 |6: 14271 } 14272 | SET_EX_OPLINE opline, REG1 14273 if (op1_info & MAY_BE_UNDEF) { 14274 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1 14275 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 14276 | EXT_CALL zend_jit_undefined_op_helper, REG0 14277 | cbz RETVALx, ->exception_handler 14278 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 14279 | b >8 14280 } 14281 |7: 14282 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14283 |8: 14284 | ldr FCARG2x, EX->func 14285 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 14286 | ldr REG0, EX->run_time_cache 14287 | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 14288 | EXT_CALL zend_jit_verify_return_slow, REG0 14289 if (!zend_jit_check_exception(Dst)) { 14290 return 0; 14291 } 14292 if (slow_check_in_cold) { 14293 | b >9 14294 |.code 14295 } 14296 } 14297 |9: 14298 return 1; 14299} 14300 14301static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14302{ 14303 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14304 14305 // TODO: support for empty() ??? 14306 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 14307 14308 if (op1_info & MAY_BE_REF) { 14309 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14310 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14311 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14312 } 14313 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 14314 |1: 14315 } 14316 14317 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 14318 if (exit_addr) { 14319 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 14320 } else if (smart_branch_opcode) { 14321 if (smart_branch_opcode == ZEND_JMPNZ) { 14322 | b =>target_label 14323 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14324 | b =>target_label2 14325 } 14326 } else { 14327 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 14328 } 14329 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 14330 if (exit_addr) { 14331 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 14332 } else if (smart_branch_opcode) { 14333 if (smart_branch_opcode != ZEND_JMPNZ) { 14334 | b =>target_label 14335 } 14336 } else { 14337 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 14338 } 14339 } else { 14340 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 14341 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 14342 | cmp TMP1w, #IS_NULL 14343 if (exit_addr) { 14344 if (smart_branch_opcode == ZEND_JMPNZ) { 14345 | bgt &exit_addr 14346 } else { 14347 | ble &exit_addr 14348 } 14349 } else if (smart_branch_opcode) { 14350 if (smart_branch_opcode == ZEND_JMPZ) { 14351 | ble =>target_label 14352 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14353 | bgt =>target_label 14354 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14355 | ble =>target_label 14356 | b =>target_label2 14357 } else { 14358 ZEND_UNREACHABLE(); 14359 } 14360 } else { 14361 | cset REG0w, gt 14362 | add REG0w, REG0w, #IS_FALSE 14363 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14364 } 14365 } 14366 14367 return 1; 14368} 14369 14370static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14371{ 14372 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14373 14374 if (opline->op1_type == IS_CONST) { 14375 zval *zv = RT_CONSTANT(opline, opline->op1); 14376 14377 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 14378 if (Z_REFCOUNTED_P(zv)) { 14379 | ADDREF_CONST zv, REG0, TMP1 14380 } 14381 } else { 14382 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14383 14384 | // ZVAL_COPY(res, value); 14385 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14386 if (opline->op1_type == IS_CV) { 14387 | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w 14388 } 14389 } 14390 | // Z_FE_POS_P(res) = 0; 14391 | MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 14392 14393 return 1; 14394} 14395 14396static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 14397{ 14398 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14399 14400 | // array = EX_VAR(opline->op1.var); 14401 | // fe_ht = Z_ARRVAL_P(array); 14402 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 14403 | // pos = Z_FE_POS_P(array); 14404 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14405 | // p = fe_ht->arData + pos; 14406 || ZEND_ASSERT(sizeof(Bucket) == 32); 14407 | mov FCARG2w, REG0w 14408 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 14409 | add FCARG2x, TMP1, FCARG2x, lsl #5 14410 |1: 14411 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14412 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14413 | cmp TMP1w, REG0w 14414 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14415 | // ZEND_VM_CONTINUE(); 14416 if (exit_addr) { 14417 if (exit_opcode == ZEND_JMP) { 14418 | bls &exit_addr 14419 } else { 14420 | bls >3 14421 } 14422 } else { 14423 | bls =>target_label 14424 } 14425 | // pos++; 14426 | add REG0w, REG0w, #1 14427 | // value_type = Z_TYPE_INFO_P(value); 14428 | // if (EXPECTED(value_type != IS_UNDEF)) { 14429 if (!exit_addr || exit_opcode == ZEND_JMP) { 14430 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w 14431 } else { 14432 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14433 } 14434 | // p++; 14435 | add FCARG2x, FCARG2x, #sizeof(Bucket) 14436 | b <1 14437 |3: 14438 14439 if (!exit_addr || exit_opcode == ZEND_JMP) { 14440 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14441 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 14442 uint32_t val_info; 14443 14444 | // Z_FE_POS_P(array) = pos + 1; 14445 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14446 14447 if (RETURN_VALUE_USED(opline)) { 14448 zend_jit_addr res_addr = RES_ADDR(); 14449 14450 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 14451 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 14452 | // if (!p->key) { 14453 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14454 | cbz REG0, >2 14455 } 14456 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 14457 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 14458 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14459 | SET_ZVAL_PTR res_addr, REG0, TMP1 14460 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] 14461 | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w 14462 | beq >1 14463 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 14464 | b >3 14465 |1: 14466 | GC_ADDREF REG0, TMP1w 14467 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14468 14469 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14470 | b >3 14471 |2: 14472 } 14473 } 14474 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14475 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 14476 | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] 14477 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14478 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14479 } 14480 |3: 14481 } 14482 14483 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 14484 if (val_info & MAY_BE_ARRAY) { 14485 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14486 } 14487 if (op1_info & MAY_BE_ARRAY_OF_REF) { 14488 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 14489 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14490 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14491 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 14492 } 14493 14494 if (opline->op2_type == IS_CV) { 14495 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 14496 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 14497 return 0; 14498 } 14499 } else { 14500 | // ZVAL_COPY(res, value); 14501 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14502 | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w 14503 } 14504 } 14505 14506 return 1; 14507} 14508 14509static int zend_jit_fetch_constant(dasm_State **Dst, 14510 const zend_op *opline, 14511 const zend_op_array *op_array, 14512 zend_ssa *ssa, 14513 const zend_ssa_op *ssa_op, 14514 zend_jit_addr res_addr) 14515{ 14516 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 14517 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 14518 uint32_t res_info = RES_INFO(); 14519 14520 | // c = CACHED_PTR(opline->extended_value); 14521 | ldr FCARG1x, EX->run_time_cache 14522 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 14523 | // if (c != NULL) 14524 | cbz REG0, >9 14525 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 14526 | // if (!IS_SPECIAL_CACHE_VAL(c)) 14527 || ZEND_ASSERT(CACHE_SPECIAL == 1); 14528 | TST_64_WITH_ONE REG0 14529 | bne >9 14530 } 14531 |8: 14532 14533 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 14534 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 14535 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 14536 int32_t exit_point; 14537 const void *exit_addr = NULL; 14538 14539 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 14540 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 14541 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 14542 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 14543 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14544 if (!exit_addr) { 14545 return 0; 14546 } 14547 res_info &= ~MAY_BE_GUARD; 14548 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 14549 14550 uint32_t type = concrete_type(res_info); 14551 14552 if (type < IS_STRING) { 14553 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 14554 } else { 14555 | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 14556 | IF_NOT_TYPE REG2w, type, &exit_addr 14557 } 14558 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 14559 if (type < IS_STRING) { 14560 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14561 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 14562 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 14563 return 0; 14564 } 14565 } else { 14566 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 14567 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 14568 } 14569 } else { 14570 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14571 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w 14572 } 14573 14574 |.cold_code 14575 |9: 14576 | // SAVE_OPLINE(); 14577 | SET_EX_OPLINE opline, REG0 14578 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 14579 | LOAD_ADDR FCARG1x, zv 14580 | LOAD_32BIT_VAL FCARG2w, opline->op1.num 14581 | EXT_CALL zend_jit_get_constant, REG0 14582 | mov REG0, RETVALx 14583 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 14584 | cbnz REG0, <8 14585 | b ->exception_handler 14586 |.code 14587 return 1; 14588} 14589 14590static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14591{ 14592 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14593 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14594 14595 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 14596 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 14597 14598 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 14599 | LOAD_ADDR FCARG1x, ht 14600 if (opline->op1_type != IS_CONST) { 14601 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14602 | EXT_CALL zend_hash_find, REG0 14603 } else { 14604 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 14605 | LOAD_ADDR FCARG2x, str 14606 | EXT_CALL zend_hash_find_known_hash, REG0 14607 } 14608 if (exit_addr) { 14609 if (smart_branch_opcode == ZEND_JMPZ) { 14610 | cbz RETVALx, &exit_addr 14611 } else { 14612 | cbnz RETVALx, &exit_addr 14613 } 14614 } else if (smart_branch_opcode) { 14615 if (smart_branch_opcode == ZEND_JMPZ) { 14616 | cbz RETVALx, =>target_label 14617 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14618 | cbnz RETVALx, =>target_label 14619 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14620 | cbz RETVALx, =>target_label 14621 | b =>target_label2 14622 } else { 14623 ZEND_UNREACHABLE(); 14624 } 14625 } else { 14626 | tst RETVALx, RETVALx 14627 | cset REG0w, ne 14628 | add REG0w, REG0w, #IS_FALSE 14629 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14630 } 14631 14632 return 1; 14633} 14634 14635static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 14636{ 14637 uint32_t offset; 14638 14639 offset = (opline->opcode == ZEND_ROPE_INIT) ? 14640 opline->result.var : 14641 opline->op1.var + opline->extended_value * sizeof(zend_string*); 14642 14643 if (opline->op2_type == IS_CONST) { 14644 zval *zv = RT_CONSTANT(opline, opline->op2); 14645 zend_string *str; 14646 14647 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14648 str = Z_STR_P(zv); 14649 | LOAD_ADDR REG0, str 14650 | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 14651 } else { 14652 zend_jit_addr op2_addr = OP2_ADDR(); 14653 14654 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14655 14656 | GET_ZVAL_PTR REG1, op2_addr, TMP1 14657 | MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1 14658 if (opline->op2_type == IS_CV) { 14659 | GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1 14660 | TRY_ADDREF op2_info, REG0w, REG1, TMP1w 14661 } 14662 } 14663 14664 if (opline->opcode == ZEND_ROPE_END) { 14665 zend_jit_addr res_addr = RES_ADDR(); 14666 14667 | ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1 14668 | LOAD_32BIT_VAL FCARG2w, opline->extended_value 14669 | EXT_CALL zend_jit_rope_end, TMP1 14670 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 14671 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14672 } 14673 14674 return 1; 14675} 14676 14677static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 14678{ 14679 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14680 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14681 14682 if (!exit_addr) { 14683 return 0; 14684 } 14685 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14686 14687 return 1; 14688} 14689 14690static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 14691{ 14692 zend_jit_addr var_addr = *var_addr_ptr; 14693 uint32_t var_info = *var_info_ptr; 14694 const void *exit_addr = NULL; 14695 14696 if (add_ref_guard || add_type_guard) { 14697 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14698 14699 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14700 if (!exit_addr) { 14701 return 0; 14702 } 14703 } 14704 14705 if (add_ref_guard) { 14706 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14707 } 14708 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 14709 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 14710 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 14711 | LOAD_ZVAL_ADDR FCARG1x, var_addr 14712 } 14713 | EXT_CALL zend_jit_unref_helper, REG0 14714 } else { 14715 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14716 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 14717 *var_addr_ptr = var_addr; 14718 } 14719 14720 if (var_type != IS_UNKNOWN) { 14721 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14722 } 14723 if (add_type_guard 14724 && var_type != IS_UNKNOWN 14725 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14726 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 14727 14728 ZEND_ASSERT(var_info & (1 << var_type)); 14729 if (var_type < IS_STRING) { 14730 var_info = (1 << var_type); 14731 } else if (var_type != IS_ARRAY) { 14732 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14733 } else { 14734 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14735 } 14736 14737 *var_info_ptr = var_info; 14738 } else { 14739 var_info &= ~MAY_BE_REF; 14740 *var_info_ptr = var_info; 14741 } 14742 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 14743 14744 return 1; 14745} 14746 14747static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 14748{ 14749 zend_jit_addr var_addr = *var_addr_ptr; 14750 uint32_t var_info = *var_info_ptr; 14751 int32_t exit_point; 14752 const void *exit_addr; 14753 14754 if (add_indirect_guard) { 14755 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14756 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14757 14758 if (!exit_addr) { 14759 return 0; 14760 } 14761 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 14762 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14763 } else { 14764 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 14765 if (opline->op1_type != IS_VAR || 14766 (opline-1)->result_type != IS_VAR || 14767 (opline-1)->result.var != opline->op1.var || 14768 (opline-1)->op2_type == IS_VAR || 14769 (opline-1)->op2_type == IS_TMP_VAR) { 14770 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14771 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 14772 | mov FCARG1x, REG0 14773 } 14774 } 14775 *var_info_ptr &= ~MAY_BE_INDIRECT; 14776 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14777 *var_addr_ptr = var_addr; 14778 14779 if (var_type != IS_UNKNOWN) { 14780 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14781 } 14782 if (!(var_type & IS_TRACE_REFERENCE) 14783 && var_type != IS_UNKNOWN 14784 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14785 exit_point = zend_jit_trace_get_exit_point(opline, 0); 14786 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14787 14788 if (!exit_addr) { 14789 return 0; 14790 } 14791 14792 | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w 14793 14794 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 14795 ZEND_ASSERT(var_info & (1 << var_type)); 14796 if (var_type < IS_STRING) { 14797 var_info = (1 << var_type); 14798 } else if (var_type != IS_ARRAY) { 14799 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14800 } else { 14801 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14802 } 14803 14804 *var_info_ptr = var_info; 14805 } 14806 14807 return 1; 14808} 14809 14810static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 14811{ 14812 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 14813 return 0; 14814 } 14815 14816 switch (opline->opcode) { 14817 case ZEND_QM_ASSIGN: 14818 case ZEND_SEND_VAR: 14819 case ZEND_ASSIGN: 14820 case ZEND_PRE_INC: 14821 case ZEND_PRE_DEC: 14822 case ZEND_POST_INC: 14823 case ZEND_POST_DEC: 14824 return 1; 14825 case ZEND_ADD: 14826 case ZEND_SUB: 14827 case ZEND_MUL: 14828 case ZEND_BW_OR: 14829 case ZEND_BW_AND: 14830 case ZEND_BW_XOR: 14831 case ZEND_SL: 14832 case ZEND_SR: 14833 if (def_var == ssa_op->result_def && 14834 use_var == ssa_op->op1_use) { 14835 return 1; 14836 } 14837 break; 14838 default: 14839 break; 14840 } 14841 return 0; 14842} 14843 14844static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 14845{ 14846 uint32_t op1_info, op2_info; 14847 14848 switch (opline->opcode) { 14849 case ZEND_SEND_VAR: 14850 case ZEND_SEND_VAL: 14851 case ZEND_SEND_VAL_EX: 14852 return (opline->op2_type != IS_CONST); 14853 case ZEND_QM_ASSIGN: 14854 case ZEND_IS_SMALLER: 14855 case ZEND_IS_SMALLER_OR_EQUAL: 14856 case ZEND_IS_EQUAL: 14857 case ZEND_IS_NOT_EQUAL: 14858 case ZEND_IS_IDENTICAL: 14859 case ZEND_IS_NOT_IDENTICAL: 14860 case ZEND_CASE: 14861 return 1; 14862 case ZEND_RETURN: 14863 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 14864 case ZEND_ASSIGN: 14865 op1_info = OP1_INFO(); 14866 op2_info = OP2_INFO(); 14867 return 14868 opline->op1_type == IS_CV && 14869 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 14870 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 14871 case ZEND_ADD: 14872 case ZEND_SUB: 14873 case ZEND_MUL: 14874 op1_info = OP1_INFO(); 14875 op2_info = OP2_INFO(); 14876 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 14877 case ZEND_BW_OR: 14878 case ZEND_BW_AND: 14879 case ZEND_BW_XOR: 14880 case ZEND_SL: 14881 case ZEND_SR: 14882 case ZEND_MOD: 14883 op1_info = OP1_INFO(); 14884 op2_info = OP2_INFO(); 14885 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 14886 case ZEND_PRE_INC: 14887 case ZEND_PRE_DEC: 14888 case ZEND_POST_INC: 14889 case ZEND_POST_DEC: 14890 op1_info = OP1_INFO(); 14891 op2_info = OP1_DEF_INFO(); 14892 return opline->op1_type == IS_CV 14893 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 14894 && (op2_info & MAY_BE_LONG); 14895 case ZEND_STRLEN: 14896 op1_info = OP1_INFO(); 14897 return (opline->op1_type & (IS_CV|IS_CONST)) 14898 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 14899 case ZEND_COUNT: 14900 op1_info = OP1_INFO(); 14901 return (opline->op1_type & (IS_CV|IS_CONST)) 14902 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 14903 case ZEND_JMPZ: 14904 case ZEND_JMPNZ: 14905 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 14906 if (!ssa->cfg.map) { 14907 return 0; 14908 } 14909 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 14910 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 14911 return 0; 14912 } 14913 } 14914 ZEND_FALLTHROUGH; 14915 case ZEND_BOOL: 14916 case ZEND_BOOL_NOT: 14917 case ZEND_JMPZNZ: 14918 case ZEND_JMPZ_EX: 14919 case ZEND_JMPNZ_EX: 14920 return 1; 14921 case ZEND_FETCH_CONSTANT: 14922 return 1; 14923 case ZEND_FETCH_DIM_R: 14924 op1_info = OP1_INFO(); 14925 op2_info = OP2_INFO(); 14926 if (trace 14927 && trace->op1_type != IS_UNKNOWN 14928 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 14929 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 14930 } 14931 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 14932 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 14933 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 14934 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 14935 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 14936 } 14937 return 0; 14938} 14939 14940static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 14941{ 14942 if (ssa->vars[var].no_val) { 14943 /* we don't need the value */ 14944 return 0; 14945 } 14946 14947 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 14948 /* Disable global register allocation, 14949 * register allocation for SSA variables connected through Phi functions 14950 */ 14951 if (ssa->vars[var].definition_phi) { 14952 return 0; 14953 } 14954 if (ssa->vars[var].phi_use_chain) { 14955 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 14956 do { 14957 if (!ssa->vars[phi->ssa_var].no_val) { 14958 return 0; 14959 } 14960 phi = zend_ssa_next_use_phi(ssa, var, phi); 14961 } while (phi); 14962 } 14963 } 14964 14965 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 14966 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 14967 /* bad type */ 14968 return 0; 14969 } 14970 14971 return 1; 14972} 14973 14974static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 14975{ 14976 if (!zend_jit_var_supports_reg(ssa, var)) { 14977 return 0; 14978 } 14979 14980 if (ssa->vars[var].definition >= 0) { 14981 uint32_t def = ssa->vars[var].definition; 14982 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 14983 return 0; 14984 } 14985 } 14986 14987 if (ssa->vars[var].use_chain >= 0) { 14988 int use = ssa->vars[var].use_chain; 14989 14990 do { 14991 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 14992 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 14993 return 0; 14994 } 14995 use = zend_ssa_next_use(ssa->ops, var, use); 14996 } while (use >= 0); 14997 } 14998 14999 return 1; 15000} 15001 15002static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15003{ 15004 uint32_t op1_info, op2_info; 15005 15006 switch (opline->opcode) { 15007 case ZEND_FETCH_DIM_R: 15008 op1_info = OP1_INFO(); 15009 op2_info = OP2_INFO(); 15010 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15011 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15012 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15013 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15014 return ZEND_REGSET(ZREG_FCARG1); 15015 } 15016 break; 15017 default: 15018 break; 15019 } 15020 15021 return ZEND_REGSET_EMPTY; 15022} 15023 15024static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15025{ 15026 uint32_t op1_info, op2_info, res_info; 15027 zend_regset regset = ZEND_REGSET_SCRATCH; 15028 15029 switch (opline->opcode) { 15030 case ZEND_NOP: 15031 case ZEND_OP_DATA: 15032 case ZEND_JMP: 15033 case ZEND_RETURN: 15034 regset = ZEND_REGSET_EMPTY; 15035 break; 15036 case ZEND_QM_ASSIGN: 15037 if (ssa_op->op1_def == current_var || 15038 ssa_op->result_def == current_var) { 15039 regset = ZEND_REGSET_EMPTY; 15040 break; 15041 } 15042 /* break missing intentionally */ 15043 case ZEND_SEND_VAL: 15044 case ZEND_SEND_VAL_EX: 15045 if (opline->op2_type == IS_CONST) { 15046 break; 15047 } 15048 if (ssa_op->op1_use == current_var) { 15049 regset = ZEND_REGSET(ZREG_REG0); 15050 break; 15051 } 15052 op1_info = OP1_INFO(); 15053 if (!(op1_info & MAY_BE_UNDEF)) { 15054 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15055 regset = ZEND_REGSET(ZREG_FPR0); 15056 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15057 regset = ZEND_REGSET(ZREG_REG0); 15058 } else { 15059 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15060 } 15061 } 15062 break; 15063 case ZEND_SEND_VAR: 15064 if (opline->op2_type == IS_CONST) { 15065 break; 15066 } 15067 if (ssa_op->op1_use == current_var || 15068 ssa_op->op1_def == current_var) { 15069 regset = ZEND_REGSET_EMPTY; 15070 break; 15071 } 15072 op1_info = OP1_INFO(); 15073 if (!(op1_info & MAY_BE_UNDEF)) { 15074 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15075 regset = ZEND_REGSET(ZREG_FPR0); 15076 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15077 } else { 15078 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15079 if (op1_info & MAY_BE_REF) { 15080 ZEND_REGSET_INCL(regset, ZREG_REG1); 15081 } 15082 } 15083 } 15084 break; 15085 case ZEND_ASSIGN: 15086 if (ssa_op->op2_use == current_var || 15087 ssa_op->op2_def == current_var || 15088 ssa_op->op1_def == current_var || 15089 ssa_op->result_def == current_var) { 15090 regset = ZEND_REGSET_EMPTY; 15091 break; 15092 } 15093 op1_info = OP1_INFO(); 15094 op2_info = OP2_INFO(); 15095 if (opline->op1_type == IS_CV 15096 && !(op2_info & MAY_BE_UNDEF) 15097 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15098 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15099 regset = ZEND_REGSET(ZREG_FPR0); 15100 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15101 regset = ZEND_REGSET(ZREG_REG0); 15102 } else { 15103 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15104 } 15105 } 15106 break; 15107 case ZEND_PRE_INC: 15108 case ZEND_PRE_DEC: 15109 case ZEND_POST_INC: 15110 case ZEND_POST_DEC: 15111 if (ssa_op->op1_use == current_var || 15112 ssa_op->op1_def == current_var || 15113 ssa_op->result_def == current_var) { 15114 regset = ZEND_REGSET_EMPTY; 15115 break; 15116 } 15117 op1_info = OP1_INFO(); 15118 if (opline->op1_type == IS_CV 15119 && (op1_info & MAY_BE_LONG) 15120 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15121 regset = ZEND_REGSET_EMPTY; 15122 if (op1_info & MAY_BE_DOUBLE) { 15123 regset = ZEND_REGSET(ZREG_FPR0); 15124 } 15125 } 15126 break; 15127 case ZEND_ADD: 15128 case ZEND_SUB: 15129 case ZEND_MUL: 15130 op1_info = OP1_INFO(); 15131 op2_info = OP2_INFO(); 15132 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15133 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15134 15135 regset = ZEND_REGSET_EMPTY; 15136 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15137 if (ssa_op->result_def != current_var && 15138 (ssa_op->op1_use != current_var || !last_use)) { 15139 ZEND_REGSET_INCL(regset, ZREG_REG0); 15140 } 15141 res_info = RES_INFO(); 15142 if (res_info & MAY_BE_DOUBLE) { 15143 ZEND_REGSET_INCL(regset, ZREG_REG0); 15144 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15145 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15146 } else if (res_info & MAY_BE_GUARD) { 15147 ZEND_REGSET_INCL(regset, ZREG_REG0); 15148 } 15149 } 15150 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15151 if (ssa_op->result_def != current_var) { 15152 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15153 } 15154 } 15155 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15156 if (zend_is_commutative(opline->opcode)) { 15157 if (ssa_op->result_def != current_var) { 15158 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15159 } 15160 } else { 15161 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15162 if (ssa_op->result_def != current_var && 15163 (ssa_op->op1_use != current_var || !last_use)) { 15164 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15165 } 15166 } 15167 } 15168 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15169 if (ssa_op->result_def != current_var && 15170 (ssa_op->op1_use != current_var || !last_use) && 15171 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15172 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15173 } 15174 } 15175 } 15176 break; 15177 case ZEND_BW_OR: 15178 case ZEND_BW_AND: 15179 case ZEND_BW_XOR: 15180 case ZEND_SL: 15181 case ZEND_SR: 15182 case ZEND_MOD: 15183 op1_info = OP1_INFO(); 15184 op2_info = OP2_INFO(); 15185 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15186 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15187 regset = ZEND_REGSET_EMPTY; 15188 if (ssa_op->result_def != current_var && 15189 (ssa_op->op1_use != current_var || !last_use)) { 15190 ZEND_REGSET_INCL(regset, ZREG_REG0); 15191 } 15192 } 15193 break; 15194 case ZEND_IS_SMALLER: 15195 case ZEND_IS_SMALLER_OR_EQUAL: 15196 case ZEND_IS_EQUAL: 15197 case ZEND_IS_NOT_EQUAL: 15198 case ZEND_IS_IDENTICAL: 15199 case ZEND_IS_NOT_IDENTICAL: 15200 case ZEND_CASE: 15201 op1_info = OP1_INFO(); 15202 op2_info = OP2_INFO(); 15203 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15204 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15205 regset = ZEND_REGSET_EMPTY; 15206 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 15207 ZEND_REGSET_INCL(regset, ZREG_REG0); 15208 } 15209 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 15210 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 15211 if (ssa_op->op1_use != current_var && 15212 ssa_op->op2_use != current_var) { 15213 ZEND_REGSET_INCL(regset, ZREG_REG0); 15214 } 15215 } 15216 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15217 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15218 } 15219 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15220 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15221 } 15222 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15223 if (ssa_op->op1_use != current_var && 15224 ssa_op->op2_use != current_var) { 15225 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15226 } 15227 } 15228 } 15229 break; 15230 case ZEND_BOOL: 15231 case ZEND_BOOL_NOT: 15232 case ZEND_JMPZ: 15233 case ZEND_JMPNZ: 15234 case ZEND_JMPZNZ: 15235 case ZEND_JMPZ_EX: 15236 case ZEND_JMPNZ_EX: 15237 op1_info = OP1_INFO(); 15238 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15239 regset = ZEND_REGSET_EMPTY; 15240 if (op1_info & MAY_BE_DOUBLE) { 15241 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15242 } 15243 if (opline->opcode == ZEND_BOOL || 15244 opline->opcode == ZEND_BOOL_NOT || 15245 opline->opcode == ZEND_JMPZ_EX || 15246 opline->opcode == ZEND_JMPNZ_EX) { 15247 ZEND_REGSET_INCL(regset, ZREG_REG0); 15248 } 15249 } 15250 break; 15251 case ZEND_DO_UCALL: 15252 case ZEND_DO_FCALL: 15253 case ZEND_DO_FCALL_BY_NAME: 15254 case ZEND_INCLUDE_OR_EVAL: 15255 case ZEND_GENERATOR_CREATE: 15256 case ZEND_YIELD: 15257 case ZEND_YIELD_FROM: 15258 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 15259 break; 15260 default: 15261 break; 15262 } 15263 15264 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 15265 if (ssa_op == ssa->ops 15266 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 15267 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 15268 ZEND_REGSET_INCL(regset, ZREG_REG0); 15269 ZEND_REGSET_INCL(regset, ZREG_REG1); 15270 } 15271 } 15272 15273 return regset; 15274} 15275 15276static size_t dasm_venners_size = 0; 15277void **dasm_labels_veneers = NULL; 15278 15279static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) 15280{ 15281 void *veneer; 15282 ptrdiff_t na; 15283 int n, m; 15284 15285 /* try to reuse veneers for global labels */ 15286 if ((ins >> 16) == DASM_REL_LG 15287 && *(b-1) < 0 15288 && dasm_labels_veneers[-*(b-1)]) { 15289 15290 veneer = dasm_labels_veneers[-*(b-1)]; 15291 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15292 n = (int)na; 15293 15294 /* check if we can jump to veneer */ 15295 if ((ptrdiff_t)n != na) { 15296 /* pass */ 15297 } else if (!(ins & 0xf800)) { /* B, BL */ 15298 if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { 15299 return n; 15300 } 15301 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15302 if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { 15303 return n; 15304 } 15305 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15306 /* pass */ 15307 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15308 /* pass */ 15309 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15310 if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { 15311 return n; 15312 } 15313 } 15314 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 15315 && (ins >> 16) == DASM_REL_A) { 15316 ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2)); 15317 15318 if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) { 15319 uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr); 15320 zend_jit_trace_info *t = zend_jit_get_current_trace_info(); 15321 15322 if (exit_point != (uint32_t)-1) { 15323 /* Use exit points table */ 15324 15325 ZEND_ASSERT(exit_point < t->exit_count); 15326 15327 veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; 15328 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15329 n = (int)na; 15330 15331 /* check if we can jump to veneer */ 15332 if ((ptrdiff_t)n != na) { 15333 ZEND_ASSERT(0); 15334 return 0; 15335 } else if (!(ins & 0xf800)) { /* B, BL */ 15336 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15337 ZEND_ASSERT(0); 15338 return 0; 15339 } 15340 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15341 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15342 ZEND_ASSERT(0); 15343 return 0; 15344 } 15345 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15346 ZEND_ASSERT(0); 15347 return 0; 15348 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15349 ZEND_ASSERT(0); 15350 return 0; 15351 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15352 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15353 ZEND_ASSERT(0); 15354 return 0; 15355 } 15356 } else { 15357 ZEND_ASSERT(0); 15358 return 0; 15359 } 15360 return n; 15361 } 15362 } 15363 } 15364 15365 veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); 15366 15367 if (veneer > dasm_end) { 15368 return 0; /* jit_buffer_size overflow */ 15369 } 15370 15371 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15372 n = (int)na; 15373 15374 /* check if we can jump to veneer */ 15375 if ((ptrdiff_t)n != na) { 15376 ZEND_ASSERT(0); 15377 return 0; 15378 } else if (!(ins & 0xf800)) { /* B, BL */ 15379 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15380 ZEND_ASSERT(0); 15381 return 0; 15382 } 15383 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15384 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15385 ZEND_ASSERT(0); 15386 return 0; 15387 } 15388 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15389 ZEND_ASSERT(0); 15390 return 0; 15391 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15392 ZEND_ASSERT(0); 15393 return 0; 15394 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15395 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15396 ZEND_ASSERT(0); 15397 return 0; 15398 } 15399 } else if ((ins & 0x8000)) { /* absolute */ 15400 ZEND_ASSERT(0); 15401 return 0; 15402 } else { 15403 ZEND_ASSERT(0); 15404 return 0; 15405 } 15406 15407 // TODO: support for long veneers (above 128MB) ??? 15408 15409 /* check if we can use B to jump from veneer */ 15410 na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; 15411 m = (int)na; 15412 if ((ptrdiff_t)m != na) { 15413 ZEND_ASSERT(0); 15414 return 0; 15415 } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { 15416 ZEND_ASSERT(0); 15417 return 0; 15418 } 15419 15420 /* generate B instruction */ 15421 *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); 15422 dasm_venners_size += 4; 15423 15424 if ((ins >> 16) == DASM_REL_LG 15425 && *(b-1) < 0) { 15426 /* reuse this veneer for the future jumps to global label */ 15427 dasm_labels_veneers[-*(b-1)] = veneer; 15428 /* Dst->globals[*(b-1)] = veneer; */ 15429 15430#ifdef HAVE_DISASM 15431 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { 15432 const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset)); 15433 15434 if (name && !offset) { 15435 if (strstr(name, "@veneer") == NULL) { 15436 char *new_name; 15437 15438 zend_spprintf(&new_name, 0, "%s@veneer", name); 15439 zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); 15440 efree(new_name); 15441 } else { 15442 zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); 15443 } 15444 } 15445 } 15446#endif 15447 } 15448 15449 return n; 15450} 15451 15452/* 15453 * Local variables: 15454 * tab-width: 4 15455 * c-basic-offset: 4 15456 * indent-tabs-mode: t 15457 * End: 15458 */ 15459