1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112const char* zend_reg_name[] = { 113#if defined(__x86_64__) || defined(_M_X64) 114 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 115 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 116 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 117 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 118#else 119 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 120 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 121#endif 122}; 123 124/* Simulate x86 fastcall */ 125#ifdef _WIN64 126# define ZREG_FCARG1 ZREG_RCX 127# define ZREG_FCARG2 ZREG_RDX 128#elif defined(__x86_64__) 129# define ZREG_FCARG1 ZREG_RDI 130# define ZREG_FCARG2 ZREG_RSI 131#else 132# define ZREG_FCARG1 ZREG_RCX 133# define ZREG_FCARG2 ZREG_RDX 134#endif 135 136|.type EX, zend_execute_data, FP 137|.type OP, zend_op 138|.type ZVAL, zval 139|.actionlist dasm_actions 140|.globals zend_lb 141|.section code, cold_code, jmp_table 142 143static void* dasm_labels[zend_lb_MAX]; 144 145#if ZTS 146static size_t tsrm_ls_cache_tcb_offset = 0; 147static size_t tsrm_tls_index; 148static size_t tsrm_tls_offset; 149#endif 150 151#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 152 153#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 154 155#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 156 157/* Not Implemented Yet */ 158|.macro NIY 159|| //ZEND_ASSERT(0); 160| int3 161|.endmacro 162 163|.macro NIY_STUB 164|| //ZEND_ASSERT(0); 165| int3 166|.endmacro 167 168|.macro ADD_HYBRID_SPAD 169||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 170| add r4, HYBRID_SPAD 171||#endif 172|.endmacro 173 174|.macro SUB_HYBRID_SPAD 175||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 176| sub r4, HYBRID_SPAD 177||#endif 178|.endmacro 179 180|.macro LOAD_ADDR, reg, addr 181| .if X64 182|| if (IS_SIGNED_32BIT(addr)) { 183| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 184|| } else { 185| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 186|| } 187| .else 188| mov reg, ((ptrdiff_t)addr) 189| .endif 190|.endmacro 191 192|.macro LOAD_TSRM_CACHE, reg 193| .if X64WIN 194| gs 195| mov reg, aword [0x58] 196| mov reg, aword [reg+tsrm_tls_index] 197| mov reg, aword [reg+tsrm_tls_offset] 198| .elif WIN 199| fs 200| mov reg, aword [0x2c] 201| mov reg, aword [reg+tsrm_tls_index] 202| mov reg, aword [reg+tsrm_tls_offset] 203| .elif X64APPLE 204| gs 205|| if (tsrm_ls_cache_tcb_offset) { 206| mov reg, aword [tsrm_ls_cache_tcb_offset] 207|| } else { 208| mov reg, aword [tsrm_tls_index] 209| mov reg, aword [reg+tsrm_tls_offset] 210|| } 211| .elif X64 212| fs 213|| if (tsrm_ls_cache_tcb_offset) { 214| mov reg, aword [tsrm_ls_cache_tcb_offset] 215|| } else { 216| mov reg, [0x8] 217| mov reg, aword [reg+tsrm_tls_index] 218| mov reg, aword [reg+tsrm_tls_offset] 219|| } 220| .else 221| gs 222|| if (tsrm_ls_cache_tcb_offset) { 223| mov reg, aword [tsrm_ls_cache_tcb_offset] 224|| } else { 225| mov reg, [0x4] 226| mov reg, aword [reg+tsrm_tls_index] 227| mov reg, aword [reg+tsrm_tls_offset] 228|| } 229| .endif 230|.endmacro 231 232|.macro LOAD_ADDR_ZTS, reg, struct, field 233| .if ZTS 234| LOAD_TSRM_CACHE reg 235| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 236| .else 237| LOAD_ADDR reg, &struct.field 238| .endif 239|.endmacro 240 241|.macro PUSH_ADDR, addr, tmp_reg 242| .if X64 243|| if (IS_SIGNED_32BIT(addr)) { 244| push ((ptrdiff_t)addr) 245|| } else { 246| mov64 tmp_reg, ((ptrdiff_t)addr) 247| push tmp_reg 248|| } 249| .else 250| push ((ptrdiff_t)addr) 251| .endif 252|.endmacro 253 254|.macro ADDR_STORE, mem, addr, tmp_reg 255| .if X64 256|| if (IS_SIGNED_32BIT(addr)) { 257| mov mem, ((ptrdiff_t)addr) 258|| } else { 259| mov64 tmp_reg, ((ptrdiff_t)addr) 260| mov mem, tmp_reg 261|| } 262| .else 263| mov mem, ((ptrdiff_t)addr) 264| .endif 265|.endmacro 266 267|.macro ADDR_CMP, mem, addr, tmp_reg 268| .if X64 269|| if (IS_SIGNED_32BIT(addr)) { 270| cmp mem, ((ptrdiff_t)addr) 271|| } else { 272| mov64 tmp_reg, ((ptrdiff_t)addr) 273| cmp mem, tmp_reg 274|| } 275| .else 276| cmp mem, ((ptrdiff_t)addr) 277| .endif 278|.endmacro 279 280|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 281| .if ZTS 282| LOAD_TSRM_CACHE tmp_reg 283| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 284| push tmp_reg 285| .else 286| PUSH_ADDR &struct.field, tmp_reg 287| .endif 288|.endmacro 289 290|.macro _MEM_OP, mem_ins, prefix, addr, op2, tmp_reg 291| .if X64 292|| if (IS_SIGNED_32BIT(addr)) { 293| mem_ins prefix [addr], op2 294|| } else { 295| mov64 tmp_reg, ((ptrdiff_t)addr) 296| mem_ins prefix [tmp_reg], op2 297|| } 298| .else 299| mem_ins prefix [addr], op2 300| .endif 301|.endmacro 302 303|.macro MEM_LOAD_OP, mem_ins, reg, prefix, addr, tmp_reg 304| .if X64 305|| if (IS_SIGNED_32BIT(addr)) { 306| mem_ins reg, prefix [addr] 307|| } else { 308| mov64 tmp_reg, ((ptrdiff_t)addr) 309| mem_ins reg, prefix [tmp_reg] 310|| } 311| .else 312| mem_ins reg, prefix [addr] 313| .endif 314|.endmacro 315 316|.macro MEM_LOAD, op1, prefix, addr, tmp_reg 317| MEM_LOAD_OP mov, op1, prefix, addr, tmp_reg 318|.endmacro 319 320|.macro _MEM_OP_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 321| .if ZTS 322| LOAD_TSRM_CACHE tmp_reg 323| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 324| .else 325| _MEM_OP mem_ins, prefix, &struct.field, op2, tmp_reg 326| .endif 327|.endmacro 328 329|.macro MEM_STORE_ZTS, prefix, struct, field, op2, tmp_reg 330| _MEM_OP_ZTS mov, prefix, struct, field, op2, tmp_reg 331|.endmacro 332 333|.macro MEM_CMP_ZTS, prefix, struct, field, op2, tmp_reg 334| _MEM_OP_ZTS cmp, prefix, struct, field, op2, tmp_reg 335|.endmacro 336 337|.macro MEM_UPDATE_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 338| _MEM_OP_ZTS mem_ins, prefix, struct, field, op2, tmp_reg 339|.endmacro 340 341|.macro MEM_LOAD_OP_ZTS, mem_ins, reg, prefix, struct, field, tmp_reg 342| .if ZTS 343| LOAD_TSRM_CACHE tmp_reg 344| mem_ins reg, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 345| .else 346| MEM_LOAD_OP mem_ins, reg, prefix, &struct.field, tmp_reg 347| .endif 348|.endmacro 349 350|.macro MEM_LOAD_ZTS, reg, prefix, struct, field, tmp_reg 351| MEM_LOAD_OP_ZTS mov, reg, prefix, struct, field, tmp_reg 352|.endmacro 353 354|.macro EXT_CALL, func, tmp_reg 355| .if X64 356|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 357| call qword &func 358|| } else { 359| LOAD_ADDR tmp_reg, func 360| call tmp_reg 361|| } 362| .else 363| call dword &func 364| .endif 365|.endmacro 366 367|.macro EXT_JMP, func, tmp_reg 368| .if X64 369|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 370| jmp qword &func 371|| } else { 372| LOAD_ADDR tmp_reg, func 373| jmp tmp_reg 374|| } 375| .else 376| jmp dword &func 377| .endif 378|.endmacro 379 380|.macro SAVE_IP 381|| if (GCC_GLOBAL_REGS) { 382| mov aword EX->opline, IP 383|| } 384|.endmacro 385 386|.macro LOAD_IP 387|| if (GCC_GLOBAL_REGS) { 388| mov IP, aword EX->opline 389|| } 390|.endmacro 391 392|.macro LOAD_IP_ADDR, addr 393|| if (GCC_GLOBAL_REGS) { 394| LOAD_ADDR IP, addr 395|| } else { 396| ADDR_STORE aword EX->opline, addr, RX 397|| } 398|.endmacro 399 400|.macro LOAD_IP_ADDR_ZTS, struct, field 401| .if ZTS 402|| if (GCC_GLOBAL_REGS) { 403| LOAD_TSRM_CACHE IP 404| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 405|| } else { 406| LOAD_TSRM_CACHE RX 407| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 408| mov aword EX->opline, RX 409|| } 410| .else 411| LOAD_IP_ADDR &struct.field 412| .endif 413|.endmacro 414 415|.macro GET_IP, reg 416|| if (GCC_GLOBAL_REGS) { 417| mov reg, IP 418|| } else { 419| mov reg, aword EX->opline 420|| } 421|.endmacro 422 423|.macro ADD_IP, val 424|| if (GCC_GLOBAL_REGS) { 425| add IP, val 426|| } else { 427| add aword EX->opline, val 428|| } 429|.endmacro 430 431|.macro JMP_IP 432|| if (GCC_GLOBAL_REGS) { 433| jmp aword [IP] 434|| } else { 435| mov r0, aword EX:FCARG1a->opline 436| jmp aword [r0] 437|| } 438|.endmacro 439 440/* In 64-bit build we compare only low 32-bits. 441 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 442 * comparison would require an additional load of 64-bit address into register. 443 * This is not a problem at all, while JIT buffer size is less than 4GB. 444 */ 445|.macro CMP_IP, addr 446|| if (GCC_GLOBAL_REGS) { 447| cmp IPl, addr 448|| } else { 449| cmp dword EX->opline, addr 450|| } 451|.endmacro 452 453|.macro LOAD_ZVAL_ADDR, reg, addr 454|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 455| LOAD_ADDR reg, Z_ZV(addr) 456|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 457|| if (Z_OFFSET(addr)) { 458| lea reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 459|| } else { 460| mov reg, Ra(Z_REG(addr)) 461|| } 462|| } else { 463|| ZEND_UNREACHABLE(); 464|| } 465|.endmacro 466 467|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 468|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 469| PUSH_ADDR Z_ZV(addr), tmp_reg 470|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 471|| if (Z_OFFSET(addr)) { 472| lea tmp_reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 473| push tmp_reg 474|| } else { 475| push Ra(Z_REG(addr)) 476|| } 477|| } else { 478|| ZEND_UNREACHABLE(); 479|| } 480|.endmacro 481 482|.macro GET_Z_TYPE_INFO, reg, zv 483| mov reg, dword [zv+offsetof(zval,u1.type_info)] 484|.endmacro 485 486|.macro SET_Z_TYPE_INFO, zv, type 487| mov dword [zv+offsetof(zval,u1.type_info)], type 488|.endmacro 489 490|.macro GET_ZVAL_TYPE, reg, addr 491|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 492| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 493|.endmacro 494 495|.macro GET_ZVAL_TYPE_INFO, reg, addr 496|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 497| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 498|.endmacro 499 500|.macro SET_ZVAL_TYPE_INFO, addr, type 501|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 502| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 503|.endmacro 504 505|.macro GET_Z_PTR, reg, zv 506| mov reg, aword [zv] 507|.endmacro 508 509|.macro GET_Z_W2, reg, zv 510| mov reg, dword [zv+4] 511|.endmacro 512 513|.macro SET_Z_W2, zv, reg 514| mov dword [zv+4], reg 515|.endmacro 516 517|.macro GET_ZVAL_PTR, reg, addr 518|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 519| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 520|.endmacro 521 522|.macro SET_ZVAL_PTR, addr, val 523|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 524| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 525|.endmacro 526 527|.macro GET_ZVAL_W2, reg, addr 528|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 529| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 530|.endmacro 531 532|.macro SET_ZVAL_W2, addr, val 533|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 534| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 535|.endmacro 536 537|.macro UNDEF_OPLINE_RESULT 538| mov r0, EX->opline 539| mov eax, dword OP:r0->result.var 540| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 541|.endmacro 542 543|.macro UNDEF_OPLINE_RESULT_IF_USED 544| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 545| jz >1 546| mov eax, dword OP:RX->result.var 547| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 548|1: 549|.endmacro 550 551|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 552|| if (CAN_USE_AVX()) { 553| avx_ins op1, op2 554|| } else { 555| sse_ins op1, op2 556|| } 557|.endmacro 558 559|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 560|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 561| MEM_LOAD_OP sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 562|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 563| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 564|| } else if (Z_MODE(addr) == IS_REG) { 565| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 566|| } else { 567|| ZEND_UNREACHABLE(); 568|| } 569|.endmacro 570 571|.macro DOUBLE_CMP, reg, addr 572|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 573| .if X64 574|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 575| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 576|| } else { 577| LOAD_ADDR r0, Z_ZV(addr) 578| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [r0] 579|| } 580| .else 581| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 582| .endif 583|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 584| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 585|| } else if (Z_MODE(addr) == IS_REG) { 586| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 587|| } else { 588|| ZEND_UNREACHABLE(); 589|| } 590|.endmacro 591 592|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg 593|| if (lval == 0) { 594|| if (CAN_USE_AVX()) { 595| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 596|| } else { 597| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 598|| } 599|| } else { 600|.if X64 601|| if (!IS_SIGNED_32BIT(lval)) { 602| mov64 Ra(tmp_reg), lval 603|| } else { 604| mov Ra(tmp_reg), lval 605|| } 606|.else 607| mov Ra(tmp_reg), lval 608|.endif 609|| if (CAN_USE_AVX()) { 610| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 611| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 612|| } else { 613| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 614| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 615|| } 616|| } 617|.endmacro 618 619|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg 620|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 621| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 622|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 623|| if (CAN_USE_AVX()) { 624| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 625| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 626|| } else { 627| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 628| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 629|| } 630|| } else if (Z_MODE(addr) == IS_REG) { 631|| if (CAN_USE_AVX()) { 632| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 633| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 634|| } else { 635| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 636| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 637|| } 638|| } else { 639|| ZEND_UNREACHABLE(); 640|| } 641|.endmacro 642 643|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr 644|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 645|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 646| .if X64 647|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 648| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 649|| } else { 650| LOAD_ADDR r0, Z_ZV(addr) 651| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 652|| } 653| .else 654| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 655| .endif 656|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 657| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 658|| } else if (Z_MODE(addr) == IS_REG) { 659| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 660|| } else { 661|| ZEND_UNREACHABLE(); 662|| } 663|| } 664|.endmacro 665 666|.macro SSE_MATH, opcode, reg, addr, tmp_reg 667|| switch (opcode) { 668|| case ZEND_ADD: 669| SSE_OP addsd, reg, addr, tmp_reg 670|| break; 671|| case ZEND_SUB: 672| SSE_OP subsd, reg, addr, tmp_reg 673|| break; 674|| case ZEND_MUL: 675| SSE_OP mulsd, reg, addr, tmp_reg 676|| break; 677|| case ZEND_DIV: 678| SSE_OP divsd, reg, addr, tmp_reg 679|| break; 680|| } 681|.endmacro 682 683|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 684|| switch (opcode) { 685|| case ZEND_ADD: 686| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 687|| break; 688|| case ZEND_SUB: 689| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 690|| break; 691|| case ZEND_MUL: 692| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 693|| break; 694|| case ZEND_DIV: 695| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 696|| break; 697|| } 698|.endmacro 699 700|.macro DOUBLE_SET_ZVAL_DVAL, addr, reg 701|| if (Z_MODE(addr) == IS_REG) { 702|| if (reg != Z_REG(addr)) { 703| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 704|| } 705|| } else { 706|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 707| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 708|| } 709|.endmacro 710 711|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 712|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 713| .if X64 714|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 715| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] 716|| } else { 717| mov64 tmp_reg, ((ptrdiff_t)Z_ZV(addr)) 718| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [tmp_reg] 719|| } 720| .else 721| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [addr] 722| .endif 723|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 724| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 725|| } else if (Z_MODE(addr) == IS_REG) { 726| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 727|| } else { 728|| ZEND_UNREACHABLE(); 729|| } 730|.endmacro 731 732|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 733|| switch (opcode) { 734|| case ZEND_ADD: 735| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 736|| break; 737|| case ZEND_SUB: 738| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 739|| break; 740|| case ZEND_MUL: 741| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 742|| break; 743|| case ZEND_DIV: 744| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 745|| break; 746|| } 747|.endmacro 748 749|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 750|| switch (opcode) { 751|| case ZEND_ADD: 752| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 753|| break; 754|| case ZEND_SUB: 755| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 756|| break; 757|| case ZEND_MUL: 758| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 759|| break; 760|| case ZEND_DIV: 761| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 762|| break; 763|| } 764|.endmacro 765 766|.macro LONG_OP, long_ins, reg, addr 767|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 768| .if X64 769|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 770|| if (reg != ZREG_R0) { 771| mov64 r0, Z_LVAL_P(Z_ZV(addr)) 772| long_ins Ra(reg), r0 773|| } else { 774| mov64 r1, Z_LVAL_P(Z_ZV(addr)) 775| long_ins Ra(reg), r1 776|| } 777|| } else { 778| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 779|| } 780| .else 781| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 782| .endif 783|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 784| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 785|| } else if (Z_MODE(addr) == IS_REG) { 786| long_ins Ra(reg), Ra(Z_REG(addr)) 787|| } else { 788|| ZEND_UNREACHABLE(); 789|| } 790|.endmacro 791 792|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 793|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 794| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 795|| } else if (Z_MODE(op1_addr) == IS_REG) { 796| long_ins Ra(Z_REG(op1_addr)), lval 797|| } else { 798|| ZEND_UNREACHABLE(); 799|| } 800|.endmacro 801 802|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 803|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 804| .if X64 805|| if (!IS_SIGNED_32BIT(lval)) { 806| mov64 r0, lval 807| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 808|| } else { 809| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 810|| } 811| .else 812| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 813| .endif 814|| } else if (Z_MODE(op1_addr) == IS_REG) { 815| .if X64 816|| if (!IS_SIGNED_32BIT(lval)) { 817| mov64 r0, lval 818| long_ins Ra(Z_REG(op1_addr)), r0 819|| } else { 820| long_ins Ra(Z_REG(op1_addr)), lval 821|| } 822| .else 823| long_ins Ra(Z_REG(op1_addr)), lval 824| .endif 825|| } else { 826|| ZEND_UNREACHABLE(); 827|| } 828|.endmacro 829 830|.macro GET_ZVAL_LVAL, reg, addr 831|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 832|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 833| xor Ra(reg), Ra(reg) 834|| } else { 835| .if X64 836|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 837| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 838|| } else { 839| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 840|| } 841| .else 842| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 843| .endif 844|| } 845|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 846| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 847|| } else if (Z_MODE(addr) == IS_REG) { 848|| if (reg != Z_REG(addr)) { 849| mov Ra(reg), Ra(Z_REG(addr)) 850|| } 851|| } else { 852|| ZEND_UNREACHABLE(); 853|| } 854|.endmacro 855 856|.macro LONG_MATH, opcode, reg, addr 857|| switch (opcode) { 858|| case ZEND_ADD: 859| LONG_OP add, reg, addr 860|| break; 861|| case ZEND_SUB: 862| LONG_OP sub, reg, addr 863|| break; 864|| case ZEND_MUL: 865| LONG_OP imul, reg, addr 866|| break; 867|| case ZEND_BW_OR: 868| LONG_OP or, reg, addr 869|| break; 870|| case ZEND_BW_AND: 871| LONG_OP and, reg, addr 872|| break; 873|| case ZEND_BW_XOR: 874| LONG_OP xor, reg, addr 875|| break; 876|| default: 877|| ZEND_UNREACHABLE(); 878|| } 879|.endmacro 880 881|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 882|| switch (opcode) { 883|| case ZEND_ADD: 884| add dst_reg, src_reg 885|| break; 886|| case ZEND_SUB: 887| sub dst_reg, src_reg 888|| break; 889|| case ZEND_MUL: 890| imul dst_reg, src_reg 891|| break; 892|| case ZEND_BW_OR: 893| or dst_reg, src_reg 894|| break; 895|| case ZEND_BW_AND: 896| and dst_reg, src_reg 897|| break; 898|| case ZEND_BW_XOR: 899| xor dst_reg, src_reg 900|| break; 901|| default: 902|| ZEND_UNREACHABLE(); 903|| } 904|.endmacro 905 906|.macro SET_ZVAL_LVAL, addr, lval 907|| if (Z_MODE(addr) == IS_REG) { 908| mov Ra(Z_REG(addr)), lval 909|| } else { 910|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 911| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 912|| } 913|.endmacro 914 915|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 916|| if (Z_TYPE_P(zv) > IS_TRUE) { 917|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 918|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 919|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 920|| if (CAN_USE_AVX()) { 921| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 922|| } else { 923| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 924|| } 925| .if X64 926|| } else if (!IS_SIGNED_32BIT(zv)) { 927| mov64 Ra(tmp_reg), ((uintptr_t)zv) 928| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 929| .endif 930|| } else { 931| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 932|| } 933| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 934|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 935|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 936| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 937| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 939| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 940|| } else { 941| .if X64 942|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 943|| if (Z_MODE(dst_addr) == IS_REG) { 944| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 945|| } else { 946| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 947| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 948|| } 949|| } else { 950| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 951|| } 952| .else 953| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 954| .endif 955|| } 956|| } 957|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 958|| if (dst_def_info == MAY_BE_DOUBLE) { 959|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 960| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 961|| } 962|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 963| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 964|| } 965|| } 966|.endmacro 967 968|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 969|| if (Z_TYPE_P(zv) > IS_TRUE) { 970|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 971|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 972|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 973|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 974|| if (CAN_USE_AVX()) { 975| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 976|| } else { 977| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 978|| } 979| .if X64 980|| } else if (!IS_SIGNED_32BIT(zv)) { 981| mov64 Ra(tmp_reg), ((uintptr_t)zv) 982| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 983| .endif 984|| } else { 985| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 986|| } 987| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 988| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 989|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 990|| if (Z_MODE(dst_addr) == IS_REG) { 991| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 992| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 993|| } else if (Z_MODE(res_addr) == IS_REG) { 994| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 995| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 996|| } else { 997| DOUBLE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 998| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 999| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1000|| } 1001|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1002|| if (Z_MODE(dst_addr) == IS_REG) { 1003| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1004| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1005|| } else { 1006| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1007| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1008|| } 1009|| } else { 1010| .if X64 1011|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1012|| if (Z_MODE(dst_addr) == IS_REG) { 1013| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1014| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1015|| } else if (Z_MODE(res_addr) == IS_REG) { 1016| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1017| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1018|| } else { 1019| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1020| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1021| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1022|| } 1023|| } else if (Z_MODE(dst_addr) == IS_REG) { 1024| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1025| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1026|| } else if (Z_MODE(res_addr) == IS_REG) { 1027| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1028| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1029|| } else { 1030| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032|| } 1033| .else 1034|| if (Z_MODE(dst_addr) == IS_REG) { 1035| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1036| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1037|| } else if (Z_MODE(res_addr) == IS_REG) { 1038| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1039| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1040|| } else { 1041| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043|| } 1044| .endif 1045|| } 1046|| } 1047|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1048|| if (dst_def_info == MAY_BE_DOUBLE) { 1049|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1050| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1051|| } 1052|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1053| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1054|| } 1055|| } 1056|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1057|| if (dst_def_info == MAY_BE_DOUBLE) { 1058| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1059|| } else { 1060| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1061|| } 1062|| } 1063|.endmacro 1064 1065/* the same as above, but "src" may overlap with "tmp_reg1" */ 1066|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1067| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1068|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1069|| !(src_info & MAY_BE_GUARD) && 1070|| has_concrete_type(src_info & MAY_BE_ANY)) { 1071|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1072|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1073|| zend_uchar type = concrete_type(src_info); 1074| SET_ZVAL_TYPE_INFO dst_addr, type 1075|| } 1076|| } 1077|| } else { 1078| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1079| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1080|| } 1081|.endmacro 1082 1083|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1084|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1085|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1086|| if (Z_MODE(src_addr) == IS_REG) { 1087|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1088| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1089|| } 1090|| } else if (Z_MODE(dst_addr) == IS_REG) { 1091| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1092|| } else { 1093| GET_ZVAL_LVAL tmp_reg2, src_addr 1094| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1095|| } 1096|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1097|| if (Z_MODE(src_addr) == IS_REG) { 1098| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1099|| } else if (Z_MODE(dst_addr) == IS_REG) { 1100| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1101|| } else { 1102| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1103| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1104|| } 1105|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1106| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1107| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1108|| } else { 1109| .if X64 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112| .else 1113|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1114| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1115| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1116| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1117| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1118|| } else { 1119| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1120| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1123|| } 1124| .endif 1125|| } 1126|| } 1127|.endmacro 1128 1129|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1130|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1131|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1132|| if (Z_MODE(src_addr) == IS_REG) { 1133|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1134| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1135|| } 1136|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1137| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1138|| } 1139|| } else if (Z_MODE(dst_addr) == IS_REG) { 1140| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1141|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1142| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1143|| } 1144|| } else if (Z_MODE(res_addr) == IS_REG) { 1145| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1146| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1147|| } else { 1148| GET_ZVAL_LVAL tmp_reg2, src_addr 1149| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1150| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1151|| } 1152|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1153|| if (Z_MODE(src_addr) == IS_REG) { 1154| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1155| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1156|| } else if (Z_MODE(dst_addr) == IS_REG) { 1157| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1158| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1159|| } else if (Z_MODE(res_addr) == IS_REG) { 1160| DOUBLE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1161| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1162|| } else { 1163| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1164| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1165| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1166|| } 1167|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1168| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1169| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1170| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1171|| } else { 1172| .if X64 1173| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1174| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1175| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1176| .else 1177|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1178| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1179| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1180| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1181| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1182| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1183| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1184|| } else { 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1187| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1188| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1189| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1190| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1191|| } 1192| .endif 1193|| } 1194|| } 1195|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| zend_uchar type = concrete_type(src_info); 1198|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1199|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1200| SET_ZVAL_TYPE_INFO dst_addr, type 1201|| } 1202|| } 1203|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1204| SET_ZVAL_TYPE_INFO res_addr, type 1205|| } 1206|| } else { 1207| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1208| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1209| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1210|| } 1211|.endmacro 1212 1213|.macro IF_UNDEF, type_reg, label 1214| test type_reg, type_reg 1215| je label 1216|.endmacro 1217 1218|.macro IF_TYPE, type, val, label 1219| cmp type, val 1220| je label 1221|.endmacro 1222 1223|.macro IF_NOT_TYPE, type, val, label 1224| cmp type, val 1225| jne label 1226|.endmacro 1227 1228|.macro IF_Z_TYPE, zv, val, label 1229| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1230|.endmacro 1231 1232|.macro IF_NOT_Z_TYPE, zv, val, label 1233| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro CMP_ZVAL_TYPE, addr, val 1237|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1238| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1239|.endmacro 1240 1241|.macro IF_ZVAL_TYPE, addr, val, label 1242|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1243| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1244|.endmacro 1245 1246|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1247|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1248| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1249|.endmacro 1250 1251|.macro IF_FLAGS, type_flags, mask, label 1252| test type_flags, mask 1253| jnz label 1254|.endmacro 1255 1256|.macro IF_NOT_FLAGS, type_flags, mask, label 1257| test type_flags, mask 1258| jz label 1259|.endmacro 1260 1261|.macro IF_REFCOUNTED, type_flags, label 1262| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1263|.endmacro 1264 1265|.macro IF_NOT_REFCOUNTED, type_flags, label 1266| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267| test type_flags, type_flags 1268| jz label 1269|.endmacro 1270 1271|.macro IF_ZVAL_FLAGS, addr, mask, label 1272|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1273| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1274|.endmacro 1275 1276|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1277|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1278| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1279|.endmacro 1280 1281|.macro IF_ZVAL_REFCOUNTED, addr, label 1282| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1283|.endmacro 1284 1285|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1286| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1291|.endmacro 1292 1293|.macro GC_ADDREF, zv 1294| add dword [zv], 1 1295|.endmacro 1296 1297|.macro GC_DELREF, zv 1298| sub dword [zv], 1 1299|.endmacro 1300 1301|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1302| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1303| jne label 1304|.endmacro 1305 1306|.macro ADDREF_CONST, zv, tmp_reg 1307| .if X64 1308|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1309| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1310| add dword [tmp_reg], 1 1311|| } else { 1312| add dword [Z_LVAL_P(zv)], 1 1313|| } 1314| .else 1315| add dword [Z_LVAL_P(zv)], 1 1316| .endif 1317|.endmacro 1318 1319|.macro ADDREF_CONST_2, zv, tmp_reg 1320| .if X64 1321|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1322| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1323| add dword [tmp_reg], 2 1324|| } else { 1325| add dword [Z_LVAL_P(zv)], 2 1326|| } 1327| .else 1328| add dword [Z_LVAL_P(zv)], 2 1329| .endif 1330|.endmacro 1331 1332|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1333|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1334|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1335| IF_NOT_REFCOUNTED type_flags_reg, >1 1336|| } 1337| GC_ADDREF value_ptr_reg 1338|1: 1339|| } 1340|.endmacro 1341 1342|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1343|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1344|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1345| IF_NOT_REFCOUNTED type_flags_reg, >1 1346|| } 1347| add dword [value_ptr_reg], 2 1348|1: 1349|| } 1350|.endmacro 1351 1352|.macro ZVAL_DEREF, reg, info 1353|| if (info & MAY_BE_REF) { 1354| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1355| GET_Z_PTR reg, reg 1356| add reg, offsetof(zend_reference, val) 1357|1: 1358|| } 1359|.endmacro 1360 1361|.macro SET_EX_OPLINE, op, tmp_reg 1362|| if (op == last_valid_opline) { 1363|| zend_jit_use_last_valid_opline(); 1364| SAVE_IP 1365|| } else { 1366| ADDR_STORE aword EX->opline, op, tmp_reg 1367|| if (!GCC_GLOBAL_REGS) { 1368|| zend_jit_reset_last_valid_opline(); 1369|| } 1370|| } 1371|.endmacro 1372 1373// zval should be in FCARG1a 1374|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1375|| do { 1376|| if (!((var_info) & MAY_BE_GUARD) 1377|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1378|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1379|| if (type == IS_STRING && !ZEND_DEBUG) { 1380| EXT_CALL _efree, r0 1381|| break; 1382|| } else if (type == IS_ARRAY) { 1383|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1384|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1385| SET_EX_OPLINE opline, r0 1386|| } 1387| EXT_CALL zend_array_destroy, r0 1388|| } else { 1389| EXT_CALL zend_jit_array_free, r0 1390|| } 1391|| break; 1392|| } else if (type == IS_OBJECT) { 1393|| if (opline) { 1394| SET_EX_OPLINE opline, r0 1395|| } 1396| EXT_CALL zend_objects_store_del, r0 1397|| break; 1398|| } 1399|| } 1400|| if (opline) { 1401| SET_EX_OPLINE opline, r0 1402|| } 1403| EXT_CALL rc_dtor_func, r0 1404|| } while(0); 1405|.endmacro 1406 1407|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1408|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1409|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1410| // if (Z_REFCOUNTED_P(cv)) { 1411|| if (cold) { 1412| IF_ZVAL_REFCOUNTED addr, >1 1413|.cold_code 1414|1: 1415|| } else { 1416| IF_NOT_ZVAL_REFCOUNTED addr, >4 1417|| } 1418|| } 1419| // if (!Z_DELREF_P(cv)) { 1420| GET_ZVAL_PTR FCARG1a, addr 1421| GC_DELREF FCARG1a 1422|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1423|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1424|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1425| jnz >3 1426|| } else { 1427| jnz >4 1428|| } 1429|| } 1430| // zval_dtor_func(r); 1431| ZVAL_DTOR_FUNC op_info, opline 1432|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1433| jmp >4 1434|| } 1435|3: 1436|| } 1437|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1438|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1439|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1440| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1441| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1442| GET_ZVAL_PTR FCARG1a, ref_addr 1443|1: 1444|| } 1445| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1446| // gc_possible_root(Z_COUNTED_P(z)) 1447| EXT_CALL gc_possible_root, r0 1448|| } 1449|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1450| jmp >4 1451|.code 1452|| } 1453|4: 1454|| } 1455|.endmacro 1456 1457|.macro FREE_OP, op_type, op, op_info, cold, opline 1458|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1459| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1460|| } 1461|.endmacro 1462 1463|.macro SEPARATE_ARRAY, addr, op_info, cold 1464|| if (RC_MAY_BE_N(op_info)) { 1465|| if (Z_REG(addr) != ZREG_FP) { 1466| GET_ZVAL_LVAL ZREG_R0, addr 1467|| if (RC_MAY_BE_1(op_info)) { 1468| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1469| jbe >2 1470|| } 1471|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1472| LOAD_ZVAL_ADDR FCARG1a, addr 1473|| } 1474| EXT_CALL zend_jit_zval_array_dup, r0 1475|2: 1476| mov FCARG1a, r0 1477|| } else { 1478| GET_ZVAL_LVAL ZREG_FCARG1, addr 1479|| if (RC_MAY_BE_1(op_info)) { 1480| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1481|| if (cold) { 1482| ja >1 1483|.cold_code 1484|1: 1485|| } else { 1486| jbe >2 1487|| } 1488|| } 1489| IF_NOT_ZVAL_REFCOUNTED addr, >1 1490| GC_DELREF FCARG1a 1491|1: 1492| EXT_CALL zend_array_dup, r0 1493| SET_ZVAL_PTR addr, r0 1494| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1495| mov FCARG1a, r0 1496|| if (RC_MAY_BE_1(op_info)) { 1497|| if (cold) { 1498| jmp >2 1499|.code 1500|| } 1501|| } 1502|2: 1503|| } 1504|| } else { 1505| GET_ZVAL_LVAL ZREG_FCARG1, addr 1506|| } 1507|.endmacro 1508 1509|.macro EFREE_REG_REFERENCE 1510||#if ZEND_DEBUG 1511| xor FCARG2a, FCARG2a // filename 1512| .if X64WIN 1513| xor CARG3d, CARG3d // lineno 1514| xor CARG4, CARG4 1515| mov aword A5, 0 1516| EXT_CALL _efree, r0 1517| .elif X64 1518| xor CARG3d, CARG3d // lineno 1519| xor CARG4, CARG4 1520| xor CARG5, CARG5 1521| EXT_CALL _efree, r0 1522| .else 1523| sub r4, 4 1524| push 0 1525| push 0 1526| push 0 // lineno 1527| EXT_CALL _efree, r0 1528| add r4, 4 1529| .endif 1530||#else 1531||#ifdef HAVE_BUILTIN_CONSTANT_P 1532| EXT_CALL _efree_32, r0 1533||#else 1534| EXT_CALL _efree, r0 1535||#endif 1536||#endif 1537|.endmacro 1538 1539|.macro EFREE_REFERENCE, ptr 1540| mov FCARG1a, ptr 1541| EFREE_REG_REFERENCE 1542|.endmacro 1543 1544|.macro EMALLOC, size, op_array, opline 1545||#if ZEND_DEBUG 1546|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1547| mov FCARG1a, size 1548| LOAD_ADDR FCARG2a, filename 1549| .if X64WIN 1550| mov CARG3d, opline->lineno 1551| xor CARG4, CARG4 1552| mov aword A5, 0 1553| EXT_CALL _emalloc, r0 1554| .elif X64 1555| mov CARG3d, opline->lineno 1556| xor CARG4, CARG4 1557| xor CARG5, CARG5 1558| EXT_CALL _emalloc, r0 1559| .else 1560| sub r4, 4 1561| push 0 1562| push 0 1563| push opline->lineno 1564| EXT_CALL _emalloc, r0 1565| add r4, 4 1566| .endif 1567||#else 1568||#ifdef HAVE_BUILTIN_CONSTANT_P 1569|| if (size > 24 && size <= 32) { 1570| EXT_CALL _emalloc_32, r0 1571|| } else { 1572| mov FCARG1a, size 1573| EXT_CALL _emalloc, r0 1574|| } 1575||#else 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578||#endif 1579||#endif 1580|.endmacro 1581 1582|.macro OBJ_RELEASE, reg, exit_label 1583| GC_DELREF Ra(reg) 1584| jne >1 1585| // zend_objects_store_del(obj); 1586|| if (reg != ZREG_FCARG1) { 1587| mov FCARG1a, Ra(reg) 1588|| } 1589| EXT_CALL zend_objects_store_del, r0 1590| jmp exit_label 1591|1: 1592| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1593| // gc_possible_root(obj) 1594|| if (reg != ZREG_FCARG1) { 1595| mov FCARG1a, Ra(reg) 1596|| } 1597| EXT_CALL gc_possible_root, r0 1598|1: 1599|.endmacro 1600 1601|.macro UNDEFINED_OFFSET, opline 1602|| if (opline == last_valid_opline) { 1603|| zend_jit_use_last_valid_opline(); 1604| call ->undefined_offset_ex 1605|| } else { 1606| SET_EX_OPLINE opline, r0 1607| call ->undefined_offset 1608|| } 1609|.endmacro 1610 1611|.macro UNDEFINED_INDEX, opline 1612|| if (opline == last_valid_opline) { 1613|| zend_jit_use_last_valid_opline(); 1614| call ->undefined_index_ex 1615|| } else { 1616| SET_EX_OPLINE opline, r0 1617| call ->undefined_index 1618|| } 1619|.endmacro 1620 1621|.macro CANNOT_ADD_ELEMENT, opline 1622|| if (opline == last_valid_opline) { 1623|| zend_jit_use_last_valid_opline(); 1624| call ->cannot_add_element_ex 1625|| } else { 1626| SET_EX_OPLINE opline, r0 1627| call ->cannot_add_element 1628|| } 1629|.endmacro 1630 1631static bool reuse_ip = 0; 1632static bool delayed_call_chain = 0; 1633static uint32_t delayed_call_level = 0; 1634static const zend_op *last_valid_opline = NULL; 1635static bool use_last_vald_opline = 0; 1636static bool track_last_valid_opline = 0; 1637static int jit_return_label = -1; 1638static uint32_t current_trace_num = 0; 1639static uint32_t allowed_opt_flags = 0; 1640 1641static void zend_jit_track_last_valid_opline(void) 1642{ 1643 use_last_vald_opline = 0; 1644 track_last_valid_opline = 1; 1645} 1646 1647static void zend_jit_use_last_valid_opline(void) 1648{ 1649 if (track_last_valid_opline) { 1650 use_last_vald_opline = 1; 1651 track_last_valid_opline = 0; 1652 } 1653} 1654 1655static bool zend_jit_trace_uses_initial_ip(void) 1656{ 1657 return use_last_vald_opline; 1658} 1659 1660static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1661{ 1662 if (!reuse_ip) { 1663 track_last_valid_opline = 0; 1664 last_valid_opline = target_opline; 1665 } 1666} 1667 1668static void zend_jit_reset_last_valid_opline(void) 1669{ 1670 track_last_valid_opline = 0; 1671 last_valid_opline = NULL; 1672} 1673 1674static void zend_jit_start_reuse_ip(void) 1675{ 1676 zend_jit_reset_last_valid_opline(); 1677 reuse_ip = 1; 1678} 1679 1680static int zend_jit_reuse_ip(dasm_State **Dst) 1681{ 1682 if (!reuse_ip) { 1683 zend_jit_start_reuse_ip(); 1684 | // call = EX(call); 1685 | mov RX, EX->call 1686 } 1687 return 1; 1688} 1689 1690static void zend_jit_stop_reuse_ip(void) 1691{ 1692 reuse_ip = 0; 1693} 1694 1695static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1696{ 1697 |->interrupt_handler: 1698 | SAVE_IP 1699 | //EG(vm_interrupt) = 0; 1700 | MEM_STORE_ZTS byte, executor_globals, vm_interrupt, 0, r0 1701 | //if (EG(timed_out)) { 1702 | MEM_CMP_ZTS byte, executor_globals, timed_out, 0, r0 1703 | je >1 1704 | //zend_timeout(); 1705 | EXT_CALL zend_timeout, r0 1706 |1: 1707 | //} else if (zend_interrupt_function) { 1708 if (zend_interrupt_function) { 1709 | //zend_interrupt_function(execute_data); 1710 |.if X64 1711 | mov CARG1, FP 1712 | EXT_CALL zend_interrupt_function, r0 1713 |.else 1714 | mov aword A1, FP 1715 | EXT_CALL zend_interrupt_function, r0 1716 |.endif 1717 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 1718 | je >1 1719 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1720 |1: 1721 | //ZEND_VM_ENTER(); 1722 | //execute_data = EG(current_execute_data); 1723 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 1724 | LOAD_IP 1725 } 1726 | //ZEND_VM_CONTINUE() 1727 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1728 | ADD_HYBRID_SPAD 1729 | JMP_IP 1730 } else if (GCC_GLOBAL_REGS) { 1731 | add r4, SPAD // stack alignment 1732 | JMP_IP 1733 } else { 1734 | mov FP, aword T2 // restore FP 1735 | mov RX, aword T3 // restore IP 1736 | add r4, NR_SPAD // stack alignment 1737 | mov r0, 1 // ZEND_VM_ENTER 1738 | ret 1739 } 1740 1741 return 1; 1742} 1743 1744static int zend_jit_exception_handler_stub(dasm_State **Dst) 1745{ 1746 |->exception_handler: 1747 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1748 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1749 1750 | ADD_HYBRID_SPAD 1751 | EXT_CALL handler, r0 1752 | JMP_IP 1753 } else { 1754 const void *handler = EG(exception_op)->handler; 1755 1756 if (GCC_GLOBAL_REGS) { 1757 | add r4, SPAD // stack alignment 1758 | EXT_JMP handler, r0 1759 } else { 1760 | mov FCARG1a, FP 1761 | EXT_CALL handler, r0 1762 | mov FP, aword T2 // restore FP 1763 | mov RX, aword T3 // restore IP 1764 | add r4, NR_SPAD // stack alignment 1765 | test eax, eax 1766 | jl >1 1767 | mov r0, 1 // ZEND_VM_ENTER 1768 |1: 1769 | ret 1770 } 1771 } 1772 1773 return 1; 1774} 1775 1776static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1777{ 1778 |->exception_handler_undef: 1779 | MEM_LOAD_ZTS r0, aword, executor_globals, opline_before_exception, r0 1780 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1781 | jz >1 1782 | mov eax, dword OP:r0->result.var 1783 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1784 |1: 1785 | jmp ->exception_handler 1786 1787 return 1; 1788} 1789 1790 1791static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1792{ 1793 |->exception_handler_free_op1_op2: 1794 | UNDEF_OPLINE_RESULT_IF_USED 1795 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1796 | je >9 1797 | mov eax, dword OP:RX->op1.var 1798 | add r0, FP 1799 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1800 |9: 1801 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1802 | je >9 1803 | mov eax, dword OP:RX->op2.var 1804 | add r0, FP 1805 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1806 |9: 1807 | jmp ->exception_handler 1808 return 1; 1809} 1810 1811static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1812{ 1813 |->exception_handler_free_op2: 1814 | MEM_LOAD_ZTS RX, aword, executor_globals, opline_before_exception, r0 1815 | UNDEF_OPLINE_RESULT_IF_USED 1816 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1817 | je >9 1818 | mov eax, dword OP:RX->op2.var 1819 | add r0, FP 1820 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1821 |9: 1822 | jmp ->exception_handler 1823 return 1; 1824} 1825 1826static int zend_jit_leave_function_stub(dasm_State **Dst) 1827{ 1828 |->leave_function_handler: 1829 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 1830 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1831 | test FCARG1d, ZEND_CALL_TOP 1832 | jnz >1 1833 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1834 | ADD_HYBRID_SPAD 1835 | JMP_IP 1836 |1: 1837 | EXT_CALL zend_jit_leave_top_func_helper, r0 1838 | ADD_HYBRID_SPAD 1839 | JMP_IP 1840 } else { 1841 if (GCC_GLOBAL_REGS) { 1842 | add r4, SPAD 1843 } else { 1844 | mov FCARG2a, FP 1845 | mov FP, aword T2 // restore FP 1846 | mov RX, aword T3 // restore IP 1847 | add r4, NR_SPAD 1848 } 1849 | test FCARG1d, ZEND_CALL_TOP 1850 | jnz >1 1851 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1852 |1: 1853 | EXT_JMP zend_jit_leave_top_func_helper, r0 1854 } 1855 1856 return 1; 1857} 1858 1859static int zend_jit_leave_throw_stub(dasm_State **Dst) 1860{ 1861 |->leave_throw_handler: 1862 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1863 if (GCC_GLOBAL_REGS) { 1864 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1865 | je >5 1866 | // EG(opline_before_exception) = opline; 1867 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1868 |5: 1869 | // opline = EG(exception_op); 1870 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1871 | // HANDLE_EXCEPTION() 1872 | jmp ->exception_handler 1873 } else { 1874 | GET_IP FCARG1a 1875 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1876 | je >5 1877 | // EG(opline_before_exception) = opline; 1878 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, FCARG1a, r0 1879 |5: 1880 | // opline = EG(exception_op); 1881 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1882 | mov FP, aword T2 // restore FP 1883 | mov RX, aword T3 // restore IP 1884 | add r4, NR_SPAD // stack alignment 1885 | mov r0, 2 // ZEND_VM_LEAVE 1886 | ret 1887 } 1888 1889 return 1; 1890} 1891 1892static int zend_jit_icall_throw_stub(dasm_State **Dst) 1893{ 1894 |->icall_throw_handler: 1895 | // zend_rethrow_exception(zend_execute_data *execute_data) 1896 | mov IP, aword EX->opline 1897 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1898 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1899 | je >1 1900 | // EG(opline_before_exception) = opline; 1901 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1902 |1: 1903 | // opline = EG(exception_op); 1904 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1905 || if (GCC_GLOBAL_REGS) { 1906 | mov aword EX->opline, IP 1907 || } 1908 | // HANDLE_EXCEPTION() 1909 | jmp ->exception_handler 1910 1911 return 1; 1912} 1913 1914static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1915{ 1916 |->throw_cannot_pass_by_ref: 1917 | mov r0, EX->opline 1918 | mov ecx, dword OP:r0->result.var 1919 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1920 | // last EX(call) frame may be delayed 1921 | cmp RX, EX->call 1922 | je >1 1923 | mov r1, EX->call 1924 | mov EX:RX->prev_execute_data, r1 1925 | mov EX->call, RX 1926 |1: 1927 | mov RX, r0 1928 | mov FCARG1d, dword OP:r0->op2.num 1929 | EXT_CALL zend_cannot_pass_by_reference, r0 1930 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1931 | jne >9 1932 | mov eax, dword OP:RX->op1.var 1933 | add r0, FP 1934 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1935 |9: 1936 | jmp ->exception_handler 1937 1938 return 1; 1939} 1940 1941static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1942{ 1943 |->undefined_offset_ex: 1944 | SAVE_IP 1945 | jmp ->undefined_offset 1946 1947 return 1; 1948} 1949 1950static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1951{ 1952 |->undefined_offset: 1953 |.if X64WIN 1954 | sub r4, 0x28 1955 |.elif X64 1956 | sub r4, 8 1957 |.else 1958 | sub r4, 12 1959 |.endif 1960 | mov r0, EX->opline 1961 | mov ecx, dword OP:r0->result.var 1962 | cmp byte OP:r0->op2_type, IS_CONST 1963 | SET_Z_TYPE_INFO FP + r1, IS_NULL 1964 | jne >2 1965 |.if X64 1966 | movsxd r1, dword OP:r0->op2.constant 1967 | add r0, r1 1968 |.else 1969 | mov r0, aword OP:r0->op2.zv 1970 |.endif 1971 | jmp >3 1972 |2: 1973 | mov eax, dword OP:r0->op2.var 1974 | add r0, FP 1975 |3: 1976 |.if X64WIN 1977 | mov CARG1, E_WARNING 1978 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 1979 | mov CARG3, aword [r0] 1980 | EXT_CALL zend_error, r0 1981 | add r4, 0x28 // stack alignment 1982 |.elif X64 1983 | mov CARG1, E_WARNING 1984 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 1985 | mov CARG3, aword [r0] 1986 | EXT_CALL zend_error, r0 1987 | add r4, 8 // stack alignment 1988 |.else 1989 | sub r4, 4 1990 | push aword [r0] 1991 | push "Undefined array key " ZEND_LONG_FMT 1992 | push E_WARNING 1993 | EXT_CALL zend_error, r0 1994 | add r4, 28 1995 |.endif 1996 | ret 1997 1998 return 1; 1999} 2000 2001static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2002{ 2003 |->undefined_index_ex: 2004 | SAVE_IP 2005 | jmp ->undefined_index 2006 2007 return 1; 2008} 2009 2010static int zend_jit_undefined_index_stub(dasm_State **Dst) 2011{ 2012 |->undefined_index: 2013 |.if X64WIN 2014 | sub r4, 0x28 2015 |.elif X64 2016 | sub r4, 8 2017 |.else 2018 | sub r4, 12 2019 |.endif 2020 | mov r0, EX->opline 2021 | mov ecx, dword OP:r0->result.var 2022 | cmp byte OP:r0->op2_type, IS_CONST 2023 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2024 | jne >2 2025 |.if X64 2026 | movsxd r1, dword OP:r0->op2.constant 2027 | add r0, r1 2028 |.else 2029 | mov r0, aword OP:r0->op2.zv 2030 |.endif 2031 | jmp >3 2032 |2: 2033 | mov eax, dword OP:r0->op2.var 2034 | add r0, FP 2035 |3: 2036 |.if X64WIN 2037 | mov CARG1, E_WARNING 2038 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2039 | mov CARG3, aword [r0] 2040 | add CARG3, offsetof(zend_string, val) 2041 | EXT_CALL zend_error, r0 2042 | add r4, 0x28 2043 |.elif X64 2044 | mov CARG1, E_WARNING 2045 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2046 | mov CARG3, aword [r0] 2047 | add CARG3, offsetof(zend_string, val) 2048 | EXT_CALL zend_error, r0 2049 | add r4, 8 2050 |.else 2051 | sub r4, 4 2052 | mov r0, aword [r0] 2053 | add r0, offsetof(zend_string, val) 2054 | push r0 2055 | push "Undefined array key \"%s\"" 2056 | push E_WARNING 2057 | EXT_CALL zend_error, r0 2058 | add r4, 28 2059 |.endif 2060 | ret 2061 2062 return 1; 2063} 2064 2065static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2066{ 2067 |->cannot_add_element_ex: 2068 | SAVE_IP 2069 | jmp ->cannot_add_element 2070 2071 return 1; 2072} 2073 2074static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2075{ 2076 |->cannot_add_element: 2077 |.if X64WIN 2078 | sub r4, 0x28 2079 |.elif X64 2080 | sub r4, 8 2081 |.else 2082 | sub r4, 12 2083 |.endif 2084 | mov r0, EX->opline 2085 | cmp byte OP:r0->result_type, IS_UNUSED 2086 | jz >1 2087 | mov eax, dword OP:r0->result.var 2088 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2089 |1: 2090 |.if X64WIN 2091 | xor CARG1, CARG1 2092 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2093 | EXT_CALL zend_throw_error, r0 2094 | add r4, 0x28 2095 |.elif X64 2096 | xor CARG1, CARG1 2097 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2098 | EXT_CALL zend_throw_error, r0 2099 | add r4, 8 2100 |.else 2101 | sub r4, 8 2102 | push "Cannot add element to the array as the next element is already occupied" 2103 | push 0 2104 | EXT_CALL zend_throw_error, r0 2105 | add r4, 28 2106 |.endif 2107 | ret 2108 2109 return 1; 2110} 2111 2112static int zend_jit_undefined_function_stub(dasm_State **Dst) 2113{ 2114 |->undefined_function: 2115 | mov r0, aword EX->opline 2116 |.if X64 2117 | xor CARG1, CARG1 2118 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2119 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2120 | mov CARG3, aword [r0 + CARG3] 2121 | add CARG3, offsetof(zend_string, val) 2122 | EXT_CALL zend_throw_error, r0 2123 |.else 2124 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2125 | mov r0, aword [r0] 2126 | add r0, offsetof(zend_string, val) 2127 | mov aword A3, r0 2128 | mov aword A2, "Call to undefined function %s()" 2129 | mov aword A1, 0 2130 | EXT_CALL zend_throw_error, r0 2131 |.endif 2132 | jmp ->exception_handler 2133 return 1; 2134} 2135 2136static int zend_jit_negative_shift_stub(dasm_State **Dst) 2137{ 2138 |->negative_shift: 2139 | mov RX, EX->opline 2140 |.if X64 2141 |.if WIN 2142 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2143 | mov CARG1, aword [CARG1] 2144 |.else 2145 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2146 |.endif 2147 | LOAD_ADDR CARG2, "Bit shift by negative number" 2148 | EXT_CALL zend_throw_error, r0 2149 |.else 2150 | sub r4, 8 2151 | push "Bit shift by negative number" 2152 |.if WIN 2153 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2154 | push aword [r0] 2155 |.else 2156 | PUSH_ADDR zend_ce_arithmetic_error, r0 2157 |.endif 2158 | EXT_CALL zend_throw_error, r0 2159 | add r4, 16 2160 |.endif 2161 | jmp ->exception_handler_free_op1_op2 2162 return 1; 2163} 2164 2165static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2166{ 2167 |->mod_by_zero: 2168 | mov RX, EX->opline 2169 |.if X64 2170 |.if WIN 2171 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2172 | mov CARG1, aword [CARG1] 2173 |.else 2174 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2175 |.endif 2176 | LOAD_ADDR CARG2, "Modulo by zero" 2177 | EXT_CALL zend_throw_error, r0 2178 |.else 2179 | sub r4, 8 2180 | push "Modulo by zero" 2181 |.if WIN 2182 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2183 | push aword [r0] 2184 |.else 2185 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2186 |.endif 2187 | EXT_CALL zend_throw_error, r0 2188 | add r4, 16 2189 |.endif 2190 | jmp ->exception_handler_free_op1_op2 2191 return 1; 2192} 2193 2194static int zend_jit_invalid_this_stub(dasm_State **Dst) 2195{ 2196 |->invalid_this: 2197 | UNDEF_OPLINE_RESULT 2198 |.if X64 2199 | xor CARG1, CARG1 2200 | LOAD_ADDR CARG2, "Using $this when not in object context" 2201 | EXT_CALL zend_throw_error, r0 2202 |.else 2203 | sub r4, 8 2204 | push "Using $this when not in object context" 2205 | push 0 2206 | EXT_CALL zend_throw_error, r0 2207 | add r4, 16 2208 |.endif 2209 | jmp ->exception_handler 2210 return 1; 2211} 2212 2213static int zend_jit_double_one_stub(dasm_State **Dst) 2214{ 2215 |->one: 2216 |.dword 0, 0x3ff00000 2217 return 1; 2218} 2219 2220static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2221{ 2222 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2223 return 1; 2224 } 2225 2226 |->hybrid_runtime_jit: 2227 | EXT_CALL zend_runtime_jit, r0 2228 | JMP_IP 2229 return 1; 2230} 2231 2232static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2233{ 2234 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2235 return 1; 2236 } 2237 2238 |->hybrid_profile_jit: 2239 | // ++zend_jit_profile_counter; 2240 | .if X64 2241 | LOAD_ADDR r0, &zend_jit_profile_counter 2242 | inc aword [r0] 2243 | .else 2244 | inc aword [&zend_jit_profile_counter] 2245 | .endif 2246 | // op_array = (zend_op_array*)EX(func); 2247 | mov r0, EX->func 2248 | // run_time_cache = EX(run_time_cache); 2249 | mov r2, EX->run_time_cache 2250 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2251 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2252 | // ++ZEND_COUNTER_INFO(op_array) 2253 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2254 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2255 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2256 return 1; 2257} 2258 2259static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2260{ 2261 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2262 return 1; 2263 } 2264 2265 |->hybrid_hot_code: 2266 | mov word [r2], ZEND_JIT_COUNTER_INIT 2267 | mov FCARG1a, FP 2268 | GET_IP FCARG2a 2269 | EXT_CALL zend_jit_hot_func, r0 2270 | JMP_IP 2271 return 1; 2272} 2273 2274/* 2275 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2276 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2277 * property disclosure and research opportunities" email 2278 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2279 * 2280 * In addition we use a variation of Knuth's multiplicative hash function 2281 * described at https://code.i-harness.com/en/q/a21ce 2282 * 2283 * uint64_t hash(uint64_t x) { 2284 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2285 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2286 * x = x ^ (x >> 31); 2287 * return x; 2288 * } 2289 * 2290 * uint_32_t hash(uint32_t x) { 2291 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2292 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2293 * x = (x >> 16) ^ x; 2294 * return x; 2295 * } 2296 * 2297 */ 2298static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2299{ 2300 | mov r0, EX->func 2301 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2302 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2303 | sub word [r2], cost 2304 | jle ->hybrid_hot_code 2305 | GET_IP r2 2306 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2307 | // divide by sizeof(zend_op) 2308 | .if X64 2309 || ZEND_ASSERT(sizeof(zend_op) == 32); 2310 | sar r2, 2 2311 | .else 2312 || ZEND_ASSERT(sizeof(zend_op) == 28); 2313 | imul r2, 0xb6db6db7 2314 | .endif 2315 | .if X64 2316 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2317 | .else 2318 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2319 | .endif 2320 return 1; 2321} 2322 2323static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2324{ 2325 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2326 return 1; 2327 } 2328 2329 |->hybrid_func_hot_counter: 2330 2331 return zend_jit_hybrid_hot_counter_stub(Dst, 2332 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2333} 2334 2335static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2336{ 2337 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2338 return 1; 2339 } 2340 2341 |->hybrid_loop_hot_counter: 2342 2343 return zend_jit_hybrid_hot_counter_stub(Dst, 2344 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2345} 2346 2347static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2348{ 2349 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2350 return 1; 2351 } 2352 2353 |->hybrid_hot_trace: 2354 | mov word [r2], ZEND_JIT_COUNTER_INIT 2355 | mov FCARG1a, FP 2356 | GET_IP FCARG2a 2357 | EXT_CALL zend_jit_trace_hot_root, r0 2358 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2359 | jl >1 2360 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2361 | LOAD_IP 2362 | JMP_IP 2363 |1: 2364 | EXT_JMP zend_jit_halt_op->handler, r0 2365 return 1; 2366} 2367 2368static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2369{ 2370 | mov r0, EX->func 2371 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2372 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2373 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2374 | sub word [r2], cost 2375 | jle ->hybrid_hot_trace 2376 | jmp aword [IP + r1] 2377 return 1; 2378} 2379 2380static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2381{ 2382 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2383 return 1; 2384 } 2385 2386 |->hybrid_func_trace_counter: 2387 2388 return zend_jit_hybrid_trace_counter_stub(Dst, 2389 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2390} 2391 2392static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2393{ 2394 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2395 return 1; 2396 } 2397 2398 |->hybrid_ret_trace_counter: 2399 2400 return zend_jit_hybrid_trace_counter_stub(Dst, 2401 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2402} 2403 2404static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2405{ 2406 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2407 return 1; 2408 } 2409 2410 |->hybrid_loop_trace_counter: 2411 2412 return zend_jit_hybrid_trace_counter_stub(Dst, 2413 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2414} 2415 2416static int zend_jit_trace_halt_stub(dasm_State **Dst) 2417{ 2418 |->trace_halt: 2419 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2420 | ADD_HYBRID_SPAD 2421 | EXT_JMP zend_jit_halt_op->handler, r0 2422 } else if (GCC_GLOBAL_REGS) { 2423 | add r4, SPAD // stack alignment 2424 | xor IP, IP // PC must be zero 2425 | ret 2426 } else { 2427 | mov FP, aword T2 // restore FP 2428 | mov RX, aword T3 // restore IP 2429 | add r4, NR_SPAD // stack alignment 2430 | mov r0, -1 // ZEND_VM_RETURN 2431 | ret 2432 } 2433 return 1; 2434} 2435 2436static int zend_jit_trace_exit_stub(dasm_State **Dst) 2437{ 2438 |->trace_exit: 2439 | 2440 | // Save CPU registers 2441 |.if X64 2442 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2443 | mov aword [r4+15*8], r15 2444 | mov aword [r4+11*8], r11 2445 | mov aword [r4+10*8], r10 2446 | mov aword [r4+9*8], r9 2447 | mov aword [r4+8*8], r8 2448 | mov aword [r4+7*8], rdi 2449 | mov aword [r4+6*8], rsi 2450 | mov aword [r4+2*8], rdx 2451 | mov aword [r4+1*8], rcx 2452 | mov aword [r4+0*8], rax 2453 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2454 | mov FCARG2a, r4 2455 | movsd qword [r4+16*8+15*8], xmm15 2456 | movsd qword [r4+16*8+14*8], xmm14 2457 | movsd qword [r4+16*8+13*8], xmm13 2458 | movsd qword [r4+16*8+12*8], xmm12 2459 | movsd qword [r4+16*8+11*8], xmm11 2460 | movsd qword [r4+16*8+10*8], xmm10 2461 | movsd qword [r4+16*8+9*8], xmm9 2462 | movsd qword [r4+16*8+8*8], xmm8 2463 | movsd qword [r4+16*8+7*8], xmm7 2464 | movsd qword [r4+16*8+6*8], xmm6 2465 | movsd qword [r4+16*8+5*8], xmm5 2466 | movsd qword [r4+16*8+4*8], xmm4 2467 | movsd qword [r4+16*8+3*8], xmm3 2468 | movsd qword [r4+16*8+2*8], xmm2 2469 | movsd qword [r4+16*8+1*8], xmm1 2470 | movsd qword [r4+16*8+0*8], xmm0 2471 |.if X64WIN 2472 | sub r4, 32 /* shadow space */ 2473 |.endif 2474 |.else 2475 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2476 | mov aword [r4+7*4], edi 2477 | mov aword [r4+2*4], edx 2478 | mov aword [r4+1*4], ecx 2479 | mov aword [r4+0*4], eax 2480 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2481 | mov FCARG2a, r4 2482 | movsd qword [r4+8*4+7*8], xmm7 2483 | movsd qword [r4+8*4+6*8], xmm6 2484 | movsd qword [r4+8*4+5*8], xmm5 2485 | movsd qword [r4+8*4+4*8], xmm4 2486 | movsd qword [r4+8*4+3*8], xmm3 2487 | movsd qword [r4+8*4+2*8], xmm2 2488 | movsd qword [r4+8*4+1*8], xmm1 2489 | movsd qword [r4+8*4+0*8], xmm0 2490 |.endif 2491 | 2492 | // EX(opline) = opline 2493 | SAVE_IP 2494 | // zend_jit_trace_exit(trace_num, exit_num) 2495 | EXT_CALL zend_jit_trace_exit, r0 2496 |.if X64WIN 2497 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2498 |.elif X64 2499 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2500 |.else 2501 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2502 |.endif 2503 2504 | test eax, eax 2505 | jne >1 2506 2507 | // execute_data = EG(current_execute_data) 2508 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2509 | // opline = EX(opline) 2510 | LOAD_IP 2511 2512 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2513 | ADD_HYBRID_SPAD 2514 | JMP_IP 2515 } else if (GCC_GLOBAL_REGS) { 2516 | add r4, SPAD // stack alignment 2517 | JMP_IP 2518 } else { 2519 | mov FP, aword T2 // restore FP 2520 | mov RX, aword T3 // restore IP 2521 | add r4, NR_SPAD // stack alignment 2522 | mov r0, 1 // ZEND_VM_ENTER 2523 | ret 2524 } 2525 2526 |1: 2527 | jl ->trace_halt 2528 2529 | // execute_data = EG(current_execute_data) 2530 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2531 | // opline = EX(opline) 2532 | LOAD_IP 2533 2534 | // check for interrupt (try to avoid this ???) 2535 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 2536 | jne ->interrupt_handler 2537 2538 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2539 | ADD_HYBRID_SPAD 2540 | mov r0, EX->func 2541 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2542 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2543 | jmp aword [IP + r0] 2544 } else if (GCC_GLOBAL_REGS) { 2545 | add r4, SPAD // stack alignment 2546 | mov r0, EX->func 2547 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2548 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2549 | jmp aword [IP + r0] 2550 } else { 2551 | mov IP, aword EX->opline 2552 | mov FCARG1a, FP 2553 | mov r0, EX->func 2554 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2555 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2556 | call aword [IP + r0] 2557 | test eax, eax 2558 | jl ->trace_halt 2559 | mov FP, aword T2 // restore FP 2560 | mov RX, aword T3 // restore IP 2561 | add r4, NR_SPAD // stack alignment 2562 | mov r0, 1 // ZEND_VM_ENTER 2563 | ret 2564 } 2565 2566 return 1; 2567} 2568 2569static int zend_jit_trace_escape_stub(dasm_State **Dst) 2570{ 2571 |->trace_escape: 2572 | 2573 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2574 | ADD_HYBRID_SPAD 2575 | JMP_IP 2576 } else if (GCC_GLOBAL_REGS) { 2577 | add r4, SPAD // stack alignment 2578 | JMP_IP 2579 } else { 2580 | mov FP, aword T2 // restore FP 2581 | mov RX, aword T3 // restore IP 2582 | add r4, NR_SPAD // stack alignment 2583 | mov r0, 1 // ZEND_VM_ENTER 2584 | ret 2585 } 2586 2587 return 1; 2588} 2589 2590/* Keep 32 exit points in a single code block */ 2591#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2592#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2593 2594static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2595{ 2596 uint32_t i; 2597 2598 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2599 | push byte i 2600 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2601 } 2602 | push byte i 2603 |// 1: 2604 | add aword [r4], n 2605 | jmp ->trace_exit 2606 2607 return 1; 2608} 2609 2610#ifdef CONTEXT_THREADED_JIT 2611static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2612{ 2613 |->context_threaded_call: 2614 | pop r0 2615 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2616 | ADD_HYBRID_SPAD 2617 | jmp aword [IP] 2618 } else if (GCC_GLOBAL_REGS) { 2619 | add r4, SPAD // stack alignment 2620 | jmp aword [IP] 2621 } else { 2622 ZEND_UNREACHABLE(); 2623 // TODO: context threading can't work without GLOBAL REGS because we have to change 2624 // the value of execute_data in execute_ex() 2625 | mov FCARG1a, FP 2626 | mov r0, aword [FP] 2627 | mov FP, aword T2 // restore FP 2628 | mov RX, aword T3 // restore IP 2629 | add r4, NR_SPAD // stack alignment 2630 | jmp aword [r0] 2631 } 2632 return 1; 2633} 2634#endif 2635 2636static int zend_jit_assign_const_stub(dasm_State **Dst) 2637{ 2638 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2639 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2640 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2641 2642 |->assign_const: 2643 |.if X64WIN 2644 | sub r4, 0x28 2645 |.elif X64 2646 | sub r4, 8 2647 |.else 2648 | sub r4, 12 2649 |.endif 2650 if (!zend_jit_assign_to_variable( 2651 Dst, NULL, 2652 var_addr, var_addr, -1, -1, 2653 IS_CONST, val_addr, val_info, 2654 0, 0)) { 2655 return 0; 2656 } 2657 |.if X64WIN 2658 | add r4, 0x28 2659 |.elif X64 2660 | add r4, 8 2661 |.else 2662 | add r4, 12 2663 |.endif 2664 | ret 2665 return 1; 2666} 2667 2668static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2669{ 2670 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2671 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2672 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2673 2674 |->assign_tmp: 2675 |.if X64WIN 2676 | sub r4, 0x28 2677 |.elif X64 2678 | sub r4, 8 2679 |.else 2680 | sub r4, 12 2681 |.endif 2682 if (!zend_jit_assign_to_variable( 2683 Dst, NULL, 2684 var_addr, var_addr, -1, -1, 2685 IS_TMP_VAR, val_addr, val_info, 2686 0, 0)) { 2687 return 0; 2688 } 2689 |.if X64WIN 2690 | add r4, 0x28 2691 |.elif X64 2692 | add r4, 8 2693 |.else 2694 | add r4, 12 2695 |.endif 2696 | ret 2697 return 1; 2698} 2699 2700static int zend_jit_assign_var_stub(dasm_State **Dst) 2701{ 2702 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2703 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2704 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2705 2706 |->assign_var: 2707 |.if X64WIN 2708 | sub r4, 0x28 2709 |.elif X64 2710 | sub r4, 8 2711 |.else 2712 | sub r4, 12 2713 |.endif 2714 if (!zend_jit_assign_to_variable( 2715 Dst, NULL, 2716 var_addr, var_addr, -1, -1, 2717 IS_VAR, val_addr, val_info, 2718 0, 0)) { 2719 return 0; 2720 } 2721 |.if X64WIN 2722 | add r4, 0x28 2723 |.elif X64 2724 | add r4, 8 2725 |.else 2726 | add r4, 12 2727 |.endif 2728 | ret 2729 return 1; 2730} 2731 2732static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2733{ 2734 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2735 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2736 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2737 2738 |->assign_cv_noref: 2739 |.if X64WIN 2740 | sub r4, 0x28 2741 |.elif X64 2742 | sub r4, 8 2743 |.else 2744 | sub r4, 12 2745 |.endif 2746 if (!zend_jit_assign_to_variable( 2747 Dst, NULL, 2748 var_addr, var_addr, -1, -1, 2749 IS_CV, val_addr, val_info, 2750 0, 0)) { 2751 return 0; 2752 } 2753 |.if X64WIN 2754 | add r4, 0x28 2755 |.elif X64 2756 | add r4, 8 2757 |.else 2758 | add r4, 12 2759 |.endif 2760 | ret 2761 return 1; 2762} 2763 2764static int zend_jit_assign_cv_stub(dasm_State **Dst) 2765{ 2766 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2767 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2768 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2769 2770 |->assign_cv: 2771 |.if X64WIN 2772 | sub r4, 0x28 2773 |.elif X64 2774 | sub r4, 8 2775 |.else 2776 | sub r4, 12 2777 |.endif 2778 if (!zend_jit_assign_to_variable( 2779 Dst, NULL, 2780 var_addr, var_addr, -1, -1, 2781 IS_CV, val_addr, val_info, 2782 0, 0)) { 2783 return 0; 2784 } 2785 |.if X64WIN 2786 | add r4, 0x28 2787 |.elif X64 2788 | add r4, 8 2789 |.else 2790 | add r4, 12 2791 |.endif 2792 | ret 2793 return 1; 2794} 2795 2796static const zend_jit_stub zend_jit_stubs[] = { 2797 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2798 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2799 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2800 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2801 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2802 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2803 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2804 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2805 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2806 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2807 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2808 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2809 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2810 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2811 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2812 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2813 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2814 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2815 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2816 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2817 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2818 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2819 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2820 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2821 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2822 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2823 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2824 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2825 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2826 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2827 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2828 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2829 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2830 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2831 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2832 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2833 JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE), 2834#ifdef CONTEXT_THREADED_JIT 2835 JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE), 2836#endif 2837}; 2838 2839#if ZTS && defined(ZEND_WIN32) 2840extern uint32_t _tls_index; 2841extern char *_tls_start; 2842extern char *_tls_end; 2843#endif 2844 2845#ifdef HAVE_GDB 2846typedef struct _Unwind_Context _Unwind_Context; 2847typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2848extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2849extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2850 2851typedef struct _zend_jit_unwind_arg { 2852 int cnt; 2853 uintptr_t cfa[3]; 2854} zend_jit_unwind_arg; 2855 2856static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2857{ 2858 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2859 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2860 arg->cnt++; 2861 if (arg->cnt == 3) { 2862 return 5; // _URC_END_OF_STACK 2863 } 2864 return 0; // _URC_NO_REASON; 2865} 2866 2867static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2868{ 2869 zend_jit_unwind_arg arg; 2870 2871 memset(&arg, 0, sizeof(arg)); 2872 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2873 if (arg.cnt == 3) { 2874 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2875 } 2876} 2877 2878extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2879 2880static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2881{ 2882 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2883 2884 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2885 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2886 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2887 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2888} 2889#endif 2890 2891static int zend_jit_setup(void) 2892{ 2893 if (!zend_cpu_supports_sse2()) { 2894 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2895 return FAILURE; 2896 } 2897 allowed_opt_flags = 0; 2898 if (zend_cpu_supports_avx()) { 2899 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2900 } 2901 2902#if ZTS 2903# ifdef _WIN64 2904 tsrm_tls_index = _tls_index * sizeof(void*); 2905 2906 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2907 /* Probably, it might be better solution */ 2908 do { 2909 void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; 2910 void *val = _tsrm_ls_cache; 2911 size_t offset = 0; 2912 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2913 2914 while (offset < size) { 2915 if (*tls_mem == val) { 2916 tsrm_tls_offset = offset; 2917 break; 2918 } 2919 tls_mem++; 2920 offset += sizeof(void*); 2921 } 2922 if (offset >= size) { 2923 // TODO: error message ??? 2924 return FAILURE; 2925 } 2926 } while(0); 2927# elif ZEND_WIN32 2928 tsrm_tls_index = _tls_index * sizeof(void*); 2929 2930 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2931 /* Probably, it might be better solution */ 2932 do { 2933 void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; 2934 void *val = _tsrm_ls_cache; 2935 size_t offset = 0; 2936 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2937 2938 while (offset < size) { 2939 if (*tls_mem == val) { 2940 tsrm_tls_offset = offset; 2941 break; 2942 } 2943 tls_mem++; 2944 offset += sizeof(void*); 2945 } 2946 if (offset >= size) { 2947 // TODO: error message ??? 2948 return FAILURE; 2949 } 2950 } while(0); 2951# elif defined(__APPLE__) && defined(__x86_64__) 2952 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2953 if (tsrm_ls_cache_tcb_offset == 0) { 2954 size_t *ti; 2955 __asm__( 2956 "leaq __tsrm_ls_cache(%%rip),%0" 2957 : "=r" (ti)); 2958 tsrm_tls_offset = ti[2]; 2959 tsrm_tls_index = ti[1] * 8; 2960 } 2961# elif defined(__GNUC__) && defined(__x86_64__) 2962 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2963 if (tsrm_ls_cache_tcb_offset == 0) { 2964#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) 2965 size_t ret; 2966 2967 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2968 : "=r" (ret)); 2969 tsrm_ls_cache_tcb_offset = ret; 2970#else 2971 size_t *ti; 2972 2973 __asm__( 2974 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2975 : "=a" (ti)); 2976 tsrm_tls_offset = ti[1]; 2977 tsrm_tls_index = ti[0] * 16; 2978#endif 2979 } 2980# elif defined(__GNUC__) && defined(__i386__) 2981 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2982 if (tsrm_ls_cache_tcb_offset == 0) { 2983#if !defined(__FreeBSD__) && !defined(__OpenBSD__) 2984 size_t ret; 2985 2986 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 2987 : "=a" (ret)); 2988 tsrm_ls_cache_tcb_offset = ret; 2989#else 2990 size_t *ti, _ebx, _ecx, _edx; 2991 2992 __asm__( 2993 "call 1f\n" 2994 ".subsection 1\n" 2995 "1:\tmovl (%%esp), %%ebx\n\t" 2996 "ret\n" 2997 ".previous\n\t" 2998 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 2999 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 3000 "call ___tls_get_addr@plt\n\t" 3001 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 3002 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 3003 tsrm_tls_offset = ti[1]; 3004 tsrm_tls_index = ti[0] * 8; 3005#endif 3006 } 3007# endif 3008#endif 3009 3010 memset(sp_adj, 0, sizeof(sp_adj)); 3011#ifdef HAVE_GDB 3012 sp_adj[SP_ADJ_RET] = sizeof(void*); 3013 |.if X64WIN 3014 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28 3015 |.elif X64 3016 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8 3017 |.else 3018 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12 3019 |.endif 3020 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3021 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 3022#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3023 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 3024#else 3025 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 3026#endif 3027 } else if (GCC_GLOBAL_REGS) { 3028 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 3029 } else { 3030 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 3031 } 3032#endif 3033 3034 return SUCCESS; 3035} 3036 3037static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 3038{ 3039 | int3 3040 return 1; 3041} 3042 3043static int zend_jit_align_func(dasm_State **Dst) 3044{ 3045 reuse_ip = 0; 3046 delayed_call_chain = 0; 3047 last_valid_opline = NULL; 3048 use_last_vald_opline = 0; 3049 track_last_valid_opline = 0; 3050 jit_return_label = -1; 3051 |.align 16 3052 return 1; 3053} 3054 3055static int zend_jit_prologue(dasm_State **Dst) 3056{ 3057 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3058 | SUB_HYBRID_SPAD 3059 } else if (GCC_GLOBAL_REGS) { 3060 | sub r4, SPAD // stack alignment 3061 } else { 3062 | sub r4, NR_SPAD // stack alignment 3063 | mov aword T2, FP // save FP 3064 | mov aword T3, RX // save IP 3065 | mov FP, FCARG1a 3066 } 3067 return 1; 3068} 3069 3070static int zend_jit_label(dasm_State **Dst, unsigned int label) 3071{ 3072 |=>label: 3073 return 1; 3074} 3075 3076static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3077{ 3078 | // call->prev_execute_data = EX(call); 3079 if (call_level == 1) { 3080 | mov aword EX:RX->prev_execute_data, 0 3081 } else { 3082 | mov r0, EX->call 3083 | mov EX:RX->prev_execute_data, r0 3084 } 3085 | // EX(call) = call; 3086 | mov EX->call, RX 3087 3088 delayed_call_chain = 0; 3089 3090 return 1; 3091} 3092 3093static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3094{ 3095 if (last_valid_opline == opline) { 3096 zend_jit_use_last_valid_opline(); 3097 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3098 zend_jit_use_last_valid_opline(); 3099 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3100 } else { 3101 | LOAD_IP_ADDR opline 3102 } 3103 zend_jit_set_last_valid_opline(opline); 3104 3105 return 1; 3106} 3107 3108static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3109{ 3110 if (last_valid_opline == opline) { 3111 zend_jit_use_last_valid_opline(); 3112 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3113 zend_jit_use_last_valid_opline(); 3114 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3115 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3116 | LOAD_ADDR RX, opline 3117 | mov aword EX->opline, RX 3118 } else { 3119 | LOAD_IP_ADDR opline 3120 } 3121 zend_jit_set_last_valid_opline(opline); 3122 3123 return 1; 3124} 3125 3126static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3127{ 3128 if (delayed_call_chain) { 3129 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3130 return 0; 3131 } 3132 } 3133 if (!zend_jit_set_ip(Dst, opline)) { 3134 return 0; 3135 } 3136 reuse_ip = 0; 3137 return 1; 3138} 3139 3140static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3141{ 3142#if 0 3143 if (!zend_jit_set_valid_ip(Dst, opline)) { 3144 return 0; 3145 } 3146 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3147 | jne ->interrupt_handler 3148#else 3149 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3150 if (exit_addr) { 3151 | jne &exit_addr 3152 } else if (last_valid_opline == opline) { 3153 || zend_jit_use_last_valid_opline(); 3154 | jne ->interrupt_handler 3155 } else { 3156 | jne >1 3157 |.cold_code 3158 |1: 3159 | LOAD_IP_ADDR opline 3160 | jmp ->interrupt_handler 3161 |.code 3162 } 3163#endif 3164 return 1; 3165} 3166 3167static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3168{ 3169 if (timeout_exit_addr) { 3170 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3171 | je =>loop_label 3172 | jmp &timeout_exit_addr 3173 } else { 3174 | jmp =>loop_label 3175 } 3176 return 1; 3177} 3178 3179static int zend_jit_check_exception(dasm_State **Dst) 3180{ 3181 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3182 | jne ->exception_handler 3183 return 1; 3184} 3185 3186static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3187{ 3188 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3189 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3190 | jne ->exception_handler_undef 3191 return 1; 3192 } 3193 return zend_jit_check_exception(Dst); 3194} 3195 3196static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3197{ 3198 zend_regset regset = ZEND_REGSET_SCRATCH; 3199 3200#if ZTS 3201 if (1) { 3202#else 3203 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3204#endif 3205 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3206 if (parent) { 3207 int i; 3208 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3209 zend_jit_trace_stack *parent_stack = 3210 parent->stack_map + 3211 parent->exit_info[exit_num].stack_offset; 3212 3213 for (i = 0; i < parent_vars_count; i++) { 3214 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3215 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3216 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3217 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { 3218 ZEND_REGSET_EXCL(regset, ZREG_R0); 3219 } 3220 } 3221 } 3222 } 3223 } 3224 3225 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3226 ZEND_REGSET_EXCL(regset, ZREG_R0); 3227 } 3228 3229 current_trace_num = trace_num; 3230 3231 | // EG(jit_trace_num) = trace_num; 3232 if (regset == ZEND_REGSET_EMPTY) { 3233 | push r0 3234 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, r0 3235 | pop r0 3236 } else { 3237 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3238 3239 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3240 (void)tmp; 3241 } 3242 3243 return 1; 3244} 3245 3246static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 3247{ 3248 |.cold_code 3249 |=>1: // end of the code 3250 |.code 3251 return 1; 3252} 3253 3254/* This taken from LuaJIT. Thanks to Mike Pall. */ 3255static uint32_t _asm_x86_inslen(const uint8_t* p) 3256{ 3257 static const uint8_t map_op1[256] = { 3258 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3259 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3260 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3261 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3262#if defined(__x86_64__) || defined(_M_X64) 3263 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3264#else 3265 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3266#endif 3267 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3268 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3269 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3270 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3271 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3272#if defined(__x86_64__) || defined(_M_X64) 3273 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3274#else 3275 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3276#endif 3277 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3278 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3279 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3280 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3281 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3282 }; 3283 static const uint8_t map_op2[256] = { 3284 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3285 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3286 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3287 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3288 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3289 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3290 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3291 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3292 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3293 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3294 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3295 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3296 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3297 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3298 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3299 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3300 }; 3301 uint32_t result = 0; 3302 uint32_t prefixes = 0; 3303 uint32_t x = map_op1[*p]; 3304 3305 for (;;) { 3306 switch (x >> 4) { 3307 case 0: 3308 return result + x + (prefixes & 4); 3309 case 1: 3310 prefixes |= x; 3311 x = map_op1[*++p]; 3312 result++; 3313 break; 3314 case 2: 3315 x = map_op2[*++p]; 3316 break; 3317 case 3: 3318 p++; 3319 goto mrm; 3320 case 4: 3321 result -= (prefixes & 2); 3322 /* fallthrough */ 3323 case 5: 3324 return result + (x & 15); 3325 case 6: /* Group 3. */ 3326 if (p[1] & 0x38) { 3327 x = 2; 3328 } else if ((prefixes & 2) && (x == 0x66)) { 3329 x = 4; 3330 } 3331 goto mrm; 3332 case 7: /* VEX c4/c5. */ 3333#if !defined(__x86_64__) && !defined(_M_X64) 3334 if (p[1] < 0xc0) { 3335 x = 2; 3336 goto mrm; 3337 } 3338#endif 3339 if (x == 0x70) { 3340 x = *++p & 0x1f; 3341 result++; 3342 if (x >= 2) { 3343 p += 2; 3344 result += 2; 3345 goto mrm; 3346 } 3347 } 3348 p++; 3349 result++; 3350 x = map_op2[*++p]; 3351 break; 3352 case 8: 3353 result -= (prefixes & 2); 3354 /* fallthrough */ 3355 case 9: 3356mrm: 3357 /* ModR/M and possibly SIB. */ 3358 result += (x & 15); 3359 x = *++p; 3360 switch (x >> 6) { 3361 case 0: 3362 if ((x & 7) == 5) { 3363 return result + 4; 3364 } 3365 break; 3366 case 1: 3367 result++; 3368 break; 3369 case 2: 3370 result += 4; 3371 break; 3372 case 3: 3373 return result; 3374 } 3375 if ((x & 7) == 4) { 3376 result++; 3377 if (x < 0x40 && (p[1] & 7) == 5) { 3378 result += 4; 3379 } 3380 } 3381 return result; 3382 } 3383 } 3384} 3385 3386typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3387typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3388 3389static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3390{ 3391 int ret = 0; 3392 uint8_t *p, *end; 3393 3394 if (jmp_table_size) { 3395 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3396 3397 do { 3398 if (*jmp_slot == from_addr) { 3399 *jmp_slot = to_addr; 3400 ret++; 3401 } 3402 jmp_slot++; 3403 } while (--jmp_table_size); 3404 } 3405 3406 p = (uint8_t*)code; 3407 end = p + size - 5; 3408 while (p < end) { 3409 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3410 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3411 ret++; 3412 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3413 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3414 ret++; 3415 } 3416 p += _asm_x86_inslen(p); 3417 } 3418#ifdef HAVE_VALGRIND 3419 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3420#endif 3421 return ret; 3422} 3423 3424static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3425{ 3426 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3427} 3428 3429static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3430{ 3431 const void *link_addr; 3432 size_t prologue_size; 3433 3434 /* Skip prologue. */ 3435 // TODO: don't hardcode this ??? 3436 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3437#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3438 prologue_size = 0; 3439#elif defined(__x86_64__) || defined(_M_X64) 3440 // sub r4, HYBRID_SPAD 3441 prologue_size = 4; 3442#else 3443 // sub r4, HYBRID_SPAD 3444 prologue_size = 3; 3445#endif 3446 } else if (GCC_GLOBAL_REGS) { 3447 // sub r4, SPAD // stack alignment 3448#if defined(__x86_64__) || defined(_M_X64) 3449 prologue_size = 4; 3450#else 3451 prologue_size = 3; 3452#endif 3453 } else { 3454 // sub r4, NR_SPAD // stack alignment 3455 // mov aword T2, FP // save FP 3456 // mov aword T3, RX // save IP 3457 // mov FP, FCARG1a 3458#if defined(__x86_64__) || defined(_M_X64) 3459 prologue_size = 17; 3460#else 3461 prologue_size = 12; 3462#endif 3463 } 3464 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3465 3466 if (timeout_exit_addr) { 3467 /* Check timeout for links to LOOP */ 3468 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3469 | je &link_addr 3470 | jmp &timeout_exit_addr 3471 } else { 3472 | jmp &link_addr 3473 } 3474 return 1; 3475} 3476 3477static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3478{ 3479#if 0 3480 | jmp ->trace_escape 3481#else 3482 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3483 | ADD_HYBRID_SPAD 3484 if (!original_handler) { 3485 | JMP_IP 3486 } else { 3487 | mov r0, EX->func 3488 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3489 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3490 | jmp aword [IP + r0] 3491 } 3492 } else if (GCC_GLOBAL_REGS) { 3493 | add r4, SPAD // stack alignment 3494 if (!original_handler) { 3495 | JMP_IP 3496 } else { 3497 | mov r0, EX->func 3498 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3499 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3500 | jmp aword [IP + r0] 3501 } 3502 } else { 3503 if (original_handler) { 3504 | mov FCARG1a, FP 3505 | mov r0, EX->func 3506 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3507 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3508 | call aword [IP + r0] 3509 } 3510 | mov FP, aword T2 // restore FP 3511 | mov RX, aword T3 // restore IP 3512 | add r4, NR_SPAD // stack alignment 3513 if (!original_handler || !opline || 3514 (opline->opcode != ZEND_RETURN 3515 && opline->opcode != ZEND_RETURN_BY_REF 3516 && opline->opcode != ZEND_GENERATOR_RETURN 3517 && opline->opcode != ZEND_GENERATOR_CREATE 3518 && opline->opcode != ZEND_YIELD 3519 && opline->opcode != ZEND_YIELD_FROM)) { 3520 | mov r0, 2 // ZEND_VM_LEAVE 3521 } 3522 | ret 3523 } 3524#endif 3525 return 1; 3526} 3527 3528static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3529{ 3530 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3531 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3532 3533 if (!exit_addr) { 3534 return 0; 3535 } 3536 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3537 3538 return 1; 3539} 3540 3541static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3542{ 3543 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3544 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3545 3546 if (!exit_addr) { 3547 return 0; 3548 } 3549 | cmp byte [FP+var+offsetof(zval, u1.v.type)], IS_STRING 3550 | jae &exit_addr 3551 3552 return 1; 3553} 3554 3555static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3556{ 3557 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3558 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3559 3560 if (!exit_addr) { 3561 return 0; 3562 } 3563 3564 | GET_ZVAL_LVAL ZREG_FCARG1, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3565 if (op_info & MAY_BE_ARRAY_PACKED) { 3566 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3567 | jz &exit_addr 3568 } else { 3569 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3570 | jnz &exit_addr 3571 } 3572 3573 return 1; 3574} 3575 3576static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3577{ 3578 zend_jit_op_array_trace_extension *jit_extension = 3579 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3580 size_t offset = jit_extension->offset; 3581 const void *handler = 3582 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3583 3584 if (!zend_jit_set_valid_ip(Dst, opline)) { 3585 return 0; 3586 } 3587 if (!GCC_GLOBAL_REGS) { 3588 | mov FCARG1a, FP 3589 } 3590 | EXT_CALL handler, r0 3591 if (may_throw 3592 && opline->opcode != ZEND_RETURN 3593 && opline->opcode != ZEND_RETURN_BY_REF) { 3594 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 3595 | jne ->exception_handler 3596 } 3597 3598 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3599 trace++; 3600 } 3601 3602 if (!GCC_GLOBAL_REGS 3603 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3604 if (opline->opcode == ZEND_RETURN || 3605 opline->opcode == ZEND_RETURN_BY_REF || 3606 opline->opcode == ZEND_DO_UCALL || 3607 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3608 opline->opcode == ZEND_DO_FCALL || 3609 opline->opcode == ZEND_GENERATOR_CREATE) { 3610 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r1 3611 } 3612 } 3613 3614 if (zend_jit_trace_may_exit(op_array, opline)) { 3615 if (opline->opcode == ZEND_RETURN || 3616 opline->opcode == ZEND_RETURN_BY_REF || 3617 opline->opcode == ZEND_GENERATOR_CREATE) { 3618 3619 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3620#if 0 3621 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3622 | cmp IP, zend_jit_halt_op 3623 | je ->trace_halt 3624#endif 3625 } else if (GCC_GLOBAL_REGS) { 3626 | test IP, IP 3627 | je ->trace_halt 3628 } else { 3629 | test eax, eax 3630 | jl ->trace_halt 3631 } 3632 } else if (opline->opcode == ZEND_EXIT || 3633 opline->opcode == ZEND_GENERATOR_RETURN || 3634 opline->opcode == ZEND_YIELD || 3635 opline->opcode == ZEND_YIELD_FROM) { 3636 | jmp ->trace_halt 3637 } 3638 if (trace->op != ZEND_JIT_TRACE_END || 3639 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3640 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3641 3642 const zend_op *next_opline = trace->opline; 3643 const zend_op *exit_opline = NULL; 3644 uint32_t exit_point; 3645 const void *exit_addr; 3646 uint32_t old_info = 0; 3647 uint32_t old_res_info = 0; 3648 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3649 3650 if (zend_is_smart_branch(opline)) { 3651 bool exit_if_true = 0; 3652 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3653 } else { 3654 switch (opline->opcode) { 3655 case ZEND_JMPZ: 3656 case ZEND_JMPNZ: 3657 case ZEND_JMPZ_EX: 3658 case ZEND_JMPNZ_EX: 3659 case ZEND_JMP_SET: 3660 case ZEND_COALESCE: 3661 case ZEND_JMP_NULL: 3662 case ZEND_FE_RESET_R: 3663 case ZEND_FE_RESET_RW: 3664 exit_opline = (trace->opline == opline + 1) ? 3665 OP_JMP_ADDR(opline, opline->op2) : 3666 opline + 1; 3667 break; 3668 case ZEND_JMPZNZ: 3669 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3670 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3671 OP_JMP_ADDR(opline, opline->op2); 3672 break; 3673 case ZEND_FE_FETCH_R: 3674 case ZEND_FE_FETCH_RW: 3675 if (opline->op2_type == IS_CV) { 3676 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3677 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3678 } 3679 exit_opline = (trace->opline == opline + 1) ? 3680 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3681 opline + 1; 3682 break; 3683 3684 } 3685 } 3686 3687 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3688 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3689 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3690 } 3691 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3692 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3693 3694 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3695 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3696 } 3697 switch (opline->opcode) { 3698 case ZEND_FE_FETCH_R: 3699 case ZEND_FE_FETCH_RW: 3700 if (opline->op2_type == IS_CV) { 3701 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3702 } 3703 break; 3704 } 3705 3706 if (!exit_addr) { 3707 return 0; 3708 } 3709 | CMP_IP next_opline 3710 | jne &exit_addr 3711 } 3712 } 3713 3714 zend_jit_set_last_valid_opline(trace->opline); 3715 3716 return 1; 3717} 3718 3719static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3720{ 3721 const void *handler; 3722 3723 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3724 handler = zend_get_opcode_handler_func(opline); 3725 } else { 3726 handler = opline->handler; 3727 } 3728 3729 if (!zend_jit_set_valid_ip(Dst, opline)) { 3730 return 0; 3731 } 3732 if (!GCC_GLOBAL_REGS) { 3733 | mov FCARG1a, FP 3734 } 3735 | EXT_CALL handler, r0 3736 if (may_throw) { 3737 zend_jit_check_exception(Dst); 3738 } 3739 3740 /* Skip the following OP_DATA */ 3741 switch (opline->opcode) { 3742 case ZEND_ASSIGN_DIM: 3743 case ZEND_ASSIGN_OBJ: 3744 case ZEND_ASSIGN_STATIC_PROP: 3745 case ZEND_ASSIGN_DIM_OP: 3746 case ZEND_ASSIGN_OBJ_OP: 3747 case ZEND_ASSIGN_STATIC_PROP_OP: 3748 case ZEND_ASSIGN_STATIC_PROP_REF: 3749 case ZEND_ASSIGN_OBJ_REF: 3750 zend_jit_set_last_valid_opline(opline + 2); 3751 break; 3752 default: 3753 zend_jit_set_last_valid_opline(opline + 1); 3754 break; 3755 } 3756 3757 return 1; 3758} 3759 3760static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3761{ 3762 if (!zend_jit_set_valid_ip(Dst, opline)) { 3763 return 0; 3764 } 3765 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3766 if (opline->opcode == ZEND_DO_UCALL || 3767 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3768 opline->opcode == ZEND_DO_FCALL || 3769 opline->opcode == ZEND_RETURN) { 3770 3771 /* Use inlined HYBRID VM handler */ 3772 const void *handler = opline->handler; 3773 3774 | ADD_HYBRID_SPAD 3775 | EXT_JMP handler, r0 3776 } else { 3777 const void *handler = zend_get_opcode_handler_func(opline); 3778 3779 | EXT_CALL handler, r0 3780 | ADD_HYBRID_SPAD 3781 | JMP_IP 3782 } 3783 } else { 3784 const void *handler = opline->handler; 3785 3786 if (GCC_GLOBAL_REGS) { 3787 | add r4, SPAD // stack alignment 3788 } else { 3789 | mov FCARG1a, FP 3790 | mov FP, aword T2 // restore FP 3791 | mov RX, aword T3 // restore IP 3792 | add r4, NR_SPAD // stack alignment 3793 } 3794 | EXT_JMP handler, r0 3795 } 3796 zend_jit_reset_last_valid_opline(); 3797 return 1; 3798} 3799 3800static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3801{ 3802 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3803 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3804 3805 if (!exit_addr) { 3806 return 0; 3807 } 3808 | CMP_IP opline 3809 | jne &exit_addr 3810 3811 zend_jit_set_last_valid_opline(opline); 3812 3813 return 1; 3814} 3815 3816static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3817{ 3818 | jmp =>target_label 3819 return 1; 3820} 3821 3822static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3823{ 3824 | CMP_IP next_opline 3825 | jne =>target_label 3826 3827 zend_jit_set_last_valid_opline(next_opline); 3828 3829 return 1; 3830} 3831 3832#ifdef CONTEXT_THREADED_JIT 3833static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3834{ 3835 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3836 if (opline->opcode == ZEND_DO_UCALL) { 3837 | call ->context_threaded_call 3838 } else { 3839 const zend_op *next_opline = opline + 1; 3840 3841 | CMP_IP next_opline 3842 | je =>next_block 3843 | call ->context_threaded_call 3844 } 3845 return 1; 3846} 3847#endif 3848 3849static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3850{ 3851#ifdef CONTEXT_THREADED_JIT 3852 return zend_jit_context_threaded_call(Dst, opline, next_block); 3853#else 3854 return zend_jit_tail_handler(Dst, opline); 3855#endif 3856} 3857 3858static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3859{ 3860 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3861 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3862 3863 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3864 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3865 if (set_type && 3866 (Z_REG(dst) != ZREG_FP || 3867 !JIT_G(current_frame) || 3868 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3869 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3870 } 3871 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3872 | DOUBLE_SET_ZVAL_DVAL dst, Z_REG(src) 3873 if (set_type && 3874 (Z_REG(dst) != ZREG_FP || 3875 !JIT_G(current_frame) || 3876 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3877 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3878 } 3879 } else { 3880 ZEND_UNREACHABLE(); 3881 } 3882 return 1; 3883} 3884 3885static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3886{ 3887 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3888 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3889 3890 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3891 | GET_ZVAL_LVAL Z_REG(dst), src 3892 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3893 | DOUBLE_GET_ZVAL_DVAL Z_REG(dst), src 3894 } else { 3895 ZEND_UNREACHABLE(); 3896 } 3897 return 1; 3898} 3899 3900static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3901{ 3902 zend_jit_addr src = ZEND_ADDR_REG(reg); 3903 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3904 3905 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3906} 3907 3908static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3909{ 3910 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3911 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3912 return zend_jit_spill_store(Dst, src, dst, info, 1); 3913 } 3914 return 1; 3915} 3916 3917static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3918{ 3919 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3920 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3921 bool set_type = 1; 3922 3923 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3924 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3925 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3926 set_type = 0; 3927 } 3928 } 3929 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3930 } 3931 return 1; 3932} 3933 3934static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3935{ 3936 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3937 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3938 3939 return zend_jit_load_reg(Dst, src, dst, info); 3940} 3941 3942static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3943{ 3944 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3945 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3946 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3947 } 3948 return 1; 3949} 3950 3951static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3952{ 3953 if (!zend_jit_same_addr(src, dst)) { 3954 if (Z_MODE(src) == IS_REG) { 3955 if (Z_MODE(dst) == IS_REG) { 3956 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3957 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3958 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3959 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3960 } else { 3961 ZEND_UNREACHABLE(); 3962 } 3963 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3964 if (!Z_LOAD(src) && !Z_STORE(src)) { 3965 if (!zend_jit_spill_store(Dst, src, dst, info, 3966 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3967 JIT_G(current_frame) == NULL || 3968 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3969 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3970 )) { 3971 return 0; 3972 } 3973 } 3974 } else { 3975 ZEND_UNREACHABLE(); 3976 } 3977 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3978 if (Z_MODE(dst) == IS_REG) { 3979 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3980 return 0; 3981 } 3982 } else { 3983 ZEND_UNREACHABLE(); 3984 } 3985 } else { 3986 ZEND_UNREACHABLE(); 3987 } 3988 } 3989 return 1; 3990} 3991 3992static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3993{ 3994 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 3995 3996 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 3997 3998 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3999 if (!zend_jit_save_call_chain(Dst, -1)) { 4000 return 0; 4001 } 4002 } 4003 4004 ZEND_ASSERT(opline); 4005 4006 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 4007 && (opline-1)->opcode != ZEND_FETCH_LIST_R 4008 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 4009 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 4010 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 4011 4012 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 4013 | GET_ZVAL_PTR r0, val_addr 4014 | GC_ADDREF r0 4015 |2: 4016 } 4017 4018 | LOAD_IP_ADDR (opline - 1) 4019 | jmp ->trace_escape 4020 |1: 4021 4022 return 1; 4023} 4024 4025static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 4026{ 4027 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 4028 4029 if (reg == ZREG_LONG_MIN_MINUS_1) { 4030 |.if X64 4031 | SET_ZVAL_LVAL dst, 0x00000000 4032 | SET_ZVAL_W2 dst, 0xc3e00000 4033 |.else 4034 | SET_ZVAL_LVAL dst, 0x00200000 4035 | SET_ZVAL_W2 dst, 0xc1e00000 4036 |.endif 4037 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4038 } else if (reg == ZREG_LONG_MIN) { 4039 |.if X64 4040 | SET_ZVAL_LVAL dst, 0x00000000 4041 | SET_ZVAL_W2 dst, 0x80000000 4042 |.else 4043 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4044 |.endif 4045 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4046 } else if (reg == ZREG_LONG_MAX) { 4047 |.if X64 4048 | SET_ZVAL_LVAL dst, 0xffffffff 4049 | SET_ZVAL_W2 dst, 0x7fffffff 4050 |.else 4051 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4052 |.endif 4053 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4054 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4055 |.if X64 4056 | SET_ZVAL_LVAL dst, 0 4057 | SET_ZVAL_W2 dst, 0x43e00000 4058 |.else 4059 | SET_ZVAL_LVAL dst, 0 4060 | SET_ZVAL_W2 dst, 0x41e00000 4061 |.endif 4062 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4063 } else if (reg == ZREG_NULL) { 4064 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4065 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4066 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4067 | GET_ZVAL_PTR r1, dst 4068 | GC_ADDREF r1 4069 |1: 4070 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 4071 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4072 4073 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4074 | TRY_ADDREF -1, ch, r2 4075 } else { 4076 ZEND_UNREACHABLE(); 4077 } 4078 return 1; 4079} 4080 4081static int zend_jit_free_trampoline(dasm_State **Dst) 4082{ 4083 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4084 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4085 | jz >1 4086 | mov FCARG1a, r0 4087 | EXT_CALL zend_jit_free_trampoline_helper, r0 4088 |1: 4089 return 1; 4090} 4091 4092static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4093{ 4094 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4095 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4096 } 4097 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4098 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4099 } 4100 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4101 return 0; 4102 } 4103 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4104 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4105 } else { 4106 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4107 } 4108 4109 if (may_overflow && 4110 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4111 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4112 int32_t exit_point; 4113 const void *exit_addr; 4114 zend_jit_trace_stack *stack; 4115 uint32_t old_op1_info, old_res_info = 0; 4116 4117 stack = JIT_G(current_frame)->stack; 4118 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4119 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4120 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4121 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4122 } else { 4123 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4124 } 4125 if (opline->result_type != IS_UNUSED) { 4126 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4127 if (opline->opcode == ZEND_PRE_INC) { 4128 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4129 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4130 } else if (opline->opcode == ZEND_PRE_DEC) { 4131 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4132 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4133 } else if (opline->opcode == ZEND_POST_INC) { 4134 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4135 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4136 } else if (opline->opcode == ZEND_POST_DEC) { 4137 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4138 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4139 } 4140 } 4141 4142 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4143 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4144 | jo &exit_addr 4145 4146 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4147 opline->result_type != IS_UNUSED) { 4148 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4149 } 4150 4151 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4152 if (opline->result_type != IS_UNUSED) { 4153 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4154 } 4155 } else if (may_overflow) { 4156 | jo >1 4157 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4158 opline->result_type != IS_UNUSED) { 4159 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4160 } 4161 |.cold_code 4162 |1: 4163 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4164 |.if X64 4165 | mov64 rax, 0x43e0000000000000 4166 | SET_ZVAL_LVAL op1_def_addr, rax 4167 |.else 4168 | SET_ZVAL_LVAL op1_def_addr, 0 4169 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4170 |.endif 4171 } else { 4172 |.if X64 4173 | mov64 rax, 0xc3e0000000000000 4174 | SET_ZVAL_LVAL op1_def_addr, rax 4175 |.else 4176 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4177 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4178 |.endif 4179 } 4180 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4181 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4182 } 4183 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4184 opline->result_type != IS_UNUSED) { 4185 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4186 } 4187 | jmp >3 4188 |.code 4189 } else { 4190 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4191 opline->result_type != IS_UNUSED) { 4192 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4193 } 4194 } 4195 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4196 |.cold_code 4197 |2: 4198 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4199 | SET_EX_OPLINE opline, r0 4200 if (op1_info & MAY_BE_UNDEF) { 4201 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4202 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4203 | mov FCARG1d, opline->op1.var 4204 | EXT_CALL zend_jit_undefined_op_helper, r0 4205 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4206 op1_info |= MAY_BE_NULL; 4207 } 4208 |2: 4209 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4210 4211 | // ZVAL_DEREF(var_ptr); 4212 if (op1_info & MAY_BE_REF) { 4213 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4214 | GET_Z_PTR FCARG1a, FCARG1a 4215 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4216 | jz >1 4217 if (RETURN_VALUE_USED(opline)) { 4218 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4219 } else { 4220 | xor FCARG2a, FCARG2a 4221 } 4222 if (opline->opcode == ZEND_PRE_INC) { 4223 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4224 } else if (opline->opcode == ZEND_PRE_DEC) { 4225 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4226 } else if (opline->opcode == ZEND_POST_INC) { 4227 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4228 } else if (opline->opcode == ZEND_POST_DEC) { 4229 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4230 } else { 4231 ZEND_UNREACHABLE(); 4232 } 4233 zend_jit_check_exception(Dst); 4234 | jmp >3 4235 |1: 4236 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4237 |2: 4238 } 4239 4240 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4241 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 4242 4243 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4244 | TRY_ADDREF op1_info, ah, r2 4245 } 4246 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4247 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4248 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4249 | EXT_CALL zend_jit_pre_inc, r0 4250 } else { 4251 | EXT_CALL increment_function, r0 4252 } 4253 } else { 4254 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4255 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4256 | EXT_CALL zend_jit_pre_dec, r0 4257 } else { 4258 | EXT_CALL decrement_function, r0 4259 } 4260 } 4261 if (may_throw) { 4262 zend_jit_check_exception(Dst); 4263 } 4264 } else { 4265 zend_reg tmp_reg; 4266 4267 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4268 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4269 } 4270 if (Z_MODE(op1_def_addr) == IS_REG) { 4271 tmp_reg = Z_REG(op1_def_addr); 4272 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4273 tmp_reg = Z_REG(op1_addr); 4274 } else { 4275 tmp_reg = ZREG_XMM0; 4276 } 4277 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 4278 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4279 if (CAN_USE_AVX()) { 4280 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4281 } else { 4282 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4283 } 4284 } else { 4285 if (CAN_USE_AVX()) { 4286 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4287 } else { 4288 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4289 } 4290 } 4291 | DOUBLE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4292 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4293 opline->result_type != IS_UNUSED) { 4294 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4295 | TRY_ADDREF op1_def_info, ah, r1 4296 } 4297 } 4298 | jmp >3 4299 |.code 4300 } 4301 |3: 4302 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4303 return 0; 4304 } 4305 if (opline->result_type != IS_UNUSED) { 4306 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4307 return 0; 4308 } 4309 } 4310 return 1; 4311} 4312 4313static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4314{ 4315 if ((opline+1)->opcode == ZEND_OP_DATA 4316 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4317 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4318 return 1; 4319 } 4320 return 4321 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4322 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4323 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4324 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4325 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4326 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4327} 4328 4329static int zend_jit_math_long_long(dasm_State **Dst, 4330 const zend_op *opline, 4331 zend_uchar opcode, 4332 zend_jit_addr op1_addr, 4333 zend_jit_addr op2_addr, 4334 zend_jit_addr res_addr, 4335 uint32_t res_info, 4336 uint32_t res_use_info, 4337 int may_overflow) 4338{ 4339 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4340 zend_reg result_reg; 4341 zend_reg tmp_reg = ZREG_R0; 4342 4343 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4344 if (may_overflow && (res_info & MAY_BE_GUARD) 4345 && JIT_G(current_frame) 4346 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4347 result_reg = ZREG_R0; 4348 } else { 4349 result_reg = Z_REG(res_addr); 4350 } 4351 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4352 result_reg = Z_REG(op1_addr); 4353 } else if (Z_REG(res_addr) != ZREG_R0) { 4354 result_reg = ZREG_R0; 4355 } else { 4356 /* ASSIGN_DIM_OP */ 4357 result_reg = ZREG_FCARG1; 4358 tmp_reg = ZREG_FCARG1; 4359 } 4360 4361 if (opcode == ZEND_MUL && 4362 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4363 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4364 if (Z_MODE(op1_addr) == IS_REG && !may_overflow) { 4365 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4366 } else { 4367 | GET_ZVAL_LVAL result_reg, op1_addr 4368 | add Ra(result_reg), Ra(result_reg) 4369 } 4370 } else if (opcode == ZEND_MUL && 4371 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4372 !may_overflow && 4373 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4374 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4375 | GET_ZVAL_LVAL result_reg, op1_addr 4376 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4377 } else if (opcode == ZEND_MUL && 4378 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4379 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4380 if (Z_MODE(op2_addr) == IS_REG && !may_overflow) { 4381 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4382 } else { 4383 | GET_ZVAL_LVAL result_reg, op2_addr 4384 | add Ra(result_reg), Ra(result_reg) 4385 } 4386 } else if (opcode == ZEND_MUL && 4387 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4388 !may_overflow && 4389 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4390 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4391 | GET_ZVAL_LVAL result_reg, op2_addr 4392 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4393 } else if (opcode == ZEND_DIV && 4394 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4395 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4396 | GET_ZVAL_LVAL result_reg, op1_addr 4397 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4398 } else if (opcode == ZEND_ADD && 4399 !may_overflow && 4400 Z_MODE(op1_addr) == IS_REG && 4401 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4402 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4403 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4404 } else if (opcode == ZEND_ADD && 4405 !may_overflow && 4406 Z_MODE(op2_addr) == IS_REG && 4407 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4408 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4409 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4410 } else if (opcode == ZEND_SUB && 4411 !may_overflow && 4412 Z_MODE(op1_addr) == IS_REG && 4413 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4414 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4415 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4416 } else { 4417 | GET_ZVAL_LVAL result_reg, op1_addr 4418 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4419 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4420 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4421 /* +/- 0 */ 4422 may_overflow = 0; 4423 } else if (same_ops && opcode != ZEND_DIV) { 4424 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4425 } else { 4426 | LONG_MATH opcode, result_reg, op2_addr 4427 } 4428 } 4429 if (may_overflow) { 4430 if (res_info & MAY_BE_GUARD) { 4431 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4432 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4433 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4434 | jo &exit_addr 4435 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4436 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4437 } 4438 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4439 | jno &exit_addr 4440 } else { 4441 ZEND_UNREACHABLE(); 4442 } 4443 } else { 4444 if (res_info & MAY_BE_LONG) { 4445 | jo >1 4446 } else { 4447 | jno >1 4448 } 4449 } 4450 } 4451 4452 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4453 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4454 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4455 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4456 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4457 } 4458 } 4459 } 4460 4461 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4462 zend_reg tmp_reg1 = ZREG_XMM0; 4463 zend_reg tmp_reg2 = ZREG_XMM1; 4464 4465 if (res_info & MAY_BE_LONG) { 4466 |.cold_code 4467 |1: 4468 } 4469 4470 do { 4471 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4472 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4473 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4474 if (opcode == ZEND_ADD) { 4475 |.if X64 4476 | mov64 rax, 0x43e0000000000000 4477 if (Z_MODE(res_addr) == IS_REG) { 4478 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax 4479 } else { 4480 | SET_ZVAL_LVAL res_addr, rax 4481 } 4482 |.else 4483 | SET_ZVAL_LVAL res_addr, 0 4484 | SET_ZVAL_W2 res_addr, 0x41e00000 4485 |.endif 4486 break; 4487 } else if (opcode == ZEND_SUB) { 4488 |.if X64 4489 | mov64 rax, 0xc3e0000000000000 4490 if (Z_MODE(res_addr) == IS_REG) { 4491 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax 4492 } else { 4493 | SET_ZVAL_LVAL res_addr, rax 4494 } 4495 |.else 4496 | SET_ZVAL_LVAL res_addr, 0x00200000 4497 | SET_ZVAL_W2 res_addr, 0xc1e00000 4498 |.endif 4499 break; 4500 } 4501 } 4502 4503 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4504 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4505 if (CAN_USE_AVX()) { 4506 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4507 } else { 4508 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4509 } 4510 | DOUBLE_SET_ZVAL_DVAL res_addr, tmp_reg1 4511 } while (0); 4512 4513 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4514 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4515 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4516 } 4517 if (res_info & MAY_BE_LONG) { 4518 | jmp >2 4519 |.code 4520 } 4521 |2: 4522 } 4523 4524 return 1; 4525} 4526 4527static int zend_jit_math_long_double(dasm_State **Dst, 4528 zend_uchar opcode, 4529 zend_jit_addr op1_addr, 4530 zend_jit_addr op2_addr, 4531 zend_jit_addr res_addr, 4532 uint32_t res_use_info) 4533{ 4534 zend_reg result_reg = 4535 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4536 zend_reg tmp_reg; 4537 4538 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4539 /* ASSIGN_DIM_OP */ 4540 tmp_reg = ZREG_R1; 4541 } else { 4542 tmp_reg = ZREG_R0; 4543 } 4544 4545 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4546 4547 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4548 /* ASSIGN_DIM_OP */ 4549 if (CAN_USE_AVX()) { 4550 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4551 } else { 4552 | SSE_MATH opcode, result_reg, op2_addr, r1 4553 } 4554 } else { 4555 if (CAN_USE_AVX()) { 4556 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4557 } else { 4558 | SSE_MATH opcode, result_reg, op2_addr, r0 4559 } 4560 } 4561 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4562 4563 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4564 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4565 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4566 } 4567 } 4568 4569 return 1; 4570} 4571 4572static int zend_jit_math_double_long(dasm_State **Dst, 4573 zend_uchar opcode, 4574 zend_jit_addr op1_addr, 4575 zend_jit_addr op2_addr, 4576 zend_jit_addr res_addr, 4577 uint32_t res_use_info) 4578{ 4579 zend_reg result_reg, tmp_reg_gp; 4580 4581 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4582 /* ASSIGN_DIM_OP */ 4583 tmp_reg_gp = ZREG_R1; 4584 } else { 4585 tmp_reg_gp = ZREG_R0; 4586 } 4587 4588 if (zend_is_commutative(opcode) 4589 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4590 if (Z_MODE(res_addr) == IS_REG) { 4591 result_reg = Z_REG(res_addr); 4592 } else { 4593 result_reg = ZREG_XMM0; 4594 } 4595 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4596 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4597 /* ASSIGN_DIM_OP */ 4598 if (CAN_USE_AVX()) { 4599 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4600 } else { 4601 | SSE_MATH opcode, result_reg, op1_addr, r1 4602 } 4603 } else { 4604 if (CAN_USE_AVX()) { 4605 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4606 } else { 4607 | SSE_MATH opcode, result_reg, op1_addr, r0 4608 } 4609 } 4610 } else { 4611 zend_reg tmp_reg; 4612 4613 if (Z_MODE(res_addr) == IS_REG) { 4614 result_reg = Z_REG(res_addr); 4615 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4616 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4617 result_reg = Z_REG(op1_addr); 4618 tmp_reg = ZREG_XMM0; 4619 } else { 4620 result_reg = ZREG_XMM0; 4621 tmp_reg = ZREG_XMM1; 4622 } 4623 if (CAN_USE_AVX()) { 4624 zend_reg op1_reg; 4625 4626 if (Z_MODE(op1_addr) == IS_REG) { 4627 op1_reg = Z_REG(op1_addr); 4628 } else { 4629 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4630 op1_reg = result_reg; 4631 } 4632 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4633 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4634 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4635 /* +/- 0 */ 4636 } else { 4637 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4638 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4639 } 4640 } else { 4641 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4642 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4643 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4644 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4645 /* +/- 0 */ 4646 } else { 4647 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4648 | SSE_MATH_REG opcode, result_reg, tmp_reg 4649 } 4650 } 4651 } 4652 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4653 4654 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4655 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4656 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4657 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4658 } 4659 } 4660 } 4661 4662 return 1; 4663} 4664 4665static int zend_jit_math_double_double(dasm_State **Dst, 4666 zend_uchar opcode, 4667 zend_jit_addr op1_addr, 4668 zend_jit_addr op2_addr, 4669 zend_jit_addr res_addr, 4670 uint32_t res_use_info) 4671{ 4672 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4673 zend_reg result_reg; 4674 4675 if (Z_MODE(res_addr) == IS_REG) { 4676 result_reg = Z_REG(res_addr); 4677 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4678 result_reg = Z_REG(op1_addr); 4679 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4680 result_reg = Z_REG(op2_addr); 4681 } else { 4682 result_reg = ZREG_XMM0; 4683 } 4684 4685 if (CAN_USE_AVX()) { 4686 zend_reg op1_reg; 4687 zend_jit_addr val_addr; 4688 4689 if (Z_MODE(op1_addr) == IS_REG) { 4690 op1_reg = Z_REG(op1_addr); 4691 val_addr = op2_addr; 4692 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4693 op1_reg = Z_REG(op2_addr); 4694 val_addr = op1_addr; 4695 } else { 4696 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4697 op1_reg = result_reg; 4698 val_addr = op2_addr; 4699 } 4700 if ((opcode == ZEND_MUL) && 4701 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4702 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4703 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4704 /* ASSIGN_DIM_OP */ 4705 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4706 } else { 4707 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4708 } 4709 } else { 4710 zend_jit_addr val_addr; 4711 4712 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4713 | DOUBLE_GET_ZVAL_DVAL result_reg, op2_addr 4714 val_addr = op1_addr; 4715 } else { 4716 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4717 val_addr = op2_addr; 4718 } 4719 if (same_ops) { 4720 | SSE_MATH_REG opcode, result_reg, result_reg 4721 } else if ((opcode == ZEND_MUL) && 4722 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4723 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4724 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4725 /* ASSIGN_DIM_OP */ 4726 | SSE_MATH opcode, result_reg, val_addr, r1 4727 } else { 4728 | SSE_MATH opcode, result_reg, val_addr, r0 4729 } 4730 } 4731 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4732 4733 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4734 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4735 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4736 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4737 } 4738 } 4739 } 4740 4741 return 1; 4742} 4743 4744static int zend_jit_math_helper(dasm_State **Dst, 4745 const zend_op *opline, 4746 zend_uchar opcode, 4747 zend_uchar op1_type, 4748 znode_op op1, 4749 zend_jit_addr op1_addr, 4750 uint32_t op1_info, 4751 zend_uchar op2_type, 4752 znode_op op2, 4753 zend_jit_addr op2_addr, 4754 uint32_t op2_info, 4755 uint32_t res_var, 4756 zend_jit_addr res_addr, 4757 uint32_t res_info, 4758 uint32_t res_use_info, 4759 int may_overflow, 4760 int may_throw) 4761/* Labels: 1,2,3,4,5,6 */ 4762{ 4763 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4764 4765 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4766 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4767 if (op1_info & MAY_BE_DOUBLE) { 4768 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4769 } else { 4770 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4771 } 4772 } 4773 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4774 if (op2_info & MAY_BE_DOUBLE) { 4775 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4776 |.cold_code 4777 |1: 4778 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4779 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4780 } 4781 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4782 return 0; 4783 } 4784 | jmp >5 4785 |.code 4786 } else { 4787 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4788 } 4789 } 4790 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4791 return 0; 4792 } 4793 if (op1_info & MAY_BE_DOUBLE) { 4794 |.cold_code 4795 |3: 4796 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4797 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4798 } 4799 if (op2_info & MAY_BE_DOUBLE) { 4800 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4801 if (!same_ops) { 4802 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4803 } else { 4804 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4805 } 4806 } 4807 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4808 return 0; 4809 } 4810 | jmp >5 4811 } 4812 if (!same_ops) { 4813 |1: 4814 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4815 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4816 } 4817 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4818 return 0; 4819 } 4820 | jmp >5 4821 } 4822 |.code 4823 } 4824 } else if ((op1_info & MAY_BE_DOUBLE) && 4825 !(op1_info & MAY_BE_LONG) && 4826 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4827 (res_info & MAY_BE_DOUBLE)) { 4828 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4829 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4830 } 4831 if (op2_info & MAY_BE_DOUBLE) { 4832 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4833 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4834 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4835 } else { 4836 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4837 } 4838 } 4839 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4840 return 0; 4841 } 4842 } 4843 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4844 if (op2_info & MAY_BE_DOUBLE) { 4845 |.cold_code 4846 } 4847 |1: 4848 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4849 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4850 } 4851 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4852 return 0; 4853 } 4854 if (op2_info & MAY_BE_DOUBLE) { 4855 | jmp >5 4856 |.code 4857 } 4858 } 4859 } else if ((op2_info & MAY_BE_DOUBLE) && 4860 !(op2_info & MAY_BE_LONG) && 4861 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4862 (res_info & MAY_BE_DOUBLE)) { 4863 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4864 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4865 } 4866 if (op1_info & MAY_BE_DOUBLE) { 4867 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4868 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4869 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4870 } else { 4871 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4872 } 4873 } 4874 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4875 return 0; 4876 } 4877 } 4878 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4879 if (op1_info & MAY_BE_DOUBLE) { 4880 |.cold_code 4881 } 4882 |1: 4883 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4884 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4885 } 4886 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4887 return 0; 4888 } 4889 if (op1_info & MAY_BE_DOUBLE) { 4890 | jmp >5 4891 |.code 4892 } 4893 } 4894 } 4895 4896 |5: 4897 4898 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4899 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4900 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4901 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4902 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4903 |.cold_code 4904 } 4905 |6: 4906 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4907 if (Z_MODE(res_addr) == IS_REG) { 4908 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4909 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4910 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4911 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4912 } 4913 if (Z_MODE(op1_addr) == IS_REG) { 4914 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4915 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4916 return 0; 4917 } 4918 op1_addr = real_addr; 4919 } 4920 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4921 } else { 4922 if (Z_MODE(op1_addr) == IS_REG) { 4923 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4924 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4925 return 0; 4926 } 4927 op1_addr = real_addr; 4928 } 4929 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4930 if (Z_MODE(res_addr) == IS_REG) { 4931 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4932 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4933 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4934 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4935 } 4936 } 4937 4938 if (Z_MODE(op2_addr) == IS_REG) { 4939 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4940 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4941 return 0; 4942 } 4943 op2_addr = real_addr; 4944 } 4945 |.if X64 4946 | LOAD_ZVAL_ADDR CARG3, op2_addr 4947 |.else 4948 | sub r4, 12 4949 | PUSH_ZVAL_ADDR op2_addr, r0 4950 |.endif 4951 | SET_EX_OPLINE opline, r0 4952 if (opcode == ZEND_ADD) { 4953 | EXT_CALL add_function, r0 4954 } else if (opcode == ZEND_SUB) { 4955 | EXT_CALL sub_function, r0 4956 } else if (opcode == ZEND_MUL) { 4957 | EXT_CALL mul_function, r0 4958 } else if (opcode == ZEND_DIV) { 4959 | EXT_CALL div_function, r0 4960 } else { 4961 ZEND_UNREACHABLE(); 4962 } 4963 |.if not(X64) 4964 | add r4, 12 4965 |.endif 4966 | FREE_OP op1_type, op1, op1_info, 0, NULL 4967 | FREE_OP op2_type, op2, op2_info, 0, NULL 4968 if (may_throw) { 4969 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4970 zend_jit_check_exception_undef_result(Dst, opline); 4971 } else { 4972 zend_jit_check_exception(Dst); 4973 } 4974 } 4975 if (Z_MODE(res_addr) == IS_REG) { 4976 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4977 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4978 return 0; 4979 } 4980 } 4981 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4982 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4983 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4984 | jmp <5 4985 |.code 4986 } 4987 } 4988 4989 return 1; 4990} 4991 4992static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4993{ 4994 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4995 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4996 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4997 4998 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4999 return 0; 5000 } 5001 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5002 return 0; 5003 } 5004 return 1; 5005} 5006 5007static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 5008{ 5009 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 5010 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5011 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5012 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG2) { 5013 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5014 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5015 } else { 5016 | GET_ZVAL_LVAL ZREG_R0, op2_addr 5017 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5018 | mov FCARG2a, r0 5019 } 5020 | EXT_CALL zend_jit_add_arrays_helper, r0 5021 | SET_ZVAL_PTR res_addr, r0 5022 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 5023 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 5024 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 5025 return 1; 5026} 5027 5028static int zend_jit_long_math_helper(dasm_State **Dst, 5029 const zend_op *opline, 5030 zend_uchar opcode, 5031 zend_uchar op1_type, 5032 znode_op op1, 5033 zend_jit_addr op1_addr, 5034 uint32_t op1_info, 5035 zend_ssa_range *op1_range, 5036 zend_uchar op2_type, 5037 znode_op op2, 5038 zend_jit_addr op2_addr, 5039 uint32_t op2_info, 5040 zend_ssa_range *op2_range, 5041 uint32_t res_var, 5042 zend_jit_addr res_addr, 5043 uint32_t res_info, 5044 uint32_t res_use_info, 5045 int may_throw) 5046/* Labels: 6 */ 5047{ 5048 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 5049 zend_reg result_reg; 5050 5051 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5052 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5053 } 5054 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5055 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5056 } 5057 5058 if (opcode == ZEND_MOD) { 5059 result_reg = ZREG_RAX; 5060 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5061 | mov aword T1, r0 // save 5062 } 5063 } else if (Z_MODE(res_addr) == IS_REG) { 5064 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5065 && opline->op2_type != IS_CONST) { 5066 result_reg = ZREG_R0; 5067 } else { 5068 result_reg = Z_REG(res_addr); 5069 } 5070 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5071 result_reg = Z_REG(op1_addr); 5072 } else if (Z_REG(res_addr) != ZREG_R0) { 5073 result_reg = ZREG_R0; 5074 } else { 5075 /* ASSIGN_DIM_OP */ 5076 if (sizeof(void*) == 4 5077 && (opcode == ZEND_SL || opcode == ZEND_SR) 5078 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5079 result_reg = ZREG_R2; 5080 } else { 5081 result_reg = ZREG_FCARG1; 5082 } 5083 } 5084 5085 if (opcode == ZEND_SL) { 5086 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5087 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5088 5089 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5090 if (EXPECTED(op2_lval > 0)) { 5091 | xor Ra(result_reg), Ra(result_reg) 5092 } else { 5093 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5094 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5095 | SET_EX_OPLINE opline, r0 5096 | jmp ->negative_shift 5097 } 5098 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5099 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5100 } else { 5101 | GET_ZVAL_LVAL result_reg, op1_addr 5102 | shl Ra(result_reg), op2_lval 5103 } 5104 } else { 5105 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5106 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5107 } 5108 if (!op2_range || 5109 op2_range->min < 0 || 5110 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5111 | cmp r1, (SIZEOF_ZEND_LONG*8) 5112 | jae >1 5113 |.cold_code 5114 |1: 5115 | cmp r1, 0 5116 | mov Ra(result_reg), 0 5117 | jg >1 5118 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5119 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5120 | SET_EX_OPLINE opline, r0 5121 | jmp ->negative_shift 5122 |.code 5123 } 5124 | GET_ZVAL_LVAL result_reg, op1_addr 5125 | shl Ra(result_reg), cl 5126 |1: 5127 } 5128 } else if (opcode == ZEND_SR) { 5129 | GET_ZVAL_LVAL result_reg, op1_addr 5130 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5131 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5132 5133 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5134 if (EXPECTED(op2_lval > 0)) { 5135 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5136 } else { 5137 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5138 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5139 | SET_EX_OPLINE opline, r0 5140 | jmp ->negative_shift 5141 } 5142 } else { 5143 | sar Ra(result_reg), op2_lval 5144 } 5145 } else { 5146 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5147 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5148 } 5149 if (!op2_range || 5150 op2_range->min < 0 || 5151 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5152 | cmp r1, (SIZEOF_ZEND_LONG*8) 5153 | jae >1 5154 |.cold_code 5155 |1: 5156 | cmp r1, 0 5157 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5158 | jg >1 5159 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5160 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5161 | SET_EX_OPLINE opline, r0 5162 | jmp ->negative_shift 5163 |.code 5164 } 5165 |1: 5166 | sar Ra(result_reg), cl 5167 } 5168 } else if (opcode == ZEND_MOD) { 5169 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5170 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5171 5172 if (op2_lval == 0) { 5173 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5174 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5175 | SET_EX_OPLINE opline, r0 5176 | jmp ->mod_by_zero 5177 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5178 zval tmp; 5179 zend_jit_addr tmp_addr; 5180 5181 /* Optimisation for mod of power of 2 */ 5182 ZVAL_LONG(&tmp, op2_lval - 1); 5183 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5184 | GET_ZVAL_LVAL result_reg, op1_addr 5185 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr 5186 } else { 5187 result_reg = ZREG_RDX; 5188 if (op2_lval == -1) { 5189 | xor Ra(result_reg), Ra(result_reg) 5190 } else { 5191 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5192 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5193 |.if X64 5194 | cqo 5195 |.else 5196 | cdq 5197 |.endif 5198 | idiv Ra(ZREG_RCX) 5199 } 5200 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5201 | mov r0, aword T1 // restore 5202 } 5203 } 5204 } else { 5205 if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5206 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5207 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5208 } else if (Z_MODE(op2_addr) == IS_REG) { 5209 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5210 } 5211 | jz >1 5212 |.cold_code 5213 |1: 5214 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5215 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5216 | SET_EX_OPLINE opline, r0 5217 | jmp ->mod_by_zero 5218 |.code 5219 } 5220 5221 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5222 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5223 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5224 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5225 } else if (Z_MODE(op2_addr) == IS_REG) { 5226 | cmp Ra(Z_REG(op2_addr)), -1 5227 } 5228 | jz >1 5229 |.cold_code 5230 |1: 5231 | SET_ZVAL_LVAL res_addr, 0 5232 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5233 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5234 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5235 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5236 } 5237 } 5238 } 5239 | jmp >5 5240 |.code 5241 } 5242 5243 result_reg = ZREG_RDX; 5244 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5245 |.if X64 5246 | cqo 5247 |.else 5248 | cdq 5249 |.endif 5250 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5251 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5252 } else if (Z_MODE(op2_addr) == IS_REG) { 5253 | idiv Ra(Z_REG(op2_addr)) 5254 } 5255 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5256 | mov r0, aword T1 // restore 5257 } 5258 } 5259 } else if (same_ops) { 5260 | GET_ZVAL_LVAL result_reg, op1_addr 5261 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5262 } else { 5263 | GET_ZVAL_LVAL result_reg, op1_addr 5264 | LONG_MATH opcode, result_reg, op2_addr 5265 } 5266 5267 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5268 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5269 } 5270 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5271 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5272 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5273 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5274 } 5275 } 5276 } 5277 5278 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5279 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5280 if ((op1_info & MAY_BE_LONG) && 5281 (op2_info & MAY_BE_LONG)) { 5282 |.cold_code 5283 } 5284 |6: 5285 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5286 if (Z_MODE(res_addr) == IS_REG) { 5287 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5288 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5289 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5290 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5291 } 5292 if (Z_MODE(op1_addr) == IS_REG) { 5293 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5294 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5295 return 0; 5296 } 5297 op1_addr = real_addr; 5298 } 5299 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5300 } else { 5301 if (Z_MODE(op1_addr) == IS_REG) { 5302 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5303 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5304 return 0; 5305 } 5306 op1_addr = real_addr; 5307 } 5308 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5309 if (Z_MODE(res_addr) == IS_REG) { 5310 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5311 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5312 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5313 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5314 } 5315 } 5316 if (Z_MODE(op2_addr) == IS_REG) { 5317 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5318 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5319 return 0; 5320 } 5321 op2_addr = real_addr; 5322 } 5323 |.if X64 5324 | LOAD_ZVAL_ADDR CARG3, op2_addr 5325 |.else 5326 | sub r4, 12 5327 | PUSH_ZVAL_ADDR op2_addr, r0 5328 |.endif 5329 | SET_EX_OPLINE opline, r0 5330 if (opcode == ZEND_BW_OR) { 5331 | EXT_CALL bitwise_or_function, r0 5332 } else if (opcode == ZEND_BW_AND) { 5333 | EXT_CALL bitwise_and_function, r0 5334 } else if (opcode == ZEND_BW_XOR) { 5335 | EXT_CALL bitwise_xor_function, r0 5336 } else if (opcode == ZEND_SL) { 5337 | EXT_CALL shift_left_function, r0 5338 } else if (opcode == ZEND_SR) { 5339 | EXT_CALL shift_right_function, r0 5340 } else if (opcode == ZEND_MOD) { 5341 | EXT_CALL mod_function, r0 5342 } else { 5343 ZEND_UNREACHABLE(); 5344 } 5345 |.if not(X64) 5346 | add r4, 12 5347 |.endif 5348 | FREE_OP op1_type, op1, op1_info, 0, NULL 5349 | FREE_OP op2_type, op2, op2_info, 0, NULL 5350 if (may_throw) { 5351 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5352 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5353 | jne ->exception_handler_free_op2 5354 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5355 zend_jit_check_exception_undef_result(Dst, opline); 5356 } else { 5357 zend_jit_check_exception(Dst); 5358 } 5359 } 5360 if (Z_MODE(res_addr) == IS_REG) { 5361 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5362 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5363 return 0; 5364 } 5365 } 5366 if ((op1_info & MAY_BE_LONG) && 5367 (op2_info & MAY_BE_LONG)) { 5368 | jmp >5 5369 |.code 5370 } 5371 } 5372 |5: 5373 5374 return 1; 5375} 5376 5377static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5378{ 5379 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5380 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5381 5382 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5383 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5384 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5385 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5386 return 0; 5387 } 5388 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5389 return 0; 5390 } 5391 return 1; 5392} 5393 5394static int zend_jit_concat_helper(dasm_State **Dst, 5395 const zend_op *opline, 5396 zend_uchar op1_type, 5397 znode_op op1, 5398 zend_jit_addr op1_addr, 5399 uint32_t op1_info, 5400 zend_uchar op2_type, 5401 znode_op op2, 5402 zend_jit_addr op2_addr, 5403 uint32_t op2_info, 5404 zend_jit_addr res_addr, 5405 int may_throw) 5406{ 5407#if 1 5408 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5409 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5410 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5411 } 5412 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5413 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5414 } 5415 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5416 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5417 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5418 } 5419 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5420 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5421 /* concatination with itself may reduce refcount */ 5422 op2_info |= MAY_BE_RC1; 5423 } else { 5424 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5425 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5426 } 5427 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5428 |.if X64 5429 | LOAD_ZVAL_ADDR CARG3, op2_addr 5430 |.else 5431 | sub r4, 12 5432 | PUSH_ZVAL_ADDR op2_addr, r0 5433 |.endif 5434 if (op1_type == IS_CV || op1_type == IS_CONST) { 5435 | EXT_CALL zend_jit_fast_concat_helper, r0 5436 } else { 5437 | EXT_CALL zend_jit_fast_concat_tmp_helper, r0 5438 } 5439 |.if not(X64) 5440 | add r4, 12 5441 |.endif 5442 } 5443 /* concatination with empty string may increase refcount */ 5444 op2_info |= MAY_BE_RCN; 5445 | FREE_OP op2_type, op2, op2_info, 0, opline 5446 |5: 5447 } 5448 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5449 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5450 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5451 |.cold_code 5452 |6: 5453 } 5454#endif 5455 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5456 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5457 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5458 } 5459 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5460 } else { 5461 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5462 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5463 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5464 } 5465 } 5466 |.if X64 5467 | LOAD_ZVAL_ADDR CARG3, op2_addr 5468 |.else 5469 | sub r4, 12 5470 | PUSH_ZVAL_ADDR op2_addr, r0 5471 |.endif 5472 | SET_EX_OPLINE opline, r0 5473 | EXT_CALL concat_function, r0 5474 |.if not(X64) 5475 | add r4, 12 5476 |.endif 5477 /* concatination with empty string may increase refcount */ 5478 op1_info |= MAY_BE_RCN; 5479 op2_info |= MAY_BE_RCN; 5480 | FREE_OP op1_type, op1, op1_info, 0, NULL 5481 | FREE_OP op2_type, op2, op2_info, 0, NULL 5482 if (may_throw) { 5483 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5484 zend_jit_check_exception_undef_result(Dst, opline); 5485 } else { 5486 zend_jit_check_exception(Dst); 5487 } 5488 } 5489#if 1 5490 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5491 | jmp <5 5492 |.code 5493 } 5494 } 5495#endif 5496 5497 return 1; 5498} 5499 5500static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5501{ 5502 zend_jit_addr op1_addr, op2_addr; 5503 5504 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5505 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5506 5507 op1_addr = OP1_ADDR(); 5508 op2_addr = OP2_ADDR(); 5509 5510 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5511} 5512 5513static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5514/* Labels: 1,2,3,4,5 */ 5515{ 5516 zend_jit_addr op2_addr = OP2_ADDR(); 5517 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5518 5519 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5520 && type == BP_VAR_R 5521 && !exit_addr) { 5522 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5523 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5524 if (!exit_addr) { 5525 return 0; 5526 } 5527 } 5528 5529 if (op2_info & MAY_BE_LONG) { 5530 bool op2_loaded = 0; 5531 bool packed_loaded = 0; 5532 bool bad_packed_key = 0; 5533 5534 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5535 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5536 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5537 } 5538 if (op1_info & MAY_BE_PACKED_GUARD) { 5539 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5540 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5541 5542 if (!exit_addr) { 5543 return 0; 5544 } 5545 if (op1_info & MAY_BE_ARRAY_PACKED) { 5546 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5547 | jz &exit_addr 5548 } else { 5549 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5550 | jnz &exit_addr 5551 } 5552 } 5553 if (type == BP_VAR_W) { 5554 | // hval = Z_LVAL_P(dim); 5555 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5556 op2_loaded = 1; 5557 } 5558 if (op1_info & MAY_BE_ARRAY_PACKED) { 5559 zend_long val = -1; 5560 5561 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5562 val = Z_LVAL_P(Z_ZV(op2_addr)); 5563 if (val >= 0 && val < HT_MAX_SIZE) { 5564 packed_loaded = 1; 5565 } else { 5566 bad_packed_key = 1; 5567 } 5568 } else { 5569 if (!op2_loaded) { 5570 | // hval = Z_LVAL_P(dim); 5571 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5572 op2_loaded = 1; 5573 } 5574 packed_loaded = 1; 5575 } 5576 5577 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { 5578 /* don't generate "fast" code for packed array */ 5579 packed_loaded = 0; 5580 } 5581 5582 if (packed_loaded) { 5583 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5584 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5585 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5586 | jz >4 // HASH_FIND 5587 } 5588 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5589 |.if X64 5590 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5591 if (val == 0) { 5592 | test r0, r0 5593 } else if (val > 0 && !op2_loaded) { 5594 | cmp r0, val 5595 } else { 5596 | cmp r0, FCARG2a 5597 } 5598 |.else 5599 if (val >= 0 && !op2_loaded) { 5600 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5601 } else { 5602 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5603 } 5604 |.endif 5605 if (type == BP_JIT_IS) { 5606 if (not_found_exit_addr) { 5607 | jbe ¬_found_exit_addr 5608 } else { 5609 | jbe >9 // NOT_FOUND 5610 } 5611 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5612 | jbe &exit_addr 5613 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5614 | jbe ¬_found_exit_addr 5615 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5616 | jbe ¬_found_exit_addr 5617 } else if (type == BP_VAR_IS && found_exit_addr) { 5618 | jbe >7 // NOT_FOUND 5619 } else { 5620 | jbe >2 // NOT_FOUND 5621 } 5622 | // _ret = &_ht->arData[_h].val; 5623 if (val >= 0) { 5624 | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] 5625 if (val != 0) { 5626 | add r0, val * sizeof(Bucket) 5627 } 5628 } else { 5629 |.if X64 5630 | mov r0, FCARG2a 5631 | shl r0, 5 5632 |.else 5633 | imul r0, FCARG2a, sizeof(Bucket) 5634 |.endif 5635 | add r0, aword [FCARG1a + offsetof(zend_array, arData)] 5636 } 5637 } 5638 } 5639 switch (type) { 5640 case BP_JIT_IS: 5641 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5642 if (packed_loaded) { 5643 | jmp >5 5644 } 5645 |4: 5646 if (!op2_loaded) { 5647 | // hval = Z_LVAL_P(dim); 5648 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5649 } 5650 if (packed_loaded) { 5651 | EXT_CALL _zend_hash_index_find, r0 5652 } else { 5653 | EXT_CALL zend_hash_index_find, r0 5654 } 5655 | test r0, r0 5656 if (not_found_exit_addr) { 5657 | jz ¬_found_exit_addr 5658 } else { 5659 | jz >9 // NOT_FOUND 5660 } 5661 if (op2_info & MAY_BE_STRING) { 5662 | jmp >5 5663 } 5664 } else if (packed_loaded) { 5665 if (op2_info & MAY_BE_STRING) { 5666 | jmp >5 5667 } 5668 } else if (not_found_exit_addr) { 5669 | jmp ¬_found_exit_addr 5670 } else { 5671 | jmp >9 // NOT_FOUND 5672 } 5673 break; 5674 case BP_VAR_R: 5675 case BP_VAR_IS: 5676 case BP_VAR_UNSET: 5677 if (packed_loaded) { 5678 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5679 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5680 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5681 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5682 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5683 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5684 } 5685 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5686 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5687 } else if (type == BP_VAR_IS && found_exit_addr) { 5688 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5689 } else { 5690 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5691 } 5692 } 5693 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5694 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5695 | jmp &exit_addr 5696 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5697 | jmp ¬_found_exit_addr 5698 } else if (type == BP_VAR_IS && found_exit_addr) { 5699 | jmp >7 // NOT_FOUND 5700 } else { 5701 | jmp >2 // NOT_FOUND 5702 } 5703 } 5704 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5705 |4: 5706 if (!op2_loaded) { 5707 | // hval = Z_LVAL_P(dim); 5708 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5709 } 5710 if (packed_loaded) { 5711 | EXT_CALL _zend_hash_index_find, r0 5712 } else { 5713 | EXT_CALL zend_hash_index_find, r0 5714 } 5715 | test r0, r0 5716 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5717 | jz &exit_addr 5718 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5719 | jz ¬_found_exit_addr 5720 } else if (type == BP_VAR_IS && found_exit_addr) { 5721 | jz >7 // NOT_FOUND 5722 } else { 5723 | jz >2 // NOT_FOUND 5724 } 5725 } 5726 |.cold_code 5727 |2: 5728 switch (type) { 5729 case BP_VAR_R: 5730 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5731 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5732 | // retval = &EG(uninitialized_zval); 5733 | UNDEFINED_OFFSET opline 5734 | jmp >9 5735 } 5736 break; 5737 case BP_VAR_IS: 5738 case BP_VAR_UNSET: 5739 if (!not_found_exit_addr && !found_exit_addr) { 5740 | // retval = &EG(uninitialized_zval); 5741 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5742 | jmp >9 5743 } 5744 break; 5745 default: 5746 ZEND_UNREACHABLE(); 5747 } 5748 |.code 5749 break; 5750 case BP_VAR_RW: 5751 if (packed_loaded && !not_found_exit_addr) { 5752 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5753 } 5754 if (!packed_loaded || 5755 !not_found_exit_addr || 5756 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5757 if (packed_loaded && not_found_exit_addr) { 5758 |.cold_code 5759 } 5760 |2: 5761 |4: 5762 if (!op2_loaded) { 5763 | // hval = Z_LVAL_P(dim); 5764 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5765 } 5766 | SET_EX_OPLINE opline, r0 5767 if (packed_loaded) { 5768 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5769 } else { 5770 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5771 } 5772 | test r0, r0 5773 if (not_found_exit_addr) { 5774 if (packed_loaded) { 5775 | jnz >8 5776 | jmp ¬_found_exit_addr 5777 |.code 5778 } else { 5779 | jz ¬_found_exit_addr 5780 } 5781 } else { 5782 | jz >9 5783 } 5784 } 5785 break; 5786 case BP_VAR_W: 5787 if (packed_loaded) { 5788 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5789 } 5790 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5791 |2: 5792 |4: 5793 if (!op2_loaded) { 5794 | // hval = Z_LVAL_P(dim); 5795 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5796 } 5797 | EXT_CALL zend_hash_index_lookup, r0 5798 } 5799 break; 5800 default: 5801 ZEND_UNREACHABLE(); 5802 } 5803 5804 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5805 | jmp >8 5806 } 5807 } 5808 5809 if (op2_info & MAY_BE_STRING) { 5810 |3: 5811 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5812 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5813 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5814 } 5815 | // offset_key = Z_STR_P(dim); 5816 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5817 | // retval = zend_hash_find(ht, offset_key); 5818 switch (type) { 5819 case BP_JIT_IS: 5820 if (opline->op2_type != IS_CONST) { 5821 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5822 | jle >1 5823 |.cold_code 5824 |1: 5825 | EXT_CALL zend_jit_symtable_find, r0 5826 | jmp >1 5827 |.code 5828 | EXT_CALL zend_hash_find, r0 5829 |1: 5830 } else { 5831 | EXT_CALL zend_hash_find_known_hash, r0 5832 } 5833 | test r0, r0 5834 if (not_found_exit_addr) { 5835 | jz ¬_found_exit_addr 5836 } else { 5837 | jz >9 // NOT_FOUND 5838 } 5839 break; 5840 case BP_VAR_R: 5841 case BP_VAR_IS: 5842 case BP_VAR_UNSET: 5843 if (opline->op2_type != IS_CONST) { 5844 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5845 | jle >1 5846 |.cold_code 5847 |1: 5848 | EXT_CALL zend_jit_symtable_find, r0 5849 | jmp >1 5850 |.code 5851 | EXT_CALL zend_hash_find, r0 5852 |1: 5853 } else { 5854 | EXT_CALL zend_hash_find_known_hash, r0 5855 } 5856 | test r0, r0 5857 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5858 | jz &exit_addr 5859 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5860 | jz ¬_found_exit_addr 5861 } else if (type == BP_VAR_IS && found_exit_addr) { 5862 | jz >7 // NOT_FOUND 5863 } else { 5864 | jz >2 // NOT_FOUND 5865 |.cold_code 5866 |2: 5867 switch (type) { 5868 case BP_VAR_R: 5869 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5870 | UNDEFINED_INDEX opline 5871 | jmp >9 5872 break; 5873 case BP_VAR_IS: 5874 case BP_VAR_UNSET: 5875 | // retval = &EG(uninitialized_zval); 5876 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5877 | jmp >9 5878 break; 5879 default: 5880 ZEND_UNREACHABLE(); 5881 } 5882 |.code 5883 } 5884 break; 5885 case BP_VAR_RW: 5886 | SET_EX_OPLINE opline, r0 5887 if (opline->op2_type != IS_CONST) { 5888 | EXT_CALL zend_jit_symtable_lookup_rw, r0 5889 } else { 5890 | EXT_CALL zend_jit_hash_lookup_rw, r0 5891 } 5892 | test r0, r0 5893 if (not_found_exit_addr) { 5894 | jz ¬_found_exit_addr 5895 } else { 5896 | jz >9 5897 } 5898 break; 5899 case BP_VAR_W: 5900 if (opline->op2_type != IS_CONST) { 5901 | EXT_CALL zend_jit_symtable_lookup_w, r0 5902 } else { 5903 | EXT_CALL zend_hash_lookup, r0 5904 } 5905 break; 5906 default: 5907 ZEND_UNREACHABLE(); 5908 } 5909 } 5910 5911 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5912 |5: 5913 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5914 | ZVAL_DEREF r0, MAY_BE_REF 5915 } 5916 | cmp byte [r0 + 8], IS_NULL 5917 if (not_found_exit_addr) { 5918 | jle ¬_found_exit_addr 5919 } else if (found_exit_addr) { 5920 | jg &found_exit_addr 5921 } else { 5922 | jle >9 // NOT FOUND 5923 } 5924 } 5925 5926 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5927 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5928 |.cold_code 5929 |3: 5930 } 5931 | SET_EX_OPLINE opline, r0 5932 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5933 switch (type) { 5934 case BP_VAR_R: 5935 |.if X64 5936 | LOAD_ZVAL_ADDR CARG3, res_addr 5937 |.else 5938 | sub r4, 12 5939 | PUSH_ZVAL_ADDR res_addr, r0 5940 |.endif 5941 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 5942 |.if not(X64) 5943 | add r4, 12 5944 |.endif 5945 | jmp >9 5946 break; 5947 case BP_JIT_IS: 5948 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 5949 | test r0, r0 5950 if (not_found_exit_addr) { 5951 | je ¬_found_exit_addr 5952 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5953 | jmp >8 5954 } 5955 } else if (found_exit_addr) { 5956 | jne &found_exit_addr 5957 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5958 | jmp >9 5959 } 5960 } else { 5961 | jne >8 5962 | jmp >9 5963 } 5964 break; 5965 case BP_VAR_IS: 5966 case BP_VAR_UNSET: 5967 |.if X64 5968 | LOAD_ZVAL_ADDR CARG3, res_addr 5969 |.else 5970 | sub r4, 12 5971 | PUSH_ZVAL_ADDR res_addr, r0 5972 |.endif 5973 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 5974 |.if not(X64) 5975 | add r4, 12 5976 |.endif 5977 | jmp >9 5978 break; 5979 case BP_VAR_RW: 5980 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 5981 | test r0, r0 5982 | jne >8 5983 | jmp >9 5984 break; 5985 case BP_VAR_W: 5986 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 5987 | test r0, r0 5988 | jne >8 5989 | jmp >9 5990 break; 5991 default: 5992 ZEND_UNREACHABLE(); 5993 } 5994 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5995 |.code 5996 } 5997 } 5998 5999 return 1; 6000} 6001 6002static int zend_jit_simple_assign(dasm_State **Dst, 6003 const zend_op *opline, 6004 zend_jit_addr var_addr, 6005 uint32_t var_info, 6006 uint32_t var_def_info, 6007 zend_uchar val_type, 6008 zend_jit_addr val_addr, 6009 uint32_t val_info, 6010 zend_jit_addr res_addr, 6011 int in_cold, 6012 int save_r1) 6013/* Labels: 1,2,3 */ 6014{ 6015 zend_reg tmp_reg; 6016 6017 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 6018 tmp_reg = ZREG_R0; 6019 } else { 6020 /* ASSIGN_DIM */ 6021 tmp_reg = ZREG_FCARG1; 6022 } 6023 6024 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6025 zval *zv = Z_ZV(val_addr); 6026 6027 if (!res_addr) { 6028 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 6029 } else { 6030 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 6031 } 6032 if (Z_REFCOUNTED_P(zv)) { 6033 if (!res_addr) { 6034 | ADDREF_CONST zv, Ra(tmp_reg) 6035 } else { 6036 | ADDREF_CONST_2 zv, Ra(tmp_reg) 6037 } 6038 } 6039 } else { 6040 if (val_info & MAY_BE_UNDEF) { 6041 if (in_cold) { 6042 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6043 } else { 6044 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6045 |.cold_code 6046 |1: 6047 } 6048 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6049 if (save_r1) { 6050 | mov aword T1, FCARG1a // save 6051 } 6052 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6053 if (res_addr) { 6054 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6055 } 6056 if (opline) { 6057 | SET_EX_OPLINE opline, Ra(tmp_reg) 6058 } 6059 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6060 | mov FCARG1d, Z_OFFSET(val_addr) 6061 | EXT_CALL zend_jit_undefined_op_helper, r0 6062 | test r0, r0 6063 | jz ->exception_handler_undef 6064 if (save_r1) { 6065 | mov FCARG1a, aword T1 // restore 6066 } 6067 | jmp >3 6068 if (in_cold) { 6069 |2: 6070 } else { 6071 |.code 6072 } 6073 } 6074 if (val_info & MAY_BE_REF) { 6075 if (val_type == IS_CV) { 6076 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6077 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6078 | LOAD_ZVAL_ADDR r2, val_addr 6079 } 6080 | ZVAL_DEREF r2, val_info 6081 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6082 } else { 6083 zend_jit_addr ref_addr; 6084 6085 if (in_cold) { 6086 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6087 } else { 6088 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6089 |.cold_code 6090 |1: 6091 } 6092 if (Z_REG(val_addr) == ZREG_R2) { 6093 | mov aword T1, r2 // save 6094 } 6095 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6096 | GET_ZVAL_PTR r2, val_addr 6097 | GC_DELREF r2 6098 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6099 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6100 if (!res_addr) { 6101 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, tmp_reg 6102 } else { 6103 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, tmp_reg 6104 } 6105 | je >2 6106 | IF_NOT_REFCOUNTED dh, >3 6107 if (!res_addr) { 6108 | GC_ADDREF Ra(tmp_reg) 6109 } else { 6110 | add dword [Ra(tmp_reg)], 2 6111 } 6112 | jmp >3 6113 |2: 6114 if (res_addr) { 6115 | IF_NOT_REFCOUNTED dh, >2 6116 | GC_ADDREF Ra(tmp_reg) 6117 |2: 6118 } 6119 if (Z_REG(val_addr) == ZREG_R2) { 6120 | mov r2, aword T1 // restore 6121 } 6122 if (save_r1) { 6123 | mov aword T1, FCARG1a // save 6124 } 6125 | EFREE_REFERENCE aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)] 6126 if (save_r1) { 6127 | mov FCARG1a, aword T1 // restore 6128 } 6129 | jmp >3 6130 if (in_cold) { 6131 |1: 6132 } else { 6133 |.code 6134 } 6135 } 6136 } 6137 6138 if (!res_addr) { 6139 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6140 } else { 6141 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6142 } 6143 6144 if (val_type == IS_CV) { 6145 if (!res_addr) { 6146 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6147 } else { 6148 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6149 } 6150 } else { 6151 if (res_addr) { 6152 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6153 } 6154 } 6155 |3: 6156 } 6157 return 1; 6158} 6159 6160static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6161 const zend_op *opline, 6162 zend_uchar val_type, 6163 zend_jit_addr val_addr, 6164 zend_jit_addr res_addr, 6165 bool check_exception) 6166{ 6167 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6168 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6169 | jnz >2 6170 |.cold_code 6171 |2: 6172 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6173 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6174 } 6175 if (opline) { 6176 | SET_EX_OPLINE opline, r0 6177 } 6178 if (val_type == IS_CONST) { 6179 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6180 } else if (val_type == IS_TMP_VAR) { 6181 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6182 } else if (val_type == IS_VAR) { 6183 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6184 } else if (val_type == IS_CV) { 6185 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6186 } else { 6187 ZEND_UNREACHABLE(); 6188 } 6189 if (res_addr) { 6190 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6191 6192 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2 6193 | TRY_ADDREF -1, ch, r2 6194 } 6195 if (check_exception) { 6196 | // if (UNEXPECTED(EG(exception) != NULL)) { 6197 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6198 | je >8 // END OF zend_jit_assign_to_variable() 6199 | jmp ->exception_handler_undef 6200 } else { 6201 | jmp >8 6202 } 6203 |.code 6204 6205 return 1; 6206} 6207 6208static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6209 const zend_op *opline, 6210 zend_jit_addr __var_use_addr, 6211 zend_jit_addr var_addr, 6212 uint32_t __var_info, 6213 uint32_t __var_def_info, 6214 zend_uchar val_type, 6215 zend_jit_addr val_addr, 6216 uint32_t val_info, 6217 zend_jit_addr __res_addr, 6218 bool __check_exception) 6219{ 6220 if (val_info & MAY_BE_UNDEF) { 6221 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6222 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6223 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6224 6225 if (!exit_addr) { 6226 return 0; 6227 } 6228 6229 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6230 } else { 6231 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6232 |.cold_code 6233 |1: 6234 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6235 if (Z_REG(var_addr) != ZREG_FP) { 6236 | mov aword T1, Ra(Z_REG(var_addr)) // save 6237 } 6238 | SET_EX_OPLINE opline, r0 6239 | mov FCARG1d, Z_OFFSET(val_addr) 6240 | EXT_CALL zend_jit_undefined_op_helper, r0 6241 if (Z_REG(var_addr) != ZREG_FP) { 6242 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6243 } 6244 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6245 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6246 } 6247 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6248 | call ->assign_const 6249 | jmp >9 6250 |.code 6251 } 6252 } 6253 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6254 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6255 } 6256 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6257 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6258 } 6259 if (opline) { 6260 | SET_EX_OPLINE opline, r0 6261 } 6262 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6263 | call ->assign_tmp 6264 } else if (val_type == IS_CONST) { 6265 | call ->assign_const 6266 } else if (val_type == IS_TMP_VAR) { 6267 | call ->assign_tmp 6268 } else if (val_type == IS_VAR) { 6269 if (!(val_info & MAY_BE_REF)) { 6270 | call ->assign_tmp 6271 } else { 6272 | call ->assign_var 6273 } 6274 } else if (val_type == IS_CV) { 6275 if (!(val_info & MAY_BE_REF)) { 6276 | call ->assign_cv_noref 6277 } else { 6278 | call ->assign_cv 6279 } 6280 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6281 |9: 6282 } 6283 } else { 6284 ZEND_UNREACHABLE(); 6285 } 6286 6287 return 1; 6288} 6289 6290static int zend_jit_assign_to_variable(dasm_State **Dst, 6291 const zend_op *opline, 6292 zend_jit_addr var_use_addr, 6293 zend_jit_addr var_addr, 6294 uint32_t var_info, 6295 uint32_t var_def_info, 6296 zend_uchar val_type, 6297 zend_jit_addr val_addr, 6298 uint32_t val_info, 6299 zend_jit_addr res_addr, 6300 bool check_exception) 6301/* Labels: 1,2,3,4,5,8 */ 6302{ 6303 int done = 0; 6304 zend_reg ref_reg, tmp_reg; 6305 6306 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6307 ref_reg = ZREG_FCARG1; 6308 tmp_reg = ZREG_R0; 6309 } else { 6310 /* ASSIGN_DIM */ 6311 ref_reg = ZREG_R0; 6312 tmp_reg = ZREG_FCARG1; 6313 } 6314 6315 if (var_info & MAY_BE_REF) { 6316 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6317 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6318 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6319 } 6320 | // if (Z_ISREF_P(variable_ptr)) { 6321 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6322 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6323 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6324 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6325 return 0; 6326 } 6327 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6328 |3: 6329 } 6330 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6331 if (RC_MAY_BE_1(var_info)) { 6332 int in_cold = 0; 6333 6334 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6335 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6336 |.cold_code 6337 |1: 6338 in_cold = 1; 6339 } 6340 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_R0) { 6341 bool keep_gc = 0; 6342 6343 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6344 if (tmp_reg == ZREG_FCARG1) { 6345 if (Z_MODE(val_addr) == IS_REG) { 6346 keep_gc = 1; 6347 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6348 keep_gc = 1; 6349 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6350 if (sizeof(void*) == 4) { 6351 keep_gc = 1; 6352 } else { 6353 zval *zv = Z_ZV(val_addr); 6354 6355 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6356 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6357 keep_gc = 1; 6358 } 6359 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6360 keep_gc = 1; 6361 } 6362 } 6363 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6364 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6365 keep_gc = 1; 6366 } 6367 } 6368 } 6369 if (!keep_gc) { 6370 | mov aword T1, Ra(tmp_reg) // save 6371 } 6372 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { 6373 return 0; 6374 } 6375 if (!keep_gc) { 6376 | mov FCARG1a, aword T1 // restore 6377 } 6378 } else { 6379 | GET_ZVAL_PTR FCARG1a, var_use_addr 6380 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { 6381 return 0; 6382 } 6383 } 6384 | GC_DELREF FCARG1a 6385 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6386 | jnz >4 6387 } else { 6388 | jnz >8 6389 } 6390 | ZVAL_DTOR_FUNC var_info, opline 6391 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6392 if (check_exception) { 6393 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6394 | je >8 6395 | jmp ->exception_handler 6396 } else { 6397 | jmp >8 6398 } 6399 } 6400 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6401 |4: 6402 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6403 | EXT_CALL gc_possible_root, r0 6404 if (in_cold) { 6405 | jmp >8 6406 } 6407 } 6408 if (in_cold) { 6409 |.code 6410 } else { 6411 done = 1; 6412 } 6413 } else /* if (RC_MAY_BE_N(var_info)) */ { 6414 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6415 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6416 } 6417 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6418 if (Z_REG(var_use_addr) != ZREG_FP) { 6419 | mov T1, Ra(Z_REG(var_use_addr)) // save 6420 } 6421 | GET_ZVAL_PTR FCARG1a, var_use_addr 6422 | GC_DELREF FCARG1a 6423 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6424 | EXT_CALL gc_possible_root, r0 6425 if (Z_REG(var_use_addr) != ZREG_FP) { 6426 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6427 } 6428 } else { 6429 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6430 | GC_DELREF Ra(tmp_reg) 6431 } 6432 |5: 6433 } 6434 } 6435 6436 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { 6437 return 0; 6438 } 6439 6440 |8: 6441 6442 return 1; 6443} 6444 6445static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6446{ 6447 zend_jit_addr op2_addr, op3_addr, res_addr; 6448 6449 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6450 op3_addr = OP1_DATA_ADDR(); 6451 if (opline->result_type == IS_UNUSED) { 6452 res_addr = 0; 6453 } else { 6454 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6455 } 6456 6457 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6458 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6459 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6460 6461 if (!exit_addr) { 6462 return 0; 6463 } 6464 6465 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6466 6467 val_info &= ~MAY_BE_UNDEF; 6468 } 6469 6470 if (op1_info & MAY_BE_REF) { 6471 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6472 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6473 | GET_Z_PTR FCARG2a, FCARG1a 6474 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6475 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6476 | jmp >3 6477 |.cold_code 6478 |2: 6479 | SET_EX_OPLINE opline, r0 6480 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6481 | test r0, r0 6482 | mov FCARG1a, r0 6483 | jne >1 6484 | jmp ->exception_handler_undef 6485 |.code 6486 |1: 6487 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6488 } 6489 6490 if (op1_info & MAY_BE_ARRAY) { 6491 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6492 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6493 } 6494 |3: 6495 | SEPARATE_ARRAY op1_addr, op1_info, 1 6496 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6497 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6498 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6499 | jg >7 6500 } 6501 | // ZVAL_ARR(container, zend_new_array(8)); 6502 if (Z_REG(op1_addr) != ZREG_FP) { 6503 | mov T1, Ra(Z_REG(op1_addr)) // save 6504 } 6505 | EXT_CALL _zend_new_array_0, r0 6506 if (Z_REG(op1_addr) != ZREG_FP) { 6507 | mov Ra(Z_REG(op1_addr)), T1 // restore 6508 } 6509 | SET_ZVAL_LVAL op1_addr, r0 6510 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6511 | mov FCARG1a, r0 6512 } 6513 6514 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6515 |6: 6516 if (opline->op2_type == IS_UNUSED) { 6517 uint32_t var_info = MAY_BE_NULL; 6518 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6519 6520 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6521 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6522 | EXT_CALL zend_hash_next_index_insert, r0 6523 | // if (UNEXPECTED(!var_ptr)) { 6524 | test r0, r0 6525 | jz >1 6526 |.cold_code 6527 |1: 6528 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6529 | CANNOT_ADD_ELEMENT opline 6530 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6531 | jmp >9 6532 |.code 6533 6534 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { 6535 return 0; 6536 } 6537 } else { 6538 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6539 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6540 6541 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6542 return 0; 6543 } 6544 6545 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6546 var_info |= MAY_BE_REF; 6547 } 6548 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6549 var_info |= MAY_BE_RC1; 6550 } 6551 6552 |8: 6553 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6554 if (opline->op1_type == IS_VAR) { 6555 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6556 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6557 return 0; 6558 } 6559 } else { 6560 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6561 return 0; 6562 } 6563 } 6564 } 6565 } 6566 6567 if (((op1_info & MAY_BE_ARRAY) && 6568 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6569 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6570 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6571 |.cold_code 6572 |7: 6573 } 6574 6575 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6576 (op1_info & MAY_BE_ARRAY)) { 6577 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6578 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6579 | jg >2 6580 } 6581 | // ZVAL_ARR(container, zend_new_array(8)); 6582 if (Z_REG(op1_addr) != ZREG_FP) { 6583 | mov T1, Ra(Z_REG(op1_addr)) // save 6584 } 6585 | EXT_CALL _zend_new_array_0, r0 6586 if (Z_REG(op1_addr) != ZREG_FP) { 6587 | mov Ra(Z_REG(op1_addr)), T1 // restore 6588 } 6589 | SET_ZVAL_LVAL op1_addr, r0 6590 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6591 | mov FCARG1a, r0 6592 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6593 | jmp <6 6594 |2: 6595 } 6596 6597 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6598 | SET_EX_OPLINE opline, r0 6599 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6600 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6601 } 6602 if (opline->op2_type == IS_UNUSED) { 6603 | xor FCARG2a, FCARG2a 6604 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6605 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6606 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6607 } else { 6608 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6609 } 6610 |.if not(X64) 6611 | sub r4, 8 6612 |.endif 6613 if (opline->result_type == IS_UNUSED) { 6614 |.if X64 6615 | xor CARG4, CARG4 6616 |.else 6617 | push 0 6618 |.endif 6619 } else { 6620 |.if X64 6621 | LOAD_ZVAL_ADDR CARG4, res_addr 6622 |.else 6623 | PUSH_ZVAL_ADDR res_addr, r0 6624 |.endif 6625 } 6626 |.if X64 6627 | LOAD_ZVAL_ADDR CARG3, op3_addr 6628 |.else 6629 | PUSH_ZVAL_ADDR op3_addr, r0 6630 |.endif 6631 | EXT_CALL zend_jit_assign_dim_helper, r0 6632 |.if not(X64) 6633 | add r4, 8 6634 |.endif 6635 6636#ifdef ZEND_JIT_USE_RC_INFERENCE 6637 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6638 /* ASSIGN_DIM may increase refcount of the value */ 6639 val_info |= MAY_BE_RCN; 6640 } 6641#endif 6642 6643 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL 6644 } 6645 6646 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6647 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6648 | jmp >9 // END 6649 } 6650 |.code 6651 } 6652 } 6653 6654#ifdef ZEND_JIT_USE_RC_INFERENCE 6655 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6656 /* ASSIGN_DIM may increase refcount of the key */ 6657 op2_info |= MAY_BE_RCN; 6658 } 6659#endif 6660 6661 |9: 6662 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6663 6664 if (may_throw) { 6665 zend_jit_check_exception(Dst); 6666 } 6667 6668 return 1; 6669} 6670 6671static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6672{ 6673 zend_jit_addr op2_addr, op3_addr, var_addr; 6674 const void *not_found_exit_addr = NULL; 6675 uint32_t var_info = MAY_BE_NULL; 6676 6677 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6678 6679 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6680 op3_addr = OP1_DATA_ADDR(); 6681 6682 if (op1_info & MAY_BE_REF) { 6683 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6684 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6685 | GET_Z_PTR FCARG2a, FCARG1a 6686 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6687 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6688 | jmp >3 6689 |.cold_code 6690 |2: 6691 | SET_EX_OPLINE opline, r0 6692 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6693 | test r0, r0 6694 | mov FCARG1a, r0 6695 | jne >1 6696 | jmp ->exception_handler_undef 6697 |.code 6698 |1: 6699 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6700 } 6701 6702 if (op1_info & MAY_BE_ARRAY) { 6703 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6704 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6705 } 6706 |3: 6707 | SEPARATE_ARRAY op1_addr, op1_info, 1 6708 } 6709 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6710 if (op1_info & MAY_BE_ARRAY) { 6711 |.cold_code 6712 |7: 6713 } 6714 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6715 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6716 | jg >7 6717 } 6718 if (Z_REG(op1_addr) != ZREG_FP) { 6719 | mov T1, Ra(Z_REG(op1_addr)) // save 6720 } 6721 if (op1_info & MAY_BE_UNDEF) { 6722 if (op1_info & MAY_BE_NULL) { 6723 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6724 } 6725 | SET_EX_OPLINE opline, r0 6726 | mov FCARG1a, opline->op1.var 6727 | EXT_CALL zend_jit_undefined_op_helper, r0 6728 |1: 6729 } 6730 | // ZVAL_ARR(container, zend_new_array(8)); 6731 | EXT_CALL _zend_new_array_0, r0 6732 if (Z_REG(op1_addr) != ZREG_FP) { 6733 | mov Ra(Z_REG(op1_addr)), T1 // restore 6734 } 6735 | SET_ZVAL_LVAL op1_addr, r0 6736 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6737 | mov FCARG1a, r0 6738 if (op1_info & MAY_BE_ARRAY) { 6739 | jmp >1 6740 |.code 6741 |1: 6742 } 6743 } 6744 6745 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6746 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6747 6748 |6: 6749 if (opline->op2_type == IS_UNUSED) { 6750 var_info = MAY_BE_NULL; 6751 6752 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6753 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6754 | EXT_CALL zend_hash_next_index_insert, r0 6755 | // if (UNEXPECTED(!var_ptr)) { 6756 | test r0, r0 6757 | jz >1 6758 |.cold_code 6759 |1: 6760 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6761 | CANNOT_ADD_ELEMENT opline 6762 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6763 | jmp >9 6764 |.code 6765 } else { 6766 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6767 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6768 var_info |= MAY_BE_REF; 6769 } 6770 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6771 var_info |= MAY_BE_RC1; 6772 } 6773 6774 if (dim_type != IS_UNKNOWN 6775 && dim_type != IS_UNDEF 6776 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6777 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6778 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6779 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6780 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6781 } 6782 6783 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6784 return 0; 6785 } 6786 6787 |8: 6788 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6789 | IF_NOT_Z_TYPE, r0, dim_type, ¬_found_exit_addr 6790 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6791 } 6792 if (var_info & MAY_BE_REF) { 6793 binary_op_type binary_op = get_binary_op(opline->extended_value); 6794 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6795 | GET_Z_PTR FCARG1a, r0 6796 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6797 | jnz >2 6798 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6799 |.cold_code 6800 |2: 6801 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6802 |.if X64 6803 | LOAD_ADDR CARG3, binary_op 6804 |.else 6805 | sub r4, 12 6806 | PUSH_ADDR binary_op, r0 6807 |.endif 6808 | SET_EX_OPLINE opline, r0 6809 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6810 |.if not(X64) 6811 | add r4, 12 6812 |.endif 6813 | jmp >9 6814 |.code 6815 |1: 6816 } 6817 } 6818 6819 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6820 switch (opline->extended_value) { 6821 case ZEND_ADD: 6822 case ZEND_SUB: 6823 case ZEND_MUL: 6824 case ZEND_DIV: 6825 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6826 1 /* may overflow */, may_throw)) { 6827 return 0; 6828 } 6829 break; 6830 case ZEND_BW_OR: 6831 case ZEND_BW_AND: 6832 case ZEND_BW_XOR: 6833 case ZEND_SL: 6834 case ZEND_SR: 6835 case ZEND_MOD: 6836 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6837 IS_CV, opline->op1, var_addr, var_info, NULL, 6838 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6839 op1_data_range, 6840 0, var_addr, var_def_info, var_info, may_throw)) { 6841 return 0; 6842 } 6843 break; 6844 case ZEND_CONCAT: 6845 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6846 may_throw)) { 6847 return 0; 6848 } 6849 break; 6850 default: 6851 ZEND_UNREACHABLE(); 6852 } 6853 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6854 } 6855 6856 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6857 binary_op_type binary_op; 6858 6859 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6860 |.cold_code 6861 |7: 6862 } 6863 6864 | SET_EX_OPLINE opline, r0 6865 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6866 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6867 } 6868 if (opline->op2_type == IS_UNUSED) { 6869 | xor FCARG2a, FCARG2a 6870 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6871 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6872 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6873 } else { 6874 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6875 } 6876 binary_op = get_binary_op(opline->extended_value); 6877 |.if X64 6878 | LOAD_ZVAL_ADDR CARG3, op3_addr 6879 | LOAD_ADDR CARG4, binary_op 6880 |.else 6881 | sub r4, 8 6882 | PUSH_ADDR binary_op, r0 6883 | PUSH_ZVAL_ADDR op3_addr, r0 6884 |.endif 6885 | EXT_CALL zend_jit_assign_dim_op_helper, r0 6886 |.if not(X64) 6887 | add r4, 8 6888 |.endif 6889 6890 |9: 6891 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL 6892 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 6893 if (may_throw) { 6894 zend_jit_check_exception(Dst); 6895 } 6896 6897 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6898 | jmp >9 // END 6899 |.code 6900 |9: 6901 } 6902 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 6903 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 6904 |.cold_code 6905 |9: 6906 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline 6907 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6908 if (may_throw) { 6909 zend_jit_check_exception(Dst); 6910 } 6911 | jmp >9 6912 |.code 6913 |9: 6914 } 6915 6916 return 1; 6917} 6918 6919static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6920{ 6921 zend_jit_addr op1_addr, op2_addr; 6922 6923 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6924 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6925 6926 op1_addr = OP1_ADDR(); 6927 op2_addr = OP2_ADDR(); 6928 6929 if (op1_info & MAY_BE_REF) { 6930 binary_op_type binary_op = get_binary_op(opline->extended_value); 6931 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6932 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 6933 | GET_Z_PTR FCARG1a, FCARG1a 6934 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6935 | jnz >2 6936 | add FCARG1a, offsetof(zend_reference, val) 6937 |.cold_code 6938 |2: 6939 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6940 |.if X64 6941 | LOAD_ADDR CARG3, binary_op 6942 |.else 6943 | sub r4, 12 6944 | PUSH_ADDR binary_op, r0 6945 |.endif 6946 | SET_EX_OPLINE opline, r0 6947 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6948 |.if not(X64) 6949 | add r4, 12 6950 |.endif 6951 zend_jit_check_exception(Dst); 6952 | jmp >9 6953 |.code 6954 |1: 6955 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6956 } 6957 6958 int result; 6959 switch (opline->extended_value) { 6960 case ZEND_ADD: 6961 case ZEND_SUB: 6962 case ZEND_MUL: 6963 case ZEND_DIV: 6964 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6965 break; 6966 case ZEND_BW_OR: 6967 case ZEND_BW_AND: 6968 case ZEND_BW_XOR: 6969 case ZEND_SL: 6970 case ZEND_SR: 6971 case ZEND_MOD: 6972 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6973 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6974 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6975 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6976 break; 6977 case ZEND_CONCAT: 6978 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6979 break; 6980 default: 6981 ZEND_UNREACHABLE(); 6982 } 6983 |9: 6984 return result; 6985} 6986 6987static int zend_jit_cmp_long_long(dasm_State **Dst, 6988 const zend_op *opline, 6989 zend_ssa_range *op1_range, 6990 zend_jit_addr op1_addr, 6991 zend_ssa_range *op2_range, 6992 zend_jit_addr op2_addr, 6993 zend_jit_addr res_addr, 6994 zend_uchar smart_branch_opcode, 6995 uint32_t target_label, 6996 uint32_t target_label2, 6997 const void *exit_addr, 6998 bool skip_comparison) 6999{ 7000 bool swap = 0; 7001 bool result; 7002 7003 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7004 if (!smart_branch_opcode || 7005 smart_branch_opcode == ZEND_JMPZ_EX || 7006 smart_branch_opcode == ZEND_JMPNZ_EX) { 7007 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7008 } 7009 if (smart_branch_opcode && !exit_addr) { 7010 if (smart_branch_opcode == ZEND_JMPZ || 7011 smart_branch_opcode == ZEND_JMPZ_EX) { 7012 if (!result) { 7013 | jmp => target_label 7014 } 7015 } else if (smart_branch_opcode == ZEND_JMPNZ || 7016 smart_branch_opcode == ZEND_JMPNZ_EX) { 7017 if (result) { 7018 | jmp => target_label 7019 } 7020 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7021 if (!result) { 7022 | jmp => target_label 7023 } else { 7024 | jmp => target_label2 7025 } 7026 } else { 7027 ZEND_UNREACHABLE(); 7028 } 7029 } 7030 return 1; 7031 } 7032 7033 if (skip_comparison) { 7034 if (Z_MODE(op1_addr) != IS_REG && 7035 (Z_MODE(op2_addr) == IS_REG || 7036 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7037 swap = 1; 7038 } 7039 } else if (Z_MODE(op1_addr) == IS_REG) { 7040 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7041 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7042 } else { 7043 | LONG_OP cmp, Z_REG(op1_addr), op2_addr 7044 } 7045 } else if (Z_MODE(op2_addr) == IS_REG) { 7046 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7047 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7048 } else { 7049 | LONG_OP cmp, Z_REG(op2_addr), op1_addr 7050 } 7051 swap = 1; 7052 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7053 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7054 swap = 1; 7055 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7056 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7057 } else { 7058 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7059 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7060 | test r0, r0 7061 } else { 7062 | LONG_OP cmp, ZREG_R0, op2_addr 7063 } 7064 } 7065 7066 if (smart_branch_opcode) { 7067 if (smart_branch_opcode == ZEND_JMPZ_EX || 7068 smart_branch_opcode == ZEND_JMPNZ_EX) { 7069 7070 switch (opline->opcode) { 7071 case ZEND_IS_EQUAL: 7072 case ZEND_IS_IDENTICAL: 7073 case ZEND_CASE: 7074 case ZEND_CASE_STRICT: 7075 | sete al 7076 break; 7077 case ZEND_IS_NOT_EQUAL: 7078 case ZEND_IS_NOT_IDENTICAL: 7079 | setne al 7080 break; 7081 case ZEND_IS_SMALLER: 7082 if (swap) { 7083 | setg al 7084 } else { 7085 | setl al 7086 } 7087 break; 7088 case ZEND_IS_SMALLER_OR_EQUAL: 7089 if (swap) { 7090 | setge al 7091 } else { 7092 | setle al 7093 } 7094 break; 7095 default: 7096 ZEND_UNREACHABLE(); 7097 } 7098 | movzx eax, al 7099 | lea eax, [eax + 2] 7100 | SET_ZVAL_TYPE_INFO res_addr, eax 7101 } 7102 if (smart_branch_opcode == ZEND_JMPZ || 7103 smart_branch_opcode == ZEND_JMPZ_EX) { 7104 switch (opline->opcode) { 7105 case ZEND_IS_EQUAL: 7106 case ZEND_IS_IDENTICAL: 7107 case ZEND_CASE: 7108 case ZEND_CASE_STRICT: 7109 if (exit_addr) { 7110 | jne &exit_addr 7111 } else { 7112 | jne => target_label 7113 } 7114 break; 7115 case ZEND_IS_NOT_EQUAL: 7116 if (exit_addr) { 7117 | je &exit_addr 7118 } else { 7119 | je => target_label 7120 } 7121 break; 7122 case ZEND_IS_NOT_IDENTICAL: 7123 if (exit_addr) { 7124 | jne &exit_addr 7125 } else { 7126 | je => target_label 7127 } 7128 break; 7129 case ZEND_IS_SMALLER: 7130 if (swap) { 7131 if (exit_addr) { 7132 | jle &exit_addr 7133 } else { 7134 | jle => target_label 7135 } 7136 } else { 7137 if (exit_addr) { 7138 | jge &exit_addr 7139 } else { 7140 | jge => target_label 7141 } 7142 } 7143 break; 7144 case ZEND_IS_SMALLER_OR_EQUAL: 7145 if (swap) { 7146 if (exit_addr) { 7147 | jl &exit_addr 7148 } else { 7149 | jl => target_label 7150 } 7151 } else { 7152 if (exit_addr) { 7153 | jg &exit_addr 7154 } else { 7155 | jg => target_label 7156 } 7157 } 7158 break; 7159 default: 7160 ZEND_UNREACHABLE(); 7161 } 7162 } else if (smart_branch_opcode == ZEND_JMPNZ || 7163 smart_branch_opcode == ZEND_JMPNZ_EX) { 7164 switch (opline->opcode) { 7165 case ZEND_IS_EQUAL: 7166 case ZEND_IS_IDENTICAL: 7167 case ZEND_CASE: 7168 case ZEND_CASE_STRICT: 7169 if (exit_addr) { 7170 | je &exit_addr 7171 } else { 7172 | je => target_label 7173 } 7174 break; 7175 case ZEND_IS_NOT_EQUAL: 7176 if (exit_addr) { 7177 | jne &exit_addr 7178 } else { 7179 | jne => target_label 7180 } 7181 break; 7182 case ZEND_IS_NOT_IDENTICAL: 7183 if (exit_addr) { 7184 | je &exit_addr 7185 } else { 7186 | jne => target_label 7187 } 7188 break; 7189 case ZEND_IS_SMALLER: 7190 if (swap) { 7191 if (exit_addr) { 7192 | jg &exit_addr 7193 } else { 7194 | jg => target_label 7195 } 7196 } else { 7197 if (exit_addr) { 7198 | jl &exit_addr 7199 } else { 7200 | jl => target_label 7201 } 7202 } 7203 break; 7204 case ZEND_IS_SMALLER_OR_EQUAL: 7205 if (swap) { 7206 if (exit_addr) { 7207 | jge &exit_addr 7208 } else { 7209 | jge => target_label 7210 } 7211 } else { 7212 if (exit_addr) { 7213 | jle &exit_addr 7214 } else { 7215 | jle => target_label 7216 } 7217 } 7218 break; 7219 default: 7220 ZEND_UNREACHABLE(); 7221 } 7222 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7223 switch (opline->opcode) { 7224 case ZEND_IS_EQUAL: 7225 case ZEND_IS_IDENTICAL: 7226 case ZEND_CASE: 7227 case ZEND_CASE_STRICT: 7228 | jne => target_label 7229 break; 7230 case ZEND_IS_NOT_EQUAL: 7231 case ZEND_IS_NOT_IDENTICAL: 7232 | je => target_label 7233 break; 7234 case ZEND_IS_SMALLER: 7235 if (swap) { 7236 | jle => target_label 7237 } else { 7238 | jge => target_label 7239 } 7240 break; 7241 case ZEND_IS_SMALLER_OR_EQUAL: 7242 if (swap) { 7243 | jl => target_label 7244 } else { 7245 | jg => target_label 7246 } 7247 break; 7248 default: 7249 ZEND_UNREACHABLE(); 7250 } 7251 | jmp => target_label2 7252 } else { 7253 ZEND_UNREACHABLE(); 7254 } 7255 } else { 7256 switch (opline->opcode) { 7257 case ZEND_IS_EQUAL: 7258 case ZEND_IS_IDENTICAL: 7259 case ZEND_CASE: 7260 case ZEND_CASE_STRICT: 7261 | sete al 7262 break; 7263 case ZEND_IS_NOT_EQUAL: 7264 case ZEND_IS_NOT_IDENTICAL: 7265 | setne al 7266 break; 7267 case ZEND_IS_SMALLER: 7268 if (swap) { 7269 | setg al 7270 } else { 7271 | setl al 7272 } 7273 break; 7274 case ZEND_IS_SMALLER_OR_EQUAL: 7275 if (swap) { 7276 | setge al 7277 } else { 7278 | setle al 7279 } 7280 break; 7281 default: 7282 ZEND_UNREACHABLE(); 7283 } 7284 | movzx eax, al 7285 | add eax, 2 7286 | SET_ZVAL_TYPE_INFO res_addr, eax 7287 } 7288 7289 return 1; 7290} 7291 7292static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7293{ 7294 if (smart_branch_opcode) { 7295 if (smart_branch_opcode == ZEND_JMPZ) { 7296 switch (opline->opcode) { 7297 case ZEND_IS_EQUAL: 7298 case ZEND_IS_IDENTICAL: 7299 case ZEND_CASE: 7300 case ZEND_CASE_STRICT: 7301 if (exit_addr) { 7302 | jne &exit_addr 7303 | jp &exit_addr 7304 } else { 7305 | jne => target_label 7306 | jp => target_label 7307 } 7308 break; 7309 case ZEND_IS_NOT_EQUAL: 7310 | jp >1 7311 if (exit_addr) { 7312 | je &exit_addr 7313 } else { 7314 | je => target_label 7315 } 7316 |1: 7317 break; 7318 case ZEND_IS_NOT_IDENTICAL: 7319 if (exit_addr) { 7320 | jne &exit_addr 7321 | jp &exit_addr 7322 } else { 7323 | jp >1 7324 | je => target_label 7325 |1: 7326 } 7327 break; 7328 case ZEND_IS_SMALLER: 7329 if (swap) { 7330 if (exit_addr) { 7331 | jbe &exit_addr 7332 } else { 7333 | jbe => target_label 7334 } 7335 } else { 7336 if (exit_addr) { 7337 | jae &exit_addr 7338 | jp &exit_addr 7339 } else { 7340 | jae => target_label 7341 | jp => target_label 7342 } 7343 } 7344 break; 7345 case ZEND_IS_SMALLER_OR_EQUAL: 7346 if (swap) { 7347 if (exit_addr) { 7348 | jb &exit_addr 7349 } else { 7350 | jb => target_label 7351 } 7352 } else { 7353 if (exit_addr) { 7354 | ja &exit_addr 7355 | jp &exit_addr 7356 } else { 7357 | ja => target_label 7358 | jp => target_label 7359 } 7360 } 7361 break; 7362 default: 7363 ZEND_UNREACHABLE(); 7364 } 7365 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7366 switch (opline->opcode) { 7367 case ZEND_IS_EQUAL: 7368 case ZEND_IS_IDENTICAL: 7369 case ZEND_CASE: 7370 case ZEND_CASE_STRICT: 7371 | jp >1 7372 if (exit_addr) { 7373 | je &exit_addr 7374 } else { 7375 | je => target_label 7376 } 7377 |1: 7378 break; 7379 case ZEND_IS_NOT_EQUAL: 7380 if (exit_addr) { 7381 | jne &exit_addr 7382 | jp &exit_addr 7383 } else { 7384 | jne => target_label 7385 | jp => target_label 7386 } 7387 break; 7388 case ZEND_IS_NOT_IDENTICAL: 7389 if (exit_addr) { 7390 | jp >1 7391 | je &exit_addr 7392 |1: 7393 } else { 7394 | jne => target_label 7395 | jp => target_label 7396 } 7397 break; 7398 case ZEND_IS_SMALLER: 7399 if (swap) { 7400 if (exit_addr) { 7401 | ja &exit_addr 7402 } else { 7403 | ja => target_label 7404 } 7405 } else { 7406 | jp >1 7407 if (exit_addr) { 7408 | jb &exit_addr 7409 } else { 7410 | jb => target_label 7411 } 7412 |1: 7413 } 7414 break; 7415 case ZEND_IS_SMALLER_OR_EQUAL: 7416 if (swap) { 7417 if (exit_addr) { 7418 | jae &exit_addr 7419 } else { 7420 | jae => target_label 7421 } 7422 } else { 7423 | jp >1 7424 if (exit_addr) { 7425 | jbe &exit_addr 7426 } else { 7427 | jbe => target_label 7428 } 7429 |1: 7430 } 7431 break; 7432 default: 7433 ZEND_UNREACHABLE(); 7434 } 7435 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7436 switch (opline->opcode) { 7437 case ZEND_IS_EQUAL: 7438 case ZEND_IS_IDENTICAL: 7439 case ZEND_CASE: 7440 case ZEND_CASE_STRICT: 7441 | jne => target_label 7442 | jp => target_label 7443 break; 7444 case ZEND_IS_NOT_EQUAL: 7445 case ZEND_IS_NOT_IDENTICAL: 7446 | jp => target_label2 7447 | je => target_label 7448 break; 7449 case ZEND_IS_SMALLER: 7450 if (swap) { 7451 | jbe => target_label 7452 } else { 7453 | jae => target_label 7454 | jp => target_label 7455 } 7456 break; 7457 case ZEND_IS_SMALLER_OR_EQUAL: 7458 if (swap) { 7459 | jb => target_label 7460 } else { 7461 | ja => target_label 7462 | jp => target_label 7463 } 7464 break; 7465 default: 7466 ZEND_UNREACHABLE(); 7467 } 7468 | jmp => target_label2 7469 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7470 switch (opline->opcode) { 7471 case ZEND_IS_EQUAL: 7472 case ZEND_IS_IDENTICAL: 7473 case ZEND_CASE: 7474 case ZEND_CASE_STRICT: 7475 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7476 | jne => target_label 7477 | jp => target_label 7478 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7479 break; 7480 case ZEND_IS_NOT_EQUAL: 7481 case ZEND_IS_NOT_IDENTICAL: 7482 | jp >1 7483 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7484 | je => target_label 7485 |1: 7486 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7487 break; 7488 case ZEND_IS_SMALLER: 7489 if (swap) { 7490 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7491 | jbe => target_label 7492 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7493 } else { 7494 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7495 | jae => target_label 7496 | jp => target_label 7497 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7498 } 7499 break; 7500 case ZEND_IS_SMALLER_OR_EQUAL: 7501 if (swap) { 7502 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7503 | jb => target_label 7504 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7505 } else { 7506 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7507 | ja => target_label 7508 | jp => target_label 7509 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7510 } 7511 break; 7512 default: 7513 ZEND_UNREACHABLE(); 7514 } 7515 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7516 switch (opline->opcode) { 7517 case ZEND_IS_EQUAL: 7518 case ZEND_IS_IDENTICAL: 7519 case ZEND_CASE: 7520 case ZEND_CASE_STRICT: 7521 | jp >1 7522 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7523 | je => target_label 7524 |1: 7525 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7526 break; 7527 case ZEND_IS_NOT_EQUAL: 7528 case ZEND_IS_NOT_IDENTICAL: 7529 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7530 | jne => target_label 7531 | jp => target_label 7532 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7533 break; 7534 case ZEND_IS_SMALLER: 7535 if (swap) { 7536 | seta al 7537 | movzx eax, al 7538 | lea eax, [eax + 2] 7539 | SET_ZVAL_TYPE_INFO res_addr, eax 7540 | ja => target_label 7541 } else { 7542 | jp >1 7543 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7544 | jb => target_label 7545 |1: 7546 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7547 } 7548 break; 7549 case ZEND_IS_SMALLER_OR_EQUAL: 7550 if (swap) { 7551 | setae al 7552 | movzx eax, al 7553 | lea eax, [eax + 2] 7554 | SET_ZVAL_TYPE_INFO res_addr, eax 7555 | jae => target_label 7556 } else { 7557 | jp >1 7558 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7559 | jbe => target_label 7560 |1: 7561 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7562 } 7563 break; 7564 default: 7565 ZEND_UNREACHABLE(); 7566 } 7567 } else { 7568 ZEND_UNREACHABLE(); 7569 } 7570 } else { 7571 switch (opline->opcode) { 7572 case ZEND_IS_EQUAL: 7573 case ZEND_IS_IDENTICAL: 7574 case ZEND_CASE: 7575 case ZEND_CASE_STRICT: 7576 | jp >1 7577 | mov eax, IS_TRUE 7578 | je >2 7579 |1: 7580 | mov eax, IS_FALSE 7581 |2: 7582 break; 7583 case ZEND_IS_NOT_EQUAL: 7584 case ZEND_IS_NOT_IDENTICAL: 7585 | jp >1 7586 | mov eax, IS_FALSE 7587 | je >2 7588 |1: 7589 | mov eax, IS_TRUE 7590 |2: 7591 break; 7592 case ZEND_IS_SMALLER: 7593 if (swap) { 7594 | seta al 7595 | movzx eax, al 7596 | add eax, 2 7597 } else { 7598 | jp >1 7599 | mov eax, IS_TRUE 7600 | jb >2 7601 |1: 7602 | mov eax, IS_FALSE 7603 |2: 7604 } 7605 break; 7606 case ZEND_IS_SMALLER_OR_EQUAL: 7607 if (swap) { 7608 | setae al 7609 | movzx eax, al 7610 | add eax, 2 7611 } else { 7612 | jp >1 7613 | mov eax, IS_TRUE 7614 | jbe >2 7615 |1: 7616 | mov eax, IS_FALSE 7617 |2: 7618 } 7619 break; 7620 default: 7621 ZEND_UNREACHABLE(); 7622 } 7623 | SET_ZVAL_TYPE_INFO res_addr, eax 7624 } 7625 7626 return 1; 7627} 7628 7629static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7630{ 7631 zend_reg tmp_reg = ZREG_XMM0; 7632 7633 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7634 | DOUBLE_CMP tmp_reg, op2_addr 7635 7636 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7637} 7638 7639static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7640{ 7641 zend_reg tmp_reg = ZREG_XMM0; 7642 7643 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7644 | DOUBLE_CMP tmp_reg, op1_addr 7645 7646 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7647} 7648 7649static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7650{ 7651 bool swap = 0; 7652 7653 if (Z_MODE(op1_addr) == IS_REG) { 7654 | DOUBLE_CMP Z_REG(op1_addr), op2_addr 7655 } else if (Z_MODE(op2_addr) == IS_REG) { 7656 | DOUBLE_CMP Z_REG(op2_addr), op1_addr 7657 swap = 1; 7658 } else { 7659 zend_reg tmp_reg = ZREG_XMM0; 7660 7661 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 7662 | DOUBLE_CMP tmp_reg, op2_addr 7663 } 7664 7665 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7666} 7667 7668static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7669{ 7670 | test, eax, eax 7671 if (smart_branch_opcode) { 7672 if (smart_branch_opcode == ZEND_JMPZ_EX || 7673 smart_branch_opcode == ZEND_JMPNZ_EX) { 7674 switch (opline->opcode) { 7675 case ZEND_IS_EQUAL: 7676 case ZEND_CASE: 7677 | sete al 7678 break; 7679 case ZEND_IS_NOT_EQUAL: 7680 | setne al 7681 break; 7682 case ZEND_IS_SMALLER: 7683 | setl al 7684 break; 7685 case ZEND_IS_SMALLER_OR_EQUAL: 7686 | setle al 7687 break; 7688 default: 7689 ZEND_UNREACHABLE(); 7690 } 7691 | movzx eax, al 7692 | lea eax, [eax + 2] 7693 | SET_ZVAL_TYPE_INFO res_addr, eax 7694 } 7695 if (smart_branch_opcode == ZEND_JMPZ || 7696 smart_branch_opcode == ZEND_JMPZ_EX) { 7697 switch (opline->opcode) { 7698 case ZEND_IS_EQUAL: 7699 case ZEND_CASE: 7700 if (exit_addr) { 7701 | jne &exit_addr 7702 } else { 7703 | jne => target_label 7704 } 7705 break; 7706 case ZEND_IS_NOT_EQUAL: 7707 if (exit_addr) { 7708 | je &exit_addr 7709 } else { 7710 | je => target_label 7711 } 7712 break; 7713 case ZEND_IS_SMALLER: 7714 if (exit_addr) { 7715 | jge &exit_addr 7716 } else { 7717 | jge => target_label 7718 } 7719 break; 7720 case ZEND_IS_SMALLER_OR_EQUAL: 7721 if (exit_addr) { 7722 | jg &exit_addr 7723 } else { 7724 | jg => target_label 7725 } 7726 break; 7727 default: 7728 ZEND_UNREACHABLE(); 7729 } 7730 } else if (smart_branch_opcode == ZEND_JMPNZ || 7731 smart_branch_opcode == ZEND_JMPNZ_EX) { 7732 switch (opline->opcode) { 7733 case ZEND_IS_EQUAL: 7734 case ZEND_CASE: 7735 if (exit_addr) { 7736 | je &exit_addr 7737 } else { 7738 | je => target_label 7739 } 7740 break; 7741 case ZEND_IS_NOT_EQUAL: 7742 if (exit_addr) { 7743 | jne &exit_addr 7744 } else { 7745 | jne => target_label 7746 } 7747 break; 7748 case ZEND_IS_SMALLER: 7749 if (exit_addr) { 7750 | jl &exit_addr 7751 } else { 7752 | jl => target_label 7753 } 7754 break; 7755 case ZEND_IS_SMALLER_OR_EQUAL: 7756 if (exit_addr) { 7757 | jle &exit_addr 7758 } else { 7759 | jle => target_label 7760 } 7761 break; 7762 default: 7763 ZEND_UNREACHABLE(); 7764 } 7765 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7766 switch (opline->opcode) { 7767 case ZEND_IS_EQUAL: 7768 case ZEND_CASE: 7769 | jne => target_label 7770 break; 7771 case ZEND_IS_NOT_EQUAL: 7772 | je => target_label 7773 break; 7774 case ZEND_IS_SMALLER: 7775 | jge => target_label 7776 break; 7777 case ZEND_IS_SMALLER_OR_EQUAL: 7778 | jg => target_label 7779 break; 7780 default: 7781 ZEND_UNREACHABLE(); 7782 } 7783 | jmp => target_label2 7784 } else { 7785 ZEND_UNREACHABLE(); 7786 } 7787 } else { 7788 switch (opline->opcode) { 7789 case ZEND_IS_EQUAL: 7790 case ZEND_CASE: 7791 | sete al 7792 break; 7793 case ZEND_IS_NOT_EQUAL: 7794 | setne al 7795 break; 7796 case ZEND_IS_SMALLER: 7797 | setl al 7798 break; 7799 case ZEND_IS_SMALLER_OR_EQUAL: 7800 | setle al 7801 break; 7802 default: 7803 ZEND_UNREACHABLE(); 7804 } 7805 | movzx eax, al 7806 | add eax, 2 7807 | SET_ZVAL_TYPE_INFO res_addr, eax 7808 } 7809 7810 return 1; 7811} 7812 7813static int zend_jit_cmp(dasm_State **Dst, 7814 const zend_op *opline, 7815 uint32_t op1_info, 7816 zend_ssa_range *op1_range, 7817 zend_jit_addr op1_addr, 7818 uint32_t op2_info, 7819 zend_ssa_range *op2_range, 7820 zend_jit_addr op2_addr, 7821 zend_jit_addr res_addr, 7822 int may_throw, 7823 zend_uchar smart_branch_opcode, 7824 uint32_t target_label, 7825 uint32_t target_label2, 7826 const void *exit_addr, 7827 bool skip_comparison) 7828{ 7829 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7830 bool has_slow; 7831 7832 has_slow = 7833 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7834 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7835 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7836 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7837 7838 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7839 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7840 if (op1_info & MAY_BE_DOUBLE) { 7841 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7842 } else { 7843 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7844 } 7845 } 7846 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7847 if (op2_info & MAY_BE_DOUBLE) { 7848 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7849 |.cold_code 7850 |3: 7851 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7852 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7853 } 7854 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7855 return 0; 7856 } 7857 | jmp >6 7858 |.code 7859 } else { 7860 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7861 } 7862 } 7863 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7864 return 0; 7865 } 7866 if (op1_info & MAY_BE_DOUBLE) { 7867 |.cold_code 7868 |4: 7869 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7870 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7871 } 7872 if (op2_info & MAY_BE_DOUBLE) { 7873 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7874 if (!same_ops) { 7875 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 7876 } else { 7877 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7878 } 7879 } 7880 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7881 return 0; 7882 } 7883 | jmp >6 7884 } 7885 if (!same_ops) { 7886 |5: 7887 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7888 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7889 } 7890 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7891 return 0; 7892 } 7893 | jmp >6 7894 } 7895 |.code 7896 } 7897 } else if ((op1_info & MAY_BE_DOUBLE) && 7898 !(op1_info & MAY_BE_LONG) && 7899 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7900 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7901 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7902 } 7903 if (op2_info & MAY_BE_DOUBLE) { 7904 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7905 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7906 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 7907 } else { 7908 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7909 } 7910 } 7911 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7912 return 0; 7913 } 7914 } 7915 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7916 if (op2_info & MAY_BE_DOUBLE) { 7917 |.cold_code 7918 } 7919 |3: 7920 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7921 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7922 } 7923 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7924 return 0; 7925 } 7926 if (op2_info & MAY_BE_DOUBLE) { 7927 | jmp >6 7928 |.code 7929 } 7930 } 7931 } else if ((op2_info & MAY_BE_DOUBLE) && 7932 !(op2_info & MAY_BE_LONG) && 7933 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7934 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7935 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7936 } 7937 if (op1_info & MAY_BE_DOUBLE) { 7938 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7939 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7940 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 7941 } else { 7942 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7943 } 7944 } 7945 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7946 return 0; 7947 } 7948 } 7949 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7950 if (op1_info & MAY_BE_DOUBLE) { 7951 |.cold_code 7952 } 7953 |3: 7954 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7955 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7956 } 7957 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7958 return 0; 7959 } 7960 if (op1_info & MAY_BE_DOUBLE) { 7961 | jmp >6 7962 |.code 7963 } 7964 } 7965 } 7966 7967 if (has_slow || 7968 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7969 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7970 if (has_slow) { 7971 |.cold_code 7972 |9: 7973 } 7974 | SET_EX_OPLINE opline, r0 7975 if (Z_MODE(op1_addr) == IS_REG) { 7976 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7977 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7978 return 0; 7979 } 7980 op1_addr = real_addr; 7981 } 7982 if (Z_MODE(op2_addr) == IS_REG) { 7983 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7984 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7985 return 0; 7986 } 7987 op2_addr = real_addr; 7988 } 7989 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7990 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 7991 | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 7992 | mov FCARG1a, opline->op1.var 7993 | EXT_CALL zend_jit_undefined_op_helper, r0 7994 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 7995 |1: 7996 } 7997 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 7998 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 7999 | mov T1, FCARG1a // save 8000 | mov FCARG1a, opline->op2.var 8001 | EXT_CALL zend_jit_undefined_op_helper, r0 8002 | mov FCARG1a, T1 // restore 8003 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8004 | jmp >2 8005 |1: 8006 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8007 |2: 8008 } else { 8009 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8010 } 8011 | EXT_CALL zend_compare, r0 8012 if ((opline->opcode != ZEND_CASE && 8013 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8014 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8015 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8016 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8017 | mov dword T1, eax // save 8018 if (opline->opcode != ZEND_CASE) { 8019 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL 8020 } 8021 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 8022 if (may_throw) { 8023 zend_jit_check_exception_undef_result(Dst, opline); 8024 } 8025 | mov eax, dword T1 // restore 8026 } else if (may_throw) { 8027#if ZTS 8028 | mov dword T1, eax // save 8029#else 8030 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8031 | mov dword T1, eax // save 8032 } 8033#endif 8034 zend_jit_check_exception_undef_result(Dst, opline); 8035#if ZTS 8036 | mov eax, dword T1 // restore 8037#else 8038 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8039 | mov eax, dword T1 // restore 8040 } 8041#endif 8042 } 8043 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8044 return 0; 8045 } 8046 if (has_slow) { 8047 | jmp >6 8048 |.code 8049 } 8050 } 8051 8052 |6: 8053 8054 return 1; 8055} 8056 8057static int zend_jit_identical(dasm_State **Dst, 8058 const zend_op *opline, 8059 uint32_t op1_info, 8060 zend_ssa_range *op1_range, 8061 zend_jit_addr op1_addr, 8062 uint32_t op2_info, 8063 zend_ssa_range *op2_range, 8064 zend_jit_addr op2_addr, 8065 zend_jit_addr res_addr, 8066 int may_throw, 8067 zend_uchar smart_branch_opcode, 8068 uint32_t target_label, 8069 uint32_t target_label2, 8070 const void *exit_addr, 8071 bool skip_comparison) 8072{ 8073 uint32_t identical_label = (uint32_t)-1; 8074 uint32_t not_identical_label = (uint32_t)-1; 8075 8076 if (smart_branch_opcode && !exit_addr) { 8077 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8078 if (smart_branch_opcode == ZEND_JMPZ) { 8079 not_identical_label = target_label; 8080 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8081 identical_label = target_label; 8082 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8083 not_identical_label = target_label; 8084 identical_label = target_label2; 8085 } else { 8086 ZEND_UNREACHABLE(); 8087 } 8088 } else { 8089 if (smart_branch_opcode == ZEND_JMPZ) { 8090 identical_label = target_label; 8091 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8092 not_identical_label = target_label; 8093 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8094 identical_label = target_label; 8095 not_identical_label = target_label2; 8096 } else { 8097 ZEND_UNREACHABLE(); 8098 } 8099 } 8100 } 8101 8102 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8103 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8104 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8105 return 0; 8106 } 8107 return 1; 8108 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8109 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8110 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8111 return 0; 8112 } 8113 return 1; 8114 } 8115 8116 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8117 op1_info |= MAY_BE_NULL; 8118 op2_info |= MAY_BE_NULL; 8119 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8120 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8121 |.cold_code 8122 |1: 8123 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8124 | SET_EX_OPLINE opline, r0 8125 | mov FCARG1d, opline->op1.var 8126 | EXT_CALL zend_jit_undefined_op_helper, r0 8127 if (may_throw) { 8128 zend_jit_check_exception_undef_result(Dst, opline); 8129 } 8130 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8131 | jmp >1 8132 |.code 8133 |1: 8134 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8135 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8136 |.cold_code 8137 |1: 8138 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8139 | SET_EX_OPLINE opline, r0 8140 | mov aword T1, FCARG1a // save 8141 | mov FCARG1d, opline->op2.var 8142 | EXT_CALL zend_jit_undefined_op_helper, r0 8143 if (may_throw) { 8144 zend_jit_check_exception_undef_result(Dst, opline); 8145 } 8146 | mov FCARG1a, aword T1 // restore 8147 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8148 | jmp >1 8149 |.code 8150 |1: 8151 } else if (op1_info & MAY_BE_UNDEF) { 8152 op1_info |= MAY_BE_NULL; 8153 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8154 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8155 |.cold_code 8156 |1: 8157 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8158 | SET_EX_OPLINE opline, r0 8159 | mov FCARG1d, opline->op1.var 8160 | EXT_CALL zend_jit_undefined_op_helper, r0 8161 if (may_throw) { 8162 zend_jit_check_exception_undef_result(Dst, opline); 8163 } 8164 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8165 | jmp >1 8166 |.code 8167 |1: 8168 if (opline->op2_type != IS_CONST) { 8169 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8170 } 8171 } else if (op2_info & MAY_BE_UNDEF) { 8172 op2_info |= MAY_BE_NULL; 8173 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8174 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8175 |.cold_code 8176 |1: 8177 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8178 | SET_EX_OPLINE opline, r0 8179 | mov FCARG1d, opline->op2.var 8180 | EXT_CALL zend_jit_undefined_op_helper, r0 8181 if (may_throw) { 8182 zend_jit_check_exception_undef_result(Dst, opline); 8183 } 8184 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8185 | jmp >1 8186 |.code 8187 |1: 8188 if (opline->op1_type != IS_CONST) { 8189 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8190 } 8191 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 8192 if (opline->op1_type != IS_CONST) { 8193 if (Z_MODE(op1_addr) == IS_REG) { 8194 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8195 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8196 return 0; 8197 } 8198 op1_addr = real_addr; 8199 } 8200 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8201 } 8202 if (opline->op2_type != IS_CONST) { 8203 if (Z_MODE(op2_addr) == IS_REG) { 8204 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8205 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8206 return 0; 8207 } 8208 op2_addr = real_addr; 8209 } 8210 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8211 } 8212 } 8213 8214 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8215 if ((opline->opcode != ZEND_CASE_STRICT && 8216 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8217 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8218 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8219 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8220 if (opline->opcode != ZEND_CASE_STRICT) { 8221 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8222 } 8223 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8224 } 8225 if (smart_branch_opcode) { 8226 if (may_throw) { 8227 zend_jit_check_exception_undef_result(Dst, opline); 8228 } 8229 if (exit_addr) { 8230 if (smart_branch_opcode == ZEND_JMPZ) { 8231 | jmp &exit_addr 8232 } 8233 } else if (not_identical_label != (uint32_t)-1) { 8234 | jmp =>not_identical_label 8235 } 8236 } else { 8237 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8238 if (may_throw) { 8239 zend_jit_check_exception(Dst); 8240 } 8241 } 8242 return 1; 8243 } 8244 8245 if (opline->op1_type & (IS_CV|IS_VAR)) { 8246 | ZVAL_DEREF FCARG1a, op1_info 8247 } 8248 if (opline->op2_type & (IS_CV|IS_VAR)) { 8249 | ZVAL_DEREF FCARG2a, op2_info 8250 } 8251 8252 if (has_concrete_type(op1_info) 8253 && has_concrete_type(op2_info) 8254 && concrete_type(op1_info) == concrete_type(op2_info) 8255 && concrete_type(op1_info) <= IS_TRUE) { 8256 if (smart_branch_opcode) { 8257 if (exit_addr) { 8258 if (smart_branch_opcode == ZEND_JMPNZ) { 8259 | jmp &exit_addr 8260 } 8261 } else if (identical_label != (uint32_t)-1) { 8262 | jmp =>identical_label 8263 } 8264 } else { 8265 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8266 } 8267 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8268 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8269 if (smart_branch_opcode) { 8270 if (exit_addr) { 8271 if (smart_branch_opcode == ZEND_JMPNZ) { 8272 | jmp &exit_addr 8273 } 8274 } else if (identical_label != (uint32_t)-1) { 8275 | jmp =>identical_label 8276 } 8277 } else { 8278 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8279 } 8280 } else { 8281 if (smart_branch_opcode) { 8282 if (exit_addr) { 8283 if (smart_branch_opcode == ZEND_JMPZ) { 8284 | jmp &exit_addr 8285 } 8286 } else if (not_identical_label != (uint32_t)-1) { 8287 | jmp =>not_identical_label 8288 } 8289 } else { 8290 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8291 } 8292 } 8293 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8294 zval *val = Z_ZV(op1_addr); 8295 8296 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8297 if (smart_branch_opcode) { 8298 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8299 | jne >8 8300 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8301 if (may_throw) { 8302 zend_jit_check_exception_undef_result(Dst, opline); 8303 } 8304 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8305 | jmp &exit_addr 8306 } else if (identical_label != (uint32_t)-1) { 8307 | jmp =>identical_label 8308 } else { 8309 | jmp >9 8310 } 8311 |8: 8312 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8313 | je &exit_addr 8314 } else if (identical_label != (uint32_t)-1) { 8315 | je =>identical_label 8316 } else { 8317 | je >9 8318 } 8319 } else { 8320 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8321 | sete al 8322 } else { 8323 | setne al 8324 } 8325 | movzx eax, al 8326 | lea eax, [eax + 2] 8327 | SET_ZVAL_TYPE_INFO res_addr, eax 8328 } 8329 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8330 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8331 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8332 if (may_throw) { 8333 zend_jit_check_exception_undef_result(Dst, opline); 8334 } 8335 } 8336 if (exit_addr) { 8337 if (smart_branch_opcode == ZEND_JMPZ) { 8338 | jmp &exit_addr 8339 } 8340 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8341 | jmp =>not_identical_label 8342 } 8343 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8344 zval *val = Z_ZV(op2_addr); 8345 8346 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8347 if (smart_branch_opcode) { 8348 if (opline->opcode != ZEND_CASE_STRICT 8349 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8350 | jne >8 8351 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8352 if (may_throw) { 8353 zend_jit_check_exception_undef_result(Dst, opline); 8354 } 8355 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8356 | jmp &exit_addr 8357 } else if (identical_label != (uint32_t)-1) { 8358 | jmp =>identical_label 8359 } else { 8360 | jmp >9 8361 } 8362 |8: 8363 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8364 | je &exit_addr 8365 } else if (identical_label != (uint32_t)-1) { 8366 | je =>identical_label 8367 } else { 8368 | je >9 8369 } 8370 } else { 8371 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8372 | sete al 8373 } else { 8374 | setne al 8375 } 8376 | movzx eax, al 8377 | lea eax, [eax + 2] 8378 | SET_ZVAL_TYPE_INFO res_addr, eax 8379 } 8380 if (opline->opcode != ZEND_CASE_STRICT 8381 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8382 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8383 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8384 if (may_throw) { 8385 zend_jit_check_exception_undef_result(Dst, opline); 8386 } 8387 } 8388 if (smart_branch_opcode) { 8389 if (exit_addr) { 8390 if (smart_branch_opcode == ZEND_JMPZ) { 8391 | jmp &exit_addr 8392 } 8393 } else if (not_identical_label != (uint32_t)-1) { 8394 | jmp =>not_identical_label 8395 } 8396 } 8397 } else { 8398 if (opline->op1_type == IS_CONST) { 8399 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8400 } 8401 if (opline->op2_type == IS_CONST) { 8402 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8403 } 8404 | EXT_CALL zend_is_identical, r0 8405 if ((opline->opcode != ZEND_CASE_STRICT && 8406 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8407 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8408 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8409 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8410 | mov aword T1, r0 // save 8411 if (opline->opcode != ZEND_CASE_STRICT) { 8412 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8413 } 8414 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8415 if (may_throw) { 8416 zend_jit_check_exception_undef_result(Dst, opline); 8417 } 8418 | mov r0, aword T1 // restore 8419 } 8420 if (smart_branch_opcode) { 8421 | test al, al 8422 if (exit_addr) { 8423 if (smart_branch_opcode == ZEND_JMPNZ) { 8424 | jnz &exit_addr 8425 } else { 8426 | jz &exit_addr 8427 } 8428 } else if (not_identical_label != (uint32_t)-1) { 8429 | jz =>not_identical_label 8430 if (identical_label != (uint32_t)-1) { 8431 | jmp =>identical_label 8432 } 8433 } else if (identical_label != (uint32_t)-1) { 8434 | jnz =>identical_label 8435 } 8436 } else { 8437 | movzx eax, al 8438 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8439 | lea eax, [eax + 2] 8440 } else { 8441 | neg eax 8442 | lea eax, [eax + 3] 8443 } 8444 | SET_ZVAL_TYPE_INFO res_addr, eax 8445 } 8446 } 8447 8448 |9: 8449 if (may_throw) { 8450 zend_jit_check_exception(Dst); 8451 } 8452 return 1; 8453} 8454 8455static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 8456{ 8457 uint32_t true_label = -1; 8458 uint32_t false_label = -1; 8459 bool set_bool = 0; 8460 bool set_bool_not = 0; 8461 bool set_delayed = 0; 8462 bool jmp_done = 0; 8463 8464 if (branch_opcode == ZEND_BOOL) { 8465 set_bool = 1; 8466 } else if (branch_opcode == ZEND_BOOL_NOT) { 8467 set_bool = 1; 8468 set_bool_not = 1; 8469 } else if (branch_opcode == ZEND_JMPZ) { 8470 false_label = target_label; 8471 } else if (branch_opcode == ZEND_JMPNZ) { 8472 true_label = target_label; 8473 } else if (branch_opcode == ZEND_JMPZNZ) { 8474 true_label = target_label2; 8475 false_label = target_label; 8476 } else if (branch_opcode == ZEND_JMPZ_EX) { 8477 set_bool = 1; 8478 false_label = target_label; 8479 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8480 set_bool = 1; 8481 true_label = target_label; 8482 } else { 8483 ZEND_UNREACHABLE(); 8484 } 8485 8486 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8487 if (zend_is_true(Z_ZV(op1_addr))) { 8488 /* Always TRUE */ 8489 if (set_bool) { 8490 if (set_bool_not) { 8491 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8492 } else { 8493 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8494 } 8495 } 8496 if (true_label != (uint32_t)-1) { 8497 | jmp =>true_label; 8498 } 8499 } else { 8500 /* Always FALSE */ 8501 if (set_bool) { 8502 if (set_bool_not) { 8503 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8504 } else { 8505 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8506 } 8507 } 8508 if (false_label != (uint32_t)-1) { 8509 | jmp =>false_label; 8510 } 8511 } 8512 return 1; 8513 } 8514 8515 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8516 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8517 | ZVAL_DEREF FCARG1a, op1_info 8518 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8519 } 8520 8521 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8522 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8523 /* Always TRUE */ 8524 if (set_bool) { 8525 if (set_bool_not) { 8526 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8527 } else { 8528 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8529 } 8530 } 8531 if (true_label != (uint32_t)-1) { 8532 | jmp =>true_label; 8533 } 8534 } else { 8535 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8536 /* Always FALSE */ 8537 if (set_bool) { 8538 if (set_bool_not) { 8539 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8540 } else { 8541 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8542 } 8543 } 8544 } else { 8545 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8546 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8547 if ((op1_info & MAY_BE_LONG) && 8548 !(op1_info & MAY_BE_UNDEF) && 8549 !set_bool) { 8550 if (exit_addr) { 8551 if (branch_opcode == ZEND_JMPNZ) { 8552 | jl >9 8553 } else { 8554 | jl &exit_addr 8555 } 8556 } else if (false_label != (uint32_t)-1) { 8557 | jl =>false_label 8558 } else { 8559 | jl >9 8560 } 8561 jmp_done = 1; 8562 } else { 8563 | jg >2 8564 } 8565 } 8566 if (!(op1_info & MAY_BE_TRUE)) { 8567 /* It's FALSE */ 8568 if (set_bool) { 8569 if (set_bool_not) { 8570 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8571 } else { 8572 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8573 } 8574 } 8575 } else { 8576 if (exit_addr) { 8577 if (set_bool) { 8578 | jne >1 8579 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8580 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8581 | jmp &exit_addr 8582 } else { 8583 | jmp >9 8584 } 8585 |1: 8586 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8587 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8588 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8589 | jne &exit_addr 8590 } 8591 } 8592 } else { 8593 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8594 | je &exit_addr 8595 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8596 | jne &exit_addr 8597 } else { 8598 | je >9 8599 } 8600 } 8601 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8602 if (set_bool) { 8603 | jne >1 8604 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8605 if (true_label != (uint32_t)-1) { 8606 | jmp =>true_label 8607 } else { 8608 | jmp >9 8609 } 8610 |1: 8611 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8612 } else { 8613 if (true_label != (uint32_t)-1) { 8614 | je =>true_label 8615 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8616 | jne =>false_label 8617 jmp_done = 1; 8618 } else { 8619 | je >9 8620 } 8621 } 8622 } else if (set_bool) { 8623 | sete al 8624 | movzx eax, al 8625 if (set_bool_not) { 8626 | neg eax 8627 | add eax, 3 8628 } else { 8629 | add eax, 2 8630 } 8631 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8632 set_delayed = 1; 8633 } else { 8634 | SET_ZVAL_TYPE_INFO res_addr, eax 8635 } 8636 } 8637 } 8638 } 8639 8640 /* It's FALSE, but may be UNDEF */ 8641 if (op1_info & MAY_BE_UNDEF) { 8642 if (op1_info & MAY_BE_ANY) { 8643 if (set_delayed) { 8644 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8645 | SET_ZVAL_TYPE_INFO res_addr, eax 8646 | jz >1 8647 } else { 8648 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8649 } 8650 |.cold_code 8651 |1: 8652 } 8653 | mov FCARG1d, opline->op1.var 8654 | SET_EX_OPLINE opline, r0 8655 | EXT_CALL zend_jit_undefined_op_helper, r0 8656 8657 if (may_throw) { 8658 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8659 return 0; 8660 } 8661 } 8662 8663 if (exit_addr) { 8664 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8665 | jmp &exit_addr 8666 } 8667 } else if (false_label != (uint32_t)-1) { 8668 | jmp =>false_label 8669 } 8670 if (op1_info & MAY_BE_ANY) { 8671 if (exit_addr) { 8672 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8673 | jmp >9 8674 } 8675 } else if (false_label == (uint32_t)-1) { 8676 | jmp >9 8677 } 8678 |.code 8679 } 8680 } 8681 8682 if (!jmp_done) { 8683 if (exit_addr) { 8684 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8685 if (op1_info & MAY_BE_LONG) { 8686 | jmp >9 8687 } 8688 } else if (op1_info & MAY_BE_LONG) { 8689 | jmp &exit_addr 8690 } 8691 } else if (false_label != (uint32_t)-1) { 8692 | jmp =>false_label 8693 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8694 | jmp >9 8695 } 8696 } 8697 } 8698 } 8699 8700 if (op1_info & MAY_BE_LONG) { 8701 |2: 8702 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8703 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8704 } 8705 if (Z_MODE(op1_addr) == IS_REG) { 8706 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8707 } else { 8708 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8709 } 8710 if (set_bool) { 8711 | setne al 8712 | movzx eax, al 8713 if (set_bool_not) { 8714 | neg eax 8715 | add eax, 3 8716 } else { 8717 | lea eax, [eax + 2] 8718 } 8719 | SET_ZVAL_TYPE_INFO res_addr, eax 8720 } 8721 if (exit_addr) { 8722 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8723 | jne &exit_addr 8724 } else { 8725 | je &exit_addr 8726 } 8727 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8728 if (true_label != (uint32_t)-1) { 8729 | jne =>true_label 8730 if (false_label != (uint32_t)-1) { 8731 | jmp =>false_label 8732 } 8733 } else { 8734 | je =>false_label 8735 } 8736 } 8737 } 8738 8739 if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8740 |2: 8741 if (CAN_USE_AVX()) { 8742 | vxorps xmm0, xmm0, xmm0 8743 } else { 8744 | xorps xmm0, xmm0 8745 } 8746 | DOUBLE_CMP ZREG_XMM0, op1_addr 8747 8748 if (set_bool) { 8749 if (exit_addr) { 8750 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8751 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8752 | jp &exit_addr 8753 | jne &exit_addr 8754 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8755 } else { 8756 | jp >1 8757 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8758 | je &exit_addr 8759 |1: 8760 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8761 } 8762 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8763 | jp >1 8764 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8765 | je => false_label 8766 |1: 8767 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8768 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8769 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8770 | jp => true_label 8771 | jne => true_label 8772 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8773 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8774 | mov eax, IS_FALSE 8775 | jp >1 8776 | jne >1 8777 | mov eax, IS_TRUE 8778 |1: 8779 | SET_ZVAL_TYPE_INFO res_addr, eax 8780 } else { // BOOL (p=>true, z=>false) 8781 | mov eax, IS_TRUE 8782 | jp >1 8783 | jne >1 8784 | mov eax, IS_FALSE 8785 |1: 8786 | SET_ZVAL_TYPE_INFO res_addr, eax 8787 } 8788 } else { 8789 if (exit_addr) { 8790 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8791 | jp &exit_addr 8792 | jne &exit_addr 8793 |1: 8794 } else { 8795 | jp >1 8796 | je &exit_addr 8797 |1: 8798 } 8799 } else { 8800 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8801 if (false_label != (uint32_t)-1 ) { 8802 | jp >1 8803 | je => false_label 8804 |1: 8805 if (true_label != (uint32_t)-1) { 8806 | jmp =>true_label 8807 } 8808 } else { 8809 | jp => true_label 8810 | jne => true_label 8811 } 8812 } 8813 } 8814 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8815 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8816 |.cold_code 8817 |2: 8818 } 8819 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8820 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8821 } 8822 | SET_EX_OPLINE opline, r0 8823 | EXT_CALL zend_is_true, r0 8824 8825 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8826 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8827 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8828 8829 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8830 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8831 } 8832 | GET_ZVAL_PTR FCARG1a, op1_addr 8833 | GC_DELREF FCARG1a 8834 | jnz >3 8835 | mov aword T1, r0 // save 8836 | ZVAL_DTOR_FUNC op1_info, opline 8837 | mov r0, aword T1 // restore 8838 |3: 8839 } 8840 if (may_throw) { 8841 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 8842 | jne ->exception_handler_undef 8843 } 8844 8845 if (set_bool) { 8846 if (set_bool_not) { 8847 | neg eax 8848 | add eax, 3 8849 } else { 8850 | add eax, 2 8851 } 8852 | SET_ZVAL_TYPE_INFO res_addr, eax 8853 if (exit_addr) { 8854 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8855 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8856 | jne &exit_addr 8857 } else { 8858 | je &exit_addr 8859 } 8860 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8861 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8862 if (true_label != (uint32_t)-1) { 8863 | jne =>true_label 8864 if (false_label != (uint32_t)-1) { 8865 | jmp =>false_label 8866 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8867 | jmp >9 8868 } 8869 } else { 8870 | je =>false_label 8871 } 8872 } 8873 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8874 | jmp >9 8875 |.code 8876 } 8877 } else { 8878 | test r0, r0 8879 if (exit_addr) { 8880 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8881 | jne &exit_addr 8882 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8883 | jmp >9 8884 } 8885 } else { 8886 | je &exit_addr 8887 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8888 | jmp >9 8889 } 8890 } 8891 } else if (true_label != (uint32_t)-1) { 8892 | jne =>true_label 8893 if (false_label != (uint32_t)-1) { 8894 | jmp =>false_label 8895 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8896 | jmp >9 8897 } 8898 } else { 8899 | je =>false_label 8900 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8901 | jmp >9 8902 } 8903 } 8904 8905 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8906 |.code 8907 } 8908 } 8909 } 8910 8911 |9: 8912 8913 return 1; 8914} 8915 8916static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8917{ 8918 if (op1_addr != op1_def_addr) { 8919 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8920 return 0; 8921 } 8922 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8923 op1_addr = op1_def_addr; 8924 } 8925 } 8926 8927 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { 8928 return 0; 8929 } 8930 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8931 return 0; 8932 } 8933 if (op1_info & MAY_BE_UNDEF) { 8934 zend_jit_check_exception(Dst); 8935 } 8936 return 1; 8937} 8938 8939static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8940{ 8941 ZEND_ASSERT(opline->op1_type == IS_CV); 8942 8943 if (op2_addr != op2_def_addr) { 8944 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8945 return 0; 8946 } 8947 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8948 op2_addr = op2_def_addr; 8949 } 8950 } 8951 8952 if (Z_MODE(op1_addr) != IS_REG 8953 && Z_MODE(op1_use_addr) == IS_REG 8954 && !Z_LOAD(op1_use_addr) 8955 && !Z_STORE(op1_use_addr)) { 8956 /* Force type update */ 8957 op1_info |= MAY_BE_UNDEF; 8958 } 8959 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8960 may_throw)) { 8961 return 0; 8962 } 8963 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8964 return 0; 8965 } 8966 if (opline->result_type != IS_UNUSED) { 8967 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8968 return 0; 8969 } 8970 } 8971 8972 return 1; 8973} 8974 8975/* copy of hidden zend_closure */ 8976typedef struct _zend_closure { 8977 zend_object std; 8978 zend_function func; 8979 zval this_ptr; 8980 zend_class_entry *called_scope; 8981 zif_handler orig_internal_handler; 8982} zend_closure; 8983 8984static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8985{ 8986 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8987 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8988 8989 if (!exit_addr) { 8990 return 0; 8991 } 8992 8993 | // Check Stack Overflow 8994 | MEM_LOAD_ZTS r1, aword, executor_globals, vm_stack_end, r0 8995 | MEM_LOAD_OP_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 8996 | cmp r1, used_stack 8997 | jb &exit_addr 8998 8999 return 1; 9000} 9001 9002static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 9003{ 9004 uint32_t used_stack; 9005 bool stack_check = 1; 9006 9007 if (func) { 9008 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9009 if ((int)used_stack <= checked_stack) { 9010 stack_check = 0; 9011 } 9012 } else { 9013 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 9014 9015 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9016 if (!is_closure) { 9017 | test byte [r0 + offsetof(zend_function, type)], 1 9018 | mov FCARG1a, used_stack 9019 | jnz >1 9020 } else { 9021 | mov FCARG1a, used_stack 9022 } 9023 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9024 | mov edx, opline->extended_value 9025 if (!is_closure) { 9026 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9027 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9028 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9029 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9030 } else { 9031 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9032 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9033 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9034 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9035 } 9036 | shl edx, 4 9037 |.if X64 9038 | movsxd r2, edx 9039 |.endif 9040 | sub FCARG1a, r2 9041 |1: 9042 } 9043 9044 zend_jit_start_reuse_ip(); 9045 9046 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9047 | MEM_LOAD_ZTS RX, aword, executor_globals, vm_stack_top, RX 9048 9049 if (stack_check) { 9050 | // Check Stack Overflow 9051 | MEM_LOAD_ZTS r2, aword, executor_globals, vm_stack_end, r2 9052 | sub r2, RX 9053 if (func) { 9054 | cmp r2, used_stack 9055 } else { 9056 | cmp r2, FCARG1a 9057 } 9058 9059 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9060 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9061 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9062 9063 if (!exit_addr) { 9064 return 0; 9065 } 9066 9067 | jb &exit_addr 9068 } else { 9069 | jb >1 9070 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9071 |.cold_code 9072 |1: 9073 if (func) { 9074 | mov FCARG1d, used_stack 9075 } 9076#ifdef _WIN32 9077 if (0) { 9078#else 9079 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9080#endif 9081 | SET_EX_OPLINE opline, r0 9082 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9083 } else { 9084 if (!is_closure) { 9085 if (func 9086 && op_array == &func->op_array 9087 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9088 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9089 | LOAD_ADDR FCARG2a, func 9090 } else { 9091 | mov FCARG2a, r0 9092 } 9093 } else { 9094 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9095 } 9096 | SET_EX_OPLINE opline, r0 9097 | EXT_CALL zend_jit_extend_stack_helper, r0 9098 } 9099 | mov RX, r0 9100 | jmp >1 9101 |.code 9102 } 9103 } 9104 9105 if (func) { 9106 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9107 } else { 9108 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9109 } 9110 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9111 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9112 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9113 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9114 } 9115#ifdef _WIN32 9116 if (0) { 9117#else 9118 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9119#endif 9120 | // call->func = func; 9121 |1: 9122 | ADDR_STORE aword EX:RX->func, func, r1 9123 } else { 9124 if (!is_closure) { 9125 | // call->func = func; 9126 if (func 9127 && op_array == &func->op_array 9128 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9129 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9130 | ADDR_STORE aword EX:RX->func, func, r1 9131 } else { 9132 | mov aword EX:RX->func, r0 9133 } 9134 } else { 9135 | // call->func = &closure->func; 9136 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9137 | mov aword EX:RX->func, r1 9138 } 9139 |1: 9140 } 9141 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9142 | // Z_PTR(call->This) = obj; 9143 | mov r1, aword T1 9144 | mov aword EX:RX->This.value.ptr, r1 9145 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 9146 | // call->call_info |= ZEND_CALL_HAS_THIS; 9147 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9148 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9149 } else { 9150 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9151 } 9152 } else { 9153 if (opline->op1_type == IS_CV) { 9154 | // GC_ADDREF(obj); 9155 | add dword [r1], 1 9156 } 9157 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9158 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9159 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9160 } else { 9161 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9162 } 9163 } 9164 } else if (!is_closure) { 9165 | // Z_CE(call->This) = called_scope; 9166 | mov aword EX:RX->This.value.ptr, 0 9167 } else { 9168 if (opline->op2_type == IS_CV) { 9169 | // GC_ADDREF(closure); 9170 | add dword [r0], 1 9171 } 9172 | // object_or_called_scope = closure->called_scope; 9173 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9174 | mov aword EX:RX->This.value.ptr, r1 9175 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9176 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9177 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9178 | and edx, ZEND_ACC_FAKE_CLOSURE 9179 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9180 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9181 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9182 | jz >1 9183 | // call_info |= ZEND_CALL_HAS_THIS; 9184 | or edx, ZEND_CALL_HAS_THIS 9185 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9186 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9187 |1: 9188 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9189 | or dword EX:RX->This.u1.type_info, edx 9190 | // Z_PTR(call->This) = object_or_called_scope; 9191 | mov aword EX:RX->This.value.ptr, r1 9192 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9193 | jnz >1 9194 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9195 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9196 |1: 9197 } 9198 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9199 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9200 return 1; 9201} 9202 9203static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9204{ 9205 int32_t exit_point; 9206 const void *exit_addr; 9207 9208 if (func->type == ZEND_INTERNAL_FUNCTION) { 9209#ifdef ZEND_WIN32 9210 // TODO: ASLR may cause different addresses in different workers ??? 9211 return 0; 9212#endif 9213 } else if (func->type == ZEND_USER_FUNCTION) { 9214 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9215 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9216 return 0; 9217 } 9218 } else { 9219 ZEND_UNREACHABLE(); 9220 return 0; 9221 } 9222 9223 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9224 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9225 if (!exit_addr) { 9226 return 0; 9227 } 9228 9229 | // call = EX(call); 9230 | mov r1, EX->call 9231 while (level > 0) { 9232 | mov r1, EX:r1->prev_execute_data 9233 level--; 9234 } 9235 9236 if (func->type == ZEND_USER_FUNCTION && 9237 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9238 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9239 !func->common.function_name)) { 9240 const zend_op *opcodes = func->op_array.opcodes; 9241 9242 | mov r1, aword EX:r1->func 9243 | .if X64 9244 || if (!IS_SIGNED_32BIT(opcodes)) { 9245 | mov64 r2, ((ptrdiff_t)opcodes) 9246 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9247 || } else { 9248 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9249 || } 9250 | .else 9251 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9252 | .endif 9253 | jne &exit_addr 9254 } else { 9255 | .if X64 9256 || if (!IS_SIGNED_32BIT(func)) { 9257 | mov64 r2, ((ptrdiff_t)func) 9258 | cmp aword EX:r1->func, r2 9259 || } else { 9260 | cmp aword EX:r1->func, func 9261 || } 9262 | .else 9263 | cmp aword EX:r1->func, func 9264 | .endif 9265 | jne &exit_addr 9266 } 9267 9268 return 1; 9269} 9270 9271static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 9272{ 9273 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9274 zend_call_info *call_info = NULL; 9275 zend_function *func = NULL; 9276 9277 if (delayed_call_chain) { 9278 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9279 return 0; 9280 } 9281 } 9282 9283 if (info) { 9284 call_info = info->callee_info; 9285 while (call_info && call_info->caller_init_opline != opline) { 9286 call_info = call_info->next_callee; 9287 } 9288 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9289 func = call_info->callee_func; 9290 } 9291 } 9292 9293 if (!func 9294 && trace 9295 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9296#ifdef _WIN32 9297 /* ASLR */ 9298 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9299 func = (zend_function*)trace->func; 9300 } 9301#else 9302 func = (zend_function*)trace->func; 9303#endif 9304 } 9305 9306#ifdef _WIN32 9307 if (0) { 9308#else 9309 if (opline->opcode == ZEND_INIT_FCALL 9310 && func 9311 && func->type == ZEND_INTERNAL_FUNCTION) { 9312#endif 9313 /* load constant address later */ 9314 } else if (func && op_array == &func->op_array) { 9315 /* recursive call */ 9316 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9317 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9318 | mov r0, EX->func 9319 } 9320 } else { 9321 | // if (CACHED_PTR(opline->result.num)) 9322 | mov r2, EX->run_time_cache 9323 | mov r0, aword [r2 + opline->result.num] 9324 | test r0, r0 9325 | jz >1 9326 |.cold_code 9327 |1: 9328 if (opline->opcode == ZEND_INIT_FCALL 9329 && func 9330 && func->type == ZEND_USER_FUNCTION 9331 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9332 | LOAD_ADDR FCARG1a, func 9333 | mov aword [r2 + opline->result.num], FCARG1a 9334 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9335 | jmp >3 9336 } else { 9337 zval *zv = RT_CONSTANT(opline, opline->op2); 9338 9339 if (opline->opcode == ZEND_INIT_FCALL) { 9340 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9341 | lea FCARG2a, aword [r2 + opline->result.num] 9342 | EXT_CALL zend_jit_find_func_helper, r0 9343 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9344 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9345 | lea FCARG2a, aword [r2 + opline->result.num] 9346 | EXT_CALL zend_jit_find_func_helper, r0 9347 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9348 | LOAD_ADDR FCARG1a, zv; 9349 | lea FCARG2a, aword [r2 + opline->result.num] 9350 | EXT_CALL zend_jit_find_ns_func_helper, r0 9351 } else { 9352 ZEND_UNREACHABLE(); 9353 } 9354 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9355 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 9356 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9357 9358 if (!exit_addr) { 9359 return 0; 9360 } 9361 9362 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9363 | test r0, r0 9364 | jnz >3 9365 } else if (func->type == ZEND_USER_FUNCTION 9366 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9367 const zend_op *opcodes = func->op_array.opcodes; 9368 9369 | .if X64 9370 || if (!IS_SIGNED_32BIT(opcodes)) { 9371 | mov64 r1, ((ptrdiff_t)opcodes) 9372 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9373 || } else { 9374 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9375 || } 9376 | .else 9377 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9378 | .endif 9379 | jz >3 9380 } else { 9381 | .if X64 9382 || if (!IS_SIGNED_32BIT(func)) { 9383 | mov64 r1, ((ptrdiff_t)func) 9384 | cmp r0, r1 9385 || } else { 9386 | cmp r0, func 9387 || } 9388 | .else 9389 | cmp r0, func 9390 | .endif 9391 | jz >3 9392 } 9393 | jmp &exit_addr 9394 } else { 9395 | test r0, r0 9396 | jnz >3 9397 | // SAVE_OPLINE(); 9398 | SET_EX_OPLINE opline, r0 9399 | jmp ->undefined_function 9400 } 9401 } 9402 |.code 9403 |3: 9404 } 9405 9406 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 9407 return 0; 9408 } 9409 9410 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9411 if (!zend_jit_save_call_chain(Dst, call_level)) { 9412 return 0; 9413 } 9414 } else { 9415 delayed_call_chain = 1; 9416 delayed_call_level = call_level; 9417 } 9418 9419 return 1; 9420} 9421 9422static int zend_jit_init_method_call(dasm_State **Dst, 9423 const zend_op *opline, 9424 uint32_t b, 9425 const zend_op_array *op_array, 9426 zend_ssa *ssa, 9427 const zend_ssa_op *ssa_op, 9428 int call_level, 9429 uint32_t op1_info, 9430 zend_jit_addr op1_addr, 9431 zend_class_entry *ce, 9432 bool ce_is_instanceof, 9433 bool on_this, 9434 bool delayed_fetch_this, 9435 zend_class_entry *trace_ce, 9436 zend_jit_trace_rec *trace, 9437 int checked_stack, 9438 bool polymorphic_side_trace) 9439{ 9440 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9441 zend_call_info *call_info = NULL; 9442 zend_function *func = NULL; 9443 zval *function_name; 9444 9445 ZEND_ASSERT(opline->op2_type == IS_CONST); 9446 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9447 9448 function_name = RT_CONSTANT(opline, opline->op2); 9449 9450 if (info) { 9451 call_info = info->callee_info; 9452 while (call_info && call_info->caller_init_opline != opline) { 9453 call_info = call_info->next_callee; 9454 } 9455 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9456 func = call_info->callee_func; 9457 } 9458 } 9459 9460 if (polymorphic_side_trace) { 9461 /* function is passed in r0 from parent_trace */ 9462 } else { 9463 if (on_this) { 9464 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9465 9466 | GET_ZVAL_PTR FCARG1a, this_addr 9467 } else { 9468 if (op1_info & MAY_BE_REF) { 9469 if (opline->op1_type == IS_CV) { 9470 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9471 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9472 } 9473 | ZVAL_DEREF FCARG1a, op1_info 9474 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 9475 } else { 9476 /* Hack: Convert reference to regular value to simplify JIT code */ 9477 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9478 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9479 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9480 | EXT_CALL zend_jit_unref_helper, r0 9481 |1: 9482 } 9483 } 9484 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9485 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9486 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9487 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9488 9489 if (!exit_addr) { 9490 return 0; 9491 } 9492 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9493 } else { 9494 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9495 |.cold_code 9496 |1: 9497 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9498 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9499 } 9500 | SET_EX_OPLINE opline, r0 9501 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9502 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9503 } else { 9504 | EXT_CALL zend_jit_invalid_method_call, r0 9505 } 9506 | jmp ->exception_handler 9507 |.code 9508 } 9509 } 9510 | GET_ZVAL_PTR FCARG1a, op1_addr 9511 } 9512 9513 if (delayed_call_chain) { 9514 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9515 return 0; 9516 } 9517 } 9518 9519 | mov aword T1, FCARG1a // save 9520 9521 if (func) { 9522 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9523 | mov r0, EX->run_time_cache 9524 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9525 | test r0, r0 9526 | jz >1 9527 } else { 9528 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9529 | mov r0, EX->run_time_cache 9530 | mov r2, aword [r0 + opline->result.num] 9531 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9532 | jnz >1 9533 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9534 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9535 } 9536 9537 |.cold_code 9538 |1: 9539 | LOAD_ADDR FCARG2a, function_name 9540 |.if X64 9541 | lea CARG3, aword T1 9542 |.else 9543 | lea r0, aword T1 9544 | sub r4, 12 9545 | push r0 9546 |.endif 9547 | SET_EX_OPLINE opline, r0 9548 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9549 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9550 } else { 9551 | EXT_CALL zend_jit_find_method_helper, r0 9552 } 9553 |.if not(X64) 9554 | add r4, 12 9555 |.endif 9556 | test r0, r0 9557 | jnz >2 9558 | jmp ->exception_handler 9559 |.code 9560 |2: 9561 } 9562 9563 if (!func 9564 && trace 9565 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9566 && trace->func 9567#ifdef _WIN32 9568 && trace->func->type != ZEND_INTERNAL_FUNCTION 9569#endif 9570 ) { 9571 int32_t exit_point; 9572 const void *exit_addr; 9573 9574 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); 9575 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9576 if (!exit_addr) { 9577 return 0; 9578 } 9579 9580 func = (zend_function*)trace->func; 9581 9582 if (func->type == ZEND_USER_FUNCTION && 9583 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9584 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9585 !func->common.function_name)) { 9586 const zend_op *opcodes = func->op_array.opcodes; 9587 9588 | .if X64 9589 || if (!IS_SIGNED_32BIT(opcodes)) { 9590 | mov64 r1, ((ptrdiff_t)opcodes) 9591 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9592 || } else { 9593 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9594 || } 9595 | .else 9596 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9597 | .endif 9598 | jne &exit_addr 9599 } else { 9600 | .if X64 9601 || if (!IS_SIGNED_32BIT(func)) { 9602 | mov64 r1, ((ptrdiff_t)func) 9603 | cmp r0, r1 9604 || } else { 9605 | cmp r0, func 9606 || } 9607 | .else 9608 | cmp r0, func 9609 | .endif 9610 | jne &exit_addr 9611 } 9612 } 9613 9614 if (!func) { 9615 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9616 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9617 | jnz >1 9618 |.cold_code 9619 |1: 9620 } 9621 9622 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9623 | mov FCARG1a, aword T1 // restore 9624 | mov FCARG2a, r0 9625 |.if X64 9626 | mov CARG3d, opline->extended_value 9627 |.else 9628 | sub r4, 12 9629 | push opline->extended_value 9630 |.endif 9631 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9632 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9633 } else { 9634 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9635 } 9636 |.if not(X64) 9637 | add r4, 12 9638 |.endif 9639 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9640 | test r0, r0 9641 | jz ->exception_handler 9642 } 9643 | mov RX, r0 9644 } 9645 9646 if (!func) { 9647 | jmp >9 9648 |.code 9649 } 9650 9651 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9652 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9653 return 0; 9654 } 9655 } 9656 9657 if (!func) { 9658 |9: 9659 } 9660 zend_jit_start_reuse_ip(); 9661 9662 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9663 if (!zend_jit_save_call_chain(Dst, call_level)) { 9664 return 0; 9665 } 9666 } else { 9667 delayed_call_chain = 1; 9668 delayed_call_level = call_level; 9669 } 9670 9671 return 1; 9672} 9673 9674static int zend_jit_init_closure_call(dasm_State **Dst, 9675 const zend_op *opline, 9676 uint32_t b, 9677 const zend_op_array *op_array, 9678 zend_ssa *ssa, 9679 const zend_ssa_op *ssa_op, 9680 int call_level, 9681 zend_jit_trace_rec *trace, 9682 int checked_stack) 9683{ 9684 zend_function *func = NULL; 9685 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9686 9687 | GET_ZVAL_PTR r0, op2_addr 9688 9689 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9690 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9691 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9692 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9693 9694 if (!exit_addr) { 9695 return 0; 9696 } 9697 9698 |.if X64 9699 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9700 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9701 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9702 || } else { 9703 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9704 || } 9705 |.else 9706 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9707 |.endif 9708 | jne &exit_addr 9709 if (ssa->var_info && ssa_op->op2_use >= 0) { 9710 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9711 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9712 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9713 } 9714 } 9715 9716 if (trace 9717 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9718 && trace->func 9719 && trace->func->type == ZEND_USER_FUNCTION) { 9720 const zend_op *opcodes; 9721 int32_t exit_point; 9722 const void *exit_addr; 9723 9724 func = (zend_function*)trace->func; 9725 opcodes = func->op_array.opcodes; 9726 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9727 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9728 if (!exit_addr) { 9729 return 0; 9730 } 9731 9732 | .if X64 9733 || if (!IS_SIGNED_32BIT(opcodes)) { 9734 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9735 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9736 || } else { 9737 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9738 || } 9739 | .else 9740 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9741 | .endif 9742 | jne &exit_addr 9743 } 9744 9745 if (delayed_call_chain) { 9746 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9747 return 0; 9748 } 9749 } 9750 9751 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9752 return 0; 9753 } 9754 9755 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9756 if (!zend_jit_save_call_chain(Dst, call_level)) { 9757 return 0; 9758 } 9759 } else { 9760 delayed_call_chain = 1; 9761 delayed_call_level = call_level; 9762 } 9763 9764 if (trace 9765 && trace->op == ZEND_JIT_TRACE_END 9766 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9767 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9768 return 0; 9769 } 9770 } 9771 9772 return 1; 9773} 9774 9775static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9776{ 9777 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9778 zend_call_info *call_info = NULL; 9779 const zend_function *func = NULL; 9780 uint32_t i; 9781 zend_jit_addr res_addr; 9782 uint32_t call_num_args = 0; 9783 bool unknown_num_args = 0; 9784 const void *exit_addr = NULL; 9785 const zend_op *prev_opline; 9786 9787 if (RETURN_VALUE_USED(opline)) { 9788 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9789 } else { 9790 /* CPU stack allocated temporary zval */ 9791 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 9792 } 9793 9794 prev_opline = opline - 1; 9795 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9796 prev_opline--; 9797 } 9798 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9799 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9800 unknown_num_args = 1; 9801 } 9802 9803 if (info) { 9804 call_info = info->callee_info; 9805 while (call_info && call_info->caller_call_opline != opline) { 9806 call_info = call_info->next_callee; 9807 } 9808 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9809 func = call_info->callee_func; 9810 } 9811 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9812 && JIT_G(current_frame) 9813 && JIT_G(current_frame)->call 9814 && !JIT_G(current_frame)->call->func) { 9815 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9816 } 9817 } 9818 if (!func) { 9819 /* resolve function at run time */ 9820 } else if (func->type == ZEND_USER_FUNCTION) { 9821 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9822 call_num_args = call_info->num_args; 9823 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9824 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9825 call_num_args = call_info->num_args; 9826 } else { 9827 ZEND_UNREACHABLE(); 9828 } 9829 9830 if (trace && !func) { 9831 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9832 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9833#ifndef ZEND_WIN32 9834 // TODO: ASLR may cause different addresses in different workers ??? 9835 func = trace->func; 9836 if (JIT_G(current_frame) && 9837 JIT_G(current_frame)->call && 9838 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9839 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9840 } else { 9841 unknown_num_args = 1; 9842 } 9843#endif 9844 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9845 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9846 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9847 func = trace->func; 9848 if (JIT_G(current_frame) && 9849 JIT_G(current_frame)->call && 9850 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9851 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9852 } else { 9853 unknown_num_args = 1; 9854 } 9855 } 9856 } 9857 } 9858 9859 bool may_have_extra_named_params = 9860 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9861 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9862 9863 if (!reuse_ip) { 9864 zend_jit_start_reuse_ip(); 9865 | // call = EX(call); 9866 | mov RX, EX->call 9867 } 9868 zend_jit_stop_reuse_ip(); 9869 9870 | // fbc = call->func; 9871 | // mov r2, EX:RX->func ??? 9872 | // SAVE_OPLINE(); 9873 | SET_EX_OPLINE opline, r0 9874 9875 if (opline->opcode == ZEND_DO_FCALL) { 9876 if (!func) { 9877 if (trace) { 9878 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9879 9880 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9881 if (!exit_addr) { 9882 return 0; 9883 } 9884 | mov r0, EX:RX->func 9885 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9886 | jnz &exit_addr 9887 } 9888 } 9889 } 9890 9891 if (!delayed_call_chain) { 9892 if (call_level == 1) { 9893 | mov aword EX->call, 0 9894 } else { 9895 | //EX(call) = call->prev_execute_data; 9896 | mov r0, EX:RX->prev_execute_data 9897 | mov EX->call, r0 9898 } 9899 } 9900 delayed_call_chain = 0; 9901 9902 | //call->prev_execute_data = execute_data; 9903 | mov EX:RX->prev_execute_data, EX 9904 9905 if (!func) { 9906 | mov r0, EX:RX->func 9907 } 9908 9909 if (opline->opcode == ZEND_DO_FCALL) { 9910 if (!func) { 9911 if (!trace) { 9912 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9913 | jnz >1 9914 |.cold_code 9915 |1: 9916 if (!GCC_GLOBAL_REGS) { 9917 | mov FCARG1a, RX 9918 } 9919 | EXT_CALL zend_jit_deprecated_helper, r0 9920 | test al, al 9921 | mov r0, EX:RX->func // reload 9922 | jne >1 9923 | jmp ->exception_handler 9924 |.code 9925 |1: 9926 } 9927 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9928 if (!GCC_GLOBAL_REGS) { 9929 | mov FCARG1a, RX 9930 } 9931 | EXT_CALL zend_jit_deprecated_helper, r0 9932 | test al, al 9933 | je ->exception_handler 9934 } 9935 } 9936 9937 if (!func 9938 && opline->opcode != ZEND_DO_UCALL 9939 && opline->opcode != ZEND_DO_ICALL) { 9940 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 9941 | jne >8 9942 } 9943 9944 if ((!func || func->type == ZEND_USER_FUNCTION) 9945 && opline->opcode != ZEND_DO_ICALL) { 9946 | // EX(call) = NULL; 9947 | mov aword EX:RX->call, 0 9948 9949 if (RETURN_VALUE_USED(opline)) { 9950 | // EX(return_value) = EX_VAR(opline->result.var); 9951 | LOAD_ZVAL_ADDR r2, res_addr 9952 | mov aword EX:RX->return_value, r2 9953 } else { 9954 | // EX(return_value) = 0; 9955 | mov aword EX:RX->return_value, 0 9956 } 9957 9958 //EX_LOAD_RUN_TIME_CACHE(op_array); 9959 if (!func || func->op_array.cache_size) { 9960 if (func && op_array == &func->op_array) { 9961 /* recursive call */ 9962 if (trace || func->op_array.cache_size > sizeof(void*)) { 9963 | mov r2, EX->run_time_cache 9964 | mov EX:RX->run_time_cache, r2 9965 } 9966 } else { 9967#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 9968 if (func) { 9969 | mov r0, EX:RX->func 9970 } 9971 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 9972 | mov r2, aword [r2] 9973#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 9974 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 9975 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 9976 | MEM_LOAD_ZTS r2, aword, compiler_globals, map_ptr_base, r1 9977 | mov r2, aword [r2 + (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)] 9978 } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9979 && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { 9980 if (func) { 9981 | mov r0, EX:RX->func 9982 } 9983 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 9984 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 9985 | mov r2, aword [r2] 9986 } else { 9987 /* the called op_array may be not persisted yet */ 9988 if (func) { 9989 | mov r0, EX:RX->func 9990 } 9991 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 9992 | test r2, 1 9993 | jz >1 9994 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 9995 |1: 9996 | mov r2, aword [r2] 9997 } 9998 } else { 9999 if (func) { 10000 | mov r0, EX:RX->func 10001 } 10002 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10003 | test r2, 1 10004 | jz >1 10005 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10006 |1: 10007 | mov r2, aword [r2] 10008 } 10009#else 10010# error "Unknown ZEND_MAP_PTR_KIND" 10011#endif 10012 | mov EX:RX->run_time_cache, r2 10013 } 10014 } 10015 10016 | // EG(current_execute_data) = execute_data; 10017 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10018 | mov FP, RX 10019 10020 | // opline = op_array->opcodes; 10021 if (func && !unknown_num_args) { 10022 10023 for (i = call_num_args; i < func->op_array.last_var; i++) { 10024 uint32_t n = EX_NUM_TO_VAR(i); 10025 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10026 } 10027 10028 if (call_num_args <= func->op_array.num_args) { 10029 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10030 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10031 uint32_t num_args; 10032 10033 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10034 if (trace) { 10035 num_args = 0; 10036 } else if (call_info) { 10037 num_args = skip_valid_arguments(op_array, ssa, call_info); 10038 } else { 10039 num_args = call_num_args; 10040 } 10041 } else { 10042 num_args = call_num_args; 10043 } 10044 if (zend_accel_in_shm(func->op_array.opcodes)) { 10045 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10046 } else { 10047 | mov r0, EX->func 10048 if (GCC_GLOBAL_REGS) { 10049 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10050 if (num_args) { 10051 | add IP, (num_args * sizeof(zend_op)) 10052 } 10053 } else { 10054 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10055 if (num_args) { 10056 | add FCARG1a, (num_args * sizeof(zend_op)) 10057 } 10058 | mov aword EX->opline, FCARG1a 10059 } 10060 } 10061 10062 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10063 && num_args >= op_array->required_num_args) { 10064 /* recursive call */ 10065 if (ZEND_OBSERVER_ENABLED) { 10066 | SAVE_IP 10067 | mov FCARG1a, FP 10068 | EXT_CALL zend_observer_fcall_begin, r0 10069 } 10070#ifdef CONTEXT_THREADED_JIT 10071 | call >1 10072 |.cold_code 10073 |1: 10074 | pop r0 10075 | jmp =>num_args 10076 |.code 10077#else 10078 | jmp =>num_args 10079#endif 10080 return 1; 10081 } 10082 } 10083 } else { 10084 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10085 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10086 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10087 | LOAD_IP_ADDR (func->op_array.opcodes) 10088 } else if (GCC_GLOBAL_REGS) { 10089 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10090 } else { 10091 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10092 | mov aword EX->opline, FCARG1a 10093 } 10094 } 10095 if (!GCC_GLOBAL_REGS) { 10096 | mov FCARG1a, FP 10097 } 10098 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10099 } 10100 } else { 10101 | // opline = op_array->opcodes 10102 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10103 | LOAD_IP_ADDR (func->op_array.opcodes) 10104 } else if (GCC_GLOBAL_REGS) { 10105 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10106 } else { 10107 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10108 | mov aword EX->opline, FCARG1a 10109 } 10110 if (func) { 10111 | // num_args = EX_NUM_ARGS(); 10112 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10113 | // if (UNEXPECTED(num_args > first_extra_arg)) 10114 | cmp ecx, (func->op_array.num_args) 10115 } else { 10116 | // first_extra_arg = op_array->num_args; 10117 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10118 | // num_args = EX_NUM_ARGS(); 10119 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10120 | // if (UNEXPECTED(num_args > first_extra_arg)) 10121 | cmp ecx, edx 10122 } 10123 | jg >1 10124 |.cold_code 10125 |1: 10126 if (!GCC_GLOBAL_REGS) { 10127 | mov FCARG1a, FP 10128 } 10129 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10130 if (!func) { 10131 | mov r0, EX->func // reload 10132 } 10133 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10134 | jmp >1 10135 |.code 10136 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10137 if (!func) { 10138 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10139 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10140 | jnz >1 10141 } 10142 | // opline += num_args; 10143 |.if X64 10144 || ZEND_ASSERT(sizeof(zend_op) == 32); 10145 | mov edx, ecx 10146 | shl r2, 5 10147 |.else 10148 | imul r2, ecx, sizeof(zend_op) 10149 |.endif 10150 | ADD_IP r2 10151 } 10152 |1: 10153 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10154 if (func) { 10155 | mov edx, (func->op_array.last_var) 10156 } else { 10157 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10158 } 10159 | sub edx, ecx 10160 | jle >3 //??? 10161 | // zval *var = EX_VAR_NUM(num_args); 10162// |.if X64 10163// | movsxd r1, ecx 10164// |.endif 10165 | shl r1, 4 10166 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10167 |2: 10168 | SET_Z_TYPE_INFO r1, IS_UNDEF 10169 | sub edx, 1 10170 | lea r1, [r1 + 16] 10171 | jne <2 10172 |3: 10173 } 10174 10175 if (ZEND_OBSERVER_ENABLED) { 10176 | SAVE_IP 10177 | mov FCARG1a, FP 10178 | EXT_CALL zend_observer_fcall_begin, r0 10179 } 10180 10181 if (trace) { 10182 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10183 | jmp >9 10184 } 10185 } else { 10186#ifdef CONTEXT_THREADED_JIT 10187 | call ->context_threaded_call 10188 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10189 | jmp >9 10190 } 10191 | call ->context_threaded_call 10192 if (!func) { 10193 | jmp >9 10194 } 10195#else 10196 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10197 | ADD_HYBRID_SPAD 10198 | JMP_IP 10199 } else if (GCC_GLOBAL_REGS) { 10200 | add r4, SPAD // stack alignment 10201 | JMP_IP 10202 } else { 10203 | mov FP, aword T2 // restore FP 10204 | mov RX, aword T3 // restore IP 10205 | add r4, NR_SPAD // stack alignment 10206 | mov r0, 1 // ZEND_VM_ENTER 10207 | ret 10208 } 10209 } 10210#endif 10211 } 10212 10213 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10214 && (opline->opcode != ZEND_DO_UCALL)) { 10215 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10216 |8: 10217 } 10218 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10219 if (!func) { 10220 if (trace) { 10221 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10222 10223 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10224 if (!exit_addr) { 10225 return 0; 10226 } 10227 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10228 | jnz &exit_addr 10229 } else { 10230 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10231 | jnz >1 10232 |.cold_code 10233 |1: 10234 if (!GCC_GLOBAL_REGS) { 10235 | mov FCARG1a, RX 10236 } 10237 | EXT_CALL zend_jit_deprecated_helper, r0 10238 | test al, al 10239 | mov r0, EX:RX->func // reload 10240 | jne >1 10241 | jmp ->exception_handler 10242 |.code 10243 |1: 10244 } 10245 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10246 if (!GCC_GLOBAL_REGS) { 10247 | mov FCARG1a, RX 10248 } 10249 | EXT_CALL zend_jit_deprecated_helper, r0 10250 | test al, al 10251 | je ->exception_handler 10252 | mov r0, EX:RX->func // reload 10253 } 10254 } 10255 10256 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10257 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10258 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10259 10260 | // EG(current_execute_data) = execute_data; 10261 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10262 10263 zend_jit_reset_last_valid_opline(); 10264 10265 | // fbc->internal_function.handler(call, ret); 10266 | mov FCARG1a, RX 10267 if (func) { 10268 | EXT_CALL func->internal_function.handler, r0 10269 } else { 10270 | call aword [r0 + offsetof(zend_internal_function, handler)] 10271 } 10272 10273 | // EG(current_execute_data) = execute_data; 10274 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 10275 10276 | // zend_vm_stack_free_args(call); 10277 if (func && !unknown_num_args) { 10278 for (i = 0; i < call_num_args; i++ ) { 10279 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 10280 uint32_t offset = EX_NUM_TO_VAR(i); 10281 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10282 } 10283 } 10284 } else { 10285 | mov FCARG1a, RX 10286 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10287 } 10288 if (may_have_extra_named_params) { 10289 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10290 | jnz >1 10291 |.cold_code 10292 |1: 10293 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10294 | EXT_CALL zend_free_extra_named_params, r0 10295 | jmp >2 10296 |.code 10297 |2: 10298 } 10299 10300 |8: 10301 if (opline->opcode == ZEND_DO_FCALL) { 10302 // TODO: optimize ??? 10303 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10304 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10305 | jnz >1 10306 |.cold_code 10307 |1: 10308 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10309 | // OBJ_RELEASE(object); 10310 | OBJ_RELEASE ZREG_FCARG1, >2 10311 | jmp >2 10312 |.code 10313 |2: 10314 } 10315 10316 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10317 !JIT_G(current_frame) || 10318 !JIT_G(current_frame)->call || 10319 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10320 prev_opline->opcode == ZEND_SEND_UNPACK || 10321 prev_opline->opcode == ZEND_SEND_ARRAY || 10322 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10323 10324 | // zend_vm_stack_free_call_frame(call); 10325 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10326 | jnz >1 10327 |.cold_code 10328 |1: 10329 | mov FCARG1a, RX 10330 | EXT_CALL zend_jit_free_call_frame, r0 10331 | jmp >1 10332 |.code 10333 } 10334 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, RX, r0 10335 |1: 10336 10337 if (!RETURN_VALUE_USED(opline)) { 10338 zend_class_entry *ce; 10339 bool ce_is_instanceof; 10340 uint32_t func_info = call_info ? 10341 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10342 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10343 10344 /* If an exception is thrown, the return_value may stay at the 10345 * original value of null. */ 10346 func_info |= MAY_BE_NULL; 10347 10348 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10349 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10350 } 10351 } 10352 10353 | // if (UNEXPECTED(EG(exception) != NULL)) { 10354 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 10355 | jne ->icall_throw_handler 10356 10357 // TODO: Can we avoid checking for interrupts after each call ??? 10358 if (trace && last_valid_opline != opline) { 10359 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10360 10361 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10362 if (!exit_addr) { 10363 return 0; 10364 } 10365 } else { 10366 exit_addr = NULL; 10367 } 10368 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10369 return 0; 10370 } 10371 10372 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10373 | LOAD_IP_ADDR (opline + 1) 10374 } else if (trace 10375 && trace->op == ZEND_JIT_TRACE_END 10376 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10377 | LOAD_IP_ADDR (opline + 1) 10378 } 10379 } 10380 10381 if (!func) { 10382 |9: 10383 } 10384 10385 return 1; 10386} 10387 10388static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10389{ 10390 uint32_t arg_num = opline->op2.num; 10391 zend_jit_addr arg_addr; 10392 10393 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10394 10395 if (!zend_jit_reuse_ip(Dst)) { 10396 return 0; 10397 } 10398 10399 if (opline->opcode == ZEND_SEND_VAL_EX) { 10400 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10401 10402 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10403 10404 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10405 && JIT_G(current_frame) 10406 && JIT_G(current_frame)->call 10407 && JIT_G(current_frame)->call->func) { 10408 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10409 /* Don't generate code that always throws exception */ 10410 return 0; 10411 } 10412 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10413 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10414 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10415 if (!exit_addr) { 10416 return 0; 10417 } 10418 | mov r0, EX:RX->func 10419 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10420 | jnz &exit_addr 10421 } else { 10422 | mov r0, EX:RX->func 10423 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10424 | jnz >1 10425 |.cold_code 10426 |1: 10427 if (Z_MODE(op1_addr) == IS_REG) { 10428 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 10429 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10430 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF 10431 } 10432 | SET_EX_OPLINE opline, r0 10433 | jmp ->throw_cannot_pass_by_ref 10434 |.code 10435 10436 } 10437 } 10438 10439 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10440 10441 if (opline->op1_type == IS_CONST) { 10442 zval *zv = RT_CONSTANT(opline, opline->op1); 10443 10444 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10445 if (Z_REFCOUNTED_P(zv)) { 10446 | ADDREF_CONST zv, r0 10447 } 10448 } else { 10449 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10450 } 10451 10452 return 1; 10453} 10454 10455static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10456{ 10457 | mov FCARG1a, EX->call 10458 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10459 | jnz >1 10460 |.cold_code 10461 |1: 10462 | SET_EX_OPLINE opline, r0 10463 | EXT_CALL zend_handle_undef_args, r0 10464 | test r0, r0 10465 | jnz ->exception_handler 10466 | jmp >2 10467 |.code 10468 |2: 10469 10470 return 1; 10471} 10472 10473static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10474{ 10475 zend_jit_addr op1_addr, arg_addr, ref_addr; 10476 10477 op1_addr = OP1_ADDR(); 10478 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10479 10480 if (!zend_jit_reuse_ip(Dst)) { 10481 return 0; 10482 } 10483 10484 if (opline->op1_type == IS_VAR) { 10485 if (op1_info & MAY_BE_INDIRECT) { 10486 | LOAD_ZVAL_ADDR r0, op1_addr 10487 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10488 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10489 | // ret = Z_INDIRECT_P(ret); 10490 | GET_Z_PTR r0, r0 10491 |1: 10492 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10493 } 10494 } else if (opline->op1_type == IS_CV) { 10495 if (op1_info & MAY_BE_UNDEF) { 10496 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10497 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10498 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10499 | jmp >2 10500 |1: 10501 } 10502 op1_info &= ~MAY_BE_UNDEF; 10503 op1_info |= MAY_BE_NULL; 10504 } 10505 } else { 10506 ZEND_UNREACHABLE(); 10507 } 10508 10509 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10510 if (op1_info & MAY_BE_REF) { 10511 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10512 | GET_ZVAL_PTR r1, op1_addr 10513 | GC_ADDREF r1 10514 | SET_ZVAL_PTR arg_addr, r1 10515 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10516 | jmp >6 10517 } 10518 |2: 10519 | // ZVAL_NEW_REF(arg, varptr); 10520 if (opline->op1_type == IS_VAR) { 10521 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10522 | LOAD_ZVAL_ADDR r0, op1_addr 10523 } 10524 | mov aword T1, r0 // save 10525 } 10526 | EMALLOC sizeof(zend_reference), op_array, opline 10527 | mov dword [r0], 2 10528 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10529 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10530 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10531 if (opline->op1_type == IS_VAR) { 10532 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10533 10534 | mov r1, aword T1 // restore 10535 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10536 | SET_ZVAL_PTR val_addr, r0 10537 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10538 } else { 10539 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10540 | SET_ZVAL_PTR op1_addr, r0 10541 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10542 } 10543 | SET_ZVAL_PTR arg_addr, r0 10544 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10545 } 10546 10547 |6: 10548 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10549 |7: 10550 10551 return 1; 10552} 10553 10554static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10555{ 10556 uint32_t arg_num = opline->op2.num; 10557 zend_jit_addr arg_addr; 10558 10559 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10560 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10561 arg_num <= MAX_ARG_FLAG_NUM); 10562 10563 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10564 10565 if (!zend_jit_reuse_ip(Dst)) { 10566 return 0; 10567 } 10568 10569 if (opline->opcode == ZEND_SEND_VAR_EX) { 10570 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10571 && JIT_G(current_frame) 10572 && JIT_G(current_frame)->call 10573 && JIT_G(current_frame)->call->func) { 10574 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10575 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10576 return 0; 10577 } 10578 return 1; 10579 } 10580 } else { 10581 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10582 10583 | mov r0, EX:RX->func 10584 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10585 | jnz >1 10586 |.cold_code 10587 |1: 10588 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10589 return 0; 10590 } 10591 | jmp >7 10592 |.code 10593 } 10594 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10595 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10596 && JIT_G(current_frame) 10597 && JIT_G(current_frame)->call 10598 && JIT_G(current_frame)->call->func) { 10599 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10600 10601 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10602 10603 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10604 if (!(op1_info & MAY_BE_REF)) { 10605 /* Don't generate code that always throws exception */ 10606 return 0; 10607 } else { 10608 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10609 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10610 if (!exit_addr) { 10611 return 0; 10612 } 10613 | cmp cl, IS_REFERENCE 10614 | jne &exit_addr 10615 } 10616 } 10617 return 1; 10618 } 10619 } else { 10620 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10621 10622 | mov r0, EX:RX->func 10623 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10624 | jnz >1 10625 |.cold_code 10626 |1: 10627 10628 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10629 10630 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10631 if (op1_info & MAY_BE_REF) { 10632 | cmp cl, IS_REFERENCE 10633 | je >7 10634 } 10635 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10636 | jnz >7 10637 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10638 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10639 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10640 if (!exit_addr) { 10641 return 0; 10642 } 10643 | jmp &exit_addr 10644 } else { 10645 | SET_EX_OPLINE opline, r0 10646 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10647 | EXT_CALL zend_jit_only_vars_by_reference, r0 10648 if (!zend_jit_check_exception(Dst)) { 10649 return 0; 10650 } 10651 | jmp >7 10652 } 10653 10654 |.code 10655 } 10656 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10657 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10658 && JIT_G(current_frame) 10659 && JIT_G(current_frame)->call 10660 && JIT_G(current_frame)->call->func) { 10661 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10662 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10663 return 0; 10664 } 10665 return 1; 10666 } 10667 } else { 10668 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10669 | jnz >1 10670 |.cold_code 10671 |1: 10672 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10673 return 0; 10674 } 10675 | jmp >7 10676 |.code 10677 } 10678 } 10679 10680 if (op1_info & MAY_BE_UNDEF) { 10681 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10682 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10683 |.cold_code 10684 |1: 10685 } 10686 10687 | SET_EX_OPLINE opline, r0 10688 | mov FCARG1d, opline->op1.var 10689 | EXT_CALL zend_jit_undefined_op_helper, r0 10690 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10691 | test r0, r0 10692 | jz ->exception_handler 10693 10694 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10695 | jmp >7 10696 |.code 10697 } else { 10698 |7: 10699 return 1; 10700 } 10701 } 10702 10703 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10704 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10705 if (op1_info & MAY_BE_REF) { 10706 | cmp cl, IS_REFERENCE 10707 | je >7 10708 } 10709 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10710 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10711 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10712 if (!exit_addr) { 10713 return 0; 10714 } 10715 | jmp &exit_addr 10716 } else { 10717 | SET_EX_OPLINE opline, r0 10718 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10719 | EXT_CALL zend_jit_only_vars_by_reference, r0 10720 if (!zend_jit_check_exception(Dst)) { 10721 return 0; 10722 } 10723 } 10724 } else { 10725 if (op1_info & MAY_BE_REF) { 10726 if (opline->op1_type == IS_CV) { 10727 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10728 10729 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10730 | ZVAL_DEREF FCARG1a, op1_info 10731 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10732 | TRY_ADDREF op1_info, ah, r2 10733 } else { 10734 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10735 10736 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10737 |.cold_code 10738 |1: 10739 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10740 | GET_ZVAL_PTR FCARG1a, op1_addr 10741 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10742 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10743 | GC_DELREF FCARG1a 10744 | je >1 10745 | IF_NOT_REFCOUNTED ah, >2 10746 | GC_ADDREF r2 10747 | jmp >2 10748 |1: 10749 | EFREE_REG_REFERENCE 10750 | jmp >2 10751 |.code 10752 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10753 |2: 10754 } 10755 } else { 10756 if (op1_addr != op1_def_addr) { 10757 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10758 return 0; 10759 } 10760 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10761 op1_addr= op1_def_addr; 10762 } 10763 } 10764 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10765 if (opline->op1_type == IS_CV) { 10766 | TRY_ADDREF op1_info, ah, r2 10767 } 10768 } 10769 } 10770 |7: 10771 10772 return 1; 10773} 10774 10775static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10776{ 10777 uint32_t arg_num = opline->op2.num; 10778 10779 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10780 && JIT_G(current_frame) 10781 && JIT_G(current_frame)->call 10782 && JIT_G(current_frame)->call->func) { 10783 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10784 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10785 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10786 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10787 || if (reuse_ip) { 10788 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10789 || } else { 10790 | mov r0, EX->call 10791 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10792 || } 10793 } 10794 } else { 10795 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10796 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10797 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10798 || if (reuse_ip) { 10799 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10800 || } else { 10801 | mov r0, EX->call 10802 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10803 || } 10804 } 10805 } 10806 } else { 10807 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10808 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10809 10810 if (!zend_jit_reuse_ip(Dst)) { 10811 return 0; 10812 } 10813 10814 | mov r0, EX:RX->func 10815 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10816 | jnz >1 10817 |.cold_code 10818 |1: 10819 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10820 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10821 | jmp >1 10822 |.code 10823 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10824 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10825 |1: 10826 } 10827 10828 return 1; 10829} 10830 10831static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10832{ 10833 if (smart_branch_opcode) { 10834 if (smart_branch_opcode == ZEND_JMPZ) { 10835 if (jmp) { 10836 | jmp >7 10837 } 10838 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10839 | jmp =>target_label 10840 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10841 | jmp =>target_label2 10842 } else { 10843 ZEND_UNREACHABLE(); 10844 } 10845 } else { 10846 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10847 10848 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 10849 if (jmp) { 10850 | jmp >7 10851 } 10852 } 10853 10854 return 1; 10855} 10856 10857static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 10858{ 10859 if (smart_branch_opcode) { 10860 if (smart_branch_opcode == ZEND_JMPZ) { 10861 | jmp =>target_label 10862 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10863 if (jmp) { 10864 | jmp >7 10865 } 10866 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10867 | jmp =>target_label 10868 } else { 10869 ZEND_UNREACHABLE(); 10870 } 10871 } else { 10872 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10873 10874 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 10875 if (jmp) { 10876 | jmp >7 10877 } 10878 } 10879 10880 return 1; 10881} 10882 10883static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10884{ 10885 uint32_t defined_label = (uint32_t)-1; 10886 uint32_t undefined_label = (uint32_t)-1; 10887 zval *zv = RT_CONSTANT(opline, opline->op1); 10888 zend_jit_addr res_addr = 0; 10889 10890 if (smart_branch_opcode && !exit_addr) { 10891 if (smart_branch_opcode == ZEND_JMPZ) { 10892 undefined_label = target_label; 10893 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10894 defined_label = target_label; 10895 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10896 undefined_label = target_label; 10897 defined_label = target_label2; 10898 } else { 10899 ZEND_UNREACHABLE(); 10900 } 10901 } 10902 10903 | // if (CACHED_PTR(opline->extended_value)) { 10904 | mov r0, EX->run_time_cache 10905 | mov r0, aword [r0 + opline->extended_value] 10906 | test r0, r0 10907 | jz >1 10908 | test r0, 0x1 10909 | jnz >4 10910 |.cold_code 10911 |4: 10912 | MEM_LOAD_ZTS FCARG1a, aword, executor_globals, zend_constants, FCARG1a 10913 | shr r0, 1 10914 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 10915 10916 if (smart_branch_opcode) { 10917 if (exit_addr) { 10918 if (smart_branch_opcode == ZEND_JMPZ) { 10919 | jz &exit_addr 10920 } else { 10921 | jz >3 10922 } 10923 } else if (undefined_label != (uint32_t)-1) { 10924 | jz =>undefined_label 10925 } else { 10926 | jz >3 10927 } 10928 } else { 10929 | jz >2 10930 } 10931 |1: 10932 | SET_EX_OPLINE opline, r0 10933 | LOAD_ADDR FCARG1a, zv 10934 | EXT_CALL zend_jit_check_constant, r0 10935 | test r0, r0 10936 if (exit_addr) { 10937 if (smart_branch_opcode == ZEND_JMPNZ) { 10938 | jz >3 10939 } else { 10940 | jnz >3 10941 } 10942 | jmp &exit_addr 10943 } else if (smart_branch_opcode) { 10944 if (undefined_label != (uint32_t)-1) { 10945 | jz =>undefined_label 10946 } else { 10947 | jz >3 10948 } 10949 if (defined_label != (uint32_t)-1) { 10950 | jmp =>defined_label 10951 } else { 10952 | jmp >3 10953 } 10954 } else { 10955 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10956 | jnz >1 10957 |2: 10958 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 10959 | jmp >3 10960 } 10961 |.code 10962 if (smart_branch_opcode) { 10963 if (exit_addr) { 10964 if (smart_branch_opcode == ZEND_JMPNZ) { 10965 | jmp &exit_addr 10966 } 10967 } else if (defined_label != (uint32_t)-1) { 10968 | jmp =>defined_label 10969 } 10970 } else { 10971 |1: 10972 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 10973 } 10974 |3: 10975 10976 return 1; 10977} 10978 10979static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10980{ 10981 uint32_t mask; 10982 zend_jit_addr op1_addr = OP1_ADDR(); 10983 10984 // TODO: support for is_resource() ??? 10985 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 10986 10987 if (op1_info & MAY_BE_UNDEF) { 10988 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10989 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10990 |.cold_code 10991 |1: 10992 } 10993 | SET_EX_OPLINE opline, r0 10994 | mov FCARG1d, opline->op1.var 10995 | EXT_CALL zend_jit_undefined_op_helper, r0 10996 zend_jit_check_exception_undef_result(Dst, opline); 10997 if (opline->extended_value & MAY_BE_NULL) { 10998 if (exit_addr) { 10999 if (smart_branch_opcode == ZEND_JMPNZ) { 11000 | jmp &exit_addr 11001 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11002 | jmp >7 11003 } 11004 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11005 return 0; 11006 } 11007 } else { 11008 if (exit_addr) { 11009 if (smart_branch_opcode == ZEND_JMPZ) { 11010 | jmp &exit_addr 11011 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11012 | jmp >7 11013 } 11014 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11015 return 0; 11016 } 11017 } 11018 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11019 |.code 11020 } 11021 } 11022 11023 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11024 mask = opline->extended_value; 11025 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11026 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11027 if (exit_addr) { 11028 if (smart_branch_opcode == ZEND_JMPNZ) { 11029 | jmp &exit_addr 11030 } 11031 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11032 return 0; 11033 } 11034 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11035 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11036 if (exit_addr) { 11037 if (smart_branch_opcode == ZEND_JMPZ) { 11038 | jmp &exit_addr 11039 } 11040 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11041 return 0; 11042 } 11043 } else { 11044 bool invert = 0; 11045 zend_uchar type; 11046 11047 switch (mask) { 11048 case MAY_BE_NULL: type = IS_NULL; break; 11049 case MAY_BE_FALSE: type = IS_FALSE; break; 11050 case MAY_BE_TRUE: type = IS_TRUE; break; 11051 case MAY_BE_LONG: type = IS_LONG; break; 11052 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11053 case MAY_BE_STRING: type = IS_STRING; break; 11054 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11055 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11056 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 11057 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 11058 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 11059 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 11060 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 11061 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 11062 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 11063 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 11064 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 11065 default: 11066 type = 0; 11067 } 11068 11069 if (op1_info & MAY_BE_REF) { 11070 | LOAD_ZVAL_ADDR r0, op1_addr 11071 | ZVAL_DEREF r0, op1_info 11072 } 11073 if (type == 0) { 11074 if (smart_branch_opcode && 11075 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11076 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11077 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11078 | // if (Z_REFCOUNTED_P(cv)) { 11079 | IF_ZVAL_REFCOUNTED op1_addr, >1 11080 |.cold_code 11081 |1: 11082 } 11083 | // if (!Z_DELREF_P(cv)) { 11084 | GET_ZVAL_PTR FCARG1a, op1_addr 11085 | GC_DELREF FCARG1a 11086 if (RC_MAY_BE_1(op1_info)) { 11087 if (RC_MAY_BE_N(op1_info)) { 11088 | jnz >3 11089 } 11090 if (op1_info & MAY_BE_REF) { 11091 | mov al, byte [r0 + 8] 11092 } else { 11093 | mov al, byte [FP + opline->op1.var + 8] 11094 } 11095 | mov byte T1, al // save 11096 | // zval_dtor_func(r); 11097 | ZVAL_DTOR_FUNC op1_info, opline 11098 | mov cl, byte T1 // restore 11099 |jmp >2 11100 } 11101 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11102 if (!RC_MAY_BE_1(op1_info)) { 11103 | jmp >3 11104 } 11105 |.code 11106 } 11107 |3: 11108 if (op1_info & MAY_BE_REF) { 11109 | mov cl, byte [r0 + 8] 11110 } else { 11111 | mov cl, byte [FP + opline->op1.var + 8] 11112 } 11113 |2: 11114 } else { 11115 if (op1_info & MAY_BE_REF) { 11116 | mov cl, byte [r0 + 8] 11117 } else { 11118 | mov cl, byte [FP + opline->op1.var + 8] 11119 } 11120 } 11121 | mov eax, 1 11122 | shl eax, cl 11123 | test eax, mask 11124 if (exit_addr) { 11125 if (smart_branch_opcode == ZEND_JMPNZ) { 11126 | jne &exit_addr 11127 } else { 11128 | je &exit_addr 11129 } 11130 } else if (smart_branch_opcode) { 11131 if (smart_branch_opcode == ZEND_JMPZ) { 11132 | je =>target_label 11133 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11134 | jne =>target_label 11135 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11136 | je =>target_label 11137 | jmp =>target_label2 11138 } else { 11139 ZEND_UNREACHABLE(); 11140 } 11141 } else { 11142 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11143 11144 | setne al 11145 | movzx eax, al 11146 | add eax, 2 11147 | SET_ZVAL_TYPE_INFO res_addr, eax 11148 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11149 } 11150 } else { 11151 if (smart_branch_opcode && 11152 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11153 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11154 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11155 | // if (Z_REFCOUNTED_P(cv)) { 11156 | IF_ZVAL_REFCOUNTED op1_addr, >1 11157 |.cold_code 11158 |1: 11159 } 11160 | // if (!Z_DELREF_P(cv)) { 11161 | GET_ZVAL_PTR FCARG1a, op1_addr 11162 | GC_DELREF FCARG1a 11163 if (RC_MAY_BE_1(op1_info)) { 11164 if (RC_MAY_BE_N(op1_info)) { 11165 | jnz >3 11166 } 11167 if (op1_info & MAY_BE_REF) { 11168 | mov al, byte [r0 + 8] 11169 } else { 11170 | mov al, byte [FP + opline->op1.var + 8] 11171 } 11172 | mov byte T1, al // save 11173 | // zval_dtor_func(r); 11174 | ZVAL_DTOR_FUNC op1_info, opline 11175 | mov cl, byte T1 // restore 11176 |jmp >2 11177 } 11178 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11179 if (!RC_MAY_BE_1(op1_info)) { 11180 | jmp >3 11181 } 11182 |.code 11183 } 11184 |3: 11185 if (op1_info & MAY_BE_REF) { 11186 | mov cl, byte [r0 + 8] 11187 } else { 11188 | mov cl, byte [FP + opline->op1.var + 8] 11189 } 11190 |2: 11191 | cmp cl, type 11192 } else { 11193 if (op1_info & MAY_BE_REF) { 11194 | cmp byte [r0 + 8], type 11195 } else { 11196 | cmp byte [FP + opline->op1.var + 8], type 11197 } 11198 } 11199 if (exit_addr) { 11200 if (invert) { 11201 if (smart_branch_opcode == ZEND_JMPNZ) { 11202 | jne &exit_addr 11203 } else { 11204 | je &exit_addr 11205 } 11206 } else { 11207 if (smart_branch_opcode == ZEND_JMPNZ) { 11208 | je &exit_addr 11209 } else { 11210 | jne &exit_addr 11211 } 11212 } 11213 } else if (smart_branch_opcode) { 11214 if (invert) { 11215 if (smart_branch_opcode == ZEND_JMPZ) { 11216 | je =>target_label 11217 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11218 | jne =>target_label 11219 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11220 | je =>target_label 11221 | jmp =>target_label2 11222 } else { 11223 ZEND_UNREACHABLE(); 11224 } 11225 } else { 11226 if (smart_branch_opcode == ZEND_JMPZ) { 11227 | jne =>target_label 11228 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11229 | je =>target_label 11230 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11231 | jne =>target_label 11232 | jmp =>target_label2 11233 } else { 11234 ZEND_UNREACHABLE(); 11235 } 11236 } 11237 } else { 11238 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11239 11240 if (invert) { 11241 | setne al 11242 } else { 11243 | sete al 11244 } 11245 | movzx eax, al 11246 | add eax, 2 11247 | SET_ZVAL_TYPE_INFO res_addr, eax 11248 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11249 } 11250 } 11251 } 11252 } 11253 11254 |7: 11255 11256 return 1; 11257} 11258 11259static int zend_jit_leave_frame(dasm_State **Dst) 11260{ 11261 | // EG(current_execute_data) = EX(prev_execute_data); 11262 | mov r0, EX->prev_execute_data 11263 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, r0, r2 11264 return 1; 11265} 11266 11267static int zend_jit_free_cvs(dasm_State **Dst) 11268{ 11269 | // EG(current_execute_data) = EX(prev_execute_data); 11270 | mov FCARG1a, EX->prev_execute_data 11271 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FCARG1a, r0 11272 | // zend_free_compiled_variables(execute_data); 11273 | mov FCARG1a, FP 11274 | EXT_CALL zend_free_compiled_variables, r0 11275 return 1; 11276} 11277 11278static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11279{ 11280 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11281 uint32_t offset = EX_NUM_TO_VAR(var); 11282 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11283 } 11284 return 1; 11285} 11286 11287static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11288{ 11289 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11290 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11291 } 11292 return 1; 11293} 11294 11295static int zend_jit_leave_func(dasm_State **Dst, 11296 const zend_op_array *op_array, 11297 const zend_op *opline, 11298 uint32_t op1_info, 11299 bool left_frame, 11300 zend_jit_trace_rec *trace, 11301 zend_jit_trace_info *trace_info, 11302 int indirect_var_access, 11303 int may_throw) 11304{ 11305 bool may_be_top_frame = 11306 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11307 !JIT_G(current_frame) || 11308 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11309 bool may_need_call_helper = 11310 indirect_var_access || /* may have symbol table */ 11311 !op_array->function_name || /* may have symbol table */ 11312 may_be_top_frame || 11313 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11314 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11315 !JIT_G(current_frame) || 11316 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11317 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11318 bool may_need_release_this = 11319 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11320 op_array->scope && 11321 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11322 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11323 !JIT_G(current_frame) || 11324 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11325 11326 if (may_need_release_this) { 11327 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11328 } 11329 if (may_need_call_helper) { 11330 if (!left_frame) { 11331 left_frame = 1; 11332 if (!zend_jit_leave_frame(Dst)) { 11333 return 0; 11334 } 11335 } 11336 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11337 if (may_need_release_this) { 11338 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11339 } else { 11340 | test dword [FP + offsetof(zend_execute_data, This.u1.type_info)], (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11341 } 11342 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11343 | jnz >1 11344 |.cold_code 11345 |1: 11346 if (!GCC_GLOBAL_REGS) { 11347 | mov FCARG1a, FP 11348 } 11349 | EXT_CALL zend_jit_leave_func_helper, r0 11350 11351 if (may_be_top_frame) { 11352 // TODO: try to avoid this check ??? 11353 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11354#if 0 11355 /* this check should be handled by the following OPLINE guard */ 11356 | cmp IP, zend_jit_halt_op 11357 | je ->trace_halt 11358#endif 11359 } else if (GCC_GLOBAL_REGS) { 11360 | test IP, IP 11361 | je ->trace_halt 11362 } else { 11363 | test eax, eax 11364 | jl ->trace_halt 11365 } 11366 } 11367 11368 if (!GCC_GLOBAL_REGS) { 11369 | // execute_data = EG(current_execute_data) 11370 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 11371 } 11372 | jmp >8 11373 |.code 11374 } else { 11375 | jnz ->leave_function_handler 11376 } 11377 } 11378 11379 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11380 if (!left_frame) { 11381 left_frame = 1; 11382 if (!zend_jit_leave_frame(Dst)) { 11383 return 0; 11384 } 11385 } 11386 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11387 | mov FCARG1a, EX->func 11388 | sub FCARG1a, sizeof(zend_object) 11389 | OBJ_RELEASE ZREG_FCARG1, >4 11390 |4: 11391 } else if (may_need_release_this) { 11392 if (!left_frame) { 11393 left_frame = 1; 11394 if (!zend_jit_leave_frame(Dst)) { 11395 return 0; 11396 } 11397 } 11398 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11399 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11400 | je >4 11401 | // zend_object *object = Z_OBJ(execute_data->This); 11402 | mov FCARG1a, EX->This.value.obj 11403 | // OBJ_RELEASE(object); 11404 | OBJ_RELEASE ZREG_FCARG1, >4 11405 |4: 11406 // TODO: avoid EG(excption) check for $this->foo() calls 11407 may_throw = 1; 11408 } 11409 11410 | // EG(vm_stack_top) = (zval*)execute_data; 11411 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, FP, r0 11412 | // execute_data = EX(prev_execute_data); 11413 | mov FP, EX->prev_execute_data 11414 11415 if (!left_frame) { 11416 | // EG(current_execute_data) = execute_data; 11417 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 11418 } 11419 11420 |9: 11421 if (trace) { 11422 if (trace->op != ZEND_JIT_TRACE_END 11423 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11424 zend_jit_reset_last_valid_opline(); 11425 } else { 11426 | LOAD_IP 11427 | ADD_IP sizeof(zend_op) 11428 } 11429 11430 |8: 11431 11432 if (trace->op == ZEND_JIT_TRACE_BACK 11433 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11434 const zend_op *next_opline = trace->opline; 11435 11436 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11437 && (op1_info & MAY_BE_RC1) 11438 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11439 /* exception might be thrown during destruction of unused return value */ 11440 | // if (EG(exception)) 11441 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11442 | jne ->leave_throw_handler 11443 } 11444 do { 11445 trace++; 11446 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11447 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11448 next_opline = trace->opline; 11449 ZEND_ASSERT(next_opline != NULL); 11450 11451 if (trace->op == ZEND_JIT_TRACE_END 11452 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11453 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11454 | CMP_IP next_opline 11455 | je =>0 // LOOP 11456#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11457 | JMP_IP 11458#else 11459 | jmp ->trace_escape 11460#endif 11461 } else { 11462 | CMP_IP next_opline 11463 | jne ->trace_escape 11464 } 11465 11466 zend_jit_set_last_valid_opline(trace->opline); 11467 11468 return 1; 11469 } else if (may_throw || 11470 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11471 && (op1_info & MAY_BE_RC1) 11472 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11473 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11474 | // if (EG(exception)) 11475 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11476 | jne ->leave_throw_handler 11477 } 11478 11479 return 1; 11480 } else { 11481 | // if (EG(exception)) 11482 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11483 | LOAD_IP 11484 | jne ->leave_throw_handler 11485 | // opline = EX(opline) + 1 11486 | ADD_IP sizeof(zend_op) 11487 } 11488 11489 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11490 | ADD_HYBRID_SPAD 11491#ifdef CONTEXT_THREADED_JIT 11492 | push aword [IP] 11493 | ret 11494#else 11495 | JMP_IP 11496#endif 11497 } else if (GCC_GLOBAL_REGS) { 11498 | add r4, SPAD // stack alignment 11499#ifdef CONTEXT_THREADED_JIT 11500 | push aword [IP] 11501 | ret 11502#else 11503 | JMP_IP 11504#endif 11505 } else { 11506#ifdef CONTEXT_THREADED_JIT 11507 ZEND_UNREACHABLE(); 11508 // TODO: context threading can't work without GLOBAL REGS because we have to change 11509 // the value of execute_data in execute_ex() 11510 | mov FCARG1a, FP 11511 | mov r0, aword [FP] 11512 | mov FP, aword T2 // restore FP 11513 | mov RX, aword T3 // restore IP 11514 | add r4, NR_SPAD // stack alignment 11515 | push aword [r0] 11516 | ret 11517#else 11518 | mov FP, aword T2 // restore FP 11519 | mov RX, aword T3 // restore IP 11520 | add r4, NR_SPAD // stack alignment 11521 | mov r0, 2 // ZEND_VM_LEAVE 11522 | ret 11523#endif 11524 } 11525 11526 return 1; 11527} 11528 11529static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11530{ 11531 zend_jit_addr ret_addr; 11532 int8_t return_value_used; 11533 11534 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11535 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11536 11537 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11538 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11539 return_value_used = 1; 11540 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11541 return_value_used = 0; 11542 } else { 11543 return_value_used = -1; 11544 } 11545 } else { 11546 return_value_used = -1; 11547 } 11548 11549 if (ZEND_OBSERVER_ENABLED) { 11550 if (Z_MODE(op1_addr) == IS_REG) { 11551 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11552 11553 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11554 return 0; 11555 } 11556 op1_addr = dst; 11557 } 11558 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11559 | mov FCARG1a, FP 11560 | SET_EX_OPLINE opline, r0 11561 | EXT_CALL zend_observer_fcall_end, r0 11562 } 11563 11564 // if (!EX(return_value)) 11565 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11566 if (return_value_used != 0) { 11567 | mov r2, EX->return_value 11568 } 11569 if (return_value_used == -1) { 11570 | test r2, r2 11571 } 11572 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11573 } else { 11574 if (return_value_used != 0) { 11575 | mov r1, EX->return_value 11576 } 11577 if (return_value_used == -1) { 11578 | test r1, r1 11579 } 11580 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11581 } 11582 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11583 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11584 if (return_value_used == -1) { 11585 | jz >1 11586 |.cold_code 11587 |1: 11588 } 11589 if (return_value_used != 1) { 11590 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11591 if (jit_return_label >= 0) { 11592 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11593 } else { 11594 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11595 } 11596 } 11597 | GET_ZVAL_PTR FCARG1a, op1_addr 11598 | GC_DELREF FCARG1a 11599 if (RC_MAY_BE_1(op1_info)) { 11600 if (RC_MAY_BE_N(op1_info)) { 11601 if (jit_return_label >= 0) { 11602 | jnz =>jit_return_label 11603 } else { 11604 | jnz >9 11605 } 11606 } 11607 | //SAVE_OPLINE() 11608 | ZVAL_DTOR_FUNC op1_info, opline 11609 | //????mov r1, EX->return_value // reload ??? 11610 } 11611 if (return_value_used == -1) { 11612 if (jit_return_label >= 0) { 11613 | jmp =>jit_return_label 11614 } else { 11615 | jmp >9 11616 } 11617 |.code 11618 } 11619 } 11620 } else if (return_value_used == -1) { 11621 if (jit_return_label >= 0) { 11622 | jz =>jit_return_label 11623 } else { 11624 | jz >9 11625 } 11626 } 11627 11628 if (return_value_used == 0) { 11629 |9: 11630 return 1; 11631 } 11632 11633 if (opline->op1_type == IS_CONST) { 11634 zval *zv = RT_CONSTANT(opline, opline->op1); 11635 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11636 if (Z_REFCOUNTED_P(zv)) { 11637 | ADDREF_CONST zv, r0 11638 } 11639 } else if (opline->op1_type == IS_TMP_VAR) { 11640 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11641 } else if (opline->op1_type == IS_CV) { 11642 if (op1_info & MAY_BE_REF) { 11643 | LOAD_ZVAL_ADDR r0, op1_addr 11644 | ZVAL_DEREF r0, op1_info 11645 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11646 } 11647 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11648 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11649 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11650 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11651 !op_array->function_name) { 11652 | TRY_ADDREF op1_info, ah, r2 11653 } else if (return_value_used != 1) { 11654 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11655 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11656 } 11657 } 11658 } else { 11659 if (op1_info & MAY_BE_REF) { 11660 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11661 11662 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11663 |.cold_code 11664 |1: 11665 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11666 | GET_ZVAL_PTR r0, op1_addr 11667 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11668 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11669 | GC_DELREF r0 11670 | je >2 11671 | // if (IS_REFCOUNTED()) 11672 if (jit_return_label >= 0) { 11673 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11674 } else { 11675 | IF_NOT_REFCOUNTED dh, >9 11676 } 11677 | // ADDREF 11678 | GET_ZVAL_PTR r2, ret_addr // reload 11679 | GC_ADDREF r2 11680 if (jit_return_label >= 0) { 11681 | jmp =>jit_return_label 11682 } else { 11683 | jmp >9 11684 } 11685 |2: 11686 | EFREE_REFERENCE r0 11687 if (jit_return_label >= 0) { 11688 | jmp =>jit_return_label 11689 } else { 11690 | jmp >9 11691 } 11692 |.code 11693 } 11694 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11695 } 11696 11697 |9: 11698 return 1; 11699} 11700 11701static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11702{ 11703 ZEND_ASSERT(type_reg == ZREG_R2); 11704 11705 |.if not(X64) 11706 || if (Z_REG(val_addr) == ZREG_R1) { 11707 | GET_ZVAL_W2 r0, val_addr 11708 || } 11709 |.endif 11710 | GET_ZVAL_PTR r1, val_addr 11711 |.if not(X64) 11712 || if (Z_REG(val_addr) != ZREG_R1) { 11713 | GET_ZVAL_W2 r0, val_addr 11714 || } 11715 |.endif 11716 | IF_NOT_REFCOUNTED dh, >2 11717 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11718 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11719 |.if not(X64) 11720 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11721 |.endif 11722 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11723 | IF_NOT_REFCOUNTED dh, >2 11724 |1: 11725 | GC_ADDREF r1 11726 |2: 11727 | SET_ZVAL_PTR res_addr, r1 11728 |.if not(X64) 11729 | SET_ZVAL_W2 res_addr, r0 11730 |.endif 11731 | SET_ZVAL_TYPE_INFO res_addr, edx 11732 11733 return 1; 11734} 11735 11736static int zend_jit_fetch_dim_read(dasm_State **Dst, 11737 const zend_op *opline, 11738 zend_ssa *ssa, 11739 const zend_ssa_op *ssa_op, 11740 uint32_t op1_info, 11741 zend_jit_addr op1_addr, 11742 bool op1_avoid_refcounting, 11743 uint32_t op2_info, 11744 uint32_t res_info, 11745 zend_jit_addr res_addr, 11746 uint8_t dim_type) 11747{ 11748 zend_jit_addr orig_op1_addr, op2_addr; 11749 const void *exit_addr = NULL; 11750 const void *not_found_exit_addr = NULL; 11751 const void *res_exit_addr = NULL; 11752 bool result_avoid_refcounting = 0; 11753 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11754 int may_throw = 0; 11755 11756 orig_op1_addr = OP1_ADDR(); 11757 op2_addr = OP2_ADDR(); 11758 11759 if (opline->opcode != ZEND_FETCH_DIM_IS 11760 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11761 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11762 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11763 if (!exit_addr) { 11764 return 0; 11765 } 11766 } 11767 11768 if ((res_info & MAY_BE_GUARD) 11769 && JIT_G(current_frame) 11770 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11771 uint32_t flags = 0; 11772 uint32_t old_op1_info = 0; 11773 uint32_t old_info; 11774 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11775 int32_t exit_point; 11776 11777 if (opline->opcode != ZEND_FETCH_LIST_R 11778 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11779 && !op1_avoid_refcounting) { 11780 flags |= ZEND_JIT_EXIT_FREE_OP1; 11781 } 11782 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11783 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11784 flags |= ZEND_JIT_EXIT_FREE_OP2; 11785 } 11786 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11787 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11788 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11789 && (ssa_op+1)->op1_use == ssa_op->result_def 11790 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11791 && zend_jit_may_avoid_refcounting(opline+1)) { 11792 result_avoid_refcounting = 1; 11793 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11794 } 11795 11796 if (op1_avoid_refcounting) { 11797 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11798 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11799 } 11800 11801 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11802 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11803 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11804 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11805 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11806 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11807 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11808 if (!res_exit_addr) { 11809 return 0; 11810 } 11811 res_info &= ~MAY_BE_GUARD; 11812 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11813 } 11814 11815 if (opline->opcode == ZEND_FETCH_DIM_IS 11816 && !(res_info & MAY_BE_NULL)) { 11817 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11818 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11819 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11820 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11821 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11822 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11823 if (!not_found_exit_addr) { 11824 return 0; 11825 } 11826 } 11827 11828 if (op1_avoid_refcounting) { 11829 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11830 } 11831 } 11832 11833 if (op1_info & MAY_BE_REF) { 11834 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11835 | ZVAL_DEREF FCARG1a, op1_info 11836 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11837 } 11838 11839 if (op1_info & MAY_BE_ARRAY) { 11840 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11841 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11842 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 11843 } else { 11844 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 11845 } 11846 } 11847 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11848 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11849 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11850 may_throw = 1; 11851 } 11852 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11853 return 0; 11854 } 11855 } 11856 11857 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11858 if (op1_info & MAY_BE_ARRAY) { 11859 |.cold_code 11860 |7: 11861 } 11862 11863 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11864 may_throw = 1; 11865 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11866 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11867 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 11868 } else { 11869 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 11870 } 11871 } 11872 | SET_EX_OPLINE opline, r0 11873 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11874 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11875 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11876 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 11877 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 11878 } else { 11879 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11880 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 11881 } 11882 | SET_ZVAL_PTR res_addr, r0 11883 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 11884 } else { 11885 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11886 |.if X64 11887 | LOAD_ZVAL_ADDR CARG3, res_addr 11888 |.else 11889 | sub r4, 12 11890 | PUSH_ZVAL_ADDR res_addr, r0 11891 |.endif 11892 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 11893 |.if not(X64) 11894 | add r4, 12 11895 |.endif 11896 } 11897 if ((op1_info & MAY_BE_ARRAY) || 11898 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11899 | jmp >9 // END 11900 } 11901 |6: 11902 } 11903 11904 if (op1_info & MAY_BE_OBJECT) { 11905 may_throw = 1; 11906 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11907 if (exit_addr) { 11908 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 11909 } else { 11910 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 11911 } 11912 } 11913 | SET_EX_OPLINE opline, r0 11914 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11915 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11916 } 11917 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11918 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11919 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 11920 } else { 11921 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11922 } 11923 |.if X64 11924 | LOAD_ZVAL_ADDR CARG3, res_addr 11925 |.else 11926 | sub r4, 12 11927 | PUSH_ZVAL_ADDR res_addr, r0 11928 |.endif 11929 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11930 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 11931 } else { 11932 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 11933 } 11934 |.if not(X64) 11935 | add r4, 12 11936 |.endif 11937 if ((op1_info & MAY_BE_ARRAY) || 11938 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11939 | jmp >9 // END 11940 } 11941 |6: 11942 } 11943 11944 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11945 | SET_EX_OPLINE opline, r0 11946 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11947 may_throw = 1; 11948 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11949 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11950 | mov FCARG1d, opline->op1.var 11951 | EXT_CALL zend_jit_undefined_op_helper, r0 11952 |1: 11953 } 11954 11955 if (op2_info & MAY_BE_UNDEF) { 11956 may_throw = 1; 11957 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 11958 | mov FCARG1d, opline->op2.var 11959 | EXT_CALL zend_jit_undefined_op_helper, r0 11960 |1: 11961 } 11962 } 11963 11964 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11965 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11966 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 11967 may_throw = 1; 11968 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 11969 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 11970 } else { 11971 | SET_EX_OPLINE opline, r0 11972 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 11973 Z_REG(op1_addr) != ZREG_FCARG1 || 11974 Z_OFFSET(op1_addr) != 0) { 11975 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11976 } 11977 } 11978 | EXT_CALL zend_jit_invalid_array_access, r0 11979 } 11980 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 11981 if (op1_info & MAY_BE_ARRAY) { 11982 | jmp >9 // END 11983 } 11984 } 11985 11986 if (op1_info & MAY_BE_ARRAY) { 11987 |.code 11988 } 11989 } 11990 11991 if (op1_info & MAY_BE_ARRAY) { 11992 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11993 11994 |8: 11995 if (res_exit_addr) { 11996 zend_uchar type = concrete_type(res_info); 11997 11998 if ((op1_info & MAY_BE_ARRAY_OF_REF) 11999 && dim_type != IS_UNKNOWN 12000 && dim_type != IS_REFERENCE) { 12001 if (type < IS_STRING) { 12002 | IF_NOT_ZVAL_TYPE val_addr, type, >1 12003 |.cold_code 12004 |1: 12005 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr 12006 | GET_Z_PTR r0, r0 12007 | add r0, offsetof(zend_reference, val) 12008 | IF_ZVAL_TYPE val_addr, type, >1 12009 | jmp &res_exit_addr 12010 |.code 12011 |1: 12012 } else { 12013 | GET_ZVAL_TYPE_INFO edx, val_addr 12014 | IF_NOT_TYPE dl, type, >1 12015 |.cold_code 12016 |1: 12017 | IF_NOT_TYPE dl, IS_REFERENCE, &res_exit_addr 12018 | GET_Z_PTR r0, r0 12019 | add r0, offsetof(zend_reference, val) 12020 | GET_ZVAL_TYPE_INFO edx, val_addr 12021 | IF_TYPE dl, type, >1 12022 | jmp &res_exit_addr 12023 |.code 12024 |1: 12025 } 12026 } else { 12027 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12028 | ZVAL_DEREF r0, MAY_BE_REF 12029 } 12030 if (type < IS_STRING) { 12031 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12032 } else { 12033 | GET_ZVAL_TYPE_INFO edx, val_addr 12034 | IF_NOT_TYPE dl, type, &res_exit_addr 12035 } 12036 } 12037 12038 | // ZVAL_COPY 12039 |7: 12040 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12041 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12042 if (type < IS_STRING) { 12043 if (Z_REG(res_addr) != ZREG_FP || 12044 JIT_G(current_frame) == NULL || 12045 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12046 | SET_ZVAL_TYPE_INFO res_addr, type 12047 } 12048 } else { 12049 | SET_ZVAL_TYPE_INFO res_addr, edx 12050 if (!result_avoid_refcounting) { 12051 | TRY_ADDREF res_info, dh, r1 12052 } 12053 } 12054 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12055 return 0; 12056 } 12057 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12058 | // ZVAL_COPY_DEREF 12059 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12060 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12061 return 0; 12062 } 12063 } else { 12064 | // ZVAL_COPY 12065 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12066 | TRY_ADDREF res_info, ch, r2 12067 } 12068 } 12069 |9: // END 12070 12071#ifdef ZEND_JIT_USE_RC_INFERENCE 12072 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12073 /* Magic offsetGet() may increase refcount of the key */ 12074 op2_info |= MAY_BE_RCN; 12075 } 12076#endif 12077 12078 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 12079 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 12080 may_throw = 1; 12081 } 12082 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12083 } 12084 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12085 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 12086 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 12087 may_throw = 1; 12088 } 12089 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12090 } 12091 } 12092 12093 if (may_throw) { 12094 if (!zend_jit_check_exception(Dst)) { 12095 return 0; 12096 } 12097 } 12098 12099 return 1; 12100} 12101 12102static int zend_jit_fetch_dim(dasm_State **Dst, 12103 const zend_op *opline, 12104 uint32_t op1_info, 12105 zend_jit_addr op1_addr, 12106 uint32_t op2_info, 12107 zend_jit_addr res_addr, 12108 uint8_t dim_type) 12109{ 12110 zend_jit_addr op2_addr; 12111 int may_throw = 0; 12112 12113 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12114 12115 if (op1_info & MAY_BE_REF) { 12116 may_throw = 1; 12117 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12118 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12119 | GET_Z_PTR FCARG2a, FCARG1a 12120 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12121 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12122 | jmp >3 12123 |.cold_code 12124 |2: 12125 | SET_EX_OPLINE opline, r0 12126 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12127 | test r0, r0 12128 | mov FCARG1a, r0 12129 | jne >1 12130 | jmp ->exception_handler_undef 12131 |.code 12132 |1: 12133 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12134 } 12135 12136 if (op1_info & MAY_BE_ARRAY) { 12137 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12138 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12139 } 12140 |3: 12141 | SEPARATE_ARRAY op1_addr, op1_info, 1 12142 } 12143 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 12144 if (op1_info & MAY_BE_ARRAY) { 12145 |.cold_code 12146 |7: 12147 } 12148 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12149 | CMP_ZVAL_TYPE op1_addr, IS_NULL 12150 | jg >7 12151 } 12152 if (Z_REG(op1_addr) != ZREG_FP) { 12153 | mov T1, Ra(Z_REG(op1_addr)) // save 12154 } 12155 if ((op1_info & MAY_BE_UNDEF) 12156 && opline->opcode == ZEND_FETCH_DIM_RW) { 12157 may_throw = 1; 12158 if (op1_info & MAY_BE_NULL) { 12159 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12160 } 12161 | SET_EX_OPLINE opline, r0 12162 | mov FCARG1a, opline->op1.var 12163 | EXT_CALL zend_jit_undefined_op_helper, r0 12164 |1: 12165 } 12166 | // ZVAL_ARR(container, zend_new_array(8)); 12167 | EXT_CALL _zend_new_array_0, r0 12168 if (Z_REG(op1_addr) != ZREG_FP) { 12169 | mov Ra(Z_REG(op1_addr)), T1 // restore 12170 } 12171 | SET_ZVAL_LVAL op1_addr, r0 12172 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12173 | mov FCARG1a, r0 12174 if (op1_info & MAY_BE_ARRAY) { 12175 | jmp >1 12176 |.code 12177 |1: 12178 } 12179 } 12180 12181 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12182 |6: 12183 if (opline->op2_type == IS_UNUSED) { 12184 may_throw = 1; 12185 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12186 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12187 | EXT_CALL zend_hash_next_index_insert, r0 12188 | // if (UNEXPECTED(!var_ptr)) { 12189 | test r0, r0 12190 | jz >1 12191 |.cold_code 12192 |1: 12193 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12194 | CANNOT_ADD_ELEMENT opline 12195 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12196 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12197 | jmp >8 12198 |.code 12199 | SET_ZVAL_PTR res_addr, r0 12200 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12201 } else { 12202 uint32_t type; 12203 12204 switch (opline->opcode) { 12205 case ZEND_FETCH_DIM_W: 12206 case ZEND_FETCH_LIST_W: 12207 type = BP_VAR_W; 12208 break; 12209 case ZEND_FETCH_DIM_RW: 12210 may_throw = 1; 12211 type = BP_VAR_RW; 12212 break; 12213 case ZEND_FETCH_DIM_UNSET: 12214 type = BP_VAR_UNSET; 12215 break; 12216 default: 12217 ZEND_UNREACHABLE(); 12218 } 12219 12220 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 12221 may_throw = 1; 12222 } 12223 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 12224 return 0; 12225 } 12226 12227 |8: 12228 | SET_ZVAL_PTR res_addr, r0 12229 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12230 12231 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12232 |.cold_code 12233 |9: 12234 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12235 | jmp >8 12236 |.code 12237 } 12238 } 12239 } 12240 12241 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12242 may_throw = 1; 12243 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12244 |.cold_code 12245 |7: 12246 } 12247 12248 | SET_EX_OPLINE opline, r0 12249 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12250 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12251 } 12252 if (opline->op2_type == IS_UNUSED) { 12253 | xor FCARG2a, FCARG2a 12254 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12255 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12256 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12257 } else { 12258 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12259 } 12260 |.if X64 12261 | LOAD_ZVAL_ADDR CARG3, res_addr 12262 |.else 12263 | sub r4, 12 12264 | PUSH_ZVAL_ADDR res_addr, r0 12265 |.endif 12266 switch (opline->opcode) { 12267 case ZEND_FETCH_DIM_W: 12268 case ZEND_FETCH_LIST_W: 12269 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12270 break; 12271 case ZEND_FETCH_DIM_RW: 12272 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12273 break; 12274// case ZEND_FETCH_DIM_UNSET: 12275// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12276// break; 12277 default: 12278 ZEND_UNREACHABLE(); 12279 } 12280 |.if not(X64) 12281 | add r4, 12 12282 |.endif 12283 12284 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12285 | jmp >8 // END 12286 |.code 12287 } 12288 } 12289 12290#ifdef ZEND_JIT_USE_RC_INFERENCE 12291 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12292 /* ASSIGN_DIM may increase refcount of the key */ 12293 op2_info |= MAY_BE_RCN; 12294 } 12295#endif 12296 12297 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 12298 && (op2_info & MAY_HAVE_DTOR) 12299 && (op2_info & MAY_BE_RC1)) { 12300 may_throw = 1; 12301 } 12302 |8: 12303 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12304 12305 if (may_throw) { 12306 if (!zend_jit_check_exception(Dst)) { 12307 return 0; 12308 } 12309 } 12310 12311 return 1; 12312} 12313 12314static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12315 const zend_op *opline, 12316 uint32_t op1_info, 12317 zend_jit_addr op1_addr, 12318 bool op1_avoid_refcounting, 12319 uint32_t op2_info, 12320 uint8_t dim_type, 12321 int may_throw, 12322 zend_uchar smart_branch_opcode, 12323 uint32_t target_label, 12324 uint32_t target_label2, 12325 const void *exit_addr) 12326{ 12327 zend_jit_addr op2_addr, res_addr; 12328 12329 // TODO: support for empty() ??? 12330 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12331 12332 op2_addr = OP2_ADDR(); 12333 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12334 12335 if (op1_info & MAY_BE_REF) { 12336 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12337 | ZVAL_DEREF FCARG1a, op1_info 12338 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12339 } 12340 12341 if (op1_info & MAY_BE_ARRAY) { 12342 const void *found_exit_addr = NULL; 12343 const void *not_found_exit_addr = NULL; 12344 12345 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12346 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12347 } 12348 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12349 if (exit_addr 12350 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12351 && !may_throw 12352 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12353 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12354 if (smart_branch_opcode == ZEND_JMPNZ) { 12355 found_exit_addr = exit_addr; 12356 } else { 12357 not_found_exit_addr = exit_addr; 12358 } 12359 } 12360 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 12361 return 0; 12362 } 12363 12364 if (found_exit_addr) { 12365 |9: 12366 return 1; 12367 } else if (not_found_exit_addr) { 12368 |8: 12369 return 1; 12370 } 12371 } 12372 12373 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12374 if (op1_info & MAY_BE_ARRAY) { 12375 |.cold_code 12376 |7: 12377 } 12378 12379 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12380 | SET_EX_OPLINE opline, r0 12381 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12382 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12383 } 12384 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12385 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12386 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12387 } else { 12388 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12389 } 12390 | EXT_CALL zend_jit_isset_dim_helper, r0 12391 | test r0, r0 12392 | jz >9 12393 if (op1_info & MAY_BE_ARRAY) { 12394 | jmp >8 12395 |.code 12396 } 12397 } else { 12398 if (op2_info & MAY_BE_UNDEF) { 12399 if (op2_info & MAY_BE_ANY) { 12400 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12401 } 12402 | SET_EX_OPLINE opline, r0 12403 | mov FCARG1d, opline->op2.var 12404 | EXT_CALL zend_jit_undefined_op_helper, r0 12405 |1: 12406 } 12407 if (op1_info & MAY_BE_ARRAY) { 12408 | jmp >9 12409 |.code 12410 } 12411 } 12412 } 12413 12414#ifdef ZEND_JIT_USE_RC_INFERENCE 12415 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12416 /* Magic offsetExists() may increase refcount of the key */ 12417 op2_info |= MAY_BE_RCN; 12418 } 12419#endif 12420 12421 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12422 |8: 12423 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12424 if (!op1_avoid_refcounting) { 12425 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12426 } 12427 if (may_throw) { 12428 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12429 return 0; 12430 } 12431 } 12432 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12433 if (exit_addr) { 12434 if (smart_branch_opcode == ZEND_JMPNZ) { 12435 | jmp &exit_addr 12436 } else { 12437 | jmp >8 12438 } 12439 } else if (smart_branch_opcode) { 12440 if (smart_branch_opcode == ZEND_JMPZ) { 12441 | jmp =>target_label2 12442 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12443 | jmp =>target_label 12444 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12445 | jmp =>target_label2 12446 } else { 12447 ZEND_UNREACHABLE(); 12448 } 12449 } else { 12450 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12451 | jmp >8 12452 } 12453 } else { 12454 | NIY // TODO: support for empty() 12455 } 12456 } 12457 12458 |9: // not found 12459 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12460 if (!op1_avoid_refcounting) { 12461 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12462 } 12463 if (may_throw) { 12464 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12465 return 0; 12466 } 12467 } 12468 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12469 if (exit_addr) { 12470 if (smart_branch_opcode == ZEND_JMPZ) { 12471 | jmp &exit_addr 12472 } 12473 } else if (smart_branch_opcode) { 12474 if (smart_branch_opcode == ZEND_JMPZ) { 12475 | jmp =>target_label 12476 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12477 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12478 | jmp =>target_label 12479 } else { 12480 ZEND_UNREACHABLE(); 12481 } 12482 } else { 12483 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12484 } 12485 } else { 12486 | NIY // TODO: support for empty() 12487 } 12488 12489 |8: 12490 12491 return 1; 12492} 12493 12494static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12495{ 12496 zend_jit_addr op1_addr = OP1_ADDR(); 12497 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12498 12499 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12500 | mov FCARG2a, EX->run_time_cache 12501 | mov r0, aword [FCARG2a + opline->extended_value] 12502 | sub r0, 1 12503 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12504 | MEM_LOAD_ZTS ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12505 |.if X64 12506 | shl r1, 5 12507 |.else 12508 | imul r1, sizeof(Bucket) 12509 |.endif 12510 | cmp r0, r1 12511 | jae >9 12512 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12513 | MEM_LOAD_OP_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12514 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12515 | // (EXPECTED(p->key == varname)) 12516 | ADDR_CMP aword [r0 + offsetof(Bucket, key)], varname, r1 12517 | jne >9 12518 | GET_Z_PTR r0, r0 12519 | GC_ADDREF r0 12520 |1: 12521 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12522 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12523 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12524 | IF_ZVAL_REFCOUNTED op1_addr, >2 12525 |.cold_code 12526 |2: 12527 } 12528 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12529 | GET_ZVAL_PTR FCARG1a, op1_addr 12530 | // ZVAL_REF(variable_ptr, ref) 12531 | SET_ZVAL_PTR op1_addr, r0 12532 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12533 | // if (GC_DELREF(garbage) == 0) 12534 | GC_DELREF FCARG1a 12535 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12536 | jnz >3 12537 } else { 12538 | jnz >5 12539 } 12540 | ZVAL_DTOR_FUNC op1_info, opline 12541 | jmp >5 12542 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12543 |3: 12544 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12545 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12546 | EXT_CALL gc_possible_root, r1 12547 | jmp >5 12548 } 12549 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12550 |.code 12551 } 12552 } 12553 12554 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12555 | // ZVAL_REF(variable_ptr, ref) 12556 | SET_ZVAL_PTR op1_addr, r0 12557 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12558 } 12559 |5: 12560 //END of handler 12561 12562 |.cold_code 12563 |9: 12564 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12565 if (opline->extended_value) { 12566 | add FCARG2a, opline->extended_value 12567 } 12568 | EXT_CALL zend_jit_fetch_global_helper, r0 12569 | jmp <1 12570 |.code 12571 12572 return 1; 12573} 12574 12575static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 12576{ 12577 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12578 bool in_cold = 0; 12579 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12580 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_R0; 12581 12582 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12583 && JIT_G(current_frame) 12584 && JIT_G(current_frame)->prev) { 12585 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12586 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 12587 12588 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 12589 return 1; 12590 } 12591 } 12592 12593 if (ZEND_ARG_SEND_MODE(arg_info)) { 12594 if (opline->opcode == ZEND_RECV_INIT) { 12595 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12596 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12597 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12598 } else { 12599 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12600 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12601 } 12602 } 12603 12604 if (type_mask != 0) { 12605 if (is_power_of_two(type_mask)) { 12606 uint32_t type_code = concrete_type(type_mask); 12607 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12608 } else { 12609 | mov edx, 1 12610 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12611 | shl edx, cl 12612 | test edx, type_mask 12613 | je >1 12614 } 12615 12616 |.cold_code 12617 |1: 12618 12619 in_cold = 1; 12620 } 12621 12622 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 12623 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12624 } 12625 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12626 | SET_EX_OPLINE opline, r0 12627 } else { 12628 | ADDR_STORE aword EX->opline, opline, r0 12629 } 12630 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12631 | EXT_CALL zend_jit_verify_arg_slow, r0 12632 12633 if (check_exception) { 12634 | test al, al 12635 if (in_cold) { 12636 | jnz >1 12637 | jmp ->exception_handler 12638 |.code 12639 |1: 12640 } else { 12641 | jz ->exception_handler 12642 } 12643 } else if (in_cold) { 12644 | jmp >1 12645 |.code 12646 |1: 12647 } 12648 12649 return 1; 12650} 12651 12652static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12653{ 12654 uint32_t arg_num = opline->op1.num; 12655 zend_arg_info *arg_info = NULL; 12656 12657 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12658 if (EXPECTED(arg_num <= op_array->num_args)) { 12659 arg_info = &op_array->arg_info[arg_num-1]; 12660 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12661 arg_info = &op_array->arg_info[op_array->num_args]; 12662 } 12663 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12664 arg_info = NULL; 12665 } 12666 } 12667 12668 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12669 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12670 if (!JIT_G(current_frame) || 12671 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 12672 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12673 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12674 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12675 12676 if (!exit_addr) { 12677 return 0; 12678 } 12679 | cmp dword EX->This.u2.num_args, arg_num 12680 | jb &exit_addr 12681 } 12682 } else { 12683 | cmp dword EX->This.u2.num_args, arg_num 12684 | jb >1 12685 |.cold_code 12686 |1: 12687 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12688 | SET_EX_OPLINE opline, r0 12689 } else { 12690 | ADDR_STORE aword EX->opline, opline, r0 12691 } 12692 | mov FCARG1a, FP 12693 | EXT_CALL zend_missing_arg_error, r0 12694 | jmp ->exception_handler 12695 |.code 12696 } 12697 } 12698 12699 if (arg_info) { 12700 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12701 return 0; 12702 } 12703 } 12704 12705 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12706 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12707 | LOAD_IP_ADDR (opline + 1) 12708 zend_jit_set_last_valid_opline(opline + 1); 12709 } 12710 } 12711 12712 return 1; 12713} 12714 12715static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12716{ 12717 uint32_t arg_num = opline->op1.num; 12718 zval *zv = RT_CONSTANT(opline, opline->op2); 12719 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12720 12721 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12722 && JIT_G(current_frame) 12723 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12724 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12725 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12726 if (Z_REFCOUNTED_P(zv)) { 12727 | ADDREF_CONST zv, r0 12728 } 12729 } 12730 } else { 12731 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12732 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12733 | cmp dword EX->This.u2.num_args, arg_num 12734 | jae >5 12735 } 12736 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12737 if (Z_REFCOUNTED_P(zv)) { 12738 | ADDREF_CONST zv, r0 12739 } 12740 } 12741 12742 if (Z_CONSTANT_P(zv)) { 12743 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12744 | SET_EX_OPLINE opline, r0 12745 } else { 12746 | ADDR_STORE aword EX->opline, opline, r0 12747 } 12748 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12749 | mov r0, EX->func 12750 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12751 | .if X64 12752 | EXT_CALL zval_update_constant_ex, r0 12753 | .else 12754 ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) 12755 | EXT_CALL zval_jit_update_constant_ex, r0 12756 ||#else 12757 | EXT_CALL zval_update_constant_ex, r0 12758 ||#endif 12759 | .endif 12760 | test al, al 12761 | jnz >1 12762 |.cold_code 12763 |1: 12764 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12765 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12766 | jmp ->exception_handler 12767 |.code 12768 } 12769 12770 |5: 12771 12772 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12773 do { 12774 zend_arg_info *arg_info; 12775 12776 if (arg_num <= op_array->num_args) { 12777 arg_info = &op_array->arg_info[arg_num-1]; 12778 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12779 arg_info = &op_array->arg_info[op_array->num_args]; 12780 } else { 12781 break; 12782 } 12783 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12784 break; 12785 } 12786 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12787 return 0; 12788 } 12789 } while (0); 12790 } 12791 12792 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12793 if (is_last) { 12794 | LOAD_IP_ADDR (opline + 1) 12795 zend_jit_set_last_valid_opline(opline + 1); 12796 } 12797 } 12798 12799 return 1; 12800} 12801 12802static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12803{ 12804 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12805 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12806 12807 if (!exit_addr) { 12808 return 0; 12809 } 12810 12811 |.if X64 12812 || if (!IS_SIGNED_32BIT(ce)) { 12813 | mov64 r0, ((ptrdiff_t)ce) 12814 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 12815 || } else { 12816 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12817 || } 12818 |.else 12819 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12820 |.endif 12821 | jne &exit_addr 12822 12823 return 1; 12824} 12825 12826static int zend_jit_fetch_obj(dasm_State **Dst, 12827 const zend_op *opline, 12828 const zend_op_array *op_array, 12829 zend_ssa *ssa, 12830 const zend_ssa_op *ssa_op, 12831 uint32_t op1_info, 12832 zend_jit_addr op1_addr, 12833 bool op1_indirect, 12834 zend_class_entry *ce, 12835 bool ce_is_instanceof, 12836 bool on_this, 12837 bool delayed_fetch_this, 12838 bool op1_avoid_refcounting, 12839 zend_class_entry *trace_ce, 12840 uint8_t prop_type, 12841 int may_throw) 12842{ 12843 zval *member; 12844 zend_property_info *prop_info; 12845 bool may_be_dynamic = 1; 12846 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12847 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12848 zend_jit_addr prop_addr; 12849 uint32_t res_info = RES_INFO(); 12850 bool type_loaded = 0; 12851 12852 ZEND_ASSERT(opline->op2_type == IS_CONST); 12853 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12854 12855 member = RT_CONSTANT(opline, opline->op2); 12856 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12857 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12858 12859 if (on_this) { 12860 | GET_ZVAL_PTR FCARG1a, this_addr 12861 } else { 12862 if (opline->op1_type == IS_VAR 12863 && opline->opcode == ZEND_FETCH_OBJ_W 12864 && (op1_info & MAY_BE_INDIRECT) 12865 && Z_REG(op1_addr) == ZREG_FP) { 12866 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12867 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 12868 | GET_Z_PTR FCARG1a, FCARG1a 12869 |1: 12870 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12871 } 12872 if (op1_info & MAY_BE_REF) { 12873 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12874 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12875 } 12876 | ZVAL_DEREF FCARG1a, op1_info 12877 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12878 } 12879 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12880 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12881 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12882 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12883 12884 if (!exit_addr) { 12885 return 0; 12886 } 12887 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12888 } else { 12889 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 12890 } 12891 } 12892 | GET_ZVAL_PTR FCARG1a, op1_addr 12893 } 12894 12895 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12896 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12897 if (prop_info) { 12898 ce = trace_ce; 12899 ce_is_instanceof = 0; 12900 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12901 if (on_this && JIT_G(current_frame) 12902 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12903 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12904 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12905 if (on_this && JIT_G(current_frame)) { 12906 JIT_G(current_frame)->ce = ce; 12907 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12908 } 12909 } else { 12910 return 0; 12911 } 12912 if (ssa->var_info && ssa_op->op1_use >= 0) { 12913 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12914 ssa->var_info[ssa_op->op1_use].ce = ce; 12915 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12916 } 12917 } 12918 } 12919 } 12920 12921 if (!prop_info) { 12922 | mov r0, EX->run_time_cache 12923 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 12924 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 12925 | jne >5 12926 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 12927 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 12928 if (may_be_dynamic) { 12929 | test r0, r0 12930 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12931 | jl >5 12932 } else { 12933 | jl >8 // dynamic property 12934 } 12935 } 12936 | mov edx, dword [FCARG1a + r0 + 8] 12937 | IF_UNDEF dl, >5 12938 | add FCARG1a, r0 12939 type_loaded = 1; 12940 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12941 if (opline->opcode == ZEND_FETCH_OBJ_W 12942 && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { 12943 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12944 12945 | mov r0, EX->run_time_cache 12946 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 12947 | test FCARG2a, FCARG2a 12948 | jnz >1 12949 |.cold_code 12950 |1: 12951 | test dword [FCARG2a + offsetof(zend_property_info, flags)], ZEND_ACC_READONLY 12952 if (flags) { 12953 | jz >3 12954 } else { 12955 | jz >4 12956 } 12957 | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >2 12958 | GET_Z_PTR r0, FCARG1a 12959 | GC_ADDREF r0 12960 | SET_ZVAL_PTR res_addr, r0 12961 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 12962 | jmp >9 12963 |2: 12964 | mov FCARG1a, FCARG2a 12965 | SET_EX_OPLINE opline, r0 12966 | EXT_CALL zend_readonly_property_modification_error, r0 12967 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 12968 | jmp >9 12969 |3: 12970 if (flags == ZEND_FETCH_DIM_WRITE) { 12971 | SET_EX_OPLINE opline, r0 12972 | EXT_CALL zend_jit_check_array_promotion, r0 12973 | jmp >9 12974 } else if (flags == ZEND_FETCH_REF) { 12975 |.if X64 12976 | LOAD_ZVAL_ADDR CARG3, res_addr 12977 |.else 12978 | sub r4, 12 12979 | PUSH_ZVAL_ADDR res_addr, r0 12980 |.endif 12981 | EXT_CALL zend_jit_create_typed_ref, r0 12982 |.if not(X64) 12983 | add r4, 12 12984 |.endif 12985 | jmp >9 12986 } else { 12987 ZEND_ASSERT(flags == 0); 12988 } 12989 |.code 12990 |4: 12991 } 12992 } else { 12993 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12994 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12995 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 12996 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 12997 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12998 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12999 13000 if (!exit_addr) { 13001 return 0; 13002 } 13003 type_loaded = 1; 13004 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13005 | IF_UNDEF dl, &exit_addr 13006 } 13007 } else { 13008 type_loaded = 1; 13009 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13010 | IF_UNDEF dl, >5 13011 } 13012 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 13013 if (!type_loaded) { 13014 type_loaded = 1; 13015 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13016 } 13017 | IF_NOT_TYPE dl, IS_OBJECT, >4 13018 | GET_ZVAL_PTR r0, prop_addr 13019 | GC_ADDREF r0 13020 | SET_ZVAL_PTR res_addr, r0 13021 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13022 | jmp >9 13023 |.cold_code 13024 |4: 13025 | LOAD_ADDR FCARG1a, prop_info 13026 | SET_EX_OPLINE opline, r0 13027 | EXT_CALL zend_readonly_property_modification_error, r0 13028 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13029 | jmp >9 13030 |.code 13031 } 13032 if (opline->opcode == ZEND_FETCH_OBJ_W 13033 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13034 && ZEND_TYPE_IS_SET(prop_info->type)) { 13035 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13036 13037 if (flags == ZEND_FETCH_DIM_WRITE) { 13038 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 13039 if (!type_loaded) { 13040 type_loaded = 1; 13041 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13042 } 13043 | cmp dl, IS_FALSE 13044 | jle >1 13045 |.cold_code 13046 |1: 13047 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13048 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13049 } 13050 | LOAD_ADDR FCARG2a, prop_info 13051 | SET_EX_OPLINE opline, r0 13052 | EXT_CALL zend_jit_check_array_promotion, r0 13053 | jmp >9 13054 |.code 13055 } 13056 } else if (flags == ZEND_FETCH_REF) { 13057 if (!type_loaded) { 13058 type_loaded = 1; 13059 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13060 } 13061 | IF_TYPE dl, IS_REFERENCE, >1 13062 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13063 | LOAD_ADDR FCARG2a, prop_info 13064 } else { 13065 int prop_info_offset = 13066 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13067 13068 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13069 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13070 | mov FCARG2a, aword[r0 + prop_info_offset] 13071 } 13072 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13073 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13074 } 13075 |.if X64 13076 | LOAD_ZVAL_ADDR CARG3, res_addr 13077 |.else 13078 | sub r4, 12 13079 | PUSH_ZVAL_ADDR res_addr, r0 13080 |.endif 13081 | EXT_CALL zend_jit_create_typed_ref, r0 13082 |.if not(X64) 13083 | add r4, 12 13084 |.endif 13085 | jmp >9 13086 |1: 13087 } else { 13088 ZEND_UNREACHABLE(); 13089 } 13090 } 13091 } 13092 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13093 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13094 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13095 } 13096 | SET_ZVAL_PTR res_addr, FCARG1a 13097 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13098 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13099 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13100 } 13101 } else { 13102 bool result_avoid_refcounting = 0; 13103 13104 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13105 uint32_t flags = 0; 13106 uint32_t old_info; 13107 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13108 int32_t exit_point; 13109 const void *exit_addr; 13110 zend_uchar type; 13111 zend_jit_addr val_addr = prop_addr; 13112 13113 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13114 && !delayed_fetch_this 13115 && !op1_avoid_refcounting) { 13116 flags = ZEND_JIT_EXIT_FREE_OP1; 13117 } 13118 13119 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13120 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13121 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13122 && (ssa_op+1)->op1_use == ssa_op->result_def 13123 && zend_jit_may_avoid_refcounting(opline+1)) { 13124 result_avoid_refcounting = 1; 13125 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13126 } 13127 13128 type = concrete_type(res_info); 13129 13130 if (prop_type != IS_UNKNOWN 13131 && prop_type != IS_UNDEF 13132 && prop_type != IS_REFERENCE 13133 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 13134 exit_point = zend_jit_trace_get_exit_point(opline, 0); 13135 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13136 } else { 13137 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13138 | LOAD_ZVAL_ADDR r0, prop_addr 13139 if (op1_avoid_refcounting) { 13140 SET_STACK_REG(JIT_G(current_frame)->stack, 13141 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13142 } 13143 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13144 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13145 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 13146 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13147 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13148 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13149 if (!exit_addr) { 13150 return 0; 13151 } 13152 13153 if (!type_loaded) { 13154 type_loaded = 1; 13155 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13156 } 13157 | // ZVAL_DEREF() 13158 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13159 | GET_Z_PTR r0, r0 13160 | add r0, offsetof(zend_reference, val) 13161 | GET_ZVAL_TYPE_INFO edx, val_addr 13162 } 13163 res_info &= ~MAY_BE_GUARD; 13164 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13165 if (type < IS_STRING) { 13166 |1: 13167 if (type_loaded) { 13168 | IF_NOT_TYPE dl, type, &exit_addr 13169 } else { 13170 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13171 } 13172 } else { 13173 if (!type_loaded) { 13174 type_loaded = 1; 13175 | GET_ZVAL_TYPE_INFO edx, val_addr 13176 } 13177 |1: 13178 | IF_NOT_TYPE dl, type, &exit_addr 13179 } 13180 | // ZVAL_COPY 13181 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13182 if (type < IS_STRING) { 13183 if (Z_REG(res_addr) != ZREG_FP || 13184 JIT_G(current_frame) == NULL || 13185 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13186 | SET_ZVAL_TYPE_INFO res_addr, type 13187 } 13188 } else { 13189 | SET_ZVAL_TYPE_INFO res_addr, edx 13190 if (!result_avoid_refcounting) { 13191 | TRY_ADDREF res_info, dh, r1 13192 } 13193 } 13194 } else { 13195 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13196 return 0; 13197 } 13198 } 13199 } 13200 13201 if (op1_avoid_refcounting) { 13202 SET_STACK_REG(JIT_G(current_frame)->stack, 13203 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13204 } 13205 13206 |.cold_code 13207 13208 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13209 |5: 13210 | SET_EX_OPLINE opline, r0 13211 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13212 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13213 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13214 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13215 } else { 13216 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13217 } 13218 | jmp >9 13219 } 13220 13221 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13222 |7: 13223 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13224 | SET_EX_OPLINE opline, r0 13225 if (opline->opcode != ZEND_FETCH_OBJ_W 13226 && (op1_info & MAY_BE_UNDEF)) { 13227 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13228 13229 if (op1_info & MAY_BE_ANY) { 13230 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13231 } 13232 | mov FCARG1d, opline->op1.var 13233 | EXT_CALL zend_jit_undefined_op_helper, r0 13234 |1: 13235 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13236 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13237 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13238 } 13239 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13240 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13241 | EXT_CALL zend_jit_invalid_property_write, r0 13242 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13243 } else { 13244 | EXT_CALL zend_jit_invalid_property_read, r0 13245 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13246 } 13247 | jmp >9 13248 } else { 13249 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13250 | jmp >9 13251 } 13252 } 13253 13254 if (!prop_info 13255 && may_be_dynamic 13256 && opline->opcode != ZEND_FETCH_OBJ_W) { 13257 |8: 13258 | mov FCARG2a, r0 13259 | SET_EX_OPLINE opline, r0 13260 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13261 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13262 } else { 13263 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13264 } 13265 | jmp >9 13266 } 13267 13268 |.code; 13269 |9: // END 13270 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13271 if (opline->op1_type == IS_VAR 13272 && opline->opcode == ZEND_FETCH_OBJ_W 13273 && (op1_info & MAY_BE_RC1)) { 13274 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13275 13276 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13277 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13278 | GC_DELREF FCARG1a 13279 | jnz >1 13280 | SET_EX_OPLINE opline, r0 13281 | EXT_CALL zend_jit_extract_helper, r0 13282 |1: 13283 } else if (!op1_avoid_refcounting) { 13284 if (on_this) { 13285 op1_info &= ~MAY_BE_RC1; 13286 } 13287 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13288 } 13289 } 13290 13291 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13292 && prop_info 13293 && (opline->opcode != ZEND_FETCH_OBJ_W || 13294 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13295 !ZEND_TYPE_IS_SET(prop_info->type)) 13296 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 13297 may_throw = 0; 13298 } 13299 13300 if (may_throw) { 13301 if (!zend_jit_check_exception(Dst)) { 13302 return 0; 13303 } 13304 } 13305 13306 return 1; 13307} 13308 13309static int zend_jit_incdec_obj(dasm_State **Dst, 13310 const zend_op *opline, 13311 const zend_op_array *op_array, 13312 zend_ssa *ssa, 13313 const zend_ssa_op *ssa_op, 13314 uint32_t op1_info, 13315 zend_jit_addr op1_addr, 13316 bool op1_indirect, 13317 zend_class_entry *ce, 13318 bool ce_is_instanceof, 13319 bool on_this, 13320 bool delayed_fetch_this, 13321 zend_class_entry *trace_ce, 13322 uint8_t prop_type) 13323{ 13324 zval *member; 13325 zend_string *name; 13326 zend_property_info *prop_info; 13327 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13328 zend_jit_addr res_addr = 0; 13329 zend_jit_addr prop_addr; 13330 bool needs_slow_path = 0; 13331 bool use_prop_guard = 0; 13332 bool may_throw = 0; 13333 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 13334 13335 ZEND_ASSERT(opline->op2_type == IS_CONST); 13336 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13337 13338 if (opline->result_type != IS_UNUSED) { 13339 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13340 } 13341 13342 member = RT_CONSTANT(opline, opline->op2); 13343 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13344 name = Z_STR_P(member); 13345 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13346 13347 if (on_this) { 13348 | GET_ZVAL_PTR FCARG1a, this_addr 13349 } else { 13350 if (opline->op1_type == IS_VAR 13351 && (op1_info & MAY_BE_INDIRECT) 13352 && Z_REG(op1_addr) == ZREG_FP) { 13353 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13354 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13355 | GET_Z_PTR FCARG1a, FCARG1a 13356 |1: 13357 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13358 } 13359 if (op1_info & MAY_BE_REF) { 13360 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13361 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13362 } 13363 | ZVAL_DEREF FCARG1a, op1_info 13364 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13365 } 13366 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13367 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13368 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13369 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13370 13371 if (!exit_addr) { 13372 return 0; 13373 } 13374 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13375 } else { 13376 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13377 |.cold_code 13378 |1: 13379 | SET_EX_OPLINE opline, r0 13380 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13381 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13382 } 13383 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13384 | EXT_CALL zend_jit_invalid_property_incdec, r0 13385 | jmp ->exception_handler 13386 |.code 13387 } 13388 } 13389 | GET_ZVAL_PTR FCARG1a, op1_addr 13390 } 13391 13392 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13393 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13394 if (prop_info) { 13395 ce = trace_ce; 13396 ce_is_instanceof = 0; 13397 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13398 if (on_this && JIT_G(current_frame) 13399 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13400 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13401 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13402 if (on_this && JIT_G(current_frame)) { 13403 JIT_G(current_frame)->ce = ce; 13404 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13405 } 13406 } else { 13407 return 0; 13408 } 13409 if (ssa->var_info && ssa_op->op1_use >= 0) { 13410 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13411 ssa->var_info[ssa_op->op1_use].ce = ce; 13412 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13413 } 13414 if (ssa->var_info && ssa_op->op1_def >= 0) { 13415 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13416 ssa->var_info[ssa_op->op1_def].ce = ce; 13417 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13418 } 13419 } 13420 } 13421 } 13422 13423 use_prop_guard = (prop_type != IS_UNKNOWN 13424 && prop_type != IS_UNDEF 13425 && prop_type != IS_REFERENCE 13426 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13427 13428 if (!prop_info) { 13429 needs_slow_path = 1; 13430 13431 | mov r0, EX->run_time_cache 13432 | mov r2, aword [r0 + opline->extended_value] 13433 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13434 | jne >7 13435 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13436 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13437 | jnz >7 13438 } 13439 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13440 | test r0, r0 13441 | jl >7 13442 if (!use_prop_guard) { 13443 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13444 } 13445 | add FCARG1a, r0 13446 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13447 } else { 13448 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13449 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13450 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13451 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13452 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13453 13454 if (!exit_addr) { 13455 return 0; 13456 } 13457 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13458 } else { 13459 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13460 needs_slow_path = 1; 13461 } 13462 } 13463 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13464 may_throw = 1; 13465 | SET_EX_OPLINE opline, r0 13466 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13467 | LOAD_ADDR FCARG2a, prop_info 13468 } else { 13469 int prop_info_offset = 13470 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13471 13472 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13473 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13474 | mov FCARG2a, aword[r0 + prop_info_offset] 13475 } 13476 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13477 if (opline->result_type == IS_UNUSED) { 13478 switch (opline->opcode) { 13479 case ZEND_PRE_INC_OBJ: 13480 case ZEND_POST_INC_OBJ: 13481 | EXT_CALL zend_jit_inc_typed_prop, r0 13482 break; 13483 case ZEND_PRE_DEC_OBJ: 13484 case ZEND_POST_DEC_OBJ: 13485 | EXT_CALL zend_jit_dec_typed_prop, r0 13486 break; 13487 default: 13488 ZEND_UNREACHABLE(); 13489 } 13490 } else { 13491 |.if X64 13492 | LOAD_ZVAL_ADDR CARG3, res_addr 13493 |.else 13494 | sub r4, 12 13495 | PUSH_ZVAL_ADDR res_addr, r0 13496 |.endif 13497 switch (opline->opcode) { 13498 case ZEND_PRE_INC_OBJ: 13499 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13500 break; 13501 case ZEND_PRE_DEC_OBJ: 13502 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13503 break; 13504 case ZEND_POST_INC_OBJ: 13505 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13506 break; 13507 case ZEND_POST_DEC_OBJ: 13508 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13509 break; 13510 default: 13511 ZEND_UNREACHABLE(); 13512 } 13513 |.if not(X64) 13514 | add r4, 12 13515 |.endif 13516 } 13517 } 13518 } 13519 13520 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13521 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13522 zend_jit_addr var_addr = prop_addr; 13523 13524 if (use_prop_guard) { 13525 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13526 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13527 13528 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 13529 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13530 } 13531 13532 if (var_info & MAY_BE_REF) { 13533 may_throw = 1; 13534 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13535 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13536 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13537 } 13538 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13539 | GET_ZVAL_PTR FCARG1a, var_addr 13540 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13541 | jnz >1 13542 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13543 |.cold_code 13544 |1: 13545 if (opline) { 13546 | SET_EX_OPLINE opline, r0 13547 } 13548 if (opline->result_type == IS_UNUSED) { 13549 | xor FCARG2a, FCARG2a 13550 } else { 13551 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13552 } 13553 switch (opline->opcode) { 13554 case ZEND_PRE_INC_OBJ: 13555 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13556 break; 13557 case ZEND_PRE_DEC_OBJ: 13558 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13559 break; 13560 case ZEND_POST_INC_OBJ: 13561 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13562 break; 13563 case ZEND_POST_DEC_OBJ: 13564 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13565 break; 13566 default: 13567 ZEND_UNREACHABLE(); 13568 } 13569 | jmp >9 13570 |.code 13571 |2: 13572 } 13573 13574 if (var_info & MAY_BE_LONG) { 13575 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13576 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13577 } 13578 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13579 if (opline->result_type != IS_UNUSED) { 13580 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13581 } 13582 } 13583 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13584 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13585 } else { 13586 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13587 } 13588 | jo >3 13589 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13590 if (opline->result_type != IS_UNUSED) { 13591 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13592 } 13593 } 13594 |.cold_code 13595 } 13596 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13597 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13598 may_throw = 1; 13599 } 13600 if (var_info & MAY_BE_LONG) { 13601 |2: 13602 } 13603 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 13604 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13605 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13606 } 13607 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13608 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13609 | TRY_ADDREF MAY_BE_ANY, ah, r2 13610 } 13611 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13612 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13613 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13614 | EXT_CALL zend_jit_pre_inc, r0 13615 } else { 13616 | EXT_CALL increment_function, r0 13617 } 13618 } else { 13619 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13620 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13621 | EXT_CALL zend_jit_pre_dec, r0 13622 } else { 13623 | EXT_CALL decrement_function, r0 13624 } 13625 } 13626 if (var_info & MAY_BE_LONG) { 13627 | jmp >4 13628 } 13629 } 13630 if (var_info & MAY_BE_LONG) { 13631 |3: 13632 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13633 |.if X64 13634 | mov64 rax, 0x43e0000000000000 13635 | SET_ZVAL_LVAL var_addr, rax 13636 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13637 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13638 | SET_ZVAL_LVAL res_addr, rax 13639 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13640 } 13641 |.else 13642 | SET_ZVAL_LVAL var_addr, 0 13643 | SET_ZVAL_W2 var_addr, 0x41e00000 13644 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13645 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13646 | SET_ZVAL_LVAL res_addr, 0 13647 | SET_ZVAL_W2 res_addr, 0x41e00000 13648 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13649 } 13650 |.endif 13651 } else { 13652 |.if X64 13653 | mov64 rax, 0xc3e0000000000000 13654 | SET_ZVAL_LVAL var_addr, rax 13655 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13656 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13657 | SET_ZVAL_LVAL res_addr, rax 13658 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13659 } 13660 |.else 13661 | SET_ZVAL_LVAL var_addr, 0x00200000 13662 | SET_ZVAL_W2 var_addr, 0xc1e00000 13663 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13664 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13665 | SET_ZVAL_LVAL res_addr, 0x00200000 13666 | SET_ZVAL_W2 res_addr, 0xc1e00000 13667 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13668 } 13669 |.endif 13670 } 13671 if (opline->result_type != IS_UNUSED 13672 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 13673 && prop_info 13674 && !ZEND_TYPE_IS_SET(prop_info->type) 13675 && (res_info & MAY_BE_GUARD) 13676 && (res_info & MAY_BE_LONG)) { 13677 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13678 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13679 int32_t exit_point; 13680 const void *exit_addr; 13681 13682 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 13683 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 13684 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13685 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 13686 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 13687 | jmp &exit_addr 13688 |.code 13689 } else { 13690 | jmp >4 13691 |.code 13692 |4: 13693 } 13694 } 13695 } 13696 13697 if (needs_slow_path) { 13698 may_throw = 1; 13699 |.cold_code 13700 |7: 13701 | SET_EX_OPLINE opline, r0 13702 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13703 | LOAD_ADDR FCARG2a, name 13704 |.if X64 13705 | mov CARG3, EX->run_time_cache 13706 | add CARG3, opline->extended_value 13707 if (opline->result_type == IS_UNUSED) { 13708 | xor CARG4, CARG4 13709 } else { 13710 | LOAD_ZVAL_ADDR CARG4, res_addr 13711 } 13712 |.else 13713 | sub r4, 8 13714 if (opline->result_type == IS_UNUSED) { 13715 | push 0 13716 } else { 13717 | PUSH_ZVAL_ADDR res_addr, r0 13718 } 13719 | mov r0, EX->run_time_cache 13720 | add r0, opline->extended_value 13721 | push r0 13722 |.endif 13723 13724 switch (opline->opcode) { 13725 case ZEND_PRE_INC_OBJ: 13726 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13727 break; 13728 case ZEND_PRE_DEC_OBJ: 13729 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13730 break; 13731 case ZEND_POST_INC_OBJ: 13732 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13733 break; 13734 case ZEND_POST_DEC_OBJ: 13735 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13736 break; 13737 default: 13738 ZEND_UNREACHABLE(); 13739 } 13740 13741 |.if not(X64) 13742 | add r4, 8 13743 |.endif 13744 13745 | jmp >9 13746 |.code 13747 } 13748 13749 |9: 13750 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13751 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13752 may_throw = 1; 13753 } 13754 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13755 } 13756 13757 if (may_throw) { 13758 if (!zend_jit_check_exception(Dst)) { 13759 return 0; 13760 } 13761 } 13762 13763 return 1; 13764} 13765 13766static int zend_jit_assign_obj_op(dasm_State **Dst, 13767 const zend_op *opline, 13768 const zend_op_array *op_array, 13769 zend_ssa *ssa, 13770 const zend_ssa_op *ssa_op, 13771 uint32_t op1_info, 13772 zend_jit_addr op1_addr, 13773 uint32_t val_info, 13774 zend_ssa_range *val_range, 13775 bool op1_indirect, 13776 zend_class_entry *ce, 13777 bool ce_is_instanceof, 13778 bool on_this, 13779 bool delayed_fetch_this, 13780 zend_class_entry *trace_ce, 13781 uint8_t prop_type) 13782{ 13783 zval *member; 13784 zend_string *name; 13785 zend_property_info *prop_info; 13786 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13787 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13788 zend_jit_addr prop_addr; 13789 bool needs_slow_path = 0; 13790 bool use_prop_guard = 0; 13791 bool may_throw = 0; 13792 binary_op_type binary_op = get_binary_op(opline->extended_value); 13793 13794 ZEND_ASSERT(opline->op2_type == IS_CONST); 13795 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13796 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13797 13798 member = RT_CONSTANT(opline, opline->op2); 13799 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13800 name = Z_STR_P(member); 13801 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13802 13803 if (on_this) { 13804 | GET_ZVAL_PTR FCARG1a, this_addr 13805 } else { 13806 if (opline->op1_type == IS_VAR 13807 && (op1_info & MAY_BE_INDIRECT) 13808 && Z_REG(op1_addr) == ZREG_FP) { 13809 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13810 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13811 | GET_Z_PTR FCARG1a, FCARG1a 13812 |1: 13813 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13814 } 13815 if (op1_info & MAY_BE_REF) { 13816 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13817 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13818 } 13819 | ZVAL_DEREF FCARG1a, op1_info 13820 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13821 } 13822 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13823 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13824 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13825 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13826 13827 if (!exit_addr) { 13828 return 0; 13829 } 13830 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13831 } else { 13832 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13833 |.cold_code 13834 |1: 13835 | SET_EX_OPLINE opline, r0 13836 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13837 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13838 } 13839 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13840 if (op1_info & MAY_BE_UNDEF) { 13841 | EXT_CALL zend_jit_invalid_property_assign_op, r0 13842 } else { 13843 | EXT_CALL zend_jit_invalid_property_assign, r0 13844 } 13845 may_throw = 1; 13846 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13847 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13848 may_throw = 1; 13849 | jmp >8 13850 } else { 13851 | jmp >9 13852 } 13853 |.code 13854 } 13855 } 13856 | GET_ZVAL_PTR FCARG1a, op1_addr 13857 } 13858 13859 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13860 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13861 if (prop_info) { 13862 ce = trace_ce; 13863 ce_is_instanceof = 0; 13864 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13865 if (on_this && JIT_G(current_frame) 13866 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13867 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13868 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13869 if (on_this && JIT_G(current_frame)) { 13870 JIT_G(current_frame)->ce = ce; 13871 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13872 } 13873 } else { 13874 return 0; 13875 } 13876 if (ssa->var_info && ssa_op->op1_use >= 0) { 13877 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13878 ssa->var_info[ssa_op->op1_use].ce = ce; 13879 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13880 } 13881 if (ssa->var_info && ssa_op->op1_def >= 0) { 13882 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13883 ssa->var_info[ssa_op->op1_def].ce = ce; 13884 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13885 } 13886 } 13887 } 13888 } 13889 13890 use_prop_guard = (prop_type != IS_UNKNOWN 13891 && prop_type != IS_UNDEF 13892 && prop_type != IS_REFERENCE 13893 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13894 13895 if (!prop_info) { 13896 needs_slow_path = 1; 13897 13898 | mov r0, EX->run_time_cache 13899 | mov r2, aword [r0 + (opline+1)->extended_value] 13900 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13901 | jne >7 13902 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13903 | cmp aword [r0 + (opline+1)->extended_value + sizeof(void*) * 2], 0 13904 | jnz >7 13905 } 13906 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 13907 | test r0, r0 13908 | jl >7 13909 if (!use_prop_guard) { 13910 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13911 } 13912 | add FCARG1a, r0 13913 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13914 } else { 13915 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13916 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13917 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13918 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13919 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13920 13921 if (!exit_addr) { 13922 return 0; 13923 } 13924 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13925 } else { 13926 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13927 needs_slow_path = 1; 13928 } 13929 } 13930 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13931 uint32_t info = val_info; 13932 13933 may_throw = 1; 13934 13935 if (opline) { 13936 | SET_EX_OPLINE opline, r0 13937 } 13938 13939 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 13940 |.cold_code 13941 |1: 13942 | GET_ZVAL_PTR FCARG1a, prop_addr 13943 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13944 | LOAD_ZVAL_ADDR FCARG2a, val_addr 13945 } 13946 |.if X64 13947 | LOAD_ADDR CARG3, binary_op 13948 |.else 13949 | sub r4, 12 13950 | PUSH_ADDR binary_op, r0 13951 |.endif 13952 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 13953 |.if not(X64) 13954 | add r4, 12 13955 |.endif 13956 | jmp >9 13957 |.code 13958 13959 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13960 13961 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13962 | LOAD_ADDR FCARG2a, prop_info 13963 } else { 13964 int prop_info_offset = 13965 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13966 13967 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13968 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13969 | mov FCARG2a, aword[r0 + prop_info_offset] 13970 } 13971 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13972 |.if X64 13973 | LOAD_ZVAL_ADDR CARG3, val_addr 13974 | LOAD_ADDR CARG4, binary_op 13975 |.else 13976 | sub r4, 8 13977 | PUSH_ADDR binary_op, r0 13978 | PUSH_ZVAL_ADDR val_addr, r0 13979 |.endif 13980 13981 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 13982 13983 |.if not(X64) 13984 | add r4, 8 13985 |.endif 13986 13987 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13988 info |= MAY_BE_RC1|MAY_BE_RCN; 13989 } 13990 13991 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 13992 } 13993 } 13994 13995 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13996 zend_jit_addr var_addr = prop_addr; 13997 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13998 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13999 14000 if (use_prop_guard) { 14001 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14002 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14003 14004 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 14005 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 14006 } 14007 14008 if (var_info & MAY_BE_REF) { 14009 may_throw = 1; 14010 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14011 | LOAD_ZVAL_ADDR r0, prop_addr 14012 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14013 | GET_ZVAL_PTR FCARG1a, var_addr 14014 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14015 | jnz >1 14016 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14017 |.cold_code 14018 |1: 14019 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14020 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14021 } 14022 if (opline) { 14023 | SET_EX_OPLINE opline, r0 14024 } 14025 |.if X64 14026 | LOAD_ADDR CARG3, binary_op 14027 |.else 14028 | sub r4, 12 14029 | PUSH_ADDR binary_op, r0 14030 |.endif 14031 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14032 |.if not(X64) 14033 | add r4, 12 14034 |.endif 14035 | jmp >9 14036 |.code 14037 |2: 14038 var_info &= ~MAY_BE_REF; 14039 } 14040 14041 switch (opline->extended_value) { 14042 case ZEND_ADD: 14043 case ZEND_SUB: 14044 case ZEND_MUL: 14045 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14046 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14047 if (opline->extended_value != ZEND_ADD || 14048 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14049 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14050 may_throw = 1; 14051 } 14052 } 14053 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14054 1 /* may overflow */, 0)) { 14055 return 0; 14056 } 14057 break; 14058 case ZEND_BW_OR: 14059 case ZEND_BW_AND: 14060 case ZEND_BW_XOR: 14061 may_throw = 1; 14062 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14063 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14064 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 14065 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 14066 may_throw = 1; 14067 } 14068 } 14069 goto long_math; 14070 case ZEND_SL: 14071 case ZEND_SR: 14072 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14073 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14074 may_throw = 1; 14075 } 14076 if ((opline+1)->op1_type != IS_CONST || 14077 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14078 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 14079 may_throw = 1; 14080 } 14081 goto long_math; 14082 case ZEND_MOD: 14083 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14084 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14085 if (opline->extended_value != ZEND_ADD || 14086 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14087 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14088 may_throw = 1; 14089 } 14090 } 14091 if ((opline+1)->op1_type != IS_CONST || 14092 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14093 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 14094 may_throw = 1; 14095 } 14096long_math: 14097 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14098 IS_CV, opline->op1, var_addr, var_info, NULL, 14099 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14100 val_range, 14101 0, var_addr, var_def_info, var_info, /* may throw */ 1)) { 14102 return 0; 14103 } 14104 break; 14105 case ZEND_CONCAT: 14106 may_throw = 1; 14107 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14108 0)) { 14109 return 0; 14110 } 14111 break; 14112 default: 14113 ZEND_UNREACHABLE(); 14114 } 14115 } 14116 14117 if (needs_slow_path) { 14118 may_throw = 1; 14119 |.cold_code 14120 |7: 14121 | SET_EX_OPLINE opline, r0 14122 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14123 | LOAD_ADDR FCARG2a, name 14124 |.if X64 14125 | LOAD_ZVAL_ADDR CARG3, val_addr 14126 | mov CARG4, EX->run_time_cache 14127 | add CARG4, (opline+1)->extended_value 14128 |.if X64WIN 14129 | LOAD_ADDR r0, binary_op 14130 | mov aword A5, r0 14131 |.else 14132 | LOAD_ADDR CARG5, binary_op 14133 |.endif 14134 |.else 14135 | sub r4, 4 14136 | PUSH_ADDR binary_op, r0 14137 | mov r0, EX->run_time_cache 14138 | add r0, (opline+1)->extended_value 14139 | push r0 14140 | PUSH_ZVAL_ADDR val_addr, r0 14141 |.endif 14142 14143 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14144 14145 |.if not(X64) 14146 | add r4, 4 14147 |.endif 14148 14149 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14150 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14151 } 14152 14153 |8: 14154 | // FREE_OP_DATA(); 14155 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14156 | jmp >9 14157 |.code 14158 } 14159 14160 |9: 14161 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14162 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 14163 may_throw = 1; 14164 } 14165 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14166 } 14167 14168 if (may_throw) { 14169 if (!zend_jit_check_exception(Dst)) { 14170 return 0; 14171 } 14172 } 14173 14174 return 1; 14175} 14176 14177static int zend_jit_assign_obj(dasm_State **Dst, 14178 const zend_op *opline, 14179 const zend_op_array *op_array, 14180 zend_ssa *ssa, 14181 const zend_ssa_op *ssa_op, 14182 uint32_t op1_info, 14183 zend_jit_addr op1_addr, 14184 uint32_t val_info, 14185 bool op1_indirect, 14186 zend_class_entry *ce, 14187 bool ce_is_instanceof, 14188 bool on_this, 14189 bool delayed_fetch_this, 14190 zend_class_entry *trace_ce, 14191 uint8_t prop_type, 14192 int may_throw) 14193{ 14194 zval *member; 14195 zend_string *name; 14196 zend_property_info *prop_info; 14197 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14198 zend_jit_addr res_addr = 0; 14199 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14200 zend_jit_addr prop_addr; 14201 bool needs_slow_path = 0; 14202 14203 if (RETURN_VALUE_USED(opline)) { 14204 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14205 } 14206 14207 ZEND_ASSERT(opline->op2_type == IS_CONST); 14208 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14209 14210 member = RT_CONSTANT(opline, opline->op2); 14211 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14212 name = Z_STR_P(member); 14213 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 14214 14215 if (on_this) { 14216 | GET_ZVAL_PTR FCARG1a, this_addr 14217 } else { 14218 if (opline->op1_type == IS_VAR 14219 && (op1_info & MAY_BE_INDIRECT) 14220 && Z_REG(op1_addr) == ZREG_FP) { 14221 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14222 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14223 | GET_Z_PTR FCARG1a, FCARG1a 14224 |1: 14225 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14226 } 14227 if (op1_info & MAY_BE_REF) { 14228 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14229 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14230 } 14231 | ZVAL_DEREF FCARG1a, op1_info 14232 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14233 } 14234 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14235 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14236 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14237 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14238 14239 if (!exit_addr) { 14240 return 0; 14241 } 14242 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14243 } else { 14244 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14245 |.cold_code 14246 |1: 14247 | SET_EX_OPLINE opline, r0 14248 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14249 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14250 } 14251 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14252 | EXT_CALL zend_jit_invalid_property_assign, r0 14253 if (RETURN_VALUE_USED(opline)) { 14254 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14255 } 14256 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14257 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14258 | jmp >7 14259 } else { 14260 | jmp >9 14261 } 14262 |.code 14263 } 14264 } 14265 | GET_ZVAL_PTR FCARG1a, op1_addr 14266 } 14267 14268 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14269 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 14270 if (prop_info) { 14271 ce = trace_ce; 14272 ce_is_instanceof = 0; 14273 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14274 if (on_this && JIT_G(current_frame) 14275 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 14276 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 14277 } else if (zend_jit_class_guard(Dst, opline, ce)) { 14278 if (on_this && JIT_G(current_frame)) { 14279 JIT_G(current_frame)->ce = ce; 14280 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 14281 } 14282 } else { 14283 return 0; 14284 } 14285 if (ssa->var_info && ssa_op->op1_use >= 0) { 14286 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14287 ssa->var_info[ssa_op->op1_use].ce = ce; 14288 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14289 } 14290 if (ssa->var_info && ssa_op->op1_def >= 0) { 14291 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14292 ssa->var_info[ssa_op->op1_def].ce = ce; 14293 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14294 } 14295 } 14296 } 14297 } 14298 14299 if (!prop_info) { 14300 needs_slow_path = 1; 14301 14302 | mov r0, EX->run_time_cache 14303 | mov r2, aword [r0 + opline->extended_value] 14304 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14305 | jne >5 14306 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14307 | mov FCARG2a, aword [r0 + opline->extended_value + sizeof(void*) * 2] 14308 } 14309 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14310 | test r0, r0 14311 | jl >5 14312 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14313 | add FCARG1a, r0 14314 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14315 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14316 | test FCARG2a, FCARG2a 14317 | jnz >1 14318 |.cold_code 14319 |1: 14320 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14321 | SET_EX_OPLINE opline, r0 14322 |.if X64 14323 | LOAD_ZVAL_ADDR CARG3, val_addr 14324 if (RETURN_VALUE_USED(opline)) { 14325 | LOAD_ZVAL_ADDR CARG4, res_addr 14326 } else { 14327 | xor CARG4, CARG4 14328 } 14329 |.else 14330 | sub r4, 8 14331 if (RETURN_VALUE_USED(opline)) { 14332 | PUSH_ZVAL_ADDR res_addr, r0 14333 } else { 14334 | push 0 14335 } 14336 | PUSH_ZVAL_ADDR val_addr, r0 14337 |.endif 14338 14339 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14340 14341 |.if not(X64) 14342 | add r4, 8 14343 |.endif 14344 14345 if ((opline+1)->op1_type == IS_CONST) { 14346 | // TODO: ??? 14347 | // if (Z_TYPE_P(value) == orig_type) { 14348 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14349 } 14350 14351 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14352 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14353 | jmp >7 14354 } else { 14355 | jmp >9 14356 } 14357 |.code 14358 } 14359 } else { 14360 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14361 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 14362 // Undefined property with magic __get()/__set() 14363 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14364 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14365 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14366 14367 if (!exit_addr) { 14368 return 0; 14369 } 14370 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14371 } else { 14372 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14373 needs_slow_path = 1; 14374 } 14375 } 14376 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14377 uint32_t info = val_info; 14378 14379 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14380 | SET_EX_OPLINE opline, r0 14381 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14382 | LOAD_ADDR FCARG2a, prop_info 14383 } else { 14384 int prop_info_offset = 14385 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14386 14387 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14388 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14389 | mov FCARG2a, aword[r0 + prop_info_offset] 14390 } 14391 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14392 |.if X64 14393 | LOAD_ZVAL_ADDR CARG3, val_addr 14394 if (RETURN_VALUE_USED(opline)) { 14395 | LOAD_ZVAL_ADDR CARG4, res_addr 14396 } else { 14397 | xor CARG4, CARG4 14398 } 14399 |.else 14400 | sub r4, 8 14401 if (RETURN_VALUE_USED(opline)) { 14402 | PUSH_ZVAL_ADDR res_addr, r0 14403 } else { 14404 | push 0 14405 } 14406 | PUSH_ZVAL_ADDR val_addr, r0 14407 |.endif 14408 14409 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14410 14411 |.if not(X64) 14412 | add r4, 8 14413 |.endif 14414 14415 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14416 info |= MAY_BE_RC1|MAY_BE_RCN; 14417 } 14418 14419 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14420 } 14421 } 14422 14423 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14424 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14425 if (opline->result_type == IS_UNUSED) { 14426 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14427 return 0; 14428 } 14429 } else { 14430 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14431 return 0; 14432 } 14433 } 14434 } 14435 14436 if (needs_slow_path) { 14437 |.cold_code 14438 |5: 14439 | SET_EX_OPLINE opline, r0 14440 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14441 | LOAD_ADDR FCARG2a, name 14442 |.if X64 14443 | LOAD_ZVAL_ADDR CARG3, val_addr 14444 | mov CARG4, EX->run_time_cache 14445 | add CARG4, opline->extended_value 14446 if (RETURN_VALUE_USED(opline)) { 14447 |.if X64WIN 14448 | LOAD_ZVAL_ADDR r0, res_addr 14449 | mov aword A5, r0 14450 |.else 14451 | LOAD_ZVAL_ADDR CARG5, res_addr 14452 |.endif 14453 } else { 14454 |.if X64WIN 14455 | mov aword A5, 0 14456 |.else 14457 | xor CARG5, CARG5 14458 |.endif 14459 } 14460 |.else 14461 | sub r4, 4 14462 if (RETURN_VALUE_USED(opline)) { 14463 | PUSH_ZVAL_ADDR res_addr, r0 14464 } else { 14465 | push 0 14466 } 14467 | mov r0, EX->run_time_cache 14468 | add r0, opline->extended_value 14469 | push r0 14470 | PUSH_ZVAL_ADDR val_addr, r0 14471 |.endif 14472 14473 | EXT_CALL zend_jit_assign_obj_helper, r0 14474 14475 |.if not(X64) 14476 | add r4, 4 14477 |.endif 14478 14479 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14480 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14481 } 14482 14483 |7: 14484 | // FREE_OP_DATA(); 14485 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14486 | jmp >9 14487 |.code 14488 } 14489 14490 |9: 14491 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14492 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14493 } 14494 14495 if (may_throw) { 14496 if (!zend_jit_check_exception(Dst)) { 14497 return 0; 14498 } 14499 } 14500 14501 return 1; 14502} 14503 14504static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14505{ 14506 zend_jit_addr op1_addr = OP1_ADDR(); 14507 14508 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14509 if (may_throw) { 14510 | SET_EX_OPLINE opline, r0 14511 } 14512 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14513 if (op1_info & MAY_BE_ARRAY) { 14514 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14515 } 14516 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14517 | cmp FCARG1d, -1 14518 | je >7 14519 | EXT_CALL zend_hash_iterator_del, r0 14520 |7: 14521 } 14522 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14523 if (may_throw) { 14524 if (!zend_jit_check_exception(Dst)) { 14525 return 0; 14526 } 14527 } 14528 } 14529 14530 return 1; 14531} 14532 14533static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14534{ 14535 if (opline->op1_type == IS_CONST) { 14536 zval *zv; 14537 size_t len; 14538 14539 zv = RT_CONSTANT(opline, opline->op1); 14540 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14541 len = Z_STRLEN_P(zv); 14542 14543 if (len > 0) { 14544 const char *str = Z_STRVAL_P(zv); 14545 14546 | SET_EX_OPLINE opline, r0 14547 |.if X64 14548 | LOAD_ADDR CARG1, str 14549 | LOAD_ADDR CARG2, len 14550 | EXT_CALL zend_write, r0 14551 |.else 14552 | mov aword A2, len 14553 | mov aword A1, str 14554 | EXT_CALL zend_write, r0 14555 |.endif 14556 if (!zend_jit_check_exception(Dst)) { 14557 return 0; 14558 } 14559 } 14560 } else { 14561 zend_jit_addr op1_addr = OP1_ADDR(); 14562 14563 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14564 14565 | SET_EX_OPLINE opline, r0 14566 | GET_ZVAL_PTR r0, op1_addr 14567 |.if X64 14568 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14569 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14570 | EXT_CALL zend_write, r0 14571 |.else 14572 | add r0, offsetof(zend_string, val) 14573 | mov aword A1, r0 14574 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14575 | mov aword A2, r0 14576 | EXT_CALL zend_write, r0 14577 |.endif 14578 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14579 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14580 } 14581 if (!zend_jit_check_exception(Dst)) { 14582 return 0; 14583 } 14584 } 14585 return 1; 14586} 14587 14588static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 14589{ 14590 if (opline->op1_type == IS_CONST) { 14591 zval *zv; 14592 size_t len; 14593 14594 zv = RT_CONSTANT(opline, opline->op1); 14595 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14596 len = Z_STRLEN_P(zv); 14597 14598 | SET_ZVAL_LVAL res_addr, len 14599 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14600 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14601 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14602 return 0; 14603 } 14604 } else { 14605 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14606 14607 if (Z_MODE(res_addr) == IS_REG) { 14608 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14609 | mov Ra(Z_REG(res_addr)), aword [Ra(Z_REG(res_addr))+offsetof(zend_string, len)] 14610 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14611 return 0; 14612 } 14613 } else { 14614 | GET_ZVAL_PTR r0, op1_addr 14615 | mov r0, aword [r0 + offsetof(zend_string, len)] 14616 | SET_ZVAL_LVAL res_addr, r0 14617 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14618 } 14619 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14620 } 14621 return 1; 14622} 14623 14624static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 14625{ 14626 if (opline->op1_type == IS_CONST) { 14627 zval *zv; 14628 zend_long count; 14629 14630 zv = RT_CONSTANT(opline, opline->op1); 14631 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 14632 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 14633 14634 | SET_ZVAL_LVAL res_addr, count 14635 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14636 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14637 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14638 return 0; 14639 } 14640 } else { 14641 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 14642 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 14643 14644 if (Z_MODE(res_addr) == IS_REG) { 14645 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14646 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14647 | mov Rd(Z_REG(res_addr)), dword [Ra(Z_REG(res_addr))+offsetof(HashTable, nNumOfElements)] 14648 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14649 return 0; 14650 } 14651 } else { 14652 | GET_ZVAL_PTR r0, op1_addr 14653 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14654 | mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)] 14655 | SET_ZVAL_LVAL res_addr, r0 14656 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14657 } 14658 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14659 } 14660 14661 if (may_throw) { 14662 return zend_jit_check_exception(Dst); 14663 } 14664 return 1; 14665} 14666 14667static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14668{ 14669 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14670 14671 | mov FCARG1a, aword EX->This.value.ptr 14672 | SET_ZVAL_PTR var_addr, FCARG1a 14673 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14674 | GC_ADDREF FCARG1a 14675 14676 return 1; 14677} 14678 14679static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 14680{ 14681 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 14682 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14683 if (!JIT_G(current_frame) || 14684 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14685 14686 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14687 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14688 14689 | cmp byte EX->This.u1.v.type, IS_OBJECT 14690 | jne &exit_addr 14691 14692 if (JIT_G(current_frame)) { 14693 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14694 } 14695 } 14696 } else { 14697 14698 | cmp byte EX->This.u1.v.type, IS_OBJECT 14699 | jne >1 14700 |.cold_code 14701 |1: 14702 | SET_EX_OPLINE opline, r0 14703 | jmp ->invalid_this 14704 |.code 14705 } 14706 } 14707 14708 if (!check_only) { 14709 if (!zend_jit_load_this(Dst, opline->result.var)) { 14710 return 0; 14711 } 14712 } 14713 14714 return 1; 14715} 14716 14717static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14718{ 14719 uint32_t count; 14720 Bucket *p; 14721 const zend_op *target; 14722 int b; 14723 int32_t exit_point; 14724 const void *exit_addr; 14725 14726 | test r0, r0 14727 if (default_label) { 14728 | jz &default_label 14729 } else if (next_opline) { 14730 | jz >3 14731 } else { 14732 | jz =>default_b 14733 } 14734 | LOAD_ADDR FCARG1a, jumptable 14735 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14736 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14737 |.if X64 14738 | cqo 14739 |.else 14740 | cdq 14741 |.endif 14742 | idiv FCARG1a 14743 |.if X64 14744 if (!IS_32BIT(dasm_end)) { 14745 | lea FCARG1a, aword [>4] 14746 | jmp aword [FCARG1a + r0] 14747 } else { 14748 | jmp aword [r0 + >4] 14749 } 14750 |.else 14751 | jmp aword [r0 + >4] 14752 |.endif 14753 |.jmp_table 14754 |.align aword 14755 |4: 14756 if (trace_info) { 14757 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14758 } 14759 14760 count = jumptable->nNumUsed; 14761 p = jumptable->arData; 14762 do { 14763 if (Z_TYPE(p->val) == IS_UNDEF) { 14764 if (default_label) { 14765 | .aword &default_label 14766 } else if (next_opline) { 14767 | .aword >3 14768 } else { 14769 | .aword =>default_b 14770 } 14771 } else { 14772 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14773 if (!next_opline) { 14774 b = ssa->cfg.map[target - op_array->opcodes]; 14775 | .aword =>b 14776 } else if (next_opline == target) { 14777 | .aword >3 14778 } else { 14779 exit_point = zend_jit_trace_get_exit_point(target, 0); 14780 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14781 | .aword &exit_addr 14782 } 14783 } 14784 p++; 14785 count--; 14786 } while (count); 14787 |.code 14788 14789 return 1; 14790} 14791 14792static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14793{ 14794 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14795 const zend_op *next_opline = NULL; 14796 14797 if (trace) { 14798 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14799 ZEND_ASSERT(trace->opline != NULL); 14800 next_opline = trace->opline; 14801 } 14802 14803 if (opline->op1_type == IS_CONST) { 14804 zval *zv = RT_CONSTANT(opline, opline->op1); 14805 zval *jump_zv = NULL; 14806 int b; 14807 14808 if (opline->opcode == ZEND_SWITCH_LONG) { 14809 if (Z_TYPE_P(zv) == IS_LONG) { 14810 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14811 } 14812 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14813 if (Z_TYPE_P(zv) == IS_STRING) { 14814 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14815 } 14816 } else if (opline->opcode == ZEND_MATCH) { 14817 if (Z_TYPE_P(zv) == IS_LONG) { 14818 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14819 } else if (Z_TYPE_P(zv) == IS_STRING) { 14820 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14821 } 14822 } else { 14823 ZEND_UNREACHABLE(); 14824 } 14825 if (next_opline) { 14826 const zend_op *target; 14827 14828 if (jump_zv != NULL) { 14829 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14830 } else { 14831 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14832 } 14833 ZEND_ASSERT(target == next_opline); 14834 } else { 14835 if (jump_zv != NULL) { 14836 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14837 } else { 14838 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14839 } 14840 | jmp =>b 14841 } 14842 } else { 14843 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14844 uint32_t op1_info = OP1_INFO(); 14845 zend_jit_addr op1_addr = OP1_ADDR(); 14846 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14847 const zend_op *target; 14848 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14849 int b; 14850 int32_t exit_point; 14851 const void *fallback_label = NULL; 14852 const void *default_label = NULL; 14853 const void *exit_addr; 14854 14855 if (next_opline) { 14856 if (next_opline != opline + 1) { 14857 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14858 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14859 } 14860 if (next_opline != default_opline) { 14861 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14862 default_label = zend_jit_trace_get_exit_addr(exit_point); 14863 } 14864 } 14865 14866 if (opline->opcode == ZEND_SWITCH_LONG) { 14867 if (op1_info & MAY_BE_LONG) { 14868 if (op1_info & MAY_BE_REF) { 14869 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 14870 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 14871 |.cold_code 14872 |1: 14873 | // ZVAL_DEREF(op) 14874 if (fallback_label) { 14875 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14876 } else { 14877 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14878 } 14879 | GET_ZVAL_PTR FCARG2a, op1_addr 14880 if (fallback_label) { 14881 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 14882 } else { 14883 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 14884 } 14885 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 14886 | jmp >2 14887 |.code 14888 |2: 14889 } else { 14890 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14891 if (fallback_label) { 14892 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 14893 } else { 14894 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14895 } 14896 } 14897 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 14898 } 14899 if (HT_IS_PACKED(jumptable)) { 14900 uint32_t count = jumptable->nNumUsed; 14901 Bucket *p = jumptable->arData; 14902 14903 | cmp FCARG2a, jumptable->nNumUsed 14904 if (default_label) { 14905 | jae &default_label 14906 } else if (next_opline) { 14907 | jae >3 14908 } else { 14909 | jae =>default_b 14910 } 14911 |.if X64 14912 if (!IS_32BIT(dasm_end)) { 14913 | lea r0, aword [>4] 14914 | jmp aword [r0 + FCARG2a * 8] 14915 } else { 14916 | jmp aword [FCARG2a * 8 + >4] 14917 } 14918 |.else 14919 | jmp aword [FCARG2a * 4 + >4] 14920 |.endif 14921 |.jmp_table 14922 |.align aword 14923 |4: 14924 if (trace_info) { 14925 trace_info->jmp_table_size += count; 14926 } 14927 p = jumptable->arData; 14928 do { 14929 if (Z_TYPE(p->val) == IS_UNDEF) { 14930 if (default_label) { 14931 | .aword &default_label 14932 } else if (next_opline) { 14933 | .aword >3 14934 } else { 14935 | .aword =>default_b 14936 } 14937 } else { 14938 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14939 if (!next_opline) { 14940 b = ssa->cfg.map[target - op_array->opcodes]; 14941 | .aword =>b 14942 } else if (next_opline == target) { 14943 | .aword >3 14944 } else { 14945 exit_point = zend_jit_trace_get_exit_point(target, 0); 14946 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14947 | .aword &exit_addr 14948 } 14949 } 14950 p++; 14951 count--; 14952 } while (count); 14953 |.code 14954 |3: 14955 } else { 14956 | LOAD_ADDR FCARG1a, jumptable 14957 | EXT_CALL zend_hash_index_find, r0 14958 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14959 return 0; 14960 } 14961 |3: 14962 } 14963 } 14964 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14965 if (op1_info & MAY_BE_STRING) { 14966 if (op1_info & MAY_BE_REF) { 14967 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 14968 | GET_ZVAL_PTR FCARG2a, op1_addr 14969 |.cold_code 14970 |1: 14971 | // ZVAL_DEREF(op) 14972 if (fallback_label) { 14973 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14974 } else { 14975 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14976 } 14977 | GET_ZVAL_PTR FCARG2a, op1_addr 14978 if (fallback_label) { 14979 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 14980 } else { 14981 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 14982 } 14983 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 14984 | jmp >2 14985 |.code 14986 |2: 14987 } else { 14988 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14989 if (fallback_label) { 14990 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 14991 } else { 14992 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 14993 } 14994 } 14995 | GET_ZVAL_PTR FCARG2a, op1_addr 14996 } 14997 | LOAD_ADDR FCARG1a, jumptable 14998 | EXT_CALL zend_hash_find, r0 14999 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15000 return 0; 15001 } 15002 |3: 15003 } 15004 } else if (opline->opcode == ZEND_MATCH) { 15005 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 15006 if (op1_info & MAY_BE_REF) { 15007 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 15008 | ZVAL_DEREF FCARG2a, op1_info 15009 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15010 } 15011 | LOAD_ADDR FCARG1a, jumptable 15012 if (op1_info & MAY_BE_LONG) { 15013 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15014 if (op1_info & MAY_BE_STRING) { 15015 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 15016 } else if (op1_info & MAY_BE_UNDEF) { 15017 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 15018 } else if (default_label) { 15019 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 15020 } else if (next_opline) { 15021 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15022 } else { 15023 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 15024 } 15025 } 15026 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15027 | EXT_CALL zend_hash_index_find, r0 15028 if (op1_info & MAY_BE_STRING) { 15029 | jmp >2 15030 } 15031 } 15032 if (op1_info & MAY_BE_STRING) { 15033 |5: 15034 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 15035 if (op1_info & MAY_BE_UNDEF) { 15036 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 15037 } else if (default_label) { 15038 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 15039 } else if (next_opline) { 15040 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15041 } else { 15042 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 15043 } 15044 } 15045 | GET_ZVAL_PTR FCARG2a, op1_addr 15046 | EXT_CALL zend_hash_find, r0 15047 } 15048 |2: 15049 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15050 return 0; 15051 } 15052 } 15053 if (op1_info & MAY_BE_UNDEF) { 15054 |6: 15055 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 15056 if (default_label) { 15057 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 15058 } else if (next_opline) { 15059 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 15060 } else { 15061 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 15062 } 15063 } 15064 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 15065 | SET_EX_OPLINE opline, r0 15066 | mov FCARG1d, opline->op1.var 15067 | EXT_CALL zend_jit_undefined_op_helper, r0 15068 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 15069 return 0; 15070 } 15071 } 15072 if (default_label) { 15073 | jmp &default_label 15074 } else if (next_opline) { 15075 | jmp >3 15076 } else { 15077 | jmp =>default_b 15078 } 15079 |3: 15080 } else { 15081 ZEND_UNREACHABLE(); 15082 } 15083 } 15084 return 1; 15085} 15086 15087static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 15088{ 15089 zend_arg_info *arg_info = &op_array->arg_info[-1]; 15090 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 15091 zend_jit_addr op1_addr = OP1_ADDR(); 15092 bool needs_slow_check = 1; 15093 bool slow_check_in_cold = 1; 15094 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 15095 15096 if (type_mask == 0) { 15097 slow_check_in_cold = 0; 15098 } else { 15099 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 15100 slow_check_in_cold = 0; 15101 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 15102 needs_slow_check = 0; 15103 } else if (is_power_of_two(type_mask)) { 15104 uint32_t type_code = concrete_type(type_mask); 15105 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 15106 } else { 15107 | mov edx, 1 15108 | GET_ZVAL_TYPE cl, op1_addr 15109 | shl edx, cl 15110 | test edx, type_mask 15111 | je >6 15112 } 15113 } 15114 if (needs_slow_check) { 15115 if (slow_check_in_cold) { 15116 |.cold_code 15117 |6: 15118 } 15119 | SET_EX_OPLINE opline, r1 15120 if (op1_info & MAY_BE_UNDEF) { 15121 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15122 | mov FCARG1a, opline->op1.var 15123 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15124 | test r0, r0 15125 | jz ->exception_handler 15126 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15127 | jmp >8 15128 } 15129 |7: 15130 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15131 |8: 15132 | mov FCARG2a, EX->func 15133 |.if X64 15134 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15135 | mov r0, EX->run_time_cache 15136 | lea CARG4, aword [r0+opline->op2.num] 15137 | EXT_CALL zend_jit_verify_return_slow, r0 15138 |.else 15139 | sub r4, 8 15140 | mov r0, EX->run_time_cache 15141 | add r0, opline->op2.num 15142 | push r0 15143 | push (ptrdiff_t)arg_info 15144 | EXT_CALL zend_jit_verify_return_slow, r0 15145 | add r4, 8 15146 |.endif 15147 if (!zend_jit_check_exception(Dst)) { 15148 return 0; 15149 } 15150 if (slow_check_in_cold) { 15151 | jmp >9 15152 |.code 15153 } 15154 } 15155 |9: 15156 return 1; 15157} 15158 15159static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15160{ 15161 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15162 15163 // TODO: support for empty() ??? 15164 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15165 15166 if (op1_info & MAY_BE_REF) { 15167 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 15168 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15169 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15170 } 15171 | ZVAL_DEREF FCARG1a, op1_info 15172 |1: 15173 } 15174 15175 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15176 if (exit_addr) { 15177 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15178 } else if (smart_branch_opcode) { 15179 if (smart_branch_opcode == ZEND_JMPNZ) { 15180 | jmp =>target_label 15181 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15182 | jmp =>target_label2 15183 } 15184 } else { 15185 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15186 } 15187 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15188 if (exit_addr) { 15189 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15190 } else if (smart_branch_opcode) { 15191 if (smart_branch_opcode != ZEND_JMPNZ) { 15192 | jmp =>target_label 15193 } 15194 } else { 15195 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15196 } 15197 } else { 15198 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15199 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15200 if (exit_addr) { 15201 if (smart_branch_opcode == ZEND_JMPNZ) { 15202 | jg &exit_addr 15203 } else { 15204 | jle &exit_addr 15205 } 15206 } else if (smart_branch_opcode) { 15207 if (smart_branch_opcode == ZEND_JMPZ) { 15208 | jle =>target_label 15209 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15210 | jg =>target_label 15211 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15212 | jle =>target_label 15213 | jmp =>target_label2 15214 } else { 15215 ZEND_UNREACHABLE(); 15216 } 15217 } else { 15218 | setg al 15219 | movzx eax, al 15220 | lea eax, [eax + IS_FALSE] 15221 | SET_ZVAL_TYPE_INFO res_addr, eax 15222 } 15223 } 15224 15225 return 1; 15226} 15227 15228static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15229{ 15230 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15231 15232 if (opline->op1_type == IS_CONST) { 15233 zval *zv = RT_CONSTANT(opline, opline->op1); 15234 15235 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15236 if (Z_REFCOUNTED_P(zv)) { 15237 | ADDREF_CONST zv, r0 15238 } 15239 } else { 15240 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15241 15242 | // ZVAL_COPY(res, value); 15243 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1 15244 if (opline->op1_type == IS_CV) { 15245 | TRY_ADDREF op1_info, ah, FCARG1a 15246 } 15247 } 15248 | // Z_FE_POS_P(res) = 0; 15249 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15250 15251 return 1; 15252} 15253 15254static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 15255{ 15256 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15257 15258 | // array = EX_VAR(opline->op1.var); 15259 | // fe_ht = Z_ARRVAL_P(array); 15260 | GET_ZVAL_PTR FCARG1a, op1_addr 15261 | // pos = Z_FE_POS_P(array); 15262 | mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15263 | // p = fe_ht->arData + pos; 15264 |.if X64 15265 || ZEND_ASSERT(sizeof(Bucket) == 32); 15266 | mov FCARG2d, eax 15267 | shl FCARG2a, 5 15268 |.else 15269 | imul FCARG2a, r0, sizeof(Bucket) 15270 |.endif 15271 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)] 15272 |1: 15273 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15274 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15275 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15276 | // ZEND_VM_CONTINUE(); 15277 if (exit_addr) { 15278 if (exit_opcode == ZEND_JMP) { 15279 | jbe &exit_addr 15280 } else { 15281 | jbe >3 15282 } 15283 } else { 15284 | jbe =>target_label 15285 } 15286 | // pos++; 15287 | add eax, 1 15288 | // value_type = Z_TYPE_INFO_P(value); 15289 | // if (EXPECTED(value_type != IS_UNDEF)) { 15290 if (!exit_addr || exit_opcode == ZEND_JMP) { 15291 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3 15292 } else { 15293 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15294 } 15295 | // p++; 15296 | add FCARG2a, sizeof(Bucket) 15297 | jmp <1 15298 |3: 15299 15300 if (!exit_addr || exit_opcode == ZEND_JMP) { 15301 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15302 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15303 uint32_t val_info; 15304 15305 | // Z_FE_POS_P(array) = pos + 1; 15306 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15307 15308 if (RETURN_VALUE_USED(opline)) { 15309 zend_jit_addr res_addr = RES_ADDR(); 15310 15311 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15312 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15313 | // if (!p->key) { 15314 | cmp aword [FCARG2a + offsetof(Bucket, key)], 0 15315 | jz >2 15316 } 15317 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15318 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15319 | mov r0, aword [FCARG2a + offsetof(Bucket, key)] 15320 | SET_ZVAL_PTR res_addr, r0 15321 | test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15322 | jz >1 15323 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15324 | jmp >3 15325 |1: 15326 | GC_ADDREF r0 15327 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15328 15329 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15330 | jmp >3 15331 |2: 15332 } 15333 } 15334 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15335 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15336 | mov r0, aword [FCARG2a + offsetof(Bucket, h)] 15337 | SET_ZVAL_LVAL res_addr, r0 15338 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15339 } 15340 |3: 15341 } 15342 15343 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15344 if (val_info & MAY_BE_ARRAY) { 15345 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15346 } 15347 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15348 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15349 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15350 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15351 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15352 } 15353 15354 if (opline->op2_type == IS_CV) { 15355 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15356 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15357 return 0; 15358 } 15359 } else { 15360 | // ZVAL_COPY(res, value); 15361 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1 15362 | TRY_ADDREF val_info, ah, FCARG1a 15363 } 15364 } 15365 15366 return 1; 15367} 15368 15369static int zend_jit_fetch_constant(dasm_State **Dst, 15370 const zend_op *opline, 15371 const zend_op_array *op_array, 15372 zend_ssa *ssa, 15373 const zend_ssa_op *ssa_op, 15374 zend_jit_addr res_addr) 15375{ 15376 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15377 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15378 uint32_t res_info = RES_INFO(); 15379 15380 | // c = CACHED_PTR(opline->extended_value); 15381 | mov FCARG1a, EX->run_time_cache 15382 | mov r0, aword [FCARG1a + opline->extended_value] 15383 | // if (c != NULL) 15384 | test r0, r0 15385 | jz >9 15386 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 15387 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15388 | test r0, CACHE_SPECIAL 15389 | jnz >9 15390 } 15391 |8: 15392 15393 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15394 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15395 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15396 int32_t exit_point; 15397 const void *exit_addr = NULL; 15398 15399 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15400 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 15401 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15402 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15403 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15404 if (!exit_addr) { 15405 return 0; 15406 } 15407 res_info &= ~MAY_BE_GUARD; 15408 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15409 15410 zend_uchar type = concrete_type(res_info); 15411 15412 if (type < IS_STRING) { 15413 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15414 } else { 15415 | GET_ZVAL_TYPE_INFO edx, const_addr 15416 | IF_NOT_TYPE dl, type, &exit_addr 15417 } 15418 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15419 if (type < IS_STRING) { 15420 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 15421 | SET_ZVAL_TYPE_INFO res_addr, type 15422 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 15423 return 0; 15424 } 15425 } else { 15426 | SET_ZVAL_TYPE_INFO res_addr, edx 15427 | TRY_ADDREF res_info, dh, r1 15428 } 15429 } else { 15430 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15431 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15432 | TRY_ADDREF MAY_BE_ANY, ah, r1 15433 } 15434 15435 |.cold_code 15436 |9: 15437 | // SAVE_OPLINE(); 15438 | SET_EX_OPLINE opline, r0 15439 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15440 | LOAD_ADDR FCARG1a, zv 15441 | mov FCARG2a, opline->op1.num 15442 | EXT_CALL zend_jit_get_constant, r0 15443 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15444 | test r0, r0 15445 | jnz <8 15446 | jmp ->exception_handler 15447 |.code 15448 15449 return 1; 15450} 15451 15452static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15453{ 15454 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15455 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15456 15457 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15458 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15459 15460 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15461 | LOAD_ADDR FCARG1a, ht 15462 if (opline->op1_type != IS_CONST) { 15463 | GET_ZVAL_PTR FCARG2a, op1_addr 15464 | EXT_CALL zend_hash_find, r0 15465 } else { 15466 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15467 | LOAD_ADDR FCARG2a, str 15468 | EXT_CALL zend_hash_find_known_hash, r0 15469 } 15470 | test r0, r0 15471 if (exit_addr) { 15472 if (smart_branch_opcode == ZEND_JMPZ) { 15473 | jz &exit_addr 15474 } else { 15475 | jnz &exit_addr 15476 } 15477 } else if (smart_branch_opcode) { 15478 if (smart_branch_opcode == ZEND_JMPZ) { 15479 | jz =>target_label 15480 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15481 | jnz =>target_label 15482 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15483 | jz =>target_label 15484 | jmp =>target_label2 15485 } else { 15486 ZEND_UNREACHABLE(); 15487 } 15488 } else { 15489 | setnz al 15490 | movzx eax, al 15491 | lea eax, [eax + IS_FALSE] 15492 | SET_ZVAL_TYPE_INFO res_addr, eax 15493 } 15494 15495 return 1; 15496} 15497 15498static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 15499{ 15500 uint32_t offset; 15501 15502 offset = (opline->opcode == ZEND_ROPE_INIT) ? 15503 opline->result.var : 15504 opline->op1.var + opline->extended_value * sizeof(zend_string*); 15505 15506 if (opline->op2_type == IS_CONST) { 15507 zval *zv = RT_CONSTANT(opline, opline->op2); 15508 zend_string *str; 15509 15510 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 15511 str = Z_STR_P(zv); 15512 | ADDR_STORE aword [FP + offset], str, r0 15513 } else { 15514 zend_jit_addr op2_addr = OP2_ADDR(); 15515 15516 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 15517 15518 | GET_ZVAL_PTR r1, op2_addr 15519 | mov aword [FP + offset], r1 15520 if (opline->op2_type == IS_CV) { 15521 | GET_ZVAL_TYPE_INFO eax, op2_addr 15522 | TRY_ADDREF op2_info, ah, r1 15523 } 15524 } 15525 15526 if (opline->opcode == ZEND_ROPE_END) { 15527 zend_jit_addr res_addr = RES_ADDR(); 15528 15529 | lea FCARG1a, [FP + opline->op1.var] 15530 | mov FCARG2d, opline->extended_value 15531 | EXT_CALL zend_jit_rope_end, r0 15532 | SET_ZVAL_PTR res_addr, r0 15533 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15534 } 15535 15536 return 1; 15537} 15538 15539static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15540{ 15541 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15542 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15543 15544 if (!exit_addr) { 15545 return 0; 15546 } 15547 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15548 15549 return 1; 15550} 15551 15552static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 15553{ 15554 zend_jit_addr var_addr = *var_addr_ptr; 15555 uint32_t var_info = *var_info_ptr; 15556 const void *exit_addr = NULL; 15557 15558 if (add_ref_guard || add_type_guard) { 15559 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15560 15561 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15562 if (!exit_addr) { 15563 return 0; 15564 } 15565 } 15566 15567 if (add_ref_guard) { 15568 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15569 } 15570 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15571 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15572 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 15573 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15574 } 15575 | EXT_CALL zend_jit_unref_helper, r0 15576 } else { 15577 | GET_ZVAL_PTR FCARG1a, var_addr 15578 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 15579 *var_addr_ptr = var_addr; 15580 } 15581 15582 if (var_type != IS_UNKNOWN) { 15583 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15584 } 15585 if (add_type_guard 15586 && var_type != IS_UNKNOWN 15587 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15588 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15589 15590 ZEND_ASSERT(var_info & (1 << var_type)); 15591 if (var_type < IS_STRING) { 15592 var_info = (1 << var_type); 15593 } else if (var_type != IS_ARRAY) { 15594 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15595 } else { 15596 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15597 } 15598 15599 *var_info_ptr = var_info; 15600 } else { 15601 var_info &= ~MAY_BE_REF; 15602 *var_info_ptr = var_info; 15603 } 15604 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15605 15606 return 1; 15607} 15608 15609static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 15610{ 15611 zend_jit_addr var_addr = *var_addr_ptr; 15612 uint32_t var_info = *var_info_ptr; 15613 int32_t exit_point; 15614 const void *exit_addr; 15615 15616 if (add_indirect_guard) { 15617 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15618 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15619 15620 if (!exit_addr) { 15621 return 0; 15622 } 15623 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15624 | GET_ZVAL_PTR FCARG1a, var_addr 15625 } else { 15626 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 15627 if (opline->op1_type != IS_VAR || 15628 (opline-1)->result_type != IS_VAR || 15629 (opline-1)->result.var != opline->op1.var || 15630 (opline-1)->op2_type == IS_VAR || 15631 (opline-1)->op2_type == IS_TMP_VAR) { 15632 | GET_ZVAL_PTR FCARG1a, var_addr 15633 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15634 | mov FCARG1a, r0 15635 } 15636 } 15637 *var_info_ptr &= ~MAY_BE_INDIRECT; 15638 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15639 *var_addr_ptr = var_addr; 15640 15641 if (var_type != IS_UNKNOWN) { 15642 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15643 } 15644 if (!(var_type & IS_TRACE_REFERENCE) 15645 && var_type != IS_UNKNOWN 15646 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15647 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15648 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15649 15650 if (!exit_addr) { 15651 return 0; 15652 } 15653 15654 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15655 15656 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15657 ZEND_ASSERT(var_info & (1 << var_type)); 15658 if (var_type < IS_STRING) { 15659 var_info = (1 << var_type); 15660 } else if (var_type != IS_ARRAY) { 15661 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15662 } else { 15663 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15664 } 15665 15666 *var_info_ptr = var_info; 15667 } 15668 15669 return 1; 15670} 15671 15672static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15673{ 15674 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15675 return 0; 15676 } 15677 15678 switch (opline->opcode) { 15679 case ZEND_QM_ASSIGN: 15680 case ZEND_SEND_VAR: 15681 case ZEND_ASSIGN: 15682 case ZEND_PRE_INC: 15683 case ZEND_PRE_DEC: 15684 case ZEND_POST_INC: 15685 case ZEND_POST_DEC: 15686 return 1; 15687 case ZEND_ADD: 15688 case ZEND_SUB: 15689 case ZEND_MUL: 15690 case ZEND_BW_OR: 15691 case ZEND_BW_AND: 15692 case ZEND_BW_XOR: 15693 if (def_var == ssa_op->result_def && 15694 use_var == ssa_op->op1_use) { 15695 return 1; 15696 } 15697 break; 15698 default: 15699 break; 15700 } 15701 return 0; 15702} 15703 15704static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15705{ 15706 uint32_t op1_info, op2_info; 15707 15708 switch (opline->opcode) { 15709 case ZEND_SEND_VAR: 15710 case ZEND_SEND_VAL: 15711 case ZEND_SEND_VAL_EX: 15712 return (opline->op2_type != IS_CONST); 15713 case ZEND_QM_ASSIGN: 15714 case ZEND_IS_SMALLER: 15715 case ZEND_IS_SMALLER_OR_EQUAL: 15716 case ZEND_IS_EQUAL: 15717 case ZEND_IS_NOT_EQUAL: 15718 case ZEND_IS_IDENTICAL: 15719 case ZEND_IS_NOT_IDENTICAL: 15720 case ZEND_CASE: 15721 return 1; 15722 case ZEND_RETURN: 15723 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15724 case ZEND_ASSIGN: 15725 op1_info = OP1_INFO(); 15726 op2_info = OP2_INFO(); 15727 return 15728 opline->op1_type == IS_CV && 15729 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15730 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15731 case ZEND_ADD: 15732 case ZEND_SUB: 15733 case ZEND_MUL: 15734 op1_info = OP1_INFO(); 15735 op2_info = OP2_INFO(); 15736 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15737 case ZEND_BW_OR: 15738 case ZEND_BW_AND: 15739 case ZEND_BW_XOR: 15740 case ZEND_SL: 15741 case ZEND_SR: 15742 case ZEND_MOD: 15743 op1_info = OP1_INFO(); 15744 op2_info = OP2_INFO(); 15745 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15746 case ZEND_PRE_INC: 15747 case ZEND_PRE_DEC: 15748 case ZEND_POST_INC: 15749 case ZEND_POST_DEC: 15750 op1_info = OP1_INFO(); 15751 op2_info = OP1_DEF_INFO(); 15752 return opline->op1_type == IS_CV 15753 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 15754 && (op2_info & MAY_BE_LONG); 15755 case ZEND_STRLEN: 15756 op1_info = OP1_INFO(); 15757 return (opline->op1_type & (IS_CV|IS_CONST)) 15758 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15759 case ZEND_COUNT: 15760 op1_info = OP1_INFO(); 15761 return (opline->op1_type & (IS_CV|IS_CONST)) 15762 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15763 case ZEND_JMPZ: 15764 case ZEND_JMPNZ: 15765 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15766 if (!ssa->cfg.map) { 15767 return 0; 15768 } 15769 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15770 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15771 return 0; 15772 } 15773 } 15774 ZEND_FALLTHROUGH; 15775 case ZEND_BOOL: 15776 case ZEND_BOOL_NOT: 15777 case ZEND_JMPZNZ: 15778 case ZEND_JMPZ_EX: 15779 case ZEND_JMPNZ_EX: 15780 return 1; 15781 case ZEND_FETCH_CONSTANT: 15782 return 1; 15783 case ZEND_FETCH_DIM_R: 15784 op1_info = OP1_INFO(); 15785 op2_info = OP2_INFO(); 15786 if (trace 15787 && trace->op1_type != IS_UNKNOWN 15788 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15789 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15790 } 15791 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15792 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15793 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15794 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15795 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15796 } 15797 return 0; 15798} 15799 15800static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15801{ 15802 if (ssa->vars[var].no_val) { 15803 /* we don't need the value */ 15804 return 0; 15805 } 15806 15807 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15808 /* Disable global register allocation, 15809 * register allocation for SSA variables connected through Phi functions 15810 */ 15811 if (ssa->vars[var].definition_phi) { 15812 return 0; 15813 } 15814 if (ssa->vars[var].phi_use_chain) { 15815 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15816 do { 15817 if (!ssa->vars[phi->ssa_var].no_val) { 15818 return 0; 15819 } 15820 phi = zend_ssa_next_use_phi(ssa, var, phi); 15821 } while (phi); 15822 } 15823 } 15824 15825 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15826 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15827 /* bad type */ 15828 return 0; 15829 } 15830 15831 return 1; 15832} 15833 15834static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15835{ 15836 if (!zend_jit_var_supports_reg(ssa, var)) { 15837 return 0; 15838 } 15839 15840 if (ssa->vars[var].definition >= 0) { 15841 uint32_t def = ssa->vars[var].definition; 15842 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15843 return 0; 15844 } 15845 } 15846 15847 if (ssa->vars[var].use_chain >= 0) { 15848 int use = ssa->vars[var].use_chain; 15849 15850 do { 15851 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15852 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15853 return 0; 15854 } 15855 use = zend_ssa_next_use(ssa->ops, var, use); 15856 } while (use >= 0); 15857 } 15858 15859 return 1; 15860} 15861 15862static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) 15863{ 15864|.if X64 15865|| if (op_type == IS_CONST) { 15866|| zval *zv = RT_CONSTANT(opline, op); 15867|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 15868|| return 1; 15869|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 15870|| return 1; 15871|| } 15872|| } 15873|.endif 15874 return 0; 15875} 15876 15877static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15878{ 15879 uint32_t op1_info, op2_info; 15880 15881 switch (opline->opcode) { 15882 case ZEND_FETCH_DIM_R: 15883 op1_info = OP1_INFO(); 15884 op2_info = OP2_INFO(); 15885 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15886 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15887 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15888 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15889 return ZEND_REGSET(ZREG_FCARG1); 15890 } 15891 break; 15892 default: 15893 break; 15894 } 15895 15896 return ZEND_REGSET_EMPTY; 15897} 15898 15899static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15900{ 15901 uint32_t op1_info, op2_info, res_info; 15902 zend_regset regset = ZEND_REGSET_SCRATCH; 15903 15904 switch (opline->opcode) { 15905 case ZEND_NOP: 15906 case ZEND_OP_DATA: 15907 case ZEND_JMP: 15908 case ZEND_RETURN: 15909 regset = ZEND_REGSET_EMPTY; 15910 break; 15911 case ZEND_QM_ASSIGN: 15912 if (ssa_op->op1_def == current_var || 15913 ssa_op->result_def == current_var) { 15914 regset = ZEND_REGSET_EMPTY; 15915 break; 15916 } 15917 /* break missing intentionally */ 15918 case ZEND_SEND_VAL: 15919 case ZEND_SEND_VAL_EX: 15920 if (opline->op2_type == IS_CONST) { 15921 break; 15922 } 15923 if (ssa_op->op1_use == current_var) { 15924 regset = ZEND_REGSET(ZREG_R0); 15925 break; 15926 } 15927 op1_info = OP1_INFO(); 15928 if (!(op1_info & MAY_BE_UNDEF)) { 15929 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15930 regset = ZEND_REGSET(ZREG_XMM0); 15931 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15932 regset = ZEND_REGSET(ZREG_R0); 15933 } else { 15934 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15935 } 15936 } 15937 break; 15938 case ZEND_SEND_VAR: 15939 if (opline->op2_type == IS_CONST) { 15940 break; 15941 } 15942 if (ssa_op->op1_use == current_var || 15943 ssa_op->op1_def == current_var) { 15944 regset = ZEND_REGSET_EMPTY; 15945 break; 15946 } 15947 op1_info = OP1_INFO(); 15948 if (!(op1_info & MAY_BE_UNDEF)) { 15949 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15950 regset = ZEND_REGSET(ZREG_XMM0); 15951 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15952 } else { 15953 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15954 if (op1_info & MAY_BE_REF) { 15955 ZEND_REGSET_INCL(regset, ZREG_R1); 15956 } 15957 } 15958 } 15959 break; 15960 case ZEND_ASSIGN: 15961 if (ssa_op->op2_use == current_var || 15962 ssa_op->op2_def == current_var || 15963 ssa_op->op1_def == current_var || 15964 ssa_op->result_def == current_var) { 15965 regset = ZEND_REGSET_EMPTY; 15966 break; 15967 } 15968 op1_info = OP1_INFO(); 15969 op2_info = OP2_INFO(); 15970 if (opline->op1_type == IS_CV 15971 && !(op2_info & MAY_BE_UNDEF) 15972 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15973 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15974 regset = ZEND_REGSET(ZREG_XMM0); 15975 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15976 regset = ZEND_REGSET(ZREG_R0); 15977 } else { 15978 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15979 } 15980 } 15981 break; 15982 case ZEND_PRE_INC: 15983 case ZEND_PRE_DEC: 15984 case ZEND_POST_INC: 15985 case ZEND_POST_DEC: 15986 if (ssa_op->op1_use == current_var || 15987 ssa_op->op1_def == current_var || 15988 ssa_op->result_def == current_var) { 15989 regset = ZEND_REGSET_EMPTY; 15990 break; 15991 } 15992 op1_info = OP1_INFO(); 15993 if (opline->op1_type == IS_CV 15994 && (op1_info & MAY_BE_LONG) 15995 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15996 regset = ZEND_REGSET_EMPTY; 15997 if (op1_info & MAY_BE_DOUBLE) { 15998 regset = ZEND_REGSET(ZREG_XMM0); 15999 } 16000 } 16001 break; 16002 case ZEND_ADD: 16003 case ZEND_SUB: 16004 case ZEND_MUL: 16005 op1_info = OP1_INFO(); 16006 op2_info = OP2_INFO(); 16007 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16008 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16009 16010 regset = ZEND_REGSET_EMPTY; 16011 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 16012 if (ssa_op->result_def != current_var && 16013 (ssa_op->op1_use != current_var || !last_use)) { 16014 ZEND_REGSET_INCL(regset, ZREG_R0); 16015 } 16016 res_info = RES_INFO(); 16017 if (res_info & MAY_BE_DOUBLE) { 16018 ZEND_REGSET_INCL(regset, ZREG_R0); 16019 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16020 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16021 } else if (res_info & MAY_BE_GUARD) { 16022 ZEND_REGSET_INCL(regset, ZREG_R0); 16023 } 16024 } 16025 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16026 if (ssa_op->result_def != current_var) { 16027 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16028 } 16029 } 16030 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16031 if (zend_is_commutative(opline->opcode)) { 16032 if (ssa_op->result_def != current_var) { 16033 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16034 } 16035 } else { 16036 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16037 if (ssa_op->result_def != current_var && 16038 (ssa_op->op1_use != current_var || !last_use)) { 16039 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16040 } 16041 } 16042 } 16043 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16044 if (ssa_op->result_def != current_var && 16045 (ssa_op->op1_use != current_var || !last_use) && 16046 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 16047 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16048 } 16049 } 16050 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16051 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16052 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16053 ZEND_REGSET_INCL(regset, ZREG_R0); 16054 } else { 16055 ZEND_REGSET_INCL(regset, ZREG_R1); 16056 } 16057 } 16058 } 16059 break; 16060 case ZEND_BW_OR: 16061 case ZEND_BW_AND: 16062 case ZEND_BW_XOR: 16063 op1_info = OP1_INFO(); 16064 op2_info = OP2_INFO(); 16065 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16066 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16067 regset = ZEND_REGSET_EMPTY; 16068 if (ssa_op->result_def != current_var && 16069 (ssa_op->op1_use != current_var || !last_use)) { 16070 ZEND_REGSET_INCL(regset, ZREG_R0); 16071 } 16072 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16073 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16074 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16075 ZEND_REGSET_INCL(regset, ZREG_R0); 16076 } else { 16077 ZEND_REGSET_INCL(regset, ZREG_R1); 16078 } 16079 } 16080 } 16081 break; 16082 case ZEND_SL: 16083 case ZEND_SR: 16084 op1_info = OP1_INFO(); 16085 op2_info = OP2_INFO(); 16086 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16087 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16088bw_op: 16089 regset = ZEND_REGSET_EMPTY; 16090 if (ssa_op->result_def != current_var && 16091 (ssa_op->op1_use != current_var || !last_use)) { 16092 ZEND_REGSET_INCL(regset, ZREG_R0); 16093 } 16094 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 16095 ZEND_REGSET_INCL(regset, ZREG_R1); 16096 } 16097 } 16098 break; 16099 case ZEND_MOD: 16100 op1_info = OP1_INFO(); 16101 op2_info = OP2_INFO(); 16102 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16103 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16104 if (opline->op2_type == IS_CONST && 16105 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 16106 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 16107 OP1_HAS_RANGE() && 16108 OP1_MIN_RANGE() >= 0) { 16109 /* MOD is going to be optimized into AND */ 16110 goto bw_op; 16111 } else { 16112 regset = ZEND_REGSET_EMPTY; 16113 ZEND_REGSET_INCL(regset, ZREG_R0); 16114 ZEND_REGSET_INCL(regset, ZREG_R2); 16115 if (opline->op2_type == IS_CONST) { 16116 ZEND_REGSET_INCL(regset, ZREG_R1); 16117 } 16118 } 16119 } 16120 break; 16121 case ZEND_IS_SMALLER: 16122 case ZEND_IS_SMALLER_OR_EQUAL: 16123 case ZEND_IS_EQUAL: 16124 case ZEND_IS_NOT_EQUAL: 16125 case ZEND_IS_IDENTICAL: 16126 case ZEND_IS_NOT_IDENTICAL: 16127 case ZEND_CASE: 16128 op1_info = OP1_INFO(); 16129 op2_info = OP2_INFO(); 16130 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16131 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16132 regset = ZEND_REGSET_EMPTY; 16133 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 16134 ZEND_REGSET_INCL(regset, ZREG_R0); 16135 } 16136 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 16137 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 16138 if (ssa_op->op1_use != current_var && 16139 ssa_op->op2_use != current_var) { 16140 ZEND_REGSET_INCL(regset, ZREG_R0); 16141 } 16142 } 16143 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16144 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16145 } 16146 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16147 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16148 } 16149 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16150 if (ssa_op->op1_use != current_var && 16151 ssa_op->op2_use != current_var) { 16152 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16153 } 16154 } 16155 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16156 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16157 ZEND_REGSET_INCL(regset, ZREG_R0); 16158 } 16159 } 16160 break; 16161 case ZEND_BOOL: 16162 case ZEND_BOOL_NOT: 16163 case ZEND_JMPZ: 16164 case ZEND_JMPNZ: 16165 case ZEND_JMPZNZ: 16166 case ZEND_JMPZ_EX: 16167 case ZEND_JMPNZ_EX: 16168 op1_info = OP1_INFO(); 16169 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16170 regset = ZEND_REGSET_EMPTY; 16171 if (op1_info & MAY_BE_DOUBLE) { 16172 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16173 } 16174 if (opline->opcode == ZEND_BOOL || 16175 opline->opcode == ZEND_BOOL_NOT || 16176 opline->opcode == ZEND_JMPZ_EX || 16177 opline->opcode == ZEND_JMPNZ_EX) { 16178 ZEND_REGSET_INCL(regset, ZREG_R0); 16179 } 16180 } 16181 break; 16182 case ZEND_DO_UCALL: 16183 case ZEND_DO_FCALL: 16184 case ZEND_DO_FCALL_BY_NAME: 16185 case ZEND_INCLUDE_OR_EVAL: 16186 case ZEND_GENERATOR_CREATE: 16187 case ZEND_YIELD: 16188 case ZEND_YIELD_FROM: 16189 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16190 break; 16191 default: 16192 break; 16193 } 16194 16195 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16196 if (ssa_op == ssa->ops 16197 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16198 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16199 ZEND_REGSET_INCL(regset, ZREG_R0); 16200 ZEND_REGSET_INCL(regset, ZREG_R1); 16201 } 16202 } 16203 16204 /* %r0 is used to check EG(vm_interrupt) */ 16205 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16206 if (ssa_op == ssa->ops 16207 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16208 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16209#if ZTS 16210 ZEND_REGSET_INCL(regset, ZREG_R0); 16211#else 16212 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16213 ZEND_REGSET_INCL(regset, ZREG_R0); 16214 } 16215#endif 16216 } 16217 } else { 16218 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16219 16220 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16221 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16222#if ZTS 16223 ZEND_REGSET_INCL(regset, ZREG_R0); 16224#else 16225 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16226 ZEND_REGSET_INCL(regset, ZREG_R0); 16227 } 16228#endif 16229 } 16230 } 16231 16232 return regset; 16233} 16234 16235/* 16236 * Local variables: 16237 * tab-width: 4 16238 * c-basic-offset: 4 16239 * indent-tabs-mode: t 16240 * End: 16241 */ 16242