1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | http://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112#include "Zend/zend_cpuinfo.h" 113#include "jit/zend_jit_x86.h" 114 115#ifdef HAVE_VALGRIND 116# include <valgrind/valgrind.h> 117#endif 118 119/* The generated code may contain tautological comparisons, ignore them. */ 120#if defined(__clang__) 121# pragma clang diagnostic push 122# pragma clang diagnostic ignored "-Wtautological-compare" 123# pragma clang diagnostic ignored "-Wstring-compare" 124#endif 125 126const char* zend_reg_name[] = { 127#if defined(__x86_64__) || defined(_M_X64) 128 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 129 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 130 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 131 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 132#else 133 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 134 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 135#endif 136}; 137 138#ifdef HAVE_GCC_GLOBAL_REGS 139# define GCC_GLOBAL_REGS 1 140#else 141# define GCC_GLOBAL_REGS 0 142#endif 143 144#if ZTS 145static size_t tsrm_ls_cache_tcb_offset = 0; 146static size_t tsrm_tls_index; 147static size_t tsrm_tls_offset; 148#endif 149 150/* By default avoid JITing inline handlers if it does not seem profitable due to lack of 151 * type information. Disabling this option allows testing some JIT handlers in the 152 * presence of try/catch blocks, which prevent SSA construction. */ 153#ifndef PROFITABILITY_CHECKS 154# define PROFITABILITY_CHECKS 1 155#endif 156 157|.type EX, zend_execute_data, FP 158|.type OP, zend_op 159|.type ZVAL, zval 160 161|.actionlist dasm_actions 162 163|.globals zend_lb 164static void* dasm_labels[zend_lb_MAX]; 165 166|.section code, cold_code, jmp_table 167 168#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 169 170#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 171 172#define BP_JIT_IS 6 173 174 175#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 176 177|.macro ADD_HYBRID_SPAD 178||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 179| add r4, HYBRID_SPAD 180||#endif 181|.endmacro 182 183|.macro SUB_HYBRID_SPAD 184||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 185| sub r4, HYBRID_SPAD 186||#endif 187|.endmacro 188 189|.macro LOAD_ADDR, reg, addr 190| .if X64 191|| if (IS_SIGNED_32BIT(addr)) { 192| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 193|| } else { 194| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 195|| } 196| .else 197| mov reg, ((ptrdiff_t)addr) 198| .endif 199|.endmacro 200 201|.macro LOAD_TSRM_CACHE, reg 202| .if X64WIN 203| gs 204| mov reg, aword [0x58] 205| mov reg, aword [reg+tsrm_tls_index] 206| mov reg, aword [reg+tsrm_tls_offset] 207| .elif WIN 208| fs 209| mov reg, aword [0x2c] 210| mov reg, aword [reg+tsrm_tls_index] 211| mov reg, aword [reg+tsrm_tls_offset] 212| .elif X64APPLE 213| gs 214|| if (tsrm_ls_cache_tcb_offset) { 215| mov reg, aword [tsrm_ls_cache_tcb_offset] 216|| } else { 217| mov reg, aword [tsrm_tls_index] 218| mov reg, aword [reg+tsrm_tls_offset] 219|| } 220| .elif X64 221| fs 222|| if (tsrm_ls_cache_tcb_offset) { 223| mov reg, aword [tsrm_ls_cache_tcb_offset] 224|| } else { 225| mov reg, [0x8] 226| mov reg, aword [reg+tsrm_tls_index] 227| mov reg, aword [reg+tsrm_tls_offset] 228|| } 229| .else 230| gs 231|| if (tsrm_ls_cache_tcb_offset) { 232| mov reg, aword [tsrm_ls_cache_tcb_offset] 233|| } else { 234| mov reg, [0x4] 235| mov reg, aword [reg+tsrm_tls_index] 236| mov reg, aword [reg+tsrm_tls_offset] 237|| } 238| .endif 239|.endmacro 240 241|.macro LOAD_ADDR_ZTS, reg, struct, field 242| .if ZTS 243| LOAD_TSRM_CACHE reg 244| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 245| .else 246| LOAD_ADDR reg, &struct.field 247| .endif 248|.endmacro 249 250|.macro ADDR_OP1, addr_ins, addr, tmp_reg 251| .if X64 252|| if (IS_SIGNED_32BIT(addr)) { 253| addr_ins ((ptrdiff_t)addr) 254|| } else { 255| mov64 tmp_reg, ((ptrdiff_t)addr) 256| addr_ins tmp_reg 257|| } 258| .else 259| addr_ins ((ptrdiff_t)addr) 260| .endif 261|.endmacro 262 263|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg 264| .if X64 265|| if (IS_SIGNED_32BIT(addr)) { 266| addr_ins op1, ((ptrdiff_t)addr) 267|| } else { 268| mov64 tmp_reg, ((ptrdiff_t)addr) 269| addr_ins op1, tmp_reg 270|| } 271| .else 272| addr_ins op1, ((ptrdiff_t)addr) 273| .endif 274|.endmacro 275 276|.macro PUSH_ADDR, addr, tmp_reg 277| ADDR_OP1 push, addr, tmp_reg 278|.endmacro 279 280|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 281| .if ZTS 282| LOAD_TSRM_CACHE tmp_reg 283| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 284| push tmp_reg 285| .else 286| ADDR_OP1 push, &struct.field, tmp_reg 287| .endif 288|.endmacro 289 290|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg 291| .if X64 292|| if (IS_SIGNED_32BIT(addr)) { 293| mem_ins prefix [addr] 294|| } else { 295| mov64 tmp_reg, ((ptrdiff_t)addr) 296| mem_ins prefix [tmp_reg] 297|| } 298| .else 299| mem_ins prefix [addr] 300| .endif 301|.endmacro 302 303|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg 304| .if X64 305|| if (IS_SIGNED_32BIT(addr)) { 306| mem_ins prefix [addr], op2 307|| } else { 308| mov64 tmp_reg, ((ptrdiff_t)addr) 309| mem_ins prefix [tmp_reg], op2 310|| } 311| .else 312| mem_ins prefix [addr], op2 313| .endif 314|.endmacro 315 316|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg 317| .if X64 318|| if (IS_SIGNED_32BIT(addr)) { 319| mem_ins op1, prefix [addr] 320|| } else { 321| mov64 tmp_reg, ((ptrdiff_t)addr) 322| mem_ins op1, prefix [tmp_reg] 323|| } 324| .else 325| mem_ins op1, prefix [addr] 326| .endif 327|.endmacro 328 329|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 330| .if ZTS 331| LOAD_TSRM_CACHE tmp_reg 332| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 333| .else 334| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg 335| .endif 336|.endmacro 337 338|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg 339| .if ZTS 340| LOAD_TSRM_CACHE tmp_reg 341| mem_ins op1, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 342| .else 343| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg 344| .endif 345|.endmacro 346 347|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg 348| .if X64 349|| if (IS_SIGNED_32BIT(addr)) { 350| mem_ins op1, op2, prefix [addr] 351|| } else { 352| mov64 tmp_reg, ((ptrdiff_t)addr) 353| mem_ins op1, op2, prefix [tmp_reg] 354|| } 355| .else 356| mem_ins op1, op2, prefix [addr] 357| .endif 358|.endmacro 359 360|.macro LOAD_BASE_ADDR, reg, base, offset 361|| if (offset) { 362| lea reg, qword [Ra(base)+offset] 363|| } else { 364| mov reg, Ra(base) 365|| } 366|.endmacro 367 368|.macro PUSH_BASE_ADDR, base, offset, tmp_reg 369|| if (offset) { 370| lea tmp_reg, qword [Ra(base)+offset] 371| push tmp_reg 372|| } else { 373| push Ra(base) 374|| } 375|.endmacro 376 377|.macro EXT_CALL, func, tmp_reg 378| .if X64 379|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 380| call qword &func 381|| } else { 382| LOAD_ADDR tmp_reg, func 383| call tmp_reg 384|| } 385| .else 386| call dword &func 387| .endif 388|.endmacro 389 390|.macro EXT_JMP, func, tmp_reg 391| .if X64 392|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 393| jmp qword &func 394|| } else { 395| LOAD_ADDR tmp_reg, func 396| jmp tmp_reg 397|| } 398| .else 399| jmp dword &func 400| .endif 401|.endmacro 402 403|.macro SAVE_IP 404|| if (GCC_GLOBAL_REGS) { 405| mov aword EX->opline, IP 406|| } 407|.endmacro 408 409|.macro LOAD_IP 410|| if (GCC_GLOBAL_REGS) { 411| mov IP, aword EX->opline 412|| } 413|.endmacro 414 415|.macro LOAD_IP_ADDR, addr 416|| if (GCC_GLOBAL_REGS) { 417| LOAD_ADDR IP, addr 418|| } else { 419| ADDR_OP2_2 mov, aword EX->opline, addr, RX 420|| } 421|.endmacro 422 423|.macro LOAD_IP_ADDR_ZTS, struct, field 424| .if ZTS 425|| if (GCC_GLOBAL_REGS) { 426| LOAD_TSRM_CACHE IP 427| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 428|| } else { 429| LOAD_TSRM_CACHE RX 430| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 431| mov aword EX->opline, RX 432|| } 433| .else 434| LOAD_IP_ADDR &struct.field 435| .endif 436|.endmacro 437 438|.macro GET_IP, reg 439|| if (GCC_GLOBAL_REGS) { 440| mov reg, IP 441|| } else { 442| mov reg, aword EX->opline 443|| } 444|.endmacro 445 446|.macro ADD_IP, val 447|| if (GCC_GLOBAL_REGS) { 448| add IP, val 449|| } else { 450| add aword EX->opline, val 451|| } 452|.endmacro 453 454|.macro JMP_IP 455|| if (GCC_GLOBAL_REGS) { 456| jmp aword [IP] 457|| } else { 458| mov r0, aword EX:FCARG1a->opline 459| jmp aword [r0] 460|| } 461|.endmacro 462 463/* In 64-bit build we compare only low 32-bits. 464 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 465 * comparison would require an additional load of 64-bit address into register. 466 * This is not a problem at all, while JIT buffer size is less than 4GB. 467 */ 468|.macro CMP_IP, addr 469|| if (GCC_GLOBAL_REGS) { 470| cmp IPl, addr 471|| } else { 472| cmp dword EX->opline, addr 473|| } 474|.endmacro 475 476|.macro LOAD_ZVAL_ADDR, reg, addr 477|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 478| LOAD_ADDR reg, Z_ZV(addr) 479|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 480| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) 481|| } else { 482|| ZEND_UNREACHABLE(); 483|| } 484|.endmacro 485 486|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 487|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 488| PUSH_ADDR Z_ZV(addr), tmp_reg 489|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 490| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg 491|| } else { 492|| ZEND_UNREACHABLE(); 493|| } 494|.endmacro 495 496|.macro GET_Z_TYPE_INFO, reg, zv 497| mov reg, dword [zv+offsetof(zval,u1.type_info)] 498|.endmacro 499 500|.macro SET_Z_TYPE_INFO, zv, type 501| mov dword [zv+offsetof(zval,u1.type_info)], type 502|.endmacro 503 504|.macro GET_ZVAL_TYPE, reg, addr 505|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 506| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 507|.endmacro 508 509|.macro GET_ZVAL_TYPE_INFO, reg, addr 510|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 511| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 512|.endmacro 513 514|.macro SET_ZVAL_TYPE_INFO, addr, type 515|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 516| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 517|.endmacro 518 519|.macro GET_Z_PTR, reg, zv 520| mov reg, aword [zv] 521|.endmacro 522 523|.macro SET_Z_PTR, zv, val 524| mov aword [zv], val 525|.endmacro 526 527|.macro GET_Z_W2, reg, zv 528| mov reg, dword [zv+4] 529|.endmacro 530 531|.macro SET_Z_W2, zv, reg 532| mov dword [zv+4], reg 533|.endmacro 534 535|.macro GET_ZVAL_PTR, reg, addr 536|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 537| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 538|.endmacro 539 540|.macro SET_ZVAL_PTR, addr, val 541|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 542| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 543|.endmacro 544 545|.macro GET_ZVAL_W2, reg, addr 546|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 547| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 548|.endmacro 549 550|.macro SET_ZVAL_W2, addr, val 551|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 552| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 553|.endmacro 554 555|.macro UNDEF_OPLINE_RESULT 556| mov r0, EX->opline 557| mov eax, dword OP:r0->result.var 558| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 559|.endmacro 560 561|.macro UNDEF_OPLINE_RESULT_IF_USED 562| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 563| jz >1 564| mov eax, dword OP:RX->result.var 565| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 566|1: 567|.endmacro 568 569|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 570|| if (CAN_USE_AVX()) { 571| avx_ins op1, op2 572|| } else { 573| sse_ins op1, op2 574|| } 575|.endmacro 576 577|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 578|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 579| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 580|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 581| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 582|| } else if (Z_MODE(addr) == IS_REG) { 583| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 584|| } else { 585|| ZEND_UNREACHABLE(); 586|| } 587|.endmacro 588 589|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr 590|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 591| .if X64 592|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 593| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 594|| } else { 595| LOAD_ADDR r0, Z_ZV(addr) 596| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0] 597|| } 598| .else 599| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 600| .endif 601|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 602| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 603|| } else if (Z_MODE(addr) == IS_REG) { 604| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 605|| } else { 606|| ZEND_UNREACHABLE(); 607|| } 608|.endmacro 609 610|.macro SSE_GET_LONG, reg, lval, tmp_reg 611|| if (lval == 0) { 612|| if (CAN_USE_AVX()) { 613| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 614|| } else { 615| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 616|| } 617|| } else { 618|.if X64 619|| if (!IS_SIGNED_32BIT(lval)) { 620| mov64 Ra(tmp_reg), lval 621|| } else { 622| mov Ra(tmp_reg), lval 623|| } 624|.else 625| mov Ra(tmp_reg), lval 626|.endif 627|| if (CAN_USE_AVX()) { 628| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 629| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 630|| } else { 631| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 632| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 633|| } 634|| } 635|.endmacro 636 637|.macro SSE_GET_ZVAL_LVAL, reg, addr, tmp_reg 638|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 639| SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 640|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 641|| if (CAN_USE_AVX()) { 642| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 643| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 644|| } else { 645| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 646| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 647|| } 648|| } else if (Z_MODE(addr) == IS_REG) { 649|| if (CAN_USE_AVX()) { 650| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 651| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 652|| } else { 653| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 654| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 655|| } 656|| } else { 657|| ZEND_UNREACHABLE(); 658|| } 659|.endmacro 660 661|.macro SSE_GET_ZVAL_DVAL, reg, addr 662|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 663|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 664| .if X64 665|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 666| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 667|| } else { 668| LOAD_ADDR r0, Z_ZV(addr) 669| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 670|| } 671| .else 672| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 673| .endif 674|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 675| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 676|| } else if (Z_MODE(addr) == IS_REG) { 677| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 678|| } else { 679|| ZEND_UNREACHABLE(); 680|| } 681|| } 682|.endmacro 683 684|.macro SSE_MATH, opcode, reg, addr, tmp_reg 685|| switch (opcode) { 686|| case ZEND_ADD: 687| SSE_OP addsd, reg, addr, tmp_reg 688|| break; 689|| case ZEND_SUB: 690| SSE_OP subsd, reg, addr, tmp_reg 691|| break; 692|| case ZEND_MUL: 693| SSE_OP mulsd, reg, addr, tmp_reg 694|| break; 695|| case ZEND_DIV: 696| SSE_OP divsd, reg, addr, tmp_reg 697|| break; 698|| } 699|.endmacro 700 701|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 702|| switch (opcode) { 703|| case ZEND_ADD: 704| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 705|| break; 706|| case ZEND_SUB: 707| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 708|| break; 709|| case ZEND_MUL: 710| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 711|| break; 712|| case ZEND_DIV: 713| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 714|| break; 715|| } 716|.endmacro 717 718|.macro SSE_SET_ZVAL_DVAL, addr, reg 719|| if (Z_MODE(addr) == IS_REG) { 720|| if (reg != Z_REG(addr)) { 721| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 722|| } 723|| } else { 724|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 725| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 726|| } 727|.endmacro 728 729|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 730|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 731| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 732|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 733| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 734|| } else if (Z_MODE(addr) == IS_REG) { 735| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 736|| } else { 737|| ZEND_UNREACHABLE(); 738|| } 739|.endmacro 740 741|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 742|| switch (opcode) { 743|| case ZEND_ADD: 744| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 745|| break; 746|| case ZEND_SUB: 747| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 748|| break; 749|| case ZEND_MUL: 750| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 751|| break; 752|| case ZEND_DIV: 753| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 754|| break; 755|| } 756|.endmacro 757 758|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 759|| switch (opcode) { 760|| case ZEND_ADD: 761| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 762|| break; 763|| case ZEND_SUB: 764| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 765|| break; 766|| case ZEND_MUL: 767| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 768|| break; 769|| case ZEND_DIV: 770| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 771|| break; 772|| } 773|.endmacro 774 775|.macro LONG_OP, long_ins, reg, addr, tmp_reg 776|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 777| .if X64 778|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 779| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr)) 780| long_ins Ra(reg), tmp_reg 781|| } else { 782| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 783|| } 784| .else 785| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 786| .endif 787|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 788| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 789|| } else if (Z_MODE(addr) == IS_REG) { 790| long_ins Ra(reg), Ra(Z_REG(addr)) 791|| } else { 792|| ZEND_UNREACHABLE(); 793|| } 794|.endmacro 795 796|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 797|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 798| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 799|| } else if (Z_MODE(op1_addr) == IS_REG) { 800| long_ins Ra(Z_REG(op1_addr)), lval 801|| } else { 802|| ZEND_UNREACHABLE(); 803|| } 804|.endmacro 805 806|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 807|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 808| .if X64 809|| if (!IS_SIGNED_32BIT(lval)) { 810| mov64 r0, lval 811| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 812|| } else { 813| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 814|| } 815| .else 816| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 817| .endif 818|| } else if (Z_MODE(op1_addr) == IS_REG) { 819| .if X64 820|| if (!IS_SIGNED_32BIT(lval)) { 821| mov64 r0, lval 822| long_ins Ra(Z_REG(op1_addr)), r0 823|| } else { 824| long_ins Ra(Z_REG(op1_addr)), lval 825|| } 826| .else 827| long_ins Ra(Z_REG(op1_addr)), lval 828| .endif 829|| } else { 830|| ZEND_UNREACHABLE(); 831|| } 832|.endmacro 833 834|.macro GET_ZVAL_LVAL, reg, addr 835|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 836|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 837| xor Ra(reg), Ra(reg) 838|| } else { 839| .if X64 840|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 841| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 842|| } else { 843| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 844|| } 845| .else 846| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 847| .endif 848|| } 849|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 850| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 851|| } else if (Z_MODE(addr) == IS_REG) { 852|| if (reg != Z_REG(addr)) { 853| mov Ra(reg), Ra(Z_REG(addr)) 854|| } 855|| } else { 856|| ZEND_UNREACHABLE(); 857|| } 858|.endmacro 859 860|.macro LONG_MATH, opcode, reg, addr, tmp_reg 861|| switch (opcode) { 862|| case ZEND_ADD: 863| LONG_OP add, reg, addr, Ra(tmp_reg) 864|| break; 865|| case ZEND_SUB: 866| LONG_OP sub, reg, addr, Ra(tmp_reg) 867|| break; 868|| case ZEND_MUL: 869| LONG_OP imul, reg, addr, Ra(tmp_reg) 870|| break; 871|| case ZEND_BW_OR: 872| LONG_OP or, reg, addr, Ra(tmp_reg) 873|| break; 874|| case ZEND_BW_AND: 875| LONG_OP and, reg, addr, Ra(tmp_reg) 876|| break; 877|| case ZEND_BW_XOR: 878| LONG_OP xor, reg, addr, Ra(tmp_reg) 879|| break; 880|| default: 881|| ZEND_UNREACHABLE(); 882|| } 883|.endmacro 884 885|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 886|| switch (opcode) { 887|| case ZEND_ADD: 888| add dst_reg, src_reg 889|| break; 890|| case ZEND_SUB: 891| sub dst_reg, src_reg 892|| break; 893|| case ZEND_MUL: 894| imul dst_reg, src_reg 895|| break; 896|| case ZEND_BW_OR: 897| or dst_reg, src_reg 898|| break; 899|| case ZEND_BW_AND: 900| and dst_reg, src_reg 901|| break; 902|| case ZEND_BW_XOR: 903| xor dst_reg, src_reg 904|| break; 905|| default: 906|| ZEND_UNREACHABLE(); 907|| } 908|.endmacro 909 910|.macro SET_ZVAL_LVAL, addr, lval 911|| if (Z_MODE(addr) == IS_REG) { 912| mov Ra(Z_REG(addr)), lval 913|| } else { 914|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 915| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 916|| } 917|.endmacro 918 919|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 920|| if (Z_TYPE_P(zv) > IS_TRUE) { 921|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 922|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 923|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 924|| if (CAN_USE_AVX()) { 925| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 926|| } else { 927| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 928|| } 929| .if X64 930|| } else if (!IS_SIGNED_32BIT(zv)) { 931| mov64 Ra(tmp_reg), ((uintptr_t)zv) 932| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 933| .endif 934|| } else { 935| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 936|| } 937| SSE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 939|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 940| SSE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 941| SSE_SET_ZVAL_DVAL dst_addr, dst_reg 942|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 943| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 944|| } else { 945| .if X64 946|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 947|| if (Z_MODE(dst_addr) == IS_REG) { 948| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 949|| } else { 950| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 951| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 952|| } 953|| } else { 954| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 955|| } 956| .else 957| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 958| .endif 959|| } 960|| } 961|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 962|| if (dst_def_info == MAY_BE_DOUBLE) { 963|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 964| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 965|| } 966|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 967| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 968|| } 969|| } 970|.endmacro 971 972|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 973|| if (Z_TYPE_P(zv) > IS_TRUE) { 974|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 975|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 976|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 977|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 978|| if (CAN_USE_AVX()) { 979| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 980|| } else { 981| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 982|| } 983| .if X64 984|| } else if (!IS_SIGNED_32BIT(zv)) { 985| mov64 Ra(tmp_reg), ((uintptr_t)zv) 986| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 987| .endif 988|| } else { 989| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 990|| } 991| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 992| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 993|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 994|| if (Z_MODE(dst_addr) == IS_REG) { 995| SSE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 996| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 997|| } else if (Z_MODE(res_addr) == IS_REG) { 998| SSE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 999| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1000|| } else { 1001| SSE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 1002| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1003| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1004|| } 1005|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1006|| if (Z_MODE(dst_addr) == IS_REG) { 1007| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1008| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1009|| } else { 1010| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1011| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1012|| } 1013|| } else { 1014| .if X64 1015|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1016|| if (Z_MODE(dst_addr) == IS_REG) { 1017| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1018| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1019|| } else if (Z_MODE(res_addr) == IS_REG) { 1020| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1021| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1022|| } else { 1023| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1024| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1025| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1026|| } 1027|| } else if (Z_MODE(dst_addr) == IS_REG) { 1028| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1029| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1030|| } else if (Z_MODE(res_addr) == IS_REG) { 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1033|| } else { 1034| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1035| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1036|| } 1037| .else 1038|| if (Z_MODE(dst_addr) == IS_REG) { 1039| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1040| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1041|| } else if (Z_MODE(res_addr) == IS_REG) { 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1044|| } else { 1045| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1046| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1047|| } 1048| .endif 1049|| } 1050|| } 1051|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1052|| if (dst_def_info == MAY_BE_DOUBLE) { 1053|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1054| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1055|| } 1056|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1057| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1058|| } 1059|| } 1060|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1061|| if (dst_def_info == MAY_BE_DOUBLE) { 1062| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1063|| } else { 1064| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1065|| } 1066|| } 1067|.endmacro 1068 1069/* the same as above, but "src" may overlap with "tmp_reg1" */ 1070|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1071| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1072|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1073|| !(src_info & MAY_BE_GUARD) && 1074|| has_concrete_type(src_info & MAY_BE_ANY)) { 1075|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1076|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1077|| zend_uchar type = concrete_type(src_info); 1078| SET_ZVAL_TYPE_INFO dst_addr, type 1079|| } 1080|| } 1081|| } else { 1082| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1083| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1084|| } 1085|.endmacro 1086 1087|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1088|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1089|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1090|| if (Z_MODE(src_addr) == IS_REG) { 1091|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1092| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1093|| } 1094|| } else if (Z_MODE(dst_addr) == IS_REG) { 1095| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1096|| } else { 1097| GET_ZVAL_LVAL tmp_reg2, src_addr 1098| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1099|| } 1100|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1101|| if (Z_MODE(src_addr) == IS_REG) { 1102| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1103|| } else if (Z_MODE(dst_addr) == IS_REG) { 1104| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1105|| } else { 1106| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1107| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1108|| } 1109|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112|| } else { 1113| .if X64 1114| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1115| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1116| .else 1117|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1118| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1119| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1120| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122|| } else { 1123| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1124| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1125| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1126| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1127|| } 1128| .endif 1129|| } 1130|| } 1131|.endmacro 1132 1133|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1134|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1135|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1136|| if (Z_MODE(src_addr) == IS_REG) { 1137|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1138| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1139|| } 1140|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1141| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1142|| } 1143|| } else if (Z_MODE(dst_addr) == IS_REG) { 1144| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1145|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1146| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1147|| } 1148|| } else if (Z_MODE(res_addr) == IS_REG) { 1149| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1150| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1151|| } else { 1152| GET_ZVAL_LVAL tmp_reg2, src_addr 1153| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1154| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1155|| } 1156|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1157|| if (Z_MODE(src_addr) == IS_REG) { 1158| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1159| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1160|| } else if (Z_MODE(dst_addr) == IS_REG) { 1161| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1162| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1163|| } else if (Z_MODE(res_addr) == IS_REG) { 1164| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1165| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1166|| } else { 1167| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1168| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1169| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1170|| } 1171|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1172| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1173| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1174| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1175|| } else { 1176| .if X64 1177| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1178| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1179| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1180| .else 1181|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1182| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1183| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1184| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1187| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1188|| } else { 1189| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1190| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1191| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1192| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1193| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1194| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1195|| } 1196| .endif 1197|| } 1198|| } 1199|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1200|| has_concrete_type(src_info & MAY_BE_ANY)) { 1201|| zend_uchar type = concrete_type(src_info); 1202|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1203|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1204| SET_ZVAL_TYPE_INFO dst_addr, type 1205|| } 1206|| } 1207|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1208| SET_ZVAL_TYPE_INFO res_addr, type 1209|| } 1210|| } else { 1211| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1212| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1213| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1214|| } 1215|.endmacro 1216 1217|.macro IF_UNDEF, type_reg, label 1218| test type_reg, type_reg 1219| je label 1220|.endmacro 1221 1222|.macro IF_TYPE, type, val, label 1223| cmp type, val 1224| je label 1225|.endmacro 1226 1227|.macro IF_NOT_TYPE, type, val, label 1228| cmp type, val 1229| jne label 1230|.endmacro 1231 1232|.macro IF_Z_TYPE, zv, val, label 1233| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro IF_NOT_Z_TYPE, zv, val, label 1237| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1238|.endmacro 1239 1240|.macro CMP_ZVAL_TYPE, addr, val 1241|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1242| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1243|.endmacro 1244 1245|.macro IF_ZVAL_TYPE, addr, val, label 1246|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1247| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1248|.endmacro 1249 1250|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1251|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1252| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1253|.endmacro 1254 1255|.macro IF_FLAGS, type_flags, mask, label 1256| test type_flags, mask 1257| jnz label 1258|.endmacro 1259 1260|.macro IF_NOT_FLAGS, type_flags, mask, label 1261| test type_flags, mask 1262| jz label 1263|.endmacro 1264 1265|.macro IF_REFCOUNTED, type_flags, label 1266| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267|.endmacro 1268 1269|.macro IF_NOT_REFCOUNTED, type_flags, label 1270| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1271| test type_flags, type_flags 1272| jz label 1273|.endmacro 1274 1275|.macro IF_ZVAL_FLAGS, addr, mask, label 1276|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1277| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1278|.endmacro 1279 1280|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1281|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1282| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1283|.endmacro 1284 1285|.macro IF_ZVAL_REFCOUNTED, addr, label 1286| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1291|.endmacro 1292 1293|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1294| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1295|.endmacro 1296 1297|.macro GC_ADDREF, zv 1298| add dword [zv], 1 1299|.endmacro 1300 1301|.macro GC_DELREF, zv 1302| sub dword [zv], 1 1303|.endmacro 1304 1305|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1306| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1307| jne label 1308|.endmacro 1309 1310|.macro ADDREF_CONST, zv, tmp_reg 1311| .if X64 1312|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1313| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1314| add dword [tmp_reg], 1 1315|| } else { 1316| add dword [Z_LVAL_P(zv)], 1 1317|| } 1318| .else 1319| add dword [Z_LVAL_P(zv)], 1 1320| .endif 1321|.endmacro 1322 1323|.macro ADDREF_CONST_2, zv, tmp_reg 1324| .if X64 1325|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1326| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1327| add dword [tmp_reg], 2 1328|| } else { 1329| add dword [Z_LVAL_P(zv)], 2 1330|| } 1331| .else 1332| add dword [Z_LVAL_P(zv)], 2 1333| .endif 1334|.endmacro 1335 1336|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1337|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1338|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1339| IF_NOT_REFCOUNTED type_flags_reg, >1 1340|| } 1341| GC_ADDREF value_ptr_reg 1342|1: 1343|| } 1344|.endmacro 1345 1346|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1347|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1348|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1349| IF_NOT_REFCOUNTED type_flags_reg, >1 1350|| } 1351| add dword [value_ptr_reg], 2 1352|1: 1353|| } 1354|.endmacro 1355 1356|.macro ZVAL_DEREF, reg, info 1357|| if (info & MAY_BE_REF) { 1358| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1359| GET_Z_PTR reg, reg 1360| add reg, offsetof(zend_reference, val) 1361|1: 1362|| } 1363|.endmacro 1364 1365|.macro SET_EX_OPLINE, op, tmp_reg 1366|| if (op == last_valid_opline) { 1367|| zend_jit_use_last_valid_opline(); 1368| SAVE_IP 1369|| } else { 1370| ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg 1371|| if (!GCC_GLOBAL_REGS) { 1372|| zend_jit_reset_last_valid_opline(); 1373|| } 1374|| } 1375|.endmacro 1376 1377// zval should be in FCARG1a 1378|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1379|| do { 1380|| if (!((var_info) & MAY_BE_GUARD) 1381|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1382|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1383|| if (type == IS_STRING && !ZEND_DEBUG) { 1384| EXT_CALL _efree, r0 1385|| break; 1386|| } else if (type == IS_ARRAY) { 1387|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1388|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1389| SET_EX_OPLINE opline, r0 1390|| } 1391| EXT_CALL zend_array_destroy, r0 1392|| } else { 1393| EXT_CALL zend_jit_array_free, r0 1394|| } 1395|| break; 1396|| } else if (type == IS_OBJECT) { 1397|| if (opline) { 1398| SET_EX_OPLINE opline, r0 1399|| } 1400| EXT_CALL zend_objects_store_del, r0 1401|| break; 1402|| } 1403|| } 1404|| if (opline) { 1405| SET_EX_OPLINE opline, r0 1406|| } 1407| EXT_CALL rc_dtor_func, r0 1408|| } while(0); 1409|.endmacro 1410 1411|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1412|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1413|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1414| // if (Z_REFCOUNTED_P(cv)) { 1415|| if (cold) { 1416| IF_ZVAL_REFCOUNTED addr, >1 1417|.cold_code 1418|1: 1419|| } else { 1420| IF_NOT_ZVAL_REFCOUNTED addr, >4 1421|| } 1422|| } 1423| // if (!Z_DELREF_P(cv)) { 1424| GET_ZVAL_PTR FCARG1a, addr 1425| GC_DELREF FCARG1a 1426|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1427|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1428|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1429| jnz >3 1430|| } else { 1431| jnz >4 1432|| } 1433|| } 1434| // zval_dtor_func(r); 1435| ZVAL_DTOR_FUNC op_info, opline 1436|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1437| jmp >4 1438|| } 1439|3: 1440|| } 1441|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1442|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1443|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); 1444| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1445| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1446| GET_ZVAL_PTR FCARG1a, ref_addr 1447|1: 1448|| } 1449| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1450| // gc_possible_root(Z_COUNTED_P(z)) 1451| EXT_CALL gc_possible_root, r0 1452|| } 1453|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1454| jmp >4 1455|.code 1456|| } 1457|4: 1458|| } 1459|.endmacro 1460 1461|.macro FREE_OP, op_type, op, op_info, cold, opline 1462|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1463| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1464|| } 1465|.endmacro 1466 1467|.macro SEPARATE_ARRAY, addr, op_info, cold 1468|| if (RC_MAY_BE_N(op_info)) { 1469|| if (Z_REG(addr) != ZREG_FP) { 1470| GET_ZVAL_LVAL ZREG_R0, addr 1471|| if (RC_MAY_BE_1(op_info)) { 1472| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1473| jbe >2 1474|| } 1475|| if (Z_REG(addr) != ZREG_FCARG1a || Z_OFFSET(addr) != 0) { 1476| LOAD_ZVAL_ADDR FCARG1a, addr 1477|| } 1478| EXT_CALL zend_jit_zval_array_dup, r0 1479|2: 1480| mov FCARG1a, r0 1481|| } else { 1482| GET_ZVAL_LVAL ZREG_FCARG1a, addr 1483|| if (RC_MAY_BE_1(op_info)) { 1484| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1485|| if (cold) { 1486| ja >1 1487|.cold_code 1488|1: 1489|| } else { 1490| jbe >2 1491|| } 1492|| } 1493| IF_NOT_ZVAL_REFCOUNTED addr, >1 1494| GC_DELREF FCARG1a 1495|1: 1496| EXT_CALL zend_array_dup, r0 1497| SET_ZVAL_PTR addr, r0 1498| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1499| mov FCARG1a, r0 1500|| if (RC_MAY_BE_1(op_info)) { 1501|| if (cold) { 1502| jmp >2 1503|.code 1504|| } 1505|| } 1506|2: 1507|| } 1508|| } else { 1509| GET_ZVAL_LVAL ZREG_FCARG1a, addr 1510|| } 1511|.endmacro 1512 1513|.macro EFREE_REG_REFERENCE 1514||#if ZEND_DEBUG 1515| xor FCARG2a, FCARG2a // filename 1516| .if X64WIN 1517| xor CARG3d, CARG3d // lineno 1518| xor CARG4, CARG4 1519| mov aword A5, 0 1520| EXT_CALL _efree, r0 1521| .elif X64 1522| xor CARG3d, CARG3d // lineno 1523| xor CARG4, CARG4 1524| xor CARG5, CARG5 1525| EXT_CALL _efree, r0 1526| .else 1527| sub r4, 4 1528| push 0 1529| push 0 1530| push 0 // lineno 1531| EXT_CALL _efree, r0 1532| add r4, 4 1533| .endif 1534||#else 1535||#ifdef HAVE_BUILTIN_CONSTANT_P 1536| EXT_CALL _efree_32, r0 1537||#else 1538| EXT_CALL _efree, r0 1539||#endif 1540||#endif 1541|.endmacro 1542 1543|.macro EFREE_REFERENCE, ptr 1544| mov FCARG1a, ptr 1545| EFREE_REG_REFERENCE 1546|.endmacro 1547 1548|.macro EMALLOC, size, op_array, opline 1549||#if ZEND_DEBUG 1550|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1551| mov FCARG1a, size 1552| LOAD_ADDR FCARG2a, filename 1553| .if X64WIN 1554| mov CARG3d, opline->lineno 1555| xor CARG4, CARG4 1556| mov aword A5, 0 1557| EXT_CALL _emalloc, r0 1558| .elif X64 1559| mov CARG3d, opline->lineno 1560| xor CARG4, CARG4 1561| xor CARG5, CARG5 1562| EXT_CALL _emalloc, r0 1563| .else 1564| sub r4, 4 1565| push 0 1566| push 0 1567| push opline->lineno 1568| EXT_CALL _emalloc, r0 1569| add r4, 4 1570| .endif 1571||#else 1572||#ifdef HAVE_BUILTIN_CONSTANT_P 1573|| if (size > 24 && size <= 32) { 1574| EXT_CALL _emalloc_32, r0 1575|| } else { 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578|| } 1579||#else 1580| mov FCARG1a, size 1581| EXT_CALL _emalloc, r0 1582||#endif 1583||#endif 1584|.endmacro 1585 1586|.macro OBJ_RELEASE, reg, exit_label 1587| GC_DELREF Ra(reg) 1588| jne >1 1589| // zend_objects_store_del(obj); 1590|| if (reg != ZREG_FCARG1a) { 1591| mov FCARG1a, Ra(reg) 1592|| } 1593| EXT_CALL zend_objects_store_del, r0 1594| jmp exit_label 1595|1: 1596| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1597| // gc_possible_root(obj) 1598|| if (reg != ZREG_FCARG1a) { 1599| mov FCARG1a, Ra(reg) 1600|| } 1601| EXT_CALL gc_possible_root, r0 1602|1: 1603|.endmacro 1604 1605|.macro UNDEFINED_OFFSET, opline 1606|| if (opline == last_valid_opline) { 1607|| zend_jit_use_last_valid_opline(); 1608| call ->undefined_offset_ex 1609|| } else { 1610| SET_EX_OPLINE opline, r0 1611| call ->undefined_offset 1612|| } 1613|.endmacro 1614 1615|.macro UNDEFINED_INDEX, opline 1616|| if (opline == last_valid_opline) { 1617|| zend_jit_use_last_valid_opline(); 1618| call ->undefined_index_ex 1619|| } else { 1620| SET_EX_OPLINE opline, r0 1621| call ->undefined_index 1622|| } 1623|.endmacro 1624 1625|.macro CANNOT_ADD_ELEMENT, opline 1626|| if (opline == last_valid_opline) { 1627|| zend_jit_use_last_valid_opline(); 1628| call ->cannot_add_element_ex 1629|| } else { 1630| SET_EX_OPLINE opline, r0 1631| call ->cannot_add_element 1632|| } 1633|.endmacro 1634 1635static zend_bool reuse_ip = 0; 1636static zend_bool delayed_call_chain = 0; 1637static uint32_t delayed_call_level = 0; 1638static const zend_op *last_valid_opline = NULL; 1639static zend_bool use_last_vald_opline = 0; 1640static zend_bool track_last_valid_opline = 0; 1641static int jit_return_label = -1; 1642static uint32_t current_trace_num = 0; 1643static uint32_t allowed_opt_flags = 0; 1644 1645static void zend_jit_track_last_valid_opline(void) 1646{ 1647 use_last_vald_opline = 0; 1648 track_last_valid_opline = 1; 1649} 1650 1651static void zend_jit_use_last_valid_opline(void) 1652{ 1653 if (track_last_valid_opline) { 1654 use_last_vald_opline = 1; 1655 track_last_valid_opline = 0; 1656 } 1657} 1658 1659static zend_bool zend_jit_trace_uses_initial_ip(void) 1660{ 1661 return use_last_vald_opline; 1662} 1663 1664static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1665{ 1666 if (!reuse_ip) { 1667 track_last_valid_opline = 0; 1668 last_valid_opline = target_opline; 1669 } 1670} 1671 1672static void zend_jit_reset_last_valid_opline(void) 1673{ 1674 track_last_valid_opline = 0; 1675 last_valid_opline = NULL; 1676} 1677 1678static void zend_jit_start_reuse_ip(void) 1679{ 1680 zend_jit_reset_last_valid_opline(); 1681 reuse_ip = 1; 1682} 1683 1684static int zend_jit_reuse_ip(dasm_State **Dst) 1685{ 1686 if (!reuse_ip) { 1687 zend_jit_start_reuse_ip(); 1688 | // call = EX(call); 1689 | mov RX, EX->call 1690 } 1691 return 1; 1692} 1693 1694static void zend_jit_stop_reuse_ip(void) 1695{ 1696 reuse_ip = 0; 1697} 1698 1699/* bit helpers */ 1700 1701/* from http://aggregate.org/MAGIC/ */ 1702static uint32_t ones32(uint32_t x) 1703{ 1704 x -= ((x >> 1) & 0x55555555); 1705 x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); 1706 x = (((x >> 4) + x) & 0x0f0f0f0f); 1707 x += (x >> 8); 1708 x += (x >> 16); 1709 return x & 0x0000003f; 1710} 1711 1712static uint32_t floor_log2(uint32_t x) 1713{ 1714 ZEND_ASSERT(x != 0); 1715 x |= (x >> 1); 1716 x |= (x >> 2); 1717 x |= (x >> 4); 1718 x |= (x >> 8); 1719 x |= (x >> 16); 1720 return ones32(x) - 1; 1721} 1722 1723static zend_bool is_power_of_two(uint32_t x) 1724{ 1725 return !(x & (x - 1)) && x != 0; 1726} 1727 1728static zend_bool has_concrete_type(uint32_t value_type) 1729{ 1730 return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); 1731} 1732 1733static uint32_t concrete_type(uint32_t value_type) 1734{ 1735 return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); 1736} 1737 1738static inline zend_bool is_signed(double d) 1739{ 1740 return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; 1741} 1742 1743static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1744{ 1745 |->interrupt_handler: 1746 | SAVE_IP 1747 | //EG(vm_interrupt) = 0; 1748 | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0 1749 | //if (EG(timed_out)) { 1750 | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0 1751 | je >1 1752 | //zend_timeout(); 1753 | EXT_CALL zend_timeout, r0 1754 |1: 1755 | //} else if (zend_interrupt_function) { 1756 if (zend_interrupt_function) { 1757 | //zend_interrupt_function(execute_data); 1758 |.if X64 1759 | mov CARG1, FP 1760 | EXT_CALL zend_interrupt_function, r0 1761 |.else 1762 | mov aword A1, FP 1763 | EXT_CALL zend_interrupt_function, r0 1764 |.endif 1765 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 1766 | je >1 1767 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1768 |1: 1769 | //ZEND_VM_ENTER(); 1770 | //execute_data = EG(current_execute_data); 1771 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 1772 | LOAD_IP 1773 } 1774 | //ZEND_VM_CONTINUE() 1775 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1776 | ADD_HYBRID_SPAD 1777 | JMP_IP 1778 } else if (GCC_GLOBAL_REGS) { 1779 | add r4, SPAD // stack alignment 1780 | JMP_IP 1781 } else { 1782 | mov FP, aword T2 // restore FP 1783 | mov RX, aword T3 // restore IP 1784 | add r4, NR_SPAD // stack alignment 1785 | mov r0, 1 // ZEND_VM_ENTER 1786 | ret 1787 } 1788 1789 return 1; 1790} 1791 1792static int zend_jit_exception_handler_stub(dasm_State **Dst) 1793{ 1794 |->exception_handler: 1795 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1796 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1797 1798 | ADD_HYBRID_SPAD 1799 | EXT_CALL handler, r0 1800 | JMP_IP 1801 } else { 1802 const void *handler = EG(exception_op)->handler; 1803 1804 if (GCC_GLOBAL_REGS) { 1805 | add r4, SPAD // stack alignment 1806 | EXT_JMP handler, r0 1807 } else { 1808 | mov FCARG1a, FP 1809 | EXT_CALL handler, r0 1810 | mov FP, aword T2 // restore FP 1811 | mov RX, aword T3 // restore IP 1812 | add r4, NR_SPAD // stack alignment 1813 | test eax, eax 1814 | jl >1 1815 | mov r0, 1 // ZEND_VM_ENTER 1816 |1: 1817 | ret 1818 } 1819 } 1820 1821 return 1; 1822} 1823 1824static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1825{ 1826 |->exception_handler_undef: 1827 | MEM_OP2_2_ZTS mov, r0, aword, executor_globals, opline_before_exception, r0 1828 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1829 | jz >1 1830 | mov eax, dword OP:r0->result.var 1831 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1832 |1: 1833 | jmp ->exception_handler 1834 1835 return 1; 1836} 1837 1838 1839static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1840{ 1841 |->exception_handler_free_op1_op2: 1842 | UNDEF_OPLINE_RESULT_IF_USED 1843 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1844 | je >9 1845 | mov eax, dword OP:RX->op1.var 1846 | add r0, FP 1847 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1848 |9: 1849 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1850 | je >9 1851 | mov eax, dword OP:RX->op2.var 1852 | add r0, FP 1853 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1854 |9: 1855 | jmp ->exception_handler 1856 return 1; 1857} 1858 1859static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1860{ 1861 |->exception_handler_free_op2: 1862 | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, opline_before_exception, r0 1863 | UNDEF_OPLINE_RESULT_IF_USED 1864 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1865 | je >9 1866 | mov eax, dword OP:RX->op2.var 1867 | add r0, FP 1868 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1869 |9: 1870 | jmp ->exception_handler 1871 return 1; 1872} 1873 1874static int zend_jit_leave_function_stub(dasm_State **Dst) 1875{ 1876 |->leave_function_handler: 1877 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1878 | test FCARG1d, ZEND_CALL_TOP 1879 | jnz >1 1880 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1881 | ADD_HYBRID_SPAD 1882 | JMP_IP 1883 |1: 1884 | EXT_CALL zend_jit_leave_top_func_helper, r0 1885 | ADD_HYBRID_SPAD 1886 | JMP_IP 1887 } else { 1888 if (GCC_GLOBAL_REGS) { 1889 | add r4, SPAD 1890 } else { 1891 | mov FCARG2a, FP 1892 | mov FP, aword T2 // restore FP 1893 | mov RX, aword T3 // restore IP 1894 | add r4, NR_SPAD 1895 } 1896 | test FCARG1d, ZEND_CALL_TOP 1897 | jnz >1 1898 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1899 |1: 1900 | EXT_JMP zend_jit_leave_top_func_helper, r0 1901 } 1902 1903 return 1; 1904} 1905 1906static int zend_jit_leave_throw_stub(dasm_State **Dst) 1907{ 1908 |->leave_throw_handler: 1909 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1910 if (GCC_GLOBAL_REGS) { 1911 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1912 | je >5 1913 | // EG(opline_before_exception) = opline; 1914 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 1915 |5: 1916 | // opline = EG(exception_op); 1917 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1918 | // HANDLE_EXCEPTION() 1919 | jmp ->exception_handler 1920 } else { 1921 | GET_IP FCARG1a 1922 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1923 | je >5 1924 | // EG(opline_before_exception) = opline; 1925 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0 1926 |5: 1927 | // opline = EG(exception_op); 1928 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1929 | mov FP, aword T2 // restore FP 1930 | mov RX, aword T3 // restore IP 1931 | add r4, NR_SPAD // stack alignment 1932 | mov r0, 2 // ZEND_VM_LEAVE 1933 | ret 1934 } 1935 1936 return 1; 1937} 1938 1939static int zend_jit_icall_throw_stub(dasm_State **Dst) 1940{ 1941 |->icall_throw_handler: 1942 | // zend_rethrow_exception(zend_execute_data *execute_data) 1943 | mov IP, aword EX->opline 1944 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1945 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1946 | je >1 1947 | // EG(opline_before_exception) = opline; 1948 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 1949 |1: 1950 | // opline = EG(exception_op); 1951 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1952 || if (GCC_GLOBAL_REGS) { 1953 | mov aword EX->opline, IP 1954 || } 1955 | // HANDLE_EXCEPTION() 1956 | jmp ->exception_handler 1957 1958 return 1; 1959} 1960 1961static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1962{ 1963 |->throw_cannot_pass_by_ref: 1964 | mov r0, EX->opline 1965 | mov ecx, dword OP:r0->result.var 1966 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1967 | // last EX(call) frame may be delayed 1968 | cmp RX, EX->call 1969 | je >1 1970 | mov r1, EX->call 1971 | mov EX:RX->prev_execute_data, r1 1972 | mov EX->call, RX 1973 |1: 1974 | mov RX, r0 1975 | mov FCARG1d, dword OP:r0->op2.num 1976 | EXT_CALL zend_cannot_pass_by_reference, r0 1977 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1978 | jne >9 1979 | mov eax, dword OP:RX->op1.var 1980 | add r0, FP 1981 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1982 |9: 1983 | jmp ->exception_handler 1984 1985 return 1; 1986} 1987 1988static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1989{ 1990 |->undefined_offset_ex: 1991 | SAVE_IP 1992 | jmp ->undefined_offset 1993 1994 return 1; 1995} 1996 1997static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1998{ 1999 |->undefined_offset: 2000 |.if X64WIN 2001 | sub r4, 0x28 2002 |.elif X64 2003 | sub r4, 8 2004 |.else 2005 | sub r4, 12 2006 |.endif 2007 | mov r0, EX->opline 2008 | mov ecx, dword OP:r0->result.var 2009 | cmp byte OP:r0->op2_type, IS_CONST 2010 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2011 | jne >2 2012 |.if X64 2013 | movsxd r1, dword OP:r0->op2.constant 2014 | add r0, r1 2015 |.else 2016 | mov r0, aword OP:r0->op2.zv 2017 |.endif 2018 | jmp >3 2019 |2: 2020 | mov eax, dword OP:r0->op2.var 2021 | add r0, FP 2022 |3: 2023 |.if X64WIN 2024 | mov CARG1, E_WARNING 2025 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2026 | mov CARG3, aword [r0] 2027 | EXT_CALL zend_error, r0 2028 | add r4, 0x28 // stack alignment 2029 |.elif X64 2030 | mov CARG1, E_WARNING 2031 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2032 | mov CARG3, aword [r0] 2033 | EXT_CALL zend_error, r0 2034 | add r4, 8 // stack alignment 2035 |.else 2036 | sub r4, 4 2037 | push aword [r0] 2038 | push "Undefined array key " ZEND_LONG_FMT 2039 | push E_WARNING 2040 | EXT_CALL zend_error, r0 2041 | add r4, 28 2042 |.endif 2043 | ret 2044 2045 return 1; 2046} 2047 2048static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2049{ 2050 |->undefined_index_ex: 2051 | SAVE_IP 2052 | jmp ->undefined_index 2053 2054 return 1; 2055} 2056 2057static int zend_jit_undefined_index_stub(dasm_State **Dst) 2058{ 2059 |->undefined_index: 2060 |.if X64WIN 2061 | sub r4, 0x28 2062 |.elif X64 2063 | sub r4, 8 2064 |.else 2065 | sub r4, 12 2066 |.endif 2067 | mov r0, EX->opline 2068 | mov ecx, dword OP:r0->result.var 2069 | cmp byte OP:r0->op2_type, IS_CONST 2070 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2071 | jne >2 2072 |.if X64 2073 | movsxd r1, dword OP:r0->op2.constant 2074 | add r0, r1 2075 |.else 2076 | mov r0, aword OP:r0->op2.zv 2077 |.endif 2078 | jmp >3 2079 |2: 2080 | mov eax, dword OP:r0->op2.var 2081 | add r0, FP 2082 |3: 2083 |.if X64WIN 2084 | mov CARG1, E_WARNING 2085 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2086 | mov CARG3, aword [r0] 2087 | add CARG3, offsetof(zend_string, val) 2088 | EXT_CALL zend_error, r0 2089 | add r4, 0x28 2090 |.elif X64 2091 | mov CARG1, E_WARNING 2092 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2093 | mov CARG3, aword [r0] 2094 | add CARG3, offsetof(zend_string, val) 2095 | EXT_CALL zend_error, r0 2096 | add r4, 8 2097 |.else 2098 | sub r4, 4 2099 | mov r0, aword [r0] 2100 | add r0, offsetof(zend_string, val) 2101 | push r0 2102 | push "Undefined array key \"%s\"" 2103 | push E_WARNING 2104 | EXT_CALL zend_error, r0 2105 | add r4, 28 2106 |.endif 2107 | ret 2108 2109 return 1; 2110} 2111 2112static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2113{ 2114 |->cannot_add_element_ex: 2115 | SAVE_IP 2116 | jmp ->cannot_add_element 2117 2118 return 1; 2119} 2120 2121static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2122{ 2123 |->cannot_add_element: 2124 |.if X64WIN 2125 | sub r4, 0x28 2126 |.elif X64 2127 | sub r4, 8 2128 |.else 2129 | sub r4, 12 2130 |.endif 2131 | mov r0, EX->opline 2132 | cmp byte OP:r0->result_type, IS_UNUSED 2133 | jz >1 2134 | mov eax, dword OP:r0->result.var 2135 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2136 |1: 2137 |.if X64WIN 2138 | xor CARG1, CARG1 2139 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2140 | EXT_CALL zend_throw_error, r0 2141 | add r4, 0x28 2142 |.elif X64 2143 | xor CARG1, CARG1 2144 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2145 | EXT_CALL zend_throw_error, r0 2146 | add r4, 8 2147 |.else 2148 | sub r4, 8 2149 | push "Cannot add element to the array as the next element is already occupied" 2150 | push 0 2151 | EXT_CALL zend_throw_error, r0 2152 | add r4, 28 2153 |.endif 2154 | ret 2155 2156 return 1; 2157} 2158 2159static int zend_jit_undefined_function_stub(dasm_State **Dst) 2160{ 2161 |->undefined_function: 2162 | mov r0, aword EX->opline 2163 |.if X64 2164 | xor CARG1, CARG1 2165 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2166 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2167 | mov CARG3, aword [r0 + CARG3] 2168 | add CARG3, offsetof(zend_string, val) 2169 | EXT_CALL zend_throw_error, r0 2170 |.else 2171 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2172 | mov r0, aword [r0] 2173 | add r0, offsetof(zend_string, val) 2174 | mov aword A3, r0 2175 | mov aword A2, "Call to undefined function %s()" 2176 | mov aword A1, 0 2177 | EXT_CALL zend_throw_error, r0 2178 |.endif 2179 | jmp ->exception_handler 2180 return 1; 2181} 2182 2183static int zend_jit_negative_shift_stub(dasm_State **Dst) 2184{ 2185 |->negative_shift: 2186 | mov RX, EX->opline 2187 |.if X64 2188 |.if WIN 2189 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2190 | mov CARG1, aword [CARG1] 2191 |.else 2192 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2193 |.endif 2194 | LOAD_ADDR CARG2, "Bit shift by negative number" 2195 | EXT_CALL zend_throw_error, r0 2196 |.else 2197 | sub r4, 8 2198 | push "Bit shift by negative number" 2199 |.if WIN 2200 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2201 | push aword [r0] 2202 |.else 2203 | PUSH_ADDR zend_ce_arithmetic_error, r0 2204 |.endif 2205 | EXT_CALL zend_throw_error, r0 2206 | add r4, 16 2207 |.endif 2208 | jmp ->exception_handler_free_op1_op2 2209 return 1; 2210} 2211 2212static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2213{ 2214 |->mod_by_zero: 2215 | mov RX, EX->opline 2216 |.if X64 2217 |.if WIN 2218 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2219 | mov CARG1, aword [CARG1] 2220 |.else 2221 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2222 |.endif 2223 | LOAD_ADDR CARG2, "Modulo by zero" 2224 | EXT_CALL zend_throw_error, r0 2225 |.else 2226 | sub r4, 8 2227 | push "Modulo by zero" 2228 |.if WIN 2229 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2230 | push aword [r0] 2231 |.else 2232 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2233 |.endif 2234 | EXT_CALL zend_throw_error, r0 2235 | add r4, 16 2236 |.endif 2237 | jmp ->exception_handler_free_op1_op2 2238 return 1; 2239} 2240 2241static int zend_jit_invalid_this_stub(dasm_State **Dst) 2242{ 2243 |->invalid_this: 2244 | UNDEF_OPLINE_RESULT 2245 |.if X64 2246 | xor CARG1, CARG1 2247 | LOAD_ADDR CARG2, "Using $this when not in object context" 2248 | EXT_CALL zend_throw_error, r0 2249 |.else 2250 | sub r4, 8 2251 | push "Using $this when not in object context" 2252 | push 0 2253 | EXT_CALL zend_throw_error, r0 2254 | add r4, 16 2255 |.endif 2256 | jmp ->exception_handler 2257 return 1; 2258} 2259 2260static int zend_jit_double_one_stub(dasm_State **Dst) 2261{ 2262 |->one: 2263 |.dword 0, 0x3ff00000 2264 return 1; 2265} 2266 2267static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2268{ 2269 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2270 return 1; 2271 } 2272 2273 |->hybrid_runtime_jit: 2274 | EXT_CALL zend_runtime_jit, r0 2275 | JMP_IP 2276 return 1; 2277} 2278 2279static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2280{ 2281 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2282 return 1; 2283 } 2284 2285 |->hybrid_profile_jit: 2286 | // ++zend_jit_profile_counter; 2287 | .if X64 2288 | LOAD_ADDR r0, &zend_jit_profile_counter 2289 | inc aword [r0] 2290 | .else 2291 | inc aword [&zend_jit_profile_counter] 2292 | .endif 2293 | // op_array = (zend_op_array*)EX(func); 2294 | mov r0, EX->func 2295 | // run_time_cache = EX(run_time_cache); 2296 | mov r2, EX->run_time_cache 2297 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2298 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2299 | // ++ZEND_COUNTER_INFO(op_array) 2300 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2301 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2302 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2303 return 1; 2304} 2305 2306static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2307{ 2308 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2309 return 1; 2310 } 2311 2312 |->hybrid_hot_code: 2313 | mov word [r2], ZEND_JIT_COUNTER_INIT 2314 | mov FCARG1a, FP 2315 | GET_IP FCARG2a 2316 | EXT_CALL zend_jit_hot_func, r0 2317 | JMP_IP 2318 return 1; 2319} 2320 2321/* 2322 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2323 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2324 * property disclosure and research opportunities" email 2325 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2326 * 2327 * In addition we use a variation of Knuth's multiplicative hash function 2328 * described at https://code.i-harness.com/en/q/a21ce 2329 * 2330 * uint64_t hash(uint64_t x) { 2331 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2332 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2333 * x = x ^ (x >> 31); 2334 * return x; 2335 * } 2336 * 2337 * uint_32_t hash(uint32_t x) { 2338 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2339 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2340 * x = (x >> 16) ^ x; 2341 * return x; 2342 * } 2343 * 2344 */ 2345static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2346{ 2347 | mov r0, EX->func 2348 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2349 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2350 | sub word [r2], cost 2351 | jle ->hybrid_hot_code 2352 | GET_IP r2 2353 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2354 | // divide by sizeof(zend_op) 2355 | .if X64 2356 || ZEND_ASSERT(sizeof(zend_op) == 32); 2357 | sar r2, 2 2358 | .else 2359 || ZEND_ASSERT(sizeof(zend_op) == 28); 2360 | imul r2, 0xb6db6db7 2361 | .endif 2362 | .if X64 2363 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2364 | .else 2365 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2366 | .endif 2367 return 1; 2368} 2369 2370static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2371{ 2372 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2373 return 1; 2374 } 2375 2376 |->hybrid_func_hot_counter: 2377 2378 return zend_jit_hybrid_hot_counter_stub(Dst, 2379 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2380} 2381 2382static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2383{ 2384 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2385 return 1; 2386 } 2387 2388 |->hybrid_loop_hot_counter: 2389 2390 return zend_jit_hybrid_hot_counter_stub(Dst, 2391 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2392} 2393 2394static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2395{ 2396 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2397 return 1; 2398 } 2399 2400 |->hybrid_hot_trace: 2401 | mov word [r2], ZEND_JIT_COUNTER_INIT 2402 | mov FCARG1a, FP 2403 | GET_IP FCARG2a 2404 | EXT_CALL zend_jit_trace_hot_root, r0 2405 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2406 | jl >1 2407 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2408 | LOAD_IP 2409 | JMP_IP 2410 |1: 2411 | EXT_JMP zend_jit_halt_op->handler, r0 2412 return 1; 2413} 2414 2415static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2416{ 2417 | mov r0, EX->func 2418 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2419 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2420 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2421 | sub word [r2], cost 2422 | jle ->hybrid_hot_trace 2423 | jmp aword [IP + r1] 2424 return 1; 2425} 2426 2427static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2428{ 2429 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2430 return 1; 2431 } 2432 2433 |->hybrid_func_trace_counter: 2434 2435 return zend_jit_hybrid_trace_counter_stub(Dst, 2436 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2437} 2438 2439static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2440{ 2441 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2442 return 1; 2443 } 2444 2445 |->hybrid_ret_trace_counter: 2446 2447 return zend_jit_hybrid_trace_counter_stub(Dst, 2448 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2449} 2450 2451static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2452{ 2453 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2454 return 1; 2455 } 2456 2457 |->hybrid_loop_trace_counter: 2458 2459 return zend_jit_hybrid_trace_counter_stub(Dst, 2460 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2461} 2462 2463static int zend_jit_trace_halt_stub(dasm_State **Dst) 2464{ 2465 |->trace_halt: 2466 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2467 | ADD_HYBRID_SPAD 2468 | EXT_JMP zend_jit_halt_op->handler, r0 2469 } else if (GCC_GLOBAL_REGS) { 2470 | add r4, SPAD // stack alignment 2471 | xor IP, IP // PC must be zero 2472 | ret 2473 } else { 2474 | mov FP, aword T2 // restore FP 2475 | mov RX, aword T3 // restore IP 2476 | add r4, NR_SPAD // stack alignment 2477 | mov r0, -1 // ZEND_VM_RETURN 2478 | ret 2479 } 2480 return 1; 2481} 2482 2483static int zend_jit_trace_exit_stub(dasm_State **Dst) 2484{ 2485 |->trace_exit: 2486 | 2487 | // Save CPU registers 2488 |.if X64 2489 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2490 | mov aword [r4+15*8], r15 2491 | mov aword [r4+11*8], r11 2492 | mov aword [r4+10*8], r10 2493 | mov aword [r4+9*8], r9 2494 | mov aword [r4+8*8], r8 2495 | mov aword [r4+7*8], rdi 2496 | mov aword [r4+6*8], rsi 2497 | mov aword [r4+2*8], rdx 2498 | mov aword [r4+1*8], rcx 2499 | mov aword [r4+0*8], rax 2500 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2501 | mov FCARG2a, r4 2502 | movsd qword [r4+16*8+15*8], xmm15 2503 | movsd qword [r4+16*8+14*8], xmm14 2504 | movsd qword [r4+16*8+13*8], xmm13 2505 | movsd qword [r4+16*8+12*8], xmm12 2506 | movsd qword [r4+16*8+11*8], xmm11 2507 | movsd qword [r4+16*8+10*8], xmm10 2508 | movsd qword [r4+16*8+9*8], xmm9 2509 | movsd qword [r4+16*8+8*8], xmm8 2510 | movsd qword [r4+16*8+7*8], xmm7 2511 | movsd qword [r4+16*8+6*8], xmm6 2512 | movsd qword [r4+16*8+5*8], xmm5 2513 | movsd qword [r4+16*8+4*8], xmm4 2514 | movsd qword [r4+16*8+3*8], xmm3 2515 | movsd qword [r4+16*8+2*8], xmm2 2516 | movsd qword [r4+16*8+1*8], xmm1 2517 | movsd qword [r4+16*8+0*8], xmm0 2518 |.if X64WIN 2519 | sub r4, 32 /* shadow space */ 2520 |.endif 2521 |.else 2522 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2523 | mov aword [r4+7*4], edi 2524 | mov aword [r4+2*4], edx 2525 | mov aword [r4+1*4], ecx 2526 | mov aword [r4+0*4], eax 2527 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2528 | mov FCARG2a, r4 2529 | movsd qword [r4+8*4+7*8], xmm7 2530 | movsd qword [r4+8*4+6*8], xmm6 2531 | movsd qword [r4+8*4+5*8], xmm5 2532 | movsd qword [r4+8*4+4*8], xmm4 2533 | movsd qword [r4+8*4+3*8], xmm3 2534 | movsd qword [r4+8*4+2*8], xmm2 2535 | movsd qword [r4+8*4+1*8], xmm1 2536 | movsd qword [r4+8*4+0*8], xmm0 2537 |.endif 2538 | 2539 | // EX(opline) = opline 2540 | SAVE_IP 2541 | // zend_jit_trace_exit(trace_num, exit_num) 2542 | EXT_CALL zend_jit_trace_exit, r0 2543 |.if X64WIN 2544 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2545 |.elif X64 2546 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2547 |.else 2548 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2549 |.endif 2550 2551 | test eax, eax 2552 | jne >1 2553 2554 | // execute_data = EG(current_execute_data) 2555 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2556 | // opline = EX(opline) 2557 | LOAD_IP 2558 2559 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2560 | ADD_HYBRID_SPAD 2561 | JMP_IP 2562 } else if (GCC_GLOBAL_REGS) { 2563 | add r4, SPAD // stack alignment 2564 | JMP_IP 2565 } else { 2566 | mov FP, aword T2 // restore FP 2567 | mov RX, aword T3 // restore IP 2568 | add r4, NR_SPAD // stack alignment 2569 | mov r0, 1 // ZEND_VM_ENTER 2570 | ret 2571 } 2572 2573 |1: 2574 | jl ->trace_halt 2575 2576 | // execute_data = EG(current_execute_data) 2577 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2578 | // opline = EX(opline) 2579 | LOAD_IP 2580 2581 | // check for interrupt (try to avoid this ???) 2582 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 2583 | jne ->interrupt_handler 2584 2585 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2586 | ADD_HYBRID_SPAD 2587 | mov r0, EX->func 2588 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2589 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2590 | jmp aword [IP + r0] 2591 } else if (GCC_GLOBAL_REGS) { 2592 | add r4, SPAD // stack alignment 2593 | mov r0, EX->func 2594 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2595 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2596 | jmp aword [IP + r0] 2597 } else { 2598 | mov IP, aword EX->opline 2599 | mov FCARG1a, FP 2600 | mov r0, EX->func 2601 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2602 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2603 | call aword [IP + r0] 2604 | test eax, eax 2605 | jl ->trace_halt 2606 | mov FP, aword T2 // restore FP 2607 | mov RX, aword T3 // restore IP 2608 | add r4, NR_SPAD // stack alignment 2609 | mov r0, 1 // ZEND_VM_ENTER 2610 | ret 2611 } 2612 2613 return 1; 2614} 2615 2616static int zend_jit_trace_escape_stub(dasm_State **Dst) 2617{ 2618 |->trace_escape: 2619 | 2620 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2621 | ADD_HYBRID_SPAD 2622 | JMP_IP 2623 } else if (GCC_GLOBAL_REGS) { 2624 | add r4, SPAD // stack alignment 2625 | JMP_IP 2626 } else { 2627 | mov FP, aword T2 // restore FP 2628 | mov RX, aword T3 // restore IP 2629 | add r4, NR_SPAD // stack alignment 2630 | mov r0, 1 // ZEND_VM_ENTER 2631 | ret 2632 } 2633 2634 return 1; 2635} 2636 2637/* Keep 32 exit points in a single code block */ 2638#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2639#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2640 2641static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2642{ 2643 uint32_t i; 2644 2645 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2646 | push byte i 2647 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2648 } 2649 | push byte i 2650 |// 1: 2651 | add aword [r4], n 2652 | jmp ->trace_exit 2653 2654 return 1; 2655} 2656 2657#ifdef CONTEXT_THREADED_JIT 2658static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2659{ 2660 |->context_threaded_call: 2661 | pop r0 2662 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2663 | ADD_HYBRID_SPAD 2664 | jmp aword [IP] 2665 } else if (GCC_GLOBAL_REGS) { 2666 | add r4, SPAD // stack alignment 2667 | jmp aword [IP] 2668 } else { 2669 ZEND_UNREACHABLE(); 2670 // TODO: context threading can't work without GLOBAL REGS because we have to change 2671 // the value of execute_data in execute_ex() 2672 | mov FCARG1a, FP 2673 | mov r0, aword [FP] 2674 | mov FP, aword T2 // restore FP 2675 | mov RX, aword T3 // restore IP 2676 | add r4, NR_SPAD // stack alignment 2677 | jmp aword [r0] 2678 } 2679 return 1; 2680} 2681#endif 2682 2683static int zend_jit_assign_to_variable(dasm_State **Dst, 2684 const zend_op *opline, 2685 zend_jit_addr var_use_addr, 2686 zend_jit_addr var_addr, 2687 uint32_t var_info, 2688 uint32_t var_def_info, 2689 zend_uchar val_type, 2690 zend_jit_addr val_addr, 2691 uint32_t val_info, 2692 zend_jit_addr res_addr, 2693 zend_bool check_exception); 2694 2695static int zend_jit_assign_const_stub(dasm_State **Dst) 2696{ 2697 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2698 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2699 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2700 2701 |->assign_const: 2702 |.if X64WIN 2703 | sub r4, 0x28 2704 |.elif X64 2705 | sub r4, 8 2706 |.else 2707 | sub r4, 12 2708 |.endif 2709 if (!zend_jit_assign_to_variable( 2710 Dst, NULL, 2711 var_addr, var_addr, -1, -1, 2712 IS_CONST, val_addr, val_info, 2713 0, 0)) { 2714 return 0; 2715 } 2716 |.if X64WIN 2717 | add r4, 0x28 2718 |.elif X64 2719 | add r4, 8 2720 |.else 2721 | add r4, 12 2722 |.endif 2723 | ret 2724 return 1; 2725} 2726 2727static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2728{ 2729 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2730 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2731 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2732 2733 |->assign_tmp: 2734 |.if X64WIN 2735 | sub r4, 0x28 2736 |.elif X64 2737 | sub r4, 8 2738 |.else 2739 | sub r4, 12 2740 |.endif 2741 if (!zend_jit_assign_to_variable( 2742 Dst, NULL, 2743 var_addr, var_addr, -1, -1, 2744 IS_TMP_VAR, val_addr, val_info, 2745 0, 0)) { 2746 return 0; 2747 } 2748 |.if X64WIN 2749 | add r4, 0x28 2750 |.elif X64 2751 | add r4, 8 2752 |.else 2753 | add r4, 12 2754 |.endif 2755 | ret 2756 return 1; 2757} 2758 2759static int zend_jit_assign_var_stub(dasm_State **Dst) 2760{ 2761 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2762 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2763 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2764 2765 |->assign_var: 2766 |.if X64WIN 2767 | sub r4, 0x28 2768 |.elif X64 2769 | sub r4, 8 2770 |.else 2771 | sub r4, 12 2772 |.endif 2773 if (!zend_jit_assign_to_variable( 2774 Dst, NULL, 2775 var_addr, var_addr, -1, -1, 2776 IS_VAR, val_addr, val_info, 2777 0, 0)) { 2778 return 0; 2779 } 2780 |.if X64WIN 2781 | add r4, 0x28 2782 |.elif X64 2783 | add r4, 8 2784 |.else 2785 | add r4, 12 2786 |.endif 2787 | ret 2788 return 1; 2789} 2790 2791static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2792{ 2793 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2794 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2795 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2796 2797 |->assign_cv_noref: 2798 |.if X64WIN 2799 | sub r4, 0x28 2800 |.elif X64 2801 | sub r4, 8 2802 |.else 2803 | sub r4, 12 2804 |.endif 2805 if (!zend_jit_assign_to_variable( 2806 Dst, NULL, 2807 var_addr, var_addr, -1, -1, 2808 IS_CV, val_addr, val_info, 2809 0, 0)) { 2810 return 0; 2811 } 2812 |.if X64WIN 2813 | add r4, 0x28 2814 |.elif X64 2815 | add r4, 8 2816 |.else 2817 | add r4, 12 2818 |.endif 2819 | ret 2820 return 1; 2821} 2822 2823static int zend_jit_assign_cv_stub(dasm_State **Dst) 2824{ 2825 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2826 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2827 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2828 2829 |->assign_cv: 2830 |.if X64WIN 2831 | sub r4, 0x28 2832 |.elif X64 2833 | sub r4, 8 2834 |.else 2835 | sub r4, 12 2836 |.endif 2837 if (!zend_jit_assign_to_variable( 2838 Dst, NULL, 2839 var_addr, var_addr, -1, -1, 2840 IS_CV, val_addr, val_info, 2841 0, 0)) { 2842 return 0; 2843 } 2844 |.if X64WIN 2845 | add r4, 0x28 2846 |.elif X64 2847 | add r4, 8 2848 |.else 2849 | add r4, 12 2850 |.endif 2851 | ret 2852 return 1; 2853} 2854 2855static const zend_jit_stub zend_jit_stubs[] = { 2856 JIT_STUB(interrupt_handler), 2857 JIT_STUB(exception_handler), 2858 JIT_STUB(exception_handler_undef), 2859 JIT_STUB(exception_handler_free_op1_op2), 2860 JIT_STUB(exception_handler_free_op2), 2861 JIT_STUB(leave_function), 2862 JIT_STUB(leave_throw), 2863 JIT_STUB(icall_throw), 2864 JIT_STUB(throw_cannot_pass_by_ref), 2865 JIT_STUB(undefined_offset), 2866 JIT_STUB(undefined_index), 2867 JIT_STUB(cannot_add_element), 2868 JIT_STUB(undefined_offset_ex), 2869 JIT_STUB(undefined_index_ex), 2870 JIT_STUB(cannot_add_element_ex), 2871 JIT_STUB(undefined_function), 2872 JIT_STUB(negative_shift), 2873 JIT_STUB(mod_by_zero), 2874 JIT_STUB(invalid_this), 2875 JIT_STUB(trace_halt), 2876 JIT_STUB(trace_exit), 2877 JIT_STUB(trace_escape), 2878 JIT_STUB(hybrid_runtime_jit), 2879 JIT_STUB(hybrid_profile_jit), 2880 JIT_STUB(hybrid_hot_code), 2881 JIT_STUB(hybrid_func_hot_counter), 2882 JIT_STUB(hybrid_loop_hot_counter), 2883 JIT_STUB(hybrid_hot_trace), 2884 JIT_STUB(hybrid_func_trace_counter), 2885 JIT_STUB(hybrid_ret_trace_counter), 2886 JIT_STUB(hybrid_loop_trace_counter), 2887 JIT_STUB(assign_const), 2888 JIT_STUB(assign_tmp), 2889 JIT_STUB(assign_var), 2890 JIT_STUB(assign_cv_noref), 2891 JIT_STUB(assign_cv), 2892 JIT_STUB(double_one), 2893#ifdef CONTEXT_THREADED_JIT 2894 JIT_STUB(context_threaded_call), 2895#endif 2896}; 2897 2898#if ZTS && defined(ZEND_WIN32) 2899extern uint32_t _tls_index; 2900extern char *_tls_start; 2901extern char *_tls_end; 2902#endif 2903 2904static int zend_jit_setup(void) 2905{ 2906 if (!zend_cpu_supports_sse2()) { 2907 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2908 return FAILURE; 2909 } 2910 allowed_opt_flags = 0; 2911 if (zend_cpu_supports_avx()) { 2912 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2913 } 2914 2915#if ZTS 2916# ifdef _WIN64 2917 tsrm_tls_index = _tls_index * sizeof(void*); 2918 2919 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2920 /* Probably, it might be better solution */ 2921 do { 2922 void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; 2923 void *val = _tsrm_ls_cache; 2924 size_t offset = 0; 2925 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2926 2927 while (offset < size) { 2928 if (*tls_mem == val) { 2929 tsrm_tls_offset = offset; 2930 break; 2931 } 2932 tls_mem++; 2933 offset += sizeof(void*); 2934 } 2935 if (offset >= size) { 2936 // TODO: error message ??? 2937 return FAILURE; 2938 } 2939 } while(0); 2940# elif ZEND_WIN32 2941 tsrm_tls_index = _tls_index * sizeof(void*); 2942 2943 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2944 /* Probably, it might be better solution */ 2945 do { 2946 void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; 2947 void *val = _tsrm_ls_cache; 2948 size_t offset = 0; 2949 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2950 2951 while (offset < size) { 2952 if (*tls_mem == val) { 2953 tsrm_tls_offset = offset; 2954 break; 2955 } 2956 tls_mem++; 2957 offset += sizeof(void*); 2958 } 2959 if (offset >= size) { 2960 // TODO: error message ??? 2961 return FAILURE; 2962 } 2963 } while(0); 2964# elif defined(__APPLE__) && defined(__x86_64__) 2965 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2966 if (tsrm_ls_cache_tcb_offset == 0) { 2967 size_t *ti; 2968 __asm__( 2969 "leaq __tsrm_ls_cache(%%rip),%0" 2970 : "=r" (ti)); 2971 tsrm_tls_offset = ti[2]; 2972 tsrm_tls_index = ti[1] * 8; 2973 } 2974# elif defined(__GNUC__) && defined(__x86_64__) 2975 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2976 if (tsrm_ls_cache_tcb_offset == 0) { 2977#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) 2978 size_t ret; 2979 2980 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2981 : "=r" (ret)); 2982 tsrm_ls_cache_tcb_offset = ret; 2983#else 2984 size_t *ti; 2985 2986 __asm__( 2987 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2988 : "=a" (ti)); 2989 tsrm_tls_offset = ti[1]; 2990 tsrm_tls_index = ti[0] * 16; 2991#endif 2992 } 2993# elif defined(__GNUC__) && defined(__i386__) 2994 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2995 if (tsrm_ls_cache_tcb_offset == 0) { 2996#if !defined(__FreeBSD__) && !defined(__OpenBSD__) 2997 size_t ret; 2998 2999 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 3000 : "=a" (ret)); 3001 tsrm_ls_cache_tcb_offset = ret; 3002#else 3003 size_t *ti, _ebx, _ecx, _edx; 3004 3005 __asm__( 3006 "call 1f\n" 3007 ".subsection 1\n" 3008 "1:\tmovl (%%esp), %%ebx\n\t" 3009 "ret\n" 3010 ".previous\n\t" 3011 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 3012 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 3013 "call ___tls_get_addr@plt\n\t" 3014 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 3015 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 3016 tsrm_tls_offset = ti[1]; 3017 tsrm_tls_index = ti[0] * 8; 3018#endif 3019 } 3020# endif 3021#endif 3022 3023 return SUCCESS; 3024} 3025 3026static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 3027{ 3028 | int3 3029 return 1; 3030} 3031 3032static int zend_jit_align_func(dasm_State **Dst) 3033{ 3034 reuse_ip = 0; 3035 delayed_call_chain = 0; 3036 last_valid_opline = NULL; 3037 use_last_vald_opline = 0; 3038 track_last_valid_opline = 0; 3039 jit_return_label = -1; 3040 |.align 16 3041 return 1; 3042} 3043 3044static int zend_jit_prologue(dasm_State **Dst) 3045{ 3046 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3047 | SUB_HYBRID_SPAD 3048 } else if (GCC_GLOBAL_REGS) { 3049 | sub r4, SPAD // stack alignment 3050 } else { 3051 | sub r4, NR_SPAD // stack alignment 3052 | mov aword T2, FP // save FP 3053 | mov aword T3, RX // save IP 3054 | mov FP, FCARG1a 3055 } 3056 return 1; 3057} 3058 3059static int zend_jit_label(dasm_State **Dst, unsigned int label) 3060{ 3061 |=>label: 3062 return 1; 3063} 3064 3065static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3066{ 3067 | // call->prev_execute_data = EX(call); 3068 if (call_level == 1) { 3069 | mov aword EX:RX->prev_execute_data, 0 3070 } else { 3071 | mov r0, EX->call 3072 | mov EX:RX->prev_execute_data, r0 3073 } 3074 | // EX(call) = call; 3075 | mov EX->call, RX 3076 3077 delayed_call_chain = 0; 3078 3079 return 1; 3080} 3081 3082static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3083{ 3084 if (last_valid_opline == opline) { 3085 zend_jit_use_last_valid_opline(); 3086 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3087 zend_jit_use_last_valid_opline(); 3088 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3089 } else { 3090 | LOAD_IP_ADDR opline 3091 } 3092 zend_jit_set_last_valid_opline(opline); 3093 3094 return 1; 3095} 3096 3097static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3098{ 3099 if (last_valid_opline == opline) { 3100 zend_jit_use_last_valid_opline(); 3101 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3102 zend_jit_use_last_valid_opline(); 3103 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3104 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3105 | LOAD_ADDR RX, opline 3106 | mov aword EX->opline, RX 3107 } else { 3108 | LOAD_IP_ADDR opline 3109 } 3110 zend_jit_set_last_valid_opline(opline); 3111 3112 return 1; 3113} 3114 3115static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3116{ 3117 if (delayed_call_chain) { 3118 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3119 return 0; 3120 } 3121 } 3122 if (!zend_jit_set_ip(Dst, opline)) { 3123 return 0; 3124 } 3125 reuse_ip = 0; 3126 return 1; 3127} 3128 3129static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3130{ 3131#if 0 3132 if (!zend_jit_set_valid_ip(Dst, opline)) { 3133 return 0; 3134 } 3135 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3136 | jne ->interrupt_handler 3137#else 3138 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3139 if (exit_addr) { 3140 | jne &exit_addr 3141 } else if (last_valid_opline == opline) { 3142 || zend_jit_use_last_valid_opline(); 3143 | jne ->interrupt_handler 3144 } else { 3145 | jne >1 3146 |.cold_code 3147 |1: 3148 | LOAD_IP_ADDR opline 3149 | jmp ->interrupt_handler 3150 |.code 3151 } 3152#endif 3153 return 1; 3154} 3155 3156static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3157{ 3158 if (timeout_exit_addr) { 3159 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3160 | je =>loop_label 3161 | jmp &timeout_exit_addr 3162 } else { 3163 | jmp =>loop_label 3164 } 3165 return 1; 3166} 3167 3168static int zend_jit_check_exception(dasm_State **Dst) 3169{ 3170 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 3171 | jne ->exception_handler 3172 return 1; 3173} 3174 3175static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3176{ 3177 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3178 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 3179 | jne ->exception_handler_undef 3180 return 1; 3181 } 3182 return zend_jit_check_exception(Dst); 3183} 3184 3185static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3186{ 3187 zend_regset regset = ZEND_REGSET_SCRATCH; 3188 3189#if ZTS 3190 if (1) { 3191#else 3192 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3193#endif 3194 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3195 if (parent) { 3196 int i; 3197 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3198 zend_jit_trace_stack *parent_stack = 3199 parent->stack_map + 3200 parent->exit_info[exit_num].stack_offset; 3201 3202 for (i = 0; i < parent_vars_count; i++) { 3203 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3204 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3205 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3206 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_R0) { 3207 ZEND_REGSET_EXCL(regset, ZREG_R0); 3208 } 3209 } 3210 } 3211 } 3212 } 3213 3214 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3215 ZEND_REGSET_EXCL(regset, ZREG_R0); 3216 } 3217 3218 current_trace_num = trace_num; 3219 3220 | // EG(jit_trace_num) = trace_num; 3221 if (regset == ZEND_REGSET_EMPTY) { 3222 | push r0 3223 | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, r0 3224 | pop r0 3225 } else { 3226 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3227 3228 | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3229 (void)tmp; 3230 } 3231 3232 return 1; 3233} 3234 3235/* This taken from LuaJIT. Thanks to Mike Pall. */ 3236static uint32_t _asm_x86_inslen(const uint8_t* p) 3237{ 3238 static const uint8_t map_op1[256] = { 3239 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3240 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3241 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3242 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3243#if defined(__x86_64__) || defined(_M_X64) 3244 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3245#else 3246 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3247#endif 3248 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3249 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3250 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3251 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3252 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3253#if defined(__x86_64__) || defined(_M_X64) 3254 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3255#else 3256 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3257#endif 3258 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3259 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3260 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3261 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3262 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3263 }; 3264 static const uint8_t map_op2[256] = { 3265 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3266 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3267 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3268 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3269 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3270 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3271 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3272 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3273 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3274 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3275 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3276 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3277 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3278 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3279 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3280 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3281 }; 3282 uint32_t result = 0; 3283 uint32_t prefixes = 0; 3284 uint32_t x = map_op1[*p]; 3285 3286 for (;;) { 3287 switch (x >> 4) { 3288 case 0: 3289 return result + x + (prefixes & 4); 3290 case 1: 3291 prefixes |= x; 3292 x = map_op1[*++p]; 3293 result++; 3294 break; 3295 case 2: 3296 x = map_op2[*++p]; 3297 break; 3298 case 3: 3299 p++; 3300 goto mrm; 3301 case 4: 3302 result -= (prefixes & 2); 3303 /* fallthrough */ 3304 case 5: 3305 return result + (x & 15); 3306 case 6: /* Group 3. */ 3307 if (p[1] & 0x38) { 3308 x = 2; 3309 } else if ((prefixes & 2) && (x == 0x66)) { 3310 x = 4; 3311 } 3312 goto mrm; 3313 case 7: /* VEX c4/c5. */ 3314#if !defined(__x86_64__) && !defined(_M_X64) 3315 if (p[1] < 0xc0) { 3316 x = 2; 3317 goto mrm; 3318 } 3319#endif 3320 if (x == 0x70) { 3321 x = *++p & 0x1f; 3322 result++; 3323 if (x >= 2) { 3324 p += 2; 3325 result += 2; 3326 goto mrm; 3327 } 3328 } 3329 p++; 3330 result++; 3331 x = map_op2[*++p]; 3332 break; 3333 case 8: 3334 result -= (prefixes & 2); 3335 /* fallthrough */ 3336 case 9: 3337mrm: 3338 /* ModR/M and possibly SIB. */ 3339 result += (x & 15); 3340 x = *++p; 3341 switch (x >> 6) { 3342 case 0: 3343 if ((x & 7) == 5) { 3344 return result + 4; 3345 } 3346 break; 3347 case 1: 3348 result++; 3349 break; 3350 case 2: 3351 result += 4; 3352 break; 3353 case 3: 3354 return result; 3355 } 3356 if ((x & 7) == 4) { 3357 result++; 3358 if (x < 0x40 && (p[1] & 7) == 5) { 3359 result += 4; 3360 } 3361 } 3362 return result; 3363 } 3364 } 3365} 3366 3367typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3368typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3369 3370static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3371{ 3372 int ret = 0; 3373 uint8_t *p, *end; 3374 3375 if (jmp_table_size) { 3376 const void **jmp_slot = (const void **)((char*)code + size); 3377 3378 size -= jmp_table_size * sizeof(void*); 3379 do { 3380 jmp_slot--; 3381 if (*jmp_slot == from_addr) { 3382 *jmp_slot = to_addr; 3383 ret++; 3384 } 3385 } while (--jmp_table_size); 3386 } 3387 3388 p = (uint8_t*)code; 3389 end = p + size - 5; 3390 while (p < end) { 3391 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3392 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3393 ret++; 3394 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3395 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3396 ret++; 3397 } 3398 p += _asm_x86_inslen(p); 3399 } 3400#ifdef HAVE_VALGRIND 3401 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3402#endif 3403 return ret; 3404} 3405 3406static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3407{ 3408 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3409} 3410 3411static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3412{ 3413 const void *link_addr; 3414 size_t prologue_size; 3415 3416 /* Skip prologue. */ 3417 // TODO: don't hardcode this ??? 3418 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3419#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3420 prologue_size = 0; 3421#elif defined(__x86_64__) || defined(_M_X64) 3422 // sub r4, HYBRID_SPAD 3423 prologue_size = 4; 3424#else 3425 // sub r4, HYBRID_SPAD 3426 prologue_size = 3; 3427#endif 3428 } else if (GCC_GLOBAL_REGS) { 3429 // sub r4, SPAD // stack alignment 3430#if defined(__x86_64__) || defined(_M_X64) 3431 prologue_size = 4; 3432#else 3433 prologue_size = 3; 3434#endif 3435 } else { 3436 // sub r4, NR_SPAD // stack alignment 3437 // mov aword T2, FP // save FP 3438 // mov aword T3, RX // save IP 3439 // mov FP, FCARG1a 3440#if defined(__x86_64__) || defined(_M_X64) 3441 prologue_size = 17; 3442#else 3443 prologue_size = 13; 3444#endif 3445 } 3446 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3447 3448 if (timeout_exit_addr) { 3449 /* Check timeout for links to LOOP */ 3450 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3451 | je &link_addr 3452 | jmp &timeout_exit_addr 3453 } else { 3454 | jmp &link_addr 3455 } 3456 return 1; 3457} 3458 3459static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler, const zend_op *opline) 3460{ 3461#if 0 3462 | jmp ->trace_escape 3463#else 3464 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3465 | ADD_HYBRID_SPAD 3466 if (!original_handler) { 3467 | JMP_IP 3468 } else { 3469 | mov r0, EX->func 3470 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3471 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3472 | jmp aword [IP + r0] 3473 } 3474 } else if (GCC_GLOBAL_REGS) { 3475 | add r4, SPAD // stack alignment 3476 if (!original_handler) { 3477 | JMP_IP 3478 } else { 3479 | mov r0, EX->func 3480 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3481 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3482 | jmp aword [IP + r0] 3483 } 3484 } else { 3485 if (original_handler) { 3486 | mov FCARG1a, FP 3487 | mov r0, EX->func 3488 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3489 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3490 | call aword [IP + r0] 3491 } 3492 | mov FP, aword T2 // restore FP 3493 | mov RX, aword T3 // restore IP 3494 | add r4, NR_SPAD // stack alignment 3495 if (!original_handler || !opline || 3496 (opline->opcode != ZEND_RETURN 3497 && opline->opcode != ZEND_RETURN_BY_REF 3498 && opline->opcode != ZEND_GENERATOR_RETURN 3499 && opline->opcode != ZEND_GENERATOR_CREATE 3500 && opline->opcode != ZEND_YIELD 3501 && opline->opcode != ZEND_YIELD_FROM)) { 3502 | mov r0, 2 // ZEND_VM_LEAVE 3503 } 3504 | ret 3505 } 3506#endif 3507 return 1; 3508} 3509 3510static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3511{ 3512 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3513 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3514 3515 if (!exit_addr) { 3516 return 0; 3517 } 3518 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3519 3520 return 1; 3521} 3522 3523static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3524{ 3525 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3526 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3527 3528 if (!exit_addr) { 3529 return 0; 3530 } 3531 3532 | GET_ZVAL_LVAL ZREG_FCARG1a, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3533 if (op_info & MAY_BE_ARRAY_PACKED) { 3534 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3535 | jz &exit_addr 3536 } else { 3537 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3538 | jnz &exit_addr 3539 } 3540 3541 return 1; 3542} 3543 3544static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3545{ 3546 zend_jit_op_array_trace_extension *jit_extension = 3547 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3548 size_t offset = jit_extension->offset; 3549 const void *handler = 3550 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3551 3552 if (!zend_jit_set_valid_ip(Dst, opline)) { 3553 return 0; 3554 } 3555 if (!GCC_GLOBAL_REGS) { 3556 | mov FCARG1a, FP 3557 } 3558 | EXT_CALL handler, r0 3559 if (may_throw 3560 && opline->opcode != ZEND_RETURN 3561 && opline->opcode != ZEND_RETURN_BY_REF) { 3562 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 3563 | jne ->exception_handler 3564 } 3565 3566 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3567 trace++; 3568 } 3569 3570 if (!GCC_GLOBAL_REGS 3571 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3572 if (opline->opcode == ZEND_RETURN || 3573 opline->opcode == ZEND_RETURN_BY_REF || 3574 opline->opcode == ZEND_DO_UCALL || 3575 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3576 opline->opcode == ZEND_DO_FCALL || 3577 opline->opcode == ZEND_GENERATOR_CREATE) { 3578 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r1 3579 } 3580 } 3581 3582 if (zend_jit_trace_may_exit(op_array, opline)) { 3583 if (opline->opcode == ZEND_RETURN || 3584 opline->opcode == ZEND_RETURN_BY_REF || 3585 opline->opcode == ZEND_GENERATOR_CREATE) { 3586 3587 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3588#if 0 3589 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3590 | cmp IP, zend_jit_halt_op 3591 | je ->trace_halt 3592#endif 3593 } else if (GCC_GLOBAL_REGS) { 3594 | test IP, IP 3595 | je ->trace_halt 3596 } else { 3597 | test eax, eax 3598 | jl ->trace_halt 3599 } 3600 } else if (opline->opcode == ZEND_EXIT || 3601 opline->opcode == ZEND_GENERATOR_RETURN || 3602 opline->opcode == ZEND_YIELD || 3603 opline->opcode == ZEND_YIELD_FROM) { 3604 | jmp ->trace_halt 3605 } 3606 if (trace->op != ZEND_JIT_TRACE_END || 3607 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3608 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3609 3610 const zend_op *next_opline = trace->opline; 3611 const zend_op *exit_opline = NULL; 3612 uint32_t exit_point; 3613 const void *exit_addr; 3614 uint32_t old_info = 0; 3615 uint32_t old_res_info = 0; 3616 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3617 3618 if (zend_is_smart_branch(opline)) { 3619 zend_bool exit_if_true = 0; 3620 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3621 } else { 3622 switch (opline->opcode) { 3623 case ZEND_JMPZ: 3624 case ZEND_JMPNZ: 3625 case ZEND_JMPZ_EX: 3626 case ZEND_JMPNZ_EX: 3627 case ZEND_JMP_SET: 3628 case ZEND_COALESCE: 3629 case ZEND_JMP_NULL: 3630 case ZEND_FE_RESET_R: 3631 case ZEND_FE_RESET_RW: 3632 exit_opline = (trace->opline == opline + 1) ? 3633 OP_JMP_ADDR(opline, opline->op2) : 3634 opline + 1; 3635 break; 3636 case ZEND_JMPZNZ: 3637 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3638 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3639 OP_JMP_ADDR(opline, opline->op2); 3640 break; 3641 case ZEND_FE_FETCH_R: 3642 case ZEND_FE_FETCH_RW: 3643 if (opline->op2_type == IS_CV) { 3644 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3645 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3646 } 3647 exit_opline = (trace->opline == opline + 1) ? 3648 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3649 opline + 1; 3650 break; 3651 3652 } 3653 } 3654 3655 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3656 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3657 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3658 } 3659 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3660 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3661 3662 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3663 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3664 } 3665 switch (opline->opcode) { 3666 case ZEND_FE_FETCH_R: 3667 case ZEND_FE_FETCH_RW: 3668 if (opline->op2_type == IS_CV) { 3669 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3670 } 3671 break; 3672 } 3673 3674 if (!exit_addr) { 3675 return 0; 3676 } 3677 | CMP_IP next_opline 3678 | jne &exit_addr 3679 } 3680 } 3681 3682 zend_jit_set_last_valid_opline(trace->opline); 3683 3684 return 1; 3685} 3686 3687static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3688{ 3689 const void *handler; 3690 3691 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3692 handler = zend_get_opcode_handler_func(opline); 3693 } else { 3694 handler = opline->handler; 3695 } 3696 3697 if (!zend_jit_set_valid_ip(Dst, opline)) { 3698 return 0; 3699 } 3700 if (!GCC_GLOBAL_REGS) { 3701 | mov FCARG1a, FP 3702 } 3703 | EXT_CALL handler, r0 3704 if (may_throw) { 3705 zend_jit_check_exception(Dst); 3706 } 3707 3708 /* Skip the following OP_DATA */ 3709 switch (opline->opcode) { 3710 case ZEND_ASSIGN_DIM: 3711 case ZEND_ASSIGN_OBJ: 3712 case ZEND_ASSIGN_STATIC_PROP: 3713 case ZEND_ASSIGN_DIM_OP: 3714 case ZEND_ASSIGN_OBJ_OP: 3715 case ZEND_ASSIGN_STATIC_PROP_OP: 3716 case ZEND_ASSIGN_STATIC_PROP_REF: 3717 case ZEND_ASSIGN_OBJ_REF: 3718 zend_jit_set_last_valid_opline(opline + 2); 3719 break; 3720 default: 3721 zend_jit_set_last_valid_opline(opline + 1); 3722 break; 3723 } 3724 3725 return 1; 3726} 3727 3728static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3729{ 3730 if (!zend_jit_set_valid_ip(Dst, opline)) { 3731 return 0; 3732 } 3733 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3734 if (opline->opcode == ZEND_DO_UCALL || 3735 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3736 opline->opcode == ZEND_DO_FCALL || 3737 opline->opcode == ZEND_RETURN) { 3738 3739 /* Use inlined HYBRID VM handler */ 3740 const void *handler = opline->handler; 3741 3742 | ADD_HYBRID_SPAD 3743 | EXT_JMP handler, r0 3744 } else { 3745 const void *handler = zend_get_opcode_handler_func(opline); 3746 3747 | EXT_CALL handler, r0 3748 | ADD_HYBRID_SPAD 3749 | JMP_IP 3750 } 3751 } else { 3752 const void *handler = opline->handler; 3753 3754 if (GCC_GLOBAL_REGS) { 3755 | add r4, SPAD // stack alignment 3756 } else { 3757 | mov FCARG1a, FP 3758 | mov FP, aword T2 // restore FP 3759 | mov RX, aword T3 // restore IP 3760 | add r4, NR_SPAD // stack alignment 3761 } 3762 | EXT_JMP handler, r0 3763 } 3764 zend_jit_reset_last_valid_opline(); 3765 return 1; 3766} 3767 3768static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3769{ 3770 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3771 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3772 3773 if (!exit_addr) { 3774 return 0; 3775 } 3776 | CMP_IP opline 3777 | jne &exit_addr 3778 3779 zend_jit_set_last_valid_opline(opline); 3780 3781 return 1; 3782} 3783 3784static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3785{ 3786 | jmp =>target_label 3787 return 1; 3788} 3789 3790static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3791{ 3792 | CMP_IP next_opline 3793 | jne =>target_label 3794 3795 zend_jit_set_last_valid_opline(next_opline); 3796 3797 return 1; 3798} 3799 3800#ifdef CONTEXT_THREADED_JIT 3801static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3802{ 3803 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3804 if (opline->opcode == ZEND_DO_UCALL) { 3805 | call ->context_threaded_call 3806 } else { 3807 const zend_op *next_opline = opline + 1; 3808 3809 | CMP_IP next_opline 3810 | je =>next_block 3811 | call ->context_threaded_call 3812 } 3813 return 1; 3814} 3815#endif 3816 3817static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3818{ 3819#ifdef CONTEXT_THREADED_JIT 3820 return zend_jit_context_threaded_call(Dst, opline, next_block); 3821#else 3822 return zend_jit_tail_handler(Dst, opline); 3823#endif 3824} 3825 3826static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type) 3827{ 3828 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3829 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3830 3831 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3832 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3833 if (set_type) { 3834 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3835 } 3836 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3837 | SSE_SET_ZVAL_DVAL dst, Z_REG(src) 3838 if (set_type) { 3839 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3840 } 3841 } else { 3842 ZEND_UNREACHABLE(); 3843 } 3844 return 1; 3845} 3846 3847static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3848{ 3849 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3850 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3851 3852 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3853 | GET_ZVAL_LVAL Z_REG(dst), src 3854 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3855 | SSE_GET_ZVAL_DVAL Z_REG(dst), src 3856 } else { 3857 ZEND_UNREACHABLE(); 3858 } 3859 return 1; 3860} 3861 3862static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, zend_bool set_type) 3863{ 3864 zend_jit_addr src = ZEND_ADDR_REG(reg); 3865 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3866 3867 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3868} 3869 3870static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3871{ 3872 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3873 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3874 return zend_jit_spill_store(Dst, src, dst, info, 1); 3875 } 3876 return 1; 3877} 3878 3879static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3880{ 3881 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3882 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3883 zend_bool set_type = 1; 3884 3885 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3886 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3887 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3888 set_type = 0; 3889 } 3890 } 3891 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3892 } 3893 return 1; 3894} 3895 3896static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3897{ 3898 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3899 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3900 3901 return zend_jit_load_reg(Dst, src, dst, info); 3902} 3903 3904static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3905{ 3906 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3907 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3908 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3909 } 3910 return 1; 3911} 3912 3913static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3914{ 3915 if (!zend_jit_same_addr(src, dst)) { 3916 if (Z_MODE(src) == IS_REG) { 3917 if (Z_MODE(dst) == IS_REG) { 3918 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3919 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3920 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3921 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3922 } else { 3923 ZEND_UNREACHABLE(); 3924 } 3925 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3926 if (!Z_LOAD(src) && !Z_STORE(src)) { 3927 if (!zend_jit_spill_store(Dst, src, dst, info, 3928 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3929 JIT_G(current_frame) == NULL || 3930 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3931 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3932 )) { 3933 return 0; 3934 } 3935 } 3936 } else { 3937 ZEND_UNREACHABLE(); 3938 } 3939 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3940 if (Z_MODE(dst) == IS_REG) { 3941 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3942 return 0; 3943 } 3944 } else { 3945 ZEND_UNREACHABLE(); 3946 } 3947 } else { 3948 ZEND_UNREACHABLE(); 3949 } 3950 } 3951 return 1; 3952} 3953 3954static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3955{ 3956 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 3957 3958 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 3959 3960 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3961 if (!zend_jit_save_call_chain(Dst, -1)) { 3962 return 0; 3963 } 3964 } 3965 3966 ZEND_ASSERT(opline); 3967 3968 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3969 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3970 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3971 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3972 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3973 3974 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 3975 | GET_ZVAL_PTR r0, val_addr 3976 | GC_ADDREF r0 3977 |2: 3978 } 3979 3980 | LOAD_IP_ADDR (opline - 1) 3981 | jmp ->trace_escape 3982 |1: 3983 3984 return 1; 3985} 3986 3987static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 3988{ 3989 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3990 3991 if (reg == ZREG_LONG_MIN_MINUS_1) { 3992 |.if X64 3993 | SET_ZVAL_LVAL dst, 0x00000000 3994 | SET_ZVAL_W2 dst, 0xc3e00000 3995 |.else 3996 | SET_ZVAL_LVAL dst, 0x00200000 3997 | SET_ZVAL_W2 dst, 0xc1e00000 3998 |.endif 3999 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4000 } else if (reg == ZREG_LONG_MIN) { 4001 |.if X64 4002 | SET_ZVAL_LVAL dst, 0x00000000 4003 | SET_ZVAL_W2 dst, 0x80000000 4004 |.else 4005 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4006 |.endif 4007 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4008 } else if (reg == ZREG_LONG_MAX) { 4009 |.if X64 4010 | SET_ZVAL_LVAL dst, 0xffffffff 4011 | SET_ZVAL_W2 dst, 0x7fffffff 4012 |.else 4013 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4014 |.endif 4015 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4016 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4017 |.if X64 4018 | SET_ZVAL_LVAL dst, 0 4019 | SET_ZVAL_W2 dst, 0x43e00000 4020 |.else 4021 | SET_ZVAL_LVAL dst, 0 4022 | SET_ZVAL_W2 dst, 0x41e00000 4023 |.endif 4024 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4025 } else if (reg == ZREG_NULL) { 4026 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4027 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4028 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4029 | GET_ZVAL_PTR r1, dst 4030 | GC_ADDREF r1 4031 |1: 4032 } else if (reg == ZREG_ZVAL_COPY_R0) { 4033 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4034 4035 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4036 | TRY_ADDREF -1, ch, r2 4037 } else { 4038 ZEND_UNREACHABLE(); 4039 } 4040 return 1; 4041} 4042 4043static int zend_jit_free_trampoline(dasm_State **Dst) 4044{ 4045 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4046 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4047 | jz >1 4048 | mov FCARG1a, r0 4049 | EXT_CALL zend_jit_free_trampoline_helper, r0 4050 |1: 4051 return 1; 4052} 4053 4054static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4055{ 4056 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4057 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4058 } 4059 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4060 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4061 } 4062 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4063 return 0; 4064 } 4065 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4066 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4067 } else { 4068 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4069 } 4070 4071 if (may_overflow && 4072 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4073 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4074 int32_t exit_point; 4075 const void *exit_addr; 4076 zend_jit_trace_stack *stack; 4077 uint32_t old_op1_info, old_res_info = 0; 4078 4079 stack = JIT_G(current_frame)->stack; 4080 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4081 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4082 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4083 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4084 } else { 4085 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4086 } 4087 if (opline->result_type != IS_UNUSED) { 4088 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4089 if (opline->opcode == ZEND_PRE_INC) { 4090 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4091 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4092 } else if (opline->opcode == ZEND_PRE_DEC) { 4093 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4094 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4095 } else if (opline->opcode == ZEND_POST_INC) { 4096 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4097 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4098 } else if (opline->opcode == ZEND_POST_DEC) { 4099 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4100 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4101 } 4102 } 4103 4104 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4105 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4106 | jo &exit_addr 4107 4108 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4109 opline->result_type != IS_UNUSED) { 4110 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4111 } 4112 4113 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4114 if (opline->result_type != IS_UNUSED) { 4115 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4116 } 4117 } else if (may_overflow) { 4118 | jo >1 4119 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4120 opline->result_type != IS_UNUSED) { 4121 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4122 } 4123 |.cold_code 4124 |1: 4125 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4126 |.if X64 4127 | mov64 rax, 0x43e0000000000000 4128 | SET_ZVAL_LVAL op1_def_addr, rax 4129 |.else 4130 | SET_ZVAL_LVAL op1_def_addr, 0 4131 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4132 |.endif 4133 } else { 4134 |.if X64 4135 | mov64 rax, 0xc3e0000000000000 4136 | SET_ZVAL_LVAL op1_def_addr, rax 4137 |.else 4138 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4139 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4140 |.endif 4141 } 4142 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4143 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4144 } 4145 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4146 opline->result_type != IS_UNUSED) { 4147 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4148 } 4149 | jmp >3 4150 |.code 4151 } else { 4152 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4153 opline->result_type != IS_UNUSED) { 4154 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4155 } 4156 } 4157 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4158 |.cold_code 4159 |2: 4160 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4161 | SET_EX_OPLINE opline, r0 4162 if (op1_info & MAY_BE_UNDEF) { 4163 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4164 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4165 | mov FCARG1d, opline->op1.var 4166 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4167 | EXT_CALL zend_jit_undefined_op_helper, r0 4168 op1_info |= MAY_BE_NULL; 4169 } 4170 |2: 4171 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4172 4173 | // ZVAL_DEREF(var_ptr); 4174 if (op1_info & MAY_BE_REF) { 4175 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4176 | GET_Z_PTR FCARG1a, FCARG1a 4177 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4178 | jz >1 4179 if (RETURN_VALUE_USED(opline)) { 4180 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4181 } else { 4182 | xor FCARG2a, FCARG2a 4183 } 4184 if (opline->opcode == ZEND_PRE_INC) { 4185 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4186 } else if (opline->opcode == ZEND_PRE_DEC) { 4187 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4188 } else if (opline->opcode == ZEND_POST_INC) { 4189 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4190 } else if (opline->opcode == ZEND_POST_DEC) { 4191 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4192 } else { 4193 ZEND_UNREACHABLE(); 4194 } 4195 zend_jit_check_exception(Dst); 4196 | jmp >3 4197 |1: 4198 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4199 |2: 4200 } 4201 4202 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4203 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 4204 4205 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4206 | TRY_ADDREF op1_info, ah, r2 4207 } 4208 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4209 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4210 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4211 | EXT_CALL zend_jit_pre_inc, r0 4212 } else { 4213 | EXT_CALL increment_function, r0 4214 } 4215 } else { 4216 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4217 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4218 | EXT_CALL zend_jit_pre_dec, r0 4219 } else { 4220 | EXT_CALL decrement_function, r0 4221 } 4222 } 4223 if (may_throw) { 4224 zend_jit_check_exception(Dst); 4225 } 4226 } else { 4227 zend_reg tmp_reg; 4228 4229 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4230 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4231 } 4232 if (Z_MODE(op1_def_addr) == IS_REG) { 4233 tmp_reg = Z_REG(op1_def_addr); 4234 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4235 tmp_reg = Z_REG(op1_addr); 4236 } else { 4237 tmp_reg = ZREG_XMM0; 4238 } 4239 | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr 4240 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4241 if (CAN_USE_AVX()) { 4242 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4243 } else { 4244 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4245 } 4246 } else { 4247 if (CAN_USE_AVX()) { 4248 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4249 } else { 4250 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4251 } 4252 } 4253 | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4254 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4255 opline->result_type != IS_UNUSED) { 4256 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4257 | TRY_ADDREF op1_def_info, ah, r1 4258 } 4259 } 4260 | jmp >3 4261 |.code 4262 } 4263 |3: 4264 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4265 return 0; 4266 } 4267 if (opline->result_type != IS_UNUSED) { 4268 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4269 return 0; 4270 } 4271 } 4272 return 1; 4273} 4274 4275static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4276{ 4277 if ((opline+1)->opcode == ZEND_OP_DATA 4278 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4279 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4280 return 1; 4281 } 4282 return 4283 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4284 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4285 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4286 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4287 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4288 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4289} 4290 4291static int zend_jit_math_long_long(dasm_State **Dst, 4292 const zend_op *opline, 4293 zend_uchar opcode, 4294 zend_jit_addr op1_addr, 4295 zend_jit_addr op2_addr, 4296 zend_jit_addr res_addr, 4297 uint32_t res_info, 4298 uint32_t res_use_info, 4299 int may_overflow) 4300{ 4301 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4302 zend_reg result_reg; 4303 zend_reg tmp_reg = ZREG_R0; 4304 4305 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4306 if (may_overflow && (res_info & MAY_BE_GUARD) 4307 && JIT_G(current_frame) 4308 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4309 result_reg = ZREG_R0; 4310 } else { 4311 result_reg = Z_REG(res_addr); 4312 } 4313 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4314 result_reg = Z_REG(op1_addr); 4315 } else if (Z_REG(res_addr) != ZREG_R0) { 4316 result_reg = ZREG_R0; 4317 } else { 4318 /* ASSIGN_DIM_OP */ 4319 result_reg = ZREG_FCARG1a; 4320 tmp_reg = ZREG_FCARG1a; 4321 } 4322 4323 if (opcode == ZEND_MUL && 4324 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4325 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4326 if (Z_MODE(op1_addr) == IS_REG && !may_overflow) { 4327 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4328 } else { 4329 | GET_ZVAL_LVAL result_reg, op1_addr 4330 | add Ra(result_reg), Ra(result_reg) 4331 } 4332 } else if (opcode == ZEND_MUL && 4333 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4334 !may_overflow && 4335 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4336 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4337 | GET_ZVAL_LVAL result_reg, op1_addr 4338 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4339 } else if (opcode == ZEND_MUL && 4340 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4341 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4342 if (Z_MODE(op2_addr) == IS_REG && !may_overflow) { 4343 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4344 } else { 4345 | GET_ZVAL_LVAL result_reg, op2_addr 4346 | add Ra(result_reg), Ra(result_reg) 4347 } 4348 } else if (opcode == ZEND_MUL && 4349 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4350 !may_overflow && 4351 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4352 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4353 | GET_ZVAL_LVAL result_reg, op2_addr 4354 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4355 } else if (opcode == ZEND_DIV && 4356 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4357 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4358 | GET_ZVAL_LVAL result_reg, op1_addr 4359 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4360 } else if (opcode == ZEND_ADD && 4361 !may_overflow && 4362 Z_MODE(op1_addr) == IS_REG && 4363 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4364 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4365 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4366 } else if (opcode == ZEND_ADD && 4367 !may_overflow && 4368 Z_MODE(op2_addr) == IS_REG && 4369 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4370 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4371 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4372 } else if (opcode == ZEND_SUB && 4373 !may_overflow && 4374 Z_MODE(op1_addr) == IS_REG && 4375 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4376 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4377 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4378 } else { 4379 | GET_ZVAL_LVAL result_reg, op1_addr 4380 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4381 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4382 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4383 /* +/- 0 */ 4384 may_overflow = 0; 4385 } else if (same_ops && opcode != ZEND_DIV) { 4386 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4387 } else { 4388 zend_reg tmp_reg; 4389 4390 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4391 tmp_reg = ZREG_R1; 4392 } else if (result_reg != ZREG_R0) { 4393 tmp_reg = ZREG_R0; 4394 } else { 4395 tmp_reg = ZREG_R1; 4396 } 4397 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 4398 (void)tmp_reg; 4399 } 4400 } 4401 if (may_overflow) { 4402 if (res_info & MAY_BE_GUARD) { 4403 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4404 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4405 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4406 | jo &exit_addr 4407 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4408 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4409 } 4410 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4411 | jno &exit_addr 4412 } else { 4413 ZEND_UNREACHABLE(); 4414 } 4415 } else { 4416 if (res_info & MAY_BE_LONG) { 4417 | jo >1 4418 } else { 4419 | jno >1 4420 } 4421 } 4422 } 4423 4424 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4425 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4426 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4427 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4428 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4429 } 4430 } 4431 } 4432 4433 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4434 zend_reg tmp_reg1 = ZREG_XMM0; 4435 zend_reg tmp_reg2 = ZREG_XMM1; 4436 4437 if (res_info & MAY_BE_LONG) { 4438 |.cold_code 4439 |1: 4440 } 4441 4442 do { 4443 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4444 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4445 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4446 if (opcode == ZEND_ADD) { 4447 |.if X64 4448 | mov64 rax, 0x43e0000000000000 4449 if (Z_MODE(res_addr) == IS_REG) { 4450 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax 4451 } else { 4452 | SET_ZVAL_LVAL res_addr, rax 4453 } 4454 |.else 4455 | SET_ZVAL_LVAL res_addr, 0 4456 | SET_ZVAL_W2 res_addr, 0x41e00000 4457 |.endif 4458 break; 4459 } else if (opcode == ZEND_SUB) { 4460 |.if X64 4461 | mov64 rax, 0xc3e0000000000000 4462 if (Z_MODE(res_addr) == IS_REG) { 4463 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax 4464 } else { 4465 | SET_ZVAL_LVAL res_addr, rax 4466 } 4467 |.else 4468 | SET_ZVAL_LVAL res_addr, 0x00200000 4469 | SET_ZVAL_W2 res_addr, 0xc1e00000 4470 |.endif 4471 break; 4472 } 4473 } 4474 4475 | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4476 | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4477 if (CAN_USE_AVX()) { 4478 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4479 } else { 4480 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4481 } 4482 | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 4483 } while (0); 4484 4485 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4486 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4487 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4488 } 4489 if (res_info & MAY_BE_LONG) { 4490 | jmp >2 4491 |.code 4492 } 4493 |2: 4494 } 4495 4496 return 1; 4497} 4498 4499static int zend_jit_math_long_double(dasm_State **Dst, 4500 zend_uchar opcode, 4501 zend_jit_addr op1_addr, 4502 zend_jit_addr op2_addr, 4503 zend_jit_addr res_addr, 4504 uint32_t res_use_info) 4505{ 4506 zend_reg result_reg = 4507 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4508 zend_reg tmp_reg; 4509 4510 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4511 /* ASSIGN_DIM_OP */ 4512 tmp_reg = ZREG_R1; 4513 } else { 4514 tmp_reg = ZREG_R0; 4515 } 4516 4517 | SSE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4518 4519 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4520 /* ASSIGN_DIM_OP */ 4521 if (CAN_USE_AVX()) { 4522 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4523 } else { 4524 | SSE_MATH opcode, result_reg, op2_addr, r1 4525 } 4526 } else { 4527 if (CAN_USE_AVX()) { 4528 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4529 } else { 4530 | SSE_MATH opcode, result_reg, op2_addr, r0 4531 } 4532 } 4533 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4534 4535 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4536 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4537 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4538 } 4539 } 4540 4541 return 1; 4542} 4543 4544static int zend_jit_math_double_long(dasm_State **Dst, 4545 zend_uchar opcode, 4546 zend_jit_addr op1_addr, 4547 zend_jit_addr op2_addr, 4548 zend_jit_addr res_addr, 4549 uint32_t res_use_info) 4550{ 4551 zend_reg result_reg, tmp_reg_gp; 4552 4553 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4554 /* ASSIGN_DIM_OP */ 4555 tmp_reg_gp = ZREG_R1; 4556 } else { 4557 tmp_reg_gp = ZREG_R0; 4558 } 4559 4560 if (zend_is_commutative(opcode) 4561 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4562 if (Z_MODE(res_addr) == IS_REG) { 4563 result_reg = Z_REG(res_addr); 4564 } else { 4565 result_reg = ZREG_XMM0; 4566 } 4567 | SSE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4568 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4569 /* ASSIGN_DIM_OP */ 4570 if (CAN_USE_AVX()) { 4571 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4572 } else { 4573 | SSE_MATH opcode, result_reg, op1_addr, r1 4574 } 4575 } else { 4576 if (CAN_USE_AVX()) { 4577 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4578 } else { 4579 | SSE_MATH opcode, result_reg, op1_addr, r0 4580 } 4581 } 4582 } else { 4583 zend_reg tmp_reg; 4584 4585 if (Z_MODE(res_addr) == IS_REG) { 4586 result_reg = Z_REG(res_addr); 4587 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4588 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4589 result_reg = Z_REG(op1_addr); 4590 tmp_reg = ZREG_XMM0; 4591 } else { 4592 result_reg = ZREG_XMM0; 4593 tmp_reg = ZREG_XMM1; 4594 } 4595 if (CAN_USE_AVX()) { 4596 zend_reg op1_reg; 4597 4598 if (Z_MODE(op1_addr) == IS_REG) { 4599 op1_reg = Z_REG(op1_addr); 4600 } else { 4601 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4602 op1_reg = result_reg; 4603 } 4604 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4605 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4606 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4607 /* +/- 0 */ 4608 } else { 4609 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4610 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4611 } 4612 } else { 4613 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4614 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4615 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4616 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4617 /* +/- 0 */ 4618 } else { 4619 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4620 | SSE_MATH_REG opcode, result_reg, tmp_reg 4621 } 4622 } 4623 } 4624 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4625 4626 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4627 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4628 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4629 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4630 } 4631 } 4632 } 4633 4634 return 1; 4635} 4636 4637static int zend_jit_math_double_double(dasm_State **Dst, 4638 zend_uchar opcode, 4639 zend_jit_addr op1_addr, 4640 zend_jit_addr op2_addr, 4641 zend_jit_addr res_addr, 4642 uint32_t res_use_info) 4643{ 4644 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4645 zend_reg result_reg; 4646 4647 if (Z_MODE(res_addr) == IS_REG) { 4648 result_reg = Z_REG(res_addr); 4649 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4650 result_reg = Z_REG(op1_addr); 4651 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4652 result_reg = Z_REG(op2_addr); 4653 } else { 4654 result_reg = ZREG_XMM0; 4655 } 4656 4657 if (CAN_USE_AVX()) { 4658 zend_reg op1_reg; 4659 zend_jit_addr val_addr; 4660 4661 if (Z_MODE(op1_addr) == IS_REG) { 4662 op1_reg = Z_REG(op1_addr); 4663 val_addr = op2_addr; 4664 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4665 op1_reg = Z_REG(op2_addr); 4666 val_addr = op1_addr; 4667 } else { 4668 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4669 op1_reg = result_reg; 4670 val_addr = op2_addr; 4671 } 4672 if ((opcode == ZEND_MUL) && 4673 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4674 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4675 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4676 /* ASSIGN_DIM_OP */ 4677 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4678 } else { 4679 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4680 } 4681 } else { 4682 zend_jit_addr val_addr; 4683 4684 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4685 | SSE_GET_ZVAL_DVAL result_reg, op2_addr 4686 val_addr = op1_addr; 4687 } else { 4688 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4689 val_addr = op2_addr; 4690 } 4691 if (same_ops) { 4692 | SSE_MATH_REG opcode, result_reg, result_reg 4693 } else if ((opcode == ZEND_MUL) && 4694 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4695 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4696 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4697 /* ASSIGN_DIM_OP */ 4698 | SSE_MATH opcode, result_reg, val_addr, r1 4699 } else { 4700 | SSE_MATH opcode, result_reg, val_addr, r0 4701 } 4702 } 4703 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4704 4705 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4706 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4707 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4708 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4709 } 4710 } 4711 } 4712 4713 return 1; 4714} 4715 4716static int zend_jit_math_helper(dasm_State **Dst, 4717 const zend_op *opline, 4718 zend_uchar opcode, 4719 zend_uchar op1_type, 4720 znode_op op1, 4721 zend_jit_addr op1_addr, 4722 uint32_t op1_info, 4723 zend_uchar op2_type, 4724 znode_op op2, 4725 zend_jit_addr op2_addr, 4726 uint32_t op2_info, 4727 uint32_t res_var, 4728 zend_jit_addr res_addr, 4729 uint32_t res_info, 4730 uint32_t res_use_info, 4731 int may_overflow, 4732 int may_throw) 4733/* Labels: 1,2,3,4,5,6 */ 4734{ 4735 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4736 4737 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4738 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4739 if (op1_info & MAY_BE_DOUBLE) { 4740 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4741 } else { 4742 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4743 } 4744 } 4745 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4746 if (op2_info & MAY_BE_DOUBLE) { 4747 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4748 |.cold_code 4749 |1: 4750 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4751 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4752 } 4753 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4754 return 0; 4755 } 4756 | jmp >5 4757 |.code 4758 } else { 4759 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4760 } 4761 } 4762 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4763 return 0; 4764 } 4765 if (op1_info & MAY_BE_DOUBLE) { 4766 |.cold_code 4767 |3: 4768 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4769 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4770 } 4771 if (op2_info & MAY_BE_DOUBLE) { 4772 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4773 if (!same_ops) { 4774 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4775 } else { 4776 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4777 } 4778 } 4779 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4780 return 0; 4781 } 4782 | jmp >5 4783 } 4784 if (!same_ops) { 4785 |1: 4786 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4787 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4788 } 4789 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4790 return 0; 4791 } 4792 | jmp >5 4793 } 4794 |.code 4795 } 4796 } else if ((op1_info & MAY_BE_DOUBLE) && 4797 !(op1_info & MAY_BE_LONG) && 4798 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4799 (res_info & MAY_BE_DOUBLE)) { 4800 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4801 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4802 } 4803 if (op2_info & MAY_BE_DOUBLE) { 4804 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4805 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4806 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4807 } else { 4808 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4809 } 4810 } 4811 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4812 return 0; 4813 } 4814 } 4815 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4816 if (op2_info & MAY_BE_DOUBLE) { 4817 |.cold_code 4818 } 4819 |1: 4820 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4821 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4822 } 4823 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4824 return 0; 4825 } 4826 if (op2_info & MAY_BE_DOUBLE) { 4827 | jmp >5 4828 |.code 4829 } 4830 } 4831 } else if ((op2_info & MAY_BE_DOUBLE) && 4832 !(op2_info & MAY_BE_LONG) && 4833 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4834 (res_info & MAY_BE_DOUBLE)) { 4835 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4836 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4837 } 4838 if (op1_info & MAY_BE_DOUBLE) { 4839 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4840 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4841 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4842 } else { 4843 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4844 } 4845 } 4846 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4847 return 0; 4848 } 4849 } 4850 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4851 if (op1_info & MAY_BE_DOUBLE) { 4852 |.cold_code 4853 } 4854 |1: 4855 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4856 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4857 } 4858 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4859 return 0; 4860 } 4861 if (op1_info & MAY_BE_DOUBLE) { 4862 | jmp >5 4863 |.code 4864 } 4865 } 4866 } 4867 4868 |5: 4869 4870 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4871 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4872 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4873 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4874 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4875 |.cold_code 4876 } 4877 |6: 4878 if (Z_MODE(res_addr) == IS_REG) { 4879 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4880 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4881 } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 4882 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4883 } 4884 if (Z_MODE(op1_addr) == IS_REG) { 4885 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4886 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4887 return 0; 4888 } 4889 op1_addr = real_addr; 4890 } 4891 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4892 if (Z_MODE(op2_addr) == IS_REG) { 4893 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4894 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4895 return 0; 4896 } 4897 op2_addr = real_addr; 4898 } 4899 |.if X64 4900 | LOAD_ZVAL_ADDR CARG3, op2_addr 4901 |.else 4902 | sub r4, 12 4903 | PUSH_ZVAL_ADDR op2_addr, r0 4904 |.endif 4905 | SET_EX_OPLINE opline, r0 4906 if (opcode == ZEND_ADD) { 4907 | EXT_CALL add_function, r0 4908 } else if (opcode == ZEND_SUB) { 4909 | EXT_CALL sub_function, r0 4910 } else if (opcode == ZEND_MUL) { 4911 | EXT_CALL mul_function, r0 4912 } else if (opcode == ZEND_DIV) { 4913 | EXT_CALL div_function, r0 4914 } else { 4915 ZEND_UNREACHABLE(); 4916 } 4917 |.if not(X64) 4918 | add r4, 12 4919 |.endif 4920 | FREE_OP op1_type, op1, op1_info, 0, opline 4921 | FREE_OP op2_type, op2, op2_info, 0, opline 4922 if (may_throw) { 4923 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4924 zend_jit_check_exception_undef_result(Dst, opline); 4925 } else { 4926 zend_jit_check_exception(Dst); 4927 } 4928 } 4929 if (Z_MODE(res_addr) == IS_REG) { 4930 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4931 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4932 return 0; 4933 } 4934 } 4935 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4936 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4937 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4938 | jmp <5 4939 |.code 4940 } 4941 } 4942 4943 return 1; 4944} 4945 4946static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4947{ 4948 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4949 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4950 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4951 4952 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4953 return 0; 4954 } 4955 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4956 return 0; 4957 } 4958 return 1; 4959} 4960 4961static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr) 4962{ 4963 zend_jit_addr op1_addr = OP1_ADDR(); 4964 zend_jit_addr op2_addr = OP2_ADDR(); 4965 4966 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 4967 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 4968 | EXT_CALL zend_jit_add_arrays_helper, r0 4969 | SET_ZVAL_PTR res_addr, r0 4970 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 4971 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 4972 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 4973 return 1; 4974} 4975 4976static int zend_jit_long_math_helper(dasm_State **Dst, 4977 const zend_op *opline, 4978 zend_uchar opcode, 4979 zend_uchar op1_type, 4980 znode_op op1, 4981 zend_jit_addr op1_addr, 4982 uint32_t op1_info, 4983 zend_ssa_range *op1_range, 4984 zend_uchar op2_type, 4985 znode_op op2, 4986 zend_jit_addr op2_addr, 4987 uint32_t op2_info, 4988 zend_ssa_range *op2_range, 4989 uint32_t res_var, 4990 zend_jit_addr res_addr, 4991 uint32_t res_info, 4992 uint32_t res_use_info, 4993 int may_throw) 4994/* Labels: 6 */ 4995{ 4996 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4997 zend_reg result_reg; 4998 4999 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5000 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5001 } 5002 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5003 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5004 } 5005 5006 if (opcode == ZEND_MOD) { 5007 result_reg = ZREG_RAX; 5008 } else if (Z_MODE(res_addr) == IS_REG) { 5009 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5010 && opline->op2_type != IS_CONST) { 5011 result_reg = ZREG_R0; 5012 } else { 5013 result_reg = Z_REG(res_addr); 5014 } 5015 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5016 result_reg = Z_REG(op1_addr); 5017 } else if (Z_REG(res_addr) != ZREG_R0) { 5018 result_reg = ZREG_R0; 5019 } else { 5020 /* ASSIGN_DIM_OP */ 5021 if (sizeof(void*) == 4 5022 && (opcode == ZEND_SL || opcode == ZEND_SR) 5023 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5024 result_reg = ZREG_R2; 5025 } else { 5026 result_reg = ZREG_FCARG1a; 5027 } 5028 } 5029 5030 if (opcode == ZEND_SL) { 5031 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5032 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5033 5034 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5035 if (EXPECTED(op2_lval > 0)) { 5036 | xor Ra(result_reg), Ra(result_reg) 5037 } else { 5038 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5039 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5040 | SET_EX_OPLINE opline, r0 5041 | jmp ->negative_shift 5042 } 5043 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5044 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5045 } else { 5046 | GET_ZVAL_LVAL result_reg, op1_addr 5047 | shl Ra(result_reg), op2_lval 5048 } 5049 } else { 5050 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5051 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5052 } 5053 if (!op2_range || 5054 op2_range->min < 0 || 5055 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5056 | cmp r1, (SIZEOF_ZEND_LONG*8) 5057 | jae >1 5058 |.cold_code 5059 |1: 5060 | cmp r1, 0 5061 | mov Ra(result_reg), 0 5062 | jg >1 5063 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5064 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5065 | SET_EX_OPLINE opline, r0 5066 | jmp ->negative_shift 5067 |.code 5068 } 5069 | GET_ZVAL_LVAL result_reg, op1_addr 5070 | shl Ra(result_reg), cl 5071 |1: 5072 } 5073 } else if (opcode == ZEND_SR) { 5074 | GET_ZVAL_LVAL result_reg, op1_addr 5075 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5076 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5077 5078 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5079 if (EXPECTED(op2_lval > 0)) { 5080 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5081 } else { 5082 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5083 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5084 | SET_EX_OPLINE opline, r0 5085 | jmp ->negative_shift 5086 } 5087 } else { 5088 | sar Ra(result_reg), op2_lval 5089 } 5090 } else { 5091 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5092 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5093 } 5094 if (!op2_range || 5095 op2_range->min < 0 || 5096 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5097 | cmp r1, (SIZEOF_ZEND_LONG*8) 5098 | jae >1 5099 |.cold_code 5100 |1: 5101 | cmp r1, 0 5102 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5103 | jg >1 5104 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5105 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5106 | SET_EX_OPLINE opline, r0 5107 | jmp ->negative_shift 5108 |.code 5109 } 5110 |1: 5111 | sar Ra(result_reg), cl 5112 } 5113 } else if (opcode == ZEND_MOD) { 5114 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5115 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5116 5117 if (op2_lval == 0) { 5118 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5119 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5120 | SET_EX_OPLINE opline, r0 5121 | jmp ->mod_by_zero 5122 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5123 zval tmp; 5124 zend_jit_addr tmp_addr; 5125 zend_reg tmp_reg; 5126 5127 /* Optimisation for mod of power of 2 */ 5128 ZVAL_LONG(&tmp, op2_lval - 1); 5129 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5130 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5131 tmp_reg = ZREG_R1; 5132 } else if (result_reg != ZREG_R0) { 5133 tmp_reg = ZREG_R0; 5134 } else { 5135 tmp_reg = ZREG_R1; 5136 } 5137 | GET_ZVAL_LVAL result_reg, op1_addr 5138 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg 5139 (void)tmp_reg; 5140 } else { 5141 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5142 | mov aword T1, r0 // save 5143 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5144 | mov aword T1, Ra(ZREG_RCX) // save 5145 } 5146 result_reg = ZREG_RDX; 5147 if (op2_lval == -1) { 5148 | xor Ra(result_reg), Ra(result_reg) 5149 } else { 5150 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5151 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5152 |.if X64 5153 | cqo 5154 |.else 5155 | cdq 5156 |.endif 5157 | idiv Ra(ZREG_RCX) 5158 } 5159 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5160 | mov r0, aword T1 // restore 5161 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5162 | mov Ra(ZREG_RCX), aword T1 // restore 5163 } 5164 } 5165 } else { 5166 if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5167 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5168 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5169 } else if (Z_MODE(op2_addr) == IS_REG) { 5170 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5171 } 5172 | jz >1 5173 |.cold_code 5174 |1: 5175 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5176 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5177 | SET_EX_OPLINE opline, r0 5178 | jmp ->mod_by_zero 5179 |.code 5180 } 5181 5182 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5183 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5184 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5185 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5186 } else if (Z_MODE(op2_addr) == IS_REG) { 5187 | cmp Ra(Z_REG(op2_addr)), -1 5188 } 5189 | jz >1 5190 |.cold_code 5191 |1: 5192 | SET_ZVAL_LVAL res_addr, 0 5193 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5194 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5195 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5196 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5197 } 5198 } 5199 } 5200 | jmp >5 5201 |.code 5202 } 5203 5204 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5205 | mov aword T1, r0 // save 5206 } 5207 result_reg = ZREG_RDX; 5208 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5209 |.if X64 5210 | cqo 5211 |.else 5212 | cdq 5213 |.endif 5214 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5215 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5216 } else if (Z_MODE(op2_addr) == IS_REG) { 5217 | idiv Ra(Z_REG(op2_addr)) 5218 } 5219 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5220 | mov r0, aword T1 // restore 5221 } 5222 } 5223 } else if (same_ops) { 5224 | GET_ZVAL_LVAL result_reg, op1_addr 5225 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5226 } else { 5227 zend_reg tmp_reg; 5228 5229 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5230 tmp_reg = ZREG_R1; 5231 } else if (result_reg != ZREG_R0) { 5232 tmp_reg = ZREG_R0; 5233 } else { 5234 tmp_reg = ZREG_R1; 5235 } 5236 | GET_ZVAL_LVAL result_reg, op1_addr 5237 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 5238 (void)tmp_reg; 5239 } 5240 5241 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5242 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5243 } 5244 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5245 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5246 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5247 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5248 } 5249 } 5250 } 5251 5252 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5253 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5254 if ((op1_info & MAY_BE_LONG) && 5255 (op2_info & MAY_BE_LONG)) { 5256 |.cold_code 5257 } 5258 |6: 5259 if (Z_MODE(res_addr) == IS_REG) { 5260 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5261 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5262 } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5263 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5264 } 5265 if (Z_MODE(op1_addr) == IS_REG) { 5266 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5267 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5268 return 0; 5269 } 5270 op1_addr = real_addr; 5271 } 5272 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5273 if (Z_MODE(op2_addr) == IS_REG) { 5274 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5275 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5276 return 0; 5277 } 5278 op2_addr = real_addr; 5279 } 5280 |.if X64 5281 | LOAD_ZVAL_ADDR CARG3, op2_addr 5282 |.else 5283 | sub r4, 12 5284 | PUSH_ZVAL_ADDR op2_addr, r0 5285 |.endif 5286 | SET_EX_OPLINE opline, r0 5287 if (opcode == ZEND_BW_OR) { 5288 | EXT_CALL bitwise_or_function, r0 5289 } else if (opcode == ZEND_BW_AND) { 5290 | EXT_CALL bitwise_and_function, r0 5291 } else if (opcode == ZEND_BW_XOR) { 5292 | EXT_CALL bitwise_xor_function, r0 5293 } else if (opcode == ZEND_SL) { 5294 | EXT_CALL shift_left_function, r0 5295 } else if (opcode == ZEND_SR) { 5296 | EXT_CALL shift_right_function, r0 5297 } else if (opcode == ZEND_MOD) { 5298 | EXT_CALL mod_function, r0 5299 } else { 5300 ZEND_UNREACHABLE(); 5301 } 5302 |.if not(X64) 5303 | add r4, 12 5304 |.endif 5305 | FREE_OP op1_type, op1, op1_info, 0, opline 5306 | FREE_OP op2_type, op2, op2_info, 0, opline 5307 if (may_throw) { 5308 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5309 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 5310 | jne ->exception_handler_free_op2 5311 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5312 zend_jit_check_exception_undef_result(Dst, opline); 5313 } else { 5314 zend_jit_check_exception(Dst); 5315 } 5316 } 5317 if (Z_MODE(res_addr) == IS_REG) { 5318 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5319 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5320 return 0; 5321 } 5322 } 5323 if ((op1_info & MAY_BE_LONG) && 5324 (op2_info & MAY_BE_LONG)) { 5325 | jmp >5 5326 |.code 5327 } 5328 } 5329 |5: 5330 5331 return 1; 5332} 5333 5334static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5335{ 5336 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5337 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5338 5339 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5340 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5341 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5342 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5343 return 0; 5344 } 5345 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5346 return 0; 5347 } 5348 return 1; 5349} 5350 5351static int zend_jit_concat_helper(dasm_State **Dst, 5352 const zend_op *opline, 5353 zend_uchar op1_type, 5354 znode_op op1, 5355 zend_jit_addr op1_addr, 5356 uint32_t op1_info, 5357 zend_uchar op2_type, 5358 znode_op op2, 5359 zend_jit_addr op2_addr, 5360 uint32_t op2_info, 5361 zend_jit_addr res_addr, 5362 int may_throw) 5363{ 5364#if 1 5365 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5366 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5367 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5368 } 5369 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5370 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5371 } 5372 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5373 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5374 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5375 } 5376 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5377 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5378 /* concatination with itself may reduce refcount */ 5379 op2_info |= MAY_BE_RC1; 5380 } else { 5381 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5382 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5383 } 5384 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5385 |.if X64 5386 | LOAD_ZVAL_ADDR CARG3, op2_addr 5387 |.else 5388 | sub r4, 12 5389 | PUSH_ZVAL_ADDR op2_addr, r0 5390 |.endif 5391 | EXT_CALL zend_jit_fast_concat_helper, r0 5392 |.if not(X64) 5393 | add r4, 12 5394 |.endif 5395 } 5396 /* concatination with empty string may increase refcount */ 5397 op1_info |= MAY_BE_RCN; 5398 op2_info |= MAY_BE_RCN; 5399 | FREE_OP op1_type, op1, op1_info, 0, opline 5400 | FREE_OP op2_type, op2, op2_info, 0, opline 5401 |5: 5402 } 5403 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5404 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5405 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5406 |.cold_code 5407 |6: 5408 } 5409#endif 5410 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5411 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5412 } 5413 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5414 |.if X64 5415 | LOAD_ZVAL_ADDR CARG3, op2_addr 5416 |.else 5417 | sub r4, 12 5418 | PUSH_ZVAL_ADDR op2_addr, r0 5419 |.endif 5420 | SET_EX_OPLINE opline, r0 5421 | EXT_CALL concat_function, r0 5422 |.if not(X64) 5423 | add r4, 12 5424 |.endif 5425 /* concatination with empty string may increase refcount */ 5426 op1_info |= MAY_BE_RCN; 5427 op2_info |= MAY_BE_RCN; 5428 | FREE_OP op1_type, op1, op1_info, 0, opline 5429 | FREE_OP op2_type, op2, op2_info, 0, opline 5430 if (may_throw) { 5431 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5432 zend_jit_check_exception_undef_result(Dst, opline); 5433 } else { 5434 zend_jit_check_exception(Dst); 5435 } 5436 } 5437#if 1 5438 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5439 | jmp <5 5440 |.code 5441 } 5442 } 5443#endif 5444 5445 return 1; 5446} 5447 5448static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5449{ 5450 zend_jit_addr op1_addr, op2_addr; 5451 5452 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5453 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5454 5455 op1_addr = OP1_ADDR(); 5456 op2_addr = OP2_ADDR(); 5457 5458 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5459} 5460 5461static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5462/* Labels: 1,2,3,4,5 */ 5463{ 5464 zend_jit_addr op2_addr = OP2_ADDR(); 5465 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5466 5467 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5468 && type == BP_VAR_R 5469 && !exit_addr) { 5470 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5471 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5472 if (!exit_addr) { 5473 return 0; 5474 } 5475 } 5476 5477 if (op2_info & MAY_BE_LONG) { 5478 zend_bool op2_loaded = 0; 5479 zend_bool packed_loaded = 0; 5480 zend_bool bad_packed_key = 0; 5481 5482 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5483 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5484 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5485 } 5486 if (op1_info & MAY_BE_PACKED_GUARD) { 5487 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5488 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5489 5490 if (!exit_addr) { 5491 return 0; 5492 } 5493 if (op1_info & MAY_BE_ARRAY_PACKED) { 5494 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5495 | jz &exit_addr 5496 } else { 5497 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5498 | jnz &exit_addr 5499 } 5500 } 5501 if (type == BP_VAR_W) { 5502 | // hval = Z_LVAL_P(dim); 5503 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5504 op2_loaded = 1; 5505 } 5506 if (op1_info & MAY_BE_ARRAY_PACKED) { 5507 zend_long val = -1; 5508 5509 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5510 val = Z_LVAL_P(Z_ZV(op2_addr)); 5511 if (val >= 0 && val < HT_MAX_SIZE) { 5512 packed_loaded = 1; 5513 } else { 5514 bad_packed_key = 1; 5515 } 5516 } else { 5517 if (!op2_loaded) { 5518 | // hval = Z_LVAL_P(dim); 5519 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5520 op2_loaded = 1; 5521 } 5522 packed_loaded = 1; 5523 } 5524 if (packed_loaded) { 5525 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5526 if (op1_info & MAY_BE_ARRAY_HASH) { 5527 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5528 | jz >4 // HASH_FIND 5529 } 5530 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5531 |.if X64 5532 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5533 if (val == 0) { 5534 | test r0, r0 5535 } else if (val > 0 && !op2_loaded) { 5536 | cmp r0, val 5537 } else { 5538 | cmp r0, FCARG2a 5539 } 5540 |.else 5541 if (val >= 0 && !op2_loaded) { 5542 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5543 } else { 5544 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5545 } 5546 |.endif 5547 if (type == BP_JIT_IS) { 5548 if (not_found_exit_addr) { 5549 | jbe ¬_found_exit_addr 5550 } else { 5551 | jbe >9 // NOT_FOUND 5552 } 5553 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5554 | jbe &exit_addr 5555 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5556 | jbe ¬_found_exit_addr 5557 } else if (type == BP_VAR_IS && found_exit_addr) { 5558 | jbe >7 // NOT_FOUND 5559 } else { 5560 | jbe >2 // NOT_FOUND 5561 } 5562 | // _ret = &_ht->arData[_h].val; 5563 if (val >= 0) { 5564 | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] 5565 if (val != 0) { 5566 | add r0, val * sizeof(Bucket) 5567 } 5568 } else { 5569 |.if X64 5570 | mov r0, FCARG2a 5571 | shl r0, 5 5572 |.else 5573 | imul r0, FCARG2a, sizeof(Bucket) 5574 |.endif 5575 | add r0, aword [FCARG1a + offsetof(zend_array, arData)] 5576 } 5577 } 5578 } 5579 switch (type) { 5580 case BP_JIT_IS: 5581 if (op1_info & MAY_BE_ARRAY_HASH) { 5582 if (packed_loaded) { 5583 | jmp >5 5584 } 5585 |4: 5586 if (!op2_loaded) { 5587 | // hval = Z_LVAL_P(dim); 5588 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5589 } 5590 if (packed_loaded) { 5591 | EXT_CALL _zend_hash_index_find, r0 5592 } else { 5593 | EXT_CALL zend_hash_index_find, r0 5594 } 5595 | test r0, r0 5596 if (not_found_exit_addr) { 5597 | jz ¬_found_exit_addr 5598 } else { 5599 | jz >9 // NOT_FOUND 5600 } 5601 if (op2_info & MAY_BE_STRING) { 5602 | jmp >5 5603 } 5604 } else if (packed_loaded) { 5605 if (op2_info & MAY_BE_STRING) { 5606 | jmp >5 5607 } 5608 } else if (not_found_exit_addr) { 5609 | jmp ¬_found_exit_addr 5610 } else { 5611 | jmp >9 // NOT_FOUND 5612 } 5613 break; 5614 case BP_VAR_R: 5615 case BP_VAR_IS: 5616 case BP_VAR_UNSET: 5617 if (packed_loaded) { 5618 if (op1_info & MAY_BE_ARRAY_HASH) { 5619 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5620 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5621 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5622 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { 5623 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5624 } 5625 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5626 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5627 } else if (type == BP_VAR_IS && found_exit_addr) { 5628 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5629 } else { 5630 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5631 } 5632 } 5633 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { 5634 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5635 | jmp &exit_addr 5636 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5637 | jmp ¬_found_exit_addr 5638 } else if (type == BP_VAR_IS && found_exit_addr) { 5639 | jmp >7 // NOT_FOUND 5640 } else { 5641 | jmp >2 // NOT_FOUND 5642 } 5643 } 5644 if (op1_info & MAY_BE_ARRAY_HASH) { 5645 |4: 5646 if (!op2_loaded) { 5647 | // hval = Z_LVAL_P(dim); 5648 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5649 } 5650 if (packed_loaded) { 5651 | EXT_CALL _zend_hash_index_find, r0 5652 } else { 5653 | EXT_CALL zend_hash_index_find, r0 5654 } 5655 | test r0, r0 5656 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5657 | jz &exit_addr 5658 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5659 | jz ¬_found_exit_addr 5660 } else if (type == BP_VAR_IS && found_exit_addr) { 5661 | jz >7 // NOT_FOUND 5662 } else { 5663 | jz >2 // NOT_FOUND 5664 } 5665 } 5666 |.cold_code 5667 |2: 5668 switch (type) { 5669 case BP_VAR_R: 5670 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5671 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5672 | // retval = &EG(uninitialized_zval); 5673 | UNDEFINED_OFFSET opline 5674 | jmp >9 5675 } 5676 break; 5677 case BP_VAR_IS: 5678 case BP_VAR_UNSET: 5679 if (!not_found_exit_addr && !found_exit_addr) { 5680 | // retval = &EG(uninitialized_zval); 5681 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5682 | jmp >9 5683 } 5684 break; 5685 default: 5686 ZEND_UNREACHABLE(); 5687 } 5688 |.code 5689 break; 5690 case BP_VAR_RW: 5691 if (packed_loaded) { 5692 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5693 } 5694 |2: 5695 |4: 5696 if (!op2_loaded) { 5697 | // hval = Z_LVAL_P(dim); 5698 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5699 } 5700 | SET_EX_OPLINE opline, r0 5701 if (packed_loaded) { 5702 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5703 } else { 5704 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5705 } 5706 | test r0, r0 5707 | jz >9 5708 break; 5709 case BP_VAR_W: 5710 if (packed_loaded) { 5711 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5712 } 5713 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded || bad_packed_key) { 5714 |2: 5715 | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); 5716 if (!op2_loaded) { 5717 | // hval = Z_LVAL_P(dim); 5718 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5719 } 5720 |.if X64 5721 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval 5722 |.else 5723 | sub r4, 12 5724 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 5725 |.endif 5726 | EXT_CALL zend_hash_index_add_new, r0 5727 |.if not(X64) 5728 | add r4, 12 5729 |.endif 5730 if (op1_info & MAY_BE_ARRAY_HASH) { 5731 | jmp >8 5732 } 5733 } 5734 if (op1_info & MAY_BE_ARRAY_HASH) { 5735 |4: 5736 if (!op2_loaded) { 5737 | // hval = Z_LVAL_P(dim); 5738 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5739 } 5740 | EXT_CALL zend_jit_hash_index_lookup_w, r0 5741 } 5742 break; 5743 default: 5744 ZEND_UNREACHABLE(); 5745 } 5746 5747 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5748 | jmp >8 5749 } 5750 } 5751 5752 if (op2_info & MAY_BE_STRING) { 5753 |3: 5754 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5755 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5756 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5757 } 5758 | // offset_key = Z_STR_P(dim); 5759 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5760 | // retval = zend_hash_find(ht, offset_key); 5761 switch (type) { 5762 case BP_JIT_IS: 5763 if (opline->op2_type != IS_CONST) { 5764 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5765 | jle >1 5766 |.cold_code 5767 |1: 5768 | EXT_CALL zend_jit_symtable_find, r0 5769 | jmp >1 5770 |.code 5771 | EXT_CALL zend_hash_find, r0 5772 |1: 5773 } else { 5774 | EXT_CALL _zend_hash_find_known_hash, r0 5775 } 5776 | test r0, r0 5777 if (not_found_exit_addr) { 5778 | jz ¬_found_exit_addr 5779 } else { 5780 | jz >9 // NOT_FOUND 5781 } 5782 | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) 5783 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 5784 | GET_Z_PTR r0, r0 5785 |1: 5786 break; 5787 case BP_VAR_R: 5788 case BP_VAR_IS: 5789 case BP_VAR_UNSET: 5790 if (opline->op2_type != IS_CONST) { 5791 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5792 | jle >1 5793 |.cold_code 5794 |1: 5795 | EXT_CALL zend_jit_symtable_find, r0 5796 | jmp >1 5797 |.code 5798 | EXT_CALL zend_hash_find, r0 5799 |1: 5800 } else { 5801 | EXT_CALL _zend_hash_find_known_hash, r0 5802 } 5803 | test r0, r0 5804 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5805 | jz &exit_addr 5806 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5807 | jz ¬_found_exit_addr 5808 } else if (type == BP_VAR_IS && found_exit_addr) { 5809 | jz >7 // NOT_FOUND 5810 } else { 5811 | jz >2 // NOT_FOUND 5812 } 5813 | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) 5814 | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW 5815 |.cold_code 5816 |1: 5817 | // retval = Z_INDIRECT_P(retval); 5818 | GET_Z_PTR r0, r0 5819 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5820 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5821 | jmp &exit_addr 5822 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5823 | jmp ¬_found_exit_addr 5824 } 5825 |2: 5826 switch (type) { 5827 case BP_VAR_R: 5828 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5829 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5830 | UNDEFINED_INDEX opline 5831 | jmp >9 5832 } 5833 break; 5834 case BP_VAR_IS: 5835 case BP_VAR_UNSET: 5836 if (!not_found_exit_addr && !found_exit_addr) { 5837 | // retval = &EG(uninitialized_zval); 5838 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5839 | jmp >9 5840 } 5841 break; 5842 default: 5843 ZEND_UNREACHABLE(); 5844 } 5845 |.code 5846 break; 5847 case BP_VAR_RW: 5848 | SET_EX_OPLINE opline, r0 5849 if (opline->op2_type != IS_CONST) { 5850 | EXT_CALL zend_jit_symtable_lookup_rw, r0 5851 } else { 5852 | EXT_CALL zend_jit_hash_lookup_rw, r0 5853 } 5854 | test r0, r0 5855 | jz >9 5856 break; 5857 case BP_VAR_W: 5858 if (opline->op2_type != IS_CONST) { 5859 | EXT_CALL zend_jit_symtable_lookup_w, r0 5860 } else { 5861 | EXT_CALL zend_jit_hash_lookup_w, r0 5862 } 5863 break; 5864 default: 5865 ZEND_UNREACHABLE(); 5866 } 5867 } 5868 5869 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5870 |5: 5871 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5872 | ZVAL_DEREF r0, MAY_BE_REF 5873 } 5874 | cmp byte [r0 + 8], IS_NULL 5875 if (not_found_exit_addr) { 5876 | jle ¬_found_exit_addr 5877 } else if (found_exit_addr) { 5878 | jg &found_exit_addr 5879 } else { 5880 | jle >9 // NOT FOUND 5881 } 5882 } 5883 5884 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5885 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5886 |.cold_code 5887 |3: 5888 } 5889 | SET_EX_OPLINE opline, r0 5890 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5891 switch (type) { 5892 case BP_VAR_R: 5893 |.if X64 5894 | LOAD_ZVAL_ADDR CARG3, res_addr 5895 |.else 5896 | sub r4, 12 5897 | PUSH_ZVAL_ADDR res_addr, r0 5898 |.endif 5899 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 5900 |.if not(X64) 5901 | add r4, 12 5902 |.endif 5903 | jmp >9 5904 break; 5905 case BP_JIT_IS: 5906 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 5907 | test r0, r0 5908 if (not_found_exit_addr) { 5909 | je ¬_found_exit_addr 5910 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5911 | jmp >8 5912 } 5913 } else if (found_exit_addr) { 5914 | jne &found_exit_addr 5915 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5916 | jmp >9 5917 } 5918 } else { 5919 | jne >8 5920 | jmp >9 5921 } 5922 break; 5923 case BP_VAR_IS: 5924 case BP_VAR_UNSET: 5925 |.if X64 5926 | LOAD_ZVAL_ADDR CARG3, res_addr 5927 |.else 5928 | sub r4, 12 5929 | PUSH_ZVAL_ADDR res_addr, r0 5930 |.endif 5931 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 5932 |.if not(X64) 5933 | add r4, 12 5934 |.endif 5935 | jmp >9 5936 break; 5937 case BP_VAR_RW: 5938 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 5939 | test r0, r0 5940 | jne >8 5941 | jmp >9 5942 break; 5943 case BP_VAR_W: 5944 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 5945 | test r0, r0 5946 | jne >8 5947 | jmp >9 5948 break; 5949 default: 5950 ZEND_UNREACHABLE(); 5951 } 5952 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5953 |.code 5954 } 5955 } 5956 5957 return 1; 5958} 5959 5960static int zend_jit_simple_assign(dasm_State **Dst, 5961 const zend_op *opline, 5962 zend_jit_addr var_addr, 5963 uint32_t var_info, 5964 uint32_t var_def_info, 5965 zend_uchar val_type, 5966 zend_jit_addr val_addr, 5967 uint32_t val_info, 5968 zend_jit_addr res_addr, 5969 int in_cold, 5970 int save_r1) 5971/* Labels: 1,2,3 */ 5972{ 5973 zend_reg tmp_reg; 5974 5975 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 5976 tmp_reg = ZREG_R0; 5977 } else { 5978 /* ASSIGN_DIM */ 5979 tmp_reg = ZREG_FCARG1a; 5980 } 5981 5982 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5983 zval *zv = Z_ZV(val_addr); 5984 5985 if (!res_addr) { 5986 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 5987 } else { 5988 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 5989 } 5990 if (Z_REFCOUNTED_P(zv)) { 5991 if (!res_addr) { 5992 | ADDREF_CONST zv, Ra(tmp_reg) 5993 } else { 5994 | ADDREF_CONST_2 zv, Ra(tmp_reg) 5995 } 5996 } 5997 } else { 5998 if (val_info & MAY_BE_UNDEF) { 5999 if (in_cold) { 6000 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6001 } else { 6002 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6003 |.cold_code 6004 |1: 6005 } 6006 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6007 if (save_r1) { 6008 | mov aword T1, FCARG1a // save 6009 } 6010 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6011 if (res_addr) { 6012 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6013 } 6014 if (opline) { 6015 | SET_EX_OPLINE opline, Ra(tmp_reg) 6016 } 6017 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6018 | mov FCARG1d, Z_OFFSET(val_addr) 6019 | EXT_CALL zend_jit_undefined_op_helper, r0 6020 | test r0, r0 6021 | jz ->exception_handler_undef 6022 if (save_r1) { 6023 | mov FCARG1a, aword T1 // restore 6024 } 6025 | jmp >3 6026 if (in_cold) { 6027 |2: 6028 } else { 6029 |.code 6030 } 6031 } 6032 if (val_info & MAY_BE_REF) { 6033 if (val_type == IS_CV) { 6034 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6035 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6036 | LOAD_ZVAL_ADDR r2, val_addr 6037 } 6038 | ZVAL_DEREF r2, val_info 6039 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6040 } else { 6041 zend_jit_addr ref_addr; 6042 6043 if (in_cold) { 6044 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6045 } else { 6046 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6047 |.cold_code 6048 |1: 6049 } 6050 if (Z_REG(val_addr) == ZREG_R2) { 6051 | mov aword T1, r2 // save 6052 } 6053 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6054 | GET_ZVAL_PTR r2, val_addr 6055 | GC_DELREF r2 6056 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6057 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6058 if (!res_addr) { 6059 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, tmp_reg 6060 } else { 6061 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, tmp_reg 6062 } 6063 | je >2 6064 | IF_NOT_REFCOUNTED dh, >3 6065 if (!res_addr) { 6066 | GC_ADDREF Ra(tmp_reg) 6067 } else { 6068 | add dword [Ra(tmp_reg)], 2 6069 } 6070 | jmp >3 6071 |2: 6072 if (res_addr) { 6073 | IF_NOT_REFCOUNTED dh, >2 6074 | GC_ADDREF Ra(tmp_reg) 6075 |2: 6076 } 6077 if (Z_REG(val_addr) == ZREG_R2) { 6078 | mov r2, aword T1 // restore 6079 } 6080 if (save_r1) { 6081 | mov aword T1, FCARG1a // save 6082 } 6083 | EFREE_REFERENCE aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)] 6084 if (save_r1) { 6085 | mov FCARG1a, aword T1 // restore 6086 } 6087 | jmp >3 6088 if (in_cold) { 6089 |1: 6090 } else { 6091 |.code 6092 } 6093 } 6094 } 6095 6096 if (!res_addr) { 6097 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6098 } else { 6099 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6100 } 6101 6102 if (val_type == IS_CV) { 6103 if (!res_addr) { 6104 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6105 } else { 6106 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6107 } 6108 } else { 6109 if (res_addr) { 6110 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6111 } 6112 } 6113 |3: 6114 } 6115 return 1; 6116} 6117 6118static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6119 const zend_op *opline, 6120 zend_uchar val_type, 6121 zend_jit_addr val_addr, 6122 zend_jit_addr res_addr, 6123 zend_bool check_exception) 6124{ 6125 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6126 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6127 | jnz >2 6128 |.cold_code 6129 |2: 6130 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 6131 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6132 } 6133 if (opline) { 6134 | SET_EX_OPLINE opline, r0 6135 } 6136 if (val_type == IS_CONST) { 6137 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6138 } else if (val_type == IS_TMP_VAR) { 6139 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6140 } else if (val_type == IS_VAR) { 6141 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6142 } else if (val_type == IS_CV) { 6143 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6144 } else { 6145 ZEND_UNREACHABLE(); 6146 } 6147 if (res_addr) { 6148 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6149 6150 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2 6151 | TRY_ADDREF -1, ch, r2 6152 } 6153 if (check_exception) { 6154 | // if (UNEXPECTED(EG(exception) != NULL)) { 6155 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 6156 | je >8 // END OF zend_jit_assign_to_variable() 6157 | jmp ->exception_handler_undef 6158 } else { 6159 | jmp >8 6160 } 6161 |.code 6162 6163 return 1; 6164} 6165 6166static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6167 const zend_op *opline, 6168 zend_jit_addr __var_use_addr, 6169 zend_jit_addr var_addr, 6170 uint32_t __var_info, 6171 uint32_t __var_def_info, 6172 zend_uchar val_type, 6173 zend_jit_addr val_addr, 6174 uint32_t val_info, 6175 zend_jit_addr __res_addr, 6176 zend_bool __check_exception) 6177{ 6178 if (val_info & MAY_BE_UNDEF) { 6179 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6180 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6181 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6182 6183 if (!exit_addr) { 6184 return 0; 6185 } 6186 6187 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6188 } else { 6189 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6190 |.cold_code 6191 |1: 6192 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6193 if (Z_REG(var_addr) != ZREG_FP) { 6194 | mov aword T1, Ra(Z_REG(var_addr)) // save 6195 } 6196 | SET_EX_OPLINE opline, r0 6197 | mov FCARG1d, Z_OFFSET(val_addr) 6198 | EXT_CALL zend_jit_undefined_op_helper, r0 6199 if (Z_REG(var_addr) != ZREG_FP) { 6200 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6201 } 6202 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 6203 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6204 } 6205 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6206 | call ->assign_const 6207 | jmp >9 6208 |.code 6209 } 6210 } 6211 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 6212 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6213 } 6214 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 6215 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6216 } 6217 if (opline) { 6218 | SET_EX_OPLINE opline, r0 6219 } 6220 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6221 | call ->assign_tmp 6222 } else if (val_type == IS_CONST) { 6223 | call ->assign_const 6224 } else if (val_type == IS_TMP_VAR) { 6225 | call ->assign_tmp 6226 } else if (val_type == IS_VAR) { 6227 if (!(val_info & MAY_BE_REF)) { 6228 | call ->assign_tmp 6229 } else { 6230 | call ->assign_var 6231 } 6232 } else if (val_type == IS_CV) { 6233 if (!(val_info & MAY_BE_REF)) { 6234 | call ->assign_cv_noref 6235 } else { 6236 | call ->assign_cv 6237 } 6238 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6239 |9: 6240 } 6241 } else { 6242 ZEND_UNREACHABLE(); 6243 } 6244 6245 return 1; 6246} 6247 6248static int zend_jit_assign_to_variable(dasm_State **Dst, 6249 const zend_op *opline, 6250 zend_jit_addr var_use_addr, 6251 zend_jit_addr var_addr, 6252 uint32_t var_info, 6253 uint32_t var_def_info, 6254 zend_uchar val_type, 6255 zend_jit_addr val_addr, 6256 uint32_t val_info, 6257 zend_jit_addr res_addr, 6258 zend_bool check_exception) 6259/* Labels: 1,2,3,4,5,8 */ 6260{ 6261 int done = 0; 6262 zend_reg ref_reg, tmp_reg; 6263 6264 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6265 ref_reg = ZREG_FCARG1a; 6266 tmp_reg = ZREG_R0; 6267 } else { 6268 /* ASSIGN_DIM */ 6269 ref_reg = ZREG_R0; 6270 tmp_reg = ZREG_FCARG1a; 6271 } 6272 6273 if (var_info & MAY_BE_REF) { 6274 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6275 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6276 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6277 } 6278 | // if (Z_ISREF_P(variable_ptr)) { 6279 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6280 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6281 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6282 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6283 return 0; 6284 } 6285 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6286 |3: 6287 } 6288 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6289 if (RC_MAY_BE_1(var_info)) { 6290 int in_cold = 0; 6291 6292 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6293 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6294 |.cold_code 6295 |1: 6296 in_cold = 1; 6297 } 6298 if (Z_REG(var_use_addr) == ZREG_FCARG1a || Z_REG(var_use_addr) == ZREG_R0) { 6299 zend_bool keep_gc = 0; 6300 6301 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6302 if (tmp_reg == ZREG_FCARG1a) { 6303 if (Z_MODE(val_addr) == IS_REG) { 6304 keep_gc = 1; 6305 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6306 keep_gc = 1; 6307 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6308 if (sizeof(void*) == 4) { 6309 keep_gc = 1; 6310 } else { 6311 zval *zv = Z_ZV(val_addr); 6312 6313 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6314 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6315 keep_gc = 1; 6316 } 6317 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6318 keep_gc = 1; 6319 } 6320 } 6321 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6322 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6323 keep_gc = 1; 6324 } 6325 } 6326 } 6327 if (!keep_gc) { 6328 | mov aword T1, Ra(tmp_reg) // save 6329 } 6330 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { 6331 return 0; 6332 } 6333 if (!keep_gc) { 6334 | mov FCARG1a, aword T1 // restore 6335 } 6336 } else { 6337 | GET_ZVAL_PTR FCARG1a, var_use_addr 6338 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { 6339 return 0; 6340 } 6341 } 6342 | GC_DELREF FCARG1a 6343 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6344 | jnz >4 6345 } else { 6346 | jnz >8 6347 } 6348 | ZVAL_DTOR_FUNC var_info, opline 6349 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6350 if (check_exception) { 6351 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 6352 | je >8 6353 | jmp ->exception_handler 6354 } else { 6355 | jmp >8 6356 } 6357 } 6358 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6359 |4: 6360 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6361 | EXT_CALL gc_possible_root, r0 6362 if (in_cold) { 6363 | jmp >8 6364 } 6365 } 6366 if (in_cold) { 6367 |.code 6368 } else { 6369 done = 1; 6370 } 6371 } else /* if (RC_MAY_BE_N(var_info)) */ { 6372 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6373 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6374 } 6375 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6376 if (Z_REG(var_use_addr) == ZREG_FP) { 6377 | mov T1, Ra(Z_REG(var_use_addr)) // save 6378 } 6379 | GET_ZVAL_PTR FCARG1a, var_use_addr 6380 | GC_DELREF FCARG1a 6381 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6382 | EXT_CALL gc_possible_root, r0 6383 if (Z_REG(var_use_addr) != ZREG_FP) { 6384 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6385 } 6386 } else { 6387 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6388 | GC_DELREF Ra(tmp_reg) 6389 } 6390 |5: 6391 } 6392 } 6393 6394 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { 6395 return 0; 6396 } 6397 6398 |8: 6399 6400 return 1; 6401} 6402 6403static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) 6404{ 6405 zend_jit_addr op2_addr, op3_addr, res_addr; 6406 6407 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6408 op3_addr = OP1_DATA_ADDR(); 6409 if (opline->result_type == IS_UNUSED) { 6410 res_addr = 0; 6411 } else { 6412 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6413 } 6414 6415 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6416 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6417 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6418 6419 if (!exit_addr) { 6420 return 0; 6421 } 6422 6423 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6424 6425 val_info &= ~MAY_BE_UNDEF; 6426 } 6427 6428 if (op1_info & MAY_BE_REF) { 6429 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6430 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6431 | GET_Z_PTR FCARG2a, FCARG1a 6432 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6433 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6434 | jmp >3 6435 |.cold_code 6436 |2: 6437 | SET_EX_OPLINE opline, r0 6438 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6439 | test r0, r0 6440 | mov FCARG1a, r0 6441 | jne >1 6442 | jmp ->exception_handler_undef 6443 |.code 6444 |1: 6445 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6446 } 6447 6448 if (op1_info & MAY_BE_ARRAY) { 6449 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6450 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6451 } 6452 |3: 6453 | SEPARATE_ARRAY op1_addr, op1_info, 1 6454 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 6455 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6456 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6457 | jg >7 6458 } 6459 | // ZVAL_ARR(container, zend_new_array(8)); 6460 if (Z_REG(op1_addr) != ZREG_FP) { 6461 | mov T1, Ra(Z_REG(op1_addr)) // save 6462 } 6463 | EXT_CALL _zend_new_array_0, r0 6464 if (Z_REG(op1_addr) != ZREG_FP) { 6465 | mov Ra(Z_REG(op1_addr)), T1 // restore 6466 } 6467 | SET_ZVAL_LVAL op1_addr, r0 6468 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6469 | mov FCARG1a, r0 6470 } 6471 6472 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6473 |6: 6474 if (opline->op2_type == IS_UNUSED) { 6475 uint32_t var_info = MAY_BE_NULL; 6476 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6477 6478 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6479 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6480 | EXT_CALL zend_hash_next_index_insert, r0 6481 | // if (UNEXPECTED(!var_ptr)) { 6482 | test r0, r0 6483 | jz >1 6484 |.cold_code 6485 |1: 6486 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6487 | CANNOT_ADD_ELEMENT opline 6488 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6489 | jmp >9 6490 |.code 6491 6492 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { 6493 return 0; 6494 } 6495 } else { 6496 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6497 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6498 6499 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { 6500 return 0; 6501 } 6502 6503 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6504 var_info |= MAY_BE_REF; 6505 } 6506 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6507 var_info |= MAY_BE_RC1; 6508 } 6509 6510 |8: 6511 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6512 if (opline->op1_type == IS_VAR) { 6513 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6514 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6515 return 0; 6516 } 6517 } else { 6518 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6519 return 0; 6520 } 6521 } 6522 } 6523 } 6524 6525 if (((op1_info & MAY_BE_ARRAY) && 6526 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || 6527 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { 6528 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6529 |.cold_code 6530 |7: 6531 } 6532 6533 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && 6534 (op1_info & MAY_BE_ARRAY)) { 6535 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6536 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6537 | jg >2 6538 } 6539 | // ZVAL_ARR(container, zend_new_array(8)); 6540 if (Z_REG(op1_addr) != ZREG_FP) { 6541 | mov T1, Ra(Z_REG(op1_addr)) // save 6542 } 6543 | EXT_CALL _zend_new_array_0, r0 6544 if (Z_REG(op1_addr) != ZREG_FP) { 6545 | mov Ra(Z_REG(op1_addr)), T1 // restore 6546 } 6547 | SET_ZVAL_LVAL op1_addr, r0 6548 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6549 | mov FCARG1a, r0 6550 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6551 | jmp <6 6552 |2: 6553 } 6554 6555 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6556 | SET_EX_OPLINE opline, r0 6557 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 6558 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6559 } 6560 if (opline->op2_type == IS_UNUSED) { 6561 | xor FCARG2a, FCARG2a 6562 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6563 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6564 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6565 } else { 6566 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6567 } 6568 |.if not(X64) 6569 | sub r4, 8 6570 |.endif 6571 if (opline->result_type == IS_UNUSED) { 6572 |.if X64 6573 | xor CARG4, CARG4 6574 |.else 6575 | push 0 6576 |.endif 6577 } else { 6578 |.if X64 6579 | LOAD_ZVAL_ADDR CARG4, res_addr 6580 |.else 6581 | PUSH_ZVAL_ADDR res_addr, r0 6582 |.endif 6583 } 6584 |.if X64 6585 | LOAD_ZVAL_ADDR CARG3, op3_addr 6586 |.else 6587 | PUSH_ZVAL_ADDR op3_addr, r0 6588 |.endif 6589 | EXT_CALL zend_jit_assign_dim_helper, r0 6590 |.if not(X64) 6591 | add r4, 8 6592 |.endif 6593 6594#ifdef ZEND_JIT_USE_RC_INFERENCE 6595 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6596 /* ASSIGN_DIM may increase refcount of the value */ 6597 val_info |= MAY_BE_RCN; 6598 } 6599#endif 6600 6601 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 6602 } 6603 6604 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6605 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6606 | jmp >9 // END 6607 } 6608 |.code 6609 } 6610 } 6611 6612#ifdef ZEND_JIT_USE_RC_INFERENCE 6613 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6614 /* ASSIGN_DIM may increase refcount of the key */ 6615 op2_info |= MAY_BE_RCN; 6616 } 6617#endif 6618 6619 |9: 6620 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6621 6622 if (may_throw) { 6623 zend_jit_check_exception(Dst); 6624 } 6625 6626 return 1; 6627} 6628 6629static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) 6630{ 6631 zend_jit_addr op2_addr, op3_addr, var_addr; 6632 6633 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6634 6635 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6636 op3_addr = OP1_DATA_ADDR(); 6637 6638 if (op1_info & MAY_BE_REF) { 6639 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6640 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6641 | GET_Z_PTR FCARG2a, FCARG1a 6642 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6643 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6644 | jmp >3 6645 |.cold_code 6646 |2: 6647 | SET_EX_OPLINE opline, r0 6648 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6649 | test r0, r0 6650 | mov FCARG1a, r0 6651 | jne >1 6652 | jmp ->exception_handler_undef 6653 |.code 6654 |1: 6655 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6656 } 6657 6658 if (op1_info & MAY_BE_ARRAY) { 6659 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6660 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6661 } 6662 |3: 6663 | SEPARATE_ARRAY op1_addr, op1_info, 1 6664 } 6665 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 6666 if (op1_info & MAY_BE_ARRAY) { 6667 |.cold_code 6668 |7: 6669 } 6670 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6671 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6672 | jg >7 6673 } 6674 if (Z_REG(op1_addr) != ZREG_FP) { 6675 | mov T1, Ra(Z_REG(op1_addr)) // save 6676 } 6677 if (op1_info & MAY_BE_UNDEF) { 6678 if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { 6679 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6680 } 6681 | SET_EX_OPLINE opline, r0 6682 | mov FCARG1a, opline->op1.var 6683 | EXT_CALL zend_jit_undefined_op_helper, r0 6684 |1: 6685 } 6686 | // ZVAL_ARR(container, zend_new_array(8)); 6687 | EXT_CALL _zend_new_array_0, r0 6688 if (Z_REG(op1_addr) != ZREG_FP) { 6689 | mov Ra(Z_REG(op1_addr)), T1 // restore 6690 } 6691 | SET_ZVAL_LVAL op1_addr, r0 6692 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6693 | mov FCARG1a, r0 6694 if (op1_info & MAY_BE_ARRAY) { 6695 | jmp >1 6696 |.code 6697 |1: 6698 } 6699 } 6700 6701 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6702 uint32_t var_info; 6703 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6704 6705 |6: 6706 if (opline->op2_type == IS_UNUSED) { 6707 var_info = MAY_BE_NULL; 6708 6709 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6710 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6711 | EXT_CALL zend_hash_next_index_insert, r0 6712 | // if (UNEXPECTED(!var_ptr)) { 6713 | test r0, r0 6714 | jz >1 6715 |.cold_code 6716 |1: 6717 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6718 | CANNOT_ADD_ELEMENT opline 6719 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6720 | jmp >9 6721 |.code 6722 } else { 6723 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6724 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6725 var_info |= MAY_BE_REF; 6726 } 6727 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6728 var_info |= MAY_BE_RC1; 6729 } 6730 6731 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { 6732 return 0; 6733 } 6734 6735 |8: 6736 if (op1_info & (MAY_BE_ARRAY_OF_REF)) { 6737 binary_op_type binary_op = get_binary_op(opline->extended_value); 6738 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6739 | GET_Z_PTR FCARG1a, r0 6740 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6741 | jnz >2 6742 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6743 |.cold_code 6744 |2: 6745 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6746 |.if X64 6747 | LOAD_ADDR CARG3, binary_op 6748 |.else 6749 | sub r4, 12 6750 | PUSH_ADDR binary_op, r0 6751 |.endif 6752 | SET_EX_OPLINE opline, r0 6753 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6754 |.if not(X64) 6755 | add r4, 12 6756 |.endif 6757 zend_jit_check_exception(Dst); 6758 | jmp >9 6759 |.code 6760 |1: 6761 } 6762 } 6763 6764 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6765 switch (opline->extended_value) { 6766 case ZEND_ADD: 6767 case ZEND_SUB: 6768 case ZEND_MUL: 6769 case ZEND_DIV: 6770 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6771 1 /* may overflow */, may_throw)) { 6772 return 0; 6773 } 6774 break; 6775 case ZEND_BW_OR: 6776 case ZEND_BW_AND: 6777 case ZEND_BW_XOR: 6778 case ZEND_SL: 6779 case ZEND_SR: 6780 case ZEND_MOD: 6781 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6782 IS_CV, opline->op1, var_addr, var_info, NULL, 6783 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6784 op1_data_range, 6785 0, var_addr, var_def_info, var_info, may_throw)) { 6786 return 0; 6787 } 6788 break; 6789 case ZEND_CONCAT: 6790 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6791 may_throw)) { 6792 return 0; 6793 } 6794 break; 6795 default: 6796 ZEND_UNREACHABLE(); 6797 } 6798 } 6799 6800 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6801 binary_op_type binary_op; 6802 6803 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6804 |.cold_code 6805 |7: 6806 } 6807 6808 | SET_EX_OPLINE opline, r0 6809 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 6810 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6811 } 6812 if (opline->op2_type == IS_UNUSED) { 6813 | xor FCARG2a, FCARG2a 6814 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6815 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6816 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6817 } else { 6818 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6819 } 6820 binary_op = get_binary_op(opline->extended_value); 6821 |.if X64 6822 | LOAD_ZVAL_ADDR CARG3, op3_addr 6823 | LOAD_ADDR CARG4, binary_op 6824 |.else 6825 | sub r4, 8 6826 | PUSH_ADDR binary_op, r0 6827 | PUSH_ZVAL_ADDR op3_addr, r0 6828 |.endif 6829 | EXT_CALL zend_jit_assign_dim_op_helper, r0 6830 |.if not(X64) 6831 | add r4, 8 6832 |.endif 6833 if (!zend_jit_check_exception(Dst)) { 6834 return 0; 6835 } 6836 6837 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6838 | jmp >9 // END 6839 |.code 6840 } 6841 } 6842 6843 |9: 6844 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6845 6846 return 1; 6847} 6848 6849static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6850{ 6851 zend_jit_addr op1_addr, op2_addr; 6852 6853 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6854 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6855 6856 op1_addr = OP1_ADDR(); 6857 op2_addr = OP2_ADDR(); 6858 6859 if (op1_info & MAY_BE_REF) { 6860 binary_op_type binary_op = get_binary_op(opline->extended_value); 6861 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6862 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 6863 | GET_Z_PTR FCARG1a, FCARG1a 6864 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6865 | jnz >2 6866 | add FCARG1a, offsetof(zend_reference, val) 6867 |.cold_code 6868 |2: 6869 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6870 |.if X64 6871 | LOAD_ADDR CARG3, binary_op 6872 |.else 6873 | sub r4, 12 6874 | PUSH_ADDR binary_op, r0 6875 |.endif 6876 | SET_EX_OPLINE opline, r0 6877 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6878 |.if not(X64) 6879 | add r4, 12 6880 |.endif 6881 zend_jit_check_exception(Dst); 6882 | jmp >9 6883 |.code 6884 |1: 6885 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6886 } 6887 6888 int result; 6889 switch (opline->extended_value) { 6890 case ZEND_ADD: 6891 case ZEND_SUB: 6892 case ZEND_MUL: 6893 case ZEND_DIV: 6894 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6895 break; 6896 case ZEND_BW_OR: 6897 case ZEND_BW_AND: 6898 case ZEND_BW_XOR: 6899 case ZEND_SL: 6900 case ZEND_SR: 6901 case ZEND_MOD: 6902 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6903 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6904 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6905 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6906 break; 6907 case ZEND_CONCAT: 6908 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6909 break; 6910 default: 6911 ZEND_UNREACHABLE(); 6912 } 6913 |9: 6914 return result; 6915} 6916 6917static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, 6918 zend_ssa_range *op1_range, 6919 zend_jit_addr op1_addr, 6920 zend_ssa_range *op2_range, 6921 zend_jit_addr op2_addr, 6922 zend_bool *result) 6923{ 6924 zend_long op1_min; 6925 zend_long op1_max; 6926 zend_long op2_min; 6927 zend_long op2_max; 6928 6929 if (op1_range) { 6930 op1_min = op1_range->min; 6931 op1_max = op1_range->max; 6932 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 6933 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); 6934 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); 6935 } else { 6936 return 0; 6937 } 6938 6939 if (op2_range) { 6940 op2_min = op2_range->min; 6941 op2_max = op2_range->max; 6942 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 6943 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); 6944 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); 6945 } else { 6946 return 0; 6947 } 6948 6949 switch (opline->opcode) { 6950 case ZEND_IS_EQUAL: 6951 case ZEND_IS_IDENTICAL: 6952 case ZEND_CASE: 6953 case ZEND_CASE_STRICT: 6954 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { 6955 *result = 1; 6956 return 1; 6957 } else if (op1_max < op2_min || op1_min > op2_max) { 6958 *result = 0; 6959 return 1; 6960 } 6961 return 0; 6962 case ZEND_IS_NOT_EQUAL: 6963 case ZEND_IS_NOT_IDENTICAL: 6964 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { 6965 *result = 0; 6966 return 1; 6967 } else if (op1_max < op2_min || op1_min > op2_max) { 6968 *result = 1; 6969 return 1; 6970 } 6971 return 0; 6972 case ZEND_IS_SMALLER: 6973 if (op1_max < op2_min) { 6974 *result = 1; 6975 return 1; 6976 } else if (op1_min >= op2_max) { 6977 *result = 0; 6978 return 1; 6979 } 6980 return 0; 6981 case ZEND_IS_SMALLER_OR_EQUAL: 6982 if (op1_max <= op2_min) { 6983 *result = 1; 6984 return 1; 6985 } else if (op1_min > op2_max) { 6986 *result = 0; 6987 return 1; 6988 } 6989 return 0; 6990 default: 6991 ZEND_UNREACHABLE(); 6992 } 6993 return 0; 6994} 6995 6996static int zend_jit_cmp_long_long(dasm_State **Dst, 6997 const zend_op *opline, 6998 zend_ssa_range *op1_range, 6999 zend_jit_addr op1_addr, 7000 zend_ssa_range *op2_range, 7001 zend_jit_addr op2_addr, 7002 zend_jit_addr res_addr, 7003 zend_uchar smart_branch_opcode, 7004 uint32_t target_label, 7005 uint32_t target_label2, 7006 const void *exit_addr, 7007 zend_bool skip_comparison) 7008{ 7009 zend_bool swap = 0; 7010 zend_bool result; 7011 7012 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7013 if (!smart_branch_opcode || 7014 smart_branch_opcode == ZEND_JMPZ_EX || 7015 smart_branch_opcode == ZEND_JMPNZ_EX) { 7016 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7017 } 7018 if (smart_branch_opcode && !exit_addr) { 7019 if (smart_branch_opcode == ZEND_JMPZ || 7020 smart_branch_opcode == ZEND_JMPZ_EX) { 7021 if (!result) { 7022 | jmp => target_label 7023 } 7024 } else if (smart_branch_opcode == ZEND_JMPNZ || 7025 smart_branch_opcode == ZEND_JMPNZ_EX) { 7026 if (result) { 7027 | jmp => target_label 7028 } 7029 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7030 if (!result) { 7031 | jmp => target_label 7032 } else { 7033 | jmp => target_label2 7034 } 7035 } else { 7036 ZEND_UNREACHABLE(); 7037 } 7038 } 7039 return 1; 7040 } 7041 7042 if (skip_comparison) { 7043 if (Z_MODE(op1_addr) != IS_REG && 7044 (Z_MODE(op2_addr) == IS_REG || 7045 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7046 swap = 1; 7047 } 7048 } else if (Z_MODE(op1_addr) == IS_REG) { 7049 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7050 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7051 } else { 7052 | LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0 7053 } 7054 } else if (Z_MODE(op2_addr) == IS_REG) { 7055 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7056 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7057 } else { 7058 | LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0 7059 } 7060 swap = 1; 7061 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7062 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7063 swap = 1; 7064 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7065 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7066 } else { 7067 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7068 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7069 | test r0, r0 7070 } else { 7071 | LONG_OP cmp, ZREG_R0, op2_addr, r0 7072 } 7073 } 7074 7075 if (smart_branch_opcode) { 7076 if (smart_branch_opcode == ZEND_JMPZ_EX || 7077 smart_branch_opcode == ZEND_JMPNZ_EX) { 7078 7079 switch (opline->opcode) { 7080 case ZEND_IS_EQUAL: 7081 case ZEND_IS_IDENTICAL: 7082 case ZEND_CASE: 7083 case ZEND_CASE_STRICT: 7084 | sete al 7085 break; 7086 case ZEND_IS_NOT_EQUAL: 7087 case ZEND_IS_NOT_IDENTICAL: 7088 | setne al 7089 break; 7090 case ZEND_IS_SMALLER: 7091 if (swap) { 7092 | setg al 7093 } else { 7094 | setl al 7095 } 7096 break; 7097 case ZEND_IS_SMALLER_OR_EQUAL: 7098 if (swap) { 7099 | setge al 7100 } else { 7101 | setle al 7102 } 7103 break; 7104 default: 7105 ZEND_UNREACHABLE(); 7106 } 7107 | movzx eax, al 7108 | lea eax, [eax + 2] 7109 | SET_ZVAL_TYPE_INFO res_addr, eax 7110 } 7111 if (smart_branch_opcode == ZEND_JMPZ || 7112 smart_branch_opcode == ZEND_JMPZ_EX) { 7113 switch (opline->opcode) { 7114 case ZEND_IS_EQUAL: 7115 case ZEND_IS_IDENTICAL: 7116 case ZEND_CASE: 7117 case ZEND_CASE_STRICT: 7118 if (exit_addr) { 7119 | jne &exit_addr 7120 } else { 7121 | jne => target_label 7122 } 7123 break; 7124 case ZEND_IS_NOT_EQUAL: 7125 if (exit_addr) { 7126 | je &exit_addr 7127 } else { 7128 | je => target_label 7129 } 7130 break; 7131 case ZEND_IS_NOT_IDENTICAL: 7132 if (exit_addr) { 7133 | jne &exit_addr 7134 } else { 7135 | je => target_label 7136 } 7137 break; 7138 case ZEND_IS_SMALLER: 7139 if (swap) { 7140 if (exit_addr) { 7141 | jle &exit_addr 7142 } else { 7143 | jle => target_label 7144 } 7145 } else { 7146 if (exit_addr) { 7147 | jge &exit_addr 7148 } else { 7149 | jge => target_label 7150 } 7151 } 7152 break; 7153 case ZEND_IS_SMALLER_OR_EQUAL: 7154 if (swap) { 7155 if (exit_addr) { 7156 | jl &exit_addr 7157 } else { 7158 | jl => target_label 7159 } 7160 } else { 7161 if (exit_addr) { 7162 | jg &exit_addr 7163 } else { 7164 | jg => target_label 7165 } 7166 } 7167 break; 7168 default: 7169 ZEND_UNREACHABLE(); 7170 } 7171 } else if (smart_branch_opcode == ZEND_JMPNZ || 7172 smart_branch_opcode == ZEND_JMPNZ_EX) { 7173 switch (opline->opcode) { 7174 case ZEND_IS_EQUAL: 7175 case ZEND_IS_IDENTICAL: 7176 case ZEND_CASE: 7177 case ZEND_CASE_STRICT: 7178 if (exit_addr) { 7179 | je &exit_addr 7180 } else { 7181 | je => target_label 7182 } 7183 break; 7184 case ZEND_IS_NOT_EQUAL: 7185 if (exit_addr) { 7186 | jne &exit_addr 7187 } else { 7188 | jne => target_label 7189 } 7190 break; 7191 case ZEND_IS_NOT_IDENTICAL: 7192 if (exit_addr) { 7193 | je &exit_addr 7194 } else { 7195 | jne => target_label 7196 } 7197 break; 7198 case ZEND_IS_SMALLER: 7199 if (swap) { 7200 if (exit_addr) { 7201 | jg &exit_addr 7202 } else { 7203 | jg => target_label 7204 } 7205 } else { 7206 if (exit_addr) { 7207 | jl &exit_addr 7208 } else { 7209 | jl => target_label 7210 } 7211 } 7212 break; 7213 case ZEND_IS_SMALLER_OR_EQUAL: 7214 if (swap) { 7215 if (exit_addr) { 7216 | jge &exit_addr 7217 } else { 7218 | jge => target_label 7219 } 7220 } else { 7221 if (exit_addr) { 7222 | jle &exit_addr 7223 } else { 7224 | jle => target_label 7225 } 7226 } 7227 break; 7228 default: 7229 ZEND_UNREACHABLE(); 7230 } 7231 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7232 switch (opline->opcode) { 7233 case ZEND_IS_EQUAL: 7234 case ZEND_IS_IDENTICAL: 7235 case ZEND_CASE: 7236 case ZEND_CASE_STRICT: 7237 | jne => target_label 7238 break; 7239 case ZEND_IS_NOT_EQUAL: 7240 case ZEND_IS_NOT_IDENTICAL: 7241 | je => target_label 7242 break; 7243 case ZEND_IS_SMALLER: 7244 if (swap) { 7245 | jle => target_label 7246 } else { 7247 | jge => target_label 7248 } 7249 break; 7250 case ZEND_IS_SMALLER_OR_EQUAL: 7251 if (swap) { 7252 | jl => target_label 7253 } else { 7254 | jg => target_label 7255 } 7256 break; 7257 default: 7258 ZEND_UNREACHABLE(); 7259 } 7260 | jmp => target_label2 7261 } else { 7262 ZEND_UNREACHABLE(); 7263 } 7264 } else { 7265 switch (opline->opcode) { 7266 case ZEND_IS_EQUAL: 7267 case ZEND_IS_IDENTICAL: 7268 case ZEND_CASE: 7269 case ZEND_CASE_STRICT: 7270 | sete al 7271 break; 7272 case ZEND_IS_NOT_EQUAL: 7273 case ZEND_IS_NOT_IDENTICAL: 7274 | setne al 7275 break; 7276 case ZEND_IS_SMALLER: 7277 if (swap) { 7278 | setg al 7279 } else { 7280 | setl al 7281 } 7282 break; 7283 case ZEND_IS_SMALLER_OR_EQUAL: 7284 if (swap) { 7285 | setge al 7286 } else { 7287 | setle al 7288 } 7289 break; 7290 default: 7291 ZEND_UNREACHABLE(); 7292 } 7293 | movzx eax, al 7294 | add eax, 2 7295 | SET_ZVAL_TYPE_INFO res_addr, eax 7296 } 7297 7298 return 1; 7299} 7300 7301static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7302{ 7303 if (smart_branch_opcode) { 7304 if (smart_branch_opcode == ZEND_JMPZ) { 7305 switch (opline->opcode) { 7306 case ZEND_IS_EQUAL: 7307 case ZEND_IS_IDENTICAL: 7308 case ZEND_CASE: 7309 case ZEND_CASE_STRICT: 7310 if (exit_addr) { 7311 | jne &exit_addr 7312 | jp &exit_addr 7313 } else { 7314 | jne => target_label 7315 | jp => target_label 7316 } 7317 break; 7318 case ZEND_IS_NOT_EQUAL: 7319 | jp >1 7320 if (exit_addr) { 7321 | je &exit_addr 7322 } else { 7323 | je => target_label 7324 } 7325 |1: 7326 break; 7327 case ZEND_IS_NOT_IDENTICAL: 7328 if (exit_addr) { 7329 | jne &exit_addr 7330 | jp &exit_addr 7331 } else { 7332 | jp >1 7333 | je => target_label 7334 |1: 7335 } 7336 break; 7337 case ZEND_IS_SMALLER: 7338 if (swap) { 7339 if (exit_addr) { 7340 | jbe &exit_addr 7341 } else { 7342 | jbe => target_label 7343 } 7344 } else { 7345 if (exit_addr) { 7346 | jae &exit_addr 7347 | jp &exit_addr 7348 } else { 7349 | jae => target_label 7350 | jp => target_label 7351 } 7352 } 7353 break; 7354 case ZEND_IS_SMALLER_OR_EQUAL: 7355 if (swap) { 7356 if (exit_addr) { 7357 | jb &exit_addr 7358 } else { 7359 | jb => target_label 7360 } 7361 } else { 7362 if (exit_addr) { 7363 | ja &exit_addr 7364 | jp &exit_addr 7365 } else { 7366 | ja => target_label 7367 | jp => target_label 7368 } 7369 } 7370 break; 7371 default: 7372 ZEND_UNREACHABLE(); 7373 } 7374 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7375 switch (opline->opcode) { 7376 case ZEND_IS_EQUAL: 7377 case ZEND_IS_IDENTICAL: 7378 case ZEND_CASE: 7379 case ZEND_CASE_STRICT: 7380 | jp >1 7381 if (exit_addr) { 7382 | je &exit_addr 7383 } else { 7384 | je => target_label 7385 } 7386 |1: 7387 break; 7388 case ZEND_IS_NOT_EQUAL: 7389 if (exit_addr) { 7390 | jne &exit_addr 7391 | jp &exit_addr 7392 } else { 7393 | jne => target_label 7394 | jp => target_label 7395 } 7396 break; 7397 case ZEND_IS_NOT_IDENTICAL: 7398 if (exit_addr) { 7399 | jp >1 7400 | je &exit_addr 7401 |1: 7402 } else { 7403 | jne => target_label 7404 | jp => target_label 7405 } 7406 break; 7407 case ZEND_IS_SMALLER: 7408 if (swap) { 7409 if (exit_addr) { 7410 | ja &exit_addr 7411 } else { 7412 | ja => target_label 7413 } 7414 } else { 7415 | jp >1 7416 if (exit_addr) { 7417 | jb &exit_addr 7418 } else { 7419 | jb => target_label 7420 } 7421 |1: 7422 } 7423 break; 7424 case ZEND_IS_SMALLER_OR_EQUAL: 7425 if (swap) { 7426 if (exit_addr) { 7427 | jae &exit_addr 7428 } else { 7429 | jae => target_label 7430 } 7431 } else { 7432 | jp >1 7433 if (exit_addr) { 7434 | jbe &exit_addr 7435 } else { 7436 | jbe => target_label 7437 } 7438 |1: 7439 } 7440 break; 7441 default: 7442 ZEND_UNREACHABLE(); 7443 } 7444 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7445 switch (opline->opcode) { 7446 case ZEND_IS_EQUAL: 7447 case ZEND_IS_IDENTICAL: 7448 case ZEND_CASE: 7449 case ZEND_CASE_STRICT: 7450 | jne => target_label 7451 | jp => target_label 7452 break; 7453 case ZEND_IS_NOT_EQUAL: 7454 case ZEND_IS_NOT_IDENTICAL: 7455 | jp => target_label2 7456 | je => target_label 7457 break; 7458 case ZEND_IS_SMALLER: 7459 if (swap) { 7460 | jbe => target_label 7461 } else { 7462 | jae => target_label 7463 | jp => target_label 7464 } 7465 break; 7466 case ZEND_IS_SMALLER_OR_EQUAL: 7467 if (swap) { 7468 | jb => target_label 7469 } else { 7470 | ja => target_label 7471 | jp => target_label 7472 } 7473 break; 7474 default: 7475 ZEND_UNREACHABLE(); 7476 } 7477 | jmp => target_label2 7478 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7479 switch (opline->opcode) { 7480 case ZEND_IS_EQUAL: 7481 case ZEND_IS_IDENTICAL: 7482 case ZEND_CASE: 7483 case ZEND_CASE_STRICT: 7484 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7485 | jne => target_label 7486 | jp => target_label 7487 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7488 break; 7489 case ZEND_IS_NOT_EQUAL: 7490 case ZEND_IS_NOT_IDENTICAL: 7491 | jp >1 7492 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7493 | je => target_label 7494 |1: 7495 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7496 break; 7497 case ZEND_IS_SMALLER: 7498 if (swap) { 7499 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7500 | jbe => target_label 7501 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7502 } else { 7503 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7504 | jae => target_label 7505 | jp => target_label 7506 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7507 } 7508 break; 7509 case ZEND_IS_SMALLER_OR_EQUAL: 7510 if (swap) { 7511 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7512 | jb => target_label 7513 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7514 } else { 7515 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7516 | ja => target_label 7517 | jp => target_label 7518 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7519 } 7520 break; 7521 default: 7522 ZEND_UNREACHABLE(); 7523 } 7524 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7525 switch (opline->opcode) { 7526 case ZEND_IS_EQUAL: 7527 case ZEND_IS_IDENTICAL: 7528 case ZEND_CASE: 7529 case ZEND_CASE_STRICT: 7530 | jp >1 7531 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7532 | je => target_label 7533 |1: 7534 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7535 break; 7536 case ZEND_IS_NOT_EQUAL: 7537 case ZEND_IS_NOT_IDENTICAL: 7538 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7539 | jne => target_label 7540 | jp => target_label 7541 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7542 break; 7543 case ZEND_IS_SMALLER: 7544 if (swap) { 7545 | seta al 7546 | movzx eax, al 7547 | lea eax, [eax + 2] 7548 | SET_ZVAL_TYPE_INFO res_addr, eax 7549 | ja => target_label 7550 } else { 7551 | jp >1 7552 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7553 | jb => target_label 7554 |1: 7555 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7556 } 7557 break; 7558 case ZEND_IS_SMALLER_OR_EQUAL: 7559 if (swap) { 7560 | setae al 7561 | movzx eax, al 7562 | lea eax, [eax + 2] 7563 | SET_ZVAL_TYPE_INFO res_addr, eax 7564 | jae => target_label 7565 } else { 7566 | jp >1 7567 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7568 | jbe => target_label 7569 |1: 7570 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7571 } 7572 break; 7573 default: 7574 ZEND_UNREACHABLE(); 7575 } 7576 } else { 7577 ZEND_UNREACHABLE(); 7578 } 7579 } else { 7580 switch (opline->opcode) { 7581 case ZEND_IS_EQUAL: 7582 case ZEND_IS_IDENTICAL: 7583 case ZEND_CASE: 7584 case ZEND_CASE_STRICT: 7585 | jp >1 7586 | mov eax, IS_TRUE 7587 | je >2 7588 |1: 7589 | mov eax, IS_FALSE 7590 |2: 7591 break; 7592 case ZEND_IS_NOT_EQUAL: 7593 case ZEND_IS_NOT_IDENTICAL: 7594 | jp >1 7595 | mov eax, IS_FALSE 7596 | je >2 7597 |1: 7598 | mov eax, IS_TRUE 7599 |2: 7600 break; 7601 case ZEND_IS_SMALLER: 7602 if (swap) { 7603 | seta al 7604 | movzx eax, al 7605 | add eax, 2 7606 } else { 7607 | jp >1 7608 | mov eax, IS_TRUE 7609 | jb >2 7610 |1: 7611 | mov eax, IS_FALSE 7612 |2: 7613 } 7614 break; 7615 case ZEND_IS_SMALLER_OR_EQUAL: 7616 if (swap) { 7617 | setae al 7618 | movzx eax, al 7619 | add eax, 2 7620 } else { 7621 | jp >1 7622 | mov eax, IS_TRUE 7623 | jbe >2 7624 |1: 7625 | mov eax, IS_FALSE 7626 |2: 7627 } 7628 break; 7629 default: 7630 ZEND_UNREACHABLE(); 7631 } 7632 | SET_ZVAL_TYPE_INFO res_addr, eax 7633 } 7634 7635 return 1; 7636} 7637 7638static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7639{ 7640 zend_reg tmp_reg = ZREG_XMM0; 7641 7642 | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7643 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr 7644 7645 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7646} 7647 7648static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7649{ 7650 zend_reg tmp_reg = ZREG_XMM0; 7651 7652 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7653 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr 7654 7655 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7656} 7657 7658static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7659{ 7660 zend_bool swap = 0; 7661 7662 if (Z_MODE(op1_addr) == IS_REG) { 7663 | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr 7664 } else if (Z_MODE(op2_addr) == IS_REG) { 7665 | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr 7666 swap = 1; 7667 } else { 7668 zend_reg tmp_reg = ZREG_XMM0; 7669 7670 | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr 7671 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr 7672 } 7673 7674 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7675} 7676 7677static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7678{ 7679 | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) 7680 if (smart_branch_opcode) { 7681 if (smart_branch_opcode == ZEND_JMPZ_EX || 7682 smart_branch_opcode == ZEND_JMPNZ_EX) { 7683 switch (opline->opcode) { 7684 case ZEND_IS_EQUAL: 7685 case ZEND_CASE: 7686 | sete al 7687 break; 7688 case ZEND_IS_NOT_EQUAL: 7689 | setne al 7690 break; 7691 case ZEND_IS_SMALLER: 7692 | setl al 7693 break; 7694 case ZEND_IS_SMALLER_OR_EQUAL: 7695 | setle al 7696 break; 7697 default: 7698 ZEND_UNREACHABLE(); 7699 } 7700 | movzx eax, al 7701 | lea eax, [eax + 2] 7702 | SET_ZVAL_TYPE_INFO res_addr, eax 7703 } 7704 if (smart_branch_opcode == ZEND_JMPZ || 7705 smart_branch_opcode == ZEND_JMPZ_EX) { 7706 switch (opline->opcode) { 7707 case ZEND_IS_EQUAL: 7708 case ZEND_CASE: 7709 if (exit_addr) { 7710 | jne &exit_addr 7711 } else { 7712 | jne => target_label 7713 } 7714 break; 7715 case ZEND_IS_NOT_EQUAL: 7716 if (exit_addr) { 7717 | je &exit_addr 7718 } else { 7719 | je => target_label 7720 } 7721 break; 7722 case ZEND_IS_SMALLER: 7723 if (exit_addr) { 7724 | jge &exit_addr 7725 } else { 7726 | jge => target_label 7727 } 7728 break; 7729 case ZEND_IS_SMALLER_OR_EQUAL: 7730 if (exit_addr) { 7731 | jg &exit_addr 7732 } else { 7733 | jg => target_label 7734 } 7735 break; 7736 default: 7737 ZEND_UNREACHABLE(); 7738 } 7739 } else if (smart_branch_opcode == ZEND_JMPNZ || 7740 smart_branch_opcode == ZEND_JMPNZ_EX) { 7741 switch (opline->opcode) { 7742 case ZEND_IS_EQUAL: 7743 case ZEND_CASE: 7744 if (exit_addr) { 7745 | je &exit_addr 7746 } else { 7747 | je => target_label 7748 } 7749 break; 7750 case ZEND_IS_NOT_EQUAL: 7751 if (exit_addr) { 7752 | jne &exit_addr 7753 } else { 7754 | jne => target_label 7755 } 7756 break; 7757 case ZEND_IS_SMALLER: 7758 if (exit_addr) { 7759 | jl &exit_addr 7760 } else { 7761 | jl => target_label 7762 } 7763 break; 7764 case ZEND_IS_SMALLER_OR_EQUAL: 7765 if (exit_addr) { 7766 | jle &exit_addr 7767 } else { 7768 | jle => target_label 7769 } 7770 break; 7771 default: 7772 ZEND_UNREACHABLE(); 7773 } 7774 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7775 switch (opline->opcode) { 7776 case ZEND_IS_EQUAL: 7777 case ZEND_CASE: 7778 | jne => target_label 7779 break; 7780 case ZEND_IS_NOT_EQUAL: 7781 | je => target_label 7782 break; 7783 case ZEND_IS_SMALLER: 7784 | jge => target_label 7785 break; 7786 case ZEND_IS_SMALLER_OR_EQUAL: 7787 | jg => target_label 7788 break; 7789 default: 7790 ZEND_UNREACHABLE(); 7791 } 7792 | jmp => target_label2 7793 } else { 7794 ZEND_UNREACHABLE(); 7795 } 7796 } else { 7797 switch (opline->opcode) { 7798 case ZEND_IS_EQUAL: 7799 case ZEND_CASE: 7800 | sete al 7801 break; 7802 case ZEND_IS_NOT_EQUAL: 7803 | setne al 7804 break; 7805 case ZEND_IS_SMALLER: 7806 | setl al 7807 break; 7808 case ZEND_IS_SMALLER_OR_EQUAL: 7809 | setle al 7810 break; 7811 default: 7812 ZEND_UNREACHABLE(); 7813 } 7814 | movzx eax, al 7815 | add eax, 2 7816 | SET_ZVAL_TYPE_INFO res_addr, eax 7817 } 7818 7819 return 1; 7820} 7821 7822static int zend_jit_cmp(dasm_State **Dst, 7823 const zend_op *opline, 7824 uint32_t op1_info, 7825 zend_ssa_range *op1_range, 7826 zend_jit_addr op1_addr, 7827 uint32_t op2_info, 7828 zend_ssa_range *op2_range, 7829 zend_jit_addr op2_addr, 7830 zend_jit_addr res_addr, 7831 int may_throw, 7832 zend_uchar smart_branch_opcode, 7833 uint32_t target_label, 7834 uint32_t target_label2, 7835 const void *exit_addr, 7836 zend_bool skip_comparison) 7837{ 7838 zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7839 zend_bool has_slow; 7840 7841 has_slow = 7842 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7843 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7844 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7845 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7846 7847 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7848 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7849 if (op1_info & MAY_BE_DOUBLE) { 7850 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7851 } else { 7852 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7853 } 7854 } 7855 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7856 if (op2_info & MAY_BE_DOUBLE) { 7857 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7858 |.cold_code 7859 |3: 7860 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7861 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7862 } 7863 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7864 return 0; 7865 } 7866 | jmp >6 7867 |.code 7868 } else { 7869 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7870 } 7871 } 7872 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7873 return 0; 7874 } 7875 if (op1_info & MAY_BE_DOUBLE) { 7876 |.cold_code 7877 |4: 7878 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7879 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7880 } 7881 if (op2_info & MAY_BE_DOUBLE) { 7882 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7883 if (!same_ops) { 7884 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 7885 } else { 7886 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7887 } 7888 } 7889 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7890 return 0; 7891 } 7892 | jmp >6 7893 } 7894 if (!same_ops) { 7895 |5: 7896 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7897 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7898 } 7899 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7900 return 0; 7901 } 7902 | jmp >6 7903 } 7904 |.code 7905 } 7906 } else if ((op1_info & MAY_BE_DOUBLE) && 7907 !(op1_info & MAY_BE_LONG) && 7908 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7909 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7910 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7911 } 7912 if (op2_info & MAY_BE_DOUBLE) { 7913 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7914 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7915 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 7916 } else { 7917 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7918 } 7919 } 7920 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7921 return 0; 7922 } 7923 } 7924 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7925 if (op2_info & MAY_BE_DOUBLE) { 7926 |.cold_code 7927 } 7928 |3: 7929 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7930 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7931 } 7932 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7933 return 0; 7934 } 7935 if (op2_info & MAY_BE_DOUBLE) { 7936 | jmp >6 7937 |.code 7938 } 7939 } 7940 } else if ((op2_info & MAY_BE_DOUBLE) && 7941 !(op2_info & MAY_BE_LONG) && 7942 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7943 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7944 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7945 } 7946 if (op1_info & MAY_BE_DOUBLE) { 7947 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7948 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7949 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 7950 } else { 7951 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7952 } 7953 } 7954 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7955 return 0; 7956 } 7957 } 7958 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7959 if (op1_info & MAY_BE_DOUBLE) { 7960 |.cold_code 7961 } 7962 |3: 7963 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7964 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7965 } 7966 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7967 return 0; 7968 } 7969 if (op1_info & MAY_BE_DOUBLE) { 7970 | jmp >6 7971 |.code 7972 } 7973 } 7974 } 7975 7976 if (has_slow || 7977 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7978 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7979 if (has_slow) { 7980 |.cold_code 7981 |9: 7982 } 7983 | SET_EX_OPLINE opline, r0 7984 if (Z_MODE(op1_addr) == IS_REG) { 7985 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7986 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7987 return 0; 7988 } 7989 op1_addr = real_addr; 7990 } 7991 if (Z_MODE(op2_addr) == IS_REG) { 7992 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7993 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7994 return 0; 7995 } 7996 op2_addr = real_addr; 7997 } 7998 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 7999 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 8000 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 8001 | mov FCARG1a, opline->op1.var 8002 | EXT_CALL zend_jit_undefined_op_helper, r0 8003 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8004 |1: 8005 } 8006 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 8007 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 8008 | mov T1, FCARG2a // save 8009 | mov FCARG1a, opline->op2.var 8010 | EXT_CALL zend_jit_undefined_op_helper, r0 8011 | mov FCARG2a, T1 // restore 8012 |.if X64 8013 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval 8014 |.else 8015 | sub r4, 12 8016 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 8017 |.endif 8018 | jmp >2 8019 |1: 8020 |.if X64 8021 | LOAD_ZVAL_ADDR CARG3, op2_addr 8022 |.else 8023 | sub r4, 12 8024 | PUSH_ZVAL_ADDR op2_addr, r0 8025 |.endif 8026 |2: 8027 } else { 8028 |.if X64 8029 | LOAD_ZVAL_ADDR CARG3, op2_addr 8030 |.else 8031 | sub r4, 12 8032 | PUSH_ZVAL_ADDR op2_addr, r0 8033 |.endif 8034 } 8035 | LOAD_ZVAL_ADDR FCARG1a, res_addr 8036 | EXT_CALL compare_function, r0 8037 |.if not(X64) 8038 | add r4, 12 8039 |.endif 8040 if (opline->opcode != ZEND_CASE) { 8041 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 8042 } 8043 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 8044 if (may_throw) { 8045 zend_jit_check_exception_undef_result(Dst, opline); 8046 } 8047 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8048 return 0; 8049 } 8050 if (has_slow) { 8051 | jmp >6 8052 |.code 8053 } 8054 } 8055 8056 |6: 8057 8058 return 1; 8059} 8060 8061static int zend_jit_identical(dasm_State **Dst, 8062 const zend_op *opline, 8063 uint32_t op1_info, 8064 zend_ssa_range *op1_range, 8065 zend_jit_addr op1_addr, 8066 uint32_t op2_info, 8067 zend_ssa_range *op2_range, 8068 zend_jit_addr op2_addr, 8069 zend_jit_addr res_addr, 8070 int may_throw, 8071 zend_uchar smart_branch_opcode, 8072 uint32_t target_label, 8073 uint32_t target_label2, 8074 const void *exit_addr, 8075 zend_bool skip_comparison) 8076{ 8077 uint32_t identical_label = (uint32_t)-1; 8078 uint32_t not_identical_label = (uint32_t)-1; 8079 8080 if (smart_branch_opcode && !exit_addr) { 8081 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8082 if (smart_branch_opcode == ZEND_JMPZ) { 8083 not_identical_label = target_label; 8084 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8085 identical_label = target_label; 8086 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8087 not_identical_label = target_label; 8088 identical_label = target_label2; 8089 } else { 8090 ZEND_UNREACHABLE(); 8091 } 8092 } else { 8093 if (smart_branch_opcode == ZEND_JMPZ) { 8094 identical_label = target_label; 8095 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8096 not_identical_label = target_label; 8097 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8098 identical_label = target_label; 8099 not_identical_label = target_label2; 8100 } else { 8101 ZEND_UNREACHABLE(); 8102 } 8103 } 8104 } 8105 8106 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8107 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8108 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8109 return 0; 8110 } 8111 return 1; 8112 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8113 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8114 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8115 return 0; 8116 } 8117 return 1; 8118 } 8119 8120 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8121 op1_info |= MAY_BE_NULL; 8122 op2_info |= MAY_BE_NULL; 8123 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8124 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8125 |.cold_code 8126 |1: 8127 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8128 | SET_EX_OPLINE opline, r0 8129 | mov FCARG1d, opline->op1.var 8130 | EXT_CALL zend_jit_undefined_op_helper, r0 8131 if (may_throw) { 8132 zend_jit_check_exception_undef_result(Dst, opline); 8133 } 8134 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8135 | jmp >1 8136 |.code 8137 |1: 8138 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8139 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8140 |.cold_code 8141 |1: 8142 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8143 | SET_EX_OPLINE opline, r0 8144 | mov aword T1, FCARG1a // save 8145 | mov FCARG1d, opline->op2.var 8146 | EXT_CALL zend_jit_undefined_op_helper, r0 8147 if (may_throw) { 8148 zend_jit_check_exception_undef_result(Dst, opline); 8149 } 8150 | mov FCARG1a, aword T1 // restore 8151 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8152 | jmp >1 8153 |.code 8154 |1: 8155 } else if (op1_info & MAY_BE_UNDEF) { 8156 op1_info |= MAY_BE_NULL; 8157 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8158 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8159 |.cold_code 8160 |1: 8161 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8162 | SET_EX_OPLINE opline, r0 8163 | mov FCARG1d, opline->op1.var 8164 | EXT_CALL zend_jit_undefined_op_helper, r0 8165 if (may_throw) { 8166 zend_jit_check_exception_undef_result(Dst, opline); 8167 } 8168 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8169 | jmp >1 8170 |.code 8171 |1: 8172 if (opline->op2_type != IS_CONST) { 8173 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8174 } 8175 } else if (op2_info & MAY_BE_UNDEF) { 8176 op2_info |= MAY_BE_NULL; 8177 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8178 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8179 |.cold_code 8180 |1: 8181 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8182 | SET_EX_OPLINE opline, r0 8183 | mov FCARG1d, opline->op2.var 8184 | EXT_CALL zend_jit_undefined_op_helper, r0 8185 if (may_throw) { 8186 zend_jit_check_exception_undef_result(Dst, opline); 8187 } 8188 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8189 | jmp >1 8190 |.code 8191 |1: 8192 if (opline->op1_type != IS_CONST) { 8193 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8194 } 8195 } else { 8196 if (opline->op1_type != IS_CONST) { 8197 if (Z_MODE(op1_addr) == IS_REG) { 8198 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8199 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8200 return 0; 8201 } 8202 op1_addr = real_addr; 8203 } 8204 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8205 } 8206 if (opline->op2_type != IS_CONST) { 8207 if (Z_MODE(op2_addr) == IS_REG) { 8208 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8209 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8210 return 0; 8211 } 8212 op2_addr = real_addr; 8213 } 8214 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8215 } 8216 } 8217 if (opline->op1_type & (IS_CV|IS_VAR)) { 8218 | ZVAL_DEREF FCARG1a, op1_info 8219 } 8220 if (opline->op2_type & (IS_CV|IS_VAR)) { 8221 | ZVAL_DEREF FCARG2a, op2_info 8222 } 8223 8224 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8225 if ((opline->opcode != ZEND_CASE_STRICT && 8226 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8227 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8228 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8229 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8230 | SET_EX_OPLINE opline, r0 8231 if (opline->opcode != ZEND_CASE_STRICT) { 8232 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8233 } 8234 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8235 } 8236 if (smart_branch_opcode) { 8237 zend_jit_check_exception_undef_result(Dst, opline); 8238 if (exit_addr) { 8239 if (smart_branch_opcode == ZEND_JMPZ) { 8240 | jmp &exit_addr 8241 } 8242 } else if (not_identical_label != (uint32_t)-1) { 8243 | jmp =>not_identical_label 8244 } 8245 } else { 8246 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8247 zend_jit_check_exception(Dst); 8248 } 8249 } else if (has_concrete_type(op1_info) && 8250 has_concrete_type(op2_info) && 8251 concrete_type(op1_info) == concrete_type(op2_info) && 8252 concrete_type(op1_info) <= IS_TRUE) { 8253 if (smart_branch_opcode) { 8254 if (exit_addr) { 8255 if (smart_branch_opcode == ZEND_JMPNZ) { 8256 | jmp &exit_addr 8257 } 8258 } else if (identical_label != (uint32_t)-1) { 8259 | jmp =>identical_label 8260 } 8261 } else { 8262 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8263 } 8264 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8265 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8266 if (smart_branch_opcode) { 8267 if (exit_addr) { 8268 if (smart_branch_opcode == ZEND_JMPNZ) { 8269 | jmp &exit_addr 8270 } 8271 } else if (identical_label != (uint32_t)-1) { 8272 | jmp =>identical_label 8273 } 8274 } else { 8275 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8276 } 8277 } else { 8278 if (smart_branch_opcode) { 8279 if (exit_addr) { 8280 if (smart_branch_opcode == ZEND_JMPZ) { 8281 | jmp &exit_addr 8282 } 8283 } else if (not_identical_label != (uint32_t)-1) { 8284 | jmp =>not_identical_label 8285 } 8286 } else { 8287 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8288 } 8289 } 8290 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8291 zval *val = Z_ZV(op1_addr); 8292 8293 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8294 if (smart_branch_opcode) { 8295 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8296 | jne >8 8297 | SET_EX_OPLINE opline, r0 8298 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8299 zend_jit_check_exception_undef_result(Dst, opline); 8300 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8301 | jmp &exit_addr 8302 } else if (identical_label != (uint32_t)-1) { 8303 | jmp =>identical_label 8304 } else { 8305 | jmp >9 8306 } 8307 |8: 8308 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8309 | je &exit_addr 8310 } else if (identical_label != (uint32_t)-1) { 8311 | je =>identical_label 8312 } else { 8313 | je >9 8314 } 8315 } else { 8316 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8317 | sete al 8318 } else { 8319 | setne al 8320 } 8321 | movzx eax, al 8322 | lea eax, [eax + 2] 8323 | SET_ZVAL_TYPE_INFO res_addr, eax 8324 } 8325 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8326 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8327 | SET_EX_OPLINE opline, r0 8328 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8329 zend_jit_check_exception_undef_result(Dst, opline); 8330 } 8331 if (exit_addr) { 8332 if (smart_branch_opcode == ZEND_JMPZ) { 8333 | jmp &exit_addr 8334 } 8335 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8336 | jmp =>not_identical_label 8337 } 8338 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8339 zval *val = Z_ZV(op2_addr); 8340 8341 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8342 if (smart_branch_opcode) { 8343 if (opline->opcode != ZEND_CASE_STRICT 8344 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8345 | jne >8 8346 | SET_EX_OPLINE opline, r0 8347 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8348 zend_jit_check_exception_undef_result(Dst, opline); 8349 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8350 | jmp &exit_addr 8351 } else if (identical_label != (uint32_t)-1) { 8352 | jmp =>identical_label 8353 } else { 8354 | jmp >9 8355 } 8356 |8: 8357 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8358 | je &exit_addr 8359 } else if (identical_label != (uint32_t)-1) { 8360 | je =>identical_label 8361 } else { 8362 | je >9 8363 } 8364 } else { 8365 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8366 | sete al 8367 } else { 8368 | setne al 8369 } 8370 | movzx eax, al 8371 | lea eax, [eax + 2] 8372 | SET_ZVAL_TYPE_INFO res_addr, eax 8373 } 8374 if (opline->opcode != ZEND_CASE_STRICT 8375 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8376 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8377 | SET_EX_OPLINE opline, r0 8378 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8379 zend_jit_check_exception_undef_result(Dst, opline); 8380 } 8381 if (smart_branch_opcode) { 8382 if (exit_addr) { 8383 if (smart_branch_opcode == ZEND_JMPZ) { 8384 | jmp &exit_addr 8385 } 8386 } else if (not_identical_label != (uint32_t)-1) { 8387 | jmp =>not_identical_label 8388 } 8389 } 8390 } else { 8391 if (opline->op1_type == IS_CONST) { 8392 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8393 } 8394 if (opline->op2_type == IS_CONST) { 8395 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8396 } 8397 | EXT_CALL zend_is_identical, r0 8398 if ((opline->opcode != ZEND_CASE_STRICT && 8399 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8400 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8401 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8402 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8403 | mov aword T1, r0 // save 8404 | SET_EX_OPLINE opline, r0 8405 if (opline->opcode != ZEND_CASE_STRICT) { 8406 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8407 } 8408 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8409 zend_jit_check_exception_undef_result(Dst, opline); 8410 | mov r0, aword T1 // restore 8411 } 8412 if (smart_branch_opcode) { 8413 | test al, al 8414 if (exit_addr) { 8415 if (smart_branch_opcode == ZEND_JMPNZ) { 8416 | jnz &exit_addr 8417 } else { 8418 | jz &exit_addr 8419 } 8420 } else if (not_identical_label != (uint32_t)-1) { 8421 | jz =>not_identical_label 8422 if (identical_label != (uint32_t)-1) { 8423 | jmp =>identical_label 8424 } 8425 } else if (identical_label != (uint32_t)-1) { 8426 | jnz =>identical_label 8427 } 8428 } else { 8429 | movzx eax, al 8430 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8431 | lea eax, [eax + 2] 8432 } else { 8433 | neg eax 8434 | lea eax, [eax + 3] 8435 } 8436 | SET_ZVAL_TYPE_INFO res_addr, eax 8437 } 8438 } 8439 8440 |9: 8441 if (may_throw) { 8442 zend_jit_check_exception(Dst); 8443 } 8444 return 1; 8445} 8446 8447static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 8448{ 8449 uint32_t true_label = -1; 8450 uint32_t false_label = -1; 8451 zend_bool set_bool = 0; 8452 zend_bool set_bool_not = 0; 8453 zend_bool set_delayed = 0; 8454 zend_bool jmp_done = 0; 8455 8456 if (branch_opcode == ZEND_BOOL) { 8457 set_bool = 1; 8458 } else if (branch_opcode == ZEND_BOOL_NOT) { 8459 set_bool = 1; 8460 set_bool_not = 1; 8461 } else if (branch_opcode == ZEND_JMPZ) { 8462 false_label = target_label; 8463 } else if (branch_opcode == ZEND_JMPNZ) { 8464 true_label = target_label; 8465 } else if (branch_opcode == ZEND_JMPZNZ) { 8466 true_label = target_label2; 8467 false_label = target_label; 8468 } else if (branch_opcode == ZEND_JMPZ_EX) { 8469 set_bool = 1; 8470 false_label = target_label; 8471 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8472 set_bool = 1; 8473 true_label = target_label; 8474 } else { 8475 ZEND_UNREACHABLE(); 8476 } 8477 8478 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8479 if (zend_is_true(Z_ZV(op1_addr))) { 8480 /* Always TRUE */ 8481 if (set_bool) { 8482 if (set_bool_not) { 8483 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8484 } else { 8485 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8486 } 8487 } 8488 if (true_label != (uint32_t)-1) { 8489 | jmp =>true_label; 8490 } 8491 } else { 8492 /* Always FALSE */ 8493 if (set_bool) { 8494 if (set_bool_not) { 8495 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8496 } else { 8497 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8498 } 8499 } 8500 if (false_label != (uint32_t)-1) { 8501 | jmp =>false_label; 8502 } 8503 } 8504 return 1; 8505 } 8506 8507 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8508 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8509 | ZVAL_DEREF FCARG1a, op1_info 8510 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 8511 } 8512 8513 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8514 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8515 /* Always TRUE */ 8516 if (set_bool) { 8517 if (set_bool_not) { 8518 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8519 } else { 8520 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8521 } 8522 } 8523 if (true_label != (uint32_t)-1) { 8524 | jmp =>true_label; 8525 } 8526 } else { 8527 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8528 /* Always FALSE */ 8529 if (set_bool) { 8530 if (set_bool_not) { 8531 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8532 } else { 8533 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8534 } 8535 } 8536 } else { 8537 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8538 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8539 if ((op1_info & MAY_BE_LONG) && 8540 !(op1_info & MAY_BE_UNDEF) && 8541 !set_bool) { 8542 if (exit_addr) { 8543 if (branch_opcode == ZEND_JMPNZ) { 8544 | jl >9 8545 } else { 8546 | jl &exit_addr 8547 } 8548 } else if (false_label != (uint32_t)-1) { 8549 | jl =>false_label 8550 } else { 8551 | jl >9 8552 } 8553 jmp_done = 1; 8554 } else { 8555 | jg >2 8556 } 8557 } 8558 if (!(op1_info & MAY_BE_TRUE)) { 8559 /* It's FALSE */ 8560 if (set_bool) { 8561 if (set_bool_not) { 8562 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8563 } else { 8564 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8565 } 8566 } 8567 } else { 8568 if (exit_addr) { 8569 if (set_bool) { 8570 | jne >1 8571 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8572 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8573 | jmp &exit_addr 8574 } else { 8575 | jmp >9 8576 } 8577 |1: 8578 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8579 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8580 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8581 | jne &exit_addr 8582 } 8583 } 8584 } else { 8585 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8586 | je &exit_addr 8587 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8588 | jne &exit_addr 8589 } else { 8590 | je >9 8591 } 8592 } 8593 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8594 if (set_bool) { 8595 | jne >1 8596 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8597 if (true_label != (uint32_t)-1) { 8598 | jmp =>true_label 8599 } else { 8600 | jmp >9 8601 } 8602 |1: 8603 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8604 } else { 8605 if (true_label != (uint32_t)-1) { 8606 | je =>true_label 8607 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8608 | jne =>false_label 8609 jmp_done = 1; 8610 } else { 8611 | je >9 8612 } 8613 } 8614 } else if (set_bool) { 8615 | sete al 8616 | movzx eax, al 8617 if (set_bool_not) { 8618 | neg eax 8619 | add eax, 3 8620 } else { 8621 | add eax, 2 8622 } 8623 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8624 set_delayed = 1; 8625 } else { 8626 | SET_ZVAL_TYPE_INFO res_addr, eax 8627 } 8628 } 8629 } 8630 } 8631 8632 /* It's FALSE, but may be UNDEF */ 8633 if (op1_info & MAY_BE_UNDEF) { 8634 if (op1_info & MAY_BE_ANY) { 8635 if (set_delayed) { 8636 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8637 | SET_ZVAL_TYPE_INFO res_addr, eax 8638 | jz >1 8639 } else { 8640 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8641 } 8642 |.cold_code 8643 |1: 8644 } 8645 | mov FCARG1d, opline->op1.var 8646 | SET_EX_OPLINE opline, r0 8647 | EXT_CALL zend_jit_undefined_op_helper, r0 8648 8649 if (may_throw) { 8650 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8651 return 0; 8652 } 8653 } 8654 8655 if (exit_addr) { 8656 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8657 | jmp &exit_addr 8658 } 8659 } else if (false_label != (uint32_t)-1) { 8660 | jmp =>false_label 8661 } 8662 if (op1_info & MAY_BE_ANY) { 8663 if (exit_addr) { 8664 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8665 | jmp >9 8666 } 8667 } else if (false_label == (uint32_t)-1) { 8668 | jmp >9 8669 } 8670 |.code 8671 } 8672 } 8673 8674 if (!jmp_done) { 8675 if (exit_addr) { 8676 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8677 if (op1_info & MAY_BE_LONG) { 8678 | jmp >9 8679 } 8680 } else if (op1_info & MAY_BE_LONG) { 8681 | jmp &exit_addr 8682 } 8683 } else if (false_label != (uint32_t)-1) { 8684 | jmp =>false_label 8685 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8686 | jmp >9 8687 } 8688 } 8689 } 8690 } 8691 8692 if (op1_info & MAY_BE_LONG) { 8693 |2: 8694 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8695 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8696 } 8697 if (Z_MODE(op1_addr) == IS_REG) { 8698 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8699 } else { 8700 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8701 } 8702 if (set_bool) { 8703 | setne al 8704 | movzx eax, al 8705 if (set_bool_not) { 8706 | neg eax 8707 | add eax, 3 8708 } else { 8709 | lea eax, [eax + 2] 8710 } 8711 | SET_ZVAL_TYPE_INFO res_addr, eax 8712 } 8713 if (exit_addr) { 8714 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8715 | jne &exit_addr 8716 } else { 8717 | je &exit_addr 8718 } 8719 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8720 if (true_label != (uint32_t)-1) { 8721 | jne =>true_label 8722 if (false_label != (uint32_t)-1) { 8723 | jmp =>false_label 8724 } 8725 } else { 8726 | je =>false_label 8727 } 8728 } 8729 } 8730 8731 if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8732 |2: 8733 if (CAN_USE_AVX()) { 8734 | vxorps xmm0, xmm0, xmm0 8735 } else { 8736 | xorps xmm0, xmm0 8737 } 8738 | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr 8739 8740 if (set_bool) { 8741 if (exit_addr) { 8742 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8743 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8744 | jp &exit_addr 8745 | jne &exit_addr 8746 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8747 } else { 8748 | jp >1 8749 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8750 | je &exit_addr 8751 |1: 8752 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8753 } 8754 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8755 | jp >1 8756 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8757 | je => false_label 8758 |1: 8759 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8760 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8761 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8762 | jp => true_label 8763 | jne => true_label 8764 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8765 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8766 | mov eax, IS_FALSE 8767 | jp >1 8768 | jne >1 8769 | mov eax, IS_TRUE 8770 |1: 8771 | SET_ZVAL_TYPE_INFO res_addr, eax 8772 } else { // BOOL (p=>true, z=>false) 8773 | mov eax, IS_TRUE 8774 | jp >1 8775 | jne >1 8776 | mov eax, IS_FALSE 8777 |1: 8778 | SET_ZVAL_TYPE_INFO res_addr, eax 8779 } 8780 } else { 8781 if (exit_addr) { 8782 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8783 | jp &exit_addr 8784 | jne &exit_addr 8785 |1: 8786 } else { 8787 | jp >1 8788 | je &exit_addr 8789 |1: 8790 } 8791 } else { 8792 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8793 if (false_label != (uint32_t)-1 ) { 8794 | jp >1 8795 | je => false_label 8796 |1: 8797 if (true_label != (uint32_t)-1) { 8798 | jmp =>true_label 8799 } 8800 } else { 8801 | jp => true_label 8802 | jne => true_label 8803 } 8804 } 8805 } 8806 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8807 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8808 |.cold_code 8809 |2: 8810 } 8811 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 8812 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8813 } 8814 | SET_EX_OPLINE opline, r0 8815 | EXT_CALL zend_is_true, r0 8816 8817 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8818 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8819 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8820 8821 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8822 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8823 } 8824 | GET_ZVAL_PTR FCARG1a, op1_addr 8825 | GC_DELREF FCARG1a 8826 | jnz >3 8827 | mov aword T1, r0 // save 8828 | ZVAL_DTOR_FUNC op1_info, opline 8829 | mov r0, aword T1 // restore 8830 |3: 8831 } 8832 if (may_throw) { 8833 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 8834 | jne ->exception_handler_undef 8835 } 8836 8837 if (set_bool) { 8838 if (set_bool_not) { 8839 | neg eax 8840 | add eax, 3 8841 } else { 8842 | add eax, 2 8843 } 8844 | SET_ZVAL_TYPE_INFO res_addr, eax 8845 if (exit_addr) { 8846 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8847 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8848 | jne &exit_addr 8849 } else { 8850 | je &exit_addr 8851 } 8852 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8853 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8854 if (true_label != (uint32_t)-1) { 8855 | jne =>true_label 8856 if (false_label != (uint32_t)-1) { 8857 | jmp =>false_label 8858 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8859 | jmp >9 8860 } 8861 } else { 8862 | je =>false_label 8863 } 8864 } 8865 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8866 | jmp >9 8867 |.code 8868 } 8869 } else { 8870 | test r0, r0 8871 if (exit_addr) { 8872 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8873 | jne &exit_addr 8874 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8875 | jmp >9 8876 } 8877 } else { 8878 | je &exit_addr 8879 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8880 | jmp >9 8881 } 8882 } 8883 } else if (true_label != (uint32_t)-1) { 8884 | jne =>true_label 8885 if (false_label != (uint32_t)-1) { 8886 | jmp =>false_label 8887 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8888 | jmp >9 8889 } 8890 } else { 8891 | je =>false_label 8892 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8893 | jmp >9 8894 } 8895 } 8896 8897 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8898 |.code 8899 } 8900 } 8901 } 8902 8903 |9: 8904 8905 return 1; 8906} 8907 8908static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8909{ 8910 if (op1_addr != op1_def_addr) { 8911 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8912 return 0; 8913 } 8914 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8915 op1_addr = op1_def_addr; 8916 } 8917 } 8918 8919 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { 8920 return 0; 8921 } 8922 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8923 return 0; 8924 } 8925 if (op1_info & MAY_BE_UNDEF) { 8926 zend_jit_check_exception(Dst); 8927 } 8928 return 1; 8929} 8930 8931static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8932{ 8933 ZEND_ASSERT(opline->op1_type == IS_CV); 8934 8935 if (op2_addr != op2_def_addr) { 8936 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8937 return 0; 8938 } 8939 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8940 op2_addr = op2_def_addr; 8941 } 8942 } 8943 8944 if (Z_MODE(op1_addr) != IS_REG 8945 && Z_MODE(op1_use_addr) == IS_REG 8946 && !Z_LOAD(op1_use_addr) 8947 && !Z_STORE(op1_use_addr)) { 8948 /* Force type update */ 8949 op1_info |= MAY_BE_UNDEF; 8950 } 8951 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8952 may_throw)) { 8953 return 0; 8954 } 8955 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8956 return 0; 8957 } 8958 if (opline->result_type != IS_UNUSED) { 8959 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8960 return 0; 8961 } 8962 } 8963 8964 return 1; 8965} 8966 8967/* copy of hidden zend_closure */ 8968typedef struct _zend_closure { 8969 zend_object std; 8970 zend_function func; 8971 zval this_ptr; 8972 zend_class_entry *called_scope; 8973 zif_handler orig_internal_handler; 8974} zend_closure; 8975 8976static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8977{ 8978 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8979 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8980 8981 if (!exit_addr) { 8982 return 0; 8983 } 8984 8985 | // Check Stack Overflow 8986 | MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0 8987 | MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 8988 | cmp r1, used_stack 8989 | jb &exit_addr 8990 8991 return 1; 8992} 8993 8994static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, zend_bool is_closure, zend_bool use_this, zend_bool stack_check) 8995{ 8996 uint32_t used_stack; 8997 8998 if (func) { 8999 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9000 } else { 9001 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 9002 9003 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9004 if (!is_closure) { 9005 | test byte [r0 + offsetof(zend_function, type)], 1 9006 | mov FCARG1a, used_stack 9007 | jnz >1 9008 } else { 9009 | mov FCARG1a, used_stack 9010 } 9011 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9012 | mov edx, opline->extended_value 9013 if (!is_closure) { 9014 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9015 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9016 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9017 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9018 } else { 9019 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9020 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9021 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9022 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9023 } 9024 | shl edx, 4 9025 |.if X64 9026 | movsxd r2, edx 9027 |.endif 9028 | sub FCARG1a, r2 9029 |1: 9030 } 9031 9032 zend_jit_start_reuse_ip(); 9033 9034 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9035 | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX 9036 9037 if (stack_check) { 9038 | // Check Stack Overflow 9039 | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 9040 | sub r2, RX 9041 if (func) { 9042 | cmp r2, used_stack 9043 } else { 9044 | cmp r2, FCARG1a 9045 } 9046 9047 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9048 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9049 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9050 9051 if (!exit_addr) { 9052 return 0; 9053 } 9054 9055 | jb &exit_addr 9056 } else { 9057 | jb >1 9058 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9059 |.cold_code 9060 |1: 9061 if (func) { 9062 | mov FCARG1d, used_stack 9063 } 9064#ifdef _WIN32 9065 if (0) { 9066#else 9067 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9068#endif 9069 | SET_EX_OPLINE opline, r0 9070 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9071 } else { 9072 if (!is_closure) { 9073 if (func 9074 && op_array == &func->op_array 9075 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9076 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9077 | LOAD_ADDR FCARG2a, func 9078 } else { 9079 | mov FCARG2a, r0 9080 } 9081 } else { 9082 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9083 } 9084 | SET_EX_OPLINE opline, r0 9085 | EXT_CALL zend_jit_extend_stack_helper, r0 9086 } 9087 | mov RX, r0 9088 | jmp >1 9089 |.code 9090 } 9091 } 9092 9093 if (func) { 9094 | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9095 } else { 9096 | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9097 } 9098 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9099 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9100 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9101 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9102 } 9103#ifdef _WIN32 9104 if (0) { 9105#else 9106 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9107#endif 9108 | // call->func = func; 9109 |1: 9110 | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 9111 } else { 9112 if (!is_closure) { 9113 | // call->func = func; 9114 if (func 9115 && op_array == &func->op_array 9116 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9117 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9118 | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 9119 } else { 9120 | mov aword EX:RX->func, r0 9121 } 9122 } else { 9123 | // call->func = &closure->func; 9124 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9125 | mov aword EX:RX->func, r1 9126 } 9127 |1: 9128 } 9129 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9130 | // Z_PTR(call->This) = obj; 9131 | mov r1, aword T1 9132 | mov aword EX:RX->This.value.ptr, r1 9133 if (opline->op1_type == IS_UNUSED || use_this) { 9134 | // call->call_info |= ZEND_CALL_HAS_THIS; 9135 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9136 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9137 } else { 9138 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9139 } 9140 } else { 9141 if (opline->op1_type == IS_CV) { 9142 | // GC_ADDREF(obj); 9143 | add dword [r1], 1 9144 } 9145 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9146 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9147 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9148 } else { 9149 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9150 } 9151 } 9152 } else if (!is_closure) { 9153 | // Z_CE(call->This) = called_scope; 9154 | mov aword EX:RX->This.value.ptr, 0 9155 } else { 9156 if (opline->op2_type == IS_CV) { 9157 | // GC_ADDREF(closure); 9158 | add dword [r0], 1 9159 } 9160 | // object_or_called_scope = closure->called_scope; 9161 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9162 | mov aword EX:RX->This.value.ptr, r1 9163 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9164 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9165 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9166 | and edx, ZEND_ACC_FAKE_CLOSURE 9167 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9168 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9169 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9170 | jz >1 9171 | // call_info |= ZEND_CALL_HAS_THIS; 9172 | or edx, ZEND_CALL_HAS_THIS 9173 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9174 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9175 |1: 9176 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9177 | or dword EX:RX->This.u1.type_info, edx 9178 | // Z_PTR(call->This) = object_or_called_scope; 9179 | mov aword EX:RX->This.value.ptr, r1 9180 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9181 | jnz >1 9182 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9183 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9184 |1: 9185 } 9186 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9187 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9188 return 1; 9189} 9190 9191static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) 9192{ 9193 int skip; 9194 9195 if (trace) { 9196 zend_jit_trace_rec *p = trace; 9197 9198 ssa_op++; 9199 while (1) { 9200 if (p->op == ZEND_JIT_TRACE_VM) { 9201 switch (p->opline->opcode) { 9202 case ZEND_SEND_ARRAY: 9203 case ZEND_SEND_USER: 9204 case ZEND_SEND_UNPACK: 9205 case ZEND_INIT_FCALL: 9206 case ZEND_INIT_METHOD_CALL: 9207 case ZEND_INIT_STATIC_METHOD_CALL: 9208 case ZEND_INIT_FCALL_BY_NAME: 9209 case ZEND_INIT_NS_FCALL_BY_NAME: 9210 case ZEND_INIT_DYNAMIC_CALL: 9211 case ZEND_NEW: 9212 case ZEND_INIT_USER_CALL: 9213 case ZEND_FAST_CALL: 9214 case ZEND_JMP: 9215 case ZEND_JMPZNZ: 9216 case ZEND_JMPZ: 9217 case ZEND_JMPNZ: 9218 case ZEND_JMPZ_EX: 9219 case ZEND_JMPNZ_EX: 9220 case ZEND_FE_RESET_R: 9221 case ZEND_FE_RESET_RW: 9222 case ZEND_JMP_SET: 9223 case ZEND_COALESCE: 9224 case ZEND_JMP_NULL: 9225 case ZEND_ASSERT_CHECK: 9226 case ZEND_CATCH: 9227 case ZEND_DECLARE_ANON_CLASS: 9228 case ZEND_FE_FETCH_R: 9229 case ZEND_FE_FETCH_RW: 9230 return 1; 9231 case ZEND_DO_ICALL: 9232 case ZEND_DO_UCALL: 9233 case ZEND_DO_FCALL_BY_NAME: 9234 case ZEND_DO_FCALL: 9235 return 0; 9236 case ZEND_SEND_VAL: 9237 case ZEND_SEND_VAR: 9238 case ZEND_SEND_VAL_EX: 9239 case ZEND_SEND_VAR_EX: 9240 case ZEND_SEND_FUNC_ARG: 9241 case ZEND_SEND_REF: 9242 case ZEND_SEND_VAR_NO_REF: 9243 case ZEND_SEND_VAR_NO_REF_EX: 9244 /* skip */ 9245 break; 9246 default: 9247 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9248 return 1; 9249 } 9250 } 9251 ssa_op += zend_jit_trace_op_len(opline); 9252 } else if (p->op == ZEND_JIT_TRACE_ENTER || 9253 p->op == ZEND_JIT_TRACE_BACK || 9254 p->op == ZEND_JIT_TRACE_END) { 9255 return 1; 9256 } 9257 p++; 9258 } 9259 } 9260 9261 if (!call_info) { 9262 const zend_op *end = op_array->opcodes + op_array->last; 9263 9264 opline++; 9265 ssa_op++; 9266 skip = (call_level == 1); 9267 while (opline != end) { 9268 if (!skip) { 9269 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9270 return 1; 9271 } 9272 } 9273 switch (opline->opcode) { 9274 case ZEND_SEND_VAL: 9275 case ZEND_SEND_VAR: 9276 case ZEND_SEND_VAL_EX: 9277 case ZEND_SEND_VAR_EX: 9278 case ZEND_SEND_FUNC_ARG: 9279 case ZEND_SEND_REF: 9280 case ZEND_SEND_VAR_NO_REF: 9281 case ZEND_SEND_VAR_NO_REF_EX: 9282 skip = 0; 9283 break; 9284 case ZEND_SEND_ARRAY: 9285 case ZEND_SEND_USER: 9286 case ZEND_SEND_UNPACK: 9287 case ZEND_INIT_FCALL: 9288 case ZEND_INIT_METHOD_CALL: 9289 case ZEND_INIT_STATIC_METHOD_CALL: 9290 case ZEND_INIT_FCALL_BY_NAME: 9291 case ZEND_INIT_NS_FCALL_BY_NAME: 9292 case ZEND_INIT_DYNAMIC_CALL: 9293 case ZEND_NEW: 9294 case ZEND_INIT_USER_CALL: 9295 case ZEND_FAST_CALL: 9296 case ZEND_JMP: 9297 case ZEND_JMPZNZ: 9298 case ZEND_JMPZ: 9299 case ZEND_JMPNZ: 9300 case ZEND_JMPZ_EX: 9301 case ZEND_JMPNZ_EX: 9302 case ZEND_FE_RESET_R: 9303 case ZEND_FE_RESET_RW: 9304 case ZEND_JMP_SET: 9305 case ZEND_COALESCE: 9306 case ZEND_JMP_NULL: 9307 case ZEND_ASSERT_CHECK: 9308 case ZEND_CATCH: 9309 case ZEND_DECLARE_ANON_CLASS: 9310 case ZEND_FE_FETCH_R: 9311 case ZEND_FE_FETCH_RW: 9312 return 1; 9313 case ZEND_DO_ICALL: 9314 case ZEND_DO_UCALL: 9315 case ZEND_DO_FCALL_BY_NAME: 9316 case ZEND_DO_FCALL: 9317 end = opline; 9318 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { 9319 /* INIT_FCALL and DO_FCALL in different BasicBlocks */ 9320 return 1; 9321 } 9322 return 0; 9323 } 9324 opline++; 9325 ssa_op++; 9326 } 9327 9328 return 1; 9329 } else { 9330 const zend_op *end = call_info->caller_call_opline; 9331 9332 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { 9333 /* INIT_FCALL and DO_FCALL in different BasicBlocks */ 9334 return 1; 9335 } 9336 9337 opline++; 9338 ssa_op++; 9339 skip = (call_level == 1); 9340 while (opline != end) { 9341 if (skip) { 9342 switch (opline->opcode) { 9343 case ZEND_SEND_VAL: 9344 case ZEND_SEND_VAR: 9345 case ZEND_SEND_VAL_EX: 9346 case ZEND_SEND_VAR_EX: 9347 case ZEND_SEND_FUNC_ARG: 9348 case ZEND_SEND_REF: 9349 case ZEND_SEND_VAR_NO_REF: 9350 case ZEND_SEND_VAR_NO_REF_EX: 9351 skip = 0; 9352 break; 9353 case ZEND_SEND_ARRAY: 9354 case ZEND_SEND_USER: 9355 case ZEND_SEND_UNPACK: 9356 return 1; 9357 } 9358 } else { 9359 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9360 return 1; 9361 } 9362 } 9363 opline++; 9364 ssa_op++; 9365 } 9366 9367 return 0; 9368 } 9369} 9370 9371static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9372{ 9373 int32_t exit_point; 9374 const void *exit_addr; 9375 9376 if (func->type == ZEND_INTERNAL_FUNCTION) { 9377#ifdef ZEND_WIN32 9378 // TODO: ASLR may cause different addresses in different workers ??? 9379 return 0; 9380#endif 9381 } else if (func->type == ZEND_USER_FUNCTION) { 9382 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9383 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9384 return 0; 9385 } 9386 } else { 9387 ZEND_UNREACHABLE(); 9388 return 0; 9389 } 9390 9391 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9392 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9393 if (!exit_addr) { 9394 return 0; 9395 } 9396 9397 | // call = EX(call); 9398 | mov r1, EX->call 9399 while (level > 0) { 9400 | mov r1, EX:r1->prev_execute_data 9401 level--; 9402 } 9403 9404 if (func->type == ZEND_USER_FUNCTION && 9405 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9406 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9407 !func->common.function_name)) { 9408 const zend_op *opcodes = func->op_array.opcodes; 9409 9410 | mov r1, aword EX:r1->func 9411 | .if X64 9412 || if (!IS_SIGNED_32BIT(opcodes)) { 9413 | mov64 r2, ((ptrdiff_t)opcodes) 9414 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9415 || } else { 9416 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9417 || } 9418 | .else 9419 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9420 | .endif 9421 | jne &exit_addr 9422 } else { 9423 | .if X64 9424 || if (!IS_SIGNED_32BIT(func)) { 9425 | mov64 r2, ((ptrdiff_t)func) 9426 | cmp aword EX:r1->func, r2 9427 || } else { 9428 | cmp aword EX:r1->func, func 9429 || } 9430 | .else 9431 | cmp aword EX:r1->func, func 9432 | .endif 9433 | jne &exit_addr 9434 } 9435 9436 return 1; 9437} 9438 9439static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, zend_bool stack_check) 9440{ 9441 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9442 zend_call_info *call_info = NULL; 9443 zend_function *func = NULL; 9444 9445 if (delayed_call_chain) { 9446 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9447 return 0; 9448 } 9449 } 9450 9451 if (info) { 9452 call_info = info->callee_info; 9453 while (call_info && call_info->caller_init_opline != opline) { 9454 call_info = call_info->next_callee; 9455 } 9456 if (call_info && call_info->callee_func) { 9457 func = call_info->callee_func; 9458 } 9459 } 9460 9461 if (!func 9462 && trace 9463 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9464#ifdef _WIN32 9465 /* ASLR */ 9466 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9467 func = (zend_function*)trace->func; 9468 } 9469#else 9470 func = (zend_function*)trace->func; 9471#endif 9472 } 9473 9474#ifdef _WIN32 9475 if (0) { 9476#else 9477 if (opline->opcode == ZEND_INIT_FCALL 9478 && func 9479 && func->type == ZEND_INTERNAL_FUNCTION) { 9480#endif 9481 /* load constant address later */ 9482 } else if (func && op_array == &func->op_array) { 9483 /* recursive call */ 9484 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9485 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9486 | mov r0, EX->func 9487 } 9488 } else { 9489 | // if (CACHED_PTR(opline->result.num)) 9490 | mov r0, EX->run_time_cache 9491 | mov r0, aword [r0 + opline->result.num] 9492 | test r0, r0 9493 | jz >1 9494 |.cold_code 9495 |1: 9496 if (opline->opcode == ZEND_INIT_FCALL 9497 && func 9498 && func->type == ZEND_USER_FUNCTION 9499 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9500 | LOAD_ADDR FCARG1a, func 9501 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9502 | mov r1, EX->run_time_cache 9503 | mov aword [r1 + opline->result.num], r0 9504 | jmp >3 9505 } else { 9506 zval *zv = RT_CONSTANT(opline, opline->op2); 9507 9508 if (opline->opcode == ZEND_INIT_FCALL) { 9509 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9510 | EXT_CALL zend_jit_find_func_helper, r0 9511 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9512 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9513 | EXT_CALL zend_jit_find_func_helper, r0 9514 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9515 | LOAD_ADDR FCARG1a, zv; 9516 | EXT_CALL zend_jit_find_ns_func_helper, r0 9517 } else { 9518 ZEND_UNREACHABLE(); 9519 } 9520 | // CACHE_PTR(opline->result.num, fbc); 9521 | mov r1, EX->run_time_cache 9522 | mov aword [r1 + opline->result.num], r0 9523 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9524 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 9525 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9526 9527 if (!exit_addr) { 9528 return 0; 9529 } 9530 9531 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9532 | test r0, r0 9533 | jnz >3 9534 } else if (func->type == ZEND_USER_FUNCTION 9535 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9536 const zend_op *opcodes = func->op_array.opcodes; 9537 9538 | .if X64 9539 || if (!IS_SIGNED_32BIT(opcodes)) { 9540 | mov64 r1, ((ptrdiff_t)opcodes) 9541 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9542 || } else { 9543 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9544 || } 9545 | .else 9546 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9547 | .endif 9548 | jz >3 9549 } else { 9550 | .if X64 9551 || if (!IS_SIGNED_32BIT(func)) { 9552 | mov64 r1, ((ptrdiff_t)func) 9553 | cmp r0, r1 9554 || } else { 9555 | cmp r0, func 9556 || } 9557 | .else 9558 | cmp r0, func 9559 | .endif 9560 | jz >3 9561 } 9562 | jmp &exit_addr 9563 } else { 9564 | test r0, r0 9565 | jnz >3 9566 | // SAVE_OPLINE(); 9567 | SET_EX_OPLINE opline, r0 9568 | jmp ->undefined_function 9569 } 9570 } 9571 |.code 9572 |3: 9573 } 9574 9575 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) { 9576 return 0; 9577 } 9578 9579 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9580 if (!zend_jit_save_call_chain(Dst, call_level)) { 9581 return 0; 9582 } 9583 } else { 9584 delayed_call_chain = 1; 9585 delayed_call_level = call_level; 9586 } 9587 9588 return 1; 9589} 9590 9591static int zend_jit_init_method_call(dasm_State **Dst, 9592 const zend_op *opline, 9593 uint32_t b, 9594 const zend_op_array *op_array, 9595 zend_ssa *ssa, 9596 const zend_ssa_op *ssa_op, 9597 int call_level, 9598 uint32_t op1_info, 9599 zend_jit_addr op1_addr, 9600 zend_class_entry *ce, 9601 zend_bool ce_is_instanceof, 9602 zend_bool use_this, 9603 zend_class_entry *trace_ce, 9604 zend_jit_trace_rec *trace, 9605 zend_bool stack_check, 9606 zend_bool polymorphic_side_trace) 9607{ 9608 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9609 zend_call_info *call_info = NULL; 9610 zend_function *func = NULL; 9611 zval *function_name; 9612 9613 ZEND_ASSERT(opline->op2_type == IS_CONST); 9614 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9615 9616 function_name = RT_CONSTANT(opline, opline->op2); 9617 9618 if (info) { 9619 call_info = info->callee_info; 9620 while (call_info && call_info->caller_init_opline != opline) { 9621 call_info = call_info->next_callee; 9622 } 9623 if (call_info && call_info->callee_func) { 9624 func = call_info->callee_func; 9625 } 9626 } 9627 9628 if (polymorphic_side_trace) { 9629 /* function is passed in r0 from parent_trace */ 9630 } else { 9631 if (opline->op1_type == IS_UNUSED || use_this) { 9632 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9633 9634 | GET_ZVAL_PTR FCARG1a, this_addr 9635 } else { 9636 if (op1_info & MAY_BE_REF) { 9637 if (opline->op1_type == IS_CV) { 9638 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 9639 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9640 } 9641 | ZVAL_DEREF FCARG1a, op1_info 9642 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 9643 } else { 9644 /* Hack: Convert reference to regular value to simplify JIT code */ 9645 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9646 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9647 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9648 | EXT_CALL zend_jit_unref_helper, r0 9649 |1: 9650 } 9651 } 9652 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9653 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9654 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9655 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9656 9657 if (!exit_addr) { 9658 return 0; 9659 } 9660 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9661 } else { 9662 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9663 |.cold_code 9664 |1: 9665 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 9666 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9667 } 9668 | SET_EX_OPLINE opline, r0 9669 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9670 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9671 } else { 9672 | EXT_CALL zend_jit_invalid_method_call, r0 9673 } 9674 | jmp ->exception_handler 9675 |.code 9676 } 9677 } 9678 | GET_ZVAL_PTR FCARG1a, op1_addr 9679 } 9680 9681 if (delayed_call_chain) { 9682 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9683 return 0; 9684 } 9685 } 9686 9687 | mov aword T1, FCARG1a // save 9688 9689 if (func) { 9690 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9691 | mov r0, EX->run_time_cache 9692 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9693 | test r0, r0 9694 | jz >1 9695 } else { 9696 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9697 | mov r0, EX->run_time_cache 9698 | mov r2, aword [r0 + opline->result.num] 9699 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9700 | jnz >1 9701 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9702 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9703 } 9704 9705 |.cold_code 9706 |1: 9707 | LOAD_ADDR FCARG2a, function_name 9708 |.if X64 9709 | lea CARG3, aword T1 9710 |.else 9711 | lea r0, aword T1 9712 | sub r4, 12 9713 | push r0 9714 |.endif 9715 | SET_EX_OPLINE opline, r0 9716 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9717 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9718 } else { 9719 | EXT_CALL zend_jit_find_method_helper, r0 9720 } 9721 |.if not(X64) 9722 | add r4, 12 9723 |.endif 9724 | test r0, r0 9725 | jnz >2 9726 | jmp ->exception_handler 9727 |.code 9728 |2: 9729 } 9730 9731 if (!func 9732 && trace 9733 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9734 && trace->func 9735#ifdef _WIN32 9736 && trace->func->type != ZEND_INTERNAL_FUNCTION 9737#endif 9738 ) { 9739 int32_t exit_point; 9740 const void *exit_addr; 9741 9742 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); 9743 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9744 if (!exit_addr) { 9745 return 0; 9746 } 9747 9748 func = (zend_function*)trace->func; 9749 9750 if (func->type == ZEND_USER_FUNCTION && 9751 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9752 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9753 !func->common.function_name)) { 9754 const zend_op *opcodes = func->op_array.opcodes; 9755 9756 | .if X64 9757 || if (!IS_SIGNED_32BIT(opcodes)) { 9758 | mov64 r1, ((ptrdiff_t)opcodes) 9759 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9760 || } else { 9761 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9762 || } 9763 | .else 9764 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9765 | .endif 9766 | jne &exit_addr 9767 } else { 9768 | .if X64 9769 || if (!IS_SIGNED_32BIT(func)) { 9770 | mov64 r1, ((ptrdiff_t)func) 9771 | cmp r0, r1 9772 || } else { 9773 | cmp r0, func 9774 || } 9775 | .else 9776 | cmp r0, func 9777 | .endif 9778 | jne &exit_addr 9779 } 9780 } 9781 9782 if (!func) { 9783 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9784 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9785 | jnz >1 9786 |.cold_code 9787 |1: 9788 } 9789 9790 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9791 | mov FCARG1a, aword T1 // restore 9792 | mov FCARG2a, r0 9793 |.if X64 9794 | mov CARG3d, opline->extended_value 9795 |.else 9796 | sub r4, 12 9797 | push opline->extended_value 9798 |.endif 9799 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9800 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9801 } else { 9802 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9803 } 9804 |.if not(X64) 9805 | add r4, 12 9806 |.endif 9807 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) { 9808 | test r0, r0 9809 | jz ->exception_handler 9810 } 9811 | mov RX, r0 9812 } 9813 9814 if (!func) { 9815 | jmp >9 9816 |.code 9817 } 9818 9819 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9820 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) { 9821 return 0; 9822 } 9823 } 9824 9825 if (!func) { 9826 |9: 9827 } 9828 zend_jit_start_reuse_ip(); 9829 9830 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9831 if (!zend_jit_save_call_chain(Dst, call_level)) { 9832 return 0; 9833 } 9834 } else { 9835 delayed_call_chain = 1; 9836 delayed_call_level = call_level; 9837 } 9838 9839 return 1; 9840} 9841 9842static int zend_jit_init_closure_call(dasm_State **Dst, 9843 const zend_op *opline, 9844 uint32_t b, 9845 const zend_op_array *op_array, 9846 zend_ssa *ssa, 9847 const zend_ssa_op *ssa_op, 9848 int call_level, 9849 zend_jit_trace_rec *trace, 9850 zend_bool stack_check) 9851{ 9852 zend_function *func = NULL; 9853 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9854 9855 | GET_ZVAL_PTR r0, op2_addr 9856 9857 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9858 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9859 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9860 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9861 9862 if (!exit_addr) { 9863 return 0; 9864 } 9865 9866 |.if X64 9867 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9868 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9869 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9870 || } else { 9871 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9872 || } 9873 |.else 9874 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9875 |.endif 9876 | jne &exit_addr 9877 if (ssa->var_info && ssa_op->op2_use >= 0) { 9878 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9879 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9880 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9881 } 9882 } 9883 9884 if (trace 9885 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9886 && trace->func 9887 && trace->func->type == ZEND_USER_FUNCTION) { 9888 const zend_op *opcodes; 9889 int32_t exit_point; 9890 const void *exit_addr; 9891 9892 func = (zend_function*)trace->func; 9893 opcodes = func->op_array.opcodes; 9894 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9895 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9896 if (!exit_addr) { 9897 return 0; 9898 } 9899 9900 | .if X64 9901 || if (!IS_SIGNED_32BIT(opcodes)) { 9902 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9903 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9904 || } else { 9905 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9906 || } 9907 | .else 9908 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9909 | .endif 9910 | jne &exit_addr 9911 } 9912 9913 if (delayed_call_chain) { 9914 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9915 return 0; 9916 } 9917 } 9918 9919 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) { 9920 return 0; 9921 } 9922 9923 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9924 if (!zend_jit_save_call_chain(Dst, call_level)) { 9925 return 0; 9926 } 9927 } else { 9928 delayed_call_chain = 1; 9929 delayed_call_level = call_level; 9930 } 9931 9932 if (trace 9933 && trace->op == ZEND_JIT_TRACE_END 9934 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9935 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9936 return 0; 9937 } 9938 } 9939 9940 return 1; 9941} 9942 9943static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) 9944{ 9945 uint32_t num_args = 0; 9946 zend_function *func = call_info->callee_func; 9947 9948 while (num_args < call_info->num_args) { 9949 zend_arg_info *arg_info = func->op_array.arg_info + num_args; 9950 9951 if (ZEND_TYPE_IS_SET(arg_info->type)) { 9952 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { 9953 zend_op *opline = call_info->arg_info[num_args].opline; 9954 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 9955 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); 9956 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { 9957 break; 9958 } 9959 } else { 9960 break; 9961 } 9962 } 9963 num_args++; 9964 } 9965 return num_args; 9966} 9967 9968static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9969{ 9970 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9971 zend_call_info *call_info = NULL; 9972 const zend_function *func = NULL; 9973 uint32_t i; 9974 zend_jit_addr res_addr; 9975 uint32_t call_num_args = 0; 9976 zend_bool unknown_num_args = 0; 9977 const void *exit_addr = NULL; 9978 const zend_op *prev_opline; 9979 9980 if (RETURN_VALUE_USED(opline)) { 9981 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9982 } else { 9983 /* CPU stack allocated temporary zval */ 9984 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 9985 } 9986 9987 prev_opline = opline - 1; 9988 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9989 prev_opline--; 9990 } 9991 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9992 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9993 unknown_num_args = 1; 9994 } 9995 9996 if (info) { 9997 call_info = info->callee_info; 9998 while (call_info && call_info->caller_call_opline != opline) { 9999 call_info = call_info->next_callee; 10000 } 10001 if (call_info && call_info->callee_func) { 10002 func = call_info->callee_func; 10003 } 10004 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 10005 && JIT_G(current_frame) 10006 && JIT_G(current_frame)->call 10007 && !JIT_G(current_frame)->call->func) { 10008 call_info = NULL; func = NULL; /* megamorphic call from trait */ 10009 } 10010 } 10011 if (!func) { 10012 /* resolve function at run time */ 10013 } else if (func->type == ZEND_USER_FUNCTION) { 10014 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 10015 call_num_args = call_info->num_args; 10016 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 10017 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 10018 call_num_args = call_info->num_args; 10019 } else { 10020 ZEND_UNREACHABLE(); 10021 } 10022 10023 if (trace && !func) { 10024 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 10025 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 10026#ifndef ZEND_WIN32 10027 // TODO: ASLR may cause different addresses in different workers ??? 10028 func = trace->func; 10029 if (JIT_G(current_frame) && 10030 JIT_G(current_frame)->call && 10031 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10032 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10033 } else { 10034 unknown_num_args = 1; 10035 } 10036#endif 10037 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 10038 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 10039 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 10040 func = trace->func; 10041 if (JIT_G(current_frame) && 10042 JIT_G(current_frame)->call && 10043 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10044 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10045 } else { 10046 unknown_num_args = 1; 10047 } 10048 } 10049 } 10050 } 10051 10052 bool may_have_extra_named_params = 10053 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 10054 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 10055 10056 if (!reuse_ip) { 10057 zend_jit_start_reuse_ip(); 10058 | // call = EX(call); 10059 | mov RX, EX->call 10060 } 10061 zend_jit_stop_reuse_ip(); 10062 10063 | // fbc = call->func; 10064 | // mov r2, EX:RX->func ??? 10065 | // SAVE_OPLINE(); 10066 | SET_EX_OPLINE opline, r0 10067 10068 if (opline->opcode == ZEND_DO_FCALL) { 10069 if (!func) { 10070 if (trace) { 10071 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10072 10073 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10074 if (!exit_addr) { 10075 return 0; 10076 } 10077 | mov r0, EX:RX->func 10078 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10079 | jnz &exit_addr 10080 } 10081 } 10082 } 10083 10084 if (!delayed_call_chain) { 10085 if (call_level == 1) { 10086 | mov aword EX->call, 0 10087 } else { 10088 | //EX(call) = call->prev_execute_data; 10089 | mov r0, EX:RX->prev_execute_data 10090 | mov EX->call, r0 10091 } 10092 } 10093 delayed_call_chain = 0; 10094 10095 | //call->prev_execute_data = execute_data; 10096 | mov EX:RX->prev_execute_data, EX 10097 10098 if (!func) { 10099 | mov r0, EX:RX->func 10100 } 10101 10102 if (opline->opcode == ZEND_DO_FCALL) { 10103 if (!func) { 10104 if (!trace) { 10105 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10106 | jnz >1 10107 |.cold_code 10108 |1: 10109 if (!GCC_GLOBAL_REGS) { 10110 | mov FCARG1a, RX 10111 } 10112 | EXT_CALL zend_jit_deprecated_helper, r0 10113 | test al, al 10114 | mov r0, EX:RX->func // reload 10115 | jne >1 10116 | jmp ->exception_handler 10117 |.code 10118 |1: 10119 } 10120 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10121 if (!GCC_GLOBAL_REGS) { 10122 | mov FCARG1a, RX 10123 } 10124 | EXT_CALL zend_jit_deprecated_helper, r0 10125 | test al, al 10126 | je ->exception_handler 10127 } 10128 } 10129 10130 if (!func 10131 && opline->opcode != ZEND_DO_UCALL 10132 && opline->opcode != ZEND_DO_ICALL) { 10133 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 10134 | jne >8 10135 } 10136 10137 if ((!func || func->type == ZEND_USER_FUNCTION) 10138 && opline->opcode != ZEND_DO_ICALL) { 10139 | // EX(call) = NULL; 10140 | mov aword EX:RX->call, 0 10141 10142 if (RETURN_VALUE_USED(opline)) { 10143 | // EX(return_value) = EX_VAR(opline->result.var); 10144 | LOAD_ZVAL_ADDR r2, res_addr 10145 | mov aword EX:RX->return_value, r2 10146 } else { 10147 | // EX(return_value) = 0; 10148 | mov aword EX:RX->return_value, 0 10149 } 10150 10151 //EX_LOAD_RUN_TIME_CACHE(op_array); 10152 if (!func || func->op_array.cache_size) { 10153 if (func && op_array == &func->op_array) { 10154 /* recursive call */ 10155 if (trace || func->op_array.cache_size > sizeof(void*)) { 10156 | mov r2, EX->run_time_cache 10157 | mov EX:RX->run_time_cache, r2 10158 } 10159 } else { 10160 if (func) { 10161 | mov r0, EX:RX->func 10162 } 10163 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10164#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 10165 | mov r2, aword [r2] 10166#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 10167 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 10168 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 10169 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10170 } else if (!zend_accel_in_shm(func->op_array.opcodes)) { 10171 /* the called op_array may be not persisted yet */ 10172 | test r2, 1 10173 | jz >1 10174 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10175 |1: 10176 } 10177 | mov r2, aword [r2] 10178 } else { 10179 | test r2, 1 10180 | jz >1 10181 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10182 |1: 10183 | mov r2, aword [r2] 10184 } 10185#else 10186# error "Unknown ZEND_MAP_PTR_KIND" 10187#endif 10188 | mov EX:RX->run_time_cache, r2 10189 } 10190 } 10191 10192 | // EG(current_execute_data) = execute_data; 10193 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 10194 | mov FP, RX 10195 10196 | // opline = op_array->opcodes; 10197 if (func && !unknown_num_args) { 10198 10199 for (i = call_num_args; i < func->op_array.last_var; i++) { 10200 uint32_t n = EX_NUM_TO_VAR(i); 10201 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10202 } 10203 10204 if (call_num_args <= func->op_array.num_args) { 10205 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10206 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10207 uint32_t num_args; 10208 10209 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10210 if (trace) { 10211 num_args = 0; 10212 } else if (call_info) { 10213 num_args = skip_valid_arguments(op_array, ssa, call_info); 10214 } else { 10215 num_args = call_num_args; 10216 } 10217 } else { 10218 num_args = call_num_args; 10219 } 10220 if (zend_accel_in_shm(func->op_array.opcodes)) { 10221 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10222 } else { 10223 | mov r0, EX->func 10224 if (GCC_GLOBAL_REGS) { 10225 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10226 if (num_args) { 10227 | add IP, (num_args * sizeof(zend_op)) 10228 } 10229 } else { 10230 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10231 if (num_args) { 10232 | add FCARG1a, (num_args * sizeof(zend_op)) 10233 } 10234 | mov aword EX->opline, FCARG1a 10235 } 10236 } 10237 10238 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10239 && num_args >= op_array->required_num_args) { 10240 /* recursive call */ 10241 if (ZEND_OBSERVER_ENABLED) { 10242 | SAVE_IP 10243 | mov FCARG1a, FP 10244 | EXT_CALL zend_observer_fcall_begin, r0 10245 } 10246#ifdef CONTEXT_THREADED_JIT 10247 | call >1 10248 |.cold_code 10249 |1: 10250 | pop r0 10251 | jmp =>num_args 10252 |.code 10253#else 10254 | jmp =>num_args 10255#endif 10256 return 1; 10257 } 10258 } 10259 } else { 10260 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10261 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10262 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10263 | LOAD_IP_ADDR (func->op_array.opcodes) 10264 } else if (GCC_GLOBAL_REGS) { 10265 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10266 } else { 10267 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10268 | mov aword EX->opline, FCARG1a 10269 } 10270 } 10271 if (!GCC_GLOBAL_REGS) { 10272 | mov FCARG1a, FP 10273 } 10274 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10275 } 10276 } else { 10277 | // opline = op_array->opcodes 10278 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10279 | LOAD_IP_ADDR (func->op_array.opcodes) 10280 } else if (GCC_GLOBAL_REGS) { 10281 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10282 } else { 10283 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10284 | mov aword EX->opline, FCARG1a 10285 } 10286 if (func) { 10287 | // num_args = EX_NUM_ARGS(); 10288 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10289 | // if (UNEXPECTED(num_args > first_extra_arg)) 10290 | cmp ecx, (func->op_array.num_args) 10291 } else { 10292 | // first_extra_arg = op_array->num_args; 10293 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10294 | // num_args = EX_NUM_ARGS(); 10295 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10296 | // if (UNEXPECTED(num_args > first_extra_arg)) 10297 | cmp ecx, edx 10298 } 10299 | jg >1 10300 |.cold_code 10301 |1: 10302 if (!GCC_GLOBAL_REGS) { 10303 | mov FCARG1a, FP 10304 } 10305 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10306 if (!func) { 10307 | mov r0, EX->func // reload 10308 } 10309 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10310 | jmp >1 10311 |.code 10312 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10313 if (!func) { 10314 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10315 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10316 | jnz >1 10317 } 10318 | // opline += num_args; 10319 |.if X64 10320 || ZEND_ASSERT(sizeof(zend_op) == 32); 10321 | mov edx, ecx 10322 | shl r2, 5 10323 |.else 10324 | imul r2, ecx, sizeof(zend_op) 10325 |.endif 10326 | ADD_IP r2 10327 } 10328 |1: 10329 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10330 if (func) { 10331 | mov edx, (func->op_array.last_var) 10332 } else { 10333 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10334 } 10335 | sub edx, ecx 10336 | jle >3 //??? 10337 | // zval *var = EX_VAR_NUM(num_args); 10338// |.if X64 10339// | movsxd r1, ecx 10340// |.endif 10341 | shl r1, 4 10342 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10343 |2: 10344 | SET_Z_TYPE_INFO r1, IS_UNDEF 10345 | sub edx, 1 10346 | lea r1, [r1 + 16] 10347 | jne <2 10348 |3: 10349 } 10350 10351 if (ZEND_OBSERVER_ENABLED) { 10352 | SAVE_IP 10353 | mov FCARG1a, FP 10354 | EXT_CALL zend_observer_fcall_begin, r0 10355 } 10356 10357 if (trace) { 10358 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10359 | jmp >9 10360 } 10361 } else { 10362#ifdef CONTEXT_THREADED_JIT 10363 | call ->context_threaded_call 10364 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10365 | jmp >9 10366 } 10367 | call ->context_threaded_call 10368 if (!func) { 10369 | jmp >9 10370 } 10371#else 10372 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10373 | ADD_HYBRID_SPAD 10374 | JMP_IP 10375 } else if (GCC_GLOBAL_REGS) { 10376 | add r4, SPAD // stack alignment 10377 | JMP_IP 10378 } else { 10379 | mov FP, aword T2 // restore FP 10380 | mov RX, aword T3 // restore IP 10381 | add r4, NR_SPAD // stack alignment 10382 | mov r0, 1 // ZEND_VM_ENTER 10383 | ret 10384 } 10385 } 10386#endif 10387 } 10388 10389 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10390 && (opline->opcode != ZEND_DO_UCALL)) { 10391 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10392 |8: 10393 } 10394 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10395 if (!func) { 10396 if (trace) { 10397 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10398 10399 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10400 if (!exit_addr) { 10401 return 0; 10402 } 10403 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10404 | jnz &exit_addr 10405 } else { 10406 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10407 | jnz >1 10408 |.cold_code 10409 |1: 10410 if (!GCC_GLOBAL_REGS) { 10411 | mov FCARG1a, RX 10412 } 10413 | EXT_CALL zend_jit_deprecated_helper, r0 10414 | test al, al 10415 | mov r0, EX:RX->func // reload 10416 | jne >1 10417 | jmp ->exception_handler 10418 |.code 10419 |1: 10420 } 10421 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10422 if (!GCC_GLOBAL_REGS) { 10423 | mov FCARG1a, RX 10424 } 10425 | EXT_CALL zend_jit_deprecated_helper, r0 10426 | test al, al 10427 | je ->exception_handler 10428 | mov r0, EX:RX->func // reload 10429 } 10430 } 10431 10432 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10433 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10434 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10435 10436 | // EG(current_execute_data) = execute_data; 10437 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 10438 10439 zend_jit_reset_last_valid_opline(); 10440 10441 | // fbc->internal_function.handler(call, ret); 10442 | mov FCARG1a, RX 10443 if (func) { 10444 | EXT_CALL func->internal_function.handler, r0 10445 } else { 10446 | call aword [r0 + offsetof(zend_internal_function, handler)] 10447 } 10448 10449 | // EG(current_execute_data) = execute_data; 10450 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 10451 10452 | // zend_vm_stack_free_args(call); 10453 if (func && !unknown_num_args) { 10454 for (i = 0; i < call_num_args; i++ ) { 10455 uint32_t offset = EX_NUM_TO_VAR(i); 10456 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10457 } 10458 } else { 10459 | mov FCARG1a, RX 10460 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10461 } 10462 if (may_have_extra_named_params) { 10463 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10464 | jnz >1 10465 |.cold_code 10466 |1: 10467 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10468 | EXT_CALL zend_free_extra_named_params, r0 10469 | jmp >2 10470 |.code 10471 |2: 10472 } 10473 10474 |8: 10475 if (opline->opcode == ZEND_DO_FCALL) { 10476 // TODO: optimize ??? 10477 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10478 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10479 | jnz >1 10480 |.cold_code 10481 |1: 10482 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10483 | // OBJ_RELEASE(object); 10484 | OBJ_RELEASE ZREG_FCARG1a, >2 10485 | jmp >2 10486 |.code 10487 |2: 10488 } 10489 10490 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10491 !JIT_G(current_frame) || 10492 !JIT_G(current_frame)->call || 10493 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10494 prev_opline->opcode == ZEND_SEND_UNPACK || 10495 prev_opline->opcode == ZEND_SEND_ARRAY || 10496 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10497 10498 | // zend_vm_stack_free_call_frame(call); 10499 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10500 | jnz >1 10501 |.cold_code 10502 |1: 10503 | mov FCARG1a, RX 10504 | EXT_CALL zend_jit_free_call_frame, r0 10505 | jmp >1 10506 |.code 10507 } 10508 | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 10509 |1: 10510 10511 if (!RETURN_VALUE_USED(opline)) { 10512 zend_class_entry *ce; 10513 zend_bool ce_is_instanceof; 10514 uint32_t func_info = call_info ? 10515 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10516 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10517 10518 /* If an exception is thrown, the return_value may stay at the 10519 * original value of null. */ 10520 func_info |= MAY_BE_NULL; 10521 10522 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10523 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10524 } 10525 } 10526 10527 | // if (UNEXPECTED(EG(exception) != NULL)) { 10528 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 10529 | jne ->icall_throw_handler 10530 10531 // TODO: Can we avoid checking for interrupts after each call ??? 10532 if (trace && last_valid_opline != opline) { 10533 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10534 10535 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10536 if (!exit_addr) { 10537 return 0; 10538 } 10539 } else { 10540 exit_addr = NULL; 10541 } 10542 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10543 return 0; 10544 } 10545 10546 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10547 | LOAD_IP_ADDR (opline + 1) 10548 } else if (trace 10549 && trace->op == ZEND_JIT_TRACE_END 10550 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10551 | LOAD_IP_ADDR (opline + 1) 10552 } 10553 } 10554 10555 if (!func) { 10556 |9: 10557 } 10558 10559 return 1; 10560} 10561 10562static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10563{ 10564 uint32_t arg_num = opline->op2.num; 10565 zend_jit_addr arg_addr; 10566 10567 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10568 10569 if (!zend_jit_reuse_ip(Dst)) { 10570 return 0; 10571 } 10572 10573 if (opline->opcode == ZEND_SEND_VAL_EX) { 10574 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10575 10576 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10577 10578 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10579 && JIT_G(current_frame) 10580 && JIT_G(current_frame)->call 10581 && JIT_G(current_frame)->call->func) { 10582 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10583 /* Don't generate code that always throws exception */ 10584 return 0; 10585 } 10586 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10587 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10588 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10589 if (!exit_addr) { 10590 return 0; 10591 } 10592 | mov r0, EX:RX->func 10593 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10594 | jnz &exit_addr 10595 } else { 10596 | mov r0, EX:RX->func 10597 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10598 | jnz >1 10599 |.cold_code 10600 |1: 10601 | SET_EX_OPLINE opline, r0 10602 | jmp ->throw_cannot_pass_by_ref 10603 |.code 10604 10605 } 10606 } 10607 10608 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10609 10610 if (opline->op1_type == IS_CONST) { 10611 zval *zv = RT_CONSTANT(opline, opline->op1); 10612 10613 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10614 if (Z_REFCOUNTED_P(zv)) { 10615 | ADDREF_CONST zv, r0 10616 } 10617 } else { 10618 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10619 } 10620 10621 return 1; 10622} 10623 10624static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10625{ 10626 | mov FCARG1a, EX->call 10627 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10628 | jnz >1 10629 |.cold_code 10630 |1: 10631 | SET_EX_OPLINE opline, r0 10632 | EXT_CALL zend_handle_undef_args, r0 10633 | test r0, r0 10634 | jnz ->exception_handler 10635 | jmp >2 10636 |.code 10637 |2: 10638 10639 return 1; 10640} 10641 10642static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10643{ 10644 zend_jit_addr op1_addr, arg_addr, ref_addr; 10645 10646 op1_addr = OP1_ADDR(); 10647 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10648 10649 if (!zend_jit_reuse_ip(Dst)) { 10650 return 0; 10651 } 10652 10653 if (opline->op1_type == IS_VAR) { 10654 if (op1_info & MAY_BE_INDIRECT) { 10655 | LOAD_ZVAL_ADDR r0, op1_addr 10656 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10657 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10658 | // ret = Z_INDIRECT_P(ret); 10659 | GET_Z_PTR r0, r0 10660 |1: 10661 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10662 } 10663 } else if (opline->op1_type == IS_CV) { 10664 if (op1_info & MAY_BE_UNDEF) { 10665 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10666 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10667 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10668 | jmp >2 10669 |1: 10670 } 10671 op1_info &= ~MAY_BE_UNDEF; 10672 op1_info |= MAY_BE_NULL; 10673 } 10674 } else { 10675 ZEND_UNREACHABLE(); 10676 } 10677 10678 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10679 if (op1_info & MAY_BE_REF) { 10680 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10681 | GET_ZVAL_PTR r1, op1_addr 10682 | GC_ADDREF r1 10683 | SET_ZVAL_PTR arg_addr, r1 10684 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10685 | jmp >6 10686 } 10687 |2: 10688 | // ZVAL_NEW_REF(arg, varptr); 10689 if (opline->op1_type == IS_VAR) { 10690 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10691 | LOAD_ZVAL_ADDR r0, op1_addr 10692 } 10693 | mov aword T1, r0 // save 10694 } 10695 | EMALLOC sizeof(zend_reference), op_array, opline 10696 | mov dword [r0], 2 10697 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10698 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10699 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10700 if (opline->op1_type == IS_VAR) { 10701 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10702 10703 | mov r1, aword T1 // restore 10704 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10705 | SET_ZVAL_PTR val_addr, r0 10706 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10707 } else { 10708 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10709 | SET_ZVAL_PTR op1_addr, r0 10710 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10711 } 10712 | SET_ZVAL_PTR arg_addr, r0 10713 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10714 } 10715 10716 |6: 10717 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10718 |7: 10719 10720 return 1; 10721} 10722 10723static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10724{ 10725 uint32_t arg_num = opline->op2.num; 10726 zend_jit_addr arg_addr; 10727 10728 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10729 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10730 arg_num <= MAX_ARG_FLAG_NUM); 10731 10732 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10733 10734 if (!zend_jit_reuse_ip(Dst)) { 10735 return 0; 10736 } 10737 10738 if (opline->opcode == ZEND_SEND_VAR_EX) { 10739 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10740 && JIT_G(current_frame) 10741 && JIT_G(current_frame)->call 10742 && JIT_G(current_frame)->call->func) { 10743 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10744 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10745 return 0; 10746 } 10747 return 1; 10748 } 10749 } else { 10750 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10751 10752 | mov r0, EX:RX->func 10753 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10754 | jnz >1 10755 |.cold_code 10756 |1: 10757 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10758 return 0; 10759 } 10760 | jmp >7 10761 |.code 10762 } 10763 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10764 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10765 && JIT_G(current_frame) 10766 && JIT_G(current_frame)->call 10767 && JIT_G(current_frame)->call->func) { 10768 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10769 10770 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10771 10772 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10773 if (!(op1_info & MAY_BE_REF)) { 10774 /* Don't generate code that always throws exception */ 10775 return 0; 10776 } else { 10777 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10778 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10779 if (!exit_addr) { 10780 return 0; 10781 } 10782 | cmp cl, IS_REFERENCE 10783 | jne &exit_addr 10784 } 10785 } 10786 return 1; 10787 } 10788 } else { 10789 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10790 10791 | mov r0, EX:RX->func 10792 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10793 | jnz >1 10794 |.cold_code 10795 |1: 10796 10797 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10798 10799 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10800 if (op1_info & MAY_BE_REF) { 10801 | cmp cl, IS_REFERENCE 10802 | je >7 10803 } 10804 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10805 | jnz >7 10806 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10807 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10808 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10809 if (!exit_addr) { 10810 return 0; 10811 } 10812 | jmp &exit_addr 10813 } else { 10814 | SET_EX_OPLINE opline, r0 10815 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10816 | EXT_CALL zend_jit_only_vars_by_reference, r0 10817 if (!zend_jit_check_exception(Dst)) { 10818 return 0; 10819 } 10820 | jmp >7 10821 } 10822 10823 |.code 10824 } 10825 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10826 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10827 && JIT_G(current_frame) 10828 && JIT_G(current_frame)->call 10829 && JIT_G(current_frame)->call->func) { 10830 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10831 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10832 return 0; 10833 } 10834 return 1; 10835 } 10836 } else { 10837 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10838 | jnz >1 10839 |.cold_code 10840 |1: 10841 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10842 return 0; 10843 } 10844 | jmp >7 10845 |.code 10846 } 10847 } 10848 10849 if (op1_info & MAY_BE_UNDEF) { 10850 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10851 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10852 |.cold_code 10853 |1: 10854 } 10855 10856 | SET_EX_OPLINE opline, r0 10857 | mov FCARG1d, opline->op1.var 10858 | EXT_CALL zend_jit_undefined_op_helper, r0 10859 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10860 | test r0, r0 10861 | jz ->exception_handler 10862 10863 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10864 | jmp >7 10865 |.code 10866 } else { 10867 |7: 10868 return 1; 10869 } 10870 } 10871 10872 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10873 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10874 if (op1_info & MAY_BE_REF) { 10875 | cmp cl, IS_REFERENCE 10876 | je >7 10877 } 10878 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10879 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10880 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10881 if (!exit_addr) { 10882 return 0; 10883 } 10884 | jmp &exit_addr 10885 } else { 10886 | SET_EX_OPLINE opline, r0 10887 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10888 | EXT_CALL zend_jit_only_vars_by_reference, r0 10889 if (!zend_jit_check_exception(Dst)) { 10890 return 0; 10891 } 10892 } 10893 } else { 10894 if (op1_info & MAY_BE_REF) { 10895 if (opline->op1_type == IS_CV) { 10896 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 10897 10898 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10899 | ZVAL_DEREF FCARG1a, op1_info 10900 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10901 | TRY_ADDREF op1_info, ah, r2 10902 } else { 10903 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); 10904 10905 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10906 |.cold_code 10907 |1: 10908 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10909 | GET_ZVAL_PTR FCARG1a, op1_addr 10910 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10911 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10912 | GC_DELREF FCARG1a 10913 | je >1 10914 | IF_NOT_REFCOUNTED ah, >2 10915 | GC_ADDREF r2 10916 | jmp >2 10917 |1: 10918 | EFREE_REG_REFERENCE 10919 | jmp >2 10920 |.code 10921 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10922 |2: 10923 } 10924 } else { 10925 if (op1_addr != op1_def_addr) { 10926 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10927 return 0; 10928 } 10929 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10930 op1_addr= op1_def_addr; 10931 } 10932 } 10933 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10934 if (opline->op1_type == IS_CV) { 10935 | TRY_ADDREF op1_info, ah, r2 10936 } 10937 } 10938 } 10939 |7: 10940 10941 return 1; 10942} 10943 10944static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10945{ 10946 uint32_t arg_num = opline->op2.num; 10947 10948 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10949 && JIT_G(current_frame) 10950 && JIT_G(current_frame)->call 10951 && JIT_G(current_frame)->call->func) { 10952 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10953 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10954 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10955 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10956 || if (reuse_ip) { 10957 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10958 || } else { 10959 | mov r0, EX->call 10960 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10961 || } 10962 } 10963 } else { 10964 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10965 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10966 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10967 || if (reuse_ip) { 10968 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10969 || } else { 10970 | mov r0, EX->call 10971 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10972 || } 10973 } 10974 } 10975 } else { 10976 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10977 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10978 10979 if (!zend_jit_reuse_ip(Dst)) { 10980 return 0; 10981 } 10982 10983 | mov r0, EX:RX->func 10984 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10985 | jnz >1 10986 |.cold_code 10987 |1: 10988 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10989 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10990 | jmp >1 10991 |.code 10992 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10993 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10994 |1: 10995 } 10996 10997 return 1; 10998} 10999 11000static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 11001{ 11002 if (smart_branch_opcode) { 11003 if (smart_branch_opcode == ZEND_JMPZ) { 11004 if (jmp) { 11005 | jmp >7 11006 } 11007 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11008 | jmp =>target_label 11009 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11010 | jmp =>target_label2 11011 } else { 11012 ZEND_UNREACHABLE(); 11013 } 11014 } else { 11015 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11016 11017 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11018 if (jmp) { 11019 | jmp >7 11020 } 11021 } 11022 11023 return 1; 11024} 11025 11026static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 11027{ 11028 if (smart_branch_opcode) { 11029 if (smart_branch_opcode == ZEND_JMPZ) { 11030 | jmp =>target_label 11031 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11032 if (jmp) { 11033 | jmp >7 11034 } 11035 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11036 | jmp =>target_label 11037 } else { 11038 ZEND_UNREACHABLE(); 11039 } 11040 } else { 11041 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11042 11043 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11044 if (jmp) { 11045 | jmp >7 11046 } 11047 } 11048 11049 return 1; 11050} 11051 11052static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11053{ 11054 uint32_t defined_label = (uint32_t)-1; 11055 uint32_t undefined_label = (uint32_t)-1; 11056 zval *zv = RT_CONSTANT(opline, opline->op1); 11057 zend_jit_addr res_addr = 0; 11058 11059 if (smart_branch_opcode && !exit_addr) { 11060 if (smart_branch_opcode == ZEND_JMPZ) { 11061 undefined_label = target_label; 11062 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11063 defined_label = target_label; 11064 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11065 undefined_label = target_label; 11066 defined_label = target_label2; 11067 } else { 11068 ZEND_UNREACHABLE(); 11069 } 11070 } 11071 11072 | // if (CACHED_PTR(opline->extended_value)) { 11073 | mov r0, EX->run_time_cache 11074 | mov r0, aword [r0 + opline->extended_value] 11075 | test r0, r0 11076 | jz >1 11077 | test r0, 0x1 11078 | jnz >4 11079 |.cold_code 11080 |4: 11081 | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a 11082 | shr r0, 1 11083 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 11084 11085 if (smart_branch_opcode) { 11086 if (exit_addr) { 11087 if (smart_branch_opcode == ZEND_JMPZ) { 11088 | jz &exit_addr 11089 } else { 11090 | jz >3 11091 } 11092 } else if (undefined_label != (uint32_t)-1) { 11093 | jz =>undefined_label 11094 } else { 11095 | jz >3 11096 } 11097 } else { 11098 | jz >2 11099 } 11100 |1: 11101 | SET_EX_OPLINE opline, r0 11102 | LOAD_ADDR FCARG1a, zv 11103 | EXT_CALL zend_jit_check_constant, r0 11104 | test r0, r0 11105 if (exit_addr) { 11106 if (smart_branch_opcode == ZEND_JMPNZ) { 11107 | jz >3 11108 } else { 11109 | jnz >3 11110 } 11111 | jmp &exit_addr 11112 } else if (smart_branch_opcode) { 11113 if (undefined_label != (uint32_t)-1) { 11114 | jz =>undefined_label 11115 } else { 11116 | jz >3 11117 } 11118 if (defined_label != (uint32_t)-1) { 11119 | jmp =>defined_label 11120 } else { 11121 | jmp >3 11122 } 11123 } else { 11124 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11125 | jnz >1 11126 |2: 11127 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11128 | jmp >3 11129 } 11130 |.code 11131 if (smart_branch_opcode) { 11132 if (exit_addr) { 11133 if (smart_branch_opcode == ZEND_JMPNZ) { 11134 | jmp &exit_addr 11135 } 11136 } else if (defined_label != (uint32_t)-1) { 11137 | jmp =>defined_label 11138 } 11139 } else { 11140 |1: 11141 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11142 } 11143 |3: 11144 11145 return 1; 11146} 11147 11148static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11149{ 11150 uint32_t mask; 11151 zend_uchar type; 11152 zend_jit_addr op1_addr = OP1_ADDR(); 11153 11154 // TODO: support for is_resource() ??? 11155 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 11156 11157 if (op1_info & MAY_BE_UNDEF) { 11158 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11159 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11160 |.cold_code 11161 |1: 11162 } 11163 | SET_EX_OPLINE opline, r0 11164 | mov FCARG1d, opline->op1.var 11165 | EXT_CALL zend_jit_undefined_op_helper, r0 11166 zend_jit_check_exception_undef_result(Dst, opline); 11167 if (opline->extended_value & MAY_BE_NULL) { 11168 if (exit_addr) { 11169 if (smart_branch_opcode == ZEND_JMPNZ) { 11170 | jmp &exit_addr 11171 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11172 | jmp >7 11173 } 11174 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11175 return 0; 11176 } 11177 } else { 11178 if (exit_addr) { 11179 if (smart_branch_opcode == ZEND_JMPZ) { 11180 | jmp &exit_addr 11181 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11182 | jmp >7 11183 } 11184 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11185 return 0; 11186 } 11187 } 11188 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11189 |.code 11190 } 11191 } 11192 11193 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11194 mask = opline->extended_value; 11195 switch (mask) { 11196 case MAY_BE_NULL: type = IS_NULL; break; 11197 case MAY_BE_FALSE: type = IS_FALSE; break; 11198 case MAY_BE_TRUE: type = IS_TRUE; break; 11199 case MAY_BE_LONG: type = IS_LONG; break; 11200 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11201 case MAY_BE_STRING: type = IS_STRING; break; 11202 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11203 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11204 default: 11205 type = 0; 11206 } 11207 11208 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11209 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11210 if (exit_addr) { 11211 if (smart_branch_opcode == ZEND_JMPNZ) { 11212 | jmp &exit_addr 11213 } 11214 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11215 return 0; 11216 } 11217 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11218 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11219 if (exit_addr) { 11220 if (smart_branch_opcode == ZEND_JMPZ) { 11221 | jmp &exit_addr 11222 } 11223 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11224 return 0; 11225 } 11226 } else { 11227 if (op1_info & MAY_BE_REF) { 11228 | LOAD_ZVAL_ADDR r0, op1_addr 11229 | ZVAL_DEREF r0, op1_info 11230 } 11231 if (type == 0) { 11232 if (smart_branch_opcode && 11233 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11234 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11235 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11236 | // if (Z_REFCOUNTED_P(cv)) { 11237 | IF_ZVAL_REFCOUNTED op1_addr, >1 11238 |.cold_code 11239 |1: 11240 } 11241 | // if (!Z_DELREF_P(cv)) { 11242 | GET_ZVAL_PTR FCARG1a, op1_addr 11243 | GC_DELREF FCARG1a 11244 if (RC_MAY_BE_1(op1_info)) { 11245 if (RC_MAY_BE_N(op1_info)) { 11246 | jnz >3 11247 } 11248 if (op1_info & MAY_BE_REF) { 11249 | mov al, byte [r0 + 8] 11250 } else { 11251 | mov al, byte [FP + opline->op1.var + 8] 11252 } 11253 | mov byte T1, al // save 11254 | // zval_dtor_func(r); 11255 | ZVAL_DTOR_FUNC op1_info, opline 11256 | mov cl, byte T1 // restore 11257 |jmp >2 11258 } 11259 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11260 if (!RC_MAY_BE_1(op1_info)) { 11261 | jmp >3 11262 } 11263 |.code 11264 } 11265 |3: 11266 if (op1_info & MAY_BE_REF) { 11267 | mov cl, byte [r0 + 8] 11268 } else { 11269 | mov cl, byte [FP + opline->op1.var + 8] 11270 } 11271 |2: 11272 } else { 11273 if (op1_info & MAY_BE_REF) { 11274 | mov cl, byte [r0 + 8] 11275 } else { 11276 | mov cl, byte [FP + opline->op1.var + 8] 11277 } 11278 } 11279 | mov eax, 1 11280 | shl eax, cl 11281 | test eax, mask 11282 if (exit_addr) { 11283 if (smart_branch_opcode == ZEND_JMPNZ) { 11284 | jne &exit_addr 11285 } else { 11286 | je &exit_addr 11287 } 11288 } else if (smart_branch_opcode) { 11289 if (smart_branch_opcode == ZEND_JMPZ) { 11290 | je =>target_label 11291 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11292 | jne =>target_label 11293 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11294 | je =>target_label 11295 | jmp =>target_label2 11296 } else { 11297 ZEND_UNREACHABLE(); 11298 } 11299 } else { 11300 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11301 11302 | setne al 11303 | movzx eax, al 11304 | add eax, 2 11305 | SET_ZVAL_TYPE_INFO res_addr, eax 11306 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11307 } 11308 } else { 11309 if (smart_branch_opcode && 11310 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11311 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11312 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11313 | // if (Z_REFCOUNTED_P(cv)) { 11314 | IF_ZVAL_REFCOUNTED op1_addr, >1 11315 |.cold_code 11316 |1: 11317 } 11318 | // if (!Z_DELREF_P(cv)) { 11319 | GET_ZVAL_PTR FCARG1a, op1_addr 11320 | GC_DELREF FCARG1a 11321 if (RC_MAY_BE_1(op1_info)) { 11322 if (RC_MAY_BE_N(op1_info)) { 11323 | jnz >3 11324 } 11325 if (op1_info & MAY_BE_REF) { 11326 | mov al, byte [r0 + 8] 11327 } else { 11328 | mov al, byte [FP + opline->op1.var + 8] 11329 } 11330 | mov byte T1, al // save 11331 | // zval_dtor_func(r); 11332 | ZVAL_DTOR_FUNC op1_info, opline 11333 | mov cl, byte T1 // restore 11334 |jmp >2 11335 } 11336 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11337 if (!RC_MAY_BE_1(op1_info)) { 11338 | jmp >3 11339 } 11340 |.code 11341 } 11342 |3: 11343 if (op1_info & MAY_BE_REF) { 11344 | mov cl, byte [r0 + 8] 11345 } else { 11346 | mov cl, byte [FP + opline->op1.var + 8] 11347 } 11348 |2: 11349 | cmp cl, type 11350 } else { 11351 if (op1_info & MAY_BE_REF) { 11352 | cmp byte [r0 + 8], type 11353 } else { 11354 | cmp byte [FP + opline->op1.var + 8], type 11355 } 11356 } 11357 if (exit_addr) { 11358 if (smart_branch_opcode == ZEND_JMPNZ) { 11359 | je &exit_addr 11360 } else { 11361 | jne &exit_addr 11362 } 11363 } else if (smart_branch_opcode) { 11364 if (smart_branch_opcode == ZEND_JMPZ) { 11365 | jne =>target_label 11366 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11367 | je =>target_label 11368 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11369 | jne =>target_label 11370 | jmp =>target_label2 11371 } else { 11372 ZEND_UNREACHABLE(); 11373 } 11374 } else { 11375 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11376 11377 | sete al 11378 | movzx eax, al 11379 | add eax, 2 11380 | SET_ZVAL_TYPE_INFO res_addr, eax 11381 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11382 } 11383 } 11384 } 11385 } 11386 11387 |7: 11388 11389 return 1; 11390} 11391 11392static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) 11393{ 11394 uint32_t j, info; 11395 11396 if (ssa->vars && ssa->var_info) { 11397 info = ssa->var_info[var].type; 11398 for (j = op_array->last_var; j < ssa->vars_count; j++) { 11399 if (ssa->vars[j].var == var) { 11400 info |= ssa->var_info[j].type; 11401 } 11402 } 11403 } else { 11404 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF | 11405 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 11406 } 11407 11408#ifdef ZEND_JIT_USE_RC_INFERENCE 11409 /* Refcount may be increased by RETURN opcode */ 11410 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { 11411 for (j = 0; j < ssa->cfg.blocks_count; j++) { 11412 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && 11413 ssa->cfg.blocks[j].len > 0) { 11414 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; 11415 11416 if (opline->opcode == ZEND_RETURN) { 11417 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { 11418 info |= MAY_BE_RCN; 11419 break; 11420 } 11421 } 11422 } 11423 } 11424 } 11425#endif 11426 11427 return info; 11428} 11429 11430static int zend_jit_leave_frame(dasm_State **Dst) 11431{ 11432 | // EG(current_execute_data) = EX(prev_execute_data); 11433 | mov r0, EX->prev_execute_data 11434 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 11435 return 1; 11436} 11437 11438static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11439{ 11440 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11441 uint32_t offset = EX_NUM_TO_VAR(var); 11442 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11443 } 11444 return 1; 11445} 11446 11447static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11448{ 11449 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11450 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11451 } 11452 return 1; 11453} 11454 11455static int zend_jit_leave_func(dasm_State **Dst, 11456 const zend_op_array *op_array, 11457 const zend_op *opline, 11458 uint32_t op1_info, 11459 zend_bool left_frame, 11460 zend_jit_trace_rec *trace, 11461 zend_jit_trace_info *trace_info, 11462 int indirect_var_access, 11463 int may_throw) 11464{ 11465 zend_bool may_be_top_frame = 11466 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11467 !JIT_G(current_frame) || 11468 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11469 zend_bool may_need_call_helper = 11470 indirect_var_access || /* may have symbol table */ 11471 !op_array->function_name || /* may have symbol table */ 11472 may_be_top_frame || 11473 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11474 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11475 !JIT_G(current_frame) || 11476 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11477 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11478 zend_bool may_need_release_this = 11479 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11480 op_array->scope && 11481 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11482 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11483 !JIT_G(current_frame) || 11484 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11485 11486 if (may_need_call_helper || may_need_release_this) { 11487 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11488 } 11489 if (may_need_call_helper) { 11490 if (!left_frame) { 11491 left_frame = 1; 11492 if (!zend_jit_leave_frame(Dst)) { 11493 return 0; 11494 } 11495 } 11496 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11497 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11498 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11499 | jnz >1 11500 |.cold_code 11501 |1: 11502 if (!GCC_GLOBAL_REGS) { 11503 | mov FCARG2a, FP 11504 } 11505 | EXT_CALL zend_jit_leave_func_helper, r0 11506 11507 if (may_be_top_frame) { 11508 // TODO: try to avoid this check ??? 11509 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11510#if 0 11511 /* this check should be handled by the following OPLINE guard */ 11512 | cmp IP, zend_jit_halt_op 11513 | je ->trace_halt 11514#endif 11515 } else if (GCC_GLOBAL_REGS) { 11516 | test IP, IP 11517 | je ->trace_halt 11518 } else { 11519 | test eax, eax 11520 | jl ->trace_halt 11521 } 11522 } 11523 11524 if (!GCC_GLOBAL_REGS) { 11525 | // execute_data = EG(current_execute_data) 11526 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 11527 } 11528 | jmp >8 11529 |.code 11530 } else { 11531 | jnz ->leave_function_handler 11532 } 11533 } 11534 11535 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11536 if (!left_frame) { 11537 left_frame = 1; 11538 if (!zend_jit_leave_frame(Dst)) { 11539 return 0; 11540 } 11541 } 11542 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11543 | mov FCARG1a, EX->func 11544 | sub FCARG1a, sizeof(zend_object) 11545 | OBJ_RELEASE ZREG_FCARG1a, >4 11546 |4: 11547 } else if (may_need_release_this) { 11548 if (!left_frame) { 11549 left_frame = 1; 11550 if (!zend_jit_leave_frame(Dst)) { 11551 return 0; 11552 } 11553 } 11554 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11555 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11556 | je >4 11557 | // zend_object *object = Z_OBJ(execute_data->This); 11558 | mov FCARG1a, EX->This.value.obj 11559 | // OBJ_RELEASE(object); 11560 | OBJ_RELEASE ZREG_FCARG1a, >4 11561 |4: 11562 // TODO: avoid EG(excption) check for $this->foo() calls 11563 may_throw = 1; 11564 } 11565 11566 | // EG(vm_stack_top) = (zval*)execute_data; 11567 | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 11568 | // execute_data = EX(prev_execute_data); 11569 | mov FP, EX->prev_execute_data 11570 11571 if (!left_frame) { 11572 | // EG(current_execute_data) = execute_data; 11573 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 11574 } 11575 11576 |9: 11577 if (trace) { 11578 if (trace->op != ZEND_JIT_TRACE_END 11579 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11580 zend_jit_reset_last_valid_opline(); 11581 } else { 11582 | LOAD_IP 11583 | ADD_IP sizeof(zend_op) 11584 } 11585 11586 |8: 11587 11588 if (trace->op == ZEND_JIT_TRACE_BACK 11589 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11590 const zend_op *next_opline = trace->opline; 11591 11592 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11593 && (op1_info & MAY_BE_RC1) 11594 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11595 /* exception might be thrown during destruction of unused return value */ 11596 | // if (EG(exception)) 11597 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11598 | jne ->leave_throw_handler 11599 } 11600 do { 11601 trace++; 11602 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11603 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11604 next_opline = trace->opline; 11605 ZEND_ASSERT(next_opline != NULL); 11606 11607 if (trace->op == ZEND_JIT_TRACE_END 11608 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11609 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11610 | CMP_IP next_opline 11611 | je =>0 // LOOP 11612#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11613 | JMP_IP 11614#else 11615 | jmp ->trace_escape 11616#endif 11617 } else { 11618 | CMP_IP next_opline 11619 | jne ->trace_escape 11620 } 11621 11622 zend_jit_set_last_valid_opline(trace->opline); 11623 11624 return 1; 11625 } else if (may_throw || 11626 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11627 && (op1_info & MAY_BE_RC1) 11628 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11629 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11630 | // if (EG(exception)) 11631 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11632 | jne ->leave_throw_handler 11633 } 11634 11635 return 1; 11636 } else { 11637 | // if (EG(exception)) 11638 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11639 | LOAD_IP 11640 | jne ->leave_throw_handler 11641 | // opline = EX(opline) + 1 11642 | ADD_IP sizeof(zend_op) 11643 } 11644 11645 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11646 | ADD_HYBRID_SPAD 11647#ifdef CONTEXT_THREADED_JIT 11648 | push aword [IP] 11649 | ret 11650#else 11651 | JMP_IP 11652#endif 11653 } else if (GCC_GLOBAL_REGS) { 11654 | add r4, SPAD // stack alignment 11655#ifdef CONTEXT_THREADED_JIT 11656 | push aword [IP] 11657 | ret 11658#else 11659 | JMP_IP 11660#endif 11661 } else { 11662#ifdef CONTEXT_THREADED_JIT 11663 ZEND_UNREACHABLE(); 11664 // TODO: context threading can't work without GLOBAL REGS because we have to change 11665 // the value of execute_data in execute_ex() 11666 | mov FCARG1a, FP 11667 | mov r0, aword [FP] 11668 | mov FP, aword T2 // restore FP 11669 | mov RX, aword T3 // restore IP 11670 | add r4, NR_SPAD // stack alignment 11671 | push aword [r0] 11672 | ret 11673#else 11674 | mov FP, aword T2 // restore FP 11675 | mov RX, aword T3 // restore IP 11676 | add r4, NR_SPAD // stack alignment 11677 | mov r0, 2 // ZEND_VM_LEAVE 11678 | ret 11679#endif 11680 } 11681 11682 return 1; 11683} 11684 11685static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11686{ 11687 zend_jit_addr ret_addr; 11688 int8_t return_value_used; 11689 11690 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11691 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11692 11693 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11694 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11695 return_value_used = 1; 11696 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11697 return_value_used = 0; 11698 } else { 11699 return_value_used = -1; 11700 } 11701 } else { 11702 return_value_used = -1; 11703 } 11704 11705 if (ZEND_OBSERVER_ENABLED) { 11706 if (Z_MODE(op1_addr) == IS_REG) { 11707 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11708 11709 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11710 return 0; 11711 } 11712 op1_addr = dst; 11713 } 11714 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11715 | mov FCARG1a, FP 11716 | SET_EX_OPLINE opline, r0 11717 | EXT_CALL zend_observer_fcall_end, r0 11718 } 11719 11720 // if (!EX(return_value)) 11721 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11722 if (return_value_used != 0) { 11723 | mov r2, EX->return_value 11724 } 11725 if (return_value_used == -1) { 11726 | test r2, r2 11727 } 11728 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11729 } else { 11730 if (return_value_used != 0) { 11731 | mov r1, EX->return_value 11732 } 11733 if (return_value_used == -1) { 11734 | test r1, r1 11735 } 11736 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11737 } 11738 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11739 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11740 if (return_value_used == -1) { 11741 | jz >1 11742 |.cold_code 11743 |1: 11744 } 11745 if (return_value_used != 1) { 11746 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11747 if (jit_return_label >= 0) { 11748 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11749 } else { 11750 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11751 } 11752 } 11753 | GET_ZVAL_PTR FCARG1a, op1_addr 11754 | GC_DELREF FCARG1a 11755 if (RC_MAY_BE_1(op1_info)) { 11756 if (RC_MAY_BE_N(op1_info)) { 11757 if (jit_return_label >= 0) { 11758 | jnz =>jit_return_label 11759 } else { 11760 | jnz >9 11761 } 11762 } 11763 | //SAVE_OPLINE() 11764 | ZVAL_DTOR_FUNC op1_info, opline 11765 | //????mov r1, EX->return_value // reload ??? 11766 } 11767 if (return_value_used == -1) { 11768 if (jit_return_label >= 0) { 11769 | jmp =>jit_return_label 11770 } else { 11771 | jmp >9 11772 } 11773 |.code 11774 } 11775 } 11776 } else if (return_value_used == -1) { 11777 if (jit_return_label >= 0) { 11778 | jz =>jit_return_label 11779 } else { 11780 | jz >9 11781 } 11782 } 11783 11784 if (return_value_used == 0) { 11785 |9: 11786 return 1; 11787 } 11788 11789 if (opline->op1_type == IS_CONST) { 11790 zval *zv = RT_CONSTANT(opline, opline->op1); 11791 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11792 if (Z_REFCOUNTED_P(zv)) { 11793 | ADDREF_CONST zv, r0 11794 } 11795 } else if (opline->op1_type == IS_TMP_VAR) { 11796 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11797 } else if (opline->op1_type == IS_CV) { 11798 if (op1_info & MAY_BE_REF) { 11799 | LOAD_ZVAL_ADDR r0, op1_addr 11800 | ZVAL_DEREF r0, op1_info 11801 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11802 } 11803 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11804 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11805 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11806 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11807 !op_array->function_name) { 11808 | TRY_ADDREF op1_info, ah, r2 11809 } else if (return_value_used != 1) { 11810 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11811 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11812 } 11813 } 11814 } else { 11815 if (op1_info & MAY_BE_REF) { 11816 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11817 11818 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11819 |.cold_code 11820 |1: 11821 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11822 | GET_ZVAL_PTR r0, op1_addr 11823 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11824 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11825 | GC_DELREF r0 11826 | je >2 11827 | // if (IS_REFCOUNTED()) 11828 if (jit_return_label >= 0) { 11829 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11830 } else { 11831 | IF_NOT_REFCOUNTED dh, >9 11832 } 11833 | // ADDREF 11834 | GET_ZVAL_PTR r2, ret_addr // reload 11835 | GC_ADDREF r2 11836 if (jit_return_label >= 0) { 11837 | jmp =>jit_return_label 11838 } else { 11839 | jmp >9 11840 } 11841 |2: 11842 | EFREE_REFERENCE r0 11843 if (jit_return_label >= 0) { 11844 | jmp =>jit_return_label 11845 } else { 11846 | jmp >9 11847 } 11848 |.code 11849 } 11850 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11851 } 11852 11853 |9: 11854 return 1; 11855} 11856 11857static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11858{ 11859 ZEND_ASSERT(type_reg == ZREG_R2); 11860 11861 |.if not(X64) 11862 || if (Z_REG(val_addr) == ZREG_R1) { 11863 | GET_ZVAL_W2 r0, val_addr 11864 || } 11865 |.endif 11866 | GET_ZVAL_PTR r1, val_addr 11867 |.if not(X64) 11868 || if (Z_REG(val_addr) != ZREG_R1) { 11869 | GET_ZVAL_W2 r0, val_addr 11870 || } 11871 |.endif 11872 | IF_NOT_REFCOUNTED dh, >2 11873 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11874 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11875 |.if not(X64) 11876 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11877 |.endif 11878 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11879 | IF_NOT_REFCOUNTED dh, >2 11880 |1: 11881 | GC_ADDREF r1 11882 |2: 11883 | SET_ZVAL_PTR res_addr, r1 11884 |.if not(X64) 11885 | SET_ZVAL_W2 res_addr, r0 11886 |.endif 11887 | SET_ZVAL_TYPE_INFO res_addr, edx 11888 11889 return 1; 11890} 11891 11892static zend_bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info) 11893{ 11894 switch (opline->opcode) { 11895 case ZEND_FETCH_OBJ_FUNC_ARG: 11896 if (!JIT_G(current_frame) || 11897 !JIT_G(current_frame)->call->func || 11898 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 11899 return 0; 11900 } 11901 /* break missing intentionally */ 11902 case ZEND_FETCH_OBJ_R: 11903 case ZEND_FETCH_OBJ_IS: 11904 if ((op1_info & MAY_BE_OBJECT) 11905 && opline->op2_type == IS_CONST 11906 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING 11907 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') { 11908 return 1; 11909 } 11910 break; 11911 case ZEND_FETCH_DIM_FUNC_ARG: 11912 if (!JIT_G(current_frame) || 11913 !JIT_G(current_frame)->call->func || 11914 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 11915 return 0; 11916 } 11917 /* break missing intentionally */ 11918 case ZEND_FETCH_DIM_R: 11919 case ZEND_FETCH_DIM_IS: 11920 return 1; 11921 case ZEND_ISSET_ISEMPTY_DIM_OBJ: 11922 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11923 return 1; 11924 } 11925 break; 11926 } 11927 return 0; 11928} 11929 11930static int zend_jit_fetch_dim_read(dasm_State **Dst, 11931 const zend_op *opline, 11932 zend_ssa *ssa, 11933 const zend_ssa_op *ssa_op, 11934 uint32_t op1_info, 11935 zend_jit_addr op1_addr, 11936 zend_bool op1_avoid_refcounting, 11937 uint32_t op2_info, 11938 uint32_t res_info, 11939 zend_jit_addr res_addr, 11940 int may_throw) 11941{ 11942 zend_jit_addr orig_op1_addr, op2_addr; 11943 const void *exit_addr = NULL; 11944 const void *not_found_exit_addr = NULL; 11945 const void *res_exit_addr = NULL; 11946 zend_bool result_avoid_refcounting = 0; 11947 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11948 11949 orig_op1_addr = OP1_ADDR(); 11950 op2_addr = OP2_ADDR(); 11951 11952 if (opline->opcode != ZEND_FETCH_DIM_IS 11953 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11954 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11955 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11956 if (!exit_addr) { 11957 return 0; 11958 } 11959 } 11960 11961 if ((res_info & MAY_BE_GUARD) 11962 && JIT_G(current_frame) 11963 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11964 uint32_t flags = 0; 11965 uint32_t old_op1_info = 0; 11966 uint32_t old_info; 11967 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11968 int32_t exit_point; 11969 11970 if (opline->opcode != ZEND_FETCH_LIST_R 11971 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11972 && !op1_avoid_refcounting) { 11973 flags |= ZEND_JIT_EXIT_FREE_OP1; 11974 } 11975 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11976 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11977 flags |= ZEND_JIT_EXIT_FREE_OP2; 11978 } 11979 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11980 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11981 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11982 && (ssa_op+1)->op1_use == ssa_op->result_def 11983 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11984 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11985 result_avoid_refcounting = 1; 11986 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11987 } 11988 11989 if (op1_avoid_refcounting) { 11990 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11991 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11992 } 11993 11994 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11995 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11996 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11997 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 11998 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11999 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12000 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12001 if (!res_exit_addr) { 12002 return 0; 12003 } 12004 res_info &= ~MAY_BE_GUARD; 12005 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12006 } 12007 12008 if (opline->opcode == ZEND_FETCH_DIM_IS 12009 && !(res_info & MAY_BE_NULL)) { 12010 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12011 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 12012 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 12013 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12014 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12015 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12016 if (!not_found_exit_addr) { 12017 return 0; 12018 } 12019 } 12020 12021 if (op1_avoid_refcounting) { 12022 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 12023 } 12024 } 12025 12026 if (op1_info & MAY_BE_REF) { 12027 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12028 | ZVAL_DEREF FCARG1a, op1_info 12029 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12030 } 12031 12032 if (op1_info & MAY_BE_ARRAY) { 12033 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12034 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 12035 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 12036 } else { 12037 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12038 } 12039 } 12040 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12041 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { 12042 return 0; 12043 } 12044 } 12045 12046 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12047 if (op1_info & MAY_BE_ARRAY) { 12048 |.cold_code 12049 |7: 12050 } 12051 12052 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 12053 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 12054 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 12055 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 12056 } else { 12057 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 12058 } 12059 } 12060 | SET_EX_OPLINE opline, r0 12061 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12062 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12063 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 12064 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 12065 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 12066 } else { 12067 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12068 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 12069 } 12070 | SET_ZVAL_PTR res_addr, r0 12071 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 12072 } else { 12073 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12074 |.if X64 12075 | LOAD_ZVAL_ADDR CARG3, res_addr 12076 |.else 12077 | sub r4, 12 12078 | PUSH_ZVAL_ADDR res_addr, r0 12079 |.endif 12080 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 12081 |.if not(X64) 12082 | add r4, 12 12083 |.endif 12084 } 12085 if ((op1_info & MAY_BE_ARRAY) || 12086 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 12087 | jmp >9 // END 12088 } 12089 |6: 12090 } 12091 12092 if (op1_info & MAY_BE_OBJECT) { 12093 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 12094 if (exit_addr) { 12095 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12096 } else { 12097 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 12098 } 12099 } 12100 | SET_EX_OPLINE opline, r0 12101 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12102 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12103 } 12104 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12105 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12106 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12107 } else { 12108 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12109 } 12110 |.if X64 12111 | LOAD_ZVAL_ADDR CARG3, res_addr 12112 |.else 12113 | sub r4, 12 12114 | PUSH_ZVAL_ADDR res_addr, r0 12115 |.endif 12116 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12117 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 12118 } else { 12119 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 12120 } 12121 |.if not(X64) 12122 | add r4, 12 12123 |.endif 12124 if ((op1_info & MAY_BE_ARRAY) || 12125 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12126 | jmp >9 // END 12127 } 12128 |6: 12129 } 12130 12131 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 12132 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12133 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 12134 | SET_EX_OPLINE opline, r0 12135 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 12136 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12137 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 12138 | mov FCARG1d, opline->op1.var 12139 | EXT_CALL zend_jit_undefined_op_helper, r0 12140 |1: 12141 } 12142 12143 if (op2_info & MAY_BE_UNDEF) { 12144 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12145 | mov FCARG1d, opline->op2.var 12146 | EXT_CALL zend_jit_undefined_op_helper, r0 12147 |1: 12148 } 12149 } 12150 12151 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 12152 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 12153 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 12154 } else { 12155 | SET_EX_OPLINE opline, r0 12156 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 12157 Z_REG(op1_addr) != ZREG_FCARG1a || 12158 Z_OFFSET(op1_addr) != 0) { 12159 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12160 } 12161 } 12162 | EXT_CALL zend_jit_invalid_array_access, r0 12163 } 12164 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12165 if (op1_info & MAY_BE_ARRAY) { 12166 | jmp >9 // END 12167 } 12168 } 12169 12170 if (op1_info & MAY_BE_ARRAY) { 12171 |.code 12172 } 12173 } 12174 12175 if (op1_info & MAY_BE_ARRAY) { 12176 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 12177 12178 |8: 12179 if (res_exit_addr) { 12180 zend_uchar type = concrete_type(res_info); 12181 12182 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12183 | ZVAL_DEREF r0, MAY_BE_REF 12184 } 12185 if (type < IS_STRING) { 12186 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12187 } else { 12188 | GET_ZVAL_TYPE_INFO edx, val_addr 12189 | IF_NOT_TYPE dl, type, &res_exit_addr 12190 } 12191 | // ZVAL_COPY 12192 |7: 12193 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12194 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12195 if (type < IS_STRING) { 12196 if (Z_REG(res_addr) != ZREG_FP || 12197 JIT_G(current_frame) == NULL || 12198 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12199 | SET_ZVAL_TYPE_INFO res_addr, type 12200 } 12201 } else { 12202 | SET_ZVAL_TYPE_INFO res_addr, edx 12203 if (!result_avoid_refcounting) { 12204 | TRY_ADDREF res_info, dh, r1 12205 } 12206 } 12207 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12208 return 0; 12209 } 12210 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12211 | // ZVAL_COPY_DEREF 12212 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12213 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12214 return 0; 12215 } 12216 } else { 12217 | // ZVAL_COPY 12218 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12219 | TRY_ADDREF res_info, ch, r2 12220 } 12221 } 12222 |9: // END 12223 12224#ifdef ZEND_JIT_USE_RC_INFERENCE 12225 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12226 /* Magic offsetGet() may increase refcount of the key */ 12227 op2_info |= MAY_BE_RCN; 12228 } 12229#endif 12230 12231 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12232 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12233 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12234 } 12235 12236 if (may_throw) { 12237 if (!zend_jit_check_exception(Dst)) { 12238 return 0; 12239 } 12240 } 12241 12242 return 1; 12243} 12244 12245static int zend_jit_fetch_dim(dasm_State **Dst, 12246 const zend_op *opline, 12247 uint32_t op1_info, 12248 zend_jit_addr op1_addr, 12249 uint32_t op2_info, 12250 zend_jit_addr res_addr, 12251 int may_throw) 12252{ 12253 zend_jit_addr op2_addr; 12254 12255 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12256 12257 if (op1_info & MAY_BE_REF) { 12258 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12259 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12260 | GET_Z_PTR FCARG2a, FCARG1a 12261 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12262 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12263 | jmp >3 12264 |.cold_code 12265 |2: 12266 | SET_EX_OPLINE opline, r0 12267 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12268 | test r0, r0 12269 | mov FCARG1a, r0 12270 | jne >1 12271 | jmp ->exception_handler_undef 12272 |.code 12273 |1: 12274 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12275 } 12276 12277 if (op1_info & MAY_BE_ARRAY) { 12278 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12279 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12280 } 12281 |3: 12282 | SEPARATE_ARRAY op1_addr, op1_info, 1 12283 } 12284 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 12285 if (op1_info & MAY_BE_ARRAY) { 12286 |.cold_code 12287 |7: 12288 } 12289 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 12290 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 12291 | jg >7 12292 } 12293 if (Z_REG(op1_addr) != ZREG_FP) { 12294 | mov T1, Ra(Z_REG(op1_addr)) // save 12295 } 12296 if ((op1_info & MAY_BE_UNDEF) 12297 && opline->opcode == ZEND_FETCH_DIM_RW) { 12298 if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { 12299 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12300 } 12301 | SET_EX_OPLINE opline, r0 12302 | mov FCARG1a, opline->op1.var 12303 | EXT_CALL zend_jit_undefined_op_helper, r0 12304 |1: 12305 } 12306 | // ZVAL_ARR(container, zend_new_array(8)); 12307 | EXT_CALL _zend_new_array_0, r0 12308 if (Z_REG(op1_addr) != ZREG_FP) { 12309 | mov Ra(Z_REG(op1_addr)), T1 // restore 12310 } 12311 | SET_ZVAL_LVAL op1_addr, r0 12312 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12313 | mov FCARG1a, r0 12314 if (op1_info & MAY_BE_ARRAY) { 12315 | jmp >1 12316 |.code 12317 |1: 12318 } 12319 } 12320 12321 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12322 |6: 12323 if (opline->op2_type == IS_UNUSED) { 12324 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12325 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12326 | EXT_CALL zend_hash_next_index_insert, r0 12327 | // if (UNEXPECTED(!var_ptr)) { 12328 | test r0, r0 12329 | jz >1 12330 |.cold_code 12331 |1: 12332 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12333 | CANNOT_ADD_ELEMENT opline 12334 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12335 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12336 | jmp >8 12337 |.code 12338 | SET_ZVAL_PTR res_addr, r0 12339 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12340 } else { 12341 uint32_t type; 12342 12343 switch (opline->opcode) { 12344 case ZEND_FETCH_DIM_W: 12345 case ZEND_FETCH_LIST_W: 12346 type = BP_VAR_W; 12347 break; 12348 case ZEND_FETCH_DIM_RW: 12349 type = BP_VAR_RW; 12350 break; 12351 case ZEND_FETCH_DIM_UNSET: 12352 type = BP_VAR_UNSET; 12353 break; 12354 default: 12355 ZEND_UNREACHABLE(); 12356 } 12357 12358 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { 12359 return 0; 12360 } 12361 12362 |8: 12363 | SET_ZVAL_PTR res_addr, r0 12364 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12365 12366 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12367 |.cold_code 12368 |9: 12369 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12370 | jmp >8 12371 |.code 12372 } 12373 } 12374 } 12375 12376 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 12377 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12378 |.cold_code 12379 |7: 12380 } 12381 12382 | SET_EX_OPLINE opline, r0 12383 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12384 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12385 } 12386 if (opline->op2_type == IS_UNUSED) { 12387 | xor FCARG2a, FCARG2a 12388 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12389 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12390 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12391 } else { 12392 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12393 } 12394 |.if X64 12395 | LOAD_ZVAL_ADDR CARG3, res_addr 12396 |.else 12397 | sub r4, 12 12398 | PUSH_ZVAL_ADDR res_addr, r0 12399 |.endif 12400 switch (opline->opcode) { 12401 case ZEND_FETCH_DIM_W: 12402 case ZEND_FETCH_LIST_W: 12403 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12404 break; 12405 case ZEND_FETCH_DIM_RW: 12406 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12407 break; 12408// case ZEND_FETCH_DIM_UNSET: 12409// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12410// break; 12411 default: 12412 ZEND_UNREACHABLE(); 12413 } 12414 |.if not(X64) 12415 | add r4, 12 12416 |.endif 12417 12418 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12419 | jmp >8 // END 12420 |.code 12421 } 12422 } 12423 12424#ifdef ZEND_JIT_USE_RC_INFERENCE 12425 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12426 /* ASSIGN_DIM may increase refcount of the key */ 12427 op2_info |= MAY_BE_RCN; 12428 } 12429#endif 12430 12431 |8: 12432 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12433 12434 if (may_throw) { 12435 if (!zend_jit_check_exception(Dst)) { 12436 return 0; 12437 } 12438 } 12439 12440 return 1; 12441} 12442 12443static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12444 const zend_op *opline, 12445 uint32_t op1_info, 12446 zend_jit_addr op1_addr, 12447 zend_bool op1_avoid_refcounting, 12448 uint32_t op2_info, 12449 int may_throw, 12450 zend_uchar smart_branch_opcode, 12451 uint32_t target_label, 12452 uint32_t target_label2, 12453 const void *exit_addr) 12454{ 12455 zend_jit_addr op2_addr, res_addr; 12456 12457 // TODO: support for empty() ??? 12458 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12459 12460 op2_addr = OP2_ADDR(); 12461 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12462 12463 if (op1_info & MAY_BE_REF) { 12464 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12465 | ZVAL_DEREF FCARG1a, op1_info 12466 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12467 } 12468 12469 if (op1_info & MAY_BE_ARRAY) { 12470 const void *found_exit_addr = NULL; 12471 const void *not_found_exit_addr = NULL; 12472 12473 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12474 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12475 } 12476 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12477 if (exit_addr 12478 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12479 && !may_throw 12480 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12481 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12482 if (smart_branch_opcode == ZEND_JMPNZ) { 12483 found_exit_addr = exit_addr; 12484 } else { 12485 not_found_exit_addr = exit_addr; 12486 } 12487 } 12488 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { 12489 return 0; 12490 } 12491 12492 if (found_exit_addr) { 12493 |9: 12494 return 1; 12495 } else if (not_found_exit_addr) { 12496 |8: 12497 return 1; 12498 } 12499 } 12500 12501 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12502 if (op1_info & MAY_BE_ARRAY) { 12503 |.cold_code 12504 |7: 12505 } 12506 12507 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12508 | SET_EX_OPLINE opline, r0 12509 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12510 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12511 } 12512 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12513 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12514 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12515 } else { 12516 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12517 } 12518 | EXT_CALL zend_jit_isset_dim_helper, r0 12519 | test r0, r0 12520 | jz >9 12521 if (op1_info & MAY_BE_ARRAY) { 12522 | jmp >8 12523 |.code 12524 } 12525 } else { 12526 if (op2_info & MAY_BE_UNDEF) { 12527 if (op2_info & MAY_BE_ANY) { 12528 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12529 } 12530 | SET_EX_OPLINE opline, r0 12531 | mov FCARG1d, opline->op2.var 12532 | EXT_CALL zend_jit_undefined_op_helper, r0 12533 |1: 12534 } 12535 if (op1_info & MAY_BE_ARRAY) { 12536 | jmp >9 12537 |.code 12538 } 12539 } 12540 } 12541 12542#ifdef ZEND_JIT_USE_RC_INFERENCE 12543 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12544 /* Magic offsetExists() may increase refcount of the key */ 12545 op2_info |= MAY_BE_RCN; 12546 } 12547#endif 12548 12549 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12550 |8: 12551 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12552 if (!op1_avoid_refcounting) { 12553 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12554 } 12555 if (may_throw) { 12556 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12557 return 0; 12558 } 12559 } 12560 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12561 if (exit_addr) { 12562 if (smart_branch_opcode == ZEND_JMPNZ) { 12563 | jmp &exit_addr 12564 } else { 12565 | jmp >8 12566 } 12567 } else if (smart_branch_opcode) { 12568 if (smart_branch_opcode == ZEND_JMPZ) { 12569 | jmp =>target_label2 12570 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12571 | jmp =>target_label 12572 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12573 | jmp =>target_label2 12574 } else { 12575 ZEND_UNREACHABLE(); 12576 } 12577 } else { 12578 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12579 | jmp >8 12580 } 12581 } else { 12582 | //???? 12583 | int3 12584 } 12585 } 12586 12587 |9: // not found 12588 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12589 if (!op1_avoid_refcounting) { 12590 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12591 } 12592 if (may_throw) { 12593 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12594 return 0; 12595 } 12596 } 12597 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12598 if (exit_addr) { 12599 if (smart_branch_opcode == ZEND_JMPZ) { 12600 | jmp &exit_addr 12601 } 12602 } else if (smart_branch_opcode) { 12603 if (smart_branch_opcode == ZEND_JMPZ) { 12604 | jmp =>target_label 12605 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12606 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12607 | jmp =>target_label 12608 } else { 12609 ZEND_UNREACHABLE(); 12610 } 12611 } else { 12612 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12613 } 12614 } else { 12615 | //???? 12616 | int3 12617 } 12618 12619 |8: 12620 12621 return 1; 12622} 12623 12624static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12625{ 12626 zend_jit_addr op1_addr = OP1_ADDR(); 12627 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12628 12629 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12630 | mov r0, EX->run_time_cache 12631 | mov r0, aword [r0 + opline->extended_value] 12632 | sub r0, 1 12633 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12634 | MEM_OP2_2_ZTS mov, ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12635 |.if X64 12636 | shl r1, 5 12637 |.else 12638 | imul r1, sizeof(Bucket) 12639 |.endif 12640 | cmp r0, r1 12641 | jae >9 12642 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12643 | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12644 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12645 | // (EXPECTED(p->key == varname)) 12646 | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], varname, r1 12647 | jne >9 12648 | GET_Z_PTR r0, r0 12649 | GC_ADDREF r0 12650 |1: 12651 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12652 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12653 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12654 | IF_ZVAL_REFCOUNTED op1_addr, >2 12655 |.cold_code 12656 |2: 12657 } 12658 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12659 | GET_ZVAL_PTR FCARG1a, op1_addr 12660 | // ZVAL_REF(variable_ptr, ref) 12661 | SET_ZVAL_PTR op1_addr, r0 12662 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12663 | // if (GC_DELREF(garbage) == 0) 12664 | GC_DELREF FCARG1a 12665 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12666 | jnz >3 12667 } else { 12668 | jnz >5 12669 } 12670 | ZVAL_DTOR_FUNC op1_info, opline 12671 | jmp >5 12672 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12673 |3: 12674 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12675 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12676 | EXT_CALL gc_possible_root, r1 12677 | jmp >5 12678 } 12679 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12680 |.code 12681 } 12682 } 12683 12684 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12685 | // ZVAL_REF(variable_ptr, ref) 12686 | SET_ZVAL_PTR op1_addr, r0 12687 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12688 } 12689 |5: 12690 //END of handler 12691 12692 |.cold_code 12693 |9: 12694 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12695 | mov FCARG2a, EX->run_time_cache 12696 if (opline->extended_value) { 12697 | add FCARG2a, opline->extended_value 12698 } 12699 | EXT_CALL zend_jit_fetch_global_helper, r0 12700 | jmp <1 12701 |.code 12702 12703 return 1; 12704} 12705 12706static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, zend_bool check_exception) 12707{ 12708 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12709 zend_bool in_cold = 0; 12710 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12711 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1a : ZREG_R0; 12712 12713 if (ZEND_ARG_SEND_MODE(arg_info)) { 12714 if (opline->opcode == ZEND_RECV_INIT) { 12715 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12716 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12717 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12718 } else { 12719 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12720 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12721 } 12722 } 12723 12724 if (type_mask != 0) { 12725 if (is_power_of_two(type_mask)) { 12726 uint32_t type_code = concrete_type(type_mask); 12727 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12728 } else { 12729 | mov edx, 1 12730 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12731 | shl edx, cl 12732 | test edx, type_mask 12733 | je >1 12734 } 12735 12736 |.cold_code 12737 |1: 12738 12739 in_cold = 1; 12740 } 12741 12742 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 12743 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12744 } 12745 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12746 | SET_EX_OPLINE opline, r0 12747 } else { 12748 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12749 } 12750 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12751 | EXT_CALL zend_jit_verify_arg_slow, r0 12752 12753 if (check_exception) { 12754 | test al, al 12755 if (in_cold) { 12756 | jnz >1 12757 | jmp ->exception_handler 12758 |.code 12759 |1: 12760 } else { 12761 | jz ->exception_handler 12762 } 12763 } else if (in_cold) { 12764 | jmp >1 12765 |.code 12766 |1: 12767 } 12768 12769 return 1; 12770} 12771 12772static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12773{ 12774 uint32_t arg_num = opline->op1.num; 12775 zend_arg_info *arg_info = NULL; 12776 12777 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12778 if (EXPECTED(arg_num <= op_array->num_args)) { 12779 arg_info = &op_array->arg_info[arg_num-1]; 12780 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12781 arg_info = &op_array->arg_info[op_array->num_args]; 12782 } 12783 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12784 arg_info = NULL; 12785 } 12786 } 12787 12788 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12789 | cmp dword EX->This.u2.num_args, arg_num 12790 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12791 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12792 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12793 12794 if (!exit_addr) { 12795 return 0; 12796 } 12797 | jb &exit_addr 12798 } else { 12799 | jb >1 12800 |.cold_code 12801 |1: 12802 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12803 | SET_EX_OPLINE opline, r0 12804 } else { 12805 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12806 } 12807 | mov FCARG1a, FP 12808 | EXT_CALL zend_missing_arg_error, r0 12809 | jmp ->exception_handler 12810 |.code 12811 } 12812 } 12813 12814 if (arg_info) { 12815 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12816 return 0; 12817 } 12818 } 12819 12820 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12821 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12822 | LOAD_IP_ADDR (opline + 1) 12823 zend_jit_set_last_valid_opline(opline + 1); 12824 } 12825 } 12826 12827 return 1; 12828} 12829 12830static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool is_last, int may_throw) 12831{ 12832 uint32_t arg_num = opline->op1.num; 12833 zval *zv = RT_CONSTANT(opline, opline->op2); 12834 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12835 12836 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12837 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12838 | cmp dword EX->This.u2.num_args, arg_num 12839 | jae >5 12840 } 12841 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12842 if (Z_REFCOUNTED_P(zv)) { 12843 | ADDREF_CONST zv, r0 12844 } 12845 12846 if (Z_CONSTANT_P(zv)) { 12847 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12848 | SET_EX_OPLINE opline, r0 12849 } else { 12850 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12851 } 12852 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12853 | mov r0, EX->func 12854 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12855 | .if X64 12856 | EXT_CALL zval_update_constant_ex, r0 12857 | .else 12858 ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) 12859 | EXT_CALL zval_jit_update_constant_ex, r0 12860 ||#else 12861 | EXT_CALL zval_update_constant_ex, r0 12862 ||#endif 12863 | .endif 12864 | test al, al 12865 | jnz >1 12866 |.cold_code 12867 |1: 12868 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12869 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12870 | jmp ->exception_handler 12871 |.code 12872 } 12873 12874 |5: 12875 12876 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12877 do { 12878 zend_arg_info *arg_info; 12879 12880 if (arg_num <= op_array->num_args) { 12881 arg_info = &op_array->arg_info[arg_num-1]; 12882 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12883 arg_info = &op_array->arg_info[op_array->num_args]; 12884 } else { 12885 break; 12886 } 12887 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12888 break; 12889 } 12890 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12891 return 0; 12892 } 12893 } while (0); 12894 } 12895 12896 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12897 if (is_last) { 12898 | LOAD_IP_ADDR (opline + 1) 12899 zend_jit_set_last_valid_opline(opline + 1); 12900 } 12901 } 12902 12903 return 1; 12904} 12905 12906static zend_property_info* zend_get_known_property_info(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) 12907{ 12908 zend_property_info *info = NULL; 12909 12910 if (!ce || 12911 !(ce->ce_flags & ZEND_ACC_LINKED) || 12912 (ce->ce_flags & ZEND_ACC_TRAIT) || 12913 ce->create_object) { 12914 return NULL; 12915 } 12916 12917 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12918 if (ce->info.user.filename != filename) { 12919 /* class declaration might be changed independently */ 12920 return NULL; 12921 } 12922 12923 if (ce->parent) { 12924 zend_class_entry *parent = ce->parent; 12925 12926 do { 12927 if (parent->type == ZEND_INTERNAL_CLASS) { 12928 break; 12929 } else if (parent->info.user.filename != filename) { 12930 /* some of parents class declarations might be changed independently */ 12931 /* TODO: this check may be not enough, because even 12932 * in the same it's possible to conditionally define 12933 * few classes with the same name, and "parent" may 12934 * change from request to request. 12935 */ 12936 return NULL; 12937 } 12938 parent = parent->parent; 12939 } while (parent); 12940 } 12941 } 12942 12943 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); 12944 if (info == NULL || 12945 !IS_VALID_PROPERTY_OFFSET(info->offset) || 12946 (info->flags & ZEND_ACC_STATIC)) { 12947 return NULL; 12948 } 12949 12950 if (!(info->flags & ZEND_ACC_PUBLIC) && 12951 (!on_this || info->ce != ce)) { 12952 return NULL; 12953 } 12954 12955 return info; 12956} 12957 12958static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) 12959{ 12960 zend_property_info *info; 12961 12962 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { 12963 return 1; 12964 } 12965 12966 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12967 if (ce->info.user.filename != filename) { 12968 /* class declaration might be changed independently */ 12969 return 1; 12970 } 12971 } 12972 12973 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); 12974 if (info == NULL || 12975 !IS_VALID_PROPERTY_OFFSET(info->offset) || 12976 (info->flags & ZEND_ACC_STATIC)) { 12977 return 1; 12978 } 12979 12980 if (!(info->flags & ZEND_ACC_PUBLIC) && 12981 (!on_this || info->ce != ce)) { 12982 return 1; 12983 } 12984 12985 return 0; 12986} 12987 12988static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12989{ 12990 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12991 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12992 12993 if (!exit_addr) { 12994 return 0; 12995 } 12996 12997 |.if X64 12998 || if (!IS_SIGNED_32BIT(ce)) { 12999 | mov64 r0, ((ptrdiff_t)ce) 13000 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 13001 || } else { 13002 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13003 || } 13004 |.else 13005 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13006 |.endif 13007 | jne &exit_addr 13008 13009 return 1; 13010} 13011 13012static int zend_jit_fetch_obj(dasm_State **Dst, 13013 const zend_op *opline, 13014 const zend_op_array *op_array, 13015 zend_ssa *ssa, 13016 const zend_ssa_op *ssa_op, 13017 uint32_t op1_info, 13018 zend_jit_addr op1_addr, 13019 zend_bool op1_indirect, 13020 zend_class_entry *ce, 13021 zend_bool ce_is_instanceof, 13022 zend_bool use_this, 13023 zend_bool op1_avoid_refcounting, 13024 zend_class_entry *trace_ce, 13025 int may_throw) 13026{ 13027 zval *member; 13028 zend_property_info *prop_info; 13029 zend_bool may_be_dynamic = 1; 13030 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13031 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13032 zend_jit_addr prop_addr; 13033 uint32_t res_info = RES_INFO(); 13034 13035 ZEND_ASSERT(opline->op2_type == IS_CONST); 13036 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13037 13038 member = RT_CONSTANT(opline, opline->op2); 13039 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13040 prop_info = zend_get_known_property_info(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13041 13042 if (opline->op1_type == IS_UNUSED || use_this) { 13043 | GET_ZVAL_PTR FCARG1a, this_addr 13044 } else { 13045 if (opline->op1_type == IS_VAR 13046 && opline->opcode == ZEND_FETCH_OBJ_W 13047 && (op1_info & MAY_BE_INDIRECT) 13048 && Z_REG(op1_addr) == ZREG_FP) { 13049 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13050 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13051 | GET_Z_PTR FCARG1a, FCARG1a 13052 |1: 13053 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13054 } 13055 if (op1_info & MAY_BE_REF) { 13056 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13057 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13058 } 13059 | ZVAL_DEREF FCARG1a, op1_info 13060 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13061 } 13062 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13063 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13064 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13065 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13066 13067 if (!exit_addr) { 13068 return 0; 13069 } 13070 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13071 } else { 13072 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 13073 } 13074 } 13075 | GET_ZVAL_PTR FCARG1a, op1_addr 13076 } 13077 13078 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13079 prop_info = zend_get_known_property_info(trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13080 if (prop_info) { 13081 ce = trace_ce; 13082 ce_is_instanceof = 0; 13083 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13084 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13085 return 0; 13086 } 13087 if (ssa->var_info && ssa_op->op1_use >= 0) { 13088 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13089 ssa->var_info[ssa_op->op1_use].ce = ce; 13090 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13091 } 13092 } 13093 } 13094 } 13095 13096 if (!prop_info) { 13097 | mov r0, EX->run_time_cache 13098 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 13099 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13100 | jne >5 13101 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 13102 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13103 if (may_be_dynamic) { 13104 | test r0, r0 13105 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13106 | jl >5 13107 } else { 13108 | jl >8 // dynamic property 13109 } 13110 } 13111 | mov edx, dword [FCARG1a + r0 + 8] 13112 | IF_UNDEF dl, >5 13113 | add FCARG1a, r0 13114 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13115 if (opline->opcode == ZEND_FETCH_OBJ_W 13116 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13117 && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { 13118 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13119 13120 | mov r0, EX->run_time_cache 13121 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 13122 | test FCARG2a, FCARG2a 13123 | jnz >1 13124 |.cold_code 13125 |1: 13126 if (flags == ZEND_FETCH_DIM_WRITE) { 13127 | SET_EX_OPLINE opline, r0 13128 | EXT_CALL zend_jit_check_array_promotion, r0 13129 | jmp >9 13130 } else if (flags == ZEND_FETCH_REF) { 13131 |.if X64 13132 | LOAD_ZVAL_ADDR CARG3, res_addr 13133 |.else 13134 | sub r4, 12 13135 | PUSH_ZVAL_ADDR res_addr, r0 13136 |.endif 13137 | EXT_CALL zend_jit_create_typed_ref, r0 13138 |.if not(X64) 13139 | add r4, 12 13140 |.endif 13141 | jmp >9 13142 } else { 13143 ZEND_UNREACHABLE(); 13144 } 13145 |.code 13146 } 13147 } else { 13148 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13149 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13150 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13151 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 13152 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 13153 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13154 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13155 13156 if (!exit_addr) { 13157 return 0; 13158 } 13159 | IF_UNDEF dl, &exit_addr 13160 } 13161 } else { 13162 | IF_UNDEF dl, >5 13163 } 13164 if (opline->opcode == ZEND_FETCH_OBJ_W 13165 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13166 && ZEND_TYPE_IS_SET(prop_info->type)) { 13167 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13168 13169 if (flags == ZEND_FETCH_DIM_WRITE) { 13170 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 13171 | cmp dl, IS_FALSE 13172 | jle >1 13173 |.cold_code 13174 |1: 13175 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13176 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13177 } 13178 | LOAD_ADDR FCARG2a, prop_info 13179 | SET_EX_OPLINE opline, r0 13180 | EXT_CALL zend_jit_check_array_promotion, r0 13181 | jmp >9 13182 |.code 13183 } 13184 } else if (flags == ZEND_FETCH_REF) { 13185 | IF_TYPE dl, IS_REFERENCE, >1 13186 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13187 | LOAD_ADDR FCARG2a, prop_info 13188 } else { 13189 int prop_info_offset = 13190 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13191 13192 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13193 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13194 | mov FCARG2a, aword[r0 + prop_info_offset] 13195 } 13196 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13197 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13198 } 13199 |.if X64 13200 | LOAD_ZVAL_ADDR CARG3, res_addr 13201 |.else 13202 | sub r4, 12 13203 | PUSH_ZVAL_ADDR res_addr, r0 13204 |.endif 13205 | EXT_CALL zend_jit_create_typed_ref, r0 13206 |.if not(X64) 13207 | add r4, 12 13208 |.endif 13209 | jmp >9 13210 |1: 13211 } else { 13212 ZEND_UNREACHABLE(); 13213 } 13214 } 13215 } 13216 if (op1_avoid_refcounting) { 13217 SET_STACK_REG(JIT_G(current_frame)->stack, 13218 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13219 } 13220 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13221 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13222 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13223 } 13224 | SET_ZVAL_PTR res_addr, FCARG1a 13225 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13226 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13227 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13228 } 13229 } else { 13230 zend_bool result_avoid_refcounting = 0; 13231 13232 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13233 uint32_t flags = 0; 13234 uint32_t old_info; 13235 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13236 int32_t exit_point; 13237 const void *exit_addr; 13238 zend_uchar type; 13239 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13240 13241 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13242 && !use_this 13243 && !op1_avoid_refcounting) { 13244 flags = ZEND_JIT_EXIT_FREE_OP1; 13245 } 13246 13247 | LOAD_ZVAL_ADDR r0, prop_addr 13248 13249 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13250 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13251 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13252 && (ssa_op+1)->op1_use == ssa_op->result_def 13253 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 13254 result_avoid_refcounting = 1; 13255 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13256 } 13257 13258 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13259 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13260 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 13261 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13262 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13263 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13264 if (!exit_addr) { 13265 return 0; 13266 } 13267 13268 res_info &= ~MAY_BE_GUARD; 13269 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13270 type = concrete_type(res_info); 13271 13272 | // ZVAL_DEREF() 13273 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13274 | GET_Z_PTR r0, r0 13275 | add r0, offsetof(zend_reference, val) 13276 if (type < IS_STRING) { 13277 |1: 13278 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13279 } else { 13280 | GET_ZVAL_TYPE_INFO edx, val_addr 13281 |1: 13282 | IF_NOT_TYPE dl, type, &exit_addr 13283 } 13284 | // ZVAL_COPY 13285 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13286 if (type < IS_STRING) { 13287 if (Z_REG(res_addr) != ZREG_FP || 13288 JIT_G(current_frame) == NULL || 13289 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13290 | SET_ZVAL_TYPE_INFO res_addr, type 13291 } 13292 } else { 13293 | SET_ZVAL_TYPE_INFO res_addr, edx 13294 if (!result_avoid_refcounting) { 13295 | TRY_ADDREF res_info, dh, r1 13296 } 13297 } 13298 } else { 13299 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13300 return 0; 13301 } 13302 } 13303 } 13304 13305 |.cold_code 13306 13307 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13308 |5: 13309 | SET_EX_OPLINE opline, r0 13310 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13311 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13312 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13313 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13314 } else { 13315 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13316 } 13317 | jmp >9 13318 } 13319 13320 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13321 |7: 13322 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13323 | SET_EX_OPLINE opline, r0 13324 if (opline->opcode != ZEND_FETCH_OBJ_W 13325 && (op1_info & MAY_BE_UNDEF)) { 13326 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13327 13328 if (op1_info & MAY_BE_ANY) { 13329 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13330 } 13331 | mov FCARG1d, opline->op1.var 13332 | EXT_CALL zend_jit_undefined_op_helper, r0 13333 |1: 13334 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13335 } else if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13336 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13337 } 13338 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13339 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13340 | EXT_CALL zend_jit_invalid_property_write, r0 13341 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13342 } else { 13343 | EXT_CALL zend_jit_invalid_property_read, r0 13344 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13345 } 13346 | jmp >9 13347 } else { 13348 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13349 | jmp >9 13350 } 13351 } 13352 13353 if (!prop_info 13354 && may_be_dynamic 13355 && opline->opcode != ZEND_FETCH_OBJ_W) { 13356 |8: 13357 | mov FCARG2a, r0 13358 | SET_EX_OPLINE opline, r0 13359 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13360 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13361 } else { 13362 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13363 } 13364 | jmp >9 13365 } 13366 13367 |.code; 13368 |9: // END 13369 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 13370 if (opline->op1_type == IS_VAR 13371 && opline->opcode == ZEND_FETCH_OBJ_W 13372 && (op1_info & MAY_BE_RC1)) { 13373 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13374 13375 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13376 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13377 | GC_DELREF FCARG1a 13378 | jnz >1 13379 | SET_EX_OPLINE opline, r0 13380 | EXT_CALL zend_jit_extract_helper, r0 13381 |1: 13382 } else if (!op1_avoid_refcounting) { 13383 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13384 } 13385 } 13386 13387 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13388 && prop_info 13389 && (opline->opcode != ZEND_FETCH_OBJ_W || 13390 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13391 !ZEND_TYPE_IS_SET(prop_info->type)) 13392 && opline->op1_type != IS_VAR 13393 && opline->op1_type != IS_TMP_VAR) { 13394 may_throw = 0; 13395 } 13396 13397 if (may_throw) { 13398 if (!zend_jit_check_exception(Dst)) { 13399 return 0; 13400 } 13401 } 13402 13403 return 1; 13404} 13405 13406static int zend_jit_incdec_obj(dasm_State **Dst, 13407 const zend_op *opline, 13408 const zend_op_array *op_array, 13409 zend_ssa *ssa, 13410 const zend_ssa_op *ssa_op, 13411 uint32_t op1_info, 13412 zend_jit_addr op1_addr, 13413 zend_bool op1_indirect, 13414 zend_class_entry *ce, 13415 zend_bool ce_is_instanceof, 13416 zend_bool use_this, 13417 zend_class_entry *trace_ce, 13418 int may_throw) 13419{ 13420 zval *member; 13421 zend_string *name; 13422 zend_property_info *prop_info; 13423 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13424 zend_jit_addr res_addr = 0; 13425 zend_jit_addr prop_addr; 13426 zend_bool needs_slow_path = 0; 13427 13428 ZEND_ASSERT(opline->op2_type == IS_CONST); 13429 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13430 13431 if (opline->result_type != IS_UNUSED) { 13432 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13433 } 13434 13435 member = RT_CONSTANT(opline, opline->op2); 13436 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13437 name = Z_STR_P(member); 13438 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13439 13440 if (opline->op1_type == IS_UNUSED || use_this) { 13441 | GET_ZVAL_PTR FCARG1a, this_addr 13442 } else { 13443 if (opline->op1_type == IS_VAR 13444 && (op1_info & MAY_BE_INDIRECT) 13445 && Z_REG(op1_addr) == ZREG_FP) { 13446 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13447 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13448 | GET_Z_PTR FCARG1a, FCARG1a 13449 |1: 13450 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13451 } 13452 if (op1_info & MAY_BE_REF) { 13453 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13454 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13455 } 13456 | ZVAL_DEREF FCARG1a, op1_info 13457 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13458 } 13459 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13460 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13461 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13462 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13463 13464 if (!exit_addr) { 13465 return 0; 13466 } 13467 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13468 } else { 13469 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13470 |.cold_code 13471 |1: 13472 | SET_EX_OPLINE opline, r0 13473 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13474 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13475 } 13476 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13477 | EXT_CALL zend_jit_invalid_property_incdec, r0 13478 | jmp ->exception_handler 13479 |.code 13480 } 13481 } 13482 | GET_ZVAL_PTR FCARG1a, op1_addr 13483 } 13484 13485 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13486 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13487 if (prop_info) { 13488 ce = trace_ce; 13489 ce_is_instanceof = 0; 13490 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13491 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13492 return 0; 13493 } 13494 if (ssa->var_info && ssa_op->op1_use >= 0) { 13495 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13496 ssa->var_info[ssa_op->op1_use].ce = ce; 13497 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13498 } 13499 if (ssa->var_info && ssa_op->op1_def >= 0) { 13500 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13501 ssa->var_info[ssa_op->op1_def].ce = ce; 13502 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13503 } 13504 } 13505 } 13506 } 13507 13508 if (!prop_info) { 13509 needs_slow_path = 1; 13510 13511 | mov r0, EX->run_time_cache 13512 | mov r2, aword [r0 + opline->extended_value] 13513 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13514 | jne >7 13515 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13516 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13517 | jnz >7 13518 } 13519 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13520 | test r0, r0 13521 | jl >7 13522 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13523 | add FCARG1a, r0 13524 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13525 } else { 13526 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13527 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13528 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13529 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13530 13531 if (!exit_addr) { 13532 return 0; 13533 } 13534 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13535 } else { 13536 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13537 needs_slow_path = 1; 13538 } 13539 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13540 | SET_EX_OPLINE opline, r0 13541 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13542 | LOAD_ADDR FCARG2a, prop_info 13543 } else { 13544 int prop_info_offset = 13545 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13546 13547 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13548 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13549 | mov FCARG2a, aword[r0 + prop_info_offset] 13550 } 13551 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13552 if (opline->result_type == IS_UNUSED) { 13553 switch (opline->opcode) { 13554 case ZEND_PRE_INC_OBJ: 13555 case ZEND_POST_INC_OBJ: 13556 | EXT_CALL zend_jit_inc_typed_prop, r0 13557 break; 13558 case ZEND_PRE_DEC_OBJ: 13559 case ZEND_POST_DEC_OBJ: 13560 | EXT_CALL zend_jit_dec_typed_prop, r0 13561 break; 13562 default: 13563 ZEND_UNREACHABLE(); 13564 } 13565 } else { 13566 |.if X64 13567 | LOAD_ZVAL_ADDR CARG3, res_addr 13568 |.else 13569 | sub r4, 12 13570 | PUSH_ZVAL_ADDR res_addr, r0 13571 |.endif 13572 switch (opline->opcode) { 13573 case ZEND_PRE_INC_OBJ: 13574 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13575 break; 13576 case ZEND_PRE_DEC_OBJ: 13577 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13578 break; 13579 case ZEND_POST_INC_OBJ: 13580 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13581 break; 13582 case ZEND_POST_DEC_OBJ: 13583 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13584 break; 13585 default: 13586 ZEND_UNREACHABLE(); 13587 } 13588 |.if not(X64) 13589 | add r4, 12 13590 |.endif 13591 } 13592 } 13593 } 13594 13595 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13596 zend_jit_addr var_addr = prop_addr; 13597 13598 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13599 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13600 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13601 } 13602 13603 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13604 | GET_ZVAL_PTR FCARG1a, var_addr 13605 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13606 | jnz >1 13607 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13608 |.cold_code 13609 |1: 13610 if (opline) { 13611 | SET_EX_OPLINE opline, r0 13612 } 13613 if (opline->result_type == IS_UNUSED) { 13614 | xor FCARG2a, FCARG2a 13615 } else { 13616 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13617 } 13618 switch (opline->opcode) { 13619 case ZEND_PRE_INC_OBJ: 13620 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13621 break; 13622 case ZEND_PRE_DEC_OBJ: 13623 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13624 break; 13625 case ZEND_POST_INC_OBJ: 13626 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13627 break; 13628 case ZEND_POST_DEC_OBJ: 13629 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13630 break; 13631 default: 13632 ZEND_UNREACHABLE(); 13633 } 13634 | jmp >9 13635 |.code 13636 13637 |2: 13638 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13639 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13640 if (opline->result_type != IS_UNUSED) { 13641 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13642 } 13643 } 13644 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13645 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13646 } else { 13647 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13648 } 13649 | jo >3 13650 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13651 if (opline->result_type != IS_UNUSED) { 13652 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13653 } 13654 } 13655 |.cold_code 13656 |2: 13657 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13658 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13659 | TRY_ADDREF MAY_BE_ANY, ah, r2 13660 } 13661 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13662 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13663 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13664 | EXT_CALL zend_jit_pre_inc, r0 13665 } else { 13666 | EXT_CALL increment_function, r0 13667 } 13668 } else { 13669 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13670 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13671 | EXT_CALL zend_jit_pre_dec, r0 13672 } else { 13673 | EXT_CALL decrement_function, r0 13674 } 13675 } 13676 | jmp >4 13677 13678 |3: 13679 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13680 |.if X64 13681 | mov64 rax, 0x43e0000000000000 13682 | SET_ZVAL_LVAL var_addr, rax 13683 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13684 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13685 | SET_ZVAL_LVAL res_addr, rax 13686 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13687 } 13688 |.else 13689 | SET_ZVAL_LVAL var_addr, 0 13690 | SET_ZVAL_W2 var_addr, 0x41e00000 13691 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13692 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13693 | SET_ZVAL_LVAL res_addr, 0 13694 | SET_ZVAL_W2 res_addr, 0x41e00000 13695 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13696 } 13697 |.endif 13698 } else { 13699 |.if X64 13700 | mov64 rax, 0xc3e0000000000000 13701 | SET_ZVAL_LVAL var_addr, rax 13702 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13703 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13704 | SET_ZVAL_LVAL res_addr, rax 13705 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13706 } 13707 |.else 13708 | SET_ZVAL_LVAL var_addr, 0x00200000 13709 | SET_ZVAL_W2 var_addr, 0xc1e00000 13710 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13711 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13712 | SET_ZVAL_LVAL res_addr, 0x00200000 13713 | SET_ZVAL_W2 res_addr, 0xc1e00000 13714 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13715 } 13716 |.endif 13717 } 13718 | jmp >4 13719 |.code 13720 |4: 13721 } 13722 13723 if (needs_slow_path) { 13724 |.cold_code 13725 |7: 13726 | SET_EX_OPLINE opline, r0 13727 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13728 | LOAD_ADDR FCARG2a, name 13729 |.if X64 13730 | mov CARG3, EX->run_time_cache 13731 | add CARG3, opline->extended_value 13732 if (opline->result_type == IS_UNUSED) { 13733 | xor CARG4, CARG4 13734 } else { 13735 | LOAD_ZVAL_ADDR CARG4, res_addr 13736 } 13737 |.else 13738 | sub r4, 8 13739 if (opline->result_type == IS_UNUSED) { 13740 | push 0 13741 } else { 13742 | PUSH_ZVAL_ADDR res_addr, r0 13743 } 13744 | mov r0, EX->run_time_cache 13745 | add r0, opline->extended_value 13746 | push r0 13747 |.endif 13748 13749 switch (opline->opcode) { 13750 case ZEND_PRE_INC_OBJ: 13751 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13752 break; 13753 case ZEND_PRE_DEC_OBJ: 13754 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13755 break; 13756 case ZEND_POST_INC_OBJ: 13757 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13758 break; 13759 case ZEND_POST_DEC_OBJ: 13760 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13761 break; 13762 default: 13763 ZEND_UNREACHABLE(); 13764 } 13765 13766 |.if not(X64) 13767 | add r4, 8 13768 |.endif 13769 13770 | jmp >9 13771 |.code 13772 } 13773 13774 |9: 13775 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 13776 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13777 } 13778 13779 if (may_throw) { 13780 if (!zend_jit_check_exception(Dst)) { 13781 return 0; 13782 } 13783 } 13784 13785 return 1; 13786} 13787 13788static int zend_jit_assign_obj_op(dasm_State **Dst, 13789 const zend_op *opline, 13790 const zend_op_array *op_array, 13791 zend_ssa *ssa, 13792 const zend_ssa_op *ssa_op, 13793 uint32_t op1_info, 13794 zend_jit_addr op1_addr, 13795 uint32_t val_info, 13796 zend_ssa_range *val_range, 13797 zend_bool op1_indirect, 13798 zend_class_entry *ce, 13799 zend_bool ce_is_instanceof, 13800 zend_bool use_this, 13801 zend_class_entry *trace_ce, 13802 int may_throw) 13803{ 13804 zval *member; 13805 zend_string *name; 13806 zend_property_info *prop_info; 13807 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13808 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13809 zend_jit_addr prop_addr; 13810 zend_bool needs_slow_path = 0; 13811 binary_op_type binary_op = get_binary_op(opline->extended_value); 13812 13813 ZEND_ASSERT(opline->op2_type == IS_CONST); 13814 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13815 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13816 13817 member = RT_CONSTANT(opline, opline->op2); 13818 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13819 name = Z_STR_P(member); 13820 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13821 13822 if (opline->op1_type == IS_UNUSED || use_this) { 13823 | GET_ZVAL_PTR FCARG1a, this_addr 13824 } else { 13825 if (opline->op1_type == IS_VAR 13826 && (op1_info & MAY_BE_INDIRECT) 13827 && Z_REG(op1_addr) == ZREG_FP) { 13828 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13829 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13830 | GET_Z_PTR FCARG1a, FCARG1a 13831 |1: 13832 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13833 } 13834 if (op1_info & MAY_BE_REF) { 13835 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13836 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13837 } 13838 | ZVAL_DEREF FCARG1a, op1_info 13839 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13840 } 13841 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13842 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13843 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13844 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13845 13846 if (!exit_addr) { 13847 return 0; 13848 } 13849 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13850 } else { 13851 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13852 |.cold_code 13853 |1: 13854 | SET_EX_OPLINE opline, r0 13855 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13856 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13857 } 13858 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13859 if (op1_info & MAY_BE_UNDEF) { 13860 | EXT_CALL zend_jit_invalid_property_assign_op, r0 13861 } else { 13862 | EXT_CALL zend_jit_invalid_property_assign, r0 13863 } 13864 may_throw = 1; 13865 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13866 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13867 | jmp >8 13868 } else { 13869 | jmp >9 13870 } 13871 |.code 13872 } 13873 } 13874 | GET_ZVAL_PTR FCARG1a, op1_addr 13875 } 13876 13877 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13878 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13879 if (prop_info) { 13880 ce = trace_ce; 13881 ce_is_instanceof = 0; 13882 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13883 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13884 return 0; 13885 } 13886 if (ssa->var_info && ssa_op->op1_use >= 0) { 13887 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13888 ssa->var_info[ssa_op->op1_use].ce = ce; 13889 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13890 } 13891 if (ssa->var_info && ssa_op->op1_def >= 0) { 13892 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13893 ssa->var_info[ssa_op->op1_def].ce = ce; 13894 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13895 } 13896 } 13897 } 13898 } 13899 13900 if (!prop_info) { 13901 needs_slow_path = 1; 13902 13903 | mov r0, EX->run_time_cache 13904 | mov r2, aword [r0 + (opline+1)->extended_value] 13905 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13906 | jne >7 13907 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13908 | cmp aword [r0 + ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2], 0 13909 | jnz >7 13910 } 13911 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 13912 | test r0, r0 13913 | jl >7 13914 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13915 | add FCARG1a, r0 13916 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13917 } else { 13918 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13919 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13920 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13921 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13922 13923 if (!exit_addr) { 13924 return 0; 13925 } 13926 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13927 } else { 13928 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13929 needs_slow_path = 1; 13930 } 13931 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13932 uint32_t info = val_info; 13933 13934 if (opline) { 13935 | SET_EX_OPLINE opline, r0 13936 } 13937 13938 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 13939 |.cold_code 13940 |1: 13941 | GET_ZVAL_PTR FCARG1a, prop_addr 13942 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 13943 | LOAD_ZVAL_ADDR FCARG2a, val_addr 13944 } 13945 |.if X64 13946 | LOAD_ADDR CARG3, binary_op 13947 |.else 13948 | sub r4, 12 13949 | PUSH_ADDR binary_op, r0 13950 |.endif 13951 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 13952 |.if not(X64) 13953 | add r4, 12 13954 |.endif 13955 | jmp >9 13956 |.code 13957 13958 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13959 13960 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13961 | LOAD_ADDR FCARG2a, prop_info 13962 } else { 13963 int prop_info_offset = 13964 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13965 13966 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13967 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13968 | mov FCARG2a, aword[r0 + prop_info_offset] 13969 } 13970 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13971 |.if X64 13972 | LOAD_ZVAL_ADDR CARG3, val_addr 13973 | LOAD_ADDR CARG4, binary_op 13974 |.else 13975 | sub r4, 8 13976 | PUSH_ADDR binary_op, r0 13977 | PUSH_ZVAL_ADDR val_addr, r0 13978 |.endif 13979 13980 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 13981 13982 |.if not(X64) 13983 | add r4, 8 13984 |.endif 13985 13986 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13987 info |= MAY_BE_RC1|MAY_BE_RCN; 13988 } 13989 13990 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline 13991 } 13992 } 13993 13994 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13995 zend_jit_addr var_addr = prop_addr; 13996 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13997 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13998 13999 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14000 | LOAD_ZVAL_ADDR r0, prop_addr 14001 14002 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14003 | GET_ZVAL_PTR FCARG1a, var_addr 14004 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14005 | jnz >1 14006 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14007 |.cold_code 14008 |1: 14009 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 14010 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14011 } 14012 if (opline) { 14013 | SET_EX_OPLINE opline, r0 14014 } 14015 |.if X64 14016 | LOAD_ADDR CARG3, binary_op 14017 |.else 14018 | sub r4, 12 14019 | PUSH_ADDR binary_op, r0 14020 |.endif 14021 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14022 |.if not(X64) 14023 | add r4, 12 14024 |.endif 14025 | jmp >9 14026 |.code 14027 |2: 14028 14029 switch (opline->extended_value) { 14030 case ZEND_ADD: 14031 case ZEND_SUB: 14032 case ZEND_MUL: 14033 case ZEND_DIV: 14034 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14035 1 /* may overflow */, 0)) { 14036 return 0; 14037 } 14038 break; 14039 case ZEND_BW_OR: 14040 case ZEND_BW_AND: 14041 case ZEND_BW_XOR: 14042 case ZEND_SL: 14043 case ZEND_SR: 14044 case ZEND_MOD: 14045 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14046 IS_CV, opline->op1, var_addr, var_info, NULL, 14047 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14048 val_range, 14049 0, var_addr, var_def_info, var_info, 0)) { 14050 return 0; 14051 } 14052 break; 14053 case ZEND_CONCAT: 14054 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14055 0)) { 14056 return 0; 14057 } 14058 break; 14059 default: 14060 ZEND_UNREACHABLE(); 14061 } 14062 } 14063 14064 if (needs_slow_path) { 14065 |.cold_code 14066 |7: 14067 | SET_EX_OPLINE opline, r0 14068 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14069 | LOAD_ADDR FCARG2a, name 14070 |.if X64 14071 | LOAD_ZVAL_ADDR CARG3, val_addr 14072 | mov CARG4, EX->run_time_cache 14073 | add CARG4, (opline+1)->extended_value 14074 |.if X64WIN 14075 | LOAD_ADDR r0, binary_op 14076 | mov aword A5, r0 14077 |.else 14078 | LOAD_ADDR CARG5, binary_op 14079 |.endif 14080 |.else 14081 | sub r4, 4 14082 | PUSH_ADDR binary_op, r0 14083 | mov r0, EX->run_time_cache 14084 | add r0, (opline+1)->extended_value 14085 | push r0 14086 | PUSH_ZVAL_ADDR val_addr, r0 14087 |.endif 14088 14089 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14090 14091 |.if not(X64) 14092 | add r4, 4 14093 |.endif 14094 14095 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14096 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14097 } 14098 14099 |8: 14100 | // FREE_OP_DATA(); 14101 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14102 | jmp >9 14103 |.code 14104 } 14105 14106 |9: 14107 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 14108 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14109 } 14110 14111 if (may_throw) { 14112 if (!zend_jit_check_exception(Dst)) { 14113 return 0; 14114 } 14115 } 14116 14117 return 1; 14118} 14119 14120static int zend_jit_assign_obj(dasm_State **Dst, 14121 const zend_op *opline, 14122 const zend_op_array *op_array, 14123 zend_ssa *ssa, 14124 const zend_ssa_op *ssa_op, 14125 uint32_t op1_info, 14126 zend_jit_addr op1_addr, 14127 uint32_t val_info, 14128 zend_bool op1_indirect, 14129 zend_class_entry *ce, 14130 zend_bool ce_is_instanceof, 14131 zend_bool use_this, 14132 zend_class_entry *trace_ce, 14133 int may_throw) 14134{ 14135 zval *member; 14136 zend_string *name; 14137 zend_property_info *prop_info; 14138 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14139 zend_jit_addr res_addr = 0; 14140 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14141 zend_jit_addr prop_addr; 14142 zend_bool needs_slow_path = 0; 14143 14144 if (RETURN_VALUE_USED(opline)) { 14145 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14146 } 14147 14148 ZEND_ASSERT(opline->op2_type == IS_CONST); 14149 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14150 14151 member = RT_CONSTANT(opline, opline->op2); 14152 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14153 name = Z_STR_P(member); 14154 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 14155 14156 if (opline->op1_type == IS_UNUSED || use_this) { 14157 | GET_ZVAL_PTR FCARG1a, this_addr 14158 } else { 14159 if (opline->op1_type == IS_VAR 14160 && (op1_info & MAY_BE_INDIRECT) 14161 && Z_REG(op1_addr) == ZREG_FP) { 14162 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14163 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14164 | GET_Z_PTR FCARG1a, FCARG1a 14165 |1: 14166 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14167 } 14168 if (op1_info & MAY_BE_REF) { 14169 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 14170 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14171 } 14172 | ZVAL_DEREF FCARG1a, op1_info 14173 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14174 } 14175 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14176 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14177 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14178 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14179 14180 if (!exit_addr) { 14181 return 0; 14182 } 14183 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14184 } else { 14185 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14186 |.cold_code 14187 |1: 14188 | SET_EX_OPLINE opline, r0 14189 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 14190 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14191 } 14192 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14193 | EXT_CALL zend_jit_invalid_property_assign, r0 14194 if (RETURN_VALUE_USED(opline)) { 14195 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14196 } 14197 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14198 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14199 | jmp >7 14200 } else { 14201 | jmp >9 14202 } 14203 |.code 14204 } 14205 } 14206 | GET_ZVAL_PTR FCARG1a, op1_addr 14207 } 14208 14209 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14210 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 14211 if (prop_info) { 14212 ce = trace_ce; 14213 ce_is_instanceof = 0; 14214 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14215 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 14216 return 0; 14217 } 14218 if (ssa->var_info && ssa_op->op1_use >= 0) { 14219 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14220 ssa->var_info[ssa_op->op1_use].ce = ce; 14221 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14222 } 14223 if (ssa->var_info && ssa_op->op1_def >= 0) { 14224 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14225 ssa->var_info[ssa_op->op1_def].ce = ce; 14226 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14227 } 14228 } 14229 } 14230 } 14231 14232 if (!prop_info) { 14233 needs_slow_path = 1; 14234 14235 | mov r0, EX->run_time_cache 14236 | mov r2, aword [r0 + opline->extended_value] 14237 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14238 | jne >5 14239 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14240 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 14241 } 14242 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14243 | test r0, r0 14244 | jl >5 14245 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14246 | add FCARG1a, r0 14247 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14248 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14249 | test FCARG2a, FCARG2a 14250 | jnz >1 14251 |.cold_code 14252 |1: 14253 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14254 | SET_EX_OPLINE opline, r0 14255 |.if X64 14256 | LOAD_ZVAL_ADDR CARG3, val_addr 14257 if (RETURN_VALUE_USED(opline)) { 14258 | LOAD_ZVAL_ADDR CARG4, res_addr 14259 } else { 14260 | xor CARG4, CARG4 14261 } 14262 |.else 14263 | sub r4, 8 14264 if (RETURN_VALUE_USED(opline)) { 14265 | PUSH_ZVAL_ADDR res_addr, r0 14266 } else { 14267 | push 0 14268 } 14269 | PUSH_ZVAL_ADDR val_addr, r0 14270 |.endif 14271 14272 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14273 14274 |.if not(X64) 14275 | add r4, 8 14276 |.endif 14277 14278 if ((opline+1)->op1_type == IS_CONST) { 14279 | // TODO: ??? 14280 | // if (Z_TYPE_P(value) == orig_type) { 14281 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14282 } 14283 14284 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14285 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14286 | jmp >7 14287 } else { 14288 | jmp >9 14289 } 14290 |.code 14291 } 14292 } else { 14293 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 14294 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) { 14295 // Undefined property with magic __get()/__set() 14296 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14297 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14298 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14299 14300 if (!exit_addr) { 14301 return 0; 14302 } 14303 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14304 } else { 14305 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14306 needs_slow_path = 1; 14307 } 14308 } 14309 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14310 uint32_t info = val_info; 14311 14312 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14313 | SET_EX_OPLINE opline, r0 14314 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14315 | LOAD_ADDR FCARG2a, prop_info 14316 } else { 14317 int prop_info_offset = 14318 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14319 14320 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14321 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14322 | mov FCARG2a, aword[r0 + prop_info_offset] 14323 } 14324 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14325 |.if X64 14326 | LOAD_ZVAL_ADDR CARG3, val_addr 14327 if (RETURN_VALUE_USED(opline)) { 14328 | LOAD_ZVAL_ADDR CARG4, res_addr 14329 } else { 14330 | xor CARG4, CARG4 14331 } 14332 |.else 14333 | sub r4, 8 14334 if (RETURN_VALUE_USED(opline)) { 14335 | PUSH_ZVAL_ADDR res_addr, r0 14336 } else { 14337 | push 0 14338 } 14339 | PUSH_ZVAL_ADDR val_addr, r0 14340 |.endif 14341 14342 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14343 14344 |.if not(X64) 14345 | add r4, 8 14346 |.endif 14347 14348 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14349 info |= MAY_BE_RC1|MAY_BE_RCN; 14350 } 14351 14352 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline 14353 } 14354 } 14355 14356 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14357 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14358 if (opline->result_type == IS_UNUSED) { 14359 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14360 return 0; 14361 } 14362 } else { 14363 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14364 return 0; 14365 } 14366 } 14367 } 14368 14369 if (needs_slow_path) { 14370 |.cold_code 14371 |5: 14372 | SET_EX_OPLINE opline, r0 14373 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14374 | LOAD_ADDR FCARG2a, name 14375 |.if X64 14376 | LOAD_ZVAL_ADDR CARG3, val_addr 14377 | mov CARG4, EX->run_time_cache 14378 | add CARG4, opline->extended_value 14379 if (RETURN_VALUE_USED(opline)) { 14380 |.if X64WIN 14381 | LOAD_ZVAL_ADDR r0, res_addr 14382 | mov aword A5, r0 14383 |.else 14384 | LOAD_ZVAL_ADDR CARG5, res_addr 14385 |.endif 14386 } else { 14387 |.if X64WIN 14388 | mov aword A5, 0 14389 |.else 14390 | xor CARG5, CARG5 14391 |.endif 14392 } 14393 |.else 14394 | sub r4, 4 14395 if (RETURN_VALUE_USED(opline)) { 14396 | PUSH_ZVAL_ADDR res_addr, r0 14397 } else { 14398 | push 0 14399 } 14400 | mov r0, EX->run_time_cache 14401 | add r0, opline->extended_value 14402 | push r0 14403 | PUSH_ZVAL_ADDR val_addr, r0 14404 |.endif 14405 14406 | EXT_CALL zend_jit_assign_obj_helper, r0 14407 14408 |.if not(X64) 14409 | add r4, 4 14410 |.endif 14411 14412 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14413 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14414 } 14415 14416 |7: 14417 | // FREE_OP_DATA(); 14418 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14419 | jmp >9 14420 |.code 14421 } 14422 14423 |9: 14424 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 14425 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14426 } 14427 14428 if (may_throw) { 14429 if (!zend_jit_check_exception(Dst)) { 14430 return 0; 14431 } 14432 } 14433 14434 return 1; 14435} 14436 14437static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14438{ 14439 zend_jit_addr op1_addr = OP1_ADDR(); 14440 14441 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14442 if (may_throw) { 14443 | SET_EX_OPLINE opline, r0 14444 } 14445 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14446 if (op1_info & MAY_BE_ARRAY) { 14447 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14448 } 14449 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14450 | cmp FCARG1d, -1 14451 | je >7 14452 | EXT_CALL zend_hash_iterator_del, r0 14453 |7: 14454 } 14455 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14456 if (may_throw) { 14457 if (!zend_jit_check_exception(Dst)) { 14458 return 0; 14459 } 14460 } 14461 } 14462 14463 return 1; 14464} 14465 14466static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14467{ 14468 if (opline->op1_type == IS_CONST) { 14469 zval *zv; 14470 size_t len; 14471 14472 zv = RT_CONSTANT(opline, opline->op1); 14473 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14474 len = Z_STRLEN_P(zv); 14475 14476 if (len > 0) { 14477 const char *str = Z_STRVAL_P(zv); 14478 14479 | SET_EX_OPLINE opline, r0 14480 |.if X64 14481 | LOAD_ADDR CARG1, str 14482 | LOAD_ADDR CARG2, len 14483 | EXT_CALL zend_write, r0 14484 |.else 14485 | mov aword A2, len 14486 | mov aword A1, str 14487 | EXT_CALL zend_write, r0 14488 |.endif 14489 if (!zend_jit_check_exception(Dst)) { 14490 return 0; 14491 } 14492 } 14493 } else { 14494 zend_jit_addr op1_addr = OP1_ADDR(); 14495 14496 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14497 14498 | SET_EX_OPLINE opline, r0 14499 | GET_ZVAL_PTR r0, op1_addr 14500 |.if X64 14501 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14502 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14503 | EXT_CALL zend_write, r0 14504 |.else 14505 | add r0, offsetof(zend_string, val) 14506 | mov aword A1, r0 14507 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14508 | mov aword A2, r0 14509 | EXT_CALL zend_write, r0 14510 |.endif 14511 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14512 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14513 } 14514 if (!zend_jit_check_exception(Dst)) { 14515 return 0; 14516 } 14517 } 14518 return 1; 14519} 14520 14521static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 14522{ 14523 zend_jit_addr res_addr = RES_ADDR(); 14524 14525 if (opline->op1_type == IS_CONST) { 14526 zval *zv; 14527 size_t len; 14528 14529 zv = RT_CONSTANT(opline, opline->op1); 14530 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14531 len = Z_STRLEN_P(zv); 14532 14533 | SET_ZVAL_LVAL res_addr, len 14534 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14535 } else { 14536 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14537 14538 | GET_ZVAL_PTR r0, op1_addr 14539 | mov r0, aword [r0 + offsetof(zend_string, len)] 14540 | SET_ZVAL_LVAL res_addr, r0 14541 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14542 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14543 } 14544 return 1; 14545} 14546 14547static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14548{ 14549 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14550 14551 | mov FCARG1a, aword EX->This.value.ptr 14552 | SET_ZVAL_PTR var_addr, FCARG1a 14553 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14554 | GC_ADDREF FCARG1a 14555 14556 return 1; 14557} 14558 14559static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool check_only) 14560{ 14561 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 14562 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14563 if (!JIT_G(current_frame) || 14564 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14565 14566 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14567 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14568 14569 | cmp byte EX->This.u1.v.type, IS_OBJECT 14570 | jne &exit_addr 14571 14572 if (JIT_G(current_frame)) { 14573 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14574 } 14575 } 14576 } else { 14577 14578 | cmp byte EX->This.u1.v.type, IS_OBJECT 14579 | jne >1 14580 |.cold_code 14581 |1: 14582 | SET_EX_OPLINE opline, r0 14583 | jmp ->invalid_this 14584 |.code 14585 } 14586 } 14587 14588 if (!check_only) { 14589 if (!zend_jit_load_this(Dst, opline->result.var)) { 14590 return 0; 14591 } 14592 } 14593 14594 return 1; 14595} 14596 14597static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14598{ 14599 uint32_t count; 14600 Bucket *p; 14601 const zend_op *target; 14602 int b; 14603 int32_t exit_point; 14604 const void *exit_addr; 14605 14606 | test r0, r0 14607 if (default_label) { 14608 | jz &default_label 14609 } else if (next_opline) { 14610 | jz >3 14611 } else { 14612 | jz =>default_b 14613 } 14614 | LOAD_ADDR FCARG1a, jumptable 14615 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14616 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14617 |.if X64 14618 | cqo 14619 |.else 14620 | cdq 14621 |.endif 14622 | idiv FCARG1a 14623 |.if X64 14624 if (!IS_32BIT(dasm_end)) { 14625 | lea FCARG1a, aword [>4] 14626 | jmp aword [FCARG1a + r0] 14627 } else { 14628 | jmp aword [r0 + >4] 14629 } 14630 |.else 14631 | jmp aword [r0 + >4] 14632 |.endif 14633 |.jmp_table 14634 |.align aword 14635 |4: 14636 if (trace_info) { 14637 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14638 } 14639 14640 count = jumptable->nNumUsed; 14641 p = jumptable->arData; 14642 do { 14643 if (Z_TYPE(p->val) == IS_UNDEF) { 14644 if (default_label) { 14645 | .aword &default_label 14646 } else if (next_opline) { 14647 | .aword >3 14648 } else { 14649 | .aword =>default_b 14650 } 14651 } else { 14652 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14653 if (!next_opline) { 14654 b = ssa->cfg.map[target - op_array->opcodes]; 14655 | .aword =>b 14656 } else if (next_opline == target) { 14657 | .aword >3 14658 } else { 14659 exit_point = zend_jit_trace_get_exit_point(target, 0); 14660 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14661 | .aword &exit_addr 14662 } 14663 } 14664 p++; 14665 count--; 14666 } while (count); 14667 |.code 14668 14669 return 1; 14670} 14671 14672static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14673{ 14674 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14675 const zend_op *next_opline = NULL; 14676 14677 if (trace) { 14678 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14679 ZEND_ASSERT(trace->opline != NULL); 14680 next_opline = trace->opline; 14681 } 14682 14683 if (opline->op1_type == IS_CONST) { 14684 zval *zv = RT_CONSTANT(opline, opline->op1); 14685 zval *jump_zv = NULL; 14686 int b; 14687 14688 if (opline->opcode == ZEND_SWITCH_LONG) { 14689 if (Z_TYPE_P(zv) == IS_LONG) { 14690 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14691 } 14692 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14693 if (Z_TYPE_P(zv) == IS_STRING) { 14694 jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); 14695 } 14696 } else if (opline->opcode == ZEND_MATCH) { 14697 if (Z_TYPE_P(zv) == IS_LONG) { 14698 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14699 } else if (Z_TYPE_P(zv) == IS_STRING) { 14700 jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); 14701 } 14702 } else { 14703 ZEND_UNREACHABLE(); 14704 } 14705 if (next_opline) { 14706 const zend_op *target; 14707 14708 if (jump_zv != NULL) { 14709 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14710 } else { 14711 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14712 } 14713 ZEND_ASSERT(target == next_opline); 14714 } else { 14715 if (jump_zv != NULL) { 14716 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14717 } else { 14718 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14719 } 14720 | jmp =>b 14721 } 14722 } else { 14723 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14724 uint32_t op1_info = OP1_INFO(); 14725 zend_jit_addr op1_addr = OP1_ADDR(); 14726 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14727 const zend_op *target; 14728 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14729 int b; 14730 int32_t exit_point; 14731 const void *fallback_label = NULL; 14732 const void *default_label = NULL; 14733 const void *exit_addr; 14734 14735 if (next_opline) { 14736 if (next_opline != opline + 1) { 14737 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14738 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14739 } 14740 if (next_opline != default_opline) { 14741 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14742 default_label = zend_jit_trace_get_exit_addr(exit_point); 14743 } 14744 } 14745 14746 if (opline->opcode == ZEND_SWITCH_LONG) { 14747 if (op1_info & MAY_BE_LONG) { 14748 if (op1_info & MAY_BE_REF) { 14749 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 14750 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 14751 |.cold_code 14752 |1: 14753 | // ZVAL_DEREF(op) 14754 if (fallback_label) { 14755 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14756 } else { 14757 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14758 } 14759 | GET_ZVAL_PTR FCARG2a, op1_addr 14760 if (fallback_label) { 14761 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 14762 } else { 14763 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 14764 } 14765 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 14766 | jmp >2 14767 |.code 14768 |2: 14769 } else { 14770 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14771 if (fallback_label) { 14772 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 14773 } else { 14774 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14775 } 14776 } 14777 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 14778 } 14779 if (HT_IS_PACKED(jumptable)) { 14780 uint32_t count = jumptable->nNumUsed; 14781 Bucket *p = jumptable->arData; 14782 14783 | cmp FCARG2a, jumptable->nNumUsed 14784 if (default_label) { 14785 | jae &default_label 14786 } else if (next_opline) { 14787 | jae >3 14788 } else { 14789 | jae =>default_b 14790 } 14791 |.if X64 14792 if (!IS_32BIT(dasm_end)) { 14793 | lea r0, aword [>4] 14794 | jmp aword [r0 + FCARG2a * 8] 14795 } else { 14796 | jmp aword [FCARG2a * 8 + >4] 14797 } 14798 |.else 14799 | jmp aword [FCARG2a * 4 + >4] 14800 |.endif 14801 |.jmp_table 14802 |.align aword 14803 |4: 14804 if (trace_info) { 14805 trace_info->jmp_table_size += count; 14806 } 14807 p = jumptable->arData; 14808 do { 14809 if (Z_TYPE(p->val) == IS_UNDEF) { 14810 if (default_label) { 14811 | .aword &default_label 14812 } else if (next_opline) { 14813 | .aword >3 14814 } else { 14815 | .aword =>default_b 14816 } 14817 } else { 14818 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14819 if (!next_opline) { 14820 b = ssa->cfg.map[target - op_array->opcodes]; 14821 | .aword =>b 14822 } else if (next_opline == target) { 14823 | .aword >3 14824 } else { 14825 exit_point = zend_jit_trace_get_exit_point(target, 0); 14826 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14827 | .aword &exit_addr 14828 } 14829 } 14830 p++; 14831 count--; 14832 } while (count); 14833 |.code 14834 |3: 14835 } else { 14836 | LOAD_ADDR FCARG1a, jumptable 14837 | EXT_CALL zend_hash_index_find, r0 14838 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14839 return 0; 14840 } 14841 |3: 14842 } 14843 } 14844 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14845 if (op1_info & MAY_BE_STRING) { 14846 if (op1_info & MAY_BE_REF) { 14847 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 14848 | GET_ZVAL_PTR FCARG2a, op1_addr 14849 |.cold_code 14850 |1: 14851 | // ZVAL_DEREF(op) 14852 if (fallback_label) { 14853 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14854 } else { 14855 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14856 } 14857 | GET_ZVAL_PTR FCARG2a, op1_addr 14858 if (fallback_label) { 14859 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 14860 } else { 14861 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 14862 } 14863 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 14864 | jmp >2 14865 |.code 14866 |2: 14867 } else { 14868 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14869 if (fallback_label) { 14870 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 14871 } else { 14872 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 14873 } 14874 } 14875 | GET_ZVAL_PTR FCARG2a, op1_addr 14876 } 14877 | LOAD_ADDR FCARG1a, jumptable 14878 | EXT_CALL zend_hash_find, r0 14879 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14880 return 0; 14881 } 14882 |3: 14883 } 14884 } else if (opline->opcode == ZEND_MATCH) { 14885 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14886 if (op1_info & MAY_BE_REF) { 14887 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 14888 | ZVAL_DEREF FCARG2a, op1_info 14889 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 14890 } 14891 | LOAD_ADDR FCARG1a, jumptable 14892 if (op1_info & MAY_BE_LONG) { 14893 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14894 if (op1_info & MAY_BE_STRING) { 14895 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 14896 } else if (op1_info & MAY_BE_UNDEF) { 14897 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 14898 } else if (default_label) { 14899 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 14900 } else if (next_opline) { 14901 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14902 } else { 14903 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 14904 } 14905 } 14906 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 14907 | EXT_CALL zend_hash_index_find, r0 14908 if (op1_info & MAY_BE_STRING) { 14909 | jmp >2 14910 } 14911 } 14912 if (op1_info & MAY_BE_STRING) { 14913 |5: 14914 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 14915 if (op1_info & MAY_BE_UNDEF) { 14916 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 14917 } else if (default_label) { 14918 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 14919 } else if (next_opline) { 14920 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 14921 } else { 14922 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 14923 } 14924 } 14925 | GET_ZVAL_PTR FCARG2a, op1_addr 14926 | EXT_CALL zend_hash_find, r0 14927 } 14928 |2: 14929 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14930 return 0; 14931 } 14932 } 14933 if (op1_info & MAY_BE_UNDEF) { 14934 |6: 14935 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 14936 if (default_label) { 14937 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 14938 } else if (next_opline) { 14939 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 14940 } else { 14941 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 14942 } 14943 } 14944 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 14945 | SET_EX_OPLINE opline, r0 14946 | mov FCARG1d, opline->op1.var 14947 | EXT_CALL zend_jit_undefined_op_helper, r0 14948 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 14949 return 0; 14950 } 14951 } 14952 if (default_label) { 14953 | jmp &default_label 14954 } else if (next_opline) { 14955 | jmp >3 14956 } else { 14957 | jmp =>default_b 14958 } 14959 |3: 14960 } else { 14961 ZEND_UNREACHABLE(); 14962 } 14963 } 14964 return 1; 14965} 14966 14967static zend_bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 14968{ 14969 zend_arg_info *arg_info = &op_array->arg_info[-1]; 14970 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 14971 zend_jit_addr op1_addr = OP1_ADDR(); 14972 zend_bool needs_slow_check = 1; 14973 zend_bool slow_check_in_cold = 1; 14974 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 14975 14976 if (type_mask == 0) { 14977 slow_check_in_cold = 0; 14978 } else { 14979 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 14980 slow_check_in_cold = 0; 14981 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 14982 needs_slow_check = 0; 14983 } else if (is_power_of_two(type_mask)) { 14984 uint32_t type_code = concrete_type(type_mask); 14985 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 14986 } else { 14987 | mov edx, 1 14988 | GET_ZVAL_TYPE cl, op1_addr 14989 | shl edx, cl 14990 | test edx, type_mask 14991 | je >6 14992 } 14993 } 14994 if (needs_slow_check) { 14995 if (slow_check_in_cold) { 14996 |.cold_code 14997 |6: 14998 } 14999 | SET_EX_OPLINE opline, r1 15000 if (op1_info & MAY_BE_UNDEF) { 15001 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15002 | mov FCARG1a, opline->op1.var 15003 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15004 | test r0, r0 15005 | jz ->exception_handler 15006 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15007 | jmp >8 15008 } 15009 |7: 15010 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15011 |8: 15012 | mov FCARG2a, EX->func 15013 |.if X64 15014 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15015 | mov r0, EX->run_time_cache 15016 | lea CARG4, aword [r0+opline->op2.num] 15017 | EXT_CALL zend_jit_verify_return_slow, r0 15018 |.else 15019 | sub r4, 8 15020 | mov r0, EX->run_time_cache 15021 | add r0, opline->op2.num 15022 | push r0 15023 | push (ptrdiff_t)arg_info 15024 | EXT_CALL zend_jit_verify_return_slow, r0 15025 | add r4, 8 15026 |.endif 15027 if (!zend_jit_check_exception(Dst)) { 15028 return 0; 15029 } 15030 if (slow_check_in_cold) { 15031 | jmp >9 15032 |.code 15033 } 15034 } 15035 |9: 15036 return 1; 15037} 15038 15039static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15040{ 15041 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15042 15043 // TODO: support for empty() ??? 15044 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15045 15046 if (op1_info & MAY_BE_REF) { 15047 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 15048 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15049 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 15050 } 15051 | ZVAL_DEREF FCARG1a, op1_info 15052 |1: 15053 } 15054 15055 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15056 if (exit_addr) { 15057 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15058 } else if (smart_branch_opcode) { 15059 if (smart_branch_opcode == ZEND_JMPNZ) { 15060 | jmp =>target_label 15061 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15062 | jmp =>target_label2 15063 } 15064 } else { 15065 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15066 } 15067 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15068 if (exit_addr) { 15069 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15070 } else if (smart_branch_opcode) { 15071 if (smart_branch_opcode != ZEND_JMPNZ) { 15072 | jmp =>target_label 15073 } 15074 } else { 15075 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15076 } 15077 } else { 15078 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15079 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15080 if (exit_addr) { 15081 if (smart_branch_opcode == ZEND_JMPNZ) { 15082 | jg &exit_addr 15083 } else { 15084 | jle &exit_addr 15085 } 15086 } else if (smart_branch_opcode) { 15087 if (smart_branch_opcode == ZEND_JMPZ) { 15088 | jle =>target_label 15089 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15090 | jg =>target_label 15091 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15092 | jle =>target_label 15093 | jmp =>target_label2 15094 } else { 15095 ZEND_UNREACHABLE(); 15096 } 15097 } else { 15098 | setg al 15099 | movzx eax, al 15100 | lea eax, [eax + IS_FALSE] 15101 | SET_ZVAL_TYPE_INFO res_addr, eax 15102 } 15103 } 15104 15105 return 1; 15106} 15107 15108static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15109{ 15110 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15111 15112 if (opline->op1_type == IS_CONST) { 15113 zval *zv = RT_CONSTANT(opline, opline->op1); 15114 15115 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15116 if (Z_REFCOUNTED_P(zv)) { 15117 | ADDREF_CONST zv, r0 15118 } 15119 } else { 15120 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15121 15122 | // ZVAL_COPY(res, value); 15123 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1a 15124 if (opline->op1_type == IS_CV) { 15125 | TRY_ADDREF op1_info, ah, FCARG1a 15126 } 15127 } 15128 | // Z_FE_POS_P(res) = 0; 15129 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15130 15131 return 1; 15132} 15133 15134static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 15135{ 15136 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15137 15138 | // array = EX_VAR(opline->op1.var); 15139 | // fe_ht = Z_ARRVAL_P(array); 15140 | GET_ZVAL_PTR FCARG2a, op1_addr 15141 | // pos = Z_FE_POS_P(array); 15142 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15143 | // p = fe_ht->arData + pos; 15144 |.if X64 15145 || ZEND_ASSERT(sizeof(Bucket) == 32); 15146 | mov eax, FCARG1d 15147 | shl r0, 5 15148 |.else 15149 | imul r0, FCARG1a, sizeof(Bucket) 15150 |.endif 15151 | add r0, aword [FCARG2a + offsetof(zend_array, arData)] 15152 |1: 15153 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15154 | cmp dword [FCARG2a + offsetof(zend_array, nNumUsed)], FCARG1d 15155 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15156 | // ZEND_VM_CONTINUE(); 15157 if (exit_addr) { 15158 if (exit_opcode == ZEND_JMP) { 15159 | jbe &exit_addr 15160 } else { 15161 | jbe >3 15162 } 15163 } else { 15164 | jbe =>target_label 15165 } 15166 | // pos++; 15167 | add FCARG1d, 1 15168 | // value_type = Z_TYPE_INFO_P(value); 15169 | // if (EXPECTED(value_type != IS_UNDEF)) { 15170 | IF_Z_TYPE r0, IS_UNDEF, >2 15171 if (!exit_addr || exit_opcode == ZEND_JMP) { 15172 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >3 15173 } else { 15174 | IF_NOT_Z_TYPE r0, IS_INDIRECT, &exit_addr 15175 } 15176 | // value = Z_INDIRECT_P(value); 15177 | GET_Z_PTR FCARG2a, r0 15178 | // value_type = Z_TYPE_INFO_P(value); 15179 | // if (EXPECTED(value_type != IS_UNDEF)) { 15180 if (!exit_addr || exit_opcode == ZEND_JMP) { 15181 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4 15182 } else { 15183 | IF_NOT_Z_TYPE r0, IS_UNDEF, &exit_addr 15184 } 15185 | GET_ZVAL_PTR FCARG2a, op1_addr // reload 15186 |2: 15187 | // p++; 15188 | add r0, sizeof(Bucket) 15189 | jmp <1 15190 |3: 15191 15192 if (!exit_addr || exit_opcode == ZEND_JMP) { 15193 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 15194 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15195 uint32_t val_info; 15196 15197 | mov FCARG2a, r0 15198 |4: 15199 | // Z_FE_POS_P(array) = pos + 1; 15200 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], FCARG1d 15201 15202 if (RETURN_VALUE_USED(opline)) { 15203 zend_jit_addr res_addr = RES_ADDR(); 15204 15205 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15206 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15207 | // if (!p->key) { 15208 | cmp aword [r0 + offsetof(Bucket, key)], 0 15209 | jz >2 15210 } 15211 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15212 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15213 | mov FCARG1a, aword [r0 + offsetof(Bucket, key)] 15214 | SET_ZVAL_PTR res_addr, FCARG1a 15215 | test dword [FCARG1a + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15216 | jz >1 15217 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15218 | jmp >3 15219 |1: 15220 | GC_ADDREF FCARG1a 15221 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15222 15223 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15224 | jmp >3 15225 |2: 15226 } 15227 } 15228 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15229 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15230 | mov FCARG1a, aword [r0 + offsetof(Bucket, h)] 15231 | SET_ZVAL_LVAL res_addr, FCARG1a 15232 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15233 } 15234 |3: 15235 } 15236 15237 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15238 if (val_info & MAY_BE_ARRAY) { 15239 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15240 } 15241 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15242 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15243 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15244 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15245 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15246 } 15247 15248 if (opline->op2_type == IS_CV) { 15249 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15250 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15251 return 0; 15252 } 15253 } else { 15254 | // ZVAL_COPY(res, value); 15255 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1a 15256 | TRY_ADDREF val_info, ah, FCARG1a 15257 } 15258 } 15259 15260 return 1; 15261} 15262 15263static int zend_jit_fetch_constant(dasm_State **Dst, 15264 const zend_op *opline, 15265 const zend_op_array *op_array, 15266 zend_ssa *ssa, 15267 const zend_ssa_op *ssa_op) 15268{ 15269 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15270 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15271 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15272 uint32_t res_info = RES_INFO(); 15273 15274 | // c = CACHED_PTR(opline->extended_value); 15275 | mov FCARG1a, EX->run_time_cache 15276 | mov r0, aword [FCARG1a + opline->extended_value] 15277 | // if (c != NULL) 15278 | test r0, r0 15279 | jz >9 15280 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15281 | test r0, CACHE_SPECIAL 15282 | jnz >9 15283 |8: 15284 15285 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15286 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15287 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15288 int32_t exit_point; 15289 const void *exit_addr = NULL; 15290 15291 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15292 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 15293 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15294 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15295 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15296 if (!exit_addr) { 15297 return 0; 15298 } 15299 res_info &= ~MAY_BE_GUARD; 15300 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15301 15302 zend_uchar type = concrete_type(res_info); 15303 15304 if (type < IS_STRING) { 15305 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15306 } else { 15307 | GET_ZVAL_TYPE_INFO edx, const_addr 15308 | IF_NOT_TYPE dl, type, &exit_addr 15309 } 15310 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15311 if (type < IS_STRING) { 15312 | SET_ZVAL_TYPE_INFO res_addr, type 15313 } else { 15314 | SET_ZVAL_TYPE_INFO res_addr, edx 15315 | TRY_ADDREF res_info, dh, r1 15316 } 15317 } else { 15318 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15319 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15320 | TRY_ADDREF MAY_BE_ANY, ah, r1 15321 } 15322 15323 |.cold_code 15324 |9: 15325 | // SAVE_OPLINE(); 15326 | SET_EX_OPLINE opline, r0 15327 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15328 | LOAD_ADDR FCARG1a, zv 15329 | mov FCARG2a, opline->op1.num 15330 | EXT_CALL zend_jit_get_constant, r0 15331 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15332 | test r0, r0 15333 | jnz <8 15334 | jmp ->exception_handler 15335 |.code 15336 15337 return 1; 15338} 15339 15340static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15341{ 15342 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15343 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15344 15345 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15346 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15347 15348 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15349 | LOAD_ADDR FCARG1a, ht 15350 if (opline->op1_type != IS_CONST) { 15351 | GET_ZVAL_PTR FCARG2a, op1_addr 15352 | EXT_CALL zend_hash_find, r0 15353 } else { 15354 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15355 | LOAD_ADDR FCARG2a, str 15356 | EXT_CALL _zend_hash_find_known_hash, r0 15357 } 15358 | test r0, r0 15359 if (exit_addr) { 15360 if (smart_branch_opcode == ZEND_JMPZ) { 15361 | jz &exit_addr 15362 } else { 15363 | jnz &exit_addr 15364 } 15365 } else if (smart_branch_opcode) { 15366 if (smart_branch_opcode == ZEND_JMPZ) { 15367 | jz =>target_label 15368 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15369 | jnz =>target_label 15370 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15371 | jz =>target_label 15372 | jmp =>target_label2 15373 } else { 15374 ZEND_UNREACHABLE(); 15375 } 15376 } else { 15377 | setnz al 15378 | movzx eax, al 15379 | lea eax, [eax + IS_FALSE] 15380 | SET_ZVAL_TYPE_INFO res_addr, eax 15381 } 15382 15383 return 1; 15384} 15385 15386static zend_bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15387{ 15388 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15389 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15390 15391 if (!exit_addr) { 15392 return 0; 15393 } 15394 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15395 15396 return 1; 15397} 15398 15399static zend_bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_ref_guard, zend_bool add_type_guard) 15400{ 15401 zend_jit_addr var_addr = *var_addr_ptr; 15402 uint32_t var_info = *var_info_ptr; 15403 const void *exit_addr = NULL; 15404 15405 if (add_ref_guard || add_type_guard) { 15406 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15407 15408 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15409 if (!exit_addr) { 15410 return 0; 15411 } 15412 } 15413 15414 if (add_ref_guard) { 15415 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15416 } 15417 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15418 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15419 if (Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 15420 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15421 } 15422 | EXT_CALL zend_jit_unref_helper, r0 15423 } else { 15424 | GET_ZVAL_PTR FCARG1a, var_addr 15425 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); 15426 *var_addr_ptr = var_addr; 15427 } 15428 15429 if (var_type != IS_UNKNOWN) { 15430 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15431 } 15432 if (add_type_guard 15433 && var_type != IS_UNKNOWN 15434 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15435 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15436 15437 ZEND_ASSERT(var_info & (1 << var_type)); 15438 if (var_type < IS_STRING) { 15439 var_info = (1 << var_type); 15440 } else if (var_type != IS_ARRAY) { 15441 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15442 } else { 15443 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15444 } 15445 15446 *var_info_ptr = var_info; 15447 } else { 15448 var_info &= ~MAY_BE_REF; 15449 *var_info_ptr = var_info; 15450 } 15451 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15452 15453 return 1; 15454} 15455 15456static zend_bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_indirect_guard) 15457{ 15458 zend_jit_addr var_addr = *var_addr_ptr; 15459 uint32_t var_info = *var_info_ptr; 15460 int32_t exit_point; 15461 const void *exit_addr; 15462 15463 if (add_indirect_guard) { 15464 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15465 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15466 15467 if (!exit_addr) { 15468 return 0; 15469 } 15470 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15471 | GET_ZVAL_PTR FCARG1a, var_addr 15472 } else { 15473 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 15474 if (opline->op1_type != IS_VAR || 15475 (opline-1)->result_type != IS_VAR || 15476 (opline-1)->result.var != opline->op1.var || 15477 (opline-1)->op2_type == IS_VAR || 15478 (opline-1)->op2_type == IS_TMP_VAR) { 15479 | GET_ZVAL_PTR FCARG1a, var_addr 15480 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15481 | mov FCARG1a, r0 15482 } 15483 } 15484 *var_info_ptr &= ~MAY_BE_INDIRECT; 15485 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 15486 *var_addr_ptr = var_addr; 15487 15488 if (var_type != IS_UNKNOWN) { 15489 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15490 } 15491 if (!(var_type & IS_TRACE_REFERENCE) 15492 && var_type != IS_UNKNOWN 15493 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15494 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15495 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15496 15497 if (!exit_addr) { 15498 return 0; 15499 } 15500 15501 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15502 15503 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15504 ZEND_ASSERT(var_info & (1 << var_type)); 15505 if (var_type < IS_STRING) { 15506 var_info = (1 << var_type); 15507 } else if (var_type != IS_ARRAY) { 15508 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15509 } else { 15510 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15511 } 15512 15513 *var_info_ptr = var_info; 15514 } 15515 15516 return 1; 15517} 15518 15519static zend_bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15520{ 15521 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15522 return 0; 15523 } 15524 15525 switch (opline->opcode) { 15526 case ZEND_QM_ASSIGN: 15527 case ZEND_SEND_VAR: 15528 case ZEND_ASSIGN: 15529 case ZEND_PRE_INC: 15530 case ZEND_PRE_DEC: 15531 case ZEND_POST_INC: 15532 case ZEND_POST_DEC: 15533 return 1; 15534 case ZEND_ADD: 15535 case ZEND_SUB: 15536 case ZEND_MUL: 15537 case ZEND_BW_OR: 15538 case ZEND_BW_AND: 15539 case ZEND_BW_XOR: 15540 if (def_var == ssa_op->result_def && 15541 use_var == ssa_op->op1_use) { 15542 return 1; 15543 } 15544 break; 15545 default: 15546 break; 15547 } 15548 return 0; 15549} 15550 15551static zend_bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15552{ 15553 uint32_t op1_info, op2_info; 15554 15555 switch (opline->opcode) { 15556 case ZEND_SEND_VAR: 15557 case ZEND_SEND_VAL: 15558 case ZEND_SEND_VAL_EX: 15559 return (opline->op2_type != IS_CONST); 15560 case ZEND_QM_ASSIGN: 15561 case ZEND_IS_SMALLER: 15562 case ZEND_IS_SMALLER_OR_EQUAL: 15563 case ZEND_IS_EQUAL: 15564 case ZEND_IS_NOT_EQUAL: 15565 case ZEND_IS_IDENTICAL: 15566 case ZEND_IS_NOT_IDENTICAL: 15567 case ZEND_CASE: 15568 return 1; 15569 case ZEND_RETURN: 15570 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15571 case ZEND_ASSIGN: 15572 op1_info = OP1_INFO(); 15573 op2_info = OP2_INFO(); 15574 return 15575 opline->op1_type == IS_CV && 15576 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15577 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15578 case ZEND_ADD: 15579 case ZEND_SUB: 15580 case ZEND_MUL: 15581 op1_info = OP1_INFO(); 15582 op2_info = OP2_INFO(); 15583 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15584 case ZEND_BW_OR: 15585 case ZEND_BW_AND: 15586 case ZEND_BW_XOR: 15587 case ZEND_SL: 15588 case ZEND_SR: 15589 case ZEND_MOD: 15590 op1_info = OP1_INFO(); 15591 op2_info = OP2_INFO(); 15592 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15593 case ZEND_PRE_INC: 15594 case ZEND_PRE_DEC: 15595 case ZEND_POST_INC: 15596 case ZEND_POST_DEC: 15597 op1_info = OP1_INFO(); 15598 return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15599 case ZEND_JMPZ: 15600 case ZEND_JMPNZ: 15601 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15602 if (!ssa->cfg.map) { 15603 return 0; 15604 } 15605 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15606 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15607 return 0; 15608 } 15609 } 15610 /* break missing intentionally */ 15611 case ZEND_BOOL: 15612 case ZEND_BOOL_NOT: 15613 case ZEND_JMPZNZ: 15614 case ZEND_JMPZ_EX: 15615 case ZEND_JMPNZ_EX: 15616 return 1; 15617 case ZEND_FETCH_DIM_R: 15618 op1_info = OP1_INFO(); 15619 op2_info = OP2_INFO(); 15620 if (trace 15621 && trace->op1_type != IS_UNKNOWN 15622 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15623 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15624 } 15625 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15626 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15627 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15628 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15629 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15630 } 15631 return 0; 15632} 15633 15634static zend_bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15635{ 15636 if (ssa->vars[var].no_val) { 15637 /* we don't need the value */ 15638 return 0; 15639 } 15640 15641 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15642 /* Disable global register allocation, 15643 * register allocation for SSA variables connected through Phi functions 15644 */ 15645 if (ssa->vars[var].definition_phi) { 15646 return 0; 15647 } 15648 if (ssa->vars[var].phi_use_chain) { 15649 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15650 do { 15651 if (!ssa->vars[phi->ssa_var].no_val) { 15652 return 0; 15653 } 15654 phi = zend_ssa_next_use_phi(ssa, var, phi); 15655 } while (phi); 15656 } 15657 } 15658 15659 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15660 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15661 /* bad type */ 15662 return 0; 15663 } 15664 15665 return 1; 15666} 15667 15668static zend_bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15669{ 15670 if (!zend_jit_var_supports_reg(ssa, var)) { 15671 return 0; 15672 } 15673 15674 if (ssa->vars[var].definition >= 0) { 15675 uint32_t def = ssa->vars[var].definition; 15676 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15677 return 0; 15678 } 15679 } 15680 15681 if (ssa->vars[var].use_chain >= 0) { 15682 int use = ssa->vars[var].use_chain; 15683 15684 do { 15685 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15686 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15687 return 0; 15688 } 15689 use = zend_ssa_next_use(ssa->ops, var, use); 15690 } while (use >= 0); 15691 } 15692 15693 return 1; 15694} 15695 15696static zend_bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) 15697{ 15698|.if X64 15699|| if (op_type == IS_CONST) { 15700|| zval *zv = RT_CONSTANT(opline, op); 15701|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 15702|| return 1; 15703|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 15704|| return 1; 15705|| } 15706|| } 15707|.endif 15708 return 0; 15709} 15710 15711static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use) 15712{ 15713 uint32_t op1_info, op2_info; 15714 15715 switch (opline->opcode) { 15716 case ZEND_FETCH_DIM_R: 15717 op1_info = OP1_INFO(); 15718 op2_info = OP2_INFO(); 15719 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15720 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15721 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15722 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15723 return ZEND_REGSET(ZREG_FCARG1a); 15724 } 15725 break; 15726 default: 15727 break; 15728 } 15729 15730 return ZEND_REGSET_EMPTY; 15731} 15732 15733static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use) 15734{ 15735 uint32_t op1_info, op2_info, res_info; 15736 zend_regset regset = ZEND_REGSET_SCRATCH; 15737 15738 switch (opline->opcode) { 15739 case ZEND_NOP: 15740 case ZEND_OP_DATA: 15741 case ZEND_JMP: 15742 case ZEND_RETURN: 15743 regset = ZEND_REGSET_EMPTY; 15744 break; 15745 case ZEND_QM_ASSIGN: 15746 if (ssa_op->op1_def == current_var || 15747 ssa_op->result_def == current_var) { 15748 regset = ZEND_REGSET_EMPTY; 15749 break; 15750 } 15751 /* break missing intentionally */ 15752 case ZEND_SEND_VAL: 15753 case ZEND_SEND_VAL_EX: 15754 if (opline->op2_type == IS_CONST) { 15755 break; 15756 } 15757 if (ssa_op->op1_use == current_var) { 15758 regset = ZEND_REGSET(ZREG_R0); 15759 break; 15760 } 15761 op1_info = OP1_INFO(); 15762 if (!(op1_info & MAY_BE_UNDEF)) { 15763 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15764 regset = ZEND_REGSET(ZREG_XMM0); 15765 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15766 regset = ZEND_REGSET(ZREG_R0); 15767 } else { 15768 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15769 } 15770 } 15771 break; 15772 case ZEND_SEND_VAR: 15773 if (opline->op2_type == IS_CONST) { 15774 break; 15775 } 15776 if (ssa_op->op1_use == current_var || 15777 ssa_op->op1_def == current_var) { 15778 regset = ZEND_REGSET_EMPTY; 15779 break; 15780 } 15781 op1_info = OP1_INFO(); 15782 if (!(op1_info & MAY_BE_UNDEF)) { 15783 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15784 regset = ZEND_REGSET(ZREG_XMM0); 15785 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15786 } else { 15787 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15788 if (op1_info & MAY_BE_REF) { 15789 ZEND_REGSET_INCL(regset, ZREG_R1); 15790 } 15791 } 15792 } 15793 break; 15794 case ZEND_ASSIGN: 15795 if (ssa_op->op2_use == current_var || 15796 ssa_op->op2_def == current_var || 15797 ssa_op->op1_def == current_var || 15798 ssa_op->result_def == current_var) { 15799 regset = ZEND_REGSET_EMPTY; 15800 break; 15801 } 15802 op1_info = OP1_INFO(); 15803 op2_info = OP2_INFO(); 15804 if (opline->op1_type == IS_CV 15805 && !(op2_info & MAY_BE_UNDEF) 15806 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15807 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15808 regset = ZEND_REGSET(ZREG_XMM0); 15809 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15810 regset = ZEND_REGSET(ZREG_R0); 15811 } else { 15812 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15813 } 15814 } 15815 break; 15816 case ZEND_PRE_INC: 15817 case ZEND_PRE_DEC: 15818 case ZEND_POST_INC: 15819 case ZEND_POST_DEC: 15820 if (ssa_op->op1_use == current_var || 15821 ssa_op->op1_def == current_var || 15822 ssa_op->result_def == current_var) { 15823 regset = ZEND_REGSET_EMPTY; 15824 break; 15825 } 15826 op1_info = OP1_INFO(); 15827 if (opline->op1_type == IS_CV 15828 && (op1_info & MAY_BE_LONG) 15829 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15830 regset = ZEND_REGSET_EMPTY; 15831 if (op1_info & MAY_BE_DOUBLE) { 15832 regset = ZEND_REGSET(ZREG_XMM0); 15833 } 15834 } 15835 break; 15836 case ZEND_ADD: 15837 case ZEND_SUB: 15838 case ZEND_MUL: 15839 op1_info = OP1_INFO(); 15840 op2_info = OP2_INFO(); 15841 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15842 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15843 15844 regset = ZEND_REGSET_EMPTY; 15845 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15846 if (ssa_op->result_def != current_var && 15847 (ssa_op->op1_use != current_var || !last_use)) { 15848 ZEND_REGSET_INCL(regset, ZREG_R0); 15849 } 15850 res_info = RES_INFO(); 15851 if (res_info & MAY_BE_DOUBLE) { 15852 ZEND_REGSET_INCL(regset, ZREG_R0); 15853 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15854 ZEND_REGSET_INCL(regset, ZREG_XMM1); 15855 } else if (res_info & MAY_BE_GUARD) { 15856 ZEND_REGSET_INCL(regset, ZREG_R0); 15857 } 15858 } 15859 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15860 if (ssa_op->result_def != current_var) { 15861 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15862 } 15863 } 15864 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15865 if (zend_is_commutative(opline->opcode)) { 15866 if (ssa_op->result_def != current_var) { 15867 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15868 } 15869 } else { 15870 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15871 if (ssa_op->result_def != current_var && 15872 (ssa_op->op1_use != current_var || !last_use)) { 15873 ZEND_REGSET_INCL(regset, ZREG_XMM1); 15874 } 15875 } 15876 } 15877 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15878 if (ssa_op->result_def != current_var && 15879 (ssa_op->op1_use != current_var || !last_use) && 15880 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15881 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15882 } 15883 } 15884 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 15885 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 15886 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 15887 ZEND_REGSET_INCL(regset, ZREG_R0); 15888 } else { 15889 ZEND_REGSET_INCL(regset, ZREG_R1); 15890 } 15891 } 15892 } 15893 break; 15894 case ZEND_BW_OR: 15895 case ZEND_BW_AND: 15896 case ZEND_BW_XOR: 15897 op1_info = OP1_INFO(); 15898 op2_info = OP2_INFO(); 15899 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15900 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15901 regset = ZEND_REGSET_EMPTY; 15902 if (ssa_op->result_def != current_var && 15903 (ssa_op->op1_use != current_var || !last_use)) { 15904 ZEND_REGSET_INCL(regset, ZREG_R0); 15905 } 15906 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 15907 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 15908 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 15909 ZEND_REGSET_INCL(regset, ZREG_R0); 15910 } else { 15911 ZEND_REGSET_INCL(regset, ZREG_R1); 15912 } 15913 } 15914 } 15915 break; 15916 case ZEND_SL: 15917 case ZEND_SR: 15918 op1_info = OP1_INFO(); 15919 op2_info = OP2_INFO(); 15920 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15921 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15922 regset = ZEND_REGSET_EMPTY; 15923 if (ssa_op->result_def != current_var && 15924 (ssa_op->op1_use != current_var || !last_use)) { 15925 ZEND_REGSET_INCL(regset, ZREG_R0); 15926 } 15927 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 15928 ZEND_REGSET_INCL(regset, ZREG_R1); 15929 } 15930 } 15931 break; 15932 case ZEND_MOD: 15933 op1_info = OP1_INFO(); 15934 op2_info = OP2_INFO(); 15935 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15936 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15937 regset = ZEND_REGSET_EMPTY; 15938 if (opline->op2_type == IS_CONST && 15939 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 15940 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 15941 OP1_HAS_RANGE() && 15942 OP1_MIN_RANGE() >= 0) { 15943 if (ssa_op->result_def != current_var && 15944 (ssa_op->op1_use != current_var || !last_use)) { 15945 ZEND_REGSET_INCL(regset, ZREG_R0); 15946 } 15947 if (sizeof(void*) == 8 15948 && !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) { 15949 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 15950 ZEND_REGSET_INCL(regset, ZREG_R0); 15951 } else { 15952 ZEND_REGSET_INCL(regset, ZREG_R1); 15953 } 15954 } 15955 } else { 15956 ZEND_REGSET_INCL(regset, ZREG_R0); 15957 ZEND_REGSET_INCL(regset, ZREG_R2); 15958 if (opline->op2_type == IS_CONST) { 15959 ZEND_REGSET_INCL(regset, ZREG_R1); 15960 } 15961 } 15962 } 15963 break; 15964 case ZEND_IS_SMALLER: 15965 case ZEND_IS_SMALLER_OR_EQUAL: 15966 case ZEND_IS_EQUAL: 15967 case ZEND_IS_NOT_EQUAL: 15968 case ZEND_IS_IDENTICAL: 15969 case ZEND_IS_NOT_IDENTICAL: 15970 case ZEND_CASE: 15971 op1_info = OP1_INFO(); 15972 op2_info = OP2_INFO(); 15973 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15974 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15975 regset = ZEND_REGSET_EMPTY; 15976 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 15977 ZEND_REGSET_INCL(regset, ZREG_R0); 15978 } 15979 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 15980 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 15981 if (ssa_op->op1_use != current_var && 15982 ssa_op->op2_use != current_var) { 15983 ZEND_REGSET_INCL(regset, ZREG_R0); 15984 } 15985 } 15986 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15987 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15988 } 15989 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15990 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15991 } 15992 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15993 if (ssa_op->op1_use != current_var && 15994 ssa_op->op2_use != current_var) { 15995 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15996 } 15997 } 15998 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 15999 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16000 ZEND_REGSET_INCL(regset, ZREG_R0); 16001 } 16002 } 16003 break; 16004 case ZEND_BOOL: 16005 case ZEND_BOOL_NOT: 16006 case ZEND_JMPZ: 16007 case ZEND_JMPNZ: 16008 case ZEND_JMPZNZ: 16009 case ZEND_JMPZ_EX: 16010 case ZEND_JMPNZ_EX: 16011 op1_info = OP1_INFO(); 16012 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16013 regset = ZEND_REGSET_EMPTY; 16014 if (op1_info & MAY_BE_DOUBLE) { 16015 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16016 } 16017 if (opline->opcode == ZEND_BOOL || 16018 opline->opcode == ZEND_BOOL_NOT || 16019 opline->opcode == ZEND_JMPZ_EX || 16020 opline->opcode == ZEND_JMPNZ_EX) { 16021 ZEND_REGSET_INCL(regset, ZREG_R0); 16022 } 16023 } 16024 break; 16025 case ZEND_DO_UCALL: 16026 case ZEND_DO_FCALL: 16027 case ZEND_DO_FCALL_BY_NAME: 16028 case ZEND_INCLUDE_OR_EVAL: 16029 case ZEND_GENERATOR_CREATE: 16030 case ZEND_YIELD: 16031 case ZEND_YIELD_FROM: 16032 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16033 break; 16034 default: 16035 break; 16036 } 16037 16038 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16039 if (ssa_op == ssa->ops 16040 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16041 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16042 ZEND_REGSET_INCL(regset, ZREG_R0); 16043 ZEND_REGSET_INCL(regset, ZREG_R1); 16044 } 16045 } 16046 16047 /* %r0 is used to check EG(vm_interrupt) */ 16048 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16049 if (ssa_op == ssa->ops 16050 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16051 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16052#if ZTS 16053 ZEND_REGSET_INCL(regset, ZREG_R0); 16054#else 16055 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16056 ZEND_REGSET_INCL(regset, ZREG_R0); 16057 } 16058#endif 16059 } 16060 } else { 16061 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16062 16063 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16064 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16065#if ZTS 16066 ZEND_REGSET_INCL(regset, ZREG_R0); 16067#else 16068 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16069 ZEND_REGSET_INCL(regset, ZREG_R0); 16070 } 16071#endif 16072 } 16073 } 16074 16075 return regset; 16076} 16077 16078#if defined(__clang__) 16079# pragma clang diagnostic pop 16080#endif 16081 16082/* 16083 * Local variables: 16084 * tab-width: 4 16085 * c-basic-offset: 4 16086 * indent-tabs-mode: t 16087 * End: 16088 */ 16089