1 /* 2 ** Instruction dispatch handling. 3 ** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h 4 */ 5 6 #ifndef _LJ_DISPATCH_H 7 #define _LJ_DISPATCH_H 8 9 #include "lj_obj.h" 10 #include "lj_bc.h" 11 #if LJ_HASJIT 12 #include "lj_jit.h" 13 #endif 14 15 #if LJ_TARGET_MIPS 16 /* Need our own global offset table for the dreaded MIPS calling conventions. */ 17 18 #ifndef _LJ_VM_H 19 LJ_ASMF int32_t LJ_FASTCALL lj_vm_modi(int32_t a, int32_t b); 20 #endif 21 22 #if LJ_SOFTFP 23 #ifndef _LJ_IRCALL_H 24 extern double __adddf3(double a, double b); 25 extern double __subdf3(double a, double b); 26 extern double __muldf3(double a, double b); 27 extern double __divdf3(double a, double b); 28 #endif 29 #define SFGOTDEF(_) _(sqrt) _(__adddf3) _(__subdf3) _(__muldf3) _(__divdf3) 30 #else 31 #define SFGOTDEF(_) 32 #endif 33 #if LJ_HASJIT 34 #define JITGOTDEF(_) _(lj_err_trace) _(lj_trace_exit) _(lj_trace_hot) 35 #else 36 #define JITGOTDEF(_) 37 #endif 38 #if LJ_HASFFI 39 #define FFIGOTDEF(_) \ 40 _(lj_meta_equal_cd) _(lj_ccallback_enter) _(lj_ccallback_leave) 41 #else 42 #define FFIGOTDEF(_) 43 #endif 44 #define GOTDEF(_) \ 45 _(floor) _(ceil) _(trunc) _(log) _(log10) _(exp) _(sin) _(cos) _(tan) \ 46 _(asin) _(acos) _(atan) _(sinh) _(cosh) _(tanh) _(frexp) _(modf) _(atan2) \ 47 _(pow) _(fmod) _(ldexp) _(lj_vm_modi) \ 48 _(lj_dispatch_call) _(lj_dispatch_ins) _(lj_dispatch_stitch) \ 49 _(lj_dispatch_profile) _(lj_err_throw) \ 50 _(lj_ffh_coroutine_wrap_err) _(lj_func_closeuv) _(lj_func_newL_gc) \ 51 _(lj_gc_barrieruv) _(lj_gc_step) _(lj_gc_step_fixtop) _(lj_meta_arith) \ 52 _(lj_meta_call) _(lj_meta_cat) _(lj_meta_comp) _(lj_meta_equal) \ 53 _(lj_meta_for) _(lj_meta_istype) _(lj_meta_len) _(lj_meta_tget) \ 54 _(lj_meta_tset) _(lj_state_growstack) _(lj_strfmt_number) \ 55 _(lj_str_new) _(lj_tab_dup) _(lj_tab_get) _(lj_tab_getinth) _(lj_tab_len) \ 56 _(lj_tab_new) _(lj_tab_newkey) _(lj_tab_next) _(lj_tab_reasize) \ 57 _(lj_tab_setinth) _(lj_buf_putstr_reverse) _(lj_buf_putstr_lower) \ 58 _(lj_buf_putstr_upper) _(lj_buf_tostr) \ 59 JITGOTDEF(_) FFIGOTDEF(_) SFGOTDEF(_) 60 61 enum { 62 #define GOTENUM(name) LJ_GOT_##name, 63 GOTDEF(GOTENUM) 64 #undef GOTENUM 65 LJ_GOT__MAX 66 }; 67 #endif 68 69 /* Type of hot counter. Must match the code in the assembler VM. */ 70 /* 16 bits are sufficient. Only 0.0015% overhead with maximum slot penalty. */ 71 typedef uint16_t HotCount; 72 73 /* Number of hot counter hash table entries (must be a power of two). */ 74 #define HOTCOUNT_SIZE 64 75 #define HOTCOUNT_PCMASK ((HOTCOUNT_SIZE-1)*sizeof(HotCount)) 76 77 /* Hotcount decrements. */ 78 #define HOTCOUNT_LOOP 2 79 #define HOTCOUNT_CALL 1 80 81 /* This solves a circular dependency problem -- bump as needed. Sigh. */ 82 #define GG_NUM_ASMFF 57 83 84 #define GG_LEN_DDISP (BC__MAX + GG_NUM_ASMFF) 85 #define GG_LEN_SDISP BC_FUNCF 86 #define GG_LEN_DISP (GG_LEN_DDISP + GG_LEN_SDISP) 87 88 /* Global state, main thread and extra fields are allocated together. */ 89 typedef struct GG_State { 90 lua_State L; /* Main thread. */ 91 global_State g; /* Global state. */ 92 #if LJ_TARGET_ARM 93 /* Make g reachable via K12 encoded DISPATCH-relative addressing. */ 94 uint8_t align1[(16-sizeof(global_State))&15]; 95 #endif 96 #if LJ_TARGET_MIPS 97 ASMFunction got[LJ_GOT__MAX]; /* Global offset table. */ 98 #endif 99 #if LJ_HASJIT 100 jit_State J; /* JIT state. */ 101 HotCount hotcount[HOTCOUNT_SIZE]; /* Hot counters. */ 102 #if LJ_TARGET_ARM 103 /* Ditto for J. */ 104 uint8_t align2[(16-sizeof(jit_State)-sizeof(HotCount)*HOTCOUNT_SIZE)&15]; 105 #endif 106 #endif 107 ASMFunction dispatch[GG_LEN_DISP]; /* Instruction dispatch tables. */ 108 BCIns bcff[GG_NUM_ASMFF]; /* Bytecode for ASM fast functions. */ 109 } GG_State; 110 111 #define GG_OFS(field) ((int)offsetof(GG_State, field)) 112 #define G2GG(gl) ((GG_State *)((char *)(gl) - GG_OFS(g))) 113 #define J2GG(j) ((GG_State *)((char *)(j) - GG_OFS(J))) 114 #define L2GG(L) (G2GG(G(L))) 115 #define J2G(J) (&J2GG(J)->g) 116 #define G2J(gl) (&G2GG(gl)->J) 117 #define L2J(L) (&L2GG(L)->J) 118 #define GG_G2J (GG_OFS(J) - GG_OFS(g)) 119 #define GG_G2DISP (GG_OFS(dispatch) - GG_OFS(g)) 120 #define GG_DISP2G (GG_OFS(g) - GG_OFS(dispatch)) 121 #define GG_DISP2J (GG_OFS(J) - GG_OFS(dispatch)) 122 #define GG_DISP2HOT (GG_OFS(hotcount) - GG_OFS(dispatch)) 123 #define GG_DISP2STATIC (GG_LEN_DDISP*(int)sizeof(ASMFunction)) 124 125 #define hotcount_get(gg, pc) \ 126 (gg)->hotcount[(u32ptr(pc)>>2) & (HOTCOUNT_SIZE-1)] 127 #define hotcount_set(gg, pc, val) \ 128 (hotcount_get((gg), (pc)) = (HotCount)(val)) 129 130 /* Dispatch table management. */ 131 LJ_FUNC void lj_dispatch_init(GG_State *GG); 132 #if LJ_HASJIT 133 LJ_FUNC void lj_dispatch_init_hotcount(global_State *g); 134 #endif 135 LJ_FUNC void lj_dispatch_update(global_State *g); 136 137 /* Instruction dispatch callback for hooks or when recording. */ 138 LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc); 139 LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns*pc); 140 #if LJ_HASJIT 141 LJ_FUNCA void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc); 142 #endif 143 #if LJ_HASPROFILE 144 LJ_FUNCA void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc); 145 #endif 146 147 #if LJ_HASFFI && !defined(_BUILDVM_H) 148 /* Save/restore errno and GetLastError() around hooks, exits and recording. */ 149 #include <errno.h> 150 #if LJ_TARGET_WINDOWS 151 #define WIN32_LEAN_AND_MEAN 152 #include <windows.h> 153 #define ERRNO_SAVE int olderr = errno; DWORD oldwerr = GetLastError(); 154 #define ERRNO_RESTORE errno = olderr; SetLastError(oldwerr); 155 #else 156 #define ERRNO_SAVE int olderr = errno; 157 #define ERRNO_RESTORE errno = olderr; 158 #endif 159 #else 160 #define ERRNO_SAVE 161 #define ERRNO_RESTORE 162 #endif 163 164 #endif 165