1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1990 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * William Jolitz. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 #include "opt_hwpmc_hooks.h" 37 #include "opt_kstack_pages.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/assym.h> 42 #include <sys/bio.h> 43 #include <sys/buf.h> 44 #include <sys/proc.h> 45 #ifdef HWPMC_HOOKS 46 #include <sys/pmckern.h> 47 #endif 48 #include <sys/errno.h> 49 #include <sys/mount.h> 50 #include <sys/mutex.h> 51 #include <sys/socket.h> 52 #include <sys/resourcevar.h> 53 #include <sys/ucontext.h> 54 #include <machine/tss.h> 55 #include <sys/vmmeter.h> 56 #include <vm/vm.h> 57 #include <vm/vm_param.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_map.h> 60 #include <sys/proc.h> 61 #include <x86/apicreg.h> 62 #include <machine/cpu.h> 63 #include <machine/pcb.h> 64 #include <machine/sigframe.h> 65 #include <machine/proc.h> 66 #include <machine/segments.h> 67 #include <machine/efi.h> 68 69 ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); 70 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); 71 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); 72 73 ASSYM(P_MD, offsetof(struct proc, p_md)); 74 ASSYM(MD_LDT, offsetof(struct mdproc, md_ldt)); 75 ASSYM(MD_LDT_SD, offsetof(struct mdproc, md_ldt_sd)); 76 77 ASSYM(MD_EFIRT_TMP, offsetof(struct mdthread, md_efirt_tmp)); 78 79 ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); 80 ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); 81 ASSYM(TD_AST, offsetof(struct thread, td_ast)); 82 ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); 83 ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags)); 84 ASSYM(TD_PROC, offsetof(struct thread, td_proc)); 85 ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); 86 ASSYM(TD_MD, offsetof(struct thread, td_md)); 87 ASSYM(TD_MD_PCB, offsetof(struct thread, td_md.md_pcb)); 88 ASSYM(TD_MD_STACK_BASE, offsetof(struct thread, td_md.md_stack_base)); 89 90 ASSYM(TDP_CALLCHAIN, TDP_CALLCHAIN); 91 ASSYM(TDP_KTHREAD, TDP_KTHREAD); 92 93 ASSYM(PAGE_SIZE, PAGE_SIZE); 94 ASSYM(NPTEPG, NPTEPG); 95 ASSYM(NPDEPG, NPDEPG); 96 ASSYM(addr_P4Tmap, addr_P4Tmap); 97 ASSYM(addr_P4Dmap, addr_P4Dmap); 98 ASSYM(addr_P5Tmap, addr_P5Tmap); 99 ASSYM(addr_P5Dmap, addr_P5Dmap); 100 ASSYM(PDESIZE, sizeof(pd_entry_t)); 101 ASSYM(PTESIZE, sizeof(pt_entry_t)); 102 ASSYM(PAGE_SHIFT, PAGE_SHIFT); 103 ASSYM(PAGE_MASK, PAGE_MASK); 104 ASSYM(PDRSHIFT, PDRSHIFT); 105 ASSYM(PDPSHIFT, PDPSHIFT); 106 ASSYM(PML4SHIFT, PML4SHIFT); 107 ASSYM(val_KPDPI, KPDPI); 108 ASSYM(val_KPML4I, KPML4I); 109 ASSYM(val_PML4PML4I, PML4PML4I); 110 ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS); 111 ASSYM(KERNBASE, KERNBASE); 112 ASSYM(KERNLOAD, KERNLOAD); 113 ASSYM(DMAP_MIN_ADDRESS, DMAP_MIN_ADDRESS); 114 ASSYM(DMAP_MAX_ADDRESS, DMAP_MAX_ADDRESS); 115 ASSYM(KSTACK_PAGES, KSTACK_PAGES); 116 117 ASSYM(PCB_R15, offsetof(struct pcb, pcb_r15)); 118 ASSYM(PCB_R14, offsetof(struct pcb, pcb_r14)); 119 ASSYM(PCB_R13, offsetof(struct pcb, pcb_r13)); 120 ASSYM(PCB_R12, offsetof(struct pcb, pcb_r12)); 121 ASSYM(PCB_RBP, offsetof(struct pcb, pcb_rbp)); 122 ASSYM(PCB_RSP, offsetof(struct pcb, pcb_rsp)); 123 ASSYM(PCB_RBX, offsetof(struct pcb, pcb_rbx)); 124 ASSYM(PCB_RIP, offsetof(struct pcb, pcb_rip)); 125 ASSYM(PCB_FSBASE, offsetof(struct pcb, pcb_fsbase)); 126 ASSYM(PCB_GSBASE, offsetof(struct pcb, pcb_gsbase)); 127 ASSYM(PCB_KGSBASE, offsetof(struct pcb, pcb_kgsbase)); 128 ASSYM(PCB_CR0, offsetof(struct pcb, pcb_cr0)); 129 ASSYM(PCB_CR2, offsetof(struct pcb, pcb_cr2)); 130 ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3)); 131 ASSYM(PCB_CR4, offsetof(struct pcb, pcb_cr4)); 132 ASSYM(PCB_DR0, offsetof(struct pcb, pcb_dr0)); 133 ASSYM(PCB_DR1, offsetof(struct pcb, pcb_dr1)); 134 ASSYM(PCB_DR2, offsetof(struct pcb, pcb_dr2)); 135 ASSYM(PCB_DR3, offsetof(struct pcb, pcb_dr3)); 136 ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6)); 137 ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7)); 138 ASSYM(PCB_GDT, offsetof(struct pcb, pcb_gdt)); 139 ASSYM(PCB_IDT, offsetof(struct pcb, pcb_idt)); 140 ASSYM(PCB_LDT, offsetof(struct pcb, pcb_ldt)); 141 ASSYM(PCB_TR, offsetof(struct pcb, pcb_tr)); 142 ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); 143 ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); 144 ASSYM(PCB_SAVED_UCR3, offsetof(struct pcb, pcb_saved_ucr3)); 145 ASSYM(PCB_TSSP, offsetof(struct pcb, pcb_tssp)); 146 ASSYM(PCB_SAVEFPU, offsetof(struct pcb, pcb_save)); 147 ASSYM(PCB_EFER, offsetof(struct pcb, pcb_efer)); 148 ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star)); 149 ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar)); 150 ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar)); 151 ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask)); 152 ASSYM(PCB_SIZE, sizeof(struct pcb)); 153 ASSYM(PCB_FULL_IRET, PCB_FULL_IRET); 154 ASSYM(PCB_DBREGS, PCB_DBREGS); 155 ASSYM(PCB_32BIT, PCB_32BIT); 156 157 ASSYM(TSS_RSP0, offsetof(struct amd64tss, tss_rsp0)); 158 159 ASSYM(TF_R15, offsetof(struct trapframe, tf_r15)); 160 ASSYM(TF_R14, offsetof(struct trapframe, tf_r14)); 161 ASSYM(TF_R13, offsetof(struct trapframe, tf_r13)); 162 ASSYM(TF_R12, offsetof(struct trapframe, tf_r12)); 163 ASSYM(TF_R11, offsetof(struct trapframe, tf_r11)); 164 ASSYM(TF_R10, offsetof(struct trapframe, tf_r10)); 165 ASSYM(TF_R9, offsetof(struct trapframe, tf_r9)); 166 ASSYM(TF_R8, offsetof(struct trapframe, tf_r8)); 167 ASSYM(TF_RDI, offsetof(struct trapframe, tf_rdi)); 168 ASSYM(TF_RSI, offsetof(struct trapframe, tf_rsi)); 169 ASSYM(TF_RBP, offsetof(struct trapframe, tf_rbp)); 170 ASSYM(TF_RBX, offsetof(struct trapframe, tf_rbx)); 171 ASSYM(TF_RDX, offsetof(struct trapframe, tf_rdx)); 172 ASSYM(TF_RCX, offsetof(struct trapframe, tf_rcx)); 173 ASSYM(TF_RAX, offsetof(struct trapframe, tf_rax)); 174 ASSYM(TF_TRAPNO, offsetof(struct trapframe, tf_trapno)); 175 ASSYM(TF_ADDR, offsetof(struct trapframe, tf_addr)); 176 ASSYM(TF_ERR, offsetof(struct trapframe, tf_err)); 177 ASSYM(TF_RIP, offsetof(struct trapframe, tf_rip)); 178 ASSYM(TF_CS, offsetof(struct trapframe, tf_cs)); 179 ASSYM(TF_RFLAGS, offsetof(struct trapframe, tf_rflags)); 180 ASSYM(TF_RSP, offsetof(struct trapframe, tf_rsp)); 181 ASSYM(TF_SS, offsetof(struct trapframe, tf_ss)); 182 ASSYM(TF_DS, offsetof(struct trapframe, tf_ds)); 183 ASSYM(TF_ES, offsetof(struct trapframe, tf_es)); 184 ASSYM(TF_FS, offsetof(struct trapframe, tf_fs)); 185 ASSYM(TF_GS, offsetof(struct trapframe, tf_gs)); 186 ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags)); 187 ASSYM(TF_SIZE, sizeof(struct trapframe)); 188 ASSYM(TF_HASSEGS, TF_HASSEGS); 189 190 ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx)); 191 ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax)); 192 ASSYM(PTI_ERR, offsetof(struct pti_frame, pti_err)); 193 ASSYM(PTI_RIP, offsetof(struct pti_frame, pti_rip)); 194 ASSYM(PTI_CS, offsetof(struct pti_frame, pti_cs)); 195 ASSYM(PTI_RFLAGS, offsetof(struct pti_frame, pti_rflags)); 196 ASSYM(PTI_RSP, offsetof(struct pti_frame, pti_rsp)); 197 ASSYM(PTI_SS, offsetof(struct pti_frame, pti_ss)); 198 ASSYM(PTI_SIZE, sizeof(struct pti_frame)); 199 200 ASSYM(SIGF_HANDLER, offsetof(struct sigframe, sf_ahu.sf_handler)); 201 ASSYM(SIGF_UC, offsetof(struct sigframe, sf_uc)); 202 ASSYM(UC_EFLAGS, offsetof(ucontext_t, uc_mcontext.mc_rflags)); 203 ASSYM(UC_RDI, offsetof(ucontext_t, uc_mcontext.mc_rdi)); 204 ASSYM(UC_RSI, offsetof(ucontext_t, uc_mcontext.mc_rsi)); 205 ASSYM(UC_RDX, offsetof(ucontext_t, uc_mcontext.mc_rdx)); 206 ASSYM(UC_RCX, offsetof(ucontext_t, uc_mcontext.mc_rcx)); 207 ASSYM(UC_R8, offsetof(ucontext_t, uc_mcontext.mc_r8)); 208 ASSYM(UC_R9, offsetof(ucontext_t, uc_mcontext.mc_r9)); 209 ASSYM(UC_RAX, offsetof(ucontext_t, uc_mcontext.mc_rax)); 210 ASSYM(UC_RBX, offsetof(ucontext_t, uc_mcontext.mc_rbx)); 211 ASSYM(UC_RBP, offsetof(ucontext_t, uc_mcontext.mc_rbp)); 212 ASSYM(UC_R10, offsetof(ucontext_t, uc_mcontext.mc_r10)); 213 ASSYM(UC_R11, offsetof(ucontext_t, uc_mcontext.mc_r11)); 214 ASSYM(UC_R12, offsetof(ucontext_t, uc_mcontext.mc_r12)); 215 ASSYM(UC_R13, offsetof(ucontext_t, uc_mcontext.mc_r13)); 216 ASSYM(UC_R14, offsetof(ucontext_t, uc_mcontext.mc_r14)); 217 ASSYM(UC_R15, offsetof(ucontext_t, uc_mcontext.mc_r15)); 218 ASSYM(UC_FS, offsetof(ucontext_t, uc_mcontext.mc_fs)); 219 ASSYM(UC_GS, offsetof(ucontext_t, uc_mcontext.mc_gs)); 220 ASSYM(UC_ES, offsetof(ucontext_t, uc_mcontext.mc_es)); 221 ASSYM(UC_DS, offsetof(ucontext_t, uc_mcontext.mc_ds)); 222 ASSYM(UC_RIP, offsetof(ucontext_t, uc_mcontext.mc_rip)); 223 ASSYM(UC_CS, offsetof(ucontext_t, uc_mcontext.mc_cs)); 224 ASSYM(UC_RFLAGS, offsetof(ucontext_t, uc_mcontext.mc_rflags)); 225 ASSYM(UC_RSP, offsetof(ucontext_t, uc_mcontext.mc_rsp)); 226 ASSYM(UC_SS, offsetof(ucontext_t, uc_mcontext.mc_ss)); 227 ASSYM(UC_FSBASE, offsetof(ucontext_t, uc_mcontext.mc_fsbase)); 228 ASSYM(UC_GSBASE, offsetof(ucontext_t, uc_mcontext.mc_gsbase)); 229 230 ASSYM(ENOENT, ENOENT); 231 ASSYM(EFAULT, EFAULT); 232 ASSYM(ENAMETOOLONG, ENAMETOOLONG); 233 ASSYM(MAXCOMLEN, MAXCOMLEN); 234 ASSYM(MAXPATHLEN, MAXPATHLEN); 235 ASSYM(PC_SIZEOF, sizeof(struct pcpu)); 236 ASSYM(PC_PRVSPACE, offsetof(struct pcpu, pc_prvspace)); 237 ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); 238 ASSYM(PC_FPCURTHREAD, offsetof(struct pcpu, pc_fpcurthread)); 239 ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread)); 240 ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); 241 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); 242 ASSYM(PC_SCRATCH_RSP, offsetof(struct pcpu, pc_scratch_rsp)); 243 ASSYM(PC_SCRATCH_RAX, offsetof(struct pcpu, pc_scratch_rax)); 244 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap)); 245 ASSYM(PC_TSSP, offsetof(struct pcpu, pc_tssp)); 246 ASSYM(PC_RSP0, offsetof(struct pcpu, pc_rsp0)); 247 ASSYM(PC_FS32P, offsetof(struct pcpu, pc_fs32p)); 248 ASSYM(PC_GS32P, offsetof(struct pcpu, pc_gs32p)); 249 ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt)); 250 ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss)); 251 ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss)); 252 ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3)); 253 ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3)); 254 ASSYM(PC_UCR3_LOAD_MASK, offsetof(struct pcpu, pc_ucr3_load_mask)); 255 ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3)); 256 ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack)); 257 ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ); 258 ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0)); 259 ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); 260 ASSYM(PC_MDS_TMP, offsetof(struct pcpu, pc_mds_tmp)); 261 ASSYM(PC_MDS_BUF, offsetof(struct pcpu, pc_mds_buf)); 262 ASSYM(PC_MDS_BUF64, offsetof(struct pcpu, pc_mds_buf64)); 263 264 ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); 265 ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL); 266 267 ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL)); 268 ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL)); 269 ASSYM(KUCSEL, GSEL(GUCODE_SEL, SEL_UPL)); 270 ASSYM(KUDSEL, GSEL(GUDATA_SEL, SEL_UPL)); 271 ASSYM(KUC32SEL, GSEL(GUCODE32_SEL, SEL_UPL)); 272 ASSYM(KUF32SEL, GSEL(GUFS32_SEL, SEL_UPL)); 273 ASSYM(KUG32SEL, GSEL(GUGS32_SEL, SEL_UPL)); 274 ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL)); 275 ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL)); 276 ASSYM(SEL_RPL_MASK, SEL_RPL_MASK); 277 278 ASSYM(__FreeBSD_version, __FreeBSD_version); 279 280 #ifdef HWPMC_HOOKS 281 ASSYM(PMC_FN_USER_CALLCHAIN, PMC_FN_USER_CALLCHAIN); 282 #endif 283 284 ASSYM(EC_EFI_STATUS, offsetof(struct efirt_callinfo, ec_efi_status)); 285 ASSYM(EC_FPTR, offsetof(struct efirt_callinfo, ec_fptr)); 286 ASSYM(EC_ARGCNT, offsetof(struct efirt_callinfo, ec_argcnt)); 287 ASSYM(EC_ARG1, offsetof(struct efirt_callinfo, ec_arg1)); 288 ASSYM(EC_ARG2, offsetof(struct efirt_callinfo, ec_arg2)); 289 ASSYM(EC_ARG3, offsetof(struct efirt_callinfo, ec_arg3)); 290 ASSYM(EC_ARG4, offsetof(struct efirt_callinfo, ec_arg4)); 291 ASSYM(EC_ARG5, offsetof(struct efirt_callinfo, ec_arg5)); 292 ASSYM(EC_RBX, offsetof(struct efirt_callinfo, ec_rbx)); 293 ASSYM(EC_RSP, offsetof(struct efirt_callinfo, ec_rsp)); 294 ASSYM(EC_RBP, offsetof(struct efirt_callinfo, ec_rbp)); 295 ASSYM(EC_R12, offsetof(struct efirt_callinfo, ec_r12)); 296 ASSYM(EC_R13, offsetof(struct efirt_callinfo, ec_r13)); 297 ASSYM(EC_R14, offsetof(struct efirt_callinfo, ec_r14)); 298 ASSYM(EC_R15, offsetof(struct efirt_callinfo, ec_r15)); 299 ASSYM(EC_RFLAGS, offsetof(struct efirt_callinfo, ec_rflags)); 300