1 /* -*- mode: asm -*- */ 2 /*- 3 * SPDX-License-Identifier: BSD-3-Clause 4 * 5 * Copyright (c) 1993 The Regents of the University of California. 6 * All rights reserved. 7 * 8 * Copyright (c) 2018 The FreeBSD Foundation 9 * All rights reserved. 10 * 11 * Portions of this software were developed by 12 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 13 * the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if defined(__i386__) 41 #include <i386/asmacros.h> 42 #else /* !__i386__ */ 43 44 #ifndef _MACHINE_ASMACROS_H_ 45 #define _MACHINE_ASMACROS_H_ 46 47 #include <sys/cdefs.h> 48 49 /* XXX too much duplication in various asm*.h's. */ 50 51 /* 52 * CNAME is used to manage the relationship between symbol names in C 53 * and the equivalent assembly language names. CNAME is given a name as 54 * it would be used in a C program. It expands to the equivalent assembly 55 * language name. 56 */ 57 #define CNAME(csym) csym 58 59 #define ALIGN_DATA .p2align 3 /* 8 byte alignment, zero filled */ 60 #define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 61 #define SUPERALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 62 63 #define GEN_ENTRY(name) ALIGN_TEXT; .globl CNAME(name); \ 64 .type CNAME(name),@function; CNAME(name): 65 #define ENTRY(name) GEN_ENTRY(name) 66 #define ALTENTRY(name) GEN_ENTRY(name) 67 #define END(name) .size name, . - name 68 69 /* 70 * Convenience for adding frame pointers to hand-coded ASM. Useful for 71 * DTrace, HWPMC, and KDB. 72 */ 73 #define PUSH_FRAME_POINTER \ 74 pushq %rbp ; \ 75 movq %rsp, %rbp ; 76 #define POP_FRAME_POINTER \ 77 popq %rbp 78 79 #ifdef LOCORE 80 /* 81 * Access per-CPU data. 82 */ 83 #define PCPU(member) %gs:PC_ ## member 84 #define PCPU_ADDR(member, reg) \ 85 movq %gs:PC_PRVSPACE, reg ; \ 86 addq $PC_ ## member, reg 87 88 /* 89 * Convenience macro for declaring interrupt entry points. 90 */ 91 #define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ 92 .type __CONCAT(X,name),@function; __CONCAT(X,name): 93 94 .macro SAVE_SEGS 95 movw %fs,TF_FS(%rsp) 96 movw %gs,TF_GS(%rsp) 97 movw %es,TF_ES(%rsp) 98 movw %ds,TF_DS(%rsp) 99 .endm 100 101 .macro MOVE_STACKS qw 102 .L.offset=0 103 .rept \qw 104 movq .L.offset(%rsp),%rdx 105 movq %rdx,.L.offset(%rax) 106 .L.offset=.L.offset+8 107 .endr 108 .endm 109 110 .macro PTI_UUENTRY has_err 111 movq PCPU(KCR3),%rax 112 movq %rax,%cr3 113 movq PCPU(RSP0),%rax 114 subq $PTI_SIZE - 8 * (1 - \has_err),%rax 115 MOVE_STACKS ((PTI_SIZE / 8) - 1 + \has_err) 116 movq %rax,%rsp 117 popq %rdx 118 popq %rax 119 .endm 120 121 .macro PTI_UENTRY has_err 122 swapgs 123 lfence 124 cmpq $~0,PCPU(UCR3) 125 je 1f 126 pushq %rax 127 pushq %rdx 128 PTI_UUENTRY \has_err 129 1: 130 .endm 131 132 .macro PTI_ENTRY name, contk, contu, has_err=0 133 ALIGN_TEXT 134 .globl X\name\()_pti 135 .type X\name\()_pti,@function 136 X\name\()_pti: 137 /* %rax, %rdx, and possibly err are not yet pushed */ 138 testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp) 139 jz \contk 140 PTI_UENTRY \has_err 141 jmp \contu 142 .endm 143 144 .macro PTI_INTRENTRY vec_name 145 SUPERALIGN_TEXT 146 .globl X\vec_name\()_pti 147 .type X\vec_name\()_pti,@function 148 X\vec_name\()_pti: 149 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */ 150 jz .L\vec_name\()_u 151 PTI_UENTRY has_err=0 152 jmp .L\vec_name\()_u 153 .endm 154 155 .macro INTR_PUSH_FRAME vec_name 156 SUPERALIGN_TEXT 157 .globl X\vec_name 158 .type X\vec_name,@function 159 X\vec_name: 160 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */ 161 jz .L\vec_name\()_u /* Yes, dont swapgs again */ 162 swapgs 163 .L\vec_name\()_u: 164 lfence 165 subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ 166 movq %rdi,TF_RDI(%rsp) 167 movq %rsi,TF_RSI(%rsp) 168 movq %rdx,TF_RDX(%rsp) 169 movq %rcx,TF_RCX(%rsp) 170 movq %r8,TF_R8(%rsp) 171 movq %r9,TF_R9(%rsp) 172 movq %rax,TF_RAX(%rsp) 173 movq %rbx,TF_RBX(%rsp) 174 movq %rbp,TF_RBP(%rsp) 175 movq %r10,TF_R10(%rsp) 176 movq %r11,TF_R11(%rsp) 177 movq %r12,TF_R12(%rsp) 178 movq %r13,TF_R13(%rsp) 179 movq %r14,TF_R14(%rsp) 180 movq %r15,TF_R15(%rsp) 181 SAVE_SEGS 182 movl $TF_HASSEGS,TF_FLAGS(%rsp) 183 pushfq 184 andq $~(PSL_D|PSL_AC),(%rsp) 185 popfq 186 testb $SEL_RPL_MASK,TF_CS(%rsp) /* come from kernel ? */ 187 jz 1f /* yes, leave PCB_FULL_IRET alone */ 188 movq PCPU(CURPCB),%r8 189 andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) 190 call handle_ibrs_entry 191 1: 192 .endm 193 194 .macro INTR_HANDLER vec_name 195 .text 196 PTI_INTRENTRY \vec_name 197 INTR_PUSH_FRAME \vec_name 198 .endm 199 200 .macro RESTORE_REGS 201 movq TF_RDI(%rsp),%rdi 202 movq TF_RSI(%rsp),%rsi 203 movq TF_RDX(%rsp),%rdx 204 movq TF_RCX(%rsp),%rcx 205 movq TF_R8(%rsp),%r8 206 movq TF_R9(%rsp),%r9 207 movq TF_RAX(%rsp),%rax 208 movq TF_RBX(%rsp),%rbx 209 movq TF_RBP(%rsp),%rbp 210 movq TF_R10(%rsp),%r10 211 movq TF_R11(%rsp),%r11 212 movq TF_R12(%rsp),%r12 213 movq TF_R13(%rsp),%r13 214 movq TF_R14(%rsp),%r14 215 movq TF_R15(%rsp),%r15 216 .endm 217 218 #ifdef KMSAN 219 /* 220 * The KMSAN runtime relies on a TLS block to track initialization and origin 221 * state for function parameters and return values. To keep this state 222 * consistent in the face of asynchronous kernel-mode traps, the runtime 223 * maintains a stack of blocks: when handling an exception or interrupt, 224 * kmsan_intr_enter() pushes the new block to be used until the handler is 225 * complete, at which point kmsan_intr_leave() restores the previous block. 226 * 227 * Thus, KMSAN_ENTER/LEAVE hooks are required only in handlers for events that 228 * may have happened while in kernel-mode. In particular, they are not required 229 * around amd64_syscall() or ast() calls. Otherwise, kmsan_intr_enter() can be 230 * called unconditionally, without distinguishing between entry from user-mode 231 * or kernel-mode. 232 */ 233 #define KMSAN_ENTER callq kmsan_intr_enter 234 #define KMSAN_LEAVE callq kmsan_intr_leave 235 #else 236 #define KMSAN_ENTER 237 #define KMSAN_LEAVE 238 #endif 239 240 #endif /* LOCORE */ 241 242 #ifdef __STDC__ 243 #define ELFNOTE(name, type, desctype, descdata...) \ 244 .pushsection .note.name, "a", @note ; \ 245 .align 4 ; \ 246 .long 2f - 1f /* namesz */ ; \ 247 .long 4f - 3f /* descsz */ ; \ 248 .long type ; \ 249 1:.asciz #name ; \ 250 2:.align 4 ; \ 251 3:desctype descdata ; \ 252 4:.align 4 ; \ 253 .popsection 254 #else /* !__STDC__, i.e. -traditional */ 255 #define ELFNOTE(name, type, desctype, descdata) \ 256 .pushsection .note.name, "a", @note ; \ 257 .align 4 ; \ 258 .long 2f - 1f /* namesz */ ; \ 259 .long 4f - 3f /* descsz */ ; \ 260 .long type ; \ 261 1:.asciz "name" ; \ 262 2:.align 4 ; \ 263 3:desctype descdata ; \ 264 4:.align 4 ; \ 265 .popsection 266 #endif /* __STDC__ */ 267 268 #endif /* !_MACHINE_ASMACROS_H_ */ 269 270 #endif /* __i386__ */ 271