1 /* -*- mode: asm -*- */ 2 /*- 3 * SPDX-License-Identifier: BSD-3-Clause 4 * 5 * Copyright (c) 1993 The Regents of the University of California. 6 * All rights reserved. 7 * 8 * Copyright (c) 2018 The FreeBSD Foundation 9 * All rights reserved. 10 * 11 * Portions of this software were developed by 12 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 13 * the FreeBSD Foundation. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * $FreeBSD$ 40 */ 41 42 #ifndef _MACHINE_ASMACROS_H_ 43 #define _MACHINE_ASMACROS_H_ 44 45 #include <sys/cdefs.h> 46 47 /* XXX too much duplication in various asm*.h's. */ 48 49 /* 50 * CNAME is used to manage the relationship between symbol names in C 51 * and the equivalent assembly language names. CNAME is given a name as 52 * it would be used in a C program. It expands to the equivalent assembly 53 * language name. 54 */ 55 #define CNAME(csym) csym 56 57 #define ALIGN_DATA .p2align 3 /* 8 byte alignment, zero filled */ 58 #define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 59 #define SUPERALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ 60 61 #define GEN_ENTRY(name) ALIGN_TEXT; .globl CNAME(name); \ 62 .type CNAME(name),@function; CNAME(name): 63 #define ENTRY(name) GEN_ENTRY(name) 64 #define ALTENTRY(name) GEN_ENTRY(name) 65 #define END(name) .size name, . - name 66 67 /* 68 * Convenience for adding frame pointers to hand-coded ASM. Useful for 69 * DTrace, HWPMC, and KDB. 70 */ 71 #define PUSH_FRAME_POINTER \ 72 pushq %rbp ; \ 73 movq %rsp, %rbp ; 74 #define POP_FRAME_POINTER \ 75 popq %rbp 76 77 #ifdef LOCORE 78 /* 79 * Access per-CPU data. 80 */ 81 #define PCPU(member) %gs:PC_ ## member 82 #define PCPU_ADDR(member, reg) \ 83 movq %gs:PC_PRVSPACE, reg ; \ 84 addq $PC_ ## member, reg 85 86 /* 87 * Convenience macro for declaring interrupt entry points. 88 */ 89 #define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ 90 .type __CONCAT(X,name),@function; __CONCAT(X,name): 91 92 .macro SAVE_SEGS 93 movw %fs,TF_FS(%rsp) 94 movw %gs,TF_GS(%rsp) 95 movw %es,TF_ES(%rsp) 96 movw %ds,TF_DS(%rsp) 97 .endm 98 99 .macro MOVE_STACKS qw 100 .L.offset=0 101 .rept \qw 102 movq .L.offset(%rsp),%rdx 103 movq %rdx,.L.offset(%rax) 104 .L.offset=.L.offset+8 105 .endr 106 .endm 107 108 .macro PTI_UUENTRY has_err 109 movq PCPU(KCR3),%rax 110 movq %rax,%cr3 111 movq PCPU(RSP0),%rax 112 subq $PTI_SIZE - 8 * (1 - \has_err),%rax 113 MOVE_STACKS ((PTI_SIZE / 8) - 1 + \has_err) 114 movq %rax,%rsp 115 popq %rdx 116 popq %rax 117 .endm 118 119 .macro PTI_UENTRY has_err 120 swapgs 121 lfence 122 cmpq $~0,PCPU(UCR3) 123 je 1f 124 pushq %rax 125 pushq %rdx 126 PTI_UUENTRY \has_err 127 1: 128 .endm 129 130 .macro PTI_ENTRY name, contk, contu, has_err=0 131 ALIGN_TEXT 132 .globl X\name\()_pti 133 .type X\name\()_pti,@function 134 X\name\()_pti: 135 /* %rax, %rdx, and possibly err are not yet pushed */ 136 testb $SEL_RPL_MASK,PTI_CS-PTI_ERR-((1-\has_err)*8)(%rsp) 137 jz \contk 138 PTI_UENTRY \has_err 139 jmp \contu 140 .endm 141 142 .macro PTI_INTRENTRY vec_name 143 SUPERALIGN_TEXT 144 .globl X\vec_name\()_pti 145 .type X\vec_name\()_pti,@function 146 X\vec_name\()_pti: 147 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* err, %rax, %rdx not pushed */ 148 jz .L\vec_name\()_u 149 PTI_UENTRY has_err=0 150 jmp .L\vec_name\()_u 151 .endm 152 153 .macro INTR_PUSH_FRAME vec_name 154 SUPERALIGN_TEXT 155 .globl X\vec_name 156 .type X\vec_name,@function 157 X\vec_name: 158 testb $SEL_RPL_MASK,PTI_CS-3*8(%rsp) /* come from kernel? */ 159 jz .L\vec_name\()_u /* Yes, dont swapgs again */ 160 swapgs 161 .L\vec_name\()_u: 162 lfence 163 subq $TF_RIP,%rsp /* skip dummy tf_err and tf_trapno */ 164 movq %rdi,TF_RDI(%rsp) 165 movq %rsi,TF_RSI(%rsp) 166 movq %rdx,TF_RDX(%rsp) 167 movq %rcx,TF_RCX(%rsp) 168 movq %r8,TF_R8(%rsp) 169 movq %r9,TF_R9(%rsp) 170 movq %rax,TF_RAX(%rsp) 171 movq %rbx,TF_RBX(%rsp) 172 movq %rbp,TF_RBP(%rsp) 173 movq %r10,TF_R10(%rsp) 174 movq %r11,TF_R11(%rsp) 175 movq %r12,TF_R12(%rsp) 176 movq %r13,TF_R13(%rsp) 177 movq %r14,TF_R14(%rsp) 178 movq %r15,TF_R15(%rsp) 179 SAVE_SEGS 180 movl $TF_HASSEGS,TF_FLAGS(%rsp) 181 pushfq 182 andq $~(PSL_D|PSL_AC),(%rsp) 183 popfq 184 testb $SEL_RPL_MASK,TF_CS(%rsp) /* come from kernel ? */ 185 jz 1f /* yes, leave PCB_FULL_IRET alone */ 186 movq PCPU(CURPCB),%r8 187 andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) 188 call handle_ibrs_entry 189 1: 190 .endm 191 192 .macro INTR_HANDLER vec_name 193 .text 194 PTI_INTRENTRY \vec_name 195 INTR_PUSH_FRAME \vec_name 196 .endm 197 198 .macro RESTORE_REGS 199 movq TF_RDI(%rsp),%rdi 200 movq TF_RSI(%rsp),%rsi 201 movq TF_RDX(%rsp),%rdx 202 movq TF_RCX(%rsp),%rcx 203 movq TF_R8(%rsp),%r8 204 movq TF_R9(%rsp),%r9 205 movq TF_RAX(%rsp),%rax 206 movq TF_RBX(%rsp),%rbx 207 movq TF_RBP(%rsp),%rbp 208 movq TF_R10(%rsp),%r10 209 movq TF_R11(%rsp),%r11 210 movq TF_R12(%rsp),%r12 211 movq TF_R13(%rsp),%r13 212 movq TF_R14(%rsp),%r14 213 movq TF_R15(%rsp),%r15 214 .endm 215 216 #ifdef KMSAN 217 /* 218 * The KMSAN runtime relies on a TLS block to track initialization and origin 219 * state for function parameters and return values. To keep this state 220 * consistent in the face of asynchronous kernel-mode traps, the runtime 221 * maintains a stack of blocks: when handling an exception or interrupt, 222 * kmsan_intr_enter() pushes the new block to be used until the handler is 223 * complete, at which point kmsan_intr_leave() restores the previous block. 224 * 225 * Thus, KMSAN_ENTER/LEAVE hooks are required only in handlers for events that 226 * may have happened while in kernel-mode. In particular, they are not required 227 * around amd64_syscall() or ast() calls. Otherwise, kmsan_intr_enter() can be 228 * called unconditionally, without distinguishing between entry from user-mode 229 * or kernel-mode. 230 */ 231 #define KMSAN_ENTER callq kmsan_intr_enter 232 #define KMSAN_LEAVE callq kmsan_intr_leave 233 #else 234 #define KMSAN_ENTER 235 #define KMSAN_LEAVE 236 #endif 237 238 #endif /* LOCORE */ 239 240 #ifdef __STDC__ 241 #define ELFNOTE(name, type, desctype, descdata...) \ 242 .pushsection .note.name ; \ 243 .align 4 ; \ 244 .long 2f - 1f /* namesz */ ; \ 245 .long 4f - 3f /* descsz */ ; \ 246 .long type ; \ 247 1:.asciz #name ; \ 248 2:.align 4 ; \ 249 3:desctype descdata ; \ 250 4:.align 4 ; \ 251 .popsection 252 #else /* !__STDC__, i.e. -traditional */ 253 #define ELFNOTE(name, type, desctype, descdata) \ 254 .pushsection .note.name ; \ 255 .align 4 ; \ 256 .long 2f - 1f /* namesz */ ; \ 257 .long 4f - 3f /* descsz */ ; \ 258 .long type ; \ 259 1:.asciz "name" ; \ 260 2:.align 4 ; \ 261 3:desctype descdata ; \ 262 4:.align 4 ; \ 263 .popsection 264 #endif /* __STDC__ */ 265 266 #endif /* !_MACHINE_ASMACROS_H_ */ 267