1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#include <machine/asmacros.h> 31#include <machine/cputypes.h> 32#include <machine/pmap.h> 33#include <machine/specialreg.h> 34 35#include "assym.inc" 36 37#define IDXSHIFT 10 38 39 .text 40 41ENTRY(sse2_pagezero) 42 pushl %ebx 43 movl 8(%esp),%ecx 44 movl %ecx,%eax 45 addl $4096,%eax 46 xor %ebx,%ebx 47 jmp 1f 48 /* 49 * The loop takes 14 bytes. Ensure that it doesn't cross a 16-byte 50 * cache line. 51 */ 52 .p2align 4,0x90 531: 54 movnti %ebx,(%ecx) 55 movnti %ebx,4(%ecx) 56 addl $8,%ecx 57 cmpl %ecx,%eax 58 jne 1b 59 sfence 60 popl %ebx 61 ret 62END(sse2_pagezero) 63 64ENTRY(i686_pagezero) 65 pushl %edi 66 pushl %ebx 67 68 movl 12(%esp),%edi 69 movl $1024,%ecx 70 71 ALIGN_TEXT 721: 73 xorl %eax,%eax 74 repe 75 scasl 76 jnz 2f 77 78 popl %ebx 79 popl %edi 80 ret 81 82 ALIGN_TEXT 83 842: 85 incl %ecx 86 subl $4,%edi 87 88 movl %ecx,%edx 89 cmpl $16,%ecx 90 91 jge 3f 92 93 movl %edi,%ebx 94 andl $0x3f,%ebx 95 shrl %ebx 96 shrl %ebx 97 movl $16,%ecx 98 subl %ebx,%ecx 99 1003: 101 subl %ecx,%edx 102 rep 103 stosl 104 105 movl %edx,%ecx 106 testl %edx,%edx 107 jnz 1b 108 109 popl %ebx 110 popl %edi 111 ret 112END(i686_pagezero) 113 114/* fillw(pat, base, cnt) */ 115ENTRY(fillw) 116 pushl %edi 117 movl 8(%esp),%eax 118 movl 12(%esp),%edi 119 movl 16(%esp),%ecx 120 rep 121 stosw 122 popl %edi 123 ret 124END(fillw) 125 126/* 127 * memmove(dst, src, cnt) (return dst) 128 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 129 */ 130ENTRY(memmove) 131 pushl %ebp 132 movl %esp,%ebp 133 pushl %esi 134 pushl %edi 135 movl 8(%ebp),%edi 136 movl 12(%ebp),%esi 1371: 138 movl 16(%ebp),%ecx 139 140 movl %edi,%eax 141 subl %esi,%eax 142 cmpl %ecx,%eax /* overlapping && src < dst? */ 143 jb 1f 144 145 shrl $2,%ecx /* copy by 32-bit words */ 146 rep 147 movsl 148 movl 16(%ebp),%ecx 149 andl $3,%ecx /* any bytes left? */ 150 rep 151 movsb 152 popl %edi 153 popl %esi 154 movl 8(%ebp),%eax /* return dst for memmove */ 155 popl %ebp 156 ret 157 158 ALIGN_TEXT 1591: 160 addl %ecx,%edi /* copy backwards */ 161 addl %ecx,%esi 162 decl %edi 163 decl %esi 164 andl $3,%ecx /* any fractional bytes? */ 165 std 166 rep 167 movsb 168 movl 16(%ebp),%ecx /* copy remainder by 32-bit words */ 169 shrl $2,%ecx 170 subl $3,%esi 171 subl $3,%edi 172 rep 173 movsl 174 popl %edi 175 popl %esi 176 cld 177 movl 8(%ebp),%eax /* return dst for memmove */ 178 popl %ebp 179 ret 180END(memmove) 181 182/* 183 * Note: memcpy does not support overlapping copies 184 */ 185ENTRY(memcpy) 186 pushl %edi 187 pushl %esi 188 movl 12(%esp),%edi 189 movl 16(%esp),%esi 190 movl 20(%esp),%ecx 191 movl %edi,%eax 192 shrl $2,%ecx /* copy by 32-bit words */ 193 rep 194 movsl 195 movl 20(%esp),%ecx 196 andl $3,%ecx /* any bytes left? */ 197 rep 198 movsb 199 popl %esi 200 popl %edi 201 ret 202END(memcpy) 203 204/* 205 * Handling of special 386 registers and descriptor tables etc 206 */ 207/* void lgdt(struct region_descriptor *rdp); */ 208ENTRY(lgdt) 209 /* reload the descriptor table */ 210 movl 4(%esp),%eax 211 lgdt (%eax) 212 213 /* flush the prefetch q */ 214 jmp 1f 215 nop 2161: 217 /* reload "stale" selectors */ 218 movl $KDSEL,%eax 219 movl %eax,%ds 220 movl %eax,%es 221 movl %eax,%gs 222 movl %eax,%ss 223 movl $KPSEL,%eax 224 movl %eax,%fs 225 226 /* reload code selector by turning return into intersegmental return */ 227 movl (%esp),%eax 228 pushl %eax 229 movl $KCSEL,4(%esp) 230 lret 231END(lgdt) 232 233/* ssdtosd(*ssdp,*sdp) */ 234ENTRY(ssdtosd) 235 pushl %ebx 236 movl 8(%esp),%ecx 237 movl 8(%ecx),%ebx 238 shll $16,%ebx 239 movl (%ecx),%edx 240 roll $16,%edx 241 movb %dh,%bl 242 movb %dl,%bh 243 rorl $8,%ebx 244 movl 4(%ecx),%eax 245 movw %ax,%dx 246 andl $0xf0000,%eax 247 orl %eax,%ebx 248 movl 12(%esp),%ecx 249 movl %edx,(%ecx) 250 movl %ebx,4(%ecx) 251 popl %ebx 252 ret 253END(ssdtosd) 254 255/* void reset_dbregs() */ 256ENTRY(reset_dbregs) 257 movl $0,%eax 258 movl %eax,%dr7 /* disable all breakpoints first */ 259 movl %eax,%dr0 260 movl %eax,%dr1 261 movl %eax,%dr2 262 movl %eax,%dr3 263 movl %eax,%dr6 264 ret 265END(reset_dbregs) 266 267/*****************************************************************************/ 268/* setjump, longjump */ 269/*****************************************************************************/ 270 271ENTRY(setjmp) 272 movl 4(%esp),%eax 273 movl %ebx,(%eax) /* save ebx */ 274 movl %esp,4(%eax) /* save esp */ 275 movl %ebp,8(%eax) /* save ebp */ 276 movl %esi,12(%eax) /* save esi */ 277 movl %edi,16(%eax) /* save edi */ 278 movl (%esp),%edx /* get rta */ 279 movl %edx,20(%eax) /* save eip */ 280 xorl %eax,%eax /* return(0); */ 281 ret 282END(setjmp) 283 284ENTRY(longjmp) 285 movl 4(%esp),%eax 286 movl (%eax),%ebx /* restore ebx */ 287 movl 4(%eax),%esp /* restore esp */ 288 movl 8(%eax),%ebp /* restore ebp */ 289 movl 12(%eax),%esi /* restore esi */ 290 movl 16(%eax),%edi /* restore edi */ 291 movl 20(%eax),%edx /* get rta */ 292 movl %edx,(%esp) /* put in return frame */ 293 xorl %eax,%eax /* return(1); */ 294 incl %eax 295 ret 296END(longjmp) 297 298/* 299 * Support for reading MSRs in the safe manner. (Instead of panic on #gp, 300 * return an error.) 301 */ 302ENTRY(rdmsr_safe) 303/* int rdmsr_safe(u_int msr, uint64_t *data) */ 304 movl PCPU(CURPCB),%ecx 305 movl $msr_onfault,PCB_ONFAULT(%ecx) 306 307 movl 4(%esp),%ecx 308 rdmsr 309 movl 8(%esp),%ecx 310 movl %eax,(%ecx) 311 movl %edx,4(%ecx) 312 xorl %eax,%eax 313 314 movl PCPU(CURPCB),%ecx 315 movl %eax,PCB_ONFAULT(%ecx) 316 317 ret 318 319/* 320 * Support for writing MSRs in the safe manner. (Instead of panic on #gp, 321 * return an error.) 322 */ 323ENTRY(wrmsr_safe) 324/* int wrmsr_safe(u_int msr, uint64_t data) */ 325 movl PCPU(CURPCB),%ecx 326 movl $msr_onfault,PCB_ONFAULT(%ecx) 327 328 movl 4(%esp),%ecx 329 movl 8(%esp),%eax 330 movl 12(%esp),%edx 331 wrmsr 332 xorl %eax,%eax 333 334 movl PCPU(CURPCB),%ecx 335 movl %eax,PCB_ONFAULT(%ecx) 336 337 ret 338 339/* 340 * MSR operations fault handler 341 */ 342 ALIGN_TEXT 343msr_onfault: 344 movl PCPU(CURPCB),%ecx 345 movl $0,PCB_ONFAULT(%ecx) 346 movl $EFAULT,%eax 347 ret 348 349 .altmacro 350 .macro rsb_seq_label l 351rsb_seq_\l: 352 .endm 353 .macro rsb_call_label l 354 call rsb_seq_\l 355 .endm 356 .macro rsb_seq count 357 ll=1 358 .rept \count 359 rsb_call_label %(ll) 360 nop 361 rsb_seq_label %(ll) 362 addl $4,%esp 363 ll=ll+1 364 .endr 365 .endm 366 367ENTRY(rsb_flush) 368 rsb_seq 32 369 ret 370 371ENTRY(handle_ibrs_entry) 372 cmpb $0,hw_ibrs_ibpb_active 373 je 1f 374 movl $MSR_IA32_SPEC_CTRL,%ecx 375 rdmsr 376 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 377 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx 378 wrmsr 379 movb $1,PCPU(IBPB_SET) 380 /* 381 * i386 does not implement SMEP. 382 */ 3831: jmp rsb_flush 384END(handle_ibrs_entry) 385 386ENTRY(handle_ibrs_exit) 387 cmpb $0,PCPU(IBPB_SET) 388 je 1f 389 movl $MSR_IA32_SPEC_CTRL,%ecx 390 rdmsr 391 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax 392 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx 393 wrmsr 394 movb $0,PCPU(IBPB_SET) 3951: ret 396END(handle_ibrs_exit) 397 398ENTRY(mds_handler_void) 399 ret 400END(mds_handler_void) 401 402ENTRY(mds_handler_verw) 403 subl $4, %esp 404 movw %ds, (%esp) 405 verw (%esp) 406 addl $4, %esp 407 ret 408END(mds_handler_verw) 409 410ENTRY(mds_handler_ivb) 411 movl %cr0, %eax 412 testb $CR0_TS, %al 413 je 1f 414 clts 4151: movl PCPU(MDS_BUF), %edx 416 movdqa %xmm0, PCPU(MDS_TMP) 417 pxor %xmm0, %xmm0 418 419 lfence 420 orpd (%edx), %xmm0 421 orpd (%edx), %xmm0 422 mfence 423 movl $40, %ecx 424 addl $16, %edx 4252: movntdq %xmm0, (%edx) 426 addl $16, %edx 427 decl %ecx 428 jnz 2b 429 mfence 430 431 movdqa PCPU(MDS_TMP),%xmm0 432 testb $CR0_TS, %al 433 je 3f 434 movl %eax, %cr0 4353: ret 436END(mds_handler_ivb) 437 438ENTRY(mds_handler_bdw) 439 movl %cr0, %eax 440 testb $CR0_TS, %al 441 je 1f 442 clts 4431: movl PCPU(MDS_BUF), %ebx 444 movdqa %xmm0, PCPU(MDS_TMP) 445 pxor %xmm0, %xmm0 446 447 movl %ebx, %edi 448 movl %ebx, %esi 449 movl $40, %ecx 4502: movntdq %xmm0, (%ebx) 451 addl $16, %ebx 452 decl %ecx 453 jnz 2b 454 mfence 455 movl $1536, %ecx 456 rep; movsb 457 lfence 458 459 movdqa PCPU(MDS_TMP),%xmm0 460 testb $CR0_TS, %al 461 je 3f 462 movl %eax, %cr0 4633: ret 464END(mds_handler_bdw) 465 466ENTRY(mds_handler_skl_sse) 467 movl %cr0, %eax 468 testb $CR0_TS, %al 469 je 1f 470 clts 4711: movl PCPU(MDS_BUF), %edi 472 movl PCPU(MDS_BUF64), %edx 473 movdqa %xmm0, PCPU(MDS_TMP) 474 pxor %xmm0, %xmm0 475 476 lfence 477 orpd (%edx), %xmm0 478 orpd (%edx), %xmm0 479 xorl %eax, %eax 4802: clflushopt 5376(%edi, %eax, 8) 481 addl $8, %eax 482 cmpl $8 * 12, %eax 483 jb 2b 484 sfence 485 movl $6144, %ecx 486 xorl %eax, %eax 487 rep; stosb 488 mfence 489 490 movdqa PCPU(MDS_TMP), %xmm0 491 testb $CR0_TS, %al 492 je 3f 493 movl %eax, %cr0 4943: ret 495END(mds_handler_skl_sse) 496 497ENTRY(mds_handler_skl_avx) 498 movl %cr0, %eax 499 testb $CR0_TS, %al 500 je 1f 501 clts 5021: movl PCPU(MDS_BUF), %edi 503 movl PCPU(MDS_BUF64), %edx 504 vmovdqa %ymm0, PCPU(MDS_TMP) 505 vpxor %ymm0, %ymm0, %ymm0 506 507 lfence 508 vorpd (%edx), %ymm0, %ymm0 509 vorpd (%edx), %ymm0, %ymm0 510 xorl %eax, %eax 5112: clflushopt 5376(%edi, %eax, 8) 512 addl $8, %eax 513 cmpl $8 * 12, %eax 514 jb 2b 515 sfence 516 movl $6144, %ecx 517 xorl %eax, %eax 518 rep; stosb 519 mfence 520 521 vmovdqa PCPU(MDS_TMP), %ymm0 522 testb $CR0_TS, %al 523 je 3f 524 movl %eax, %cr0 5253: ret 526END(mds_handler_skl_avx) 527 528ENTRY(mds_handler_skl_avx512) 529 movl %cr0, %eax 530 testb $CR0_TS, %al 531 je 1f 532 clts 5331: movl PCPU(MDS_BUF), %edi 534 movl PCPU(MDS_BUF64), %edx 535 vmovdqa64 %zmm0, PCPU(MDS_TMP) 536 vpxord %zmm0, %zmm0, %zmm0 537 538 lfence 539 vorpd (%edx), %zmm0, %zmm0 540 vorpd (%edx), %zmm0, %zmm0 541 xorl %eax, %eax 5422: clflushopt 5376(%edi, %eax, 8) 543 addl $8, %eax 544 cmpl $8 * 12, %eax 545 jb 2b 546 sfence 547 movl $6144, %ecx 548 xorl %eax, %eax 549 rep; stosb 550 mfence 551 552 vmovdqa64 PCPU(MDS_TMP), %zmm0 553 testb $CR0_TS, %al 554 je 3f 555 movl %eax, %cr0 5563: ret 557END(mds_handler_skl_avx512) 558 559ENTRY(mds_handler_silvermont) 560 movl %cr0, %eax 561 testb $CR0_TS, %al 562 je 1f 563 clts 5641: movl PCPU(MDS_BUF), %edx 565 movdqa %xmm0, PCPU(MDS_TMP) 566 pxor %xmm0, %xmm0 567 568 movl $16, %ecx 5692: movntdq %xmm0, (%edx) 570 addl $16, %edx 571 decl %ecx 572 jnz 2b 573 mfence 574 575 movdqa PCPU(MDS_TMP),%xmm0 576 testb $CR0_TS, %al 577 je 3f 578 movl %eax, %cr0 5793: ret 580END(mds_handler_silvermont) 581 582ENTRY(cpu_sync_core) 583 popl %eax 584 pushfl 585 pushl %cs 586 pushl %eax 587 iretl 588END(cpu_sync_core) 589