1#! /usr/bin/env perl 2# Copyright 2005-2018 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the OpenSSL license (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9 10$flavour = shift; 11$output = shift; 12if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 13 14$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 15 16$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 17( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 18( $xlate="${dir}perlasm/x86_64-xlate.pl" and -f $xlate) or 19die "can't locate x86_64-xlate.pl"; 20 21open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; 22*STDOUT=*OUT; 23 24($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order 25 ("%rdi","%rsi","%rdx","%rcx"); # Unix order 26 27print<<___; 28.extern OPENSSL_cpuid_setup 29.hidden OPENSSL_cpuid_setup 30.section .init 31 call OPENSSL_cpuid_setup 32 33.hidden OPENSSL_ia32cap_P 34.comm OPENSSL_ia32cap_P,16,4 35 36.text 37 38.globl OPENSSL_atomic_add 39.type OPENSSL_atomic_add,\@abi-omnipotent 40.align 16 41OPENSSL_atomic_add: 42 movl ($arg1),%eax 43.Lspin: leaq ($arg2,%rax),%r8 44 .byte 0xf0 # lock 45 cmpxchgl %r8d,($arg1) 46 jne .Lspin 47 movl %r8d,%eax 48 .byte 0x48,0x98 # cltq/cdqe 49 ret 50.size OPENSSL_atomic_add,.-OPENSSL_atomic_add 51 52.globl OPENSSL_rdtsc 53.type OPENSSL_rdtsc,\@abi-omnipotent 54.align 16 55OPENSSL_rdtsc: 56 rdtsc 57 shl \$32,%rdx 58 or %rdx,%rax 59 ret 60.size OPENSSL_rdtsc,.-OPENSSL_rdtsc 61 62.globl OPENSSL_ia32_cpuid 63.type OPENSSL_ia32_cpuid,\@function,1 64.align 16 65OPENSSL_ia32_cpuid: 66.cfi_startproc 67 mov %rbx,%r8 # save %rbx 68.cfi_register %rbx,%r8 69 70 xor %eax,%eax 71 mov %rax,8(%rdi) # clear extended feature flags 72 cpuid 73 mov %eax,%r11d # max value for standard query level 74 75 xor %eax,%eax 76 cmp \$0x756e6547,%ebx # "Genu" 77 setne %al 78 mov %eax,%r9d 79 cmp \$0x49656e69,%edx # "ineI" 80 setne %al 81 or %eax,%r9d 82 cmp \$0x6c65746e,%ecx # "ntel" 83 setne %al 84 or %eax,%r9d # 0 indicates Intel CPU 85 jz .Lintel 86 87 cmp \$0x68747541,%ebx # "Auth" 88 setne %al 89 mov %eax,%r10d 90 cmp \$0x69746E65,%edx # "enti" 91 setne %al 92 or %eax,%r10d 93 cmp \$0x444D4163,%ecx # "cAMD" 94 setne %al 95 or %eax,%r10d # 0 indicates AMD CPU 96 jnz .Lintel 97 98 # AMD specific 99 mov \$0x80000000,%eax 100 cpuid 101 cmp \$0x80000001,%eax 102 jb .Lintel 103 mov %eax,%r10d 104 mov \$0x80000001,%eax 105 cpuid 106 or %ecx,%r9d 107 and \$0x00000801,%r9d # isolate AMD XOP bit, 1<<11 108 109 cmp \$0x80000008,%r10d 110 jb .Lintel 111 112 mov \$0x80000008,%eax 113 cpuid 114 movzb %cl,%r10 # number of cores - 1 115 inc %r10 # number of cores 116 117 mov \$1,%eax 118 cpuid 119 bt \$28,%edx # test hyper-threading bit 120 jnc .Lgeneric 121 shr \$16,%ebx # number of logical processors 122 cmp %r10b,%bl 123 ja .Lgeneric 124 and \$0xefffffff,%edx # ~(1<<28) 125 jmp .Lgeneric 126 127.Lintel: 128 cmp \$4,%r11d 129 mov \$-1,%r10d 130 jb .Lnocacheinfo 131 132 mov \$4,%eax 133 mov \$0,%ecx # query L1D 134 cpuid 135 mov %eax,%r10d 136 shr \$14,%r10d 137 and \$0xfff,%r10d # number of cores -1 per L1D 138 139.Lnocacheinfo: 140 mov \$1,%eax 141 cpuid 142 movd %eax,%xmm0 # put aside processor id 143 and \$0xbfefffff,%edx # force reserved bits to 0 144 cmp \$0,%r9d 145 jne .Lnotintel 146 or \$0x40000000,%edx # set reserved bit#30 on Intel CPUs 147 and \$15,%ah 148 cmp \$15,%ah # examine Family ID 149 jne .LnotP4 150 or \$0x00100000,%edx # set reserved bit#20 to engage RC4_CHAR 151.LnotP4: 152 cmp \$6,%ah 153 jne .Lnotintel 154 and \$0x0fff0ff0,%eax 155 cmp \$0x00050670,%eax # Knights Landing 156 je .Lknights 157 cmp \$0x00080650,%eax # Knights Mill (according to sde) 158 jne .Lnotintel 159.Lknights: 160 and \$0xfbffffff,%ecx # clear XSAVE flag to mimic Silvermont 161 162.Lnotintel: 163 bt \$28,%edx # test hyper-threading bit 164 jnc .Lgeneric 165 and \$0xefffffff,%edx # ~(1<<28) 166 cmp \$0,%r10d 167 je .Lgeneric 168 169 or \$0x10000000,%edx # 1<<28 170 shr \$16,%ebx 171 cmp \$1,%bl # see if cache is shared 172 ja .Lgeneric 173 and \$0xefffffff,%edx # ~(1<<28) 174.Lgeneric: 175 and \$0x00000800,%r9d # isolate AMD XOP flag 176 and \$0xfffff7ff,%ecx 177 or %ecx,%r9d # merge AMD XOP flag 178 179 mov %edx,%r10d # %r9d:%r10d is copy of %ecx:%edx 180 181 cmp \$7,%r11d 182 jb .Lno_extended_info 183 mov \$7,%eax 184 xor %ecx,%ecx 185 cpuid 186 bt \$26,%r9d # check XSAVE bit, cleared on Knights 187 jc .Lnotknights 188 and \$0xfff7ffff,%ebx # clear ADCX/ADOX flag 189.Lnotknights: 190 movd %xmm0,%eax # restore processor id 191 and \$0x0fff0ff0,%eax 192 cmp \$0x00050650,%eax # Skylake-X 193 jne .Lnotskylakex 194 and \$0xfffeffff,%ebx # ~(1<<16) 195 # suppress AVX512F flag on Skylake-X 196.Lnotskylakex: 197 mov %ebx,8(%rdi) # save extended feature flags 198 mov %ecx,12(%rdi) 199.Lno_extended_info: 200 201 bt \$27,%r9d # check OSXSAVE bit 202 jnc .Lclear_avx 203 xor %ecx,%ecx # XCR0 204 .byte 0x0f,0x01,0xd0 # xgetbv 205 and \$0xe6,%eax # isolate XMM, YMM and ZMM state support 206 cmp \$0xe6,%eax 207 je .Ldone 208 andl \$0x3fdeffff,8(%rdi) # ~(1<<31|1<<30|1<<21|1<<16) 209 # clear AVX512F+BW+VL+FIMA, all of 210 # them are EVEX-encoded, which requires 211 # ZMM state support even if one uses 212 # only XMM and YMM :-( 213 and \$6,%eax # isolate XMM and YMM state support 214 cmp \$6,%eax 215 je .Ldone 216.Lclear_avx: 217 mov \$0xefffe7ff,%eax # ~(1<<28|1<<12|1<<11) 218 and %eax,%r9d # clear AVX, FMA and AMD XOP bits 219 mov \$0x3fdeffdf,%eax # ~(1<<31|1<<30|1<<21|1<<16|1<<5) 220 and %eax,8(%rdi) # clear AVX2 and AVX512* bits 221.Ldone: 222 shl \$32,%r9 223 mov %r10d,%eax 224 mov %r8,%rbx # restore %rbx 225.cfi_restore %rbx 226 or %r9,%rax 227 ret 228.cfi_endproc 229.size OPENSSL_ia32_cpuid,.-OPENSSL_ia32_cpuid 230 231.globl OPENSSL_cleanse 232.type OPENSSL_cleanse,\@abi-omnipotent 233.align 16 234OPENSSL_cleanse: 235 xor %rax,%rax 236 cmp \$15,$arg2 237 jae .Lot 238 cmp \$0,$arg2 239 je .Lret 240.Little: 241 mov %al,($arg1) 242 sub \$1,$arg2 243 lea 1($arg1),$arg1 244 jnz .Little 245.Lret: 246 ret 247.align 16 248.Lot: 249 test \$7,$arg1 250 jz .Laligned 251 mov %al,($arg1) 252 lea -1($arg2),$arg2 253 lea 1($arg1),$arg1 254 jmp .Lot 255.Laligned: 256 mov %rax,($arg1) 257 lea -8($arg2),$arg2 258 test \$-8,$arg2 259 lea 8($arg1),$arg1 260 jnz .Laligned 261 cmp \$0,$arg2 262 jne .Little 263 ret 264.size OPENSSL_cleanse,.-OPENSSL_cleanse 265 266.globl CRYPTO_memcmp 267.type CRYPTO_memcmp,\@abi-omnipotent 268.align 16 269CRYPTO_memcmp: 270 xor %rax,%rax 271 xor %r10,%r10 272 cmp \$0,$arg3 273 je .Lno_data 274 cmp \$16,$arg3 275 jne .Loop_cmp 276 mov ($arg1),%r10 277 mov 8($arg1),%r11 278 mov \$1,$arg3 279 xor ($arg2),%r10 280 xor 8($arg2),%r11 281 or %r11,%r10 282 cmovnz $arg3,%rax 283 ret 284 285.align 16 286.Loop_cmp: 287 mov ($arg1),%r10b 288 lea 1($arg1),$arg1 289 xor ($arg2),%r10b 290 lea 1($arg2),$arg2 291 or %r10b,%al 292 dec $arg3 293 jnz .Loop_cmp 294 neg %rax 295 shr \$63,%rax 296.Lno_data: 297 ret 298.size CRYPTO_memcmp,.-CRYPTO_memcmp 299___ 300 301print<<___ if (!$win64); 302.globl OPENSSL_wipe_cpu 303.type OPENSSL_wipe_cpu,\@abi-omnipotent 304.align 16 305OPENSSL_wipe_cpu: 306 pxor %xmm0,%xmm0 307 pxor %xmm1,%xmm1 308 pxor %xmm2,%xmm2 309 pxor %xmm3,%xmm3 310 pxor %xmm4,%xmm4 311 pxor %xmm5,%xmm5 312 pxor %xmm6,%xmm6 313 pxor %xmm7,%xmm7 314 pxor %xmm8,%xmm8 315 pxor %xmm9,%xmm9 316 pxor %xmm10,%xmm10 317 pxor %xmm11,%xmm11 318 pxor %xmm12,%xmm12 319 pxor %xmm13,%xmm13 320 pxor %xmm14,%xmm14 321 pxor %xmm15,%xmm15 322 xorq %rcx,%rcx 323 xorq %rdx,%rdx 324 xorq %rsi,%rsi 325 xorq %rdi,%rdi 326 xorq %r8,%r8 327 xorq %r9,%r9 328 xorq %r10,%r10 329 xorq %r11,%r11 330 leaq 8(%rsp),%rax 331 ret 332.size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu 333___ 334print<<___ if ($win64); 335.globl OPENSSL_wipe_cpu 336.type OPENSSL_wipe_cpu,\@abi-omnipotent 337.align 16 338OPENSSL_wipe_cpu: 339 pxor %xmm0,%xmm0 340 pxor %xmm1,%xmm1 341 pxor %xmm2,%xmm2 342 pxor %xmm3,%xmm3 343 pxor %xmm4,%xmm4 344 pxor %xmm5,%xmm5 345 xorq %rcx,%rcx 346 xorq %rdx,%rdx 347 xorq %r8,%r8 348 xorq %r9,%r9 349 xorq %r10,%r10 350 xorq %r11,%r11 351 leaq 8(%rsp),%rax 352 ret 353.size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu 354___ 355{ 356my $out="%r10"; 357my $cnt="%rcx"; 358my $max="%r11"; 359my $lasttick="%r8d"; 360my $lastdiff="%r9d"; 361my $redzone=win64?8:-8; 362 363print<<___; 364.globl OPENSSL_instrument_bus 365.type OPENSSL_instrument_bus,\@abi-omnipotent 366.align 16 367OPENSSL_instrument_bus: 368 mov $arg1,$out # tribute to Win64 369 mov $arg2,$cnt 370 mov $arg2,$max 371 372 rdtsc # collect 1st tick 373 mov %eax,$lasttick # lasttick = tick 374 mov \$0,$lastdiff # lastdiff = 0 375 clflush ($out) 376 .byte 0xf0 # lock 377 add $lastdiff,($out) 378 jmp .Loop 379.align 16 380.Loop: rdtsc 381 mov %eax,%edx 382 sub $lasttick,%eax 383 mov %edx,$lasttick 384 mov %eax,$lastdiff 385 clflush ($out) 386 .byte 0xf0 # lock 387 add %eax,($out) 388 lea 4($out),$out 389 sub \$1,$cnt 390 jnz .Loop 391 392 mov $max,%rax 393 ret 394.size OPENSSL_instrument_bus,.-OPENSSL_instrument_bus 395 396.globl OPENSSL_instrument_bus2 397.type OPENSSL_instrument_bus2,\@abi-omnipotent 398.align 16 399OPENSSL_instrument_bus2: 400 mov $arg1,$out # tribute to Win64 401 mov $arg2,$cnt 402 mov $arg3,$max 403 mov $cnt,$redzone(%rsp) 404 405 rdtsc # collect 1st tick 406 mov %eax,$lasttick # lasttick = tick 407 mov \$0,$lastdiff # lastdiff = 0 408 409 clflush ($out) 410 .byte 0xf0 # lock 411 add $lastdiff,($out) 412 413 rdtsc # collect 1st diff 414 mov %eax,%edx 415 sub $lasttick,%eax # diff 416 mov %edx,$lasttick # lasttick = tick 417 mov %eax,$lastdiff # lastdiff = diff 418.Loop2: 419 clflush ($out) 420 .byte 0xf0 # lock 421 add %eax,($out) # accumulate diff 422 423 sub \$1,$max 424 jz .Ldone2 425 426 rdtsc 427 mov %eax,%edx 428 sub $lasttick,%eax # diff 429 mov %edx,$lasttick # lasttick = tick 430 cmp $lastdiff,%eax 431 mov %eax,$lastdiff # lastdiff = diff 432 mov \$0,%edx 433 setne %dl 434 sub %rdx,$cnt # conditional --$cnt 435 lea ($out,%rdx,4),$out # conditional ++$out 436 jnz .Loop2 437 438.Ldone2: 439 mov $redzone(%rsp),%rax 440 sub $cnt,%rax 441 ret 442.size OPENSSL_instrument_bus2,.-OPENSSL_instrument_bus2 443___ 444} 445 446sub gen_random { 447my $rdop = shift; 448print<<___; 449.globl OPENSSL_ia32_${rdop}_bytes 450.type OPENSSL_ia32_${rdop}_bytes,\@abi-omnipotent 451.align 16 452OPENSSL_ia32_${rdop}_bytes: 453 xor %rax, %rax # return value 454 cmp \$0,$arg2 455 je .Ldone_${rdop}_bytes 456 457 mov \$8,%r11 458.Loop_${rdop}_bytes: 459 ${rdop} %r10 460 jc .Lbreak_${rdop}_bytes 461 dec %r11 462 jnz .Loop_${rdop}_bytes 463 jmp .Ldone_${rdop}_bytes 464 465.align 16 466.Lbreak_${rdop}_bytes: 467 cmp \$8,$arg2 468 jb .Ltail_${rdop}_bytes 469 mov %r10,($arg1) 470 lea 8($arg1),$arg1 471 add \$8,%rax 472 sub \$8,$arg2 473 jz .Ldone_${rdop}_bytes 474 mov \$8,%r11 475 jmp .Loop_${rdop}_bytes 476 477.align 16 478.Ltail_${rdop}_bytes: 479 mov %r10b,($arg1) 480 lea 1($arg1),$arg1 481 inc %rax 482 shr \$8,%r10 483 dec $arg2 484 jnz .Ltail_${rdop}_bytes 485 486.Ldone_${rdop}_bytes: 487 xor %r10,%r10 # Clear sensitive data from register 488 ret 489.size OPENSSL_ia32_${rdop}_bytes,.-OPENSSL_ia32_${rdop}_bytes 490___ 491} 492gen_random("rdrand"); 493gen_random("rdseed"); 494 495close STDOUT; # flush 496