1######################################################################## 2# Implement fast SHA-256 with AVX2 instructions. (x86_64) 3# 4# Copyright (C) 2013 Intel Corporation. 5# 6# Authors: 7# James Guilford <james.guilford@intel.com> 8# Kirk Yap <kirk.s.yap@intel.com> 9# Tim Chen <tim.c.chen@linux.intel.com> 10# 11# This software is available to you under a choice of one of two 12# licenses. You may choose to be licensed under the terms of the GNU 13# General Public License (GPL) Version 2, available from the file 14# COPYING in the main directory of this source tree, or the 15# OpenIB.org BSD license below: 16# 17# Redistribution and use in source and binary forms, with or 18# without modification, are permitted provided that the following 19# conditions are met: 20# 21# - Redistributions of source code must retain the above 22# copyright notice, this list of conditions and the following 23# disclaimer. 24# 25# - Redistributions in binary form must reproduce the above 26# copyright notice, this list of conditions and the following 27# disclaimer in the documentation and/or other materials 28# provided with the distribution. 29# 30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37# SOFTWARE. 38# 39######################################################################## 40# 41# This code is described in an Intel White-Paper: 42# "Fast SHA-256 Implementations on Intel Architecture Processors" 43# 44# To find it, surf to http://www.intel.com/p/en_US/embedded 45# and search for that title. 46# 47######################################################################## 48# This code schedules 2 blocks at a time, with 4 lanes per block 49######################################################################## 50 51#ifdef CONFIG_AS_AVX2 52#include <linux/linkage.h> 53 54## assume buffers not aligned 55#define VMOVDQ vmovdqu 56 57################################ Define Macros 58 59# addm [mem], reg 60# Add reg to mem using reg-mem add and store 61.macro addm p1 p2 62 add \p1, \p2 63 mov \p2, \p1 64.endm 65 66################################ 67 68X0 = %ymm4 69X1 = %ymm5 70X2 = %ymm6 71X3 = %ymm7 72 73# XMM versions of above 74XWORD0 = %xmm4 75XWORD1 = %xmm5 76XWORD2 = %xmm6 77XWORD3 = %xmm7 78 79XTMP0 = %ymm0 80XTMP1 = %ymm1 81XTMP2 = %ymm2 82XTMP3 = %ymm3 83XTMP4 = %ymm8 84XFER = %ymm9 85XTMP5 = %ymm11 86 87SHUF_00BA = %ymm10 # shuffle xBxA -> 00BA 88SHUF_DC00 = %ymm12 # shuffle xDxC -> DC00 89BYTE_FLIP_MASK = %ymm13 90 91X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK 92 93NUM_BLKS = %rdx # 3rd arg 94INP = %rsi # 2nd arg 95CTX = %rdi # 1st arg 96c = %ecx 97d = %r8d 98e = %edx # clobbers NUM_BLKS 99y3 = %esi # clobbers INP 100 101SRND = CTX # SRND is same register as CTX 102 103a = %eax 104b = %ebx 105f = %r9d 106g = %r10d 107h = %r11d 108old_h = %r11d 109 110T1 = %r12d 111y0 = %r13d 112y1 = %r14d 113y2 = %r15d 114 115 116_XFER_SIZE = 2*64*4 # 2 blocks, 64 rounds, 4 bytes/round 117_XMM_SAVE_SIZE = 0 118_INP_END_SIZE = 8 119_INP_SIZE = 8 120_CTX_SIZE = 8 121_RSP_SIZE = 8 122 123_XFER = 0 124_XMM_SAVE = _XFER + _XFER_SIZE 125_INP_END = _XMM_SAVE + _XMM_SAVE_SIZE 126_INP = _INP_END + _INP_END_SIZE 127_CTX = _INP + _INP_SIZE 128_RSP = _CTX + _CTX_SIZE 129STACK_SIZE = _RSP + _RSP_SIZE 130 131# rotate_Xs 132# Rotate values of symbols X0...X3 133.macro rotate_Xs 134 X_ = X0 135 X0 = X1 136 X1 = X2 137 X2 = X3 138 X3 = X_ 139.endm 140 141# ROTATE_ARGS 142# Rotate values of symbols a...h 143.macro ROTATE_ARGS 144 old_h = h 145 TMP_ = h 146 h = g 147 g = f 148 f = e 149 e = d 150 d = c 151 c = b 152 b = a 153 a = TMP_ 154.endm 155 156.macro FOUR_ROUNDS_AND_SCHED disp 157################################### RND N + 0 ############################ 158 159 mov a, y3 # y3 = a # MAJA 160 rorx $25, e, y0 # y0 = e >> 25 # S1A 161 rorx $11, e, y1 # y1 = e >> 11 # S1B 162 163 addl \disp(%rsp, SRND), h # h = k + w + h # -- 164 or c, y3 # y3 = a|c # MAJA 165 vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] 166 mov f, y2 # y2 = f # CH 167 rorx $13, a, T1 # T1 = a >> 13 # S0B 168 169 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 170 xor g, y2 # y2 = f^g # CH 171 vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1 172 rorx $6, e, y1 # y1 = (e >> 6) # S1 173 174 and e, y2 # y2 = (f^g)&e # CH 175 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 176 rorx $22, a, y1 # y1 = a >> 22 # S0A 177 add h, d # d = k + w + h + d # -- 178 179 and b, y3 # y3 = (a|c)&b # MAJA 180 vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] 181 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 182 rorx $2, a, T1 # T1 = (a >> 2) # S0 183 184 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 185 vpsrld $7, XTMP1, XTMP2 186 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 187 mov a, T1 # T1 = a # MAJB 188 and c, T1 # T1 = a&c # MAJB 189 190 add y0, y2 # y2 = S1 + CH # -- 191 vpslld $(32-7), XTMP1, XTMP3 192 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 193 add y1, h # h = k + w + h + S0 # -- 194 195 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 196 vpor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 197 198 vpsrld $18, XTMP1, XTMP2 199 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 200 add y3, h # h = t1 + S0 + MAJ # -- 201 202 203 ROTATE_ARGS 204 205################################### RND N + 1 ############################ 206 207 mov a, y3 # y3 = a # MAJA 208 rorx $25, e, y0 # y0 = e >> 25 # S1A 209 rorx $11, e, y1 # y1 = e >> 11 # S1B 210 offset = \disp + 1*4 211 addl offset(%rsp, SRND), h # h = k + w + h # -- 212 or c, y3 # y3 = a|c # MAJA 213 214 215 vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 216 mov f, y2 # y2 = f # CH 217 rorx $13, a, T1 # T1 = a >> 13 # S0B 218 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 219 xor g, y2 # y2 = f^g # CH 220 221 222 rorx $6, e, y1 # y1 = (e >> 6) # S1 223 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 224 rorx $22, a, y1 # y1 = a >> 22 # S0A 225 and e, y2 # y2 = (f^g)&e # CH 226 add h, d # d = k + w + h + d # -- 227 228 vpslld $(32-18), XTMP1, XTMP1 229 and b, y3 # y3 = (a|c)&b # MAJA 230 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 231 232 vpxor XTMP1, XTMP3, XTMP3 233 rorx $2, a, T1 # T1 = (a >> 2) # S0 234 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 235 236 vpxor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 ^ W[-15] ror 18 237 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 238 mov a, T1 # T1 = a # MAJB 239 and c, T1 # T1 = a&c # MAJB 240 add y0, y2 # y2 = S1 + CH # -- 241 242 vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 243 vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} 244 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 245 add y1, h # h = k + w + h + S0 # -- 246 247 vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 248 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 249 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 250 add y3, h # h = t1 + S0 + MAJ # -- 251 252 vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} 253 254 255 ROTATE_ARGS 256 257################################### RND N + 2 ############################ 258 259 mov a, y3 # y3 = a # MAJA 260 rorx $25, e, y0 # y0 = e >> 25 # S1A 261 offset = \disp + 2*4 262 addl offset(%rsp, SRND), h # h = k + w + h # -- 263 264 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} 265 rorx $11, e, y1 # y1 = e >> 11 # S1B 266 or c, y3 # y3 = a|c # MAJA 267 mov f, y2 # y2 = f # CH 268 xor g, y2 # y2 = f^g # CH 269 270 rorx $13, a, T1 # T1 = a >> 13 # S0B 271 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 272 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} 273 and e, y2 # y2 = (f^g)&e # CH 274 275 rorx $6, e, y1 # y1 = (e >> 6) # S1 276 vpxor XTMP3, XTMP2, XTMP2 277 add h, d # d = k + w + h + d # -- 278 and b, y3 # y3 = (a|c)&b # MAJA 279 280 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 281 rorx $22, a, y1 # y1 = a >> 22 # S0A 282 vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} 283 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 284 285 vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} 286 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 287 rorx $2, a ,T1 # T1 = (a >> 2) # S0 288 vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} 289 290 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 291 mov a, T1 # T1 = a # MAJB 292 and c, T1 # T1 = a&c # MAJB 293 add y0, y2 # y2 = S1 + CH # -- 294 vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} 295 296 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 297 add y1,h # h = k + w + h + S0 # -- 298 add y2,d # d = k + w + h + d + S1 + CH = d + t1 # -- 299 add y2,h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 300 301 add y3,h # h = t1 + S0 + MAJ # -- 302 303 304 ROTATE_ARGS 305 306################################### RND N + 3 ############################ 307 308 mov a, y3 # y3 = a # MAJA 309 rorx $25, e, y0 # y0 = e >> 25 # S1A 310 rorx $11, e, y1 # y1 = e >> 11 # S1B 311 offset = \disp + 3*4 312 addl offset(%rsp, SRND), h # h = k + w + h # -- 313 or c, y3 # y3 = a|c # MAJA 314 315 316 vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} 317 mov f, y2 # y2 = f # CH 318 rorx $13, a, T1 # T1 = a >> 13 # S0B 319 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 320 xor g, y2 # y2 = f^g # CH 321 322 323 vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} 324 rorx $6, e, y1 # y1 = (e >> 6) # S1 325 and e, y2 # y2 = (f^g)&e # CH 326 add h, d # d = k + w + h + d # -- 327 and b, y3 # y3 = (a|c)&b # MAJA 328 329 vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} 330 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 331 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 332 333 vpxor XTMP3, XTMP2, XTMP2 334 rorx $22, a, y1 # y1 = a >> 22 # S0A 335 add y0, y2 # y2 = S1 + CH # -- 336 337 vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} 338 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 339 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 340 341 rorx $2, a, T1 # T1 = (a >> 2) # S0 342 vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} 343 344 vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} 345 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 346 mov a, T1 # T1 = a # MAJB 347 and c, T1 # T1 = a&c # MAJB 348 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 349 350 add y1, h # h = k + w + h + S0 # -- 351 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 352 add y3, h # h = t1 + S0 + MAJ # -- 353 354 ROTATE_ARGS 355 rotate_Xs 356.endm 357 358.macro DO_4ROUNDS disp 359################################### RND N + 0 ########################### 360 361 mov f, y2 # y2 = f # CH 362 rorx $25, e, y0 # y0 = e >> 25 # S1A 363 rorx $11, e, y1 # y1 = e >> 11 # S1B 364 xor g, y2 # y2 = f^g # CH 365 366 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 367 rorx $6, e, y1 # y1 = (e >> 6) # S1 368 and e, y2 # y2 = (f^g)&e # CH 369 370 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 371 rorx $13, a, T1 # T1 = a >> 13 # S0B 372 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 373 rorx $22, a, y1 # y1 = a >> 22 # S0A 374 mov a, y3 # y3 = a # MAJA 375 376 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 377 rorx $2, a, T1 # T1 = (a >> 2) # S0 378 addl \disp(%rsp, SRND), h # h = k + w + h # -- 379 or c, y3 # y3 = a|c # MAJA 380 381 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 382 mov a, T1 # T1 = a # MAJB 383 and b, y3 # y3 = (a|c)&b # MAJA 384 and c, T1 # T1 = a&c # MAJB 385 add y0, y2 # y2 = S1 + CH # -- 386 387 388 add h, d # d = k + w + h + d # -- 389 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 390 add y1, h # h = k + w + h + S0 # -- 391 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 392 393 ROTATE_ARGS 394 395################################### RND N + 1 ########################### 396 397 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 398 mov f, y2 # y2 = f # CH 399 rorx $25, e, y0 # y0 = e >> 25 # S1A 400 rorx $11, e, y1 # y1 = e >> 11 # S1B 401 xor g, y2 # y2 = f^g # CH 402 403 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 404 rorx $6, e, y1 # y1 = (e >> 6) # S1 405 and e, y2 # y2 = (f^g)&e # CH 406 add y3, old_h # h = t1 + S0 + MAJ # -- 407 408 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 409 rorx $13, a, T1 # T1 = a >> 13 # S0B 410 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 411 rorx $22, a, y1 # y1 = a >> 22 # S0A 412 mov a, y3 # y3 = a # MAJA 413 414 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 415 rorx $2, a, T1 # T1 = (a >> 2) # S0 416 offset = 4*1 + \disp 417 addl offset(%rsp, SRND), h # h = k + w + h # -- 418 or c, y3 # y3 = a|c # MAJA 419 420 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 421 mov a, T1 # T1 = a # MAJB 422 and b, y3 # y3 = (a|c)&b # MAJA 423 and c, T1 # T1 = a&c # MAJB 424 add y0, y2 # y2 = S1 + CH # -- 425 426 427 add h, d # d = k + w + h + d # -- 428 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 429 add y1, h # h = k + w + h + S0 # -- 430 431 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 432 433 ROTATE_ARGS 434 435################################### RND N + 2 ############################## 436 437 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 438 mov f, y2 # y2 = f # CH 439 rorx $25, e, y0 # y0 = e >> 25 # S1A 440 rorx $11, e, y1 # y1 = e >> 11 # S1B 441 xor g, y2 # y2 = f^g # CH 442 443 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 444 rorx $6, e, y1 # y1 = (e >> 6) # S1 445 and e, y2 # y2 = (f^g)&e # CH 446 add y3, old_h # h = t1 + S0 + MAJ # -- 447 448 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 449 rorx $13, a, T1 # T1 = a >> 13 # S0B 450 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 451 rorx $22, a, y1 # y1 = a >> 22 # S0A 452 mov a, y3 # y3 = a # MAJA 453 454 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 455 rorx $2, a, T1 # T1 = (a >> 2) # S0 456 offset = 4*2 + \disp 457 addl offset(%rsp, SRND), h # h = k + w + h # -- 458 or c, y3 # y3 = a|c # MAJA 459 460 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 461 mov a, T1 # T1 = a # MAJB 462 and b, y3 # y3 = (a|c)&b # MAJA 463 and c, T1 # T1 = a&c # MAJB 464 add y0, y2 # y2 = S1 + CH # -- 465 466 467 add h, d # d = k + w + h + d # -- 468 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 469 add y1, h # h = k + w + h + S0 # -- 470 471 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 472 473 ROTATE_ARGS 474 475################################### RND N + 3 ########################### 476 477 add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 478 mov f, y2 # y2 = f # CH 479 rorx $25, e, y0 # y0 = e >> 25 # S1A 480 rorx $11, e, y1 # y1 = e >> 11 # S1B 481 xor g, y2 # y2 = f^g # CH 482 483 xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 484 rorx $6, e, y1 # y1 = (e >> 6) # S1 485 and e, y2 # y2 = (f^g)&e # CH 486 add y3, old_h # h = t1 + S0 + MAJ # -- 487 488 xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 489 rorx $13, a, T1 # T1 = a >> 13 # S0B 490 xor g, y2 # y2 = CH = ((f^g)&e)^g # CH 491 rorx $22, a, y1 # y1 = a >> 22 # S0A 492 mov a, y3 # y3 = a # MAJA 493 494 xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 495 rorx $2, a, T1 # T1 = (a >> 2) # S0 496 offset = 4*3 + \disp 497 addl offset(%rsp, SRND), h # h = k + w + h # -- 498 or c, y3 # y3 = a|c # MAJA 499 500 xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 501 mov a, T1 # T1 = a # MAJB 502 and b, y3 # y3 = (a|c)&b # MAJA 503 and c, T1 # T1 = a&c # MAJB 504 add y0, y2 # y2 = S1 + CH # -- 505 506 507 add h, d # d = k + w + h + d # -- 508 or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ 509 add y1, h # h = k + w + h + S0 # -- 510 511 add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- 512 513 514 add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- 515 516 add y3, h # h = t1 + S0 + MAJ # -- 517 518 ROTATE_ARGS 519 520.endm 521 522######################################################################## 523## void sha256_transform_rorx(void *input_data, UINT32 digest[8], UINT64 num_blks) 524## arg 1 : pointer to digest 525## arg 2 : pointer to input data 526## arg 3 : Num blocks 527######################################################################## 528.text 529SYM_FUNC_START(sha256_transform_rorx) 530.align 32 531 pushq %rbx 532 pushq %r12 533 pushq %r13 534 pushq %r14 535 pushq %r15 536 537 mov %rsp, %rax 538 subq $STACK_SIZE, %rsp 539 and $-32, %rsp # align rsp to 32 byte boundary 540 mov %rax, _RSP(%rsp) 541 542 543 shl $6, NUM_BLKS # convert to bytes 544 jz done_hash 545 lea -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block 546 mov NUM_BLKS, _INP_END(%rsp) 547 548 cmp NUM_BLKS, INP 549 je only_one_block 550 551 ## load initial digest 552 mov (CTX), a 553 mov 4*1(CTX), b 554 mov 4*2(CTX), c 555 mov 4*3(CTX), d 556 mov 4*4(CTX), e 557 mov 4*5(CTX), f 558 mov 4*6(CTX), g 559 mov 4*7(CTX), h 560 561 vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 562 vmovdqa _SHUF_00BA(%rip), SHUF_00BA 563 vmovdqa _SHUF_DC00(%rip), SHUF_DC00 564 565 mov CTX, _CTX(%rsp) 566 567loop0: 568 ## Load first 16 dwords from two blocks 569 VMOVDQ 0*32(INP),XTMP0 570 VMOVDQ 1*32(INP),XTMP1 571 VMOVDQ 2*32(INP),XTMP2 572 VMOVDQ 3*32(INP),XTMP3 573 574 ## byte swap data 575 vpshufb BYTE_FLIP_MASK, XTMP0, XTMP0 576 vpshufb BYTE_FLIP_MASK, XTMP1, XTMP1 577 vpshufb BYTE_FLIP_MASK, XTMP2, XTMP2 578 vpshufb BYTE_FLIP_MASK, XTMP3, XTMP3 579 580 ## transpose data into high/low halves 581 vperm2i128 $0x20, XTMP2, XTMP0, X0 582 vperm2i128 $0x31, XTMP2, XTMP0, X1 583 vperm2i128 $0x20, XTMP3, XTMP1, X2 584 vperm2i128 $0x31, XTMP3, XTMP1, X3 585 586last_block_enter: 587 add $64, INP 588 mov INP, _INP(%rsp) 589 590 ## schedule 48 input dwords, by doing 3 rounds of 12 each 591 xor SRND, SRND 592 593.align 16 594loop1: 595 vpaddd K256+0*32(SRND), X0, XFER 596 vmovdqa XFER, 0*32+_XFER(%rsp, SRND) 597 FOUR_ROUNDS_AND_SCHED _XFER + 0*32 598 599 vpaddd K256+1*32(SRND), X0, XFER 600 vmovdqa XFER, 1*32+_XFER(%rsp, SRND) 601 FOUR_ROUNDS_AND_SCHED _XFER + 1*32 602 603 vpaddd K256+2*32(SRND), X0, XFER 604 vmovdqa XFER, 2*32+_XFER(%rsp, SRND) 605 FOUR_ROUNDS_AND_SCHED _XFER + 2*32 606 607 vpaddd K256+3*32(SRND), X0, XFER 608 vmovdqa XFER, 3*32+_XFER(%rsp, SRND) 609 FOUR_ROUNDS_AND_SCHED _XFER + 3*32 610 611 add $4*32, SRND 612 cmp $3*4*32, SRND 613 jb loop1 614 615loop2: 616 ## Do last 16 rounds with no scheduling 617 vpaddd K256+0*32(SRND), X0, XFER 618 vmovdqa XFER, 0*32+_XFER(%rsp, SRND) 619 DO_4ROUNDS _XFER + 0*32 620 621 vpaddd K256+1*32(SRND), X1, XFER 622 vmovdqa XFER, 1*32+_XFER(%rsp, SRND) 623 DO_4ROUNDS _XFER + 1*32 624 add $2*32, SRND 625 626 vmovdqa X2, X0 627 vmovdqa X3, X1 628 629 cmp $4*4*32, SRND 630 jb loop2 631 632 mov _CTX(%rsp), CTX 633 mov _INP(%rsp), INP 634 635 addm (4*0)(CTX),a 636 addm (4*1)(CTX),b 637 addm (4*2)(CTX),c 638 addm (4*3)(CTX),d 639 addm (4*4)(CTX),e 640 addm (4*5)(CTX),f 641 addm (4*6)(CTX),g 642 addm (4*7)(CTX),h 643 644 cmp _INP_END(%rsp), INP 645 ja done_hash 646 647 #### Do second block using previously scheduled results 648 xor SRND, SRND 649.align 16 650loop3: 651 DO_4ROUNDS _XFER + 0*32 + 16 652 DO_4ROUNDS _XFER + 1*32 + 16 653 add $2*32, SRND 654 cmp $4*4*32, SRND 655 jb loop3 656 657 mov _CTX(%rsp), CTX 658 mov _INP(%rsp), INP 659 add $64, INP 660 661 addm (4*0)(CTX),a 662 addm (4*1)(CTX),b 663 addm (4*2)(CTX),c 664 addm (4*3)(CTX),d 665 addm (4*4)(CTX),e 666 addm (4*5)(CTX),f 667 addm (4*6)(CTX),g 668 addm (4*7)(CTX),h 669 670 cmp _INP_END(%rsp), INP 671 jb loop0 672 ja done_hash 673 674do_last_block: 675 VMOVDQ 0*16(INP),XWORD0 676 VMOVDQ 1*16(INP),XWORD1 677 VMOVDQ 2*16(INP),XWORD2 678 VMOVDQ 3*16(INP),XWORD3 679 680 vpshufb X_BYTE_FLIP_MASK, XWORD0, XWORD0 681 vpshufb X_BYTE_FLIP_MASK, XWORD1, XWORD1 682 vpshufb X_BYTE_FLIP_MASK, XWORD2, XWORD2 683 vpshufb X_BYTE_FLIP_MASK, XWORD3, XWORD3 684 685 jmp last_block_enter 686 687only_one_block: 688 689 ## load initial digest 690 mov (4*0)(CTX),a 691 mov (4*1)(CTX),b 692 mov (4*2)(CTX),c 693 mov (4*3)(CTX),d 694 mov (4*4)(CTX),e 695 mov (4*5)(CTX),f 696 mov (4*6)(CTX),g 697 mov (4*7)(CTX),h 698 699 vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK 700 vmovdqa _SHUF_00BA(%rip), SHUF_00BA 701 vmovdqa _SHUF_DC00(%rip), SHUF_DC00 702 703 mov CTX, _CTX(%rsp) 704 jmp do_last_block 705 706done_hash: 707 708 mov _RSP(%rsp), %rsp 709 710 popq %r15 711 popq %r14 712 popq %r13 713 popq %r12 714 popq %rbx 715 ret 716SYM_FUNC_END(sha256_transform_rorx) 717 718.section .rodata.cst512.K256, "aM", @progbits, 512 719.align 64 720K256: 721 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 722 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 723 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 724 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 725 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 726 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 727 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 728 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 729 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 730 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc 731 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 732 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da 733 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 734 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 735 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 736 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 737 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 738 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 739 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 740 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 741 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 742 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 743 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 744 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 745 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 746 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 747 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 748 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 749 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 750 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 751 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 752 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 753 754.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 755.align 32 756PSHUFFLE_BYTE_FLIP_MASK: 757 .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203 758 759# shuffle xBxA -> 00BA 760.section .rodata.cst32._SHUF_00BA, "aM", @progbits, 32 761.align 32 762_SHUF_00BA: 763 .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100 764 765# shuffle xDxC -> DC00 766.section .rodata.cst32._SHUF_DC00, "aM", @progbits, 32 767.align 32 768_SHUF_DC00: 769 .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF 770 771#endif 772