1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * AES-NI + SSE2 implementation of AEGIS-128 4 * 5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com> 6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. 7 */ 8 9#include <linux/linkage.h> 10#include <asm/frame.h> 11 12#define STATE0 %xmm0 13#define STATE1 %xmm1 14#define STATE2 %xmm2 15#define STATE3 %xmm3 16#define STATE4 %xmm4 17#define KEY %xmm5 18#define MSG %xmm5 19#define T0 %xmm6 20#define T1 %xmm7 21 22#define STATEP %rdi 23#define LEN %rsi 24#define SRC %rdx 25#define DST %rcx 26 27.section .rodata.cst16.aegis128_const, "aM", @progbits, 32 28.align 16 29.Laegis128_const_0: 30 .byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d 31 .byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62 32.Laegis128_const_1: 33 .byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1 34 .byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd 35 36.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16 37.align 16 38.Laegis128_counter: 39 .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 40 .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f 41 42.text 43 44/* 45 * aegis128_update 46 * input: 47 * STATE[0-4] - input state 48 * output: 49 * STATE[0-4] - output state (shifted positions) 50 * changed: 51 * T0 52 */ 53.macro aegis128_update 54 movdqa STATE4, T0 55 aesenc STATE0, STATE4 56 aesenc STATE1, STATE0 57 aesenc STATE2, STATE1 58 aesenc STATE3, STATE2 59 aesenc T0, STATE3 60.endm 61 62/* 63 * __load_partial: internal ABI 64 * input: 65 * LEN - bytes 66 * SRC - src 67 * output: 68 * MSG - message block 69 * changed: 70 * T0 71 * %r8 72 * %r9 73 */ 74SYM_FUNC_START_LOCAL(__load_partial) 75 xor %r9d, %r9d 76 pxor MSG, MSG 77 78 mov LEN, %r8 79 and $0x1, %r8 80 jz .Lld_partial_1 81 82 mov LEN, %r8 83 and $0x1E, %r8 84 add SRC, %r8 85 mov (%r8), %r9b 86 87.Lld_partial_1: 88 mov LEN, %r8 89 and $0x2, %r8 90 jz .Lld_partial_2 91 92 mov LEN, %r8 93 and $0x1C, %r8 94 add SRC, %r8 95 shl $0x10, %r9 96 mov (%r8), %r9w 97 98.Lld_partial_2: 99 mov LEN, %r8 100 and $0x4, %r8 101 jz .Lld_partial_4 102 103 mov LEN, %r8 104 and $0x18, %r8 105 add SRC, %r8 106 shl $32, %r9 107 mov (%r8), %r8d 108 xor %r8, %r9 109 110.Lld_partial_4: 111 movq %r9, MSG 112 113 mov LEN, %r8 114 and $0x8, %r8 115 jz .Lld_partial_8 116 117 mov LEN, %r8 118 and $0x10, %r8 119 add SRC, %r8 120 pslldq $8, MSG 121 movq (%r8), T0 122 pxor T0, MSG 123 124.Lld_partial_8: 125 ret 126SYM_FUNC_END(__load_partial) 127 128/* 129 * __store_partial: internal ABI 130 * input: 131 * LEN - bytes 132 * DST - dst 133 * output: 134 * T0 - message block 135 * changed: 136 * %r8 137 * %r9 138 * %r10 139 */ 140SYM_FUNC_START_LOCAL(__store_partial) 141 mov LEN, %r8 142 mov DST, %r9 143 144 movq T0, %r10 145 146 cmp $8, %r8 147 jl .Lst_partial_8 148 149 mov %r10, (%r9) 150 psrldq $8, T0 151 movq T0, %r10 152 153 sub $8, %r8 154 add $8, %r9 155 156.Lst_partial_8: 157 cmp $4, %r8 158 jl .Lst_partial_4 159 160 mov %r10d, (%r9) 161 shr $32, %r10 162 163 sub $4, %r8 164 add $4, %r9 165 166.Lst_partial_4: 167 cmp $2, %r8 168 jl .Lst_partial_2 169 170 mov %r10w, (%r9) 171 shr $0x10, %r10 172 173 sub $2, %r8 174 add $2, %r9 175 176.Lst_partial_2: 177 cmp $1, %r8 178 jl .Lst_partial_1 179 180 mov %r10b, (%r9) 181 182.Lst_partial_1: 183 ret 184SYM_FUNC_END(__store_partial) 185 186/* 187 * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv); 188 */ 189SYM_FUNC_START(crypto_aegis128_aesni_init) 190 FRAME_BEGIN 191 192 /* load IV: */ 193 movdqu (%rdx), T1 194 195 /* load key: */ 196 movdqa (%rsi), KEY 197 pxor KEY, T1 198 movdqa T1, STATE0 199 movdqa KEY, STATE3 200 movdqa KEY, STATE4 201 202 /* load the constants: */ 203 movdqa .Laegis128_const_0, STATE2 204 movdqa .Laegis128_const_1, STATE1 205 pxor STATE2, STATE3 206 pxor STATE1, STATE4 207 208 /* update 10 times with KEY / KEY xor IV: */ 209 aegis128_update; pxor KEY, STATE4 210 aegis128_update; pxor T1, STATE3 211 aegis128_update; pxor KEY, STATE2 212 aegis128_update; pxor T1, STATE1 213 aegis128_update; pxor KEY, STATE0 214 aegis128_update; pxor T1, STATE4 215 aegis128_update; pxor KEY, STATE3 216 aegis128_update; pxor T1, STATE2 217 aegis128_update; pxor KEY, STATE1 218 aegis128_update; pxor T1, STATE0 219 220 /* store the state: */ 221 movdqu STATE0, 0x00(STATEP) 222 movdqu STATE1, 0x10(STATEP) 223 movdqu STATE2, 0x20(STATEP) 224 movdqu STATE3, 0x30(STATEP) 225 movdqu STATE4, 0x40(STATEP) 226 227 FRAME_END 228 ret 229SYM_FUNC_END(crypto_aegis128_aesni_init) 230 231/* 232 * void crypto_aegis128_aesni_ad(void *state, unsigned int length, 233 * const void *data); 234 */ 235SYM_FUNC_START(crypto_aegis128_aesni_ad) 236 FRAME_BEGIN 237 238 cmp $0x10, LEN 239 jb .Lad_out 240 241 /* load the state: */ 242 movdqu 0x00(STATEP), STATE0 243 movdqu 0x10(STATEP), STATE1 244 movdqu 0x20(STATEP), STATE2 245 movdqu 0x30(STATEP), STATE3 246 movdqu 0x40(STATEP), STATE4 247 248 mov SRC, %r8 249 and $0xF, %r8 250 jnz .Lad_u_loop 251 252.align 8 253.Lad_a_loop: 254 movdqa 0x00(SRC), MSG 255 aegis128_update 256 pxor MSG, STATE4 257 sub $0x10, LEN 258 cmp $0x10, LEN 259 jl .Lad_out_1 260 261 movdqa 0x10(SRC), MSG 262 aegis128_update 263 pxor MSG, STATE3 264 sub $0x10, LEN 265 cmp $0x10, LEN 266 jl .Lad_out_2 267 268 movdqa 0x20(SRC), MSG 269 aegis128_update 270 pxor MSG, STATE2 271 sub $0x10, LEN 272 cmp $0x10, LEN 273 jl .Lad_out_3 274 275 movdqa 0x30(SRC), MSG 276 aegis128_update 277 pxor MSG, STATE1 278 sub $0x10, LEN 279 cmp $0x10, LEN 280 jl .Lad_out_4 281 282 movdqa 0x40(SRC), MSG 283 aegis128_update 284 pxor MSG, STATE0 285 sub $0x10, LEN 286 cmp $0x10, LEN 287 jl .Lad_out_0 288 289 add $0x50, SRC 290 jmp .Lad_a_loop 291 292.align 8 293.Lad_u_loop: 294 movdqu 0x00(SRC), MSG 295 aegis128_update 296 pxor MSG, STATE4 297 sub $0x10, LEN 298 cmp $0x10, LEN 299 jl .Lad_out_1 300 301 movdqu 0x10(SRC), MSG 302 aegis128_update 303 pxor MSG, STATE3 304 sub $0x10, LEN 305 cmp $0x10, LEN 306 jl .Lad_out_2 307 308 movdqu 0x20(SRC), MSG 309 aegis128_update 310 pxor MSG, STATE2 311 sub $0x10, LEN 312 cmp $0x10, LEN 313 jl .Lad_out_3 314 315 movdqu 0x30(SRC), MSG 316 aegis128_update 317 pxor MSG, STATE1 318 sub $0x10, LEN 319 cmp $0x10, LEN 320 jl .Lad_out_4 321 322 movdqu 0x40(SRC), MSG 323 aegis128_update 324 pxor MSG, STATE0 325 sub $0x10, LEN 326 cmp $0x10, LEN 327 jl .Lad_out_0 328 329 add $0x50, SRC 330 jmp .Lad_u_loop 331 332 /* store the state: */ 333.Lad_out_0: 334 movdqu STATE0, 0x00(STATEP) 335 movdqu STATE1, 0x10(STATEP) 336 movdqu STATE2, 0x20(STATEP) 337 movdqu STATE3, 0x30(STATEP) 338 movdqu STATE4, 0x40(STATEP) 339 FRAME_END 340 ret 341 342.Lad_out_1: 343 movdqu STATE4, 0x00(STATEP) 344 movdqu STATE0, 0x10(STATEP) 345 movdqu STATE1, 0x20(STATEP) 346 movdqu STATE2, 0x30(STATEP) 347 movdqu STATE3, 0x40(STATEP) 348 FRAME_END 349 ret 350 351.Lad_out_2: 352 movdqu STATE3, 0x00(STATEP) 353 movdqu STATE4, 0x10(STATEP) 354 movdqu STATE0, 0x20(STATEP) 355 movdqu STATE1, 0x30(STATEP) 356 movdqu STATE2, 0x40(STATEP) 357 FRAME_END 358 ret 359 360.Lad_out_3: 361 movdqu STATE2, 0x00(STATEP) 362 movdqu STATE3, 0x10(STATEP) 363 movdqu STATE4, 0x20(STATEP) 364 movdqu STATE0, 0x30(STATEP) 365 movdqu STATE1, 0x40(STATEP) 366 FRAME_END 367 ret 368 369.Lad_out_4: 370 movdqu STATE1, 0x00(STATEP) 371 movdqu STATE2, 0x10(STATEP) 372 movdqu STATE3, 0x20(STATEP) 373 movdqu STATE4, 0x30(STATEP) 374 movdqu STATE0, 0x40(STATEP) 375 FRAME_END 376 ret 377 378.Lad_out: 379 FRAME_END 380 ret 381SYM_FUNC_END(crypto_aegis128_aesni_ad) 382 383.macro encrypt_block a s0 s1 s2 s3 s4 i 384 movdq\a (\i * 0x10)(SRC), MSG 385 movdqa MSG, T0 386 pxor \s1, T0 387 pxor \s4, T0 388 movdqa \s2, T1 389 pand \s3, T1 390 pxor T1, T0 391 movdq\a T0, (\i * 0x10)(DST) 392 393 aegis128_update 394 pxor MSG, \s4 395 396 sub $0x10, LEN 397 cmp $0x10, LEN 398 jl .Lenc_out_\i 399.endm 400 401/* 402 * void crypto_aegis128_aesni_enc(void *state, unsigned int length, 403 * const void *src, void *dst); 404 */ 405SYM_FUNC_START(crypto_aegis128_aesni_enc) 406 FRAME_BEGIN 407 408 cmp $0x10, LEN 409 jb .Lenc_out 410 411 /* load the state: */ 412 movdqu 0x00(STATEP), STATE0 413 movdqu 0x10(STATEP), STATE1 414 movdqu 0x20(STATEP), STATE2 415 movdqu 0x30(STATEP), STATE3 416 movdqu 0x40(STATEP), STATE4 417 418 mov SRC, %r8 419 or DST, %r8 420 and $0xF, %r8 421 jnz .Lenc_u_loop 422 423.align 8 424.Lenc_a_loop: 425 encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0 426 encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1 427 encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2 428 encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3 429 encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4 430 431 add $0x50, SRC 432 add $0x50, DST 433 jmp .Lenc_a_loop 434 435.align 8 436.Lenc_u_loop: 437 encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0 438 encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1 439 encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2 440 encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3 441 encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4 442 443 add $0x50, SRC 444 add $0x50, DST 445 jmp .Lenc_u_loop 446 447 /* store the state: */ 448.Lenc_out_0: 449 movdqu STATE4, 0x00(STATEP) 450 movdqu STATE0, 0x10(STATEP) 451 movdqu STATE1, 0x20(STATEP) 452 movdqu STATE2, 0x30(STATEP) 453 movdqu STATE3, 0x40(STATEP) 454 FRAME_END 455 ret 456 457.Lenc_out_1: 458 movdqu STATE3, 0x00(STATEP) 459 movdqu STATE4, 0x10(STATEP) 460 movdqu STATE0, 0x20(STATEP) 461 movdqu STATE1, 0x30(STATEP) 462 movdqu STATE2, 0x40(STATEP) 463 FRAME_END 464 ret 465 466.Lenc_out_2: 467 movdqu STATE2, 0x00(STATEP) 468 movdqu STATE3, 0x10(STATEP) 469 movdqu STATE4, 0x20(STATEP) 470 movdqu STATE0, 0x30(STATEP) 471 movdqu STATE1, 0x40(STATEP) 472 FRAME_END 473 ret 474 475.Lenc_out_3: 476 movdqu STATE1, 0x00(STATEP) 477 movdqu STATE2, 0x10(STATEP) 478 movdqu STATE3, 0x20(STATEP) 479 movdqu STATE4, 0x30(STATEP) 480 movdqu STATE0, 0x40(STATEP) 481 FRAME_END 482 ret 483 484.Lenc_out_4: 485 movdqu STATE0, 0x00(STATEP) 486 movdqu STATE1, 0x10(STATEP) 487 movdqu STATE2, 0x20(STATEP) 488 movdqu STATE3, 0x30(STATEP) 489 movdqu STATE4, 0x40(STATEP) 490 FRAME_END 491 ret 492 493.Lenc_out: 494 FRAME_END 495 ret 496SYM_FUNC_END(crypto_aegis128_aesni_enc) 497 498/* 499 * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length, 500 * const void *src, void *dst); 501 */ 502SYM_FUNC_START(crypto_aegis128_aesni_enc_tail) 503 FRAME_BEGIN 504 505 /* load the state: */ 506 movdqu 0x00(STATEP), STATE0 507 movdqu 0x10(STATEP), STATE1 508 movdqu 0x20(STATEP), STATE2 509 movdqu 0x30(STATEP), STATE3 510 movdqu 0x40(STATEP), STATE4 511 512 /* encrypt message: */ 513 call __load_partial 514 515 movdqa MSG, T0 516 pxor STATE1, T0 517 pxor STATE4, T0 518 movdqa STATE2, T1 519 pand STATE3, T1 520 pxor T1, T0 521 522 call __store_partial 523 524 aegis128_update 525 pxor MSG, STATE4 526 527 /* store the state: */ 528 movdqu STATE4, 0x00(STATEP) 529 movdqu STATE0, 0x10(STATEP) 530 movdqu STATE1, 0x20(STATEP) 531 movdqu STATE2, 0x30(STATEP) 532 movdqu STATE3, 0x40(STATEP) 533 534 FRAME_END 535 ret 536SYM_FUNC_END(crypto_aegis128_aesni_enc_tail) 537 538.macro decrypt_block a s0 s1 s2 s3 s4 i 539 movdq\a (\i * 0x10)(SRC), MSG 540 pxor \s1, MSG 541 pxor \s4, MSG 542 movdqa \s2, T1 543 pand \s3, T1 544 pxor T1, MSG 545 movdq\a MSG, (\i * 0x10)(DST) 546 547 aegis128_update 548 pxor MSG, \s4 549 550 sub $0x10, LEN 551 cmp $0x10, LEN 552 jl .Ldec_out_\i 553.endm 554 555/* 556 * void crypto_aegis128_aesni_dec(void *state, unsigned int length, 557 * const void *src, void *dst); 558 */ 559SYM_FUNC_START(crypto_aegis128_aesni_dec) 560 FRAME_BEGIN 561 562 cmp $0x10, LEN 563 jb .Ldec_out 564 565 /* load the state: */ 566 movdqu 0x00(STATEP), STATE0 567 movdqu 0x10(STATEP), STATE1 568 movdqu 0x20(STATEP), STATE2 569 movdqu 0x30(STATEP), STATE3 570 movdqu 0x40(STATEP), STATE4 571 572 mov SRC, %r8 573 or DST, %r8 574 and $0xF, %r8 575 jnz .Ldec_u_loop 576 577.align 8 578.Ldec_a_loop: 579 decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0 580 decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1 581 decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2 582 decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3 583 decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4 584 585 add $0x50, SRC 586 add $0x50, DST 587 jmp .Ldec_a_loop 588 589.align 8 590.Ldec_u_loop: 591 decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0 592 decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1 593 decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2 594 decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3 595 decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4 596 597 add $0x50, SRC 598 add $0x50, DST 599 jmp .Ldec_u_loop 600 601 /* store the state: */ 602.Ldec_out_0: 603 movdqu STATE4, 0x00(STATEP) 604 movdqu STATE0, 0x10(STATEP) 605 movdqu STATE1, 0x20(STATEP) 606 movdqu STATE2, 0x30(STATEP) 607 movdqu STATE3, 0x40(STATEP) 608 FRAME_END 609 ret 610 611.Ldec_out_1: 612 movdqu STATE3, 0x00(STATEP) 613 movdqu STATE4, 0x10(STATEP) 614 movdqu STATE0, 0x20(STATEP) 615 movdqu STATE1, 0x30(STATEP) 616 movdqu STATE2, 0x40(STATEP) 617 FRAME_END 618 ret 619 620.Ldec_out_2: 621 movdqu STATE2, 0x00(STATEP) 622 movdqu STATE3, 0x10(STATEP) 623 movdqu STATE4, 0x20(STATEP) 624 movdqu STATE0, 0x30(STATEP) 625 movdqu STATE1, 0x40(STATEP) 626 FRAME_END 627 ret 628 629.Ldec_out_3: 630 movdqu STATE1, 0x00(STATEP) 631 movdqu STATE2, 0x10(STATEP) 632 movdqu STATE3, 0x20(STATEP) 633 movdqu STATE4, 0x30(STATEP) 634 movdqu STATE0, 0x40(STATEP) 635 FRAME_END 636 ret 637 638.Ldec_out_4: 639 movdqu STATE0, 0x00(STATEP) 640 movdqu STATE1, 0x10(STATEP) 641 movdqu STATE2, 0x20(STATEP) 642 movdqu STATE3, 0x30(STATEP) 643 movdqu STATE4, 0x40(STATEP) 644 FRAME_END 645 ret 646 647.Ldec_out: 648 FRAME_END 649 ret 650SYM_FUNC_END(crypto_aegis128_aesni_dec) 651 652/* 653 * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length, 654 * const void *src, void *dst); 655 */ 656SYM_FUNC_START(crypto_aegis128_aesni_dec_tail) 657 FRAME_BEGIN 658 659 /* load the state: */ 660 movdqu 0x00(STATEP), STATE0 661 movdqu 0x10(STATEP), STATE1 662 movdqu 0x20(STATEP), STATE2 663 movdqu 0x30(STATEP), STATE3 664 movdqu 0x40(STATEP), STATE4 665 666 /* decrypt message: */ 667 call __load_partial 668 669 pxor STATE1, MSG 670 pxor STATE4, MSG 671 movdqa STATE2, T1 672 pand STATE3, T1 673 pxor T1, MSG 674 675 movdqa MSG, T0 676 call __store_partial 677 678 /* mask with byte count: */ 679 movq LEN, T0 680 punpcklbw T0, T0 681 punpcklbw T0, T0 682 punpcklbw T0, T0 683 punpcklbw T0, T0 684 movdqa .Laegis128_counter, T1 685 pcmpgtb T1, T0 686 pand T0, MSG 687 688 aegis128_update 689 pxor MSG, STATE4 690 691 /* store the state: */ 692 movdqu STATE4, 0x00(STATEP) 693 movdqu STATE0, 0x10(STATEP) 694 movdqu STATE1, 0x20(STATEP) 695 movdqu STATE2, 0x30(STATEP) 696 movdqu STATE3, 0x40(STATEP) 697 698 FRAME_END 699 ret 700SYM_FUNC_END(crypto_aegis128_aesni_dec_tail) 701 702/* 703 * void crypto_aegis128_aesni_final(void *state, void *tag_xor, 704 * u64 assoclen, u64 cryptlen); 705 */ 706SYM_FUNC_START(crypto_aegis128_aesni_final) 707 FRAME_BEGIN 708 709 /* load the state: */ 710 movdqu 0x00(STATEP), STATE0 711 movdqu 0x10(STATEP), STATE1 712 movdqu 0x20(STATEP), STATE2 713 movdqu 0x30(STATEP), STATE3 714 movdqu 0x40(STATEP), STATE4 715 716 /* prepare length block: */ 717 movq %rdx, MSG 718 movq %rcx, T0 719 pslldq $8, T0 720 pxor T0, MSG 721 psllq $3, MSG /* multiply by 8 (to get bit count) */ 722 723 pxor STATE3, MSG 724 725 /* update state: */ 726 aegis128_update; pxor MSG, STATE4 727 aegis128_update; pxor MSG, STATE3 728 aegis128_update; pxor MSG, STATE2 729 aegis128_update; pxor MSG, STATE1 730 aegis128_update; pxor MSG, STATE0 731 aegis128_update; pxor MSG, STATE4 732 aegis128_update; pxor MSG, STATE3 733 734 /* xor tag: */ 735 movdqu (%rsi), MSG 736 737 pxor STATE0, MSG 738 pxor STATE1, MSG 739 pxor STATE2, MSG 740 pxor STATE3, MSG 741 pxor STATE4, MSG 742 743 movdqu MSG, (%rsi) 744 745 FRAME_END 746 ret 747SYM_FUNC_END(crypto_aegis128_aesni_final) 748