1#! /usr/bin/env perl 2# Copyright 2011-2020 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the OpenSSL license (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9 10###################################################################### 11## Constant-time SSSE3 AES core implementation. 12## version 0.1 13## 14## By Mike Hamburg (Stanford University), 2009 15## Public domain. 16## 17## For details see http://shiftleft.org/papers/vector_aes/ and 18## http://crypto.stanford.edu/vpaes/. 19 20###################################################################### 21# September 2011. 22# 23# Port vpaes-x86_64.pl as 32-bit "almost" drop-in replacement for 24# aes-586.pl. "Almost" refers to the fact that AES_cbc_encrypt 25# doesn't handle partial vectors (doesn't have to if called from 26# EVP only). "Drop-in" implies that this module doesn't share key 27# schedule structure with the original nor does it make assumption 28# about its alignment... 29# 30# Performance summary. aes-586.pl column lists large-block CBC 31# encrypt/decrypt/with-hyper-threading-off(*) results in cycles per 32# byte processed with 128-bit key, and vpaes-x86.pl column - [also 33# large-block CBC] encrypt/decrypt. 34# 35# aes-586.pl vpaes-x86.pl 36# 37# Core 2(**) 28.1/41.4/18.3 21.9/25.2(***) 38# Nehalem 27.9/40.4/18.1 10.2/11.9 39# Atom 70.7/92.1/60.1 61.1/75.4(***) 40# Silvermont 45.4/62.9/24.1 49.2/61.1(***) 41# 42# (*) "Hyper-threading" in the context refers rather to cache shared 43# among multiple cores, than to specifically Intel HTT. As vast 44# majority of contemporary cores share cache, slower code path 45# is common place. In other words "with-hyper-threading-off" 46# results are presented mostly for reference purposes. 47# 48# (**) "Core 2" refers to initial 65nm design, a.k.a. Conroe. 49# 50# (***) Less impressive improvement on Core 2 and Atom is due to slow 51# pshufb, yet it's respectable +28%/64% improvement on Core 2 52# and +15% on Atom (as implied, over "hyper-threading-safe" 53# code path). 54# 55# <appro@openssl.org> 56 57$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 58push(@INC,"${dir}","${dir}../../perlasm"); 59require "x86asm.pl"; 60 61$output = pop; 62open OUT,">$output"; 63*STDOUT=*OUT; 64 65&asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386"); 66 67$PREFIX="vpaes"; 68 69my ($round, $base, $magic, $key, $const, $inp, $out)= 70 ("eax", "ebx", "ecx", "edx","ebp", "esi","edi"); 71 72&static_label("_vpaes_consts"); 73&static_label("_vpaes_schedule_low_round"); 74 75&set_label("_vpaes_consts",64); 76$k_inv=-0x30; # inv, inva 77 &data_word(0x0D080180,0x0E05060F,0x0A0B0C02,0x04070309); 78 &data_word(0x0F0B0780,0x01040A06,0x02050809,0x030D0E0C); 79 80$k_s0F=-0x10; # s0F 81 &data_word(0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F,0x0F0F0F0F); 82 83$k_ipt=0x00; # input transform (lo, hi) 84 &data_word(0x5A2A7000,0xC2B2E898,0x52227808,0xCABAE090); 85 &data_word(0x317C4D00,0x4C01307D,0xB0FDCC81,0xCD80B1FC); 86 87$k_sb1=0x20; # sb1u, sb1t 88 &data_word(0xCB503E00,0xB19BE18F,0x142AF544,0xA5DF7A6E); 89 &data_word(0xFAE22300,0x3618D415,0x0D2ED9EF,0x3BF7CCC1); 90$k_sb2=0x40; # sb2u, sb2t 91 &data_word(0x0B712400,0xE27A93C6,0xBC982FCD,0x5EB7E955); 92 &data_word(0x0AE12900,0x69EB8840,0xAB82234A,0xC2A163C8); 93$k_sbo=0x60; # sbou, sbot 94 &data_word(0x6FBDC700,0xD0D26D17,0xC502A878,0x15AABF7A); 95 &data_word(0x5FBB6A00,0xCFE474A5,0x412B35FA,0x8E1E90D1); 96 97$k_mc_forward=0x80; # mc_forward 98 &data_word(0x00030201,0x04070605,0x080B0A09,0x0C0F0E0D); 99 &data_word(0x04070605,0x080B0A09,0x0C0F0E0D,0x00030201); 100 &data_word(0x080B0A09,0x0C0F0E0D,0x00030201,0x04070605); 101 &data_word(0x0C0F0E0D,0x00030201,0x04070605,0x080B0A09); 102 103$k_mc_backward=0xc0; # mc_backward 104 &data_word(0x02010003,0x06050407,0x0A09080B,0x0E0D0C0F); 105 &data_word(0x0E0D0C0F,0x02010003,0x06050407,0x0A09080B); 106 &data_word(0x0A09080B,0x0E0D0C0F,0x02010003,0x06050407); 107 &data_word(0x06050407,0x0A09080B,0x0E0D0C0F,0x02010003); 108 109$k_sr=0x100; # sr 110 &data_word(0x03020100,0x07060504,0x0B0A0908,0x0F0E0D0C); 111 &data_word(0x0F0A0500,0x030E0904,0x07020D08,0x0B06010C); 112 &data_word(0x0B020900,0x0F060D04,0x030A0108,0x070E050C); 113 &data_word(0x070A0D00,0x0B0E0104,0x0F020508,0x0306090C); 114 115$k_rcon=0x140; # rcon 116 &data_word(0xAF9DEEB6,0x1F8391B9,0x4D7C7D81,0x702A9808); 117 118$k_s63=0x150; # s63: all equal to 0x63 transformed 119 &data_word(0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B,0x5B5B5B5B); 120 121$k_opt=0x160; # output transform 122 &data_word(0xD6B66000,0xFF9F4929,0xDEBE6808,0xF7974121); 123 &data_word(0x50BCEC00,0x01EDBD51,0xB05C0CE0,0xE10D5DB1); 124 125$k_deskew=0x180; # deskew tables: inverts the sbox's "skew" 126 &data_word(0x47A4E300,0x07E4A340,0x5DBEF91A,0x1DFEB95A); 127 &data_word(0x83EA6900,0x5F36B5DC,0xF49D1E77,0x2841C2AB); 128## 129## Decryption stuff 130## Key schedule constants 131## 132$k_dksd=0x1a0; # decryption key schedule: invskew x*D 133 &data_word(0xA3E44700,0xFEB91A5D,0x5A1DBEF9,0x0740E3A4); 134 &data_word(0xB5368300,0x41C277F4,0xAB289D1E,0x5FDC69EA); 135$k_dksb=0x1c0; # decryption key schedule: invskew x*B 136 &data_word(0x8550D500,0x9A4FCA1F,0x1CC94C99,0x03D65386); 137 &data_word(0xB6FC4A00,0x115BEDA7,0x7E3482C8,0xD993256F); 138$k_dkse=0x1e0; # decryption key schedule: invskew x*E + 0x63 139 &data_word(0x1FC9D600,0xD5031CCA,0x994F5086,0x53859A4C); 140 &data_word(0x4FDC7BE8,0xA2319605,0x20B31487,0xCD5EF96A); 141$k_dks9=0x200; # decryption key schedule: invskew x*9 142 &data_word(0x7ED9A700,0xB6116FC8,0x82255BFC,0x4AED9334); 143 &data_word(0x27143300,0x45765162,0xE9DAFDCE,0x8BB89FAC); 144 145## 146## Decryption stuff 147## Round function constants 148## 149$k_dipt=0x220; # decryption input transform 150 &data_word(0x0B545F00,0x0F505B04,0x114E451A,0x154A411E); 151 &data_word(0x60056500,0x86E383E6,0xF491F194,0x12771772); 152 153$k_dsb9=0x240; # decryption sbox output *9*u, *9*t 154 &data_word(0x9A86D600,0x851C0353,0x4F994CC9,0xCAD51F50); 155 &data_word(0xECD74900,0xC03B1789,0xB2FBA565,0x725E2C9E); 156$k_dsbd=0x260; # decryption sbox output *D*u, *D*t 157 &data_word(0xE6B1A200,0x7D57CCDF,0x882A4439,0xF56E9B13); 158 &data_word(0x24C6CB00,0x3CE2FAF7,0x15DEEFD3,0x2931180D); 159$k_dsbb=0x280; # decryption sbox output *B*u, *B*t 160 &data_word(0x96B44200,0xD0226492,0xB0F2D404,0x602646F6); 161 &data_word(0xCD596700,0xC19498A6,0x3255AA6B,0xF3FF0C3E); 162$k_dsbe=0x2a0; # decryption sbox output *E*u, *E*t 163 &data_word(0x26D4D000,0x46F29296,0x64B4F6B0,0x22426004); 164 &data_word(0xFFAAC100,0x0C55A6CD,0x98593E32,0x9467F36B); 165$k_dsbo=0x2c0; # decryption sbox final output 166 &data_word(0x7EF94000,0x1387EA53,0xD4943E2D,0xC7AA6DB9); 167 &data_word(0x93441D00,0x12D7560F,0xD8C58E9C,0xCA4B8159); 168&asciz ("Vector Permutation AES for x86/SSSE3, Mike Hamburg (Stanford University)"); 169&align (64); 170 171&function_begin_B("_vpaes_preheat"); 172 &add ($const,&DWP(0,"esp")); 173 &movdqa ("xmm7",&QWP($k_inv,$const)); 174 &movdqa ("xmm6",&QWP($k_s0F,$const)); 175 &ret (); 176&function_end_B("_vpaes_preheat"); 177 178## 179## _aes_encrypt_core 180## 181## AES-encrypt %xmm0. 182## 183## Inputs: 184## %xmm0 = input 185## %xmm6-%xmm7 as in _vpaes_preheat 186## (%edx) = scheduled keys 187## 188## Output in %xmm0 189## Clobbers %xmm1-%xmm5, %eax, %ebx, %ecx, %edx 190## 191## 192&function_begin_B("_vpaes_encrypt_core"); 193 &mov ($magic,16); 194 &mov ($round,&DWP(240,$key)); 195 &movdqa ("xmm1","xmm6") 196 &movdqa ("xmm2",&QWP($k_ipt,$const)); 197 &pandn ("xmm1","xmm0"); 198 &pand ("xmm0","xmm6"); 199 &movdqu ("xmm5",&QWP(0,$key)); 200 &pshufb ("xmm2","xmm0"); 201 &movdqa ("xmm0",&QWP($k_ipt+16,$const)); 202 &pxor ("xmm2","xmm5"); 203 &psrld ("xmm1",4); 204 &add ($key,16); 205 &pshufb ("xmm0","xmm1"); 206 &lea ($base,&DWP($k_mc_backward,$const)); 207 &pxor ("xmm0","xmm2"); 208 &jmp (&label("enc_entry")); 209 210 211&set_label("enc_loop",16); 212 # middle of middle round 213 &movdqa ("xmm4",&QWP($k_sb1,$const)); # 4 : sb1u 214 &movdqa ("xmm0",&QWP($k_sb1+16,$const));# 0 : sb1t 215 &pshufb ("xmm4","xmm2"); # 4 = sb1u 216 &pshufb ("xmm0","xmm3"); # 0 = sb1t 217 &pxor ("xmm4","xmm5"); # 4 = sb1u + k 218 &movdqa ("xmm5",&QWP($k_sb2,$const)); # 4 : sb2u 219 &pxor ("xmm0","xmm4"); # 0 = A 220 &movdqa ("xmm1",&QWP(-0x40,$base,$magic));# .Lk_mc_forward[] 221 &pshufb ("xmm5","xmm2"); # 4 = sb2u 222 &movdqa ("xmm2",&QWP($k_sb2+16,$const));# 2 : sb2t 223 &movdqa ("xmm4",&QWP(0,$base,$magic)); # .Lk_mc_backward[] 224 &pshufb ("xmm2","xmm3"); # 2 = sb2t 225 &movdqa ("xmm3","xmm0"); # 3 = A 226 &pxor ("xmm2","xmm5"); # 2 = 2A 227 &pshufb ("xmm0","xmm1"); # 0 = B 228 &add ($key,16); # next key 229 &pxor ("xmm0","xmm2"); # 0 = 2A+B 230 &pshufb ("xmm3","xmm4"); # 3 = D 231 &add ($magic,16); # next mc 232 &pxor ("xmm3","xmm0"); # 3 = 2A+B+D 233 &pshufb ("xmm0","xmm1"); # 0 = 2B+C 234 &and ($magic,0x30); # ... mod 4 235 &sub ($round,1); # nr-- 236 &pxor ("xmm0","xmm3"); # 0 = 2A+3B+C+D 237 238&set_label("enc_entry"); 239 # top of round 240 &movdqa ("xmm1","xmm6"); # 1 : i 241 &movdqa ("xmm5",&QWP($k_inv+16,$const));# 2 : a/k 242 &pandn ("xmm1","xmm0"); # 1 = i<<4 243 &psrld ("xmm1",4); # 1 = i 244 &pand ("xmm0","xmm6"); # 0 = k 245 &pshufb ("xmm5","xmm0"); # 2 = a/k 246 &movdqa ("xmm3","xmm7"); # 3 : 1/i 247 &pxor ("xmm0","xmm1"); # 0 = j 248 &pshufb ("xmm3","xmm1"); # 3 = 1/i 249 &movdqa ("xmm4","xmm7"); # 4 : 1/j 250 &pxor ("xmm3","xmm5"); # 3 = iak = 1/i + a/k 251 &pshufb ("xmm4","xmm0"); # 4 = 1/j 252 &movdqa ("xmm2","xmm7"); # 2 : 1/iak 253 &pxor ("xmm4","xmm5"); # 4 = jak = 1/j + a/k 254 &pshufb ("xmm2","xmm3"); # 2 = 1/iak 255 &movdqa ("xmm3","xmm7"); # 3 : 1/jak 256 &pxor ("xmm2","xmm0"); # 2 = io 257 &pshufb ("xmm3","xmm4"); # 3 = 1/jak 258 &movdqu ("xmm5",&QWP(0,$key)); 259 &pxor ("xmm3","xmm1"); # 3 = jo 260 &jnz (&label("enc_loop")); 261 262 # middle of last round 263 &movdqa ("xmm4",&QWP($k_sbo,$const)); # 3 : sbou .Lk_sbo 264 &movdqa ("xmm0",&QWP($k_sbo+16,$const));# 3 : sbot .Lk_sbo+16 265 &pshufb ("xmm4","xmm2"); # 4 = sbou 266 &pxor ("xmm4","xmm5"); # 4 = sb1u + k 267 &pshufb ("xmm0","xmm3"); # 0 = sb1t 268 &movdqa ("xmm1",&QWP(0x40,$base,$magic));# .Lk_sr[] 269 &pxor ("xmm0","xmm4"); # 0 = A 270 &pshufb ("xmm0","xmm1"); 271 &ret (); 272&function_end_B("_vpaes_encrypt_core"); 273 274## 275## Decryption core 276## 277## Same API as encryption core. 278## 279&function_begin_B("_vpaes_decrypt_core"); 280 &lea ($base,&DWP($k_dsbd,$const)); 281 &mov ($round,&DWP(240,$key)); 282 &movdqa ("xmm1","xmm6"); 283 &movdqa ("xmm2",&QWP($k_dipt-$k_dsbd,$base)); 284 &pandn ("xmm1","xmm0"); 285 &mov ($magic,$round); 286 &psrld ("xmm1",4) 287 &movdqu ("xmm5",&QWP(0,$key)); 288 &shl ($magic,4); 289 &pand ("xmm0","xmm6"); 290 &pshufb ("xmm2","xmm0"); 291 &movdqa ("xmm0",&QWP($k_dipt-$k_dsbd+16,$base)); 292 &xor ($magic,0x30); 293 &pshufb ("xmm0","xmm1"); 294 &and ($magic,0x30); 295 &pxor ("xmm2","xmm5"); 296 &movdqa ("xmm5",&QWP($k_mc_forward+48,$const)); 297 &pxor ("xmm0","xmm2"); 298 &add ($key,16); 299 &lea ($magic,&DWP($k_sr-$k_dsbd,$base,$magic)); 300 &jmp (&label("dec_entry")); 301 302&set_label("dec_loop",16); 303## 304## Inverse mix columns 305## 306 &movdqa ("xmm4",&QWP(-0x20,$base)); # 4 : sb9u 307 &movdqa ("xmm1",&QWP(-0x10,$base)); # 0 : sb9t 308 &pshufb ("xmm4","xmm2"); # 4 = sb9u 309 &pshufb ("xmm1","xmm3"); # 0 = sb9t 310 &pxor ("xmm0","xmm4"); 311 &movdqa ("xmm4",&QWP(0,$base)); # 4 : sbdu 312 &pxor ("xmm0","xmm1"); # 0 = ch 313 &movdqa ("xmm1",&QWP(0x10,$base)); # 0 : sbdt 314 315 &pshufb ("xmm4","xmm2"); # 4 = sbdu 316 &pshufb ("xmm0","xmm5"); # MC ch 317 &pshufb ("xmm1","xmm3"); # 0 = sbdt 318 &pxor ("xmm0","xmm4"); # 4 = ch 319 &movdqa ("xmm4",&QWP(0x20,$base)); # 4 : sbbu 320 &pxor ("xmm0","xmm1"); # 0 = ch 321 &movdqa ("xmm1",&QWP(0x30,$base)); # 0 : sbbt 322 323 &pshufb ("xmm4","xmm2"); # 4 = sbbu 324 &pshufb ("xmm0","xmm5"); # MC ch 325 &pshufb ("xmm1","xmm3"); # 0 = sbbt 326 &pxor ("xmm0","xmm4"); # 4 = ch 327 &movdqa ("xmm4",&QWP(0x40,$base)); # 4 : sbeu 328 &pxor ("xmm0","xmm1"); # 0 = ch 329 &movdqa ("xmm1",&QWP(0x50,$base)); # 0 : sbet 330 331 &pshufb ("xmm4","xmm2"); # 4 = sbeu 332 &pshufb ("xmm0","xmm5"); # MC ch 333 &pshufb ("xmm1","xmm3"); # 0 = sbet 334 &pxor ("xmm0","xmm4"); # 4 = ch 335 &add ($key,16); # next round key 336 &palignr("xmm5","xmm5",12); 337 &pxor ("xmm0","xmm1"); # 0 = ch 338 &sub ($round,1); # nr-- 339 340&set_label("dec_entry"); 341 # top of round 342 &movdqa ("xmm1","xmm6"); # 1 : i 343 &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k 344 &pandn ("xmm1","xmm0"); # 1 = i<<4 345 &pand ("xmm0","xmm6"); # 0 = k 346 &psrld ("xmm1",4); # 1 = i 347 &pshufb ("xmm2","xmm0"); # 2 = a/k 348 &movdqa ("xmm3","xmm7"); # 3 : 1/i 349 &pxor ("xmm0","xmm1"); # 0 = j 350 &pshufb ("xmm3","xmm1"); # 3 = 1/i 351 &movdqa ("xmm4","xmm7"); # 4 : 1/j 352 &pxor ("xmm3","xmm2"); # 3 = iak = 1/i + a/k 353 &pshufb ("xmm4","xmm0"); # 4 = 1/j 354 &pxor ("xmm4","xmm2"); # 4 = jak = 1/j + a/k 355 &movdqa ("xmm2","xmm7"); # 2 : 1/iak 356 &pshufb ("xmm2","xmm3"); # 2 = 1/iak 357 &movdqa ("xmm3","xmm7"); # 3 : 1/jak 358 &pxor ("xmm2","xmm0"); # 2 = io 359 &pshufb ("xmm3","xmm4"); # 3 = 1/jak 360 &movdqu ("xmm0",&QWP(0,$key)); 361 &pxor ("xmm3","xmm1"); # 3 = jo 362 &jnz (&label("dec_loop")); 363 364 # middle of last round 365 &movdqa ("xmm4",&QWP(0x60,$base)); # 3 : sbou 366 &pshufb ("xmm4","xmm2"); # 4 = sbou 367 &pxor ("xmm4","xmm0"); # 4 = sb1u + k 368 &movdqa ("xmm0",&QWP(0x70,$base)); # 0 : sbot 369 &movdqa ("xmm2",&QWP(0,$magic)); 370 &pshufb ("xmm0","xmm3"); # 0 = sb1t 371 &pxor ("xmm0","xmm4"); # 0 = A 372 &pshufb ("xmm0","xmm2"); 373 &ret (); 374&function_end_B("_vpaes_decrypt_core"); 375 376######################################################## 377## ## 378## AES key schedule ## 379## ## 380######################################################## 381&function_begin_B("_vpaes_schedule_core"); 382 &add ($const,&DWP(0,"esp")); 383 &movdqu ("xmm0",&QWP(0,$inp)); # load key (unaligned) 384 &movdqa ("xmm2",&QWP($k_rcon,$const)); # load rcon 385 386 # input transform 387 &movdqa ("xmm3","xmm0"); 388 &lea ($base,&DWP($k_ipt,$const)); 389 &movdqa (&QWP(4,"esp"),"xmm2"); # xmm8 390 &call ("_vpaes_schedule_transform"); 391 &movdqa ("xmm7","xmm0"); 392 393 &test ($out,$out); 394 &jnz (&label("schedule_am_decrypting")); 395 396 # encrypting, output zeroth round key after transform 397 &movdqu (&QWP(0,$key),"xmm0"); 398 &jmp (&label("schedule_go")); 399 400&set_label("schedule_am_decrypting"); 401 # decrypting, output zeroth round key after shiftrows 402 &movdqa ("xmm1",&QWP($k_sr,$const,$magic)); 403 &pshufb ("xmm3","xmm1"); 404 &movdqu (&QWP(0,$key),"xmm3"); 405 &xor ($magic,0x30); 406 407&set_label("schedule_go"); 408 &cmp ($round,192); 409 &ja (&label("schedule_256")); 410 &je (&label("schedule_192")); 411 # 128: fall though 412 413## 414## .schedule_128 415## 416## 128-bit specific part of key schedule. 417## 418## This schedule is really simple, because all its parts 419## are accomplished by the subroutines. 420## 421&set_label("schedule_128"); 422 &mov ($round,10); 423 424&set_label("loop_schedule_128"); 425 &call ("_vpaes_schedule_round"); 426 &dec ($round); 427 &jz (&label("schedule_mangle_last")); 428 &call ("_vpaes_schedule_mangle"); # write output 429 &jmp (&label("loop_schedule_128")); 430 431## 432## .aes_schedule_192 433## 434## 192-bit specific part of key schedule. 435## 436## The main body of this schedule is the same as the 128-bit 437## schedule, but with more smearing. The long, high side is 438## stored in %xmm7 as before, and the short, low side is in 439## the high bits of %xmm6. 440## 441## This schedule is somewhat nastier, however, because each 442## round produces 192 bits of key material, or 1.5 round keys. 443## Therefore, on each cycle we do 2 rounds and produce 3 round 444## keys. 445## 446&set_label("schedule_192",16); 447 &movdqu ("xmm0",&QWP(8,$inp)); # load key part 2 (very unaligned) 448 &call ("_vpaes_schedule_transform"); # input transform 449 &movdqa ("xmm6","xmm0"); # save short part 450 &pxor ("xmm4","xmm4"); # clear 4 451 &movhlps("xmm6","xmm4"); # clobber low side with zeros 452 &mov ($round,4); 453 454&set_label("loop_schedule_192"); 455 &call ("_vpaes_schedule_round"); 456 &palignr("xmm0","xmm6",8); 457 &call ("_vpaes_schedule_mangle"); # save key n 458 &call ("_vpaes_schedule_192_smear"); 459 &call ("_vpaes_schedule_mangle"); # save key n+1 460 &call ("_vpaes_schedule_round"); 461 &dec ($round); 462 &jz (&label("schedule_mangle_last")); 463 &call ("_vpaes_schedule_mangle"); # save key n+2 464 &call ("_vpaes_schedule_192_smear"); 465 &jmp (&label("loop_schedule_192")); 466 467## 468## .aes_schedule_256 469## 470## 256-bit specific part of key schedule. 471## 472## The structure here is very similar to the 128-bit 473## schedule, but with an additional "low side" in 474## %xmm6. The low side's rounds are the same as the 475## high side's, except no rcon and no rotation. 476## 477&set_label("schedule_256",16); 478 &movdqu ("xmm0",&QWP(16,$inp)); # load key part 2 (unaligned) 479 &call ("_vpaes_schedule_transform"); # input transform 480 &mov ($round,7); 481 482&set_label("loop_schedule_256"); 483 &call ("_vpaes_schedule_mangle"); # output low result 484 &movdqa ("xmm6","xmm0"); # save cur_lo in xmm6 485 486 # high round 487 &call ("_vpaes_schedule_round"); 488 &dec ($round); 489 &jz (&label("schedule_mangle_last")); 490 &call ("_vpaes_schedule_mangle"); 491 492 # low round. swap xmm7 and xmm6 493 &pshufd ("xmm0","xmm0",0xFF); 494 &movdqa (&QWP(20,"esp"),"xmm7"); 495 &movdqa ("xmm7","xmm6"); 496 &call ("_vpaes_schedule_low_round"); 497 &movdqa ("xmm7",&QWP(20,"esp")); 498 499 &jmp (&label("loop_schedule_256")); 500 501## 502## .aes_schedule_mangle_last 503## 504## Mangler for last round of key schedule 505## Mangles %xmm0 506## when encrypting, outputs out(%xmm0) ^ 63 507## when decrypting, outputs unskew(%xmm0) 508## 509## Always called right before return... jumps to cleanup and exits 510## 511&set_label("schedule_mangle_last",16); 512 # schedule last round key from xmm0 513 &lea ($base,&DWP($k_deskew,$const)); 514 &test ($out,$out); 515 &jnz (&label("schedule_mangle_last_dec")); 516 517 # encrypting 518 &movdqa ("xmm1",&QWP($k_sr,$const,$magic)); 519 &pshufb ("xmm0","xmm1"); # output permute 520 &lea ($base,&DWP($k_opt,$const)); # prepare to output transform 521 &add ($key,32); 522 523&set_label("schedule_mangle_last_dec"); 524 &add ($key,-16); 525 &pxor ("xmm0",&QWP($k_s63,$const)); 526 &call ("_vpaes_schedule_transform"); # output transform 527 &movdqu (&QWP(0,$key),"xmm0"); # save last key 528 529 # cleanup 530 &pxor ("xmm0","xmm0"); 531 &pxor ("xmm1","xmm1"); 532 &pxor ("xmm2","xmm2"); 533 &pxor ("xmm3","xmm3"); 534 &pxor ("xmm4","xmm4"); 535 &pxor ("xmm5","xmm5"); 536 &pxor ("xmm6","xmm6"); 537 &pxor ("xmm7","xmm7"); 538 &ret (); 539&function_end_B("_vpaes_schedule_core"); 540 541## 542## .aes_schedule_192_smear 543## 544## Smear the short, low side in the 192-bit key schedule. 545## 546## Inputs: 547## %xmm7: high side, b a x y 548## %xmm6: low side, d c 0 0 549## %xmm13: 0 550## 551## Outputs: 552## %xmm6: b+c+d b+c 0 0 553## %xmm0: b+c+d b+c b a 554## 555&function_begin_B("_vpaes_schedule_192_smear"); 556 &pshufd ("xmm1","xmm6",0x80); # d c 0 0 -> c 0 0 0 557 &pshufd ("xmm0","xmm7",0xFE); # b a _ _ -> b b b a 558 &pxor ("xmm6","xmm1"); # -> c+d c 0 0 559 &pxor ("xmm1","xmm1"); 560 &pxor ("xmm6","xmm0"); # -> b+c+d b+c b a 561 &movdqa ("xmm0","xmm6"); 562 &movhlps("xmm6","xmm1"); # clobber low side with zeros 563 &ret (); 564&function_end_B("_vpaes_schedule_192_smear"); 565 566## 567## .aes_schedule_round 568## 569## Runs one main round of the key schedule on %xmm0, %xmm7 570## 571## Specifically, runs subbytes on the high dword of %xmm0 572## then rotates it by one byte and xors into the low dword of 573## %xmm7. 574## 575## Adds rcon from low byte of %xmm8, then rotates %xmm8 for 576## next rcon. 577## 578## Smears the dwords of %xmm7 by xoring the low into the 579## second low, result into third, result into highest. 580## 581## Returns results in %xmm7 = %xmm0. 582## Clobbers %xmm1-%xmm5. 583## 584&function_begin_B("_vpaes_schedule_round"); 585 # extract rcon from xmm8 586 &movdqa ("xmm2",&QWP(8,"esp")); # xmm8 587 &pxor ("xmm1","xmm1"); 588 &palignr("xmm1","xmm2",15); 589 &palignr("xmm2","xmm2",15); 590 &pxor ("xmm7","xmm1"); 591 592 # rotate 593 &pshufd ("xmm0","xmm0",0xFF); 594 &palignr("xmm0","xmm0",1); 595 596 # fall through... 597 &movdqa (&QWP(8,"esp"),"xmm2"); # xmm8 598 599 # low round: same as high round, but no rotation and no rcon. 600&set_label("_vpaes_schedule_low_round"); 601 # smear xmm7 602 &movdqa ("xmm1","xmm7"); 603 &pslldq ("xmm7",4); 604 &pxor ("xmm7","xmm1"); 605 &movdqa ("xmm1","xmm7"); 606 &pslldq ("xmm7",8); 607 &pxor ("xmm7","xmm1"); 608 &pxor ("xmm7",&QWP($k_s63,$const)); 609 610 # subbyte 611 &movdqa ("xmm4",&QWP($k_s0F,$const)); 612 &movdqa ("xmm5",&QWP($k_inv,$const)); # 4 : 1/j 613 &movdqa ("xmm1","xmm4"); 614 &pandn ("xmm1","xmm0"); 615 &psrld ("xmm1",4); # 1 = i 616 &pand ("xmm0","xmm4"); # 0 = k 617 &movdqa ("xmm2",&QWP($k_inv+16,$const));# 2 : a/k 618 &pshufb ("xmm2","xmm0"); # 2 = a/k 619 &pxor ("xmm0","xmm1"); # 0 = j 620 &movdqa ("xmm3","xmm5"); # 3 : 1/i 621 &pshufb ("xmm3","xmm1"); # 3 = 1/i 622 &pxor ("xmm3","xmm2"); # 3 = iak = 1/i + a/k 623 &movdqa ("xmm4","xmm5"); # 4 : 1/j 624 &pshufb ("xmm4","xmm0"); # 4 = 1/j 625 &pxor ("xmm4","xmm2"); # 4 = jak = 1/j + a/k 626 &movdqa ("xmm2","xmm5"); # 2 : 1/iak 627 &pshufb ("xmm2","xmm3"); # 2 = 1/iak 628 &pxor ("xmm2","xmm0"); # 2 = io 629 &movdqa ("xmm3","xmm5"); # 3 : 1/jak 630 &pshufb ("xmm3","xmm4"); # 3 = 1/jak 631 &pxor ("xmm3","xmm1"); # 3 = jo 632 &movdqa ("xmm4",&QWP($k_sb1,$const)); # 4 : sbou 633 &pshufb ("xmm4","xmm2"); # 4 = sbou 634 &movdqa ("xmm0",&QWP($k_sb1+16,$const));# 0 : sbot 635 &pshufb ("xmm0","xmm3"); # 0 = sb1t 636 &pxor ("xmm0","xmm4"); # 0 = sbox output 637 638 # add in smeared stuff 639 &pxor ("xmm0","xmm7"); 640 &movdqa ("xmm7","xmm0"); 641 &ret (); 642&function_end_B("_vpaes_schedule_round"); 643 644## 645## .aes_schedule_transform 646## 647## Linear-transform %xmm0 according to tables at (%ebx) 648## 649## Output in %xmm0 650## Clobbers %xmm1, %xmm2 651## 652&function_begin_B("_vpaes_schedule_transform"); 653 &movdqa ("xmm2",&QWP($k_s0F,$const)); 654 &movdqa ("xmm1","xmm2"); 655 &pandn ("xmm1","xmm0"); 656 &psrld ("xmm1",4); 657 &pand ("xmm0","xmm2"); 658 &movdqa ("xmm2",&QWP(0,$base)); 659 &pshufb ("xmm2","xmm0"); 660 &movdqa ("xmm0",&QWP(16,$base)); 661 &pshufb ("xmm0","xmm1"); 662 &pxor ("xmm0","xmm2"); 663 &ret (); 664&function_end_B("_vpaes_schedule_transform"); 665 666## 667## .aes_schedule_mangle 668## 669## Mangle xmm0 from (basis-transformed) standard version 670## to our version. 671## 672## On encrypt, 673## xor with 0x63 674## multiply by circulant 0,1,1,1 675## apply shiftrows transform 676## 677## On decrypt, 678## xor with 0x63 679## multiply by "inverse mixcolumns" circulant E,B,D,9 680## deskew 681## apply shiftrows transform 682## 683## 684## Writes out to (%edx), and increments or decrements it 685## Keeps track of round number mod 4 in %ecx 686## Preserves xmm0 687## Clobbers xmm1-xmm5 688## 689&function_begin_B("_vpaes_schedule_mangle"); 690 &movdqa ("xmm4","xmm0"); # save xmm0 for later 691 &movdqa ("xmm5",&QWP($k_mc_forward,$const)); 692 &test ($out,$out); 693 &jnz (&label("schedule_mangle_dec")); 694 695 # encrypting 696 &add ($key,16); 697 &pxor ("xmm4",&QWP($k_s63,$const)); 698 &pshufb ("xmm4","xmm5"); 699 &movdqa ("xmm3","xmm4"); 700 &pshufb ("xmm4","xmm5"); 701 &pxor ("xmm3","xmm4"); 702 &pshufb ("xmm4","xmm5"); 703 &pxor ("xmm3","xmm4"); 704 705 &jmp (&label("schedule_mangle_both")); 706 707&set_label("schedule_mangle_dec",16); 708 # inverse mix columns 709 &movdqa ("xmm2",&QWP($k_s0F,$const)); 710 &lea ($inp,&DWP($k_dksd,$const)); 711 &movdqa ("xmm1","xmm2"); 712 &pandn ("xmm1","xmm4"); 713 &psrld ("xmm1",4); # 1 = hi 714 &pand ("xmm4","xmm2"); # 4 = lo 715 716 &movdqa ("xmm2",&QWP(0,$inp)); 717 &pshufb ("xmm2","xmm4"); 718 &movdqa ("xmm3",&QWP(0x10,$inp)); 719 &pshufb ("xmm3","xmm1"); 720 &pxor ("xmm3","xmm2"); 721 &pshufb ("xmm3","xmm5"); 722 723 &movdqa ("xmm2",&QWP(0x20,$inp)); 724 &pshufb ("xmm2","xmm4"); 725 &pxor ("xmm2","xmm3"); 726 &movdqa ("xmm3",&QWP(0x30,$inp)); 727 &pshufb ("xmm3","xmm1"); 728 &pxor ("xmm3","xmm2"); 729 &pshufb ("xmm3","xmm5"); 730 731 &movdqa ("xmm2",&QWP(0x40,$inp)); 732 &pshufb ("xmm2","xmm4"); 733 &pxor ("xmm2","xmm3"); 734 &movdqa ("xmm3",&QWP(0x50,$inp)); 735 &pshufb ("xmm3","xmm1"); 736 &pxor ("xmm3","xmm2"); 737 &pshufb ("xmm3","xmm5"); 738 739 &movdqa ("xmm2",&QWP(0x60,$inp)); 740 &pshufb ("xmm2","xmm4"); 741 &pxor ("xmm2","xmm3"); 742 &movdqa ("xmm3",&QWP(0x70,$inp)); 743 &pshufb ("xmm3","xmm1"); 744 &pxor ("xmm3","xmm2"); 745 746 &add ($key,-16); 747 748&set_label("schedule_mangle_both"); 749 &movdqa ("xmm1",&QWP($k_sr,$const,$magic)); 750 &pshufb ("xmm3","xmm1"); 751 &add ($magic,-16); 752 &and ($magic,0x30); 753 &movdqu (&QWP(0,$key),"xmm3"); 754 &ret (); 755&function_end_B("_vpaes_schedule_mangle"); 756 757# 758# Interface to OpenSSL 759# 760&function_begin("${PREFIX}_set_encrypt_key"); 761 &mov ($inp,&wparam(0)); # inp 762 &lea ($base,&DWP(-56,"esp")); 763 &mov ($round,&wparam(1)); # bits 764 &and ($base,-16); 765 &mov ($key,&wparam(2)); # key 766 &xchg ($base,"esp"); # alloca 767 &mov (&DWP(48,"esp"),$base); 768 769 &mov ($base,$round); 770 &shr ($base,5); 771 &add ($base,5); 772 &mov (&DWP(240,$key),$base); # AES_KEY->rounds = nbits/32+5; 773 &mov ($magic,0x30); 774 &mov ($out,0); 775 776 &lea ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point"))); 777 &call ("_vpaes_schedule_core"); 778&set_label("pic_point"); 779 780 &mov ("esp",&DWP(48,"esp")); 781 &xor ("eax","eax"); 782&function_end("${PREFIX}_set_encrypt_key"); 783 784&function_begin("${PREFIX}_set_decrypt_key"); 785 &mov ($inp,&wparam(0)); # inp 786 &lea ($base,&DWP(-56,"esp")); 787 &mov ($round,&wparam(1)); # bits 788 &and ($base,-16); 789 &mov ($key,&wparam(2)); # key 790 &xchg ($base,"esp"); # alloca 791 &mov (&DWP(48,"esp"),$base); 792 793 &mov ($base,$round); 794 &shr ($base,5); 795 &add ($base,5); 796 &mov (&DWP(240,$key),$base); # AES_KEY->rounds = nbits/32+5; 797 &shl ($base,4); 798 &lea ($key,&DWP(16,$key,$base)); 799 800 &mov ($out,1); 801 &mov ($magic,$round); 802 &shr ($magic,1); 803 &and ($magic,32); 804 &xor ($magic,32); # nbist==192?0:32; 805 806 &lea ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point"))); 807 &call ("_vpaes_schedule_core"); 808&set_label("pic_point"); 809 810 &mov ("esp",&DWP(48,"esp")); 811 &xor ("eax","eax"); 812&function_end("${PREFIX}_set_decrypt_key"); 813 814&function_begin("${PREFIX}_encrypt"); 815 &lea ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point"))); 816 &call ("_vpaes_preheat"); 817&set_label("pic_point"); 818 &mov ($inp,&wparam(0)); # inp 819 &lea ($base,&DWP(-56,"esp")); 820 &mov ($out,&wparam(1)); # out 821 &and ($base,-16); 822 &mov ($key,&wparam(2)); # key 823 &xchg ($base,"esp"); # alloca 824 &mov (&DWP(48,"esp"),$base); 825 826 &movdqu ("xmm0",&QWP(0,$inp)); 827 &call ("_vpaes_encrypt_core"); 828 &movdqu (&QWP(0,$out),"xmm0"); 829 830 &mov ("esp",&DWP(48,"esp")); 831&function_end("${PREFIX}_encrypt"); 832 833&function_begin("${PREFIX}_decrypt"); 834 &lea ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point"))); 835 &call ("_vpaes_preheat"); 836&set_label("pic_point"); 837 &mov ($inp,&wparam(0)); # inp 838 &lea ($base,&DWP(-56,"esp")); 839 &mov ($out,&wparam(1)); # out 840 &and ($base,-16); 841 &mov ($key,&wparam(2)); # key 842 &xchg ($base,"esp"); # alloca 843 &mov (&DWP(48,"esp"),$base); 844 845 &movdqu ("xmm0",&QWP(0,$inp)); 846 &call ("_vpaes_decrypt_core"); 847 &movdqu (&QWP(0,$out),"xmm0"); 848 849 &mov ("esp",&DWP(48,"esp")); 850&function_end("${PREFIX}_decrypt"); 851 852&function_begin("${PREFIX}_cbc_encrypt"); 853 &mov ($inp,&wparam(0)); # inp 854 &mov ($out,&wparam(1)); # out 855 &mov ($round,&wparam(2)); # len 856 &mov ($key,&wparam(3)); # key 857 &sub ($round,16); 858 &jc (&label("cbc_abort")); 859 &lea ($base,&DWP(-56,"esp")); 860 &mov ($const,&wparam(4)); # ivp 861 &and ($base,-16); 862 &mov ($magic,&wparam(5)); # enc 863 &xchg ($base,"esp"); # alloca 864 &movdqu ("xmm1",&QWP(0,$const)); # load IV 865 &sub ($out,$inp); 866 &mov (&DWP(48,"esp"),$base); 867 868 &mov (&DWP(0,"esp"),$out); # save out 869 &mov (&DWP(4,"esp"),$key) # save key 870 &mov (&DWP(8,"esp"),$const); # save ivp 871 &mov ($out,$round); # $out works as $len 872 873 &lea ($const,&DWP(&label("_vpaes_consts")."+0x30-".&label("pic_point"))); 874 &call ("_vpaes_preheat"); 875&set_label("pic_point"); 876 &cmp ($magic,0); 877 &je (&label("cbc_dec_loop")); 878 &jmp (&label("cbc_enc_loop")); 879 880&set_label("cbc_enc_loop",16); 881 &movdqu ("xmm0",&QWP(0,$inp)); # load input 882 &pxor ("xmm0","xmm1"); # inp^=iv 883 &call ("_vpaes_encrypt_core"); 884 &mov ($base,&DWP(0,"esp")); # restore out 885 &mov ($key,&DWP(4,"esp")); # restore key 886 &movdqa ("xmm1","xmm0"); 887 &movdqu (&QWP(0,$base,$inp),"xmm0"); # write output 888 &lea ($inp,&DWP(16,$inp)); 889 &sub ($out,16); 890 &jnc (&label("cbc_enc_loop")); 891 &jmp (&label("cbc_done")); 892 893&set_label("cbc_dec_loop",16); 894 &movdqu ("xmm0",&QWP(0,$inp)); # load input 895 &movdqa (&QWP(16,"esp"),"xmm1"); # save IV 896 &movdqa (&QWP(32,"esp"),"xmm0"); # save future IV 897 &call ("_vpaes_decrypt_core"); 898 &mov ($base,&DWP(0,"esp")); # restore out 899 &mov ($key,&DWP(4,"esp")); # restore key 900 &pxor ("xmm0",&QWP(16,"esp")); # out^=iv 901 &movdqa ("xmm1",&QWP(32,"esp")); # load next IV 902 &movdqu (&QWP(0,$base,$inp),"xmm0"); # write output 903 &lea ($inp,&DWP(16,$inp)); 904 &sub ($out,16); 905 &jnc (&label("cbc_dec_loop")); 906 907&set_label("cbc_done"); 908 &mov ($base,&DWP(8,"esp")); # restore ivp 909 &mov ("esp",&DWP(48,"esp")); 910 &movdqu (&QWP(0,$base),"xmm1"); # write IV 911&set_label("cbc_abort"); 912&function_end("${PREFIX}_cbc_encrypt"); 913 914&asm_finish(); 915 916close STDOUT or die "error closing STDOUT: $!"; 917