1/** 2 * Author......: See docs/credits.txt 3 * License.....: MIT 4 */ 5 6#define NEW_SIMD_CODE 7 8#ifdef KERNEL_STATIC 9#include "inc_vendor.h" 10#include "inc_types.h" 11#include "inc_platform.cl" 12#include "inc_common.cl" 13#include "inc_simd.cl" 14#include "inc_hash_whirlpool.cl" 15#include "inc_cipher_aes.cl" 16#include "inc_cipher_twofish.cl" 17#include "inc_cipher_serpent.cl" 18#endif 19 20typedef struct tc 21{ 22 u32 salt_buf[32]; 23 u32 data_buf[112]; 24 u32 keyfile_buf16[16]; 25 u32 keyfile_buf32[32]; 26 u32 keyfile_enabled; 27 u32 signature; 28 29 keyboard_layout_mapping_t keyboard_layout_mapping_buf[256]; 30 int keyboard_layout_mapping_cnt; 31 32} tc_t; 33 34#ifdef KERNEL_STATIC 35#include "inc_truecrypt_crc32.cl" 36#include "inc_truecrypt_xts.cl" 37#include "inc_truecrypt_keyfile.cl" 38#endif 39 40typedef struct tc_tmp 41{ 42 u32 ipad[16]; 43 u32 opad[16]; 44 45 u32 dgst[64]; 46 u32 out[64]; 47 48} tc_tmp_t; 49 50DECLSPEC void hmac_whirlpool_run_V (u32x *w0, u32x *w1, u32x *w2, u32x *w3, u32x *ipad, u32x *opad, u32x *digest, SHM_TYPE u64 *s_MT0, SHM_TYPE u64 *s_MT1, SHM_TYPE u64 *s_MT2, SHM_TYPE u64 *s_MT3, SHM_TYPE u64 *s_MT4, SHM_TYPE u64 *s_MT5, SHM_TYPE u64 *s_MT6, SHM_TYPE u64 *s_MT7) 51{ 52 digest[ 0] = ipad[ 0]; 53 digest[ 1] = ipad[ 1]; 54 digest[ 2] = ipad[ 2]; 55 digest[ 3] = ipad[ 3]; 56 digest[ 4] = ipad[ 4]; 57 digest[ 5] = ipad[ 5]; 58 digest[ 6] = ipad[ 6]; 59 digest[ 7] = ipad[ 7]; 60 digest[ 8] = ipad[ 8]; 61 digest[ 9] = ipad[ 9]; 62 digest[10] = ipad[10]; 63 digest[11] = ipad[11]; 64 digest[12] = ipad[12]; 65 digest[13] = ipad[13]; 66 digest[14] = ipad[14]; 67 digest[15] = ipad[15]; 68 69 whirlpool_transform_vector (w0, w1, w2, w3, digest, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 70 71 w0[0] = 0x80000000; 72 w0[1] = 0; 73 w0[2] = 0; 74 w0[3] = 0; 75 w1[0] = 0; 76 w1[1] = 0; 77 w1[2] = 0; 78 w1[3] = 0; 79 w2[0] = 0; 80 w2[1] = 0; 81 w2[2] = 0; 82 w2[3] = 0; 83 w3[0] = 0; 84 w3[1] = 0; 85 w3[2] = 0; 86 w3[3] = (64 + 64) * 8; 87 88 whirlpool_transform_vector (w0, w1, w2, w3, digest, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 89 90 w0[0] = digest[ 0]; 91 w0[1] = digest[ 1]; 92 w0[2] = digest[ 2]; 93 w0[3] = digest[ 3]; 94 w1[0] = digest[ 4]; 95 w1[1] = digest[ 5]; 96 w1[2] = digest[ 6]; 97 w1[3] = digest[ 7]; 98 w2[0] = digest[ 8]; 99 w2[1] = digest[ 9]; 100 w2[2] = digest[10]; 101 w2[3] = digest[11]; 102 w3[0] = digest[12]; 103 w3[1] = digest[13]; 104 w3[2] = digest[14]; 105 w3[3] = digest[15]; 106 107 digest[ 0] = opad[ 0]; 108 digest[ 1] = opad[ 1]; 109 digest[ 2] = opad[ 2]; 110 digest[ 3] = opad[ 3]; 111 digest[ 4] = opad[ 4]; 112 digest[ 5] = opad[ 5]; 113 digest[ 6] = opad[ 6]; 114 digest[ 7] = opad[ 7]; 115 digest[ 8] = opad[ 8]; 116 digest[ 9] = opad[ 9]; 117 digest[10] = opad[10]; 118 digest[11] = opad[11]; 119 digest[12] = opad[12]; 120 digest[13] = opad[13]; 121 digest[14] = opad[14]; 122 digest[15] = opad[15]; 123 124 whirlpool_transform_vector (w0, w1, w2, w3, digest, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 125 126 w0[0] = 0x80000000; 127 w0[1] = 0; 128 w0[2] = 0; 129 w0[3] = 0; 130 w1[0] = 0; 131 w1[1] = 0; 132 w1[2] = 0; 133 w1[3] = 0; 134 w2[0] = 0; 135 w2[1] = 0; 136 w2[2] = 0; 137 w2[3] = 0; 138 w3[0] = 0; 139 w3[1] = 0; 140 w3[2] = 0; 141 w3[3] = (64 + 64) * 8; 142 143 whirlpool_transform_vector (w0, w1, w2, w3, digest, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 144} 145 146KERNEL_FQ void m06231_init (KERN_ATTR_TMPS_ESALT (tc_tmp_t, tc_t)) 147{ 148 const u64 gid = get_global_id (0); 149 const u64 lid = get_local_id (0); 150 const u64 lsz = get_local_size (0); 151 152 /** 153 * keyboard layout shared 154 */ 155 156 const int keyboard_layout_mapping_cnt = esalt_bufs[DIGESTS_OFFSET].keyboard_layout_mapping_cnt; 157 158 LOCAL_VK keyboard_layout_mapping_t s_keyboard_layout_mapping_buf[256]; 159 160 for (u32 i = lid; i < 256; i += lsz) 161 { 162 s_keyboard_layout_mapping_buf[i] = esalt_bufs[DIGESTS_OFFSET].keyboard_layout_mapping_buf[i]; 163 } 164 165 SYNC_THREADS (); 166 167 /** 168 * Whirlpool shared 169 */ 170 171 #ifdef REAL_SHM 172 173 LOCAL_VK u64 s_MT0[256]; 174 LOCAL_VK u64 s_MT1[256]; 175 LOCAL_VK u64 s_MT2[256]; 176 LOCAL_VK u64 s_MT3[256]; 177 LOCAL_VK u64 s_MT4[256]; 178 LOCAL_VK u64 s_MT5[256]; 179 LOCAL_VK u64 s_MT6[256]; 180 LOCAL_VK u64 s_MT7[256]; 181 182 for (u32 i = lid; i < 256; i += lsz) 183 { 184 s_MT0[i] = MT0[i]; 185 s_MT1[i] = MT1[i]; 186 s_MT2[i] = MT2[i]; 187 s_MT3[i] = MT3[i]; 188 s_MT4[i] = MT4[i]; 189 s_MT5[i] = MT5[i]; 190 s_MT6[i] = MT6[i]; 191 s_MT7[i] = MT7[i]; 192 } 193 194 SYNC_THREADS (); 195 196 #else 197 198 CONSTANT_AS u64a *s_MT0 = MT0; 199 CONSTANT_AS u64a *s_MT1 = MT1; 200 CONSTANT_AS u64a *s_MT2 = MT2; 201 CONSTANT_AS u64a *s_MT3 = MT3; 202 CONSTANT_AS u64a *s_MT4 = MT4; 203 CONSTANT_AS u64a *s_MT5 = MT5; 204 CONSTANT_AS u64a *s_MT6 = MT6; 205 CONSTANT_AS u64a *s_MT7 = MT7; 206 207 #endif 208 209 if (gid >= gid_max) return; 210 211 /** 212 * base 213 */ 214 215 u32 w[32]; 216 217 w[ 0] = pws[gid].i[ 0]; 218 w[ 1] = pws[gid].i[ 1]; 219 w[ 2] = pws[gid].i[ 2]; 220 w[ 3] = pws[gid].i[ 3]; 221 w[ 4] = pws[gid].i[ 4]; 222 w[ 5] = pws[gid].i[ 5]; 223 w[ 6] = pws[gid].i[ 6]; 224 w[ 7] = pws[gid].i[ 7]; 225 w[ 8] = pws[gid].i[ 8]; 226 w[ 9] = pws[gid].i[ 9]; 227 w[10] = pws[gid].i[10]; 228 w[11] = pws[gid].i[11]; 229 w[12] = pws[gid].i[12]; 230 w[13] = pws[gid].i[13]; 231 w[14] = pws[gid].i[14]; 232 w[15] = pws[gid].i[15]; 233 w[16] = pws[gid].i[16]; 234 w[17] = pws[gid].i[17]; 235 w[18] = pws[gid].i[18]; 236 w[19] = pws[gid].i[19]; 237 w[20] = pws[gid].i[20]; 238 w[21] = pws[gid].i[21]; 239 w[22] = pws[gid].i[22]; 240 w[23] = pws[gid].i[23]; 241 w[24] = pws[gid].i[24]; 242 w[25] = pws[gid].i[25]; 243 w[26] = pws[gid].i[26]; 244 w[27] = pws[gid].i[27]; 245 w[28] = pws[gid].i[28]; 246 w[29] = pws[gid].i[29]; 247 w[30] = pws[gid].i[30]; 248 w[31] = pws[gid].i[31]; 249 250 u32 pw_len = pws[gid].pw_len; 251 252 hc_execute_keyboard_layout_mapping (w, pw_len, s_keyboard_layout_mapping_buf, keyboard_layout_mapping_cnt); 253 254 pw_len = hc_apply_keyfile_tc (w, pw_len, &esalt_bufs[DIGESTS_OFFSET]); 255 256 whirlpool_hmac_ctx_t whirlpool_hmac_ctx; 257 258 whirlpool_hmac_init_swap (&whirlpool_hmac_ctx, w, pw_len, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 259 260 tmps[gid].ipad[ 0] = whirlpool_hmac_ctx.ipad.h[ 0]; 261 tmps[gid].ipad[ 1] = whirlpool_hmac_ctx.ipad.h[ 1]; 262 tmps[gid].ipad[ 2] = whirlpool_hmac_ctx.ipad.h[ 2]; 263 tmps[gid].ipad[ 3] = whirlpool_hmac_ctx.ipad.h[ 3]; 264 tmps[gid].ipad[ 4] = whirlpool_hmac_ctx.ipad.h[ 4]; 265 tmps[gid].ipad[ 5] = whirlpool_hmac_ctx.ipad.h[ 5]; 266 tmps[gid].ipad[ 6] = whirlpool_hmac_ctx.ipad.h[ 6]; 267 tmps[gid].ipad[ 7] = whirlpool_hmac_ctx.ipad.h[ 7]; 268 tmps[gid].ipad[ 8] = whirlpool_hmac_ctx.ipad.h[ 8]; 269 tmps[gid].ipad[ 9] = whirlpool_hmac_ctx.ipad.h[ 9]; 270 tmps[gid].ipad[10] = whirlpool_hmac_ctx.ipad.h[10]; 271 tmps[gid].ipad[11] = whirlpool_hmac_ctx.ipad.h[11]; 272 tmps[gid].ipad[12] = whirlpool_hmac_ctx.ipad.h[12]; 273 tmps[gid].ipad[13] = whirlpool_hmac_ctx.ipad.h[13]; 274 tmps[gid].ipad[14] = whirlpool_hmac_ctx.ipad.h[14]; 275 tmps[gid].ipad[15] = whirlpool_hmac_ctx.ipad.h[15]; 276 277 tmps[gid].opad[ 0] = whirlpool_hmac_ctx.opad.h[ 0]; 278 tmps[gid].opad[ 1] = whirlpool_hmac_ctx.opad.h[ 1]; 279 tmps[gid].opad[ 2] = whirlpool_hmac_ctx.opad.h[ 2]; 280 tmps[gid].opad[ 3] = whirlpool_hmac_ctx.opad.h[ 3]; 281 tmps[gid].opad[ 4] = whirlpool_hmac_ctx.opad.h[ 4]; 282 tmps[gid].opad[ 5] = whirlpool_hmac_ctx.opad.h[ 5]; 283 tmps[gid].opad[ 6] = whirlpool_hmac_ctx.opad.h[ 6]; 284 tmps[gid].opad[ 7] = whirlpool_hmac_ctx.opad.h[ 7]; 285 tmps[gid].opad[ 8] = whirlpool_hmac_ctx.opad.h[ 8]; 286 tmps[gid].opad[ 9] = whirlpool_hmac_ctx.opad.h[ 9]; 287 tmps[gid].opad[10] = whirlpool_hmac_ctx.opad.h[10]; 288 tmps[gid].opad[11] = whirlpool_hmac_ctx.opad.h[11]; 289 tmps[gid].opad[12] = whirlpool_hmac_ctx.opad.h[12]; 290 tmps[gid].opad[13] = whirlpool_hmac_ctx.opad.h[13]; 291 tmps[gid].opad[14] = whirlpool_hmac_ctx.opad.h[14]; 292 tmps[gid].opad[15] = whirlpool_hmac_ctx.opad.h[15]; 293 294 whirlpool_hmac_update_global_swap (&whirlpool_hmac_ctx, esalt_bufs[DIGESTS_OFFSET].salt_buf, 64); 295 296 for (u32 i = 0, j = 1; i < 16; i += 16, j += 1) 297 { 298 whirlpool_hmac_ctx_t whirlpool_hmac_ctx2 = whirlpool_hmac_ctx; 299 300 u32 w0[4]; 301 u32 w1[4]; 302 u32 w2[4]; 303 u32 w3[4]; 304 305 w0[0] = j; 306 w0[1] = 0; 307 w0[2] = 0; 308 w0[3] = 0; 309 w1[0] = 0; 310 w1[1] = 0; 311 w1[2] = 0; 312 w1[3] = 0; 313 w2[0] = 0; 314 w2[1] = 0; 315 w2[2] = 0; 316 w2[3] = 0; 317 w3[0] = 0; 318 w3[1] = 0; 319 w3[2] = 0; 320 w3[3] = 0; 321 322 whirlpool_hmac_update_64 (&whirlpool_hmac_ctx2, w0, w1, w2, w3, 4); 323 324 whirlpool_hmac_final (&whirlpool_hmac_ctx2); 325 326 tmps[gid].dgst[i + 0] = whirlpool_hmac_ctx2.opad.h[ 0]; 327 tmps[gid].dgst[i + 1] = whirlpool_hmac_ctx2.opad.h[ 1]; 328 tmps[gid].dgst[i + 2] = whirlpool_hmac_ctx2.opad.h[ 2]; 329 tmps[gid].dgst[i + 3] = whirlpool_hmac_ctx2.opad.h[ 3]; 330 tmps[gid].dgst[i + 4] = whirlpool_hmac_ctx2.opad.h[ 4]; 331 tmps[gid].dgst[i + 5] = whirlpool_hmac_ctx2.opad.h[ 5]; 332 tmps[gid].dgst[i + 6] = whirlpool_hmac_ctx2.opad.h[ 6]; 333 tmps[gid].dgst[i + 7] = whirlpool_hmac_ctx2.opad.h[ 7]; 334 tmps[gid].dgst[i + 8] = whirlpool_hmac_ctx2.opad.h[ 8]; 335 tmps[gid].dgst[i + 9] = whirlpool_hmac_ctx2.opad.h[ 9]; 336 tmps[gid].dgst[i + 10] = whirlpool_hmac_ctx2.opad.h[10]; 337 tmps[gid].dgst[i + 11] = whirlpool_hmac_ctx2.opad.h[11]; 338 tmps[gid].dgst[i + 12] = whirlpool_hmac_ctx2.opad.h[12]; 339 tmps[gid].dgst[i + 13] = whirlpool_hmac_ctx2.opad.h[13]; 340 tmps[gid].dgst[i + 14] = whirlpool_hmac_ctx2.opad.h[14]; 341 tmps[gid].dgst[i + 15] = whirlpool_hmac_ctx2.opad.h[15]; 342 343 tmps[gid].out[i + 0] = tmps[gid].dgst[i + 0]; 344 tmps[gid].out[i + 1] = tmps[gid].dgst[i + 1]; 345 tmps[gid].out[i + 2] = tmps[gid].dgst[i + 2]; 346 tmps[gid].out[i + 3] = tmps[gid].dgst[i + 3]; 347 tmps[gid].out[i + 4] = tmps[gid].dgst[i + 4]; 348 tmps[gid].out[i + 5] = tmps[gid].dgst[i + 5]; 349 tmps[gid].out[i + 6] = tmps[gid].dgst[i + 6]; 350 tmps[gid].out[i + 7] = tmps[gid].dgst[i + 7]; 351 tmps[gid].out[i + 8] = tmps[gid].dgst[i + 8]; 352 tmps[gid].out[i + 9] = tmps[gid].dgst[i + 9]; 353 tmps[gid].out[i + 10] = tmps[gid].dgst[i + 10]; 354 tmps[gid].out[i + 11] = tmps[gid].dgst[i + 11]; 355 tmps[gid].out[i + 12] = tmps[gid].dgst[i + 12]; 356 tmps[gid].out[i + 13] = tmps[gid].dgst[i + 13]; 357 tmps[gid].out[i + 14] = tmps[gid].dgst[i + 14]; 358 tmps[gid].out[i + 15] = tmps[gid].dgst[i + 15]; 359 } 360} 361 362KERNEL_FQ void m06231_loop (KERN_ATTR_TMPS_ESALT (tc_tmp_t, tc_t)) 363{ 364 /** 365 * Whirlpool shared 366 */ 367 368 const u64 gid = get_global_id (0); 369 const u64 lid = get_local_id (0); 370 const u64 lsz = get_local_size (0); 371 372 #ifdef REAL_SHM 373 374 LOCAL_VK u64 s_MT0[256]; 375 LOCAL_VK u64 s_MT1[256]; 376 LOCAL_VK u64 s_MT2[256]; 377 LOCAL_VK u64 s_MT3[256]; 378 LOCAL_VK u64 s_MT4[256]; 379 LOCAL_VK u64 s_MT5[256]; 380 LOCAL_VK u64 s_MT6[256]; 381 LOCAL_VK u64 s_MT7[256]; 382 383 for (u32 i = lid; i < 256; i += lsz) 384 { 385 s_MT0[i] = MT0[i]; 386 s_MT1[i] = MT1[i]; 387 s_MT2[i] = MT2[i]; 388 s_MT3[i] = MT3[i]; 389 s_MT4[i] = MT4[i]; 390 s_MT5[i] = MT5[i]; 391 s_MT6[i] = MT6[i]; 392 s_MT7[i] = MT7[i]; 393 } 394 395 SYNC_THREADS (); 396 397 #else 398 399 CONSTANT_AS u64a *s_MT0 = MT0; 400 CONSTANT_AS u64a *s_MT1 = MT1; 401 CONSTANT_AS u64a *s_MT2 = MT2; 402 CONSTANT_AS u64a *s_MT3 = MT3; 403 CONSTANT_AS u64a *s_MT4 = MT4; 404 CONSTANT_AS u64a *s_MT5 = MT5; 405 CONSTANT_AS u64a *s_MT6 = MT6; 406 CONSTANT_AS u64a *s_MT7 = MT7; 407 408 #endif 409 410 if ((gid * VECT_SIZE) >= gid_max) return; 411 412 u32x ipad[16]; 413 u32x opad[16]; 414 415 ipad[ 0] = packv (tmps, ipad, gid, 0); 416 ipad[ 1] = packv (tmps, ipad, gid, 1); 417 ipad[ 2] = packv (tmps, ipad, gid, 2); 418 ipad[ 3] = packv (tmps, ipad, gid, 3); 419 ipad[ 4] = packv (tmps, ipad, gid, 4); 420 ipad[ 5] = packv (tmps, ipad, gid, 5); 421 ipad[ 6] = packv (tmps, ipad, gid, 6); 422 ipad[ 7] = packv (tmps, ipad, gid, 7); 423 ipad[ 8] = packv (tmps, ipad, gid, 8); 424 ipad[ 9] = packv (tmps, ipad, gid, 9); 425 ipad[10] = packv (tmps, ipad, gid, 10); 426 ipad[11] = packv (tmps, ipad, gid, 11); 427 ipad[12] = packv (tmps, ipad, gid, 12); 428 ipad[13] = packv (tmps, ipad, gid, 13); 429 ipad[14] = packv (tmps, ipad, gid, 14); 430 ipad[15] = packv (tmps, ipad, gid, 15); 431 432 opad[ 0] = packv (tmps, opad, gid, 0); 433 opad[ 1] = packv (tmps, opad, gid, 1); 434 opad[ 2] = packv (tmps, opad, gid, 2); 435 opad[ 3] = packv (tmps, opad, gid, 3); 436 opad[ 4] = packv (tmps, opad, gid, 4); 437 opad[ 5] = packv (tmps, opad, gid, 5); 438 opad[ 6] = packv (tmps, opad, gid, 6); 439 opad[ 7] = packv (tmps, opad, gid, 7); 440 opad[ 8] = packv (tmps, opad, gid, 8); 441 opad[ 9] = packv (tmps, opad, gid, 9); 442 opad[10] = packv (tmps, opad, gid, 10); 443 opad[11] = packv (tmps, opad, gid, 11); 444 opad[12] = packv (tmps, opad, gid, 12); 445 opad[13] = packv (tmps, opad, gid, 13); 446 opad[14] = packv (tmps, opad, gid, 14); 447 opad[15] = packv (tmps, opad, gid, 15); 448 449 for (u32 i = 0; i < 16; i += 16) 450 { 451 u32x dgst[16]; 452 u32x out[16]; 453 454 dgst[ 0] = packv (tmps, dgst, gid, i + 0); 455 dgst[ 1] = packv (tmps, dgst, gid, i + 1); 456 dgst[ 2] = packv (tmps, dgst, gid, i + 2); 457 dgst[ 3] = packv (tmps, dgst, gid, i + 3); 458 dgst[ 4] = packv (tmps, dgst, gid, i + 4); 459 dgst[ 5] = packv (tmps, dgst, gid, i + 5); 460 dgst[ 6] = packv (tmps, dgst, gid, i + 6); 461 dgst[ 7] = packv (tmps, dgst, gid, i + 7); 462 dgst[ 8] = packv (tmps, dgst, gid, i + 8); 463 dgst[ 9] = packv (tmps, dgst, gid, i + 9); 464 dgst[10] = packv (tmps, dgst, gid, i + 10); 465 dgst[11] = packv (tmps, dgst, gid, i + 11); 466 dgst[12] = packv (tmps, dgst, gid, i + 12); 467 dgst[13] = packv (tmps, dgst, gid, i + 13); 468 dgst[14] = packv (tmps, dgst, gid, i + 14); 469 dgst[15] = packv (tmps, dgst, gid, i + 15); 470 471 out[ 0] = packv (tmps, out, gid, i + 0); 472 out[ 1] = packv (tmps, out, gid, i + 1); 473 out[ 2] = packv (tmps, out, gid, i + 2); 474 out[ 3] = packv (tmps, out, gid, i + 3); 475 out[ 4] = packv (tmps, out, gid, i + 4); 476 out[ 5] = packv (tmps, out, gid, i + 5); 477 out[ 6] = packv (tmps, out, gid, i + 6); 478 out[ 7] = packv (tmps, out, gid, i + 7); 479 out[ 8] = packv (tmps, out, gid, i + 8); 480 out[ 9] = packv (tmps, out, gid, i + 9); 481 out[10] = packv (tmps, out, gid, i + 10); 482 out[11] = packv (tmps, out, gid, i + 11); 483 out[12] = packv (tmps, out, gid, i + 12); 484 out[13] = packv (tmps, out, gid, i + 13); 485 out[14] = packv (tmps, out, gid, i + 14); 486 out[15] = packv (tmps, out, gid, i + 15); 487 488 for (u32 j = 0; j < loop_cnt; j++) 489 { 490 u32x w0[4]; 491 u32x w1[4]; 492 u32x w2[4]; 493 u32x w3[4]; 494 495 w0[0] = dgst[ 0]; 496 w0[1] = dgst[ 1]; 497 w0[2] = dgst[ 2]; 498 w0[3] = dgst[ 3]; 499 w1[0] = dgst[ 4]; 500 w1[1] = dgst[ 5]; 501 w1[2] = dgst[ 6]; 502 w1[3] = dgst[ 7]; 503 w2[0] = dgst[ 8]; 504 w2[1] = dgst[ 9]; 505 w2[2] = dgst[10]; 506 w2[3] = dgst[11]; 507 w3[0] = dgst[12]; 508 w3[1] = dgst[13]; 509 w3[2] = dgst[14]; 510 w3[3] = dgst[15]; 511 512 hmac_whirlpool_run_V (w0, w1, w2, w3, ipad, opad, dgst, s_MT0, s_MT1, s_MT2, s_MT3, s_MT4, s_MT5, s_MT6, s_MT7); 513 514 out[ 0] ^= dgst[ 0]; 515 out[ 1] ^= dgst[ 1]; 516 out[ 2] ^= dgst[ 2]; 517 out[ 3] ^= dgst[ 3]; 518 out[ 4] ^= dgst[ 4]; 519 out[ 5] ^= dgst[ 5]; 520 out[ 6] ^= dgst[ 6]; 521 out[ 7] ^= dgst[ 7]; 522 out[ 8] ^= dgst[ 8]; 523 out[ 9] ^= dgst[ 9]; 524 out[10] ^= dgst[10]; 525 out[11] ^= dgst[11]; 526 out[12] ^= dgst[12]; 527 out[13] ^= dgst[13]; 528 out[14] ^= dgst[14]; 529 out[15] ^= dgst[15]; 530 } 531 532 unpackv (tmps, dgst, gid, i + 0, dgst[ 0]); 533 unpackv (tmps, dgst, gid, i + 1, dgst[ 1]); 534 unpackv (tmps, dgst, gid, i + 2, dgst[ 2]); 535 unpackv (tmps, dgst, gid, i + 3, dgst[ 3]); 536 unpackv (tmps, dgst, gid, i + 4, dgst[ 4]); 537 unpackv (tmps, dgst, gid, i + 5, dgst[ 5]); 538 unpackv (tmps, dgst, gid, i + 6, dgst[ 6]); 539 unpackv (tmps, dgst, gid, i + 7, dgst[ 7]); 540 unpackv (tmps, dgst, gid, i + 8, dgst[ 8]); 541 unpackv (tmps, dgst, gid, i + 9, dgst[ 9]); 542 unpackv (tmps, dgst, gid, i + 10, dgst[10]); 543 unpackv (tmps, dgst, gid, i + 11, dgst[11]); 544 unpackv (tmps, dgst, gid, i + 12, dgst[12]); 545 unpackv (tmps, dgst, gid, i + 13, dgst[13]); 546 unpackv (tmps, dgst, gid, i + 14, dgst[14]); 547 unpackv (tmps, dgst, gid, i + 15, dgst[15]); 548 549 unpackv (tmps, out, gid, i + 0, out[ 0]); 550 unpackv (tmps, out, gid, i + 1, out[ 1]); 551 unpackv (tmps, out, gid, i + 2, out[ 2]); 552 unpackv (tmps, out, gid, i + 3, out[ 3]); 553 unpackv (tmps, out, gid, i + 4, out[ 4]); 554 unpackv (tmps, out, gid, i + 5, out[ 5]); 555 unpackv (tmps, out, gid, i + 6, out[ 6]); 556 unpackv (tmps, out, gid, i + 7, out[ 7]); 557 unpackv (tmps, out, gid, i + 8, out[ 8]); 558 unpackv (tmps, out, gid, i + 9, out[ 9]); 559 unpackv (tmps, out, gid, i + 10, out[10]); 560 unpackv (tmps, out, gid, i + 11, out[11]); 561 unpackv (tmps, out, gid, i + 12, out[12]); 562 unpackv (tmps, out, gid, i + 13, out[13]); 563 unpackv (tmps, out, gid, i + 14, out[14]); 564 unpackv (tmps, out, gid, i + 15, out[15]); 565 } 566} 567 568KERNEL_FQ void m06231_comp (KERN_ATTR_TMPS_ESALT (tc_tmp_t, tc_t)) 569{ 570 const u64 gid = get_global_id (0); 571 const u64 lid = get_local_id (0); 572 const u64 lsz = get_local_size (0); 573 574 /** 575 * aes shared 576 */ 577 578 #ifdef REAL_SHM 579 580 LOCAL_VK u32 s_td0[256]; 581 LOCAL_VK u32 s_td1[256]; 582 LOCAL_VK u32 s_td2[256]; 583 LOCAL_VK u32 s_td3[256]; 584 LOCAL_VK u32 s_td4[256]; 585 586 LOCAL_VK u32 s_te0[256]; 587 LOCAL_VK u32 s_te1[256]; 588 LOCAL_VK u32 s_te2[256]; 589 LOCAL_VK u32 s_te3[256]; 590 LOCAL_VK u32 s_te4[256]; 591 592 for (u32 i = lid; i < 256; i += lsz) 593 { 594 s_td0[i] = td0[i]; 595 s_td1[i] = td1[i]; 596 s_td2[i] = td2[i]; 597 s_td3[i] = td3[i]; 598 s_td4[i] = td4[i]; 599 600 s_te0[i] = te0[i]; 601 s_te1[i] = te1[i]; 602 s_te2[i] = te2[i]; 603 s_te3[i] = te3[i]; 604 s_te4[i] = te4[i]; 605 } 606 607 SYNC_THREADS (); 608 609 #else 610 611 CONSTANT_AS u32a *s_td0 = td0; 612 CONSTANT_AS u32a *s_td1 = td1; 613 CONSTANT_AS u32a *s_td2 = td2; 614 CONSTANT_AS u32a *s_td3 = td3; 615 CONSTANT_AS u32a *s_td4 = td4; 616 617 CONSTANT_AS u32a *s_te0 = te0; 618 CONSTANT_AS u32a *s_te1 = te1; 619 CONSTANT_AS u32a *s_te2 = te2; 620 CONSTANT_AS u32a *s_te3 = te3; 621 CONSTANT_AS u32a *s_te4 = te4; 622 623 #endif 624 625 if (gid >= gid_max) return; 626 627 u32 ukey1[8]; 628 629 ukey1[0] = hc_swap32_S (tmps[gid].out[ 0]); 630 ukey1[1] = hc_swap32_S (tmps[gid].out[ 1]); 631 ukey1[2] = hc_swap32_S (tmps[gid].out[ 2]); 632 ukey1[3] = hc_swap32_S (tmps[gid].out[ 3]); 633 ukey1[4] = hc_swap32_S (tmps[gid].out[ 4]); 634 ukey1[5] = hc_swap32_S (tmps[gid].out[ 5]); 635 ukey1[6] = hc_swap32_S (tmps[gid].out[ 6]); 636 ukey1[7] = hc_swap32_S (tmps[gid].out[ 7]); 637 638 u32 ukey2[8]; 639 640 ukey2[0] = hc_swap32_S (tmps[gid].out[ 8]); 641 ukey2[1] = hc_swap32_S (tmps[gid].out[ 9]); 642 ukey2[2] = hc_swap32_S (tmps[gid].out[10]); 643 ukey2[3] = hc_swap32_S (tmps[gid].out[11]); 644 ukey2[4] = hc_swap32_S (tmps[gid].out[12]); 645 ukey2[5] = hc_swap32_S (tmps[gid].out[13]); 646 ukey2[6] = hc_swap32_S (tmps[gid].out[14]); 647 ukey2[7] = hc_swap32_S (tmps[gid].out[15]); 648 649 if (verify_header_serpent (esalt_bufs[0].data_buf, esalt_bufs[0].signature, ukey1, ukey2) == 1) 650 { 651 if (hc_atomic_inc (&hashes_shown[0]) == 0) 652 { 653 mark_hash (plains_buf, d_return_buf, SALT_POS, digests_cnt, 0, 0, gid, 0, 0, 0); 654 } 655 } 656 657 if (verify_header_twofish (esalt_bufs[0].data_buf, esalt_bufs[0].signature, ukey1, ukey2) == 1) 658 { 659 if (hc_atomic_inc (&hashes_shown[0]) == 0) 660 { 661 mark_hash (plains_buf, d_return_buf, SALT_POS, digests_cnt, 0, 0, gid, 0, 0, 0); 662 } 663 } 664 665 if (verify_header_aes (esalt_bufs[0].data_buf, esalt_bufs[0].signature, ukey1, ukey2, s_te0, s_te1, s_te2, s_te3, s_te4, s_td0, s_td1, s_td2, s_td3, s_td4) == 1) 666 { 667 if (hc_atomic_inc (&hashes_shown[0]) == 0) 668 { 669 mark_hash (plains_buf, d_return_buf, SALT_POS, digests_cnt, 0, 0, gid, 0, 0, 0); 670 } 671 } 672} 673