1 2 #include <stdint.h> 3 #include <stdlib.h> 4 #include <string.h> 5 6 #include "core.h" 7 #include "crypto_stream_chacha20.h" 8 #include "private/common.h" 9 #include "private/sse2_64_32.h" 10 #include "utils.h" 11 12 #if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ 13 defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) 14 15 # ifdef __GNUC__ 16 # pragma GCC target("sse2") 17 # pragma GCC target("ssse3") 18 # pragma GCC target("sse4.1") 19 # pragma GCC target("avx2") 20 # endif 21 22 # include <emmintrin.h> 23 # include <immintrin.h> 24 # include <smmintrin.h> 25 # include <tmmintrin.h> 26 27 # include "../stream_chacha20.h" 28 # include "chacha20_dolbeau-avx2.h" 29 30 # define ROUNDS 20 31 32 typedef struct chacha_ctx { 33 uint32_t input[16]; 34 } chacha_ctx; 35 36 static void 37 chacha_keysetup(chacha_ctx *ctx, const uint8_t *k) 38 { 39 ctx->input[0] = 0x61707865; 40 ctx->input[1] = 0x3320646e; 41 ctx->input[2] = 0x79622d32; 42 ctx->input[3] = 0x6b206574; 43 ctx->input[4] = LOAD32_LE(k + 0); 44 ctx->input[5] = LOAD32_LE(k + 4); 45 ctx->input[6] = LOAD32_LE(k + 8); 46 ctx->input[7] = LOAD32_LE(k + 12); 47 ctx->input[8] = LOAD32_LE(k + 16); 48 ctx->input[9] = LOAD32_LE(k + 20); 49 ctx->input[10] = LOAD32_LE(k + 24); 50 ctx->input[11] = LOAD32_LE(k + 28); 51 } 52 53 static void 54 chacha_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) 55 { 56 ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter + 0); 57 ctx->input[13] = counter == NULL ? 0 : LOAD32_LE(counter + 4); 58 ctx->input[14] = LOAD32_LE(iv + 0); 59 ctx->input[15] = LOAD32_LE(iv + 4); 60 } 61 62 static void 63 chacha_ietf_ivsetup(chacha_ctx *ctx, const uint8_t *iv, const uint8_t *counter) 64 { 65 ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter); 66 ctx->input[13] = LOAD32_LE(iv + 0); 67 ctx->input[14] = LOAD32_LE(iv + 4); 68 ctx->input[15] = LOAD32_LE(iv + 8); 69 } 70 71 static void 72 chacha20_encrypt_bytes(chacha_ctx *ctx, const uint8_t *m, uint8_t *c, 73 unsigned long long bytes) 74 { 75 uint32_t * const x = &ctx->input[0]; 76 77 if (!bytes) { 78 return; /* LCOV_EXCL_LINE */ 79 } 80 if (bytes > crypto_stream_chacha20_MESSAGEBYTES_MAX) { 81 sodium_misuse(); 82 } 83 # include "u8.h" 84 # include "u4.h" 85 # include "u1.h" 86 # include "u0.h" 87 } 88 89 static int 90 stream_ref(unsigned char *c, unsigned long long clen, const unsigned char *n, 91 const unsigned char *k) 92 { 93 struct chacha_ctx ctx; 94 95 if (!clen) { 96 return 0; 97 } 98 COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); 99 chacha_keysetup(&ctx, k); 100 chacha_ivsetup(&ctx, n, NULL); 101 memset(c, 0, clen); 102 chacha20_encrypt_bytes(&ctx, c, c, clen); 103 sodium_memzero(&ctx, sizeof ctx); 104 105 return 0; 106 } 107 108 static int 109 stream_ietf_ref(unsigned char *c, unsigned long long clen, 110 const unsigned char *n, const unsigned char *k) 111 { 112 struct chacha_ctx ctx; 113 114 if (!clen) { 115 return 0; 116 } 117 COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); 118 chacha_keysetup(&ctx, k); 119 chacha_ietf_ivsetup(&ctx, n, NULL); 120 memset(c, 0, clen); 121 chacha20_encrypt_bytes(&ctx, c, c, clen); 122 sodium_memzero(&ctx, sizeof ctx); 123 124 return 0; 125 } 126 127 static int 128 stream_ref_xor_ic(unsigned char *c, const unsigned char *m, 129 unsigned long long mlen, const unsigned char *n, uint64_t ic, 130 const unsigned char *k) 131 { 132 struct chacha_ctx ctx; 133 uint8_t ic_bytes[8]; 134 uint32_t ic_high; 135 uint32_t ic_low; 136 137 if (!mlen) { 138 return 0; 139 } 140 ic_high = (uint32_t) (ic >> 32); 141 ic_low = (uint32_t) ic; 142 STORE32_LE(&ic_bytes[0], ic_low); 143 STORE32_LE(&ic_bytes[4], ic_high); 144 chacha_keysetup(&ctx, k); 145 chacha_ivsetup(&ctx, n, ic_bytes); 146 chacha20_encrypt_bytes(&ctx, m, c, mlen); 147 sodium_memzero(&ctx, sizeof ctx); 148 149 return 0; 150 } 151 152 static int 153 stream_ietf_ref_xor_ic(unsigned char *c, const unsigned char *m, 154 unsigned long long mlen, const unsigned char *n, 155 uint32_t ic, const unsigned char *k) 156 { 157 struct chacha_ctx ctx; 158 uint8_t ic_bytes[4]; 159 160 if (!mlen) { 161 return 0; 162 } 163 STORE32_LE(ic_bytes, ic); 164 chacha_keysetup(&ctx, k); 165 chacha_ietf_ivsetup(&ctx, n, ic_bytes); 166 chacha20_encrypt_bytes(&ctx, m, c, mlen); 167 sodium_memzero(&ctx, sizeof ctx); 168 169 return 0; 170 } 171 172 struct crypto_stream_chacha20_implementation 173 crypto_stream_chacha20_dolbeau_avx2_implementation = { 174 SODIUM_C99(.stream =) stream_ref, 175 SODIUM_C99(.stream_ietf =) stream_ietf_ref, 176 SODIUM_C99(.stream_xor_ic =) stream_ref_xor_ic, 177 SODIUM_C99(.stream_ietf_xor_ic =) stream_ietf_ref_xor_ic 178 }; 179 180 #endif 181