1// Copyright 2016 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// +build ignore 6 7package aes 8 9import ( 10 "crypto/cipher" 11 subtleoverlap "crypto/internal/subtle" 12 "crypto/subtle" 13 "encoding/binary" 14 "errors" 15 "internal/cpu" 16) 17 18// This file contains two implementations of AES-GCM. The first implementation 19// (gcmAsm) uses the KMCTR instruction to encrypt using AES in counter mode and 20// the KIMD instruction for GHASH. The second implementation (gcmKMA) uses the 21// newer KMA instruction which performs both operations. 22 23// gcmCount represents a 16-byte big-endian count value. 24type gcmCount [16]byte 25 26// inc increments the rightmost 32-bits of the count value by 1. 27func (x *gcmCount) inc() { 28 binary.BigEndian.PutUint32(x[len(x)-4:], binary.BigEndian.Uint32(x[len(x)-4:])+1) 29} 30 31// gcmLengths writes len0 || len1 as big-endian values to a 16-byte array. 32func gcmLengths(len0, len1 uint64) [16]byte { 33 v := [16]byte{} 34 binary.BigEndian.PutUint64(v[0:], len0) 35 binary.BigEndian.PutUint64(v[8:], len1) 36 return v 37} 38 39// gcmHashKey represents the 16-byte hash key required by the GHASH algorithm. 40type gcmHashKey [16]byte 41 42type gcmAsm struct { 43 block *aesCipherAsm 44 hashKey gcmHashKey 45 nonceSize int 46 tagSize int 47} 48 49const ( 50 gcmBlockSize = 16 51 gcmTagSize = 16 52 gcmMinimumTagSize = 12 // NIST SP 800-38D recommends tags with 12 or more bytes. 53 gcmStandardNonceSize = 12 54) 55 56var errOpen = errors.New("cipher: message authentication failed") 57 58// Assert that aesCipherAsm implements the gcmAble interface. 59var _ gcmAble = (*aesCipherAsm)(nil) 60 61// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only 62// called by crypto/cipher.NewGCM via the gcmAble interface. 63func (c *aesCipherAsm) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) { 64 var hk gcmHashKey 65 c.Encrypt(hk[:], hk[:]) 66 g := gcmAsm{ 67 block: c, 68 hashKey: hk, 69 nonceSize: nonceSize, 70 tagSize: tagSize, 71 } 72 if cpu.S390X.HasAESGCM { 73 g := gcmKMA{g} 74 return &g, nil 75 } 76 return &g, nil 77} 78 79func (g *gcmAsm) NonceSize() int { 80 return g.nonceSize 81} 82 83func (g *gcmAsm) Overhead() int { 84 return g.tagSize 85} 86 87// sliceForAppend takes a slice and a requested number of bytes. It returns a 88// slice with the contents of the given slice followed by that many bytes and a 89// second slice that aliases into it and contains only the extra bytes. If the 90// original slice has sufficient capacity then no allocation is performed. 91func sliceForAppend(in []byte, n int) (head, tail []byte) { 92 if total := len(in) + n; cap(in) >= total { 93 head = in[:total] 94 } else { 95 head = make([]byte, total) 96 copy(head, in) 97 } 98 tail = head[len(in):] 99 return 100} 101 102// ghash uses the GHASH algorithm to hash data with the given key. The initial 103// hash value is given by hash which will be updated with the new hash value. 104// The length of data must be a multiple of 16-bytes. 105//go:noescape 106func ghash(key *gcmHashKey, hash *[16]byte, data []byte) 107 108// paddedGHASH pads data with zeroes until its length is a multiple of 109// 16-bytes. It then calculates a new value for hash using the GHASH algorithm. 110func (g *gcmAsm) paddedGHASH(hash *[16]byte, data []byte) { 111 siz := len(data) &^ 0xf // align size to 16-bytes 112 if siz > 0 { 113 ghash(&g.hashKey, hash, data[:siz]) 114 data = data[siz:] 115 } 116 if len(data) > 0 { 117 var s [16]byte 118 copy(s[:], data) 119 ghash(&g.hashKey, hash, s[:]) 120 } 121} 122 123// cryptBlocksGCM encrypts src using AES in counter mode using the given 124// function code and key. The rightmost 32-bits of the counter are incremented 125// between each block as required by the GCM spec. The initial counter value 126// is given by cnt, which is updated with the value of the next counter value 127// to use. 128// 129// The lengths of both dst and buf must be greater than or equal to the length 130// of src. buf may be partially or completely overwritten during the execution 131// of the function. 132//go:noescape 133func cryptBlocksGCM(fn code, key, dst, src, buf []byte, cnt *gcmCount) 134 135// counterCrypt encrypts src using AES in counter mode and places the result 136// into dst. cnt is the initial count value and will be updated with the next 137// count value. The length of dst must be greater than or equal to the length 138// of src. 139func (g *gcmAsm) counterCrypt(dst, src []byte, cnt *gcmCount) { 140 // Copying src into a buffer improves performance on some models when 141 // src and dst point to the same underlying array. We also need a 142 // buffer for counter values. 143 var ctrbuf, srcbuf [2048]byte 144 for len(src) >= 16 { 145 siz := len(src) 146 if len(src) > len(ctrbuf) { 147 siz = len(ctrbuf) 148 } 149 siz &^= 0xf // align siz to 16-bytes 150 copy(srcbuf[:], src[:siz]) 151 cryptBlocksGCM(g.block.function, g.block.key, dst[:siz], srcbuf[:siz], ctrbuf[:], cnt) 152 src = src[siz:] 153 dst = dst[siz:] 154 } 155 if len(src) > 0 { 156 var x [16]byte 157 g.block.Encrypt(x[:], cnt[:]) 158 for i := range src { 159 dst[i] = src[i] ^ x[i] 160 } 161 cnt.inc() 162 } 163} 164 165// deriveCounter computes the initial GCM counter state from the given nonce. 166// See NIST SP 800-38D, section 7.1. 167func (g *gcmAsm) deriveCounter(nonce []byte) gcmCount { 168 // GCM has two modes of operation with respect to the initial counter 169 // state: a "fast path" for 96-bit (12-byte) nonces, and a "slow path" 170 // for nonces of other lengths. For a 96-bit nonce, the nonce, along 171 // with a four-byte big-endian counter starting at one, is used 172 // directly as the starting counter. For other nonce sizes, the counter 173 // is computed by passing it through the GHASH function. 174 var counter gcmCount 175 if len(nonce) == gcmStandardNonceSize { 176 copy(counter[:], nonce) 177 counter[gcmBlockSize-1] = 1 178 } else { 179 var hash [16]byte 180 g.paddedGHASH(&hash, nonce) 181 lens := gcmLengths(0, uint64(len(nonce))*8) 182 g.paddedGHASH(&hash, lens[:]) 183 copy(counter[:], hash[:]) 184 } 185 return counter 186} 187 188// auth calculates GHASH(ciphertext, additionalData), masks the result with 189// tagMask and writes the result to out. 190func (g *gcmAsm) auth(out, ciphertext, additionalData []byte, tagMask *[gcmTagSize]byte) { 191 var hash [16]byte 192 g.paddedGHASH(&hash, additionalData) 193 g.paddedGHASH(&hash, ciphertext) 194 lens := gcmLengths(uint64(len(additionalData))*8, uint64(len(ciphertext))*8) 195 g.paddedGHASH(&hash, lens[:]) 196 197 copy(out, hash[:]) 198 for i := range out { 199 out[i] ^= tagMask[i] 200 } 201} 202 203// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for 204// details. 205func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte { 206 if len(nonce) != g.nonceSize { 207 panic("crypto/cipher: incorrect nonce length given to GCM") 208 } 209 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 210 panic("crypto/cipher: message too large for GCM") 211 } 212 213 ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize) 214 if subtleoverlap.InexactOverlap(out[:len(plaintext)], plaintext) { 215 panic("crypto/cipher: invalid buffer overlap") 216 } 217 218 counter := g.deriveCounter(nonce) 219 220 var tagMask [gcmBlockSize]byte 221 g.block.Encrypt(tagMask[:], counter[:]) 222 counter.inc() 223 224 var tagOut [gcmTagSize]byte 225 g.counterCrypt(out, plaintext, &counter) 226 g.auth(tagOut[:], out[:len(plaintext)], data, &tagMask) 227 copy(out[len(plaintext):], tagOut[:]) 228 229 return ret 230} 231 232// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface 233// for details. 234func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 235 if len(nonce) != g.nonceSize { 236 panic("crypto/cipher: incorrect nonce length given to GCM") 237 } 238 // Sanity check to prevent the authentication from always succeeding if an implementation 239 // leaves tagSize uninitialized, for example. 240 if g.tagSize < gcmMinimumTagSize { 241 panic("crypto/cipher: incorrect GCM tag size") 242 } 243 if len(ciphertext) < g.tagSize { 244 return nil, errOpen 245 } 246 if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) { 247 return nil, errOpen 248 } 249 250 tag := ciphertext[len(ciphertext)-g.tagSize:] 251 ciphertext = ciphertext[:len(ciphertext)-g.tagSize] 252 253 counter := g.deriveCounter(nonce) 254 255 var tagMask [gcmBlockSize]byte 256 g.block.Encrypt(tagMask[:], counter[:]) 257 counter.inc() 258 259 var expectedTag [gcmTagSize]byte 260 g.auth(expectedTag[:], ciphertext, data, &tagMask) 261 262 ret, out := sliceForAppend(dst, len(ciphertext)) 263 if subtleoverlap.InexactOverlap(out, ciphertext) { 264 panic("crypto/cipher: invalid buffer overlap") 265 } 266 267 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 { 268 // The AESNI code decrypts and authenticates concurrently, and 269 // so overwrites dst in the event of a tag mismatch. That 270 // behavior is mimicked here in order to be consistent across 271 // platforms. 272 for i := range out { 273 out[i] = 0 274 } 275 return nil, errOpen 276 } 277 278 g.counterCrypt(out, ciphertext, &counter) 279 return ret, nil 280} 281 282// gcmKMA implements the cipher.AEAD interface using the KMA instruction. It should 283// only be used if hasKMA is true. 284type gcmKMA struct { 285 gcmAsm 286} 287 288// flags for the KMA instruction 289const ( 290 kmaHS = 1 << 10 // hash subkey supplied 291 kmaLAAD = 1 << 9 // last series of additional authenticated data 292 kmaLPC = 1 << 8 // last series of plaintext or ciphertext blocks 293 kmaDecrypt = 1 << 7 // decrypt 294) 295 296// kmaGCM executes the encryption or decryption operation given by fn. The tag 297// will be calculated and written to tag. cnt should contain the current 298// counter state and will be overwritten with the updated counter state. 299// TODO(mundaym): could pass in hash subkey 300//go:noescape 301func kmaGCM(fn code, key, dst, src, aad []byte, tag *[16]byte, cnt *gcmCount) 302 303// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for 304// details. 305func (g *gcmKMA) Seal(dst, nonce, plaintext, data []byte) []byte { 306 if len(nonce) != g.nonceSize { 307 panic("crypto/cipher: incorrect nonce length given to GCM") 308 } 309 if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize { 310 panic("crypto/cipher: message too large for GCM") 311 } 312 313 ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize) 314 if subtleoverlap.InexactOverlap(out[:len(plaintext)], plaintext) { 315 panic("crypto/cipher: invalid buffer overlap") 316 } 317 318 counter := g.deriveCounter(nonce) 319 fc := g.block.function | kmaLAAD | kmaLPC 320 321 var tag [gcmTagSize]byte 322 kmaGCM(fc, g.block.key, out[:len(plaintext)], plaintext, data, &tag, &counter) 323 copy(out[len(plaintext):], tag[:]) 324 325 return ret 326} 327 328// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface 329// for details. 330func (g *gcmKMA) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) { 331 if len(nonce) != g.nonceSize { 332 panic("crypto/cipher: incorrect nonce length given to GCM") 333 } 334 if len(ciphertext) < g.tagSize { 335 return nil, errOpen 336 } 337 if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) { 338 return nil, errOpen 339 } 340 341 tag := ciphertext[len(ciphertext)-g.tagSize:] 342 ciphertext = ciphertext[:len(ciphertext)-g.tagSize] 343 ret, out := sliceForAppend(dst, len(ciphertext)) 344 if subtleoverlap.InexactOverlap(out, ciphertext) { 345 panic("crypto/cipher: invalid buffer overlap") 346 } 347 348 if g.tagSize < gcmMinimumTagSize { 349 panic("crypto/cipher: incorrect GCM tag size") 350 } 351 352 counter := g.deriveCounter(nonce) 353 fc := g.block.function | kmaLAAD | kmaLPC | kmaDecrypt 354 355 var expectedTag [gcmTagSize]byte 356 kmaGCM(fc, g.block.key, out[:len(ciphertext)], ciphertext, data, &expectedTag, &counter) 357 358 if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 { 359 // The AESNI code decrypts and authenticates concurrently, and 360 // so overwrites dst in the event of a tag mismatch. That 361 // behavior is mimicked here in order to be consistent across 362 // platforms. 363 for i := range out { 364 out[i] = 0 365 } 366 return nil, errOpen 367 } 368 369 return ret, nil 370} 371