1// Copyright 2019 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build ignore
6// +build ppc64le
7
8package aes
9
10import (
11	"crypto/cipher"
12	"crypto/subtle"
13	"encoding/binary"
14	"errors"
15)
16
17// This file implements GCM using an optimized GHASH function.
18
19//go:noescape
20func gcmInit(productTable *[256]byte, h []byte)
21
22//go:noescape
23func gcmHash(output []byte, productTable *[256]byte, inp []byte, len int)
24
25//go:noescape
26func gcmMul(output []byte, productTable *[256]byte)
27
28const (
29	gcmCounterSize       = 16
30	gcmBlockSize         = 16
31	gcmTagSize           = 16
32	gcmStandardNonceSize = 12
33)
34
35var errOpen = errors.New("cipher: message authentication failed")
36
37// Assert that aesCipherGCM implements the gcmAble interface.
38var _ gcmAble = (*aesCipherAsm)(nil)
39
40type gcmAsm struct {
41	cipher *aesCipherAsm
42	// ks is the key schedule, the length of which depends on the size of
43	// the AES key.
44	ks []uint32
45	// productTable contains pre-computed multiples of the binary-field
46	// element used in GHASH.
47	productTable [256]byte
48	// nonceSize contains the expected size of the nonce, in bytes.
49	nonceSize int
50	// tagSize contains the size of the tag, in bytes.
51	tagSize int
52}
53
54// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only
55// called by crypto/cipher.NewGCM via the gcmAble interface.
56func (c *aesCipherAsm) NewGCM(nonceSize, tagSize int) (cipher.AEAD, error) {
57	g := &gcmAsm{cipher: c, ks: c.enc, nonceSize: nonceSize, tagSize: tagSize}
58
59	hle := make([]byte, gcmBlockSize)
60	c.Encrypt(hle, hle)
61
62	// Reverse the bytes in each 8 byte chunk
63	// Load little endian, store big endian
64	h1 := binary.LittleEndian.Uint64(hle[:8])
65	h2 := binary.LittleEndian.Uint64(hle[8:])
66	binary.BigEndian.PutUint64(hle[:8], h1)
67	binary.BigEndian.PutUint64(hle[8:], h2)
68	gcmInit(&g.productTable, hle)
69
70	return g, nil
71}
72
73func (g *gcmAsm) NonceSize() int {
74	return g.nonceSize
75}
76
77func (g *gcmAsm) Overhead() int {
78	return g.tagSize
79}
80
81func sliceForAppend(in []byte, n int) (head, tail []byte) {
82	if total := len(in) + n; cap(in) >= total {
83		head = in[:total]
84	} else {
85		head = make([]byte, total)
86		copy(head, in)
87	}
88	tail = head[len(in):]
89	return
90}
91
92// deriveCounter computes the initial GCM counter state from the given nonce.
93func (g *gcmAsm) deriveCounter(counter *[gcmBlockSize]byte, nonce []byte) {
94	if len(nonce) == gcmStandardNonceSize {
95		copy(counter[:], nonce)
96		counter[gcmBlockSize-1] = 1
97	} else {
98		var hash [16]byte
99		g.paddedGHASH(&hash, nonce)
100		lens := gcmLengths(0, uint64(len(nonce))*8)
101		g.paddedGHASH(&hash, lens[:])
102		copy(counter[:], hash[:])
103	}
104}
105
106// counterCrypt encrypts in using AES in counter mode and places the result
107// into out. counter is the initial count value and will be updated with the next
108// count value. The length of out must be greater than or equal to the length
109// of in.
110func (g *gcmAsm) counterCrypt(out, in []byte, counter *[gcmBlockSize]byte) {
111	var mask [gcmBlockSize]byte
112
113	for len(in) >= gcmBlockSize {
114		// Hint to avoid bounds check
115		_, _ = in[15], out[15]
116		g.cipher.Encrypt(mask[:], counter[:])
117		gcmInc32(counter)
118
119		// XOR 16 bytes each loop iteration in 8 byte chunks
120		in0 := binary.LittleEndian.Uint64(in[0:])
121		in1 := binary.LittleEndian.Uint64(in[8:])
122		m0 := binary.LittleEndian.Uint64(mask[:8])
123		m1 := binary.LittleEndian.Uint64(mask[8:])
124		binary.LittleEndian.PutUint64(out[:8], in0^m0)
125		binary.LittleEndian.PutUint64(out[8:], in1^m1)
126		out = out[16:]
127		in = in[16:]
128	}
129
130	if len(in) > 0 {
131		g.cipher.Encrypt(mask[:], counter[:])
132		gcmInc32(counter)
133		// XOR leftover bytes
134		for i, inb := range in {
135			out[i] = inb ^ mask[i]
136		}
137	}
138}
139
140// increments the rightmost 32-bits of the count value by 1.
141func gcmInc32(counterBlock *[16]byte) {
142	c := counterBlock[len(counterBlock)-4:]
143	x := binary.BigEndian.Uint32(c) + 1
144	binary.BigEndian.PutUint32(c, x)
145}
146
147// paddedGHASH pads data with zeroes until its length is a multiple of
148// 16-bytes. It then calculates a new value for hash using the ghash
149// algorithm.
150func (g *gcmAsm) paddedGHASH(hash *[16]byte, data []byte) {
151	if siz := len(data) - (len(data) % gcmBlockSize); siz > 0 {
152		gcmHash(hash[:], &g.productTable, data[:], siz)
153		data = data[siz:]
154	}
155	if len(data) > 0 {
156		var s [16]byte
157		copy(s[:], data)
158		gcmHash(hash[:], &g.productTable, s[:], len(s))
159	}
160}
161
162// auth calculates GHASH(ciphertext, additionalData), masks the result with
163// tagMask and writes the result to out.
164func (g *gcmAsm) auth(out, ciphertext, aad []byte, tagMask *[gcmTagSize]byte) {
165	var hash [16]byte
166	g.paddedGHASH(&hash, aad)
167	g.paddedGHASH(&hash, ciphertext)
168	lens := gcmLengths(uint64(len(aad))*8, uint64(len(ciphertext))*8)
169	g.paddedGHASH(&hash, lens[:])
170
171	copy(out, hash[:])
172	for i := range out {
173		out[i] ^= tagMask[i]
174	}
175}
176
177// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for
178// details.
179func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte {
180	if len(nonce) != g.nonceSize {
181		panic("cipher: incorrect nonce length given to GCM")
182	}
183	if uint64(len(plaintext)) > ((1<<32)-2)*BlockSize {
184		panic("cipher: message too large for GCM")
185	}
186
187	ret, out := sliceForAppend(dst, len(plaintext)+g.tagSize)
188
189	var counter, tagMask [gcmBlockSize]byte
190	g.deriveCounter(&counter, nonce)
191
192	g.cipher.Encrypt(tagMask[:], counter[:])
193	gcmInc32(&counter)
194
195	g.counterCrypt(out, plaintext, &counter)
196	g.auth(out[len(plaintext):], out[:len(plaintext)], data, &tagMask)
197
198	return ret
199}
200
201// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface
202// for details.
203func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
204	if len(nonce) != g.nonceSize {
205		panic("cipher: incorrect nonce length given to GCM")
206	}
207	if len(ciphertext) < g.tagSize {
208		return nil, errOpen
209	}
210	if uint64(len(ciphertext)) > ((1<<32)-2)*uint64(BlockSize)+uint64(g.tagSize) {
211		return nil, errOpen
212	}
213
214	tag := ciphertext[len(ciphertext)-g.tagSize:]
215	ciphertext = ciphertext[:len(ciphertext)-g.tagSize]
216
217	var counter, tagMask [gcmBlockSize]byte
218	g.deriveCounter(&counter, nonce)
219
220	g.cipher.Encrypt(tagMask[:], counter[:])
221	gcmInc32(&counter)
222
223	var expectedTag [gcmTagSize]byte
224	g.auth(expectedTag[:], ciphertext, data, &tagMask)
225
226	ret, out := sliceForAppend(dst, len(ciphertext))
227
228	if subtle.ConstantTimeCompare(expectedTag[:g.tagSize], tag) != 1 {
229		for i := range out {
230			out[i] = 0
231		}
232		return nil, errOpen
233	}
234
235	g.counterCrypt(out, ciphertext, &counter)
236	return ret, nil
237}
238
239func gcmLengths(len0, len1 uint64) [16]byte {
240	return [16]byte{
241		byte(len0 >> 56),
242		byte(len0 >> 48),
243		byte(len0 >> 40),
244		byte(len0 >> 32),
245		byte(len0 >> 24),
246		byte(len0 >> 16),
247		byte(len0 >> 8),
248		byte(len0),
249		byte(len1 >> 56),
250		byte(len1 >> 48),
251		byte(len1 >> 40),
252		byte(len1 >> 32),
253		byte(len1 >> 24),
254		byte(len1 >> 16),
255		byte(len1 >> 8),
256		byte(len1),
257	}
258}
259