1 /*
2  * Copyright (c) 2018 Thomas Pornin <pornin@bolet.org>
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining
5  * a copy of this software and associated documentation files (the
6  * "Software"), to deal in the Software without restriction, including
7  * without limitation the rights to use, copy, modify, merge, publish,
8  * distribute, sublicense, and/or sell copies of the Software, and to
9  * permit persons to whom the Software is furnished to do so, subject to
10  * the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be
13  * included in all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 
25 #include "inner.h"
26 
27 /* see inner.h */
28 uint32_t
29 br_rsa_pss_sig_unpad(const br_hash_class *hf_data,
30 	const br_hash_class *hf_mgf1,
31 	const unsigned char *hash, size_t salt_len,
32 	const br_rsa_public_key *pk, unsigned char *x)
33 {
34 	size_t u, xlen, hash_len;
35 	br_hash_compat_context hc;
36 	unsigned char *seed, *salt;
37 	unsigned char tmp[64];
38 	uint32_t r, n_bitlen;
39 
40 	hash_len = br_digest_size(hf_data);
41 
42 	/*
43 	 * Value r will be set to a non-zero value is any test fails.
44 	 */
45 	r = 0;
46 
47 	/*
48 	 * The value bit length (as an integer) must be strictly less than
49 	 * that of the modulus.
50 	 */
51 	for (u = 0; u < pk->nlen; u ++) {
52 		if (pk->n[u] != 0) {
53 			break;
54 		}
55 	}
56 	if (u == pk->nlen) {
57 		return 0;
58 	}
59 	n_bitlen = BIT_LENGTH(pk->n[u]) + ((uint32_t)(pk->nlen - u - 1) << 3);
60 	n_bitlen --;
61 	if ((n_bitlen & 7) == 0) {
62 		r |= *x ++;
63 	} else {
64 		r |= x[0] & (0xFF << (n_bitlen & 7));
65 	}
66 	xlen = (n_bitlen + 7) >> 3;
67 
68 	/*
69 	 * Check that the modulus is large enough for the hash value
70 	 * length combined with the intended salt length.
71 	 */
72 	if (hash_len > xlen || salt_len > xlen
73 		|| (hash_len + salt_len + 2) > xlen)
74 	{
75 		return 0;
76 	}
77 
78 	/*
79 	 * Check value of rightmost byte.
80 	 */
81 	r |= x[xlen - 1] ^ 0xBC;
82 
83 	/*
84 	 * Generate the mask and XOR it into the first bytes to reveal PS;
85 	 * we must also mask out the leading bits.
86 	 */
87 	seed = x + xlen - hash_len - 1;
88 	br_mgf1_xor(x, xlen - hash_len - 1, hf_mgf1, seed, hash_len);
89 	if ((n_bitlen & 7) != 0) {
90 		x[0] &= 0xFF >> (8 - (n_bitlen & 7));
91 	}
92 
93 	/*
94 	 * Check that all padding bytes have the expected value.
95 	 */
96 	for (u = 0; u < (xlen - hash_len - salt_len - 2); u ++) {
97 		r |= x[u];
98 	}
99 	r |= x[xlen - hash_len - salt_len - 2] ^ 0x01;
100 
101 	/*
102 	 * Recompute H.
103 	 */
104 	salt = x + xlen - hash_len - salt_len - 1;
105 	hf_data->init(&hc.vtable);
106 	memset(tmp, 0, 8);
107 	hf_data->update(&hc.vtable, tmp, 8);
108 	hf_data->update(&hc.vtable, hash, hash_len);
109 	hf_data->update(&hc.vtable, salt, salt_len);
110 	hf_data->out(&hc.vtable, tmp);
111 
112 	/*
113 	 * Check that the recomputed H value matches the one appearing
114 	 * in the string.
115 	 */
116 	for (u = 0; u < hash_len; u ++) {
117 		r |= tmp[u] ^ x[(xlen - salt_len - 1) + u];
118 	}
119 
120 	return EQ0(r);
121 }
122