xref: /freebsd/contrib/wpa/src/tls/pkcs1.c (revision aa0a1e58)
1 /*
2  * PKCS #1 (RSA Encryption)
3  * Copyright (c) 2006-2009, Jouni Malinen <j@w1.fi>
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as
7  * published by the Free Software Foundation.
8  *
9  * Alternatively, this software may be distributed under the terms of BSD
10  * license.
11  *
12  * See README and COPYING for more details.
13  */
14 
15 #include "includes.h"
16 
17 #include "common.h"
18 #include "rsa.h"
19 #include "pkcs1.h"
20 
21 
22 static int pkcs1_generate_encryption_block(u8 block_type, size_t modlen,
23 					   const u8 *in, size_t inlen,
24 					   u8 *out, size_t *outlen)
25 {
26 	size_t ps_len;
27 	u8 *pos;
28 
29 	/*
30 	 * PKCS #1 v1.5, 8.1:
31 	 *
32 	 * EB = 00 || BT || PS || 00 || D
33 	 * BT = 00 or 01 for private-key operation; 02 for public-key operation
34 	 * PS = k-3-||D||; at least eight octets
35 	 * (BT=0: PS=0x00, BT=1: PS=0xff, BT=2: PS=pseudorandom non-zero)
36 	 * k = length of modulus in octets (modlen)
37 	 */
38 
39 	if (modlen < 12 || modlen > *outlen || inlen > modlen - 11) {
40 		wpa_printf(MSG_DEBUG, "PKCS #1: %s - Invalid buffer "
41 			   "lengths (modlen=%lu outlen=%lu inlen=%lu)",
42 			   __func__, (unsigned long) modlen,
43 			   (unsigned long) *outlen,
44 			   (unsigned long) inlen);
45 		return -1;
46 	}
47 
48 	pos = out;
49 	*pos++ = 0x00;
50 	*pos++ = block_type; /* BT */
51 	ps_len = modlen - inlen - 3;
52 	switch (block_type) {
53 	case 0:
54 		os_memset(pos, 0x00, ps_len);
55 		pos += ps_len;
56 		break;
57 	case 1:
58 		os_memset(pos, 0xff, ps_len);
59 		pos += ps_len;
60 		break;
61 	case 2:
62 		if (os_get_random(pos, ps_len) < 0) {
63 			wpa_printf(MSG_DEBUG, "PKCS #1: %s - Failed to get "
64 				   "random data for PS", __func__);
65 			return -1;
66 		}
67 		while (ps_len--) {
68 			if (*pos == 0x00)
69 				*pos = 0x01;
70 			pos++;
71 		}
72 		break;
73 	default:
74 		wpa_printf(MSG_DEBUG, "PKCS #1: %s - Unsupported block type "
75 			   "%d", __func__, block_type);
76 		return -1;
77 	}
78 	*pos++ = 0x00;
79 	os_memcpy(pos, in, inlen); /* D */
80 
81 	return 0;
82 }
83 
84 
85 int pkcs1_encrypt(int block_type, struct crypto_rsa_key *key,
86 		  int use_private, const u8 *in, size_t inlen,
87 		  u8 *out, size_t *outlen)
88 {
89 	size_t modlen;
90 
91 	modlen = crypto_rsa_get_modulus_len(key);
92 
93 	if (pkcs1_generate_encryption_block(block_type, modlen, in, inlen,
94 					    out, outlen) < 0)
95 		return -1;
96 
97 	return crypto_rsa_exptmod(out, modlen, out, outlen, key, use_private);
98 }
99 
100 
101 int pkcs1_v15_private_key_decrypt(struct crypto_rsa_key *key,
102 				  const u8 *in, size_t inlen,
103 				  u8 *out, size_t *outlen)
104 {
105 	int res;
106 	u8 *pos, *end;
107 
108 	res = crypto_rsa_exptmod(in, inlen, out, outlen, key, 1);
109 	if (res)
110 		return res;
111 
112 	if (*outlen < 2 || out[0] != 0 || out[1] != 2)
113 		return -1;
114 
115 	/* Skip PS (pseudorandom non-zero octets) */
116 	pos = out + 2;
117 	end = out + *outlen;
118 	while (*pos && pos < end)
119 		pos++;
120 	if (pos == end)
121 		return -1;
122 	pos++;
123 
124 	*outlen -= pos - out;
125 
126 	/* Strip PKCS #1 header */
127 	os_memmove(out, pos, *outlen);
128 
129 	return 0;
130 }
131 
132 
133 int pkcs1_decrypt_public_key(struct crypto_rsa_key *key,
134 			     const u8 *crypt, size_t crypt_len,
135 			     u8 *plain, size_t *plain_len)
136 {
137 	size_t len;
138 	u8 *pos;
139 
140 	len = *plain_len;
141 	if (crypto_rsa_exptmod(crypt, crypt_len, plain, &len, key, 0) < 0)
142 		return -1;
143 
144 	/*
145 	 * PKCS #1 v1.5, 8.1:
146 	 *
147 	 * EB = 00 || BT || PS || 00 || D
148 	 * BT = 00 or 01
149 	 * PS = k-3-||D|| times (00 if BT=00) or (FF if BT=01)
150 	 * k = length of modulus in octets
151 	 */
152 
153 	if (len < 3 + 8 + 16 /* min hash len */ ||
154 	    plain[0] != 0x00 || (plain[1] != 0x00 && plain[1] != 0x01)) {
155 		wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature EB "
156 			   "structure");
157 		return -1;
158 	}
159 
160 	pos = plain + 3;
161 	if (plain[1] == 0x00) {
162 		/* BT = 00 */
163 		if (plain[2] != 0x00) {
164 			wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature "
165 				   "PS (BT=00)");
166 			return -1;
167 		}
168 		while (pos + 1 < plain + len && *pos == 0x00 && pos[1] == 0x00)
169 			pos++;
170 	} else {
171 		/* BT = 01 */
172 		if (plain[2] != 0xff) {
173 			wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature "
174 				   "PS (BT=01)");
175 			return -1;
176 		}
177 		while (pos < plain + len && *pos == 0xff)
178 			pos++;
179 	}
180 
181 	if (pos - plain - 2 < 8) {
182 		/* PKCS #1 v1.5, 8.1: At least eight octets long PS */
183 		wpa_printf(MSG_INFO, "LibTomCrypt: Too short signature "
184 			   "padding");
185 		return -1;
186 	}
187 
188 	if (pos + 16 /* min hash len */ >= plain + len || *pos != 0x00) {
189 		wpa_printf(MSG_INFO, "LibTomCrypt: Invalid signature EB "
190 			   "structure (2)");
191 		return -1;
192 	}
193 	pos++;
194 	len -= pos - plain;
195 
196 	/* Strip PKCS #1 header */
197 	os_memmove(plain, pos, len);
198 	*plain_len = len;
199 
200 	return 0;
201 }
202