xref: /openbsd/lib/libcrypto/evp/e_rc2.c (revision 9ea232b5)
1 /* $OpenBSD: e_rc2.c,v 1.27 2024/01/07 15:42:57 tb Exp $ */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 
59 #include <limits.h>
60 #include <stdio.h>
61 
62 #include <openssl/opensslconf.h>
63 
64 #ifndef OPENSSL_NO_RC2
65 
66 #include <openssl/err.h>
67 #include <openssl/evp.h>
68 #include <openssl/objects.h>
69 #include <openssl/rc2.h>
70 
71 #include "evp_local.h"
72 
73 static int rc2_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
74     const unsigned char *iv, int enc);
75 static int rc2_meth_to_magic(EVP_CIPHER_CTX *ctx);
76 static int rc2_magic_to_meth(int i);
77 static int rc2_set_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type);
78 static int rc2_get_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type);
79 static int rc2_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr);
80 
81 typedef struct {
82 	int key_bits;	/* effective key bits */
83 	RC2_KEY ks;	/* key schedule */
84 } EVP_RC2_KEY;
85 
86 #define data(ctx)	((EVP_RC2_KEY *)(ctx)->cipher_data)
87 
88 static int
89 rc2_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
90 {
91 	size_t chunk = LONG_MAX & ~0xff;
92 
93 	while (inl >= chunk) {
94 		RC2_cbc_encrypt(in, out, (long)chunk, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
95 		inl -= chunk;
96 		in += chunk;
97 		out += chunk;
98 	}
99 
100 	if (inl)
101 		RC2_cbc_encrypt(in, out, (long)inl, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
102 
103 	return 1;
104 }
105 
106 static int
107 rc2_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
108 {
109 	size_t chunk = LONG_MAX & ~0xff;
110 
111 	if (inl < chunk)
112 		chunk = inl;
113 
114 	while (inl && inl >= chunk) {
115 		RC2_cfb64_encrypt(in, out, (long)chunk, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num, ctx->encrypt);
116 		inl -= chunk;
117 		in += chunk;
118 		out += chunk;
119 		if (inl < chunk)
120 			chunk = inl;
121 	}
122 
123 	return 1;
124 }
125 
126 static int
127 rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
128 {
129 	size_t i, bl;
130 
131 	bl = ctx->cipher->block_size;
132 
133 	if (inl < bl)
134 		return 1;
135 
136 	inl -= bl;
137 
138 	for (i = 0; i <= inl; i += bl)
139 		RC2_ecb_encrypt(in + i, out + i, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->encrypt);
140 
141 	return 1;
142 }
143 
144 static int
145 rc2_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
146 {
147 	size_t chunk = LONG_MAX & ~0xff;
148 
149 	while (inl >= chunk) {
150 		RC2_ofb64_encrypt(in, out, (long)chunk, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
151 		inl -= chunk;
152 		in += chunk;
153 		out += chunk;
154 	}
155 
156 	if (inl)
157 		RC2_ofb64_encrypt(in, out, (long)inl, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
158 
159 	return 1;
160 }
161 
162 static const EVP_CIPHER rc2_cbc = {
163 	.nid = NID_rc2_cbc,
164 	.block_size = 8,
165 	.key_len = RC2_KEY_LENGTH,
166 	.iv_len = 8,
167 	.flags = EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT | EVP_CIPH_CBC_MODE,
168 	.init = rc2_init_key,
169 	.do_cipher = rc2_cbc_cipher,
170 	.cleanup = NULL,
171 	.ctx_size = sizeof(EVP_RC2_KEY),
172 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
173 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
174 	.ctrl = rc2_ctrl,
175 };
176 
177 const EVP_CIPHER *
178 EVP_rc2_cbc(void)
179 {
180 	return &rc2_cbc;
181 }
182 
183 static const EVP_CIPHER rc2_cfb64 = {
184 	.nid = NID_rc2_cfb64,
185 	.block_size = 1,
186 	.key_len = RC2_KEY_LENGTH,
187 	.iv_len = 8,
188 	.flags = EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT | EVP_CIPH_CFB_MODE,
189 	.init = rc2_init_key,
190 	.do_cipher = rc2_cfb64_cipher,
191 	.cleanup = NULL,
192 	.ctx_size = sizeof(EVP_RC2_KEY),
193 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
194 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
195 	.ctrl = rc2_ctrl,
196 };
197 
198 const EVP_CIPHER *
199 EVP_rc2_cfb64(void)
200 {
201 	return &rc2_cfb64;
202 }
203 
204 static const EVP_CIPHER rc2_ofb = {
205 	.nid = NID_rc2_ofb64,
206 	.block_size = 1,
207 	.key_len = RC2_KEY_LENGTH,
208 	.iv_len = 8,
209 	.flags = EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT | EVP_CIPH_OFB_MODE,
210 	.init = rc2_init_key,
211 	.do_cipher = rc2_ofb_cipher,
212 	.cleanup = NULL,
213 	.ctx_size = sizeof(EVP_RC2_KEY),
214 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
215 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
216 	.ctrl = rc2_ctrl,
217 };
218 
219 const EVP_CIPHER *
220 EVP_rc2_ofb(void)
221 {
222 	return &rc2_ofb;
223 }
224 
225 static const EVP_CIPHER rc2_ecb = {
226 	.nid = NID_rc2_ecb,
227 	.block_size = 8,
228 	.key_len = RC2_KEY_LENGTH,
229 	.iv_len = 0,
230 	.flags = EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT | EVP_CIPH_ECB_MODE,
231 	.init = rc2_init_key,
232 	.do_cipher = rc2_ecb_cipher,
233 	.cleanup = NULL,
234 	.ctx_size = sizeof(EVP_RC2_KEY),
235 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
236 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
237 	.ctrl = rc2_ctrl,
238 };
239 
240 const EVP_CIPHER *
241 EVP_rc2_ecb(void)
242 {
243 	return &rc2_ecb;
244 }
245 
246 #define RC2_40_MAGIC	0xa0
247 #define RC2_64_MAGIC	0x78
248 #define RC2_128_MAGIC	0x3a
249 
250 static const EVP_CIPHER r2_64_cbc_cipher = {
251 	.nid = NID_rc2_64_cbc,
252 	.block_size = 8,
253 	.key_len = 8,
254 	.iv_len = 8,
255 	.flags = EVP_CIPH_CBC_MODE | EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT,
256 	.init = rc2_init_key,
257 	.do_cipher = rc2_cbc_cipher,
258 	.cleanup = NULL,
259 	.ctx_size = sizeof(EVP_RC2_KEY),
260 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
261 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
262 	.ctrl = rc2_ctrl,
263 };
264 
265 static const EVP_CIPHER r2_40_cbc_cipher = {
266 	.nid = NID_rc2_40_cbc,
267 	.block_size = 8,
268 	.key_len = 5,
269 	.iv_len = 8,
270 	.flags = EVP_CIPH_CBC_MODE | EVP_CIPH_VARIABLE_LENGTH | EVP_CIPH_CTRL_INIT,
271 	.init = rc2_init_key,
272 	.do_cipher = rc2_cbc_cipher,
273 	.cleanup = NULL,
274 	.ctx_size = sizeof(EVP_RC2_KEY),
275 	.set_asn1_parameters = rc2_set_asn1_type_and_iv,
276 	.get_asn1_parameters = rc2_get_asn1_type_and_iv,
277 	.ctrl = rc2_ctrl,
278 };
279 
280 const EVP_CIPHER *
281 EVP_rc2_64_cbc(void)
282 {
283 	return (&r2_64_cbc_cipher);
284 }
285 
286 const EVP_CIPHER *
287 EVP_rc2_40_cbc(void)
288 {
289 	return (&r2_40_cbc_cipher);
290 }
291 
292 static int
293 rc2_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
294     const unsigned char *iv, int enc)
295 {
296 	RC2_set_key(&data(ctx)->ks, EVP_CIPHER_CTX_key_length(ctx),
297 	    key, data(ctx)->key_bits);
298 	return 1;
299 }
300 
301 static int
302 rc2_meth_to_magic(EVP_CIPHER_CTX *e)
303 {
304 	int i;
305 
306 	if (EVP_CIPHER_CTX_ctrl(e, EVP_CTRL_GET_RC2_KEY_BITS, 0, &i) <= 0)
307 		return (0);
308 	if (i == 128)
309 		return (RC2_128_MAGIC);
310 	else if (i == 64)
311 		return (RC2_64_MAGIC);
312 	else if (i == 40)
313 		return (RC2_40_MAGIC);
314 	else
315 		return (0);
316 }
317 
318 static int
319 rc2_magic_to_meth(int i)
320 {
321 	if (i == RC2_128_MAGIC)
322 		return 128;
323 	else if (i == RC2_64_MAGIC)
324 		return 64;
325 	else if (i == RC2_40_MAGIC)
326 		return 40;
327 	else {
328 		EVPerror(EVP_R_UNSUPPORTED_KEY_SIZE);
329 		return (0);
330 	}
331 }
332 
333 static int
334 rc2_get_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
335 {
336 	long num = 0;
337 	int i = 0;
338 	int key_bits;
339 	int l;
340 	unsigned char iv[EVP_MAX_IV_LENGTH];
341 
342 	if (type != NULL) {
343 		l = EVP_CIPHER_CTX_iv_length(c);
344 		if (l < 0 || l > sizeof(iv)) {
345 			EVPerror(EVP_R_IV_TOO_LARGE);
346 			return -1;
347 		}
348 		i = ASN1_TYPE_get_int_octetstring(type, &num, iv, l);
349 		if (i != l)
350 			return (-1);
351 		key_bits = rc2_magic_to_meth((int)num);
352 		if (!key_bits)
353 			return (-1);
354 		if (i > 0 && !EVP_CipherInit_ex(c, NULL, NULL, NULL, iv, -1))
355 			return -1;
356 		if (EVP_CIPHER_CTX_ctrl(c, EVP_CTRL_SET_RC2_KEY_BITS,
357 		    key_bits, NULL) <= 0)
358 			return -1;
359 		if (!EVP_CIPHER_CTX_set_key_length(c, key_bits / 8))
360 			return -1;
361 	}
362 	return (i);
363 }
364 
365 static int
366 rc2_set_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
367 {
368 	long num;
369 	int i = 0, j;
370 
371 	if (type != NULL) {
372 		num = rc2_meth_to_magic(c);
373 		j = EVP_CIPHER_CTX_iv_length(c);
374 		if (j < 0 || j > sizeof(c->oiv))
375 			return 0;
376 		i = ASN1_TYPE_set_int_octetstring(type, num, c->oiv, j);
377 	}
378 	return (i);
379 }
380 
381 static int
382 rc2_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
383 {
384 	switch (type) {
385 	case EVP_CTRL_INIT:
386 		data(c)->key_bits = EVP_CIPHER_CTX_key_length(c) * 8;
387 		return 1;
388 
389 	case EVP_CTRL_GET_RC2_KEY_BITS:
390 		*(int *)ptr = data(c)->key_bits;
391 		return 1;
392 
393 	case EVP_CTRL_SET_RC2_KEY_BITS:
394 		if (arg > 0) {
395 			data(c)->key_bits = arg;
396 			return 1;
397 		}
398 		return 0;
399 
400 #ifdef PBE_PRF_TEST
401 	case EVP_CTRL_PBE_PRF_NID:
402 		*(int *)ptr = NID_hmacWithMD5;
403 		return 1;
404 #endif
405 
406 	default:
407 		return -1;
408 	}
409 }
410 
411 #endif
412