xref: /freebsd/contrib/telnet/libtelnet/enc_des.c (revision a0ee8cc6)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 #if 0
35 #ifndef lint
36 static const char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 5/30/95";
37 #endif /* not lint */
38 #endif
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
41 
42 #ifdef	ENCRYPTION
43 # ifdef	AUTHENTICATION
44 #include <arpa/telnet.h>
45 #include <openssl/des.h>
46 #include <stdio.h>
47 #include <stdlib.h>
48 #include <string.h>
49 
50 #include "encrypt.h"
51 #include "key-proto.h"
52 #include "misc-proto.h"
53 
54 extern int encrypt_debug_mode;
55 
56 #define	CFB	0
57 #define	OFB	1
58 
59 #define	NO_SEND_IV	1
60 #define	NO_RECV_IV	2
61 #define	NO_KEYID	4
62 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
63 #define	SUCCESS		0
64 #define	FAILED		-1
65 
66 
67 struct fb {
68 	Block krbdes_key;
69 	Schedule krbdes_sched;
70 	Block temp_feed;
71 	unsigned char fb_feed[64];
72 	int need_start;
73 	int state[2];
74 	int keyid[2];
75 	struct stinfo {
76 		Block		str_output;
77 		Block		str_feed;
78 		Block		str_iv;
79 		Block		str_ikey;
80 		Schedule	str_sched;
81 		int		str_index;
82 		int		str_flagshift;
83 	} streams[2];
84 };
85 
86 static struct fb fb[2];
87 
88 struct keyidlist {
89 	const char *keyid;
90 	int	keyidlen;
91 	char	*key;
92 	int	keylen;
93 	int	flags;
94 } keyidlist [] = {
95 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
96 	{ 0, 0, 0, 0, 0 }
97 };
98 
99 #define	KEYFLAG_MASK	03
100 
101 #define	KEYFLAG_NOINIT	00
102 #define	KEYFLAG_INIT	01
103 #define	KEYFLAG_OK	02
104 #define	KEYFLAG_BAD	03
105 
106 #define	KEYFLAG_SHIFT	2
107 
108 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
109 
110 #define	FB64_IV		1
111 #define	FB64_IV_OK	2
112 #define	FB64_IV_BAD	3
113 
114 
115 void fb64_stream_iv(Block, struct stinfo *);
116 void fb64_init(struct fb *);
117 static int fb64_start(struct fb *, int, int);
118 int fb64_is(unsigned char *, int, struct fb *);
119 int fb64_reply(unsigned char *, int, struct fb *);
120 static void fb64_session(Session_Key *, int, struct fb *);
121 void fb64_stream_key(Block, struct stinfo *);
122 int fb64_keyid(int, unsigned char *, int *, struct fb *);
123 
124 void
125 cfb64_init(int server __unused)
126 {
127 	fb64_init(&fb[CFB]);
128 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
129 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
130 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
131 }
132 
133 void
134 ofb64_init(int server __unused)
135 {
136 	fb64_init(&fb[OFB]);
137 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
138 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
139 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
140 }
141 
142 void
143 fb64_init(struct fb *fbp)
144 {
145 	memset((void *)fbp, 0, sizeof(*fbp));
146 	fbp->state[0] = fbp->state[1] = FAILED;
147 	fbp->fb_feed[0] = IAC;
148 	fbp->fb_feed[1] = SB;
149 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
150 	fbp->fb_feed[3] = ENCRYPT_IS;
151 }
152 
153 /*
154  * Returns:
155  *	-1: some error.  Negotiation is done, encryption not ready.
156  *	 0: Successful, initial negotiation all done.
157  *	 1: successful, negotiation not done yet.
158  *	 2: Not yet.  Other things (like getting the key from
159  *	    Kerberos) have to happen before we can continue.
160  */
161 int
162 cfb64_start(int dir, int server)
163 {
164 	return(fb64_start(&fb[CFB], dir, server));
165 }
166 
167 int
168 ofb64_start(int dir, int server)
169 {
170 	return(fb64_start(&fb[OFB], dir, server));
171 }
172 
173 static int
174 fb64_start(struct fb *fbp, int dir, int server __unused)
175 {
176 	size_t x;
177 	unsigned char *p;
178 	int state;
179 
180 	switch (dir) {
181 	case DIR_DECRYPT:
182 		/*
183 		 * This is simply a request to have the other side
184 		 * start output (our input).  He will negotiate an
185 		 * IV so we need not look for it.
186 		 */
187 		state = fbp->state[dir-1];
188 		if (state == FAILED)
189 			state = IN_PROGRESS;
190 		break;
191 
192 	case DIR_ENCRYPT:
193 		state = fbp->state[dir-1];
194 		if (state == FAILED)
195 			state = IN_PROGRESS;
196 		else if ((state & NO_SEND_IV) == 0)
197 			break;
198 
199 		if (!VALIDKEY(fbp->krbdes_key)) {
200 			fbp->need_start = 1;
201 			break;
202 		}
203 		state &= ~NO_SEND_IV;
204 		state |= NO_RECV_IV;
205 		if (encrypt_debug_mode)
206 			printf("Creating new feed\r\n");
207 		/*
208 		 * Create a random feed and send it over.
209 		 */
210 		des_random_key((Block *)fbp->temp_feed);
211 		des_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
212 				fbp->krbdes_sched, 1);
213 		p = fbp->fb_feed + 3;
214 		*p++ = ENCRYPT_IS;
215 		p++;
216 		*p++ = FB64_IV;
217 		for (x = 0; x < sizeof(Block); ++x) {
218 			if ((*p++ = fbp->temp_feed[x]) == IAC)
219 				*p++ = IAC;
220 		}
221 		*p++ = IAC;
222 		*p++ = SE;
223 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
224 		net_write(fbp->fb_feed, p - fbp->fb_feed);
225 		break;
226 	default:
227 		return(FAILED);
228 	}
229 	return(fbp->state[dir-1] = state);
230 }
231 
232 /*
233  * Returns:
234  *	-1: some error.  Negotiation is done, encryption not ready.
235  *	 0: Successful, initial negotiation all done.
236  *	 1: successful, negotiation not done yet.
237  */
238 int
239 cfb64_is(unsigned char *data, int cnt)
240 {
241 	return(fb64_is(data, cnt, &fb[CFB]));
242 }
243 
244 int
245 ofb64_is(unsigned char *data, int cnt)
246 {
247 	return(fb64_is(data, cnt, &fb[OFB]));
248 }
249 
250 int
251 fb64_is(unsigned char *data, int cnt, struct fb *fbp)
252 {
253 	unsigned char *p;
254 	int state = fbp->state[DIR_DECRYPT-1];
255 
256 	if (cnt-- < 1)
257 		goto failure;
258 
259 	switch (*data++) {
260 	case FB64_IV:
261 		if (cnt != sizeof(Block)) {
262 			if (encrypt_debug_mode)
263 				printf("CFB64: initial vector failed on size\r\n");
264 			state = FAILED;
265 			goto failure;
266 		}
267 
268 		if (encrypt_debug_mode)
269 			printf("CFB64: initial vector received\r\n");
270 
271 		if (encrypt_debug_mode)
272 			printf("Initializing Decrypt stream\r\n");
273 
274 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
275 
276 		p = fbp->fb_feed + 3;
277 		*p++ = ENCRYPT_REPLY;
278 		p++;
279 		*p++ = FB64_IV_OK;
280 		*p++ = IAC;
281 		*p++ = SE;
282 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
283 		net_write(fbp->fb_feed, p - fbp->fb_feed);
284 
285 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
286 		break;
287 
288 	default:
289 		if (encrypt_debug_mode) {
290 			printf("Unknown option type: %d\r\n", *(data-1));
291 			printd(data, cnt);
292 			printf("\r\n");
293 		}
294 		/* FALL THROUGH */
295 	failure:
296 		/*
297 		 * We failed.  Send an FB64_IV_BAD option
298 		 * to the other side so it will know that
299 		 * things failed.
300 		 */
301 		p = fbp->fb_feed + 3;
302 		*p++ = ENCRYPT_REPLY;
303 		p++;
304 		*p++ = FB64_IV_BAD;
305 		*p++ = IAC;
306 		*p++ = SE;
307 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
308 		net_write(fbp->fb_feed, p - fbp->fb_feed);
309 
310 		break;
311 	}
312 	return(fbp->state[DIR_DECRYPT-1] = state);
313 }
314 
315 /*
316  * Returns:
317  *	-1: some error.  Negotiation is done, encryption not ready.
318  *	 0: Successful, initial negotiation all done.
319  *	 1: successful, negotiation not done yet.
320  */
321 int
322 cfb64_reply(unsigned char *data, int cnt)
323 {
324 	return(fb64_reply(data, cnt, &fb[CFB]));
325 }
326 
327 int
328 ofb64_reply(unsigned char *data, int cnt)
329 {
330 	return(fb64_reply(data, cnt, &fb[OFB]));
331 }
332 
333 int
334 fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
335 {
336 	int state = fbp->state[DIR_ENCRYPT-1];
337 
338 	if (cnt-- < 1)
339 		goto failure;
340 
341 	switch (*data++) {
342 	case FB64_IV_OK:
343 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
344 		if (state == FAILED)
345 			state = IN_PROGRESS;
346 		state &= ~NO_RECV_IV;
347 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
348 		break;
349 
350 	case FB64_IV_BAD:
351 		memset(fbp->temp_feed, 0, sizeof(Block));
352 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
353 		state = FAILED;
354 		break;
355 
356 	default:
357 		if (encrypt_debug_mode) {
358 			printf("Unknown option type: %d\r\n", data[-1]);
359 			printd(data, cnt);
360 			printf("\r\n");
361 		}
362 		/* FALL THROUGH */
363 	failure:
364 		state = FAILED;
365 		break;
366 	}
367 	return(fbp->state[DIR_ENCRYPT-1] = state);
368 }
369 
370 void
371 cfb64_session(Session_Key *key, int server)
372 {
373 	fb64_session(key, server, &fb[CFB]);
374 }
375 
376 void
377 ofb64_session(Session_Key *key, int server)
378 {
379 	fb64_session(key, server, &fb[OFB]);
380 }
381 
382 static void
383 fb64_session(Session_Key *key, int server, struct fb *fbp)
384 {
385 	if (!key || key->type != SK_DES) {
386 		if (encrypt_debug_mode)
387 			printf("Can't set krbdes's session key (%d != %d)\r\n",
388 				key ? key->type : -1, SK_DES);
389 		return;
390 	}
391 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
392 
393 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
394 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
395 
396 	des_key_sched((Block *)fbp->krbdes_key, fbp->krbdes_sched);
397 	/*
398 	 * Now look to see if krbdes_start() was was waiting for
399 	 * the key to show up.  If so, go ahead an call it now
400 	 * that we have the key.
401 	 */
402 	if (fbp->need_start) {
403 		fbp->need_start = 0;
404 		fb64_start(fbp, DIR_ENCRYPT, server);
405 	}
406 }
407 
408 /*
409  * We only accept a keyid of 0.  If we get a keyid of
410  * 0, then mark the state as SUCCESS.
411  */
412 int
413 cfb64_keyid(int dir, unsigned char *kp, int *lenp)
414 {
415 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
416 }
417 
418 int
419 ofb64_keyid(int dir, unsigned char *kp, int *lenp)
420 {
421 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
422 }
423 
424 int
425 fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
426 {
427 	int state = fbp->state[dir-1];
428 
429 	if (*lenp != 1 || (*kp != '\0')) {
430 		*lenp = 0;
431 		return(state);
432 	}
433 
434 	if (state == FAILED)
435 		state = IN_PROGRESS;
436 
437 	state &= ~NO_KEYID;
438 
439 	return(fbp->state[dir-1] = state);
440 }
441 
442 void
443 fb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen, const char *type)
444 {
445 	char lbuf[32];
446 	int i;
447 	char *cp;
448 
449 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
450 	buflen -= 1;
451 
452 	switch(data[2]) {
453 	case FB64_IV:
454 		sprintf(lbuf, "%s_IV", type);
455 		cp = lbuf;
456 		goto common;
457 
458 	case FB64_IV_OK:
459 		sprintf(lbuf, "%s_IV_OK", type);
460 		cp = lbuf;
461 		goto common;
462 
463 	case FB64_IV_BAD:
464 		sprintf(lbuf, "%s_IV_BAD", type);
465 		cp = lbuf;
466 		goto common;
467 
468 	default:
469 		sprintf(lbuf, " %d (unknown)", data[2]);
470 		cp = lbuf;
471 	common:
472 		for (; (buflen > 0) && (*buf = *cp++); buf++)
473 			buflen--;
474 		for (i = 3; i < cnt; i++) {
475 			sprintf(lbuf, " %d", data[i]);
476 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
477 				buflen--;
478 		}
479 		break;
480 	}
481 }
482 
483 void
484 cfb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
485 {
486 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
487 }
488 
489 void
490 ofb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
491 {
492 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
493 }
494 
495 void
496 fb64_stream_iv(Block seed, struct stinfo *stp)
497 {
498 
499 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
500 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
501 
502 	des_key_sched((Block *)stp->str_ikey, stp->str_sched);
503 
504 	stp->str_index = sizeof(Block);
505 }
506 
507 void
508 fb64_stream_key(Block key, struct stinfo *stp)
509 {
510 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
511 	des_key_sched((Block *)key, stp->str_sched);
512 
513 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
514 
515 	stp->str_index = sizeof(Block);
516 }
517 
518 /*
519  * DES 64 bit Cipher Feedback
520  *
521  *     key --->+-----+
522  *          +->| DES |--+
523  *          |  +-----+  |
524  *	    |           v
525  *  INPUT --(--------->(+)+---> DATA
526  *          |             |
527  *	    +-------------+
528  *
529  *
530  * Given:
531  *	iV: Initial vector, 64 bits (8 bytes) long.
532  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
533  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
534  *
535  *	V0 = DES(iV, key)
536  *	On = Dn ^ Vn
537  *	V(n+1) = DES(On, key)
538  */
539 
540 void
541 cfb64_encrypt(unsigned char *s, int c)
542 {
543 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
544 	int idx;
545 
546 	idx = stp->str_index;
547 	while (c-- > 0) {
548 		if (idx == sizeof(Block)) {
549 			Block b;
550 			des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
551 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
552 			idx = 0;
553 		}
554 
555 		/* On encryption, we store (feed ^ data) which is cypher */
556 		*s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
557 		s++;
558 		idx++;
559 	}
560 	stp->str_index = idx;
561 }
562 
563 int
564 cfb64_decrypt(int data)
565 {
566 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
567 	int idx;
568 
569 	if (data == -1) {
570 		/*
571 		 * Back up one byte.  It is assumed that we will
572 		 * never back up more than one byte.  If we do, this
573 		 * may or may not work.
574 		 */
575 		if (stp->str_index)
576 			--stp->str_index;
577 		return(0);
578 	}
579 
580 	idx = stp->str_index++;
581 	if (idx == sizeof(Block)) {
582 		Block b;
583 		des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
584 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
585 		stp->str_index = 1;	/* Next time will be 1 */
586 		idx = 0;		/* But now use 0 */
587 	}
588 
589 	/* On decryption we store (data) which is cypher. */
590 	stp->str_output[idx] = data;
591 	return(data ^ stp->str_feed[idx]);
592 }
593 
594 /*
595  * DES 64 bit Output Feedback
596  *
597  * key --->+-----+
598  *	+->| DES |--+
599  *	|  +-----+  |
600  *	+-----------+
601  *	            v
602  *  INPUT -------->(+) ----> DATA
603  *
604  * Given:
605  *	iV: Initial vector, 64 bits (8 bytes) long.
606  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
607  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
608  *
609  *	V0 = DES(iV, key)
610  *	V(n+1) = DES(Vn, key)
611  *	On = Dn ^ Vn
612  */
613 void
614 ofb64_encrypt(unsigned char *s, int c)
615 {
616 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
617 	int idx;
618 
619 	idx = stp->str_index;
620 	while (c-- > 0) {
621 		if (idx == sizeof(Block)) {
622 			Block b;
623 			des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
624 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
625 			idx = 0;
626 		}
627 		*s++ ^= stp->str_feed[idx];
628 		idx++;
629 	}
630 	stp->str_index = idx;
631 }
632 
633 int
634 ofb64_decrypt(int data)
635 {
636 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
637 	int idx;
638 
639 	if (data == -1) {
640 		/*
641 		 * Back up one byte.  It is assumed that we will
642 		 * never back up more than one byte.  If we do, this
643 		 * may or may not work.
644 		 */
645 		if (stp->str_index)
646 			--stp->str_index;
647 		return(0);
648 	}
649 
650 	idx = stp->str_index++;
651 	if (idx == sizeof(Block)) {
652 		Block b;
653 		des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
654 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
655 		stp->str_index = 1;	/* Next time will be 1 */
656 		idx = 0;		/* But now use 0 */
657 	}
658 
659 	return(data ^ stp->str_feed[idx]);
660 }
661 # endif	/* AUTHENTICATION */
662 #endif	/* ENCRYPTION */
663