xref: /dragonfly/lib/libtelnet/enc_des.c (revision dcd37f7d)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. All advertising materials mentioning features or use of this software
14  *    must display the following acknowledgement:
15  *	This product includes software developed by the University of
16  *	California, Berkeley and its contributors.
17  * 4. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  *
33  * $FreeBSD: src/crypto/telnet/libtelnet/enc_des.c,v 1.3.2.2 2003/02/14 22:38:13 nectar Exp $
34  * $DragonFly: src/crypto/telnet/libtelnet/enc_des.c,v 1.2 2003/06/17 04:24:37 dillon Exp $
35  *
36  * @(#)enc_des.c	8.3 (Berkeley) 5/30/95
37  * $FreeBSD: src/crypto/telnet/libtelnet/enc_des.c,v 1.3.2.2 2003/02/14 22:38:13 nectar Exp $
38  */
39 
40 #ifdef	ENCRYPTION
41 # ifdef	AUTHENTICATION
42 #include <arpa/telnet.h>
43 #include <openssl/des.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 
48 #include "encrypt.h"
49 #include "key-proto.h"
50 #include "misc-proto.h"
51 
52 extern int encrypt_debug_mode;
53 
54 #define	CFB	0
55 #define	OFB	1
56 
57 #define	NO_SEND_IV	1
58 #define	NO_RECV_IV	2
59 #define	NO_KEYID	4
60 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
61 #define	SUCCESS		0
62 #define	FAILED		-1
63 
64 
65 struct fb {
66 	Block krbdes_key;
67 	Schedule krbdes_sched;
68 	Block temp_feed;
69 	unsigned char fb_feed[64];
70 	int need_start;
71 	int state[2];
72 	int keyid[2];
73 	struct stinfo {
74 		Block		str_output;
75 		Block		str_feed;
76 		Block		str_iv;
77 		Block		str_ikey;
78 		Schedule	str_sched;
79 		int		str_index;
80 		int		str_flagshift;
81 	} streams[2];
82 };
83 
84 static struct fb fb[2];
85 
86 struct keyidlist {
87 	const char *keyid;
88 	int	keyidlen;
89 	char	*key;
90 	int	keylen;
91 	int	flags;
92 } keyidlist [] = {
93 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
94 	{ 0, 0, 0, 0, 0 }
95 };
96 
97 #define	KEYFLAG_MASK	03
98 
99 #define	KEYFLAG_NOINIT	00
100 #define	KEYFLAG_INIT	01
101 #define	KEYFLAG_OK	02
102 #define	KEYFLAG_BAD	03
103 
104 #define	KEYFLAG_SHIFT	2
105 
106 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
107 
108 #define	FB64_IV		1
109 #define	FB64_IV_OK	2
110 #define	FB64_IV_BAD	3
111 
112 
113 void fb64_stream_iv(Block, struct stinfo *);
114 void fb64_init(struct fb *);
115 static int fb64_start(struct fb *, int, int);
116 int fb64_is(unsigned char *, int, struct fb *);
117 int fb64_reply(unsigned char *, int, struct fb *);
118 static void fb64_session(Session_Key *, int, struct fb *);
119 void fb64_stream_key(Block, struct stinfo *);
120 int fb64_keyid(int, unsigned char *, int *, struct fb *);
121 
122 void
123 cfb64_init(int server __unused)
124 {
125 	fb64_init(&fb[CFB]);
126 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
127 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
128 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
129 }
130 
131 void
132 ofb64_init(int server __unused)
133 {
134 	fb64_init(&fb[OFB]);
135 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
136 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
137 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
138 }
139 
140 void
141 fb64_init(struct fb *fbp)
142 {
143 	memset((void *)fbp, 0, sizeof(*fbp));
144 	fbp->state[0] = fbp->state[1] = FAILED;
145 	fbp->fb_feed[0] = IAC;
146 	fbp->fb_feed[1] = SB;
147 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
148 	fbp->fb_feed[3] = ENCRYPT_IS;
149 }
150 
151 /*
152  * Returns:
153  *	-1: some error.  Negotiation is done, encryption not ready.
154  *	 0: Successful, initial negotiation all done.
155  *	 1: successful, negotiation not done yet.
156  *	 2: Not yet.  Other things (like getting the key from
157  *	    Kerberos) have to happen before we can continue.
158  */
159 int
160 cfb64_start(int dir, int server)
161 {
162 	return(fb64_start(&fb[CFB], dir, server));
163 }
164 
165 int
166 ofb64_start(int dir, int server)
167 {
168 	return(fb64_start(&fb[OFB], dir, server));
169 }
170 
171 static int
172 fb64_start(struct fb *fbp, int dir, int server __unused)
173 {
174 	size_t x;
175 	unsigned char *p;
176 	int state;
177 
178 	switch (dir) {
179 	case DIR_DECRYPT:
180 		/*
181 		 * This is simply a request to have the other side
182 		 * start output (our input).  He will negotiate an
183 		 * IV so we need not look for it.
184 		 */
185 		state = fbp->state[dir-1];
186 		if (state == FAILED)
187 			state = IN_PROGRESS;
188 		break;
189 
190 	case DIR_ENCRYPT:
191 		state = fbp->state[dir-1];
192 		if (state == FAILED)
193 			state = IN_PROGRESS;
194 		else if ((state & NO_SEND_IV) == 0)
195 			break;
196 
197 		if (!VALIDKEY(fbp->krbdes_key)) {
198 			fbp->need_start = 1;
199 			break;
200 		}
201 		state &= ~NO_SEND_IV;
202 		state |= NO_RECV_IV;
203 		if (encrypt_debug_mode)
204 			printf("Creating new feed\r\n");
205 		/*
206 		 * Create a random feed and send it over.
207 		 */
208 		des_random_key((Block *)fbp->temp_feed);
209 		des_ecb_encrypt((Block *)fbp->temp_feed, (Block *)fbp->temp_feed,
210 				fbp->krbdes_sched, 1);
211 		p = fbp->fb_feed + 3;
212 		*p++ = ENCRYPT_IS;
213 		p++;
214 		*p++ = FB64_IV;
215 		for (x = 0; x < sizeof(Block); ++x) {
216 			if ((*p++ = fbp->temp_feed[x]) == IAC)
217 				*p++ = IAC;
218 		}
219 		*p++ = IAC;
220 		*p++ = SE;
221 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
222 		net_write(fbp->fb_feed, p - fbp->fb_feed);
223 		break;
224 	default:
225 		return(FAILED);
226 	}
227 	return(fbp->state[dir-1] = state);
228 }
229 
230 /*
231  * Returns:
232  *	-1: some error.  Negotiation is done, encryption not ready.
233  *	 0: Successful, initial negotiation all done.
234  *	 1: successful, negotiation not done yet.
235  */
236 int
237 cfb64_is(unsigned char *data, int cnt)
238 {
239 	return(fb64_is(data, cnt, &fb[CFB]));
240 }
241 
242 int
243 ofb64_is(unsigned char *data, int cnt)
244 {
245 	return(fb64_is(data, cnt, &fb[OFB]));
246 }
247 
248 int
249 fb64_is(unsigned char *data, int cnt, struct fb *fbp)
250 {
251 	unsigned char *p;
252 	int state = fbp->state[DIR_DECRYPT-1];
253 
254 	if (cnt-- < 1)
255 		goto failure;
256 
257 	switch (*data++) {
258 	case FB64_IV:
259 		if (cnt != sizeof(Block)) {
260 			if (encrypt_debug_mode)
261 				printf("CFB64: initial vector failed on size\r\n");
262 			state = FAILED;
263 			goto failure;
264 		}
265 
266 		if (encrypt_debug_mode)
267 			printf("CFB64: initial vector received\r\n");
268 
269 		if (encrypt_debug_mode)
270 			printf("Initializing Decrypt stream\r\n");
271 
272 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
273 
274 		p = fbp->fb_feed + 3;
275 		*p++ = ENCRYPT_REPLY;
276 		p++;
277 		*p++ = FB64_IV_OK;
278 		*p++ = IAC;
279 		*p++ = SE;
280 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
281 		net_write(fbp->fb_feed, p - fbp->fb_feed);
282 
283 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
284 		break;
285 
286 	default:
287 		if (encrypt_debug_mode) {
288 			printf("Unknown option type: %d\r\n", *(data-1));
289 			printd(data, cnt);
290 			printf("\r\n");
291 		}
292 		/* FALL THROUGH */
293 	failure:
294 		/*
295 		 * We failed.  Send an FB64_IV_BAD option
296 		 * to the other side so it will know that
297 		 * things failed.
298 		 */
299 		p = fbp->fb_feed + 3;
300 		*p++ = ENCRYPT_REPLY;
301 		p++;
302 		*p++ = FB64_IV_BAD;
303 		*p++ = IAC;
304 		*p++ = SE;
305 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
306 		net_write(fbp->fb_feed, p - fbp->fb_feed);
307 
308 		break;
309 	}
310 	return(fbp->state[DIR_DECRYPT-1] = state);
311 }
312 
313 /*
314  * Returns:
315  *	-1: some error.  Negotiation is done, encryption not ready.
316  *	 0: Successful, initial negotiation all done.
317  *	 1: successful, negotiation not done yet.
318  */
319 int
320 cfb64_reply(unsigned char *data, int cnt)
321 {
322 	return(fb64_reply(data, cnt, &fb[CFB]));
323 }
324 
325 int
326 ofb64_reply(unsigned char *data, int cnt)
327 {
328 	return(fb64_reply(data, cnt, &fb[OFB]));
329 }
330 
331 int
332 fb64_reply(unsigned char *data, int cnt, struct fb *fbp)
333 {
334 	int state = fbp->state[DIR_ENCRYPT-1];
335 
336 	if (cnt-- < 1)
337 		goto failure;
338 
339 	switch (*data++) {
340 	case FB64_IV_OK:
341 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
342 		if (state == FAILED)
343 			state = IN_PROGRESS;
344 		state &= ~NO_RECV_IV;
345 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
346 		break;
347 
348 	case FB64_IV_BAD:
349 		memset(fbp->temp_feed, 0, sizeof(Block));
350 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
351 		state = FAILED;
352 		break;
353 
354 	default:
355 		if (encrypt_debug_mode) {
356 			printf("Unknown option type: %d\r\n", data[-1]);
357 			printd(data, cnt);
358 			printf("\r\n");
359 		}
360 		/* FALL THROUGH */
361 	failure:
362 		state = FAILED;
363 		break;
364 	}
365 	return(fbp->state[DIR_ENCRYPT-1] = state);
366 }
367 
368 void
369 cfb64_session(Session_Key *key, int server)
370 {
371 	fb64_session(key, server, &fb[CFB]);
372 }
373 
374 void
375 ofb64_session(Session_Key *key, int server)
376 {
377 	fb64_session(key, server, &fb[OFB]);
378 }
379 
380 static void
381 fb64_session(Session_Key *key, int server, struct fb *fbp)
382 {
383 	if (!key || key->type != SK_DES) {
384 		if (encrypt_debug_mode)
385 			printf("Can't set krbdes's session key (%d != %d)\r\n",
386 				key ? key->type : -1, SK_DES);
387 		return;
388 	}
389 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
390 
391 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
392 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
393 
394 	des_key_sched((Block *)fbp->krbdes_key, fbp->krbdes_sched);
395 	/*
396 	 * Now look to see if krbdes_start() was was waiting for
397 	 * the key to show up.  If so, go ahead an call it now
398 	 * that we have the key.
399 	 */
400 	if (fbp->need_start) {
401 		fbp->need_start = 0;
402 		fb64_start(fbp, DIR_ENCRYPT, server);
403 	}
404 }
405 
406 /*
407  * We only accept a keyid of 0.  If we get a keyid of
408  * 0, then mark the state as SUCCESS.
409  */
410 int
411 cfb64_keyid(int dir, unsigned char *kp, int *lenp)
412 {
413 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
414 }
415 
416 int
417 ofb64_keyid(int dir, unsigned char *kp, int *lenp)
418 {
419 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
420 }
421 
422 int
423 fb64_keyid(int dir, unsigned char *kp, int *lenp, struct fb *fbp)
424 {
425 	int state = fbp->state[dir-1];
426 
427 	if (*lenp != 1 || (*kp != '\0')) {
428 		*lenp = 0;
429 		return(state);
430 	}
431 
432 	if (state == FAILED)
433 		state = IN_PROGRESS;
434 
435 	state &= ~NO_KEYID;
436 
437 	return(fbp->state[dir-1] = state);
438 }
439 
440 void
441 fb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen, const char *type)
442 {
443 	char lbuf[32];
444 	int i;
445 	char *cp;
446 
447 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
448 	buflen -= 1;
449 
450 	switch(data[2]) {
451 	case FB64_IV:
452 		sprintf(lbuf, "%s_IV", type);
453 		cp = lbuf;
454 		goto common;
455 
456 	case FB64_IV_OK:
457 		sprintf(lbuf, "%s_IV_OK", type);
458 		cp = lbuf;
459 		goto common;
460 
461 	case FB64_IV_BAD:
462 		sprintf(lbuf, "%s_IV_BAD", type);
463 		cp = lbuf;
464 		goto common;
465 
466 	default:
467 		sprintf(lbuf, " %d (unknown)", data[2]);
468 		cp = lbuf;
469 	common:
470 		for (; (buflen > 0) && (*buf = *cp++); buf++)
471 			buflen--;
472 		for (i = 3; i < cnt; i++) {
473 			sprintf(lbuf, " %d", data[i]);
474 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
475 				buflen--;
476 		}
477 		break;
478 	}
479 }
480 
481 void
482 cfb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
483 {
484 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
485 }
486 
487 void
488 ofb64_printsub(unsigned char *data, int cnt, unsigned char *buf, int buflen)
489 {
490 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
491 }
492 
493 void
494 fb64_stream_iv(Block seed, struct stinfo *stp)
495 {
496 
497 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
498 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
499 
500 	des_key_sched((Block *)stp->str_ikey, stp->str_sched);
501 
502 	stp->str_index = sizeof(Block);
503 }
504 
505 void
506 fb64_stream_key(Block key, struct stinfo *stp)
507 {
508 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
509 	des_key_sched((Block *)key, stp->str_sched);
510 
511 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
512 
513 	stp->str_index = sizeof(Block);
514 }
515 
516 /*
517  * DES 64 bit Cipher Feedback
518  *
519  *     key --->+-----+
520  *          +->| DES |--+
521  *          |  +-----+  |
522  *	    |           v
523  *  INPUT --(--------->(+)+---> DATA
524  *          |             |
525  *	    +-------------+
526  *
527  *
528  * Given:
529  *	iV: Initial vector, 64 bits (8 bytes) long.
530  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
531  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
532  *
533  *	V0 = DES(iV, key)
534  *	On = Dn ^ Vn
535  *	V(n+1) = DES(On, key)
536  */
537 
538 void
539 cfb64_encrypt(unsigned char *s, int c)
540 {
541 	struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
542 	int idx;
543 
544 	idx = stp->str_index;
545 	while (c-- > 0) {
546 		if (idx == sizeof(Block)) {
547 			Block b;
548 			des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
549 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
550 			idx = 0;
551 		}
552 
553 		/* On encryption, we store (feed ^ data) which is cypher */
554 		*s = stp->str_output[idx] = (stp->str_feed[idx] ^ *s);
555 		s++;
556 		idx++;
557 	}
558 	stp->str_index = idx;
559 }
560 
561 int
562 cfb64_decrypt(int data)
563 {
564 	struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
565 	int idx;
566 
567 	if (data == -1) {
568 		/*
569 		 * Back up one byte.  It is assumed that we will
570 		 * never back up more than one byte.  If we do, this
571 		 * may or may not work.
572 		 */
573 		if (stp->str_index)
574 			--stp->str_index;
575 		return(0);
576 	}
577 
578 	idx = stp->str_index++;
579 	if (idx == sizeof(Block)) {
580 		Block b;
581 		des_ecb_encrypt((Block *)stp->str_output, (Block *)b, stp->str_sched, 1);
582 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
583 		stp->str_index = 1;	/* Next time will be 1 */
584 		idx = 0;		/* But now use 0 */
585 	}
586 
587 	/* On decryption we store (data) which is cypher. */
588 	stp->str_output[idx] = data;
589 	return(data ^ stp->str_feed[idx]);
590 }
591 
592 /*
593  * DES 64 bit Output Feedback
594  *
595  * key --->+-----+
596  *	+->| DES |--+
597  *	|  +-----+  |
598  *	+-----------+
599  *	            v
600  *  INPUT -------->(+) ----> DATA
601  *
602  * Given:
603  *	iV: Initial vector, 64 bits (8 bytes) long.
604  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
605  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
606  *
607  *	V0 = DES(iV, key)
608  *	V(n+1) = DES(Vn, key)
609  *	On = Dn ^ Vn
610  */
611 void
612 ofb64_encrypt(unsigned char *s, int c)
613 {
614 	struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
615 	int idx;
616 
617 	idx = stp->str_index;
618 	while (c-- > 0) {
619 		if (idx == sizeof(Block)) {
620 			Block b;
621 			des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
622 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
623 			idx = 0;
624 		}
625 		*s++ ^= stp->str_feed[idx];
626 		idx++;
627 	}
628 	stp->str_index = idx;
629 }
630 
631 int
632 ofb64_decrypt(int data)
633 {
634 	struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
635 	int idx;
636 
637 	if (data == -1) {
638 		/*
639 		 * Back up one byte.  It is assumed that we will
640 		 * never back up more than one byte.  If we do, this
641 		 * may or may not work.
642 		 */
643 		if (stp->str_index)
644 			--stp->str_index;
645 		return(0);
646 	}
647 
648 	idx = stp->str_index++;
649 	if (idx == sizeof(Block)) {
650 		Block b;
651 		des_ecb_encrypt((Block *)stp->str_feed, (Block *)b, stp->str_sched, 1);
652 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
653 		stp->str_index = 1;	/* Next time will be 1 */
654 		idx = 0;		/* But now use 0 */
655 	}
656 
657 	return(data ^ stp->str_feed[idx]);
658 }
659 # endif	/* AUTHENTICATION */
660 #endif	/* ENCRYPTION */
661