xref: /original-bsd/lib/libtelnet/enc_des.c (revision 3b43aa51)
1 /*-
2  * Copyright (c) 1991 The Regents of the University of California.
3  * All rights reserved.
4  *
5  * %sccs.include.redist.c%
6  */
7 
8 #ifndef lint
9 static char sccsid[] = "@(#)enc_des.c	5.2 (Berkeley) 12/18/92";
10 #endif /* not lint */
11 
12 #if defined(AUTHENTICATION) && defined(ENCRYPTION) && defined(DES_ENCRYPTION)
13 #include <arpa/telnet.h>
14 #include <stdio.h>
15 #ifdef	__STDC__
16 #include <stdlib.h>
17 #endif
18 
19 #include "encrypt.h"
20 #include "key-proto.h"
21 #include "misc-proto.h"
22 
23 extern encrypt_debug_mode;
24 
25 #define	CFB	0
26 #define	OFB	1
27 
28 #define	NO_SEND_IV	1
29 #define	NO_RECV_IV	2
30 #define	NO_KEYID	4
31 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
32 #define	SUCCESS		0
33 #define	FAILED		-1
34 
35 
36 struct fb {
37 	Block krbdes_key;
38 	Schedule krbdes_sched;
39 	Block temp_feed;
40 	unsigned char fb_feed[64];
41 	int need_start;
42 	int state[2];
43 	int keyid[2];
44 	int once;
45 	struct stinfo {
46 		Block		str_output;
47 		Block		str_feed;
48 		Block		str_iv;
49 		Block		str_ikey;
50 		Schedule	str_sched;
51 		int		str_index;
52 		int		str_flagshift;
53 	} streams[2];
54 };
55 
56 static struct fb fb[2];
57 
58 struct keyidlist {
59 	char	*keyid;
60 	int	keyidlen;
61 	char	*key;
62 	int	keylen;
63 	int	flags;
64 } keyidlist [] = {
65 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
66 	{ 0, 0, 0, 0, 0 }
67 };
68 
69 #define	KEYFLAG_MASK	03
70 
71 #define	KEYFLAG_NOINIT	00
72 #define	KEYFLAG_INIT	01
73 #define	KEYFLAG_OK	02
74 #define	KEYFLAG_BAD	03
75 
76 #define	KEYFLAG_SHIFT	2
77 
78 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
79 
80 #define	FB64_IV		1
81 #define	FB64_IV_OK	2
82 #define	FB64_IV_BAD	3
83 
84 
85 void fb64_stream_iv P((Block, struct stinfo *));
86 void fb64_init P((struct fb *));
87 int fb64_start P((struct fb *, int, int));
88 int fb64_is P((unsigned char *, int, struct fb *));
89 int fb64_reply P((unsigned char *, int, struct fb *));
90 void fb64_session P((Session_Key *, int, struct fb *));
91 void fb64_stream_key P((Block, struct stinfo *));
92 int fb64_keyid P((int, unsigned char *, int *, struct fb *));
93 
94 	void
95 cfb64_init(server)
96 	int server;
97 {
98 	fb64_init(&fb[CFB]);
99 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
100 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
101 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
102 }
103 
104 	void
105 ofb64_init(server)
106 	int server;
107 {
108 	fb64_init(&fb[OFB]);
109 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
110 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
111 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
112 }
113 
114 	void
115 fb64_init(fbp)
116 	register struct fb *fbp;
117 {
118 	bzero((void *)fbp, sizeof(*fbp));
119 	fbp->state[0] = fbp->state[1] = FAILED;
120 	fbp->fb_feed[0] = IAC;
121 	fbp->fb_feed[1] = SB;
122 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
123 	fbp->fb_feed[3] = ENCRYPT_IS;
124 }
125 
126 /*
127  * Returns:
128  *	-1: some error.  Negotiation is done, encryption not ready.
129  *	 0: Successful, initial negotiation all done.
130  *	 1: successful, negotiation not done yet.
131  *	 2: Not yet.  Other things (like getting the key from
132  *	    Kerberos) have to happen before we can continue.
133  */
134 	int
135 cfb64_start(dir, server)
136 	int dir;
137 	int server;
138 {
139 	return(fb64_start(&fb[CFB], dir, server));
140 }
141 	int
142 ofb64_start(dir, server)
143 	int dir;
144 	int server;
145 {
146 	return(fb64_start(&fb[OFB], dir, server));
147 }
148 
149 	static int
150 fb64_start(fbp, dir, server)
151 	struct fb *fbp;
152 	int dir;
153 	int server;
154 {
155 	Block b;
156 	int x;
157 	unsigned char *p;
158 	register int state;
159 
160 	switch (dir) {
161 	case DIR_DECRYPT:
162 		/*
163 		 * This is simply a request to have the other side
164 		 * start output (our input).  He will negotiate an
165 		 * IV so we need not look for it.
166 		 */
167 		state = fbp->state[dir-1];
168 		if (state == FAILED)
169 			state = IN_PROGRESS;
170 		break;
171 
172 	case DIR_ENCRYPT:
173 		state = fbp->state[dir-1];
174 		if (state == FAILED)
175 			state = IN_PROGRESS;
176 		else if ((state & NO_SEND_IV) == 0)
177 			break;
178 
179 		if (!VALIDKEY(fbp->krbdes_key)) {
180 			fbp->need_start = 1;
181 			break;
182 		}
183 		state &= ~NO_SEND_IV;
184 		state |= NO_RECV_IV;
185 		if (encrypt_debug_mode)
186 			printf("Creating new feed\r\n");
187 		/*
188 		 * Create a random feed and send it over.
189 		 */
190 		des_new_random_key(fbp->temp_feed);
191 		des_ecb_encrypt(fbp->temp_feed, fbp->temp_feed,
192 				fbp->krbdes_sched, 1);
193 		p = fbp->fb_feed + 3;
194 		*p++ = ENCRYPT_IS;
195 		p++;
196 		*p++ = FB64_IV;
197 		for (x = 0; x < sizeof(Block); ++x) {
198 			if ((*p++ = fbp->temp_feed[x]) == IAC)
199 				*p++ = IAC;
200 		}
201 		*p++ = IAC;
202 		*p++ = SE;
203 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
204 		net_write(fbp->fb_feed, p - fbp->fb_feed);
205 		break;
206 	default:
207 		return(FAILED);
208 	}
209 	return(fbp->state[dir-1] = state);
210 }
211 
212 /*
213  * Returns:
214  *	-1: some error.  Negotiation is done, encryption not ready.
215  *	 0: Successful, initial negotiation all done.
216  *	 1: successful, negotiation not done yet.
217  */
218 	int
219 cfb64_is(data, cnt)
220 	unsigned char *data;
221 	int cnt;
222 {
223 	return(fb64_is(data, cnt, &fb[CFB]));
224 }
225 	int
226 ofb64_is(data, cnt)
227 	unsigned char *data;
228 	int cnt;
229 {
230 	return(fb64_is(data, cnt, &fb[OFB]));
231 }
232 
233 	int
234 fb64_is(data, cnt, fbp)
235 	unsigned char *data;
236 	int cnt;
237 	struct fb *fbp;
238 {
239 	int x;
240 	unsigned char *p;
241 	Block b;
242 	register int state = fbp->state[DIR_DECRYPT-1];
243 
244 	if (cnt-- < 1)
245 		goto failure;
246 
247 	switch (*data++) {
248 	case FB64_IV:
249 		if (cnt != sizeof(Block)) {
250 			if (encrypt_debug_mode)
251 				printf("CFB64: initial vector failed on size\r\n");
252 			state = FAILED;
253 			goto failure;
254 		}
255 
256 		if (encrypt_debug_mode)
257 			printf("CFB64: initial vector received\r\n");
258 
259 		if (encrypt_debug_mode)
260 			printf("Initializing Decrypt stream\r\n");
261 
262 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
263 
264 		p = fbp->fb_feed + 3;
265 		*p++ = ENCRYPT_REPLY;
266 		p++;
267 		*p++ = FB64_IV_OK;
268 		*p++ = IAC;
269 		*p++ = SE;
270 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
271 		net_write(fbp->fb_feed, p - fbp->fb_feed);
272 
273 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
274 		break;
275 
276 	default:
277 		if (encrypt_debug_mode) {
278 			printf("Unknown option type: %d\r\n", *(data-1));
279 			printd(data, cnt);
280 			printf("\r\n");
281 		}
282 		/* FALL THROUGH */
283 	failure:
284 		/*
285 		 * We failed.  Send an FB64_IV_BAD option
286 		 * to the other side so it will know that
287 		 * things failed.
288 		 */
289 		p = fbp->fb_feed + 3;
290 		*p++ = ENCRYPT_REPLY;
291 		p++;
292 		*p++ = FB64_IV_BAD;
293 		*p++ = IAC;
294 		*p++ = SE;
295 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
296 		net_write(fbp->fb_feed, p - fbp->fb_feed);
297 
298 		break;
299 	}
300 	return(fbp->state[DIR_DECRYPT-1] = state);
301 }
302 
303 /*
304  * Returns:
305  *	-1: some error.  Negotiation is done, encryption not ready.
306  *	 0: Successful, initial negotiation all done.
307  *	 1: successful, negotiation not done yet.
308  */
309 	int
310 cfb64_reply(data, cnt)
311 	unsigned char *data;
312 	int cnt;
313 {
314 	return(fb64_reply(data, cnt, &fb[CFB]));
315 }
316 	int
317 ofb64_reply(data, cnt)
318 	unsigned char *data;
319 	int cnt;
320 {
321 	return(fb64_reply(data, cnt, &fb[OFB]));
322 }
323 
324 
325 	int
326 fb64_reply(data, cnt, fbp)
327 	unsigned char *data;
328 	int cnt;
329 	struct fb *fbp;
330 {
331 	int x;
332 	unsigned char *p;
333 	Block b;
334 	register int state = fbp->state[DIR_ENCRYPT-1];
335 
336 	if (cnt-- < 1)
337 		goto failure;
338 
339 	switch (*data++) {
340 	case FB64_IV_OK:
341 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
342 		if (state == FAILED)
343 			state = IN_PROGRESS;
344 		state &= ~NO_RECV_IV;
345 		encrypt_send_keyid(DIR_ENCRYPT, "\0", 1, 1);
346 		break;
347 
348 	case FB64_IV_BAD:
349 		bzero(fbp->temp_feed, sizeof(Block));
350 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
351 		state = FAILED;
352 		break;
353 
354 	default:
355 		if (encrypt_debug_mode) {
356 			printf("Unknown option type: %d\r\n", data[-1]);
357 			printd(data, cnt);
358 			printf("\r\n");
359 		}
360 		/* FALL THROUGH */
361 	failure:
362 		state = FAILED;
363 		break;
364 	}
365 	return(fbp->state[DIR_ENCRYPT-1] = state);
366 }
367 
368 	void
369 cfb64_session(key, server)
370 	Session_Key *key;
371 	int server;
372 {
373 	fb64_session(key, server, &fb[CFB]);
374 }
375 
376 	void
377 ofb64_session(key, server)
378 	Session_Key *key;
379 	int server;
380 {
381 	fb64_session(key, server, &fb[OFB]);
382 }
383 
384 	static void
385 fb64_session(key, server, fbp)
386 	Session_Key *key;
387 	int server;
388 	struct fb *fbp;
389 {
390 
391 	if (!key || key->type != SK_DES) {
392 		if (encrypt_debug_mode)
393 			printf("Can't set krbdes's session key (%d != %d)\r\n",
394 				key ? key->type : -1, SK_DES);
395 		return;
396 	}
397 	bcopy((void *)key->data, (void *)fbp->krbdes_key, sizeof(Block));
398 
399 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
400 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
401 
402 	if (fbp->once == 0) {
403 		des_set_random_generator_seed(fbp->krbdes_key);
404 		fbp->once = 1;
405 	}
406 	des_key_sched(fbp->krbdes_key, fbp->krbdes_sched);
407 	/*
408 	 * Now look to see if krbdes_start() was was waiting for
409 	 * the key to show up.  If so, go ahead an call it now
410 	 * that we have the key.
411 	 */
412 	if (fbp->need_start) {
413 		fbp->need_start = 0;
414 		fb64_start(fbp, DIR_ENCRYPT, server);
415 	}
416 }
417 
418 /*
419  * We only accept a keyid of 0.  If we get a keyid of
420  * 0, then mark the state as SUCCESS.
421  */
422 	int
423 cfb64_keyid(dir, kp, lenp)
424 	int dir, *lenp;
425 	unsigned char *kp;
426 {
427 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
428 }
429 
430 	int
431 ofb64_keyid(dir, kp, lenp)
432 	int dir, *lenp;
433 	unsigned char *kp;
434 {
435 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
436 }
437 
438 	int
439 fb64_keyid(dir, kp, lenp, fbp)
440 	int dir, *lenp;
441 	unsigned char *kp;
442 	struct fb *fbp;
443 {
444 	register int state = fbp->state[dir-1];
445 
446 	if (*lenp != 1 || (*kp != '\0')) {
447 		*lenp = 0;
448 		return(state);
449 	}
450 
451 	if (state == FAILED)
452 		state = IN_PROGRESS;
453 
454 	state &= ~NO_KEYID;
455 
456 	return(fbp->state[dir-1] = state);
457 }
458 
459 	void
460 fb64_printsub(data, cnt, buf, buflen, type)
461 	unsigned char *data, *buf, *type;
462 	int cnt, buflen;
463 {
464 	char lbuf[32];
465 	register int i;
466 	char *cp;
467 
468 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
469 	buflen -= 1;
470 
471 	switch(data[2]) {
472 	case FB64_IV:
473 		sprintf(lbuf, "%s_IV", type);
474 		cp = lbuf;
475 		goto common;
476 
477 	case FB64_IV_OK:
478 		sprintf(lbuf, "%s_IV_OK", type);
479 		cp = lbuf;
480 		goto common;
481 
482 	case FB64_IV_BAD:
483 		sprintf(lbuf, "%s_IV_BAD", type);
484 		cp = lbuf;
485 		goto common;
486 
487 	default:
488 		sprintf(lbuf, " %d (unknown)", data[2]);
489 		cp = lbuf;
490 	common:
491 		for (; (buflen > 0) && (*buf = *cp++); buf++)
492 			buflen--;
493 		for (i = 3; i < cnt; i++) {
494 			sprintf(lbuf, " %d", data[i]);
495 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
496 				buflen--;
497 		}
498 		break;
499 	}
500 }
501 
502 	void
503 cfb64_printsub(data, cnt, buf, buflen)
504 	unsigned char *data, *buf;
505 	int cnt, buflen;
506 {
507 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
508 }
509 
510 	void
511 ofb64_printsub(data, cnt, buf, buflen)
512 	unsigned char *data, *buf;
513 	int cnt, buflen;
514 {
515 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
516 }
517 
518 	void
519 fb64_stream_iv(seed, stp)
520 	Block seed;
521 	register struct stinfo *stp;
522 {
523 
524 	bcopy((void *)seed, (void *)stp->str_iv, sizeof(Block));
525 	bcopy((void *)seed, (void *)stp->str_output, sizeof(Block));
526 
527 	des_key_sched(stp->str_ikey, stp->str_sched);
528 
529 	stp->str_index = sizeof(Block);
530 }
531 
532 	void
533 fb64_stream_key(key, stp)
534 	Block key;
535 	register struct stinfo *stp;
536 {
537 	bcopy((void *)key, (void *)stp->str_ikey, sizeof(Block));
538 	des_key_sched(key, stp->str_sched);
539 
540 	bcopy((void *)stp->str_iv, (void *)stp->str_output, sizeof(Block));
541 
542 	stp->str_index = sizeof(Block);
543 }
544 
545 /*
546  * DES 64 bit Cipher Feedback
547  *
548  *     key --->+-----+
549  *          +->| DES |--+
550  *          |  +-----+  |
551  *	    |           v
552  *  INPUT --(--------->(+)+---> DATA
553  *          |             |
554  *	    +-------------+
555  *
556  *
557  * Given:
558  *	iV: Initial vector, 64 bits (8 bytes) long.
559  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
560  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
561  *
562  *	V0 = DES(iV, key)
563  *	On = Dn ^ Vn
564  *	V(n+1) = DES(On, key)
565  */
566 
567 	void
568 cfb64_encrypt(s, c)
569 	register unsigned char *s;
570 	int c;
571 {
572 	register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
573 	register int index;
574 
575 	index = stp->str_index;
576 	while (c-- > 0) {
577 		if (index == sizeof(Block)) {
578 			Block b;
579 			des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
580 			bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
581 			index = 0;
582 		}
583 
584 		/* On encryption, we store (feed ^ data) which is cypher */
585 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
586 		s++;
587 		index++;
588 	}
589 	stp->str_index = index;
590 }
591 
592 	int
593 cfb64_decrypt(data)
594 	int data;
595 {
596 	register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
597 	int index;
598 
599 	if (data == -1) {
600 		/*
601 		 * Back up one byte.  It is assumed that we will
602 		 * never back up more than one byte.  If we do, this
603 		 * may or may not work.
604 		 */
605 		if (stp->str_index)
606 			--stp->str_index;
607 		return(0);
608 	}
609 
610 	index = stp->str_index++;
611 	if (index == sizeof(Block)) {
612 		Block b;
613 		des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
614 		bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
615 		stp->str_index = 1;	/* Next time will be 1 */
616 		index = 0;		/* But now use 0 */
617 	}
618 
619 	/* On decryption we store (data) which is cypher. */
620 	stp->str_output[index] = data;
621 	return(data ^ stp->str_feed[index]);
622 }
623 
624 /*
625  * DES 64 bit Output Feedback
626  *
627  * key --->+-----+
628  *	+->| DES |--+
629  *	|  +-----+  |
630  *	+-----------+
631  *	            v
632  *  INPUT -------->(+) ----> DATA
633  *
634  * Given:
635  *	iV: Initial vector, 64 bits (8 bytes) long.
636  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
637  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
638  *
639  *	V0 = DES(iV, key)
640  *	V(n+1) = DES(Vn, key)
641  *	On = Dn ^ Vn
642  */
643 	void
644 ofb64_encrypt(s, c)
645 	register unsigned char *s;
646 	int c;
647 {
648 	register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
649 	register int index;
650 
651 	index = stp->str_index;
652 	while (c-- > 0) {
653 		if (index == sizeof(Block)) {
654 			Block b;
655 			des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
656 			bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
657 			index = 0;
658 		}
659 		*s++ ^= stp->str_feed[index];
660 		index++;
661 	}
662 	stp->str_index = index;
663 }
664 
665 	int
666 ofb64_decrypt(data)
667 	int data;
668 {
669 	register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
670 	int index;
671 
672 	if (data == -1) {
673 		/*
674 		 * Back up one byte.  It is assumed that we will
675 		 * never back up more than one byte.  If we do, this
676 		 * may or may not work.
677 		 */
678 		if (stp->str_index)
679 			--stp->str_index;
680 		return(0);
681 	}
682 
683 	index = stp->str_index++;
684 	if (index == sizeof(Block)) {
685 		Block b;
686 		des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
687 		bcopy((void *)b, (void *)stp->str_feed, sizeof(Block));
688 		stp->str_index = 1;	/* Next time will be 1 */
689 		index = 0;		/* But now use 0 */
690 	}
691 
692 	return(data ^ stp->str_feed[index]);
693 }
694 #endif
695