xref: /original-bsd/lib/libtelnet/enc_des.c (revision c4f3b704)
1 /*-
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * %sccs.include.redist.c%
6  */
7 
8 #ifndef lint
9 static char sccsid[] = "@(#)enc_des.c	8.3 (Berkeley) 05/30/95";
10 #endif /* not lint */
11 
12 #ifdef	ENCRYPTION
13 # ifdef	AUTHENTICATION
14 #  ifdef DES_ENCRYPTION
15 #include <arpa/telnet.h>
16 #include <stdio.h>
17 #ifdef	__STDC__
18 #include <stdlib.h>
19 #endif
20 
21 #include "encrypt.h"
22 #include "key-proto.h"
23 #include "misc-proto.h"
24 
25 extern encrypt_debug_mode;
26 
27 #define	CFB	0
28 #define	OFB	1
29 
30 #define	NO_SEND_IV	1
31 #define	NO_RECV_IV	2
32 #define	NO_KEYID	4
33 #define	IN_PROGRESS	(NO_SEND_IV|NO_RECV_IV|NO_KEYID)
34 #define	SUCCESS		0
35 #define	FAILED		-1
36 
37 
38 struct fb {
39 	Block krbdes_key;
40 	Schedule krbdes_sched;
41 	Block temp_feed;
42 	unsigned char fb_feed[64];
43 	int need_start;
44 	int state[2];
45 	int keyid[2];
46 	int once;
47 	struct stinfo {
48 		Block		str_output;
49 		Block		str_feed;
50 		Block		str_iv;
51 		Block		str_ikey;
52 		Schedule	str_sched;
53 		int		str_index;
54 		int		str_flagshift;
55 	} streams[2];
56 };
57 
58 static struct fb fb[2];
59 
60 struct keyidlist {
61 	char	*keyid;
62 	int	keyidlen;
63 	char	*key;
64 	int	keylen;
65 	int	flags;
66 } keyidlist [] = {
67 	{ "\0", 1, 0, 0, 0 },		/* default key of zero */
68 	{ 0, 0, 0, 0, 0 }
69 };
70 
71 #define	KEYFLAG_MASK	03
72 
73 #define	KEYFLAG_NOINIT	00
74 #define	KEYFLAG_INIT	01
75 #define	KEYFLAG_OK	02
76 #define	KEYFLAG_BAD	03
77 
78 #define	KEYFLAG_SHIFT	2
79 
80 #define	SHIFT_VAL(a,b)	(KEYFLAG_SHIFT*((a)+((b)*2)))
81 
82 #define	FB64_IV		1
83 #define	FB64_IV_OK	2
84 #define	FB64_IV_BAD	3
85 
86 
87 void fb64_stream_iv P((Block, struct stinfo *));
88 void fb64_init P((struct fb *));
89 static int fb64_start P((struct fb *, int, int));
90 int fb64_is P((unsigned char *, int, struct fb *));
91 int fb64_reply P((unsigned char *, int, struct fb *));
92 static void fb64_session P((Session_Key *, int, struct fb *));
93 void fb64_stream_key P((Block, struct stinfo *));
94 int fb64_keyid P((int, unsigned char *, int *, struct fb *));
95 
96 	void
cfb64_init(server)97 cfb64_init(server)
98 	int server;
99 {
100 	fb64_init(&fb[CFB]);
101 	fb[CFB].fb_feed[4] = ENCTYPE_DES_CFB64;
102 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, CFB);
103 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, CFB);
104 }
105 
106 	void
ofb64_init(server)107 ofb64_init(server)
108 	int server;
109 {
110 	fb64_init(&fb[OFB]);
111 	fb[OFB].fb_feed[4] = ENCTYPE_DES_OFB64;
112 	fb[CFB].streams[0].str_flagshift = SHIFT_VAL(0, OFB);
113 	fb[CFB].streams[1].str_flagshift = SHIFT_VAL(1, OFB);
114 }
115 
116 	void
fb64_init(fbp)117 fb64_init(fbp)
118 	register struct fb *fbp;
119 {
120 	memset((void *)fbp, 0, sizeof(*fbp));
121 	fbp->state[0] = fbp->state[1] = FAILED;
122 	fbp->fb_feed[0] = IAC;
123 	fbp->fb_feed[1] = SB;
124 	fbp->fb_feed[2] = TELOPT_ENCRYPT;
125 	fbp->fb_feed[3] = ENCRYPT_IS;
126 }
127 
128 /*
129  * Returns:
130  *	-1: some error.  Negotiation is done, encryption not ready.
131  *	 0: Successful, initial negotiation all done.
132  *	 1: successful, negotiation not done yet.
133  *	 2: Not yet.  Other things (like getting the key from
134  *	    Kerberos) have to happen before we can continue.
135  */
136 	int
cfb64_start(dir,server)137 cfb64_start(dir, server)
138 	int dir;
139 	int server;
140 {
141 	return(fb64_start(&fb[CFB], dir, server));
142 }
143 	int
ofb64_start(dir,server)144 ofb64_start(dir, server)
145 	int dir;
146 	int server;
147 {
148 	return(fb64_start(&fb[OFB], dir, server));
149 }
150 
151 	static int
fb64_start(fbp,dir,server)152 fb64_start(fbp, dir, server)
153 	struct fb *fbp;
154 	int dir;
155 	int server;
156 {
157 	Block b;
158 	int x;
159 	unsigned char *p;
160 	register int state;
161 
162 	switch (dir) {
163 	case DIR_DECRYPT:
164 		/*
165 		 * This is simply a request to have the other side
166 		 * start output (our input).  He will negotiate an
167 		 * IV so we need not look for it.
168 		 */
169 		state = fbp->state[dir-1];
170 		if (state == FAILED)
171 			state = IN_PROGRESS;
172 		break;
173 
174 	case DIR_ENCRYPT:
175 		state = fbp->state[dir-1];
176 		if (state == FAILED)
177 			state = IN_PROGRESS;
178 		else if ((state & NO_SEND_IV) == 0)
179 			break;
180 
181 		if (!VALIDKEY(fbp->krbdes_key)) {
182 			fbp->need_start = 1;
183 			break;
184 		}
185 		state &= ~NO_SEND_IV;
186 		state |= NO_RECV_IV;
187 		if (encrypt_debug_mode)
188 			printf("Creating new feed\r\n");
189 		/*
190 		 * Create a random feed and send it over.
191 		 */
192 		des_new_random_key(fbp->temp_feed);
193 		des_ecb_encrypt(fbp->temp_feed, fbp->temp_feed,
194 				fbp->krbdes_sched, 1);
195 		p = fbp->fb_feed + 3;
196 		*p++ = ENCRYPT_IS;
197 		p++;
198 		*p++ = FB64_IV;
199 		for (x = 0; x < sizeof(Block); ++x) {
200 			if ((*p++ = fbp->temp_feed[x]) == IAC)
201 				*p++ = IAC;
202 		}
203 		*p++ = IAC;
204 		*p++ = SE;
205 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
206 		net_write(fbp->fb_feed, p - fbp->fb_feed);
207 		break;
208 	default:
209 		return(FAILED);
210 	}
211 	return(fbp->state[dir-1] = state);
212 }
213 
214 /*
215  * Returns:
216  *	-1: some error.  Negotiation is done, encryption not ready.
217  *	 0: Successful, initial negotiation all done.
218  *	 1: successful, negotiation not done yet.
219  */
220 	int
cfb64_is(data,cnt)221 cfb64_is(data, cnt)
222 	unsigned char *data;
223 	int cnt;
224 {
225 	return(fb64_is(data, cnt, &fb[CFB]));
226 }
227 	int
ofb64_is(data,cnt)228 ofb64_is(data, cnt)
229 	unsigned char *data;
230 	int cnt;
231 {
232 	return(fb64_is(data, cnt, &fb[OFB]));
233 }
234 
235 	int
fb64_is(data,cnt,fbp)236 fb64_is(data, cnt, fbp)
237 	unsigned char *data;
238 	int cnt;
239 	struct fb *fbp;
240 {
241 	int x;
242 	unsigned char *p;
243 	Block b;
244 	register int state = fbp->state[DIR_DECRYPT-1];
245 
246 	if (cnt-- < 1)
247 		goto failure;
248 
249 	switch (*data++) {
250 	case FB64_IV:
251 		if (cnt != sizeof(Block)) {
252 			if (encrypt_debug_mode)
253 				printf("CFB64: initial vector failed on size\r\n");
254 			state = FAILED;
255 			goto failure;
256 		}
257 
258 		if (encrypt_debug_mode)
259 			printf("CFB64: initial vector received\r\n");
260 
261 		if (encrypt_debug_mode)
262 			printf("Initializing Decrypt stream\r\n");
263 
264 		fb64_stream_iv((void *)data, &fbp->streams[DIR_DECRYPT-1]);
265 
266 		p = fbp->fb_feed + 3;
267 		*p++ = ENCRYPT_REPLY;
268 		p++;
269 		*p++ = FB64_IV_OK;
270 		*p++ = IAC;
271 		*p++ = SE;
272 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
273 		net_write(fbp->fb_feed, p - fbp->fb_feed);
274 
275 		state = fbp->state[DIR_DECRYPT-1] = IN_PROGRESS;
276 		break;
277 
278 	default:
279 		if (encrypt_debug_mode) {
280 			printf("Unknown option type: %d\r\n", *(data-1));
281 			printd(data, cnt);
282 			printf("\r\n");
283 		}
284 		/* FALL THROUGH */
285 	failure:
286 		/*
287 		 * We failed.  Send an FB64_IV_BAD option
288 		 * to the other side so it will know that
289 		 * things failed.
290 		 */
291 		p = fbp->fb_feed + 3;
292 		*p++ = ENCRYPT_REPLY;
293 		p++;
294 		*p++ = FB64_IV_BAD;
295 		*p++ = IAC;
296 		*p++ = SE;
297 		printsub('>', &fbp->fb_feed[2], p - &fbp->fb_feed[2]);
298 		net_write(fbp->fb_feed, p - fbp->fb_feed);
299 
300 		break;
301 	}
302 	return(fbp->state[DIR_DECRYPT-1] = state);
303 }
304 
305 /*
306  * Returns:
307  *	-1: some error.  Negotiation is done, encryption not ready.
308  *	 0: Successful, initial negotiation all done.
309  *	 1: successful, negotiation not done yet.
310  */
311 	int
cfb64_reply(data,cnt)312 cfb64_reply(data, cnt)
313 	unsigned char *data;
314 	int cnt;
315 {
316 	return(fb64_reply(data, cnt, &fb[CFB]));
317 }
318 	int
ofb64_reply(data,cnt)319 ofb64_reply(data, cnt)
320 	unsigned char *data;
321 	int cnt;
322 {
323 	return(fb64_reply(data, cnt, &fb[OFB]));
324 }
325 
326 
327 	int
fb64_reply(data,cnt,fbp)328 fb64_reply(data, cnt, fbp)
329 	unsigned char *data;
330 	int cnt;
331 	struct fb *fbp;
332 {
333 	int x;
334 	unsigned char *p;
335 	Block b;
336 	register int state = fbp->state[DIR_ENCRYPT-1];
337 
338 	if (cnt-- < 1)
339 		goto failure;
340 
341 	switch (*data++) {
342 	case FB64_IV_OK:
343 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
344 		if (state == FAILED)
345 			state = IN_PROGRESS;
346 		state &= ~NO_RECV_IV;
347 		encrypt_send_keyid(DIR_ENCRYPT, (unsigned char *)"\0", 1, 1);
348 		break;
349 
350 	case FB64_IV_BAD:
351 		memset(fbp->temp_feed, 0, sizeof(Block));
352 		fb64_stream_iv(fbp->temp_feed, &fbp->streams[DIR_ENCRYPT-1]);
353 		state = FAILED;
354 		break;
355 
356 	default:
357 		if (encrypt_debug_mode) {
358 			printf("Unknown option type: %d\r\n", data[-1]);
359 			printd(data, cnt);
360 			printf("\r\n");
361 		}
362 		/* FALL THROUGH */
363 	failure:
364 		state = FAILED;
365 		break;
366 	}
367 	return(fbp->state[DIR_ENCRYPT-1] = state);
368 }
369 
370 	void
cfb64_session(key,server)371 cfb64_session(key, server)
372 	Session_Key *key;
373 	int server;
374 {
375 	fb64_session(key, server, &fb[CFB]);
376 }
377 
378 	void
ofb64_session(key,server)379 ofb64_session(key, server)
380 	Session_Key *key;
381 	int server;
382 {
383 	fb64_session(key, server, &fb[OFB]);
384 }
385 
386 	static void
fb64_session(key,server,fbp)387 fb64_session(key, server, fbp)
388 	Session_Key *key;
389 	int server;
390 	struct fb *fbp;
391 {
392 
393 	if (!key || key->type != SK_DES) {
394 		if (encrypt_debug_mode)
395 			printf("Can't set krbdes's session key (%d != %d)\r\n",
396 				key ? key->type : -1, SK_DES);
397 		return;
398 	}
399 	memmove((void *)fbp->krbdes_key, (void *)key->data, sizeof(Block));
400 
401 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_ENCRYPT-1]);
402 	fb64_stream_key(fbp->krbdes_key, &fbp->streams[DIR_DECRYPT-1]);
403 
404 	if (fbp->once == 0) {
405 		des_set_random_generator_seed(fbp->krbdes_key);
406 		fbp->once = 1;
407 	}
408 	des_key_sched(fbp->krbdes_key, fbp->krbdes_sched);
409 	/*
410 	 * Now look to see if krbdes_start() was was waiting for
411 	 * the key to show up.  If so, go ahead an call it now
412 	 * that we have the key.
413 	 */
414 	if (fbp->need_start) {
415 		fbp->need_start = 0;
416 		fb64_start(fbp, DIR_ENCRYPT, server);
417 	}
418 }
419 
420 /*
421  * We only accept a keyid of 0.  If we get a keyid of
422  * 0, then mark the state as SUCCESS.
423  */
424 	int
cfb64_keyid(dir,kp,lenp)425 cfb64_keyid(dir, kp, lenp)
426 	int dir, *lenp;
427 	unsigned char *kp;
428 {
429 	return(fb64_keyid(dir, kp, lenp, &fb[CFB]));
430 }
431 
432 	int
ofb64_keyid(dir,kp,lenp)433 ofb64_keyid(dir, kp, lenp)
434 	int dir, *lenp;
435 	unsigned char *kp;
436 {
437 	return(fb64_keyid(dir, kp, lenp, &fb[OFB]));
438 }
439 
440 	int
fb64_keyid(dir,kp,lenp,fbp)441 fb64_keyid(dir, kp, lenp, fbp)
442 	int dir, *lenp;
443 	unsigned char *kp;
444 	struct fb *fbp;
445 {
446 	register int state = fbp->state[dir-1];
447 
448 	if (*lenp != 1 || (*kp != '\0')) {
449 		*lenp = 0;
450 		return(state);
451 	}
452 
453 	if (state == FAILED)
454 		state = IN_PROGRESS;
455 
456 	state &= ~NO_KEYID;
457 
458 	return(fbp->state[dir-1] = state);
459 }
460 
461 	void
fb64_printsub(data,cnt,buf,buflen,type)462 fb64_printsub(data, cnt, buf, buflen, type)
463 	unsigned char *data, *buf, *type;
464 	int cnt, buflen;
465 {
466 	char lbuf[32];
467 	register int i;
468 	char *cp;
469 
470 	buf[buflen-1] = '\0';		/* make sure it's NULL terminated */
471 	buflen -= 1;
472 
473 	switch(data[2]) {
474 	case FB64_IV:
475 		sprintf(lbuf, "%s_IV", type);
476 		cp = lbuf;
477 		goto common;
478 
479 	case FB64_IV_OK:
480 		sprintf(lbuf, "%s_IV_OK", type);
481 		cp = lbuf;
482 		goto common;
483 
484 	case FB64_IV_BAD:
485 		sprintf(lbuf, "%s_IV_BAD", type);
486 		cp = lbuf;
487 		goto common;
488 
489 	default:
490 		sprintf(lbuf, " %d (unknown)", data[2]);
491 		cp = lbuf;
492 	common:
493 		for (; (buflen > 0) && (*buf = *cp++); buf++)
494 			buflen--;
495 		for (i = 3; i < cnt; i++) {
496 			sprintf(lbuf, " %d", data[i]);
497 			for (cp = lbuf; (buflen > 0) && (*buf = *cp++); buf++)
498 				buflen--;
499 		}
500 		break;
501 	}
502 }
503 
504 	void
cfb64_printsub(data,cnt,buf,buflen)505 cfb64_printsub(data, cnt, buf, buflen)
506 	unsigned char *data, *buf;
507 	int cnt, buflen;
508 {
509 	fb64_printsub(data, cnt, buf, buflen, "CFB64");
510 }
511 
512 	void
ofb64_printsub(data,cnt,buf,buflen)513 ofb64_printsub(data, cnt, buf, buflen)
514 	unsigned char *data, *buf;
515 	int cnt, buflen;
516 {
517 	fb64_printsub(data, cnt, buf, buflen, "OFB64");
518 }
519 
520 	void
fb64_stream_iv(seed,stp)521 fb64_stream_iv(seed, stp)
522 	Block seed;
523 	register struct stinfo *stp;
524 {
525 
526 	memmove((void *)stp->str_iv, (void *)seed, sizeof(Block));
527 	memmove((void *)stp->str_output, (void *)seed, sizeof(Block));
528 
529 	des_key_sched(stp->str_ikey, stp->str_sched);
530 
531 	stp->str_index = sizeof(Block);
532 }
533 
534 	void
fb64_stream_key(key,stp)535 fb64_stream_key(key, stp)
536 	Block key;
537 	register struct stinfo *stp;
538 {
539 	memmove((void *)stp->str_ikey, (void *)key, sizeof(Block));
540 	des_key_sched(key, stp->str_sched);
541 
542 	memmove((void *)stp->str_output, (void *)stp->str_iv, sizeof(Block));
543 
544 	stp->str_index = sizeof(Block);
545 }
546 
547 /*
548  * DES 64 bit Cipher Feedback
549  *
550  *     key --->+-----+
551  *          +->| DES |--+
552  *          |  +-----+  |
553  *	    |           v
554  *  INPUT --(--------->(+)+---> DATA
555  *          |             |
556  *	    +-------------+
557  *
558  *
559  * Given:
560  *	iV: Initial vector, 64 bits (8 bytes) long.
561  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
562  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
563  *
564  *	V0 = DES(iV, key)
565  *	On = Dn ^ Vn
566  *	V(n+1) = DES(On, key)
567  */
568 
569 	void
cfb64_encrypt(s,c)570 cfb64_encrypt(s, c)
571 	register unsigned char *s;
572 	int c;
573 {
574 	register struct stinfo *stp = &fb[CFB].streams[DIR_ENCRYPT-1];
575 	register int index;
576 
577 	index = stp->str_index;
578 	while (c-- > 0) {
579 		if (index == sizeof(Block)) {
580 			Block b;
581 			des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
582 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
583 			index = 0;
584 		}
585 
586 		/* On encryption, we store (feed ^ data) which is cypher */
587 		*s = stp->str_output[index] = (stp->str_feed[index] ^ *s);
588 		s++;
589 		index++;
590 	}
591 	stp->str_index = index;
592 }
593 
594 	int
cfb64_decrypt(data)595 cfb64_decrypt(data)
596 	int data;
597 {
598 	register struct stinfo *stp = &fb[CFB].streams[DIR_DECRYPT-1];
599 	int index;
600 
601 	if (data == -1) {
602 		/*
603 		 * Back up one byte.  It is assumed that we will
604 		 * never back up more than one byte.  If we do, this
605 		 * may or may not work.
606 		 */
607 		if (stp->str_index)
608 			--stp->str_index;
609 		return(0);
610 	}
611 
612 	index = stp->str_index++;
613 	if (index == sizeof(Block)) {
614 		Block b;
615 		des_ecb_encrypt(stp->str_output, b, stp->str_sched, 1);
616 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
617 		stp->str_index = 1;	/* Next time will be 1 */
618 		index = 0;		/* But now use 0 */
619 	}
620 
621 	/* On decryption we store (data) which is cypher. */
622 	stp->str_output[index] = data;
623 	return(data ^ stp->str_feed[index]);
624 }
625 
626 /*
627  * DES 64 bit Output Feedback
628  *
629  * key --->+-----+
630  *	+->| DES |--+
631  *	|  +-----+  |
632  *	+-----------+
633  *	            v
634  *  INPUT -------->(+) ----> DATA
635  *
636  * Given:
637  *	iV: Initial vector, 64 bits (8 bytes) long.
638  *	Dn: the nth chunk of 64 bits (8 bytes) of data to encrypt (decrypt).
639  *	On: the nth chunk of 64 bits (8 bytes) of encrypted (decrypted) output.
640  *
641  *	V0 = DES(iV, key)
642  *	V(n+1) = DES(Vn, key)
643  *	On = Dn ^ Vn
644  */
645 	void
ofb64_encrypt(s,c)646 ofb64_encrypt(s, c)
647 	register unsigned char *s;
648 	int c;
649 {
650 	register struct stinfo *stp = &fb[OFB].streams[DIR_ENCRYPT-1];
651 	register int index;
652 
653 	index = stp->str_index;
654 	while (c-- > 0) {
655 		if (index == sizeof(Block)) {
656 			Block b;
657 			des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
658 			memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
659 			index = 0;
660 		}
661 		*s++ ^= stp->str_feed[index];
662 		index++;
663 	}
664 	stp->str_index = index;
665 }
666 
667 	int
ofb64_decrypt(data)668 ofb64_decrypt(data)
669 	int data;
670 {
671 	register struct stinfo *stp = &fb[OFB].streams[DIR_DECRYPT-1];
672 	int index;
673 
674 	if (data == -1) {
675 		/*
676 		 * Back up one byte.  It is assumed that we will
677 		 * never back up more than one byte.  If we do, this
678 		 * may or may not work.
679 		 */
680 		if (stp->str_index)
681 			--stp->str_index;
682 		return(0);
683 	}
684 
685 	index = stp->str_index++;
686 	if (index == sizeof(Block)) {
687 		Block b;
688 		des_ecb_encrypt(stp->str_feed, b, stp->str_sched, 1);
689 		memmove((void *)stp->str_feed, (void *)b, sizeof(Block));
690 		stp->str_index = 1;	/* Next time will be 1 */
691 		index = 0;		/* But now use 0 */
692 	}
693 
694 	return(data ^ stp->str_feed[index]);
695 }
696 #  endif /* DES_ENCRYPTION */
697 # endif	/* AUTHENTICATION */
698 #endif	/* ENCRYPTION */
699