1 /*
2 * alg2268.c - implementation of the algorithm in RFC 2268
3 *
4 * This Source Code Form is subject to the terms of the Mozilla Public
5 * License, v. 2.0. If a copy of the MPL was not distributed with this
6 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7
8 #ifdef FREEBL_NO_DEPEND
9 #include "../stubs.h"
10 #endif
11
12 #include "../blapi.h"
13 #include "../blapii.h"
14 #include "secerr.h"
15 #ifdef XP_UNIX_XXX
16 #include <stddef.h> /* for ptrdiff_t */
17 #endif
18
19 /*
20 ** RC2 symmetric block cypher
21 */
22
23 typedef SECStatus(rc2Func)(RC2Context *cx, unsigned char *output,
24 const unsigned char *input, unsigned int inputLen);
25
26 /* forward declarations */
27 static rc2Func rc2_EncryptECB;
28 static rc2Func rc2_DecryptECB;
29 static rc2Func rc2_EncryptCBC;
30 static rc2Func rc2_DecryptCBC;
31
32 typedef union {
33 PRUint32 l[2];
34 PRUint16 s[4];
35 PRUint8 b[8];
36 } RC2Block;
37
38 struct RC2ContextStr {
39 union {
40 PRUint8 Kb[128];
41 PRUint16 Kw[64];
42 } u;
43 RC2Block iv;
44 rc2Func *enc;
45 rc2Func *dec;
46 };
47
48 #define B u.Kb
49 #define K u.Kw
50 #define BYTESWAP(x) ((x) << 8 | (x) >> 8)
51 #define SWAPK(i) cx->K[i] = (tmpS = cx->K[i], BYTESWAP(tmpS))
52 #define RC2_BLOCK_SIZE 8
53
54 #define LOAD_HARD(R) \
55 R[0] = (PRUint16)input[1] << 8 | input[0]; \
56 R[1] = (PRUint16)input[3] << 8 | input[2]; \
57 R[2] = (PRUint16)input[5] << 8 | input[4]; \
58 R[3] = (PRUint16)input[7] << 8 | input[6];
59 #define LOAD_EASY(R) \
60 R[0] = ((PRUint16 *)input)[0]; \
61 R[1] = ((PRUint16 *)input)[1]; \
62 R[2] = ((PRUint16 *)input)[2]; \
63 R[3] = ((PRUint16 *)input)[3];
64 #define STORE_HARD(R) \
65 output[0] = (PRUint8)(R[0]); \
66 output[1] = (PRUint8)(R[0] >> 8); \
67 output[2] = (PRUint8)(R[1]); \
68 output[3] = (PRUint8)(R[1] >> 8); \
69 output[4] = (PRUint8)(R[2]); \
70 output[5] = (PRUint8)(R[2] >> 8); \
71 output[6] = (PRUint8)(R[3]); \
72 output[7] = (PRUint8)(R[3] >> 8);
73 #define STORE_EASY(R) \
74 ((PRUint16 *)output)[0] = R[0]; \
75 ((PRUint16 *)output)[1] = R[1]; \
76 ((PRUint16 *)output)[2] = R[2]; \
77 ((PRUint16 *)output)[3] = R[3];
78
79 #if defined(NSS_X86_OR_X64)
80 #define LOAD(R) LOAD_EASY(R)
81 #define STORE(R) STORE_EASY(R)
82 #elif !defined(IS_LITTLE_ENDIAN)
83 #define LOAD(R) LOAD_HARD(R)
84 #define STORE(R) STORE_HARD(R)
85 #else
86 #define LOAD(R) \
87 if ((ptrdiff_t)input & 1) { \
88 LOAD_HARD(R) \
89 } else { \
90 LOAD_EASY(R) \
91 }
92 #define STORE(R) \
93 if ((ptrdiff_t)input & 1) { \
94 STORE_HARD(R) \
95 } else { \
96 STORE_EASY(R) \
97 }
98 #endif
99
100 static const PRUint8 S[256] = {
101 0331, 0170, 0371, 0304, 0031, 0335, 0265, 0355, 0050, 0351, 0375, 0171, 0112, 0240, 0330, 0235,
102 0306, 0176, 0067, 0203, 0053, 0166, 0123, 0216, 0142, 0114, 0144, 0210, 0104, 0213, 0373, 0242,
103 0027, 0232, 0131, 0365, 0207, 0263, 0117, 0023, 0141, 0105, 0155, 0215, 0011, 0201, 0175, 0062,
104 0275, 0217, 0100, 0353, 0206, 0267, 0173, 0013, 0360, 0225, 0041, 0042, 0134, 0153, 0116, 0202,
105 0124, 0326, 0145, 0223, 0316, 0140, 0262, 0034, 0163, 0126, 0300, 0024, 0247, 0214, 0361, 0334,
106 0022, 0165, 0312, 0037, 0073, 0276, 0344, 0321, 0102, 0075, 0324, 0060, 0243, 0074, 0266, 0046,
107 0157, 0277, 0016, 0332, 0106, 0151, 0007, 0127, 0047, 0362, 0035, 0233, 0274, 0224, 0103, 0003,
108 0370, 0021, 0307, 0366, 0220, 0357, 0076, 0347, 0006, 0303, 0325, 0057, 0310, 0146, 0036, 0327,
109 0010, 0350, 0352, 0336, 0200, 0122, 0356, 0367, 0204, 0252, 0162, 0254, 0065, 0115, 0152, 0052,
110 0226, 0032, 0322, 0161, 0132, 0025, 0111, 0164, 0113, 0237, 0320, 0136, 0004, 0030, 0244, 0354,
111 0302, 0340, 0101, 0156, 0017, 0121, 0313, 0314, 0044, 0221, 0257, 0120, 0241, 0364, 0160, 0071,
112 0231, 0174, 0072, 0205, 0043, 0270, 0264, 0172, 0374, 0002, 0066, 0133, 0045, 0125, 0227, 0061,
113 0055, 0135, 0372, 0230, 0343, 0212, 0222, 0256, 0005, 0337, 0051, 0020, 0147, 0154, 0272, 0311,
114 0323, 0000, 0346, 0317, 0341, 0236, 0250, 0054, 0143, 0026, 0001, 0077, 0130, 0342, 0211, 0251,
115 0015, 0070, 0064, 0033, 0253, 0063, 0377, 0260, 0273, 0110, 0014, 0137, 0271, 0261, 0315, 0056,
116 0305, 0363, 0333, 0107, 0345, 0245, 0234, 0167, 0012, 0246, 0040, 0150, 0376, 0177, 0301, 0255
117 };
118
119 RC2Context *
RC2_AllocateContext(void)120 RC2_AllocateContext(void)
121 {
122 return PORT_ZNew(RC2Context);
123 }
124 SECStatus
RC2_InitContext(RC2Context * cx,const unsigned char * key,unsigned int len,const unsigned char * input,int mode,unsigned int efLen8,unsigned int unused)125 RC2_InitContext(RC2Context *cx, const unsigned char *key, unsigned int len,
126 const unsigned char *input, int mode, unsigned int efLen8,
127 unsigned int unused)
128 {
129 PRUint8 *L, *L2;
130 int i;
131 #if !defined(IS_LITTLE_ENDIAN)
132 PRUint16 tmpS;
133 #endif
134 PRUint8 tmpB;
135
136 if (!key || !cx || !len || len > (sizeof cx->B) ||
137 efLen8 > (sizeof cx->B)) {
138 PORT_SetError(SEC_ERROR_INVALID_ARGS);
139 return SECFailure;
140 }
141 if (mode == NSS_RC2) {
142 /* groovy */
143 } else if (mode == NSS_RC2_CBC) {
144 if (!input) {
145 PORT_SetError(SEC_ERROR_INVALID_ARGS);
146 return SECFailure;
147 }
148 } else {
149 PORT_SetError(SEC_ERROR_INVALID_ARGS);
150 return SECFailure;
151 }
152
153 if (mode == NSS_RC2_CBC) {
154 cx->enc = &rc2_EncryptCBC;
155 cx->dec = &rc2_DecryptCBC;
156 LOAD(cx->iv.s);
157 } else {
158 cx->enc = &rc2_EncryptECB;
159 cx->dec = &rc2_DecryptECB;
160 }
161
162 /* Step 0. Copy key into table. */
163 memcpy(cx->B, key, len);
164
165 /* Step 1. Compute all values to the right of the key. */
166 L2 = cx->B;
167 L = L2 + len;
168 tmpB = L[-1];
169 for (i = (sizeof cx->B) - len; i > 0; --i) {
170 *L++ = tmpB = S[(PRUint8)(tmpB + *L2++)];
171 }
172
173 /* step 2. Adjust left most byte of effective key. */
174 i = (sizeof cx->B) - efLen8;
175 L = cx->B + i;
176 *L = tmpB = S[*L]; /* mask is always 0xff */
177
178 /* step 3. Recompute all values to the left of effective key. */
179 L2 = --L + efLen8;
180 while (L >= cx->B) {
181 *L-- = tmpB = S[tmpB ^ *L2--];
182 }
183
184 #if !defined(IS_LITTLE_ENDIAN)
185 for (i = 63; i >= 0; --i) {
186 SWAPK(i); /* candidate for unrolling */
187 }
188 #endif
189 return SECSuccess;
190 }
191
192 /*
193 ** Create a new RC2 context suitable for RC2 encryption/decryption.
194 ** "key" raw key data
195 ** "len" the number of bytes of key data
196 ** "iv" is the CBC initialization vector (if mode is NSS_RC2_CBC)
197 ** "mode" one of NSS_RC2 or NSS_RC2_CBC
198 ** "effectiveKeyLen" in bytes, not bits.
199 **
200 ** When mode is set to NSS_RC2_CBC the RC2 cipher is run in "cipher block
201 ** chaining" mode.
202 */
203 RC2Context *
RC2_CreateContext(const unsigned char * key,unsigned int len,const unsigned char * iv,int mode,unsigned efLen8)204 RC2_CreateContext(const unsigned char *key, unsigned int len,
205 const unsigned char *iv, int mode, unsigned efLen8)
206 {
207 RC2Context *cx = PORT_ZNew(RC2Context);
208 if (cx) {
209 SECStatus rv = RC2_InitContext(cx, key, len, iv, mode, efLen8, 0);
210 if (rv != SECSuccess) {
211 RC2_DestroyContext(cx, PR_TRUE);
212 cx = NULL;
213 }
214 }
215 return cx;
216 }
217
218 /*
219 ** Destroy an RC2 encryption/decryption context.
220 ** "cx" the context
221 ** "freeit" if PR_TRUE then free the object as well as its sub-objects
222 */
223 void
RC2_DestroyContext(RC2Context * cx,PRBool freeit)224 RC2_DestroyContext(RC2Context *cx, PRBool freeit)
225 {
226 if (cx) {
227 memset(cx, 0, sizeof *cx);
228 if (freeit) {
229 PORT_Free(cx);
230 }
231 }
232 }
233
234 #define ROL(x, k) (x << k | x >> (16 - k))
235 #define MIX(j) \
236 R0 = R0 + cx->K[4 * j + 0] + (R3 & R2) + (~R3 & R1); \
237 R0 = ROL(R0, 1); \
238 R1 = R1 + cx->K[4 * j + 1] + (R0 & R3) + (~R0 & R2); \
239 R1 = ROL(R1, 2); \
240 R2 = R2 + cx->K[4 * j + 2] + (R1 & R0) + (~R1 & R3); \
241 R2 = ROL(R2, 3); \
242 R3 = R3 + cx->K[4 * j + 3] + (R2 & R1) + (~R2 & R0); \
243 R3 = ROL(R3, 5)
244 #define MASH \
245 R0 = R0 + cx->K[R3 & 63]; \
246 R1 = R1 + cx->K[R0 & 63]; \
247 R2 = R2 + cx->K[R1 & 63]; \
248 R3 = R3 + cx->K[R2 & 63]
249
250 /* Encrypt one block */
251 static void
rc2_Encrypt1Block(RC2Context * cx,RC2Block * output,RC2Block * input)252 rc2_Encrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
253 {
254 register PRUint16 R0, R1, R2, R3;
255
256 /* step 1. Initialize input. */
257 R0 = input->s[0];
258 R1 = input->s[1];
259 R2 = input->s[2];
260 R3 = input->s[3];
261
262 /* step 2. Expand Key (already done, in context) */
263 /* step 3. j = 0 */
264 /* step 4. Perform 5 mixing rounds. */
265
266 MIX(0);
267 MIX(1);
268 MIX(2);
269 MIX(3);
270 MIX(4);
271
272 /* step 5. Perform 1 mashing round. */
273 MASH;
274
275 /* step 6. Perform 6 mixing rounds. */
276
277 MIX(5);
278 MIX(6);
279 MIX(7);
280 MIX(8);
281 MIX(9);
282 MIX(10);
283
284 /* step 7. Perform 1 mashing round. */
285 MASH;
286
287 /* step 8. Perform 5 mixing rounds. */
288
289 MIX(11);
290 MIX(12);
291 MIX(13);
292 MIX(14);
293 MIX(15);
294
295 /* output results */
296 output->s[0] = R0;
297 output->s[1] = R1;
298 output->s[2] = R2;
299 output->s[3] = R3;
300 }
301
302 #define ROR(x, k) (x >> k | x << (16 - k))
303 #define R_MIX(j) \
304 R3 = ROR(R3, 5); \
305 R3 = R3 - cx->K[4 * j + 3] - (R2 & R1) - (~R2 & R0); \
306 R2 = ROR(R2, 3); \
307 R2 = R2 - cx->K[4 * j + 2] - (R1 & R0) - (~R1 & R3); \
308 R1 = ROR(R1, 2); \
309 R1 = R1 - cx->K[4 * j + 1] - (R0 & R3) - (~R0 & R2); \
310 R0 = ROR(R0, 1); \
311 R0 = R0 - cx->K[4 * j + 0] - (R3 & R2) - (~R3 & R1)
312 #define R_MASH \
313 R3 = R3 - cx->K[R2 & 63]; \
314 R2 = R2 - cx->K[R1 & 63]; \
315 R1 = R1 - cx->K[R0 & 63]; \
316 R0 = R0 - cx->K[R3 & 63]
317
318 /* Encrypt one block */
319 static void
rc2_Decrypt1Block(RC2Context * cx,RC2Block * output,RC2Block * input)320 rc2_Decrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
321 {
322 register PRUint16 R0, R1, R2, R3;
323
324 /* step 1. Initialize input. */
325 R0 = input->s[0];
326 R1 = input->s[1];
327 R2 = input->s[2];
328 R3 = input->s[3];
329
330 /* step 2. Expand Key (already done, in context) */
331 /* step 3. j = 63 */
332 /* step 4. Perform 5 r_mixing rounds. */
333 R_MIX(15);
334 R_MIX(14);
335 R_MIX(13);
336 R_MIX(12);
337 R_MIX(11);
338
339 /* step 5. Perform 1 r_mashing round. */
340 R_MASH;
341
342 /* step 6. Perform 6 r_mixing rounds. */
343 R_MIX(10);
344 R_MIX(9);
345 R_MIX(8);
346 R_MIX(7);
347 R_MIX(6);
348 R_MIX(5);
349
350 /* step 7. Perform 1 r_mashing round. */
351 R_MASH;
352
353 /* step 8. Perform 5 r_mixing rounds. */
354 R_MIX(4);
355 R_MIX(3);
356 R_MIX(2);
357 R_MIX(1);
358 R_MIX(0);
359
360 /* output results */
361 output->s[0] = R0;
362 output->s[1] = R1;
363 output->s[2] = R2;
364 output->s[3] = R3;
365 }
366
367 static SECStatus NO_SANITIZE_ALIGNMENT
rc2_EncryptECB(RC2Context * cx,unsigned char * output,const unsigned char * input,unsigned int inputLen)368 rc2_EncryptECB(RC2Context *cx, unsigned char *output,
369 const unsigned char *input, unsigned int inputLen)
370 {
371 RC2Block iBlock;
372
373 while (inputLen > 0) {
374 LOAD(iBlock.s)
375 rc2_Encrypt1Block(cx, &iBlock, &iBlock);
376 STORE(iBlock.s)
377 output += RC2_BLOCK_SIZE;
378 input += RC2_BLOCK_SIZE;
379 inputLen -= RC2_BLOCK_SIZE;
380 }
381 return SECSuccess;
382 }
383
384 static SECStatus NO_SANITIZE_ALIGNMENT
rc2_DecryptECB(RC2Context * cx,unsigned char * output,const unsigned char * input,unsigned int inputLen)385 rc2_DecryptECB(RC2Context *cx, unsigned char *output,
386 const unsigned char *input, unsigned int inputLen)
387 {
388 RC2Block iBlock;
389
390 while (inputLen > 0) {
391 LOAD(iBlock.s)
392 rc2_Decrypt1Block(cx, &iBlock, &iBlock);
393 STORE(iBlock.s)
394 output += RC2_BLOCK_SIZE;
395 input += RC2_BLOCK_SIZE;
396 inputLen -= RC2_BLOCK_SIZE;
397 }
398 return SECSuccess;
399 }
400
401 static SECStatus NO_SANITIZE_ALIGNMENT
rc2_EncryptCBC(RC2Context * cx,unsigned char * output,const unsigned char * input,unsigned int inputLen)402 rc2_EncryptCBC(RC2Context *cx, unsigned char *output,
403 const unsigned char *input, unsigned int inputLen)
404 {
405 RC2Block iBlock;
406
407 while (inputLen > 0) {
408
409 LOAD(iBlock.s)
410 iBlock.l[0] ^= cx->iv.l[0];
411 iBlock.l[1] ^= cx->iv.l[1];
412 rc2_Encrypt1Block(cx, &iBlock, &iBlock);
413 cx->iv = iBlock;
414 STORE(iBlock.s)
415 output += RC2_BLOCK_SIZE;
416 input += RC2_BLOCK_SIZE;
417 inputLen -= RC2_BLOCK_SIZE;
418 }
419 return SECSuccess;
420 }
421
422 static SECStatus NO_SANITIZE_ALIGNMENT
rc2_DecryptCBC(RC2Context * cx,unsigned char * output,const unsigned char * input,unsigned int inputLen)423 rc2_DecryptCBC(RC2Context *cx, unsigned char *output,
424 const unsigned char *input, unsigned int inputLen)
425 {
426 RC2Block iBlock;
427 RC2Block oBlock;
428
429 while (inputLen > 0) {
430 LOAD(iBlock.s)
431 rc2_Decrypt1Block(cx, &oBlock, &iBlock);
432 oBlock.l[0] ^= cx->iv.l[0];
433 oBlock.l[1] ^= cx->iv.l[1];
434 cx->iv = iBlock;
435 STORE(oBlock.s)
436 output += RC2_BLOCK_SIZE;
437 input += RC2_BLOCK_SIZE;
438 inputLen -= RC2_BLOCK_SIZE;
439 }
440 return SECSuccess;
441 }
442
443 /*
444 ** Perform RC2 encryption.
445 ** "cx" the context
446 ** "output" the output buffer to store the encrypted data.
447 ** "outputLen" how much data is stored in "output". Set by the routine
448 ** after some data is stored in output.
449 ** "maxOutputLen" the maximum amount of data that can ever be
450 ** stored in "output"
451 ** "input" the input data
452 ** "inputLen" the amount of input data
453 */
454 SECStatus
RC2_Encrypt(RC2Context * cx,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)455 RC2_Encrypt(RC2Context *cx, unsigned char *output,
456 unsigned int *outputLen, unsigned int maxOutputLen,
457 const unsigned char *input, unsigned int inputLen)
458 {
459 SECStatus rv = SECSuccess;
460 if (inputLen) {
461 if (inputLen % RC2_BLOCK_SIZE) {
462 PORT_SetError(SEC_ERROR_INPUT_LEN);
463 return SECFailure;
464 }
465 if (maxOutputLen < inputLen) {
466 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
467 return SECFailure;
468 }
469 rv = (*cx->enc)(cx, output, input, inputLen);
470 }
471 if (rv == SECSuccess) {
472 *outputLen = inputLen;
473 }
474 return rv;
475 }
476
477 /*
478 ** Perform RC2 decryption.
479 ** "cx" the context
480 ** "output" the output buffer to store the decrypted data.
481 ** "outputLen" how much data is stored in "output". Set by the routine
482 ** after some data is stored in output.
483 ** "maxOutputLen" the maximum amount of data that can ever be
484 ** stored in "output"
485 ** "input" the input data
486 ** "inputLen" the amount of input data
487 */
488 SECStatus
RC2_Decrypt(RC2Context * cx,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)489 RC2_Decrypt(RC2Context *cx, unsigned char *output,
490 unsigned int *outputLen, unsigned int maxOutputLen,
491 const unsigned char *input, unsigned int inputLen)
492 {
493 SECStatus rv = SECSuccess;
494 if (inputLen) {
495 if (inputLen % RC2_BLOCK_SIZE) {
496 PORT_SetError(SEC_ERROR_INPUT_LEN);
497 return SECFailure;
498 }
499 if (maxOutputLen < inputLen) {
500 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
501 return SECFailure;
502 }
503 rv = (*cx->dec)(cx, output, input, inputLen);
504 }
505 if (rv == SECSuccess) {
506 *outputLen = inputLen;
507 }
508 return rv;
509 }
510