1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 /*
6 * RSA PKCS#1 v2.1 (RFC 3447) operations
7 */
8
9 #ifdef FREEBL_NO_DEPEND
10 #include "stubs.h"
11 #endif
12
13 #include "secerr.h"
14
15 #include "blapi.h"
16 #include "secitem.h"
17 #include "blapii.h"
18
19 #define RSA_BLOCK_MIN_PAD_LEN 8
20 #define RSA_BLOCK_FIRST_OCTET 0x00
21 #define RSA_BLOCK_PRIVATE_PAD_OCTET 0xff
22 #define RSA_BLOCK_AFTER_PAD_OCTET 0x00
23
24 /*
25 * RSA block types
26 *
27 * The values of RSA_BlockPrivate and RSA_BlockPublic are fixed.
28 * The value of RSA_BlockRaw isn't fixed by definition, but we are keeping
29 * the value that NSS has been using in the past.
30 */
31 typedef enum {
32 RSA_BlockPrivate = 1, /* pad for a private-key operation */
33 RSA_BlockPublic = 2, /* pad for a public-key operation */
34 RSA_BlockRaw = 4 /* simply justify the block appropriately */
35 } RSA_BlockType;
36
37 /* Needed for RSA-PSS functions */
38 static const unsigned char eightZeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 };
39
40 /* Constant time comparison of a single byte.
41 * Returns 1 iff a == b, otherwise returns 0.
42 * Note: For ranges of bytes, use constantTimeCompare.
43 */
44 static unsigned char
constantTimeEQ8(unsigned char a,unsigned char b)45 constantTimeEQ8(unsigned char a, unsigned char b)
46 {
47 unsigned char c = ~((a - b) | (b - a));
48 c >>= 7;
49 return c;
50 }
51
52 /* Constant time comparison of a range of bytes.
53 * Returns 1 iff len bytes of a are identical to len bytes of b, otherwise
54 * returns 0.
55 */
56 static unsigned char
constantTimeCompare(const unsigned char * a,const unsigned char * b,unsigned int len)57 constantTimeCompare(const unsigned char *a,
58 const unsigned char *b,
59 unsigned int len)
60 {
61 unsigned char tmp = 0;
62 unsigned int i;
63 for (i = 0; i < len; ++i, ++a, ++b)
64 tmp |= *a ^ *b;
65 return constantTimeEQ8(0x00, tmp);
66 }
67
68 /* Constant time conditional.
69 * Returns a if c is 1, or b if c is 0. The result is undefined if c is
70 * not 0 or 1.
71 */
72 static unsigned int
constantTimeCondition(unsigned int c,unsigned int a,unsigned int b)73 constantTimeCondition(unsigned int c,
74 unsigned int a,
75 unsigned int b)
76 {
77 return (~(c - 1) & a) | ((c - 1) & b);
78 }
79
80 static unsigned int
rsa_modulusLen(SECItem * modulus)81 rsa_modulusLen(SECItem *modulus)
82 {
83 unsigned char byteZero = modulus->data[0];
84 unsigned int modLen = modulus->len - !byteZero;
85 return modLen;
86 }
87
88 static unsigned int
rsa_modulusBits(SECItem * modulus)89 rsa_modulusBits(SECItem *modulus)
90 {
91 unsigned char byteZero = modulus->data[0];
92 unsigned int numBits = (modulus->len - 1) * 8;
93
94 if (byteZero == 0) {
95 numBits -= 8;
96 byteZero = modulus->data[1];
97 }
98
99 while (byteZero > 0) {
100 numBits++;
101 byteZero >>= 1;
102 }
103
104 return numBits;
105 }
106
107 /*
108 * Format one block of data for public/private key encryption using
109 * the rules defined in PKCS #1.
110 */
111 static unsigned char *
rsa_FormatOneBlock(unsigned modulusLen,RSA_BlockType blockType,SECItem * data)112 rsa_FormatOneBlock(unsigned modulusLen,
113 RSA_BlockType blockType,
114 SECItem *data)
115 {
116 unsigned char *block;
117 unsigned char *bp;
118 unsigned int padLen;
119 int i, j;
120 SECStatus rv;
121
122 block = (unsigned char *)PORT_Alloc(modulusLen);
123 if (block == NULL)
124 return NULL;
125
126 bp = block;
127
128 /*
129 * All RSA blocks start with two octets:
130 * 0x00 || BlockType
131 */
132 *bp++ = RSA_BLOCK_FIRST_OCTET;
133 *bp++ = (unsigned char)blockType;
134
135 switch (blockType) {
136
137 /*
138 * Blocks intended for private-key operation.
139 */
140 case RSA_BlockPrivate: /* preferred method */
141 /*
142 * 0x00 || BT || Pad || 0x00 || ActualData
143 * 1 1 padLen 1 data->len
144 * padLen must be at least RSA_BLOCK_MIN_PAD_LEN (8) bytes.
145 * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
146 */
147 padLen = modulusLen - data->len - 3;
148 PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
149 if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
150 PORT_Free(block);
151 return NULL;
152 }
153 PORT_Memset(bp, RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
154 bp += padLen;
155 *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
156 PORT_Memcpy(bp, data->data, data->len);
157 break;
158
159 /*
160 * Blocks intended for public-key operation.
161 */
162 case RSA_BlockPublic:
163 /*
164 * 0x00 || BT || Pad || 0x00 || ActualData
165 * 1 1 padLen 1 data->len
166 * Pad is 8 or more non-zero random bytes.
167 *
168 * Build the block left to right.
169 * Fill the entire block from Pad to the end with random bytes.
170 * Use the bytes after Pad as a supply of extra random bytes from
171 * which to find replacements for the zero bytes in Pad.
172 * If we need more than that, refill the bytes after Pad with
173 * new random bytes as necessary.
174 */
175
176 padLen = modulusLen - (data->len + 3);
177 PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
178 if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
179 PORT_Free(block);
180 return NULL;
181 }
182 j = modulusLen - 2;
183 rv = RNG_GenerateGlobalRandomBytes(bp, j);
184 if (rv == SECSuccess) {
185 for (i = 0; i < padLen;) {
186 unsigned char repl;
187 /* Pad with non-zero random data. */
188 if (bp[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
189 ++i;
190 continue;
191 }
192 if (j <= padLen) {
193 rv = RNG_GenerateGlobalRandomBytes(bp + padLen,
194 modulusLen - (2 + padLen));
195 if (rv != SECSuccess)
196 break;
197 j = modulusLen - 2;
198 }
199 do {
200 repl = bp[--j];
201 } while (repl == RSA_BLOCK_AFTER_PAD_OCTET && j > padLen);
202 if (repl != RSA_BLOCK_AFTER_PAD_OCTET) {
203 bp[i++] = repl;
204 }
205 }
206 }
207 if (rv != SECSuccess) {
208 PORT_Free(block);
209 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
210 return NULL;
211 }
212 bp += padLen;
213 *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
214 PORT_Memcpy(bp, data->data, data->len);
215 break;
216
217 default:
218 PORT_Assert(0);
219 PORT_Free(block);
220 return NULL;
221 }
222
223 return block;
224 }
225
226 static SECStatus
rsa_FormatBlock(SECItem * result,unsigned modulusLen,RSA_BlockType blockType,SECItem * data)227 rsa_FormatBlock(SECItem *result,
228 unsigned modulusLen,
229 RSA_BlockType blockType,
230 SECItem *data)
231 {
232 switch (blockType) {
233 case RSA_BlockPrivate:
234 case RSA_BlockPublic:
235 /*
236 * 0x00 || BT || Pad || 0x00 || ActualData
237 *
238 * The "3" below is the first octet + the second octet + the 0x00
239 * octet that always comes just before the ActualData.
240 */
241 if (data->len > (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))) {
242 return SECFailure;
243 }
244 result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
245 if (result->data == NULL) {
246 result->len = 0;
247 return SECFailure;
248 }
249 result->len = modulusLen;
250
251 break;
252
253 case RSA_BlockRaw:
254 /*
255 * Pad || ActualData
256 * Pad is zeros. The application is responsible for recovering
257 * the actual data.
258 */
259 if (data->len > modulusLen) {
260 return SECFailure;
261 }
262 result->data = (unsigned char *)PORT_ZAlloc(modulusLen);
263 result->len = modulusLen;
264 PORT_Memcpy(result->data + (modulusLen - data->len),
265 data->data, data->len);
266 break;
267
268 default:
269 PORT_Assert(0);
270 result->data = NULL;
271 result->len = 0;
272 return SECFailure;
273 }
274
275 return SECSuccess;
276 }
277
278 /*
279 * Mask generation function MGF1 as defined in PKCS #1 v2.1 / RFC 3447.
280 */
281 static SECStatus
MGF1(HASH_HashType hashAlg,unsigned char * mask,unsigned int maskLen,const unsigned char * mgfSeed,unsigned int mgfSeedLen)282 MGF1(HASH_HashType hashAlg,
283 unsigned char *mask,
284 unsigned int maskLen,
285 const unsigned char *mgfSeed,
286 unsigned int mgfSeedLen)
287 {
288 unsigned int digestLen;
289 PRUint32 counter;
290 PRUint32 rounds;
291 unsigned char *tempHash;
292 unsigned char *temp;
293 const SECHashObject *hash;
294 void *hashContext;
295 unsigned char C[4];
296 SECStatus rv = SECSuccess;
297
298 hash = HASH_GetRawHashObject(hashAlg);
299 if (hash == NULL) {
300 return SECFailure;
301 }
302
303 hashContext = (*hash->create)();
304 rounds = (maskLen + hash->length - 1) / hash->length;
305 for (counter = 0; counter < rounds; counter++) {
306 C[0] = (unsigned char)((counter >> 24) & 0xff);
307 C[1] = (unsigned char)((counter >> 16) & 0xff);
308 C[2] = (unsigned char)((counter >> 8) & 0xff);
309 C[3] = (unsigned char)(counter & 0xff);
310
311 /* This could be optimized when the clone functions in
312 * rawhash.c are implemented. */
313 (*hash->begin)(hashContext);
314 (*hash->update)(hashContext, mgfSeed, mgfSeedLen);
315 (*hash->update)(hashContext, C, sizeof C);
316
317 tempHash = mask + counter * hash->length;
318 if (counter != (rounds - 1)) {
319 (*hash->end)(hashContext, tempHash, &digestLen, hash->length);
320 } else { /* we're in the last round and need to cut the hash */
321 temp = (unsigned char *)PORT_Alloc(hash->length);
322 if (!temp) {
323 rv = SECFailure;
324 goto done;
325 }
326 (*hash->end)(hashContext, temp, &digestLen, hash->length);
327 PORT_Memcpy(tempHash, temp, maskLen - counter * hash->length);
328 PORT_Free(temp);
329 }
330 }
331
332 done:
333 (*hash->destroy)(hashContext, PR_TRUE);
334 return rv;
335 }
336
337 /* XXX Doesn't set error code */
338 SECStatus
RSA_SignRaw(RSAPrivateKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * data,unsigned int dataLen)339 RSA_SignRaw(RSAPrivateKey *key,
340 unsigned char *output,
341 unsigned int *outputLen,
342 unsigned int maxOutputLen,
343 const unsigned char *data,
344 unsigned int dataLen)
345 {
346 SECStatus rv = SECSuccess;
347 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
348 SECItem formatted;
349 SECItem unformatted;
350
351 if (maxOutputLen < modulusLen)
352 return SECFailure;
353
354 unformatted.len = dataLen;
355 unformatted.data = (unsigned char *)data;
356 formatted.data = NULL;
357 rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
358 if (rv != SECSuccess)
359 goto done;
360
361 rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
362 *outputLen = modulusLen;
363
364 done:
365 if (formatted.data != NULL)
366 PORT_ZFree(formatted.data, modulusLen);
367 return rv;
368 }
369
370 /* XXX Doesn't set error code */
371 SECStatus
RSA_CheckSignRaw(RSAPublicKey * key,const unsigned char * sig,unsigned int sigLen,const unsigned char * hash,unsigned int hashLen)372 RSA_CheckSignRaw(RSAPublicKey *key,
373 const unsigned char *sig,
374 unsigned int sigLen,
375 const unsigned char *hash,
376 unsigned int hashLen)
377 {
378 SECStatus rv;
379 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
380 unsigned char *buffer;
381
382 if (sigLen != modulusLen)
383 goto failure;
384 if (hashLen > modulusLen)
385 goto failure;
386
387 buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
388 if (!buffer)
389 goto failure;
390
391 rv = RSA_PublicKeyOp(key, buffer, sig);
392 if (rv != SECSuccess)
393 goto loser;
394
395 /*
396 * make sure we get the same results
397 */
398 /* XXX(rsleevi): Constant time */
399 /* NOTE: should we verify the leading zeros? */
400 if (PORT_Memcmp(buffer + (modulusLen - hashLen), hash, hashLen) != 0)
401 goto loser;
402
403 PORT_Free(buffer);
404 return SECSuccess;
405
406 loser:
407 PORT_Free(buffer);
408 failure:
409 return SECFailure;
410 }
411
412 /* XXX Doesn't set error code */
413 SECStatus
RSA_CheckSignRecoverRaw(RSAPublicKey * key,unsigned char * data,unsigned int * dataLen,unsigned int maxDataLen,const unsigned char * sig,unsigned int sigLen)414 RSA_CheckSignRecoverRaw(RSAPublicKey *key,
415 unsigned char *data,
416 unsigned int *dataLen,
417 unsigned int maxDataLen,
418 const unsigned char *sig,
419 unsigned int sigLen)
420 {
421 SECStatus rv;
422 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
423
424 if (sigLen != modulusLen)
425 goto failure;
426 if (maxDataLen < modulusLen)
427 goto failure;
428
429 rv = RSA_PublicKeyOp(key, data, sig);
430 if (rv != SECSuccess)
431 goto failure;
432
433 *dataLen = modulusLen;
434 return SECSuccess;
435
436 failure:
437 return SECFailure;
438 }
439
440 /* XXX Doesn't set error code */
441 SECStatus
RSA_EncryptRaw(RSAPublicKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)442 RSA_EncryptRaw(RSAPublicKey *key,
443 unsigned char *output,
444 unsigned int *outputLen,
445 unsigned int maxOutputLen,
446 const unsigned char *input,
447 unsigned int inputLen)
448 {
449 SECStatus rv;
450 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
451 SECItem formatted;
452 SECItem unformatted;
453
454 formatted.data = NULL;
455 if (maxOutputLen < modulusLen)
456 goto failure;
457
458 unformatted.len = inputLen;
459 unformatted.data = (unsigned char *)input;
460 formatted.data = NULL;
461 rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
462 if (rv != SECSuccess)
463 goto failure;
464
465 rv = RSA_PublicKeyOp(key, output, formatted.data);
466 if (rv != SECSuccess)
467 goto failure;
468
469 PORT_ZFree(formatted.data, modulusLen);
470 *outputLen = modulusLen;
471 return SECSuccess;
472
473 failure:
474 if (formatted.data != NULL)
475 PORT_ZFree(formatted.data, modulusLen);
476 return SECFailure;
477 }
478
479 /* XXX Doesn't set error code */
480 SECStatus
RSA_DecryptRaw(RSAPrivateKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)481 RSA_DecryptRaw(RSAPrivateKey *key,
482 unsigned char *output,
483 unsigned int *outputLen,
484 unsigned int maxOutputLen,
485 const unsigned char *input,
486 unsigned int inputLen)
487 {
488 SECStatus rv;
489 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
490
491 if (modulusLen > maxOutputLen)
492 goto failure;
493 if (inputLen != modulusLen)
494 goto failure;
495
496 rv = RSA_PrivateKeyOp(key, output, input);
497 if (rv != SECSuccess)
498 goto failure;
499
500 *outputLen = modulusLen;
501 return SECSuccess;
502
503 failure:
504 return SECFailure;
505 }
506
507 /*
508 * Decodes an EME-OAEP encoded block, validating the encoding in constant
509 * time.
510 * Described in RFC 3447, section 7.1.2.
511 * input contains the encoded block, after decryption.
512 * label is the optional value L that was associated with the message.
513 * On success, the original message and message length will be stored in
514 * output and outputLen.
515 */
516 static SECStatus
eme_oaep_decode(unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * label,unsigned int labelLen)517 eme_oaep_decode(unsigned char *output,
518 unsigned int *outputLen,
519 unsigned int maxOutputLen,
520 const unsigned char *input,
521 unsigned int inputLen,
522 HASH_HashType hashAlg,
523 HASH_HashType maskHashAlg,
524 const unsigned char *label,
525 unsigned int labelLen)
526 {
527 const SECHashObject *hash;
528 void *hashContext;
529 SECStatus rv = SECFailure;
530 unsigned char labelHash[HASH_LENGTH_MAX];
531 unsigned int i;
532 unsigned int maskLen;
533 unsigned int paddingOffset;
534 unsigned char *mask = NULL;
535 unsigned char *tmpOutput = NULL;
536 unsigned char isGood;
537 unsigned char foundPaddingEnd;
538
539 hash = HASH_GetRawHashObject(hashAlg);
540
541 /* 1.c */
542 if (inputLen < (hash->length * 2) + 2) {
543 PORT_SetError(SEC_ERROR_INPUT_LEN);
544 return SECFailure;
545 }
546
547 /* Step 3.a - Generate lHash */
548 hashContext = (*hash->create)();
549 if (hashContext == NULL) {
550 PORT_SetError(SEC_ERROR_NO_MEMORY);
551 return SECFailure;
552 }
553 (*hash->begin)(hashContext);
554 if (labelLen > 0)
555 (*hash->update)(hashContext, label, labelLen);
556 (*hash->end)(hashContext, labelHash, &i, sizeof(labelHash));
557 (*hash->destroy)(hashContext, PR_TRUE);
558
559 tmpOutput = (unsigned char *)PORT_Alloc(inputLen);
560 if (tmpOutput == NULL) {
561 PORT_SetError(SEC_ERROR_NO_MEMORY);
562 goto done;
563 }
564
565 maskLen = inputLen - hash->length - 1;
566 mask = (unsigned char *)PORT_Alloc(maskLen);
567 if (mask == NULL) {
568 PORT_SetError(SEC_ERROR_NO_MEMORY);
569 goto done;
570 }
571
572 PORT_Memcpy(tmpOutput, input, inputLen);
573
574 /* 3.c - Generate seedMask */
575 MGF1(maskHashAlg, mask, hash->length, &tmpOutput[1 + hash->length],
576 inputLen - hash->length - 1);
577 /* 3.d - Unmask seed */
578 for (i = 0; i < hash->length; ++i)
579 tmpOutput[1 + i] ^= mask[i];
580
581 /* 3.e - Generate dbMask */
582 MGF1(maskHashAlg, mask, maskLen, &tmpOutput[1], hash->length);
583 /* 3.f - Unmask DB */
584 for (i = 0; i < maskLen; ++i)
585 tmpOutput[1 + hash->length + i] ^= mask[i];
586
587 /* 3.g - Compare Y, lHash, and PS in constant time
588 * Warning: This code is timing dependent and must not disclose which of
589 * these were invalid.
590 */
591 paddingOffset = 0;
592 isGood = 1;
593 foundPaddingEnd = 0;
594
595 /* Compare Y */
596 isGood &= constantTimeEQ8(0x00, tmpOutput[0]);
597
598 /* Compare lHash and lHash' */
599 isGood &= constantTimeCompare(&labelHash[0],
600 &tmpOutput[1 + hash->length],
601 hash->length);
602
603 /* Compare that the padding is zero or more zero octets, followed by a
604 * 0x01 octet */
605 for (i = 1 + (hash->length * 2); i < inputLen; ++i) {
606 unsigned char isZero = constantTimeEQ8(0x00, tmpOutput[i]);
607 unsigned char isOne = constantTimeEQ8(0x01, tmpOutput[i]);
608 /* non-constant time equivalent:
609 * if (tmpOutput[i] == 0x01 && !foundPaddingEnd)
610 * paddingOffset = i;
611 */
612 paddingOffset = constantTimeCondition(isOne & ~foundPaddingEnd, i,
613 paddingOffset);
614 /* non-constant time equivalent:
615 * if (tmpOutput[i] == 0x01)
616 * foundPaddingEnd = true;
617 *
618 * Note: This may yield false positives, as it will be set whenever
619 * a 0x01 byte is encountered. If there was bad padding (eg:
620 * 0x03 0x02 0x01), foundPaddingEnd will still be set to true, and
621 * paddingOffset will still be set to 2.
622 */
623 foundPaddingEnd = constantTimeCondition(isOne, 1, foundPaddingEnd);
624 /* non-constant time equivalent:
625 * if (tmpOutput[i] != 0x00 && tmpOutput[i] != 0x01 &&
626 * !foundPaddingEnd) {
627 * isGood = false;
628 * }
629 *
630 * Note: This may yield false positives, as a message (and padding)
631 * that is entirely zeros will result in isGood still being true. Thus
632 * it's necessary to check foundPaddingEnd is positive below.
633 */
634 isGood = constantTimeCondition(~foundPaddingEnd & ~isZero, 0, isGood);
635 }
636
637 /* While both isGood and foundPaddingEnd may have false positives, they
638 * cannot BOTH have false positives. If both are not true, then an invalid
639 * message was received. Note, this comparison must still be done in constant
640 * time so as not to leak either condition.
641 */
642 if (!(isGood & foundPaddingEnd)) {
643 PORT_SetError(SEC_ERROR_BAD_DATA);
644 goto done;
645 }
646
647 /* End timing dependent code */
648
649 ++paddingOffset; /* Skip the 0x01 following the end of PS */
650
651 *outputLen = inputLen - paddingOffset;
652 if (*outputLen > maxOutputLen) {
653 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
654 goto done;
655 }
656
657 if (*outputLen)
658 PORT_Memcpy(output, &tmpOutput[paddingOffset], *outputLen);
659 rv = SECSuccess;
660
661 done:
662 if (mask)
663 PORT_ZFree(mask, maskLen);
664 if (tmpOutput)
665 PORT_ZFree(tmpOutput, inputLen);
666 return rv;
667 }
668
669 /*
670 * Generate an EME-OAEP encoded block for encryption
671 * Described in RFC 3447, section 7.1.1
672 * We use input instead of M for the message to be encrypted
673 * label is the optional value L to be associated with the message.
674 */
675 static SECStatus
eme_oaep_encode(unsigned char * em,unsigned int emLen,const unsigned char * input,unsigned int inputLen,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * label,unsigned int labelLen,const unsigned char * seed,unsigned int seedLen)676 eme_oaep_encode(unsigned char *em,
677 unsigned int emLen,
678 const unsigned char *input,
679 unsigned int inputLen,
680 HASH_HashType hashAlg,
681 HASH_HashType maskHashAlg,
682 const unsigned char *label,
683 unsigned int labelLen,
684 const unsigned char *seed,
685 unsigned int seedLen)
686 {
687 const SECHashObject *hash;
688 void *hashContext;
689 SECStatus rv;
690 unsigned char *mask;
691 unsigned int reservedLen;
692 unsigned int dbMaskLen;
693 unsigned int i;
694
695 hash = HASH_GetRawHashObject(hashAlg);
696 PORT_Assert(seed == NULL || seedLen == hash->length);
697
698 /* Step 1.b */
699 reservedLen = (2 * hash->length) + 2;
700 if (emLen < reservedLen || inputLen > (emLen - reservedLen)) {
701 PORT_SetError(SEC_ERROR_INPUT_LEN);
702 return SECFailure;
703 }
704
705 /*
706 * From RFC 3447, Section 7.1
707 * +----------+---------+-------+
708 * DB = | lHash | PS | M |
709 * +----------+---------+-------+
710 * |
711 * +----------+ V
712 * | seed |--> MGF ---> xor
713 * +----------+ |
714 * | |
715 * +--+ V |
716 * |00| xor <----- MGF <-----|
717 * +--+ | |
718 * | | |
719 * V V V
720 * +--+----------+----------------------------+
721 * EM = |00|maskedSeed| maskedDB |
722 * +--+----------+----------------------------+
723 *
724 * We use mask to hold the result of the MGF functions, and all other
725 * values are generated in their final resting place.
726 */
727 *em = 0x00;
728
729 /* Step 2.a - Generate lHash */
730 hashContext = (*hash->create)();
731 if (hashContext == NULL) {
732 PORT_SetError(SEC_ERROR_NO_MEMORY);
733 return SECFailure;
734 }
735 (*hash->begin)(hashContext);
736 if (labelLen > 0)
737 (*hash->update)(hashContext, label, labelLen);
738 (*hash->end)(hashContext, &em[1 + hash->length], &i, hash->length);
739 (*hash->destroy)(hashContext, PR_TRUE);
740
741 /* Step 2.b - Generate PS */
742 if (emLen - reservedLen - inputLen > 0) {
743 PORT_Memset(em + 1 + (hash->length * 2), 0x00,
744 emLen - reservedLen - inputLen);
745 }
746
747 /* Step 2.c. - Generate DB
748 * DB = lHash || PS || 0x01 || M
749 * Note that PS and lHash have already been placed into em at their
750 * appropriate offsets. This just copies M into place
751 */
752 em[emLen - inputLen - 1] = 0x01;
753 if (inputLen)
754 PORT_Memcpy(em + emLen - inputLen, input, inputLen);
755
756 if (seed == NULL) {
757 /* Step 2.d - Generate seed */
758 rv = RNG_GenerateGlobalRandomBytes(em + 1, hash->length);
759 if (rv != SECSuccess) {
760 return rv;
761 }
762 } else {
763 /* For Known Answer Tests, copy the supplied seed. */
764 PORT_Memcpy(em + 1, seed, seedLen);
765 }
766
767 /* Step 2.e - Generate dbMask*/
768 dbMaskLen = emLen - hash->length - 1;
769 mask = (unsigned char *)PORT_Alloc(dbMaskLen);
770 if (mask == NULL) {
771 PORT_SetError(SEC_ERROR_NO_MEMORY);
772 return SECFailure;
773 }
774 MGF1(maskHashAlg, mask, dbMaskLen, em + 1, hash->length);
775 /* Step 2.f - Compute maskedDB*/
776 for (i = 0; i < dbMaskLen; ++i)
777 em[1 + hash->length + i] ^= mask[i];
778
779 /* Step 2.g - Generate seedMask */
780 MGF1(maskHashAlg, mask, hash->length, &em[1 + hash->length], dbMaskLen);
781 /* Step 2.h - Compute maskedSeed */
782 for (i = 0; i < hash->length; ++i)
783 em[1 + i] ^= mask[i];
784
785 PORT_ZFree(mask, dbMaskLen);
786 return SECSuccess;
787 }
788
789 SECStatus
RSA_EncryptOAEP(RSAPublicKey * key,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * label,unsigned int labelLen,const unsigned char * seed,unsigned int seedLen,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)790 RSA_EncryptOAEP(RSAPublicKey *key,
791 HASH_HashType hashAlg,
792 HASH_HashType maskHashAlg,
793 const unsigned char *label,
794 unsigned int labelLen,
795 const unsigned char *seed,
796 unsigned int seedLen,
797 unsigned char *output,
798 unsigned int *outputLen,
799 unsigned int maxOutputLen,
800 const unsigned char *input,
801 unsigned int inputLen)
802 {
803 SECStatus rv = SECFailure;
804 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
805 unsigned char *oaepEncoded = NULL;
806
807 if (maxOutputLen < modulusLen) {
808 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
809 return SECFailure;
810 }
811
812 if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
813 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
814 return SECFailure;
815 }
816
817 if ((labelLen == 0 && label != NULL) ||
818 (labelLen > 0 && label == NULL)) {
819 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
820 return SECFailure;
821 }
822
823 oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
824 if (oaepEncoded == NULL) {
825 PORT_SetError(SEC_ERROR_NO_MEMORY);
826 return SECFailure;
827 }
828 rv = eme_oaep_encode(oaepEncoded, modulusLen, input, inputLen,
829 hashAlg, maskHashAlg, label, labelLen, seed, seedLen);
830 if (rv != SECSuccess)
831 goto done;
832
833 rv = RSA_PublicKeyOp(key, output, oaepEncoded);
834 if (rv != SECSuccess)
835 goto done;
836 *outputLen = modulusLen;
837
838 done:
839 PORT_Free(oaepEncoded);
840 return rv;
841 }
842
843 SECStatus
RSA_DecryptOAEP(RSAPrivateKey * key,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * label,unsigned int labelLen,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)844 RSA_DecryptOAEP(RSAPrivateKey *key,
845 HASH_HashType hashAlg,
846 HASH_HashType maskHashAlg,
847 const unsigned char *label,
848 unsigned int labelLen,
849 unsigned char *output,
850 unsigned int *outputLen,
851 unsigned int maxOutputLen,
852 const unsigned char *input,
853 unsigned int inputLen)
854 {
855 SECStatus rv = SECFailure;
856 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
857 unsigned char *oaepEncoded = NULL;
858
859 if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
860 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
861 return SECFailure;
862 }
863
864 if (inputLen != modulusLen) {
865 PORT_SetError(SEC_ERROR_INPUT_LEN);
866 return SECFailure;
867 }
868
869 if ((labelLen == 0 && label != NULL) ||
870 (labelLen > 0 && label == NULL)) {
871 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
872 return SECFailure;
873 }
874
875 oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
876 if (oaepEncoded == NULL) {
877 PORT_SetError(SEC_ERROR_NO_MEMORY);
878 return SECFailure;
879 }
880
881 rv = RSA_PrivateKeyOpDoubleChecked(key, oaepEncoded, input);
882 if (rv != SECSuccess) {
883 goto done;
884 }
885 rv = eme_oaep_decode(output, outputLen, maxOutputLen, oaepEncoded,
886 modulusLen, hashAlg, maskHashAlg, label,
887 labelLen);
888
889 done:
890 if (oaepEncoded)
891 PORT_ZFree(oaepEncoded, modulusLen);
892 return rv;
893 }
894
895 /* XXX Doesn't set error code */
896 SECStatus
RSA_EncryptBlock(RSAPublicKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)897 RSA_EncryptBlock(RSAPublicKey *key,
898 unsigned char *output,
899 unsigned int *outputLen,
900 unsigned int maxOutputLen,
901 const unsigned char *input,
902 unsigned int inputLen)
903 {
904 SECStatus rv;
905 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
906 SECItem formatted;
907 SECItem unformatted;
908
909 formatted.data = NULL;
910 if (maxOutputLen < modulusLen)
911 goto failure;
912
913 unformatted.len = inputLen;
914 unformatted.data = (unsigned char *)input;
915 formatted.data = NULL;
916 rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPublic,
917 &unformatted);
918 if (rv != SECSuccess)
919 goto failure;
920
921 rv = RSA_PublicKeyOp(key, output, formatted.data);
922 if (rv != SECSuccess)
923 goto failure;
924
925 PORT_ZFree(formatted.data, modulusLen);
926 *outputLen = modulusLen;
927 return SECSuccess;
928
929 failure:
930 if (formatted.data != NULL)
931 PORT_ZFree(formatted.data, modulusLen);
932 return SECFailure;
933 }
934
935 /* XXX Doesn't set error code */
936 SECStatus
RSA_DecryptBlock(RSAPrivateKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)937 RSA_DecryptBlock(RSAPrivateKey *key,
938 unsigned char *output,
939 unsigned int *outputLen,
940 unsigned int maxOutputLen,
941 const unsigned char *input,
942 unsigned int inputLen)
943 {
944 PRInt8 rv;
945 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
946 unsigned int i;
947 unsigned char *buffer = NULL;
948 unsigned int outLen = 0;
949 unsigned int copyOutLen = modulusLen - 11;
950
951 if (inputLen != modulusLen || modulusLen < 10) {
952 return SECFailure;
953 }
954
955 if (copyOutLen > maxOutputLen) {
956 copyOutLen = maxOutputLen;
957 }
958
959 // Allocate enough space to decrypt + copyOutLen to allow copying outLen later.
960 buffer = PORT_ZAlloc(modulusLen + 1 + copyOutLen);
961 if (!buffer) {
962 return SECFailure;
963 }
964
965 // rv is 0 if everything is going well and 1 if an error occurs.
966 rv = RSA_PrivateKeyOp(key, buffer, input) != SECSuccess;
967 rv |= (buffer[0] != RSA_BLOCK_FIRST_OCTET) |
968 (buffer[1] != (unsigned char)RSA_BlockPublic);
969
970 // There have to be at least 8 bytes of padding.
971 for (i = 2; i < 10; i++) {
972 rv |= buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET;
973 }
974
975 for (i = 10; i < modulusLen; i++) {
976 unsigned int newLen = modulusLen - i - 1;
977 unsigned int c = (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) & (outLen == 0);
978 outLen = constantTimeCondition(c, newLen, outLen);
979 }
980 rv |= outLen == 0;
981 rv |= outLen > maxOutputLen;
982
983 // Note that output is set even if SECFailure is returned.
984 PORT_Memcpy(output, buffer + modulusLen - outLen, copyOutLen);
985 *outputLen = constantTimeCondition(outLen > maxOutputLen, maxOutputLen,
986 outLen);
987
988 PORT_Free(buffer);
989
990 for (i = 1; i < sizeof(rv) * 8; i <<= 1) {
991 rv |= rv << i;
992 }
993 return (SECStatus)rv;
994 }
995
996 /*
997 * Encode a RSA-PSS signature.
998 * Described in RFC 3447, section 9.1.1.
999 * We use mHash instead of M as input.
1000 * emBits from the RFC is just modBits - 1, see section 8.1.1.
1001 * We only support MGF1 as the MGF.
1002 */
1003 static SECStatus
emsa_pss_encode(unsigned char * em,unsigned int emLen,unsigned int emBits,const unsigned char * mHash,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * salt,unsigned int saltLen)1004 emsa_pss_encode(unsigned char *em,
1005 unsigned int emLen,
1006 unsigned int emBits,
1007 const unsigned char *mHash,
1008 HASH_HashType hashAlg,
1009 HASH_HashType maskHashAlg,
1010 const unsigned char *salt,
1011 unsigned int saltLen)
1012 {
1013 const SECHashObject *hash;
1014 void *hash_context;
1015 unsigned char *dbMask;
1016 unsigned int dbMaskLen;
1017 unsigned int i;
1018 SECStatus rv;
1019
1020 hash = HASH_GetRawHashObject(hashAlg);
1021 dbMaskLen = emLen - hash->length - 1;
1022
1023 /* Step 3 */
1024 if (emLen < hash->length + saltLen + 2) {
1025 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
1026 return SECFailure;
1027 }
1028
1029 /* Step 4 */
1030 if (salt == NULL) {
1031 rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - saltLen], saltLen);
1032 if (rv != SECSuccess) {
1033 return rv;
1034 }
1035 } else {
1036 PORT_Memcpy(&em[dbMaskLen - saltLen], salt, saltLen);
1037 }
1038
1039 /* Step 5 + 6 */
1040 /* Compute H and store it at its final location &em[dbMaskLen]. */
1041 hash_context = (*hash->create)();
1042 if (hash_context == NULL) {
1043 PORT_SetError(SEC_ERROR_NO_MEMORY);
1044 return SECFailure;
1045 }
1046 (*hash->begin)(hash_context);
1047 (*hash->update)(hash_context, eightZeros, 8);
1048 (*hash->update)(hash_context, mHash, hash->length);
1049 (*hash->update)(hash_context, &em[dbMaskLen - saltLen], saltLen);
1050 (*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
1051 (*hash->destroy)(hash_context, PR_TRUE);
1052
1053 /* Step 7 + 8 */
1054 PORT_Memset(em, 0, dbMaskLen - saltLen - 1);
1055 em[dbMaskLen - saltLen - 1] = 0x01;
1056
1057 /* Step 9 */
1058 dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
1059 if (dbMask == NULL) {
1060 PORT_SetError(SEC_ERROR_NO_MEMORY);
1061 return SECFailure;
1062 }
1063 MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);
1064
1065 /* Step 10 */
1066 for (i = 0; i < dbMaskLen; i++)
1067 em[i] ^= dbMask[i];
1068 PORT_Free(dbMask);
1069
1070 /* Step 11 */
1071 em[0] &= 0xff >> (8 * emLen - emBits);
1072
1073 /* Step 12 */
1074 em[emLen - 1] = 0xbc;
1075
1076 return SECSuccess;
1077 }
1078
1079 /*
1080 * Verify a RSA-PSS signature.
1081 * Described in RFC 3447, section 9.1.2.
1082 * We use mHash instead of M as input.
1083 * emBits from the RFC is just modBits - 1, see section 8.1.2.
1084 * We only support MGF1 as the MGF.
1085 */
1086 static SECStatus
emsa_pss_verify(const unsigned char * mHash,const unsigned char * em,unsigned int emLen,unsigned int emBits,HASH_HashType hashAlg,HASH_HashType maskHashAlg,unsigned int saltLen)1087 emsa_pss_verify(const unsigned char *mHash,
1088 const unsigned char *em,
1089 unsigned int emLen,
1090 unsigned int emBits,
1091 HASH_HashType hashAlg,
1092 HASH_HashType maskHashAlg,
1093 unsigned int saltLen)
1094 {
1095 const SECHashObject *hash;
1096 void *hash_context;
1097 unsigned char *db;
1098 unsigned char *H_; /* H' from the RFC */
1099 unsigned int i;
1100 unsigned int dbMaskLen;
1101 unsigned int zeroBits;
1102 SECStatus rv;
1103
1104 hash = HASH_GetRawHashObject(hashAlg);
1105 dbMaskLen = emLen - hash->length - 1;
1106
1107 /* Step 3 + 4 */
1108 if ((emLen < (hash->length + saltLen + 2)) ||
1109 (em[emLen - 1] != 0xbc)) {
1110 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1111 return SECFailure;
1112 }
1113
1114 /* Step 6 */
1115 zeroBits = 8 * emLen - emBits;
1116 if (em[0] >> (8 - zeroBits)) {
1117 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1118 return SECFailure;
1119 }
1120
1121 /* Step 7 */
1122 db = (unsigned char *)PORT_Alloc(dbMaskLen);
1123 if (db == NULL) {
1124 PORT_SetError(SEC_ERROR_NO_MEMORY);
1125 return SECFailure;
1126 }
1127 /* &em[dbMaskLen] points to H, used as mgfSeed */
1128 MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);
1129
1130 /* Step 8 */
1131 for (i = 0; i < dbMaskLen; i++) {
1132 db[i] ^= em[i];
1133 }
1134
1135 /* Step 9 */
1136 db[0] &= 0xff >> zeroBits;
1137
1138 /* Step 10 */
1139 for (i = 0; i < (dbMaskLen - saltLen - 1); i++) {
1140 if (db[i] != 0) {
1141 PORT_Free(db);
1142 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1143 return SECFailure;
1144 }
1145 }
1146 if (db[dbMaskLen - saltLen - 1] != 0x01) {
1147 PORT_Free(db);
1148 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1149 return SECFailure;
1150 }
1151
1152 /* Step 12 + 13 */
1153 H_ = (unsigned char *)PORT_Alloc(hash->length);
1154 if (H_ == NULL) {
1155 PORT_Free(db);
1156 PORT_SetError(SEC_ERROR_NO_MEMORY);
1157 return SECFailure;
1158 }
1159 hash_context = (*hash->create)();
1160 if (hash_context == NULL) {
1161 PORT_Free(db);
1162 PORT_Free(H_);
1163 PORT_SetError(SEC_ERROR_NO_MEMORY);
1164 return SECFailure;
1165 }
1166 (*hash->begin)(hash_context);
1167 (*hash->update)(hash_context, eightZeros, 8);
1168 (*hash->update)(hash_context, mHash, hash->length);
1169 (*hash->update)(hash_context, &db[dbMaskLen - saltLen], saltLen);
1170 (*hash->end)(hash_context, H_, &i, hash->length);
1171 (*hash->destroy)(hash_context, PR_TRUE);
1172
1173 PORT_Free(db);
1174
1175 /* Step 14 */
1176 if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
1177 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1178 rv = SECFailure;
1179 } else {
1180 rv = SECSuccess;
1181 }
1182
1183 PORT_Free(H_);
1184 return rv;
1185 }
1186
1187 SECStatus
RSA_SignPSS(RSAPrivateKey * key,HASH_HashType hashAlg,HASH_HashType maskHashAlg,const unsigned char * salt,unsigned int saltLength,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)1188 RSA_SignPSS(RSAPrivateKey *key,
1189 HASH_HashType hashAlg,
1190 HASH_HashType maskHashAlg,
1191 const unsigned char *salt,
1192 unsigned int saltLength,
1193 unsigned char *output,
1194 unsigned int *outputLen,
1195 unsigned int maxOutputLen,
1196 const unsigned char *input,
1197 unsigned int inputLen)
1198 {
1199 SECStatus rv = SECSuccess;
1200 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
1201 unsigned int modulusBits = rsa_modulusBits(&key->modulus);
1202 unsigned int emLen = modulusLen;
1203 unsigned char *pssEncoded, *em;
1204
1205 if (maxOutputLen < modulusLen) {
1206 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
1207 return SECFailure;
1208 }
1209
1210 if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
1211 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
1212 return SECFailure;
1213 }
1214
1215 pssEncoded = em = (unsigned char *)PORT_Alloc(modulusLen);
1216 if (pssEncoded == NULL) {
1217 PORT_SetError(SEC_ERROR_NO_MEMORY);
1218 return SECFailure;
1219 }
1220
1221 /* len(em) == ceil((modulusBits - 1) / 8). */
1222 if (modulusBits % 8 == 1) {
1223 em[0] = 0;
1224 emLen--;
1225 em++;
1226 }
1227 rv = emsa_pss_encode(em, emLen, modulusBits - 1, input, hashAlg,
1228 maskHashAlg, salt, saltLength);
1229 if (rv != SECSuccess)
1230 goto done;
1231
1232 // This sets error codes upon failure.
1233 rv = RSA_PrivateKeyOpDoubleChecked(key, output, pssEncoded);
1234 *outputLen = modulusLen;
1235
1236 done:
1237 PORT_Free(pssEncoded);
1238 return rv;
1239 }
1240
1241 SECStatus
RSA_CheckSignPSS(RSAPublicKey * key,HASH_HashType hashAlg,HASH_HashType maskHashAlg,unsigned int saltLength,const unsigned char * sig,unsigned int sigLen,const unsigned char * hash,unsigned int hashLen)1242 RSA_CheckSignPSS(RSAPublicKey *key,
1243 HASH_HashType hashAlg,
1244 HASH_HashType maskHashAlg,
1245 unsigned int saltLength,
1246 const unsigned char *sig,
1247 unsigned int sigLen,
1248 const unsigned char *hash,
1249 unsigned int hashLen)
1250 {
1251 SECStatus rv;
1252 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
1253 unsigned int modulusBits = rsa_modulusBits(&key->modulus);
1254 unsigned int emLen = modulusLen;
1255 unsigned char *buffer, *em;
1256
1257 if (sigLen != modulusLen) {
1258 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1259 return SECFailure;
1260 }
1261
1262 if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
1263 PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
1264 return SECFailure;
1265 }
1266
1267 buffer = em = (unsigned char *)PORT_Alloc(modulusLen);
1268 if (!buffer) {
1269 PORT_SetError(SEC_ERROR_NO_MEMORY);
1270 return SECFailure;
1271 }
1272
1273 rv = RSA_PublicKeyOp(key, buffer, sig);
1274 if (rv != SECSuccess) {
1275 PORT_Free(buffer);
1276 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1277 return SECFailure;
1278 }
1279
1280 /* len(em) == ceil((modulusBits - 1) / 8). */
1281 if (modulusBits % 8 == 1) {
1282 emLen--;
1283 em++;
1284 }
1285 rv = emsa_pss_verify(hash, em, emLen, modulusBits - 1, hashAlg,
1286 maskHashAlg, saltLength);
1287
1288 PORT_Free(buffer);
1289 return rv;
1290 }
1291
1292 SECStatus
RSA_Sign(RSAPrivateKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * input,unsigned int inputLen)1293 RSA_Sign(RSAPrivateKey *key,
1294 unsigned char *output,
1295 unsigned int *outputLen,
1296 unsigned int maxOutputLen,
1297 const unsigned char *input,
1298 unsigned int inputLen)
1299 {
1300 SECStatus rv = SECFailure;
1301 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
1302 SECItem formatted = { siBuffer, NULL, 0 };
1303 SECItem unformatted = { siBuffer, (unsigned char *)input, inputLen };
1304
1305 if (maxOutputLen < modulusLen) {
1306 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
1307 goto done;
1308 }
1309
1310 rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPrivate,
1311 &unformatted);
1312 if (rv != SECSuccess) {
1313 PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
1314 goto done;
1315 }
1316
1317 // This sets error codes upon failure.
1318 rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
1319 *outputLen = modulusLen;
1320
1321 done:
1322 if (formatted.data != NULL) {
1323 PORT_ZFree(formatted.data, modulusLen);
1324 }
1325 return rv;
1326 }
1327
1328 SECStatus
RSA_CheckSign(RSAPublicKey * key,const unsigned char * sig,unsigned int sigLen,const unsigned char * data,unsigned int dataLen)1329 RSA_CheckSign(RSAPublicKey *key,
1330 const unsigned char *sig,
1331 unsigned int sigLen,
1332 const unsigned char *data,
1333 unsigned int dataLen)
1334 {
1335 SECStatus rv = SECFailure;
1336 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
1337 unsigned int i;
1338 unsigned char *buffer = NULL;
1339
1340 if (sigLen != modulusLen) {
1341 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1342 goto done;
1343 }
1344
1345 /*
1346 * 0x00 || BT || Pad || 0x00 || ActualData
1347 *
1348 * The "3" below is the first octet + the second octet + the 0x00
1349 * octet that always comes just before the ActualData.
1350 */
1351 if (dataLen > modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)) {
1352 PORT_SetError(SEC_ERROR_BAD_DATA);
1353 goto done;
1354 }
1355
1356 buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
1357 if (!buffer) {
1358 PORT_SetError(SEC_ERROR_NO_MEMORY);
1359 goto done;
1360 }
1361
1362 if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
1363 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1364 goto done;
1365 }
1366
1367 /*
1368 * check the padding that was used
1369 */
1370 if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
1371 buffer[1] != (unsigned char)RSA_BlockPrivate) {
1372 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1373 goto done;
1374 }
1375 for (i = 2; i < modulusLen - dataLen - 1; i++) {
1376 if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
1377 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1378 goto done;
1379 }
1380 }
1381 if (buffer[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
1382 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1383 goto done;
1384 }
1385
1386 /*
1387 * make sure we get the same results
1388 */
1389 if (PORT_Memcmp(buffer + modulusLen - dataLen, data, dataLen) == 0) {
1390 rv = SECSuccess;
1391 }
1392
1393 done:
1394 if (buffer) {
1395 PORT_Free(buffer);
1396 }
1397 return rv;
1398 }
1399
1400 SECStatus
RSA_CheckSignRecover(RSAPublicKey * key,unsigned char * output,unsigned int * outputLen,unsigned int maxOutputLen,const unsigned char * sig,unsigned int sigLen)1401 RSA_CheckSignRecover(RSAPublicKey *key,
1402 unsigned char *output,
1403 unsigned int *outputLen,
1404 unsigned int maxOutputLen,
1405 const unsigned char *sig,
1406 unsigned int sigLen)
1407 {
1408 SECStatus rv = SECFailure;
1409 unsigned int modulusLen = rsa_modulusLen(&key->modulus);
1410 unsigned int i;
1411 unsigned char *buffer = NULL;
1412
1413 if (sigLen != modulusLen) {
1414 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1415 goto done;
1416 }
1417
1418 buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
1419 if (!buffer) {
1420 PORT_SetError(SEC_ERROR_NO_MEMORY);
1421 goto done;
1422 }
1423
1424 if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
1425 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1426 goto done;
1427 }
1428
1429 *outputLen = 0;
1430
1431 /*
1432 * check the padding that was used
1433 */
1434 if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
1435 buffer[1] != (unsigned char)RSA_BlockPrivate) {
1436 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1437 goto done;
1438 }
1439 for (i = 2; i < modulusLen; i++) {
1440 if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
1441 *outputLen = modulusLen - i - 1;
1442 break;
1443 }
1444 if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
1445 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1446 goto done;
1447 }
1448 }
1449 if (*outputLen == 0) {
1450 PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
1451 goto done;
1452 }
1453 if (*outputLen > maxOutputLen) {
1454 PORT_SetError(SEC_ERROR_OUTPUT_LEN);
1455 goto done;
1456 }
1457
1458 PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
1459 rv = SECSuccess;
1460
1461 done:
1462 if (buffer) {
1463 PORT_Free(buffer);
1464 }
1465 return rv;
1466 }
1467