1# Test hashlib module
2#
3# $Id$
4#
5#  Copyright (C) 2005-2010   Gregory P. Smith (greg@krypto.org)
6#  Licensed to PSF under a Contributor Agreement.
7#
8
9import array
10from binascii import unhexlify
11import hashlib
12import importlib
13import itertools
14import os
15import sys
16import sysconfig
17import threading
18import unittest
19import warnings
20from test import support
21from test.support import _4G, bigmemtest
22from test.support.import_helper import import_fresh_module
23from test.support import threading_helper
24from test.support import warnings_helper
25from http.client import HTTPException
26
27# Were we compiled --with-pydebug or with #define Py_DEBUG?
28COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
29
30# default builtin hash module
31default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'}
32# --with-builtin-hashlib-hashes override
33builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
34if builtin_hashes is None:
35    builtin_hashes = default_builtin_hashes
36else:
37    builtin_hashes = {
38        m.strip() for m in builtin_hashes.strip('"').lower().split(",")
39    }
40
41# hashlib with and without OpenSSL backend for PBKDF2
42# only import builtin_hashlib when all builtin hashes are available.
43# Otherwise import prints noise on stderr
44openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
45if builtin_hashes == default_builtin_hashes:
46    builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
47else:
48    builtin_hashlib = None
49
50try:
51    from _hashlib import HASH, HASHXOF, openssl_md_meth_names
52except ImportError:
53    HASH = None
54    HASHXOF = None
55    openssl_md_meth_names = frozenset()
56
57try:
58    import _blake2
59except ImportError:
60    _blake2 = None
61
62requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')
63
64
65def hexstr(s):
66    assert isinstance(s, bytes), repr(s)
67    h = "0123456789abcdef"
68    r = ''
69    for i in s:
70        r += h[(i >> 4) & 0xF] + h[i & 0xF]
71    return r
72
73
74URL = "http://www.pythontest.net/hashlib/{}.txt"
75
76def read_vectors(hash_name):
77    url = URL.format(hash_name)
78    try:
79        testdata = support.open_urlresource(url, encoding="utf-8")
80    except (OSError, HTTPException):
81        raise unittest.SkipTest("Could not retrieve {}".format(url))
82    with testdata:
83        for line in testdata:
84            line = line.strip()
85            if line.startswith('#') or not line:
86                continue
87            parts = line.split(',')
88            parts[0] = bytes.fromhex(parts[0])
89            yield parts
90
91
92class HashLibTestCase(unittest.TestCase):
93    supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
94                             'sha224', 'SHA224', 'sha256', 'SHA256',
95                             'sha384', 'SHA384', 'sha512', 'SHA512',
96                             'blake2b', 'blake2s',
97                             'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
98                             'shake_128', 'shake_256')
99
100    shakes = {'shake_128', 'shake_256'}
101
102    # Issue #14693: fallback modules are always compiled under POSIX
103    _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG
104
105    def _conditional_import_module(self, module_name):
106        """Import a module and return a reference to it or None on failure."""
107        try:
108            return importlib.import_module(module_name)
109        except ModuleNotFoundError as error:
110            if self._warn_on_extension_import and module_name in builtin_hashes:
111                warnings.warn('Did a C extension fail to compile? %s' % error)
112        return None
113
114    def __init__(self, *args, **kwargs):
115        algorithms = set()
116        for algorithm in self.supported_hash_names:
117            algorithms.add(algorithm.lower())
118
119        _blake2 = self._conditional_import_module('_blake2')
120        if _blake2:
121            algorithms.update({'blake2b', 'blake2s'})
122
123        self.constructors_to_test = {}
124        for algorithm in algorithms:
125            self.constructors_to_test[algorithm] = set()
126
127        # For each algorithm, test the direct constructor and the use
128        # of hashlib.new given the algorithm name.
129        for algorithm, constructors in self.constructors_to_test.items():
130            constructors.add(getattr(hashlib, algorithm))
131            def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
132                if data is None:
133                    return hashlib.new(_alg, **kwargs)
134                return hashlib.new(_alg, data, **kwargs)
135            constructors.add(_test_algorithm_via_hashlib_new)
136
137        _hashlib = self._conditional_import_module('_hashlib')
138        self._hashlib = _hashlib
139        if _hashlib:
140            # These two algorithms should always be present when this module
141            # is compiled.  If not, something was compiled wrong.
142            self.assertTrue(hasattr(_hashlib, 'openssl_md5'))
143            self.assertTrue(hasattr(_hashlib, 'openssl_sha1'))
144            for algorithm, constructors in self.constructors_to_test.items():
145                constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
146                if constructor:
147                    try:
148                        constructor()
149                    except ValueError:
150                        # default constructor blocked by crypto policy
151                        pass
152                    else:
153                        constructors.add(constructor)
154
155        def add_builtin_constructor(name):
156            constructor = getattr(hashlib, "__get_builtin_constructor")(name)
157            self.constructors_to_test[name].add(constructor)
158
159        _md5 = self._conditional_import_module('_md5')
160        if _md5:
161            add_builtin_constructor('md5')
162        _sha1 = self._conditional_import_module('_sha1')
163        if _sha1:
164            add_builtin_constructor('sha1')
165        _sha256 = self._conditional_import_module('_sha256')
166        if _sha256:
167            add_builtin_constructor('sha224')
168            add_builtin_constructor('sha256')
169        _sha512 = self._conditional_import_module('_sha512')
170        if _sha512:
171            add_builtin_constructor('sha384')
172            add_builtin_constructor('sha512')
173        if _blake2:
174            add_builtin_constructor('blake2s')
175            add_builtin_constructor('blake2b')
176
177        _sha3 = self._conditional_import_module('_sha3')
178        if _sha3:
179            add_builtin_constructor('sha3_224')
180            add_builtin_constructor('sha3_256')
181            add_builtin_constructor('sha3_384')
182            add_builtin_constructor('sha3_512')
183            add_builtin_constructor('shake_128')
184            add_builtin_constructor('shake_256')
185
186        super(HashLibTestCase, self).__init__(*args, **kwargs)
187
188    @property
189    def hash_constructors(self):
190        constructors = self.constructors_to_test.values()
191        return itertools.chain.from_iterable(constructors)
192
193    @property
194    def is_fips_mode(self):
195        if hasattr(self._hashlib, "get_fips_mode"):
196            return self._hashlib.get_fips_mode()
197        else:
198            return None
199
200    def test_hash_array(self):
201        a = array.array("b", range(10))
202        for cons in self.hash_constructors:
203            c = cons(a, usedforsecurity=False)
204            if c.name in self.shakes:
205                c.hexdigest(16)
206            else:
207                c.hexdigest()
208
209    def test_algorithms_guaranteed(self):
210        self.assertEqual(hashlib.algorithms_guaranteed,
211            set(_algo for _algo in self.supported_hash_names
212                  if _algo.islower()))
213
214    def test_algorithms_available(self):
215        self.assertTrue(set(hashlib.algorithms_guaranteed).
216                            issubset(hashlib.algorithms_available))
217
218    def test_usedforsecurity_true(self):
219        hashlib.new("sha256", usedforsecurity=True)
220        if self.is_fips_mode:
221            self.skipTest("skip in FIPS mode")
222        for cons in self.hash_constructors:
223            cons(usedforsecurity=True)
224            cons(b'', usedforsecurity=True)
225        hashlib.new("md5", usedforsecurity=True)
226        hashlib.md5(usedforsecurity=True)
227        if self._hashlib is not None:
228            self._hashlib.new("md5", usedforsecurity=True)
229            self._hashlib.openssl_md5(usedforsecurity=True)
230
231    def test_usedforsecurity_false(self):
232        hashlib.new("sha256", usedforsecurity=False)
233        for cons in self.hash_constructors:
234            cons(usedforsecurity=False)
235            cons(b'', usedforsecurity=False)
236        hashlib.new("md5", usedforsecurity=False)
237        hashlib.md5(usedforsecurity=False)
238        if self._hashlib is not None:
239            self._hashlib.new("md5", usedforsecurity=False)
240            self._hashlib.openssl_md5(usedforsecurity=False)
241
242    def test_unknown_hash(self):
243        self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam')
244        self.assertRaises(TypeError, hashlib.new, 1)
245
246    def test_new_upper_to_lower(self):
247        self.assertEqual(hashlib.new("SHA256").name, "sha256")
248
249    def test_get_builtin_constructor(self):
250        get_builtin_constructor = getattr(hashlib,
251                                          '__get_builtin_constructor')
252        builtin_constructor_cache = getattr(hashlib,
253                                            '__builtin_constructor_cache')
254        self.assertRaises(ValueError, get_builtin_constructor, 'test')
255        try:
256            import _md5
257        except ImportError:
258            self.skipTest("_md5 module not available")
259        # This forces an ImportError for "import _md5" statements
260        sys.modules['_md5'] = None
261        # clear the cache
262        builtin_constructor_cache.clear()
263        try:
264            self.assertRaises(ValueError, get_builtin_constructor, 'md5')
265        finally:
266            if '_md5' in locals():
267                sys.modules['_md5'] = _md5
268            else:
269                del sys.modules['_md5']
270        self.assertRaises(TypeError, get_builtin_constructor, 3)
271        constructor = get_builtin_constructor('md5')
272        self.assertIs(constructor, _md5.md5)
273        self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5'])
274
275    def test_hexdigest(self):
276        for cons in self.hash_constructors:
277            h = cons(usedforsecurity=False)
278            if h.name in self.shakes:
279                self.assertIsInstance(h.digest(16), bytes)
280                self.assertEqual(hexstr(h.digest(16)), h.hexdigest(16))
281            else:
282                self.assertIsInstance(h.digest(), bytes)
283                self.assertEqual(hexstr(h.digest()), h.hexdigest())
284
285    def test_digest_length_overflow(self):
286        # See issue #34922
287        large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)
288        for cons in self.hash_constructors:
289            h = cons(usedforsecurity=False)
290            if h.name not in self.shakes:
291                continue
292            if HASH is not None and isinstance(h, HASH):
293                # _hashopenssl's take a size_t
294                continue
295            for digest in h.digest, h.hexdigest:
296                self.assertRaises(ValueError, digest, -10)
297                for length in large_sizes:
298                    with self.assertRaises((ValueError, OverflowError)):
299                        digest(length)
300
301    def test_name_attribute(self):
302        for cons in self.hash_constructors:
303            h = cons(usedforsecurity=False)
304            self.assertIsInstance(h.name, str)
305            if h.name in self.supported_hash_names:
306                self.assertIn(h.name, self.supported_hash_names)
307            else:
308                self.assertNotIn(h.name, self.supported_hash_names)
309            self.assertEqual(
310                h.name,
311                hashlib.new(h.name, usedforsecurity=False).name
312            )
313
314    def test_large_update(self):
315        aas = b'a' * 128
316        bees = b'b' * 127
317        cees = b'c' * 126
318        dees = b'd' * 2048 #  HASHLIB_GIL_MINSIZE
319
320        for cons in self.hash_constructors:
321            m1 = cons(usedforsecurity=False)
322            m1.update(aas)
323            m1.update(bees)
324            m1.update(cees)
325            m1.update(dees)
326            if m1.name in self.shakes:
327                args = (16,)
328            else:
329                args = ()
330
331            m2 = cons(usedforsecurity=False)
332            m2.update(aas + bees + cees + dees)
333            self.assertEqual(m1.digest(*args), m2.digest(*args))
334
335            m3 = cons(aas + bees + cees + dees, usedforsecurity=False)
336            self.assertEqual(m1.digest(*args), m3.digest(*args))
337
338            # verify copy() doesn't touch original
339            m4 = cons(aas + bees + cees, usedforsecurity=False)
340            m4_digest = m4.digest(*args)
341            m4_copy = m4.copy()
342            m4_copy.update(dees)
343            self.assertEqual(m1.digest(*args), m4_copy.digest(*args))
344            self.assertEqual(m4.digest(*args), m4_digest)
345
346    def check(self, name, data, hexdigest, shake=False, **kwargs):
347        length = len(hexdigest)//2
348        hexdigest = hexdigest.lower()
349        constructors = self.constructors_to_test[name]
350        # 2 is for hashlib.name(...) and hashlib.new(name, ...)
351        self.assertGreaterEqual(len(constructors), 2)
352        for hash_object_constructor in constructors:
353            m = hash_object_constructor(data, **kwargs)
354            computed = m.hexdigest() if not shake else m.hexdigest(length)
355            self.assertEqual(
356                    computed, hexdigest,
357                    "Hash algorithm %s constructed using %s returned hexdigest"
358                    " %r for %d byte input data that should have hashed to %r."
359                    % (name, hash_object_constructor,
360                       computed, len(data), hexdigest))
361            computed = m.digest() if not shake else m.digest(length)
362            digest = bytes.fromhex(hexdigest)
363            self.assertEqual(computed, digest)
364            if not shake:
365                self.assertEqual(len(digest), m.digest_size)
366
367    def check_no_unicode(self, algorithm_name):
368        # Unicode objects are not allowed as input.
369        constructors = self.constructors_to_test[algorithm_name]
370        for hash_object_constructor in constructors:
371            self.assertRaises(TypeError, hash_object_constructor, 'spam')
372
373    def test_no_unicode(self):
374        self.check_no_unicode('md5')
375        self.check_no_unicode('sha1')
376        self.check_no_unicode('sha224')
377        self.check_no_unicode('sha256')
378        self.check_no_unicode('sha384')
379        self.check_no_unicode('sha512')
380
381    @requires_blake2
382    def test_no_unicode_blake2(self):
383        self.check_no_unicode('blake2b')
384        self.check_no_unicode('blake2s')
385
386    def test_no_unicode_sha3(self):
387        self.check_no_unicode('sha3_224')
388        self.check_no_unicode('sha3_256')
389        self.check_no_unicode('sha3_384')
390        self.check_no_unicode('sha3_512')
391        self.check_no_unicode('shake_128')
392        self.check_no_unicode('shake_256')
393
394    def check_blocksize_name(self, name, block_size=0, digest_size=0,
395                             digest_length=None):
396        constructors = self.constructors_to_test[name]
397        for hash_object_constructor in constructors:
398            m = hash_object_constructor(usedforsecurity=False)
399            self.assertEqual(m.block_size, block_size)
400            self.assertEqual(m.digest_size, digest_size)
401            if digest_length:
402                self.assertEqual(len(m.digest(digest_length)),
403                                 digest_length)
404                self.assertEqual(len(m.hexdigest(digest_length)),
405                                 2*digest_length)
406            else:
407                self.assertEqual(len(m.digest()), digest_size)
408                self.assertEqual(len(m.hexdigest()), 2*digest_size)
409            self.assertEqual(m.name, name)
410            # split for sha3_512 / _sha3.sha3 object
411            self.assertIn(name.split("_")[0], repr(m))
412
413    def test_blocksize_name(self):
414        self.check_blocksize_name('md5', 64, 16)
415        self.check_blocksize_name('sha1', 64, 20)
416        self.check_blocksize_name('sha224', 64, 28)
417        self.check_blocksize_name('sha256', 64, 32)
418        self.check_blocksize_name('sha384', 128, 48)
419        self.check_blocksize_name('sha512', 128, 64)
420
421    def test_blocksize_name_sha3(self):
422        self.check_blocksize_name('sha3_224', 144, 28)
423        self.check_blocksize_name('sha3_256', 136, 32)
424        self.check_blocksize_name('sha3_384', 104, 48)
425        self.check_blocksize_name('sha3_512', 72, 64)
426        self.check_blocksize_name('shake_128', 168, 0, 32)
427        self.check_blocksize_name('shake_256', 136, 0, 64)
428
429    def check_sha3(self, name, capacity, rate, suffix):
430        constructors = self.constructors_to_test[name]
431        for hash_object_constructor in constructors:
432            m = hash_object_constructor()
433            if HASH is not None and isinstance(m, HASH):
434                # _hashopenssl's variant does not have extra SHA3 attributes
435                continue
436            self.assertEqual(capacity + rate, 1600)
437            self.assertEqual(m._capacity_bits, capacity)
438            self.assertEqual(m._rate_bits, rate)
439            self.assertEqual(m._suffix, suffix)
440
441    def test_extra_sha3(self):
442        self.check_sha3('sha3_224', 448, 1152, b'\x06')
443        self.check_sha3('sha3_256', 512, 1088, b'\x06')
444        self.check_sha3('sha3_384', 768, 832, b'\x06')
445        self.check_sha3('sha3_512', 1024, 576, b'\x06')
446        self.check_sha3('shake_128', 256, 1344, b'\x1f')
447        self.check_sha3('shake_256', 512, 1088, b'\x1f')
448
449    @requires_blake2
450    def test_blocksize_name_blake2(self):
451        self.check_blocksize_name('blake2b', 128, 64)
452        self.check_blocksize_name('blake2s', 64, 32)
453
454    def test_case_md5_0(self):
455        self.check(
456            'md5', b'', 'd41d8cd98f00b204e9800998ecf8427e',
457            usedforsecurity=False
458        )
459
460    def test_case_md5_1(self):
461        self.check(
462            'md5', b'abc', '900150983cd24fb0d6963f7d28e17f72',
463            usedforsecurity=False
464        )
465
466    def test_case_md5_2(self):
467        self.check(
468            'md5',
469            b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
470            'd174ab98d277d9f5a5611c2c9f419d9f',
471            usedforsecurity=False
472        )
473
474    @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems')
475    @bigmemtest(size=_4G + 5, memuse=1, dry_run=False)
476    def test_case_md5_huge(self, size):
477        self.check('md5', b'A'*size, 'c9af2dff37468ce5dfee8f2cfc0a9c6d')
478
479    @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems')
480    @bigmemtest(size=_4G - 1, memuse=1, dry_run=False)
481    def test_case_md5_uintmax(self, size):
482        self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
483
484    # use the three examples from Federal Information Processing Standards
485    # Publication 180-1, Secure Hash Standard,  1995 April 17
486    # http://www.itl.nist.gov/div897/pubs/fip180-1.htm
487
488    def test_case_sha1_0(self):
489        self.check('sha1', b"",
490                   "da39a3ee5e6b4b0d3255bfef95601890afd80709")
491
492    def test_case_sha1_1(self):
493        self.check('sha1', b"abc",
494                   "a9993e364706816aba3e25717850c26c9cd0d89d")
495
496    def test_case_sha1_2(self):
497        self.check('sha1',
498                   b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
499                   "84983e441c3bd26ebaae4aa1f95129e5e54670f1")
500
501    def test_case_sha1_3(self):
502        self.check('sha1', b"a" * 1000000,
503                   "34aa973cd4c4daa4f61eeb2bdbad27316534016f")
504
505
506    # use the examples from Federal Information Processing Standards
507    # Publication 180-2, Secure Hash Standard,  2002 August 1
508    # http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf
509
510    def test_case_sha224_0(self):
511        self.check('sha224', b"",
512          "d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f")
513
514    def test_case_sha224_1(self):
515        self.check('sha224', b"abc",
516          "23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7")
517
518    def test_case_sha224_2(self):
519        self.check('sha224',
520          b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
521          "75388b16512776cc5dba5da1fd890150b0c6455cb4f58b1952522525")
522
523    def test_case_sha224_3(self):
524        self.check('sha224', b"a" * 1000000,
525          "20794655980c91d8bbb4c1ea97618a4bf03f42581948b2ee4ee7ad67")
526
527
528    def test_case_sha256_0(self):
529        self.check('sha256', b"",
530          "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
531
532    def test_case_sha256_1(self):
533        self.check('sha256', b"abc",
534          "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
535
536    def test_case_sha256_2(self):
537        self.check('sha256',
538          b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
539          "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")
540
541    def test_case_sha256_3(self):
542        self.check('sha256', b"a" * 1000000,
543          "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0")
544
545
546    def test_case_sha384_0(self):
547        self.check('sha384', b"",
548          "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da"+
549          "274edebfe76f65fbd51ad2f14898b95b")
550
551    def test_case_sha384_1(self):
552        self.check('sha384', b"abc",
553          "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed"+
554          "8086072ba1e7cc2358baeca134c825a7")
555
556    def test_case_sha384_2(self):
557        self.check('sha384',
558                   b"abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn"+
559                   b"hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu",
560          "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712"+
561          "fcc7c71a557e2db966c3e9fa91746039")
562
563    def test_case_sha384_3(self):
564        self.check('sha384', b"a" * 1000000,
565          "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b"+
566          "07b8b3dc38ecc4ebae97ddd87f3d8985")
567
568
569    def test_case_sha512_0(self):
570        self.check('sha512', b"",
571          "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce"+
572          "47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e")
573
574    def test_case_sha512_1(self):
575        self.check('sha512', b"abc",
576          "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a"+
577          "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
578
579    def test_case_sha512_2(self):
580        self.check('sha512',
581                   b"abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn"+
582                   b"hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu",
583          "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018"+
584          "501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909")
585
586    def test_case_sha512_3(self):
587        self.check('sha512', b"a" * 1000000,
588          "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+
589          "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b")
590
591    def check_blake2(self, constructor, salt_size, person_size, key_size,
592                     digest_size, max_offset):
593        self.assertEqual(constructor.SALT_SIZE, salt_size)
594        for i in range(salt_size + 1):
595            constructor(salt=b'a' * i)
596        salt = b'a' * (salt_size + 1)
597        self.assertRaises(ValueError, constructor, salt=salt)
598
599        self.assertEqual(constructor.PERSON_SIZE, person_size)
600        for i in range(person_size+1):
601            constructor(person=b'a' * i)
602        person = b'a' * (person_size + 1)
603        self.assertRaises(ValueError, constructor, person=person)
604
605        self.assertEqual(constructor.MAX_DIGEST_SIZE, digest_size)
606        for i in range(1, digest_size + 1):
607            constructor(digest_size=i)
608        self.assertRaises(ValueError, constructor, digest_size=-1)
609        self.assertRaises(ValueError, constructor, digest_size=0)
610        self.assertRaises(ValueError, constructor, digest_size=digest_size+1)
611
612        self.assertEqual(constructor.MAX_KEY_SIZE, key_size)
613        for i in range(key_size+1):
614            constructor(key=b'a' * i)
615        key = b'a' * (key_size + 1)
616        self.assertRaises(ValueError, constructor, key=key)
617        self.assertEqual(constructor().hexdigest(),
618                         constructor(key=b'').hexdigest())
619
620        for i in range(0, 256):
621            constructor(fanout=i)
622        self.assertRaises(ValueError, constructor, fanout=-1)
623        self.assertRaises(ValueError, constructor, fanout=256)
624
625        for i in range(1, 256):
626            constructor(depth=i)
627        self.assertRaises(ValueError, constructor, depth=-1)
628        self.assertRaises(ValueError, constructor, depth=0)
629        self.assertRaises(ValueError, constructor, depth=256)
630
631        for i in range(0, 256):
632            constructor(node_depth=i)
633        self.assertRaises(ValueError, constructor, node_depth=-1)
634        self.assertRaises(ValueError, constructor, node_depth=256)
635
636        for i in range(0, digest_size + 1):
637            constructor(inner_size=i)
638        self.assertRaises(ValueError, constructor, inner_size=-1)
639        self.assertRaises(ValueError, constructor, inner_size=digest_size+1)
640
641        constructor(leaf_size=0)
642        constructor(leaf_size=(1<<32)-1)
643        self.assertRaises(ValueError, constructor, leaf_size=-1)
644        self.assertRaises(OverflowError, constructor, leaf_size=1<<32)
645
646        constructor(node_offset=0)
647        constructor(node_offset=max_offset)
648        self.assertRaises(ValueError, constructor, node_offset=-1)
649        self.assertRaises(OverflowError, constructor, node_offset=max_offset+1)
650
651        self.assertRaises(TypeError, constructor, data=b'')
652        self.assertRaises(TypeError, constructor, string=b'')
653        self.assertRaises(TypeError, constructor, '')
654
655        constructor(
656            b'',
657            key=b'',
658            salt=b'',
659            person=b'',
660            digest_size=17,
661            fanout=1,
662            depth=1,
663            leaf_size=256,
664            node_offset=512,
665            node_depth=1,
666            inner_size=7,
667            last_node=True
668        )
669
670    def blake2_rfc7693(self, constructor, md_len, in_len):
671        def selftest_seq(length, seed):
672            mask = (1<<32)-1
673            a = (0xDEAD4BAD * seed) & mask
674            b = 1
675            out = bytearray(length)
676            for i in range(length):
677                t = (a + b) & mask
678                a, b = b, t
679                out[i] = (t >> 24) & 0xFF
680            return out
681        outer = constructor(digest_size=32)
682        for outlen in md_len:
683            for inlen in in_len:
684                indata = selftest_seq(inlen, inlen)
685                key = selftest_seq(outlen, outlen)
686                unkeyed = constructor(indata, digest_size=outlen)
687                outer.update(unkeyed.digest())
688                keyed = constructor(indata, key=key, digest_size=outlen)
689                outer.update(keyed.digest())
690        return outer.hexdigest()
691
692    @requires_blake2
693    def test_blake2b(self):
694        self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1)
695        b2b_md_len = [20, 32, 48, 64]
696        b2b_in_len = [0, 3, 128, 129, 255, 1024]
697        self.assertEqual(
698            self.blake2_rfc7693(hashlib.blake2b, b2b_md_len, b2b_in_len),
699            "c23a7800d98123bd10f506c61e29da5603d763b8bbad2e737f5e765a7bccd475")
700
701    @requires_blake2
702    def test_case_blake2b_0(self):
703        self.check('blake2b', b"",
704          "786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419"+
705          "d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce")
706
707    @requires_blake2
708    def test_case_blake2b_1(self):
709        self.check('blake2b', b"abc",
710          "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+
711          "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923")
712
713    @requires_blake2
714    def test_case_blake2b_all_parameters(self):
715        # This checks that all the parameters work in general, and also that
716        # parameter byte order doesn't get confused on big endian platforms.
717        self.check('blake2b', b"foo",
718          "920568b0c5873b2f0ab67bedb6cf1b2b",
719          digest_size=16,
720          key=b"bar",
721          salt=b"baz",
722          person=b"bing",
723          fanout=2,
724          depth=3,
725          leaf_size=4,
726          node_offset=5,
727          node_depth=6,
728          inner_size=7,
729          last_node=True)
730
731    @requires_blake2
732    def test_blake2b_vectors(self):
733        for msg, key, md in read_vectors('blake2b'):
734            key = bytes.fromhex(key)
735            self.check('blake2b', msg, md, key=key)
736
737    @requires_blake2
738    def test_blake2s(self):
739        self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1)
740        b2s_md_len = [16, 20, 28, 32]
741        b2s_in_len = [0, 3, 64, 65, 255, 1024]
742        self.assertEqual(
743            self.blake2_rfc7693(hashlib.blake2s, b2s_md_len, b2s_in_len),
744            "6a411f08ce25adcdfb02aba641451cec53c598b24f4fc787fbdc88797f4c1dfe")
745
746    @requires_blake2
747    def test_case_blake2s_0(self):
748        self.check('blake2s', b"",
749          "69217a3079908094e11121d042354a7c1f55b6482ca1a51e1b250dfd1ed0eef9")
750
751    @requires_blake2
752    def test_case_blake2s_1(self):
753        self.check('blake2s', b"abc",
754          "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982")
755
756    @requires_blake2
757    def test_case_blake2s_all_parameters(self):
758        # This checks that all the parameters work in general, and also that
759        # parameter byte order doesn't get confused on big endian platforms.
760        self.check('blake2s', b"foo",
761          "bf2a8f7fe3c555012a6f8046e646bc75",
762          digest_size=16,
763          key=b"bar",
764          salt=b"baz",
765          person=b"bing",
766          fanout=2,
767          depth=3,
768          leaf_size=4,
769          node_offset=5,
770          node_depth=6,
771          inner_size=7,
772          last_node=True)
773
774    @requires_blake2
775    def test_blake2s_vectors(self):
776        for msg, key, md in read_vectors('blake2s'):
777            key = bytes.fromhex(key)
778            self.check('blake2s', msg, md, key=key)
779
780    def test_case_sha3_224_0(self):
781        self.check('sha3_224', b"",
782          "6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7")
783
784    def test_case_sha3_224_vector(self):
785        for msg, md in read_vectors('sha3_224'):
786            self.check('sha3_224', msg, md)
787
788    def test_case_sha3_256_0(self):
789        self.check('sha3_256', b"",
790          "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a")
791
792    def test_case_sha3_256_vector(self):
793        for msg, md in read_vectors('sha3_256'):
794            self.check('sha3_256', msg, md)
795
796    def test_case_sha3_384_0(self):
797        self.check('sha3_384', b"",
798          "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2a"+
799          "c3713831264adb47fb6bd1e058d5f004")
800
801    def test_case_sha3_384_vector(self):
802        for msg, md in read_vectors('sha3_384'):
803            self.check('sha3_384', msg, md)
804
805    def test_case_sha3_512_0(self):
806        self.check('sha3_512', b"",
807          "a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a6"+
808          "15b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26")
809
810    def test_case_sha3_512_vector(self):
811        for msg, md in read_vectors('sha3_512'):
812            self.check('sha3_512', msg, md)
813
814    def test_case_shake_128_0(self):
815        self.check('shake_128', b"",
816          "7f9c2ba4e88f827d616045507605853ed73b8093f6efbc88eb1a6eacfa66ef26",
817          True)
818        self.check('shake_128', b"", "7f9c", True)
819
820    def test_case_shake128_vector(self):
821        for msg, md in read_vectors('shake_128'):
822            self.check('shake_128', msg, md, True)
823
824    def test_case_shake_256_0(self):
825        self.check('shake_256', b"",
826          "46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f",
827          True)
828        self.check('shake_256', b"", "46b9", True)
829
830    def test_case_shake256_vector(self):
831        for msg, md in read_vectors('shake_256'):
832            self.check('shake_256', msg, md, True)
833
834    def test_gil(self):
835        # Check things work fine with an input larger than the size required
836        # for multithreaded operation (which is hardwired to 2048).
837        gil_minsize = 2048
838
839        for cons in self.hash_constructors:
840            m = cons(usedforsecurity=False)
841            m.update(b'1')
842            m.update(b'#' * gil_minsize)
843            m.update(b'1')
844
845            m = cons(b'x' * gil_minsize, usedforsecurity=False)
846            m.update(b'1')
847
848        m = hashlib.sha256()
849        m.update(b'1')
850        m.update(b'#' * gil_minsize)
851        m.update(b'1')
852        self.assertEqual(
853            m.hexdigest(),
854            '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
855        )
856
857        m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1')
858        self.assertEqual(
859            m.hexdigest(),
860            '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
861        )
862
863    @threading_helper.reap_threads
864    def test_threaded_hashing(self):
865        # Updating the same hash object from several threads at once
866        # using data chunk sizes containing the same byte sequences.
867        #
868        # If the internal locks are working to prevent multiple
869        # updates on the same object from running at once, the resulting
870        # hash will be the same as doing it single threaded upfront.
871        hasher = hashlib.sha1()
872        num_threads = 5
873        smallest_data = b'swineflu'
874        data = smallest_data * 200000
875        expected_hash = hashlib.sha1(data*num_threads).hexdigest()
876
877        def hash_in_chunks(chunk_size):
878            index = 0
879            while index < len(data):
880                hasher.update(data[index:index + chunk_size])
881                index += chunk_size
882
883        threads = []
884        for threadnum in range(num_threads):
885            chunk_size = len(data) // (10 ** threadnum)
886            self.assertGreater(chunk_size, 0)
887            self.assertEqual(chunk_size % len(smallest_data), 0)
888            thread = threading.Thread(target=hash_in_chunks,
889                                      args=(chunk_size,))
890            threads.append(thread)
891
892        for thread in threads:
893            thread.start()
894        for thread in threads:
895            thread.join()
896
897        self.assertEqual(expected_hash, hasher.hexdigest())
898
899    def test_get_fips_mode(self):
900        fips_mode = self.is_fips_mode
901        if fips_mode is not None:
902            self.assertIsInstance(fips_mode, int)
903
904    @support.cpython_only
905    def test_disallow_instantiation(self):
906        for algorithm, constructors in self.constructors_to_test.items():
907            if algorithm.startswith(("sha3_", "shake", "blake")):
908                # _sha3 and _blake types can be instantiated
909                continue
910            # all other types have DISALLOW_INSTANTIATION
911            for constructor in constructors:
912                # In FIPS mode some algorithms are not available raising ValueError
913                try:
914                    h = constructor()
915                except ValueError:
916                    continue
917                with self.subTest(constructor=constructor):
918                    support.check_disallow_instantiation(self, type(h))
919
920    @unittest.skipUnless(HASH is not None, 'need _hashlib')
921    def test_hash_disallow_instantiation(self):
922        # internal types like _hashlib.HASH are not constructable
923        support.check_disallow_instantiation(self, HASH)
924        support.check_disallow_instantiation(self, HASHXOF)
925
926    def test_readonly_types(self):
927        for algorithm, constructors in self.constructors_to_test.items():
928            # all other types have DISALLOW_INSTANTIATION
929            for constructor in constructors:
930                # In FIPS mode some algorithms are not available raising ValueError
931                try:
932                    hash_type = type(constructor())
933                except ValueError:
934                    continue
935                with self.subTest(hash_type=hash_type):
936                    with self.assertRaisesRegex(TypeError, "immutable type"):
937                        hash_type.value = False
938
939
940class KDFTests(unittest.TestCase):
941
942    pbkdf2_test_vectors = [
943        (b'password', b'salt', 1, None),
944        (b'password', b'salt', 2, None),
945        (b'password', b'salt', 4096, None),
946        # too slow, it takes over a minute on a fast CPU.
947        #(b'password', b'salt', 16777216, None),
948        (b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt',
949         4096, -1),
950        (b'pass\0word', b'sa\0lt', 4096, 16),
951    ]
952
953    scrypt_test_vectors = [
954        (b'', b'', 16, 1, 1, unhexlify('77d6576238657b203b19ca42c18a0497f16b4844e3074ae8dfdffa3fede21442fcd0069ded0948f8326a753a0fc81f17e8d3e0fb2e0d3628cf35e20c38d18906')),
955        (b'password', b'NaCl', 1024, 8, 16, unhexlify('fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b3731622eaf30d92e22a3886ff109279d9830dac727afb94a83ee6d8360cbdfa2cc0640')),
956        (b'pleaseletmein', b'SodiumChloride', 16384, 8, 1, unhexlify('7023bdcb3afd7348461c06cd81fd38ebfda8fbba904f8e3ea9b543f6545da1f2d5432955613f0fcf62d49705242a9af9e61e85dc0d651e40dfcf017b45575887')),
957   ]
958
959    pbkdf2_results = {
960        "sha1": [
961            # official test vectors from RFC 6070
962            (bytes.fromhex('0c60c80f961f0e71f3a9b524af6012062fe037a6'), None),
963            (bytes.fromhex('ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'), None),
964            (bytes.fromhex('4b007901b765489abead49d926f721d065a429c1'), None),
965            #(bytes.fromhex('eefe3d61cd4da4e4e9945b3d6ba2158c2634e984'), None),
966            (bytes.fromhex('3d2eec4fe41c849b80c8d83662c0e44a8b291a964c'
967                           'f2f07038'), 25),
968            (bytes.fromhex('56fa6aa75548099dcc37d7f03425e0c3'), None),],
969        "sha256": [
970            (bytes.fromhex('120fb6cffcf8b32c43e7225256c4f837'
971                           'a86548c92ccc35480805987cb70be17b'), None),
972            (bytes.fromhex('ae4d0c95af6b46d32d0adff928f06dd0'
973                           '2a303f8ef3c251dfd6e2d85a95474c43'), None),
974            (bytes.fromhex('c5e478d59288c841aa530db6845c4c8d'
975                           '962893a001ce4e11a4963873aa98134a'), None),
976            #(bytes.fromhex('cf81c66fe8cfc04d1f31ecb65dab4089'
977            #               'f7f179e89b3b0bcb17ad10e3ac6eba46'), None),
978            (bytes.fromhex('348c89dbcbd32b2f32d814b8116e84cf2b17'
979                           '347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9'), 40),
980            (bytes.fromhex('89b69d0516f829893c696226650a8687'), None),],
981        "sha512": [
982            (bytes.fromhex('867f70cf1ade02cff3752599a3a53dc4af34c7a669815ae5'
983                           'd513554e1c8cf252c02d470a285a0501bad999bfe943c08f'
984                           '050235d7d68b1da55e63f73b60a57fce'), None),
985            (bytes.fromhex('e1d9c16aa681708a45f5c7c4e215ceb66e011a2e9f004071'
986                           '3f18aefdb866d53cf76cab2868a39b9f7840edce4fef5a82'
987                           'be67335c77a6068e04112754f27ccf4e'), None),
988            (bytes.fromhex('d197b1b33db0143e018b12f3d1d1479e6cdebdcc97c5c0f8'
989                           '7f6902e072f457b5143f30602641b3d55cd335988cb36b84'
990                           '376060ecd532e039b742a239434af2d5'), None),
991            (bytes.fromhex('8c0511f4c6e597c6ac6315d8f0362e225f3c501495ba23b8'
992                           '68c005174dc4ee71115b59f9e60cd9532fa33e0f75aefe30'
993                           '225c583a186cd82bd4daea9724a3d3b8'), 64),
994            (bytes.fromhex('9d9e9c4cd21fe4be24d5b8244c759665'), None),],
995    }
996
997    def _test_pbkdf2_hmac(self, pbkdf2, supported):
998        for digest_name, results in self.pbkdf2_results.items():
999            if digest_name not in supported:
1000                continue
1001            for i, vector in enumerate(self.pbkdf2_test_vectors):
1002                password, salt, rounds, dklen = vector
1003                expected, overwrite_dklen = results[i]
1004                if overwrite_dklen:
1005                    dklen = overwrite_dklen
1006                out = pbkdf2(digest_name, password, salt, rounds, dklen)
1007                self.assertEqual(out, expected,
1008                                 (digest_name, password, salt, rounds, dklen))
1009                out = pbkdf2(digest_name, memoryview(password),
1010                             memoryview(salt), rounds, dklen)
1011                self.assertEqual(out, expected)
1012                out = pbkdf2(digest_name, bytearray(password),
1013                             bytearray(salt), rounds, dklen)
1014                self.assertEqual(out, expected)
1015                if dklen is None:
1016                    out = pbkdf2(digest_name, password, salt, rounds)
1017                    self.assertEqual(out, expected,
1018                                     (digest_name, password, salt, rounds))
1019
1020        with self.assertRaisesRegex(ValueError, 'unsupported hash type'):
1021            pbkdf2('unknown', b'pass', b'salt', 1)
1022
1023        if 'sha1' in supported:
1024            self.assertRaises(
1025                TypeError, pbkdf2, b'sha1', b'pass', b'salt', 1
1026            )
1027            self.assertRaises(
1028                TypeError, pbkdf2, 'sha1', 'pass', 'salt', 1
1029            )
1030            self.assertRaises(
1031                ValueError, pbkdf2, 'sha1', b'pass', b'salt', 0
1032            )
1033            self.assertRaises(
1034                ValueError, pbkdf2, 'sha1', b'pass', b'salt', -1
1035            )
1036            self.assertRaises(
1037                ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, 0
1038            )
1039            self.assertRaises(
1040                ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, -1
1041            )
1042            out = pbkdf2(hash_name='sha1', password=b'password', salt=b'salt',
1043                iterations=1, dklen=None)
1044            self.assertEqual(out, self.pbkdf2_results['sha1'][0][0])
1045
1046    @unittest.skipIf(builtin_hashlib is None, "test requires builtin_hashlib")
1047    def test_pbkdf2_hmac_py(self):
1048        with warnings_helper.check_warnings():
1049            self._test_pbkdf2_hmac(
1050                builtin_hashlib.pbkdf2_hmac, builtin_hashes
1051            )
1052
1053    @unittest.skipUnless(hasattr(openssl_hashlib, 'pbkdf2_hmac'),
1054                     '   test requires OpenSSL > 1.0')
1055    def test_pbkdf2_hmac_c(self):
1056        self._test_pbkdf2_hmac(openssl_hashlib.pbkdf2_hmac, openssl_md_meth_names)
1057
1058    @unittest.skipUnless(hasattr(hashlib, 'scrypt'),
1059                     '   test requires OpenSSL > 1.1')
1060    def test_scrypt(self):
1061        for password, salt, n, r, p, expected in self.scrypt_test_vectors:
1062            result = hashlib.scrypt(password, salt=salt, n=n, r=r, p=p)
1063            self.assertEqual(result, expected)
1064
1065        # this values should work
1066        hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1)
1067        # password and salt must be bytes-like
1068        with self.assertRaises(TypeError):
1069            hashlib.scrypt('password', salt=b'salt', n=2, r=8, p=1)
1070        with self.assertRaises(TypeError):
1071            hashlib.scrypt(b'password', salt='salt', n=2, r=8, p=1)
1072        # require keyword args
1073        with self.assertRaises(TypeError):
1074            hashlib.scrypt(b'password')
1075        with self.assertRaises(TypeError):
1076            hashlib.scrypt(b'password', b'salt')
1077        with self.assertRaises(TypeError):
1078            hashlib.scrypt(b'password', 2, 8, 1, salt=b'salt')
1079        for n in [-1, 0, 1, None]:
1080            with self.assertRaises((ValueError, OverflowError, TypeError)):
1081                hashlib.scrypt(b'password', salt=b'salt', n=n, r=8, p=1)
1082        for r in [-1, 0, None]:
1083            with self.assertRaises((ValueError, OverflowError, TypeError)):
1084                hashlib.scrypt(b'password', salt=b'salt', n=2, r=r, p=1)
1085        for p in [-1, 0, None]:
1086            with self.assertRaises((ValueError, OverflowError, TypeError)):
1087                hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=p)
1088        for maxmem in [-1, None]:
1089            with self.assertRaises((ValueError, OverflowError, TypeError)):
1090                hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1,
1091                               maxmem=maxmem)
1092        for dklen in [-1, None]:
1093            with self.assertRaises((ValueError, OverflowError, TypeError)):
1094                hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1,
1095                               dklen=dklen)
1096
1097    def test_normalized_name(self):
1098        self.assertNotIn("blake2b512", hashlib.algorithms_available)
1099        self.assertNotIn("sha3-512", hashlib.algorithms_available)
1100
1101
1102if __name__ == "__main__":
1103    unittest.main()
1104