1from __future__ import print_function
2from __future__ import division
3
4try:
5    from urllib.request import urlopen
6except ImportError:
7    from urllib2 import urlopen
8
9from mpi4py.futures import MPIPoolExecutor
10
11URLS = [
12    'http://www.google.com/',
13    'http://www.apple.com/',
14    'http://www.ibm.com/',
15    'http://www.slashdot.org/',
16    'http://www.python.org/',
17    'http://www.bing.com/',
18    'http://www.facebook.com/',
19    'http://www.yahoo.com/',
20    'http://www.youtube.com/',
21    'http://www.blogger.com/',
22]
23
24def load_url(url):
25    return url, urlopen(url).read()
26
27def test_crawl():
28    with MPIPoolExecutor(10) as executor:
29        for url, content in executor.map(load_url, URLS,
30                                         timeout=10, unordered=True):
31            print('%-25s: %6.2f KiB' % (url, len(content)/(1 << 10)))
32
33if __name__ == '__main__':
34    test_crawl()
35