1from __future__ import print_function 2from __future__ import division 3 4try: 5 from urllib.request import urlopen 6except ImportError: 7 from urllib2 import urlopen 8 9from mpi4py.futures import MPIPoolExecutor 10 11URLS = [ 12 'http://www.google.com/', 13 'http://www.apple.com/', 14 'http://www.ibm.com/', 15 'http://www.slashdot.org/', 16 'http://www.python.org/', 17 'http://www.bing.com/', 18 'http://www.facebook.com/', 19 'http://www.yahoo.com/', 20 'http://www.youtube.com/', 21 'http://www.blogger.com/', 22] 23 24def load_url(url): 25 return url, urlopen(url).read() 26 27def test_crawl(): 28 with MPIPoolExecutor(10) as executor: 29 for url, content in executor.map(load_url, URLS, 30 timeout=10, unordered=True): 31 print('%-25s: %6.2f KiB' % (url, len(content)/(1 << 10))) 32 33if __name__ == '__main__': 34 test_crawl() 35