1#!/usr/bin/python -t 2 3# This library is free software; you can redistribute it and/or 4# modify it under the terms of the GNU Lesser General Public 5# License as published by the Free Software Foundation; either 6# version 2.1 of the License, or (at your option) any later version. 7# 8# This library is distributed in the hope that it will be useful, 9# but WITHOUT ANY WARRANTY; without even the implied warranty of 10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11# Lesser General Public License for more details. 12# 13# You should have received a copy of the GNU Lesser General Public 14# License along with this library; if not, write to the 15# Free Software Foundation, Inc., 16# 59 Temple Place, Suite 330, 17# Boston, MA 02111-1307 USA 18 19# This file is part of urlgrabber, a high-level cross-protocol url-grabber 20# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko 21 22from __future__ import print_function 23 24import sys 25import os 26from os.path import dirname, join as joinpath 27import tempfile 28import time 29 30import urlgrabber.grabber as grabber 31from urlgrabber.grabber import URLGrabber, urlgrab, urlopen, urlread 32from urlgrabber.progress import text_progress_meter 33 34tempsrc = '/tmp/ug-test-src' 35tempdst = '/tmp/ug-test-dst' 36 37# this isn't used but forces a proxy handler to be 38# added when creating the urllib2 opener. 39proxies = { 'http' : 'http://localhost' } 40DEBUG=0 41 42def main(): 43 speedtest(1024) # 1KB 44 speedtest(10 * 1024) # 10 KB 45 speedtest(100 * 1024) # 100 KB 46 speedtest(1000 * 1024) # 1,000 KB (almost 1MB) 47 #speedtest(10000 * 1024) # 10,000 KB (almost 10MB) 48 # remove temp files 49 os.unlink(tempsrc) 50 os.unlink(tempdst) 51 52def setuptemp(size): 53 if DEBUG: print('writing %d KB to temporary file (%s).' % (size / 1024, tempsrc)) 54 file = open(tempsrc, 'w', 1024) 55 chars = '0123456789' 56 for i in range(size): 57 file.write(chars[i % 10]) 58 file.flush() 59 file.close() 60 61def speedtest(size): 62 setuptemp(size) 63 full_times = [] 64 raw_times = [] 65 none_times = [] 66 throttle = 2**40 # throttle to 1 TB/s :) 67 68 try: 69 from urlgrabber.progress import text_progress_meter 70 except ImportError as e: 71 tpm = None 72 print('not using progress meter') 73 else: 74 tpm = text_progress_meter(fo=open('/dev/null', 'w')) 75 76 # to address concerns that the overhead from the progress meter 77 # and throttling slow things down, we do this little test. 78 # 79 # using this test, you get the FULL overhead of the progress 80 # meter and throttling, without the benefit: the meter is directed 81 # to /dev/null and the throttle bandwidth is set EXTREMELY high. 82 # 83 # note: it _is_ even slower to direct the progress meter to a real 84 # tty or file, but I'm just interested in the overhead from _this_ 85 # module. 86 87 # get it nicely cached before we start comparing 88 if DEBUG: print('pre-caching') 89 for i in range(100): 90 urlgrab(tempsrc, tempdst, copy_local=1, throttle=None, proxies=proxies) 91 92 if DEBUG: print('running speed test.') 93 reps = 500 94 for i in range(reps): 95 if DEBUG: 96 print('\r%4i/%-4i' % (i+1, reps), end=' ') 97 sys.stdout.flush() 98 t = time.time() 99 urlgrab(tempsrc, tempdst, 100 copy_local=1, progress_obj=tpm, 101 throttle=throttle, proxies=proxies) 102 full_times.append(1000 * (time.time() - t)) 103 104 t = time.time() 105 urlgrab(tempsrc, tempdst, 106 copy_local=1, progress_obj=None, 107 throttle=None, proxies=proxies) 108 raw_times.append(1000 * (time.time() - t)) 109 110 t = time.time() 111 in_fo = open(tempsrc) 112 out_fo = open(tempdst, 'wb') 113 while True: 114 s = in_fo.read(1024 * 8) 115 if not s: break 116 out_fo.write(s) 117 in_fo.close() 118 out_fo.close() 119 none_times.append(1000 * (time.time() - t)) 120 121 if DEBUG: print('\r') 122 123 print("%d KB Results:" % (size / 1024)) 124 print_result('full', full_times) 125 print_result('raw', raw_times) 126 print_result('none', none_times) 127 128def print_result(label, result_list): 129 format = '[%4s] mean: %6.3f ms, median: %6.3f ms, ' \ 130 'min: %6.3f ms, max: %6.3f ms' 131 result_list.sort() 132 mean = 0.0 133 for i in result_list: mean += i 134 mean = mean/len(result_list) 135 median = result_list[int(len(result_list)/2)] 136 print(format % (label, mean, median, result_list[0], result_list[-1])) 137 138if __name__ == '__main__': 139 main() 140