1#!/usr/bin/python -t
2
3#   This library is free software; you can redistribute it and/or
4#   modify it under the terms of the GNU Lesser General Public
5#   License as published by the Free Software Foundation; either
6#   version 2.1 of the License, or (at your option) any later version.
7#
8#   This library is distributed in the hope that it will be useful,
9#   but WITHOUT ANY WARRANTY; without even the implied warranty of
10#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11#   Lesser General Public License for more details.
12#
13#   You should have received a copy of the GNU Lesser General Public
14#   License along with this library; if not, write to the
15#      Free Software Foundation, Inc.,
16#      59 Temple Place, Suite 330,
17#      Boston, MA  02111-1307  USA
18
19# This file is part of urlgrabber, a high-level cross-protocol url-grabber
20# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
21
22from __future__ import print_function
23
24import sys
25import os
26from os.path import dirname, join as joinpath
27import tempfile
28import time
29
30import urlgrabber.grabber as grabber
31from urlgrabber.grabber import URLGrabber, urlgrab, urlopen, urlread
32from urlgrabber.progress import text_progress_meter
33
34tempsrc = '/tmp/ug-test-src'
35tempdst = '/tmp/ug-test-dst'
36
37# this isn't used but forces a proxy handler to be
38# added when creating the urllib2 opener.
39proxies = { 'http' : 'http://localhost' }
40DEBUG=0
41
42def main():
43    speedtest(1024)         # 1KB
44    speedtest(10 * 1024)    # 10 KB
45    speedtest(100 * 1024)   # 100 KB
46    speedtest(1000 * 1024)  # 1,000 KB (almost 1MB)
47    #speedtest(10000 * 1024) # 10,000 KB (almost 10MB)
48    # remove temp files
49    os.unlink(tempsrc)
50    os.unlink(tempdst)
51
52def setuptemp(size):
53    if DEBUG: print('writing %d KB to temporary file (%s).' % (size / 1024, tempsrc))
54    file = open(tempsrc, 'w', 1024)
55    chars = '0123456789'
56    for i in range(size):
57        file.write(chars[i % 10])
58    file.flush()
59    file.close()
60
61def speedtest(size):
62    setuptemp(size)
63    full_times = []
64    raw_times = []
65    none_times = []
66    throttle = 2**40 # throttle to 1 TB/s   :)
67
68    try:
69        from urlgrabber.progress import text_progress_meter
70    except ImportError as e:
71        tpm = None
72        print('not using progress meter')
73    else:
74        tpm = text_progress_meter(fo=open('/dev/null', 'w'))
75
76    # to address concerns that the overhead from the progress meter
77    # and throttling slow things down, we do this little test.
78    #
79    # using this test, you get the FULL overhead of the progress
80    # meter and throttling, without the benefit: the meter is directed
81    # to /dev/null and the throttle bandwidth is set EXTREMELY high.
82    #
83    # note: it _is_ even slower to direct the progress meter to a real
84    # tty or file, but I'm just interested in the overhead from _this_
85    # module.
86
87    # get it nicely cached before we start comparing
88    if DEBUG: print('pre-caching')
89    for i in range(100):
90        urlgrab(tempsrc, tempdst, copy_local=1, throttle=None, proxies=proxies)
91
92    if DEBUG: print('running speed test.')
93    reps = 500
94    for i in range(reps):
95        if DEBUG:
96            print('\r%4i/%-4i' % (i+1, reps), end=' ')
97            sys.stdout.flush()
98        t = time.time()
99        urlgrab(tempsrc, tempdst,
100                copy_local=1, progress_obj=tpm,
101                throttle=throttle, proxies=proxies)
102        full_times.append(1000 * (time.time() - t))
103
104        t = time.time()
105        urlgrab(tempsrc, tempdst,
106                copy_local=1, progress_obj=None,
107                throttle=None, proxies=proxies)
108        raw_times.append(1000 * (time.time() - t))
109
110        t = time.time()
111        in_fo = open(tempsrc)
112        out_fo = open(tempdst, 'wb')
113        while True:
114            s = in_fo.read(1024 * 8)
115            if not s: break
116            out_fo.write(s)
117        in_fo.close()
118        out_fo.close()
119        none_times.append(1000 * (time.time() - t))
120
121    if DEBUG: print('\r')
122
123    print("%d KB Results:" % (size / 1024))
124    print_result('full', full_times)
125    print_result('raw', raw_times)
126    print_result('none', none_times)
127
128def print_result(label, result_list):
129    format = '[%4s] mean: %6.3f ms, median: %6.3f ms, ' \
130             'min: %6.3f ms, max: %6.3f ms'
131    result_list.sort()
132    mean = 0.0
133    for i in result_list: mean += i
134    mean = mean/len(result_list)
135    median = result_list[int(len(result_list)/2)]
136    print(format % (label, mean, median, result_list[0], result_list[-1]))
137
138if __name__ == '__main__':
139    main()
140