1"""
2A spider that generate light requests to meassure QPS troughput
3
4usage:
5
6    scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
7
8"""
9
10from scrapy.spiders import Spider
11from scrapy.http import Request
12
13
14class QPSSpider(Spider):
15
16    name = 'qps'
17    benchurl = 'http://localhost:8880/'
18
19    # Max concurrency is limited by global CONCURRENT_REQUESTS setting
20    max_concurrent_requests = 8
21    # Requests per second goal
22    qps = None # same as: 1 / download_delay
23    download_delay = None
24    # time in seconds to delay server responses
25    latency = None
26    # number of slots to create
27    slots = 1
28
29    def __init__(self, *a, **kw):
30        super().__init__(*a, **kw)
31        if self.qps is not None:
32            self.qps = float(self.qps)
33            self.download_delay = 1 / self.qps
34        elif self.download_delay is not None:
35            self.download_delay = float(self.download_delay)
36
37    def start_requests(self):
38        url = self.benchurl
39        if self.latency is not None:
40            url += f'?latency={self.latency}'
41
42        slots = int(self.slots)
43        if slots > 1:
44            urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)]
45        else:
46            urls = [url]
47
48        idx = 0
49        while True:
50            url = urls[idx % len(urls)]
51            yield Request(url, dont_filter=True)
52            idx += 1
53
54    def parse(self, response):
55        pass
56