1""" 2A spider that generate light requests to meassure QPS troughput 3 4usage: 5 6 scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3 7 8""" 9 10from scrapy.spiders import Spider 11from scrapy.http import Request 12 13 14class QPSSpider(Spider): 15 16 name = 'qps' 17 benchurl = 'http://localhost:8880/' 18 19 # Max concurrency is limited by global CONCURRENT_REQUESTS setting 20 max_concurrent_requests = 8 21 # Requests per second goal 22 qps = None # same as: 1 / download_delay 23 download_delay = None 24 # time in seconds to delay server responses 25 latency = None 26 # number of slots to create 27 slots = 1 28 29 def __init__(self, *a, **kw): 30 super().__init__(*a, **kw) 31 if self.qps is not None: 32 self.qps = float(self.qps) 33 self.download_delay = 1 / self.qps 34 elif self.download_delay is not None: 35 self.download_delay = float(self.download_delay) 36 37 def start_requests(self): 38 url = self.benchurl 39 if self.latency is not None: 40 url += f'?latency={self.latency}' 41 42 slots = int(self.slots) 43 if slots > 1: 44 urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] 45 else: 46 urls = [url] 47 48 idx = 0 49 while True: 50 url = urls[idx % len(urls)] 51 yield Request(url, dont_filter=True) 52 idx += 1 53 54 def parse(self, response): 55 pass 56