1import json 2import time 3from gzip import GzipFile 4from io import StringIO 5from urllib.request import Request 6import requests 7from fsgs.network import openretro_url_prefix, is_http_url 8from fsgs.res import gettext 9 10# FIXME: Overlap in code/functionality with GameDatabaseSynchronizer 11 12 13class SynchronizerBase(object): 14 def __init__(self, context, on_status=None, stop_check=None): 15 self.context = context 16 self.on_status = on_status 17 self._stop_check = stop_check 18 19 @staticmethod 20 def bytes_to_int(m): 21 return m[0] << 24 | m[1] << 16 | m[2] << 8 | m[3] 22 23 def stop_check(self): 24 if self._stop_check: 25 return self._stop_check() 26 27 def set_status(self, title, status=""): 28 if self.on_status: 29 self.on_status((title, status)) 30 31 def url_prefix(self): 32 return openretro_url_prefix() 33 34 def auth(self): 35 return self.context.username, self.context.password 36 37 def url(self, url): 38 if not is_http_url(url): 39 url = "{0}{1}".format(self.url_prefix(), url) 40 return url 41 42 def fetch_json_attempt(self, url): 43 print("[HTTP] {}".format(self.url(url))) 44 r = requests.get(self.url(url), auth=self.auth()) 45 r.raise_for_status() 46 return r.json() 47 48 def fetch_data_attempt(self, url, accept_gzip_encoding=False): 49 print("[HTTP] {}".format(self.url(url))) 50 r = requests.get(self.url(url), auth=self.auth()) 51 r.raise_for_status() 52 return r.content 53 54 def fetch_json(self, url): 55 for i in range(20): 56 try: 57 return self.fetch_json_attempt(url) 58 except Exception as e: 59 print(e) 60 sleep_time = 2.0 + i * 0.3 61 # FIXME: change second {0} to {1} 62 self.set_status( 63 gettext( 64 "Download failed (attempt {0}) - retrying in {0} " 65 "seconds" 66 ).format(i + 1, int(sleep_time)) 67 ) 68 for _ in range(int(sleep_time) * 10): 69 time.sleep(0.1) 70 if self.stop_check(): 71 return 72 self.set_status( 73 gettext("Retrying last operation (attempt {0})").format( 74 i + 1 75 ) 76 ) 77 return self.fetch_json_attempt(url) 78 79 def fetch_data(self, url): 80 for i in range(10): 81 try: 82 return self.fetch_data_attempt(url) 83 except Exception as e: 84 print(e) 85 sleep_time = 2.0 + i * 0.3 86 # FIXME: change second {0} to {1} 87 self.set_status( 88 gettext( 89 "Download failed (attempt {0}) - retrying in {0} " 90 "seconds" 91 ).format(i + 1, int(sleep_time)) 92 ) 93 for _ in range(int(sleep_time) * 10): 94 time.sleep(0.1) 95 if self.stop_check(): 96 return 97 self.set_status( 98 gettext("Retrying last operation (attempt {0})").format( 99 i + 1 100 ) 101 ) 102 return self.fetch_data_attempt(url) 103