1""" 2Periodically update bundled versions. 3""" 4 5from __future__ import absolute_import, unicode_literals 6 7import json 8import logging 9import os 10import ssl 11import subprocess 12import sys 13from datetime import datetime, timedelta 14from itertools import groupby 15from shutil import copy2 16from textwrap import dedent 17from threading import Thread 18 19from six.moves.urllib.error import URLError 20from six.moves.urllib.request import urlopen 21 22from virtualenv.app_data import AppDataDiskFolder 23from virtualenv.info import PY2 24from virtualenv.util.path import Path 25from virtualenv.util.subprocess import CREATE_NO_WINDOW, Popen 26 27from ..wheels.embed import BUNDLE_SUPPORT 28from ..wheels.util import Wheel 29 30if PY2: 31 # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread 32 # Failed to import _strptime because the import lock is held by 33 try: 34 import _strptime # noqa 35 except ImportError: # pragma: no cov 36 pass # pragma: no cov 37 38 39def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update): 40 if do_periodic_update: 41 handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data) 42 43 now = datetime.now() 44 45 u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) 46 u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False 47 for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]): 48 version = next(group) # use only latest patch version per minor, earlier assumed to be buggy 49 if wheel is not None and Path(version.filename).name == wheel.name: 50 break 51 if u_log.periodic is False or (u_log_older_than_hour and version.use(now)): 52 updated_wheel = Wheel(app_data.house / version.filename) 53 logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) 54 wheel = updated_wheel 55 break 56 57 return wheel 58 59 60def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data): 61 embed_update_log = app_data.embed_update_log(distribution, for_py_version) 62 u_log = UpdateLog.from_dict(embed_update_log.read()) 63 if u_log.needs_update: 64 u_log.periodic = True 65 u_log.started = datetime.now() 66 embed_update_log.write(u_log.to_dict()) 67 trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True) 68 69 70DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" 71 72 73def dump_datetime(value): 74 return None if value is None else value.strftime(DATETIME_FMT) 75 76 77def load_datetime(value): 78 return None if value is None else datetime.strptime(value, DATETIME_FMT) 79 80 81class NewVersion(object): 82 def __init__(self, filename, found_date, release_date): 83 self.filename = filename 84 self.found_date = found_date 85 self.release_date = release_date 86 87 @classmethod 88 def from_dict(cls, dictionary): 89 return cls( 90 filename=dictionary["filename"], 91 found_date=load_datetime(dictionary["found_date"]), 92 release_date=load_datetime(dictionary["release_date"]), 93 ) 94 95 def to_dict(self): 96 return { 97 "filename": self.filename, 98 "release_date": dump_datetime(self.release_date), 99 "found_date": dump_datetime(self.found_date), 100 } 101 102 def use(self, now): 103 compare_from = self.release_date or self.found_date 104 return now - compare_from >= timedelta(days=28) 105 106 def __repr__(self): 107 return "{}(filename={}), found_date={}, release_date={})".format( 108 self.__class__.__name__, 109 self.filename, 110 self.found_date, 111 self.release_date, 112 ) 113 114 def __eq__(self, other): 115 return type(self) == type(other) and all( 116 getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"] 117 ) 118 119 def __ne__(self, other): 120 return not (self == other) 121 122 @property 123 def wheel(self): 124 return Wheel(Path(self.filename)) 125 126 127class UpdateLog(object): 128 def __init__(self, started, completed, versions, periodic): 129 self.started = started 130 self.completed = completed 131 self.versions = versions 132 self.periodic = periodic 133 134 @classmethod 135 def from_dict(cls, dictionary): 136 if dictionary is None: 137 dictionary = {} 138 return cls( 139 load_datetime(dictionary.get("started")), 140 load_datetime(dictionary.get("completed")), 141 [NewVersion.from_dict(v) for v in dictionary.get("versions", [])], 142 dictionary.get("periodic"), 143 ) 144 145 @classmethod 146 def from_app_data(cls, app_data, distribution, for_py_version): 147 raw_json = app_data.embed_update_log(distribution, for_py_version).read() 148 return cls.from_dict(raw_json) 149 150 def to_dict(self): 151 return { 152 "started": dump_datetime(self.started), 153 "completed": dump_datetime(self.completed), 154 "periodic": self.periodic, 155 "versions": [r.to_dict() for r in self.versions], 156 } 157 158 @property 159 def needs_update(self): 160 now = datetime.now() 161 if self.completed is None: # never completed 162 return self._check_start(now) 163 else: 164 if now - self.completed <= timedelta(days=14): 165 return False 166 return self._check_start(now) 167 168 def _check_start(self, now): 169 return self.started is None or now - self.started > timedelta(hours=1) 170 171 172def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic): 173 wheel_path = None if wheel is None else str(wheel.path) 174 cmd = [ 175 sys.executable, 176 "-c", 177 dedent( 178 """ 179 from virtualenv.report import setup_report, MAX_LEVEL 180 from virtualenv.seed.wheels.periodic_update import do_update 181 setup_report(MAX_LEVEL, show_pid=True) 182 do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) 183 """, 184 ) 185 .strip() 186 .format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic), 187 ] 188 debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1") 189 pipe = None if debug else subprocess.PIPE 190 kwargs = {"stdout": pipe, "stderr": pipe} 191 if not debug and sys.platform == "win32": 192 kwargs["creationflags"] = CREATE_NO_WINDOW 193 process = Popen(cmd, **kwargs) 194 logging.info( 195 "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", 196 distribution, 197 "" if wheel is None else "=={}".format(wheel.version), 198 for_py_version, 199 process.pid, 200 ) 201 if debug: 202 process.communicate() # on purpose not called to make it a background process 203 204 205def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): 206 versions = None 207 try: 208 versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) 209 finally: 210 logging.debug("done %s %s with %s", distribution, for_py_version, versions) 211 return versions 212 213 214def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs): 215 from virtualenv.seed.wheels import acquire 216 217 wheel_filename = None if embed_filename is None else Path(embed_filename) 218 embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple 219 app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data 220 search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs] 221 wheelhouse = app_data.house 222 embed_update_log = app_data.embed_update_log(distribution, for_py_version) 223 u_log = UpdateLog.from_dict(embed_update_log.read()) 224 now = datetime.now() 225 if wheel_filename is not None: 226 dest = wheelhouse / wheel_filename.name 227 if not dest.exists(): 228 copy2(str(wheel_filename), str(wheelhouse)) 229 last, last_version, versions = None, None, [] 230 while last is None or not last.use(now): 231 download_time = datetime.now() 232 dest = acquire.download_wheel( 233 distribution=distribution, 234 version_spec=None if last_version is None else "<{}".format(last_version), 235 for_py_version=for_py_version, 236 search_dirs=search_dirs, 237 app_data=app_data, 238 to_folder=wheelhouse, 239 ) 240 if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): 241 break 242 release_date = release_date_for_wheel_path(dest.path) 243 last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) 244 logging.info("detected %s in %s", last, datetime.now() - download_time) 245 versions.append(last) 246 last_wheel = Wheel(Path(last.filename)) 247 last_version = last_wheel.version 248 if embed_version is not None: 249 if embed_version >= last_wheel.version_tuple: # stop download if we reach the embed version 250 break 251 u_log.periodic = periodic 252 if not u_log.periodic: 253 u_log.started = now 254 u_log.versions = versions + u_log.versions 255 u_log.completed = datetime.now() 256 embed_update_log.write(u_log.to_dict()) 257 return versions 258 259 260def release_date_for_wheel_path(dest): 261 wheel = Wheel(dest) 262 # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json, 263 # see https://warehouse.pypa.io/api-reference/json/ for more details 264 content = _pypi_get_distribution_info_cached(wheel.distribution) 265 if content is not None: 266 try: 267 upload_time = content["releases"][wheel.version][0]["upload_time"] 268 return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S") 269 except Exception as exception: 270 logging.error("could not load release date %s because %r", content, exception) 271 return None 272 273 274def _request_context(): 275 yield None 276 # fallback to non verified HTTPS (the information we request is not sensitive, so fallback) 277 yield ssl._create_unverified_context() # noqa 278 279 280_PYPI_CACHE = {} 281 282 283def _pypi_get_distribution_info_cached(distribution): 284 if distribution not in _PYPI_CACHE: 285 _PYPI_CACHE[distribution] = _pypi_get_distribution_info(distribution) 286 return _PYPI_CACHE[distribution] 287 288 289def _pypi_get_distribution_info(distribution): 290 content, url = None, "https://pypi.org/pypi/{}/json".format(distribution) 291 try: 292 for context in _request_context(): 293 try: 294 with urlopen(url, context=context) as file_handler: 295 content = json.load(file_handler) 296 break 297 except URLError as exception: 298 logging.error("failed to access %s because %r", url, exception) 299 except Exception as exception: 300 logging.error("failed to access %s because %r", url, exception) 301 return content 302 303 304def manual_upgrade(app_data): 305 threads = [] 306 307 for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items(): 308 # load extra search dir for the given for_py 309 for distribution in distribution_to_package.keys(): 310 thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version)) 311 thread.start() 312 threads.append(thread) 313 314 for thread in threads: 315 thread.join() 316 317 318def _run_manual_upgrade(app_data, distribution, for_py_version): 319 start = datetime.now() 320 from .bundle import from_bundle 321 322 current = from_bundle( 323 distribution=distribution, 324 version=None, 325 for_py_version=for_py_version, 326 search_dirs=[], 327 app_data=app_data, 328 do_periodic_update=False, 329 ) 330 logging.warning( 331 "upgrade %s for python %s with current %s", 332 distribution, 333 for_py_version, 334 "" if current is None else current.name, 335 ) 336 versions = do_update( 337 distribution=distribution, 338 for_py_version=for_py_version, 339 embed_filename=current.path, 340 app_data=app_data, 341 search_dirs=[], 342 periodic=False, 343 ) 344 msg = "upgraded %s for python %s in %s {}".format( 345 "new entries found:\n%s" if versions else "no new versions found", 346 ) 347 args = [ 348 distribution, 349 for_py_version, 350 datetime.now() - start, 351 ] 352 if versions: 353 args.append("\n".join("\t{}".format(v) for v in versions)) 354 logging.warning(msg, *args) 355 356 357__all__ = ( 358 "periodic_update", 359 "do_update", 360 "manual_upgrade", 361 "NewVersion", 362 "UpdateLog", 363 "load_datetime", 364 "dump_datetime", 365 "trigger_update", 366 "release_date_for_wheel_path", 367) 368