1"""
2Periodically update bundled versions.
3"""
4
5from __future__ import absolute_import, unicode_literals
6
7import json
8import logging
9import os
10import ssl
11import subprocess
12import sys
13from datetime import datetime, timedelta
14from itertools import groupby
15from shutil import copy2
16from textwrap import dedent
17from threading import Thread
18
19from six.moves.urllib.error import URLError
20from six.moves.urllib.request import urlopen
21
22from virtualenv.app_data import AppDataDiskFolder
23from virtualenv.info import PY2
24from virtualenv.util.path import Path
25from virtualenv.util.subprocess import CREATE_NO_WINDOW, Popen
26
27from ..wheels.embed import BUNDLE_SUPPORT
28from ..wheels.util import Wheel
29
30if PY2:
31    # on Python 2 datetime.strptime throws the error below if the import did not trigger on main thread
32    # Failed to import _strptime because the import lock is held by
33    try:
34        import _strptime  # noqa
35    except ImportError:  # pragma: no cov
36        pass  # pragma: no cov
37
38
39def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data, do_periodic_update):
40    if do_periodic_update:
41        handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data)
42
43    now = datetime.now()
44
45    u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version)
46    u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False
47    for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]):
48        version = next(group)  # use only latest patch version per minor, earlier assumed to be buggy
49        if wheel is not None and Path(version.filename).name == wheel.name:
50            break
51        if u_log.periodic is False or (u_log_older_than_hour and version.use(now)):
52            updated_wheel = Wheel(app_data.house / version.filename)
53            logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel)
54            wheel = updated_wheel
55            break
56
57    return wheel
58
59
60def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data):
61    embed_update_log = app_data.embed_update_log(distribution, for_py_version)
62    u_log = UpdateLog.from_dict(embed_update_log.read())
63    if u_log.needs_update:
64        u_log.periodic = True
65        u_log.started = datetime.now()
66        embed_update_log.write(u_log.to_dict())
67        trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic=True)
68
69
70DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ"
71
72
73def dump_datetime(value):
74    return None if value is None else value.strftime(DATETIME_FMT)
75
76
77def load_datetime(value):
78    return None if value is None else datetime.strptime(value, DATETIME_FMT)
79
80
81class NewVersion(object):
82    def __init__(self, filename, found_date, release_date):
83        self.filename = filename
84        self.found_date = found_date
85        self.release_date = release_date
86
87    @classmethod
88    def from_dict(cls, dictionary):
89        return cls(
90            filename=dictionary["filename"],
91            found_date=load_datetime(dictionary["found_date"]),
92            release_date=load_datetime(dictionary["release_date"]),
93        )
94
95    def to_dict(self):
96        return {
97            "filename": self.filename,
98            "release_date": dump_datetime(self.release_date),
99            "found_date": dump_datetime(self.found_date),
100        }
101
102    def use(self, now):
103        compare_from = self.release_date or self.found_date
104        return now - compare_from >= timedelta(days=28)
105
106    def __repr__(self):
107        return "{}(filename={}), found_date={}, release_date={})".format(
108            self.__class__.__name__,
109            self.filename,
110            self.found_date,
111            self.release_date,
112        )
113
114    def __eq__(self, other):
115        return type(self) == type(other) and all(
116            getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"]
117        )
118
119    def __ne__(self, other):
120        return not (self == other)
121
122    @property
123    def wheel(self):
124        return Wheel(Path(self.filename))
125
126
127class UpdateLog(object):
128    def __init__(self, started, completed, versions, periodic):
129        self.started = started
130        self.completed = completed
131        self.versions = versions
132        self.periodic = periodic
133
134    @classmethod
135    def from_dict(cls, dictionary):
136        if dictionary is None:
137            dictionary = {}
138        return cls(
139            load_datetime(dictionary.get("started")),
140            load_datetime(dictionary.get("completed")),
141            [NewVersion.from_dict(v) for v in dictionary.get("versions", [])],
142            dictionary.get("periodic"),
143        )
144
145    @classmethod
146    def from_app_data(cls, app_data, distribution, for_py_version):
147        raw_json = app_data.embed_update_log(distribution, for_py_version).read()
148        return cls.from_dict(raw_json)
149
150    def to_dict(self):
151        return {
152            "started": dump_datetime(self.started),
153            "completed": dump_datetime(self.completed),
154            "periodic": self.periodic,
155            "versions": [r.to_dict() for r in self.versions],
156        }
157
158    @property
159    def needs_update(self):
160        now = datetime.now()
161        if self.completed is None:  # never completed
162            return self._check_start(now)
163        else:
164            if now - self.completed <= timedelta(days=14):
165                return False
166            return self._check_start(now)
167
168    def _check_start(self, now):
169        return self.started is None or now - self.started > timedelta(hours=1)
170
171
172def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, periodic):
173    wheel_path = None if wheel is None else str(wheel.path)
174    cmd = [
175        sys.executable,
176        "-c",
177        dedent(
178            """
179        from virtualenv.report import setup_report, MAX_LEVEL
180        from virtualenv.seed.wheels.periodic_update import do_update
181        setup_report(MAX_LEVEL, show_pid=True)
182        do_update({!r}, {!r}, {!r}, {!r}, {!r}, {!r})
183        """,
184        )
185        .strip()
186        .format(distribution, for_py_version, wheel_path, str(app_data), [str(p) for p in search_dirs], periodic),
187    ]
188    debug = os.environ.get(str("_VIRTUALENV_PERIODIC_UPDATE_INLINE")) == str("1")
189    pipe = None if debug else subprocess.PIPE
190    kwargs = {"stdout": pipe, "stderr": pipe}
191    if not debug and sys.platform == "win32":
192        kwargs["creationflags"] = CREATE_NO_WINDOW
193    process = Popen(cmd, **kwargs)
194    logging.info(
195        "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d",
196        distribution,
197        "" if wheel is None else "=={}".format(wheel.version),
198        for_py_version,
199        process.pid,
200    )
201    if debug:
202        process.communicate()  # on purpose not called to make it a background process
203
204
205def do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic):
206    versions = None
207    try:
208        versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs)
209    finally:
210        logging.debug("done %s %s with %s", distribution, for_py_version, versions)
211    return versions
212
213
214def _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs):
215    from virtualenv.seed.wheels import acquire
216
217    wheel_filename = None if embed_filename is None else Path(embed_filename)
218    embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple
219    app_data = AppDataDiskFolder(app_data) if isinstance(app_data, str) else app_data
220    search_dirs = [Path(p) if isinstance(p, str) else p for p in search_dirs]
221    wheelhouse = app_data.house
222    embed_update_log = app_data.embed_update_log(distribution, for_py_version)
223    u_log = UpdateLog.from_dict(embed_update_log.read())
224    now = datetime.now()
225    if wheel_filename is not None:
226        dest = wheelhouse / wheel_filename.name
227        if not dest.exists():
228            copy2(str(wheel_filename), str(wheelhouse))
229    last, last_version, versions = None, None, []
230    while last is None or not last.use(now):
231        download_time = datetime.now()
232        dest = acquire.download_wheel(
233            distribution=distribution,
234            version_spec=None if last_version is None else "<{}".format(last_version),
235            for_py_version=for_py_version,
236            search_dirs=search_dirs,
237            app_data=app_data,
238            to_folder=wheelhouse,
239        )
240        if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name):
241            break
242        release_date = release_date_for_wheel_path(dest.path)
243        last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time)
244        logging.info("detected %s in %s", last, datetime.now() - download_time)
245        versions.append(last)
246        last_wheel = Wheel(Path(last.filename))
247        last_version = last_wheel.version
248        if embed_version is not None:
249            if embed_version >= last_wheel.version_tuple:  # stop download if we reach the embed version
250                break
251    u_log.periodic = periodic
252    if not u_log.periodic:
253        u_log.started = now
254    u_log.versions = versions + u_log.versions
255    u_log.completed = datetime.now()
256    embed_update_log.write(u_log.to_dict())
257    return versions
258
259
260def release_date_for_wheel_path(dest):
261    wheel = Wheel(dest)
262    # the most accurate is to ask PyPi - e.g. https://pypi.org/pypi/pip/json,
263    # see https://warehouse.pypa.io/api-reference/json/ for more details
264    content = _pypi_get_distribution_info_cached(wheel.distribution)
265    if content is not None:
266        try:
267            upload_time = content["releases"][wheel.version][0]["upload_time"]
268            return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S")
269        except Exception as exception:
270            logging.error("could not load release date %s because %r", content, exception)
271    return None
272
273
274def _request_context():
275    yield None
276    # fallback to non verified HTTPS (the information we request is not sensitive, so fallback)
277    yield ssl._create_unverified_context()  # noqa
278
279
280_PYPI_CACHE = {}
281
282
283def _pypi_get_distribution_info_cached(distribution):
284    if distribution not in _PYPI_CACHE:
285        _PYPI_CACHE[distribution] = _pypi_get_distribution_info(distribution)
286    return _PYPI_CACHE[distribution]
287
288
289def _pypi_get_distribution_info(distribution):
290    content, url = None, "https://pypi.org/pypi/{}/json".format(distribution)
291    try:
292        for context in _request_context():
293            try:
294                with urlopen(url, context=context) as file_handler:
295                    content = json.load(file_handler)
296                break
297            except URLError as exception:
298                logging.error("failed to access %s because %r", url, exception)
299    except Exception as exception:
300        logging.error("failed to access %s because %r", url, exception)
301    return content
302
303
304def manual_upgrade(app_data):
305    threads = []
306
307    for for_py_version, distribution_to_package in BUNDLE_SUPPORT.items():
308        # load extra search dir for the given for_py
309        for distribution in distribution_to_package.keys():
310            thread = Thread(target=_run_manual_upgrade, args=(app_data, distribution, for_py_version))
311            thread.start()
312            threads.append(thread)
313
314    for thread in threads:
315        thread.join()
316
317
318def _run_manual_upgrade(app_data, distribution, for_py_version):
319    start = datetime.now()
320    from .bundle import from_bundle
321
322    current = from_bundle(
323        distribution=distribution,
324        version=None,
325        for_py_version=for_py_version,
326        search_dirs=[],
327        app_data=app_data,
328        do_periodic_update=False,
329    )
330    logging.warning(
331        "upgrade %s for python %s with current %s",
332        distribution,
333        for_py_version,
334        "" if current is None else current.name,
335    )
336    versions = do_update(
337        distribution=distribution,
338        for_py_version=for_py_version,
339        embed_filename=current.path,
340        app_data=app_data,
341        search_dirs=[],
342        periodic=False,
343    )
344    msg = "upgraded %s for python %s in %s {}".format(
345        "new entries found:\n%s" if versions else "no new versions found",
346    )
347    args = [
348        distribution,
349        for_py_version,
350        datetime.now() - start,
351    ]
352    if versions:
353        args.append("\n".join("\t{}".format(v) for v in versions))
354    logging.warning(msg, *args)
355
356
357__all__ = (
358    "periodic_update",
359    "do_update",
360    "manual_upgrade",
361    "NewVersion",
362    "UpdateLog",
363    "load_datetime",
364    "dump_datetime",
365    "trigger_update",
366    "release_date_for_wheel_path",
367)
368