1""" 2The PythonInfo contains information about a concrete instance of a Python interpreter 3 4Note: this file is also used to query target interpreters, so can only use standard library methods 5""" 6from __future__ import absolute_import, print_function 7 8import json 9import logging 10import os 11import platform 12import re 13import sys 14import sysconfig 15import warnings 16from collections import OrderedDict, namedtuple 17from string import digits 18 19VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) 20 21 22def _get_path_extensions(): 23 return list(OrderedDict.fromkeys([""] + os.environ.get("PATHEXT", "").lower().split(os.pathsep))) 24 25 26EXTENSIONS = _get_path_extensions() 27_CONF_VAR_RE = re.compile(r"\{\w+\}") 28 29 30class PythonInfo(object): 31 """Contains information for a Python interpreter""" 32 33 def __init__(self): 34 def u(v): 35 return v.decode("utf-8") if isinstance(v, bytes) else v 36 37 def abs_path(v): 38 return None if v is None else os.path.abspath(v) # unroll relative elements from path (e.g. ..) 39 40 # qualifies the python 41 self.platform = u(sys.platform) 42 self.implementation = u(platform.python_implementation()) 43 if self.implementation == "PyPy": 44 self.pypy_version_info = tuple(u(i) for i in sys.pypy_version_info) 45 46 # this is a tuple in earlier, struct later, unify to our own named tuple 47 self.version_info = VersionInfo(*list(u(i) for i in sys.version_info)) 48 self.architecture = 64 if sys.maxsize > 2 ** 32 else 32 49 50 self.version = u(sys.version) 51 self.os = u(os.name) 52 53 # information about the prefix - determines python home 54 self.prefix = u(abs_path(getattr(sys, "prefix", None))) # prefix we think 55 self.base_prefix = u(abs_path(getattr(sys, "base_prefix", None))) # venv 56 self.real_prefix = u(abs_path(getattr(sys, "real_prefix", None))) # old virtualenv 57 58 # information about the exec prefix - dynamic stdlib modules 59 self.base_exec_prefix = u(abs_path(getattr(sys, "base_exec_prefix", None))) 60 self.exec_prefix = u(abs_path(getattr(sys, "exec_prefix", None))) 61 62 self.executable = u(abs_path(sys.executable)) # the executable we were invoked via 63 self.original_executable = u(abs_path(self.executable)) # the executable as known by the interpreter 64 self.system_executable = self._fast_get_system_executable() # the executable we are based of (if available) 65 66 try: 67 __import__("venv") 68 has = True 69 except ImportError: 70 has = False 71 self.has_venv = has 72 self.path = [u(i) for i in sys.path] 73 self.file_system_encoding = u(sys.getfilesystemencoding()) 74 self.stdout_encoding = u(getattr(sys.stdout, "encoding", None)) 75 76 self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()} 77 # https://bugs.python.org/issue22199 78 makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) 79 self.sysconfig = { 80 u(k): u(v) 81 for k, v in [ 82 # a list of content to store from sysconfig 83 ("makefile_filename", makefile()), 84 ] 85 if k is not None 86 } 87 88 config_var_keys = set() 89 for element in self.sysconfig_paths.values(): 90 for k in _CONF_VAR_RE.findall(element): 91 config_var_keys.add(u(k[1:-1])) 92 config_var_keys.add("PYTHONFRAMEWORK") 93 94 self.sysconfig_vars = {u(i): u(sysconfig.get_config_var(i) or "") for i in config_var_keys} 95 if self.implementation == "PyPy" and sys.version_info.major == 2: 96 self.sysconfig_vars[u"implementation_lower"] = u"python" 97 98 self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()} 99 confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()} 100 self.system_stdlib = self.sysconfig_path("stdlib", confs) 101 self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) 102 self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) 103 self._creators = None 104 105 def _fast_get_system_executable(self): 106 """Try to get the system executable by just looking at properties""" 107 if self.real_prefix or ( 108 self.base_prefix is not None and self.base_prefix != self.prefix 109 ): # if this is a virtual environment 110 if self.real_prefix is None: 111 base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us 112 if base_executable is not None: # use the saved system executable if present 113 if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us 114 return base_executable 115 return None # in this case we just can't tell easily without poking around FS and calling them, bail 116 # if we're not in a virtual environment, this is already a system python, so return the original executable 117 # note we must choose the original and not the pure executable as shim scripts might throw us off 118 return self.original_executable 119 120 def install_path(self, key): 121 result = self.distutils_install.get(key) 122 if result is None: # use sysconfig if distutils is unavailable 123 # set prefixes to empty => result is relative from cwd 124 prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix 125 config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()} 126 result = self.sysconfig_path(key, config_var=config_var).lstrip(os.sep) 127 # A hack for https://github.com/pypa/virtualenv/issues/2208 128 if result.startswith(u"local/"): 129 return result[6:] 130 return result 131 132 @staticmethod 133 def _distutils_install(): 134 # use distutils primarily because that's what pip does 135 # https://github.com/pypa/pip/blob/main/src/pip/_internal/locations.py#L95 136 # note here we don't import Distribution directly to allow setuptools to patch it 137 with warnings.catch_warnings(): # disable warning for PEP-632 138 warnings.simplefilter("ignore") 139 try: 140 from distutils import dist 141 from distutils.command.install import SCHEME_KEYS 142 except ImportError: # if removed or not installed ignore 143 return {} 144 145 d = dist.Distribution({"script_args": "--no-user-cfg"}) # conf files not parsed so they do not hijack paths 146 if hasattr(sys, "_framework"): 147 sys._framework = None # disable macOS static paths for framework 148 i = d.get_command_obj("install", create=True) 149 i.prefix = os.sep # paths generated are relative to prefix that contains the path sep, this makes it relative 150 i.finalize_options() 151 result = {key: (getattr(i, "install_{}".format(key))[1:]).lstrip(os.sep) for key in SCHEME_KEYS} 152 return result 153 154 @property 155 def version_str(self): 156 return ".".join(str(i) for i in self.version_info[0:3]) 157 158 @property 159 def version_release_str(self): 160 return ".".join(str(i) for i in self.version_info[0:2]) 161 162 @property 163 def python_name(self): 164 version_info = self.version_info 165 return "python{}.{}".format(version_info.major, version_info.minor) 166 167 @property 168 def is_old_virtualenv(self): 169 return self.real_prefix is not None 170 171 @property 172 def is_venv(self): 173 return self.base_prefix is not None and self.version_info.major == 3 174 175 def sysconfig_path(self, key, config_var=None, sep=os.sep): 176 pattern = self.sysconfig_paths[key] 177 if config_var is None: 178 config_var = self.sysconfig_vars 179 else: 180 base = {k: v for k, v in self.sysconfig_vars.items()} 181 base.update(config_var) 182 config_var = base 183 return pattern.format(**config_var).replace(u"/", sep) 184 185 def creators(self, refresh=False): 186 if self._creators is None or refresh is True: 187 from virtualenv.run.plugin.creators import CreatorSelector 188 189 self._creators = CreatorSelector.for_interpreter(self) 190 return self._creators 191 192 @property 193 def system_include(self): 194 path = self.sysconfig_path( 195 "include", 196 {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}, 197 ) 198 if not os.path.exists(path): # some broken packaging don't respect the sysconfig, fallback to distutils path 199 # the pattern include the distribution name too at the end, remove that via the parent call 200 fallback = os.path.join(self.prefix, os.path.dirname(self.install_path("headers"))) 201 if os.path.exists(fallback): 202 path = fallback 203 return path 204 205 @property 206 def system_prefix(self): 207 return self.real_prefix or self.base_prefix or self.prefix 208 209 @property 210 def system_exec_prefix(self): 211 return self.real_prefix or self.base_exec_prefix or self.exec_prefix 212 213 def __unicode__(self): 214 content = repr(self) 215 if sys.version_info == 2: 216 content = content.decode("utf-8") 217 return content 218 219 def __repr__(self): 220 return "{}({!r})".format( 221 self.__class__.__name__, 222 {k: v for k, v in self.__dict__.items() if not k.startswith("_")}, 223 ) 224 225 def __str__(self): 226 content = "{}({})".format( 227 self.__class__.__name__, 228 ", ".join( 229 "{}={}".format(k, v) 230 for k, v in ( 231 ("spec", self.spec), 232 ( 233 "system" 234 if self.system_executable is not None and self.system_executable != self.executable 235 else None, 236 self.system_executable, 237 ), 238 ( 239 "original" 240 if ( 241 self.original_executable != self.system_executable 242 and self.original_executable != self.executable 243 ) 244 else None, 245 self.original_executable, 246 ), 247 ("exe", self.executable), 248 ("platform", self.platform), 249 ("version", repr(self.version)), 250 ("encoding_fs_io", "{}-{}".format(self.file_system_encoding, self.stdout_encoding)), 251 ) 252 if k is not None 253 ), 254 ) 255 return content 256 257 @property 258 def spec(self): 259 return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) 260 261 @classmethod 262 def clear_cache(cls, app_data): 263 # this method is not used by itself, so here and called functions can import stuff locally 264 from virtualenv.discovery.cached_py_info import clear 265 266 clear(app_data) 267 cls._cache_exe_discovery.clear() 268 269 def satisfies(self, spec, impl_must_match): 270 """check if a given specification can be satisfied by the this python interpreter instance""" 271 if spec.path: 272 if self.executable == os.path.abspath(spec.path): 273 return True # if the path is a our own executable path we're done 274 if not spec.is_abs: 275 # if path set, and is not our original executable name, this does not match 276 basename = os.path.basename(self.original_executable) 277 spec_path = spec.path 278 if sys.platform == "win32": 279 basename, suffix = os.path.splitext(basename) 280 if spec_path.endswith(suffix): 281 spec_path = spec_path[: -len(suffix)] 282 if basename != spec_path: 283 return False 284 285 if impl_must_match: 286 if spec.implementation is not None and spec.implementation.lower() != self.implementation.lower(): 287 return False 288 289 if spec.architecture is not None and spec.architecture != self.architecture: 290 return False 291 292 for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)): 293 if req is not None and our is not None and our != req: 294 return False 295 return True 296 297 _current_system = None 298 _current = None 299 300 @classmethod 301 def current(cls, app_data=None): 302 """ 303 This locates the current host interpreter information. This might be different than what we run into in case 304 the host python has been upgraded from underneath us. 305 """ 306 if cls._current is None: 307 cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False) 308 return cls._current 309 310 @classmethod 311 def current_system(cls, app_data=None): 312 """ 313 This locates the current host interpreter information. This might be different than what we run into in case 314 the host python has been upgraded from underneath us. 315 """ 316 if cls._current_system is None: 317 cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True) 318 return cls._current_system 319 320 def _to_json(self): 321 # don't save calculated paths, as these are non primitive types 322 return json.dumps(self._to_dict(), indent=2) 323 324 def _to_dict(self): 325 data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)} 326 # noinspection PyProtectedMember 327 data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary 328 return data 329 330 @classmethod 331 def from_exe(cls, exe, app_data=None, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None): 332 """Given a path to an executable get the python information""" 333 # this method is not used by itself, so here and called functions can import stuff locally 334 from virtualenv.discovery.cached_py_info import from_exe 335 336 env = os.environ if env is None else env 337 proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache) 338 # noinspection PyProtectedMember 339 if isinstance(proposed, PythonInfo) and resolve_to_host: 340 try: 341 proposed = proposed._resolve_to_system(app_data, proposed) 342 except Exception as exception: 343 if raise_on_error: 344 raise exception 345 logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) 346 proposed = None 347 return proposed 348 349 @classmethod 350 def _from_json(cls, payload): 351 # the dictionary unroll here is to protect against pypy bug of interpreter crashing 352 raw = json.loads(payload) 353 return cls._from_dict({k: v for k, v in raw.items()}) 354 355 @classmethod 356 def _from_dict(cls, data): 357 data["version_info"] = VersionInfo(**data["version_info"]) # restore this to a named tuple structure 358 result = cls() 359 result.__dict__ = {k: v for k, v in data.items()} 360 return result 361 362 @classmethod 363 def _resolve_to_system(cls, app_data, target): 364 start_executable = target.executable 365 prefixes = OrderedDict() 366 while target.system_executable is None: 367 prefix = target.real_prefix or target.base_prefix or target.prefix 368 if prefix in prefixes: 369 if len(prefixes) == 1: 370 # if we're linking back to ourselves accept ourselves with a WARNING 371 logging.info("%r links back to itself via prefixes", target) 372 target.system_executable = target.executable 373 break 374 for at, (p, t) in enumerate(prefixes.items(), start=1): 375 logging.error("%d: prefix=%s, info=%r", at, p, t) 376 logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) 377 raise RuntimeError("prefixes are causing a circle {}".format("|".join(prefixes.keys()))) 378 prefixes[prefix] = target 379 target = target.discover_exe(app_data, prefix=prefix, exact=False) 380 if target.executable != target.system_executable: 381 target = cls.from_exe(target.system_executable, app_data) 382 target.executable = start_executable 383 return target 384 385 _cache_exe_discovery = {} 386 387 def discover_exe(self, app_data, prefix, exact=True, env=None): 388 key = prefix, exact 389 if key in self._cache_exe_discovery and prefix: 390 logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) 391 return self._cache_exe_discovery[key] 392 logging.debug("discover exe for %s in %s", self, prefix) 393 # we don't know explicitly here, do some guess work - our executable name should tell 394 possible_names = self._find_possible_exe_names() 395 possible_folders = self._find_possible_folders(prefix) 396 discovered = [] 397 env = os.environ if env is None else env 398 for folder in possible_folders: 399 for name in possible_names: 400 info = self._check_exe(app_data, folder, name, exact, discovered, env) 401 if info is not None: 402 self._cache_exe_discovery[key] = info 403 return info 404 if exact is False and discovered: 405 info = self._select_most_likely(discovered, self) 406 folders = os.pathsep.join(possible_folders) 407 self._cache_exe_discovery[key] = info 408 logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) 409 return info 410 msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) 411 raise RuntimeError(msg) 412 413 def _check_exe(self, app_data, folder, name, exact, discovered, env): 414 exe_path = os.path.join(folder, name) 415 if not os.path.exists(exe_path): 416 return None 417 info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env) 418 if info is None: # ignore if for some reason we can't query 419 return None 420 for item in ["implementation", "architecture", "version_info"]: 421 found = getattr(info, item) 422 searched = getattr(self, item) 423 if found != searched: 424 if item == "version_info": 425 found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) 426 executable = info.executable 427 logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) 428 if exact is False: 429 discovered.append(info) 430 break 431 else: 432 return info 433 return None 434 435 @staticmethod 436 def _select_most_likely(discovered, target): 437 # no exact match found, start relaxing our requirements then to facilitate system package upgrades that 438 # could cause this (when using copy strategy of the host python) 439 def sort_by(info): 440 # we need to setup some priority of traits, this is as follows: 441 # implementation, major, minor, micro, architecture, tag, serial 442 matches = [ 443 info.implementation == target.implementation, 444 info.version_info.major == target.version_info.major, 445 info.version_info.minor == target.version_info.minor, 446 info.architecture == target.architecture, 447 info.version_info.micro == target.version_info.micro, 448 info.version_info.releaselevel == target.version_info.releaselevel, 449 info.version_info.serial == target.version_info.serial, 450 ] 451 priority = sum((1 << pos if match else 0) for pos, match in enumerate(reversed(matches))) 452 return priority 453 454 sorted_discovered = sorted(discovered, key=sort_by, reverse=True) # sort by priority in decreasing order 455 most_likely = sorted_discovered[0] 456 return most_likely 457 458 def _find_possible_folders(self, inside_folder): 459 candidate_folder = OrderedDict() 460 executables = OrderedDict() 461 executables[os.path.realpath(self.executable)] = None 462 executables[self.executable] = None 463 executables[os.path.realpath(self.original_executable)] = None 464 executables[self.original_executable] = None 465 for exe in executables.keys(): 466 base = os.path.dirname(exe) 467 # following path pattern of the current 468 if base.startswith(self.prefix): 469 relative = base[len(self.prefix) :] 470 candidate_folder["{}{}".format(inside_folder, relative)] = None 471 472 # or at root level 473 candidate_folder[inside_folder] = None 474 return list(i for i in candidate_folder.keys() if os.path.exists(i)) 475 476 def _find_possible_exe_names(self): 477 name_candidate = OrderedDict() 478 for name in self._possible_base(): 479 for at in (3, 2, 1, 0): 480 version = ".".join(str(i) for i in self.version_info[:at]) 481 for arch in ["-{}".format(self.architecture), ""]: 482 for ext in EXTENSIONS: 483 candidate = "{}{}{}{}".format(name, version, arch, ext) 484 name_candidate[candidate] = None 485 return list(name_candidate.keys()) 486 487 def _possible_base(self): 488 possible_base = OrderedDict() 489 basename = os.path.splitext(os.path.basename(self.executable))[0].rstrip(digits) 490 possible_base[basename] = None 491 possible_base[self.implementation] = None 492 # python is always the final option as in practice is used by multiple implementation as exe name 493 if "python" in possible_base: 494 del possible_base["python"] 495 possible_base["python"] = None 496 for base in possible_base: 497 lower = base.lower() 498 yield lower 499 from virtualenv.info import fs_is_case_sensitive 500 501 if fs_is_case_sensitive(): 502 if base != lower: 503 yield base 504 upper = base.upper() 505 if upper != base: 506 yield upper 507 508 509if __name__ == "__main__": 510 # dump a JSON representation of the current python 511 # noinspection PyProtectedMember 512 print(PythonInfo()._to_json()) 513