1import functools 2import logging 3import os 4import pathlib 5import sys 6import sysconfig 7from typing import Any, Dict, Iterator, List, Optional, Tuple 8 9from pip._internal.models.scheme import SCHEME_KEYS, Scheme 10from pip._internal.utils.compat import WINDOWS 11from pip._internal.utils.deprecation import deprecated 12from pip._internal.utils.virtualenv import running_under_virtualenv 13 14from . import _distutils, _sysconfig 15from .base import ( 16 USER_CACHE_DIR, 17 get_major_minor_version, 18 get_src_prefix, 19 is_osx_framework, 20 site_packages, 21 user_site, 22) 23 24__all__ = [ 25 "USER_CACHE_DIR", 26 "get_bin_prefix", 27 "get_bin_user", 28 "get_major_minor_version", 29 "get_platlib", 30 "get_prefixed_libs", 31 "get_purelib", 32 "get_scheme", 33 "get_src_prefix", 34 "site_packages", 35 "user_site", 36] 37 38 39logger = logging.getLogger(__name__) 40 41if os.environ.get("_PIP_LOCATIONS_NO_WARN_ON_MISMATCH"): 42 _MISMATCH_LEVEL = logging.DEBUG 43else: 44 _MISMATCH_LEVEL = logging.WARNING 45 46_PLATLIBDIR: str = getattr(sys, "platlibdir", "lib") 47 48 49def _looks_like_bpo_44860() -> bool: 50 """The resolution to bpo-44860 will change this incorrect platlib. 51 52 See <https://bugs.python.org/issue44860>. 53 """ 54 from distutils.command.install import INSTALL_SCHEMES # type: ignore 55 56 try: 57 unix_user_platlib = INSTALL_SCHEMES["unix_user"]["platlib"] 58 except KeyError: 59 return False 60 return unix_user_platlib == "$usersite" 61 62 63def _looks_like_red_hat_patched_platlib_purelib(scheme: Dict[str, str]) -> bool: 64 platlib = scheme["platlib"] 65 if "/lib64/" not in platlib: 66 return False 67 unpatched = platlib.replace("/lib64/", "/lib/") 68 return unpatched.replace("$platbase/", "$base/") == scheme["purelib"] 69 70 71@functools.lru_cache(maxsize=None) 72def _looks_like_red_hat_lib() -> bool: 73 """Red Hat patches platlib in unix_prefix and unix_home, but not purelib. 74 75 This is the only way I can see to tell a Red Hat-patched Python. 76 """ 77 from distutils.command.install import INSTALL_SCHEMES # type: ignore 78 79 return all( 80 k in INSTALL_SCHEMES 81 and _looks_like_red_hat_patched_platlib_purelib(INSTALL_SCHEMES[k]) 82 for k in ("unix_prefix", "unix_home") 83 ) 84 85 86@functools.lru_cache(maxsize=None) 87def _looks_like_debian_scheme() -> bool: 88 """Debian adds two additional schemes.""" 89 from distutils.command.install import INSTALL_SCHEMES # type: ignore 90 91 return "deb_system" in INSTALL_SCHEMES and "unix_local" in INSTALL_SCHEMES 92 93 94@functools.lru_cache(maxsize=None) 95def _looks_like_red_hat_scheme() -> bool: 96 """Red Hat patches ``sys.prefix`` and ``sys.exec_prefix``. 97 98 Red Hat's ``00251-change-user-install-location.patch`` changes the install 99 command's ``prefix`` and ``exec_prefix`` to append ``"/local"``. This is 100 (fortunately?) done quite unconditionally, so we create a default command 101 object without any configuration to detect this. 102 """ 103 from distutils.command.install import install 104 from distutils.dist import Distribution 105 106 cmd: Any = install(Distribution()) 107 cmd.finalize_options() 108 return ( 109 cmd.exec_prefix == f"{os.path.normpath(sys.exec_prefix)}/local" 110 and cmd.prefix == f"{os.path.normpath(sys.prefix)}/local" 111 ) 112 113 114@functools.lru_cache(maxsize=None) 115def _looks_like_msys2_mingw_scheme() -> bool: 116 """MSYS2 patches distutils and sysconfig to use a UNIX-like scheme. 117 118 However, MSYS2 incorrectly patches sysconfig ``nt`` scheme. The fix is 119 likely going to be included in their 3.10 release, so we ignore the warning. 120 See msys2/MINGW-packages#9319. 121 122 MSYS2 MINGW's patch uses lowercase ``"lib"`` instead of the usual uppercase, 123 and is missing the final ``"site-packages"``. 124 """ 125 paths = sysconfig.get_paths("nt", expand=False) 126 return all( 127 "Lib" not in p and "lib" in p and not p.endswith("site-packages") 128 for p in (paths[key] for key in ("platlib", "purelib")) 129 ) 130 131 132def _fix_abiflags(parts: Tuple[str]) -> Iterator[str]: 133 ldversion = sysconfig.get_config_var("LDVERSION") 134 abiflags: str = getattr(sys, "abiflags", None) 135 136 # LDVERSION does not end with sys.abiflags. Just return the path unchanged. 137 if not ldversion or not abiflags or not ldversion.endswith(abiflags): 138 yield from parts 139 return 140 141 # Strip sys.abiflags from LDVERSION-based path components. 142 for part in parts: 143 if part.endswith(ldversion): 144 part = part[: (0 - len(abiflags))] 145 yield part 146 147 148@functools.lru_cache(maxsize=None) 149def _warn_mismatched(old: pathlib.Path, new: pathlib.Path, *, key: str) -> None: 150 issue_url = "https://github.com/pypa/pip/issues/10151" 151 message = ( 152 "Value for %s does not match. Please report this to <%s>" 153 "\ndistutils: %s" 154 "\nsysconfig: %s" 155 ) 156 logger.log(_MISMATCH_LEVEL, message, key, issue_url, old, new) 157 158 159def _warn_if_mismatch(old: pathlib.Path, new: pathlib.Path, *, key: str) -> bool: 160 if old == new: 161 return False 162 _warn_mismatched(old, new, key=key) 163 return True 164 165 166@functools.lru_cache(maxsize=None) 167def _log_context( 168 *, 169 user: bool = False, 170 home: Optional[str] = None, 171 root: Optional[str] = None, 172 prefix: Optional[str] = None, 173) -> None: 174 parts = [ 175 "Additional context:", 176 "user = %r", 177 "home = %r", 178 "root = %r", 179 "prefix = %r", 180 ] 181 182 logger.log(_MISMATCH_LEVEL, "\n".join(parts), user, home, root, prefix) 183 184 185def get_scheme( 186 dist_name: str, 187 user: bool = False, 188 home: Optional[str] = None, 189 root: Optional[str] = None, 190 isolated: bool = False, 191 prefix: Optional[str] = None, 192) -> Scheme: 193 old = _distutils.get_scheme( 194 dist_name, 195 user=user, 196 home=home, 197 root=root, 198 isolated=isolated, 199 prefix=prefix, 200 ) 201 new = _sysconfig.get_scheme( 202 dist_name, 203 user=user, 204 home=home, 205 root=root, 206 isolated=isolated, 207 prefix=prefix, 208 ) 209 210 warning_contexts = [] 211 for k in SCHEME_KEYS: 212 old_v = pathlib.Path(getattr(old, k)) 213 new_v = pathlib.Path(getattr(new, k)) 214 215 if old_v == new_v: 216 continue 217 218 # distutils incorrectly put PyPy packages under ``site-packages/python`` 219 # in the ``posix_home`` scheme, but PyPy devs said they expect the 220 # directory name to be ``pypy`` instead. So we treat this as a bug fix 221 # and not warn about it. See bpo-43307 and python/cpython#24628. 222 skip_pypy_special_case = ( 223 sys.implementation.name == "pypy" 224 and home is not None 225 and k in ("platlib", "purelib") 226 and old_v.parent == new_v.parent 227 and old_v.name.startswith("python") 228 and new_v.name.startswith("pypy") 229 ) 230 if skip_pypy_special_case: 231 continue 232 233 # sysconfig's ``osx_framework_user`` does not include ``pythonX.Y`` in 234 # the ``include`` value, but distutils's ``headers`` does. We'll let 235 # CPython decide whether this is a bug or feature. See bpo-43948. 236 skip_osx_framework_user_special_case = ( 237 user 238 and is_osx_framework() 239 and k == "headers" 240 and old_v.parent.parent == new_v.parent 241 and old_v.parent.name.startswith("python") 242 ) 243 if skip_osx_framework_user_special_case: 244 continue 245 246 # On Red Hat and derived Linux distributions, distutils is patched to 247 # use "lib64" instead of "lib" for platlib. 248 if k == "platlib" and _looks_like_red_hat_lib(): 249 continue 250 251 # On Python 3.9+, sysconfig's posix_user scheme sets platlib against 252 # sys.platlibdir, but distutils's unix_user incorrectly coninutes 253 # using the same $usersite for both platlib and purelib. This creates a 254 # mismatch when sys.platlibdir is not "lib". 255 skip_bpo_44860 = ( 256 user 257 and k == "platlib" 258 and not WINDOWS 259 and sys.version_info >= (3, 9) 260 and _PLATLIBDIR != "lib" 261 and _looks_like_bpo_44860() 262 ) 263 if skip_bpo_44860: 264 continue 265 266 # Both Debian and Red Hat patch Python to place the system site under 267 # /usr/local instead of /usr. Debian also places lib in dist-packages 268 # instead of site-packages, but the /usr/local check should cover it. 269 skip_linux_system_special_case = ( 270 not (user or home or prefix or running_under_virtualenv()) 271 and old_v.parts[1:3] == ("usr", "local") 272 and len(new_v.parts) > 1 273 and new_v.parts[1] == "usr" 274 and (len(new_v.parts) < 3 or new_v.parts[2] != "local") 275 and (_looks_like_red_hat_scheme() or _looks_like_debian_scheme()) 276 ) 277 if skip_linux_system_special_case: 278 continue 279 280 # On Python 3.7 and earlier, sysconfig does not include sys.abiflags in 281 # the "pythonX.Y" part of the path, but distutils does. 282 skip_sysconfig_abiflag_bug = ( 283 sys.version_info < (3, 8) 284 and not WINDOWS 285 and k in ("headers", "platlib", "purelib") 286 and tuple(_fix_abiflags(old_v.parts)) == new_v.parts 287 ) 288 if skip_sysconfig_abiflag_bug: 289 continue 290 291 # MSYS2 MINGW's sysconfig patch does not include the "site-packages" 292 # part of the path. This is incorrect and will be fixed in MSYS. 293 skip_msys2_mingw_bug = ( 294 WINDOWS and k in ("platlib", "purelib") and _looks_like_msys2_mingw_scheme() 295 ) 296 if skip_msys2_mingw_bug: 297 continue 298 299 warning_contexts.append((old_v, new_v, f"scheme.{k}")) 300 301 if not warning_contexts: 302 return old 303 304 # Check if this path mismatch is caused by distutils config files. Those 305 # files will no longer work once we switch to sysconfig, so this raises a 306 # deprecation message for them. 307 default_old = _distutils.distutils_scheme( 308 dist_name, 309 user, 310 home, 311 root, 312 isolated, 313 prefix, 314 ignore_config_files=True, 315 ) 316 if any(default_old[k] != getattr(old, k) for k in SCHEME_KEYS): 317 deprecated( 318 "Configuring installation scheme with distutils config files " 319 "is deprecated and will no longer work in the near future. If you " 320 "are using a Homebrew or Linuxbrew Python, please see discussion " 321 "at https://github.com/Homebrew/homebrew-core/issues/76621", 322 replacement=None, 323 gone_in=None, 324 ) 325 return old 326 327 # Post warnings about this mismatch so user can report them back. 328 for old_v, new_v, key in warning_contexts: 329 _warn_mismatched(old_v, new_v, key=key) 330 _log_context(user=user, home=home, root=root, prefix=prefix) 331 332 return old 333 334 335def get_bin_prefix() -> str: 336 old = _distutils.get_bin_prefix() 337 new = _sysconfig.get_bin_prefix() 338 if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="bin_prefix"): 339 _log_context() 340 return old 341 342 343def get_bin_user() -> str: 344 return _sysconfig.get_scheme("", user=True).scripts 345 346 347def _looks_like_deb_system_dist_packages(value: str) -> bool: 348 """Check if the value is Debian's APT-controlled dist-packages. 349 350 Debian's ``distutils.sysconfig.get_python_lib()`` implementation returns the 351 default package path controlled by APT, but does not patch ``sysconfig`` to 352 do the same. This is similar to the bug worked around in ``get_scheme()``, 353 but here the default is ``deb_system`` instead of ``unix_local``. Ultimately 354 we can't do anything about this Debian bug, and this detection allows us to 355 skip the warning when needed. 356 """ 357 if not _looks_like_debian_scheme(): 358 return False 359 if value == "/usr/lib/python3/dist-packages": 360 return True 361 return False 362 363 364def get_purelib() -> str: 365 """Return the default pure-Python lib location.""" 366 old = _distutils.get_purelib() 367 new = _sysconfig.get_purelib() 368 if _looks_like_deb_system_dist_packages(old): 369 return old 370 if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="purelib"): 371 _log_context() 372 return old 373 374 375def get_platlib() -> str: 376 """Return the default platform-shared lib location.""" 377 old = _distutils.get_platlib() 378 new = _sysconfig.get_platlib() 379 if _looks_like_deb_system_dist_packages(old): 380 return old 381 if _warn_if_mismatch(pathlib.Path(old), pathlib.Path(new), key="platlib"): 382 _log_context() 383 return old 384 385 386def get_prefixed_libs(prefix: str) -> List[str]: 387 """Return the lib locations under ``prefix``.""" 388 old_pure, old_plat = _distutils.get_prefixed_libs(prefix) 389 new_pure, new_plat = _sysconfig.get_prefixed_libs(prefix) 390 391 warned = [ 392 _warn_if_mismatch( 393 pathlib.Path(old_pure), 394 pathlib.Path(new_pure), 395 key="prefixed-purelib", 396 ), 397 _warn_if_mismatch( 398 pathlib.Path(old_plat), 399 pathlib.Path(new_plat), 400 key="prefixed-platlib", 401 ), 402 ] 403 if any(warned): 404 _log_context(prefix=prefix) 405 406 if old_pure == old_plat: 407 return [old_pure] 408 return [old_pure, old_plat] 409