1import collections 2import logging 3import os 4from typing import ( 5 Container, 6 Dict, 7 Iterable, 8 Iterator, 9 List, 10 NamedTuple, 11 Optional, 12 Set, 13 Union, 14) 15 16from pip._vendor.packaging.requirements import Requirement 17from pip._vendor.packaging.utils import canonicalize_name 18from pip._vendor.packaging.version import Version 19 20from pip._internal.exceptions import BadCommand, InstallationError 21from pip._internal.metadata import BaseDistribution, get_environment 22from pip._internal.req.constructors import ( 23 install_req_from_editable, 24 install_req_from_line, 25) 26from pip._internal.req.req_file import COMMENT_RE 27from pip._internal.utils.direct_url_helpers import direct_url_as_pep440_direct_reference 28 29logger = logging.getLogger(__name__) 30 31 32class _EditableInfo(NamedTuple): 33 requirement: Optional[str] 34 editable: bool 35 comments: List[str] 36 37 38def freeze( 39 requirement=None, # type: Optional[List[str]] 40 local_only=False, # type: bool 41 user_only=False, # type: bool 42 paths=None, # type: Optional[List[str]] 43 isolated=False, # type: bool 44 exclude_editable=False, # type: bool 45 skip=() # type: Container[str] 46): 47 # type: (...) -> Iterator[str] 48 installations = {} # type: Dict[str, FrozenRequirement] 49 50 dists = get_environment(paths).iter_installed_distributions( 51 local_only=local_only, 52 skip=(), 53 user_only=user_only, 54 ) 55 for dist in dists: 56 req = FrozenRequirement.from_dist(dist) 57 if exclude_editable and req.editable: 58 continue 59 installations[req.canonical_name] = req 60 61 if requirement: 62 # the options that don't get turned into an InstallRequirement 63 # should only be emitted once, even if the same option is in multiple 64 # requirements files, so we need to keep track of what has been emitted 65 # so that we don't emit it again if it's seen again 66 emitted_options = set() # type: Set[str] 67 # keep track of which files a requirement is in so that we can 68 # give an accurate warning if a requirement appears multiple times. 69 req_files = collections.defaultdict(list) # type: Dict[str, List[str]] 70 for req_file_path in requirement: 71 with open(req_file_path) as req_file: 72 for line in req_file: 73 if (not line.strip() or 74 line.strip().startswith('#') or 75 line.startswith(( 76 '-r', '--requirement', 77 '-f', '--find-links', 78 '-i', '--index-url', 79 '--pre', 80 '--trusted-host', 81 '--process-dependency-links', 82 '--extra-index-url', 83 '--use-feature'))): 84 line = line.rstrip() 85 if line not in emitted_options: 86 emitted_options.add(line) 87 yield line 88 continue 89 90 if line.startswith('-e') or line.startswith('--editable'): 91 if line.startswith('-e'): 92 line = line[2:].strip() 93 else: 94 line = line[len('--editable'):].strip().lstrip('=') 95 line_req = install_req_from_editable( 96 line, 97 isolated=isolated, 98 ) 99 else: 100 line_req = install_req_from_line( 101 COMMENT_RE.sub('', line).strip(), 102 isolated=isolated, 103 ) 104 105 if not line_req.name: 106 logger.info( 107 "Skipping line in requirement file [%s] because " 108 "it's not clear what it would install: %s", 109 req_file_path, line.strip(), 110 ) 111 logger.info( 112 " (add #egg=PackageName to the URL to avoid" 113 " this warning)" 114 ) 115 else: 116 line_req_canonical_name = canonicalize_name( 117 line_req.name) 118 if line_req_canonical_name not in installations: 119 # either it's not installed, or it is installed 120 # but has been processed already 121 if not req_files[line_req.name]: 122 logger.warning( 123 "Requirement file [%s] contains %s, but " 124 "package %r is not installed", 125 req_file_path, 126 COMMENT_RE.sub('', line).strip(), 127 line_req.name 128 ) 129 else: 130 req_files[line_req.name].append(req_file_path) 131 else: 132 yield str(installations[ 133 line_req_canonical_name]).rstrip() 134 del installations[line_req_canonical_name] 135 req_files[line_req.name].append(req_file_path) 136 137 # Warn about requirements that were included multiple times (in a 138 # single requirements file or in different requirements files). 139 for name, files in req_files.items(): 140 if len(files) > 1: 141 logger.warning("Requirement %s included multiple times [%s]", 142 name, ', '.join(sorted(set(files)))) 143 144 yield( 145 '## The following requirements were added by ' 146 'pip freeze:' 147 ) 148 for installation in sorted( 149 installations.values(), key=lambda x: x.name.lower()): 150 if installation.canonical_name not in skip: 151 yield str(installation).rstrip() 152 153 154def _format_as_name_version(dist: BaseDistribution) -> str: 155 if isinstance(dist.version, Version): 156 return f"{dist.raw_name}=={dist.version}" 157 return f"{dist.raw_name}==={dist.version}" 158 159 160def _get_editable_info(dist: BaseDistribution) -> _EditableInfo: 161 """ 162 Compute and return values (req, editable, comments) for use in 163 FrozenRequirement.from_dist(). 164 """ 165 if not dist.editable: 166 return _EditableInfo(requirement=None, editable=False, comments=[]) 167 if dist.location is None: 168 display = _format_as_name_version(dist) 169 logger.warning("Editable requirement not found on disk: %s", display) 170 return _EditableInfo( 171 requirement=None, 172 editable=True, 173 comments=[f"# Editable install not found ({display})"], 174 ) 175 176 location = os.path.normcase(os.path.abspath(dist.location)) 177 178 from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs 179 180 vcs_backend = vcs.get_backend_for_dir(location) 181 182 if vcs_backend is None: 183 display = _format_as_name_version(dist) 184 logger.debug( 185 'No VCS found for editable requirement "%s" in: %r', display, 186 location, 187 ) 188 return _EditableInfo( 189 requirement=location, 190 editable=True, 191 comments=[f'# Editable install with no version control ({display})'], 192 ) 193 194 vcs_name = type(vcs_backend).__name__ 195 196 try: 197 req = vcs_backend.get_src_requirement(location, dist.raw_name) 198 except RemoteNotFoundError: 199 display = _format_as_name_version(dist) 200 return _EditableInfo( 201 requirement=location, 202 editable=True, 203 comments=[f'# Editable {vcs_name} install with no remote ({display})'], 204 ) 205 except RemoteNotValidError as ex: 206 display = _format_as_name_version(dist) 207 return _EditableInfo( 208 requirement=location, 209 editable=True, 210 comments=[ 211 f"# Editable {vcs_name} install ({display}) with either a deleted " 212 f"local remote or invalid URI:", 213 f"# '{ex.url}'", 214 ], 215 ) 216 217 except BadCommand: 218 logger.warning( 219 'cannot determine version of editable source in %s ' 220 '(%s command not found in path)', 221 location, 222 vcs_backend.name, 223 ) 224 return _EditableInfo(requirement=None, editable=True, comments=[]) 225 226 except InstallationError as exc: 227 logger.warning( 228 "Error when trying to get requirement for VCS system %s, " 229 "falling back to uneditable format", exc 230 ) 231 else: 232 return _EditableInfo(requirement=req, editable=True, comments=[]) 233 234 logger.warning('Could not determine repository location of %s', location) 235 236 return _EditableInfo( 237 requirement=None, 238 editable=False, 239 comments=['## !! Could not determine repository location'], 240 ) 241 242 243class FrozenRequirement: 244 def __init__(self, name, req, editable, comments=()): 245 # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None 246 self.name = name 247 self.canonical_name = canonicalize_name(name) 248 self.req = req 249 self.editable = editable 250 self.comments = comments 251 252 @classmethod 253 def from_dist(cls, dist: BaseDistribution) -> "FrozenRequirement": 254 # TODO `get_requirement_info` is taking care of editable requirements. 255 # TODO This should be refactored when we will add detection of 256 # editable that provide .dist-info metadata. 257 req, editable, comments = _get_editable_info(dist) 258 if req is None and not editable: 259 # if PEP 610 metadata is present, attempt to use it 260 direct_url = dist.direct_url 261 if direct_url: 262 req = direct_url_as_pep440_direct_reference( 263 direct_url, dist.raw_name 264 ) 265 comments = [] 266 if req is None: 267 # name==version requirement 268 req = _format_as_name_version(dist) 269 270 return cls(dist.raw_name, req, editable, comments=comments) 271 272 def __str__(self): 273 # type: () -> str 274 req = self.req 275 if self.editable: 276 req = f'-e {req}' 277 return '\n'.join(list(self.comments) + [str(req)]) + '\n' 278