1import collections 2import logging 3import os 4 5from pip._vendor.packaging.utils import canonicalize_name 6from pip._vendor.pkg_resources import RequirementParseError 7 8from pip._internal.exceptions import BadCommand, InstallationError 9from pip._internal.req.constructors import ( 10 install_req_from_editable, 11 install_req_from_line, 12) 13from pip._internal.req.req_file import COMMENT_RE 14from pip._internal.utils.direct_url_helpers import ( 15 direct_url_as_pep440_direct_reference, 16 dist_get_direct_url, 17) 18from pip._internal.utils.misc import dist_is_editable, get_installed_distributions 19from pip._internal.utils.typing import MYPY_CHECK_RUNNING 20 21if MYPY_CHECK_RUNNING: 22 from typing import ( 23 Container, 24 Dict, 25 Iterable, 26 Iterator, 27 List, 28 Optional, 29 Set, 30 Tuple, 31 Union, 32 ) 33 34 from pip._vendor.pkg_resources import Distribution, Requirement 35 36 RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] 37 38 39logger = logging.getLogger(__name__) 40 41 42def freeze( 43 requirement=None, # type: Optional[List[str]] 44 find_links=None, # type: Optional[List[str]] 45 local_only=False, # type: bool 46 user_only=False, # type: bool 47 paths=None, # type: Optional[List[str]] 48 isolated=False, # type: bool 49 exclude_editable=False, # type: bool 50 skip=() # type: Container[str] 51): 52 # type: (...) -> Iterator[str] 53 find_links = find_links or [] 54 55 for link in find_links: 56 yield f'-f {link}' 57 installations = {} # type: Dict[str, FrozenRequirement] 58 59 for dist in get_installed_distributions( 60 local_only=local_only, 61 skip=(), 62 user_only=user_only, 63 paths=paths 64 ): 65 try: 66 req = FrozenRequirement.from_dist(dist) 67 except RequirementParseError as exc: 68 # We include dist rather than dist.project_name because the 69 # dist string includes more information, like the version and 70 # location. We also include the exception message to aid 71 # troubleshooting. 72 logger.warning( 73 'Could not generate requirement for distribution %r: %s', 74 dist, exc 75 ) 76 continue 77 if exclude_editable and req.editable: 78 continue 79 installations[req.canonical_name] = req 80 81 if requirement: 82 # the options that don't get turned into an InstallRequirement 83 # should only be emitted once, even if the same option is in multiple 84 # requirements files, so we need to keep track of what has been emitted 85 # so that we don't emit it again if it's seen again 86 emitted_options = set() # type: Set[str] 87 # keep track of which files a requirement is in so that we can 88 # give an accurate warning if a requirement appears multiple times. 89 req_files = collections.defaultdict(list) # type: Dict[str, List[str]] 90 for req_file_path in requirement: 91 with open(req_file_path) as req_file: 92 for line in req_file: 93 if (not line.strip() or 94 line.strip().startswith('#') or 95 line.startswith(( 96 '-r', '--requirement', 97 '-f', '--find-links', 98 '-i', '--index-url', 99 '--pre', 100 '--trusted-host', 101 '--process-dependency-links', 102 '--extra-index-url', 103 '--use-feature'))): 104 line = line.rstrip() 105 if line not in emitted_options: 106 emitted_options.add(line) 107 yield line 108 continue 109 110 if line.startswith('-e') or line.startswith('--editable'): 111 if line.startswith('-e'): 112 line = line[2:].strip() 113 else: 114 line = line[len('--editable'):].strip().lstrip('=') 115 line_req = install_req_from_editable( 116 line, 117 isolated=isolated, 118 ) 119 else: 120 line_req = install_req_from_line( 121 COMMENT_RE.sub('', line).strip(), 122 isolated=isolated, 123 ) 124 125 if not line_req.name: 126 logger.info( 127 "Skipping line in requirement file [%s] because " 128 "it's not clear what it would install: %s", 129 req_file_path, line.strip(), 130 ) 131 logger.info( 132 " (add #egg=PackageName to the URL to avoid" 133 " this warning)" 134 ) 135 else: 136 line_req_canonical_name = canonicalize_name( 137 line_req.name) 138 if line_req_canonical_name not in installations: 139 # either it's not installed, or it is installed 140 # but has been processed already 141 if not req_files[line_req.name]: 142 logger.warning( 143 "Requirement file [%s] contains %s, but " 144 "package %r is not installed", 145 req_file_path, 146 COMMENT_RE.sub('', line).strip(), 147 line_req.name 148 ) 149 else: 150 req_files[line_req.name].append(req_file_path) 151 else: 152 yield str(installations[ 153 line_req_canonical_name]).rstrip() 154 del installations[line_req_canonical_name] 155 req_files[line_req.name].append(req_file_path) 156 157 # Warn about requirements that were included multiple times (in a 158 # single requirements file or in different requirements files). 159 for name, files in req_files.items(): 160 if len(files) > 1: 161 logger.warning("Requirement %s included multiple times [%s]", 162 name, ', '.join(sorted(set(files)))) 163 164 yield( 165 '## The following requirements were added by ' 166 'pip freeze:' 167 ) 168 for installation in sorted( 169 installations.values(), key=lambda x: x.name.lower()): 170 if installation.canonical_name not in skip: 171 yield str(installation).rstrip() 172 173 174def get_requirement_info(dist): 175 # type: (Distribution) -> RequirementInfo 176 """ 177 Compute and return values (req, editable, comments) for use in 178 FrozenRequirement.from_dist(). 179 """ 180 if not dist_is_editable(dist): 181 return (None, False, []) 182 183 location = os.path.normcase(os.path.abspath(dist.location)) 184 185 from pip._internal.vcs import RemoteNotFoundError, vcs 186 vcs_backend = vcs.get_backend_for_dir(location) 187 188 if vcs_backend is None: 189 req = dist.as_requirement() 190 logger.debug( 191 'No VCS found for editable requirement "%s" in: %r', req, 192 location, 193 ) 194 comments = [ 195 f'# Editable install with no version control ({req})' 196 ] 197 return (location, True, comments) 198 199 try: 200 req = vcs_backend.get_src_requirement(location, dist.project_name) 201 except RemoteNotFoundError: 202 req = dist.as_requirement() 203 comments = [ 204 '# Editable {} install with no remote ({})'.format( 205 type(vcs_backend).__name__, req, 206 ) 207 ] 208 return (location, True, comments) 209 210 except BadCommand: 211 logger.warning( 212 'cannot determine version of editable source in %s ' 213 '(%s command not found in path)', 214 location, 215 vcs_backend.name, 216 ) 217 return (None, True, []) 218 219 except InstallationError as exc: 220 logger.warning( 221 "Error when trying to get requirement for VCS system %s, " 222 "falling back to uneditable format", exc 223 ) 224 else: 225 return (req, True, []) 226 227 logger.warning( 228 'Could not determine repository location of %s', location 229 ) 230 comments = ['## !! Could not determine repository location'] 231 232 return (None, False, comments) 233 234 235class FrozenRequirement: 236 def __init__(self, name, req, editable, comments=()): 237 # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None 238 self.name = name 239 self.canonical_name = canonicalize_name(name) 240 self.req = req 241 self.editable = editable 242 self.comments = comments 243 244 @classmethod 245 def from_dist(cls, dist): 246 # type: (Distribution) -> FrozenRequirement 247 # TODO `get_requirement_info` is taking care of editable requirements. 248 # TODO This should be refactored when we will add detection of 249 # editable that provide .dist-info metadata. 250 req, editable, comments = get_requirement_info(dist) 251 if req is None and not editable: 252 # if PEP 610 metadata is present, attempt to use it 253 direct_url = dist_get_direct_url(dist) 254 if direct_url: 255 req = direct_url_as_pep440_direct_reference( 256 direct_url, dist.project_name 257 ) 258 comments = [] 259 if req is None: 260 # name==version requirement 261 req = dist.as_requirement() 262 263 return cls(dist.project_name, req, editable, comments=comments) 264 265 def __str__(self): 266 # type: () -> str 267 req = self.req 268 if self.editable: 269 req = f'-e {req}' 270 return '\n'.join(list(self.comments) + [str(req)]) + '\n' 271