1from __future__ import absolute_import 2 3import collections 4import logging 5import os 6 7from pip._vendor import six 8from pip._vendor.packaging.utils import canonicalize_name 9from pip._vendor.pkg_resources import RequirementParseError 10 11from pip._internal.exceptions import BadCommand, InstallationError 12from pip._internal.req.constructors import ( 13 install_req_from_editable, 14 install_req_from_line, 15) 16from pip._internal.req.req_file import COMMENT_RE 17from pip._internal.utils.direct_url_helpers import ( 18 direct_url_as_pep440_direct_reference, 19 dist_get_direct_url, 20) 21from pip._internal.utils.misc import dist_is_editable, get_installed_distributions 22from pip._internal.utils.typing import MYPY_CHECK_RUNNING 23 24if MYPY_CHECK_RUNNING: 25 from typing import ( 26 Container, 27 Dict, 28 Iterable, 29 Iterator, 30 List, 31 Optional, 32 Set, 33 Tuple, 34 Union, 35 ) 36 37 from pip._vendor.pkg_resources import Distribution, Requirement 38 39 from pip._internal.cache import WheelCache 40 41 RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] 42 43 44logger = logging.getLogger(__name__) 45 46 47def freeze( 48 requirement=None, # type: Optional[List[str]] 49 find_links=None, # type: Optional[List[str]] 50 local_only=False, # type: bool 51 user_only=False, # type: bool 52 paths=None, # type: Optional[List[str]] 53 isolated=False, # type: bool 54 wheel_cache=None, # type: Optional[WheelCache] 55 exclude_editable=False, # type: bool 56 skip=() # type: Container[str] 57): 58 # type: (...) -> Iterator[str] 59 find_links = find_links or [] 60 61 for link in find_links: 62 yield '-f {}'.format(link) 63 installations = {} # type: Dict[str, FrozenRequirement] 64 65 for dist in get_installed_distributions( 66 local_only=local_only, 67 skip=(), 68 user_only=user_only, 69 paths=paths 70 ): 71 try: 72 req = FrozenRequirement.from_dist(dist) 73 except RequirementParseError as exc: 74 # We include dist rather than dist.project_name because the 75 # dist string includes more information, like the version and 76 # location. We also include the exception message to aid 77 # troubleshooting. 78 logger.warning( 79 'Could not generate requirement for distribution %r: %s', 80 dist, exc 81 ) 82 continue 83 if exclude_editable and req.editable: 84 continue 85 installations[req.canonical_name] = req 86 87 if requirement: 88 # the options that don't get turned into an InstallRequirement 89 # should only be emitted once, even if the same option is in multiple 90 # requirements files, so we need to keep track of what has been emitted 91 # so that we don't emit it again if it's seen again 92 emitted_options = set() # type: Set[str] 93 # keep track of which files a requirement is in so that we can 94 # give an accurate warning if a requirement appears multiple times. 95 req_files = collections.defaultdict(list) # type: Dict[str, List[str]] 96 for req_file_path in requirement: 97 with open(req_file_path) as req_file: 98 for line in req_file: 99 if (not line.strip() or 100 line.strip().startswith('#') or 101 line.startswith(( 102 '-r', '--requirement', 103 '-f', '--find-links', 104 '-i', '--index-url', 105 '--pre', 106 '--trusted-host', 107 '--process-dependency-links', 108 '--extra-index-url', 109 '--use-feature'))): 110 line = line.rstrip() 111 if line not in emitted_options: 112 emitted_options.add(line) 113 yield line 114 continue 115 116 if line.startswith('-e') or line.startswith('--editable'): 117 if line.startswith('-e'): 118 line = line[2:].strip() 119 else: 120 line = line[len('--editable'):].strip().lstrip('=') 121 line_req = install_req_from_editable( 122 line, 123 isolated=isolated, 124 ) 125 else: 126 line_req = install_req_from_line( 127 COMMENT_RE.sub('', line).strip(), 128 isolated=isolated, 129 ) 130 131 if not line_req.name: 132 logger.info( 133 "Skipping line in requirement file [%s] because " 134 "it's not clear what it would install: %s", 135 req_file_path, line.strip(), 136 ) 137 logger.info( 138 " (add #egg=PackageName to the URL to avoid" 139 " this warning)" 140 ) 141 else: 142 line_req_canonical_name = canonicalize_name( 143 line_req.name) 144 if line_req_canonical_name not in installations: 145 # either it's not installed, or it is installed 146 # but has been processed already 147 if not req_files[line_req.name]: 148 logger.warning( 149 "Requirement file [%s] contains %s, but " 150 "package %r is not installed", 151 req_file_path, 152 COMMENT_RE.sub('', line).strip(), 153 line_req.name 154 ) 155 else: 156 req_files[line_req.name].append(req_file_path) 157 else: 158 yield str(installations[ 159 line_req_canonical_name]).rstrip() 160 del installations[line_req_canonical_name] 161 req_files[line_req.name].append(req_file_path) 162 163 # Warn about requirements that were included multiple times (in a 164 # single requirements file or in different requirements files). 165 for name, files in six.iteritems(req_files): 166 if len(files) > 1: 167 logger.warning("Requirement %s included multiple times [%s]", 168 name, ', '.join(sorted(set(files)))) 169 170 yield( 171 '## The following requirements were added by ' 172 'pip freeze:' 173 ) 174 for installation in sorted( 175 installations.values(), key=lambda x: x.name.lower()): 176 if installation.canonical_name not in skip: 177 yield str(installation).rstrip() 178 179 180def get_requirement_info(dist): 181 # type: (Distribution) -> RequirementInfo 182 """ 183 Compute and return values (req, editable, comments) for use in 184 FrozenRequirement.from_dist(). 185 """ 186 if not dist_is_editable(dist): 187 return (None, False, []) 188 189 location = os.path.normcase(os.path.abspath(dist.location)) 190 191 from pip._internal.vcs import RemoteNotFoundError, vcs 192 vcs_backend = vcs.get_backend_for_dir(location) 193 194 if vcs_backend is None: 195 req = dist.as_requirement() 196 logger.debug( 197 'No VCS found for editable requirement "%s" in: %r', req, 198 location, 199 ) 200 comments = [ 201 '# Editable install with no version control ({})'.format(req) 202 ] 203 return (location, True, comments) 204 205 try: 206 req = vcs_backend.get_src_requirement(location, dist.project_name) 207 except RemoteNotFoundError: 208 req = dist.as_requirement() 209 comments = [ 210 '# Editable {} install with no remote ({})'.format( 211 type(vcs_backend).__name__, req, 212 ) 213 ] 214 return (location, True, comments) 215 216 except BadCommand: 217 logger.warning( 218 'cannot determine version of editable source in %s ' 219 '(%s command not found in path)', 220 location, 221 vcs_backend.name, 222 ) 223 return (None, True, []) 224 225 except InstallationError as exc: 226 logger.warning( 227 "Error when trying to get requirement for VCS system %s, " 228 "falling back to uneditable format", exc 229 ) 230 else: 231 if req is not None: 232 return (req, True, []) 233 234 logger.warning( 235 'Could not determine repository location of %s', location 236 ) 237 comments = ['## !! Could not determine repository location'] 238 239 return (None, False, comments) 240 241 242class FrozenRequirement(object): 243 def __init__(self, name, req, editable, comments=()): 244 # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None 245 self.name = name 246 self.canonical_name = canonicalize_name(name) 247 self.req = req 248 self.editable = editable 249 self.comments = comments 250 251 @classmethod 252 def from_dist(cls, dist): 253 # type: (Distribution) -> FrozenRequirement 254 # TODO `get_requirement_info` is taking care of editable requirements. 255 # TODO This should be refactored when we will add detection of 256 # editable that provide .dist-info metadata. 257 req, editable, comments = get_requirement_info(dist) 258 if req is None and not editable: 259 # if PEP 610 metadata is present, attempt to use it 260 direct_url = dist_get_direct_url(dist) 261 if direct_url: 262 req = direct_url_as_pep440_direct_reference( 263 direct_url, dist.project_name 264 ) 265 comments = [] 266 if req is None: 267 # name==version requirement 268 req = dist.as_requirement() 269 270 return cls(dist.project_name, req, editable, comments=comments) 271 272 def __str__(self): 273 # type: () -> str 274 req = self.req 275 if self.editable: 276 req = '-e {}'.format(req) 277 return '\n'.join(list(self.comments) + [str(req)]) + '\n' 278