1from __future__ import absolute_import 2 3import collections 4import logging 5import os 6 7from pip._vendor import six 8from pip._vendor.packaging.utils import canonicalize_name 9from pip._vendor.pkg_resources import RequirementParseError 10 11from pip._internal.exceptions import BadCommand, InstallationError 12from pip._internal.req.constructors import ( 13 install_req_from_editable, 14 install_req_from_line, 15) 16from pip._internal.req.req_file import COMMENT_RE 17from pip._internal.utils.direct_url_helpers import ( 18 direct_url_as_pep440_direct_reference, 19 dist_get_direct_url, 20) 21from pip._internal.utils.misc import ( 22 dist_is_editable, 23 get_installed_distributions, 24) 25from pip._internal.utils.typing import MYPY_CHECK_RUNNING 26 27if MYPY_CHECK_RUNNING: 28 from typing import ( 29 Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union 30 ) 31 from pip._internal.cache import WheelCache 32 from pip._vendor.pkg_resources import ( 33 Distribution, Requirement 34 ) 35 36 RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] 37 38 39logger = logging.getLogger(__name__) 40 41 42def freeze( 43 requirement=None, # type: Optional[List[str]] 44 find_links=None, # type: Optional[List[str]] 45 local_only=False, # type: bool 46 user_only=False, # type: bool 47 paths=None, # type: Optional[List[str]] 48 isolated=False, # type: bool 49 wheel_cache=None, # type: Optional[WheelCache] 50 exclude_editable=False, # type: bool 51 skip=() # type: Container[str] 52): 53 # type: (...) -> Iterator[str] 54 find_links = find_links or [] 55 56 for link in find_links: 57 yield '-f {}'.format(link) 58 installations = {} # type: Dict[str, FrozenRequirement] 59 60 for dist in get_installed_distributions( 61 local_only=local_only, 62 skip=(), 63 user_only=user_only, 64 paths=paths 65 ): 66 try: 67 req = FrozenRequirement.from_dist(dist) 68 except RequirementParseError as exc: 69 # We include dist rather than dist.project_name because the 70 # dist string includes more information, like the version and 71 # location. We also include the exception message to aid 72 # troubleshooting. 73 logger.warning( 74 'Could not generate requirement for distribution %r: %s', 75 dist, exc 76 ) 77 continue 78 if exclude_editable and req.editable: 79 continue 80 installations[req.canonical_name] = req 81 82 if requirement: 83 # the options that don't get turned into an InstallRequirement 84 # should only be emitted once, even if the same option is in multiple 85 # requirements files, so we need to keep track of what has been emitted 86 # so that we don't emit it again if it's seen again 87 emitted_options = set() # type: Set[str] 88 # keep track of which files a requirement is in so that we can 89 # give an accurate warning if a requirement appears multiple times. 90 req_files = collections.defaultdict(list) # type: Dict[str, List[str]] 91 for req_file_path in requirement: 92 with open(req_file_path) as req_file: 93 for line in req_file: 94 if (not line.strip() or 95 line.strip().startswith('#') or 96 line.startswith(( 97 '-r', '--requirement', 98 '-f', '--find-links', 99 '-i', '--index-url', 100 '--pre', 101 '--trusted-host', 102 '--process-dependency-links', 103 '--extra-index-url', 104 '--use-feature'))): 105 line = line.rstrip() 106 if line not in emitted_options: 107 emitted_options.add(line) 108 yield line 109 continue 110 111 if line.startswith('-e') or line.startswith('--editable'): 112 if line.startswith('-e'): 113 line = line[2:].strip() 114 else: 115 line = line[len('--editable'):].strip().lstrip('=') 116 line_req = install_req_from_editable( 117 line, 118 isolated=isolated, 119 ) 120 else: 121 line_req = install_req_from_line( 122 COMMENT_RE.sub('', line).strip(), 123 isolated=isolated, 124 ) 125 126 if not line_req.name: 127 logger.info( 128 "Skipping line in requirement file [%s] because " 129 "it's not clear what it would install: %s", 130 req_file_path, line.strip(), 131 ) 132 logger.info( 133 " (add #egg=PackageName to the URL to avoid" 134 " this warning)" 135 ) 136 else: 137 line_req_canonical_name = canonicalize_name( 138 line_req.name) 139 if line_req_canonical_name not in installations: 140 # either it's not installed, or it is installed 141 # but has been processed already 142 if not req_files[line_req.name]: 143 logger.warning( 144 "Requirement file [%s] contains %s, but " 145 "package %r is not installed", 146 req_file_path, 147 COMMENT_RE.sub('', line).strip(), 148 line_req.name 149 ) 150 else: 151 req_files[line_req.name].append(req_file_path) 152 else: 153 yield str(installations[ 154 line_req_canonical_name]).rstrip() 155 del installations[line_req_canonical_name] 156 req_files[line_req.name].append(req_file_path) 157 158 # Warn about requirements that were included multiple times (in a 159 # single requirements file or in different requirements files). 160 for name, files in six.iteritems(req_files): 161 if len(files) > 1: 162 logger.warning("Requirement %s included multiple times [%s]", 163 name, ', '.join(sorted(set(files)))) 164 165 yield( 166 '## The following requirements were added by ' 167 'pip freeze:' 168 ) 169 for installation in sorted( 170 installations.values(), key=lambda x: x.name.lower()): 171 if installation.canonical_name not in skip: 172 yield str(installation).rstrip() 173 174 175def get_requirement_info(dist): 176 # type: (Distribution) -> RequirementInfo 177 """ 178 Compute and return values (req, editable, comments) for use in 179 FrozenRequirement.from_dist(). 180 """ 181 if not dist_is_editable(dist): 182 return (None, False, []) 183 184 location = os.path.normcase(os.path.abspath(dist.location)) 185 186 from pip._internal.vcs import vcs, RemoteNotFoundError 187 vcs_backend = vcs.get_backend_for_dir(location) 188 189 if vcs_backend is None: 190 req = dist.as_requirement() 191 logger.debug( 192 'No VCS found for editable requirement "%s" in: %r', req, 193 location, 194 ) 195 comments = [ 196 '# Editable install with no version control ({})'.format(req) 197 ] 198 return (location, True, comments) 199 200 try: 201 req = vcs_backend.get_src_requirement(location, dist.project_name) 202 except RemoteNotFoundError: 203 req = dist.as_requirement() 204 comments = [ 205 '# Editable {} install with no remote ({})'.format( 206 type(vcs_backend).__name__, req, 207 ) 208 ] 209 return (location, True, comments) 210 211 except BadCommand: 212 logger.warning( 213 'cannot determine version of editable source in %s ' 214 '(%s command not found in path)', 215 location, 216 vcs_backend.name, 217 ) 218 return (None, True, []) 219 220 except InstallationError as exc: 221 logger.warning( 222 "Error when trying to get requirement for VCS system %s, " 223 "falling back to uneditable format", exc 224 ) 225 else: 226 if req is not None: 227 return (req, True, []) 228 229 logger.warning( 230 'Could not determine repository location of %s', location 231 ) 232 comments = ['## !! Could not determine repository location'] 233 234 return (None, False, comments) 235 236 237class FrozenRequirement(object): 238 def __init__(self, name, req, editable, comments=()): 239 # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None 240 self.name = name 241 self.canonical_name = canonicalize_name(name) 242 self.req = req 243 self.editable = editable 244 self.comments = comments 245 246 @classmethod 247 def from_dist(cls, dist): 248 # type: (Distribution) -> FrozenRequirement 249 # TODO `get_requirement_info` is taking care of editable requirements. 250 # TODO This should be refactored when we will add detection of 251 # editable that provide .dist-info metadata. 252 req, editable, comments = get_requirement_info(dist) 253 if req is None and not editable: 254 # if PEP 610 metadata is present, attempt to use it 255 direct_url = dist_get_direct_url(dist) 256 if direct_url: 257 req = direct_url_as_pep440_direct_reference( 258 direct_url, dist.project_name 259 ) 260 comments = [] 261 if req is None: 262 # name==version requirement 263 req = dist.as_requirement() 264 265 return cls(dist.project_name, req, editable, comments=comments) 266 267 def __str__(self): 268 # type: () -> str 269 req = self.req 270 if self.editable: 271 req = '-e {}'.format(req) 272 return '\n'.join(list(self.comments) + [str(req)]) + '\n' 273