1from __future__ import absolute_import 2 3import collections 4import logging 5import os 6import re 7 8from pip._vendor import six 9from pip._vendor.packaging.utils import canonicalize_name 10from pip._vendor.pkg_resources import RequirementParseError 11 12from pip._internal.exceptions import BadCommand, InstallationError 13from pip._internal.req.constructors import ( 14 install_req_from_editable, install_req_from_line, 15) 16from pip._internal.req.req_file import COMMENT_RE 17from pip._internal.utils.misc import ( 18 dist_is_editable, get_installed_distributions, 19) 20from pip._internal.utils.typing import MYPY_CHECK_RUNNING 21 22if MYPY_CHECK_RUNNING: 23 from typing import ( 24 Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union 25 ) 26 from pip._internal.cache import WheelCache 27 from pip._vendor.pkg_resources import ( 28 Distribution, Requirement 29 ) 30 31 RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] 32 33 34logger = logging.getLogger(__name__) 35 36 37def freeze( 38 requirement=None, # type: Optional[List[str]] 39 find_links=None, # type: Optional[List[str]] 40 local_only=None, # type: Optional[bool] 41 user_only=None, # type: Optional[bool] 42 paths=None, # type: Optional[List[str]] 43 skip_regex=None, # type: Optional[str] 44 isolated=False, # type: bool 45 wheel_cache=None, # type: Optional[WheelCache] 46 exclude_editable=False, # type: bool 47 skip=() # type: Container[str] 48): 49 # type: (...) -> Iterator[str] 50 find_links = find_links or [] 51 skip_match = None 52 53 if skip_regex: 54 skip_match = re.compile(skip_regex).search 55 56 for link in find_links: 57 yield '-f %s' % link 58 installations = {} # type: Dict[str, FrozenRequirement] 59 for dist in get_installed_distributions(local_only=local_only, 60 skip=(), 61 user_only=user_only, 62 paths=paths): 63 try: 64 req = FrozenRequirement.from_dist(dist) 65 except RequirementParseError as exc: 66 # We include dist rather than dist.project_name because the 67 # dist string includes more information, like the version and 68 # location. We also include the exception message to aid 69 # troubleshooting. 70 logger.warning( 71 'Could not generate requirement for distribution %r: %s', 72 dist, exc 73 ) 74 continue 75 if exclude_editable and req.editable: 76 continue 77 installations[req.name] = req 78 79 if requirement: 80 # the options that don't get turned into an InstallRequirement 81 # should only be emitted once, even if the same option is in multiple 82 # requirements files, so we need to keep track of what has been emitted 83 # so that we don't emit it again if it's seen again 84 emitted_options = set() # type: Set[str] 85 # keep track of which files a requirement is in so that we can 86 # give an accurate warning if a requirement appears multiple times. 87 req_files = collections.defaultdict(list) # type: Dict[str, List[str]] 88 for req_file_path in requirement: 89 with open(req_file_path) as req_file: 90 for line in req_file: 91 if (not line.strip() or 92 line.strip().startswith('#') or 93 (skip_match and skip_match(line)) or 94 line.startswith(( 95 '-r', '--requirement', 96 '-Z', '--always-unzip', 97 '-f', '--find-links', 98 '-i', '--index-url', 99 '--pre', 100 '--trusted-host', 101 '--process-dependency-links', 102 '--extra-index-url'))): 103 line = line.rstrip() 104 if line not in emitted_options: 105 emitted_options.add(line) 106 yield line 107 continue 108 109 if line.startswith('-e') or line.startswith('--editable'): 110 if line.startswith('-e'): 111 line = line[2:].strip() 112 else: 113 line = line[len('--editable'):].strip().lstrip('=') 114 line_req = install_req_from_editable( 115 line, 116 isolated=isolated, 117 wheel_cache=wheel_cache, 118 ) 119 else: 120 line_req = install_req_from_line( 121 COMMENT_RE.sub('', line).strip(), 122 isolated=isolated, 123 wheel_cache=wheel_cache, 124 ) 125 126 if not line_req.name: 127 logger.info( 128 "Skipping line in requirement file [%s] because " 129 "it's not clear what it would install: %s", 130 req_file_path, line.strip(), 131 ) 132 logger.info( 133 " (add #egg=PackageName to the URL to avoid" 134 " this warning)" 135 ) 136 elif line_req.name not in installations: 137 # either it's not installed, or it is installed 138 # but has been processed already 139 if not req_files[line_req.name]: 140 logger.warning( 141 "Requirement file [%s] contains %s, but " 142 "package %r is not installed", 143 req_file_path, 144 COMMENT_RE.sub('', line).strip(), line_req.name 145 ) 146 else: 147 req_files[line_req.name].append(req_file_path) 148 else: 149 yield str(installations[line_req.name]).rstrip() 150 del installations[line_req.name] 151 req_files[line_req.name].append(req_file_path) 152 153 # Warn about requirements that were included multiple times (in a 154 # single requirements file or in different requirements files). 155 for name, files in six.iteritems(req_files): 156 if len(files) > 1: 157 logger.warning("Requirement %s included multiple times [%s]", 158 name, ', '.join(sorted(set(files)))) 159 160 yield( 161 '## The following requirements were added by ' 162 'pip freeze:' 163 ) 164 for installation in sorted( 165 installations.values(), key=lambda x: x.name.lower()): 166 if canonicalize_name(installation.name) not in skip: 167 yield str(installation).rstrip() 168 169 170def get_requirement_info(dist): 171 # type: (Distribution) -> RequirementInfo 172 """ 173 Compute and return values (req, editable, comments) for use in 174 FrozenRequirement.from_dist(). 175 """ 176 if not dist_is_editable(dist): 177 return (None, False, []) 178 179 location = os.path.normcase(os.path.abspath(dist.location)) 180 181 from pip._internal.vcs import vcs, RemoteNotFoundError 182 vcs_backend = vcs.get_backend_for_dir(location) 183 184 if vcs_backend is None: 185 req = dist.as_requirement() 186 logger.debug( 187 'No VCS found for editable requirement "%s" in: %r', req, 188 location, 189 ) 190 comments = [ 191 '# Editable install with no version control ({})'.format(req) 192 ] 193 return (location, True, comments) 194 195 try: 196 req = vcs_backend.get_src_requirement(location, dist.project_name) 197 except RemoteNotFoundError: 198 req = dist.as_requirement() 199 comments = [ 200 '# Editable {} install with no remote ({})'.format( 201 type(vcs_backend).__name__, req, 202 ) 203 ] 204 return (location, True, comments) 205 206 except BadCommand: 207 logger.warning( 208 'cannot determine version of editable source in %s ' 209 '(%s command not found in path)', 210 location, 211 vcs_backend.name, 212 ) 213 return (None, True, []) 214 215 except InstallationError as exc: 216 logger.warning( 217 "Error when trying to get requirement for VCS system %s, " 218 "falling back to uneditable format", exc 219 ) 220 else: 221 if req is not None: 222 return (req, True, []) 223 224 logger.warning( 225 'Could not determine repository location of %s', location 226 ) 227 comments = ['## !! Could not determine repository location'] 228 229 return (None, False, comments) 230 231 232class FrozenRequirement(object): 233 def __init__(self, name, req, editable, comments=()): 234 # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None 235 self.name = name 236 self.req = req 237 self.editable = editable 238 self.comments = comments 239 240 @classmethod 241 def from_dist(cls, dist): 242 # type: (Distribution) -> FrozenRequirement 243 req, editable, comments = get_requirement_info(dist) 244 if req is None: 245 req = dist.as_requirement() 246 247 return cls(dist.project_name, req, editable, comments=comments) 248 249 def __str__(self): 250 req = self.req 251 if self.editable: 252 req = '-e %s' % req 253 return '\n'.join(list(self.comments) + [str(req)]) + '\n' 254