1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7from distutils.version import LooseVersion 8import hashlib 9import logging 10from mozbuild.base import ( 11 BuildEnvironmentNotFoundException, 12 MozbuildObject, 13) 14import mozfile 15import mozpack.path as mozpath 16import os 17import re 18import subprocess 19import sys 20 21class VendorRust(MozbuildObject): 22 def get_cargo_path(self): 23 try: 24 return self.substs['CARGO'] 25 except (BuildEnvironmentNotFoundException, KeyError): 26 # Default if this tree isn't configured. 27 import which 28 return which.which('cargo') 29 30 def check_cargo_version(self, cargo): 31 ''' 32 Ensure that cargo is new enough. cargo 0.12 added support 33 for source replacement, which is required for vendoring to work. 34 ''' 35 out = subprocess.check_output([cargo, '--version']).splitlines()[0] 36 if not out.startswith('cargo'): 37 return False 38 return LooseVersion(out.split()[1]) >= b'0.13' 39 40 def check_cargo_vendor_version(self, cargo): 41 ''' 42 Ensure that cargo-vendor is new enough. cargo-vendor 0.1.13 and newer 43 strips out .cargo-ok, .orig and .rej files, and deals with [patch] 44 replacements in Cargo.toml files which we want. 45 ''' 46 for l in subprocess.check_output([cargo, 'install', '--list']).splitlines(): 47 # The line looks like one of the following: 48 # cargo-vendor v0.1.12: 49 # cargo-vendor v0.1.12 (file:///path/to/local/build/cargo-vendor): 50 # and we want to extract the version part of it 51 m = re.match('cargo-vendor v((\d+\.)*\d+)', l) 52 if m: 53 version = m.group(1) 54 return LooseVersion(version) >= b'0.1.13' 55 return False 56 57 def check_modified_files(self): 58 ''' 59 Ensure that there aren't any uncommitted changes to files 60 in the working copy, since we're going to change some state 61 on the user. Allow changes to Cargo.{toml,lock} since that's 62 likely to be a common use case. 63 ''' 64 modified = [f for f in self.repository.get_changed_files('M') if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] 65 if modified: 66 self.log(logging.ERROR, 'modified_files', {}, 67 '''You have uncommitted changes to the following files: 68 69{files} 70 71Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. 72'''.format(files='\n'.join(sorted(modified)))) 73 sys.exit(1) 74 75 def check_openssl(self): 76 ''' 77 Set environment flags for building with openssl. 78 79 MacOS doesn't include openssl, but the openssl-sys crate used by 80 mach-vendor expects one of the system. It's common to have one 81 installed in /usr/local/opt/openssl by homebrew, but custom link 82 flags are necessary to build against it. 83 ''' 84 85 test_paths = ['/usr/include', '/usr/local/include'] 86 if any([os.path.exists(os.path.join(path, 'openssl/ssl.h')) for path in test_paths]): 87 # Assume we can use one of these system headers. 88 return None 89 90 if os.path.exists('/usr/local/opt/openssl/include/openssl/ssl.h'): 91 # Found a likely homebrew install. 92 self.log(logging.INFO, 'openssl', {}, 93 'Using OpenSSL in /usr/local/opt/openssl') 94 return { 95 'OPENSSL_INCLUDE_DIR': '/usr/local/opt/openssl/include', 96 'OPENSSL_LIB_DIR': '/usr/local/opt/openssl/lib', 97 } 98 99 self.log(logging.ERROR, 'openssl', {}, "OpenSSL not found!") 100 return None 101 102 def _ensure_cargo(self): 103 ''' 104 Ensures all the necessary cargo bits are installed. 105 106 Returns the path to cargo if successful, None otherwise. 107 ''' 108 cargo = self.get_cargo_path() 109 if not self.check_cargo_version(cargo): 110 self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') 111 return None 112 else: 113 self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough') 114 have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) 115 if not have_vendor: 116 self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor (this may take a few minutes)...') 117 env = self.check_openssl() 118 self.run_process(args=[cargo, 'install', 'cargo-vendor'], 119 append_env=env) 120 elif not self.check_cargo_vendor_version(cargo): 121 self.log(logging.INFO, 'cargo_vendor', {}, 'cargo-vendor >= 0.1.12 required; force-reinstalling (this may take a few minutes)...') 122 env = self.check_openssl() 123 self.run_process(args=[cargo, 'install', '--force', 'cargo-vendor'], 124 append_env=env) 125 else: 126 self.log(logging.DEBUG, 'cargo_vendor', {}, 'sufficiently new cargo-vendor is already installed') 127 128 return cargo 129 130 def _check_licenses(self, vendor_dir): 131 # A whitelist of acceptable license identifiers for the 132 # packages.license field from https://spdx.org/licenses/. Cargo 133 # documentation claims that values are checked against the above 134 # list and that multiple entries can be separated by '/'. We 135 # choose to list all combinations instead for the sake of 136 # completeness and because some entries below obviously do not 137 # conform to the format prescribed in the documentation. 138 # 139 # It is insufficient to have additions to this whitelist reviewed 140 # solely by a build peer; any additions must be checked by somebody 141 # competent to review licensing minutiae. 142 LICENSE_WHITELIST = [ 143 'Apache-2.0', 144 'Apache-2.0 / MIT', 145 'Apache-2.0/MIT', 146 'Apache-2 / MIT', 147 'BSD-3-Clause', # bindgen (only used at build time) 148 'CC0-1.0', 149 'ISC', 150 'ISC/Apache-2.0', 151 'MIT', 152 'MIT / Apache-2.0', 153 'MIT/Apache-2.0', 154 'MIT OR Apache-2.0', 155 'MPL-2.0', 156 'Unlicense/MIT', 157 ] 158 159 # This whitelist should only be used for packages that use a 160 # license-file and for which the license-file entry has been 161 # reviewed. The table is keyed by package names and maps to the 162 # sha256 hash of the license file that we reviewed. 163 # 164 # As above, it is insufficient to have additions to this whitelist 165 # reviewed solely by a build peer; any additions must be checked by 166 # somebody competent to review licensing minutiae. 167 LICENSE_FILE_PACKAGE_WHITELIST = { 168 # MIT 169 'deque': '6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb', 170 } 171 172 LICENSE_LINE_RE = re.compile(r'\s*license\s*=\s*"([^"]+)"') 173 LICENSE_FILE_LINE_RE = re.compile(r'\s*license[-_]file\s*=\s*"([^"]+)"') 174 175 def check_package(package): 176 self.log(logging.DEBUG, 'package_check', {}, 177 'Checking license for {}'.format(package)) 178 179 toml_file = os.path.join(vendor_dir, package, 'Cargo.toml') 180 181 # pytoml is not sophisticated enough to parse Cargo.toml files 182 # with [target.'cfg(...)'.dependencies sections, so we resort 183 # to scanning individual lines. 184 with open(toml_file, 'r') as f: 185 license_lines = [l for l in f if l.strip().startswith(b'license')] 186 license_matches = list(filter(lambda x: x, [LICENSE_LINE_RE.match(l) for l in license_lines])) 187 license_file_matches = list(filter(lambda x: x, [LICENSE_FILE_LINE_RE.match(l) for l in license_lines])) 188 189 # License information is optional for crates to provide, but 190 # we require it. 191 if not license_matches and not license_file_matches: 192 self.log(logging.ERROR, 'package_no_license', {}, 193 'package {} does not provide a license'.format(package)) 194 return False 195 196 # The Cargo.toml spec suggests that crates should either have 197 # `license` or `license-file`, but not both. We might as well 198 # be defensive about that, though. 199 if len(license_matches) > 1 or len(license_file_matches) > 1 or \ 200 license_matches and license_file_matches: 201 self.log(logging.ERROR, 'package_many_licenses', {}, 202 'package {} provides too many licenses'.format(package)) 203 return False 204 205 if license_matches: 206 license = license_matches[0].group(1) 207 self.log(logging.DEBUG, 'package_license', {}, 208 'has license {}'.format(license)) 209 210 if license not in LICENSE_WHITELIST: 211 self.log(logging.ERROR, 'package_license_error', {}, 212 '''Package {} has a non-approved license: {}. 213 214Please request license review on the package's license. If the package's license 215is approved, please add it to the whitelist of suitable licenses. 216'''.format(package, license)) 217 return False 218 else: 219 license_file = license_file_matches[0].group(1) 220 self.log(logging.DEBUG, 'package_license_file', {}, 221 'has license-file {}'.format(license_file)) 222 223 if package not in LICENSE_FILE_PACKAGE_WHITELIST: 224 self.log(logging.ERROR, 'package_license_file_unknown', {}, 225 '''Package {} has an unreviewed license file: {}. 226 227Please request review on the provided license; if approved, the package can be added 228to the whitelist of packages whose licenses are suitable. 229'''.format(package, license_file)) 230 return False 231 232 approved_hash = LICENSE_FILE_PACKAGE_WHITELIST[package] 233 license_contents = open(os.path.join(vendor_dir, package, license_file), 'r').read() 234 current_hash = hashlib.sha256(license_contents).hexdigest() 235 if current_hash != approved_hash: 236 self.log(logging.ERROR, 'package_license_file_mismatch', {}, 237 '''Package {} has changed its license file: {} (hash {}). 238 239Please request review on the provided license; if approved, please update the 240license file's hash. 241'''.format(package, license_file, current_hash)) 242 return False 243 244 return True 245 246 # Force all of the packages to be checked for license information 247 # before reducing via `all`, so all license issues are found in a 248 # single `mach vendor rust` invocation. 249 results = [check_package(p) for p in os.listdir(vendor_dir)] 250 return all(results) 251 252 def vendor(self, ignore_modified=False, 253 build_peers_said_large_imports_were_ok=False): 254 self.populate_logger() 255 self.log_manager.enable_unstructured() 256 if not ignore_modified: 257 self.check_modified_files() 258 259 cargo = self._ensure_cargo() 260 if not cargo: 261 return 262 263 relative_vendor_dir = 'third_party/rust' 264 vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir) 265 self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir) 266 mozfile.remove(vendor_dir) 267 268 # We use check_call instead of mozprocess to ensure errors are displayed. 269 # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462 270 subprocess.check_call([cargo, 'update', '-p', 'gkrust'], cwd=self.topsrcdir) 271 272 subprocess.check_call([cargo, 'vendor', '--quiet', '--no-delete', '--sync', 'Cargo.lock'] + [vendor_dir], cwd=self.topsrcdir) 273 274 if not self._check_licenses(vendor_dir): 275 self.log(logging.ERROR, 'license_check_failed', {}, 276 '''The changes from `mach vendor rust` will NOT be added to version control.''') 277 sys.exit(1) 278 279 self.repository.add_remove_files(vendor_dir) 280 281 # 100k is a reasonable upper bound on source file size. 282 FILESIZE_LIMIT = 100 * 1024 283 large_files = set() 284 cumulative_added_size = 0 285 for f in self.repository.get_changed_files('A'): 286 path = mozpath.join(self.topsrcdir, f) 287 size = os.stat(path).st_size 288 cumulative_added_size += size 289 if size > FILESIZE_LIMIT: 290 large_files.add(f) 291 292 # Forcefully complain about large files being added, as history has 293 # shown that large-ish files typically are not needed. 294 if large_files and not build_peers_said_large_imports_were_ok: 295 self.log(logging.ERROR, 'filesize_check', {}, 296 '''The following files exceed the filesize limit of {size}: 297 298{files} 299 300Please find a way to reduce the sizes of these files or talk to a build 301peer about the particular large files you are adding. 302 303The changes from `mach vendor rust` will NOT be added to version control. 304'''.format(files='\n'.join(sorted(large_files)), size=FILESIZE_LIMIT)) 305 self.repository.forget_add_remove_files(vendor_dir) 306 sys.exit(1) 307 308 # Only warn for large imports, since we may just have large code 309 # drops from time to time (e.g. importing features into m-c). 310 SIZE_WARN_THRESHOLD = 5 * 1024 * 1024 311 if cumulative_added_size >= SIZE_WARN_THRESHOLD: 312 self.log(logging.WARN, 'filesize_check', {}, 313 '''Your changes add {size} bytes of added files. 314 315Please consider finding ways to reduce the size of the vendored packages. 316For instance, check the vendored packages for unusually large test or 317benchmark files that don't need to be published to crates.io and submit 318a pull request upstream to ignore those files when publishing.'''.format(size=cumulative_added_size)) 319