1#!/usr/local/bin/python3.8 2# vim:fileencoding=utf-8 3# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net> 4 5import glob 6import json 7import os 8import shutil 9import stat 10import subprocess 11import sys 12import tempfile 13import zipfile 14 15from bypy.constants import PREFIX, PYTHON, SW, python_major_minor_version 16from bypy.freeze import ( 17 extract_extension_modules, freeze_python, path_to_freeze_dir 18) 19from bypy.macos_sign import ( 20 codesign, create_entitlements_file, make_certificate_useable, notarize_app, 21 verify_signature 22) 23from bypy.utils import ( 24 current_dir, mkdtemp, py_compile, run_shell, timeit, walk 25) 26 27iv = globals()['init_env'] 28kitty_constants = iv['kitty_constants'] 29self_dir = os.path.dirname(os.path.abspath(__file__)) 30join = os.path.join 31basename = os.path.basename 32dirname = os.path.dirname 33abspath = os.path.abspath 34APPNAME = kitty_constants['appname'] 35VERSION = kitty_constants['version'] 36py_ver = '.'.join(map(str, python_major_minor_version())) 37 38 39def flush(func): 40 def ff(*args, **kwargs): 41 sys.stdout.flush() 42 sys.stderr.flush() 43 ret = func(*args, **kwargs) 44 sys.stdout.flush() 45 sys.stderr.flush() 46 return ret 47 48 return ff 49 50 51def flipwritable(fn, mode=None): 52 """ 53 Flip the writability of a file and return the old mode. Returns None 54 if the file is already writable. 55 """ 56 if os.access(fn, os.W_OK): 57 return None 58 old_mode = os.stat(fn).st_mode 59 os.chmod(fn, stat.S_IWRITE | old_mode) 60 return old_mode 61 62 63STRIPCMD = ('/usr/bin/strip', '-x', '-S', '-') 64 65 66def strip_files(files, argv_max=(256 * 1024)): 67 """ 68 Strip a list of files 69 """ 70 tostrip = [(fn, flipwritable(fn)) for fn in files if os.path.exists(fn)] 71 while tostrip: 72 cmd = list(STRIPCMD) 73 flips = [] 74 pathlen = sum(len(s) + 1 for s in cmd) 75 while pathlen < argv_max: 76 if not tostrip: 77 break 78 added, flip = tostrip.pop() 79 pathlen += len(added) + 1 80 cmd.append(added) 81 flips.append((added, flip)) 82 else: 83 cmd.pop() 84 tostrip.append(flips.pop()) 85 os.spawnv(os.P_WAIT, cmd[0], cmd) 86 for args in flips: 87 flipwritable(*args) 88 89 90def files_in(folder): 91 for record in os.walk(folder): 92 for f in record[-1]: 93 yield os.path.join(record[0], f) 94 95 96def expand_dirs(items, exclude=lambda x: x.endswith('.so')): 97 items = set(items) 98 dirs = set(x for x in items if os.path.isdir(x)) 99 items.difference_update(dirs) 100 for x in dirs: 101 items.update({y for y in files_in(x) if not exclude(y)}) 102 return items 103 104 105def do_sign(app_dir): 106 with current_dir(os.path.join(app_dir, 'Contents')): 107 # Sign all .so files 108 so_files = {x for x in files_in('.') if x.endswith('.so')} 109 codesign(so_files) 110 # Sign everything else in Frameworks 111 with current_dir('Frameworks'): 112 fw = set(glob.glob('*.framework')) 113 codesign(fw) 114 items = set(os.listdir('.')) - fw 115 codesign(expand_dirs(items)) 116 117 # Now sign the main app 118 codesign(app_dir) 119 verify_signature(app_dir) 120 121 122def sign_app(app_dir, notarize): 123 # Copied from iTerm2: https://github.com/gnachman/iTerm2/blob/master/iTerm2.entitlements 124 create_entitlements_file({ 125 'com.apple.security.automation.apple-events': True, 126 'com.apple.security.cs.allow-jit': True, 127 'com.apple.security.device.audio-input': True, 128 'com.apple.security.device.camera': True, 129 'com.apple.security.personal-information.addressbook': True, 130 'com.apple.security.personal-information.calendars': True, 131 'com.apple.security.personal-information.location': True, 132 'com.apple.security.personal-information.photos-library': True, 133 }) 134 with make_certificate_useable(): 135 do_sign(app_dir) 136 if notarize: 137 notarize_app(app_dir) 138 139 140class Freeze(object): 141 142 FID = '@executable_path/../Frameworks' 143 144 def __init__(self, build_dir, dont_strip=False, sign_installers=False, notarize=False, skip_tests=False): 145 self.build_dir = build_dir 146 self.skip_tests = skip_tests 147 self.sign_installers = sign_installers 148 self.notarize = notarize 149 self.dont_strip = dont_strip 150 self.contents_dir = join(self.build_dir, 'Contents') 151 self.resources_dir = join(self.contents_dir, 'Resources') 152 self.frameworks_dir = join(self.contents_dir, 'Frameworks') 153 self.to_strip = [] 154 self.warnings = [] 155 self.py_ver = py_ver 156 self.python_stdlib = join(self.resources_dir, 'Python', 'lib', 'python' + self.py_ver) 157 self.site_packages = self.python_stdlib # hack to avoid needing to add site-packages to path 158 self.obj_dir = mkdtemp('launchers-') 159 160 self.run() 161 162 def run_shell(self): 163 with current_dir(self.contents_dir): 164 run_shell() 165 166 def run(self): 167 ret = 0 168 self.add_python_framework() 169 self.add_site_packages() 170 self.add_stdlib() 171 self.add_misc_libraries() 172 self.freeze_python() 173 self.add_ca_certs() 174 if not self.dont_strip: 175 self.strip_files() 176 if not self.skip_tests: 177 self.run_tests() 178 # self.run_shell() 179 180 ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION) 181 182 return ret 183 184 @flush 185 def add_ca_certs(self): 186 print('\nDownloading CA certs...') 187 from urllib.request import urlopen 188 cdata = urlopen(kitty_constants['cacerts_url']).read() 189 dest = os.path.join(self.contents_dir, 'Resources', 'cacert.pem') 190 with open(dest, 'wb') as f: 191 f.write(cdata) 192 193 @flush 194 def strip_files(self): 195 print('\nStripping files...') 196 strip_files(self.to_strip) 197 198 @flush 199 def run_tests(self): 200 iv['run_tests'](os.path.join(self.contents_dir, 'MacOS', 'kitty')) 201 202 @flush 203 def set_id(self, path_to_lib, new_id): 204 old_mode = flipwritable(path_to_lib) 205 subprocess.check_call( 206 ['install_name_tool', '-id', new_id, path_to_lib]) 207 if old_mode is not None: 208 flipwritable(path_to_lib, old_mode) 209 210 @flush 211 def get_dependencies(self, path_to_lib): 212 install_name = subprocess.check_output( 213 ['otool', '-D', path_to_lib]).decode('utf-8').splitlines()[-1].strip() 214 raw = subprocess.check_output(['otool', '-L', path_to_lib]).decode('utf-8') 215 for line in raw.splitlines(): 216 if 'compatibility' not in line or line.strip().endswith(':'): 217 continue 218 idx = line.find('(') 219 path = line[:idx].strip() 220 yield path, path == install_name 221 222 @flush 223 def get_local_dependencies(self, path_to_lib): 224 for x, is_id in self.get_dependencies(path_to_lib): 225 for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'): 226 if x.startswith(y): 227 if y == PREFIX + '/python/Python.framework/': 228 y = PREFIX + '/python/' 229 yield x, x[len(y):], is_id 230 break 231 232 @flush 233 def change_dep(self, old_dep, new_dep, is_id, path_to_lib): 234 cmd = ['-id', new_dep] if is_id else ['-change', old_dep, new_dep] 235 subprocess.check_call(['install_name_tool'] + cmd + [path_to_lib]) 236 237 @flush 238 def fix_dependencies_in_lib(self, path_to_lib): 239 self.to_strip.append(path_to_lib) 240 old_mode = flipwritable(path_to_lib) 241 for dep, bname, is_id in self.get_local_dependencies(path_to_lib): 242 ndep = self.FID + '/' + bname 243 self.change_dep(dep, ndep, is_id, path_to_lib) 244 ldeps = list(self.get_local_dependencies(path_to_lib)) 245 if ldeps: 246 print('\nFailed to fix dependencies in', path_to_lib) 247 print('Remaining local dependencies:', ldeps) 248 raise SystemExit(1) 249 if old_mode is not None: 250 flipwritable(path_to_lib, old_mode) 251 252 @flush 253 def add_python_framework(self): 254 print('\nAdding Python framework') 255 src = join(PREFIX + '/python', 'Python.framework') 256 x = join(self.frameworks_dir, 'Python.framework') 257 curr = os.path.realpath(join(src, 'Versions', 'Current')) 258 currd = join(x, 'Versions', basename(curr)) 259 rd = join(currd, 'Resources') 260 os.makedirs(rd) 261 shutil.copy2(join(curr, 'Resources', 'Info.plist'), rd) 262 shutil.copy2(join(curr, 'Python'), currd) 263 self.set_id( 264 join(currd, 'Python'), 265 self.FID + '/Python.framework/Versions/%s/Python' % basename(curr)) 266 # The following is needed for codesign 267 with current_dir(x): 268 os.symlink(basename(curr), 'Versions/Current') 269 for y in ('Python', 'Resources'): 270 os.symlink('Versions/Current/%s' % y, y) 271 272 @flush 273 def install_dylib(self, path, set_id=True): 274 shutil.copy2(path, self.frameworks_dir) 275 if set_id: 276 self.set_id( 277 join(self.frameworks_dir, basename(path)), 278 self.FID + '/' + basename(path)) 279 self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path))) 280 281 @flush 282 def add_misc_libraries(self): 283 for x in ( 284 'sqlite3.0', 285 'z.1', 286 'harfbuzz.0', 287 'png16.16', 288 'lcms2.2', 289 'crypto.1.1', 290 'ssl.1.1', 291 ): 292 print('\nAdding', x) 293 x = 'lib%s.dylib' % x 294 src = join(PREFIX, 'lib', x) 295 shutil.copy2(src, self.frameworks_dir) 296 dest = join(self.frameworks_dir, x) 297 self.set_id(dest, self.FID + '/' + x) 298 self.fix_dependencies_in_lib(dest) 299 300 @flush 301 def add_package_dir(self, x, dest=None): 302 def ignore(root, files): 303 ans = [] 304 for y in files: 305 ext = os.path.splitext(y)[1] 306 if ext not in ('', '.py', '.so') or \ 307 (not ext and not os.path.isdir(join(root, y))): 308 ans.append(y) 309 310 return ans 311 312 if dest is None: 313 dest = self.site_packages 314 dest = join(dest, basename(x)) 315 shutil.copytree(x, dest, symlinks=True, ignore=ignore) 316 for f in walk(dest): 317 if f.endswith('.so'): 318 self.fix_dependencies_in_lib(f) 319 320 @flush 321 def add_stdlib(self): 322 print('\nAdding python stdlib') 323 src = PREFIX + '/python/Python.framework/Versions/Current/lib/python' + self.py_ver 324 dest = self.python_stdlib 325 if not os.path.exists(dest): 326 os.makedirs(dest) 327 for x in os.listdir(src): 328 if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk', 329 'lib-old', 'idlelib', 'plat-mac', 'plat-darwin', 330 'site.py', 'distutils', 'turtledemo', 'tkinter'): 331 continue 332 x = join(src, x) 333 if os.path.isdir(x): 334 self.add_package_dir(x, dest) 335 elif os.path.splitext(x)[1] in ('.so', '.py'): 336 shutil.copy2(x, dest) 337 dest2 = join(dest, basename(x)) 338 if dest2.endswith('.so'): 339 self.fix_dependencies_in_lib(dest2) 340 341 @flush 342 def freeze_python(self): 343 print('\nFreezing python') 344 kitty_dir = join(self.resources_dir, 'kitty') 345 bases = ('kitty', 'kittens', 'kitty_tests') 346 for x in bases: 347 dest = os.path.join(self.python_stdlib, x) 348 os.rename(os.path.join(kitty_dir, x), dest) 349 if x == 'kitty': 350 shutil.rmtree(os.path.join(dest, 'launcher')) 351 os.rename(os.path.join(kitty_dir, '__main__.py'), os.path.join(self.python_stdlib, 'kitty_main.py')) 352 shutil.rmtree(os.path.join(kitty_dir, '__pycache__')) 353 pdir = os.path.join(dirname(self.python_stdlib), 'kitty-extensions') 354 os.mkdir(pdir) 355 print('Extracting extension modules from', self.python_stdlib, 'to', pdir) 356 ext_map = extract_extension_modules(self.python_stdlib, pdir) 357 shutil.copy(os.path.join(os.path.dirname(self_dir), 'site.py'), os.path.join(self.python_stdlib, 'site.py')) 358 for x in bases: 359 iv['sanitize_source_folder'](os.path.join(self.python_stdlib, x)) 360 self.compile_py_modules() 361 freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM', remove_pyc_files=True) 362 iv['build_frozen_launcher']([path_to_freeze_dir(), self.obj_dir]) 363 os.rename(join(dirname(self.contents_dir), 'bin', 'kitty'), join(self.contents_dir, 'MacOS', 'kitty')) 364 shutil.rmtree(join(dirname(self.contents_dir), 'bin')) 365 self.fix_dependencies_in_lib(join(self.contents_dir, 'MacOS', 'kitty')) 366 for f in walk(pdir): 367 if f.endswith('.so') or f.endswith('.dylib'): 368 self.fix_dependencies_in_lib(f) 369 370 @flush 371 def add_site_packages(self): 372 print('\nAdding site-packages') 373 os.makedirs(self.site_packages) 374 sys_path = json.loads(subprocess.check_output([ 375 PYTHON, '-c', 'import sys, json; json.dump(sys.path, sys.stdout)'])) 376 paths = reversed(tuple(map(abspath, [x for x in sys_path if x.startswith('/') and not x.startswith('/Library/')]))) 377 upaths = [] 378 for x in paths: 379 if x not in upaths and (x.endswith('.egg') or x.endswith('/site-packages')): 380 upaths.append(x) 381 for x in upaths: 382 print('\t', x) 383 tdir = None 384 try: 385 if not os.path.isdir(x): 386 zf = zipfile.ZipFile(x) 387 tdir = tempfile.mkdtemp() 388 zf.extractall(tdir) 389 x = tdir 390 self.add_modules_from_dir(x) 391 self.add_packages_from_dir(x) 392 finally: 393 if tdir is not None: 394 shutil.rmtree(tdir) 395 self.remove_bytecode(self.site_packages) 396 397 @flush 398 def add_modules_from_dir(self, src): 399 for x in glob.glob(join(src, '*.py')) + glob.glob(join(src, '*.so')): 400 shutil.copy2(x, self.site_packages) 401 if x.endswith('.so'): 402 self.fix_dependencies_in_lib(x) 403 404 @flush 405 def add_packages_from_dir(self, src): 406 for x in os.listdir(src): 407 x = join(src, x) 408 if os.path.isdir(x) and os.path.exists(join(x, '__init__.py')): 409 if self.filter_package(basename(x)): 410 continue 411 self.add_package_dir(x) 412 413 @flush 414 def filter_package(self, name): 415 return name in ('Cython', 'modulegraph', 'macholib', 'py2app', 416 'bdist_mpkg', 'altgraph') 417 418 @flush 419 def remove_bytecode(self, dest): 420 for x in os.walk(dest): 421 root = x[0] 422 for f in x[-1]: 423 if os.path.splitext(f) == '.pyc': 424 os.remove(join(root, f)) 425 426 @flush 427 def compile_py_modules(self): 428 self.remove_bytecode(join(self.resources_dir, 'Python')) 429 py_compile(join(self.resources_dir, 'Python')) 430 431 @flush 432 def makedmg(self, d, volname, format='ULFO'): 433 ''' Copy a directory d into a dmg named volname ''' 434 print('\nMaking dmg...') 435 sys.stdout.flush() 436 destdir = os.path.join(SW, 'dist') 437 try: 438 shutil.rmtree(destdir) 439 except FileNotFoundError: 440 pass 441 os.mkdir(destdir) 442 dmg = os.path.join(destdir, volname + '.dmg') 443 if os.path.exists(dmg): 444 os.unlink(dmg) 445 tdir = tempfile.mkdtemp() 446 appdir = os.path.join(tdir, os.path.basename(d)) 447 shutil.copytree(d, appdir, symlinks=True) 448 if self.sign_installers: 449 with timeit() as times: 450 sign_app(appdir, self.notarize) 451 print('Signing completed in %d minutes %d seconds' % tuple(times)) 452 os.symlink('/Applications', os.path.join(tdir, 'Applications')) 453 size_in_mb = int( 454 subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8') 455 .split()[0]) / 1024. 456 cmd = [ 457 '/usr/bin/hdiutil', 'create', '-srcfolder', tdir, '-volname', 458 volname, '-format', format 459 ] 460 if 190 < size_in_mb < 250: 461 # We need -size 255m because of a bug in hdiutil. When the size of 462 # srcfolder is close to 200MB hdiutil fails with 463 # diskimages-helper: resize request is above maximum size allowed. 464 cmd += ['-size', '255m'] 465 print('\nCreating dmg...') 466 with timeit() as times: 467 subprocess.check_call(cmd + [dmg]) 468 print('dmg created in %d minutes and %d seconds' % tuple(times)) 469 shutil.rmtree(tdir) 470 size = os.stat(dmg).st_size / (1024 * 1024.) 471 print('\nInstaller size: %.2fMB\n' % size) 472 return dmg 473 474 475def main(): 476 args = globals()['args'] 477 ext_dir = globals()['ext_dir'] 478 Freeze( 479 os.path.join(ext_dir, kitty_constants['appname'] + '.app'), 480 dont_strip=args.dont_strip, 481 sign_installers=args.sign_installers, 482 notarize=args.notarize, 483 skip_tests=args.skip_tests 484 ) 485 486 487if __name__ == '__main__': 488 main() 489