1#!/usr/local/bin/python3.8
2# vim:fileencoding=utf-8
3# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
4
5import glob
6import json
7import os
8import shutil
9import stat
10import subprocess
11import sys
12import tempfile
13import zipfile
14
15from bypy.constants import PREFIX, PYTHON, SW, python_major_minor_version
16from bypy.freeze import (
17    extract_extension_modules, freeze_python, path_to_freeze_dir
18)
19from bypy.macos_sign import (
20    codesign, create_entitlements_file, make_certificate_useable, notarize_app,
21    verify_signature
22)
23from bypy.utils import (
24    current_dir, mkdtemp, py_compile, run_shell, timeit, walk
25)
26
27iv = globals()['init_env']
28kitty_constants = iv['kitty_constants']
29self_dir = os.path.dirname(os.path.abspath(__file__))
30join = os.path.join
31basename = os.path.basename
32dirname = os.path.dirname
33abspath = os.path.abspath
34APPNAME = kitty_constants['appname']
35VERSION = kitty_constants['version']
36py_ver = '.'.join(map(str, python_major_minor_version()))
37
38
39def flush(func):
40    def ff(*args, **kwargs):
41        sys.stdout.flush()
42        sys.stderr.flush()
43        ret = func(*args, **kwargs)
44        sys.stdout.flush()
45        sys.stderr.flush()
46        return ret
47
48    return ff
49
50
51def flipwritable(fn, mode=None):
52    """
53    Flip the writability of a file and return the old mode. Returns None
54    if the file is already writable.
55    """
56    if os.access(fn, os.W_OK):
57        return None
58    old_mode = os.stat(fn).st_mode
59    os.chmod(fn, stat.S_IWRITE | old_mode)
60    return old_mode
61
62
63STRIPCMD = ('/usr/bin/strip', '-x', '-S', '-')
64
65
66def strip_files(files, argv_max=(256 * 1024)):
67    """
68    Strip a list of files
69    """
70    tostrip = [(fn, flipwritable(fn)) for fn in files if os.path.exists(fn)]
71    while tostrip:
72        cmd = list(STRIPCMD)
73        flips = []
74        pathlen = sum(len(s) + 1 for s in cmd)
75        while pathlen < argv_max:
76            if not tostrip:
77                break
78            added, flip = tostrip.pop()
79            pathlen += len(added) + 1
80            cmd.append(added)
81            flips.append((added, flip))
82        else:
83            cmd.pop()
84            tostrip.append(flips.pop())
85        os.spawnv(os.P_WAIT, cmd[0], cmd)
86        for args in flips:
87            flipwritable(*args)
88
89
90def files_in(folder):
91    for record in os.walk(folder):
92        for f in record[-1]:
93            yield os.path.join(record[0], f)
94
95
96def expand_dirs(items, exclude=lambda x: x.endswith('.so')):
97    items = set(items)
98    dirs = set(x for x in items if os.path.isdir(x))
99    items.difference_update(dirs)
100    for x in dirs:
101        items.update({y for y in files_in(x) if not exclude(y)})
102    return items
103
104
105def do_sign(app_dir):
106    with current_dir(os.path.join(app_dir, 'Contents')):
107        # Sign all .so files
108        so_files = {x for x in files_in('.') if x.endswith('.so')}
109        codesign(so_files)
110        # Sign everything else in Frameworks
111        with current_dir('Frameworks'):
112            fw = set(glob.glob('*.framework'))
113            codesign(fw)
114            items = set(os.listdir('.')) - fw
115            codesign(expand_dirs(items))
116
117    # Now sign the main app
118    codesign(app_dir)
119    verify_signature(app_dir)
120
121
122def sign_app(app_dir, notarize):
123    # Copied from iTerm2: https://github.com/gnachman/iTerm2/blob/master/iTerm2.entitlements
124    create_entitlements_file({
125        'com.apple.security.automation.apple-events': True,
126        'com.apple.security.cs.allow-jit': True,
127        'com.apple.security.device.audio-input': True,
128        'com.apple.security.device.camera': True,
129        'com.apple.security.personal-information.addressbook': True,
130        'com.apple.security.personal-information.calendars': True,
131        'com.apple.security.personal-information.location': True,
132        'com.apple.security.personal-information.photos-library': True,
133    })
134    with make_certificate_useable():
135        do_sign(app_dir)
136        if notarize:
137            notarize_app(app_dir)
138
139
140class Freeze(object):
141
142    FID = '@executable_path/../Frameworks'
143
144    def __init__(self, build_dir, dont_strip=False, sign_installers=False, notarize=False, skip_tests=False):
145        self.build_dir = build_dir
146        self.skip_tests = skip_tests
147        self.sign_installers = sign_installers
148        self.notarize = notarize
149        self.dont_strip = dont_strip
150        self.contents_dir = join(self.build_dir, 'Contents')
151        self.resources_dir = join(self.contents_dir, 'Resources')
152        self.frameworks_dir = join(self.contents_dir, 'Frameworks')
153        self.to_strip = []
154        self.warnings = []
155        self.py_ver = py_ver
156        self.python_stdlib = join(self.resources_dir, 'Python', 'lib', 'python' + self.py_ver)
157        self.site_packages = self.python_stdlib  # hack to avoid needing to add site-packages to path
158        self.obj_dir = mkdtemp('launchers-')
159
160        self.run()
161
162    def run_shell(self):
163        with current_dir(self.contents_dir):
164            run_shell()
165
166    def run(self):
167        ret = 0
168        self.add_python_framework()
169        self.add_site_packages()
170        self.add_stdlib()
171        self.add_misc_libraries()
172        self.freeze_python()
173        self.add_ca_certs()
174        if not self.dont_strip:
175            self.strip_files()
176        if not self.skip_tests:
177            self.run_tests()
178        # self.run_shell()
179
180        ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION)
181
182        return ret
183
184    @flush
185    def add_ca_certs(self):
186        print('\nDownloading CA certs...')
187        from urllib.request import urlopen
188        cdata = urlopen(kitty_constants['cacerts_url']).read()
189        dest = os.path.join(self.contents_dir, 'Resources', 'cacert.pem')
190        with open(dest, 'wb') as f:
191            f.write(cdata)
192
193    @flush
194    def strip_files(self):
195        print('\nStripping files...')
196        strip_files(self.to_strip)
197
198    @flush
199    def run_tests(self):
200        iv['run_tests'](os.path.join(self.contents_dir, 'MacOS', 'kitty'))
201
202    @flush
203    def set_id(self, path_to_lib, new_id):
204        old_mode = flipwritable(path_to_lib)
205        subprocess.check_call(
206            ['install_name_tool', '-id', new_id, path_to_lib])
207        if old_mode is not None:
208            flipwritable(path_to_lib, old_mode)
209
210    @flush
211    def get_dependencies(self, path_to_lib):
212        install_name = subprocess.check_output(
213            ['otool', '-D', path_to_lib]).decode('utf-8').splitlines()[-1].strip()
214        raw = subprocess.check_output(['otool', '-L', path_to_lib]).decode('utf-8')
215        for line in raw.splitlines():
216            if 'compatibility' not in line or line.strip().endswith(':'):
217                continue
218            idx = line.find('(')
219            path = line[:idx].strip()
220            yield path, path == install_name
221
222    @flush
223    def get_local_dependencies(self, path_to_lib):
224        for x, is_id in self.get_dependencies(path_to_lib):
225            for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'):
226                if x.startswith(y):
227                    if y == PREFIX + '/python/Python.framework/':
228                        y = PREFIX + '/python/'
229                    yield x, x[len(y):], is_id
230                    break
231
232    @flush
233    def change_dep(self, old_dep, new_dep, is_id, path_to_lib):
234        cmd = ['-id', new_dep] if is_id else ['-change', old_dep, new_dep]
235        subprocess.check_call(['install_name_tool'] + cmd + [path_to_lib])
236
237    @flush
238    def fix_dependencies_in_lib(self, path_to_lib):
239        self.to_strip.append(path_to_lib)
240        old_mode = flipwritable(path_to_lib)
241        for dep, bname, is_id in self.get_local_dependencies(path_to_lib):
242            ndep = self.FID + '/' + bname
243            self.change_dep(dep, ndep, is_id, path_to_lib)
244        ldeps = list(self.get_local_dependencies(path_to_lib))
245        if ldeps:
246            print('\nFailed to fix dependencies in', path_to_lib)
247            print('Remaining local dependencies:', ldeps)
248            raise SystemExit(1)
249        if old_mode is not None:
250            flipwritable(path_to_lib, old_mode)
251
252    @flush
253    def add_python_framework(self):
254        print('\nAdding Python framework')
255        src = join(PREFIX + '/python', 'Python.framework')
256        x = join(self.frameworks_dir, 'Python.framework')
257        curr = os.path.realpath(join(src, 'Versions', 'Current'))
258        currd = join(x, 'Versions', basename(curr))
259        rd = join(currd, 'Resources')
260        os.makedirs(rd)
261        shutil.copy2(join(curr, 'Resources', 'Info.plist'), rd)
262        shutil.copy2(join(curr, 'Python'), currd)
263        self.set_id(
264            join(currd, 'Python'),
265            self.FID + '/Python.framework/Versions/%s/Python' % basename(curr))
266        # The following is needed for codesign
267        with current_dir(x):
268            os.symlink(basename(curr), 'Versions/Current')
269            for y in ('Python', 'Resources'):
270                os.symlink('Versions/Current/%s' % y, y)
271
272    @flush
273    def install_dylib(self, path, set_id=True):
274        shutil.copy2(path, self.frameworks_dir)
275        if set_id:
276            self.set_id(
277                join(self.frameworks_dir, basename(path)),
278                self.FID + '/' + basename(path))
279        self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path)))
280
281    @flush
282    def add_misc_libraries(self):
283        for x in (
284                'sqlite3.0',
285                'z.1',
286                'harfbuzz.0',
287                'png16.16',
288                'lcms2.2',
289                'crypto.1.1',
290                'ssl.1.1',
291        ):
292            print('\nAdding', x)
293            x = 'lib%s.dylib' % x
294            src = join(PREFIX, 'lib', x)
295            shutil.copy2(src, self.frameworks_dir)
296            dest = join(self.frameworks_dir, x)
297            self.set_id(dest, self.FID + '/' + x)
298            self.fix_dependencies_in_lib(dest)
299
300    @flush
301    def add_package_dir(self, x, dest=None):
302        def ignore(root, files):
303            ans = []
304            for y in files:
305                ext = os.path.splitext(y)[1]
306                if ext not in ('', '.py', '.so') or \
307                        (not ext and not os.path.isdir(join(root, y))):
308                    ans.append(y)
309
310            return ans
311
312        if dest is None:
313            dest = self.site_packages
314        dest = join(dest, basename(x))
315        shutil.copytree(x, dest, symlinks=True, ignore=ignore)
316        for f in walk(dest):
317            if f.endswith('.so'):
318                self.fix_dependencies_in_lib(f)
319
320    @flush
321    def add_stdlib(self):
322        print('\nAdding python stdlib')
323        src = PREFIX + '/python/Python.framework/Versions/Current/lib/python' + self.py_ver
324        dest = self.python_stdlib
325        if not os.path.exists(dest):
326            os.makedirs(dest)
327        for x in os.listdir(src):
328            if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk',
329                     'lib-old', 'idlelib', 'plat-mac', 'plat-darwin',
330                     'site.py', 'distutils', 'turtledemo', 'tkinter'):
331                continue
332            x = join(src, x)
333            if os.path.isdir(x):
334                self.add_package_dir(x, dest)
335            elif os.path.splitext(x)[1] in ('.so', '.py'):
336                shutil.copy2(x, dest)
337                dest2 = join(dest, basename(x))
338                if dest2.endswith('.so'):
339                    self.fix_dependencies_in_lib(dest2)
340
341    @flush
342    def freeze_python(self):
343        print('\nFreezing python')
344        kitty_dir = join(self.resources_dir, 'kitty')
345        bases = ('kitty', 'kittens', 'kitty_tests')
346        for x in bases:
347            dest = os.path.join(self.python_stdlib, x)
348            os.rename(os.path.join(kitty_dir, x), dest)
349            if x == 'kitty':
350                shutil.rmtree(os.path.join(dest, 'launcher'))
351        os.rename(os.path.join(kitty_dir, '__main__.py'), os.path.join(self.python_stdlib, 'kitty_main.py'))
352        shutil.rmtree(os.path.join(kitty_dir, '__pycache__'))
353        pdir = os.path.join(dirname(self.python_stdlib), 'kitty-extensions')
354        os.mkdir(pdir)
355        print('Extracting extension modules from', self.python_stdlib, 'to', pdir)
356        ext_map = extract_extension_modules(self.python_stdlib, pdir)
357        shutil.copy(os.path.join(os.path.dirname(self_dir), 'site.py'), os.path.join(self.python_stdlib, 'site.py'))
358        for x in bases:
359            iv['sanitize_source_folder'](os.path.join(self.python_stdlib, x))
360        self.compile_py_modules()
361        freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM', remove_pyc_files=True)
362        iv['build_frozen_launcher']([path_to_freeze_dir(), self.obj_dir])
363        os.rename(join(dirname(self.contents_dir), 'bin', 'kitty'), join(self.contents_dir, 'MacOS', 'kitty'))
364        shutil.rmtree(join(dirname(self.contents_dir), 'bin'))
365        self.fix_dependencies_in_lib(join(self.contents_dir, 'MacOS', 'kitty'))
366        for f in walk(pdir):
367            if f.endswith('.so') or f.endswith('.dylib'):
368                self.fix_dependencies_in_lib(f)
369
370    @flush
371    def add_site_packages(self):
372        print('\nAdding site-packages')
373        os.makedirs(self.site_packages)
374        sys_path = json.loads(subprocess.check_output([
375            PYTHON, '-c', 'import sys, json; json.dump(sys.path, sys.stdout)']))
376        paths = reversed(tuple(map(abspath, [x for x in sys_path if x.startswith('/') and not x.startswith('/Library/')])))
377        upaths = []
378        for x in paths:
379            if x not in upaths and (x.endswith('.egg') or x.endswith('/site-packages')):
380                upaths.append(x)
381        for x in upaths:
382            print('\t', x)
383            tdir = None
384            try:
385                if not os.path.isdir(x):
386                    zf = zipfile.ZipFile(x)
387                    tdir = tempfile.mkdtemp()
388                    zf.extractall(tdir)
389                    x = tdir
390                self.add_modules_from_dir(x)
391                self.add_packages_from_dir(x)
392            finally:
393                if tdir is not None:
394                    shutil.rmtree(tdir)
395        self.remove_bytecode(self.site_packages)
396
397    @flush
398    def add_modules_from_dir(self, src):
399        for x in glob.glob(join(src, '*.py')) + glob.glob(join(src, '*.so')):
400            shutil.copy2(x, self.site_packages)
401            if x.endswith('.so'):
402                self.fix_dependencies_in_lib(x)
403
404    @flush
405    def add_packages_from_dir(self, src):
406        for x in os.listdir(src):
407            x = join(src, x)
408            if os.path.isdir(x) and os.path.exists(join(x, '__init__.py')):
409                if self.filter_package(basename(x)):
410                    continue
411                self.add_package_dir(x)
412
413    @flush
414    def filter_package(self, name):
415        return name in ('Cython', 'modulegraph', 'macholib', 'py2app',
416                        'bdist_mpkg', 'altgraph')
417
418    @flush
419    def remove_bytecode(self, dest):
420        for x in os.walk(dest):
421            root = x[0]
422            for f in x[-1]:
423                if os.path.splitext(f) == '.pyc':
424                    os.remove(join(root, f))
425
426    @flush
427    def compile_py_modules(self):
428        self.remove_bytecode(join(self.resources_dir, 'Python'))
429        py_compile(join(self.resources_dir, 'Python'))
430
431    @flush
432    def makedmg(self, d, volname, format='ULFO'):
433        ''' Copy a directory d into a dmg named volname '''
434        print('\nMaking dmg...')
435        sys.stdout.flush()
436        destdir = os.path.join(SW, 'dist')
437        try:
438            shutil.rmtree(destdir)
439        except FileNotFoundError:
440            pass
441        os.mkdir(destdir)
442        dmg = os.path.join(destdir, volname + '.dmg')
443        if os.path.exists(dmg):
444            os.unlink(dmg)
445        tdir = tempfile.mkdtemp()
446        appdir = os.path.join(tdir, os.path.basename(d))
447        shutil.copytree(d, appdir, symlinks=True)
448        if self.sign_installers:
449            with timeit() as times:
450                sign_app(appdir, self.notarize)
451            print('Signing completed in %d minutes %d seconds' % tuple(times))
452        os.symlink('/Applications', os.path.join(tdir, 'Applications'))
453        size_in_mb = int(
454            subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8')
455            .split()[0]) / 1024.
456        cmd = [
457            '/usr/bin/hdiutil', 'create', '-srcfolder', tdir, '-volname',
458            volname, '-format', format
459        ]
460        if 190 < size_in_mb < 250:
461            # We need -size 255m because of a bug in hdiutil. When the size of
462            # srcfolder is close to 200MB hdiutil fails with
463            # diskimages-helper: resize request is above maximum size allowed.
464            cmd += ['-size', '255m']
465        print('\nCreating dmg...')
466        with timeit() as times:
467            subprocess.check_call(cmd + [dmg])
468        print('dmg created in %d minutes and %d seconds' % tuple(times))
469        shutil.rmtree(tdir)
470        size = os.stat(dmg).st_size / (1024 * 1024.)
471        print('\nInstaller size: %.2fMB\n' % size)
472        return dmg
473
474
475def main():
476    args = globals()['args']
477    ext_dir = globals()['ext_dir']
478    Freeze(
479        os.path.join(ext_dir, kitty_constants['appname'] + '.app'),
480        dont_strip=args.dont_strip,
481        sign_installers=args.sign_installers,
482        notarize=args.notarize,
483        skip_tests=args.skip_tests
484    )
485
486
487if __name__ == '__main__':
488    main()
489