1#!/usr/bin/python -u -OO
2
3import os
4from optparse import OptionParser
5from bockbuild.util.util import *
6from bockbuild.util.csproj import *
7from bockbuild.environment import Environment
8from bockbuild.package import *
9from bockbuild.profile import Profile
10import collections
11import hashlib
12import itertools
13import traceback
14from collections import namedtuple
15
16ProfileDesc = namedtuple ('Profile', 'name description path modes')
17
18global active_profile, bockbuild
19active_profile = None
20bockbuild = None
21
22def find_profiles (base_path):
23    assert Profile.loaded == None
24
25    search_path = first_existing(['%s/bockbuild' % base_path, '%s/packaging' % base_path])
26    sys.path.append(search_path)
27    profiles = []
28    resolved_names = []
29    while True:
30        progress_made = False
31        for path in iterate_dir (search_path, with_dirs=True):
32            file = '%s/profile.py' % path
33            if os.path.isdir (path) and os.path.isfile (file):
34                name = os.path.basename (path)
35                if name in resolved_names:
36                    continue
37
38                fail = None
39                profile = None
40                try:
41                    execfile(file, globals())
42                    if not Profile.loaded:
43                        fail = 'No profile loaded'
44                    profile = Profile.loaded
45                except Exception as e:
46                    fail = e
47                finally:
48                    Profile.loaded = None
49
50                if not fail:
51                    profile = Profile.loaded
52                    Profile.loaded = None
53                    progress_made = True
54                    description = ""
55                    if hasattr(profile.__class__, 'description'):
56                        description = profile.__class__.description
57                    profiles.append (ProfileDesc (name = name, description = description, path = path, modes = ""))
58                    resolved_names.append(name)
59                else:
60                    warn(fail)
61
62        if not progress_made:
63            break
64    assert Profile.loaded == None
65    return profiles
66
67class Bockbuild:
68
69    def run(self):
70        self.name = 'bockbuild'
71        self.root = os.path.dirname (os.path.abspath(__file__)) # Bockbuild system root
72        self.execution_root = os.getcwd()
73        self.resources = set([os.path.realpath(
74            os.path.join(self.root, 'packages'))]) # list of paths on where to look for packages, patches, etc.
75
76        config.state_root = self.root # root path for all storage; artifacts, build I/O, cache, storage and output
77        config.protected_git_repos.append (self.root)
78        config.absolute_root = os.path.commonprefix([self.root, self.execution_root])
79
80        self.build_root = os.path.join(config.state_root, 'builds')
81        self.staged_prefix = os.path.join(config.state_root, 'stage')
82        self.toolchain_root = os.path.join(config.state_root, 'toolchain')
83        self.artifact_root = os.path.join(config.state_root, 'artifacts')
84        self.package_root = os.path.join(config.state_root, 'distribution')
85        self.scratch = os.path.join(config.state_root, 'scratch')
86        self.logs = os.path.join(config.state_root, 'logs')
87        self.env_file = os.path.join(config.state_root, 'last-successful-build.env')
88        self.source_cache = os.getenv('BOCKBUILD_SOURCE_CACHE') or os.path.realpath(
89            os.path.join(config.state_root, 'cache'))
90        self.cpu_count = get_cpu_count()
91        self.host = get_host()
92        self.uname = backtick('uname -a')
93
94        self.full_rebuild = False
95
96        self.toolchain = []
97
98        find_git(self)
99        self.bockbuild_rev = git_shortid(self, self.root)
100        self.profile_root = git_rootdir (self, self.execution_root)
101        self.profiles = find_profiles (self.profile_root)
102
103        for profile in self.profiles:
104            self.resources.add(profile.path)
105
106        loginit('bockbuild (%s)' % (self.bockbuild_rev))
107        info('cmd: %s' % ' '.join(sys.argv))
108
109        if len (sys.argv) < 2:
110            info ('Profiles in %s --' % self.git ('config --get remote.origin.url', self.profile_root)[0])
111            info(map (lambda x: '\t%s: %s' % (x.name, x.description), self.profiles))
112            finish (exit_codes.FAILURE)
113
114        global active_profile
115        Package.profile = active_profile = self.load_profile (sys.argv[1])
116
117        self.parser = self.init_parser()
118        self.cmd_options, self.cmd_args = self.parser.parse_args(sys.argv[2:])
119
120        self.packages_to_build = self.cmd_args or active_profile.packages
121
122
123        active_profile.setup()
124        self.verbose = self.cmd_options.verbose
125        config.verbose = self.cmd_options.verbose
126        self.arch = self.cmd_options.arch
127        self.unsafe = self.cmd_options.unsafe
128        config.trace = self.cmd_options.trace
129        self.tracked_env = []
130
131
132
133        ensure_dir(self.source_cache, purge=False)
134        ensure_dir(self.artifact_root, purge=False)
135        ensure_dir(self.build_root, purge=False)
136        ensure_dir(self.scratch, purge=True)
137        ensure_dir(self.logs, purge=False)
138
139        self.build()
140
141    def init_parser(self):
142        parser = OptionParser(
143            usage='usage: %prog [options] [package_names...]')
144        parser.add_option('--build',
145                          action='store_true', dest='do_build', default=True,
146                          help='build the profile')
147        parser.add_option('--package',
148                          action='store_true', dest='do_package', default=False,
149                          help='package the profile')
150        parser.add_option('--verbose',
151                          action='store_true', dest='verbose', default=False,
152                          help='show all build output (e.g. configure, make)')
153        parser.add_option('-d', '--debug', default=False,
154                          action='store_true', dest='debug',
155                          help='Build with debug flags enabled')
156        parser.add_option('-e', '--environment', default=False,
157                          action='store_true', dest='dump_environment',
158                          help='Dump the profile environment as a shell-sourceable list of exports ')
159        parser.add_option('-r', '--release', default=False,
160                          action='store_true', dest='release_build',
161                          help='Whether or not this build is a release build')
162        parser.add_option('', '--csproj-env', default=False,
163                          action='store_true', dest='dump_environment_csproj',
164                          help='Dump the profile environment xml formarted for use in .csproj files')
165        parser.add_option('', '--csproj-insert', default=None,
166                          action='store', dest='csproj_file',
167                          help='Inserts the profile environment variables into VS/MonoDevelop .csproj files')
168        parser.add_option('', '--arch', default='default',
169                          action='store', dest='arch',
170                          help='Select the target architecture(s) for the package')
171        parser.add_option('', '--shell', default=False,
172                          action='store_true', dest='shell',
173                          help='Get an shell with the package environment')
174        parser.add_option('', '--unsafe', default=False,
175                          action='store_true', dest='unsafe',
176                          help='Prevents full rebuilds when a build environment change is detected. Useful for debugging.')
177        parser.add_option('', '--trace', default=False,
178                          action='store_true', dest='trace',
179                          help='Enable tracing (for diagnosing bockbuild problems')
180
181        return parser
182
183    def build_distribution(self, packages, dest, stage, arch):
184        # TODO: full relocation means that we shouldn't need dest at this stage
185        build_list = []
186        stage_invalidated = False #if anything is dirty we flush the stageination path and fill it again
187
188        if self.full_rebuild:
189            ensure_dir (stage, purge = True)
190
191        progress('Fetching packages')
192        for package in packages.values():
193            package.build_artifact = os.path.join(
194                self.artifact_root, '%s-%s' % (package.name, arch))
195            package.buildstring_file = package.build_artifact + '.buildstring'
196            package.log = os.path.join(self.logs, package.name + '.log')
197            if os.path.exists(package.log):
198                delete(package.log)
199
200            package.source_dir_name = expand_macros(package.source_dir_name, package)
201            workspace_path = os.path.join(self.build_root, package.source_dir_name)
202            package.fetch(workspace_path)
203
204            if self.full_rebuild:
205                package.request_build('Full rebuild')
206
207            elif not os.path.exists(package.build_artifact):
208                package.request_build('No artifact')
209
210            elif is_changed(package.buildstring, package.buildstring_file):
211                package.request_build('Updated')
212
213            if package.needs_build:
214                build_list.append(package)
215                stage_invalidated = True
216
217        verbose('%d packages need building:' % len(build_list))
218        verbose(['%s (%s)' % (x.name, x.needs_build) for x in build_list])
219
220        if stage_invalidated:
221            ensure_dir (stage, purge = True)
222            for package in packages.values():
223                package.deploy_requests.append (stage)
224
225        for package in packages.values():
226            package.start_build(arch, dest, stage)
227            # make artifact in scratch
228            # delete artifact + buildstring
229            with open(package.buildstring_file, 'w') as output:
230                output.write('\n'.join(package.buildstring))
231
232    def build(self):
233        profile = active_profile
234        env = profile.env
235
236        if self.cmd_options.dump_environment:
237            env.compile()
238            env.dump()
239            sys.exit(0)
240
241        if self.cmd_options.dump_environment_csproj:
242            # specify to use our GAC, else MonoDevelop would
243            # use its own
244            env.set('MONO_GAC_PREFIX', self.staged_prefix)
245
246            env.compile()
247            env.dump_csproj()
248            sys.exit(0)
249
250        if self.cmd_options.csproj_file is not None:
251            env.set('MONO_GAC_PREFIX', self.staged_prefix)
252            env.compile()
253            env.write_csproj(self.cmd_options.csproj_file)
254            sys.exit(0)
255
256        profile.toolchain_packages = collections.OrderedDict()
257        for source in self.toolchain:
258            package = self.load_package(source)
259            profile.toolchain_packages[package.name] = package
260
261        profile.release_packages = collections.OrderedDict()
262        for source in self.packages_to_build:
263            package = self.load_package(source)
264            profile.release_packages[package.name] = package
265
266        profile.setup_release()
267
268        if self.track_env():
269            if self.unsafe:
270                warn('Build environment changed, but overriding full rebuild!')
271            else:
272                info('Build environment changed, full rebuild triggered')
273                self.full_rebuild = True
274                ensure_dir(self.build_root, purge=True)
275
276        if self.cmd_options.shell:
277            title('Shell')
278            self.shell()
279
280        if self.cmd_options.do_build:
281            title('Building toolchain')
282            self.build_distribution(
283                profile.toolchain_packages, self.toolchain_root, self.toolchain_root, arch='toolchain')
284
285            title('Building release')
286            self.build_distribution(
287                profile.release_packages, profile.prefix, self.staged_prefix, arch=self.arch)
288
289            # update env
290            with open(self.env_file, 'w') as output:
291                output.write('\n'.join(self.tracked_env))
292
293        if self.cmd_options.do_package:
294            title('Packaging')
295            protect_dir(self.staged_prefix)
296            ensure_dir(self.package_root, True)
297
298            run_shell('rsync -aPq %s/* %s' %
299                      (self.staged_prefix, self.package_root), False)
300            unprotect_dir(self.package_root)
301
302            profile.process_release(self.package_root)
303            profile.package()
304
305        finish(exit_codes.SUCCESS)
306
307    def track_env(self):
308        env = active_profile.env
309        env.compile()
310        env.export()
311        self.env_script = os.path.join(
312            self.root, self.profile_name) + '_env.sh'
313        env.write_source_script(self.env_script)
314
315        self.tracked_env.extend(env.serialize())
316        return is_changed(self.tracked_env, self.env_file)
317
318    def load_package(self, source):
319        if isinstance(source, Package):  # package can already be loaded in the source list
320            return source
321
322        fullpath = None
323        for i in self.resources:
324            candidate_fullpath = os.path.join(i, source + '.py')
325            if os.path.exists(candidate_fullpath):
326                if fullpath is not None:
327                    error ('Package "%s" resolved in multiple locations (search paths: %s' % (source, self.resources))
328                fullpath = candidate_fullpath
329
330        if not fullpath:
331            error("Package '%s' not found ('search paths: %s')" % (source, self.resources))
332
333        Package.last_instance = None
334
335        trace(fullpath)
336        execfile(fullpath, globals())
337
338        if Package.last_instance is None:
339            error('%s does not provide a valid package.' % source)
340
341        new_package = Package.last_instance
342        new_package._path = fullpath
343        return new_package
344
345    def load_profile(self, source):
346        if Profile.loaded:
347            error ('A profile is already loaded: %s' % Profile.loaded)
348        path = None
349        for profile in self.profiles:
350            if profile.name == source:
351                path = profile.path
352
353        if path == None:
354            if isinstance(source, Profile):  # package can already be loaded in the source list
355                Profile.loaded = source
356            else:
357                error("Profile '%s' not found" % source)
358
359        fullpath = os.path.join(path, 'profile.py')
360
361        if not os.path.exists(fullpath):
362            error("Profile '%s' not found" % source)
363
364        sys.path.append (path)
365        self.resources.add (path)
366        execfile(fullpath, globals())
367        Profile.loaded.attach (self)
368
369        if Profile.loaded is None:
370            error('%s does not provide a valid profile (developers: ensure Profile.attach() is called.)' % source)
371
372        if Profile.loaded.bockbuild is None:
373            error ('Profile init is invalid: Failed to attach to bockbuild object')
374
375        new_profile = Profile.loaded
376        new_profile._path = fullpath
377        new_profile.directory = path
378
379        new_profile.git_root = git_rootdir (self, os.path.dirname (path))
380        config.protected_git_repos.append (new_profile.git_root)
381        self.profile_name = source
382        return new_profile
383
384if __name__ == "__main__":
385    try:
386        bockbuild = Bockbuild()
387        bockbuild.run()
388    except Exception as e:
389        exc_type, exc_value, exc_traceback = sys.exc_info()
390        error('%s (%s)' % (e, exc_type.__name__), more_output=True)
391        error(('%s:%s @%s\t\t"%s"' % p for p in traceback.extract_tb(
392            exc_traceback)[-5:]))
393    except KeyboardInterrupt:
394        error('Interrupted.')
395    finally:
396        if config.exit_code == exit_codes.NOTSET:
397            print 'spurious sys.exit() call'
398        if config.exit_code == exit_codes.SUCCESS:
399            logprint('\n** %s **\n' % 'Goodbye!', bcolors.BOLD)
400        sys.exit (config.exit_code)
401