1#!/usr/bin/python -u -OO 2 3import os 4from optparse import OptionParser 5from bockbuild.util.util import * 6from bockbuild.util.csproj import * 7from bockbuild.environment import Environment 8from bockbuild.package import * 9from bockbuild.profile import Profile 10import collections 11import hashlib 12import itertools 13import traceback 14from collections import namedtuple 15 16ProfileDesc = namedtuple ('Profile', 'name description path modes') 17 18global active_profile, bockbuild 19active_profile = None 20bockbuild = None 21 22def find_profiles (base_path): 23 assert Profile.loaded == None 24 25 search_path = first_existing(['%s/bockbuild' % base_path, '%s/packaging' % base_path]) 26 sys.path.append(search_path) 27 profiles = [] 28 resolved_names = [] 29 while True: 30 progress_made = False 31 for path in iterate_dir (search_path, with_dirs=True): 32 file = '%s/profile.py' % path 33 if os.path.isdir (path) and os.path.isfile (file): 34 name = os.path.basename (path) 35 if name in resolved_names: 36 continue 37 38 fail = None 39 profile = None 40 try: 41 execfile(file, globals()) 42 if not Profile.loaded: 43 fail = 'No profile loaded' 44 profile = Profile.loaded 45 except Exception as e: 46 fail = e 47 finally: 48 Profile.loaded = None 49 50 if not fail: 51 profile = Profile.loaded 52 Profile.loaded = None 53 progress_made = True 54 description = "" 55 if hasattr(profile.__class__, 'description'): 56 description = profile.__class__.description 57 profiles.append (ProfileDesc (name = name, description = description, path = path, modes = "")) 58 resolved_names.append(name) 59 else: 60 warn(fail) 61 62 if not progress_made: 63 break 64 assert Profile.loaded == None 65 return profiles 66 67class Bockbuild: 68 69 def run(self): 70 self.name = 'bockbuild' 71 self.root = os.path.dirname (os.path.abspath(__file__)) # Bockbuild system root 72 self.execution_root = os.getcwd() 73 self.resources = set([os.path.realpath( 74 os.path.join(self.root, 'packages'))]) # list of paths on where to look for packages, patches, etc. 75 76 config.state_root = self.root # root path for all storage; artifacts, build I/O, cache, storage and output 77 config.protected_git_repos.append (self.root) 78 config.absolute_root = os.path.commonprefix([self.root, self.execution_root]) 79 80 self.build_root = os.path.join(config.state_root, 'builds') 81 self.staged_prefix = os.path.join(config.state_root, 'stage') 82 self.toolchain_root = os.path.join(config.state_root, 'toolchain') 83 self.artifact_root = os.path.join(config.state_root, 'artifacts') 84 self.package_root = os.path.join(config.state_root, 'distribution') 85 self.scratch = os.path.join(config.state_root, 'scratch') 86 self.logs = os.path.join(config.state_root, 'logs') 87 self.env_file = os.path.join(config.state_root, 'last-successful-build.env') 88 self.source_cache = os.getenv('BOCKBUILD_SOURCE_CACHE') or os.path.realpath( 89 os.path.join(config.state_root, 'cache')) 90 self.cpu_count = get_cpu_count() 91 self.host = get_host() 92 self.uname = backtick('uname -a') 93 94 self.full_rebuild = False 95 96 self.toolchain = [] 97 98 find_git(self) 99 self.bockbuild_rev = git_shortid(self, self.root) 100 self.profile_root = git_rootdir (self, self.execution_root) 101 self.profiles = find_profiles (self.profile_root) 102 103 for profile in self.profiles: 104 self.resources.add(profile.path) 105 106 loginit('bockbuild (%s)' % (self.bockbuild_rev)) 107 info('cmd: %s' % ' '.join(sys.argv)) 108 109 if len (sys.argv) < 2: 110 info ('Profiles in %s --' % self.git ('config --get remote.origin.url', self.profile_root)[0]) 111 info(map (lambda x: '\t%s: %s' % (x.name, x.description), self.profiles)) 112 finish (exit_codes.FAILURE) 113 114 global active_profile 115 Package.profile = active_profile = self.load_profile (sys.argv[1]) 116 117 self.parser = self.init_parser() 118 self.cmd_options, self.cmd_args = self.parser.parse_args(sys.argv[2:]) 119 120 self.packages_to_build = self.cmd_args or active_profile.packages 121 122 123 active_profile.setup() 124 self.verbose = self.cmd_options.verbose 125 config.verbose = self.cmd_options.verbose 126 self.arch = self.cmd_options.arch 127 self.unsafe = self.cmd_options.unsafe 128 config.trace = self.cmd_options.trace 129 self.tracked_env = [] 130 131 132 133 ensure_dir(self.source_cache, purge=False) 134 ensure_dir(self.artifact_root, purge=False) 135 ensure_dir(self.build_root, purge=False) 136 ensure_dir(self.scratch, purge=True) 137 ensure_dir(self.logs, purge=False) 138 139 self.build() 140 141 def init_parser(self): 142 parser = OptionParser( 143 usage='usage: %prog [options] [package_names...]') 144 parser.add_option('--build', 145 action='store_true', dest='do_build', default=True, 146 help='build the profile') 147 parser.add_option('--package', 148 action='store_true', dest='do_package', default=False, 149 help='package the profile') 150 parser.add_option('--verbose', 151 action='store_true', dest='verbose', default=False, 152 help='show all build output (e.g. configure, make)') 153 parser.add_option('-d', '--debug', default=False, 154 action='store_true', dest='debug', 155 help='Build with debug flags enabled') 156 parser.add_option('-e', '--environment', default=False, 157 action='store_true', dest='dump_environment', 158 help='Dump the profile environment as a shell-sourceable list of exports ') 159 parser.add_option('-r', '--release', default=False, 160 action='store_true', dest='release_build', 161 help='Whether or not this build is a release build') 162 parser.add_option('', '--csproj-env', default=False, 163 action='store_true', dest='dump_environment_csproj', 164 help='Dump the profile environment xml formarted for use in .csproj files') 165 parser.add_option('', '--csproj-insert', default=None, 166 action='store', dest='csproj_file', 167 help='Inserts the profile environment variables into VS/MonoDevelop .csproj files') 168 parser.add_option('', '--arch', default='default', 169 action='store', dest='arch', 170 help='Select the target architecture(s) for the package') 171 parser.add_option('', '--shell', default=False, 172 action='store_true', dest='shell', 173 help='Get an shell with the package environment') 174 parser.add_option('', '--unsafe', default=False, 175 action='store_true', dest='unsafe', 176 help='Prevents full rebuilds when a build environment change is detected. Useful for debugging.') 177 parser.add_option('', '--trace', default=False, 178 action='store_true', dest='trace', 179 help='Enable tracing (for diagnosing bockbuild problems') 180 181 return parser 182 183 def build_distribution(self, packages, dest, stage, arch): 184 # TODO: full relocation means that we shouldn't need dest at this stage 185 build_list = [] 186 stage_invalidated = False #if anything is dirty we flush the stageination path and fill it again 187 188 if self.full_rebuild: 189 ensure_dir (stage, purge = True) 190 191 progress('Fetching packages') 192 for package in packages.values(): 193 package.build_artifact = os.path.join( 194 self.artifact_root, '%s-%s' % (package.name, arch)) 195 package.buildstring_file = package.build_artifact + '.buildstring' 196 package.log = os.path.join(self.logs, package.name + '.log') 197 if os.path.exists(package.log): 198 delete(package.log) 199 200 package.source_dir_name = expand_macros(package.source_dir_name, package) 201 workspace_path = os.path.join(self.build_root, package.source_dir_name) 202 package.fetch(workspace_path) 203 204 if self.full_rebuild: 205 package.request_build('Full rebuild') 206 207 elif not os.path.exists(package.build_artifact): 208 package.request_build('No artifact') 209 210 elif is_changed(package.buildstring, package.buildstring_file): 211 package.request_build('Updated') 212 213 if package.needs_build: 214 build_list.append(package) 215 stage_invalidated = True 216 217 verbose('%d packages need building:' % len(build_list)) 218 verbose(['%s (%s)' % (x.name, x.needs_build) for x in build_list]) 219 220 if stage_invalidated: 221 ensure_dir (stage, purge = True) 222 for package in packages.values(): 223 package.deploy_requests.append (stage) 224 225 for package in packages.values(): 226 package.start_build(arch, dest, stage) 227 # make artifact in scratch 228 # delete artifact + buildstring 229 with open(package.buildstring_file, 'w') as output: 230 output.write('\n'.join(package.buildstring)) 231 232 def build(self): 233 profile = active_profile 234 env = profile.env 235 236 if self.cmd_options.dump_environment: 237 env.compile() 238 env.dump() 239 sys.exit(0) 240 241 if self.cmd_options.dump_environment_csproj: 242 # specify to use our GAC, else MonoDevelop would 243 # use its own 244 env.set('MONO_GAC_PREFIX', self.staged_prefix) 245 246 env.compile() 247 env.dump_csproj() 248 sys.exit(0) 249 250 if self.cmd_options.csproj_file is not None: 251 env.set('MONO_GAC_PREFIX', self.staged_prefix) 252 env.compile() 253 env.write_csproj(self.cmd_options.csproj_file) 254 sys.exit(0) 255 256 profile.toolchain_packages = collections.OrderedDict() 257 for source in self.toolchain: 258 package = self.load_package(source) 259 profile.toolchain_packages[package.name] = package 260 261 profile.release_packages = collections.OrderedDict() 262 for source in self.packages_to_build: 263 package = self.load_package(source) 264 profile.release_packages[package.name] = package 265 266 profile.setup_release() 267 268 if self.track_env(): 269 if self.unsafe: 270 warn('Build environment changed, but overriding full rebuild!') 271 else: 272 info('Build environment changed, full rebuild triggered') 273 self.full_rebuild = True 274 ensure_dir(self.build_root, purge=True) 275 276 if self.cmd_options.shell: 277 title('Shell') 278 self.shell() 279 280 if self.cmd_options.do_build: 281 title('Building toolchain') 282 self.build_distribution( 283 profile.toolchain_packages, self.toolchain_root, self.toolchain_root, arch='toolchain') 284 285 title('Building release') 286 self.build_distribution( 287 profile.release_packages, profile.prefix, self.staged_prefix, arch=self.arch) 288 289 # update env 290 with open(self.env_file, 'w') as output: 291 output.write('\n'.join(self.tracked_env)) 292 293 if self.cmd_options.do_package: 294 title('Packaging') 295 protect_dir(self.staged_prefix) 296 ensure_dir(self.package_root, True) 297 298 run_shell('rsync -aPq %s/* %s' % 299 (self.staged_prefix, self.package_root), False) 300 unprotect_dir(self.package_root) 301 302 profile.process_release(self.package_root) 303 profile.package() 304 305 finish(exit_codes.SUCCESS) 306 307 def track_env(self): 308 env = active_profile.env 309 env.compile() 310 env.export() 311 self.env_script = os.path.join( 312 self.root, self.profile_name) + '_env.sh' 313 env.write_source_script(self.env_script) 314 315 self.tracked_env.extend(env.serialize()) 316 return is_changed(self.tracked_env, self.env_file) 317 318 def load_package(self, source): 319 if isinstance(source, Package): # package can already be loaded in the source list 320 return source 321 322 fullpath = None 323 for i in self.resources: 324 candidate_fullpath = os.path.join(i, source + '.py') 325 if os.path.exists(candidate_fullpath): 326 if fullpath is not None: 327 error ('Package "%s" resolved in multiple locations (search paths: %s' % (source, self.resources)) 328 fullpath = candidate_fullpath 329 330 if not fullpath: 331 error("Package '%s' not found ('search paths: %s')" % (source, self.resources)) 332 333 Package.last_instance = None 334 335 trace(fullpath) 336 execfile(fullpath, globals()) 337 338 if Package.last_instance is None: 339 error('%s does not provide a valid package.' % source) 340 341 new_package = Package.last_instance 342 new_package._path = fullpath 343 return new_package 344 345 def load_profile(self, source): 346 if Profile.loaded: 347 error ('A profile is already loaded: %s' % Profile.loaded) 348 path = None 349 for profile in self.profiles: 350 if profile.name == source: 351 path = profile.path 352 353 if path == None: 354 if isinstance(source, Profile): # package can already be loaded in the source list 355 Profile.loaded = source 356 else: 357 error("Profile '%s' not found" % source) 358 359 fullpath = os.path.join(path, 'profile.py') 360 361 if not os.path.exists(fullpath): 362 error("Profile '%s' not found" % source) 363 364 sys.path.append (path) 365 self.resources.add (path) 366 execfile(fullpath, globals()) 367 Profile.loaded.attach (self) 368 369 if Profile.loaded is None: 370 error('%s does not provide a valid profile (developers: ensure Profile.attach() is called.)' % source) 371 372 if Profile.loaded.bockbuild is None: 373 error ('Profile init is invalid: Failed to attach to bockbuild object') 374 375 new_profile = Profile.loaded 376 new_profile._path = fullpath 377 new_profile.directory = path 378 379 new_profile.git_root = git_rootdir (self, os.path.dirname (path)) 380 config.protected_git_repos.append (new_profile.git_root) 381 self.profile_name = source 382 return new_profile 383 384if __name__ == "__main__": 385 try: 386 bockbuild = Bockbuild() 387 bockbuild.run() 388 except Exception as e: 389 exc_type, exc_value, exc_traceback = sys.exc_info() 390 error('%s (%s)' % (e, exc_type.__name__), more_output=True) 391 error(('%s:%s @%s\t\t"%s"' % p for p in traceback.extract_tb( 392 exc_traceback)[-5:])) 393 except KeyboardInterrupt: 394 error('Interrupted.') 395 finally: 396 if config.exit_code == exit_codes.NOTSET: 397 print 'spurious sys.exit() call' 398 if config.exit_code == exit_codes.SUCCESS: 399 logprint('\n** %s **\n' % 'Goodbye!', bcolors.BOLD) 400 sys.exit (config.exit_code) 401