1#!/usr/bin/env python 2# encoding: utf-8 3# Remote Builds tool using rsync+ssh 4 5__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>" 6__copyright__ = "Jérôme Carretero, 2013" 7 8""" 9Simple Remote Builds 10******************** 11 12This tool is an *experimental* tool (meaning, do not even try to pollute 13the waf bug tracker with bugs in here, contact me directly) providing simple 14remote builds. 15 16It uses rsync and ssh to perform the remote builds. 17It is intended for performing cross-compilation on platforms where 18a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product 19does not exist (eg. Windows builds using Visual Studio) or simply not installed. 20This tool sends the sources and the waf script to the remote host, 21and commands the usual waf execution. 22 23There are alternatives to using this tool, such as setting up shared folders, 24logging on to remote machines, and building on the shared folders. 25Electing one method or another depends on the size of the program. 26 27 28Usage 29===== 30 311. Set your wscript file so it includes a list of variants, 32 e.g.:: 33 34 from waflib import Utils 35 top = '.' 36 out = 'build' 37 38 variants = [ 39 'linux_64_debug', 40 'linux_64_release', 41 'linux_32_debug', 42 'linux_32_release', 43 ] 44 45 from waflib.extras import remote 46 47 def options(opt): 48 # normal stuff from here on 49 opt.load('compiler_c') 50 51 def configure(conf): 52 if not conf.variant: 53 return 54 # normal stuff from here on 55 conf.load('compiler_c') 56 57 def build(bld): 58 if not bld.variant: 59 return 60 # normal stuff from here on 61 bld(features='c cprogram', target='app', source='main.c') 62 63 642. Build the waf file, so it includes this tool, and put it in the current 65 directory 66 67 .. code:: bash 68 69 ./waf-light --tools=remote 70 713. Set the host names to access the hosts: 72 73 .. code:: bash 74 75 export REMOTE_QNX=user@kiunix 76 774. Setup the ssh server and ssh keys 78 79 The ssh key should not be protected by a password, or it will prompt for it every time. 80 Create the key on the client: 81 82 .. code:: bash 83 84 ssh-keygen -t rsa -f foo.rsa 85 86 Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys), 87 and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys) 88 89 A separate key for the build processes can be set in the environment variable WAF_SSH_KEY. 90 The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so 91 be warned to use this feature on internal networks only (MITM). 92 93 .. code:: bash 94 95 export WAF_SSH_KEY=~/foo.rsa 96 975. Perform the build: 98 99 .. code:: bash 100 101 waf configure_all build_all --remote 102 103""" 104 105 106import getpass, os, re, sys 107from collections import OrderedDict 108from waflib import Context, Options, Utils, ConfigSet 109 110from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext 111from waflib.Configure import ConfigurationContext 112 113 114is_remote = False 115if '--remote' in sys.argv: 116 is_remote = True 117 sys.argv.remove('--remote') 118 119class init(Context.Context): 120 """ 121 Generates the *_all commands 122 """ 123 cmd = 'init' 124 fun = 'init' 125 def execute(self): 126 for x in list(Context.g_module.variants): 127 self.make_variant(x) 128 lst = ['remote'] 129 for k in Options.commands: 130 if k.endswith('_all'): 131 name = k.replace('_all', '') 132 for x in Context.g_module.variants: 133 lst.append('%s_%s' % (name, x)) 134 else: 135 lst.append(k) 136 del Options.commands[:] 137 Options.commands += lst 138 139 def make_variant(self, x): 140 for y in (BuildContext, CleanContext, InstallContext, UninstallContext): 141 name = y.__name__.replace('Context','').lower() 142 class tmp(y): 143 cmd = name + '_' + x 144 fun = 'build' 145 variant = x 146 class tmp(ConfigurationContext): 147 cmd = 'configure_' + x 148 fun = 'configure' 149 variant = x 150 def __init__(self, **kw): 151 ConfigurationContext.__init__(self, **kw) 152 self.setenv(x) 153 154class remote(BuildContext): 155 cmd = 'remote' 156 fun = 'build' 157 158 def get_ssh_hosts(self): 159 lst = [] 160 for v in Context.g_module.variants: 161 self.env.HOST = self.login_to_host(self.variant_to_login(v)) 162 cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env) 163 out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH) 164 lst.append(out.strip()) 165 return lst 166 167 def setup_private_ssh_key(self): 168 """ 169 When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory 170 Make sure that the ssh key does not prompt for a password 171 """ 172 key = os.environ.get('WAF_SSH_KEY', '') 173 if not key: 174 return 175 if not os.path.isfile(key): 176 self.fatal('Key in WAF_SSH_KEY must point to a valid file') 177 self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh') 178 self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts') 179 self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key)) 180 self.ssh_config = os.path.join(self.ssh_dir, 'config') 181 for x in self.ssh_hosts, self.ssh_key, self.ssh_config: 182 if not os.path.isfile(x): 183 if not os.path.isdir(self.ssh_dir): 184 os.makedirs(self.ssh_dir) 185 Utils.writef(self.ssh_key, Utils.readf(key), 'wb') 186 os.chmod(self.ssh_key, 448) 187 188 Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts())) 189 os.chmod(self.ssh_key, 448) 190 191 Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb') 192 os.chmod(self.ssh_config, 448) 193 self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key] 194 self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh') 195 196 def skip_unbuildable_variant(self): 197 # skip variants that cannot be built on this OS 198 for k in Options.commands: 199 a, _, b = k.partition('_') 200 if b in Context.g_module.variants: 201 c, _, _ = b.partition('_') 202 if c != Utils.unversioned_sys_platform(): 203 Options.commands.remove(k) 204 205 def login_to_host(self, login): 206 return re.sub(r'(\w+@)', '', login) 207 208 def variant_to_login(self, variant): 209 """linux_32_debug -> search env.LINUX_32 and then env.LINUX""" 210 x = variant[:variant.rfind('_')] 211 ret = os.environ.get('REMOTE_' + x.upper(), '') 212 if not ret: 213 x = x[:x.find('_')] 214 ret = os.environ.get('REMOTE_' + x.upper(), '') 215 if not ret: 216 ret = '%s@localhost' % getpass.getuser() 217 return ret 218 219 def execute(self): 220 global is_remote 221 if not is_remote: 222 self.skip_unbuildable_variant() 223 else: 224 BuildContext.execute(self) 225 226 def restore(self): 227 self.top_dir = os.path.abspath(Context.g_module.top) 228 self.srcnode = self.root.find_node(self.top_dir) 229 self.path = self.srcnode 230 231 self.out_dir = os.path.join(self.top_dir, Context.g_module.out) 232 self.bldnode = self.root.make_node(self.out_dir) 233 self.bldnode.mkdir() 234 235 self.env = ConfigSet.ConfigSet() 236 237 def extract_groups_of_builds(self): 238 """Return a dict mapping each variants to the commands to build""" 239 self.vgroups = {} 240 for x in reversed(Options.commands): 241 _, _, variant = x.partition('_') 242 if variant in Context.g_module.variants: 243 try: 244 dct = self.vgroups[variant] 245 except KeyError: 246 dct = self.vgroups[variant] = OrderedDict() 247 try: 248 dct[variant].append(x) 249 except KeyError: 250 dct[variant] = [x] 251 Options.commands.remove(x) 252 253 def custom_options(self, login): 254 try: 255 return Context.g_module.host_options[login] 256 except (AttributeError, KeyError): 257 return {} 258 259 def recurse(self, *k, **kw): 260 self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx') 261 self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh') 262 self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan') 263 try: 264 self.env.WAF = getattr(Context.g_module, 'waf') 265 except AttributeError: 266 try: 267 os.stat('waf') 268 except KeyError: 269 self.fatal('Put a waf file in the directory (./waf-light --tools=remote)') 270 else: 271 self.env.WAF = './waf' 272 273 self.extract_groups_of_builds() 274 self.setup_private_ssh_key() 275 for k, v in self.vgroups.items(): 276 task = self(rule=rsync_and_ssh, always=True) 277 task.env.login = self.variant_to_login(k) 278 279 task.env.commands = [] 280 for opt, value in v.items(): 281 task.env.commands += value 282 task.env.variant = task.env.commands[0].partition('_')[2] 283 for opt, value in self.custom_options(k): 284 task.env[opt] = value 285 self.jobs = len(self.vgroups) 286 287 def make_mkdir_command(self, task): 288 return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env) 289 290 def make_send_command(self, task): 291 return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env) 292 293 def make_exec_command(self, task): 294 txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"''' 295 return Utils.subst_vars(txt, task.env) 296 297 def make_save_command(self, task): 298 return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env) 299 300def rsync_and_ssh(task): 301 302 # remove a warning 303 task.uid_ = id(task) 304 305 bld = task.generator.bld 306 307 task.env.user, _, _ = task.env.login.partition('@') 308 task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant))) 309 task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir) 310 task.env.local_dir = bld.srcnode.abspath() + '/' 311 312 task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant) 313 task.env.build_dir = bld.bldnode.abspath() 314 315 ret = task.exec_command(bld.make_mkdir_command(task)) 316 if ret: 317 return ret 318 ret = task.exec_command(bld.make_send_command(task)) 319 if ret: 320 return ret 321 ret = task.exec_command(bld.make_exec_command(task)) 322 if ret: 323 return ret 324 ret = task.exec_command(bld.make_save_command(task)) 325 if ret: 326 return ret 327 328