1# benchmark -- automated system for testing distcc correctness 2# and performance on various source trees. 3 4# Copyright (C) 2002, 2003 by Martin Pool 5# Copyright 2008 Google Inc. 6# 7# This program is free software; you can redistribute it and/or 8# modify it under the terms of the GNU General Public License 9# as published by the Free Software Foundation; either version 2 10# of the License, or (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program; if not, write to the Free Software 19# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, 20# USA. 21 22import commands 23import os 24import shutil 25import stat 26import sys 27import tempfile 28 29import buildutil 30 31STANDARD_CC_NAMES = ['cc', 'gcc'] 32STANDARD_CXX_NAMES = ['cxx', 'c++', 'g++' ] 33 34def _find_executable(name): 35 (rs, output) = commands.getstatusoutput('which "%s"' % name) 36 if rs: 37 sys.exit("Could not determine location of '%s'" % name) 38 return output.strip() 39 40class CompilerSpec: 41 """Describes a compiler/make setup. 42 43 Used to define different situations such as local compilation, and 44 various degrees of parallelism.""" 45 46 def __init__(self, where, cc, cxx, prefix='', make_opts='', 47 pump_cmd='', num_hosts=1, host_opts='', 48 name=None): 49 """Constructor: 50 51 Args: 52 where: 'local', 'dist', 'lzo', or 'pump' 53 cc: location of the C compiler 54 cxx: location of the C++ 55 prefix: a string, either 'distcc ' or '' 56 make_opts: options to make, such as '-j120' 57 host_opts: for appending to hosts in DISTCC_HOSTS 58 such as ',lzo,cpp' 59 name: a string 60 """ 61 self.where = where 62 self.real_cc = _find_executable(cc) 63 self.real_cxx = _find_executable(cxx) 64 self.cc = prefix + self.real_cc 65 self.cxx = prefix + self.real_cxx 66 self.make_opts = make_opts 67 self.host_opts = host_opts 68 self.pump_cmd = pump_cmd 69 self.num_hosts = num_hosts 70 self.host_opts = host_opts 71 self.name = name or (self.pump_cmd + self.real_cc + "__" + 72 self.make_opts).replace(' ', '_') 73 74 def prepare_shell_script_farm(self, farm_dir, masquerade): 75 """Prepare farm directory for masquerading. 76 77 Assume the compiler is not local. Each standard name, such as 78 'cc', is used for form a shell script, named 'cc', that 79 contains the line 'distcc /my/path/gcc "$@"', where 80 '/my/path/gcc' is the value of the compiler.gcc field. 81 82 If the compiler is local, then the same procedure is followed 83 except that 'distcc' is omitted from the command line. 84 """ 85 assert os.path.isdir(farm_dir) 86 assert os.path.isabs(farm_dir) 87 88 def make_shell_script(name, compiler_path, where): 89 fd = open(os.path.join(farm_dir, name), 'w') 90 fd.write('#!/bin/sh\n%s%s "$@"' 91 % (where != 'local' and 'distcc ' or '', 92 compiler_path)) 93 fd.close() 94 os.chmod(os.path.join(farm_dir, name), 95 stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) 96 97 for generic_name in STANDARD_CC_NAMES: 98 make_shell_script(generic_name, self.real_cc, self.where) 99 100 for generic_name in STANDARD_CXX_NAMES: 101 make_shell_script(generic_name, self.real_cxx, self.where) 102 103 # Make shell wrapper to help manual debugging. 104 fd = open(masquerade, 'w') 105 fd.write("""\ 106#!/bin/sh 107# Execute $@, but force 'cc' and 'cxx'" to be those in the farm of 108# masquerading scripts. Each script in turn executes 'distcc' with the actual 109# compiler specified with the benchmark.py command. 110PATH=%s:"$PATH" "$@" 111""" % farm_dir) 112 fd.close() 113 os.chmod(masquerade, 114 stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) 115 116 117def default_compilers(cc, cxx): 118 return [parse_compiler_opt('local,h1,j1', cc, cxx), 119 parse_compiler_opt('dist,h10,j20', cc, cxx), 120 parse_compiler_opt('dist,h10,j40', cc, cxx), 121 parse_compiler_opt('pump,h10,j20', cc, cxx), 122 parse_compiler_opt('pump,h10,j40', cc, cxx), 123 ] 124 125def parse_compiler_opt(optarg, cc, cxx): 126 """Parse command-line specification of a compiler (-c/--compiler). 127 128 XXX: I don't really know what the best syntax for this is. For 129 the moment, it is "local", "dist", "lzo", or "pump", followed by ",h" 130 and the number of hosts to use, followed by ",j" and the number 131 of jobs to use (for the -j option to make). 132 """ 133 where, hosts, jobs = optarg.split(',') 134 if hosts.startswith("h"): 135 hosts = int(hosts[1:]) 136 if not os.getenv("DISTCC_HOSTS"): 137 raise ValueError, "You must set DISTCC_HOSTS before running benchmarks" 138 max_hosts = buildutil.count_hosts(os.getenv("DISTCC_HOSTS")) 139 if hosts > max_hosts: 140 print ("Warning: can't use %d hosts: DISTCC_HOSTS only has %d" % 141 (hosts, max_hosts)) 142 hosts = max_hosts 143 else: 144 raise ValueError, ("invalid compiler option: " 145 "expecting '...,h<NUMBER OF HOSTS>,...', found %s" 146 % `hosts`) 147 if jobs.startswith("j"): 148 jobs = int(jobs[1:]) 149 else: 150 raise ValueError, ("invalid compiler option: " 151 "expecting '...,j<NUMBER OF JOBS>', found %s" 152 % `jobs`) 153 if where == 'local': 154 return CompilerSpec(where=where, 155 name='local_%02d' % jobs, 156 cc=cc, 157 cxx=cxx, 158 num_hosts=1, 159 make_opts='-j%d' % jobs) 160 elif where == 'dist': 161 return CompilerSpec(where=where, 162 name='dist_h%02d_j%02d' % (hosts, jobs), 163 cc=cc, 164 cxx=cxx, 165 prefix='distcc ', 166 num_hosts=hosts, 167 make_opts='-j%d' % jobs) 168 elif where == 'lzo': 169 return CompilerSpec(where=where, 170 name='lzo_h%02d_j%02d' % (hosts, jobs), 171 cc=cc, 172 cxx=cxx, 173 prefix='distcc ', 174 num_hosts=hosts, 175 host_opts=",lzo", 176 make_opts='-j%d' % jobs) 177 elif where == 'pump': 178 return CompilerSpec(where=where, 179 name='pump_h%02d_j%02d' % (hosts, jobs), 180 cc=cc, 181 cxx=cxx, 182 prefix='distcc ', 183 pump_cmd='pump ', 184 num_hosts=hosts, 185 host_opts=",cpp,lzo", 186 make_opts='-j%d' % jobs) 187 else: 188 raise ValueError, ("invalid compiler option: don't understand %s" 189 % `where`) 190 191 192def prepare_shell_script_farm(compiler, farm_dir, masquerade): 193 compiler.prepare_shell_script_farm(farm_dir, masquerade) 194