1from .IOH_function import IOH_function, custom_IOH_function 2from .IOH_logger import IOH_logger 3from .IOH_Utils import runParallelFunction 4 5from itertools import product 6from functools import partial 7from multiprocessing import cpu_count 8 9import numpy as np 10 11def _run_default(alg, fid, dim, iid, precision, suite, repetitions, observing, 12 location, foldername, dat, cdat, idat, tdat_base, tdat_exp, 13 parameters, dynamic_attrs, static_attrs): 14 '''A helper function for parallellization of the IOHexperimter. 15 ''' 16 name = alg.__class__.__name__ 17 info = "Run using the IOHexperimenter in python, beta version" 18 f = IOH_function(fid, dim, iid, precision, suite) 19 if observing: 20 logger = IOH_logger(location, foldername, name, info) 21 logger.set_tracking_options(dat, cdat, idat, tdat_base, tdat_exp) 22 if parameters is not None: 23 logger.track_parameters(alg, parameters) 24 if dynamic_attrs is not None: 25 logger.set_dynamic_attributes(dynamic_attrs) 26 if static_attrs is not None: 27 logger.set_static_attributes(static_attrs) 28 f.add_logger(logger) 29 30 for rep in range(repetitions): 31 np.random.seed = rep 32 if rep > 0: 33 f.reset() 34 alg(f) 35 print(fid, f.evaluations, f.best_so_far_precision) 36 37 f.clear_logger() 38 39def _run_custom(alg, function, fname, dim, suite, repetitions, observing, 40 location, foldername, dat, cdat, idat, tdat_base, tdat_exp, 41 parameters, dynamic_attrs, static_attrs): 42 '''A helper function for parallellization of the IOHexperimter. 43 ''' 44 name = alg.__class__.__name__ 45 info = "Run using the IOHexperimenter in python, beta version" 46 f = custom_IOH_function(function, fname, dim) 47 48 if observing: 49 logger = IOH_logger(location, foldername, name, info) 50 logger.set_tracking_options(dat, cdat, idat, tdat_base, tdat_exp) 51 if parameters is not None: 52 logger.track_parameters(alg, parameters) 53 if dynamic_attrs is not None: 54 logger.set_dynamic_attributes(dynamic_attrs) 55 if static_attrs is not None: 56 logger.set_static_attributes(static_attrs) 57 f.add_logger(logger) 58 59 for rep in range(repetitions): 60 np.random.seed = rep 61 if rep > 0: 62 f.reset() 63 alg(f) 64 print(f.fid, f.evaluations, f.best_so_far_precision) 65 f.clear_logger() 66 67class IOHexperimenter(): 68 '''An interfact to easily run a set of algorithms on multiple functions from the IOHexperimenter. 69 ''' 70 def __init__(self): 71 '''Create an IOHexperimenter object for benchmarking a set of algorithms on multiple functions 72 73 Initialize the functions to use by calling 'initialize_PBO', 'initialize_BBOB' or 'initialize_custom' 74 Set up parallellization by calling 'set_parallell' 75 76 ''' 77 self.location = None 78 self.foldername = None 79 self.dat = True 80 self.cdat = False 81 self.idat = 0 82 self.tdat_base = [0] 83 self.tdat_exp = 0 84 self.observing = False 85 self.parameters = None 86 self.dynamic_attrs = None 87 self.static_attrs = None 88 self.precision = 0 89 self.parallel_settings = None 90 91 def initialize_PBO(self, fids, iids, dims, repetitions): 92 '''Initialize to a set of functions for the PBO suite 93 94 Parameters 95 ---------- 96 fids: 97 List of the function numbers within the selected suite. Alternatively, the names of the functions as used in IOHexperimenter. 98 dims: 99 List of the dimensions (number of variables) of the problem 100 iids: 101 List of the instance IDs of the problem 102 repetitions: 103 Number of repetitions on each function instance 104 ''' 105 self.fids = fids 106 self.iids = iids 107 self.dims = dims 108 self.repetitions = repetitions 109 self.suite = "PBO" 110 111 def initialize_BBOB(self, fids, iids, dims, repetitions, precision = 1e-8): 112 '''Initialize to a set of functions for the BBOB suite 113 114 Parameters 115 ---------- 116 fids: 117 List of the function numbers within the selected suite. Alternatively, the names of the functions as used in IOHexperimenter. 118 dims: 119 List of the dimensions (number of variables) of the problem 120 iids: 121 List of the instance IDs of the problem 122 repetitions: 123 Number of repetitions on each function instance 124 target_precision: 125 Optional, how close to the optimum the problem is considered 'solved' 126 ''' 127 self.fids = fids 128 self.iids = iids 129 self.dims = dims 130 self.repetitions = repetitions 131 self.precision = precision 132 self.suite = "BBOB" 133 134 def initialize_custom(self, functions, fnames, fdims, repetitions, suite = "No Suite"): 135 '''Initialize to a set of custom functions 136 137 Parameters 138 ---------- 139 functions: 140 List of functions to use 141 fnames: 142 List of names corresponding to the provided functions 143 fdims: 144 List of dimensionalities corresponding to the provided functions 145 repetitions: 146 Number of repetitions on each function instance 147 suite: 148 Optional, a name for the suite of the provided functions 149 ''' 150 self.functions = functions 151 self.fnames = fnames 152 self.fdims = fdims 153 self.repetitions = repetitions 154 self.suite = suite 155 156 def set_logger_location(self, location, foldername = "run"): 157 '''Set the location options for the logger of the suite 158 Parameters 159 ---------- 160 location: 161 The directory in which to store the results 162 foldername: 163 The name of the folder to create to store the results 164 165 ''' 166 self.location = location 167 self.foldername = foldername 168 self.observing = True 169 170 def set_parallel(self, parallel, version = "joblib", timeout = 30, num_threads = None): 171 '''Set the parallellization options for the experiments 172 173 Parameters 174 ---------- 175 parallel: 176 Boolean. Whether or not to use parallell execution 177 version: 178 Which parallellization library to use. Options are: 179 - 'multiprocessing' 180 - 'MPI' (using the schwimmbad library) 181 - 'joblib' (recommended) 182 - 'pebble' (can timeout functions which take too long) 183 num_threads: 184 How many threads to use. Defaults to all available ones from 'cpu_count' 185 timeout: 186 If using pebble, this sets the timeout in seconds after which to cancel the execution 187 ''' 188 self.parallel_settings = { 189 "evaluate_parallel" : parallel, "use_MPI" : version == "MPI", 190 "use_pebble" : version == "pebble", "timeout" : timeout, 191 "use_joblib" : version == "joblib", 192 "num_threads" : num_threads if num_threads is not None else cpu_count() 193 } 194 195 def set_logger_options(self, dat = True, cdat = False, idat = 0, tdat_base = [0], tdat_exp = 0): 196 '''Set which datafiles should be stored by the logger 197 198 Parameters 199 ---------- 200 dat: 201 Whether or not to store .dat-files (one line per improvement of objective) 202 cdat: 203 Whether or not to store .cdat-files (one line for each evaluation of objective) 204 idat: 205 Integer, if non-zero, an .idat file will be made which records ever i-th evaluation of the objective 206 tdat_base: 207 Base values for the tdat-file 208 tdat_exp: 209 Exponent-values for the tdat-file 210 ''' 211 self.dat = dat 212 self.cdat = cdat 213 self.idat = idat 214 self.tdat_base = tdat_base 215 self.tdat_exp = tdat_exp 216 217 def set_parameter_tracking(self, parameters): 218 '''Set which parameters should be tracked during the benchmarking procedure 219 220 Only usable when 'set_logger_location' has been called previously 221 222 Parameters 223 ---------- 224 parameters: 225 List of the names of the parameters to track. All 'parameters'-variables need to be accessible by 226 using algorithm.parameter (recommended to use @property) for each of the algorithms to be benchmarked 227 ''' 228 self.parameters = parameters 229 230 def set_dynamic_tracking(self, attributes): 231 '''Set which attributes should be tracked for each algorithm run (stored at the end of the run in the meta-information) 232 233 Only usable when 'set_logger_location' has been called previously 234 235 Parameters 236 ---------- 237 attributes: 238 List of the names of the parameters to track. All 'attributes'-variables need to be accessible by 239 using algorithm.parameter (recommended to use @property) for each of the algorithms to be benchmarked 240 ''' 241 self.dynamic_attrs = attributes 242 243 def set_static_tracking(self, attributes): 244 '''Set which static attributes should be stored in the meta-information of the benchmark data 245 246 Only usable when 'set_logger_location' has been called previously 247 248 Parameters 249 ---------- 250 attributes: 251 List of the names of the parameters to track. All 'attributes'-variables need to be accessible by 252 using algorithm.parameter (recommended to use @property) for each of the algorithms to be benchmarked 253 ''' 254 self.static_attrs = attributes 255 256 def __call__(self, algorithms): 257 '''Run the experiment with the given set of algorithms 258 259 Parameters 260 ---------- 261 algorithms: 262 A list containing algorithm functions. 263 These will get as input only one argument, of type IOH_function. 264 ''' 265 if self.suite in ["PBO", "BBOB"]: 266 arguments = product(algorithms, self.fids, self.dims, self.iids) 267 partial_run = partial(_run_default, precision = self.precision, suite = self.suite, 268 repetitions = self.repetitions, observing = self.observing, 269 location = self.location, foldername = self.foldername, 270 dat = self.dat, cdat = self.cdat, 271 idat = self.idat, tdat_base = self.tdat_base, 272 tdat_exp = self.tdat_exp, parameters = self.parameters, 273 dynamic_attrs = self.dynamic_attrs, static_attrs = self.static_attrs) 274 else: 275 arguments = product(algorithms, self.functions, self.fnames, self.fdims) 276 partial_run = partial(_run_custom, suite = self.suite, 277 repetitions = self.repetitions, observing = self.observing, 278 location = self.location, foldername = self.foldername, 279 dat = self.dat, cdat = self.cdat, 280 idat = self.idat, tdat_base = self.tdat_base, 281 tdat_exp = self.tdat_exp, parameters = self.parameters, 282 dynamic_attrs = self.dynamic_attrs, static_attrs = self.static_attrs) 283 284 results = runParallelFunction(partial_run, arguments, self.parallel_settings) 285 return results