1# ---------------------------------------------------------------------------------------------------- 2# 3# Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. 4# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5# 6# This code is free software; you can redistribute it and/or modify it 7# under the terms of the GNU General Public License version 2 only, as 8# published by the Free Software Foundation. 9# 10# This code is distributed in the hope that it will be useful, but WITHOUT 11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13# version 2 for more details (a copy is included in the LICENSE file that 14# accompanied this code). 15# 16# You should have received a copy of the GNU General Public License version 17# 2 along with this work; if not, write to the Free Software Foundation, 18# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19# 20# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21# or visit www.oracle.com if you need additional information or have any 22# questions. 23# 24# ---------------------------------------------------------------------------------------------------- 25 26from outputparser import OutputParser, ValuesMatcher 27import re, mx, mx_graal, os, sys, StringIO, subprocess 28from os.path import isfile, join, exists 29 30gc = 'UseSerialGC' 31 32dacapoSanityWarmup = { 33 'avrora': [0, 0, 3, 6, 13], 34 'batik': [0, 0, 5, 5, 20], 35 'eclipse': [0, 0, 0, 0, 0], 36 'fop': [4, 8, 10, 20, 30], 37 'h2': [0, 0, 5, 5, 8], 38 'jython': [0, 0, 5, 10, 13], 39 'luindex': [0, 0, 5, 10, 10], 40 'lusearch': [0, 4, 5, 5, 8], 41 'pmd': [0, 0, 5, 10, 13], 42 'sunflow': [0, 2, 5, 10, 15], 43 'tomcat': [0, 0, 5, 10, 15], 44 'tradebeans': [0, 0, 5, 10, 13], 45 'tradesoap': [0, 0, 5, 10, 15], 46 'xalan': [0, 0, 5, 10, 18], 47} 48 49dacapoScalaSanityWarmup = { 50 'actors': [0, 0, 2, 5, 5], 51 'apparat': [0, 0, 2, 5, 5], 52 'factorie': [0, 0, 2, 5, 5], 53 'kiama': [0, 4, 3, 13, 15], 54 'scalac': [0, 0, 5, 15, 20], 55 'scaladoc': [0, 0, 5, 15, 15], 56 'scalap': [0, 0, 5, 15, 20], 57 'scalariform':[0, 0, 6, 15, 20], 58 'scalatest': [0, 0, 2, 10, 12], 59 'scalaxb': [0, 0, 5, 15, 25], 60# (gdub) specs sometimes returns a non-zero value event though there is no apparent failure 61 'specs': [0, 0, 0, 0, 0], 62 'tmt': [0, 0, 3, 10, 12] 63} 64 65dacapoGateBuildLevels = { 66 'avrora': ['product', 'fastdebug', 'debug'], 67 'batik': ['product', 'fastdebug', 'debug'], 68 # (lewurm): does not work with JDK8 69 'eclipse': [], 70 'fop': ['fastdebug', 'debug'], 71 'h2': ['product', 'fastdebug', 'debug'], 72 'jython': ['product', 'fastdebug', 'debug'], 73 'luindex': ['product', 'fastdebug', 'debug'], 74 'lusearch': ['product'], 75 'pmd': ['product', 'fastdebug', 'debug'], 76 'sunflow': ['fastdebug', 'debug'], 77 'tomcat': ['product', 'fastdebug', 'debug'], 78 'tradebeans': ['product', 'fastdebug', 'debug'], 79 # tradesoap is too unreliable for the gate, often crashing with concurrency problems: 80 # http://sourceforge.net/p/dacapobench/bugs/99/ 81 'tradesoap': [], 82 'xalan': ['product', 'fastdebug', 'debug'], 83} 84 85dacapoScalaGateBuildLevels = { 86 'actors': ['product', 'fastdebug', 'debug'], 87 'apparat': ['product', 'fastdebug', 'debug'], 88 'factorie': ['product', 'fastdebug', 'debug'], 89 'kiama': ['fastdebug', 'debug'], 90 'scalac': ['product', 'fastdebug', 'debug'], 91 'scaladoc': ['product', 'fastdebug', 'debug'], 92 'scalap': ['product', 'fastdebug', 'debug'], 93 'scalariform':['product', 'fastdebug', 'debug'], 94 'scalatest': ['product', 'fastdebug', 'debug'], 95 'scalaxb': ['product', 'fastdebug', 'debug'], 96 'specs': ['product', 'fastdebug', 'debug'], 97 'tmt': ['product', 'fastdebug', 'debug'], 98} 99 100specjvm2008Names = [ 101 'startup.helloworld', 102 'startup.compiler.compiler', 103 'startup.compiler.sunflow', 104 'startup.compress', 105 'startup.crypto.aes', 106 'startup.crypto.rsa', 107 'startup.crypto.signverify', 108 'startup.mpegaudio', 109 'startup.scimark.fft', 110 'startup.scimark.lu', 111 'startup.scimark.monte_carlo', 112 'startup.scimark.sor', 113 'startup.scimark.sparse', 114 'startup.serial', 115 'startup.sunflow', 116 'startup.xml.transform', 117 'startup.xml.validation', 118 'compiler.compiler', 119 'compiler.sunflow', 120 'compress', 121 'crypto.aes', 122 'crypto.rsa', 123 'crypto.signverify', 124 'derby', 125 'mpegaudio', 126 'scimark.fft.large', 127 'scimark.lu.large', 128 'scimark.sor.large', 129 'scimark.sparse.large', 130 'scimark.fft.small', 131 'scimark.lu.small', 132 'scimark.sor.small', 133 'scimark.sparse.small', 134 'scimark.monte_carlo', 135 'serial', 136 'sunflow', 137 'xml.transform', 138 'xml.validation' 139] 140 141def _noneAsEmptyList(a): 142 if a is None: 143 return [] 144 return a 145 146class SanityCheckLevel: 147 Fast, Gate, Normal, Extensive, Benchmark = range(5) 148 149def getSPECjbb2005(benchArgs=None): 150 benchArgs = [] if benchArgs is None else benchArgs 151 152 specjbb2005 = mx.get_env('SPECJBB2005') 153 if specjbb2005 is None or not exists(join(specjbb2005, 'jbb.jar')): 154 mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory') 155 156 score = re.compile(r"^Valid run, Score is (?P<score>[0-9]+)$", re.MULTILINE) 157 error = re.compile(r"VALIDATION ERROR") 158 success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE) 159 matcher = ValuesMatcher(score, {'group' : 'SPECjbb2005', 'name' : 'score', 'score' : '<score>'}) 160 classpath = ['jbb.jar', 'check.jar'] 161 return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+' + gc, '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) 162 163def getSPECjbb2013(benchArgs=None): 164 165 specjbb2013 = mx.get_env('SPECJBB2013') 166 if specjbb2013 is None or not exists(join(specjbb2013, 'specjbb2013.jar')): 167 mx.abort('Please set the SPECJBB2013 environment variable to a SPECjbb2013 directory') 168 169 jops = re.compile(r"^RUN RESULT: hbIR \(max attempted\) = [0-9]+, hbIR \(settled\) = [0-9]+, max-jOPS = (?P<max>[0-9]+), critical-jOPS = (?P<critical>[0-9]+)$", re.MULTILINE) 170 # error? 171 success = re.compile(r"org.spec.jbb.controller: Run finished", re.MULTILINE) 172 matcherMax = ValuesMatcher(jops, {'group' : 'SPECjbb2013', 'name' : 'max', 'score' : '<max>'}) 173 matcherCritical = ValuesMatcher(jops, {'group' : 'SPECjbb2013', 'name' : 'critical', 'score' : '<critical>'}) 174 return Test("SPECjbb2013", ['-jar', 'specjbb2013.jar', '-m', 'composite'] + 175 _noneAsEmptyList(benchArgs), [success], [], [matcherCritical, matcherMax], 176 vmOpts=['-Xmx6g', '-Xms6g', '-Xmn3g', '-XX:+UseParallelOldGC', '-XX:-UseAdaptiveSizePolicy', '-XX:-UseBiasedLocking', '-XX:-UseCompressedOops'], defaultCwd=specjbb2013) 177 178def getSPECjbb2015(benchArgs=None): 179 180 specjbb2015 = mx.get_env('SPECJBB2015') 181 if specjbb2015 is None or not exists(join(specjbb2015, 'specjbb2015.jar')): 182 mx.abort('Please set the SPECJBB2015 environment variable to a SPECjbb2015 directory') 183 184 jops = re.compile(r"^RUN RESULT: hbIR \(max attempted\) = [0-9]+, hbIR \(settled\) = [0-9]+, max-jOPS = (?P<max>[0-9]+), critical-jOPS = (?P<critical>[0-9]+)$", re.MULTILINE) 185 # error? 186 success = re.compile(r"org.spec.jbb.controller: Run finished", re.MULTILINE) 187 matcherMax = ValuesMatcher(jops, {'group' : 'SPECjbb2015', 'name' : 'max', 'score' : '<max>'}) 188 matcherCritical = ValuesMatcher(jops, {'group' : 'SPECjbb2015', 'name' : 'critical', 'score' : '<critical>'}) 189 return Test("SPECjbb2015", ['-jar', 'specjbb2015.jar', '-m', 'composite'] + 190 _noneAsEmptyList(benchArgs), [success], [], [matcherCritical, matcherMax], 191 vmOpts=['-Xmx6g', '-Xms6g', '-Xmn3g', '-XX:+UseParallelOldGC', '-XX:-UseAdaptiveSizePolicy', '-XX:-UseBiasedLocking', '-XX:-UseCompressedOops'], defaultCwd=specjbb2015) 192 193def getSPECjvm2008(benchArgs=None): 194 195 specjvm2008 = mx.get_env('SPECJVM2008') 196 if specjvm2008 is None or not exists(join(specjvm2008, 'SPECjvm2008.jar')): 197 mx.abort('Please set the SPECJVM2008 environment variable to a SPECjvm2008 directory') 198 199 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE) 200 error = re.compile(r"^Errors in benchmark: ", re.MULTILINE) 201 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart 202 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE) 203 matcher = ValuesMatcher(score, {'group' : 'SPECjvm2008', 'name' : '<benchmark>', 'score' : '<score>'}) 204 205 return Test("SPECjvm2008", ['-jar', 'SPECjvm2008.jar'] + _noneAsEmptyList(benchArgs), [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+' + gc, '-XX:-UseCompressedOops'], defaultCwd=specjvm2008) 206 207def getDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=None, extraVmArguments=None): 208 checks = [] 209 210 for (bench, ns) in dacapoSanityWarmup.items(): 211 if ns[level] > 0: 212 if gateBuildLevel is None or gateBuildLevel in dacapoGateBuildLevels[bench]: 213 checks.append(getDacapo(bench, ['-n', str(ns[level])] + _noneAsEmptyList(dacapoArgs), extraVmArguments=extraVmArguments)) 214 215 return checks 216 217def getDacapo(name, dacapoArgs=None, extraVmArguments=None): 218 dacapo = mx.get_env('DACAPO_CP') 219 if dacapo is None: 220 l = mx.library('DACAPO', False) 221 if l is not None: 222 dacapo = l.get_path(True) 223 else: 224 mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library') 225 226 if not isfile(dacapo) or not dacapo.endswith('.jar'): 227 mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) 228 229 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====", re.MULTILINE) 230 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====", re.MULTILINE) 231 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 232 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") 233 234 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'}) 235 dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'}) 236 237 # Use ipv4 stack for dacapos; tomcat+solaris+ipv6_interface fails (see also: JDK-8072384) 238 return Test("DaCapo-" + name, ['-jar', mx._cygpathU2W(dacapo), name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail], 239 [dacapoMatcher, dacapoMatcher1], 240 ['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops', "-Djava.net.preferIPv4Stack=true", '-G:+ExitVMOnException'] + 241 _noneAsEmptyList(extraVmArguments)) 242 243def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=None, extraVmArguments=None): 244 checks = [] 245 246 for (bench, ns) in dacapoScalaSanityWarmup.items(): 247 if ns[level] > 0: 248 if gateBuildLevel is None or gateBuildLevel in dacapoScalaGateBuildLevels[bench]: 249 checks.append(getScalaDacapo(bench, ['-n', str(ns[level])] + _noneAsEmptyList(dacapoArgs), extraVmArguments=extraVmArguments)) 250 251 return checks 252 253def getScalaDacapo(name, dacapoArgs=None, extraVmArguments=None): 254 dacapo = mx.get_env('DACAPO_SCALA_CP') 255 if dacapo is None: 256 l = mx.library('DACAPO_SCALA', False) 257 if l is not None: 258 dacapo = l.get_path(True) 259 else: 260 mx.abort('Scala DaCapo 0.1.0 jar file must be specified with DACAPO_SCALA_CP environment variable or as DACAPO_SCALA library') 261 262 if not isfile(dacapo) or not dacapo.endswith('.jar'): 263 mx.abort('Specified Scala DaCapo jar file does not exist or is not a jar file: ' + dacapo) 264 265 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====", re.MULTILINE) 266 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====", re.MULTILINE) 267 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 268 269 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : "Scala-DaCapo", 'name' : '<benchmark>', 'score' : '<time>'}) 270 271 return Test("Scala-DaCapo-" + name, ['-jar', mx._cygpathU2W(dacapo), name] + _noneAsEmptyList(dacapoArgs), [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+' + gc, '-XX:-UseCompressedOops'] + _noneAsEmptyList(extraVmArguments)) 272 273def getBootstraps(): 274 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms( \(compiled (?P<methods>[0-9]+) methods\))?") 275 scoreMatcher = ValuesMatcher(time, {'group' : 'Bootstrap', 'name' : 'BootstrapTime', 'score' : '<time>'}) 276 methodMatcher = ValuesMatcher(time, {'group' : 'Bootstrap', 'name' : 'BootstrapMethods', 'score' : '<methods>'}) 277 scoreMatcherBig = ValuesMatcher(time, {'group' : 'Bootstrap-bigHeap', 'name' : 'BootstrapTime', 'score' : '<time>'}) 278 methodMatcherBig = ValuesMatcher(time, {'group' : 'Bootstrap-bigHeap', 'name' : 'BootstrapMethods', 'score' : '<methods>'}) 279 280 tests = [] 281 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher, methodMatcher], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) 282 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig, methodMatcherBig], vmOpts=['-Xms2g'], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) 283 return tests 284 285class CTWMode: 286 Full, NoInline = range(2) 287 288def getCTW(vm, mode): 289 time = re.compile(r"CompileTheWorld : Done \([0-9]+ classes, [0-9]+ methods, (?P<time>[0-9]+) ms\)") 290 scoreMatcher = ValuesMatcher(time, {'group' : 'CompileTheWorld', 'name' : 'CompileTime', 'score' : '<time>'}) 291 292 jre = os.environ.get('JAVA_HOME') 293 if exists(join(jre, 'jre')): 294 jre = join(jre, 'jre') 295 rtjar = join(jre, 'lib', 'rt.jar') 296 297 298 args = ['-XX:+CompileTheWorld', '-Xbootclasspath/p:' + rtjar] 299 if vm == 'jvmci': 300 args += ['-XX:+BootstrapGraal'] 301 if mode >= CTWMode.NoInline: 302 if not mx_graal.isJVMCIEnabled(vm): 303 args.append('-XX:-Inline') 304 else: 305 args.append('-G:CompileTheWordConfig=-Inline') 306 307 return Test("CompileTheWorld", args, successREs=[time], scoreMatchers=[scoreMatcher], benchmarkCompilationRate=False) 308 309 310class Tee: 311 def __init__(self): 312 self.output = StringIO.StringIO() 313 def eat(self, line): 314 self.output.write(line) 315 sys.stdout.write(line) 316 317""" 318Encapsulates a single program that is a sanity test and/or a benchmark. 319""" 320class Test: 321 def __init__(self, name, cmd, successREs=None, failureREs=None, scoreMatchers=None, vmOpts=None, defaultCwd=None, ignoredVMs=None, benchmarkCompilationRate=False): 322 323 self.name = name 324 self.successREs = _noneAsEmptyList(successREs) 325 self.failureREs = _noneAsEmptyList(failureREs) + [re.compile(r"Exception occurred in scope: ")] 326 self.scoreMatchers = _noneAsEmptyList(scoreMatchers) 327 self.vmOpts = _noneAsEmptyList(vmOpts) 328 self.cmd = cmd 329 self.defaultCwd = defaultCwd 330 self.ignoredVMs = _noneAsEmptyList(ignoredVMs) 331 self.benchmarkCompilationRate = benchmarkCompilationRate 332 if benchmarkCompilationRate: 333 self.vmOpts = self.vmOpts + ['-XX:+CITime'] 334 335 def __str__(self): 336 return self.name 337 338 def test(self, vm, cwd=None, extraVmOpts=None, vmbuild=None): 339 """ 340 Run this program as a sanity test. 341 """ 342 if vm in self.ignoredVMs: 343 return True 344 if cwd is None: 345 cwd = self.defaultCwd 346 parser = OutputParser() 347 jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)") 348 parser.addMatcher(ValuesMatcher(jvmError, {'jvmError' : '<jvmerror>'})) 349 350 for successRE in self.successREs: 351 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'})) 352 for failureRE in self.failureREs: 353 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'})) 354 355 tee = Tee() 356 retcode = mx_graal.run_vm(self.vmOpts + _noneAsEmptyList(extraVmOpts) + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, debugLevel=vmbuild) 357 output = tee.output.getvalue() 358 valueMaps = parser.parse(output) 359 360 if len(valueMaps) == 0: 361 return False 362 363 record = {} 364 for valueMap in valueMaps: 365 for key, value in valueMap.items(): 366 if record.has_key(key) and record[key] != value: 367 mx.abort('Inconsistant values returned by test machers : ' + str(valueMaps)) 368 record[key] = value 369 370 jvmErrorFile = record.get('jvmError') 371 if jvmErrorFile: 372 mx.log('/!\\JVM Error : dumping error log...') 373 with open(jvmErrorFile, 'rb') as fp: 374 mx.log(fp.read()) 375 os.unlink(jvmErrorFile) 376 return False 377 378 if record.get('failed') == '1': 379 return False 380 381 return retcode == 0 and record.get('passed') == '1' 382 383 def bench(self, vm, cwd=None, extraVmOpts=None, vmbuild=None): 384 """ 385 Run this program as a benchmark. 386 """ 387 if vm in self.ignoredVMs: 388 return {} 389 if cwd is None: 390 cwd = self.defaultCwd 391 parser = OutputParser() 392 393 for successRE in self.successREs: 394 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'})) 395 for failureRE in self.failureREs: 396 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'})) 397 for scoreMatcher in self.scoreMatchers: 398 parser.addMatcher(scoreMatcher) 399 400 if self.benchmarkCompilationRate: 401 if vm == 'jvmci': 402 bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)") 403 ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)") 404 parser.addMatcher(ValuesMatcher(bps, {'group' : 'ParsedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) 405 parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) 406 else: 407 ibps = re.compile(r"(?P<compiler>[\w]+) compilation speed: +(?P<rate>[0-9]+) bytes/s {standard") 408 parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : '<compiler>:' + self.name, 'score' : '<rate>'})) 409 410 startDelim = 'START: ' + self.name 411 endDelim = 'END: ' + self.name 412 413 outputfile = os.environ.get('BENCH_OUTPUT', None) 414 if outputfile: 415 # Used only to debug output parsing 416 with open(outputfile) as fp: 417 output = fp.read() 418 start = output.find(startDelim) 419 end = output.find(endDelim, start) 420 if start == -1 and end == -1: 421 return {} 422 output = output[start + len(startDelim + os.linesep): end] 423 mx.log(startDelim) 424 mx.log(output) 425 mx.log(endDelim) 426 else: 427 tee = Tee() 428 mx.log(startDelim) 429 if mx_graal.run_vm(self.vmOpts + _noneAsEmptyList(extraVmOpts) + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, debugLevel=vmbuild) != 0: 430 mx.abort("Benchmark failed (non-zero retcode)") 431 mx.log(endDelim) 432 output = tee.output.getvalue() 433 434 groups = {} 435 passed = False 436 for valueMap in parser.parse(output): 437 assert (valueMap.has_key('name') and valueMap.has_key('score') and valueMap.has_key('group')) or valueMap.has_key('passed') or valueMap.has_key('failed'), valueMap 438 if valueMap.get('failed') == '1': 439 mx.abort("Benchmark failed") 440 if valueMap.get('passed') == '1': 441 passed = True 442 groupName = valueMap.get('group') 443 if groupName: 444 group = groups.setdefault(groupName, {}) 445 name = valueMap.get('name') 446 score = valueMap.get('score') 447 if name and score: 448 group[name] = score 449 450 if not passed: 451 mx.abort("Benchmark failed (not passed)") 452 453 return groups 454