1#!/usr/bin/env python3 2# 3# steadystate_tests.py 4# 5# Test option parsing and functonality for fio's steady state detection feature. 6# 7# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio 8# 9# REQUIREMENTS 10# Python 2.6+ 11# SciPy 12# 13# KNOWN ISSUES 14# only option parsing and read tests are carried out 15# On Windows this script works under Cygwin but not from cmd.exe 16# On Windows I encounter frequent fio problems generating JSON output (nothing to decode) 17# min runtime: 18# if ss attained: min runtime = ss_dur + ss_ramp 19# if not attained: runtime = timeout 20 21from __future__ import absolute_import 22from __future__ import print_function 23import os 24import sys 25import json 26import pprint 27import argparse 28import subprocess 29from scipy import stats 30 31def parse_args(): 32 parser = argparse.ArgumentParser() 33 parser.add_argument('fio', help='path to fio executable') 34 args = parser.parse_args() 35 36 return args 37 38 39def check(data, iops, slope, pct, limit, dur, criterion): 40 measurement = 'iops' if iops else 'bw' 41 data = data[measurement] 42 mean = sum(data) / len(data) 43 if slope: 44 x = list(range(len(data))) 45 m, intercept, r_value, p_value, std_err = stats.linregress(x,data) 46 m = abs(m) 47 if pct: 48 target = (m / mean * 100) if mean != 0 else 0 49 criterion = criterion[:-1] 50 else: 51 target = m 52 else: 53 maxdev = 0 54 for x in data: 55 maxdev = max(abs(mean-x), maxdev) 56 if pct: 57 target = maxdev / mean * 100 58 criterion = criterion[:-1] 59 else: 60 target = maxdev 61 62 criterion = float(criterion) 63 if criterion == 0.0: 64 objsame = False 65 else: 66 objsame = abs(target - criterion) / criterion < 0.005 67 return (objsame, target < limit, mean, target) 68 69 70if __name__ == '__main__': 71 args = parse_args() 72 73 pp = pprint.PrettyPrinter(indent=4) 74 75 passed = 0 76 failed = 0 77 78# 79# test option parsing 80# 81 parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], 82 'output': "set steady state IOPS threshold to 10.000000" }, 83 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], 84 'output': "set steady state threshold to 10.000000%" }, 85 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], 86 'output': "set steady state threshold to 0.100000%" }, 87 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], 88 'output': "set steady state threshold to 10.000000%" }, 89 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], 90 'output': "set steady state threshold to 0.100000%" }, 91 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], 92 'output': "set steady state BW threshold to 12" }, 93 ] 94 for test in parsing: 95 output = subprocess.check_output([args.fio] + test['args']) 96 if test['output'] in output.decode(): 97 print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])) 98 passed = passed + 1 99 else: 100 print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])) 101 failed = failed + 1 102 103# 104# test some read workloads 105# 106# if ss active and attained, 107# check that runtime is less than job time 108# check criteria 109# how to check ramp time? 110# 111# if ss inactive 112# check that runtime is what was specified 113# 114 reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, 115 {'s': False, 'timeout': 20, 'numjobs': 2}, 116 {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, 117 {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, 118 ] 119 120 jobnum = 0 121 for job in reads: 122 123 tf = "steadystate_job{0}.json".format(jobnum) 124 parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ] 125 parameters.extend([ "--thread", 126 "--output-format=json", 127 "--output={0}".format(tf), 128 "--ioengine=null", 129 "--size=1G", 130 "--rw=randrw", 131 "--rwmixread=100", 132 "--stonewall", 133 "--group_reporting", 134 "--numjobs={0}".format(job['numjobs']), 135 "--time_based", 136 "--runtime={0}".format(job['timeout']) ]) 137 if job['s']: 138 if job['iops']: 139 ss = 'iops' 140 else: 141 ss = 'bw' 142 if job['slope']: 143 ss += "_slope" 144 ss += ":" + str(job['ss_limit']) 145 if job['pct']: 146 ss += '%' 147 parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), 148 '--ss={0}'.format(ss), 149 '--ss_ramp={0}'.format(job['ss_ramp']) ]) 150 151 output = subprocess.call([args.fio] + parameters) 152 with open(tf, 'r') as source: 153 jsondata = json.loads(source.read()) 154 source.close() 155 156 for jsonjob in jsondata['jobs']: 157 line = "{0}".format(jsonjob['job options']['name']) 158 if job['s']: 159 if jsonjob['steadystate']['attained'] == 1: 160 # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit 161 mintime = (job['ss_dur'] + job['ss_ramp']) * 1000 162 actual = jsonjob['read']['runtime'] 163 if mintime > actual: 164 line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp']) 165 failed = failed + 1 166 else: 167 line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) 168 objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], 169 iops=job['iops'], 170 slope=job['slope'], 171 pct=job['pct'], 172 limit=job['ss_limit'], 173 dur=job['ss_dur'], 174 criterion=jsonjob['steadystate']['criterion']) 175 if not objsame: 176 line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) 177 failed = failed + 1 178 else: 179 if met: 180 line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit']) 181 passed = passed + 1 182 else: 183 line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit']) 184 failed = failed + 1 185 else: 186 # check runtime, confirm criterion calculation, and confirm that criterion was not met 187 expected = job['timeout'] * 1000 188 actual = jsonjob['read']['runtime'] 189 if abs(expected - actual) > 50: 190 line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) 191 else: 192 line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) 193 objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], 194 iops=job['iops'], 195 slope=job['slope'], 196 pct=job['pct'], 197 limit=job['ss_limit'], 198 dur=job['ss_dur'], 199 criterion=jsonjob['steadystate']['criterion']) 200 if not objsame: 201 if actual > (job['ss_dur'] + job['ss_ramp'])*1000: 202 line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) 203 failed = failed + 1 204 else: 205 line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion']) 206 passed = passed + 1 207 else: 208 if met: 209 line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit']) 210 failed = failed + 1 211 else: 212 line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit']) 213 passed = passed + 1 214 else: 215 expected = job['timeout'] * 1000 216 actual = jsonjob['read']['runtime'] 217 if abs(expected - actual) > 50: 218 result = 'FAILED ' 219 failed = failed + 1 220 else: 221 result = 'PASSED ' 222 passed = passed + 1 223 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) 224 print(line) 225 if 'steadystate' in jsonjob: 226 pp.pprint(jsonjob['steadystate']) 227 jobnum += 1 228 229 print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed)) 230 sys.exit(failed) 231