1#!/usr/bin/env python3
2#
3# steadystate_tests.py
4#
5# Test option parsing and functonality for fio's steady state detection feature.
6#
7# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
8#
9# REQUIREMENTS
10# Python 2.6+
11# SciPy
12#
13# KNOWN ISSUES
14# only option parsing and read tests are carried out
15# On Windows this script works under Cygwin but not from cmd.exe
16# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
17# min runtime:
18# if ss attained: min runtime = ss_dur + ss_ramp
19# if not attained: runtime = timeout
20
21from __future__ import absolute_import
22from __future__ import print_function
23import os
24import sys
25import json
26import pprint
27import argparse
28import subprocess
29from scipy import stats
30
31def parse_args():
32    parser = argparse.ArgumentParser()
33    parser.add_argument('fio', help='path to fio executable')
34    args = parser.parse_args()
35
36    return args
37
38
39def check(data, iops, slope, pct, limit, dur, criterion):
40    measurement = 'iops' if iops else 'bw'
41    data = data[measurement]
42    mean = sum(data) / len(data)
43    if slope:
44        x = list(range(len(data)))
45        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
46        m = abs(m)
47        if pct:
48            target = (m / mean * 100) if mean != 0 else 0
49            criterion = criterion[:-1]
50        else:
51            target = m
52    else:
53        maxdev = 0
54        for x in data:
55            maxdev = max(abs(mean-x), maxdev)
56        if pct:
57            target = maxdev / mean * 100
58            criterion = criterion[:-1]
59        else:
60            target = maxdev
61
62    criterion = float(criterion)
63    if criterion == 0.0:
64        objsame = False
65    else:
66        objsame = abs(target - criterion) / criterion < 0.005
67    return (objsame, target < limit, mean, target)
68
69
70if __name__ == '__main__':
71    args = parse_args()
72
73    pp = pprint.PrettyPrinter(indent=4)
74
75    passed = 0
76    failed = 0
77
78#
79# test option parsing
80#
81    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
82                  'output': "set steady state IOPS threshold to 10.000000" },
83                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
84                  'output': "set steady state threshold to 10.000000%" },
85                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
86                  'output': "set steady state threshold to 0.100000%" },
87                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
88                  'output': "set steady state threshold to 10.000000%" },
89                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
90                  'output': "set steady state threshold to 0.100000%" },
91                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
92                  'output': "set steady state BW threshold to 12" },
93              ]
94    for test in parsing:
95        output = subprocess.check_output([args.fio] + test['args'])
96        if test['output'] in output.decode():
97            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
98            passed = passed + 1
99        else:
100            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
101            failed = failed + 1
102
103#
104# test some read workloads
105#
106# if ss active and attained,
107#   check that runtime is less than job time
108#   check criteria
109#   how to check ramp time?
110#
111# if ss inactive
112#   check that runtime is what was specified
113#
114    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
115              {'s': False, 'timeout': 20, 'numjobs': 2},
116              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
117              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
118            ]
119
120    jobnum = 0
121    for job in reads:
122
123        tf = "steadystate_job{0}.json".format(jobnum)
124        parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
125        parameters.extend([ "--thread",
126                            "--output-format=json",
127                            "--output={0}".format(tf),
128                            "--ioengine=null",
129                            "--size=1G",
130                            "--rw=randrw",
131                            "--rwmixread=100",
132                            "--stonewall",
133                            "--group_reporting",
134                            "--numjobs={0}".format(job['numjobs']),
135                            "--time_based",
136                            "--runtime={0}".format(job['timeout']) ])
137        if job['s']:
138           if job['iops']:
139               ss = 'iops'
140           else:
141               ss = 'bw'
142           if job['slope']:
143               ss += "_slope"
144           ss += ":" + str(job['ss_limit'])
145           if job['pct']:
146               ss += '%'
147           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
148                               '--ss={0}'.format(ss),
149                               '--ss_ramp={0}'.format(job['ss_ramp']) ])
150
151        output = subprocess.call([args.fio] + parameters)
152        with open(tf, 'r') as source:
153            jsondata = json.loads(source.read())
154            source.close()
155
156        for jsonjob in jsondata['jobs']:
157            line = "{0}".format(jsonjob['job options']['name'])
158            if job['s']:
159                if jsonjob['steadystate']['attained'] == 1:
160                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
161                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
162                    actual = jsonjob['read']['runtime']
163                    if mintime > actual:
164                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
165                        failed = failed + 1
166                    else:
167                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
168                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
169                            iops=job['iops'],
170                            slope=job['slope'],
171                            pct=job['pct'],
172                            limit=job['ss_limit'],
173                            dur=job['ss_dur'],
174                            criterion=jsonjob['steadystate']['criterion'])
175                        if not objsame:
176                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
177                            failed = failed + 1
178                        else:
179                            if met:
180                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
181                                passed = passed + 1
182                            else:
183                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
184                                failed = failed + 1
185                else:
186                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
187                    expected = job['timeout'] * 1000
188                    actual = jsonjob['read']['runtime']
189                    if abs(expected - actual) > 50:
190                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
191                    else:
192                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
193                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
194                            iops=job['iops'],
195                            slope=job['slope'],
196                            pct=job['pct'],
197                            limit=job['ss_limit'],
198                            dur=job['ss_dur'],
199                            criterion=jsonjob['steadystate']['criterion'])
200                        if not objsame:
201                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
202                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
203                                failed = failed + 1
204                            else:
205                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
206                                passed = passed + 1
207                        else:
208                            if met:
209                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
210                                failed = failed + 1
211                            else:
212                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
213                                passed = passed + 1
214            else:
215                expected = job['timeout'] * 1000
216                actual = jsonjob['read']['runtime']
217                if abs(expected - actual) > 50:
218                    result = 'FAILED '
219                    failed = failed + 1
220                else:
221                    result = 'PASSED '
222                    passed = passed + 1
223                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
224            print(line)
225            if 'steadystate' in jsonjob:
226                pp.pprint(jsonjob['steadystate'])
227        jobnum += 1
228
229    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
230    sys.exit(failed)
231