1#!/usr/bin/env python
2################################################################################
3# Copyright 2021 Intel Corporation
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16################################################################################
17
18import sys, os, subprocess
19
20import argparse
21from argparse import RawTextHelpFormatter
22
23# add parent dir to sys.path to make verbose_converter visible for test
24current_dir = os.path.dirname(os.path.realpath(__file__))
25parent_dir = os.path.dirname(current_dir)
26sys.path.append(parent_dir)
27
28import verbose_converter
29from src import benchdnn_generator as benchdnn_gen
30
31status = {'SUCCESS': 0, 'FAILED': 1}
32
33
34def convert_dir_benchdnn2verbose(dir):
35    return {
36        'FWD_D': 'forward_training',
37        'FWD_B': 'forward_training',
38        'FWD_I': 'forward_inference',
39        'BWD_D': 'backward_data',
40        'BWD_W': 'backward_weights',
41        'BWD_DW': 'backward'
42    }.get(dir)
43
44
45def generate_verbose(path_to_benchdnn, driver, batch):
46    benchdnn_exe = path_to_benchdnn + '/benchdnn'
47    sub_env = os.environ.copy()
48    sub_env['ONEDNN_VERBOSE'] = '1'
49    sub_env['ONEDNN_PRIMITIVE_CACHE_CAPACITY'] = '0'
50    sub_args = [
51        benchdnn_exe, f"--{driver}", f"--mode=R", f"-v1", f"--batch={batch}"
52    ]
53    try:
54        sub = subprocess.run(sub_args,
55                             capture_output=True,
56                             text=True,
57                             env=sub_env)
58    except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e:
59        return [
60            status.get('FAILED'), f"subprocess.run() raised exception: " + \
61                f"{e.stdout}"
62        ], ""
63    except BaseException as e:
64        return [
65            status.get('FAILED'), f"subprocess.run() raised exception: " + \
66                f"{e.args}\n{e.stdout}"
67        ], ""
68    if sub.returncode != 0:
69        # most likely converter generated incorrect batch file
70        return [status.get('FAILED'), \
71                f"subprocess.run() returned {sub.returncode},\n" + \
72                f"args: {sub_args}\nstderr: {sub.stderr}"], ""
73    v = ''
74    # strip benchdnn and primitive execution time from verbose
75    v_str = sub.stdout.splitlines()
76    benchdnn_prop_kind = None
77    benchdnn_with_rt_dims = False
78    for l in v_str:
79        if l.find("run: ") != -1:
80            # detect prop kind in benchdnn log
81            dir = '--dir='
82            dir_start = l.find(dir)
83            if dir_start != -1:
84                dir_end = l.find(' ', dir_start)
85                benchdnn_prop_kind = convert_dir_benchdnn2verbose(
86                    l[dir_start + len(dir):dir_end])
87            else:
88                benchdnn_prop_kind = None
89            # detect runtime dims
90            rt_mask = '--runtime_dims_masks='
91            rt_mask_start = l.find(rt_mask)
92            if rt_mask_start != -1:
93                rt_mask_end = l.find(' ', rt_mask_start)
94                benchdnn_rt_dims = l[rt_mask_start + len(rt_mask):rt_mask_end]
95                if benchdnn_rt_dims != '0:0':
96                    benchdnn_with_rt_dims = True
97            else:
98                benchdnn_with_rt_dims = False
99
100        # detect driver
101        l_s = l.split(',')
102        d = benchdnn_gen.convert_driver(l_s[3]) if len(l_s) > 3 else ''
103        if len(l_s) > 3 and l_s[0] == 'onednn_verbose' and d == driver:
104            # filter out additional forward calls
105            verbose_prop_kind = l_s[5]
106            if benchdnn_prop_kind != None and verbose_prop_kind != benchdnn_prop_kind:
107                continue
108            # filter out cases with runtime dims
109            if benchdnn_with_rt_dims:
110                continue
111
112            # remove time
113            l_wo_time = "".join(f + ',' for f in l.split(',')[0:-1])[0:-1]
114
115            v += l_wo_time + '\n'
116
117    return [status.get('SUCCESS'), ''], v
118
119
120def generate_batch(verbose, driver):
121    verbose = verbose.splitlines()
122    s, data = verbose_converter.convert(verbose_level=0,
123                                        parser='oneDNN',
124                                        input=verbose,
125                                        action='generate',
126                                        generator='benchdnn',
127                                        split_output=True)
128    if s != status.get('SUCCESS'):
129        return [s, f"verbose_converter.convert() returned {s}"], ""
130
131    filename = "test.generated"
132    for key, value in data.items():
133        # remove -- from driver name
134        driver_filename = key + '.' + filename
135        of = open(driver_filename, 'w')
136        print(value, file=of)
137    return [s, ''], driver + '.' + filename
138
139
140def compare(driver, ref_v, comp_v):
141    ref_lines = ref_v.splitlines()
142    ref_lines = [l for l in ref_lines if driver in l]
143    comp_lines = comp_v.splitlines()
144    len(comp_lines)
145    comp_lines = [l for l in comp_lines if driver in l]
146    len(comp_lines)
147
148    for r, c in zip(ref_lines, comp_lines):
149        if r != c:
150            ref_log_filename = f"{driver}.reference.log"
151            com_log_filename = f"{driver}.computed.log"
152            ref_log = open(ref_log_filename, 'w')
153            com_log = open(com_log_filename, 'w')
154            print(ref_v, file=ref_log)
155            print(comp_v, file=com_log)
156            return status.get('FAILED'), \
157                    f"verboses do not match,\nref: {r}\ncom: {c}"
158
159    return status.get('SUCCESS'), ''
160
161
162def test(path_to_benchdnn, driver, batch):
163    s, ref_verbose = generate_verbose(path_to_benchdnn, driver, batch)
164    if s[0] != status.get('SUCCESS'):
165        return s
166    # XXX: Maybe generate batch and run becndhnn for each verbose line
167    # separately to detect error on case level and not on batch level?
168    # The reason behind testing on batch level is that ref_verbose generator
169    # might introduce multiple verbose lines for single line in batch file
170    s, gen_batch = generate_batch(ref_verbose, driver)
171    if s[0] != status.get('SUCCESS'):
172        return s
173    s, verbose = generate_verbose(path_to_benchdnn, driver, gen_batch)
174    if s[0] != status.get('SUCCESS'):
175        return s
176
177    return compare(driver, ref_verbose, verbose)
178
179
180def main():
181    realpath = os.path.dirname(os.path.realpath(__file__))
182    print(realpath)
183    realpath_benchdnn = realpath + '/../../../build/tests/benchdnn'
184    args_parser = argparse.ArgumentParser(description='benchdnn test',
185                                          formatter_class=RawTextHelpFormatter)
186    args_parser.add_argument('-d',
187                             '--dataset',
188                             default=realpath + '/' + 'dataset_simple',
189                             help='input with benchdnn batch files')
190    args_parser.add_argument('-b',
191                             '--benchdnn_path',
192                             default=realpath_benchdnn,
193                             help='Path to benchdnn executable')
194    args_parser.add_argument('-i',
195                             '--inputs_path',
196                             default=realpath_benchdnn + '/' + 'inputs',
197                             help='Path to benchdnn batch files')
198    args = args_parser.parse_args()
199
200    with open(args.dataset, 'r') as dataset:
201        for case in dataset.readlines():
202            if case[0] != '#' and case[0] != '\n':
203                [driver, batch] = case.split(',')
204                batch = batch.split('\n')[0]
205                batch_file_path = args.inputs_path + '/' + driver + '/' + batch
206                s = test(args.benchdnn_path, driver, batch_file_path)
207                s_str = 'PASSED' if s[0] == status.get('SUCCESS') else 'FAILED'
208                print(f"BENCHDNN TEST: {driver}, {batch}: {s_str} " + s[1])
209
210    return status.get('SUCCESS')
211
212
213if __name__ == "__main__":
214    main()
215