1# -*- coding: utf-8 -*- 2 3# Copyright (c) 2017-2020 Intel Corporation 4# 5# Permission is hereby granted, free of charge, to any person obtaining a copy 6# of this software and associated documentation files (the "Software"), to deal 7# in the Software without restriction, including without limitation the rights 8# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9# copies of the Software, and to permit persons to whom the Software is 10# furnished to do so, subject to the following conditions: 11# 12# The above copyright notice and this permission notice shall be included in all 13# copies or substantial portions of the Software. 14# 15# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21# SOFTWARE. 22 23import json 24import datetime 25import collections 26import itertools 27 28from pathlib import Path 29 30from . import objects, run, platform 31 32 33class ValidationError(Exception): 34 pass 35 36 37class CaseLogger(object): 38 def __init__(self, fn, cfg): 39 self.fn = fn 40 self.cfg = cfg 41 42 def log(self, msg): 43 with self.fn.open('a+') as f: 44 f.write(msg + '\n') 45 46 def dump_header(self): 47 for key, value in self.cfg.environment.items(): 48 self.log("{:>8s}: {}".format(key, value)) 49 50 self.log(" started: {}".format(datetime.datetime.now())) 51 52 def separator(self): 53 self.log('-' * 78) 54 55 56class Test(object): 57 def __init__(self, fn, base, cfg, args): 58 self.cases = [] 59 self.cfg = cfg 60 self.test_type = None 61 self.base_dir = Path(base) 62 self.device = args.device 63 64 fn = Path(fn) 65 66 self.config = json.loads(fn.read_text(), object_pairs_hook=collections.OrderedDict) 67 self.name = fn.stem 68 69 self.results = self.base_dir / 'results' / self.name 70 71 self.generate_cases() 72 73 self.runner = run.Runner(dict(), cfg) 74 75 def clear_results(self): 76 if self.results.exists(): 77 for fn in self.results.glob('*.*'): 78 fn.unlink() 79 80 def remove_generated(self, results, folder): 81 for fn in results.keys(): 82 try: 83 (folder / fn).unlink() 84 except Exception: 85 pass 86 87 def exec_test_tool(self, case_id, params, workdir, log): 88 if self.test_type == 'decode': 89 return self.runner.sample_decode(case_id, params, workdir, log) 90 elif self.test_type == 'encode': 91 return self.runner.sample_encode(case_id, params, workdir, log) 92 elif self.test_type == 'transcode': 93 return self.runner.sample_multi_transcode(case_id, params, workdir, log) 94 elif self.test_type == 'vpp': 95 return self.runner.sample_vpp(case_id, params, workdir, log) 96 97 def run(self): 98 self.clear_results() 99 self.results.mkdir(parents=True, exist_ok=True) 100 total = passed = 0 101 102 details = { 103 'test': self.name, 104 'cases': [] 105 } 106 for i, case in enumerate(self.cases, 1): 107 log = CaseLogger(self.results / "{:04d}.log".format(i), self.cfg) 108 109 error = None 110 111 total += 1 112 print(" {:04d}".format(i), end="") 113 results = self.exec_test_tool(i, case, self.results, log) 114 115 if results: 116 passed += 1 117 log.log('PASS') 118 else: 119 error = "fail" 120 log.log('FAIL') 121 self.remove_generated(results, self.results) 122 log.separator() 123 res = { 124 'id': '{:04d}'.format(i), 125 } 126 if error: 127 print(' - FAIL') 128 res['status'] = 'FAIL' 129 # Print log in case of the failure 130 with log.fn.open('r') as f: 131 print(f.read()) 132 res['error'] = error 133 log.log(error) 134 res['artifacts'] = results 135 else: 136 print(' - ok') 137 log.log('PASS') 138 res['status'] = 'PASS' 139 res['artifacts'] = results 140 141 details['cases'].append(res) 142 143 log.log('\nfinisned: {}'.format(datetime.datetime.now())) 144 145 return (total, passed, details) 146 147 def generate_cases(self): 148 self.platforms = self.config.get('platforms', None) 149 device_name = Path(self.device).name 150 if self.platforms is not None: 151 if not platform.check_platform(platform.get_current_device_id(device_name), self.platforms): 152 raise platform.UnsupportedPlatform("test is disabled for current platform") 153 154 self.test_type = self.config.get('type', None) 155 if self.test_type not in ['decode', 'encode', 'transcode', 'vpp']: 156 raise ValidationError("unknown test type") 157 158 # let's generate preliminary cases list 159 keys = [] 160 values = [] 161 162 for key, val in self.config.items(): 163 if key == 'type': 164 continue 165 166 keys.append(key) 167 if isinstance(val, list): 168 values.append(val) 169 else: 170 values.append([val]) 171 172 for vals in itertools.product(*values): 173 case = collections.OrderedDict(zip(keys, vals)) 174 case['device'] = self.device 175 176 if 'stream' not in case: 177 if self.test_type != 'transcode': 178 raise ValidationError("stream is not defined") 179 else: 180 # process common options 181 case['stream'] = self.cfg.stream_by_name(case['stream']) 182 183 if 'codec' in case: 184 case['codec'] = objects.Encoder(case['codec']) 185 if case['codec'].codec == 'jpeg': 186 if 'quality' not in case: 187 raise ValidationError("undefined JPEG quality") 188 if 'target_usage' in case: 189 raise ValidationError("JPEG encoder does not support target usage") 190 if 'bitrate' in case or 'qp' in case: 191 raise ValidationError("JPEG encoder does not support bitrate or QP setting") 192 193 else: 194 if 'target_usage' in case: 195 case['target_usage'] = objects.TargetUsage(case['target_usage']) 196 197 if 'bitrate' not in case and 'qp' not in case: 198 raise ValidationError("undefined bitrate or QP") 199 if 'bitrate' in case and 'qp' in case: 200 raise ValidationError("both bitrate and QP defined") 201 202 if 'parfile' in case: 203 case['parfile'] = objects.ParFile(case['parfile'], self.base_dir, self.cfg) 204 205 if self.test_type == 'transcode' and 'parfile' not in case: 206 raise ValidationError("unknown parfile for transcode test") 207 208 if self.test_type == 'encode' and 'codec' not in case: 209 raise ValidationError("unknown codec for encode test") 210 211 self.cases.append(case) 212 213