1from __future__ import absolute_import 2 3import unittest 4 5from vmaf.config import VmafConfig 6from vmaf.core.asset import Asset 7from vmaf.core.quality_runner import VmafossExecQualityRunner 8from vmaf.core.result_store import FileSystemResultStore 9from vmaf.tools.misc import MyTestCase 10 11from test.testutil import set_default_576_324_videos_for_testing, set_default_576_324_10bit_videos_for_testing, \ 12 set_default_576_324_12bit_videos_for_testing, set_default_576_324_16bit_videos_for_testing, \ 13 set_default_576_324_10bit_videos_for_testing_b 14 15__copyright__ = "Copyright 2016-2020, Netflix, Inc." 16__license__ = "BSD+Patent" 17 18 19class VmafossexecQualityRunnerTest(MyTestCase): 20 21 def setUp(self): 22 super().setUp() 23 self.result_store = FileSystemResultStore() 24 25 def tearDown(self): 26 if hasattr(self, 'runner'): 27 self.runner.remove_results() 28 pass 29 super().tearDown() 30 31 def test_run_vmafossexec_runner(self): 32 33 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 34 35 self.runner = VmafossExecQualityRunner( 36 [asset, asset_original], 37 None, fifo_mode=True, 38 delete_workdir=True, 39 result_store=None, 40 ) 41 self.runner.run(parallelize=True) 42 43 results = self.runner.results 44 45 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 46 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 47 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 48 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 49 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 50 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345149030293786, places=4) 51 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 52 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 53 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632406874999999, places=4) 54 55 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.00000001415, places=4) 56 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.99999972612, places=4) 57 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.999999465724, places=4) 58 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999399683, places=4) 59 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 60 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 61 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 62 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 63 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 64 65 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.66890519623612, places=4) 66 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.946416604585025, places=4) 67 68 def test_run_vmafossexec_runner_float_ex(self): 69 70 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 71 72 self.runner = VmafossExecQualityRunner( 73 [asset, asset_original], 74 None, fifo_mode=True, 75 delete_workdir=True, 76 result_store=None, 77 optional_dict={'model_filepath': VmafConfig.model_path("vmaf_float_v0.6.1.json")} 78 ) 79 self.runner.run(parallelize=True) 80 81 results = self.runner.results 82 83 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.3634208125, places=4) 84 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.7666474166666667, places=4) 85 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.8628533333333334, places=4) 86 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9159719583333334, places=4) 87 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.895352291666667, places=4) 88 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345148541666667, places=4) 89 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 90 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 91 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632406874999999, places=4) 92 93 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 94 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.9999998541666666, places=4) 95 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.9999996041666667, places=4) 96 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 97 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.895352291666667, places=4) 98 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 99 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 100 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 101 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 102 103 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.68425208333333, places=4) 104 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.94641666666666, places=4) 105 106 def test_run_vmafossexec_runner_with_thread(self): 107 108 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 109 110 self.runner = VmafossExecQualityRunner( 111 [asset, asset_original], 112 None, fifo_mode=True, 113 delete_workdir=True, 114 result_store=None, 115 optional_dict={'thread': 3} 116 ) 117 self.runner.run(parallelize=True) 118 119 results = self.runner.results 120 121 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.66890519623612, places=4) 122 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.946416604585025, places=4) 123 124 def test_run_vmafossexec_runner_with_subsample(self): 125 126 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 127 128 self.runner = VmafossExecQualityRunner( 129 [asset, asset_original], 130 None, fifo_mode=True, 131 delete_workdir=True, 132 result_store=None, 133 optional_dict={'subsample': 5} 134 ) 135 self.runner.run(parallelize=True) 136 137 results = self.runner.results 138 139 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.9294514, places=4) 140 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.7428478, places=4) 141 142 def test_run_vmafossexec_runner_with_phone_score(self): 143 144 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 145 146 self.runner = VmafossExecQualityRunner( 147 [asset, asset_original], 148 None, fifo_mode=True, 149 delete_workdir=True, 150 result_store=None, 151 optional_dict={ 152 'enable_transform_score': True, 153 } 154 ) 155 self.runner.run(parallelize=True) 156 157 results = self.runner.results 158 159 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 92.52240502083333, places=4) 160 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4) 161 162 def test_run_vmafossexec_runner_norm_type_none(self): 163 164 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 165 166 self.runner = VmafossExecQualityRunner( 167 [asset, asset_original], 168 None, fifo_mode=True, 169 delete_workdir=True, 170 result_store=None, 171 optional_dict={ 172 'model_filepath':VmafConfig.model_path("other_models", "nflxtrain_norm_type_none.json"), 173 }, 174 ) 175 self.runner.run(parallelize=True) 176 177 results = self.runner.results 178 179 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'],0.363420458333, places=4) 180 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4) 181 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4) 182 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4) 183 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) 184 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345148541666667, places=4) 185 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 186 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 187 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4) 188 189 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 190 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'],0.999999958333, places=4) 191 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'],0.999999416667, places=4) 192 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4) 193 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) 194 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 195 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 196 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 197 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 198 199 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 74.25323125, places=4) 200 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 77.996338095161946, places=4) 201 202 def test_run_vmafossexec_runner_yuv422p10le(self): 203 204 ref_path, dis_path, asset, asset_original = set_default_576_324_10bit_videos_for_testing() 205 206 self.runner = VmafossExecQualityRunner( 207 [asset, asset_original], 208 None, fifo_mode=True, 209 delete_workdir=True, 210 result_store=None, 211 ) 212 self.runner.run(parallelize=True) 213 214 results = self.runner.results 215 216 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.3636620625, places=4) 217 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.7674953125, places=4) 218 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.8631078125, places=4) 219 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9157200833333333, places=4) 220 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.895352291666667, places=4) 221 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345148541666667, places=4) 222 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.780577083333331, places=4) 223 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 224 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632406874999999, places=4) 225 226 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 227 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 228 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 229 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 230 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.895352291666667, places=4) 231 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 232 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 72.0, places=4) 233 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 234 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 235 236 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.66890489583334, places=4) 237 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.94641666666666, places=4) 238 239 def test_run_vmafossexec_runner_yuv420p10le_b(self): 240 241 ref_path, dis_path, asset, asset_original = set_default_576_324_10bit_videos_for_testing_b() 242 243 self.runner = VmafossExecQualityRunner( 244 [asset, asset_original], 245 None, fifo_mode=True, 246 delete_workdir=True, 247 result_store=None, 248 ) 249 self.runner.run(parallelize=True) 250 251 results = self.runner.results 252 253 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.4330893333333334, places=4) 254 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.830613, places=4) 255 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 256 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.945896, places=4) 257 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 258 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9517763333333334, places=4) 259 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 32.57143333333333, places=4) # pypsnr: 32.57145231892744 260 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.8978630000000001, places=4) 261 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9747490000000001, places=4) 262 263 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 264 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 265 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 266 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 267 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 268 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 269 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 72.0, places=4) 270 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 271 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 272 273 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 82.56523033333333, places=4) 274 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.142826, places=4) 275 276 def test_run_vmafossexec_runner_yuv420p12le(self): 277 278 ref_path, dis_path, asset, asset_original = set_default_576_324_12bit_videos_for_testing() 279 280 self.runner = VmafossExecQualityRunner( 281 [asset, asset_original], 282 None, fifo_mode=True, 283 delete_workdir=True, 284 result_store=None, 285 ) 286 self.runner.run(parallelize=True) 287 288 results = self.runner.results 289 290 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.4330893333333334, places=4) 291 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.830613, places=4) 292 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 293 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.945896, places=4) 294 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 295 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9517763333333334, places=4) 296 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 32.57783333333334, places=4) # pypsnr: 32.577817940053734 297 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.8978630000000001, places=4) 298 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9747490000000001, places=4) 299 300 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 301 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 302 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 303 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 304 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 305 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 306 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 84.0, places=4) 307 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 308 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 309 310 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 82.56523033333333, places=4) 311 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.142826, places=4) 312 313 def test_run_vmafossexec_runner_yuv420p16le(self): 314 ref_path, dis_path, asset, asset_original = set_default_576_324_16bit_videos_for_testing() 315 316 self.runner = VmafossExecQualityRunner( 317 [asset, asset_original], 318 None, fifo_mode=True, 319 delete_workdir=True, 320 result_store=None, 321 ) 322 self.runner.run(parallelize=True) 323 324 results = self.runner.results 325 326 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.4330893333333334, places=4) 327 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.830613, places=4) 328 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 329 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.945896, places=4) 330 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 331 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9517763333333334, places=4) 332 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 32.579806000000005, places=4) 333 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.8978630000000001, places=4) 334 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9747490000000001, places=4) 335 336 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 337 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 338 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 339 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 340 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 2.8104600000000004, places=4) 341 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 342 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 108.0, places=4) 343 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 344 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 345 346 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 82.56523033333333, places=4) 347 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.142826, places=4) 348 349 def test_run_vmafossexec_runner_yuv420p10le_sparks(self): 350 351 ref_path = VmafConfig.test_resource_path("yuv", "sparks_ref_480x270.yuv42010le.yuv") 352 dis_path = VmafConfig.test_resource_path("yuv", "sparks_dis_480x270.yuv42010le.yuv") 353 asset = Asset(dataset="test", content_id=0, asset_id=0, 354 workdir_root=VmafConfig.workdir_path(), 355 ref_path=ref_path, 356 dis_path=dis_path, 357 asset_dict={'width': 480, 'height': 270, 358 'yuv_type': 'yuv420p10le'}) 359 360 asset_original = Asset(dataset="test", content_id=0, asset_id=1, 361 workdir_root=VmafConfig.workdir_path(), 362 ref_path=ref_path, 363 dis_path=ref_path, 364 asset_dict={'width': 480, 'height': 270, 365 'yuv_type': 'yuv420p10le'}) 366 367 self.runner = VmafossExecQualityRunner( 368 [asset, asset_original], 369 None, fifo_mode=True, 370 delete_workdir=True, 371 result_store=None, 372 ) 373 self.runner.run(parallelize=True) 374 375 results = self.runner.results 376 377 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.9240746, places=4) 378 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.9968371999999999, places=4) 379 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.9987575999999999, places=4) 380 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9993221999999999, places=4) 381 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 0.7523685999999999, places=4) 382 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9981770000000001, places=4) 383 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 48.81622, places=4) 384 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.99566, places=4) 385 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9993778000000001, places=4) 386 387 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 388 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.9999990000000001, places=4) 389 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.9999990000000001, places=4) 390 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.9999990000000001, places=4) 391 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 0.7523685999999999, places=4) 392 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 393 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 72.0, places=4) 394 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 395 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 396 397 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 97.90069380000001, places=4) 398 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 98.47175940000001, places=4) 399 400 def test_run_vmafossexec_runner_with_transform_score(self): 401 402 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 403 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 404 asset = Asset(dataset="test", content_id=0, asset_id=0, 405 workdir_root=VmafConfig.workdir_path(), 406 ref_path=ref_path, 407 dis_path=dis_path, 408 asset_dict={'width':1920, 'height':1080}) 409 410 self.runner = VmafossExecQualityRunner( 411 [asset], 412 None, fifo_mode=True, 413 delete_workdir=True, 414 optional_dict={ 415 'model_filepath': VmafConfig.test_resource_path("test_model_transform_add40.json"), 416 'enable_transform_score': True, 417 }, 418 result_store=self.result_store, 419 ) 420 self.runner.run(parallelize=True) 421 422 results = self.runner.results 423 424 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.0, places=4) 425 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.0, places=4) 426 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.0, places=4) 427 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.0, places=4) 428 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 12.5548366667, places=4) 429 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 430 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 431 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 432 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 433 434 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 32.757433333333331, places=4) 435 436 def test_run_vmafossexec_runner_with_transform_score_2(self): 437 438 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 439 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 440 asset = Asset(dataset="test", content_id=0, asset_id=0, 441 workdir_root=VmafConfig.workdir_path(), 442 ref_path=ref_path, 443 dis_path=dis_path, 444 asset_dict={'width':1920, 'height':1080}) 445 446 self.runner = VmafossExecQualityRunner( 447 [asset], 448 None, fifo_mode=True, 449 delete_workdir=True, 450 optional_dict={ 451 'model_filepath': VmafConfig.test_resource_path("test_model_transform_add40_outltein.json"), 452 'enable_transform_score': True, 453 'disable_clip_score': True, 454 }, 455 result_store=self.result_store, 456 ) 457 self.runner.run(parallelize=True) 458 459 results = self.runner.results 460 461 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.0, places=4) 462 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.0, places=4) 463 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.0, places=4) 464 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.0, places=4) 465 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 12.5548366667, places=4) 466 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 467 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 468 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 469 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 470 471 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], -7.2425766666666673, places=4) 472 473 def test_run_vmafossexec_runner_with_transform_score_disabled(self): 474 475 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 476 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 477 asset = Asset(dataset="test", content_id=0, asset_id=0, 478 workdir_root=VmafConfig.workdir_path(), 479 ref_path=ref_path, 480 dis_path=dis_path, 481 asset_dict={'width':1920, 'height':1080}) 482 483 self.runner = VmafossExecQualityRunner( 484 [asset], 485 None, fifo_mode=True, 486 delete_workdir=True, 487 optional_dict={ 488 'model_filepath':VmafConfig.test_resource_path("test_model_transform_add40.json"), 489 'enable_transform_score':False, 490 }, 491 result_store=self.result_store, 492 ) 493 self.runner.run(parallelize=True) 494 495 results = self.runner.results 496 497 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.0, places=4) 498 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.0, places=4) 499 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.0, places=4) 500 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.0, places=4) 501 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 12.5548366667, places=4) 502 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 503 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 504 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 505 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 506 507 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 0.0, places=4) 508 509 def test_run_vmafossexec_runner_with_transform_for_phone(self): 510 511 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 512 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 513 asset = Asset(dataset="test", content_id=0, asset_id=0, 514 workdir_root=VmafConfig.workdir_path(), 515 ref_path=ref_path, 516 dis_path=dis_path, 517 asset_dict={'width': 1920, 'height': 1080}) 518 519 self.runner = VmafossExecQualityRunner( 520 [asset], 521 None, fifo_mode=True, 522 delete_workdir=True, 523 optional_dict={ 524 'enable_transform_score': True, 525 }, 526 result_store=self.result_store, 527 ) 528 self.runner.run(parallelize=True) 529 530 results = self.runner.results 531 532 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.0, places=4) 533 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.0, places=4) 534 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.0, places=4) 535 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.0, places=4) 536 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 12.554711666666668, places=4) 537 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.053996566666666669, places=4) 538 539 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 14.982747333333334, places=4) 540 541 def test_run_vmafossexec_runner_with_phone_model(self): 542 543 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 544 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 545 asset = Asset(dataset="test", content_id=0, asset_id=0, 546 workdir_root=VmafConfig.workdir_path(), 547 ref_path=ref_path, 548 dis_path=dis_path, 549 asset_dict={'width': 1920, 'height': 1080}) 550 551 self.runner = VmafossExecQualityRunner( 552 [asset], 553 None, fifo_mode=True, 554 delete_workdir=True, 555 optional_dict={ 556 'phone_model': True, 557 }, 558 result_store=self.result_store, 559 ) 560 self.runner.run(parallelize=True) 561 562 results = self.runner.results 563 564 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.0, places=4) 565 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.0, places=4) 566 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.0, places=4) 567 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.0, places=4) 568 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 12.554711666666668, places=4) 569 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.053996566666666669, places=4) 570 571 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 14.982747333333334, places=4) 572 573 def test_run_vmafossexec_runner_disable_avx_precise(self): 574 575 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 576 577 self.runner = VmafossExecQualityRunner( 578 [asset, asset_original], 579 None, fifo_mode=True, 580 delete_workdir=True, 581 result_store=None, 582 optional_dict={'disable_avx': True} 583 ) 584 self.runner.run(parallelize=True) 585 586 results = self.runner.results 587 588 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 589 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 590 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 591 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 592 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 593 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345149030293786, places=4) 594 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 595 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 596 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632406874999999, places=4) 597 598 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.00000001415, places=4) 599 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.99999972612, places=4) 600 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.999999465724, places=4) 601 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999399683, places=4) 602 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 603 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 604 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 605 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 606 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 607 608 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.66890519623612, places=4) 609 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.946416604585025, places=4) 610 611 def test_run_vmafossexec_runner_with_motion2(self): 612 613 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 614 615 self.runner = VmafossExecQualityRunner( 616 [asset, asset_original], 617 None, fifo_mode=True, 618 delete_workdir=True, 619 result_store=None, 620 optional_dict={ 621 'model_filepath':VmafConfig.test_resource_path("test_motion2.json") 622 }, 623 ) 624 self.runner.run(parallelize=True) 625 626 results = self.runner.results 627 628 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) 629 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4) 630 631 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.04982583333, places=4) 632 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953522916666672, places=4) 633 634 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 78.51201666666667, places=4) 635 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 97.089554166666673, places=4) 636 637 def test_run_vmafossexec_runner_with_ci(self): 638 639 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 640 641 self.runner = VmafossExecQualityRunner( 642 [asset, asset_original], 643 None, fifo_mode=True, 644 delete_workdir=True, 645 result_store=None, 646 optional_dict={ 647 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.json"), 648 'ci': True 649 }, 650 ) 651 self.runner.run(parallelize=True) 652 653 results = self.runner.results 654 655 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.363420458333, places=4) 656 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.766647520833, places=4) 657 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.862854708333, places=4) 658 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.915971791667, places=4) 659 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 660 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345148541666667, places=4) 661 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 662 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 663 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632498125, places=4) 664 665 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 666 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 0.999999958333, places=4) 667 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 0.999999416667, places=4) 668 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 0.999999208333, places=4) 669 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion2_score'], 3.8953518541666665, places=4) 670 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm2_score'], 1.0, places=4) 671 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 672 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 673 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 674 675 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.42800833333332, places=3) 676 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4) 677 678 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_bagging_score'], 73.0913125, places=3) 679 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_bagging_score'], 99.79000416666668, places=4) 680 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_stddev_score'], 1.1982760416666667, places=3) 681 682 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_stddev_score'], 1.3028828125, places=4) 683 684 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_ci_p95_lo_score'], 70.81472708333332, places=3) 685 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_ci_p95_lo_score'], 94.79667083333334, places=4) 686 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_ci_p95_hi_score'], 74.83768541666666, places=3) 687 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_ci_p95_hi_score'], 99.99736666666666, places=4) 688 689 # per model score checks 690 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 73.25864166666666, places=3) 691 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 70.37462916666668, places=3) 692 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 71.58177291666668, places=3) 693 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 73.14455, places=3) 694 695 def test_run_vmafossexec_runner_with_ci_and_custom_model(self): 696 697 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 698 699 self.runner = VmafossExecQualityRunner( 700 [asset, asset_original], 701 None, fifo_mode=True, 702 delete_workdir=True, 703 result_store=None, 704 optional_dict={ 705 'model_filepath': VmafConfig.test_resource_path('model', 'vmafplus_v0.5.2boot_test.json'), 706 'ci': True 707 }, 708 ) 709 self.runner.run(parallelize=True) 710 711 results = self.runner.results 712 713 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 75.42800833333332, places=3) 714 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.958047916666672, places=4) 715 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_bagging_score'], 75.11496458333333, places=3) 716 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_bagging_score'], 99.96504855577571, places=4) 717 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_stddev_score'], 0.6812993325967104, places=3) 718 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_stddev_score'], 0.03947607207290399, places=4) 719 720 def test_run_vmafossexec_runner_with_ci_and_phone_model(self): 721 722 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 723 724 self.runner = VmafossExecQualityRunner( 725 [asset, asset_original], 726 None, fifo_mode=True, 727 delete_workdir=True, 728 result_store=None, 729 optional_dict={ 730 'model_filepath': VmafConfig.model_path("vmaf_rb_v0.6.3", "vmaf_rb_v0.6.3.json"), 731 'phone_model': True, 732 'ci': True, 733 }, 734 ) 735 self.runner.run(parallelize=True) 736 737 results = self.runner.results 738 739 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 91.71303333333334, places=4) 740 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 100.0, places=4) 741 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_bagging_score'], 90.12363958333333, places=3) 742 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_bagging_score'], 100.0, places=4) 743 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_stddev_score'], 0.8371132083333332, places=3) 744 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vmaf_stddev_score'], 0.0, places=4) 745 746 # per model score checks 747 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0001_score'], 90.24346041666666, places=3) 748 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0002_score'], 88.17759791666667, places=3) 749 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0003_score'], 89.04172708333333, places=3) 750 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vmaf_0020_score'], 90.1585875, places=3) 751 752 def test_run_vmafossexec_runner_vmafplus1080model(self): 753 754 ref_path = VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv") 755 dis_path = VmafConfig.test_resource_path("yuv", "src01_hrc01_576x324.yuv") 756 asset = Asset(dataset="test", content_id=0, asset_id=1, 757 workdir_root=VmafConfig.workdir_path(), 758 ref_path=ref_path, 759 dis_path=dis_path, 760 asset_dict={'width': 576, 'height': 324, 761 }) 762 763 asset_original = Asset(dataset="test", content_id=0, asset_id=2, 764 workdir_root=VmafConfig.workdir_path(), 765 ref_path=ref_path, 766 dis_path=ref_path, 767 asset_dict={'width': 576, 'height': 324, 768 }) 769 770 self.runner = VmafossExecQualityRunner( 771 [asset, asset_original], 772 None, fifo_mode=True, 773 delete_workdir=True, 774 result_store=None, 775 optional_dict={ 776 'model_filepath': VmafConfig.test_resource_path('vmafplus_laptop_480source_vmafplusv2_sa.json') 777 }, 778 ) 779 self.runner.run(parallelize=True) 780 781 results = self.runner.results 782 783 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.36342048943884936, places=4) 784 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.76664754213485187, places=4) 785 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.86285466690193247, places=4) 786 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.91597177803640772, places=4) 787 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion_score'], 4.0498253541666669, places=4) 788 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale0_score'], 0.90791927083333324, places=4) 789 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale1_score'], 0.8938705625000001, places=4) 790 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale2_score'], 0.9300123749999999, places=4) 791 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm_scale3_score'], 0.9649663541666667, places=4) 792 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.755066666666664, places=4) 793 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 794 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.96324981249999997, places=4) 795 796 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale0_score'], 1.0, places=4) 797 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale1_score'], 1.0, places=4) 798 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale2_score'], 1.0, places=4) 799 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_vif_scale3_score'], 1.0, places=4) 800 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_motion_score'], 4.0498253541666669, places=4) 801 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm_scale0_score'], 1.0, places=4) 802 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm_scale1_score'], 1.0, places=4) 803 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm_scale2_score'], 1.0, places=4) 804 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_adm_scale3_score'], 1.0, places=4) 805 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_psnr_score'], 60.0, places=4) 806 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ssim_score'], 1.0, places=4) 807 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_ms_ssim_score'], 1.0, places=4) 808 809 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 35.4220125, places=4) 810 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 65.559652083333333, places=4) 811 812 def test_run_vmafossexec_runner_akiyo_multiply_no_enhn_gain_model_float(self): 813 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 814 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 815 asset = Asset(dataset="test", content_id=0, asset_id=0, 816 workdir_root=VmafConfig.workdir_path(), 817 ref_path=ref_path, 818 dis_path=dis_path, 819 asset_dict={'width': 352, 'height': 288}) 820 821 self.runner = VmafossExecQualityRunner( 822 [asset], 823 None, fifo_mode=True, 824 delete_workdir=True, 825 result_store=None, 826 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_float_v0.6.1neg.json")} 827 ) 828 self.runner.run(parallelize=True) 829 830 results = self.runner.results 831 832 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 833 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 834 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 835 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 836 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 837 838 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 88.032956, places=2) # 132.78849246495625 839 840 def test_run_vmafossexec_runner_akiyo_multiply_no_enhn_gain_model_json(self): 841 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 842 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 843 asset = Asset(dataset="test", content_id=0, asset_id=0, 844 workdir_root=VmafConfig.workdir_path(), 845 ref_path=ref_path, 846 dis_path=dis_path, 847 asset_dict={'width': 352, 'height': 288}) 848 849 self.runner = VmafossExecQualityRunner( 850 [asset], 851 None, fifo_mode=False, 852 delete_workdir=True, 853 result_store=None, 854 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_v0.6.1neg.json")} 855 ) 856 self.runner.run(parallelize=False) 857 858 results = self.runner.results 859 860 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 861 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 862 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 863 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 864 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 865 866 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 88.030463, places=4) # 132.78849246495625 867 868 def test_run_parallel_vmafossexec_runner_with_repeated_assets(self): 869 870 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 871 872 self.runner = VmafossExecQualityRunner( 873 [asset, asset_original, asset, asset], 874 None, fifo_mode=True, 875 delete_workdir=True, 876 result_store=None 877 ) 878 self.runner.run(parallelize=True) 879 results = self.runner.results 880 881 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 76.66890489583334, places=3) 882 self.assertAlmostEqual(results[1]['VMAFOSSEXEC_score'], 99.946416666666664, places=4) 883 self.assertAlmostEqual(results[2]['VMAFOSSEXEC_score'], 76.66890489583334, places=3) 884 self.assertAlmostEqual(results[3]['VMAFOSSEXEC_score'], 76.66890489583334, places=3) 885 886 def test_run_vmafossexec_runner_4k(self): 887 888 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 889 890 self.runner = VmafossExecQualityRunner( 891 [asset], 892 None, fifo_mode=True, 893 delete_workdir=True, 894 result_store=None, 895 optional_dict={'model_filepath': VmafConfig.model_path("vmaf_4k_v0.6.1.json")} 896 ) 897 self.runner.run(parallelize=True) 898 899 results = self.runner.results 900 901 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale0_score'], 0.3636620625, places=4) 902 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale1_score'], 0.7674953125, places=4) 903 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale2_score'], 0.8631078125, places=4) 904 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_vif_scale3_score'], 0.9157200833333333, places=4) 905 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_motion2_score'], 3.895352291666667, places=4) 906 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_adm2_score'], 0.9345148541666667, places=4) 907 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_psnr_score'], 30.7550666667, places=4) 908 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ssim_score'], 0.86322654166666657, places=4) 909 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_ms_ssim_score'], 0.9632406874999999, places=4) 910 911 self.assertAlmostEqual(results[0]['VMAFOSSEXEC_score'], 84.95064735416668, places=4) 912 913 914class VmafossexecQualityRunnerSubsamplingTest(MyTestCase): 915 916 def setUp(self): 917 super().setUp() 918 self.result_store = FileSystemResultStore() 919 920 def tearDown(self): 921 if hasattr(self, 'runner0'): 922 self.runner0.remove_results() 923 if hasattr(self, 'runner'): 924 self.runner.remove_results() 925 super().tearDown() 926 927 def test_run_vmafossexec_runner_with_subsample2(self): 928 929 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 930 931 subsample = 5 932 933 self.runner0 = VmafossExecQualityRunner( 934 [asset, asset_original], 935 None, fifo_mode=True, 936 delete_workdir=True, 937 result_store=None, 938 optional_dict={} 939 ) 940 self.runner0.run(parallelize=True) 941 results0 = self.runner0.results 942 943 self.runner = VmafossExecQualityRunner( 944 [asset, asset_original], 945 None, fifo_mode=True, 946 delete_workdir=True, 947 result_store=None, 948 optional_dict={'subsample': subsample} 949 ) 950 self.runner.run(parallelize=True) 951 results = self.runner.results 952 953 for i in range(48): 954 if i % subsample == 0: 955 self.assertAlmostEqual(results0[0]['VMAFOSSEXEC_scores'][i], results[0]['VMAFOSSEXEC_scores'][i // subsample], places=7) 956 self.assertAlmostEqual(results0[1]['VMAFOSSEXEC_scores'][i], results[1]['VMAFOSSEXEC_scores'][i // subsample], places=7) 957 958 959class QualityRunnerVersionTest(unittest.TestCase): 960 961 def test_vmafossexec_quality_runner_version(self): 962 self.assertEqual(VmafossExecQualityRunner.VERSION, 'F0.2.7-0.6.1') 963 self.assertEqual(VmafossExecQualityRunner.ALGO_VERSION, 2) 964 965 966if __name__ == '__main__': 967 unittest.main(verbosity=2) 968