1from __future__ import absolute_import 2 3import unittest 4 5from vmaf.config import VmafConfig 6from vmaf.core.asset import Asset 7from vmaf.core.quality_runner import VmafexecQualityRunner, VmafossExecQualityRunner 8from vmaf.core.result_store import FileSystemResultStore 9from test.testutil import set_default_576_324_videos_for_testing, \ 10 set_default_576_324_10bit_videos_for_testing, set_default_576_324_10bit_videos_for_testing_b, \ 11 set_default_576_324_12bit_videos_for_testing, set_default_576_324_16bit_videos_for_testing 12from vmaf.tools.misc import MyTestCase 13 14__copyright__ = "Copyright 2016-2020, Netflix, Inc." 15__license__ = "BSD+Patent" 16 17 18class VmafexecQualityRunnerTest(MyTestCase): 19 20 def setUp(self): 21 super().setUp() 22 self.result_store = FileSystemResultStore() 23 24 def tearDown(self): 25 if hasattr(self, 'runner'): 26 self.runner.remove_results() 27 pass 28 super().tearDown() 29 30 def test_run_vmafexec_runner_matched_to_vmafossexec(self): 31 32 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 33 34 self.runner = VmafexecQualityRunner( 35 [asset, asset_original], 36 None, fifo_mode=True, 37 delete_workdir=True, 38 result_store=None, 39 optional_dict={ 40 'float_psnr': True, 41 'float_ssim': True, 42 'float_ms_ssim': True, 43 } 44 ) 45 self.runner.run(parallelize=True) 46 47 results = self.runner.results 48 49 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 50 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 51 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 52 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 53 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 54 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 55 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 30.7550666667, places=4) 56 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.86322654166666657, places=4) 57 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9632406874999999, places=4) 58 59 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 60 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'],0.99999972612, places=4) 61 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'],0.999999465724, places=4) 62 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.999999399683, places=4) 63 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 64 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 65 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 60.0, places=4) 66 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 67 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 68 69 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890519623612, places=4) 70 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.946416604585025, places=4) 71 72 def test_run_vmafexec_runner_float_fex(self): 73 74 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 75 76 self.runner = VmafexecQualityRunner( 77 [asset, asset_original], 78 None, fifo_mode=True, 79 delete_workdir=True, 80 result_store=None, 81 optional_dict={ 82 'float_psnr': True, 83 'float_ssim': True, 84 'float_ms_ssim': True, 85 'model_filepath': VmafConfig.model_path("vmaf_float_v0.6.1.json") 86 } 87 ) 88 self.runner.run(parallelize=True) 89 90 results = self.runner.results 91 92 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3634208125, places=4) 93 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7666474166666667, places=4) 94 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8628533333333334, places=4) 95 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9159719583333334, places=4) 96 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 97 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345148541666667, places=4) 98 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 30.7550666667, places=4) 99 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.86322654166666657, places=4) 100 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9632406874999999, places=4) 101 102 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 103 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'],0.9999998541666666, places=4) 104 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'],0.9999996041666667, places=4) 105 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 106 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 107 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 108 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 60.0, places=4) 109 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 110 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 111 112 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.68425579166666, places=4) 113 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.94641666666666, places=4) 114 115 def test_run_vmafexec_runner_motion_force_zero(self): 116 117 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 118 119 self.runner = VmafexecQualityRunner( 120 [asset, asset_original], 121 None, fifo_mode=True, 122 delete_workdir=True, 123 result_store=None, 124 optional_dict={ 125 'motion_force_zero': True, 126 'model_filepath': VmafConfig.model_path("vmaf_v0.6.1.json") 127 } 128 ) 129 self.runner.run(parallelize=True) 130 131 results = self.runner.results 132 133 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 134 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 135 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 136 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 137 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_force_0_score'], 0.0, places=4) 138 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 139 140 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 141 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 142 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 143 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 144 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_force_0_score'], 0.0, places=4) 145 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 146 147 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 72.32054995833333, places=4) # 76.68425579166666 148 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 97.42843597916665, places=4) # 99.94641666666666 149 150 self.assertEqual(len(results[0]['VMAFEXEC_motion2_force_0_scores']), 48) 151 self.assertEqual(len(results[1]['VMAFEXEC_motion2_force_0_scores']), 48) 152 153 def test_run_vmafexec_runner_motion_force_zero2(self): 154 155 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 156 157 self.runner = VmafexecQualityRunner( 158 [asset, asset_original], 159 None, fifo_mode=False, 160 delete_workdir=True, 161 result_store=None, 162 optional_dict={ 163 'model_filepath': VmafConfig.model_path("other_models", "vmaf_v0.6.1mfz.json") 164 } 165 ) 166 self.runner.run(parallelize=False) 167 168 results = self.runner.results 169 170 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 171 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 172 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 173 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 174 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_force_0_score'], 0.0, places=4) 175 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 176 177 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 178 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 179 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 180 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 181 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_force_0_score'], 0.0, places=4) 182 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 183 184 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 72.32054995833333, places=4) # 76.68425579166666 185 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 97.42843597916665, places=4) # 99.94641666666666 186 187 self.assertEqual(len(results[0]['VMAFEXEC_motion2_force_0_scores']), 48) 188 self.assertEqual(len(results[1]['VMAFEXEC_motion2_force_0_scores']), 48) 189 190 def test_run_vmafexec_runner_fixed_psnr(self): 191 192 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 193 194 self.runner = VmafexecQualityRunner( 195 [asset, asset_original], 196 None, fifo_mode=True, 197 delete_workdir=True, 198 result_store=None, 199 optional_dict={ 200 'psnr': True, 201 'no_prediction': True, 202 } 203 ) 204 self.runner.run(parallelize=True) 205 206 results = self.runner.results 207 208 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_y_score'], 30.755063979166668, places=4) 209 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_cb_score'], 38.4494410625, places=4) 210 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_cr_score'], 40.99191027083334, places=4) 211 212 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_y_score'], 60.0, places=4) 213 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_cb_score'], 60.0, places=4) 214 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_cr_score'], 60.0, places=4) 215 216 def test_run_vmafexec_runner_n_threads(self): 217 218 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 219 220 self.runner = VmafexecQualityRunner( 221 [asset, asset_original], 222 None, fifo_mode=True, 223 delete_workdir=True, 224 result_store=None, 225 optional_dict={ 226 'float_psnr': True, 227 'float_ssim': True, 228 'float_ms_ssim': True, 229 'psnr': True, 230 'n_threads': 4, 231 } 232 ) 233 self.runner.run(parallelize=True) 234 235 results = self.runner.results 236 237 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 238 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 239 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 240 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 241 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 242 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 243 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 30.7550666667, places=4) 244 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.86322654166666657, places=4) 245 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9632406874999999, places=4) 246 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_y_score'], 30.755063979166668, places=4) 247 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_cb_score'], 38.4494410625, places=4) 248 self.assertAlmostEqual(results[0]['VMAFEXEC_psnr_cr_score'], 40.99191027083334, places=4) 249 250 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 251 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 252 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 253 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 254 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 255 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 256 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 60.0, places=4) 257 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 258 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 259 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_y_score'], 60.0, places=4) 260 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_cb_score'], 60.0, places=4) 261 self.assertAlmostEqual(results[1]['VMAFEXEC_psnr_cr_score'], 60.0, places=4) 262 263 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890489583334, places=4) 264 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.94641666666666, places=4) 265 266 def test_run_vmafexec_runner_set_custom_models(self): 267 268 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 269 270 self.runner = VmafexecQualityRunner( 271 [asset, asset_original], 272 None, fifo_mode=True, 273 delete_workdir=True, 274 result_store=None, 275 optional_dict={ 276 'models': [ 277 'path={}:name=custom_vmaf_0'.format(VmafConfig.model_path("vmaf_float_v0.6.1.json")), 278 'path={}:name=custom_vmaf_1'.format(VmafConfig.model_path("vmaf_float_v0.6.1.json")), 279 ] 280 } 281 ) 282 self.runner.run(parallelize=True) 283 284 results = self.runner.results 285 286 self.assertAlmostEqual(results[0]['VMAFEXEC_custom_vmaf_0_score'], 76.68425579166666, places=4) 287 self.assertAlmostEqual(results[0]['VMAFEXEC_custom_vmaf_1_score'], 76.68425579166666, places=4) 288 self.assertAlmostEqual(results[1]['VMAFEXEC_custom_vmaf_0_score'], 99.94641666666666, places=4) 289 self.assertAlmostEqual(results[1]['VMAFEXEC_custom_vmaf_1_score'], 99.94641666666666, places=4) 290 291 def test_run_vmafexec_runner_set_custom_models_enable_transform(self): 292 293 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 294 295 self.runner = VmafexecQualityRunner( 296 [asset, asset_original], 297 None, fifo_mode=True, 298 delete_workdir=True, 299 result_store=None, 300 optional_dict={ 301 'models': [ 302 'path={}:name=standvmaf'.format(VmafConfig.model_path("vmaf_float_v0.6.1.json")), 303 'path={}:name=phonevmaf:enable_transform'.format(VmafConfig.model_path("vmaf_float_v0.6.1.json")), 304 ] 305 } 306 ) 307 self.runner.run(parallelize=True) 308 309 results = self.runner.results 310 311 self.assertAlmostEqual(results[0]['VMAFEXEC_standvmaf_score'], 76.68425579166666, places=4) 312 self.assertAlmostEqual(results[0]['VMAFEXEC_phonevmaf_score'], 92.53270047916665, places=4) 313 self.assertAlmostEqual(results[1]['VMAFEXEC_standvmaf_score'], 99.94641666666666, places=4) 314 self.assertAlmostEqual(results[1]['VMAFEXEC_phonevmaf_score'], 100.0, places=4) 315 316 def test_run_vmafexec_runner_disable_avx(self): 317 318 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 319 320 self.runner = VmafexecQualityRunner( 321 [asset, asset_original], 322 None, fifo_mode=True, 323 delete_workdir=True, 324 result_store=None, 325 optional_dict={ 326 'disable_avx': True 327 } 328 ) 329 self.runner.run(parallelize=True) 330 331 results = self.runner.results 332 333 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 334 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 335 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 336 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 337 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 338 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 339 340 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 341 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.99999972612, places=4) 342 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.999999465724, places=4) 343 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.999999399683, places=4) 344 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 345 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 346 347 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890519623612, places=4) 348 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.946416604585025, places=4) 349 350 def test_run_parallel_vmafexec_runner_with_repeated_assets(self): 351 352 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 353 354 self.runner = VmafexecQualityRunner( 355 [asset, asset_original, asset, asset], 356 None, fifo_mode=True, 357 delete_workdir=True, 358 result_store=None 359 ) 360 self.runner.run(parallelize=True) 361 results = self.runner.results 362 363 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890519623612, places=3) 364 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.946416666666664, places=4) 365 self.assertAlmostEqual(results[2]['VMAFEXEC_score'], 76.66890519623612, places=3) 366 self.assertAlmostEqual(results[3]['VMAFEXEC_score'], 76.66890519623612, places=3) 367 368 def test_run_vmafexec_runner_yuv422p10le(self): 369 370 ref_path, dis_path, asset, asset_original = set_default_576_324_10bit_videos_for_testing() 371 372 self.runner = VmafexecQualityRunner( 373 [asset, asset_original], 374 None, fifo_mode=True, 375 delete_workdir=True, 376 result_store=None, 377 optional_dict={ 378 'float_psnr': True, 379 'float_ssim': True, 380 'float_ms_ssim': True, 381 } 382 ) 383 self.runner.run(parallelize=True) 384 385 results = self.runner.results 386 387 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'],0.3636620625, places=4) 388 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674953125, places=4) 389 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631078125, places=4) 390 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200833333333, places=4) 391 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 392 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345148541666667, places=4) 393 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 30.780577083333331, places=4) 394 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.86322654166666657, places=4) 395 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9632406874999999, places=4) 396 397 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 398 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 399 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 400 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 401 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 402 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 403 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 72.0, places=4) 404 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 405 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 406 407 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890489583334, places=4) 408 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.94641666666666, places=4) 409 410 def test_run_vmafexec_runner_yuv420p10le_b(self): 411 412 ref_path, dis_path, asset, asset_original = set_default_576_324_10bit_videos_for_testing_b() 413 414 self.runner = VmafexecQualityRunner( 415 [asset, asset_original], 416 None, fifo_mode=True, 417 delete_workdir=True, 418 result_store=None, 419 optional_dict={ 420 'float_psnr': True, 421 'float_ssim': True, 422 'float_ms_ssim': True, 423 } 424 ) 425 self.runner.run(parallelize=True) 426 427 results = self.runner.results 428 429 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.4330893333333334, places=4) 430 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.830613, places=4) 431 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 432 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.945896, places=4) 433 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 434 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9517763333333334, places=4) 435 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 32.57143333333333, places=4) 436 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.8978630000000001, places=4) 437 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9747490000000001, places=4) 438 439 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 440 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 441 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 442 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 443 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 444 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 445 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 72.0, places=4) 446 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 447 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 448 449 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 82.56523033333333, places=4) 450 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.142826, places=4) 451 452 def test_run_vmafexec_runner_yuv420p12le(self): 453 454 ref_path, dis_path, asset, asset_original = set_default_576_324_12bit_videos_for_testing() 455 456 self.runner = VmafexecQualityRunner( 457 [asset, asset_original], 458 None, fifo_mode=True, 459 delete_workdir=True, 460 result_store=None, 461 optional_dict={ 462 'float_psnr': True, 463 'float_ssim': True, 464 'float_ms_ssim': True, 465 } 466 ) 467 self.runner.run(parallelize=True) 468 469 results = self.runner.results 470 471 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'],0.4330893333333334, places=4) 472 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.830613, places=4) 473 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 474 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.945896, places=4) 475 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 476 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9517763333333334, places=4) 477 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 32.577818, places=4) 478 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.8978630000000001, places=4) 479 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9747490000000001, places=4) 480 481 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 482 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 483 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 484 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 485 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 486 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 487 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 84.0, places=4) 488 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 489 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 490 491 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 82.56523033333333, places=4) 492 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.142826, places=4) 493 494 def test_run_vmafexec_runner_yuv420p16le(self): 495 496 ref_path, dis_path, asset, asset_original = set_default_576_324_16bit_videos_for_testing() 497 498 self.runner = VmafexecQualityRunner( 499 [asset, asset_original], 500 None, fifo_mode=True, 501 delete_workdir=True, 502 result_store=None, 503 optional_dict={ 504 'float_psnr': True, 505 'float_ssim': True, 506 'float_ms_ssim': True, 507 } 508 ) 509 self.runner.run(parallelize=True) 510 511 results = self.runner.results 512 513 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'],0.4330893333333334, places=4) 514 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.830613, places=4) 515 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.9072123333333333, places=4) 516 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.945896, places=4) 517 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 518 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9517763333333334, places=4) 519 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 32.579806000000005, places=4) 520 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.8978630000000001, places=4) 521 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9747490000000001, places=4) 522 523 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 524 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999998541666666, places=4) 525 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999996041666667, places=4) 526 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999991458333334, places=4) 527 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 2.8104600000000004, places=4) 528 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 529 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 108.0, places=4) 530 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 531 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 532 533 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 82.56523033333333, places=4) 534 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.142826, places=4) 535 536 def test_run_vmafexec_runner_yuv420p10le_sparks(self): 537 538 ref_path = VmafConfig.test_resource_path("yuv", "sparks_ref_480x270.yuv42010le.yuv") 539 dis_path = VmafConfig.test_resource_path("yuv", "sparks_dis_480x270.yuv42010le.yuv") 540 asset = Asset(dataset="test", content_id=0, asset_id=0, 541 workdir_root=VmafConfig.workdir_path(), 542 ref_path=ref_path, 543 dis_path=dis_path, 544 asset_dict={'width': 480, 'height': 270, 545 'yuv_type': 'yuv420p10le'}) 546 547 asset_original = Asset(dataset="test", content_id=0, asset_id=1, 548 workdir_root=VmafConfig.workdir_path(), 549 ref_path=ref_path, 550 dis_path=ref_path, 551 asset_dict={'width': 480, 'height': 270, 552 'yuv_type': 'yuv420p10le'}) 553 554 self.runner = VmafexecQualityRunner( 555 [asset, asset_original], 556 None, fifo_mode=True, 557 delete_workdir=True, 558 result_store=None, 559 optional_dict={ 560 'float_psnr': True, 561 'float_ssim': True, 562 'float_ms_ssim': True, 563 } 564 ) 565 self.runner.run(parallelize=True) 566 567 results = self.runner.results 568 569 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.9240746, places=4) 570 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.9968371999999999, places=4) 571 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.9987575999999999, places=4) 572 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9993221999999999, places=4) 573 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 0.7523685999999999, places=4) 574 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9981770000000001, places=4) 575 self.assertAlmostEqual(results[0]['VMAFEXEC_float_psnr_score'], 48.81622, places=4) 576 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ssim_score'], 0.99566, places=4) 577 self.assertAlmostEqual(results[0]['VMAFEXEC_float_ms_ssim_score'], 0.9993778000000001, places=4) 578 579 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 580 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'], 0.9999990000000001, places=4) 581 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'], 0.9999990000000001, places=4) 582 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.9999990000000001, places=4) 583 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 0.7523685999999999, places=4) 584 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 585 self.assertAlmostEqual(results[1]['VMAFEXEC_float_psnr_score'], 72.0, places=4) 586 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ssim_score'], 1.0, places=4) 587 self.assertAlmostEqual(results[1]['VMAFEXEC_float_ms_ssim_score'], 1.0, places=4) 588 589 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 97.90069380000001, places=4) 590 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 98.47175940000001, places=4) 591 592 def test_run_vmafexec_compare_directly_with_ossexec_420_8bit(self): 593 594 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 595 596 self.runner = VmafexecQualityRunner( 597 [asset, asset_original], 598 None, fifo_mode=True, 599 delete_workdir=True, 600 result_store=None, 601 optional_dict={ 602 'float_psnr': True, 603 'float_ssim': True, 604 'float_ms_ssim': True, 605 'psnr': True, 606 } 607 ) 608 self.runner.run(parallelize=True) 609 610 results_rc = self.runner.results 611 612 self.runner = VmafossExecQualityRunner( 613 [asset, asset_original], 614 None, fifo_mode=True, 615 delete_workdir=True, 616 result_store=None, 617 ) 618 self.runner.run(parallelize=True) 619 620 results = self.runner.results 621 622 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale0_score'], results[0]['VMAFOSSEXEC_vif_scale0_score'], places=5) 623 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale1_score'], results[0]['VMAFOSSEXEC_vif_scale1_score'], places=5) 624 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale2_score'], results[0]['VMAFOSSEXEC_vif_scale2_score'], places=5) 625 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale3_score'], results[0]['VMAFOSSEXEC_vif_scale3_score'], places=5) 626 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_motion2_score'], results[0]['VMAFOSSEXEC_motion2_score'], places=5) 627 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_adm2_score'], results[0]['VMAFOSSEXEC_adm2_score'], places=5) 628 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_psnr_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 629 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ssim_score'], results[0]['VMAFOSSEXEC_ssim_score'], places=5) 630 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ms_ssim_score'], results[0]['VMAFOSSEXEC_ms_ssim_score'], places=5) 631 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_y_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 632 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cb_score'], 38.4494410625, places=4) 633 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cr_score'], 40.99191027083334, places=4) 634 635 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale0_score'], results[1]['VMAFOSSEXEC_vif_scale0_score'], places=5) 636 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale1_score'], results[1]['VMAFOSSEXEC_vif_scale1_score'], places=5) 637 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale2_score'], results[1]['VMAFOSSEXEC_vif_scale2_score'], places=5) 638 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale3_score'], results[1]['VMAFOSSEXEC_vif_scale3_score'], places=5) 639 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_motion2_score'], results[1]['VMAFOSSEXEC_motion2_score'], places=5) 640 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_adm2_score'], results[1]['VMAFOSSEXEC_adm2_score'], places=5) 641 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_psnr_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 642 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ssim_score'], results[1]['VMAFOSSEXEC_ssim_score'], places=5) 643 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ms_ssim_score'], results[1]['VMAFOSSEXEC_ms_ssim_score'], places=5) 644 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_y_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 645 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cb_score'], 60.0, places=4) 646 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cr_score'], 60.0, places=4) 647 648 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_score'], results[0]['VMAFOSSEXEC_score'], places=4) 649 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_score'], results[1]['VMAFOSSEXEC_score'], places=4) 650 651 def test_run_vmafexec_compare_directly_with_ossexec_420_10bit(self): 652 653 ref_path = VmafConfig.test_resource_path("yuv", "sparks_ref_480x270.yuv42010le.yuv") 654 dis_path = VmafConfig.test_resource_path("yuv", "sparks_dis_480x270.yuv42010le.yuv") 655 asset = Asset(dataset="test", content_id=0, asset_id=0, 656 workdir_root=VmafConfig.workdir_path(), 657 ref_path=ref_path, 658 dis_path=dis_path, 659 asset_dict={'width': 480, 'height': 270, 660 'yuv_type': 'yuv420p10le'}) 661 662 asset_original = Asset(dataset="test", content_id=0, asset_id=1, 663 workdir_root=VmafConfig.workdir_path(), 664 ref_path=ref_path, 665 dis_path=ref_path, 666 asset_dict={'width': 480, 'height': 270, 667 'yuv_type': 'yuv420p10le'}) 668 669 self.runner = VmafexecQualityRunner( 670 [asset, asset_original], 671 None, fifo_mode=True, 672 delete_workdir=True, 673 result_store=None, 674 optional_dict={ 675 'float_psnr': True, 676 'float_ssim': True, 677 'float_ms_ssim': True, 678 'psnr': True, 679 } 680 ) 681 self.runner.run(parallelize=True) 682 683 results_rc = self.runner.results 684 685 self.runner = VmafossExecQualityRunner( 686 [asset, asset_original], 687 None, fifo_mode=True, 688 delete_workdir=True, 689 result_store=None, 690 ) 691 self.runner.run(parallelize=True) 692 693 results = self.runner.results 694 695 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale0_score'], results[0]['VMAFOSSEXEC_vif_scale0_score'], places=5) 696 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale1_score'], results[0]['VMAFOSSEXEC_vif_scale1_score'], places=5) 697 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale2_score'], results[0]['VMAFOSSEXEC_vif_scale2_score'], places=5) 698 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale3_score'], results[0]['VMAFOSSEXEC_vif_scale3_score'], places=5) 699 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_motion2_score'], results[0]['VMAFOSSEXEC_motion2_score'], places=5) 700 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_adm2_score'], results[0]['VMAFOSSEXEC_adm2_score'], places=5) 701 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_psnr_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 702 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ssim_score'], results[0]['VMAFOSSEXEC_ssim_score'], places=5) 703 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ms_ssim_score'], results[0]['VMAFOSSEXEC_ms_ssim_score'], places=5) 704 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_y_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 705 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cb_score'], 50.2879106, places=4) 706 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cr_score'], 49.740259, places=4) 707 708 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale0_score'], results[1]['VMAFOSSEXEC_vif_scale0_score'], places=5) 709 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale1_score'], results[1]['VMAFOSSEXEC_vif_scale1_score'], places=5) 710 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale2_score'], results[1]['VMAFOSSEXEC_vif_scale2_score'], places=5) 711 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale3_score'], results[1]['VMAFOSSEXEC_vif_scale3_score'], places=5) 712 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_motion2_score'], results[1]['VMAFOSSEXEC_motion2_score'], places=5) 713 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_adm2_score'], results[1]['VMAFOSSEXEC_adm2_score'], places=5) 714 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_psnr_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 715 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ssim_score'], results[1]['VMAFOSSEXEC_ssim_score'], places=5) 716 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ms_ssim_score'], results[1]['VMAFOSSEXEC_ms_ssim_score'], places=5) 717 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_y_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 718 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cb_score'], 72.0, places=4) 719 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cr_score'], 72.0, places=4) 720 721 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_score'], results[0]['VMAFOSSEXEC_score'], places=4) 722 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_score'], results[1]['VMAFOSSEXEC_score'], places=4) 723 724 def test_run_vmafexec_compare_directly_with_ossexec_422_10bit(self): 725 726 ref_path, dis_path, asset, asset_original = set_default_576_324_10bit_videos_for_testing() 727 728 self.runner = VmafexecQualityRunner( 729 [asset, asset_original], 730 None, fifo_mode=True, 731 delete_workdir=True, 732 result_store=None, 733 optional_dict={ 734 'float_psnr': True, 735 'float_ssim': True, 736 'float_ms_ssim': True, 737 'psnr': True, 738 } 739 ) 740 self.runner.run(parallelize=True) 741 742 results_rc = self.runner.results 743 744 self.runner = VmafossExecQualityRunner( 745 [asset, asset_original], 746 None, fifo_mode=True, 747 delete_workdir=True, 748 result_store=None, 749 ) 750 self.runner.run(parallelize=True) 751 752 results = self.runner.results 753 754 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale0_score'], results[0]['VMAFOSSEXEC_vif_scale0_score'], places=5) 755 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale1_score'], results[0]['VMAFOSSEXEC_vif_scale1_score'], places=5) 756 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale2_score'], results[0]['VMAFOSSEXEC_vif_scale2_score'], places=5) 757 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_vif_scale3_score'], results[0]['VMAFOSSEXEC_vif_scale3_score'], places=5) 758 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_motion2_score'], results[0]['VMAFOSSEXEC_motion2_score'], places=5) 759 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_adm2_score'], results[0]['VMAFOSSEXEC_adm2_score'], places=5) 760 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_psnr_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 761 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ssim_score'], results[0]['VMAFOSSEXEC_ssim_score'], places=5) 762 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_float_ms_ssim_score'], results[0]['VMAFOSSEXEC_ms_ssim_score'], places=5) 763 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_y_score'], results[0]['VMAFOSSEXEC_psnr_score'], places=4) 764 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cb_score'], 38.7698320625, places=4) 765 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_psnr_cr_score'], 41.284188416666666, places=4) 766 767 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale0_score'], results[1]['VMAFOSSEXEC_vif_scale0_score'], places=5) 768 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale1_score'], results[1]['VMAFOSSEXEC_vif_scale1_score'], places=5) 769 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale2_score'], results[1]['VMAFOSSEXEC_vif_scale2_score'], places=5) 770 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_vif_scale3_score'], results[1]['VMAFOSSEXEC_vif_scale3_score'], places=5) 771 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_motion2_score'], results[1]['VMAFOSSEXEC_motion2_score'], places=5) 772 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_adm2_score'], results[1]['VMAFOSSEXEC_adm2_score'], places=5) 773 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_psnr_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 774 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ssim_score'], results[1]['VMAFOSSEXEC_ssim_score'], places=5) 775 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_float_ms_ssim_score'], results[1]['VMAFOSSEXEC_ms_ssim_score'], places=5) 776 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_y_score'], results[1]['VMAFOSSEXEC_psnr_score'], places=4) 777 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cb_score'], 72.0, places=4) 778 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_psnr_cr_score'], 72.0, places=4) 779 780 self.assertAlmostEqual(results_rc[0]['VMAFEXEC_score'], results[0]['VMAFOSSEXEC_score'], places=4) 781 self.assertAlmostEqual(results_rc[1]['VMAFEXEC_score'], results[1]['VMAFOSSEXEC_score'], places=4) 782 783 def test_run_vmafexec_runner_float_moment(self): 784 785 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 786 787 self.runner = VmafexecQualityRunner( 788 [asset, asset_original], 789 None, fifo_mode=False, 790 delete_workdir=True, 791 result_store=None, 792 optional_dict={ 793 'float_moment': True, 794 'no_prediction': True, 795 } 796 ) 797 self.runner.run(parallelize=False) 798 799 results = self.runner.results 800 801 self.assertAlmostEqual(results[0]['VMAFEXEC_float_moment_ref1st_score'], 59.788567354166666, places=4) 802 self.assertAlmostEqual(results[0]['VMAFEXEC_float_moment_dis1st_score'], 61.332006625, places=4) 803 self.assertAlmostEqual(results[0]['VMAFEXEC_float_moment_ref2nd_score'], 4696.668388125, places=4) 804 self.assertAlmostEqual(results[0]['VMAFEXEC_float_moment_dis2nd_score'], 4798.659574041666, places=4) 805 806 def test_run_vmafexec_runner_akiyo_multiply(self): 807 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 808 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 809 asset = Asset(dataset="test", content_id=0, asset_id=0, 810 workdir_root=VmafConfig.workdir_path(), 811 ref_path=ref_path, 812 dis_path=dis_path, 813 asset_dict={'width': 352, 'height': 288}) 814 815 self.runner = VmafexecQualityRunner( 816 [asset], 817 None, fifo_mode=True, 818 delete_workdir=True, 819 result_store=None, 820 optional_dict={'disable_clip_score': True} 821 ) 822 self.runner.run(parallelize=True) 823 824 results = self.runner.results 825 826 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 1.116691, places=4) # 1.116691484215469 827 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 1.052403, places=4) # 1.0522544319369052 828 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 1.070149, places=4) # 1.0705609423182443 829 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 1.072518, places=4) # 1.0731529493098957 830 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 1.072512, places=4) # 1.0728060231246508 831 832 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 132.732952, places=3) # 132.78849246495625 833 834 def test_run_vmafexec_runner_akiyo_multiply_with_feature_enhn_gain_limit(self): 835 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 836 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 837 asset = Asset(dataset="test", content_id=0, asset_id=0, 838 workdir_root=VmafConfig.workdir_path(), 839 ref_path=ref_path, 840 dis_path=dis_path, 841 asset_dict={'width': 352, 'height': 288}) 842 843 self.runner = VmafexecQualityRunner( 844 [asset], 845 None, fifo_mode=True, 846 delete_workdir=True, 847 result_store=None, 848 optional_dict={'disable_clip_score': True, 'adm_enhn_gain_limit': 1.0, 'vif_enhn_gain_limit': 1.0} 849 ) 850 self.runner.run(parallelize=True) 851 852 results = self.runner.results 853 854 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 855 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 856 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 857 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 858 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 859 860 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 88.030463, places=4) # 132.78849246495625 861 862 def test_run_vmafexec_runner_akiyo_multiply_with_feature_enhn_gain_limit_custom(self): 863 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 864 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 865 asset = Asset(dataset="test", content_id=0, asset_id=0, 866 workdir_root=VmafConfig.workdir_path(), 867 ref_path=ref_path, 868 dis_path=dis_path, 869 asset_dict={'width': 352, 'height': 288}) 870 871 self.runner = VmafexecQualityRunner( 872 [asset], 873 None, fifo_mode=True, 874 delete_workdir=True, 875 result_store=None, 876 optional_dict={'disable_clip_score': True, 'adm_enhn_gain_limit': 1.2, 'vif_enhn_gain_limit': 1.1} 877 ) 878 self.runner.run(parallelize=True) 879 880 results = self.runner.results 881 882 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1.2_score'], 1.116595, places=4) # 1.116691484215469 883 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1.1_score'], 1.029842, places=4) # 1.0522544319369052 884 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1.1_score'], 1.046578, places=4) # 1.0705609423182443 885 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1.1_score'], 1.04852, places=4) # 1.0731529493098957 886 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1.1_score'], 1.04892, places=4) # 1.0728060231246508 887 888 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 129.474226, places=3) # 132.78849246495625 889 890 def test_run_vmafexec_runner_akiyo_multiply_disable_enhn_gain(self): 891 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 892 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 893 asset = Asset(dataset="test", content_id=0, asset_id=0, 894 workdir_root=VmafConfig.workdir_path(), 895 ref_path=ref_path, 896 dis_path=dis_path, 897 asset_dict={'width': 352, 'height': 288}) 898 899 self.runner = VmafexecQualityRunner( 900 [asset], 901 None, fifo_mode=True, 902 delete_workdir=True, 903 result_store=None, 904 optional_dict={'disable_clip_score': True, 'disable_enhn_gain': True} 905 ) 906 self.runner.run(parallelize=True) 907 908 results = self.runner.results 909 910 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 911 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 912 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 913 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 914 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 915 916 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 88.030463, places=4) # 132.78849246495625 917 918 def test_run_vmafexec_runner_akiyo_multiply_no_enhn_gain_model(self): 919 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 920 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 921 asset = Asset(dataset="test", content_id=0, asset_id=0, 922 workdir_root=VmafConfig.workdir_path(), 923 ref_path=ref_path, 924 dis_path=dis_path, 925 asset_dict={'width': 352, 'height': 288}) 926 927 self.runner = VmafexecQualityRunner( 928 [asset], 929 None, fifo_mode=True, 930 delete_workdir=True, 931 result_store=None, 932 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_v0.6.1neg.json")} 933 ) 934 self.runner.run(parallelize=True) 935 936 results = self.runner.results 937 938 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 939 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 940 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 941 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 942 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 943 944 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 88.030463, places=4) # 132.78849246495625 945 946 def test_run_vmafexec_runner_akiyo_multiply_no_enhn_gain_model_and_cmd_options(self): 947 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 948 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 949 asset = Asset(dataset="test", content_id=0, asset_id=0, 950 workdir_root=VmafConfig.workdir_path(), 951 ref_path=ref_path, 952 dis_path=dis_path, 953 asset_dict={'width': 352, 'height': 288}) 954 955 self.runner = VmafexecQualityRunner( 956 [asset], 957 None, fifo_mode=True, 958 delete_workdir=True, 959 result_store=None, 960 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_v0.6.1neg.json"), 961 'adm_enhn_gain_limit': 1.2 } 962 ) 963 self.runner.run(parallelize=False) 964 965 results = self.runner.results 966 967 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1.2_score'], 1.116595, places=4) # 1.116691484215469 968 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 969 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 970 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 971 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 972 973 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 122.804272, places=3) # 132.78849246495625 974 975 def test_run_vmafexec_runner_akiyo_multiply_no_enhn_gain_model_and_cmd_options_illegal(self): 976 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 977 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 978 asset = Asset(dataset="test", content_id=0, asset_id=0, 979 workdir_root=VmafConfig.workdir_path(), 980 ref_path=ref_path, 981 dis_path=dis_path, 982 asset_dict={'width': 352, 'height': 288}) 983 984 self.runner = VmafexecQualityRunner( 985 [asset], 986 None, fifo_mode=False, 987 delete_workdir=True, 988 result_store=None, 989 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_float_v0.6.1neg.json"), 990 'adm_enhn_gain_limit': 0.9} 991 ) 992 with self.assertRaises(AssertionError, 993 msg="adm_enhn_gain_limit is below 1 but the exception is not raised"): 994 self.runner.run(parallelize=False) 995 996 def test_run_vmafexec_runner_akiyo_multiply_no_enhn_gain_model_and_cmd_options_illegal2(self): 997 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 998 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 999 asset = Asset(dataset="test", content_id=0, asset_id=0, 1000 workdir_root=VmafConfig.workdir_path(), 1001 ref_path=ref_path, 1002 dis_path=dis_path, 1003 asset_dict={'width': 352, 'height': 288}) 1004 1005 self.runner = VmafexecQualityRunner( 1006 [asset], 1007 None, fifo_mode=False, 1008 delete_workdir=True, 1009 result_store=None, 1010 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_float_v0.6.1neg.json"), 1011 'vif_enhn_gain_limit': 0.9} 1012 ) 1013 with self.assertRaises(AssertionError, 1014 msg="vif_enhn_gain_limit is below 1 but the exception is not raised"): 1015 self.runner.run(parallelize=False) 1016 1017 def test_run_vmafexec_runner_akiyo_multiply_with_feature_enhn_gain_limit_b_v063(self): 1018 ref_path = VmafConfig.test_resource_path("yuv", "refp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 1019 dis_path = VmafConfig.test_resource_path("yuv", "disp_vmaf_hacking_investigation_0_0_akiyo_cif_notyuv_0to0_identity_vs_akiyo_cif_notyuv_0to0_multiply_q_352x288") 1020 asset = Asset(dataset="test", content_id=0, asset_id=0, 1021 workdir_root=VmafConfig.workdir_path(), 1022 ref_path=ref_path, 1023 dis_path=dis_path, 1024 asset_dict={'width': 352, 'height': 288}) 1025 1026 self.runner = VmafexecQualityRunner( 1027 [asset], 1028 None, fifo_mode=False, 1029 delete_workdir=True, 1030 result_store=None, 1031 optional_dict={'disable_clip_score': True, 1032 'model_filepath': VmafConfig.model_path("vmaf_float_b_v0.6.3", "vmaf_float_b_v0.6.3.json"), 1033 'adm_enhn_gain_limit': 1.0, 'vif_enhn_gain_limit': 1.0} 1034 ) 1035 self.runner.run(parallelize=False) 1036 1037 results = self.runner.results 1038 1039 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_egl_1_score'], 0.9574308606115118, places=4) # 1.116691484215469 1040 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_egl_1_score'], 0.983699512450884, places=4) # 1.0522544319369052 1041 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_egl_1_score'], 0.9974276726830457, places=4) # 1.0705609423182443 1042 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_egl_1_score'], 0.9984692380091739, places=4) # 1.0731529493098957 1043 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_egl_1_score'], 0.999146211879154, places=4) # 1.0728060231246508 1044 1045 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 88.4895, places=2) # 88.032956 1046 1047 def test_run_vmafexec_runner_use_default_built_in_model(self): 1048 1049 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 1050 1051 self.runner = VmafexecQualityRunner( 1052 [asset, asset_original], 1053 None, fifo_mode=True, 1054 delete_workdir=True, 1055 result_store=None, 1056 optional_dict={ 1057 'use_default_built_in_model': True 1058 } 1059 ) 1060 self.runner.run(parallelize=True) 1061 1062 results = self.runner.results 1063 1064 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620710647402, places=4) 1065 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674952820232231, places=4) 1066 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631077727416296, places=4) 1067 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200890843669, places=4) 1068 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 1069 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345149030293786, places=4) 1070 1071 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale0_score'], 1.0, places=4) 1072 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale1_score'],0.99999972612, places=4) 1073 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale2_score'],0.999999465724, places=4) 1074 self.assertAlmostEqual(results[1]['VMAFEXEC_vif_scale3_score'], 0.999999399683, places=4) 1075 self.assertAlmostEqual(results[1]['VMAFEXEC_motion2_score'], 3.8953518541666665, places=4) 1076 self.assertAlmostEqual(results[1]['VMAFEXEC_adm2_score'], 1.0, places=4) 1077 1078 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 76.66890519623612, places=4) 1079 self.assertAlmostEqual(results[1]['VMAFEXEC_score'], 99.946416604585025, places=4) 1080 1081 def test_run_vmafexec_runner_akiyo_multiply_4k_model(self): 1082 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 1083 1084 self.runner = VmafexecQualityRunner( 1085 [asset], 1086 None, fifo_mode=True, 1087 delete_workdir=True, 1088 result_store=None, 1089 optional_dict={'disable_clip_score': True, 'model_filepath': VmafConfig.model_path("vmaf_4k_v0.6.1.json")} 1090 ) 1091 self.runner.run(parallelize=True) 1092 1093 results = self.runner.results 1094 1095 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.3636620625, places=4) 1096 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.7674953125, places=4) 1097 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.8631078125, places=4) 1098 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.9157200833333333, places=4) 1099 self.assertAlmostEqual(results[0]['VMAFEXEC_motion2_score'], 3.895352291666667, places=4) 1100 self.assertAlmostEqual(results[0]['VMAFEXEC_adm2_score'], 0.9345148541666667, places=4) 1101 1102 self.assertAlmostEqual(results[0]['VMAFEXEC_score'], 84.95064735416668, places=4) 1103 1104 def test_run_vmaf_runner_with_transform_score(self): 1105 1106 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 1107 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 1108 asset = Asset(dataset="test", content_id=0, asset_id=0, 1109 workdir_root=VmafConfig.workdir_path(), 1110 ref_path=ref_path, 1111 dis_path=dis_path, 1112 asset_dict={'width': 1920, 'height': 1080}) 1113 1114 self.runner = VmafexecQualityRunner( 1115 [asset], 1116 None, fifo_mode=True, 1117 delete_workdir=True, 1118 optional_dict={ 1119 'models': [ 1120 'path={}:name=vmat:enable_transform'.format(VmafConfig.test_resource_path("test_model_transform_add40.json")), 1121 ], 1122 }, 1123 result_store=self.result_store, 1124 ) 1125 self.runner.run(parallelize=True) 1126 1127 results = self.runner.results 1128 1129 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.0, places=4) 1130 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.0, places=4) 1131 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.0, places=4) 1132 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.0, places=4) 1133 self.assertAlmostEqual(results[0]['VMAFEXEC_motion_score'], 12.5548366667, places=4) 1134 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 1135 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 1136 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 1137 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 1138 1139 self.assertAlmostEqual(results[0]['VMAFEXEC_vmat_score'], 32.757433750978919, places=4) 1140 1141 def test_run_vmaf_runner_with_transform_score_2(self): 1142 1143 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 1144 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 1145 asset = Asset(dataset="test", content_id=0, asset_id=0, 1146 workdir_root=VmafConfig.workdir_path(), 1147 ref_path=ref_path, 1148 dis_path=dis_path, 1149 asset_dict={'width': 1920, 'height': 1080}) 1150 1151 self.runner = VmafexecQualityRunner( 1152 [asset], 1153 None, fifo_mode=True, 1154 delete_workdir=True, 1155 optional_dict={ 1156 'models': [ 1157 'path={}:name=vmat:enable_transform:disable_clip'.format(VmafConfig.test_resource_path("test_model_transform_add40_outltein.json")), 1158 ], 1159 }, 1160 result_store=self.result_store, 1161 ) 1162 self.runner.run(parallelize=True) 1163 1164 results = self.runner.results 1165 1166 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.0, places=4) 1167 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.0, places=4) 1168 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.0, places=4) 1169 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.0, places=4) 1170 self.assertAlmostEqual(results[0]['VMAFEXEC_motion_score'], 12.5548366667, places=4) 1171 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 1172 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 1173 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 1174 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 1175 1176 self.assertAlmostEqual(results[0]['VMAFEXEC_vmat_score'], -7.2425662490210838, places=4) 1177 1178 def test_run_vmaf_runner_with_transform_score3(self): 1179 1180 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 1181 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 1182 asset = Asset(dataset="test", content_id=0, asset_id=0, 1183 workdir_root=VmafConfig.workdir_path(), 1184 ref_path=ref_path, 1185 dis_path=dis_path, 1186 asset_dict={'width': 1920, 'height': 1080}) 1187 1188 self.runner = VmafexecQualityRunner( 1189 [asset], 1190 None, fifo_mode=True, 1191 delete_workdir=True, 1192 optional_dict={ 1193 'models': [ 1194 'path={}:name=vmat:enable_transform'.format(VmafConfig.test_resource_path("test_model_transform_add40_piecewiselinear.json")), 1195 ], 1196 }, 1197 result_store=self.result_store, 1198 ) 1199 self.runner.run(parallelize=True) 1200 1201 results = self.runner.results 1202 1203 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.0, places=4) 1204 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.0, places=4) 1205 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.0, places=4) 1206 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.0, places=4) 1207 self.assertAlmostEqual(results[0]['VMAFEXEC_motion_score'], 12.5548366667, places=4) 1208 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 1209 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 1210 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 1211 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 1212 1213 self.assertAlmostEqual(results[0]['VMAFEXEC_vmat_score'], 8.262602639723815, places=4) 1214 1215 def test_run_vmaf_runner_with_transform_score4(self): 1216 1217 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 1218 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 1219 asset = Asset(dataset="test", content_id=0, asset_id=0, 1220 workdir_root=VmafConfig.workdir_path(), 1221 ref_path=ref_path, 1222 dis_path=dis_path, 1223 asset_dict={'width': 1920, 'height': 1080}) 1224 1225 self.runner = VmafexecQualityRunner( 1226 [asset], 1227 None, fifo_mode=False, 1228 delete_workdir=True, 1229 optional_dict={ 1230 'models': [ 1231 'path={}:name=vmat'.format(VmafConfig.test_resource_path("test_model_transform_add40_piecewiselinear_forced.json")), 1232 ], 1233 }, 1234 result_store=self.result_store, 1235 ) 1236 self.runner.run(parallelize=False) 1237 1238 results = self.runner.results 1239 1240 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale0_score'], 0.0, places=4) 1241 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale1_score'], 0.0, places=4) 1242 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale2_score'], 0.0, places=4) 1243 self.assertAlmostEqual(results[0]['VMAFEXEC_vif_scale3_score'], 0.0, places=4) 1244 self.assertAlmostEqual(results[0]['VMAFEXEC_motion_score'], 12.5548366667, places=4) 1245 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale0_score'], 0.23738393128710478, places=4) 1246 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale1_score'], 0.08524788663335138, places=4) 1247 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale2_score'], 0.024058909404945077, places=4) 1248 self.assertAlmostEqual(results[0]['VMAFEXEC_adm_scale3_score'], 0.018034879735107798, places=4) 1249 1250 self.assertAlmostEqual(results[0]['VMAFEXEC_vmat_score'], 8.262602639723815, places=4) 1251 1252 def test_run_vmaf_runner_with_transform_score_both_specified(self): 1253 1254 ref_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_0_0.yuv") 1255 dis_path = VmafConfig.test_resource_path("yuv", "checkerboard_1920_1080_10_3_10_0.yuv") 1256 asset = Asset(dataset="test", content_id=0, asset_id=0, 1257 workdir_root=VmafConfig.workdir_path(), 1258 ref_path=ref_path, 1259 dis_path=dis_path, 1260 asset_dict={'width': 1920, 'height': 1080}) 1261 1262 self.runner = VmafexecQualityRunner( 1263 [asset], 1264 None, fifo_mode=False, 1265 delete_workdir=True, 1266 optional_dict={ 1267 'models': [ 1268 'path={}:name=vmat:enable_transform'.format(VmafConfig.test_resource_path("test_model_transform_add40_piecewiselinear_forced.json")), 1269 ], 1270 }, 1271 result_store=self.result_store, 1272 ) 1273 self.runner.run(parallelize=False) 1274 1275 results = self.runner.results 1276 1277 self.assertAlmostEqual(results[0]['VMAFEXEC_vmat_score'], 8.262602639723815, places=4) 1278 1279 1280class VmafexecQualityRunnerSubsamplingTest(MyTestCase): 1281 1282 def setUp(self): 1283 super().setUp() 1284 self.result_store = FileSystemResultStore() 1285 1286 def tearDown(self): 1287 if hasattr(self, 'runner0'): 1288 self.runner0.remove_results() 1289 if hasattr(self, 'runner'): 1290 self.runner.remove_results() 1291 super().tearDown() 1292 1293 def test_run_vmafexec_runner_with_subsample2(self): 1294 1295 ref_path, dis_path, asset, asset_original = set_default_576_324_videos_for_testing() 1296 1297 subsample = 5 1298 1299 self.runner0 = VmafexecQualityRunner( 1300 [asset, asset_original], 1301 None, fifo_mode=True, 1302 delete_workdir=True, 1303 result_store=None, 1304 optional_dict={} 1305 ) 1306 self.runner0.run() 1307 results0 = self.runner0.results 1308 1309 self.runner = VmafexecQualityRunner( 1310 [asset, asset_original], 1311 None, fifo_mode=True, 1312 delete_workdir=True, 1313 result_store=None, 1314 optional_dict={'subsample': subsample} 1315 ) 1316 self.runner.run(parallelize=True) 1317 results = self.runner.results 1318 1319 for i in range(48): 1320 if i % subsample == 0: 1321 self.assertAlmostEqual(results0[0]['VMAFEXEC_scores'][i], results[0]['VMAFEXEC_scores'][i // subsample], places=7) 1322 self.assertAlmostEqual(results0[1]['VMAFEXEC_scores'][i], results[1]['VMAFEXEC_scores'][i // subsample], places=7) 1323 1324 1325class QualityRunnerVersionTest(unittest.TestCase): 1326 1327 def test_vmafexec_quality_runner_version(self): 1328 self.assertEqual(VmafexecQualityRunner.VERSION, 'F0.2.7-0.6.1') 1329 self.assertEqual(VmafexecQualityRunner.ALGO_VERSION, 2) 1330 1331 1332if __name__ == '__main__': 1333 unittest.main(verbosity=2) 1334