1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include "third_party/googletest/src/googletest/include/gtest/gtest.h"
13 #include "test/codec_factory.h"
14 #include "test/encode_test_driver.h"
15 #include "test/i420_video_source.h"
16 #include "test/util.h"
17
18 namespace {
19
20 const int kMaxErrorFrames = 12;
21 const int kMaxDroppableFrames = 12;
22
23 class ErrorResilienceTestLarge
24 : public ::libaom_test::CodecTestWithParam<libaom_test::TestMode>,
25 public ::libaom_test::EncoderTest {
26 protected:
ErrorResilienceTestLarge()27 ErrorResilienceTestLarge()
28 : EncoderTest(GET_PARAM(0)), psnr_(0.0), nframes_(0), mismatch_psnr_(0.0),
29 mismatch_nframes_(0), encoding_mode_(GET_PARAM(1)) {
30 Reset();
31 }
32
~ErrorResilienceTestLarge()33 virtual ~ErrorResilienceTestLarge() {}
34
Reset()35 void Reset() {
36 error_nframes_ = 0;
37 droppable_nframes_ = 0;
38 pattern_switch_ = 0;
39 }
40
SetUp()41 virtual void SetUp() {
42 InitializeConfig();
43 SetMode(encoding_mode_);
44 }
45
BeginPassHook(unsigned int)46 virtual void BeginPassHook(unsigned int /*pass*/) {
47 psnr_ = 0.0;
48 nframes_ = 0;
49 mismatch_psnr_ = 0.0;
50 mismatch_nframes_ = 0;
51 }
52
PSNRPktHook(const aom_codec_cx_pkt_t * pkt)53 virtual void PSNRPktHook(const aom_codec_cx_pkt_t *pkt) {
54 psnr_ += pkt->data.psnr.psnr[0];
55 nframes_++;
56 }
57
PreEncodeFrameHook(libaom_test::VideoSource * video)58 virtual void PreEncodeFrameHook(libaom_test::VideoSource *video) {
59 frame_flags_ &=
60 ~(AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF | AOM_EFLAG_NO_UPD_ARF);
61 if (droppable_nframes_ > 0 &&
62 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
63 for (unsigned int i = 0; i < droppable_nframes_; ++i) {
64 if (droppable_frames_[i] == video->frame()) {
65 std::cout << "Encoding droppable frame: " << droppable_frames_[i]
66 << "\n";
67 frame_flags_ |= (AOM_EFLAG_NO_UPD_LAST | AOM_EFLAG_NO_UPD_GF |
68 AOM_EFLAG_NO_UPD_ARF);
69 return;
70 }
71 }
72 }
73 }
74
GetAveragePsnr() const75 double GetAveragePsnr() const {
76 if (nframes_) return psnr_ / nframes_;
77 return 0.0;
78 }
79
GetAverageMismatchPsnr() const80 double GetAverageMismatchPsnr() const {
81 if (mismatch_nframes_) return mismatch_psnr_ / mismatch_nframes_;
82 return 0.0;
83 }
84
DoDecode() const85 virtual bool DoDecode() const {
86 if (error_nframes_ > 0 &&
87 (cfg_.g_pass == AOM_RC_LAST_PASS || cfg_.g_pass == AOM_RC_ONE_PASS)) {
88 for (unsigned int i = 0; i < error_nframes_; ++i) {
89 if (error_frames_[i] == nframes_ - 1) {
90 std::cout << " Skipping decoding frame: "
91 << error_frames_[i] << "\n";
92 return 0;
93 }
94 }
95 }
96 return 1;
97 }
98
MismatchHook(const aom_image_t * img1,const aom_image_t * img2)99 virtual void MismatchHook(const aom_image_t *img1, const aom_image_t *img2) {
100 double mismatch_psnr = compute_psnr(img1, img2);
101 mismatch_psnr_ += mismatch_psnr;
102 ++mismatch_nframes_;
103 // std::cout << "Mismatch frame psnr: " << mismatch_psnr << "\n";
104 ::libaom_test::EncoderTest::MismatchHook(img1, img2);
105 }
106
SetErrorFrames(int num,unsigned int * list)107 void SetErrorFrames(int num, unsigned int *list) {
108 if (num > kMaxErrorFrames)
109 num = kMaxErrorFrames;
110 else if (num < 0)
111 num = 0;
112 error_nframes_ = num;
113 for (unsigned int i = 0; i < error_nframes_; ++i)
114 error_frames_[i] = list[i];
115 }
116
SetDroppableFrames(int num,unsigned int * list)117 void SetDroppableFrames(int num, unsigned int *list) {
118 if (num > kMaxDroppableFrames)
119 num = kMaxDroppableFrames;
120 else if (num < 0)
121 num = 0;
122 droppable_nframes_ = num;
123 for (unsigned int i = 0; i < droppable_nframes_; ++i)
124 droppable_frames_[i] = list[i];
125 }
126
GetMismatchFrames()127 unsigned int GetMismatchFrames() { return mismatch_nframes_; }
128
SetPatternSwitch(int frame_switch)129 void SetPatternSwitch(int frame_switch) { pattern_switch_ = frame_switch; }
130
131 private:
132 double psnr_;
133 unsigned int nframes_;
134 unsigned int error_nframes_;
135 unsigned int droppable_nframes_;
136 unsigned int pattern_switch_;
137 double mismatch_psnr_;
138 unsigned int mismatch_nframes_;
139 unsigned int error_frames_[kMaxErrorFrames];
140 unsigned int droppable_frames_[kMaxDroppableFrames];
141 libaom_test::TestMode encoding_mode_;
142 };
143
TEST_P(ErrorResilienceTestLarge,OnVersusOff)144 TEST_P(ErrorResilienceTestLarge, OnVersusOff) {
145 const aom_rational timebase = { 33333333, 1000000000 };
146 cfg_.g_timebase = timebase;
147 cfg_.rc_target_bitrate = 2000;
148 cfg_.g_lag_in_frames = 10;
149
150 init_flags_ = AOM_CODEC_USE_PSNR;
151
152 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
153 timebase.den, timebase.num, 0, 12);
154
155 // Error resilient mode OFF.
156 cfg_.g_error_resilient = 0;
157 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
158 const double psnr_resilience_off = GetAveragePsnr();
159 EXPECT_GT(psnr_resilience_off, 25.0);
160
161 // Error resilient mode ON.
162 cfg_.g_error_resilient = 1;
163 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
164 const double psnr_resilience_on = GetAveragePsnr();
165 EXPECT_GT(psnr_resilience_on, 25.0);
166
167 // Test that turning on error resilient mode hurts by 10% at most.
168 if (psnr_resilience_off > 0.0) {
169 const double psnr_ratio = psnr_resilience_on / psnr_resilience_off;
170 EXPECT_GE(psnr_ratio, 0.9);
171 EXPECT_LE(psnr_ratio, 1.1);
172 }
173 }
174
175 // Check for successful decoding and no encoder/decoder mismatch
176 // if we lose (i.e., drop before decoding) a set of droppable
177 // frames (i.e., frames that don't update any reference buffers).
178 // Check both isolated and consecutive loss.
TEST_P(ErrorResilienceTestLarge,DropFramesWithoutRecovery)179 TEST_P(ErrorResilienceTestLarge, DropFramesWithoutRecovery) {
180 const aom_rational timebase = { 33333333, 1000000000 };
181 cfg_.g_timebase = timebase;
182 cfg_.rc_target_bitrate = 500;
183 // FIXME(debargha): Fix this to work for any lag.
184 // Currently this test only works for lag = 0
185 cfg_.g_lag_in_frames = 0;
186
187 init_flags_ = AOM_CODEC_USE_PSNR;
188
189 libaom_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288,
190 timebase.den, timebase.num, 0, 20);
191
192 // Error resilient mode ON.
193 cfg_.g_error_resilient = 1;
194 cfg_.kf_mode = AOM_KF_DISABLED;
195
196 // Set an arbitrary set of error frames same as droppable frames.
197 // In addition to isolated loss/drop, add a long consecutive series
198 // (of size 9) of dropped frames.
199 unsigned int num_droppable_frames = 5;
200 unsigned int droppable_frame_list[] = { 5, 10, 13, 16, 19 };
201 SetDroppableFrames(num_droppable_frames, droppable_frame_list);
202 SetErrorFrames(num_droppable_frames, droppable_frame_list);
203 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
204 // Test that no mismatches have been found
205 std::cout << " Mismatch frames: " << GetMismatchFrames() << "\n";
206 EXPECT_EQ(GetMismatchFrames(), (unsigned int)0);
207
208 // Reset previously set of error/droppable frames.
209 Reset();
210
211 #if 0
212 // TODO(jkoleszar): This test is disabled for the time being as too
213 // sensitive. It's not clear how to set a reasonable threshold for
214 // this behavior.
215
216 // Now set an arbitrary set of error frames that are non-droppable
217 unsigned int num_error_frames = 3;
218 unsigned int error_frame_list[] = {3, 10, 20};
219 SetErrorFrames(num_error_frames, error_frame_list);
220 ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
221
222 // Test that dropping an arbitrary set of inter frames does not hurt too much
223 // Note the Average Mismatch PSNR is the average of the PSNR between
224 // decoded frame and encoder's version of the same frame for all frames
225 // with mismatch.
226 const double psnr_resilience_mismatch = GetAverageMismatchPsnr();
227 std::cout << " Mismatch PSNR: "
228 << psnr_resilience_mismatch << "\n";
229 EXPECT_GT(psnr_resilience_mismatch, 20.0);
230 #endif
231 }
232
233 AV1_INSTANTIATE_TEST_CASE(ErrorResilienceTestLarge, ONE_PASS_TEST_MODES);
234 } // namespace
235