1 /*
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "media/base/videoadapter.h"
12
13 #include <algorithm>
14 #include <cmath>
15 #include <cstdlib>
16 #include <limits>
17 #include <utility>
18
19 #include "api/optional.h"
20 #include "media/base/mediaconstants.h"
21 #include "media/base/videocommon.h"
22 #include "rtc_base/arraysize.h"
23 #include "rtc_base/checks.h"
24 #include "rtc_base/logging.h"
25
26 namespace {
27 struct Fraction {
28 int numerator;
29 int denominator;
30
31 // Determines number of output pixels if both width and height of an input of
32 // |input_pixels| pixels is scaled with the fraction numerator / denominator.
scale_pixel_count__anon1d1e83ec0111::Fraction33 int scale_pixel_count(int input_pixels) {
34 return (numerator * numerator * input_pixels) / (denominator * denominator);
35 }
36 };
37
38 // Round |value_to_round| to a multiple of |multiple|. Prefer rounding upwards,
39 // but never more than |max_value|.
roundUp(int value_to_round,int multiple,int max_value)40 int roundUp(int value_to_round, int multiple, int max_value) {
41 const int rounded_value =
42 (value_to_round + multiple - 1) / multiple * multiple;
43 return rounded_value <= max_value ? rounded_value
44 : (max_value / multiple * multiple);
45 }
46
47 // Generates a scale factor that makes |input_pixels| close to |target_pixels|,
48 // but no higher than |max_pixels|.
FindScale(int input_pixels,int target_pixels,int max_pixels)49 Fraction FindScale(int input_pixels, int target_pixels, int max_pixels) {
50 // This function only makes sense for a positive target.
51 RTC_DCHECK_GT(target_pixels, 0);
52 RTC_DCHECK_GT(max_pixels, 0);
53 RTC_DCHECK_GE(max_pixels, target_pixels);
54
55 // Don't scale up original.
56 if (target_pixels >= input_pixels)
57 return Fraction{1, 1};
58
59 Fraction current_scale = Fraction{1, 1};
60 Fraction best_scale = Fraction{1, 1};
61 // The minimum (absolute) difference between the number of output pixels and
62 // the target pixel count.
63 int min_pixel_diff = std::numeric_limits<int>::max();
64 if (input_pixels <= max_pixels) {
65 // Start condition for 1/1 case, if it is less than max.
66 min_pixel_diff = std::abs(input_pixels - target_pixels);
67 }
68
69 // Alternately scale down by 2/3 and 3/4. This results in fractions which are
70 // effectively scalable. For instance, starting at 1280x720 will result in
71 // the series (3/4) => 960x540, (1/2) => 640x360, (3/8) => 480x270,
72 // (1/4) => 320x180, (3/16) => 240x125, (1/8) => 160x90.
73 while (current_scale.scale_pixel_count(input_pixels) > target_pixels) {
74 if (current_scale.numerator % 3 == 0 &&
75 current_scale.denominator % 2 == 0) {
76 // Multiply by 2/3.
77 current_scale.numerator /= 3;
78 current_scale.denominator /= 2;
79 } else {
80 // Multiply by 3/4.
81 current_scale.numerator *= 3;
82 current_scale.denominator *= 4;
83 }
84
85 int output_pixels = current_scale.scale_pixel_count(input_pixels);
86 if (output_pixels <= max_pixels) {
87 int diff = std::abs(target_pixels - output_pixels);
88 if (diff < min_pixel_diff) {
89 min_pixel_diff = diff;
90 best_scale = current_scale;
91 }
92 }
93 }
94
95 return best_scale;
96 }
97 } // namespace
98
99 namespace cricket {
100
VideoAdapter(int required_resolution_alignment)101 VideoAdapter::VideoAdapter(int required_resolution_alignment)
102 : frames_in_(0),
103 frames_out_(0),
104 frames_scaled_(0),
105 adaption_changes_(0),
106 previous_width_(0),
107 previous_height_(0),
108 required_resolution_alignment_(required_resolution_alignment),
109 resolution_request_target_pixel_count_(std::numeric_limits<int>::max()),
110 resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
111 max_framerate_request_(std::numeric_limits<int>::max()) {}
112
VideoAdapter()113 VideoAdapter::VideoAdapter() : VideoAdapter(1) {}
114
~VideoAdapter()115 VideoAdapter::~VideoAdapter() {}
116
KeepFrame(int64_t in_timestamp_ns)117 bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
118 rtc::CritScope cs(&critical_section_);
119 if (max_framerate_request_ <= 0)
120 return false;
121
122 int64_t frame_interval_ns =
123 requested_format_ ? requested_format_->interval : 0;
124
125 // If |max_framerate_request_| is not set, it will default to maxint, which
126 // will lead to a frame_interval_ns rounded to 0.
127 frame_interval_ns = std::max<int64_t>(
128 frame_interval_ns, rtc::kNumNanosecsPerSec / max_framerate_request_);
129
130 if (frame_interval_ns <= 0) {
131 // Frame rate throttling not enabled.
132 return true;
133 }
134
135 if (next_frame_timestamp_ns_) {
136 // Time until next frame should be outputted.
137 const int64_t time_until_next_frame_ns =
138 (*next_frame_timestamp_ns_ - in_timestamp_ns);
139
140 // Continue if timestamp is within expected range.
141 if (std::abs(time_until_next_frame_ns) < 2 * frame_interval_ns) {
142 // Drop if a frame shouldn't be outputted yet.
143 if (time_until_next_frame_ns > 0)
144 return false;
145 // Time to output new frame.
146 *next_frame_timestamp_ns_ += frame_interval_ns;
147 return true;
148 }
149 }
150
151 // First timestamp received or timestamp is way outside expected range, so
152 // reset. Set first timestamp target to just half the interval to prefer
153 // keeping frames in case of jitter.
154 next_frame_timestamp_ns_ = in_timestamp_ns + frame_interval_ns / 2;
155 return true;
156 }
157
AdaptFrameResolution(int in_width,int in_height,int64_t in_timestamp_ns,int * cropped_width,int * cropped_height,int * out_width,int * out_height)158 bool VideoAdapter::AdaptFrameResolution(int in_width,
159 int in_height,
160 int64_t in_timestamp_ns,
161 int* cropped_width,
162 int* cropped_height,
163 int* out_width,
164 int* out_height) {
165 rtc::CritScope cs(&critical_section_);
166 ++frames_in_;
167
168 // The max output pixel count is the minimum of the requests from
169 // OnOutputFormatRequest and OnResolutionRequest.
170 int max_pixel_count = resolution_request_max_pixel_count_;
171 if (scale_) {
172 // We calculate the scaled pixel count from the in_width and in_height,
173 // which is the input resolution. We then take the minimum of the scaled
174 // resolution and the current max_pixel_count. This will allow the
175 // quality scaler to reduce the resolution in response to load, but we
176 // will never go above the requested scaled resolution.
177 int scaled_pixel_count = (in_width*in_height/scale_resolution_by_)/scale_resolution_by_;
178 max_pixel_count = std::min(max_pixel_count, scaled_pixel_count);
179 }
180
181 if (requested_format_) {
182 max_pixel_count = std::min(
183 max_pixel_count, requested_format_->width * requested_format_->height);
184 }
185 int target_pixel_count =
186 std::min(resolution_request_target_pixel_count_, max_pixel_count);
187
188 // Drop the input frame if necessary.
189 if (max_pixel_count <= 0 || !KeepFrame(in_timestamp_ns)) {
190 // Show VAdapt log every 90 frames dropped. (3 seconds)
191 if ((frames_in_ - frames_out_) % 90 == 0) {
192 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
193 // in default calls.
194 RTC_LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
195 << " / out " << frames_out_ << " / in " << frames_in_
196 << " Changes: " << adaption_changes_
197 << " Input: " << in_width << "x" << in_height
198 << " timestamp: " << in_timestamp_ns << " Output: i"
199 << (requested_format_ ? requested_format_->interval : 0);
200 }
201
202 // Drop frame.
203 return false;
204 }
205
206 // Calculate how the input should be cropped.
207 if (!requested_format_ ||
208 requested_format_->width == 0 || requested_format_->height == 0) {
209 *cropped_width = in_width;
210 *cropped_height = in_height;
211 } else {
212 // Adjust |requested_format_| orientation to match input.
213 if ((in_width > in_height) !=
214 (requested_format_->width > requested_format_->height)) {
215 std::swap(requested_format_->width, requested_format_->height);
216 }
217 const float requested_aspect =
218 requested_format_->width /
219 static_cast<float>(requested_format_->height);
220 *cropped_width =
221 std::min(in_width, static_cast<int>(in_height * requested_aspect));
222 *cropped_height =
223 std::min(in_height, static_cast<int>(in_width / requested_aspect));
224 }
225 const Fraction scale = FindScale((*cropped_width) * (*cropped_height),
226 target_pixel_count, max_pixel_count);
227 // Adjust cropping slightly to get even integer output size and a perfect
228 // scale factor. Make sure the resulting dimensions are aligned correctly
229 // to be nice to hardware encoders.
230 *cropped_width =
231 roundUp(*cropped_width,
232 scale.denominator * required_resolution_alignment_, in_width);
233 *cropped_height =
234 roundUp(*cropped_height,
235 scale.denominator * required_resolution_alignment_, in_height);
236 RTC_DCHECK_EQ(0, *cropped_width % scale.denominator);
237 RTC_DCHECK_EQ(0, *cropped_height % scale.denominator);
238
239 // Calculate final output size.
240 *out_width = *cropped_width / scale.denominator * scale.numerator;
241 *out_height = *cropped_height / scale.denominator * scale.numerator;
242 RTC_DCHECK_EQ(0, *out_width % required_resolution_alignment_);
243 RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_);
244
245 ++frames_out_;
246 if (scale.numerator != scale.denominator)
247 ++frames_scaled_;
248
249 if ((previous_width_ || scale_) && (previous_width_ != *out_width ||
250 previous_height_ != *out_height)) {
251 ++adaption_changes_;
252 RTC_LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_
253 << " / out " << frames_out_ << " / in " << frames_in_
254 << " Changes: " << adaption_changes_
255 << " Input: " << in_width << "x" << in_height
256 << " Scale: " << scale.numerator << "/"
257 << scale.denominator << " Output: " << *out_width << "x"
258 << *out_height << " i"
259 << (requested_format_ ? requested_format_->interval : 0);
260 }
261
262 previous_width_ = *out_width;
263 previous_height_ = *out_height;
264
265 return true;
266 }
267
OnOutputFormatRequest(const VideoFormat & format)268 void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
269 rtc::CritScope cs(&critical_section_);
270 requested_format_ = format;
271 next_frame_timestamp_ns_ = rtc::nullopt;
272 }
273
OnResolutionFramerateRequest(const rtc::Optional<int> & target_pixel_count,int max_pixel_count,int max_framerate_fps)274 void VideoAdapter::OnResolutionFramerateRequest(
275 const rtc::Optional<int>& target_pixel_count,
276 int max_pixel_count,
277 int max_framerate_fps) {
278 rtc::CritScope cs(&critical_section_);
279 resolution_request_max_pixel_count_ = max_pixel_count;
280 resolution_request_target_pixel_count_ =
281 target_pixel_count.value_or(resolution_request_max_pixel_count_);
282 max_framerate_request_ = max_framerate_fps;
283 }
284
OnScaleResolutionBy(rtc::Optional<float> scale_resolution_by)285 void VideoAdapter::OnScaleResolutionBy(
286 rtc::Optional<float> scale_resolution_by) {
287 rtc::CritScope cs(&critical_section_);
288 scale_resolution_by_ = scale_resolution_by.value_or(1.0);
289 RTC_DCHECK_GE(scale_resolution_by_, 1.0);
290 scale_ = static_cast<bool>(scale_resolution_by);
291 }
292
293 } // namespace cricket
294