1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/include/video_codec_initializer.h"
12 
13 #include <stdint.h>
14 #include <string.h>
15 
16 #include <algorithm>
17 
18 #include "absl/types/optional.h"
19 #include "api/scoped_refptr.h"
20 #include "api/units/data_rate.h"
21 #include "api/video/video_bitrate_allocation.h"
22 #include "api/video_codecs/video_encoder.h"
23 #include "modules/video_coding/codecs/vp9/svc_config.h"
24 #include "modules/video_coding/include/video_coding_defines.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/experiments/min_video_bitrate_experiment.h"
27 #include "rtc_base/logging.h"
28 #include "rtc_base/numerics/safe_conversions.h"
29 
30 namespace webrtc {
31 
SetupCodec(const VideoEncoderConfig & config,const std::vector<VideoStream> & streams,VideoCodec * codec)32 bool VideoCodecInitializer::SetupCodec(const VideoEncoderConfig& config,
33                                        const std::vector<VideoStream>& streams,
34                                        VideoCodec* codec) {
35   if (config.codec_type == kVideoCodecMultiplex) {
36     VideoEncoderConfig associated_config = config.Copy();
37     associated_config.codec_type = kVideoCodecVP9;
38     if (!SetupCodec(associated_config, streams, codec)) {
39       RTC_LOG(LS_ERROR) << "Failed to create stereo encoder configuration.";
40       return false;
41     }
42     codec->codecType = kVideoCodecMultiplex;
43     return true;
44   }
45 
46   *codec = VideoEncoderConfigToVideoCodec(config, streams);
47   return true;
48 }
49 
50 // TODO(sprang): Split this up and separate the codec specific parts.
VideoEncoderConfigToVideoCodec(const VideoEncoderConfig & config,const std::vector<VideoStream> & streams)51 VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
52     const VideoEncoderConfig& config,
53     const std::vector<VideoStream>& streams) {
54   static const int kEncoderMinBitrateKbps = 30;
55   RTC_DCHECK(!streams.empty());
56   RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
57 
58   VideoCodec video_codec;
59   memset(&video_codec, 0, sizeof(video_codec));
60   video_codec.codecType = config.codec_type;
61 
62   switch (config.content_type) {
63     case VideoEncoderConfig::ContentType::kRealtimeVideo:
64       video_codec.mode = VideoCodecMode::kRealtimeVideo;
65       break;
66     case VideoEncoderConfig::ContentType::kScreen:
67       video_codec.mode = VideoCodecMode::kScreensharing;
68       break;
69   }
70 
71   video_codec.legacy_conference_mode =
72       config.content_type == VideoEncoderConfig::ContentType::kScreen &&
73       config.legacy_conference_mode;
74 
75   video_codec.numberOfSimulcastStreams =
76       static_cast<unsigned char>(streams.size());
77   video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
78   bool codec_active = false;
79   // Active configuration might not be fully copied to |streams| for SVC yet.
80   // Therefore the |config| is checked here.
81   for (const VideoStream& stream : config.simulcast_layers) {
82     if (stream.active) {
83       codec_active = true;
84       break;
85     }
86   }
87   // Set active for the entire video codec for the non simulcast case.
88   video_codec.active = codec_active;
89   if (video_codec.minBitrate < kEncoderMinBitrateKbps)
90     video_codec.minBitrate = kEncoderMinBitrateKbps;
91   video_codec.timing_frame_thresholds = {kDefaultTimingFramesDelayMs,
92                                          kDefaultOutlierFrameSizePercent};
93   RTC_DCHECK_LE(streams.size(), kMaxSimulcastStreams);
94 
95   int max_framerate = 0;
96 
97   for (size_t i = 0; i < streams.size(); ++i) {
98     SpatialLayer* sim_stream = &video_codec.simulcastStream[i];
99     RTC_DCHECK_GT(streams[i].width, 0);
100     RTC_DCHECK_GT(streams[i].height, 0);
101     RTC_DCHECK_GT(streams[i].max_framerate, 0);
102     RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
103     RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
104     RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
105     RTC_DCHECK_GE(streams[i].max_qp, 0);
106 
107     sim_stream->width = static_cast<uint16_t>(streams[i].width);
108     sim_stream->height = static_cast<uint16_t>(streams[i].height);
109     sim_stream->maxFramerate = streams[i].max_framerate;
110     sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
111     sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
112     sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
113     sim_stream->qpMax = streams[i].max_qp;
114     sim_stream->numberOfTemporalLayers =
115         static_cast<unsigned char>(streams[i].num_temporal_layers.value_or(1));
116     sim_stream->active = streams[i].active;
117 
118     video_codec.width =
119         std::max(video_codec.width, static_cast<uint16_t>(streams[i].width));
120     video_codec.height =
121         std::max(video_codec.height, static_cast<uint16_t>(streams[i].height));
122     video_codec.minBitrate =
123         std::min(static_cast<uint16_t>(video_codec.minBitrate),
124                  static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000));
125     video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
126     video_codec.qpMax = std::max(video_codec.qpMax,
127                                  static_cast<unsigned int>(streams[i].max_qp));
128     max_framerate = std::max(max_framerate, streams[i].max_framerate);
129   }
130 
131   if (video_codec.maxBitrate == 0) {
132     // Unset max bitrate -> cap to one bit per pixel.
133     video_codec.maxBitrate =
134         (video_codec.width * video_codec.height * video_codec.maxFramerate) /
135         1000;
136   }
137   if (video_codec.maxBitrate < kEncoderMinBitrateKbps)
138     video_codec.maxBitrate = kEncoderMinBitrateKbps;
139 
140   video_codec.maxFramerate = max_framerate;
141 
142   // Set codec specific options
143   if (config.encoder_specific_settings)
144     config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec);
145 
146   switch (video_codec.codecType) {
147     case kVideoCodecVP8: {
148       if (!config.encoder_specific_settings) {
149         *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings();
150       }
151 
152       video_codec.VP8()->numberOfTemporalLayers = static_cast<unsigned char>(
153           streams.back().num_temporal_layers.value_or(
154               video_codec.VP8()->numberOfTemporalLayers));
155       RTC_DCHECK_GE(video_codec.VP8()->numberOfTemporalLayers, 1);
156       RTC_DCHECK_LE(video_codec.VP8()->numberOfTemporalLayers,
157                     kMaxTemporalStreams);
158 
159       break;
160     }
161     case kVideoCodecVP9: {
162       // Force the first stream to always be active.
163       video_codec.simulcastStream[0].active = codec_active;
164 
165       if (!config.encoder_specific_settings) {
166         *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings();
167       }
168 
169       video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
170           streams.back().num_temporal_layers.value_or(
171               video_codec.VP9()->numberOfTemporalLayers));
172       RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1);
173       RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers,
174                     kMaxTemporalStreams);
175 
176       RTC_DCHECK(config.spatial_layers.empty() ||
177                  config.spatial_layers.size() ==
178                      video_codec.VP9()->numberOfSpatialLayers);
179 
180       std::vector<SpatialLayer> spatial_layers;
181       if (!config.spatial_layers.empty()) {
182         // Layering is set explicitly.
183         spatial_layers = config.spatial_layers;
184       } else {
185         size_t first_active_layer = 0;
186         for (size_t spatial_idx = 0;
187              spatial_idx < config.simulcast_layers.size(); ++spatial_idx) {
188           if (config.simulcast_layers[spatial_idx].active) {
189             first_active_layer = spatial_idx;
190             break;
191           }
192         }
193 
194         spatial_layers = GetSvcConfig(
195             video_codec.width, video_codec.height, video_codec.maxFramerate,
196             first_active_layer, video_codec.VP9()->numberOfSpatialLayers,
197             video_codec.VP9()->numberOfTemporalLayers,
198             video_codec.mode == VideoCodecMode::kScreensharing);
199 
200         // If there was no request for spatial layering, don't limit bitrate
201         // of single spatial layer.
202         const bool no_spatial_layering =
203             video_codec.VP9()->numberOfSpatialLayers <= 1;
204         if (no_spatial_layering) {
205           // Use codec's bitrate limits.
206           spatial_layers.back().minBitrate = video_codec.minBitrate;
207           spatial_layers.back().targetBitrate = video_codec.maxBitrate;
208           spatial_layers.back().maxBitrate = video_codec.maxBitrate;
209         }
210 
211         for (size_t spatial_idx = first_active_layer;
212              spatial_idx < config.simulcast_layers.size() &&
213              spatial_idx < spatial_layers.size() + first_active_layer;
214              ++spatial_idx) {
215           spatial_layers[spatial_idx - first_active_layer].active =
216               config.simulcast_layers[spatial_idx].active;
217         }
218       }
219 
220       RTC_DCHECK(!spatial_layers.empty());
221       for (size_t i = 0; i < spatial_layers.size(); ++i) {
222         video_codec.spatialLayers[i] = spatial_layers[i];
223       }
224 
225       // The top spatial layer dimensions may not be equal to the input
226       // resolution because of the rounding or explicit configuration.
227       // This difference must be propagated to the stream configuration.
228       video_codec.width = spatial_layers.back().width;
229       video_codec.height = spatial_layers.back().height;
230       video_codec.simulcastStream[0].width = spatial_layers.back().width;
231       video_codec.simulcastStream[0].height = spatial_layers.back().height;
232 
233       // Update layering settings.
234       video_codec.VP9()->numberOfSpatialLayers =
235           static_cast<unsigned char>(spatial_layers.size());
236       RTC_DCHECK_GE(video_codec.VP9()->numberOfSpatialLayers, 1);
237       RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers,
238                     kMaxSpatialLayers);
239 
240       video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
241           spatial_layers.back().numberOfTemporalLayers);
242       RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1);
243       RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers,
244                     kMaxTemporalStreams);
245 
246       break;
247     }
248     case kVideoCodecH264: {
249       if (!config.encoder_specific_settings)
250         *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();
251       video_codec.H264()->numberOfTemporalLayers = static_cast<unsigned char>(
252           streams.back().num_temporal_layers.value_or(
253               video_codec.H264()->numberOfTemporalLayers));
254       RTC_DCHECK_GE(video_codec.H264()->numberOfTemporalLayers, 1);
255       RTC_DCHECK_LE(video_codec.H264()->numberOfTemporalLayers,
256                     kMaxTemporalStreams);
257       break;
258     }
259     default:
260       // TODO(pbos): Support encoder_settings codec-agnostically.
261       RTC_DCHECK(!config.encoder_specific_settings)
262           << "Encoder-specific settings for codec type not wired up.";
263       break;
264   }
265 
266   const absl::optional<DataRate> experimental_min_bitrate =
267       GetExperimentalMinVideoBitrate(video_codec.codecType);
268   if (experimental_min_bitrate) {
269     const int experimental_min_bitrate_kbps =
270         rtc::saturated_cast<int>(experimental_min_bitrate->kbps());
271     video_codec.minBitrate = experimental_min_bitrate_kbps;
272     video_codec.simulcastStream[0].minBitrate = experimental_min_bitrate_kbps;
273     if (video_codec.codecType == kVideoCodecVP9) {
274       video_codec.spatialLayers[0].minBitrate = experimental_min_bitrate_kbps;
275     }
276   }
277 
278   return video_codec;
279 }
280 
281 }  // namespace webrtc
282