1 /*
2  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/include/video_codec_initializer.h"
12 
13 #include <stdint.h>
14 #include <string.h>
15 
16 #include <algorithm>
17 
18 #include "absl/types/optional.h"
19 #include "api/scoped_refptr.h"
20 #include "api/units/data_rate.h"
21 #include "api/video/video_bitrate_allocation.h"
22 #include "api/video_codecs/video_encoder.h"
23 #include "modules/video_coding/codecs/vp9/svc_config.h"
24 #include "modules/video_coding/include/video_coding_defines.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/experiments/min_video_bitrate_experiment.h"
27 #include "rtc_base/logging.h"
28 #include "rtc_base/numerics/safe_conversions.h"
29 
30 namespace webrtc {
31 
SetupCodec(const VideoEncoderConfig & config,const std::vector<VideoStream> & streams,VideoCodec * codec)32 bool VideoCodecInitializer::SetupCodec(const VideoEncoderConfig& config,
33                                        const std::vector<VideoStream>& streams,
34                                        VideoCodec* codec) {
35   if (config.codec_type == kVideoCodecMultiplex) {
36     VideoEncoderConfig associated_config = config.Copy();
37     associated_config.codec_type = kVideoCodecVP9;
38     if (!SetupCodec(associated_config, streams, codec)) {
39       RTC_LOG(LS_ERROR) << "Failed to create stereo encoder configuration.";
40       return false;
41     }
42     codec->codecType = kVideoCodecMultiplex;
43     return true;
44   }
45 
46   *codec = VideoEncoderConfigToVideoCodec(config, streams);
47   return true;
48 }
49 
50 // TODO(sprang): Split this up and separate the codec specific parts.
VideoEncoderConfigToVideoCodec(const VideoEncoderConfig & config,const std::vector<VideoStream> & streams)51 VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
52     const VideoEncoderConfig& config,
53     const std::vector<VideoStream>& streams) {
54   static const int kEncoderMinBitrateKbps = 30;
55   RTC_DCHECK(!streams.empty());
56   RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
57 
58   VideoCodec video_codec;
59   memset(&video_codec, 0, sizeof(video_codec));
60   video_codec.codecType = config.codec_type;
61 
62   switch (config.content_type) {
63     case VideoEncoderConfig::ContentType::kRealtimeVideo:
64       video_codec.mode = VideoCodecMode::kRealtimeVideo;
65       break;
66     case VideoEncoderConfig::ContentType::kScreen:
67       video_codec.mode = VideoCodecMode::kScreensharing;
68       break;
69   }
70 
71   // TODO(nisse): The plType field should be deleted. Luckily, our
72   // callers don't need it.
73   video_codec.plType = 0;
74   video_codec.numberOfSimulcastStreams =
75       static_cast<unsigned char>(streams.size());
76   video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
77   bool codec_active = false;
78   for (const VideoStream& stream : streams) {
79     if (stream.active) {
80       codec_active = true;
81       break;
82     }
83   }
84   // Set active for the entire video codec for the non simulcast case.
85   video_codec.active = codec_active;
86   if (video_codec.minBitrate < kEncoderMinBitrateKbps)
87     video_codec.minBitrate = kEncoderMinBitrateKbps;
88   video_codec.timing_frame_thresholds = {kDefaultTimingFramesDelayMs,
89                                          kDefaultOutlierFrameSizePercent};
90   RTC_DCHECK_LE(streams.size(), kMaxSimulcastStreams);
91 
92   int max_framerate = 0;
93 
94   for (size_t i = 0; i < streams.size(); ++i) {
95     SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
96     RTC_DCHECK_GT(streams[i].width, 0);
97     RTC_DCHECK_GT(streams[i].height, 0);
98     RTC_DCHECK_GT(streams[i].max_framerate, 0);
99     RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
100     RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
101     RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
102     RTC_DCHECK_GE(streams[i].max_qp, 0);
103 
104     sim_stream->width = static_cast<uint16_t>(streams[i].width);
105     sim_stream->height = static_cast<uint16_t>(streams[i].height);
106     sim_stream->maxFramerate = streams[i].max_framerate;
107     sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
108     sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
109     sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
110     sim_stream->qpMax = streams[i].max_qp;
111     sim_stream->numberOfTemporalLayers =
112         static_cast<unsigned char>(streams[i].num_temporal_layers.value_or(1));
113     sim_stream->active = streams[i].active;
114 
115     video_codec.width =
116         std::max(video_codec.width, static_cast<uint16_t>(streams[i].width));
117     video_codec.height =
118         std::max(video_codec.height, static_cast<uint16_t>(streams[i].height));
119     video_codec.minBitrate =
120         std::min(static_cast<uint16_t>(video_codec.minBitrate),
121                  static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000));
122     video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
123     video_codec.qpMax = std::max(video_codec.qpMax,
124                                  static_cast<unsigned int>(streams[i].max_qp));
125     max_framerate = std::max(max_framerate, streams[i].max_framerate);
126   }
127 
128   if (video_codec.maxBitrate == 0) {
129     // Unset max bitrate -> cap to one bit per pixel.
130     video_codec.maxBitrate =
131         (video_codec.width * video_codec.height * video_codec.maxFramerate) /
132         1000;
133   }
134   if (video_codec.maxBitrate < kEncoderMinBitrateKbps)
135     video_codec.maxBitrate = kEncoderMinBitrateKbps;
136 
137   video_codec.maxFramerate = max_framerate;
138 
139   // Set codec specific options
140   if (config.encoder_specific_settings)
141     config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec);
142 
143   switch (video_codec.codecType) {
144     case kVideoCodecVP8: {
145       if (!config.encoder_specific_settings) {
146         *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings();
147       }
148 
149       video_codec.VP8()->numberOfTemporalLayers = static_cast<unsigned char>(
150           streams.back().num_temporal_layers.value_or(
151               video_codec.VP8()->numberOfTemporalLayers));
152       RTC_DCHECK_GE(video_codec.VP8()->numberOfTemporalLayers, 1);
153       RTC_DCHECK_LE(video_codec.VP8()->numberOfTemporalLayers,
154                     kMaxTemporalStreams);
155 
156       break;
157     }
158     case kVideoCodecVP9: {
159       // Force the first stream to always be active.
160       video_codec.simulcastStream[0].active = codec_active;
161 
162       if (!config.encoder_specific_settings) {
163         *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings();
164       }
165 
166       video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
167           streams.back().num_temporal_layers.value_or(
168               video_codec.VP9()->numberOfTemporalLayers));
169       RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1);
170       RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers,
171                     kMaxTemporalStreams);
172 
173       RTC_DCHECK(config.spatial_layers.empty() ||
174                  config.spatial_layers.size() ==
175                      video_codec.VP9()->numberOfSpatialLayers);
176 
177       std::vector<SpatialLayer> spatial_layers;
178       if (!config.spatial_layers.empty()) {
179         // Layering is set explicitly.
180         spatial_layers = config.spatial_layers;
181       } else {
182         size_t min_required_layers = 0;
183         // Need at least enough layers for the first active one to be present.
184         for (size_t spatial_idx = 0;
185              spatial_idx < config.simulcast_layers.size(); ++spatial_idx) {
186           if (config.simulcast_layers[spatial_idx].active) {
187             min_required_layers = spatial_idx + 1;
188             break;
189           }
190         }
191 
192         spatial_layers = GetSvcConfig(
193             video_codec.width, video_codec.height, video_codec.maxFramerate,
194             min_required_layers, video_codec.VP9()->numberOfSpatialLayers,
195             video_codec.VP9()->numberOfTemporalLayers,
196             video_codec.mode == VideoCodecMode::kScreensharing);
197 
198         // If there was no request for spatial layering, don't limit bitrate
199         // of single spatial layer.
200         const bool no_spatial_layering =
201             video_codec.VP9()->numberOfSpatialLayers <= 1;
202         if (no_spatial_layering) {
203           // Use codec's bitrate limits.
204           spatial_layers.back().minBitrate = video_codec.minBitrate;
205           spatial_layers.back().targetBitrate = video_codec.maxBitrate;
206           spatial_layers.back().maxBitrate = video_codec.maxBitrate;
207         }
208 
209         for (size_t spatial_idx = 0;
210              spatial_idx < config.simulcast_layers.size() &&
211              spatial_idx < spatial_layers.size();
212              ++spatial_idx) {
213           spatial_layers[spatial_idx].active =
214               config.simulcast_layers[spatial_idx].active;
215         }
216       }
217 
218       RTC_DCHECK(!spatial_layers.empty());
219       for (size_t i = 0; i < spatial_layers.size(); ++i) {
220         video_codec.spatialLayers[i] = spatial_layers[i];
221       }
222 
223       // Update layering settings.
224       video_codec.VP9()->numberOfSpatialLayers =
225           static_cast<unsigned char>(spatial_layers.size());
226       RTC_DCHECK_GE(video_codec.VP9()->numberOfSpatialLayers, 1);
227       RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers,
228                     kMaxSpatialLayers);
229 
230       video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
231           spatial_layers.back().numberOfTemporalLayers);
232       RTC_DCHECK_GE(video_codec.VP9()->numberOfTemporalLayers, 1);
233       RTC_DCHECK_LE(video_codec.VP9()->numberOfTemporalLayers,
234                     kMaxTemporalStreams);
235 
236       break;
237     }
238     case kVideoCodecH264: {
239       if (!config.encoder_specific_settings)
240         *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();
241       video_codec.H264()->numberOfTemporalLayers = static_cast<unsigned char>(
242           streams.back().num_temporal_layers.value_or(
243               video_codec.H264()->numberOfTemporalLayers));
244       RTC_DCHECK_GE(video_codec.H264()->numberOfTemporalLayers, 1);
245       RTC_DCHECK_LE(video_codec.H264()->numberOfTemporalLayers,
246                     kMaxTemporalStreams);
247       break;
248     }
249     default:
250       // TODO(pbos): Support encoder_settings codec-agnostically.
251       RTC_DCHECK(!config.encoder_specific_settings)
252           << "Encoder-specific settings for codec type not wired up.";
253       break;
254   }
255 
256   const absl::optional<DataRate> experimental_min_bitrate =
257       GetExperimentalMinVideoBitrate(video_codec.codecType);
258   if (experimental_min_bitrate) {
259     const int experimental_min_bitrate_kbps =
260         rtc::saturated_cast<int>(experimental_min_bitrate->kbps());
261     video_codec.minBitrate = experimental_min_bitrate_kbps;
262     video_codec.simulcastStream[0].minBitrate = experimental_min_bitrate_kbps;
263     if (video_codec.codecType == kVideoCodecVP9) {
264       video_codec.spatialLayers[0].minBitrate = experimental_min_bitrate_kbps;
265     }
266   }
267 
268   return video_codec;
269 }
270 
271 }  // namespace webrtc
272