1 /*
2 * Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #ifndef MEDIA_BASE_MEDIA_CHANNEL_H_
12 #define MEDIA_BASE_MEDIA_CHANNEL_H_
13
14 #include <map>
15 #include <memory>
16 #include <string>
17 #include <utility>
18 #include <vector>
19
20 #include "absl/types/optional.h"
21 #include "api/audio_codecs/audio_encoder.h"
22 #include "api/audio_options.h"
23 #include "api/crypto/frame_decryptor_interface.h"
24 #include "api/crypto/frame_encryptor_interface.h"
25 #include "api/frame_transformer_interface.h"
26 #include "api/media_stream_interface.h"
27 #include "api/rtc_error.h"
28 #include "api/rtp_parameters.h"
29 #include "api/transport/rtp/rtp_source.h"
30 #include "api/video/video_content_type.h"
31 #include "api/video/video_sink_interface.h"
32 #include "api/video/video_source_interface.h"
33 #include "api/video/video_timing.h"
34 #include "api/video_codecs/video_encoder_config.h"
35 #include "call/video_receive_stream.h"
36 #include "common_video/include/quality_limitation_reason.h"
37 #include "media/base/codec.h"
38 #include "media/base/delayable.h"
39 #include "media/base/media_config.h"
40 #include "media/base/media_constants.h"
41 #include "media/base/stream_params.h"
42 #include "modules/audio_processing/include/audio_processing_statistics.h"
43 #include "modules/rtp_rtcp/include/report_block_data.h"
44 #include "rtc_base/async_packet_socket.h"
45 #include "rtc_base/buffer.h"
46 #include "rtc_base/callback.h"
47 #include "rtc_base/copy_on_write_buffer.h"
48 #include "rtc_base/dscp.h"
49 #include "rtc_base/logging.h"
50 #include "rtc_base/network_route.h"
51 #include "rtc_base/socket.h"
52 #include "rtc_base/string_encode.h"
53 #include "rtc_base/strings/string_builder.h"
54 #include "rtc_base/synchronization/mutex.h"
55 #include "rtc_base/third_party/sigslot/sigslot.h"
56
57 namespace rtc {
58 class Timing;
59 }
60
61 namespace webrtc {
62 class AudioSinkInterface;
63 class VideoFrame;
64 } // namespace webrtc
65
66 namespace cricket {
67
68 class AudioSource;
69 class VideoCapturer;
70 struct RtpHeader;
71 struct VideoFormat;
72
73 const int kScreencastDefaultFps = 5;
74
75 template <class T>
ToStringIfSet(const char * key,const absl::optional<T> & val)76 static std::string ToStringIfSet(const char* key,
77 const absl::optional<T>& val) {
78 std::string str;
79 if (val) {
80 str = key;
81 str += ": ";
82 str += val ? rtc::ToString(*val) : "";
83 str += ", ";
84 }
85 return str;
86 }
87
88 template <class T>
VectorToString(const std::vector<T> & vals)89 static std::string VectorToString(const std::vector<T>& vals) {
90 rtc::StringBuilder ost; // no-presubmit-check TODO(webrtc:8982)
91 ost << "[";
92 for (size_t i = 0; i < vals.size(); ++i) {
93 if (i > 0) {
94 ost << ", ";
95 }
96 ost << vals[i].ToString();
97 }
98 ost << "]";
99 return ost.Release();
100 }
101
102 // Options that can be applied to a VideoMediaChannel or a VideoMediaEngine.
103 // Used to be flags, but that makes it hard to selectively apply options.
104 // We are moving all of the setting of options to structs like this,
105 // but some things currently still use flags.
106 struct VideoOptions {
107 VideoOptions();
108 ~VideoOptions();
109
SetAllVideoOptions110 void SetAll(const VideoOptions& change) {
111 SetFrom(&video_noise_reduction, change.video_noise_reduction);
112 SetFrom(&screencast_min_bitrate_kbps, change.screencast_min_bitrate_kbps);
113 SetFrom(&is_screencast, change.is_screencast);
114 }
115
116 bool operator==(const VideoOptions& o) const {
117 return video_noise_reduction == o.video_noise_reduction &&
118 screencast_min_bitrate_kbps == o.screencast_min_bitrate_kbps &&
119 is_screencast == o.is_screencast;
120 }
121 bool operator!=(const VideoOptions& o) const { return !(*this == o); }
122
ToStringVideoOptions123 std::string ToString() const {
124 rtc::StringBuilder ost;
125 ost << "VideoOptions {";
126 ost << ToStringIfSet("noise reduction", video_noise_reduction);
127 ost << ToStringIfSet("screencast min bitrate kbps",
128 screencast_min_bitrate_kbps);
129 ost << ToStringIfSet("is_screencast ", is_screencast);
130 ost << "}";
131 return ost.Release();
132 }
133
134 // Enable denoising? This flag comes from the getUserMedia
135 // constraint 'googNoiseReduction', and WebRtcVideoEngine passes it
136 // on to the codec options. Disabled by default.
137 absl::optional<bool> video_noise_reduction;
138 // Force screencast to use a minimum bitrate. This flag comes from
139 // the PeerConnection constraint 'googScreencastMinBitrate'. It is
140 // copied to the encoder config by WebRtcVideoChannel.
141 absl::optional<int> screencast_min_bitrate_kbps;
142 // Set by screencast sources. Implies selection of encoding settings
143 // suitable for screencast. Most likely not the right way to do
144 // things, e.g., screencast of a text document and screencast of a
145 // youtube video have different needs.
146 absl::optional<bool> is_screencast;
147 webrtc::VideoTrackInterface::ContentHint content_hint;
148
149 private:
150 template <typename T>
SetFromVideoOptions151 static void SetFrom(absl::optional<T>* s, const absl::optional<T>& o) {
152 if (o) {
153 *s = o;
154 }
155 }
156 };
157
158 class MediaChannel : public sigslot::has_slots<> {
159 public:
160 class NetworkInterface {
161 public:
162 enum SocketType { ST_RTP, ST_RTCP };
163 virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet,
164 const rtc::PacketOptions& options) = 0;
165 virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
166 const rtc::PacketOptions& options) = 0;
167 virtual int SetOption(SocketType type,
168 rtc::Socket::Option opt,
169 int option) = 0;
~NetworkInterface()170 virtual ~NetworkInterface() {}
171 };
172
173 explicit MediaChannel(const MediaConfig& config);
174 MediaChannel();
175 ~MediaChannel() override;
176
177 virtual cricket::MediaType media_type() const = 0;
178
179 // Sets the abstract interface class for sending RTP/RTCP data.
180 virtual void SetInterface(NetworkInterface* iface)
181 RTC_LOCKS_EXCLUDED(network_interface_mutex_);
182 // Called when a RTP packet is received.
183 virtual void OnPacketReceived(rtc::CopyOnWriteBuffer packet,
184 int64_t packet_time_us) = 0;
185 // Called when the socket's ability to send has changed.
186 virtual void OnReadyToSend(bool ready) = 0;
187 // Called when the network route used for sending packets changed.
188 virtual void OnNetworkRouteChanged(
189 const std::string& transport_name,
190 const rtc::NetworkRoute& network_route) = 0;
191 // Creates a new outgoing media stream with SSRCs and CNAME as described
192 // by sp.
193 virtual bool AddSendStream(const StreamParams& sp) = 0;
194 // Removes an outgoing media stream.
195 // SSRC must be the first SSRC of the media stream if the stream uses
196 // multiple SSRCs. In the case of an ssrc of 0, the possibly cached
197 // StreamParams is removed.
198 virtual bool RemoveSendStream(uint32_t ssrc) = 0;
199 // Creates a new incoming media stream with SSRCs, CNAME as described
200 // by sp. In the case of a sp without SSRCs, the unsignaled sp is cached
201 // to be used later for unsignaled streams received.
202 virtual bool AddRecvStream(const StreamParams& sp) = 0;
203 // Removes an incoming media stream.
204 // ssrc must be the first SSRC of the media stream if the stream uses
205 // multiple SSRCs.
206 virtual bool RemoveRecvStream(uint32_t ssrc) = 0;
207 // Resets any cached StreamParams for an unsignaled RecvStream, and removes
208 // any existing unsignaled streams.
209 virtual void ResetUnsignaledRecvStream() = 0;
210 // Returns the absoulte sendtime extension id value from media channel.
211 virtual int GetRtpSendTimeExtnId() const;
212 // Set the frame encryptor to use on all outgoing frames. This is optional.
213 // This pointers lifetime is managed by the set of RtpSender it is attached
214 // to.
215 // TODO(benwright) make pure virtual once internal supports it.
216 virtual void SetFrameEncryptor(
217 uint32_t ssrc,
218 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
219 // Set the frame decryptor to use on all incoming frames. This is optional.
220 // This pointers lifetimes is managed by the set of RtpReceivers it is
221 // attached to.
222 // TODO(benwright) make pure virtual once internal supports it.
223 virtual void SetFrameDecryptor(
224 uint32_t ssrc,
225 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
226
227 // Enable network condition based codec switching.
228 virtual void SetVideoCodecSwitchingEnabled(bool enabled);
229
230 // Base method to send packet using NetworkInterface.
SendPacket(rtc::CopyOnWriteBuffer * packet,const rtc::PacketOptions & options)231 bool SendPacket(rtc::CopyOnWriteBuffer* packet,
232 const rtc::PacketOptions& options) {
233 return DoSendPacket(packet, false, options);
234 }
235
SendRtcp(rtc::CopyOnWriteBuffer * packet,const rtc::PacketOptions & options)236 bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
237 const rtc::PacketOptions& options) {
238 return DoSendPacket(packet, true, options);
239 }
240
SetOption(NetworkInterface::SocketType type,rtc::Socket::Option opt,int option)241 int SetOption(NetworkInterface::SocketType type,
242 rtc::Socket::Option opt,
243 int option) RTC_LOCKS_EXCLUDED(network_interface_mutex_) {
244 webrtc::MutexLock lock(&network_interface_mutex_);
245 return SetOptionLocked(type, opt, option);
246 }
247
248 // Corresponds to the SDP attribute extmap-allow-mixed, see RFC8285.
249 // Set to true if it's allowed to mix one- and two-byte RTP header extensions
250 // in the same stream. The setter and getter must only be called from
251 // worker_thread.
SetExtmapAllowMixed(bool extmap_allow_mixed)252 void SetExtmapAllowMixed(bool extmap_allow_mixed) {
253 extmap_allow_mixed_ = extmap_allow_mixed;
254 }
ExtmapAllowMixed()255 bool ExtmapAllowMixed() const { return extmap_allow_mixed_; }
256
257 virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0;
258 virtual webrtc::RTCError SetRtpSendParameters(
259 uint32_t ssrc,
260 const webrtc::RtpParameters& parameters) = 0;
261
262 virtual void SetEncoderToPacketizerFrameTransformer(
263 uint32_t ssrc,
264 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
265 virtual void SetDepacketizerToDecoderFrameTransformer(
266 uint32_t ssrc,
267 rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
268
269 protected:
SetOptionLocked(NetworkInterface::SocketType type,rtc::Socket::Option opt,int option)270 int SetOptionLocked(NetworkInterface::SocketType type,
271 rtc::Socket::Option opt,
272 int option)
273 RTC_EXCLUSIVE_LOCKS_REQUIRED(network_interface_mutex_) {
274 if (!network_interface_)
275 return -1;
276 return network_interface_->SetOption(type, opt, option);
277 }
278
DscpEnabled()279 bool DscpEnabled() const { return enable_dscp_; }
280
281 // This is the DSCP value used for both RTP and RTCP channels if DSCP is
282 // enabled. It can be changed at any time via |SetPreferredDscp|.
PreferredDscp()283 rtc::DiffServCodePoint PreferredDscp() const
284 RTC_LOCKS_EXCLUDED(network_interface_mutex_) {
285 webrtc::MutexLock lock(&network_interface_mutex_);
286 return preferred_dscp_;
287 }
288
SetPreferredDscp(rtc::DiffServCodePoint preferred_dscp)289 int SetPreferredDscp(rtc::DiffServCodePoint preferred_dscp)
290 RTC_LOCKS_EXCLUDED(network_interface_mutex_) {
291 webrtc::MutexLock lock(&network_interface_mutex_);
292 if (preferred_dscp == preferred_dscp_) {
293 return 0;
294 }
295 preferred_dscp_ = preferred_dscp;
296 return UpdateDscp();
297 }
298
299 private:
300 // Apply the preferred DSCP setting to the underlying network interface RTP
301 // and RTCP channels. If DSCP is disabled, then apply the default DSCP value.
UpdateDscp()302 int UpdateDscp() RTC_EXCLUSIVE_LOCKS_REQUIRED(network_interface_mutex_) {
303 rtc::DiffServCodePoint value =
304 enable_dscp_ ? preferred_dscp_ : rtc::DSCP_DEFAULT;
305 int ret =
306 SetOptionLocked(NetworkInterface::ST_RTP, rtc::Socket::OPT_DSCP, value);
307 if (ret == 0) {
308 ret = SetOptionLocked(NetworkInterface::ST_RTCP, rtc::Socket::OPT_DSCP,
309 value);
310 }
311 return ret;
312 }
313
DoSendPacket(rtc::CopyOnWriteBuffer * packet,bool rtcp,const rtc::PacketOptions & options)314 bool DoSendPacket(rtc::CopyOnWriteBuffer* packet,
315 bool rtcp,
316 const rtc::PacketOptions& options)
317 RTC_LOCKS_EXCLUDED(network_interface_mutex_) {
318 webrtc::MutexLock lock(&network_interface_mutex_);
319 if (!network_interface_)
320 return false;
321
322 return (!rtcp) ? network_interface_->SendPacket(packet, options)
323 : network_interface_->SendRtcp(packet, options);
324 }
325
326 const bool enable_dscp_;
327 // |network_interface_| can be accessed from the worker_thread and
328 // from any MediaEngine threads. This critical section is to protect accessing
329 // of network_interface_ object.
330 mutable webrtc::Mutex network_interface_mutex_;
331 NetworkInterface* network_interface_
332 RTC_GUARDED_BY(network_interface_mutex_) = nullptr;
333 rtc::DiffServCodePoint preferred_dscp_
334 RTC_GUARDED_BY(network_interface_mutex_) = rtc::DSCP_DEFAULT;
335 bool extmap_allow_mixed_ = false;
336 };
337
338 // The stats information is structured as follows:
339 // Media are represented by either MediaSenderInfo or MediaReceiverInfo.
340 // Media contains a vector of SSRC infos that are exclusively used by this
341 // media. (SSRCs shared between media streams can't be represented.)
342
343 // Information about an SSRC.
344 // This data may be locally recorded, or received in an RTCP SR or RR.
345 struct SsrcSenderInfo {
346 uint32_t ssrc = 0;
347 double timestamp = 0.0; // NTP timestamp, represented as seconds since epoch.
348 };
349
350 struct SsrcReceiverInfo {
351 uint32_t ssrc = 0;
352 double timestamp = 0.0;
353 };
354
355 struct MediaSenderInfo {
356 MediaSenderInfo();
357 ~MediaSenderInfo();
add_ssrcMediaSenderInfo358 void add_ssrc(const SsrcSenderInfo& stat) { local_stats.push_back(stat); }
359 // Temporary utility function for call sites that only provide SSRC.
360 // As more info is added into SsrcSenderInfo, this function should go away.
add_ssrcMediaSenderInfo361 void add_ssrc(uint32_t ssrc) {
362 SsrcSenderInfo stat;
363 stat.ssrc = ssrc;
364 add_ssrc(stat);
365 }
366 // Utility accessor for clients that are only interested in ssrc numbers.
ssrcsMediaSenderInfo367 std::vector<uint32_t> ssrcs() const {
368 std::vector<uint32_t> retval;
369 for (std::vector<SsrcSenderInfo>::const_iterator it = local_stats.begin();
370 it != local_stats.end(); ++it) {
371 retval.push_back(it->ssrc);
372 }
373 return retval;
374 }
375 // Returns true if the media has been connected.
connectedMediaSenderInfo376 bool connected() const { return local_stats.size() > 0; }
377 // Utility accessor for clients that make the assumption only one ssrc
378 // exists per media.
379 // This will eventually go away.
380 // Call sites that compare this to zero should use connected() instead.
381 // https://bugs.webrtc.org/8694
ssrcMediaSenderInfo382 uint32_t ssrc() const {
383 if (connected()) {
384 return local_stats[0].ssrc;
385 } else {
386 return 0;
387 }
388 }
389 // https://w3c.github.io/webrtc-stats/#dom-rtcsentrtpstreamstats-bytessent
390 int64_t payload_bytes_sent = 0;
391 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-headerbytessent
392 int64_t header_and_padding_bytes_sent = 0;
393 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedbytessent
394 uint64_t retransmitted_bytes_sent = 0;
395 int packets_sent = 0;
396 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedpacketssent
397 uint64_t retransmitted_packets_sent = 0;
398 int packets_lost = 0;
399 float fraction_lost = 0.0f;
400 int64_t rtt_ms = 0;
401 std::string codec_name;
402 absl::optional<int> codec_payload_type;
403 std::vector<SsrcSenderInfo> local_stats;
404 std::vector<SsrcReceiverInfo> remote_stats;
405 // A snapshot of the most recent Report Block with additional data of interest
406 // to statistics. Used to implement RTCRemoteInboundRtpStreamStats. Within
407 // this list, the ReportBlockData::RTCPReportBlock::source_ssrc(), which is
408 // the SSRC of the corresponding outbound RTP stream, is unique.
409 std::vector<webrtc::ReportBlockData> report_block_datas;
410 };
411
412 struct MediaReceiverInfo {
413 MediaReceiverInfo();
414 ~MediaReceiverInfo();
add_ssrcMediaReceiverInfo415 void add_ssrc(const SsrcReceiverInfo& stat) { local_stats.push_back(stat); }
416 // Temporary utility function for call sites that only provide SSRC.
417 // As more info is added into SsrcSenderInfo, this function should go away.
add_ssrcMediaReceiverInfo418 void add_ssrc(uint32_t ssrc) {
419 SsrcReceiverInfo stat;
420 stat.ssrc = ssrc;
421 add_ssrc(stat);
422 }
ssrcsMediaReceiverInfo423 std::vector<uint32_t> ssrcs() const {
424 std::vector<uint32_t> retval;
425 for (std::vector<SsrcReceiverInfo>::const_iterator it = local_stats.begin();
426 it != local_stats.end(); ++it) {
427 retval.push_back(it->ssrc);
428 }
429 return retval;
430 }
431 // Returns true if the media has been connected.
connectedMediaReceiverInfo432 bool connected() const { return local_stats.size() > 0; }
433 // Utility accessor for clients that make the assumption only one ssrc
434 // exists per media.
435 // This will eventually go away.
436 // Call sites that compare this to zero should use connected();
437 // https://bugs.webrtc.org/8694
ssrcMediaReceiverInfo438 uint32_t ssrc() const {
439 if (connected()) {
440 return local_stats[0].ssrc;
441 } else {
442 return 0;
443 }
444 }
445
446 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-bytesreceived
447 int64_t payload_bytes_rcvd = 0;
448 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-headerbytesreceived
449 int64_t header_and_padding_bytes_rcvd = 0;
450 int packets_rcvd = 0;
451 int packets_lost = 0;
452 // The timestamp at which the last packet was received, i.e. the time of the
453 // local clock when it was received - not the RTP timestamp of that packet.
454 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp
455 absl::optional<int64_t> last_packet_received_timestamp_ms;
456 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp
457 absl::optional<int64_t> estimated_playout_ntp_timestamp_ms;
458 std::string codec_name;
459 absl::optional<int> codec_payload_type;
460 std::vector<SsrcReceiverInfo> local_stats;
461 std::vector<SsrcSenderInfo> remote_stats;
462 };
463
464 struct VoiceSenderInfo : public MediaSenderInfo {
465 VoiceSenderInfo();
466 ~VoiceSenderInfo();
467 int jitter_ms = 0;
468 // Current audio level, expressed linearly [0,32767].
469 int audio_level = 0;
470 // See description of "totalAudioEnergy" in the WebRTC stats spec:
471 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
472 double total_input_energy = 0.0;
473 double total_input_duration = 0.0;
474 bool typing_noise_detected = false;
475 webrtc::ANAStats ana_statistics;
476 webrtc::AudioProcessingStats apm_statistics;
477 };
478
479 struct VoiceReceiverInfo : public MediaReceiverInfo {
480 VoiceReceiverInfo();
481 ~VoiceReceiverInfo();
482 int jitter_ms = 0;
483 int jitter_buffer_ms = 0;
484 int jitter_buffer_preferred_ms = 0;
485 int delay_estimate_ms = 0;
486 int audio_level = 0;
487 // Stats below correspond to similarly-named fields in the WebRTC stats spec.
488 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats
489 double total_output_energy = 0.0;
490 uint64_t total_samples_received = 0;
491 double total_output_duration = 0.0;
492 uint64_t concealed_samples = 0;
493 uint64_t silent_concealed_samples = 0;
494 uint64_t concealment_events = 0;
495 double jitter_buffer_delay_seconds = 0.0;
496 uint64_t jitter_buffer_emitted_count = 0;
497 double jitter_buffer_target_delay_seconds = 0.0;
498 uint64_t inserted_samples_for_deceleration = 0;
499 uint64_t removed_samples_for_acceleration = 0;
500 uint64_t fec_packets_received = 0;
501 uint64_t fec_packets_discarded = 0;
502 // Stats below correspond to similarly-named fields in the WebRTC stats spec.
503 // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats
504 uint64_t packets_discarded = 0;
505 // Stats below DO NOT correspond directly to anything in the WebRTC stats
506 // fraction of synthesized audio inserted through expansion.
507 float expand_rate = 0.0f;
508 // fraction of synthesized speech inserted through expansion.
509 float speech_expand_rate = 0.0f;
510 // fraction of data out of secondary decoding, including FEC and RED.
511 float secondary_decoded_rate = 0.0f;
512 // Fraction of secondary data, including FEC and RED, that is discarded.
513 // Discarding of secondary data can be caused by the reception of the primary
514 // data, obsoleting the secondary data. It can also be caused by early
515 // or late arrival of secondary data. This metric is the percentage of
516 // discarded secondary data since last query of receiver info.
517 float secondary_discarded_rate = 0.0f;
518 // Fraction of data removed through time compression.
519 float accelerate_rate = 0.0f;
520 // Fraction of data inserted through time stretching.
521 float preemptive_expand_rate = 0.0f;
522 int decoding_calls_to_silence_generator = 0;
523 int decoding_calls_to_neteq = 0;
524 int decoding_normal = 0;
525 // TODO(alexnarest): Consider decoding_neteq_plc for consistency
526 int decoding_plc = 0;
527 int decoding_codec_plc = 0;
528 int decoding_cng = 0;
529 int decoding_plc_cng = 0;
530 int decoding_muted_output = 0;
531 // Estimated capture start time in NTP time in ms.
532 int64_t capture_start_ntp_time_ms = -1;
533 // Count of the number of buffer flushes.
534 uint64_t jitter_buffer_flushes = 0;
535 // Number of samples expanded due to delayed packets.
536 uint64_t delayed_packet_outage_samples = 0;
537 // Arrival delay of received audio packets.
538 double relative_packet_arrival_delay_seconds = 0.0;
539 // Count and total duration of audio interruptions (loss-concealement periods
540 // longer than 150 ms).
541 int32_t interruption_count = 0;
542 int32_t total_interruption_duration_ms = 0;
543 // Remote outbound stats derived by the received RTCP sender reports.
544 // https://w3c.github.io/webrtc-stats/#remoteoutboundrtpstats-dict*
545 absl::optional<int64_t> last_sender_report_timestamp_ms;
546 absl::optional<int64_t> last_sender_report_remote_timestamp_ms;
547 uint32_t sender_reports_packets_sent = 0;
548 uint64_t sender_reports_bytes_sent = 0;
549 uint64_t sender_reports_reports_count = 0;
550 };
551
552 struct VideoSenderInfo : public MediaSenderInfo {
553 VideoSenderInfo();
554 ~VideoSenderInfo();
555 std::vector<SsrcGroup> ssrc_groups;
556 std::string encoder_implementation_name;
557 int firs_rcvd = 0;
558 int plis_rcvd = 0;
559 int nacks_rcvd = 0;
560 int send_frame_width = 0;
561 int send_frame_height = 0;
562 int framerate_input = 0;
563 int framerate_sent = 0;
564 int aggregated_framerate_sent = 0;
565 int nominal_bitrate = 0;
566 int adapt_reason = 0;
567 int adapt_changes = 0;
568 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason
569 webrtc::QualityLimitationReason quality_limitation_reason =
570 webrtc::QualityLimitationReason::kNone;
571 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations
572 std::map<webrtc::QualityLimitationReason, int64_t>
573 quality_limitation_durations_ms;
574 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges
575 uint32_t quality_limitation_resolution_changes = 0;
576 int avg_encode_ms = 0;
577 int encode_usage_percent = 0;
578 uint32_t frames_encoded = 0;
579 uint32_t key_frames_encoded = 0;
580 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime
581 uint64_t total_encode_time_ms = 0;
582 // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget
583 uint64_t total_encoded_bytes_target = 0;
584 uint64_t total_packet_send_delay_ms = 0;
585 bool has_entered_low_resolution = false;
586 absl::optional<uint64_t> qp_sum;
587 webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;
588 uint32_t frames_sent = 0;
589 // https://w3c.github.io/webrtc-stats/#dom-rtcvideosenderstats-hugeframessent
590 uint32_t huge_frames_sent = 0;
591 uint32_t aggregated_huge_frames_sent = 0;
592 absl::optional<std::string> rid;
593 };
594
595 struct VideoReceiverInfo : public MediaReceiverInfo {
596 VideoReceiverInfo();
597 ~VideoReceiverInfo();
598 std::vector<SsrcGroup> ssrc_groups;
599 std::string decoder_implementation_name;
600 int packets_concealed = 0;
601 int firs_sent = 0;
602 int plis_sent = 0;
603 int nacks_sent = 0;
604 int frame_width = 0;
605 int frame_height = 0;
606 int framerate_rcvd = 0;
607 int framerate_decoded = 0;
608 int framerate_output = 0;
609 // Framerate as sent to the renderer.
610 int framerate_render_input = 0;
611 // Framerate that the renderer reports.
612 int framerate_render_output = 0;
613 uint32_t frames_received = 0;
614 uint32_t frames_dropped = 0;
615 uint32_t frames_decoded = 0;
616 uint32_t key_frames_decoded = 0;
617 uint32_t frames_rendered = 0;
618 absl::optional<uint64_t> qp_sum;
619 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime
620 uint64_t total_decode_time_ms = 0;
621 double total_inter_frame_delay = 0;
622 double total_squared_inter_frame_delay = 0;
623 int64_t interframe_delay_max_ms = -1;
624 uint32_t freeze_count = 0;
625 uint32_t pause_count = 0;
626 uint32_t total_freezes_duration_ms = 0;
627 uint32_t total_pauses_duration_ms = 0;
628 uint32_t total_frames_duration_ms = 0;
629 double sum_squared_frame_durations = 0.0;
630
631 webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;
632
633 // All stats below are gathered per-VideoReceiver, but some will be correlated
634 // across MediaStreamTracks. NOTE(hta): when sinking stats into per-SSRC
635 // structures, reflect this in the new layout.
636
637 // Current frame decode latency.
638 int decode_ms = 0;
639 // Maximum observed frame decode latency.
640 int max_decode_ms = 0;
641 // Jitter (network-related) latency.
642 int jitter_buffer_ms = 0;
643 // Jitter (network-related) latency (cumulative).
644 // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferdelay
645 double jitter_buffer_delay_seconds = 0;
646 // Number of observations for cumulative jitter latency.
647 // https://w3c.github.io/webrtc-stats/#dom-rtcvideoreceiverstats-jitterbufferemittedcount
648 uint64_t jitter_buffer_emitted_count = 0;
649 // Requested minimum playout latency.
650 int min_playout_delay_ms = 0;
651 // Requested latency to account for rendering delay.
652 int render_delay_ms = 0;
653 // Target overall delay: network+decode+render, accounting for
654 // min_playout_delay_ms.
655 int target_delay_ms = 0;
656 // Current overall delay, possibly ramping towards target_delay_ms.
657 int current_delay_ms = 0;
658
659 // Estimated capture start time in NTP time in ms.
660 int64_t capture_start_ntp_time_ms = -1;
661
662 // First frame received to first frame decoded latency.
663 int64_t first_frame_received_to_decoded_ms = -1;
664
665 // Timing frame info: all important timestamps for a full lifetime of a
666 // single 'timing frame'.
667 absl::optional<webrtc::TimingFrameInfo> timing_frame_info;
668 };
669
670 struct DataSenderInfo : public MediaSenderInfo {
671 uint32_t ssrc = 0;
672 };
673
674 struct DataReceiverInfo : public MediaReceiverInfo {
675 uint32_t ssrc = 0;
676 };
677
678 struct BandwidthEstimationInfo {
679 int available_send_bandwidth = 0;
680 int available_recv_bandwidth = 0;
681 int target_enc_bitrate = 0;
682 int actual_enc_bitrate = 0;
683 int retransmit_bitrate = 0;
684 int transmit_bitrate = 0;
685 int64_t bucket_delay = 0;
686 };
687
688 // Maps from payload type to |RtpCodecParameters|.
689 typedef std::map<int, webrtc::RtpCodecParameters> RtpCodecParametersMap;
690
691 struct VoiceMediaInfo {
692 VoiceMediaInfo();
693 ~VoiceMediaInfo();
ClearVoiceMediaInfo694 void Clear() {
695 senders.clear();
696 receivers.clear();
697 send_codecs.clear();
698 receive_codecs.clear();
699 }
700 std::vector<VoiceSenderInfo> senders;
701 std::vector<VoiceReceiverInfo> receivers;
702 RtpCodecParametersMap send_codecs;
703 RtpCodecParametersMap receive_codecs;
704 int32_t device_underrun_count = 0;
705 };
706
707 struct VideoMediaInfo {
708 VideoMediaInfo();
709 ~VideoMediaInfo();
ClearVideoMediaInfo710 void Clear() {
711 senders.clear();
712 aggregated_senders.clear();
713 receivers.clear();
714 send_codecs.clear();
715 receive_codecs.clear();
716 }
717 // Each sender info represents one "outbound-rtp" stream.In non - simulcast,
718 // this means one info per RtpSender but if simulcast is used this means
719 // one info per simulcast layer.
720 std::vector<VideoSenderInfo> senders;
721 // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's
722 // "track" stats. If simulcast is used, instead of having one sender info per
723 // simulcast layer, the metrics of all layers of an RtpSender are aggregated
724 // into a single sender info per RtpSender.
725 std::vector<VideoSenderInfo> aggregated_senders;
726 std::vector<VideoReceiverInfo> receivers;
727 RtpCodecParametersMap send_codecs;
728 RtpCodecParametersMap receive_codecs;
729 };
730
731 struct DataMediaInfo {
732 DataMediaInfo();
733 ~DataMediaInfo();
ClearDataMediaInfo734 void Clear() {
735 senders.clear();
736 receivers.clear();
737 }
738 std::vector<DataSenderInfo> senders;
739 std::vector<DataReceiverInfo> receivers;
740 };
741
742 struct RtcpParameters {
743 bool reduced_size = false;
744 bool remote_estimate = false;
745 };
746
747 template <class Codec>
748 struct RtpParameters {
749 virtual ~RtpParameters() = default;
750
751 std::vector<Codec> codecs;
752 std::vector<webrtc::RtpExtension> extensions;
753 // For a send stream this is true if we've neogtiated a send direction,
754 // for a receive stream this is true if we've negotiated a receive direction.
755 bool is_stream_active = true;
756
757 // TODO(pthatcher): Add streams.
758 RtcpParameters rtcp;
759
ToStringRtpParameters760 std::string ToString() const {
761 rtc::StringBuilder ost;
762 ost << "{";
763 const char* separator = "";
764 for (const auto& entry : ToStringMap()) {
765 ost << separator << entry.first << ": " << entry.second;
766 separator = ", ";
767 }
768 ost << "}";
769 return ost.Release();
770 }
771
772 protected:
ToStringMapRtpParameters773 virtual std::map<std::string, std::string> ToStringMap() const {
774 return {{"codecs", VectorToString(codecs)},
775 {"extensions", VectorToString(extensions)}};
776 }
777 };
778
779 // TODO(deadbeef): Rename to RtpSenderParameters, since they're intended to
780 // encapsulate all the parameters needed for an RtpSender.
781 template <class Codec>
782 struct RtpSendParameters : RtpParameters<Codec> {
783 int max_bandwidth_bps = -1;
784 // This is the value to be sent in the MID RTP header extension (if the header
785 // extension in included in the list of extensions).
786 std::string mid;
787 bool extmap_allow_mixed = false;
788
789 protected:
ToStringMapRtpSendParameters790 std::map<std::string, std::string> ToStringMap() const override {
791 auto params = RtpParameters<Codec>::ToStringMap();
792 params["max_bandwidth_bps"] = rtc::ToString(max_bandwidth_bps);
793 params["mid"] = (mid.empty() ? "<not set>" : mid);
794 params["extmap-allow-mixed"] = extmap_allow_mixed ? "true" : "false";
795 return params;
796 }
797 };
798
799 struct AudioSendParameters : RtpSendParameters<AudioCodec> {
800 AudioSendParameters();
801 ~AudioSendParameters() override;
802 AudioOptions options;
803
804 protected:
805 std::map<std::string, std::string> ToStringMap() const override;
806 };
807
808 struct AudioRecvParameters : RtpParameters<AudioCodec> {};
809
810 class VoiceMediaChannel : public MediaChannel, public Delayable {
811 public:
VoiceMediaChannel()812 VoiceMediaChannel() {}
VoiceMediaChannel(const MediaConfig & config)813 explicit VoiceMediaChannel(const MediaConfig& config)
814 : MediaChannel(config) {}
~VoiceMediaChannel()815 ~VoiceMediaChannel() override {}
816
817 cricket::MediaType media_type() const override;
818 virtual bool SetSendParameters(const AudioSendParameters& params) = 0;
819 virtual bool SetRecvParameters(const AudioRecvParameters& params) = 0;
820 // Get the receive parameters for the incoming stream identified by |ssrc|.
821 virtual webrtc::RtpParameters GetRtpReceiveParameters(
822 uint32_t ssrc) const = 0;
823 // Retrieve the receive parameters for the default receive
824 // stream, which is used when SSRCs are not signaled.
825 virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
826 // Starts or stops playout of received audio.
827 virtual void SetPlayout(bool playout) = 0;
828 // Starts or stops sending (and potentially capture) of local audio.
829 virtual void SetSend(bool send) = 0;
830 // Configure stream for sending.
831 virtual bool SetAudioSend(uint32_t ssrc,
832 bool enable,
833 const AudioOptions* options,
834 AudioSource* source) = 0;
835 // Set speaker output volume of the specified ssrc.
836 virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0;
837 // Set speaker output volume for future unsignaled streams.
838 virtual bool SetDefaultOutputVolume(double volume) = 0;
839 // Returns if the telephone-event has been negotiated.
840 virtual bool CanInsertDtmf() = 0;
841 // Send a DTMF |event|. The DTMF out-of-band signal will be used.
842 // The |ssrc| should be either 0 or a valid send stream ssrc.
843 // The valid value for the |event| are 0 to 15 which corresponding to
844 // DTMF event 0-9, *, #, A-D.
845 virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0;
846 // Gets quality stats for the channel.
847 virtual bool GetStats(VoiceMediaInfo* info,
848 bool get_and_clear_legacy_stats) = 0;
849
850 virtual void SetRawAudioSink(
851 uint32_t ssrc,
852 std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
853 virtual void SetDefaultRawAudioSink(
854 std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
855
856 virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
857 };
858
859 // TODO(deadbeef): Rename to VideoSenderParameters, since they're intended to
860 // encapsulate all the parameters needed for a video RtpSender.
861 struct VideoSendParameters : RtpSendParameters<VideoCodec> {
862 VideoSendParameters();
863 ~VideoSendParameters() override;
864 // Use conference mode? This flag comes from the remote
865 // description's SDP line 'a=x-google-flag:conference', copied over
866 // by VideoChannel::SetRemoteContent_w, and ultimately used by
867 // conference mode screencast logic in
868 // WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig.
869 // The special screencast behaviour is disabled by default.
870 bool conference_mode = false;
871
872 protected:
873 std::map<std::string, std::string> ToStringMap() const override;
874 };
875
876 // TODO(deadbeef): Rename to VideoReceiverParameters, since they're intended to
877 // encapsulate all the parameters needed for a video RtpReceiver.
878 struct VideoRecvParameters : RtpParameters<VideoCodec> {};
879
880 class VideoMediaChannel : public MediaChannel, public Delayable {
881 public:
VideoMediaChannel()882 VideoMediaChannel() {}
VideoMediaChannel(const MediaConfig & config)883 explicit VideoMediaChannel(const MediaConfig& config)
884 : MediaChannel(config) {}
~VideoMediaChannel()885 ~VideoMediaChannel() override {}
886
887 cricket::MediaType media_type() const override;
888 virtual bool SetSendParameters(const VideoSendParameters& params) = 0;
889 virtual bool SetRecvParameters(const VideoRecvParameters& params) = 0;
890 // Get the receive parameters for the incoming stream identified by |ssrc|.
891 virtual webrtc::RtpParameters GetRtpReceiveParameters(
892 uint32_t ssrc) const = 0;
893 // Retrieve the receive parameters for the default receive
894 // stream, which is used when SSRCs are not signaled.
895 virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
896 // Gets the currently set codecs/payload types to be used for outgoing media.
897 virtual bool GetSendCodec(VideoCodec* send_codec) = 0;
898 // Starts or stops transmission (and potentially capture) of local video.
899 virtual bool SetSend(bool send) = 0;
900 // Configure stream for sending and register a source.
901 // The |ssrc| must correspond to a registered send stream.
902 virtual bool SetVideoSend(
903 uint32_t ssrc,
904 const VideoOptions* options,
905 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) = 0;
906 // Sets the sink object to be used for the specified stream.
907 virtual bool SetSink(uint32_t ssrc,
908 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
909 // The sink is used for the 'default' stream.
910 virtual void SetDefaultSink(
911 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
912 // This fills the "bitrate parts" (rtx, video bitrate) of the
913 // BandwidthEstimationInfo, since that part that isn't possible to get
914 // through webrtc::Call::GetStats, as they are statistics of the send
915 // streams.
916 // TODO(holmer): We should change this so that either BWE graphs doesn't
917 // need access to bitrates of the streams, or change the (RTC)StatsCollector
918 // so that it's getting the send stream stats separately by calling
919 // GetStats(), and merges with BandwidthEstimationInfo by itself.
920 virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0;
921 // Gets quality stats for the channel.
922 virtual bool GetStats(VideoMediaInfo* info) = 0;
923 // Set recordable encoded frame callback for |ssrc|
924 virtual void SetRecordableEncodedFrameCallback(
925 uint32_t ssrc,
926 std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
927 // Clear recordable encoded frame callback for |ssrc|
928 virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
929 // Cause generation of a keyframe for |ssrc|
930 virtual void GenerateKeyFrame(uint32_t ssrc) = 0;
931
932 virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
933 };
934
935 enum DataMessageType {
936 // Chrome-Internal use only. See SctpDataMediaChannel for the actual PPID
937 // values.
938 DMT_NONE = 0,
939 DMT_CONTROL = 1,
940 DMT_BINARY = 2,
941 DMT_TEXT = 3,
942 };
943
944 // Info about data received in DataMediaChannel. For use in
945 // DataMediaChannel::SignalDataReceived and in all of the signals that
946 // signal fires, on up the chain.
947 struct ReceiveDataParams {
948 // The in-packet stream indentifier.
949 // RTP data channels use SSRCs, SCTP data channels use SIDs.
950 union {
951 uint32_t ssrc;
952 int sid = 0;
953 };
954 // The type of message (binary, text, or control).
955 DataMessageType type = DMT_TEXT;
956 // A per-stream value incremented per packet in the stream.
957 int seq_num = 0;
958 // A per-stream value monotonically increasing with time.
959 int timestamp = 0;
960 };
961
962 struct SendDataParams {
963 // The in-packet stream indentifier.
964 // RTP data channels use SSRCs, SCTP data channels use SIDs.
965 union {
966 uint32_t ssrc;
967 int sid = 0;
968 };
969 // The type of message (binary, text, or control).
970 DataMessageType type = DMT_TEXT;
971
972 // TODO(pthatcher): Make |ordered| and |reliable| true by default?
973 // For SCTP, whether to send messages flagged as ordered or not.
974 // If false, messages can be received out of order.
975 bool ordered = false;
976 // For SCTP, whether the messages are sent reliably or not.
977 // If false, messages may be lost.
978 bool reliable = false;
979 // For SCTP, if reliable == false, provide partial reliability by
980 // resending up to this many times. Either count or millis
981 // is supported, not both at the same time.
982 int max_rtx_count = 0;
983 // For SCTP, if reliable == false, provide partial reliability by
984 // resending for up to this many milliseconds. Either count or millis
985 // is supported, not both at the same time.
986 int max_rtx_ms = 0;
987 };
988
989 enum SendDataResult { SDR_SUCCESS, SDR_ERROR, SDR_BLOCK };
990
991 struct DataSendParameters : RtpSendParameters<DataCodec> {};
992
993 struct DataRecvParameters : RtpParameters<DataCodec> {};
994
995 class DataMediaChannel : public MediaChannel {
996 public:
997 DataMediaChannel();
998 explicit DataMediaChannel(const MediaConfig& config);
999 ~DataMediaChannel() override;
1000
1001 cricket::MediaType media_type() const override;
1002 virtual bool SetSendParameters(const DataSendParameters& params) = 0;
1003 virtual bool SetRecvParameters(const DataRecvParameters& params) = 0;
1004
1005 // RtpParameter methods are not supported for Data channel.
1006 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
1007 webrtc::RTCError SetRtpSendParameters(
1008 uint32_t ssrc,
1009 const webrtc::RtpParameters& parameters) override;
1010
1011 // TODO(pthatcher): Implement this.
1012 virtual bool GetStats(DataMediaInfo* info);
1013
1014 virtual bool SetSend(bool send) = 0;
1015 virtual bool SetReceive(bool receive) = 0;
1016
OnNetworkRouteChanged(const std::string & transport_name,const rtc::NetworkRoute & network_route)1017 void OnNetworkRouteChanged(const std::string& transport_name,
1018 const rtc::NetworkRoute& network_route) override {}
1019
1020 virtual bool SendData(const SendDataParams& params,
1021 const rtc::CopyOnWriteBuffer& payload,
1022 SendDataResult* result = NULL) = 0;
1023 // Signals when data is received (params, data, len)
1024 sigslot::signal3<const ReceiveDataParams&, const char*, size_t>
1025 SignalDataReceived;
1026 // Signal when the media channel is ready to send the stream. Arguments are:
1027 // writable(bool)
1028 sigslot::signal1<bool> SignalReadyToSend;
1029 };
1030
1031 } // namespace cricket
1032
1033 #endif // MEDIA_BASE_MEDIA_CHANNEL_H_
1034