1 #include "GroupInstanceCustomImpl.h"
2
3 #include <memory>
4 #include <iomanip>
5
6 #include "Instance.h"
7 #include "VideoCaptureInterfaceImpl.h"
8 #include "VideoCapturerInterface.h"
9 #include "CodecSelectHelper.h"
10 #include "Message.h"
11 #include "platform/PlatformInterface.h"
12 #include "StaticThreads.h"
13 #include "GroupNetworkManager.h"
14
15 #include "api/audio_codecs/audio_decoder_factory_template.h"
16 #include "api/audio_codecs/audio_encoder_factory_template.h"
17 #include "api/audio_codecs/opus/audio_decoder_opus.h"
18 #include "api/audio_codecs/opus/audio_decoder_multi_channel_opus.h"
19 #include "api/audio_codecs/opus/audio_encoder_opus.h"
20 #include "api/audio_codecs/L16/audio_decoder_L16.h"
21 #include "api/audio_codecs/L16/audio_encoder_L16.h"
22 #include "api/task_queue/default_task_queue_factory.h"
23 #include "media/engine/webrtc_media_engine.h"
24 #include "system_wrappers/include/field_trial.h"
25 #include "api/video/builtin_video_bitrate_allocator_factory.h"
26 #include "call/call.h"
27 #include "modules/rtp_rtcp/source/rtp_utility.h"
28 #include "api/call/audio_sink.h"
29 #include "modules/audio_processing/audio_buffer.h"
30 #include "absl/strings/match.h"
31 #include "modules/audio_processing/agc2/vad_with_level.h"
32 #include "pc/channel_manager.h"
33 #include "media/base/rtp_data_engine.h"
34 #include "audio/audio_state.h"
35 #include "modules/audio_coding/neteq/default_neteq_factory.h"
36 #include "modules/audio_coding/include/audio_coding_module.h"
37 #include "common_audio/include/audio_util.h"
38 #include "modules/audio_device/include/audio_device_data_observer.h"
39 #include "common_audio/resampler/include/resampler.h"
40
41 #include "AudioFrame.h"
42 #include "ThreadLocalObject.h"
43 #include "Manager.h"
44 #include "NetworkManager.h"
45 #include "VideoCaptureInterfaceImpl.h"
46 #include "platform/PlatformInterface.h"
47 #include "LogSinkImpl.h"
48 #include "CodecSelectHelper.h"
49 #include "AudioStreamingPart.h"
50 #include "VideoStreamingPart.h"
51 #include "AudioDeviceHelper.h"
52 #include "FakeAudioDeviceModule.h"
53 #include "StreamingMediaContext.h"
54
55 #include <mutex>
56 #include <random>
57 #include <sstream>
58 #include <iostream>
59
60
61 #ifndef USE_RNNOISE
62 #define USE_RNNOISE 1
63 #endif
64
65 #if USE_RNNOISE
66 #include "rnnoise.h"
67 #endif
68
69 #include "GroupJoinPayloadInternal.h"
70
71 #include "third-party/json11.hpp"
72
73 namespace tgcalls {
74
75 namespace {
76
77 template <typename Out>
splitString(const std::string & s,char delim,Out result)78 void splitString(const std::string &s, char delim, Out result) {
79 std::istringstream iss(s);
80 std::string item;
81 while (std::getline(iss, item, delim)) {
82 *result++ = item;
83 }
84 }
85
splitString(const std::string & s,char delim)86 std::vector<std::string> splitString(const std::string &s, char delim) {
87 std::vector<std::string> elems;
88 splitString(s, delim, std::back_inserter(elems));
89 return elems;
90 }
91
stringToInt(std::string const & string)92 static int stringToInt(std::string const &string) {
93 std::stringstream stringStream(string);
94 int value = 0;
95 stringStream >> value;
96 return value;
97 }
98
intToString(int value)99 static std::string intToString(int value) {
100 std::ostringstream stringStream;
101 stringStream << value;
102 return stringStream.str();
103 }
104
uint32ToString(uint32_t value)105 static std::string uint32ToString(uint32_t value) {
106 std::ostringstream stringStream;
107 stringStream << value;
108 return stringStream.str();
109 }
110
stringToUInt32(std::string const & string)111 static uint32_t stringToUInt32(std::string const &string) {
112 std::stringstream stringStream(string);
113 uint32_t value = 0;
114 stringStream >> value;
115 return value;
116 }
117
stringToUInt16(std::string const & string)118 static uint16_t stringToUInt16(std::string const &string) {
119 std::stringstream stringStream(string);
120 uint16_t value = 0;
121 stringStream >> value;
122 return value;
123 }
124
formatTimestampMillis(int64_t timestamp)125 static std::string formatTimestampMillis(int64_t timestamp) {
126 std::ostringstream stringStream;
127 stringStream << std::fixed << std::setprecision(3) << (double)timestamp / 1000.0;
128 return stringStream.str();
129 }
130
GetVideoCaptureAssumingSameThread(VideoCaptureInterface * videoCapture)131 static VideoCaptureInterfaceObject *GetVideoCaptureAssumingSameThread(VideoCaptureInterface *videoCapture) {
132 return videoCapture
133 ? static_cast<VideoCaptureInterfaceImpl*>(videoCapture)->object()->getSyncAssumingSameThread()
134 : nullptr;
135 }
136
137 struct OutgoingVideoFormat {
138 cricket::VideoCodec videoCodec;
139 cricket::VideoCodec rtxCodec;
140 };
141
addDefaultFeedbackParams(cricket::VideoCodec * codec)142 static void addDefaultFeedbackParams(cricket::VideoCodec *codec) {
143 // Don't add any feedback params for RED and ULPFEC.
144 if (codec->name == cricket::kRedCodecName || codec->name == cricket::kUlpfecCodecName) {
145 return;
146 }
147 codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamRemb, cricket::kParamValueEmpty));
148 codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc, cricket::kParamValueEmpty));
149 // Don't add any more feedback params for FLEXFEC.
150 if (codec->name == cricket::kFlexfecCodecName) {
151 return;
152 }
153 codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamCcm, cricket::kRtcpFbCcmParamFir));
154 codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kParamValueEmpty));
155 codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kRtcpFbNackParamPli));
156 }
157
158 struct H264FormatParameters {
159 std::string profileLevelId;
160 std::string packetizationMode;
161 std::string levelAssymetryAllowed;
162 };
163
parseH264FormatParameters(webrtc::SdpVideoFormat const & format)164 H264FormatParameters parseH264FormatParameters(webrtc::SdpVideoFormat const &format) {
165 H264FormatParameters result;
166
167 for (const auto ¶meter : format.parameters) {
168 if (parameter.first == "profile-level-id") {
169 result.profileLevelId = parameter.second;
170 } else if (parameter.first == "packetization-mode") {
171 result.packetizationMode = parameter.second;
172 } else if (parameter.first == "level-asymmetry-allowed") {
173 result.levelAssymetryAllowed = parameter.second;
174 }
175 }
176
177 return result;
178 }
179
getH264ProfileLevelIdPriority(std::string const & profileLevelId)180 static int getH264ProfileLevelIdPriority(std::string const &profileLevelId) {
181 if (profileLevelId == cricket::kH264ProfileLevelConstrainedHigh) {
182 return 0;
183 } else if (profileLevelId == cricket::kH264ProfileLevelConstrainedBaseline) {
184 return 1;
185 } else {
186 return 2;
187 }
188 }
189
getH264PacketizationModePriority(std::string const & packetizationMode)190 static int getH264PacketizationModePriority(std::string const &packetizationMode) {
191 if (packetizationMode == "1") {
192 return 0;
193 } else {
194 return 1;
195 }
196 }
197
getH264LevelAssymetryAllowedPriority(std::string const & levelAssymetryAllowed)198 static int getH264LevelAssymetryAllowedPriority(std::string const &levelAssymetryAllowed) {
199 if (levelAssymetryAllowed == "1") {
200 return 0;
201 } else {
202 return 1;
203 }
204 }
205
filterSupportedVideoFormats(std::vector<webrtc::SdpVideoFormat> const & formats)206 static std::vector<webrtc::SdpVideoFormat> filterSupportedVideoFormats(std::vector<webrtc::SdpVideoFormat> const &formats) {
207 std::vector<webrtc::SdpVideoFormat> filteredFormats;
208
209 std::vector<std::string> filterCodecNames = {
210 cricket::kVp8CodecName,
211 cricket::kVp9CodecName,
212 cricket::kH264CodecName
213 };
214
215 std::vector<webrtc::SdpVideoFormat> vp9Formats;
216 std::vector<webrtc::SdpVideoFormat> h264Formats;
217
218 for (const auto &format : formats) {
219 if (std::find(filterCodecNames.begin(), filterCodecNames.end(), format.name) == filterCodecNames.end()) {
220 continue;
221 }
222
223 if (format.name == cricket::kVp9CodecName) {
224 vp9Formats.push_back(format);
225 } else if (format.name == cricket::kH264CodecName) {
226 h264Formats.push_back(format);
227 } else {
228 filteredFormats.push_back(format);
229 }
230 }
231
232 if (!vp9Formats.empty()) {
233 bool added = false;
234 for (const auto &format : vp9Formats) {
235 if (added) {
236 break;
237 }
238 for (const auto ¶meter : format.parameters) {
239 if (parameter.first == "profile-id") {
240 if (parameter.second == "0") {
241 filteredFormats.push_back(format);
242 added = true;
243 break;
244 }
245 }
246 }
247 }
248
249 if (!added) {
250 filteredFormats.push_back(vp9Formats[0]);
251 }
252 }
253
254 if (!h264Formats.empty()) {
255 std::sort(h264Formats.begin(), h264Formats.end(), [](const webrtc::SdpVideoFormat &lhs, const webrtc::SdpVideoFormat &rhs) {
256 auto lhsParameters = parseH264FormatParameters(lhs);
257 auto rhsParameters = parseH264FormatParameters(rhs);
258
259 int lhsLevelIdPriority = getH264ProfileLevelIdPriority(lhsParameters.profileLevelId);
260 int lhsPacketizationModePriority = getH264PacketizationModePriority(lhsParameters.packetizationMode);
261 int lhsLevelAssymetryAllowedPriority = getH264LevelAssymetryAllowedPriority(lhsParameters.levelAssymetryAllowed);
262
263 int rhsLevelIdPriority = getH264ProfileLevelIdPriority(rhsParameters.profileLevelId);
264 int rhsPacketizationModePriority = getH264PacketizationModePriority(rhsParameters.packetizationMode);
265 int rhsLevelAssymetryAllowedPriority = getH264LevelAssymetryAllowedPriority(rhsParameters.levelAssymetryAllowed);
266
267 if (lhsLevelIdPriority != rhsLevelIdPriority) {
268 return lhsLevelIdPriority < rhsLevelIdPriority;
269 }
270 if (lhsPacketizationModePriority != rhsPacketizationModePriority) {
271 return lhsPacketizationModePriority < rhsPacketizationModePriority;
272 }
273 if (lhsLevelAssymetryAllowedPriority != rhsLevelAssymetryAllowedPriority) {
274 return lhsLevelAssymetryAllowedPriority < rhsLevelAssymetryAllowedPriority;
275 }
276
277 return true;
278 });
279
280 filteredFormats.push_back(h264Formats[0]);
281 }
282
283 return filteredFormats;
284 }
285
assignPayloadTypes(std::vector<webrtc::SdpVideoFormat> const & formats)286 static std::vector<OutgoingVideoFormat> assignPayloadTypes(std::vector<webrtc::SdpVideoFormat> const &formats) {
287 if (formats.empty()) {
288 return {};
289 }
290
291 constexpr int kFirstDynamicPayloadType = 100;
292 constexpr int kLastDynamicPayloadType = 127;
293
294 int payload_type = kFirstDynamicPayloadType;
295
296 std::vector<OutgoingVideoFormat> result;
297
298 std::vector<std::string> filterCodecNames = {
299 cricket::kVp8CodecName,
300 cricket::kVp9CodecName,
301 cricket::kH264CodecName,
302 };
303
304 for (const auto &codecName : filterCodecNames) {
305 for (const auto &format : formats) {
306 if (format.name != codecName) {
307 continue;
308 }
309
310 cricket::VideoCodec codec(format);
311 codec.id = payload_type;
312 addDefaultFeedbackParams(&codec);
313
314 OutgoingVideoFormat resultFormat;
315
316 resultFormat.videoCodec = codec;
317
318 // Increment payload type.
319 ++payload_type;
320 if (payload_type > kLastDynamicPayloadType) {
321 RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
322 break;
323 }
324
325 // Add associated RTX codec for non-FEC codecs.
326 if (!absl::EqualsIgnoreCase(codec.name, cricket::kUlpfecCodecName) &&
327 !absl::EqualsIgnoreCase(codec.name, cricket::kFlexfecCodecName)) {
328 resultFormat.rtxCodec = cricket::VideoCodec::CreateRtxCodec(payload_type, codec.id);
329
330 // Increment payload type.
331 ++payload_type;
332 if (payload_type > kLastDynamicPayloadType) {
333 RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
334 break;
335 }
336 }
337
338 result.push_back(std::move(resultFormat));
339 }
340 }
341
342 return result;
343 }
344
345 struct VideoSsrcs {
346 struct SimulcastLayer {
347 uint32_t ssrc = 0;
348 uint32_t fidSsrc = 0;
349
SimulcastLayertgcalls::__anon2cee46520111::VideoSsrcs::SimulcastLayer350 SimulcastLayer(uint32_t ssrc_, uint32_t fidSsrc_) :
351 ssrc(ssrc_), fidSsrc(fidSsrc_) {
352 }
353
SimulcastLayertgcalls::__anon2cee46520111::VideoSsrcs::SimulcastLayer354 SimulcastLayer(const SimulcastLayer &other) :
355 ssrc(other.ssrc), fidSsrc(other.fidSsrc) {
356 }
357 };
358
359 std::vector<SimulcastLayer> simulcastLayers;
360
VideoSsrcstgcalls::__anon2cee46520111::VideoSsrcs361 VideoSsrcs() {
362 }
363
VideoSsrcstgcalls::__anon2cee46520111::VideoSsrcs364 VideoSsrcs(const VideoSsrcs &other) :
365 simulcastLayers(other.simulcastLayers) {
366 }
367 };
368
369 struct InternalGroupLevelValue {
370 GroupLevelValue value;
371 int64_t timestamp = 0;
372 };
373
374 struct ChannelId {
375 uint32_t networkSsrc = 0;
376 uint32_t actualSsrc = 0;
377
ChannelIdtgcalls::__anon2cee46520111::ChannelId378 ChannelId(uint32_t networkSsrc_, uint32_t actualSsrc_) :
379 networkSsrc(networkSsrc_),
380 actualSsrc(actualSsrc_) {
381 }
382
ChannelIdtgcalls::__anon2cee46520111::ChannelId383 explicit ChannelId(uint32_t networkSsrc_) :
384 networkSsrc(networkSsrc_),
385 actualSsrc(networkSsrc_) {
386 }
387
operator <tgcalls::__anon2cee46520111::ChannelId388 bool operator <(const ChannelId& rhs) const {
389 if (networkSsrc != rhs.networkSsrc) {
390 return networkSsrc < rhs.networkSsrc;
391 }
392 return actualSsrc < rhs.actualSsrc;
393 }
394
nametgcalls::__anon2cee46520111::ChannelId395 std::string name() {
396 if (networkSsrc == actualSsrc) {
397 return uint32ToString(networkSsrc);
398 } else {
399 return uint32ToString(networkSsrc) + "to" + uint32ToString(actualSsrc);
400 }
401 }
402 };
403
404 struct VideoChannelId {
405 std::string endpointId;
406
VideoChannelIdtgcalls::__anon2cee46520111::VideoChannelId407 explicit VideoChannelId(std::string const &endpointId_) :
408 endpointId(endpointId_) {
409 }
410
operator <tgcalls::__anon2cee46520111::VideoChannelId411 bool operator <(const VideoChannelId& rhs) const {
412 return endpointId < rhs.endpointId;
413 }
414 };
415
416 struct ChannelSsrcInfo {
417 enum class Type {
418 Audio,
419 Video
420 };
421
422 Type type = Type::Audio;
423 std::vector<uint32_t> allSsrcs;
424 std::string videoEndpointId;
425 };
426
427 struct RequestedMediaChannelDescriptions {
428 std::shared_ptr<RequestMediaChannelDescriptionTask> task;
429 std::vector<uint32_t> ssrcs;
430
RequestedMediaChannelDescriptionstgcalls::__anon2cee46520111::RequestedMediaChannelDescriptions431 RequestedMediaChannelDescriptions(std::shared_ptr<RequestMediaChannelDescriptionTask> task_, std::vector<uint32_t> ssrcs_) :
432 task(task_), ssrcs(std::move(ssrcs_)) {
433 }
434 };
435
436 static const int kVadResultHistoryLength = 8;
437
438 class VadHistory {
439 private:
440 float _vadResultHistory[kVadResultHistoryLength];
441
442 public:
VadHistory()443 VadHistory() {
444 for (int i = 0; i < kVadResultHistoryLength; i++) {
445 _vadResultHistory[i] = 0.0f;
446 }
447 }
448
~VadHistory()449 ~VadHistory() {
450 }
451
update(float vadProbability)452 bool update(float vadProbability) {
453 for (int i = 1; i < kVadResultHistoryLength; i++) {
454 _vadResultHistory[i - 1] = _vadResultHistory[i];
455 }
456 _vadResultHistory[kVadResultHistoryLength - 1] = vadProbability;
457
458 float movingAverage = 0.0f;
459 for (int i = 0; i < kVadResultHistoryLength; i++) {
460 movingAverage += _vadResultHistory[i];
461 }
462 movingAverage /= (float)kVadResultHistoryLength;
463
464 bool vadResult = false;
465 if (movingAverage > 0.8f) {
466 vadResult = true;
467 }
468
469 return vadResult;
470 }
471 };
472
473 class CombinedVad {
474 private:
475 webrtc::VadLevelAnalyzer _vadWithLevel;
476 VadHistory _history;
477
478 public:
CombinedVad()479 CombinedVad() {
480 }
481
~CombinedVad()482 ~CombinedVad() {
483 }
484
update(webrtc::AudioBuffer * buffer)485 bool update(webrtc::AudioBuffer *buffer) {
486 if (buffer->num_channels() <= 0) {
487 return _history.update(0.0f);
488 }
489 webrtc::AudioFrameView<float> frameView(buffer->channels(), buffer->num_channels(), buffer->num_frames());
490 float peak = 0.0f;
491 for (const auto &x : frameView.channel(0)) {
492 peak = std::max(std::fabs(x), peak);
493 }
494 if (peak <= 0.01f) {
495 return _history.update(false);
496 }
497
498 auto result = _vadWithLevel.AnalyzeFrame(frameView);
499
500 return _history.update(result.speech_probability);
501 }
502
update()503 bool update() {
504 return _history.update(0.0f);
505 }
506 };
507
508 class SparseVad {
509 public:
SparseVad()510 SparseVad() {
511 }
512
update(webrtc::AudioBuffer * buffer)513 bool update(webrtc::AudioBuffer *buffer) {
514 _sampleCount += buffer->num_frames();
515 if (_sampleCount < 400) {
516 return _currentValue;
517 }
518 _sampleCount = 0;
519
520 _currentValue = _vad.update(buffer);
521
522 return _currentValue;
523 }
524
525 private:
526 CombinedVad _vad;
527 bool _currentValue = false;
528 size_t _sampleCount = 0;
529 };
530
531 class AudioSinkImpl: public webrtc::AudioSinkInterface {
532 public:
533 struct Update {
534 float level = 0.0f;
535 bool hasSpeech = false;
536
Updatetgcalls::__anon2cee46520111::AudioSinkImpl::Update537 Update(float level_, bool hasSpech_) :
538 level(level_), hasSpeech(hasSpech_) {
539 }
540
Updatetgcalls::__anon2cee46520111::AudioSinkImpl::Update541 Update(const Update &other) :
542 level(other.level), hasSpeech(other.hasSpeech) {
543 }
544 };
545
546 public:
AudioSinkImpl(std::function<void (Update)> update,ChannelId channel_id,std::function<void (uint32_t,const AudioFrame &)> onAudioFrame)547 AudioSinkImpl(std::function<void(Update)> update,
548 ChannelId channel_id, std::function<void(uint32_t, const AudioFrame &)> onAudioFrame) :
549 _update(update), _channel_id(channel_id), _onAudioFrame(std::move(onAudioFrame)) {
550 }
551
~AudioSinkImpl()552 virtual ~AudioSinkImpl() {
553 }
554
OnData(const Data & audio)555 virtual void OnData(const Data& audio) override {
556 if (_onAudioFrame) {
557 AudioFrame frame;
558 frame.audio_samples = audio.data;
559 frame.num_samples = audio.samples_per_channel;
560 frame.bytes_per_sample = 2;
561 frame.num_channels = audio.channels;
562 frame.samples_per_sec = audio.sample_rate;
563 frame.elapsed_time_ms = 0;
564 frame.ntp_time_ms = 0;
565 _onAudioFrame(_channel_id.actualSsrc, frame);
566 }
567 if (_update && audio.channels == 1) {
568 const int16_t *samples = (const int16_t *)audio.data;
569 int numberOfSamplesInFrame = (int)audio.samples_per_channel;
570
571 int16_t currentPeak = 0;
572 for (int i = 0; i < numberOfSamplesInFrame; i++) {
573 int16_t sample = samples[i];
574 if (sample < 0) {
575 sample = -sample;
576 }
577 if (_peak < sample) {
578 _peak = sample;
579 }
580 if (currentPeak < sample) {
581 currentPeak = sample;
582 }
583 _peakCount += 1;
584 }
585
586 /*bool vadResult = false;
587 if (currentPeak > 10) {
588 webrtc::AudioBuffer buffer(audio.sample_rate, 1, 48000, 1, 48000, 1);
589 webrtc::StreamConfig config(audio.sample_rate, 1);
590 buffer.CopyFrom(samples, config);
591
592 vadResult = _vad.update(&buffer);
593 } else {
594 vadResult = _vad.update();
595 }*/
596
597 if (_peakCount >= 4400) {
598 float level = ((float)(_peak)) / 8000.0f;
599 _peak = 0;
600 _peakCount = 0;
601 _update(Update(level, level >= 1.0f));
602 }
603 }
604 }
605
606 private:
607 std::function<void(Update)> _update;
608 ChannelId _channel_id;
609 std::function<void(uint32_t, const AudioFrame &)> _onAudioFrame;
610
611 int _peakCount = 0;
612 uint16_t _peak = 0;
613
614 CombinedVad _vad;
615
616 };
617
618 class VideoSinkImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
619 public:
VideoSinkImpl(std::string const & endpointId)620 VideoSinkImpl(std::string const &endpointId) :
621 _endpointId(endpointId) {
622 }
623
~VideoSinkImpl()624 virtual ~VideoSinkImpl() {
625 }
626
OnFrame(const webrtc::VideoFrame & frame)627 virtual void OnFrame(const webrtc::VideoFrame& frame) override {
628 std::unique_lock<std::mutex> lock{ _mutex };
629 int64_t timestamp = rtc::TimeMillis();
630 if (_lastFrame) {
631 if (_lastFrame->video_frame_buffer()->width() != frame.video_frame_buffer()->width()) {
632 int64_t deltaTime = std::abs(_lastFrameSizeChangeTimestamp - timestamp);
633 if (deltaTime < 200) {
634 RTC_LOG(LS_WARNING) << "VideoSinkImpl: frequent frame size change detected for " << _endpointId << ": " << _lastFrameSizeChangeHeight << " -> " << _lastFrame->video_frame_buffer()->height() << " -> " << frame.video_frame_buffer()->height() << " in " << deltaTime << " ms";
635 }
636
637 _lastFrameSizeChangeHeight = _lastFrame->video_frame_buffer()->height();
638 _lastFrameSizeChangeTimestamp = timestamp;
639 }
640 } else {
641 _lastFrameSizeChangeHeight = 0;
642 _lastFrameSizeChangeTimestamp = timestamp;
643 }
644 _lastFrame = frame;
645 for (int i = (int)(_sinks.size()) - 1; i >= 0; i--) {
646 auto strong = _sinks[i].lock();
647 if (!strong) {
648 _sinks.erase(_sinks.begin() + i);
649 } else {
650 strong->OnFrame(frame);
651 }
652 }
653 }
654
OnDiscardedFrame()655 virtual void OnDiscardedFrame() override {
656 std::unique_lock<std::mutex> lock{ _mutex };
657 for (int i = (int)(_sinks.size()) - 1; i >= 0; i--) {
658 auto strong = _sinks[i].lock();
659 if (!strong) {
660 _sinks.erase(_sinks.begin() + i);
661 } else {
662 strong->OnDiscardedFrame();
663 }
664 }
665 }
666
addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl)667 void addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl) {
668 std::unique_lock<std::mutex> lock{ _mutex };
669 _sinks.push_back(impl);
670 if (_lastFrame) {
671 auto strong = impl.lock();
672 if (strong) {
673 strong->OnFrame(_lastFrame.value());
674 }
675 }
676 }
677
getSinks()678 std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> getSinks() {
679 return _sinks;
680 }
681
682 private:
683 std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> _sinks;
684 absl::optional<webrtc::VideoFrame> _lastFrame;
685 std::mutex _mutex;
686 int64_t _lastFrameSizeChangeTimestamp = 0;
687 int _lastFrameSizeChangeHeight = 0;
688 std::string _endpointId;
689
690 };
691
692 struct NoiseSuppressionConfiguration {
NoiseSuppressionConfigurationtgcalls::__anon2cee46520111::NoiseSuppressionConfiguration693 NoiseSuppressionConfiguration(bool isEnabled_) :
694 isEnabled(isEnabled_) {
695
696 }
697
698 bool isEnabled = false;
699 };
700
701 #if USE_RNNOISE
702 class AudioCapturePostProcessor : public webrtc::CustomProcessing {
703 public:
AudioCapturePostProcessor(std::function<void (GroupLevelValue const &)> updated,std::shared_ptr<NoiseSuppressionConfiguration> noiseSuppressionConfiguration,std::vector<float> * externalAudioSamples,webrtc::Mutex * externalAudioSamplesMutex)704 AudioCapturePostProcessor(std::function<void(GroupLevelValue const &)> updated, std::shared_ptr<NoiseSuppressionConfiguration> noiseSuppressionConfiguration, std::vector<float> *externalAudioSamples, webrtc::Mutex *externalAudioSamplesMutex) :
705 _updated(updated),
706 _noiseSuppressionConfiguration(noiseSuppressionConfiguration),
707 _externalAudioSamples(externalAudioSamples),
708 _externalAudioSamplesMutex(externalAudioSamplesMutex) {
709 int frameSize = rnnoise_get_frame_size();
710 _frameSamples.resize(frameSize);
711
712 _denoiseState = rnnoise_create(nullptr);
713 }
714
~AudioCapturePostProcessor()715 virtual ~AudioCapturePostProcessor() {
716 if (_denoiseState) {
717 rnnoise_destroy(_denoiseState);
718 }
719 }
720
721 private:
Initialize(int sample_rate_hz,int num_channels)722 virtual void Initialize(int sample_rate_hz, int num_channels) override {
723 }
724
Process(webrtc::AudioBuffer * buffer)725 virtual void Process(webrtc::AudioBuffer *buffer) override {
726 if (!buffer) {
727 return;
728 }
729 if (buffer->num_channels() != 1) {
730 return;
731 }
732 if (!_denoiseState) {
733 return;
734 }
735 if (buffer->num_frames() != _frameSamples.size()) {
736 return;
737 }
738
739 float sourcePeak = 0.0f;
740 float *sourceSamples = buffer->channels()[0];
741 for (int i = 0; i < _frameSamples.size(); i++) {
742 sourcePeak = std::max(std::fabs(sourceSamples[i]), sourcePeak);
743 }
744
745 if (_noiseSuppressionConfiguration->isEnabled) {
746 float vadProbability = 0.0f;
747 if (sourcePeak >= 0.01f) {
748 vadProbability = rnnoise_process_frame(_denoiseState, _frameSamples.data(), buffer->channels()[0]);
749 if (_noiseSuppressionConfiguration->isEnabled) {
750 memcpy(buffer->channels()[0], _frameSamples.data(), _frameSamples.size() * sizeof(float));
751 }
752 }
753
754 float peak = 0;
755 int peakCount = 0;
756 const float *samples = buffer->channels_const()[0];
757 for (int i = 0; i < buffer->num_frames(); i++) {
758 float sample = samples[i];
759 if (sample < 0) {
760 sample = -sample;
761 }
762 if (peak < sample) {
763 peak = sample;
764 }
765 peakCount += 1;
766 }
767
768 bool vadStatus = _history.update(vadProbability);
769
770 _peakCount += peakCount;
771 if (_peak < peak) {
772 _peak = peak;
773 }
774 if (_peakCount >= 4400) {
775 float level = _peak / 4000.0f;
776 _peak = 0;
777 _peakCount = 0;
778
779 _updated(GroupLevelValue{
780 level,
781 vadStatus,
782 });
783 }
784 } else {
785 float peak = 0;
786 int peakCount = 0;
787 const float *samples = buffer->channels_const()[0];
788 for (int i = 0; i < buffer->num_frames(); i++) {
789 float sample = samples[i];
790 if (sample < 0) {
791 sample = -sample;
792 }
793 if (peak < sample) {
794 peak = sample;
795 }
796 peakCount += 1;
797 }
798
799 _peakCount += peakCount;
800 if (_peak < peak) {
801 _peak = peak;
802 }
803 if (_peakCount >= 1200) {
804 float level = _peak / 8000.0f;
805 _peak = 0;
806 _peakCount = 0;
807
808 _updated(GroupLevelValue{
809 level,
810 level >= 1.0f,
811 });
812 }
813 }
814
815 if (_externalAudioSamplesMutex && _externalAudioSamples) {
816 _externalAudioSamplesMutex->Lock();
817 if (!_externalAudioSamples->empty()) {
818 float *bufferData = buffer->channels()[0];
819 int takenSamples = 0;
820 for (int i = 0; i < _externalAudioSamples->size() && i < _frameSamples.size(); i++) {
821 float sample = (*_externalAudioSamples)[i];
822 sample += bufferData[i];
823 sample = std::min(sample, 32768.f);
824 sample = std::max(sample, -32768.f);
825 bufferData[i] = sample;
826 takenSamples++;
827 }
828 if (takenSamples != 0) {
829 _externalAudioSamples->erase(_externalAudioSamples->begin(), _externalAudioSamples->begin() + takenSamples);
830 }
831 }
832 _externalAudioSamplesMutex->Unlock();
833 }
834 }
835
ToString() const836 virtual std::string ToString() const override {
837 return "CustomPostProcessing";
838 }
839
SetRuntimeSetting(webrtc::AudioProcessing::RuntimeSetting setting)840 virtual void SetRuntimeSetting(webrtc::AudioProcessing::RuntimeSetting setting) override {
841 }
842
843 private:
844 std::function<void(GroupLevelValue const &)> _updated;
845 std::shared_ptr<NoiseSuppressionConfiguration> _noiseSuppressionConfiguration;
846
847 DenoiseState *_denoiseState = nullptr;
848 std::vector<float> _frameSamples;
849 int32_t _peakCount = 0;
850 float _peak = 0;
851 VadHistory _history;
852 SparseVad _vad;
853
854 std::vector<float> *_externalAudioSamples = nullptr;
855 webrtc::Mutex *_externalAudioSamplesMutex = nullptr;
856 };
857 #endif
858
859 class ExternalAudioRecorder : public FakeAudioDeviceModule::Recorder {
860 public:
ExternalAudioRecorder(std::vector<float> * externalAudioSamples,webrtc::Mutex * externalAudioSamplesMutex)861 ExternalAudioRecorder(std::vector<float> *externalAudioSamples, webrtc::Mutex *externalAudioSamplesMutex) :
862 _externalAudioSamples(externalAudioSamples),
863 _externalAudioSamplesMutex(externalAudioSamplesMutex) {
864 _samples.resize(480);
865 }
866
~ExternalAudioRecorder()867 virtual ~ExternalAudioRecorder() {
868 }
869
Record()870 virtual AudioFrame Record() override {
871 AudioFrame result;
872
873 _externalAudioSamplesMutex->Lock();
874 if (!_externalAudioSamples->empty() && _externalAudioSamples->size() >= 480) {
875 size_t takenSamples = std::min(_samples.size(), _externalAudioSamples->size());
876 webrtc::FloatS16ToS16(_externalAudioSamples->data(), takenSamples, _samples.data());
877
878 result.num_samples = takenSamples;
879
880 if (takenSamples != 0) {
881 _externalAudioSamples->erase(_externalAudioSamples->begin(), _externalAudioSamples->begin() + takenSamples);
882 }
883 } else {
884 result.num_samples = 0;
885 }
886 _externalAudioSamplesMutex->Unlock();
887
888 result.audio_samples = _samples.data();
889 result.bytes_per_sample = 2;
890 result.num_channels = 1;
891 result.samples_per_sec = 48000;
892 result.elapsed_time_ms = 0;
893 result.ntp_time_ms = 0;
894
895 return result;
896 }
897
WaitForUs()898 virtual int32_t WaitForUs() override {
899 _externalAudioSamplesMutex->Lock();
900 _externalAudioSamplesMutex->Unlock();
901
902 return 1000;
903 }
904
905 private:
906 std::vector<float> *_externalAudioSamples = nullptr;
907 webrtc::Mutex *_externalAudioSamplesMutex = nullptr;
908 std::vector<int16_t> _samples;
909 };
910
911 class IncomingAudioChannel : public sigslot::has_slots<> {
912 public:
IncomingAudioChannel(cricket::ChannelManager * channelManager,webrtc::Call * call,webrtc::RtpTransport * rtpTransport,rtc::UniqueRandomIdGenerator * randomIdGenerator,bool isRawPcm,ChannelId ssrc,std::function<void (AudioSinkImpl::Update)> && onAudioLevelUpdated,std::function<void (uint32_t,const AudioFrame &)> onAudioFrame,std::shared_ptr<Threads> threads)913 IncomingAudioChannel(
914 cricket::ChannelManager *channelManager,
915 webrtc::Call *call,
916 webrtc::RtpTransport *rtpTransport,
917 rtc::UniqueRandomIdGenerator *randomIdGenerator,
918 bool isRawPcm,
919 ChannelId ssrc,
920 std::function<void(AudioSinkImpl::Update)> &&onAudioLevelUpdated,
921 std::function<void(uint32_t, const AudioFrame &)> onAudioFrame,
922 std::shared_ptr<Threads> threads) :
923 _threads(threads),
924 _ssrc(ssrc),
925 _channelManager(channelManager),
926 _call(call) {
927 _creationTimestamp = rtc::TimeMillis();
928
929 threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, rtpTransport, ssrc, onAudioFrame = std::move(onAudioFrame), onAudioLevelUpdated = std::move(onAudioLevelUpdated), randomIdGenerator, isRawPcm]() mutable {
930 cricket::AudioOptions audioOptions;
931 audioOptions.audio_jitter_buffer_fast_accelerate = true;
932 audioOptions.audio_jitter_buffer_min_delay_ms = 50;
933
934 std::string streamId = std::string("stream") + ssrc.name();
935
936 _audioChannel = _channelManager->CreateVoiceChannel(_call, cricket::MediaConfig(), rtpTransport, _threads->getWorkerThread(), std::string("audio") + uint32ToString(ssrc.networkSsrc), false, GroupNetworkManager::getDefaulCryptoOptions(), randomIdGenerator, audioOptions);
937
938 const uint8_t opusPTimeMs = 120;
939
940 cricket::AudioCodec opusCodec(111, "opus", 48000, 0, 2);
941 opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1);
942 opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs);
943
944 cricket::AudioCodec pcmCodec(112, "l16", 48000, 0, 1);
945
946 auto outgoingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
947 if (!isRawPcm) {
948 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
949 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
950 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
951 }
952 outgoingAudioDescription->set_rtcp_mux(true);
953 outgoingAudioDescription->set_rtcp_reduced_size(true);
954 outgoingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
955 outgoingAudioDescription->set_codecs({ opusCodec, pcmCodec });
956 outgoingAudioDescription->set_bandwidth(1300000);
957
958 auto incomingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
959 if (!isRawPcm) {
960 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
961 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
962 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
963 }
964 incomingAudioDescription->set_rtcp_mux(true);
965 incomingAudioDescription->set_rtcp_reduced_size(true);
966 incomingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
967 incomingAudioDescription->set_codecs({ opusCodec, pcmCodec });
968 incomingAudioDescription->set_bandwidth(1300000);
969 cricket::StreamParams streamParams = cricket::StreamParams::CreateLegacy(ssrc.networkSsrc);
970 streamParams.set_stream_ids({ streamId });
971 incomingAudioDescription->AddStream(streamParams);
972
973 _audioChannel->SetLocalContent(outgoingAudioDescription.get(), webrtc::SdpType::kOffer, nullptr);
974 _audioChannel->SetRemoteContent(incomingAudioDescription.get(), webrtc::SdpType::kAnswer, nullptr);
975 _audioChannel->SetPayloadTypeDemuxingEnabled(false);
976
977 outgoingAudioDescription.reset();
978 incomingAudioDescription.reset();
979
980 if (_ssrc.actualSsrc != 1) {
981 std::unique_ptr<AudioSinkImpl> audioLevelSink(new AudioSinkImpl(std::move(onAudioLevelUpdated), _ssrc, std::move(onAudioFrame)));
982 _audioChannel->media_channel()->SetRawAudioSink(ssrc.networkSsrc, std::move(audioLevelSink));
983 }
984 });
985
986 //_audioChannel->SignalSentPacket().connect(this, &IncomingAudioChannel::OnSentPacket_w);
987
988 _audioChannel->Enable(true);
989 }
990
~IncomingAudioChannel()991 ~IncomingAudioChannel() {
992 //_audioChannel->SignalSentPacket().disconnect(this);
993 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
994 _channelManager->DestroyVoiceChannel(_audioChannel);
995 _audioChannel = nullptr;
996 });
997 }
998
setVolume(double value)999 void setVolume(double value) {
1000 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, value]() {
1001 _audioChannel->media_channel()->SetOutputVolume(_ssrc.networkSsrc, value);
1002 });
1003 }
1004
updateActivity()1005 void updateActivity() {
1006 _activityTimestamp = rtc::TimeMillis();
1007 }
1008
getActivity()1009 int64_t getActivity() {
1010 return _activityTimestamp;
1011 }
1012
1013 private:
OnSentPacket_w(const rtc::SentPacket & sent_packet)1014 void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
1015 _call->OnSentPacket(sent_packet);
1016 }
1017
1018 private:
1019 std::shared_ptr<Threads> _threads;
1020 ChannelId _ssrc;
1021 // Memory is managed by _channelManager
1022 cricket::VoiceChannel *_audioChannel = nullptr;
1023 // Memory is managed externally
1024 cricket::ChannelManager *_channelManager = nullptr;
1025 webrtc::Call *_call = nullptr;
1026 int64_t _creationTimestamp = 0;
1027 int64_t _activityTimestamp = 0;
1028 };
1029
1030 class IncomingVideoChannel : public sigslot::has_slots<> {
1031 public:
IncomingVideoChannel(cricket::ChannelManager * channelManager,webrtc::Call * call,webrtc::RtpTransport * rtpTransport,rtc::UniqueRandomIdGenerator * randomIdGenerator,std::vector<webrtc::SdpVideoFormat> const & availableVideoFormats,GroupJoinVideoInformation sharedVideoInformation,uint32_t audioSsrc,VideoChannelDescription::Quality minQuality,VideoChannelDescription::Quality maxQuality,GroupParticipantVideoInformation const & description,std::shared_ptr<Threads> threads)1032 IncomingVideoChannel(
1033 cricket::ChannelManager *channelManager,
1034 webrtc::Call *call,
1035 webrtc::RtpTransport *rtpTransport,
1036 rtc::UniqueRandomIdGenerator *randomIdGenerator,
1037 std::vector<webrtc::SdpVideoFormat> const &availableVideoFormats,
1038 GroupJoinVideoInformation sharedVideoInformation,
1039 uint32_t audioSsrc,
1040 VideoChannelDescription::Quality minQuality,
1041 VideoChannelDescription::Quality maxQuality,
1042 GroupParticipantVideoInformation const &description,
1043 std::shared_ptr<Threads> threads) :
1044 _threads(threads),
1045 _endpointId(description.endpointId),
1046 _channelManager(channelManager),
1047 _call(call),
1048 _requestedMinQuality(minQuality),
1049 _requestedMaxQuality(maxQuality) {
1050 _videoSink.reset(new VideoSinkImpl(_endpointId));
1051
1052 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, rtpTransport, &availableVideoFormats, &description, randomIdGenerator]() mutable {
1053 uint32_t mid = randomIdGenerator->GenerateId();
1054 std::string streamId = std::string("video") + uint32ToString(mid);
1055
1056 _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory();
1057
1058 auto payloadTypes = assignPayloadTypes(availableVideoFormats);
1059 std::vector<cricket::VideoCodec> codecs;
1060 for (const auto &payloadType : payloadTypes) {
1061 codecs.push_back(payloadType.videoCodec);
1062 codecs.push_back(payloadType.rtxCodec);
1063 }
1064
1065 auto outgoingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1066 outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1067 outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1068 outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kVideoRotationUri, 13));
1069 outgoingVideoDescription->set_rtcp_mux(true);
1070 outgoingVideoDescription->set_rtcp_reduced_size(true);
1071 outgoingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1072 outgoingVideoDescription->set_codecs(codecs);
1073 outgoingVideoDescription->set_bandwidth(1300000);
1074
1075 cricket::StreamParams videoRecvStreamParams;
1076
1077 std::vector<uint32_t> allSsrcs;
1078 for (const auto &group : description.ssrcGroups) {
1079 for (auto ssrc : group.ssrcs) {
1080 if (std::find(allSsrcs.begin(), allSsrcs.end(), ssrc) == allSsrcs.end()) {
1081 allSsrcs.push_back(ssrc);
1082 }
1083 }
1084
1085 if (group.semantics == "SIM") {
1086 if (_mainVideoSsrc == 0) {
1087 _mainVideoSsrc = group.ssrcs[0];
1088 }
1089 }
1090
1091 cricket::SsrcGroup parsedGroup(group.semantics, group.ssrcs);
1092 videoRecvStreamParams.ssrc_groups.push_back(parsedGroup);
1093 }
1094 videoRecvStreamParams.ssrcs = allSsrcs;
1095
1096 if (_mainVideoSsrc == 0) {
1097 if (description.ssrcGroups.size() == 1) {
1098 _mainVideoSsrc = description.ssrcGroups[0].ssrcs[0];
1099 }
1100 }
1101
1102 videoRecvStreamParams.cname = "cname";
1103 videoRecvStreamParams.set_stream_ids({ streamId });
1104
1105 auto incomingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1106 incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1107 incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1108 incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kVideoRotationUri, 13));
1109 incomingVideoDescription->set_rtcp_mux(true);
1110 incomingVideoDescription->set_rtcp_reduced_size(true);
1111 incomingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1112 incomingVideoDescription->set_codecs(codecs);
1113 incomingVideoDescription->set_bandwidth(1300000);
1114
1115 incomingVideoDescription->AddStream(videoRecvStreamParams);
1116
1117 _videoChannel = _channelManager->CreateVideoChannel(_call, cricket::MediaConfig(), rtpTransport, _threads->getWorkerThread(), std::string("video") + uint32ToString(mid), false, GroupNetworkManager::getDefaulCryptoOptions(), randomIdGenerator, cricket::VideoOptions(), _videoBitrateAllocatorFactory.get());
1118
1119 _videoChannel->SetLocalContent(outgoingVideoDescription.get(), webrtc::SdpType::kOffer, nullptr);
1120 _videoChannel->SetRemoteContent(incomingVideoDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1121 _videoChannel->SetPayloadTypeDemuxingEnabled(false);
1122 _videoChannel->media_channel()->SetSink(_mainVideoSsrc, _videoSink.get());
1123 _videoChannel->Enable(true);
1124 });
1125
1126 //_videoChannel->SignalSentPacket().connect(this, &IncomingVideoChannel::OnSentPacket_w);
1127 }
1128
~IncomingVideoChannel()1129 ~IncomingVideoChannel() {
1130 //_videoChannel->SignalSentPacket().disconnect(this);
1131 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1132 _videoChannel->Enable(false);
1133 _channelManager->DestroyVideoChannel(_videoChannel);
1134 _videoChannel = nullptr;
1135 });
1136 }
1137
addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl)1138 void addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl) {
1139 _videoSink->addSink(impl);
1140 }
1141
getSinks()1142 std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> getSinks() {
1143 return _videoSink->getSinks();
1144 }
1145
endpointId()1146 std::string const &endpointId() {
1147 return _endpointId;
1148 }
1149
requestedMinQuality()1150 VideoChannelDescription::Quality requestedMinQuality() {
1151 return _requestedMinQuality;
1152 }
1153
requestedMaxQuality()1154 VideoChannelDescription::Quality requestedMaxQuality() {
1155 return _requestedMaxQuality;
1156 }
1157
setRequstedMinQuality(VideoChannelDescription::Quality quality)1158 void setRequstedMinQuality(VideoChannelDescription::Quality quality) {
1159 _requestedMinQuality = quality;
1160 }
1161
setRequstedMaxQuality(VideoChannelDescription::Quality quality)1162 void setRequstedMaxQuality(VideoChannelDescription::Quality quality) {
1163 _requestedMaxQuality = quality;
1164 }
1165
setStats(absl::optional<GroupInstanceStats::IncomingVideoStats> stats)1166 void setStats(absl::optional<GroupInstanceStats::IncomingVideoStats> stats) {
1167 _stats = stats;
1168 }
1169
getStats()1170 absl::optional<GroupInstanceStats::IncomingVideoStats> getStats() {
1171 return _stats;
1172 }
1173
1174 private:
OnSentPacket_w(const rtc::SentPacket & sent_packet)1175 void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
1176 //_call->OnSentPacket(sent_packet);
1177 }
1178
1179 private:
1180 std::shared_ptr<Threads> _threads;
1181 uint32_t _mainVideoSsrc = 0;
1182 std::string _endpointId;
1183 std::unique_ptr<VideoSinkImpl> _videoSink;
1184 std::vector<GroupJoinPayloadVideoSourceGroup> _ssrcGroups;
1185 std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> _videoBitrateAllocatorFactory;
1186 // Memory is managed by _channelManager
1187 cricket::VideoChannel *_videoChannel;
1188 // Memory is managed externally
1189 cricket::ChannelManager *_channelManager = nullptr;
1190 webrtc::Call *_call = nullptr;
1191
1192 VideoChannelDescription::Quality _requestedMinQuality = VideoChannelDescription::Quality::Thumbnail;
1193 VideoChannelDescription::Quality _requestedMaxQuality = VideoChannelDescription::Quality::Thumbnail;
1194
1195 absl::optional<GroupInstanceStats::IncomingVideoStats> _stats;
1196 };
1197
1198 class MissingSsrcPacketBuffer {
1199 public:
MissingSsrcPacketBuffer(int limit)1200 MissingSsrcPacketBuffer(int limit) :
1201 _limit(limit) {
1202 }
1203
~MissingSsrcPacketBuffer()1204 ~MissingSsrcPacketBuffer() {
1205 }
1206
add(uint32_t ssrc,rtc::CopyOnWriteBuffer const & packet)1207 void add(uint32_t ssrc, rtc::CopyOnWriteBuffer const &packet) {
1208 if (_packets.size() == _limit) {
1209 _packets.erase(_packets.begin());
1210 }
1211 _packets.push_back(std::make_pair(ssrc, packet));
1212 }
1213
get(uint32_t ssrc)1214 std::vector<rtc::CopyOnWriteBuffer> get(uint32_t ssrc) {
1215 std::vector<rtc::CopyOnWriteBuffer> result;
1216 for (auto it = _packets.begin(); it != _packets.end(); ) {
1217 if (it->first == ssrc) {
1218 result.push_back(it->second);
1219 _packets.erase(it);
1220 } else {
1221 it++;
1222 }
1223 }
1224 return result;
1225 }
1226
1227 private:
1228 int _limit = 0;
1229 std::vector<std::pair<uint32_t, rtc::CopyOnWriteBuffer>> _packets;
1230
1231 };
1232
1233 class RequestedBroadcastPart {
1234 public:
1235 int64_t timestamp = 0;
1236 std::shared_ptr<BroadcastPartTask> task;
1237
RequestedBroadcastPart(int64_t timestamp_,std::shared_ptr<BroadcastPartTask> task_)1238 explicit RequestedBroadcastPart(int64_t timestamp_, std::shared_ptr<BroadcastPartTask> task_) :
1239 timestamp(timestamp_), task(task_) {
1240 }
1241 };
1242
1243 struct DecodedBroadcastPart {
1244 struct DecodedBroadcastPartChannel {
1245 uint32_t ssrc = 0;
1246 std::vector<int16_t> pcmData;
1247 };
1248
DecodedBroadcastParttgcalls::__anon2cee46520111::DecodedBroadcastPart1249 DecodedBroadcastPart(int numSamples_, std::vector<DecodedBroadcastPartChannel> &&_channels) :
1250 numSamples(numSamples_), channels(std::move(_channels)) {
1251 }
1252
1253 int numSamples = 0;
1254 std::vector<DecodedBroadcastPartChannel> channels;
1255 };
1256
videoCaptureToGetVideoSource(std::shared_ptr<VideoCaptureInterface> videoCapture)1257 std::function<webrtc::VideoTrackSourceInterface*()> videoCaptureToGetVideoSource(std::shared_ptr<VideoCaptureInterface> videoCapture) {
1258 return [videoCapture]() {
1259 VideoCaptureInterfaceObject *videoCaptureImpl = GetVideoCaptureAssumingSameThread(videoCapture.get());
1260 return videoCaptureImpl ? videoCaptureImpl->source() : nullptr;
1261 };
1262 }
1263
1264 class AudioDeviceDataObserverShared {
1265 public:
AudioDeviceDataObserverShared()1266 AudioDeviceDataObserverShared() {
1267 }
1268
~AudioDeviceDataObserverShared()1269 ~AudioDeviceDataObserverShared() {
1270 }
1271
setStreamingContext(std::shared_ptr<StreamingMediaContext> streamingContext)1272 void setStreamingContext(std::shared_ptr<StreamingMediaContext> streamingContext) {
1273 _mutex.Lock();
1274 _streamingContext = streamingContext;
1275 _mutex.Unlock();
1276 }
1277
mixAudio(int16_t * audio_samples,const size_t num_samples,const size_t num_channels,const uint32_t samples_per_sec)1278 void mixAudio(int16_t *audio_samples, const size_t num_samples, const size_t num_channels, const uint32_t samples_per_sec) {
1279
1280 _mutex.Lock();
1281 const auto context = _streamingContext;
1282 _mutex.Unlock();
1283
1284 if (context) {
1285 if (_samplesToResample.size() < 480 * num_channels) {
1286 _samplesToResample.resize(480 * num_channels);
1287 }
1288 memset(_samplesToResample.data(), 0, _samplesToResample.size() * sizeof(int16_t));
1289
1290 context->getAudio(_samplesToResample.data(), 480, num_channels, 48000);
1291
1292 if (_resamplerFrequency != samples_per_sec || _resamplerNumChannels != num_channels) {
1293 _resamplerFrequency = samples_per_sec;
1294 _resamplerNumChannels = num_channels;
1295 _resampler = std::make_unique<webrtc::Resampler>(48000, samples_per_sec, num_channels);
1296 }
1297
1298 size_t outLen = 0;
1299 _resampler->Push(_samplesToResample.data(), _samplesToResample.size(), (int16_t *)audio_samples, num_samples * num_channels, outLen);
1300 }
1301 }
1302
1303 private:
1304 webrtc::Mutex _mutex;
1305 std::unique_ptr<webrtc::Resampler> _resampler;
1306 uint32_t _resamplerFrequency = 0;
1307 size_t _resamplerNumChannels = 0;
1308 std::vector<int16_t> _samplesToResample;
1309 std::shared_ptr<StreamingMediaContext> _streamingContext;
1310 };
1311
1312 class AudioDeviceDataObserverImpl : public webrtc::AudioDeviceDataObserver {
1313 public:
AudioDeviceDataObserverImpl(std::shared_ptr<AudioDeviceDataObserverShared> shared)1314 AudioDeviceDataObserverImpl(std::shared_ptr<AudioDeviceDataObserverShared> shared) :
1315 _shared(shared) {
1316 }
1317
~AudioDeviceDataObserverImpl()1318 virtual ~AudioDeviceDataObserverImpl() {
1319 }
1320
OnCaptureData(const void * audio_samples,const size_t num_samples,const size_t bytes_per_sample,const size_t num_channels,const uint32_t samples_per_sec)1321 virtual void OnCaptureData(const void* audio_samples,
1322 const size_t num_samples,
1323 const size_t bytes_per_sample,
1324 const size_t num_channels,
1325 const uint32_t samples_per_sec) override {
1326 }
1327
OnRenderData(const void * audio_samples,const size_t num_samples,const size_t bytes_per_sample,const size_t num_channels,const uint32_t samples_per_sec)1328 virtual void OnRenderData(const void* audio_samples,
1329 const size_t num_samples,
1330 const size_t bytes_per_sample,
1331 const size_t num_channels,
1332 const uint32_t samples_per_sec) override {
1333 if (bytes_per_sample != num_channels * 2) {
1334 return;
1335 }
1336 if (samples_per_sec % 100 != 0) {
1337 return;
1338 }
1339 if (num_samples != samples_per_sec / 100) {
1340 return;
1341 }
1342
1343 if (_shared) {
1344 _shared->mixAudio((int16_t *)audio_samples, num_samples, num_channels, samples_per_sec);
1345 }
1346 }
1347
1348 private:
1349 std::shared_ptr<AudioDeviceDataObserverShared> _shared;
1350 };
1351
1352 } // namespace
1353
1354 class GroupInstanceCustomInternal : public sigslot::has_slots<>, public std::enable_shared_from_this<GroupInstanceCustomInternal> {
1355 public:
GroupInstanceCustomInternal(GroupInstanceDescriptor && descriptor,std::shared_ptr<Threads> threads)1356 GroupInstanceCustomInternal(GroupInstanceDescriptor &&descriptor, std::shared_ptr<Threads> threads) :
1357 _threads(std::move(threads)),
1358 _networkStateUpdated(descriptor.networkStateUpdated),
1359 _audioLevelsUpdated(descriptor.audioLevelsUpdated),
1360 _onAudioFrame(descriptor.onAudioFrame),
1361 _requestMediaChannelDescriptions(descriptor.requestMediaChannelDescriptions),
1362 _requestCurrentTime(descriptor.requestCurrentTime),
1363 _requestAudioBroadcastPart(descriptor.requestAudioBroadcastPart),
1364 _requestVideoBroadcastPart(descriptor.requestVideoBroadcastPart),
1365 _videoCapture(descriptor.videoCapture),
1366 _videoCaptureSink(new VideoSinkImpl("VideoCapture")),
1367 _getVideoSource(descriptor.getVideoSource),
1368 _disableIncomingChannels(descriptor.disableIncomingChannels),
1369 _useDummyChannel(descriptor.useDummyChannel),
1370 _outgoingAudioBitrateKbit(descriptor.outgoingAudioBitrateKbit),
1371 _disableOutgoingAudioProcessing(descriptor.disableOutgoingAudioProcessing),
1372 _minOutgoingVideoBitrateKbit(descriptor.minOutgoingVideoBitrateKbit),
1373 _videoContentType(descriptor.videoContentType),
1374 _videoCodecPreferences(std::move(descriptor.videoCodecPreferences)),
1375 _eventLog(std::make_unique<webrtc::RtcEventLogNull>()),
1376 _taskQueueFactory(webrtc::CreateDefaultTaskQueueFactory()),
1377 _createAudioDeviceModule(descriptor.createAudioDeviceModule),
1378 _initialInputDeviceId(std::move(descriptor.initialInputDeviceId)),
1379 _initialOutputDeviceId(std::move(descriptor.initialOutputDeviceId)),
1380 _missingPacketBuffer(50) {
1381 assert(_threads->getMediaThread()->IsCurrent());
1382
1383 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this] {
1384 _workerThreadSafery = webrtc::PendingTaskSafetyFlag::Create();
1385 });
1386 _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this] {
1387 _networkThreadSafery = webrtc::PendingTaskSafetyFlag::Create();
1388 });
1389
1390 if (_videoCapture) {
1391 assert(!_getVideoSource);
1392 _getVideoSource = videoCaptureToGetVideoSource(std::move(descriptor.videoCapture));
1393 }
1394 generateSsrcs();
1395
1396 _noiseSuppressionConfiguration = std::make_shared<NoiseSuppressionConfiguration>(descriptor.initialEnableNoiseSuppression);
1397
1398 _externalAudioRecorder.reset(new ExternalAudioRecorder(&_externalAudioSamples, &_externalAudioSamplesMutex));
1399 }
1400
~GroupInstanceCustomInternal()1401 ~GroupInstanceCustomInternal() {
1402 _incomingAudioChannels.clear();
1403 _incomingVideoChannels.clear();
1404 _serverBandwidthProbingVideoSsrc.reset();
1405
1406 destroyOutgoingAudioChannel();
1407 destroyOutgoingVideoChannel();
1408
1409 _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1410 _rtpTransport->SignalSentPacket.disconnect(this);
1411 _rtpTransport->SignalRtcpPacketReceived.disconnect(this);
1412 });
1413
1414 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1415 _channelManager = nullptr;
1416 if (_audioDeviceModule) {
1417 _audioDeviceModule->Stop();
1418 _audioDeviceModule = nullptr;
1419 }
1420 _call.reset();
1421 });
1422 }
1423
start()1424 void start() {
1425 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1426
1427 webrtc::field_trial::InitFieldTrialsFromString(
1428 "WebRTC-Audio-Allocation/min:32kbps,max:32kbps/"
1429 "WebRTC-Audio-OpusMinPacketLossRate/Enabled-1/"
1430 "WebRTC-TaskQueuePacer/Enabled/"
1431 "WebRTC-VP8ConferenceTemporalLayers/1/"
1432 "WebRTC-Audio-MinimizeResamplingOnMobile/Enabled/"
1433 "WebRTC-BweLossExperiment/Enabled/"
1434 );
1435
1436 _networkManager.reset(new ThreadLocalObject<GroupNetworkManager>(_threads->getNetworkThread(), [weak, threads = _threads] () mutable {
1437 return new GroupNetworkManager(
1438 [=](const GroupNetworkManager::State &state) {
1439 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [=] {
1440 const auto strong = weak.lock();
1441 if (!strong) {
1442 return;
1443 }
1444 strong->setIsRtcConnected(state.isReadyToSendData);
1445 });
1446 },
1447 [=](rtc::CopyOnWriteBuffer const &message, bool isUnresolved) {
1448 if (!isUnresolved) {
1449 return;
1450 }
1451 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, message, isUnresolved]() mutable {
1452 if (const auto strong = weak.lock()) {
1453 strong->receivePacket(message, isUnresolved);
1454 }
1455 });
1456 },
1457 [=](bool isDataChannelOpen) {
1458 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, isDataChannelOpen]() mutable {
1459 if (const auto strong = weak.lock()) {
1460 strong->updateIsDataChannelOpen(isDataChannelOpen);
1461 }
1462 });
1463 },
1464 [=](std::string const &message) {
1465 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, message]() {
1466 if (const auto strong = weak.lock()) {
1467 strong->receiveDataChannelMessage(message);
1468 }
1469 });
1470 },
1471 [=](uint32_t ssrc, uint8_t audioLevel, bool isSpeech) {
1472 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, ssrc, audioLevel, isSpeech]() {
1473 if (const auto strong = weak.lock()) {
1474 strong->updateSsrcAudioLevel(ssrc, audioLevel, isSpeech);
1475 }
1476 });
1477 }, threads);
1478 }));
1479
1480 #if USE_RNNOISE
1481 std::unique_ptr<AudioCapturePostProcessor> audioProcessor = nullptr;
1482 #endif
1483 if (_videoContentType != VideoContentType::Screencast) {
1484 PlatformInterface::SharedInstance()->configurePlatformAudio();
1485
1486 #if USE_RNNOISE
1487 audioProcessor = std::make_unique<AudioCapturePostProcessor>([weak, threads = _threads](GroupLevelValue const &level) {
1488 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, level](){
1489 auto strong = weak.lock();
1490 if (!strong) {
1491 return;
1492 }
1493 strong->_myAudioLevel = level;
1494 });
1495 }, _noiseSuppressionConfiguration, nullptr, nullptr);
1496 #endif
1497 }
1498
1499 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this
1500 #if USE_RNNOISE
1501 , audioProcessor = std::move(audioProcessor)
1502 #endif
1503 ]() mutable {
1504 cricket::MediaEngineDependencies mediaDeps;
1505 mediaDeps.task_queue_factory = _taskQueueFactory.get();
1506 mediaDeps.audio_encoder_factory = webrtc::CreateAudioEncoderFactory<webrtc::AudioEncoderOpus, webrtc::AudioEncoderL16>();
1507 mediaDeps.audio_decoder_factory = webrtc::CreateAudioDecoderFactory<webrtc::AudioDecoderOpus, webrtc::AudioDecoderL16>();
1508
1509 mediaDeps.video_encoder_factory = PlatformInterface::SharedInstance()->makeVideoEncoderFactory();
1510 mediaDeps.video_decoder_factory = PlatformInterface::SharedInstance()->makeVideoDecoderFactory();
1511
1512 #if USE_RNNOISE
1513 if (_audioLevelsUpdated && audioProcessor) {
1514 webrtc::AudioProcessingBuilder builder;
1515 builder.SetCapturePostProcessing(std::move(audioProcessor));
1516
1517 mediaDeps.audio_processing = builder.Create();
1518 }
1519 #endif
1520
1521 _audioDeviceDataObserverShared = std::make_shared<AudioDeviceDataObserverShared>();
1522
1523 _audioDeviceModule = createAudioDeviceModule();
1524 if (!_audioDeviceModule) {
1525 return;
1526 }
1527 mediaDeps.adm = _audioDeviceModule;
1528
1529 _availableVideoFormats = filterSupportedVideoFormats(mediaDeps.video_encoder_factory->GetSupportedFormats());
1530
1531 std::unique_ptr<cricket::MediaEngineInterface> mediaEngine = cricket::CreateMediaEngine(std::move(mediaDeps));
1532
1533 _channelManager = cricket::ChannelManager::Create(
1534 std::move(mediaEngine),
1535 std::make_unique<cricket::RtpDataEngine>(),
1536 true,
1537 _threads->getWorkerThread(),
1538 _threads->getNetworkThread()
1539 );
1540 });
1541
1542 setAudioInputDevice(_initialInputDeviceId);
1543 setAudioOutputDevice(_initialOutputDeviceId);
1544
1545 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1546 webrtc::Call::Config callConfig(_eventLog.get(), _threads->getNetworkThread());
1547 callConfig.task_queue_factory = _taskQueueFactory.get();
1548 callConfig.trials = &_fieldTrials;
1549 callConfig.audio_state = _channelManager->media_engine()->voice().GetAudioState();
1550 _call.reset(webrtc::Call::Create(callConfig, _threads->getSharedModuleThread()));
1551 });
1552
1553 _uniqueRandomIdGenerator.reset(new rtc::UniqueRandomIdGenerator());
1554
1555 _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1556 _rtpTransport = _networkManager->getSyncAssumingSameThread()->getRtpTransport();
1557 _rtpTransport->SignalSentPacket.connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1558 _rtpTransport->SignalRtcpPacketReceived.connect(this, &GroupInstanceCustomInternal::OnRtcpPacketReceived_n);
1559 });
1560
1561 _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory();
1562
1563 if (_audioLevelsUpdated) {
1564 beginLevelsTimer(100);
1565 }
1566
1567 if (_getVideoSource) {
1568 setVideoSource(_getVideoSource, true);
1569 }
1570
1571 if (_useDummyChannel && _videoContentType != VideoContentType::Screencast) {
1572 addIncomingAudioChannel(ChannelId(1), true);
1573 }
1574
1575 if (_videoContentType == VideoContentType::Screencast) {
1576 setIsMuted(false);
1577 }
1578
1579 /*if (_videoContentType != VideoContentType::Screencast) {
1580 createOutgoingAudioChannel();
1581 }*/
1582
1583 beginNetworkStatusTimer(0);
1584 //beginAudioChannelCleanupTimer(0);
1585
1586 adjustBitratePreferences(true);
1587
1588 beginRemoteConstraintsUpdateTimer(5000);
1589 }
1590
destroyOutgoingVideoChannel()1591 void destroyOutgoingVideoChannel() {
1592 if (!_outgoingVideoChannel) {
1593 return;
1594 }
1595 //_outgoingVideoChannel->SignalSentPacket().disconnect(this);
1596 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1597 _outgoingVideoChannel->Enable(false);
1598 _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, nullptr);
1599 _channelManager->DestroyVideoChannel(_outgoingVideoChannel);
1600 });
1601 _outgoingVideoChannel = nullptr;
1602 }
1603
createOutgoingVideoChannel()1604 void createOutgoingVideoChannel() {
1605 if (_outgoingVideoChannel
1606 || _videoContentType == VideoContentType::None) {
1607 return;
1608 }
1609 configureVideoParams();
1610
1611 if (!_selectedPayloadType) {
1612 RTC_LOG(LS_ERROR) << "Could not select payload type.";
1613 return;
1614 }
1615
1616 cricket::VideoOptions videoOptions;
1617 if (_videoContentType == VideoContentType::Screencast) {
1618 videoOptions.is_screencast = true;
1619 }
1620 _outgoingVideoChannel = _channelManager->CreateVideoChannel(_call.get(), cricket::MediaConfig(), _rtpTransport, _threads->getWorkerThread(), "1", false, GroupNetworkManager::getDefaulCryptoOptions(), _uniqueRandomIdGenerator.get(), videoOptions, _videoBitrateAllocatorFactory.get());
1621
1622 if (!_outgoingVideoChannel) {
1623 RTC_LOG(LS_ERROR) << "Could not create outgoing video channel.";
1624 return;
1625 }
1626
1627 cricket::StreamParams videoSendStreamParams;
1628
1629 std::vector<uint32_t> simulcastGroupSsrcs;
1630 std::vector<cricket::SsrcGroup> fidGroups;
1631 for (const auto &layer : _outgoingVideoSsrcs.simulcastLayers) {
1632 simulcastGroupSsrcs.push_back(layer.ssrc);
1633
1634 videoSendStreamParams.ssrcs.push_back(layer.ssrc);
1635 videoSendStreamParams.ssrcs.push_back(layer.fidSsrc);
1636
1637 cricket::SsrcGroup fidGroup(cricket::kFidSsrcGroupSemantics, { layer.ssrc, layer.fidSsrc });
1638 fidGroups.push_back(fidGroup);
1639 }
1640 if (simulcastGroupSsrcs.size() > 1) {
1641 cricket::SsrcGroup simulcastGroup(cricket::kSimSsrcGroupSemantics, simulcastGroupSsrcs);
1642 videoSendStreamParams.ssrc_groups.push_back(simulcastGroup);
1643
1644 GroupJoinPayloadVideoSourceGroup payloadSimulcastGroup;
1645 payloadSimulcastGroup.semantics = "SIM";
1646 payloadSimulcastGroup.ssrcs = simulcastGroupSsrcs;
1647 }
1648
1649 for (auto fidGroup : fidGroups) {
1650 videoSendStreamParams.ssrc_groups.push_back(fidGroup);
1651
1652 GroupJoinPayloadVideoSourceGroup payloadFidGroup;
1653 payloadFidGroup.semantics = "FID";
1654 payloadFidGroup.ssrcs = fidGroup.ssrcs;
1655 }
1656
1657 videoSendStreamParams.cname = "cname";
1658
1659 auto outgoingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1660 for (const auto &extension : _videoExtensionMap) {
1661 outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(extension.second, extension.first));
1662 }
1663 outgoingVideoDescription->set_rtcp_mux(true);
1664 outgoingVideoDescription->set_rtcp_reduced_size(true);
1665 outgoingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1666 outgoingVideoDescription->set_codecs({ _selectedPayloadType->videoCodec, _selectedPayloadType->rtxCodec });
1667 outgoingVideoDescription->set_bandwidth(1300000);
1668 outgoingVideoDescription->AddStream(videoSendStreamParams);
1669
1670 auto incomingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1671 for (const auto &extension : _videoExtensionMap) {
1672 incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(extension.second, extension.first));
1673 }
1674 incomingVideoDescription->set_rtcp_mux(true);
1675 incomingVideoDescription->set_rtcp_reduced_size(true);
1676 incomingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1677 incomingVideoDescription->set_codecs({ _selectedPayloadType->videoCodec, _selectedPayloadType->rtxCodec });
1678 incomingVideoDescription->set_bandwidth(1300000);
1679
1680 _outgoingVideoChannel->SetRemoteContent(incomingVideoDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1681 _outgoingVideoChannel->SetLocalContent(outgoingVideoDescription.get(), webrtc::SdpType::kOffer, nullptr);
1682 _outgoingVideoChannel->SetPayloadTypeDemuxingEnabled(false);
1683
1684 //_outgoingVideoChannel->SignalSentPacket().connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1685
1686 adjustVideoSendParams();
1687 updateVideoSend();
1688 }
1689
adjustVideoSendParams()1690 void adjustVideoSendParams() {
1691 if (!_outgoingVideoChannel) {
1692 return;
1693 }
1694
1695 if (_videoContentType == VideoContentType::Screencast) {
1696 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1697 webrtc::RtpParameters rtpParameters = _outgoingVideoChannel->media_channel()->GetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc);
1698 if (rtpParameters.encodings.size() == 3) {
1699 for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1700 if (i == 0) {
1701 rtpParameters.encodings[i].min_bitrate_bps = 50000;
1702 rtpParameters.encodings[i].max_bitrate_bps = 100000;
1703 rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1704 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 180;
1705 } else if (i == 1) {
1706 rtpParameters.encodings[i].min_bitrate_bps = 150000;
1707 rtpParameters.encodings[i].max_bitrate_bps = 200000;
1708 rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1709 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 360;
1710 } else if (i == 2) {
1711 rtpParameters.encodings[i].min_bitrate_bps = 300000;
1712 rtpParameters.encodings[i].max_bitrate_bps = 800000 + 100000;
1713 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 720;
1714 }
1715 }
1716 } else if (rtpParameters.encodings.size() == 2) {
1717 for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1718 if (i == 0) {
1719 rtpParameters.encodings[i].min_bitrate_bps = 50000;
1720 rtpParameters.encodings[i].max_bitrate_bps = 100000;
1721 rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1722 } else if (i == 1) {
1723 rtpParameters.encodings[i].min_bitrate_bps = 200000;
1724 rtpParameters.encodings[i].max_bitrate_bps = 900000 + 100000;
1725 }
1726 }
1727 } else {
1728 rtpParameters.encodings[0].max_bitrate_bps = (800000 + 100000) * 2;
1729 }
1730
1731 _outgoingVideoChannel->media_channel()->SetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, rtpParameters);
1732 });
1733 } else {
1734 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1735 webrtc::RtpParameters rtpParameters = _outgoingVideoChannel->media_channel()->GetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc);
1736 if (rtpParameters.encodings.size() == 3) {
1737 for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1738 if (i == 0) {
1739 rtpParameters.encodings[i].min_bitrate_bps = 50000;
1740 rtpParameters.encodings[i].max_bitrate_bps = 60000;
1741 rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1742 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 180;
1743 } else if (i == 1) {
1744 rtpParameters.encodings[i].min_bitrate_bps = 100000;
1745 rtpParameters.encodings[i].max_bitrate_bps = 110000;
1746 rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1747 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 360;
1748 } else if (i == 2) {
1749 rtpParameters.encodings[i].min_bitrate_bps = 300000;
1750 rtpParameters.encodings[i].max_bitrate_bps = 800000 + 100000;
1751 rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 720;
1752 }
1753 }
1754 } else if (rtpParameters.encodings.size() == 2) {
1755 for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1756 if (i == 0) {
1757 rtpParameters.encodings[i].min_bitrate_bps = 50000;
1758 rtpParameters.encodings[i].max_bitrate_bps = 100000;
1759 rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1760 } else if (i == 1) {
1761 rtpParameters.encodings[i].min_bitrate_bps = 200000;
1762 rtpParameters.encodings[i].max_bitrate_bps = 900000 + 100000;
1763 }
1764 }
1765 } else {
1766 rtpParameters.encodings[0].max_bitrate_bps = (800000 + 100000) * 2;
1767 }
1768
1769 _outgoingVideoChannel->media_channel()->SetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, rtpParameters);
1770 });
1771 }
1772 }
1773
updateVideoSend()1774 void updateVideoSend() {
1775 if (!_outgoingVideoChannel) {
1776 return;
1777 }
1778
1779 webrtc::VideoTrackSourceInterface *videoSource = _getVideoSource ? _getVideoSource() : nullptr;
1780 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, videoSource]() {
1781 if (_getVideoSource) {
1782 _outgoingVideoChannel->Enable(true);
1783 _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, videoSource);
1784 } else {
1785 _outgoingVideoChannel->Enable(false);
1786 _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, nullptr);
1787 }
1788 });
1789 }
1790
destroyOutgoingAudioChannel()1791 void destroyOutgoingAudioChannel() {
1792 if (!_outgoingAudioChannel) {
1793 return;
1794 }
1795
1796 //_outgoingAudioChannel->SignalSentPacket().disconnect(this);
1797 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1798 _outgoingAudioChannel->media_channel()->SetAudioSend(_outgoingAudioSsrc, false, nullptr, &_audioSource);
1799 _outgoingAudioChannel->Enable(false);
1800 _channelManager->DestroyVoiceChannel(_outgoingAudioChannel);
1801 });
1802 _outgoingAudioChannel = nullptr;
1803 }
1804
createOutgoingAudioChannel()1805 void createOutgoingAudioChannel() {
1806 if (_outgoingAudioChannel) {
1807 return;
1808 }
1809
1810 cricket::AudioOptions audioOptions;
1811 if (_disableOutgoingAudioProcessing || _videoContentType == VideoContentType::Screencast) {
1812 audioOptions.echo_cancellation = false;
1813 audioOptions.noise_suppression = false;
1814 audioOptions.auto_gain_control = false;
1815 audioOptions.highpass_filter = false;
1816 audioOptions.typing_detection = false;
1817 audioOptions.experimental_agc = false;
1818 audioOptions.experimental_ns = false;
1819 audioOptions.residual_echo_detector = false;
1820 } else {
1821 audioOptions.echo_cancellation = true;
1822 audioOptions.noise_suppression = true;
1823 audioOptions.experimental_ns = true;
1824 audioOptions.residual_echo_detector = true;
1825 }
1826
1827 std::vector<std::string> streamIds;
1828 streamIds.push_back("1");
1829
1830 _outgoingAudioChannel = _channelManager->CreateVoiceChannel(_call.get(), cricket::MediaConfig(), _rtpTransport, _threads->getWorkerThread(), "0", false, GroupNetworkManager::getDefaulCryptoOptions(), _uniqueRandomIdGenerator.get(), audioOptions);
1831
1832 const uint8_t opusMinBitrateKbps = _outgoingAudioBitrateKbit;
1833 const uint8_t opusMaxBitrateKbps = _outgoingAudioBitrateKbit;
1834 const uint8_t opusStartBitrateKbps = _outgoingAudioBitrateKbit;
1835 const uint8_t opusPTimeMs = 120;
1836
1837 cricket::AudioCodec opusCodec(111, "opus", 48000, 0, 2);
1838 opusCodec.AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc));
1839 opusCodec.SetParam(cricket::kCodecParamMinBitrate, opusMinBitrateKbps);
1840 opusCodec.SetParam(cricket::kCodecParamStartBitrate, opusStartBitrateKbps);
1841 opusCodec.SetParam(cricket::kCodecParamMaxBitrate, opusMaxBitrateKbps);
1842 opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1);
1843 opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs);
1844
1845 auto outgoingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
1846 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
1847 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1848 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1849 outgoingAudioDescription->set_rtcp_mux(true);
1850 outgoingAudioDescription->set_rtcp_reduced_size(true);
1851 outgoingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1852 outgoingAudioDescription->set_codecs({ opusCodec });
1853 outgoingAudioDescription->set_bandwidth(1300000);
1854 outgoingAudioDescription->AddStream(cricket::StreamParams::CreateLegacy(_outgoingAudioSsrc));
1855
1856 auto incomingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
1857 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
1858 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1859 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1860 incomingAudioDescription->set_rtcp_mux(true);
1861 incomingAudioDescription->set_rtcp_reduced_size(true);
1862 incomingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1863 incomingAudioDescription->set_codecs({ opusCodec });
1864 incomingAudioDescription->set_bandwidth(1300000);
1865
1866 _outgoingAudioChannel->SetLocalContent(outgoingAudioDescription.get(), webrtc::SdpType::kOffer, nullptr);
1867 _outgoingAudioChannel->SetRemoteContent(incomingAudioDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1868 _outgoingAudioChannel->SetPayloadTypeDemuxingEnabled(false);
1869
1870 //_outgoingAudioChannel->SignalSentPacket().connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1871
1872 _outgoingAudioChannel->Enable(true);
1873
1874 onUpdatedIsMuted();
1875
1876 adjustBitratePreferences(false);
1877 }
1878
stop()1879 void stop() {
1880 }
1881
updateSsrcAudioLevel(uint32_t ssrc,uint8_t audioLevel,bool isSpeech)1882 void updateSsrcAudioLevel(uint32_t ssrc, uint8_t audioLevel, bool isSpeech) {
1883 float mappedLevelDb = ((float)audioLevel) / (float)(0x7f);
1884
1885 //mappedLevelDb = fabs(1.0f - mappedLevelDb);
1886 //float mappedLevel = pow(10.0f, mappedLevelDb * 0.1f);
1887
1888 //printf("mappedLevelDb: %f, mappedLevel: %f\n", mappedLevelDb, mappedLevel);
1889
1890 float mappedLevel = (fabs(1.0f - mappedLevelDb)) * 1.0f;
1891
1892 auto it = _audioLevels.find(ChannelId(ssrc));
1893 if (it != _audioLevels.end()) {
1894 it->second.value.level = fmax(it->second.value.level, mappedLevel);
1895 if (isSpeech) {
1896 it->second.value.voice = true;
1897 }
1898 it->second.timestamp = rtc::TimeMillis();
1899 } else {
1900 InternalGroupLevelValue updated;
1901 updated.value.level = mappedLevel;
1902 updated.value.voice = isSpeech;
1903 updated.timestamp = rtc::TimeMillis();
1904 _audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
1905 }
1906
1907 auto audioChannel = _incomingAudioChannels.find(ChannelId(ssrc));
1908 if (audioChannel != _incomingAudioChannels.end()) {
1909 audioChannel->second->updateActivity();
1910 }
1911 }
1912
beginLevelsTimer(int timeoutMs)1913 void beginLevelsTimer(int timeoutMs) {
1914 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1915 _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
1916 auto strong = weak.lock();
1917 if (!strong) {
1918 return;
1919 }
1920
1921 //int64_t timestamp = rtc::TimeMillis();
1922 //int64_t maxSampleTimeout = 400;
1923
1924 GroupLevelsUpdate levelsUpdate;
1925 levelsUpdate.updates.reserve(strong->_audioLevels.size() + 1);
1926 for (auto &it : strong->_audioLevels) {
1927 /*if (it.second.value.level < 0.001f) {
1928 continue;
1929 }
1930 if (it.second.timestamp <= timestamp - maxSampleTimeout) {
1931 continue;
1932 }*/
1933
1934 uint32_t effectiveSsrc = it.first.actualSsrc;
1935 if (std::find_if(levelsUpdate.updates.begin(), levelsUpdate.updates.end(), [&](GroupLevelUpdate const &item) {
1936 return item.ssrc == effectiveSsrc;
1937 }) != levelsUpdate.updates.end()) {
1938 continue;
1939 }
1940 levelsUpdate.updates.push_back(GroupLevelUpdate{
1941 effectiveSsrc,
1942 it.second.value,
1943 });
1944 if (it.second.value.level > 0.001f) {
1945 auto audioChannel = strong->_incomingAudioChannels.find(it.first);
1946 if (audioChannel != strong->_incomingAudioChannels.end()) {
1947 audioChannel->second->updateActivity();
1948 }
1949 }
1950
1951 //it.second.value.level *= 0.5f;
1952 //it.second.value.voice = false;
1953 }
1954
1955 strong->_audioLevels.clear();
1956
1957 auto myAudioLevel = strong->_myAudioLevel;
1958 myAudioLevel.isMuted = strong->_isMuted;
1959 levelsUpdate.updates.push_back(GroupLevelUpdate{ 0, myAudioLevel });
1960
1961 if (strong->_audioLevelsUpdated) {
1962 strong->_audioLevelsUpdated(levelsUpdate);
1963 }
1964
1965 bool isSpeech = myAudioLevel.voice && !myAudioLevel.isMuted;
1966 strong->_networkManager->perform(RTC_FROM_HERE, [isSpeech = isSpeech](GroupNetworkManager *networkManager) {
1967 networkManager->setOutgoingVoiceActivity(isSpeech);
1968 });
1969
1970 strong->beginLevelsTimer(100);
1971 }, timeoutMs);
1972 }
1973
beginAudioChannelCleanupTimer(int delayMs)1974 void beginAudioChannelCleanupTimer(int delayMs) {
1975 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1976 _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
1977 auto strong = weak.lock();
1978 if (!strong) {
1979 return;
1980 }
1981
1982 auto timestamp = rtc::TimeMillis();
1983
1984 std::vector<ChannelId> removeChannels;
1985 for (const auto &it : strong->_incomingAudioChannels) {
1986 if (it.first.networkSsrc == 1) {
1987 continue;
1988 }
1989 auto activity = it.second->getActivity();
1990 if (activity < timestamp - 1000) {
1991 removeChannels.push_back(it.first);
1992 }
1993 }
1994
1995 for (const auto &channelId : removeChannels) {
1996 strong->removeIncomingAudioChannel(channelId);
1997 }
1998
1999 strong->beginAudioChannelCleanupTimer(500);
2000 }, delayMs);
2001 }
2002
beginRemoteConstraintsUpdateTimer(int delayMs)2003 void beginRemoteConstraintsUpdateTimer(int delayMs) {
2004 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2005 _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
2006 auto strong = weak.lock();
2007 if (!strong) {
2008 return;
2009 }
2010
2011 strong->maybeUpdateRemoteVideoConstraints();
2012
2013 strong->beginRemoteConstraintsUpdateTimer(5000);
2014 }, delayMs);
2015 }
2016
beginNetworkStatusTimer(int delayMs)2017 void beginNetworkStatusTimer(int delayMs) {
2018 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2019 _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
2020 auto strong = weak.lock();
2021 if (!strong) {
2022 return;
2023 }
2024
2025 if (strong->_connectionMode == GroupConnectionMode::GroupConnectionModeBroadcast || strong->_broadcastEnabledUntilRtcIsConnectedAtTimestamp) {
2026 strong->updateBroadcastNetworkStatus();
2027 }
2028
2029 strong->beginNetworkStatusTimer(500);
2030 }, delayMs);
2031 }
2032
updateBroadcastNetworkStatus()2033 void updateBroadcastNetworkStatus() {
2034 bool isBroadcastConnected = true;
2035
2036 if (isBroadcastConnected != _isBroadcastConnected) {
2037 _isBroadcastConnected = isBroadcastConnected;
2038 updateIsConnected();
2039 }
2040 }
2041
configureVideoParams()2042 void configureVideoParams() {
2043 if (!_sharedVideoInformation) {
2044 return;
2045 }
2046 if (_selectedPayloadType) {
2047 // Already configured.
2048 return;
2049 }
2050
2051 _availablePayloadTypes = assignPayloadTypes(_availableVideoFormats);
2052 if (_availablePayloadTypes.empty()) {
2053 return;
2054 }
2055
2056 for (const auto &payloadType : _availablePayloadTypes) {
2057 GroupJoinPayloadVideoPayloadType payload;
2058 payload.id = payloadType.videoCodec.id;
2059 payload.name = payloadType.videoCodec.name;
2060 payload.clockrate = payloadType.videoCodec.clockrate;
2061 payload.channels = 0;
2062
2063 std::vector<GroupJoinPayloadVideoPayloadType::FeedbackType> feedbackTypes;
2064
2065 GroupJoinPayloadVideoPayloadType::FeedbackType fbGoogRemb;
2066 fbGoogRemb.type = "goog-remb";
2067 feedbackTypes.push_back(fbGoogRemb);
2068
2069 GroupJoinPayloadVideoPayloadType::FeedbackType fbTransportCc;
2070 fbTransportCc.type = "transport-cc";
2071 feedbackTypes.push_back(fbTransportCc);
2072
2073 GroupJoinPayloadVideoPayloadType::FeedbackType fbCcmFir;
2074 fbCcmFir.type = "ccm";
2075 fbCcmFir.subtype = "fir";
2076 feedbackTypes.push_back(fbCcmFir);
2077
2078 GroupJoinPayloadVideoPayloadType::FeedbackType fbNack;
2079 fbNack.type = "nack";
2080 feedbackTypes.push_back(fbNack);
2081
2082 GroupJoinPayloadVideoPayloadType::FeedbackType fbNackPli;
2083 fbNackPli.type = "nack";
2084 fbNackPli.subtype = "pli";
2085 feedbackTypes.push_back(fbNackPli);
2086
2087 payload.feedbackTypes = feedbackTypes;
2088 payload.parameters = {};
2089
2090 _videoPayloadTypes.push_back(std::move(payload));
2091
2092 GroupJoinPayloadVideoPayloadType rtxPayload;
2093 rtxPayload.id = payloadType.rtxCodec.id;
2094 rtxPayload.name = payloadType.rtxCodec.name;
2095 rtxPayload.clockrate = payloadType.rtxCodec.clockrate;
2096 rtxPayload.parameters.push_back(std::make_pair("apt", intToString(payloadType.videoCodec.id)));
2097 _videoPayloadTypes.push_back(std::move(rtxPayload));
2098 }
2099
2100 std::vector<std::string> codecPriorities;
2101 for (const auto name : _videoCodecPreferences) {
2102 std::string codecName;
2103 switch (name) {
2104 case VideoCodecName::VP8: {
2105 codecName = cricket::kVp8CodecName;
2106 break;
2107 }
2108 case VideoCodecName::VP9: {
2109 codecName = cricket::kVp9CodecName;
2110 break;
2111 }
2112 case VideoCodecName::H264: {
2113 codecName = cricket::kH264CodecName;
2114 break;
2115 }
2116 default: {
2117 break;
2118 }
2119 }
2120 if (codecName.size() != 0) {
2121 codecPriorities.push_back(std::move(codecName));
2122 }
2123 }
2124 std::vector<std::string> defaultCodecPriorities = {
2125 cricket::kVp8CodecName,
2126 cricket::kVp9CodecName
2127 };
2128
2129 bool enableH264 = false;
2130 for (const auto &payloadType : _sharedVideoInformation->payloadTypes) {
2131 if (payloadType.name == cricket::kH264CodecName) {
2132 enableH264 = true;
2133 break;
2134 }
2135 }
2136 if (enableH264) {
2137 defaultCodecPriorities.insert(defaultCodecPriorities.begin(), cricket::kH264CodecName);
2138 }
2139
2140 for (const auto &name : defaultCodecPriorities) {
2141 if (std::find(codecPriorities.begin(), codecPriorities.end(), name) == codecPriorities.end()) {
2142 codecPriorities.push_back(name);
2143 }
2144 }
2145
2146 for (const auto &codecName : codecPriorities) {
2147 if (_selectedPayloadType) {
2148 break;
2149 }
2150 for (const auto &payloadType : _availablePayloadTypes) {
2151 if (payloadType.videoCodec.name == codecName) {
2152 _selectedPayloadType = payloadType;
2153 break;
2154 }
2155 }
2156 }
2157 if (!_selectedPayloadType) {
2158 return;
2159 }
2160
2161 _videoExtensionMap.emplace_back(2, webrtc::RtpExtension::kAbsSendTimeUri);
2162 _videoExtensionMap.emplace_back(3, webrtc::RtpExtension::kTransportSequenceNumberUri);
2163 _videoExtensionMap.emplace_back(13, webrtc::RtpExtension::kVideoRotationUri);
2164 }
2165
OnSentPacket_w(const rtc::SentPacket & sent_packet)2166 void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
2167 _call->OnSentPacket(sent_packet);
2168 }
2169
OnRtcpPacketReceived_n(rtc::CopyOnWriteBuffer * buffer,int64_t packet_time_us)2170 void OnRtcpPacketReceived_n(rtc::CopyOnWriteBuffer *buffer, int64_t packet_time_us) {
2171 rtc::CopyOnWriteBuffer packet = *buffer;
2172 _threads->getWorkerThread()->PostTask(ToQueuedTask(_workerThreadSafery, [this, packet, packet_time_us] {
2173 if (_call) {
2174 _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, packet_time_us);
2175 }
2176 }));
2177 }
2178
adjustBitratePreferences(bool resetStartBitrate)2179 void adjustBitratePreferences(bool resetStartBitrate) {
2180 webrtc::BitrateConstraints preferences;
2181 webrtc::BitrateSettings settings;
2182 if (_getVideoSource) {
2183 settings.min_bitrate_bps = _minOutgoingVideoBitrateKbit * 1024;
2184 if (resetStartBitrate) {
2185 preferences.start_bitrate_bps = std::max(preferences.min_bitrate_bps, 400 * 1000);
2186 }
2187 if (_videoContentType == VideoContentType::Screencast) {
2188 preferences.max_bitrate_bps = std::max(preferences.min_bitrate_bps, (1020 + 32) * 1000);
2189 } else {
2190 preferences.max_bitrate_bps = std::max(preferences.min_bitrate_bps, (1020 + 32) * 1000);
2191 }
2192 } else {
2193 preferences.min_bitrate_bps = 32000;
2194 if (resetStartBitrate) {
2195 preferences.start_bitrate_bps = 32000;
2196 }
2197 preferences.max_bitrate_bps = 32000;
2198 }
2199
2200 settings.min_bitrate_bps = preferences.min_bitrate_bps;
2201 settings.start_bitrate_bps = preferences.start_bitrate_bps;
2202 settings.max_bitrate_bps = preferences.max_bitrate_bps;
2203
2204 _call->GetTransportControllerSend()->SetSdpBitrateParameters(preferences);
2205 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&]() {
2206 _call->SetClientBitratePreferences(settings);
2207 });
2208 }
2209
setIsRtcConnected(bool isConnected)2210 void setIsRtcConnected(bool isConnected) {
2211 if (_isRtcConnected == isConnected) {
2212 return;
2213 }
2214 _isRtcConnected = isConnected;
2215
2216 RTC_LOG(LS_INFO) << formatTimestampMillis(rtc::TimeMillis()) << ": " << "setIsRtcConnected: " << _isRtcConnected;
2217
2218 if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp) {
2219 _broadcastEnabledUntilRtcIsConnectedAtTimestamp = absl::nullopt;
2220
2221 if (_streamingContext) {
2222 _streamingContext.reset();
2223 _audioDeviceDataObserverShared->setStreamingContext(nullptr);
2224 }
2225 }
2226
2227 updateIsConnected();
2228 }
2229
updateIsConnected()2230 void updateIsConnected() {
2231 bool isEffectivelyConnected = false;
2232 bool isTransitioningFromBroadcastToRtc = false;
2233 switch (_connectionMode) {
2234 case GroupConnectionMode::GroupConnectionModeNone: {
2235 isEffectivelyConnected = false;
2236 if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp && _isBroadcastConnected) {
2237 isEffectivelyConnected = true;
2238 isTransitioningFromBroadcastToRtc = true;
2239 }
2240 break;
2241 }
2242 case GroupConnectionMode::GroupConnectionModeRtc: {
2243 isEffectivelyConnected = _isRtcConnected;
2244 if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp && _isBroadcastConnected) {
2245 isEffectivelyConnected = true;
2246 isTransitioningFromBroadcastToRtc = true;
2247 }
2248 break;
2249 }
2250 case GroupConnectionMode::GroupConnectionModeBroadcast: {
2251 isEffectivelyConnected = _isBroadcastConnected;
2252 break;
2253 }
2254 }
2255
2256 GroupNetworkState effectiveNetworkState;
2257 effectiveNetworkState.isConnected = isEffectivelyConnected;
2258 effectiveNetworkState.isTransitioningFromBroadcastToRtc = isTransitioningFromBroadcastToRtc;
2259
2260 if (_effectiveNetworkState.isConnected != effectiveNetworkState.isConnected || _effectiveNetworkState.isTransitioningFromBroadcastToRtc != effectiveNetworkState.isTransitioningFromBroadcastToRtc) {
2261 _effectiveNetworkState = effectiveNetworkState;
2262
2263 if (_networkStateUpdated) {
2264 _networkStateUpdated(_effectiveNetworkState);
2265 }
2266 }
2267 }
2268
updateIsDataChannelOpen(bool isDataChannelOpen)2269 void updateIsDataChannelOpen(bool isDataChannelOpen) {
2270 if (_isDataChannelOpen == isDataChannelOpen) {
2271 return;
2272 }
2273 _isDataChannelOpen = isDataChannelOpen;
2274
2275 if (_isDataChannelOpen) {
2276 maybeUpdateRemoteVideoConstraints();
2277 }
2278 }
2279
receivePacket(rtc::CopyOnWriteBuffer const & packet,bool isUnresolved)2280 void receivePacket(rtc::CopyOnWriteBuffer const &packet, bool isUnresolved) {
2281 if (packet.size() >= 4) {
2282 if (packet.data()[0] == 0x13 && packet.data()[1] == 0x88 && packet.data()[2] == 0x13 && packet.data()[3] == 0x88) {
2283 // SCTP packet header (source port 5000, destination port 5000)
2284 return;
2285 }
2286 }
2287
2288 webrtc::RtpUtility::RtpHeaderParser rtpParser(packet.data(), packet.size());
2289
2290 webrtc::RTPHeader header;
2291 if (rtpParser.RTCP()) {
2292 if (!rtpParser.ParseRtcp(&header)) {
2293 RTC_LOG(LS_INFO) << "Could not parse rtcp header";
2294 return;
2295 }
2296
2297 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, packet]() {
2298 _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, -1);
2299 });
2300 } else {
2301 if (!rtpParser.Parse(&header)) {
2302 // Probably a data channel message
2303 return;
2304 }
2305
2306 if (header.ssrc == _outgoingAudioSsrc) {
2307 return;
2308 }
2309
2310 auto ssrcInfo = _channelBySsrc.find(header.ssrc);
2311 if (ssrcInfo == _channelBySsrc.end()) {
2312 // opus
2313 if (header.payloadType == 111) {
2314 maybeRequestUnknownSsrc(header.ssrc);
2315 _missingPacketBuffer.add(header.ssrc, packet);
2316 }
2317 } else {
2318 switch (ssrcInfo->second.type) {
2319 case ChannelSsrcInfo::Type::Audio: {
2320 const auto it = _incomingAudioChannels.find(ChannelId(header.ssrc));
2321 if (it != _incomingAudioChannels.end()) {
2322 it->second->updateActivity();
2323 }
2324
2325 break;
2326 }
2327 case ChannelSsrcInfo::Type::Video: {
2328 break;
2329 }
2330 default: {
2331 break;
2332 }
2333 }
2334 }
2335 }
2336 }
2337
receiveRtcpPacket(rtc::CopyOnWriteBuffer const & packet,int64_t timestamp)2338 void receiveRtcpPacket(rtc::CopyOnWriteBuffer const &packet, int64_t timestamp) {
2339 _threads->getWorkerThread()->PostTask(RTC_FROM_HERE, [this, packet, timestamp]() {
2340 _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, timestamp);
2341 });
2342 }
2343
receiveDataChannelMessage(std::string const & message)2344 void receiveDataChannelMessage(std::string const &message) {
2345 std::string parsingError;
2346 auto json = json11::Json::parse(message, parsingError);
2347 if (json.type() != json11::Json::OBJECT) {
2348 RTC_LOG(LS_WARNING) << "receiveDataChannelMessage: error parsing message: " << parsingError;
2349 return;
2350 }
2351
2352 if (json.is_object()) {
2353 const auto colibriClass = json.object_items().find("colibriClass");
2354 if (colibriClass != json.object_items().end() && colibriClass->second.is_string()) {
2355 const auto messageType = colibriClass->second.string_value();
2356 if (messageType == "SenderVideoConstraints") {
2357 const auto videoConstraints = json.object_items().find("videoConstraints");
2358 if (videoConstraints != json.object_items().end() && videoConstraints->second.is_object()) {
2359 const auto idealHeight = videoConstraints->second.object_items().find("idealHeight");
2360 if (idealHeight != videoConstraints->second.object_items().end() && idealHeight->second.is_number()) {
2361 int outgoingVideoConstraint = idealHeight->second.int_value();
2362 if (_outgoingVideoConstraint != outgoingVideoConstraint) {
2363 if (_outgoingVideoConstraint > outgoingVideoConstraint) {
2364 _pendingOutgoingVideoConstraint = outgoingVideoConstraint;
2365
2366 int requestId = _pendingOutgoingVideoConstraintRequestId;
2367 _pendingOutgoingVideoConstraintRequestId += 1;
2368
2369 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2370 _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak, requestId]() {
2371 auto strong = weak.lock();
2372 if (!strong) {
2373 return;
2374 }
2375 if (strong->_pendingOutgoingVideoConstraint != -1 && strong->_pendingOutgoingVideoConstraintRequestId == requestId) {
2376 if (strong->_outgoingVideoConstraint != strong->_pendingOutgoingVideoConstraint) {
2377 strong->_outgoingVideoConstraint = strong->_pendingOutgoingVideoConstraint;
2378 strong->adjustVideoSendParams();
2379 }
2380 strong->_pendingOutgoingVideoConstraint = -1;
2381 }
2382 }, 2000);
2383 } else {
2384 _pendingOutgoingVideoConstraint = -1;
2385 _pendingOutgoingVideoConstraintRequestId += 1;
2386 _outgoingVideoConstraint = outgoingVideoConstraint;
2387 adjustVideoSendParams();
2388 }
2389 }
2390 }
2391 }
2392 } else if (messageType == "DebugMessage") {
2393 const auto message = json.object_items().find("message");
2394 if (message != json.object_items().end() && message->second.is_string()) {
2395 std::vector<std::string> parts = splitString(message->second.string_value(), '\n');
2396 for (const auto &part : parts) {
2397 std::string cleanString = part;
2398 std::size_t index = cleanString.find("=");
2399 if (index == std::string::npos) {
2400 continue;
2401 }
2402 cleanString.erase(cleanString.begin(), cleanString.begin() + index + 1);
2403
2404 index = cleanString.find("target=");
2405 if (index == std::string::npos) {
2406 continue;
2407 }
2408
2409 std::string endpointId = cleanString.substr(0, index);
2410 cleanString.erase(cleanString.begin(), cleanString.begin() + index + 7);
2411
2412 index = cleanString.find("p/");
2413 if (index == std::string::npos) {
2414 continue;
2415 }
2416
2417 std::string targetQuality = cleanString.substr(0, index);
2418 cleanString.erase(cleanString.begin(), cleanString.begin() + index + 2);
2419
2420 index = cleanString.find("ideal=");
2421 if (index == std::string::npos) {
2422 continue;
2423 }
2424
2425 cleanString.erase(cleanString.begin(), cleanString.begin() + index + 6);
2426
2427 index = cleanString.find("p/");
2428 if (index == std::string::npos) {
2429 continue;
2430 }
2431
2432 std::string availableQuality = cleanString.substr(0, index);
2433
2434 for (const auto &it : _incomingVideoChannels) {
2435 if (it.second->endpointId() == endpointId) {
2436 GroupInstanceStats::IncomingVideoStats incomingVideoStats;
2437 incomingVideoStats.receivingQuality = stringToInt(targetQuality);
2438 incomingVideoStats.availableQuality = stringToInt(availableQuality);
2439 it.second->setStats(incomingVideoStats);
2440 }
2441 }
2442 }
2443 }
2444 }
2445 }
2446 }
2447 }
2448
maybeRequestUnknownSsrc(uint32_t ssrc)2449 void maybeRequestUnknownSsrc(uint32_t ssrc) {
2450 if (!_requestMediaChannelDescriptions) {
2451 MediaChannelDescription description;
2452 description.audioSsrc = ssrc;
2453 processMediaChannelDescriptionsResponse(-1, {description});
2454 return;
2455 }
2456
2457 for (const auto &it : _requestedMediaChannelDescriptions) {
2458 if (std::find(it.second.ssrcs.begin(), it.second.ssrcs.end(), ssrc) != it.second.ssrcs.end()) {
2459 return;
2460 }
2461 }
2462
2463 int requestId = _nextMediaChannelDescriptionsRequestId;
2464 _nextMediaChannelDescriptionsRequestId += 1;
2465
2466 std::vector<uint32_t> requestSsrcs = { ssrc };
2467
2468 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2469 auto task = _requestMediaChannelDescriptions(requestSsrcs, [weak, threads = _threads, requestId](std::vector<MediaChannelDescription> &&descriptions) {
2470 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, requestId, descriptions = std::move(descriptions)]() mutable {
2471 auto strong = weak.lock();
2472 if (!strong) {
2473 return;
2474 }
2475
2476 strong->processMediaChannelDescriptionsResponse(requestId, descriptions);
2477 });
2478 });
2479 _requestedMediaChannelDescriptions.insert(std::make_pair(requestId, RequestedMediaChannelDescriptions(task, std::move(requestSsrcs))));
2480 }
2481
processMediaChannelDescriptionsResponse(int requestId,std::vector<MediaChannelDescription> const & descriptions)2482 void processMediaChannelDescriptionsResponse(int requestId, std::vector<MediaChannelDescription> const &descriptions) {
2483 _requestedMediaChannelDescriptions.erase(requestId);
2484
2485 if (_disableIncomingChannels) {
2486 return;
2487 }
2488
2489 for (const auto &description : descriptions) {
2490 switch (description.type) {
2491 case MediaChannelDescription::Type::Audio: {
2492 if (description.audioSsrc != 0) {
2493 addIncomingAudioChannel(ChannelId(description.audioSsrc));
2494 }
2495 break;
2496 }
2497 case MediaChannelDescription::Type::Video: {
2498 break;
2499 }
2500 default: {
2501 break;
2502 }
2503 }
2504 }
2505 }
2506
maybeDeliverBufferedPackets(uint32_t ssrc)2507 void maybeDeliverBufferedPackets(uint32_t ssrc) {
2508 // TODO: Re-enable after implementing custom transport
2509 /*auto packets = _missingPacketBuffer.get(ssrc);
2510 if (packets.size() != 0) {
2511 auto it = _ssrcMapping.find(ssrc);
2512 if (it != _ssrcMapping.end()) {
2513 for (const auto &packet : packets) {
2514 _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this, packet]() {
2515 _rtpTransport->DemuxPacketInternal(packet, -1);
2516 });
2517 }
2518 }
2519 }*/
2520 }
2521
maybeUpdateRemoteVideoConstraints()2522 void maybeUpdateRemoteVideoConstraints() {
2523 if (!_isDataChannelOpen) {
2524 return;
2525 }
2526
2527 std::string pinnedEndpoint;
2528
2529 json11::Json::object json;
2530 json.insert(std::make_pair("colibriClass", json11::Json("ReceiverVideoConstraints")));
2531
2532 json11::Json::object defaultConstraints;
2533 defaultConstraints.insert(std::make_pair("maxHeight", json11::Json(0)));
2534 json.insert(std::make_pair("defaultConstraints", json11::Json(std::move(defaultConstraints))));
2535
2536 json11::Json::array onStageEndpoints;
2537 json11::Json::object constraints;
2538
2539 for (const auto &incomingVideoChannel : _incomingVideoChannels) {
2540 json11::Json::object selectedConstraint;
2541
2542 switch (incomingVideoChannel.second->requestedMinQuality()) {
2543 case VideoChannelDescription::Quality::Full: {
2544 selectedConstraint.insert(std::make_pair("minHeight", json11::Json(720)));
2545 break;
2546 }
2547 case VideoChannelDescription::Quality::Medium: {
2548 selectedConstraint.insert(std::make_pair("minHeight", json11::Json(360)));
2549 break;
2550 }
2551 case VideoChannelDescription::Quality::Thumbnail: {
2552 selectedConstraint.insert(std::make_pair("minHeight", json11::Json(180)));
2553 break;
2554 }
2555 default: {
2556 break;
2557 }
2558 }
2559 switch (incomingVideoChannel.second->requestedMaxQuality()) {
2560 case VideoChannelDescription::Quality::Full: {
2561 onStageEndpoints.push_back(json11::Json(incomingVideoChannel.first.endpointId));
2562 selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(720)));
2563 break;
2564 }
2565 case VideoChannelDescription::Quality::Medium: {
2566 selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(360)));
2567 break;
2568 }
2569 case VideoChannelDescription::Quality::Thumbnail: {
2570 selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(180)));
2571 break;
2572 }
2573 default: {
2574 break;
2575 }
2576 }
2577
2578 constraints.insert(std::make_pair(incomingVideoChannel.first.endpointId, json11::Json(std::move(selectedConstraint))));
2579 }
2580
2581 json.insert(std::make_pair("onStageEndpoints", json11::Json(std::move(onStageEndpoints))));
2582 json.insert(std::make_pair("constraints", json11::Json(std::move(constraints))));
2583
2584 std::string result = json11::Json(std::move(json)).dump();
2585 _networkManager->perform(RTC_FROM_HERE, [result = std::move(result)](GroupNetworkManager *networkManager) {
2586 networkManager->sendDataChannelMessage(result);
2587 });
2588 }
2589
setConnectionMode(GroupConnectionMode connectionMode,bool keepBroadcastIfWasEnabled)2590 void setConnectionMode(GroupConnectionMode connectionMode, bool keepBroadcastIfWasEnabled) {
2591 if (_connectionMode != connectionMode || connectionMode == GroupConnectionMode::GroupConnectionModeNone) {
2592 GroupConnectionMode previousMode = _connectionMode;
2593 _connectionMode = connectionMode;
2594 onConnectionModeUpdated(previousMode, keepBroadcastIfWasEnabled);
2595 }
2596 }
2597
onConnectionModeUpdated(GroupConnectionMode previousMode,bool keepBroadcastIfWasEnabled)2598 void onConnectionModeUpdated(GroupConnectionMode previousMode, bool keepBroadcastIfWasEnabled) {
2599 RTC_CHECK(_connectionMode != previousMode || _connectionMode == GroupConnectionMode::GroupConnectionModeNone);
2600
2601 if (previousMode == GroupConnectionMode::GroupConnectionModeRtc) {
2602 _networkManager->perform(RTC_FROM_HERE, [](GroupNetworkManager *networkManager) {
2603 networkManager->stop();
2604 });
2605 } else if (previousMode == GroupConnectionMode::GroupConnectionModeBroadcast) {
2606 if (keepBroadcastIfWasEnabled) {
2607 _broadcastEnabledUntilRtcIsConnectedAtTimestamp = rtc::TimeMillis();
2608 } else {
2609 if (_streamingContext) {
2610 _streamingContext.reset();
2611 _audioDeviceDataObserverShared->setStreamingContext(nullptr);
2612 }
2613 }
2614 }
2615
2616 if (_connectionMode == GroupConnectionMode::GroupConnectionModeNone) {
2617 destroyOutgoingAudioChannel();
2618 destroyOutgoingVideoChannel();
2619
2620 // Regenerate and reconfigure.
2621 generateSsrcs();
2622
2623 if (!_isMuted) {
2624 createOutgoingAudioChannel();
2625 }
2626 createOutgoingVideoChannel();
2627 }
2628
2629 switch (_connectionMode) {
2630 case GroupConnectionMode::GroupConnectionModeNone: {
2631 break;
2632 }
2633 case GroupConnectionMode::GroupConnectionModeRtc: {
2634 _networkManager->perform(RTC_FROM_HERE, [](GroupNetworkManager *networkManager) {
2635 networkManager->start();
2636 });
2637 break;
2638 }
2639 case GroupConnectionMode::GroupConnectionModeBroadcast: {
2640 _isBroadcastConnected = false;
2641
2642 if (!_streamingContext) {
2643 StreamingMediaContext::StreamingMediaContextArguments arguments;
2644 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2645 arguments.threads = _threads;
2646 arguments.requestCurrentTime = _requestCurrentTime;
2647 arguments.requestAudioBroadcastPart = _requestAudioBroadcastPart;
2648 arguments.requestVideoBroadcastPart = _requestVideoBroadcastPart;
2649 arguments.updateAudioLevel = [weak, threads = _threads](uint32_t ssrc, float level, bool isSpeech) {
2650 assert(threads->getMediaThread()->IsCurrent());
2651
2652 auto strong = weak.lock();
2653 if (!strong) {
2654 return;
2655 }
2656
2657 InternalGroupLevelValue updated;
2658 updated.value.level = level;
2659 updated.value.voice = isSpeech;
2660 updated.timestamp = rtc::TimeMillis();
2661 strong->_audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
2662 };
2663 _streamingContext = std::make_shared<StreamingMediaContext>(std::move(arguments));
2664
2665 for (const auto &it : _pendingVideoSinks) {
2666 for (const auto &sink : it.second) {
2667 _streamingContext->addVideoSink(it.first.endpointId, sink);
2668 }
2669 }
2670
2671 for (const auto &it : _volumeBySsrc) {
2672 _streamingContext->setVolume(it.first, it.second);
2673 }
2674
2675 std::vector<StreamingMediaContext::VideoChannel> streamingVideoChannels;
2676 for (const auto &it : _pendingRequestedVideo) {
2677 streamingVideoChannels.emplace_back(it.maxQuality, it.endpointId);
2678 }
2679 _streamingContext->setActiveVideoChannels(streamingVideoChannels);
2680
2681 _audioDeviceDataObserverShared->setStreamingContext(_streamingContext);
2682 }
2683
2684 break;
2685 }
2686 default: {
2687 RTC_FATAL() << "Unknown connectionMode";
2688 break;
2689 }
2690 }
2691
2692 updateIsConnected();
2693 }
2694
generateSsrcs()2695 void generateSsrcs() {
2696 auto generator = std::mt19937(std::random_device()());
2697 auto distribution = std::uniform_int_distribution<uint32_t>();
2698 do {
2699 _outgoingAudioSsrc = distribution(generator) & 0x7fffffffU;
2700 } while (!_outgoingAudioSsrc);
2701
2702 uint32_t outgoingVideoSsrcBase = _outgoingAudioSsrc + 1;
2703 int numVideoSimulcastLayers = 3;
2704 if (_videoContentType == VideoContentType::Screencast) {
2705 numVideoSimulcastLayers = 2;
2706 }
2707 _outgoingVideoSsrcs.simulcastLayers.clear();
2708 for (int layerIndex = 0; layerIndex < numVideoSimulcastLayers; layerIndex++) {
2709 _outgoingVideoSsrcs.simulcastLayers.push_back(VideoSsrcs::SimulcastLayer(outgoingVideoSsrcBase + layerIndex * 2 + 0, outgoingVideoSsrcBase + layerIndex * 2 + 1));
2710 }
2711
2712 _videoSourceGroups.clear();
2713
2714 std::vector<uint32_t> simulcastGroupSsrcs;
2715 std::vector<cricket::SsrcGroup> fidGroups;
2716 for (const auto &layer : _outgoingVideoSsrcs.simulcastLayers) {
2717 simulcastGroupSsrcs.push_back(layer.ssrc);
2718
2719 cricket::SsrcGroup fidGroup(cricket::kFidSsrcGroupSemantics, { layer.ssrc, layer.fidSsrc });
2720 fidGroups.push_back(fidGroup);
2721 }
2722 if (simulcastGroupSsrcs.size() > 1) {
2723 cricket::SsrcGroup simulcastGroup(cricket::kSimSsrcGroupSemantics, simulcastGroupSsrcs);
2724
2725 GroupJoinPayloadVideoSourceGroup payloadSimulcastGroup;
2726 payloadSimulcastGroup.semantics = "SIM";
2727 payloadSimulcastGroup.ssrcs = simulcastGroupSsrcs;
2728 _videoSourceGroups.push_back(payloadSimulcastGroup);
2729 }
2730
2731 for (auto fidGroup : fidGroups) {
2732 GroupJoinPayloadVideoSourceGroup payloadFidGroup;
2733 payloadFidGroup.semantics = "FID";
2734 payloadFidGroup.ssrcs = fidGroup.ssrcs;
2735 _videoSourceGroups.push_back(payloadFidGroup);
2736 }
2737 }
2738
emitJoinPayload(std::function<void (GroupJoinPayload const &)> completion)2739 void emitJoinPayload(std::function<void(GroupJoinPayload const &)> completion) {
2740 _networkManager->perform(RTC_FROM_HERE, [outgoingAudioSsrc = _outgoingAudioSsrc, /*videoPayloadTypes = _videoPayloadTypes, videoExtensionMap = _videoExtensionMap, */videoSourceGroups = _videoSourceGroups, videoContentType = _videoContentType, completion](GroupNetworkManager *networkManager) {
2741 GroupJoinInternalPayload payload;
2742
2743 payload.audioSsrc = outgoingAudioSsrc;
2744
2745 if (videoContentType != VideoContentType::None) {
2746 GroupParticipantVideoInformation videoInformation;
2747 videoInformation.ssrcGroups = videoSourceGroups;
2748 payload.videoInformation = std::move(videoInformation);
2749 }
2750
2751 GroupJoinTransportDescription transportDescription;
2752
2753 auto localIceParameters = networkManager->getLocalIceParameters();
2754 transportDescription.ufrag = localIceParameters.ufrag;
2755 transportDescription.pwd = localIceParameters.pwd;
2756
2757 auto localFingerprint = networkManager->getLocalFingerprint();
2758 if (localFingerprint) {
2759 GroupJoinTransportDescription::Fingerprint serializedFingerprint;
2760 serializedFingerprint.hash = localFingerprint->algorithm;
2761 serializedFingerprint.fingerprint = localFingerprint->GetRfc4572Fingerprint();
2762 serializedFingerprint.setup = "passive";
2763 transportDescription.fingerprints.push_back(std::move(serializedFingerprint));
2764 }
2765
2766 payload.transport = std::move(transportDescription);
2767
2768 GroupJoinPayload result;
2769 result.audioSsrc = payload.audioSsrc;
2770 result.json = payload.serialize();
2771 completion(result);
2772 });
2773 }
2774
setVideoSource(std::function<webrtc::VideoTrackSourceInterface * ()> getVideoSource,bool isInitializing)2775 void setVideoSource(std::function<webrtc::VideoTrackSourceInterface*()> getVideoSource, bool isInitializing) {
2776 bool resetBitrate = (!_getVideoSource) != (!getVideoSource) && !isInitializing;
2777 if (!isInitializing && _getVideoSource && getVideoSource && getVideoSource() == _getVideoSource()) {
2778 return;
2779 }
2780
2781 _getVideoSource = std::move(getVideoSource);
2782 updateVideoSend();
2783 if (resetBitrate) {
2784 adjustBitratePreferences(true);
2785 }
2786 }
2787
setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture,bool isInitializing)2788 void setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture, bool isInitializing) {
2789 _videoCapture = videoCapture;
2790 setVideoSource(videoCaptureToGetVideoSource(std::move(videoCapture)), isInitializing);
2791 }
2792
setAudioOutputDevice(const std::string & id)2793 void setAudioOutputDevice(const std::string &id) {
2794 #ifndef WEBRTC_IOS
2795 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&] {
2796 SetAudioOutputDeviceById(_audioDeviceModule.get(), id);
2797 });
2798 #endif // WEBRTC_IOS
2799 }
2800
setAudioInputDevice(const std::string & id)2801 void setAudioInputDevice(const std::string &id) {
2802 #ifndef WEBRTC_IOS
2803 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&] {
2804 SetAudioInputDeviceById(_audioDeviceModule.get(), id);
2805 });
2806 #endif // WEBRTC_IOS
2807 }
2808
addExternalAudioSamples(std::vector<uint8_t> && samples)2809 void addExternalAudioSamples(std::vector<uint8_t> &&samples) {
2810 if (samples.size() % 2 != 0) {
2811 return;
2812 }
2813 _externalAudioSamplesMutex.Lock();
2814
2815 size_t previousSize = _externalAudioSamples.size();
2816 _externalAudioSamples.resize(_externalAudioSamples.size() + samples.size() / 2);
2817 webrtc::S16ToFloatS16((const int16_t *)samples.data(), samples.size() / 2, _externalAudioSamples.data() + previousSize);
2818
2819 if (_externalAudioSamples.size() > 2 * 48000) {
2820 _externalAudioSamples.erase(_externalAudioSamples.begin(), _externalAudioSamples.begin() + (_externalAudioSamples.size() - 2 * 48000));
2821 }
2822
2823 _externalAudioSamplesMutex.Unlock();
2824 }
2825
setJoinResponsePayload(std::string const & payload)2826 void setJoinResponsePayload(std::string const &payload) {
2827 RTC_LOG(LS_INFO) << formatTimestampMillis(rtc::TimeMillis()) << ": " << "setJoinResponsePayload";
2828
2829 auto parsedPayload = GroupJoinResponsePayload::parse(payload);
2830 if (!parsedPayload) {
2831 RTC_LOG(LS_ERROR) << "Could not parse json response payload";
2832 return;
2833 }
2834
2835 _sharedVideoInformation = parsedPayload->videoInformation;
2836
2837 _serverBandwidthProbingVideoSsrc.reset();
2838
2839 if (parsedPayload->videoInformation && parsedPayload->videoInformation->serverVideoBandwidthProbingSsrc) {
2840 setServerBandwidthProbingChannelSsrc(parsedPayload->videoInformation->serverVideoBandwidthProbingSsrc);
2841 }
2842
2843 _networkManager->perform(RTC_FROM_HERE, [parsedTransport = parsedPayload->transport](GroupNetworkManager *networkManager) {
2844 PeerIceParameters remoteIceParameters;
2845 remoteIceParameters.ufrag = parsedTransport.ufrag;
2846 remoteIceParameters.pwd = parsedTransport.pwd;
2847
2848 std::vector<cricket::Candidate> iceCandidates;
2849 for (auto const &candidate : parsedTransport.candidates) {
2850 rtc::SocketAddress address(candidate.ip, stringToInt(candidate.port));
2851
2852 cricket::Candidate parsedCandidate(
2853 /*component=*/stringToInt(candidate.component),
2854 /*protocol=*/candidate.protocol,
2855 /*address=*/address,
2856 /*priority=*/stringToUInt32(candidate.priority),
2857 /*username=*/parsedTransport.ufrag,
2858 /*password=*/parsedTransport.pwd,
2859 /*type=*/candidate.type,
2860 /*generation=*/stringToUInt32(candidate.generation),
2861 /*foundation=*/candidate.foundation,
2862 /*network_id=*/stringToUInt16(candidate.network),
2863 /*network_cost=*/0
2864 );
2865 iceCandidates.push_back(parsedCandidate);
2866 }
2867
2868 std::unique_ptr<rtc::SSLFingerprint> fingerprint;
2869 if (parsedTransport.fingerprints.size() != 0) {
2870 fingerprint = rtc::SSLFingerprint::CreateUniqueFromRfc4572(parsedTransport.fingerprints[0].hash, parsedTransport.fingerprints[0].fingerprint);
2871 }
2872
2873 networkManager->setRemoteParams(remoteIceParameters, iceCandidates, fingerprint.get());
2874 });
2875
2876 configureVideoParams();
2877 createOutgoingVideoChannel();
2878
2879 adjustBitratePreferences(true);
2880
2881 if (!_pendingRequestedVideo.empty()) {
2882 setRequestedVideoChannels(std::move(_pendingRequestedVideo));
2883 _pendingRequestedVideo.clear();
2884 }
2885 }
2886
setServerBandwidthProbingChannelSsrc(uint32_t probingSsrc)2887 void setServerBandwidthProbingChannelSsrc(uint32_t probingSsrc) {
2888 RTC_CHECK(probingSsrc);
2889
2890 if (!_sharedVideoInformation || _availablePayloadTypes.empty()) {
2891 return;
2892 }
2893
2894 GroupParticipantVideoInformation videoInformation;
2895
2896 GroupJoinPayloadVideoSourceGroup sourceGroup;
2897 sourceGroup.ssrcs.push_back(probingSsrc);
2898 sourceGroup.semantics = "SIM";
2899
2900 videoInformation.ssrcGroups.push_back(std::move(sourceGroup));
2901
2902 _serverBandwidthProbingVideoSsrc.reset(new IncomingVideoChannel(
2903 _channelManager.get(),
2904 _call.get(),
2905 _rtpTransport,
2906 _uniqueRandomIdGenerator.get(),
2907 _availableVideoFormats,
2908 _sharedVideoInformation.value(),
2909 123456,
2910 VideoChannelDescription::Quality::Thumbnail,
2911 VideoChannelDescription::Quality::Thumbnail,
2912 videoInformation,
2913 _threads
2914 ));
2915
2916 ChannelSsrcInfo mapping;
2917 mapping.type = ChannelSsrcInfo::Type::Video;
2918 mapping.allSsrcs.push_back(probingSsrc);
2919 _channelBySsrc.insert(std::make_pair(probingSsrc, std::move(mapping)));
2920 }
2921
removeSsrcs(std::vector<uint32_t> ssrcs)2922 void removeSsrcs(std::vector<uint32_t> ssrcs) {
2923 }
2924
removeIncomingVideoSource(uint32_t ssrc)2925 void removeIncomingVideoSource(uint32_t ssrc) {
2926 }
2927
setIsMuted(bool isMuted)2928 void setIsMuted(bool isMuted) {
2929 if (_isMuted == isMuted) {
2930 return;
2931 }
2932 _isMuted = isMuted;
2933
2934 if (!_isMuted && !_outgoingAudioChannel) {
2935 createOutgoingAudioChannel();
2936 }
2937
2938 onUpdatedIsMuted();
2939 }
2940
onUpdatedIsMuted()2941 void onUpdatedIsMuted() {
2942 if (_outgoingAudioChannel) {
2943 _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
2944 _outgoingAudioChannel->media_channel()->SetAudioSend(_outgoingAudioSsrc, !_isMuted, nullptr, &_audioSource);
2945 _outgoingAudioChannel->Enable(!_isMuted);
2946 });
2947 }
2948 }
2949
setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled)2950 void setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled) {
2951 _noiseSuppressionConfiguration->isEnabled = isNoiseSuppressionEnabled;
2952 }
2953
addIncomingVideoOutput(std::string const & endpointId,std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink)2954 void addIncomingVideoOutput(std::string const &endpointId, std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
2955 if (_sharedVideoInformation && endpointId == _sharedVideoInformation->endpointId) {
2956 if (_videoCapture) {
2957 _videoCaptureSink->addSink(sink);
2958 _videoCapture->setOutput(_videoCaptureSink);
2959 }
2960 } else {
2961 auto it = _incomingVideoChannels.find(VideoChannelId(endpointId));
2962 if (it != _incomingVideoChannels.end()) {
2963 it->second->addSink(sink);
2964 } else {
2965 _pendingVideoSinks[VideoChannelId(endpointId)].push_back(sink);
2966 }
2967
2968 if (_streamingContext) {
2969 _streamingContext->addVideoSink(endpointId, sink);
2970 }
2971 }
2972 }
2973
addIncomingAudioChannel(ChannelId ssrc,bool isRawPcm=false)2974 void addIncomingAudioChannel(ChannelId ssrc, bool isRawPcm = false) {
2975 if (_incomingAudioChannels.find(ssrc) != _incomingAudioChannels.end()) {
2976 return;
2977 }
2978
2979 if (_incomingAudioChannels.size() > 5) {
2980 auto timestamp = rtc::TimeMillis();
2981
2982 int64_t minActivity = INT64_MAX;
2983 ChannelId minActivityChannelId(0, 0);
2984
2985 for (const auto &it : _incomingAudioChannels) {
2986 if (it.first.networkSsrc == 1) {
2987 continue;
2988 }
2989 auto activity = it.second->getActivity();
2990 if (activity < minActivity && activity < timestamp - 1000) {
2991 minActivity = activity;
2992 minActivityChannelId = it.first;
2993 }
2994 }
2995
2996 if (minActivityChannelId.networkSsrc != 0) {
2997 removeIncomingAudioChannel(minActivityChannelId);
2998 }
2999
3000 if (_incomingAudioChannels.size() > 5) {
3001 // Wait until there is a channel that hasn't been active in 1 second
3002 return;
3003 }
3004 }
3005
3006 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
3007
3008 std::function<void(AudioSinkImpl::Update)> onAudioSinkUpdate;
3009 if (ssrc.actualSsrc != ssrc.networkSsrc) {
3010 if (_audioLevelsUpdated) {
3011 onAudioSinkUpdate = [weak, ssrc = ssrc, threads = _threads](AudioSinkImpl::Update update) {
3012 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, ssrc, update]() {
3013 auto strong = weak.lock();
3014 if (!strong) {
3015 return;
3016 }
3017
3018 auto it = strong->_audioLevels.find(ChannelId(ssrc));
3019 if (it != strong->_audioLevels.end()) {
3020 it->second.value.level = fmax(it->second.value.level, update.level);
3021 if (update.hasSpeech) {
3022 it->second.value.voice = true;
3023 }
3024 it->second.timestamp = rtc::TimeMillis();
3025 } else {
3026 InternalGroupLevelValue updated;
3027 updated.value.level = update.level;
3028 updated.value.voice = update.hasSpeech;
3029 updated.timestamp = rtc::TimeMillis();
3030 strong->_audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
3031 }
3032 });
3033 };
3034 }
3035 }
3036
3037 std::unique_ptr<IncomingAudioChannel> channel(new IncomingAudioChannel(
3038 _channelManager.get(),
3039 _call.get(),
3040 _rtpTransport,
3041 _uniqueRandomIdGenerator.get(),
3042 isRawPcm,
3043 ssrc,
3044 std::move(onAudioSinkUpdate),
3045 _onAudioFrame,
3046 _threads
3047 ));
3048
3049 auto volume = _volumeBySsrc.find(ssrc.actualSsrc);
3050 if (volume != _volumeBySsrc.end()) {
3051 channel->setVolume(volume->second);
3052 }
3053
3054 _incomingAudioChannels.insert(std::make_pair(ssrc, std::move(channel)));
3055
3056 auto currentMapping = _channelBySsrc.find(ssrc.networkSsrc);
3057 if (currentMapping != _channelBySsrc.end()) {
3058 if (currentMapping->second.type == ChannelSsrcInfo::Type::Audio) {
3059 if (std::find(currentMapping->second.allSsrcs.begin(), currentMapping->second.allSsrcs.end(), ssrc.networkSsrc) == currentMapping->second.allSsrcs.end()) {
3060 currentMapping->second.allSsrcs.push_back(ssrc.networkSsrc);
3061 }
3062 }
3063 } else {
3064 ChannelSsrcInfo mapping;
3065 mapping.type = ChannelSsrcInfo::Type::Audio;
3066 mapping.allSsrcs.push_back(ssrc.networkSsrc);
3067 _channelBySsrc.insert(std::make_pair(ssrc.networkSsrc, std::move(mapping)));
3068 }
3069
3070 maybeDeliverBufferedPackets(ssrc.networkSsrc);
3071
3072 adjustBitratePreferences(false);
3073 }
3074
removeIncomingAudioChannel(ChannelId const & channelId)3075 void removeIncomingAudioChannel(ChannelId const &channelId) {
3076 const auto it = _incomingAudioChannels.find(channelId);
3077 if (it != _incomingAudioChannels.end()) {
3078 _incomingAudioChannels.erase(it);
3079 }
3080
3081 auto currentMapping = _channelBySsrc.find(channelId.networkSsrc);
3082 if (currentMapping != _channelBySsrc.end()) {
3083 if (currentMapping->second.type == ChannelSsrcInfo::Type::Audio) {
3084 auto ssrcs = currentMapping->second.allSsrcs;
3085 for (auto ssrc : ssrcs) {
3086 auto it = _channelBySsrc.find(ssrc);
3087 if (it != _channelBySsrc.end()) {
3088 _channelBySsrc.erase(it);
3089 }
3090 }
3091 }
3092 }
3093 }
3094
addIncomingVideoChannel(uint32_t audioSsrc,GroupParticipantVideoInformation const & videoInformation,VideoChannelDescription::Quality minQuality,VideoChannelDescription::Quality maxQuality)3095 void addIncomingVideoChannel(uint32_t audioSsrc, GroupParticipantVideoInformation const &videoInformation, VideoChannelDescription::Quality minQuality, VideoChannelDescription::Quality maxQuality) {
3096 if (!_sharedVideoInformation) {
3097 return;
3098 }
3099 if (_incomingVideoChannels.find(VideoChannelId(videoInformation.endpointId)) != _incomingVideoChannels.end()) {
3100 return;
3101 }
3102
3103 const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
3104
3105 std::unique_ptr<IncomingVideoChannel> channel(new IncomingVideoChannel(
3106 _channelManager.get(),
3107 _call.get(),
3108 _rtpTransport,
3109 _uniqueRandomIdGenerator.get(),
3110 _availableVideoFormats,
3111 _sharedVideoInformation.value(),
3112 audioSsrc,
3113 minQuality,
3114 maxQuality,
3115 videoInformation,
3116 _threads
3117 ));
3118
3119 const auto pendingSinks = _pendingVideoSinks.find(VideoChannelId(videoInformation.endpointId));
3120 if (pendingSinks != _pendingVideoSinks.end()) {
3121 for (const auto &sink : pendingSinks->second) {
3122 channel->addSink(sink);
3123 }
3124
3125 _pendingVideoSinks.erase(pendingSinks);
3126 }
3127
3128 _incomingVideoChannels.insert(std::make_pair(VideoChannelId(videoInformation.endpointId), std::move(channel)));
3129
3130 std::vector<uint32_t> allSsrcs;
3131 for (const auto &group : videoInformation.ssrcGroups) {
3132 for (auto ssrc : group.ssrcs) {
3133 if (std::find(allSsrcs.begin(), allSsrcs.end(), ssrc) == allSsrcs.end()) {
3134 allSsrcs.push_back(ssrc);
3135 }
3136 }
3137 }
3138
3139 for (auto ssrc : allSsrcs) {
3140 ChannelSsrcInfo mapping;
3141 mapping.type = ChannelSsrcInfo::Type::Video;
3142 mapping.allSsrcs = allSsrcs;
3143 mapping.videoEndpointId = videoInformation.endpointId;
3144 _channelBySsrc.insert(std::make_pair(ssrc, std::move(mapping)));
3145 }
3146
3147 for (auto ssrc : allSsrcs) {
3148 maybeDeliverBufferedPackets(ssrc);
3149 }
3150
3151 adjustBitratePreferences(false);
3152 }
3153
setVolume(uint32_t ssrc,double volume)3154 void setVolume(uint32_t ssrc, double volume) {
3155 auto current = _volumeBySsrc.find(ssrc);
3156 if (current != _volumeBySsrc.end() && std::abs(current->second - volume) < 0.0001) {
3157 return;
3158 }
3159
3160 _volumeBySsrc[ssrc] = volume;
3161
3162 auto it = _incomingAudioChannels.find(ChannelId(ssrc));
3163 if (it != _incomingAudioChannels.end()) {
3164 it->second->setVolume(volume);
3165 }
3166
3167 it = _incomingAudioChannels.find(ChannelId(ssrc + 1000, ssrc));
3168 if (it != _incomingAudioChannels.end()) {
3169 it->second->setVolume(volume);
3170 }
3171
3172 if (_streamingContext) {
3173 _streamingContext->setVolume(ssrc, volume);
3174 }
3175 }
3176
setRequestedVideoChannels(std::vector<VideoChannelDescription> && requestedVideoChannels)3177 void setRequestedVideoChannels(std::vector<VideoChannelDescription> &&requestedVideoChannels) {
3178 if (_streamingContext) {
3179 std::vector<StreamingMediaContext::VideoChannel> streamingVideoChannels;
3180 for (const auto &it : requestedVideoChannels) {
3181 streamingVideoChannels.emplace_back(it.maxQuality, it.endpointId);
3182 }
3183 _streamingContext->setActiveVideoChannels(streamingVideoChannels);
3184 }
3185
3186 if (!_sharedVideoInformation) {
3187 _pendingRequestedVideo = std::move(requestedVideoChannels);
3188 return;
3189 }
3190 bool updated = false;
3191 std::vector<std::string> allEndpointIds;
3192
3193 for (const auto &description : requestedVideoChannels) {
3194 if (_sharedVideoInformation && _sharedVideoInformation->endpointId == description.endpointId) {
3195 continue;
3196 }
3197
3198 GroupParticipantVideoInformation videoInformation;
3199 videoInformation.endpointId = description.endpointId;
3200 for (const auto &group : description.ssrcGroups) {
3201 GroupJoinPayloadVideoSourceGroup parsedGroup;
3202 parsedGroup.semantics = group.semantics;
3203 parsedGroup.ssrcs = group.ssrcs;
3204 videoInformation.ssrcGroups.push_back(std::move(parsedGroup));
3205 }
3206
3207 allEndpointIds.push_back(videoInformation.endpointId);
3208
3209 auto current = _incomingVideoChannels.find(VideoChannelId(videoInformation.endpointId));
3210 if (current != _incomingVideoChannels.end()) {
3211 if (current->second->requestedMinQuality() != description.minQuality || current->second->requestedMaxQuality() != description.maxQuality) {
3212 current->second->setRequstedMinQuality(description.minQuality);
3213 current->second->setRequstedMaxQuality(description.maxQuality);
3214 updated = true;
3215 }
3216 continue;
3217 }
3218
3219 addIncomingVideoChannel(description.audioSsrc, videoInformation, description.minQuality, description.maxQuality);
3220 updated = true;
3221 }
3222
3223 std::vector<std::string> removeEndpointIds;
3224 for (const auto &it : _incomingVideoChannels) {
3225 if (std::find(allEndpointIds.begin(), allEndpointIds.end(), it.first.endpointId) == allEndpointIds.end()) {
3226 removeEndpointIds.push_back(it.first.endpointId);
3227 updated = true;
3228 }
3229 }
3230
3231 for (const auto &endpointId : removeEndpointIds) {
3232 const auto it = _incomingVideoChannels.find(VideoChannelId(endpointId));
3233 if (it != _incomingVideoChannels.end()) {
3234 auto sinks = it->second->getSinks();
3235 for (const auto &sink : sinks) {
3236 _pendingVideoSinks[VideoChannelId(endpointId)].push_back(sink);
3237 }
3238 _incomingVideoChannels.erase(it);
3239 }
3240 }
3241
3242 if (updated) {
3243 maybeUpdateRemoteVideoConstraints();
3244 }
3245 }
3246
getStats(std::function<void (GroupInstanceStats)> completion)3247 void getStats(std::function<void(GroupInstanceStats)> completion) {
3248 GroupInstanceStats result;
3249
3250 for (const auto &it : _incomingVideoChannels) {
3251 const auto videoStats = it.second->getStats();
3252 if (videoStats) {
3253 result.incomingVideoStats.push_back(std::make_pair(it.second->endpointId(), videoStats.value()));
3254 }
3255 }
3256
3257 completion(result);
3258 }
3259
3260 private:
createAudioDeviceModule()3261 rtc::scoped_refptr<WrappedAudioDeviceModule> createAudioDeviceModule() {
3262 auto audioDeviceDataObserverShared = _audioDeviceDataObserverShared;
3263 const auto create = [&](webrtc::AudioDeviceModule::AudioLayer layer) {
3264 return webrtc::AudioDeviceModule::Create(
3265 layer,
3266 _taskQueueFactory.get());
3267 };
3268 const auto check = [&](const rtc::scoped_refptr<webrtc::AudioDeviceModule> &result) -> rtc::scoped_refptr<WrappedAudioDeviceModule> {
3269 if (!result) {
3270 return nullptr;
3271 }
3272
3273 auto audioDeviceObserver = std::make_unique<AudioDeviceDataObserverImpl>(audioDeviceDataObserverShared);
3274 auto module = webrtc::CreateAudioDeviceWithDataObserver(result, std::move(audioDeviceObserver));
3275
3276 if (module->Init() == 0) {
3277 return PlatformInterface::SharedInstance()->wrapAudioDeviceModule(module);
3278 } else {
3279 return nullptr;
3280 }
3281 };
3282 if (_createAudioDeviceModule) {
3283 if (const auto result = check(_createAudioDeviceModule(_taskQueueFactory.get()))) {
3284 return result;
3285 }
3286 } else if (_videoContentType == VideoContentType::Screencast) {
3287 FakeAudioDeviceModule::Options options;
3288 options.num_channels = 1;
3289 return check(FakeAudioDeviceModule::Creator(nullptr, _externalAudioRecorder, options)(_taskQueueFactory.get()));
3290 }
3291 return check(create(webrtc::AudioDeviceModule::kPlatformDefaultAudio));
3292 }
3293
3294 private:
3295 std::shared_ptr<Threads> _threads;
3296 GroupConnectionMode _connectionMode = GroupConnectionMode::GroupConnectionModeNone;
3297
3298 std::function<void(GroupNetworkState)> _networkStateUpdated;
3299 std::function<void(GroupLevelsUpdate const &)> _audioLevelsUpdated;
3300 std::function<void(uint32_t, const AudioFrame &)> _onAudioFrame;
3301 std::function<std::shared_ptr<RequestMediaChannelDescriptionTask>(std::vector<uint32_t> const &, std::function<void(std::vector<MediaChannelDescription> &&)>)> _requestMediaChannelDescriptions;
3302 std::function<std::shared_ptr<BroadcastPartTask>(std::function<void(int64_t)>)> _requestCurrentTime;
3303 std::function<std::shared_ptr<BroadcastPartTask>(int64_t, int64_t, std::function<void(BroadcastPart &&)>)> _requestAudioBroadcastPart;
3304 std::function<std::shared_ptr<BroadcastPartTask>(int64_t, int64_t, int32_t, VideoChannelDescription::Quality, std::function<void(BroadcastPart &&)>)> _requestVideoBroadcastPart;
3305 std::shared_ptr<VideoCaptureInterface> _videoCapture;
3306 std::shared_ptr<VideoSinkImpl> _videoCaptureSink;
3307 std::function<webrtc::VideoTrackSourceInterface*()> _getVideoSource;
3308 bool _disableIncomingChannels = false;
3309 bool _useDummyChannel{true};
3310 int _outgoingAudioBitrateKbit{32};
3311 bool _disableOutgoingAudioProcessing{false};
3312 int _minOutgoingVideoBitrateKbit{100};
3313 VideoContentType _videoContentType{VideoContentType::None};
3314 std::vector<VideoCodecName> _videoCodecPreferences;
3315
3316 int _nextMediaChannelDescriptionsRequestId = 0;
3317 std::map<int, RequestedMediaChannelDescriptions> _requestedMediaChannelDescriptions;
3318
3319 std::unique_ptr<ThreadLocalObject<GroupNetworkManager>> _networkManager;
3320
3321 std::unique_ptr<webrtc::RtcEventLogNull> _eventLog;
3322 std::unique_ptr<webrtc::TaskQueueFactory> _taskQueueFactory;
3323 std::unique_ptr<cricket::MediaEngineInterface> _mediaEngine;
3324 std::unique_ptr<webrtc::Call> _call;
3325 webrtc::FieldTrialBasedConfig _fieldTrials;
3326 webrtc::LocalAudioSinkAdapter _audioSource;
3327 std::shared_ptr<AudioDeviceDataObserverShared> _audioDeviceDataObserverShared;
3328 rtc::scoped_refptr<WrappedAudioDeviceModule> _audioDeviceModule;
3329 std::function<rtc::scoped_refptr<webrtc::AudioDeviceModule>(webrtc::TaskQueueFactory*)> _createAudioDeviceModule;
3330 std::string _initialInputDeviceId;
3331 std::string _initialOutputDeviceId;
3332
3333 // _outgoingAudioChannel memory is managed by _channelManager
3334 cricket::VoiceChannel *_outgoingAudioChannel = nullptr;
3335 uint32_t _outgoingAudioSsrc = 0;
3336
3337 std::vector<webrtc::SdpVideoFormat> _availableVideoFormats;
3338 std::vector<OutgoingVideoFormat> _availablePayloadTypes;
3339 absl::optional<OutgoingVideoFormat> _selectedPayloadType;
3340
3341 std::vector<GroupJoinPayloadVideoPayloadType> _videoPayloadTypes;
3342 std::vector<std::pair<uint32_t, std::string>> _videoExtensionMap;
3343 std::vector<GroupJoinPayloadVideoSourceGroup> _videoSourceGroups;
3344
3345 std::unique_ptr<rtc::UniqueRandomIdGenerator> _uniqueRandomIdGenerator;
3346 webrtc::RtpTransport *_rtpTransport = nullptr;
3347 std::unique_ptr<cricket::ChannelManager> _channelManager;
3348
3349 std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> _videoBitrateAllocatorFactory;
3350 // _outgoingVideoChannel memory is managed by _channelManager
3351 cricket::VideoChannel *_outgoingVideoChannel = nullptr;
3352 VideoSsrcs _outgoingVideoSsrcs;
3353 int _outgoingVideoConstraint = 720;
3354 int _pendingOutgoingVideoConstraint = -1;
3355 int _pendingOutgoingVideoConstraintRequestId = 0;
3356
3357 std::map<ChannelId, InternalGroupLevelValue> _audioLevels;
3358 GroupLevelValue _myAudioLevel;
3359
3360 bool _isMuted = true;
3361 std::shared_ptr<NoiseSuppressionConfiguration> _noiseSuppressionConfiguration;
3362
3363 MissingSsrcPacketBuffer _missingPacketBuffer;
3364 std::map<uint32_t, ChannelSsrcInfo> _channelBySsrc;
3365 std::map<uint32_t, double> _volumeBySsrc;
3366 std::map<ChannelId, std::unique_ptr<IncomingAudioChannel>> _incomingAudioChannels;
3367 std::map<VideoChannelId, std::unique_ptr<IncomingVideoChannel>> _incomingVideoChannels;
3368
3369 std::map<VideoChannelId, std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>>> _pendingVideoSinks;
3370 std::vector<VideoChannelDescription> _pendingRequestedVideo;
3371
3372 std::unique_ptr<IncomingVideoChannel> _serverBandwidthProbingVideoSsrc;
3373
3374 absl::optional<GroupJoinVideoInformation> _sharedVideoInformation;
3375
3376 std::vector<float> _externalAudioSamples;
3377 webrtc::Mutex _externalAudioSamplesMutex;
3378 std::shared_ptr<ExternalAudioRecorder> _externalAudioRecorder;
3379
3380 bool _isRtcConnected = false;
3381 bool _isBroadcastConnected = false;
3382 absl::optional<int64_t> _broadcastEnabledUntilRtcIsConnectedAtTimestamp;
3383 bool _isDataChannelOpen = false;
3384 GroupNetworkState _effectiveNetworkState;
3385
3386 std::shared_ptr<StreamingMediaContext> _streamingContext;
3387
3388 rtc::scoped_refptr<webrtc::PendingTaskSafetyFlag> _workerThreadSafery;
3389 rtc::scoped_refptr<webrtc::PendingTaskSafetyFlag> _networkThreadSafery;
3390 };
3391
GroupInstanceCustomImpl(GroupInstanceDescriptor && descriptor)3392 GroupInstanceCustomImpl::GroupInstanceCustomImpl(GroupInstanceDescriptor &&descriptor) {
3393 if (descriptor.config.need_log) {
3394 _logSink = std::make_unique<LogSinkImpl>(descriptor.config.logPath);
3395 rtc::LogMessage::SetLogToStderr(true);
3396 } else {
3397 rtc::LogMessage::SetLogToStderr(false);
3398 }
3399 rtc::LogMessage::LogToDebug(rtc::LS_INFO);
3400 if (_logSink) {
3401 rtc::LogMessage::AddLogToStream(_logSink.get(), rtc::LS_INFO);
3402 }
3403
3404 _threads = descriptor.threads;
3405 _internal.reset(new ThreadLocalObject<GroupInstanceCustomInternal>(_threads->getMediaThread(), [descriptor = std::move(descriptor), threads = _threads]() mutable {
3406 return new GroupInstanceCustomInternal(std::move(descriptor), threads);
3407 }));
3408 _internal->perform(RTC_FROM_HERE, [](GroupInstanceCustomInternal *internal) {
3409 internal->start();
3410 });
3411 }
3412
~GroupInstanceCustomImpl()3413 GroupInstanceCustomImpl::~GroupInstanceCustomImpl() {
3414 if (_logSink) {
3415 rtc::LogMessage::RemoveLogToStream(_logSink.get());
3416 }
3417 _internal.reset();
3418
3419 // Wait until _internal is destroyed
3420 _threads->getMediaThread()->Invoke<void>(RTC_FROM_HERE, [] {});
3421 }
3422
stop()3423 void GroupInstanceCustomImpl::stop() {
3424 _internal->perform(RTC_FROM_HERE, [](GroupInstanceCustomInternal *internal) {
3425 internal->stop();
3426 });
3427 }
3428
setConnectionMode(GroupConnectionMode connectionMode,bool keepBroadcastIfWasEnabled)3429 void GroupInstanceCustomImpl::setConnectionMode(GroupConnectionMode connectionMode, bool keepBroadcastIfWasEnabled) {
3430 _internal->perform(RTC_FROM_HERE, [connectionMode, keepBroadcastIfWasEnabled](GroupInstanceCustomInternal *internal) {
3431 internal->setConnectionMode(connectionMode, keepBroadcastIfWasEnabled);
3432 });
3433 }
3434
emitJoinPayload(std::function<void (GroupJoinPayload const &)> completion)3435 void GroupInstanceCustomImpl::emitJoinPayload(std::function<void(GroupJoinPayload const &)> completion) {
3436 _internal->perform(RTC_FROM_HERE, [completion](GroupInstanceCustomInternal *internal) {
3437 internal->emitJoinPayload(completion);
3438 });
3439 }
3440
setJoinResponsePayload(std::string const & payload)3441 void GroupInstanceCustomImpl::setJoinResponsePayload(std::string const &payload) {
3442 _internal->perform(RTC_FROM_HERE, [payload](GroupInstanceCustomInternal *internal) {
3443 internal->setJoinResponsePayload(payload);
3444 });
3445 }
3446
removeSsrcs(std::vector<uint32_t> ssrcs)3447 void GroupInstanceCustomImpl::removeSsrcs(std::vector<uint32_t> ssrcs) {
3448 _internal->perform(RTC_FROM_HERE, [ssrcs = std::move(ssrcs)](GroupInstanceCustomInternal *internal) mutable {
3449 internal->removeSsrcs(ssrcs);
3450 });
3451 }
3452
removeIncomingVideoSource(uint32_t ssrc)3453 void GroupInstanceCustomImpl::removeIncomingVideoSource(uint32_t ssrc) {
3454 _internal->perform(RTC_FROM_HERE, [ssrc](GroupInstanceCustomInternal *internal) mutable {
3455 internal->removeIncomingVideoSource(ssrc);
3456 });
3457 }
3458
setIsMuted(bool isMuted)3459 void GroupInstanceCustomImpl::setIsMuted(bool isMuted) {
3460 _internal->perform(RTC_FROM_HERE, [isMuted](GroupInstanceCustomInternal *internal) {
3461 internal->setIsMuted(isMuted);
3462 });
3463 }
3464
setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled)3465 void GroupInstanceCustomImpl::setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled) {
3466 _internal->perform(RTC_FROM_HERE, [isNoiseSuppressionEnabled](GroupInstanceCustomInternal *internal) {
3467 internal->setIsNoiseSuppressionEnabled(isNoiseSuppressionEnabled);
3468 });
3469 }
3470
setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture)3471 void GroupInstanceCustomImpl::setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture) {
3472 _internal->perform(RTC_FROM_HERE, [videoCapture](GroupInstanceCustomInternal *internal) {
3473 internal->setVideoCapture(videoCapture, false);
3474 });
3475 }
3476
setVideoSource(std::function<webrtc::VideoTrackSourceInterface * ()> getVideoSource)3477 void GroupInstanceCustomImpl::setVideoSource(std::function<webrtc::VideoTrackSourceInterface*()> getVideoSource) {
3478 _internal->perform(RTC_FROM_HERE, [getVideoSource](GroupInstanceCustomInternal *internal) {
3479 internal->setVideoSource(getVideoSource, false);
3480 });
3481 }
3482
setAudioOutputDevice(std::string id)3483 void GroupInstanceCustomImpl::setAudioOutputDevice(std::string id) {
3484 _internal->perform(RTC_FROM_HERE, [id](GroupInstanceCustomInternal *internal) {
3485 internal->setAudioOutputDevice(id);
3486 });
3487 }
3488
setAudioInputDevice(std::string id)3489 void GroupInstanceCustomImpl::setAudioInputDevice(std::string id) {
3490 _internal->perform(RTC_FROM_HERE, [id](GroupInstanceCustomInternal *internal) {
3491 internal->setAudioInputDevice(id);
3492 });
3493 }
3494
addExternalAudioSamples(std::vector<uint8_t> && samples)3495 void GroupInstanceCustomImpl::addExternalAudioSamples(std::vector<uint8_t> &&samples) {
3496 _internal->perform(RTC_FROM_HERE, [samples = std::move(samples)](GroupInstanceCustomInternal *internal) mutable {
3497 internal->addExternalAudioSamples(std::move(samples));
3498 });
3499 }
3500
addIncomingVideoOutput(std::string const & endpointId,std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink)3501 void GroupInstanceCustomImpl::addIncomingVideoOutput(std::string const &endpointId, std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
3502 _internal->perform(RTC_FROM_HERE, [endpointId, sink](GroupInstanceCustomInternal *internal) mutable {
3503 internal->addIncomingVideoOutput(endpointId, sink);
3504 });
3505 }
3506
setVolume(uint32_t ssrc,double volume)3507 void GroupInstanceCustomImpl::setVolume(uint32_t ssrc, double volume) {
3508 _internal->perform(RTC_FROM_HERE, [ssrc, volume](GroupInstanceCustomInternal *internal) {
3509 internal->setVolume(ssrc, volume);
3510 });
3511 }
3512
setRequestedVideoChannels(std::vector<VideoChannelDescription> && requestedVideoChannels)3513 void GroupInstanceCustomImpl::setRequestedVideoChannels(std::vector<VideoChannelDescription> &&requestedVideoChannels) {
3514 _internal->perform(RTC_FROM_HERE, [requestedVideoChannels = std::move(requestedVideoChannels)](GroupInstanceCustomInternal *internal) mutable {
3515 internal->setRequestedVideoChannels(std::move(requestedVideoChannels));
3516 });
3517 }
3518
getStats(std::function<void (GroupInstanceStats)> completion)3519 void GroupInstanceCustomImpl::getStats(std::function<void(GroupInstanceStats)> completion) {
3520 _internal->perform(RTC_FROM_HERE, [completion = std::move(completion)](GroupInstanceCustomInternal *internal) mutable {
3521 internal->getStats(completion);
3522 });
3523 }
3524
getAudioDevices(AudioDevice::Type type)3525 std::vector<GroupInstanceInterface::AudioDevice> GroupInstanceInterface::getAudioDevices(AudioDevice::Type type) {
3526 auto result = std::vector<AudioDevice>();
3527 #ifdef WEBRTC_LINUX //Not needed for ios, and some crl::sync stuff is needed for windows
3528 const auto resolve = [&] {
3529 const auto queueFactory = webrtc::CreateDefaultTaskQueueFactory();
3530 const auto info = webrtc::AudioDeviceModule::Create(
3531 webrtc::AudioDeviceModule::kPlatformDefaultAudio,
3532 queueFactory.get());
3533 if (!info || info->Init() < 0) {
3534 return;
3535 }
3536 const auto count = type == AudioDevice::Type::Input ? info->RecordingDevices() : info->PlayoutDevices();
3537 if (count <= 0) {
3538 return;
3539 }
3540 for (auto i = int16_t(); i != count; ++i) {
3541 char name[webrtc::kAdmMaxDeviceNameSize + 1] = { 0 };
3542 char id[webrtc::kAdmMaxGuidSize + 1] = { 0 };
3543 if (type == AudioDevice::Type::Input) {
3544 info->RecordingDeviceName(i, name, id);
3545 } else {
3546 info->PlayoutDeviceName(i, name, id);
3547 }
3548 result.push_back({ id, name });
3549 }
3550 };
3551 resolve();
3552 #endif
3553 return result;
3554 }
3555
3556 } // namespace tgcalls
3557