1 #include "GroupInstanceCustomImpl.h"
2 
3 #include <memory>
4 #include <iomanip>
5 
6 #include "Instance.h"
7 #include "VideoCaptureInterfaceImpl.h"
8 #include "VideoCapturerInterface.h"
9 #include "CodecSelectHelper.h"
10 #include "Message.h"
11 #include "platform/PlatformInterface.h"
12 #include "StaticThreads.h"
13 #include "GroupNetworkManager.h"
14 
15 #include "api/audio_codecs/audio_decoder_factory_template.h"
16 #include "api/audio_codecs/audio_encoder_factory_template.h"
17 #include "api/audio_codecs/opus/audio_decoder_opus.h"
18 #include "api/audio_codecs/opus/audio_decoder_multi_channel_opus.h"
19 #include "api/audio_codecs/opus/audio_encoder_opus.h"
20 #include "api/audio_codecs/L16/audio_decoder_L16.h"
21 #include "api/audio_codecs/L16/audio_encoder_L16.h"
22 #include "api/task_queue/default_task_queue_factory.h"
23 #include "media/engine/webrtc_media_engine.h"
24 #include "system_wrappers/include/field_trial.h"
25 #include "api/video/builtin_video_bitrate_allocator_factory.h"
26 #include "call/call.h"
27 #include "modules/rtp_rtcp/source/rtp_utility.h"
28 #include "api/call/audio_sink.h"
29 #include "modules/audio_processing/audio_buffer.h"
30 #include "absl/strings/match.h"
31 #include "modules/audio_processing/agc2/vad_with_level.h"
32 #include "pc/channel_manager.h"
33 #include "media/base/rtp_data_engine.h"
34 #include "audio/audio_state.h"
35 #include "modules/audio_coding/neteq/default_neteq_factory.h"
36 #include "modules/audio_coding/include/audio_coding_module.h"
37 #include "common_audio/include/audio_util.h"
38 #include "modules/audio_device/include/audio_device_data_observer.h"
39 #include "common_audio/resampler/include/resampler.h"
40 
41 #include "AudioFrame.h"
42 #include "ThreadLocalObject.h"
43 #include "Manager.h"
44 #include "NetworkManager.h"
45 #include "VideoCaptureInterfaceImpl.h"
46 #include "platform/PlatformInterface.h"
47 #include "LogSinkImpl.h"
48 #include "CodecSelectHelper.h"
49 #include "AudioStreamingPart.h"
50 #include "VideoStreamingPart.h"
51 #include "AudioDeviceHelper.h"
52 #include "FakeAudioDeviceModule.h"
53 #include "StreamingMediaContext.h"
54 
55 #include <mutex>
56 #include <random>
57 #include <sstream>
58 #include <iostream>
59 
60 
61 #ifndef USE_RNNOISE
62 #define USE_RNNOISE 1
63 #endif
64 
65 #if USE_RNNOISE
66 #include "rnnoise.h"
67 #endif
68 
69 #include "GroupJoinPayloadInternal.h"
70 
71 #include "third-party/json11.hpp"
72 
73 namespace tgcalls {
74 
75 namespace {
76 
77 template <typename Out>
splitString(const std::string & s,char delim,Out result)78 void splitString(const std::string &s, char delim, Out result) {
79     std::istringstream iss(s);
80     std::string item;
81     while (std::getline(iss, item, delim)) {
82         *result++ = item;
83     }
84 }
85 
splitString(const std::string & s,char delim)86 std::vector<std::string> splitString(const std::string &s, char delim) {
87     std::vector<std::string> elems;
88     splitString(s, delim, std::back_inserter(elems));
89     return elems;
90 }
91 
stringToInt(std::string const & string)92 static int stringToInt(std::string const &string) {
93     std::stringstream stringStream(string);
94     int value = 0;
95     stringStream >> value;
96     return value;
97 }
98 
intToString(int value)99 static std::string intToString(int value) {
100     std::ostringstream stringStream;
101     stringStream << value;
102     return stringStream.str();
103 }
104 
uint32ToString(uint32_t value)105 static std::string uint32ToString(uint32_t value) {
106     std::ostringstream stringStream;
107     stringStream << value;
108     return stringStream.str();
109 }
110 
stringToUInt32(std::string const & string)111 static uint32_t stringToUInt32(std::string const &string) {
112     std::stringstream stringStream(string);
113     uint32_t value = 0;
114     stringStream >> value;
115     return value;
116 }
117 
stringToUInt16(std::string const & string)118 static uint16_t stringToUInt16(std::string const &string) {
119     std::stringstream stringStream(string);
120     uint16_t value = 0;
121     stringStream >> value;
122     return value;
123 }
124 
formatTimestampMillis(int64_t timestamp)125 static std::string formatTimestampMillis(int64_t timestamp) {
126     std::ostringstream stringStream;
127     stringStream << std::fixed << std::setprecision(3) << (double)timestamp / 1000.0;
128     return stringStream.str();
129 }
130 
GetVideoCaptureAssumingSameThread(VideoCaptureInterface * videoCapture)131 static VideoCaptureInterfaceObject *GetVideoCaptureAssumingSameThread(VideoCaptureInterface *videoCapture) {
132     return videoCapture
133         ? static_cast<VideoCaptureInterfaceImpl*>(videoCapture)->object()->getSyncAssumingSameThread()
134         : nullptr;
135 }
136 
137 struct OutgoingVideoFormat {
138     cricket::VideoCodec videoCodec;
139     cricket::VideoCodec rtxCodec;
140 };
141 
addDefaultFeedbackParams(cricket::VideoCodec * codec)142 static void addDefaultFeedbackParams(cricket::VideoCodec *codec) {
143     // Don't add any feedback params for RED and ULPFEC.
144     if (codec->name == cricket::kRedCodecName || codec->name == cricket::kUlpfecCodecName) {
145         return;
146     }
147     codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamRemb, cricket::kParamValueEmpty));
148     codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc, cricket::kParamValueEmpty));
149     // Don't add any more feedback params for FLEXFEC.
150     if (codec->name == cricket::kFlexfecCodecName) {
151         return;
152     }
153     codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamCcm, cricket::kRtcpFbCcmParamFir));
154     codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kParamValueEmpty));
155     codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kRtcpFbNackParamPli));
156 }
157 
158 struct H264FormatParameters {
159     std::string profileLevelId;
160     std::string packetizationMode;
161     std::string levelAssymetryAllowed;
162 };
163 
parseH264FormatParameters(webrtc::SdpVideoFormat const & format)164 H264FormatParameters parseH264FormatParameters(webrtc::SdpVideoFormat const &format) {
165     H264FormatParameters result;
166 
167     for (const auto &parameter : format.parameters) {
168         if (parameter.first == "profile-level-id") {
169             result.profileLevelId = parameter.second;
170         } else if (parameter.first == "packetization-mode") {
171             result.packetizationMode = parameter.second;
172         } else if (parameter.first == "level-asymmetry-allowed") {
173             result.levelAssymetryAllowed = parameter.second;
174         }
175     }
176 
177     return result;
178 }
179 
getH264ProfileLevelIdPriority(std::string const & profileLevelId)180 static int getH264ProfileLevelIdPriority(std::string const &profileLevelId) {
181     if (profileLevelId == cricket::kH264ProfileLevelConstrainedHigh) {
182         return 0;
183     } else if (profileLevelId == cricket::kH264ProfileLevelConstrainedBaseline) {
184         return 1;
185     } else {
186         return 2;
187     }
188 }
189 
getH264PacketizationModePriority(std::string const & packetizationMode)190 static int getH264PacketizationModePriority(std::string const &packetizationMode) {
191     if (packetizationMode == "1") {
192         return 0;
193     } else {
194         return 1;
195     }
196 }
197 
getH264LevelAssymetryAllowedPriority(std::string const & levelAssymetryAllowed)198 static int getH264LevelAssymetryAllowedPriority(std::string const &levelAssymetryAllowed) {
199     if (levelAssymetryAllowed == "1") {
200         return 0;
201     } else {
202         return 1;
203     }
204 }
205 
filterSupportedVideoFormats(std::vector<webrtc::SdpVideoFormat> const & formats)206 static std::vector<webrtc::SdpVideoFormat> filterSupportedVideoFormats(std::vector<webrtc::SdpVideoFormat> const &formats) {
207     std::vector<webrtc::SdpVideoFormat> filteredFormats;
208 
209     std::vector<std::string> filterCodecNames = {
210         cricket::kVp8CodecName,
211         cricket::kVp9CodecName,
212         cricket::kH264CodecName
213     };
214 
215     std::vector<webrtc::SdpVideoFormat> vp9Formats;
216     std::vector<webrtc::SdpVideoFormat> h264Formats;
217 
218     for (const auto &format : formats) {
219         if (std::find(filterCodecNames.begin(), filterCodecNames.end(), format.name) == filterCodecNames.end()) {
220             continue;
221         }
222 
223         if (format.name == cricket::kVp9CodecName) {
224             vp9Formats.push_back(format);
225         } else if (format.name == cricket::kH264CodecName) {
226             h264Formats.push_back(format);
227         } else {
228             filteredFormats.push_back(format);
229         }
230     }
231 
232     if (!vp9Formats.empty()) {
233         bool added = false;
234         for (const auto &format : vp9Formats) {
235             if (added) {
236                 break;
237             }
238             for (const auto &parameter : format.parameters) {
239                 if (parameter.first == "profile-id") {
240                     if (parameter.second == "0") {
241                         filteredFormats.push_back(format);
242                         added = true;
243                         break;
244                     }
245                 }
246             }
247         }
248 
249         if (!added) {
250             filteredFormats.push_back(vp9Formats[0]);
251         }
252     }
253 
254     if (!h264Formats.empty()) {
255         std::sort(h264Formats.begin(), h264Formats.end(), [](const webrtc::SdpVideoFormat &lhs, const webrtc::SdpVideoFormat &rhs) {
256             auto lhsParameters = parseH264FormatParameters(lhs);
257             auto rhsParameters = parseH264FormatParameters(rhs);
258 
259             int lhsLevelIdPriority = getH264ProfileLevelIdPriority(lhsParameters.profileLevelId);
260             int lhsPacketizationModePriority = getH264PacketizationModePriority(lhsParameters.packetizationMode);
261             int lhsLevelAssymetryAllowedPriority = getH264LevelAssymetryAllowedPriority(lhsParameters.levelAssymetryAllowed);
262 
263             int rhsLevelIdPriority = getH264ProfileLevelIdPriority(rhsParameters.profileLevelId);
264             int rhsPacketizationModePriority = getH264PacketizationModePriority(rhsParameters.packetizationMode);
265             int rhsLevelAssymetryAllowedPriority = getH264LevelAssymetryAllowedPriority(rhsParameters.levelAssymetryAllowed);
266 
267             if (lhsLevelIdPriority != rhsLevelIdPriority) {
268                 return lhsLevelIdPriority < rhsLevelIdPriority;
269             }
270             if (lhsPacketizationModePriority != rhsPacketizationModePriority) {
271                 return lhsPacketizationModePriority < rhsPacketizationModePriority;
272             }
273             if (lhsLevelAssymetryAllowedPriority != rhsLevelAssymetryAllowedPriority) {
274                 return lhsLevelAssymetryAllowedPriority < rhsLevelAssymetryAllowedPriority;
275             }
276 
277             return true;
278         });
279 
280         filteredFormats.push_back(h264Formats[0]);
281     }
282 
283     return filteredFormats;
284 }
285 
assignPayloadTypes(std::vector<webrtc::SdpVideoFormat> const & formats)286 static std::vector<OutgoingVideoFormat> assignPayloadTypes(std::vector<webrtc::SdpVideoFormat> const &formats) {
287     if (formats.empty()) {
288         return {};
289     }
290 
291     constexpr int kFirstDynamicPayloadType = 100;
292     constexpr int kLastDynamicPayloadType = 127;
293 
294     int payload_type = kFirstDynamicPayloadType;
295 
296     std::vector<OutgoingVideoFormat> result;
297 
298     std::vector<std::string> filterCodecNames = {
299         cricket::kVp8CodecName,
300         cricket::kVp9CodecName,
301         cricket::kH264CodecName,
302     };
303 
304     for (const auto &codecName : filterCodecNames) {
305         for (const auto &format : formats) {
306             if (format.name != codecName) {
307                 continue;
308             }
309 
310             cricket::VideoCodec codec(format);
311             codec.id = payload_type;
312             addDefaultFeedbackParams(&codec);
313 
314             OutgoingVideoFormat resultFormat;
315 
316             resultFormat.videoCodec = codec;
317 
318             // Increment payload type.
319             ++payload_type;
320             if (payload_type > kLastDynamicPayloadType) {
321                 RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
322                 break;
323             }
324 
325             // Add associated RTX codec for non-FEC codecs.
326             if (!absl::EqualsIgnoreCase(codec.name, cricket::kUlpfecCodecName) &&
327                 !absl::EqualsIgnoreCase(codec.name, cricket::kFlexfecCodecName)) {
328                 resultFormat.rtxCodec = cricket::VideoCodec::CreateRtxCodec(payload_type, codec.id);
329 
330                 // Increment payload type.
331                 ++payload_type;
332                 if (payload_type > kLastDynamicPayloadType) {
333                     RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest.";
334                     break;
335                 }
336             }
337 
338             result.push_back(std::move(resultFormat));
339         }
340     }
341 
342     return result;
343 }
344 
345 struct VideoSsrcs {
346     struct SimulcastLayer {
347         uint32_t ssrc = 0;
348         uint32_t fidSsrc = 0;
349 
SimulcastLayertgcalls::__anon2cee46520111::VideoSsrcs::SimulcastLayer350         SimulcastLayer(uint32_t ssrc_, uint32_t fidSsrc_) :
351             ssrc(ssrc_), fidSsrc(fidSsrc_) {
352         }
353 
SimulcastLayertgcalls::__anon2cee46520111::VideoSsrcs::SimulcastLayer354         SimulcastLayer(const SimulcastLayer &other) :
355             ssrc(other.ssrc), fidSsrc(other.fidSsrc) {
356         }
357     };
358 
359     std::vector<SimulcastLayer> simulcastLayers;
360 
VideoSsrcstgcalls::__anon2cee46520111::VideoSsrcs361     VideoSsrcs() {
362     }
363 
VideoSsrcstgcalls::__anon2cee46520111::VideoSsrcs364     VideoSsrcs(const VideoSsrcs &other) :
365         simulcastLayers(other.simulcastLayers) {
366     }
367 };
368 
369 struct InternalGroupLevelValue {
370     GroupLevelValue value;
371     int64_t timestamp = 0;
372 };
373 
374 struct ChannelId {
375   uint32_t networkSsrc = 0;
376   uint32_t actualSsrc = 0;
377 
ChannelIdtgcalls::__anon2cee46520111::ChannelId378   ChannelId(uint32_t networkSsrc_, uint32_t actualSsrc_) :
379       networkSsrc(networkSsrc_),
380       actualSsrc(actualSsrc_) {
381   }
382 
ChannelIdtgcalls::__anon2cee46520111::ChannelId383   explicit ChannelId(uint32_t networkSsrc_) :
384       networkSsrc(networkSsrc_),
385       actualSsrc(networkSsrc_) {
386   }
387 
operator <tgcalls::__anon2cee46520111::ChannelId388   bool operator <(const ChannelId& rhs) const {
389     if (networkSsrc != rhs.networkSsrc) {
390       return networkSsrc < rhs.networkSsrc;
391     }
392     return actualSsrc < rhs.actualSsrc;
393   }
394 
nametgcalls::__anon2cee46520111::ChannelId395   std::string name() {
396     if (networkSsrc == actualSsrc) {
397       return uint32ToString(networkSsrc);
398     } else {
399       return uint32ToString(networkSsrc) + "to" + uint32ToString(actualSsrc);
400     }
401   }
402 };
403 
404 struct VideoChannelId {
405     std::string endpointId;
406 
VideoChannelIdtgcalls::__anon2cee46520111::VideoChannelId407     explicit VideoChannelId(std::string const &endpointId_) :
408     endpointId(endpointId_) {
409     }
410 
operator <tgcalls::__anon2cee46520111::VideoChannelId411     bool operator <(const VideoChannelId& rhs) const {
412       return endpointId < rhs.endpointId;
413     }
414 };
415 
416 struct ChannelSsrcInfo {
417     enum class Type {
418         Audio,
419         Video
420     };
421 
422     Type type = Type::Audio;
423     std::vector<uint32_t> allSsrcs;
424     std::string videoEndpointId;
425 };
426 
427 struct RequestedMediaChannelDescriptions {
428     std::shared_ptr<RequestMediaChannelDescriptionTask> task;
429     std::vector<uint32_t> ssrcs;
430 
RequestedMediaChannelDescriptionstgcalls::__anon2cee46520111::RequestedMediaChannelDescriptions431     RequestedMediaChannelDescriptions(std::shared_ptr<RequestMediaChannelDescriptionTask> task_, std::vector<uint32_t> ssrcs_) :
432     task(task_), ssrcs(std::move(ssrcs_)) {
433     }
434 };
435 
436 static const int kVadResultHistoryLength = 8;
437 
438 class VadHistory {
439 private:
440     float _vadResultHistory[kVadResultHistoryLength];
441 
442 public:
VadHistory()443     VadHistory() {
444         for (int i = 0; i < kVadResultHistoryLength; i++) {
445             _vadResultHistory[i] = 0.0f;
446         }
447     }
448 
~VadHistory()449     ~VadHistory() {
450     }
451 
update(float vadProbability)452     bool update(float vadProbability) {
453         for (int i = 1; i < kVadResultHistoryLength; i++) {
454             _vadResultHistory[i - 1] = _vadResultHistory[i];
455         }
456         _vadResultHistory[kVadResultHistoryLength - 1] = vadProbability;
457 
458         float movingAverage = 0.0f;
459         for (int i = 0; i < kVadResultHistoryLength; i++) {
460             movingAverage += _vadResultHistory[i];
461         }
462         movingAverage /= (float)kVadResultHistoryLength;
463 
464         bool vadResult = false;
465         if (movingAverage > 0.8f) {
466             vadResult = true;
467         }
468 
469         return vadResult;
470     }
471 };
472 
473 class CombinedVad {
474 private:
475     webrtc::VadLevelAnalyzer _vadWithLevel;
476     VadHistory _history;
477 
478 public:
CombinedVad()479     CombinedVad() {
480     }
481 
~CombinedVad()482     ~CombinedVad() {
483     }
484 
update(webrtc::AudioBuffer * buffer)485     bool update(webrtc::AudioBuffer *buffer) {
486         if (buffer->num_channels() <= 0) {
487             return _history.update(0.0f);
488         }
489         webrtc::AudioFrameView<float> frameView(buffer->channels(), buffer->num_channels(), buffer->num_frames());
490         float peak = 0.0f;
491         for (const auto &x : frameView.channel(0)) {
492             peak = std::max(std::fabs(x), peak);
493         }
494         if (peak <= 0.01f) {
495             return _history.update(false);
496         }
497 
498         auto result = _vadWithLevel.AnalyzeFrame(frameView);
499 
500         return _history.update(result.speech_probability);
501     }
502 
update()503     bool update() {
504         return _history.update(0.0f);
505     }
506 };
507 
508 class SparseVad {
509 public:
SparseVad()510     SparseVad() {
511     }
512 
update(webrtc::AudioBuffer * buffer)513     bool update(webrtc::AudioBuffer *buffer) {
514         _sampleCount += buffer->num_frames();
515         if (_sampleCount < 400) {
516             return _currentValue;
517         }
518         _sampleCount = 0;
519 
520         _currentValue = _vad.update(buffer);
521 
522         return _currentValue;
523     }
524 
525 private:
526     CombinedVad _vad;
527     bool _currentValue = false;
528     size_t _sampleCount = 0;
529 };
530 
531 class AudioSinkImpl: public webrtc::AudioSinkInterface {
532 public:
533     struct Update {
534         float level = 0.0f;
535         bool hasSpeech = false;
536 
Updatetgcalls::__anon2cee46520111::AudioSinkImpl::Update537         Update(float level_, bool hasSpech_) :
538             level(level_), hasSpeech(hasSpech_) {
539         }
540 
Updatetgcalls::__anon2cee46520111::AudioSinkImpl::Update541         Update(const Update &other) :
542             level(other.level), hasSpeech(other.hasSpeech) {
543         }
544     };
545 
546 public:
AudioSinkImpl(std::function<void (Update)> update,ChannelId channel_id,std::function<void (uint32_t,const AudioFrame &)> onAudioFrame)547     AudioSinkImpl(std::function<void(Update)> update,
548         ChannelId channel_id, std::function<void(uint32_t, const AudioFrame &)> onAudioFrame) :
549     _update(update), _channel_id(channel_id), _onAudioFrame(std::move(onAudioFrame)) {
550     }
551 
~AudioSinkImpl()552     virtual ~AudioSinkImpl() {
553     }
554 
OnData(const Data & audio)555     virtual void OnData(const Data& audio) override {
556       if (_onAudioFrame) {
557         AudioFrame frame;
558         frame.audio_samples = audio.data;
559         frame.num_samples = audio.samples_per_channel;
560         frame.bytes_per_sample = 2;
561         frame.num_channels = audio.channels;
562         frame.samples_per_sec = audio.sample_rate;
563         frame.elapsed_time_ms = 0;
564         frame.ntp_time_ms = 0;
565         _onAudioFrame(_channel_id.actualSsrc, frame);
566       }
567       if (_update && audio.channels == 1) {
568             const int16_t *samples = (const int16_t *)audio.data;
569             int numberOfSamplesInFrame = (int)audio.samples_per_channel;
570 
571             int16_t currentPeak = 0;
572             for (int i = 0; i < numberOfSamplesInFrame; i++) {
573                 int16_t sample = samples[i];
574                 if (sample < 0) {
575                     sample = -sample;
576                 }
577                 if (_peak < sample) {
578                     _peak = sample;
579                 }
580                 if (currentPeak < sample) {
581                     currentPeak = sample;
582                 }
583                 _peakCount += 1;
584             }
585 
586             /*bool vadResult = false;
587             if (currentPeak > 10) {
588                 webrtc::AudioBuffer buffer(audio.sample_rate, 1, 48000, 1, 48000, 1);
589                 webrtc::StreamConfig config(audio.sample_rate, 1);
590                 buffer.CopyFrom(samples, config);
591 
592                 vadResult = _vad.update(&buffer);
593             } else {
594                 vadResult = _vad.update();
595             }*/
596 
597             if (_peakCount >= 4400) {
598                 float level = ((float)(_peak)) / 8000.0f;
599                 _peak = 0;
600                 _peakCount = 0;
601                 _update(Update(level, level >= 1.0f));
602             }
603         }
604     }
605 
606 private:
607     std::function<void(Update)> _update;
608     ChannelId _channel_id;
609     std::function<void(uint32_t, const AudioFrame &)> _onAudioFrame;
610 
611   int _peakCount = 0;
612     uint16_t _peak = 0;
613 
614     CombinedVad _vad;
615 
616 };
617 
618 class VideoSinkImpl : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
619 public:
VideoSinkImpl(std::string const & endpointId)620     VideoSinkImpl(std::string const &endpointId) :
621     _endpointId(endpointId) {
622     }
623 
~VideoSinkImpl()624     virtual ~VideoSinkImpl() {
625     }
626 
OnFrame(const webrtc::VideoFrame & frame)627     virtual void OnFrame(const webrtc::VideoFrame& frame) override {
628         std::unique_lock<std::mutex> lock{ _mutex };
629         int64_t timestamp = rtc::TimeMillis();
630         if (_lastFrame) {
631             if (_lastFrame->video_frame_buffer()->width() != frame.video_frame_buffer()->width()) {
632                 int64_t deltaTime = std::abs(_lastFrameSizeChangeTimestamp - timestamp);
633                 if (deltaTime < 200) {
634                     RTC_LOG(LS_WARNING) << "VideoSinkImpl: frequent frame size change detected for " << _endpointId << ": " << _lastFrameSizeChangeHeight << " -> " << _lastFrame->video_frame_buffer()->height() << " -> " << frame.video_frame_buffer()->height() << " in " << deltaTime << " ms";
635                 }
636 
637                 _lastFrameSizeChangeHeight = _lastFrame->video_frame_buffer()->height();
638                 _lastFrameSizeChangeTimestamp = timestamp;
639             }
640         } else {
641             _lastFrameSizeChangeHeight = 0;
642             _lastFrameSizeChangeTimestamp = timestamp;
643         }
644         _lastFrame = frame;
645         for (int i = (int)(_sinks.size()) - 1; i >= 0; i--) {
646             auto strong = _sinks[i].lock();
647             if (!strong) {
648                 _sinks.erase(_sinks.begin() + i);
649             } else {
650                 strong->OnFrame(frame);
651             }
652         }
653     }
654 
OnDiscardedFrame()655     virtual void OnDiscardedFrame() override {
656         std::unique_lock<std::mutex> lock{ _mutex };
657         for (int i = (int)(_sinks.size()) - 1; i >= 0; i--) {
658             auto strong = _sinks[i].lock();
659             if (!strong) {
660                 _sinks.erase(_sinks.begin() + i);
661             } else {
662                 strong->OnDiscardedFrame();
663             }
664         }
665     }
666 
addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl)667     void addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl) {
668         std::unique_lock<std::mutex> lock{ _mutex };
669         _sinks.push_back(impl);
670         if (_lastFrame) {
671             auto strong = impl.lock();
672             if (strong) {
673                 strong->OnFrame(_lastFrame.value());
674             }
675         }
676     }
677 
getSinks()678     std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> getSinks() {
679         return _sinks;
680     }
681 
682 private:
683     std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> _sinks;
684     absl::optional<webrtc::VideoFrame> _lastFrame;
685     std::mutex _mutex;
686     int64_t _lastFrameSizeChangeTimestamp = 0;
687     int _lastFrameSizeChangeHeight = 0;
688     std::string _endpointId;
689 
690 };
691 
692 struct NoiseSuppressionConfiguration {
NoiseSuppressionConfigurationtgcalls::__anon2cee46520111::NoiseSuppressionConfiguration693     NoiseSuppressionConfiguration(bool isEnabled_) :
694     isEnabled(isEnabled_) {
695 
696     }
697 
698     bool isEnabled = false;
699 };
700 
701 #if USE_RNNOISE
702 class AudioCapturePostProcessor : public webrtc::CustomProcessing {
703 public:
AudioCapturePostProcessor(std::function<void (GroupLevelValue const &)> updated,std::shared_ptr<NoiseSuppressionConfiguration> noiseSuppressionConfiguration,std::vector<float> * externalAudioSamples,webrtc::Mutex * externalAudioSamplesMutex)704     AudioCapturePostProcessor(std::function<void(GroupLevelValue const &)> updated, std::shared_ptr<NoiseSuppressionConfiguration> noiseSuppressionConfiguration, std::vector<float> *externalAudioSamples, webrtc::Mutex *externalAudioSamplesMutex) :
705     _updated(updated),
706     _noiseSuppressionConfiguration(noiseSuppressionConfiguration),
707     _externalAudioSamples(externalAudioSamples),
708     _externalAudioSamplesMutex(externalAudioSamplesMutex) {
709         int frameSize = rnnoise_get_frame_size();
710         _frameSamples.resize(frameSize);
711 
712         _denoiseState = rnnoise_create(nullptr);
713     }
714 
~AudioCapturePostProcessor()715     virtual ~AudioCapturePostProcessor() {
716         if (_denoiseState) {
717             rnnoise_destroy(_denoiseState);
718         }
719     }
720 
721 private:
Initialize(int sample_rate_hz,int num_channels)722     virtual void Initialize(int sample_rate_hz, int num_channels) override {
723     }
724 
Process(webrtc::AudioBuffer * buffer)725     virtual void Process(webrtc::AudioBuffer *buffer) override {
726         if (!buffer) {
727             return;
728         }
729         if (buffer->num_channels() != 1) {
730             return;
731         }
732         if (!_denoiseState) {
733             return;
734         }
735         if (buffer->num_frames() != _frameSamples.size()) {
736             return;
737         }
738 
739         float sourcePeak = 0.0f;
740         float *sourceSamples = buffer->channels()[0];
741         for (int i = 0; i < _frameSamples.size(); i++) {
742             sourcePeak = std::max(std::fabs(sourceSamples[i]), sourcePeak);
743         }
744 
745         if (_noiseSuppressionConfiguration->isEnabled) {
746             float vadProbability = 0.0f;
747             if (sourcePeak >= 0.01f) {
748                 vadProbability = rnnoise_process_frame(_denoiseState, _frameSamples.data(), buffer->channels()[0]);
749                 if (_noiseSuppressionConfiguration->isEnabled) {
750                     memcpy(buffer->channels()[0], _frameSamples.data(), _frameSamples.size() * sizeof(float));
751                 }
752             }
753 
754             float peak = 0;
755             int peakCount = 0;
756             const float *samples = buffer->channels_const()[0];
757             for (int i = 0; i < buffer->num_frames(); i++) {
758                 float sample = samples[i];
759                 if (sample < 0) {
760                     sample = -sample;
761                 }
762                 if (peak < sample) {
763                     peak = sample;
764                 }
765                 peakCount += 1;
766             }
767 
768             bool vadStatus = _history.update(vadProbability);
769 
770             _peakCount += peakCount;
771             if (_peak < peak) {
772                 _peak = peak;
773             }
774             if (_peakCount >= 4400) {
775                 float level = _peak / 4000.0f;
776                 _peak = 0;
777                 _peakCount = 0;
778 
779                 _updated(GroupLevelValue{
780                     level,
781                     vadStatus,
782                 });
783             }
784         } else {
785             float peak = 0;
786             int peakCount = 0;
787             const float *samples = buffer->channels_const()[0];
788             for (int i = 0; i < buffer->num_frames(); i++) {
789                 float sample = samples[i];
790                 if (sample < 0) {
791                     sample = -sample;
792                 }
793                 if (peak < sample) {
794                     peak = sample;
795                 }
796                 peakCount += 1;
797             }
798 
799             _peakCount += peakCount;
800             if (_peak < peak) {
801                 _peak = peak;
802             }
803             if (_peakCount >= 1200) {
804                 float level = _peak / 8000.0f;
805                 _peak = 0;
806                 _peakCount = 0;
807 
808                 _updated(GroupLevelValue{
809                     level,
810                     level >= 1.0f,
811                 });
812             }
813         }
814 
815         if (_externalAudioSamplesMutex && _externalAudioSamples) {
816             _externalAudioSamplesMutex->Lock();
817             if (!_externalAudioSamples->empty()) {
818                 float *bufferData = buffer->channels()[0];
819                 int takenSamples = 0;
820                 for (int i = 0; i < _externalAudioSamples->size() && i < _frameSamples.size(); i++) {
821                     float sample = (*_externalAudioSamples)[i];
822                     sample += bufferData[i];
823                     sample = std::min(sample, 32768.f);
824                     sample = std::max(sample, -32768.f);
825                     bufferData[i] = sample;
826                     takenSamples++;
827                 }
828                 if (takenSamples != 0) {
829                     _externalAudioSamples->erase(_externalAudioSamples->begin(), _externalAudioSamples->begin() + takenSamples);
830                 }
831             }
832             _externalAudioSamplesMutex->Unlock();
833         }
834     }
835 
ToString() const836     virtual std::string ToString() const override {
837         return "CustomPostProcessing";
838     }
839 
SetRuntimeSetting(webrtc::AudioProcessing::RuntimeSetting setting)840     virtual void SetRuntimeSetting(webrtc::AudioProcessing::RuntimeSetting setting) override {
841     }
842 
843 private:
844     std::function<void(GroupLevelValue const &)> _updated;
845     std::shared_ptr<NoiseSuppressionConfiguration> _noiseSuppressionConfiguration;
846 
847     DenoiseState *_denoiseState = nullptr;
848     std::vector<float> _frameSamples;
849     int32_t _peakCount = 0;
850     float _peak = 0;
851     VadHistory _history;
852     SparseVad _vad;
853 
854     std::vector<float> *_externalAudioSamples = nullptr;
855     webrtc::Mutex *_externalAudioSamplesMutex = nullptr;
856 };
857 #endif
858 
859 class ExternalAudioRecorder : public FakeAudioDeviceModule::Recorder {
860 public:
ExternalAudioRecorder(std::vector<float> * externalAudioSamples,webrtc::Mutex * externalAudioSamplesMutex)861     ExternalAudioRecorder(std::vector<float> *externalAudioSamples, webrtc::Mutex *externalAudioSamplesMutex) :
862     _externalAudioSamples(externalAudioSamples),
863     _externalAudioSamplesMutex(externalAudioSamplesMutex) {
864         _samples.resize(480);
865     }
866 
~ExternalAudioRecorder()867     virtual ~ExternalAudioRecorder() {
868     }
869 
Record()870     virtual AudioFrame Record() override {
871         AudioFrame result;
872 
873         _externalAudioSamplesMutex->Lock();
874         if (!_externalAudioSamples->empty() && _externalAudioSamples->size() >= 480) {
875             size_t takenSamples = std::min(_samples.size(), _externalAudioSamples->size());
876             webrtc::FloatS16ToS16(_externalAudioSamples->data(), takenSamples, _samples.data());
877 
878             result.num_samples = takenSamples;
879 
880             if (takenSamples != 0) {
881                 _externalAudioSamples->erase(_externalAudioSamples->begin(), _externalAudioSamples->begin() + takenSamples);
882             }
883         } else {
884             result.num_samples = 0;
885         }
886         _externalAudioSamplesMutex->Unlock();
887 
888         result.audio_samples = _samples.data();
889         result.bytes_per_sample = 2;
890         result.num_channels = 1;
891         result.samples_per_sec = 48000;
892         result.elapsed_time_ms = 0;
893         result.ntp_time_ms = 0;
894 
895         return result;
896     }
897 
WaitForUs()898     virtual int32_t WaitForUs() override {
899         _externalAudioSamplesMutex->Lock();
900         _externalAudioSamplesMutex->Unlock();
901 
902         return 1000;
903     }
904 
905 private:
906     std::vector<float> *_externalAudioSamples = nullptr;
907     webrtc::Mutex *_externalAudioSamplesMutex = nullptr;
908     std::vector<int16_t> _samples;
909 };
910 
911 class IncomingAudioChannel : public sigslot::has_slots<> {
912 public:
IncomingAudioChannel(cricket::ChannelManager * channelManager,webrtc::Call * call,webrtc::RtpTransport * rtpTransport,rtc::UniqueRandomIdGenerator * randomIdGenerator,bool isRawPcm,ChannelId ssrc,std::function<void (AudioSinkImpl::Update)> && onAudioLevelUpdated,std::function<void (uint32_t,const AudioFrame &)> onAudioFrame,std::shared_ptr<Threads> threads)913     IncomingAudioChannel(
914         cricket::ChannelManager *channelManager,
915         webrtc::Call *call,
916         webrtc::RtpTransport *rtpTransport,
917         rtc::UniqueRandomIdGenerator *randomIdGenerator,
918         bool isRawPcm,
919         ChannelId ssrc,
920         std::function<void(AudioSinkImpl::Update)> &&onAudioLevelUpdated,
921         std::function<void(uint32_t, const AudioFrame &)> onAudioFrame,
922         std::shared_ptr<Threads> threads) :
923     _threads(threads),
924     _ssrc(ssrc),
925     _channelManager(channelManager),
926     _call(call) {
927         _creationTimestamp = rtc::TimeMillis();
928 
929         threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, rtpTransport, ssrc, onAudioFrame = std::move(onAudioFrame), onAudioLevelUpdated = std::move(onAudioLevelUpdated), randomIdGenerator, isRawPcm]() mutable {
930             cricket::AudioOptions audioOptions;
931             audioOptions.audio_jitter_buffer_fast_accelerate = true;
932             audioOptions.audio_jitter_buffer_min_delay_ms = 50;
933 
934             std::string streamId = std::string("stream") + ssrc.name();
935 
936             _audioChannel = _channelManager->CreateVoiceChannel(_call, cricket::MediaConfig(), rtpTransport, _threads->getWorkerThread(), std::string("audio") + uint32ToString(ssrc.networkSsrc), false, GroupNetworkManager::getDefaulCryptoOptions(), randomIdGenerator, audioOptions);
937 
938             const uint8_t opusPTimeMs = 120;
939 
940             cricket::AudioCodec opusCodec(111, "opus", 48000, 0, 2);
941             opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1);
942             opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs);
943 
944             cricket::AudioCodec pcmCodec(112, "l16", 48000, 0, 1);
945 
946             auto outgoingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
947             if (!isRawPcm) {
948                 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
949                 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
950                 outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
951             }
952             outgoingAudioDescription->set_rtcp_mux(true);
953             outgoingAudioDescription->set_rtcp_reduced_size(true);
954             outgoingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
955             outgoingAudioDescription->set_codecs({ opusCodec, pcmCodec });
956             outgoingAudioDescription->set_bandwidth(1300000);
957 
958             auto incomingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
959             if (!isRawPcm) {
960                 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
961                 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
962                 incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
963             }
964             incomingAudioDescription->set_rtcp_mux(true);
965             incomingAudioDescription->set_rtcp_reduced_size(true);
966             incomingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
967             incomingAudioDescription->set_codecs({ opusCodec, pcmCodec });
968             incomingAudioDescription->set_bandwidth(1300000);
969             cricket::StreamParams streamParams = cricket::StreamParams::CreateLegacy(ssrc.networkSsrc);
970             streamParams.set_stream_ids({ streamId });
971             incomingAudioDescription->AddStream(streamParams);
972 
973             _audioChannel->SetLocalContent(outgoingAudioDescription.get(), webrtc::SdpType::kOffer, nullptr);
974             _audioChannel->SetRemoteContent(incomingAudioDescription.get(), webrtc::SdpType::kAnswer, nullptr);
975             _audioChannel->SetPayloadTypeDemuxingEnabled(false);
976 
977             outgoingAudioDescription.reset();
978             incomingAudioDescription.reset();
979 
980             if (_ssrc.actualSsrc != 1) {
981                 std::unique_ptr<AudioSinkImpl> audioLevelSink(new AudioSinkImpl(std::move(onAudioLevelUpdated), _ssrc, std::move(onAudioFrame)));
982                 _audioChannel->media_channel()->SetRawAudioSink(ssrc.networkSsrc, std::move(audioLevelSink));
983             }
984         });
985 
986         //_audioChannel->SignalSentPacket().connect(this, &IncomingAudioChannel::OnSentPacket_w);
987 
988         _audioChannel->Enable(true);
989     }
990 
~IncomingAudioChannel()991     ~IncomingAudioChannel() {
992         //_audioChannel->SignalSentPacket().disconnect(this);
993         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
994             _channelManager->DestroyVoiceChannel(_audioChannel);
995             _audioChannel = nullptr;
996         });
997     }
998 
setVolume(double value)999     void setVolume(double value) {
1000         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, value]() {
1001             _audioChannel->media_channel()->SetOutputVolume(_ssrc.networkSsrc, value);
1002         });
1003     }
1004 
updateActivity()1005     void updateActivity() {
1006         _activityTimestamp = rtc::TimeMillis();
1007     }
1008 
getActivity()1009     int64_t getActivity() {
1010         return _activityTimestamp;
1011     }
1012 
1013 private:
OnSentPacket_w(const rtc::SentPacket & sent_packet)1014     void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
1015         _call->OnSentPacket(sent_packet);
1016     }
1017 
1018 private:
1019     std::shared_ptr<Threads> _threads;
1020     ChannelId _ssrc;
1021     // Memory is managed by _channelManager
1022     cricket::VoiceChannel *_audioChannel = nullptr;
1023     // Memory is managed externally
1024     cricket::ChannelManager *_channelManager = nullptr;
1025     webrtc::Call *_call = nullptr;
1026     int64_t _creationTimestamp = 0;
1027     int64_t _activityTimestamp = 0;
1028 };
1029 
1030 class IncomingVideoChannel : public sigslot::has_slots<> {
1031 public:
IncomingVideoChannel(cricket::ChannelManager * channelManager,webrtc::Call * call,webrtc::RtpTransport * rtpTransport,rtc::UniqueRandomIdGenerator * randomIdGenerator,std::vector<webrtc::SdpVideoFormat> const & availableVideoFormats,GroupJoinVideoInformation sharedVideoInformation,uint32_t audioSsrc,VideoChannelDescription::Quality minQuality,VideoChannelDescription::Quality maxQuality,GroupParticipantVideoInformation const & description,std::shared_ptr<Threads> threads)1032     IncomingVideoChannel(
1033         cricket::ChannelManager *channelManager,
1034         webrtc::Call *call,
1035         webrtc::RtpTransport *rtpTransport,
1036         rtc::UniqueRandomIdGenerator *randomIdGenerator,
1037         std::vector<webrtc::SdpVideoFormat> const &availableVideoFormats,
1038         GroupJoinVideoInformation sharedVideoInformation,
1039         uint32_t audioSsrc,
1040         VideoChannelDescription::Quality minQuality,
1041         VideoChannelDescription::Quality maxQuality,
1042         GroupParticipantVideoInformation const &description,
1043         std::shared_ptr<Threads> threads) :
1044     _threads(threads),
1045     _endpointId(description.endpointId),
1046     _channelManager(channelManager),
1047     _call(call),
1048     _requestedMinQuality(minQuality),
1049     _requestedMaxQuality(maxQuality) {
1050         _videoSink.reset(new VideoSinkImpl(_endpointId));
1051 
1052         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, rtpTransport, &availableVideoFormats, &description, randomIdGenerator]() mutable {
1053             uint32_t mid = randomIdGenerator->GenerateId();
1054             std::string streamId = std::string("video") + uint32ToString(mid);
1055 
1056             _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory();
1057 
1058             auto payloadTypes = assignPayloadTypes(availableVideoFormats);
1059             std::vector<cricket::VideoCodec> codecs;
1060             for (const auto &payloadType : payloadTypes) {
1061                 codecs.push_back(payloadType.videoCodec);
1062                 codecs.push_back(payloadType.rtxCodec);
1063             }
1064 
1065             auto outgoingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1066             outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1067             outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1068             outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kVideoRotationUri, 13));
1069             outgoingVideoDescription->set_rtcp_mux(true);
1070             outgoingVideoDescription->set_rtcp_reduced_size(true);
1071             outgoingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1072             outgoingVideoDescription->set_codecs(codecs);
1073             outgoingVideoDescription->set_bandwidth(1300000);
1074 
1075             cricket::StreamParams videoRecvStreamParams;
1076 
1077             std::vector<uint32_t> allSsrcs;
1078             for (const auto &group : description.ssrcGroups) {
1079                 for (auto ssrc : group.ssrcs) {
1080                     if (std::find(allSsrcs.begin(), allSsrcs.end(), ssrc) == allSsrcs.end()) {
1081                         allSsrcs.push_back(ssrc);
1082                     }
1083                 }
1084 
1085                 if (group.semantics == "SIM") {
1086                     if (_mainVideoSsrc == 0) {
1087                         _mainVideoSsrc = group.ssrcs[0];
1088                     }
1089                 }
1090 
1091                 cricket::SsrcGroup parsedGroup(group.semantics, group.ssrcs);
1092                 videoRecvStreamParams.ssrc_groups.push_back(parsedGroup);
1093             }
1094             videoRecvStreamParams.ssrcs = allSsrcs;
1095 
1096             if (_mainVideoSsrc == 0) {
1097                 if (description.ssrcGroups.size() == 1) {
1098                     _mainVideoSsrc = description.ssrcGroups[0].ssrcs[0];
1099                 }
1100             }
1101 
1102             videoRecvStreamParams.cname = "cname";
1103             videoRecvStreamParams.set_stream_ids({ streamId });
1104 
1105             auto incomingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1106             incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1107             incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1108             incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kVideoRotationUri, 13));
1109             incomingVideoDescription->set_rtcp_mux(true);
1110             incomingVideoDescription->set_rtcp_reduced_size(true);
1111             incomingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1112             incomingVideoDescription->set_codecs(codecs);
1113             incomingVideoDescription->set_bandwidth(1300000);
1114 
1115             incomingVideoDescription->AddStream(videoRecvStreamParams);
1116 
1117             _videoChannel = _channelManager->CreateVideoChannel(_call, cricket::MediaConfig(), rtpTransport, _threads->getWorkerThread(), std::string("video") + uint32ToString(mid), false, GroupNetworkManager::getDefaulCryptoOptions(), randomIdGenerator, cricket::VideoOptions(), _videoBitrateAllocatorFactory.get());
1118 
1119             _videoChannel->SetLocalContent(outgoingVideoDescription.get(), webrtc::SdpType::kOffer, nullptr);
1120             _videoChannel->SetRemoteContent(incomingVideoDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1121             _videoChannel->SetPayloadTypeDemuxingEnabled(false);
1122             _videoChannel->media_channel()->SetSink(_mainVideoSsrc, _videoSink.get());
1123             _videoChannel->Enable(true);
1124         });
1125 
1126         //_videoChannel->SignalSentPacket().connect(this, &IncomingVideoChannel::OnSentPacket_w);
1127     }
1128 
~IncomingVideoChannel()1129     ~IncomingVideoChannel() {
1130         //_videoChannel->SignalSentPacket().disconnect(this);
1131         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1132             _videoChannel->Enable(false);
1133             _channelManager->DestroyVideoChannel(_videoChannel);
1134             _videoChannel = nullptr;
1135         });
1136     }
1137 
addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl)1138     void addSink(std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> impl) {
1139         _videoSink->addSink(impl);
1140     }
1141 
getSinks()1142     std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>> getSinks() {
1143         return _videoSink->getSinks();
1144     }
1145 
endpointId()1146     std::string const &endpointId() {
1147         return _endpointId;
1148     }
1149 
requestedMinQuality()1150     VideoChannelDescription::Quality requestedMinQuality() {
1151         return _requestedMinQuality;
1152     }
1153 
requestedMaxQuality()1154     VideoChannelDescription::Quality requestedMaxQuality() {
1155         return _requestedMaxQuality;
1156     }
1157 
setRequstedMinQuality(VideoChannelDescription::Quality quality)1158     void setRequstedMinQuality(VideoChannelDescription::Quality quality) {
1159         _requestedMinQuality = quality;
1160     }
1161 
setRequstedMaxQuality(VideoChannelDescription::Quality quality)1162     void setRequstedMaxQuality(VideoChannelDescription::Quality quality) {
1163         _requestedMaxQuality = quality;
1164     }
1165 
setStats(absl::optional<GroupInstanceStats::IncomingVideoStats> stats)1166     void setStats(absl::optional<GroupInstanceStats::IncomingVideoStats> stats) {
1167         _stats = stats;
1168     }
1169 
getStats()1170     absl::optional<GroupInstanceStats::IncomingVideoStats> getStats() {
1171         return _stats;
1172     }
1173 
1174 private:
OnSentPacket_w(const rtc::SentPacket & sent_packet)1175     void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
1176         //_call->OnSentPacket(sent_packet);
1177     }
1178 
1179 private:
1180     std::shared_ptr<Threads> _threads;
1181     uint32_t _mainVideoSsrc = 0;
1182     std::string _endpointId;
1183     std::unique_ptr<VideoSinkImpl> _videoSink;
1184     std::vector<GroupJoinPayloadVideoSourceGroup> _ssrcGroups;
1185     std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> _videoBitrateAllocatorFactory;
1186     // Memory is managed by _channelManager
1187     cricket::VideoChannel *_videoChannel;
1188     // Memory is managed externally
1189     cricket::ChannelManager *_channelManager = nullptr;
1190     webrtc::Call *_call = nullptr;
1191 
1192     VideoChannelDescription::Quality _requestedMinQuality = VideoChannelDescription::Quality::Thumbnail;
1193     VideoChannelDescription::Quality _requestedMaxQuality = VideoChannelDescription::Quality::Thumbnail;
1194 
1195     absl::optional<GroupInstanceStats::IncomingVideoStats> _stats;
1196 };
1197 
1198 class MissingSsrcPacketBuffer {
1199 public:
MissingSsrcPacketBuffer(int limit)1200     MissingSsrcPacketBuffer(int limit) :
1201     _limit(limit) {
1202     }
1203 
~MissingSsrcPacketBuffer()1204     ~MissingSsrcPacketBuffer() {
1205     }
1206 
add(uint32_t ssrc,rtc::CopyOnWriteBuffer const & packet)1207     void add(uint32_t ssrc, rtc::CopyOnWriteBuffer const &packet) {
1208         if (_packets.size() == _limit) {
1209             _packets.erase(_packets.begin());
1210         }
1211         _packets.push_back(std::make_pair(ssrc, packet));
1212     }
1213 
get(uint32_t ssrc)1214     std::vector<rtc::CopyOnWriteBuffer> get(uint32_t ssrc) {
1215         std::vector<rtc::CopyOnWriteBuffer> result;
1216         for (auto it = _packets.begin(); it != _packets.end(); ) {
1217             if (it->first == ssrc) {
1218                 result.push_back(it->second);
1219                 _packets.erase(it);
1220             } else {
1221                 it++;
1222             }
1223         }
1224         return result;
1225     }
1226 
1227 private:
1228     int _limit = 0;
1229     std::vector<std::pair<uint32_t, rtc::CopyOnWriteBuffer>> _packets;
1230 
1231 };
1232 
1233 class RequestedBroadcastPart {
1234 public:
1235     int64_t timestamp = 0;
1236     std::shared_ptr<BroadcastPartTask> task;
1237 
RequestedBroadcastPart(int64_t timestamp_,std::shared_ptr<BroadcastPartTask> task_)1238     explicit RequestedBroadcastPart(int64_t timestamp_, std::shared_ptr<BroadcastPartTask> task_) :
1239         timestamp(timestamp_), task(task_) {
1240     }
1241 };
1242 
1243 struct DecodedBroadcastPart {
1244     struct DecodedBroadcastPartChannel {
1245         uint32_t ssrc = 0;
1246         std::vector<int16_t> pcmData;
1247     };
1248 
DecodedBroadcastParttgcalls::__anon2cee46520111::DecodedBroadcastPart1249     DecodedBroadcastPart(int numSamples_, std::vector<DecodedBroadcastPartChannel> &&_channels) :
1250         numSamples(numSamples_), channels(std::move(_channels)) {
1251     }
1252 
1253     int numSamples = 0;
1254     std::vector<DecodedBroadcastPartChannel> channels;
1255 };
1256 
videoCaptureToGetVideoSource(std::shared_ptr<VideoCaptureInterface> videoCapture)1257 std::function<webrtc::VideoTrackSourceInterface*()> videoCaptureToGetVideoSource(std::shared_ptr<VideoCaptureInterface> videoCapture) {
1258   return [videoCapture]() {
1259     VideoCaptureInterfaceObject *videoCaptureImpl = GetVideoCaptureAssumingSameThread(videoCapture.get());
1260     return videoCaptureImpl ? videoCaptureImpl->source() : nullptr;
1261   };
1262 }
1263 
1264 class AudioDeviceDataObserverShared {
1265 public:
AudioDeviceDataObserverShared()1266     AudioDeviceDataObserverShared() {
1267     }
1268 
~AudioDeviceDataObserverShared()1269     ~AudioDeviceDataObserverShared() {
1270     }
1271 
setStreamingContext(std::shared_ptr<StreamingMediaContext> streamingContext)1272     void setStreamingContext(std::shared_ptr<StreamingMediaContext> streamingContext) {
1273         _mutex.Lock();
1274         _streamingContext = streamingContext;
1275         _mutex.Unlock();
1276     }
1277 
mixAudio(int16_t * audio_samples,const size_t num_samples,const size_t num_channels,const uint32_t samples_per_sec)1278     void mixAudio(int16_t *audio_samples, const size_t num_samples, const size_t num_channels, const uint32_t samples_per_sec) {
1279 
1280         _mutex.Lock();
1281         const auto context = _streamingContext;
1282         _mutex.Unlock();
1283 
1284         if (context) {
1285             if (_samplesToResample.size() < 480 * num_channels) {
1286                 _samplesToResample.resize(480 * num_channels);
1287             }
1288             memset(_samplesToResample.data(), 0, _samplesToResample.size() * sizeof(int16_t));
1289 
1290             context->getAudio(_samplesToResample.data(), 480, num_channels, 48000);
1291 
1292             if (_resamplerFrequency != samples_per_sec || _resamplerNumChannels != num_channels) {
1293                 _resamplerFrequency = samples_per_sec;
1294                 _resamplerNumChannels = num_channels;
1295                 _resampler = std::make_unique<webrtc::Resampler>(48000, samples_per_sec, num_channels);
1296             }
1297 
1298             size_t outLen = 0;
1299             _resampler->Push(_samplesToResample.data(), _samplesToResample.size(), (int16_t *)audio_samples, num_samples * num_channels, outLen);
1300         }
1301     }
1302 
1303 private:
1304     webrtc::Mutex _mutex;
1305     std::unique_ptr<webrtc::Resampler> _resampler;
1306     uint32_t _resamplerFrequency = 0;
1307     size_t _resamplerNumChannels = 0;
1308     std::vector<int16_t> _samplesToResample;
1309     std::shared_ptr<StreamingMediaContext> _streamingContext;
1310 };
1311 
1312 class AudioDeviceDataObserverImpl : public webrtc::AudioDeviceDataObserver {
1313 public:
AudioDeviceDataObserverImpl(std::shared_ptr<AudioDeviceDataObserverShared> shared)1314     AudioDeviceDataObserverImpl(std::shared_ptr<AudioDeviceDataObserverShared> shared) :
1315     _shared(shared) {
1316     }
1317 
~AudioDeviceDataObserverImpl()1318     virtual ~AudioDeviceDataObserverImpl() {
1319     }
1320 
OnCaptureData(const void * audio_samples,const size_t num_samples,const size_t bytes_per_sample,const size_t num_channels,const uint32_t samples_per_sec)1321     virtual void OnCaptureData(const void* audio_samples,
1322                                const size_t num_samples,
1323                                const size_t bytes_per_sample,
1324                                const size_t num_channels,
1325                                const uint32_t samples_per_sec) override {
1326     }
1327 
OnRenderData(const void * audio_samples,const size_t num_samples,const size_t bytes_per_sample,const size_t num_channels,const uint32_t samples_per_sec)1328     virtual void OnRenderData(const void* audio_samples,
1329                               const size_t num_samples,
1330                               const size_t bytes_per_sample,
1331                               const size_t num_channels,
1332                               const uint32_t samples_per_sec) override {
1333         if (bytes_per_sample != num_channels * 2) {
1334             return;
1335         }
1336         if (samples_per_sec % 100 != 0) {
1337             return;
1338         }
1339         if (num_samples != samples_per_sec / 100) {
1340             return;
1341         }
1342 
1343         if (_shared) {
1344             _shared->mixAudio((int16_t *)audio_samples, num_samples, num_channels, samples_per_sec);
1345         }
1346     }
1347 
1348 private:
1349     std::shared_ptr<AudioDeviceDataObserverShared> _shared;
1350 };
1351 
1352 } // namespace
1353 
1354 class GroupInstanceCustomInternal : public sigslot::has_slots<>, public std::enable_shared_from_this<GroupInstanceCustomInternal> {
1355 public:
GroupInstanceCustomInternal(GroupInstanceDescriptor && descriptor,std::shared_ptr<Threads> threads)1356     GroupInstanceCustomInternal(GroupInstanceDescriptor &&descriptor, std::shared_ptr<Threads> threads) :
1357     _threads(std::move(threads)),
1358     _networkStateUpdated(descriptor.networkStateUpdated),
1359     _audioLevelsUpdated(descriptor.audioLevelsUpdated),
1360     _onAudioFrame(descriptor.onAudioFrame),
1361     _requestMediaChannelDescriptions(descriptor.requestMediaChannelDescriptions),
1362     _requestCurrentTime(descriptor.requestCurrentTime),
1363     _requestAudioBroadcastPart(descriptor.requestAudioBroadcastPart),
1364     _requestVideoBroadcastPart(descriptor.requestVideoBroadcastPart),
1365     _videoCapture(descriptor.videoCapture),
1366     _videoCaptureSink(new VideoSinkImpl("VideoCapture")),
1367     _getVideoSource(descriptor.getVideoSource),
1368     _disableIncomingChannels(descriptor.disableIncomingChannels),
1369     _useDummyChannel(descriptor.useDummyChannel),
1370     _outgoingAudioBitrateKbit(descriptor.outgoingAudioBitrateKbit),
1371     _disableOutgoingAudioProcessing(descriptor.disableOutgoingAudioProcessing),
1372     _minOutgoingVideoBitrateKbit(descriptor.minOutgoingVideoBitrateKbit),
1373     _videoContentType(descriptor.videoContentType),
1374     _videoCodecPreferences(std::move(descriptor.videoCodecPreferences)),
1375     _eventLog(std::make_unique<webrtc::RtcEventLogNull>()),
1376     _taskQueueFactory(webrtc::CreateDefaultTaskQueueFactory()),
1377     _createAudioDeviceModule(descriptor.createAudioDeviceModule),
1378     _initialInputDeviceId(std::move(descriptor.initialInputDeviceId)),
1379     _initialOutputDeviceId(std::move(descriptor.initialOutputDeviceId)),
1380     _missingPacketBuffer(50) {
1381         assert(_threads->getMediaThread()->IsCurrent());
1382 
1383         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this] {
1384             _workerThreadSafery = webrtc::PendingTaskSafetyFlag::Create();
1385         });
1386         _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this] {
1387             _networkThreadSafery = webrtc::PendingTaskSafetyFlag::Create();
1388         });
1389 
1390         if (_videoCapture) {
1391           assert(!_getVideoSource);
1392           _getVideoSource = videoCaptureToGetVideoSource(std::move(descriptor.videoCapture));
1393         }
1394         generateSsrcs();
1395 
1396         _noiseSuppressionConfiguration = std::make_shared<NoiseSuppressionConfiguration>(descriptor.initialEnableNoiseSuppression);
1397 
1398         _externalAudioRecorder.reset(new ExternalAudioRecorder(&_externalAudioSamples, &_externalAudioSamplesMutex));
1399     }
1400 
~GroupInstanceCustomInternal()1401     ~GroupInstanceCustomInternal() {
1402         _incomingAudioChannels.clear();
1403         _incomingVideoChannels.clear();
1404         _serverBandwidthProbingVideoSsrc.reset();
1405 
1406         destroyOutgoingAudioChannel();
1407         destroyOutgoingVideoChannel();
1408 
1409         _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1410             _rtpTransport->SignalSentPacket.disconnect(this);
1411             _rtpTransport->SignalRtcpPacketReceived.disconnect(this);
1412         });
1413 
1414         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1415             _channelManager = nullptr;
1416             if (_audioDeviceModule) {
1417                 _audioDeviceModule->Stop();
1418                 _audioDeviceModule = nullptr;
1419             }
1420             _call.reset();
1421         });
1422     }
1423 
start()1424     void start() {
1425         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1426 
1427         webrtc::field_trial::InitFieldTrialsFromString(
1428             "WebRTC-Audio-Allocation/min:32kbps,max:32kbps/"
1429             "WebRTC-Audio-OpusMinPacketLossRate/Enabled-1/"
1430             "WebRTC-TaskQueuePacer/Enabled/"
1431             "WebRTC-VP8ConferenceTemporalLayers/1/"
1432             "WebRTC-Audio-MinimizeResamplingOnMobile/Enabled/"
1433             "WebRTC-BweLossExperiment/Enabled/"
1434         );
1435 
1436         _networkManager.reset(new ThreadLocalObject<GroupNetworkManager>(_threads->getNetworkThread(), [weak, threads = _threads] () mutable {
1437             return new GroupNetworkManager(
1438                 [=](const GroupNetworkManager::State &state) {
1439                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [=] {
1440                         const auto strong = weak.lock();
1441                         if (!strong) {
1442                             return;
1443                         }
1444                         strong->setIsRtcConnected(state.isReadyToSendData);
1445                     });
1446                 },
1447                 [=](rtc::CopyOnWriteBuffer const &message, bool isUnresolved) {
1448                     if (!isUnresolved) {
1449                         return;
1450                     }
1451                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, message, isUnresolved]() mutable {
1452                         if (const auto strong = weak.lock()) {
1453                             strong->receivePacket(message, isUnresolved);
1454                         }
1455                     });
1456                 },
1457                 [=](bool isDataChannelOpen) {
1458                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, isDataChannelOpen]() mutable {
1459                         if (const auto strong = weak.lock()) {
1460                             strong->updateIsDataChannelOpen(isDataChannelOpen);
1461                         }
1462                     });
1463                 },
1464                 [=](std::string const &message) {
1465                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, message]() {
1466                         if (const auto strong = weak.lock()) {
1467                             strong->receiveDataChannelMessage(message);
1468                         }
1469                     });
1470                 },
1471                 [=](uint32_t ssrc, uint8_t audioLevel, bool isSpeech) {
1472                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, ssrc, audioLevel, isSpeech]() {
1473                         if (const auto strong = weak.lock()) {
1474                             strong->updateSsrcAudioLevel(ssrc, audioLevel, isSpeech);
1475                         }
1476                     });
1477                 }, threads);
1478         }));
1479 
1480     #if USE_RNNOISE
1481         std::unique_ptr<AudioCapturePostProcessor> audioProcessor = nullptr;
1482     #endif
1483         if (_videoContentType != VideoContentType::Screencast) {
1484             PlatformInterface::SharedInstance()->configurePlatformAudio();
1485 
1486     #if USE_RNNOISE
1487             audioProcessor = std::make_unique<AudioCapturePostProcessor>([weak, threads = _threads](GroupLevelValue const &level) {
1488                 threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, level](){
1489                     auto strong = weak.lock();
1490                     if (!strong) {
1491                         return;
1492                     }
1493                     strong->_myAudioLevel = level;
1494                 });
1495             }, _noiseSuppressionConfiguration, nullptr, nullptr);
1496     #endif
1497         }
1498 
1499         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this
1500     #if USE_RNNOISE
1501 			, audioProcessor = std::move(audioProcessor)
1502     #endif
1503           ]() mutable {
1504             cricket::MediaEngineDependencies mediaDeps;
1505             mediaDeps.task_queue_factory = _taskQueueFactory.get();
1506             mediaDeps.audio_encoder_factory = webrtc::CreateAudioEncoderFactory<webrtc::AudioEncoderOpus, webrtc::AudioEncoderL16>();
1507             mediaDeps.audio_decoder_factory = webrtc::CreateAudioDecoderFactory<webrtc::AudioDecoderOpus, webrtc::AudioDecoderL16>();
1508 
1509             mediaDeps.video_encoder_factory = PlatformInterface::SharedInstance()->makeVideoEncoderFactory();
1510             mediaDeps.video_decoder_factory = PlatformInterface::SharedInstance()->makeVideoDecoderFactory();
1511 
1512     #if USE_RNNOISE
1513             if (_audioLevelsUpdated && audioProcessor) {
1514                 webrtc::AudioProcessingBuilder builder;
1515                 builder.SetCapturePostProcessing(std::move(audioProcessor));
1516 
1517                 mediaDeps.audio_processing = builder.Create();
1518             }
1519     #endif
1520 
1521             _audioDeviceDataObserverShared = std::make_shared<AudioDeviceDataObserverShared>();
1522 
1523             _audioDeviceModule = createAudioDeviceModule();
1524             if (!_audioDeviceModule) {
1525                 return;
1526             }
1527             mediaDeps.adm = _audioDeviceModule;
1528 
1529             _availableVideoFormats = filterSupportedVideoFormats(mediaDeps.video_encoder_factory->GetSupportedFormats());
1530 
1531             std::unique_ptr<cricket::MediaEngineInterface> mediaEngine = cricket::CreateMediaEngine(std::move(mediaDeps));
1532 
1533             _channelManager = cricket::ChannelManager::Create(
1534                 std::move(mediaEngine),
1535                 std::make_unique<cricket::RtpDataEngine>(),
1536                 true,
1537                 _threads->getWorkerThread(),
1538                 _threads->getNetworkThread()
1539             );
1540         });
1541 
1542         setAudioInputDevice(_initialInputDeviceId);
1543         setAudioOutputDevice(_initialOutputDeviceId);
1544 
1545         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1546             webrtc::Call::Config callConfig(_eventLog.get(), _threads->getNetworkThread());
1547             callConfig.task_queue_factory = _taskQueueFactory.get();
1548             callConfig.trials = &_fieldTrials;
1549             callConfig.audio_state = _channelManager->media_engine()->voice().GetAudioState();
1550             _call.reset(webrtc::Call::Create(callConfig, _threads->getSharedModuleThread()));
1551         });
1552 
1553         _uniqueRandomIdGenerator.reset(new rtc::UniqueRandomIdGenerator());
1554 
1555         _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1556             _rtpTransport = _networkManager->getSyncAssumingSameThread()->getRtpTransport();
1557             _rtpTransport->SignalSentPacket.connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1558             _rtpTransport->SignalRtcpPacketReceived.connect(this, &GroupInstanceCustomInternal::OnRtcpPacketReceived_n);
1559         });
1560 
1561         _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory();
1562 
1563         if (_audioLevelsUpdated) {
1564             beginLevelsTimer(100);
1565         }
1566 
1567         if (_getVideoSource) {
1568             setVideoSource(_getVideoSource, true);
1569         }
1570 
1571         if (_useDummyChannel && _videoContentType != VideoContentType::Screencast) {
1572             addIncomingAudioChannel(ChannelId(1), true);
1573         }
1574 
1575         if (_videoContentType == VideoContentType::Screencast) {
1576             setIsMuted(false);
1577         }
1578 
1579         /*if (_videoContentType != VideoContentType::Screencast) {
1580             createOutgoingAudioChannel();
1581         }*/
1582 
1583         beginNetworkStatusTimer(0);
1584         //beginAudioChannelCleanupTimer(0);
1585 
1586         adjustBitratePreferences(true);
1587 
1588         beginRemoteConstraintsUpdateTimer(5000);
1589     }
1590 
destroyOutgoingVideoChannel()1591     void destroyOutgoingVideoChannel() {
1592         if (!_outgoingVideoChannel) {
1593             return;
1594         }
1595         //_outgoingVideoChannel->SignalSentPacket().disconnect(this);
1596         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1597             _outgoingVideoChannel->Enable(false);
1598             _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, nullptr);
1599             _channelManager->DestroyVideoChannel(_outgoingVideoChannel);
1600         });
1601 		_outgoingVideoChannel = nullptr;
1602     }
1603 
createOutgoingVideoChannel()1604     void createOutgoingVideoChannel() {
1605         if (_outgoingVideoChannel
1606             || _videoContentType == VideoContentType::None) {
1607             return;
1608         }
1609         configureVideoParams();
1610 
1611         if (!_selectedPayloadType) {
1612             RTC_LOG(LS_ERROR) << "Could not select payload type.";
1613             return;
1614         }
1615 
1616         cricket::VideoOptions videoOptions;
1617         if (_videoContentType == VideoContentType::Screencast) {
1618             videoOptions.is_screencast = true;
1619         }
1620         _outgoingVideoChannel = _channelManager->CreateVideoChannel(_call.get(), cricket::MediaConfig(), _rtpTransport, _threads->getWorkerThread(), "1", false, GroupNetworkManager::getDefaulCryptoOptions(), _uniqueRandomIdGenerator.get(), videoOptions, _videoBitrateAllocatorFactory.get());
1621 
1622         if (!_outgoingVideoChannel) {
1623             RTC_LOG(LS_ERROR) << "Could not create outgoing video channel.";
1624             return;
1625         }
1626 
1627         cricket::StreamParams videoSendStreamParams;
1628 
1629         std::vector<uint32_t> simulcastGroupSsrcs;
1630         std::vector<cricket::SsrcGroup> fidGroups;
1631         for (const auto &layer : _outgoingVideoSsrcs.simulcastLayers) {
1632             simulcastGroupSsrcs.push_back(layer.ssrc);
1633 
1634             videoSendStreamParams.ssrcs.push_back(layer.ssrc);
1635             videoSendStreamParams.ssrcs.push_back(layer.fidSsrc);
1636 
1637             cricket::SsrcGroup fidGroup(cricket::kFidSsrcGroupSemantics, { layer.ssrc, layer.fidSsrc });
1638             fidGroups.push_back(fidGroup);
1639         }
1640         if (simulcastGroupSsrcs.size() > 1) {
1641             cricket::SsrcGroup simulcastGroup(cricket::kSimSsrcGroupSemantics, simulcastGroupSsrcs);
1642             videoSendStreamParams.ssrc_groups.push_back(simulcastGroup);
1643 
1644             GroupJoinPayloadVideoSourceGroup payloadSimulcastGroup;
1645             payloadSimulcastGroup.semantics = "SIM";
1646             payloadSimulcastGroup.ssrcs = simulcastGroupSsrcs;
1647         }
1648 
1649         for (auto fidGroup : fidGroups) {
1650             videoSendStreamParams.ssrc_groups.push_back(fidGroup);
1651 
1652             GroupJoinPayloadVideoSourceGroup payloadFidGroup;
1653             payloadFidGroup.semantics = "FID";
1654             payloadFidGroup.ssrcs = fidGroup.ssrcs;
1655         }
1656 
1657         videoSendStreamParams.cname = "cname";
1658 
1659         auto outgoingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1660         for (const auto &extension : _videoExtensionMap) {
1661             outgoingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(extension.second, extension.first));
1662         }
1663         outgoingVideoDescription->set_rtcp_mux(true);
1664         outgoingVideoDescription->set_rtcp_reduced_size(true);
1665         outgoingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1666         outgoingVideoDescription->set_codecs({ _selectedPayloadType->videoCodec, _selectedPayloadType->rtxCodec });
1667         outgoingVideoDescription->set_bandwidth(1300000);
1668         outgoingVideoDescription->AddStream(videoSendStreamParams);
1669 
1670         auto incomingVideoDescription = std::make_unique<cricket::VideoContentDescription>();
1671         for (const auto &extension : _videoExtensionMap) {
1672             incomingVideoDescription->AddRtpHeaderExtension(webrtc::RtpExtension(extension.second, extension.first));
1673         }
1674         incomingVideoDescription->set_rtcp_mux(true);
1675         incomingVideoDescription->set_rtcp_reduced_size(true);
1676         incomingVideoDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1677         incomingVideoDescription->set_codecs({ _selectedPayloadType->videoCodec, _selectedPayloadType->rtxCodec });
1678         incomingVideoDescription->set_bandwidth(1300000);
1679 
1680         _outgoingVideoChannel->SetRemoteContent(incomingVideoDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1681         _outgoingVideoChannel->SetLocalContent(outgoingVideoDescription.get(), webrtc::SdpType::kOffer, nullptr);
1682         _outgoingVideoChannel->SetPayloadTypeDemuxingEnabled(false);
1683 
1684         //_outgoingVideoChannel->SignalSentPacket().connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1685 
1686         adjustVideoSendParams();
1687         updateVideoSend();
1688     }
1689 
adjustVideoSendParams()1690     void adjustVideoSendParams() {
1691         if (!_outgoingVideoChannel) {
1692             return;
1693         }
1694 
1695         if (_videoContentType == VideoContentType::Screencast) {
1696             _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1697                 webrtc::RtpParameters rtpParameters = _outgoingVideoChannel->media_channel()->GetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc);
1698                 if (rtpParameters.encodings.size() == 3) {
1699                     for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1700                         if (i == 0) {
1701                             rtpParameters.encodings[i].min_bitrate_bps = 50000;
1702                             rtpParameters.encodings[i].max_bitrate_bps = 100000;
1703                             rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1704                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 180;
1705                         } else if (i == 1) {
1706                             rtpParameters.encodings[i].min_bitrate_bps = 150000;
1707                             rtpParameters.encodings[i].max_bitrate_bps = 200000;
1708                             rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1709                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 360;
1710                         } else if (i == 2) {
1711                             rtpParameters.encodings[i].min_bitrate_bps = 300000;
1712                             rtpParameters.encodings[i].max_bitrate_bps = 800000 + 100000;
1713                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 720;
1714                         }
1715                     }
1716                 } else if (rtpParameters.encodings.size() == 2) {
1717                     for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1718                         if (i == 0) {
1719                             rtpParameters.encodings[i].min_bitrate_bps = 50000;
1720                             rtpParameters.encodings[i].max_bitrate_bps = 100000;
1721                             rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1722                         } else if (i == 1) {
1723                             rtpParameters.encodings[i].min_bitrate_bps = 200000;
1724                             rtpParameters.encodings[i].max_bitrate_bps = 900000 + 100000;
1725                         }
1726                     }
1727                 } else {
1728                     rtpParameters.encodings[0].max_bitrate_bps = (800000 + 100000) * 2;
1729                 }
1730 
1731                 _outgoingVideoChannel->media_channel()->SetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, rtpParameters);
1732             });
1733         } else {
1734             _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1735                 webrtc::RtpParameters rtpParameters = _outgoingVideoChannel->media_channel()->GetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc);
1736                 if (rtpParameters.encodings.size() == 3) {
1737                     for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1738                         if (i == 0) {
1739                             rtpParameters.encodings[i].min_bitrate_bps = 50000;
1740                             rtpParameters.encodings[i].max_bitrate_bps = 60000;
1741                             rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1742                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 180;
1743                         } else if (i == 1) {
1744                             rtpParameters.encodings[i].min_bitrate_bps = 100000;
1745                             rtpParameters.encodings[i].max_bitrate_bps = 110000;
1746                             rtpParameters.encodings[i].scale_resolution_down_by = 2.0;
1747                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 360;
1748                         } else if (i == 2) {
1749                             rtpParameters.encodings[i].min_bitrate_bps = 300000;
1750                             rtpParameters.encodings[i].max_bitrate_bps = 800000 + 100000;
1751                             rtpParameters.encodings[i].active = _outgoingVideoConstraint >= 720;
1752                         }
1753                     }
1754                 } else if (rtpParameters.encodings.size() == 2) {
1755                     for (int i = 0; i < (int)rtpParameters.encodings.size(); i++) {
1756                         if (i == 0) {
1757                             rtpParameters.encodings[i].min_bitrate_bps = 50000;
1758                             rtpParameters.encodings[i].max_bitrate_bps = 100000;
1759                             rtpParameters.encodings[i].scale_resolution_down_by = 4.0;
1760                         } else if (i == 1) {
1761                             rtpParameters.encodings[i].min_bitrate_bps = 200000;
1762                             rtpParameters.encodings[i].max_bitrate_bps = 900000 + 100000;
1763                         }
1764                     }
1765                 } else {
1766                     rtpParameters.encodings[0].max_bitrate_bps = (800000 + 100000) * 2;
1767                 }
1768 
1769                 _outgoingVideoChannel->media_channel()->SetRtpSendParameters(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, rtpParameters);
1770             });
1771         }
1772     }
1773 
updateVideoSend()1774     void updateVideoSend() {
1775         if (!_outgoingVideoChannel) {
1776             return;
1777         }
1778 
1779         webrtc::VideoTrackSourceInterface *videoSource = _getVideoSource ? _getVideoSource() : nullptr;
1780         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, videoSource]() {
1781             if (_getVideoSource) {
1782                 _outgoingVideoChannel->Enable(true);
1783                 _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, videoSource);
1784             } else {
1785                 _outgoingVideoChannel->Enable(false);
1786                 _outgoingVideoChannel->media_channel()->SetVideoSend(_outgoingVideoSsrcs.simulcastLayers[0].ssrc, nullptr, nullptr);
1787             }
1788         });
1789     }
1790 
destroyOutgoingAudioChannel()1791     void destroyOutgoingAudioChannel() {
1792         if (!_outgoingAudioChannel) {
1793             return;
1794         }
1795 
1796         //_outgoingAudioChannel->SignalSentPacket().disconnect(this);
1797         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
1798             _outgoingAudioChannel->media_channel()->SetAudioSend(_outgoingAudioSsrc, false, nullptr, &_audioSource);
1799             _outgoingAudioChannel->Enable(false);
1800             _channelManager->DestroyVoiceChannel(_outgoingAudioChannel);
1801         });
1802         _outgoingAudioChannel = nullptr;
1803     }
1804 
createOutgoingAudioChannel()1805     void createOutgoingAudioChannel() {
1806         if (_outgoingAudioChannel) {
1807             return;
1808         }
1809 
1810         cricket::AudioOptions audioOptions;
1811         if (_disableOutgoingAudioProcessing || _videoContentType == VideoContentType::Screencast) {
1812             audioOptions.echo_cancellation = false;
1813             audioOptions.noise_suppression = false;
1814             audioOptions.auto_gain_control = false;
1815             audioOptions.highpass_filter = false;
1816             audioOptions.typing_detection = false;
1817             audioOptions.experimental_agc = false;
1818             audioOptions.experimental_ns = false;
1819             audioOptions.residual_echo_detector = false;
1820         } else {
1821             audioOptions.echo_cancellation = true;
1822             audioOptions.noise_suppression = true;
1823             audioOptions.experimental_ns = true;
1824             audioOptions.residual_echo_detector = true;
1825         }
1826 
1827         std::vector<std::string> streamIds;
1828         streamIds.push_back("1");
1829 
1830         _outgoingAudioChannel = _channelManager->CreateVoiceChannel(_call.get(), cricket::MediaConfig(), _rtpTransport, _threads->getWorkerThread(), "0", false, GroupNetworkManager::getDefaulCryptoOptions(), _uniqueRandomIdGenerator.get(), audioOptions);
1831 
1832         const uint8_t opusMinBitrateKbps = _outgoingAudioBitrateKbit;
1833         const uint8_t opusMaxBitrateKbps = _outgoingAudioBitrateKbit;
1834         const uint8_t opusStartBitrateKbps = _outgoingAudioBitrateKbit;
1835         const uint8_t opusPTimeMs = 120;
1836 
1837         cricket::AudioCodec opusCodec(111, "opus", 48000, 0, 2);
1838         opusCodec.AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc));
1839         opusCodec.SetParam(cricket::kCodecParamMinBitrate, opusMinBitrateKbps);
1840         opusCodec.SetParam(cricket::kCodecParamStartBitrate, opusStartBitrateKbps);
1841         opusCodec.SetParam(cricket::kCodecParamMaxBitrate, opusMaxBitrateKbps);
1842         opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1);
1843         opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs);
1844 
1845         auto outgoingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
1846         outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
1847         outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1848         outgoingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1849         outgoingAudioDescription->set_rtcp_mux(true);
1850         outgoingAudioDescription->set_rtcp_reduced_size(true);
1851         outgoingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kSendOnly);
1852         outgoingAudioDescription->set_codecs({ opusCodec });
1853         outgoingAudioDescription->set_bandwidth(1300000);
1854         outgoingAudioDescription->AddStream(cricket::StreamParams::CreateLegacy(_outgoingAudioSsrc));
1855 
1856         auto incomingAudioDescription = std::make_unique<cricket::AudioContentDescription>();
1857         incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAudioLevelUri, 1));
1858         incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kAbsSendTimeUri, 2));
1859         incomingAudioDescription->AddRtpHeaderExtension(webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, 3));
1860         incomingAudioDescription->set_rtcp_mux(true);
1861         incomingAudioDescription->set_rtcp_reduced_size(true);
1862         incomingAudioDescription->set_direction(webrtc::RtpTransceiverDirection::kRecvOnly);
1863         incomingAudioDescription->set_codecs({ opusCodec });
1864         incomingAudioDescription->set_bandwidth(1300000);
1865 
1866         _outgoingAudioChannel->SetLocalContent(outgoingAudioDescription.get(), webrtc::SdpType::kOffer, nullptr);
1867         _outgoingAudioChannel->SetRemoteContent(incomingAudioDescription.get(), webrtc::SdpType::kAnswer, nullptr);
1868         _outgoingAudioChannel->SetPayloadTypeDemuxingEnabled(false);
1869 
1870         //_outgoingAudioChannel->SignalSentPacket().connect(this, &GroupInstanceCustomInternal::OnSentPacket_w);
1871 
1872         _outgoingAudioChannel->Enable(true);
1873 
1874         onUpdatedIsMuted();
1875 
1876         adjustBitratePreferences(false);
1877     }
1878 
stop()1879     void stop() {
1880     }
1881 
updateSsrcAudioLevel(uint32_t ssrc,uint8_t audioLevel,bool isSpeech)1882     void updateSsrcAudioLevel(uint32_t ssrc, uint8_t audioLevel, bool isSpeech) {
1883         float mappedLevelDb = ((float)audioLevel) / (float)(0x7f);
1884 
1885         //mappedLevelDb = fabs(1.0f - mappedLevelDb);
1886         //float mappedLevel = pow(10.0f, mappedLevelDb * 0.1f);
1887 
1888         //printf("mappedLevelDb: %f, mappedLevel: %f\n", mappedLevelDb, mappedLevel);
1889 
1890         float mappedLevel = (fabs(1.0f - mappedLevelDb)) * 1.0f;
1891 
1892         auto it = _audioLevels.find(ChannelId(ssrc));
1893         if (it != _audioLevels.end()) {
1894             it->second.value.level = fmax(it->second.value.level, mappedLevel);
1895             if (isSpeech) {
1896                 it->second.value.voice = true;
1897             }
1898             it->second.timestamp = rtc::TimeMillis();
1899         } else {
1900             InternalGroupLevelValue updated;
1901             updated.value.level = mappedLevel;
1902             updated.value.voice = isSpeech;
1903             updated.timestamp = rtc::TimeMillis();
1904             _audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
1905         }
1906 
1907         auto audioChannel = _incomingAudioChannels.find(ChannelId(ssrc));
1908         if (audioChannel != _incomingAudioChannels.end()) {
1909             audioChannel->second->updateActivity();
1910         }
1911     }
1912 
beginLevelsTimer(int timeoutMs)1913     void beginLevelsTimer(int timeoutMs) {
1914         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1915         _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
1916             auto strong = weak.lock();
1917             if (!strong) {
1918                 return;
1919             }
1920 
1921             //int64_t timestamp = rtc::TimeMillis();
1922             //int64_t maxSampleTimeout = 400;
1923 
1924             GroupLevelsUpdate levelsUpdate;
1925             levelsUpdate.updates.reserve(strong->_audioLevels.size() + 1);
1926             for (auto &it : strong->_audioLevels) {
1927                 /*if (it.second.value.level < 0.001f) {
1928                     continue;
1929                 }
1930                 if (it.second.timestamp <= timestamp - maxSampleTimeout) {
1931                     continue;
1932                 }*/
1933 
1934                 uint32_t effectiveSsrc = it.first.actualSsrc;
1935                 if (std::find_if(levelsUpdate.updates.begin(), levelsUpdate.updates.end(), [&](GroupLevelUpdate const &item) {
1936                     return item.ssrc == effectiveSsrc;
1937                 }) != levelsUpdate.updates.end()) {
1938                     continue;
1939                 }
1940                 levelsUpdate.updates.push_back(GroupLevelUpdate{
1941                     effectiveSsrc,
1942                     it.second.value,
1943                     });
1944                 if (it.second.value.level > 0.001f) {
1945                     auto audioChannel = strong->_incomingAudioChannels.find(it.first);
1946                     if (audioChannel != strong->_incomingAudioChannels.end()) {
1947                         audioChannel->second->updateActivity();
1948                     }
1949                 }
1950 
1951                 //it.second.value.level *= 0.5f;
1952                 //it.second.value.voice = false;
1953             }
1954 
1955             strong->_audioLevels.clear();
1956 
1957             auto myAudioLevel = strong->_myAudioLevel;
1958             myAudioLevel.isMuted = strong->_isMuted;
1959             levelsUpdate.updates.push_back(GroupLevelUpdate{ 0, myAudioLevel });
1960 
1961             if (strong->_audioLevelsUpdated) {
1962                 strong->_audioLevelsUpdated(levelsUpdate);
1963             }
1964 
1965             bool isSpeech = myAudioLevel.voice && !myAudioLevel.isMuted;
1966             strong->_networkManager->perform(RTC_FROM_HERE, [isSpeech = isSpeech](GroupNetworkManager *networkManager) {
1967                 networkManager->setOutgoingVoiceActivity(isSpeech);
1968             });
1969 
1970             strong->beginLevelsTimer(100);
1971         }, timeoutMs);
1972     }
1973 
beginAudioChannelCleanupTimer(int delayMs)1974     void beginAudioChannelCleanupTimer(int delayMs) {
1975         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
1976         _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
1977             auto strong = weak.lock();
1978             if (!strong) {
1979                 return;
1980             }
1981 
1982             auto timestamp = rtc::TimeMillis();
1983 
1984             std::vector<ChannelId> removeChannels;
1985             for (const auto &it : strong->_incomingAudioChannels) {
1986                 if (it.first.networkSsrc == 1) {
1987                     continue;
1988                 }
1989                 auto activity = it.second->getActivity();
1990                 if (activity < timestamp - 1000) {
1991                     removeChannels.push_back(it.first);
1992                 }
1993             }
1994 
1995             for (const auto &channelId : removeChannels) {
1996                 strong->removeIncomingAudioChannel(channelId);
1997             }
1998 
1999             strong->beginAudioChannelCleanupTimer(500);
2000         }, delayMs);
2001     }
2002 
beginRemoteConstraintsUpdateTimer(int delayMs)2003     void beginRemoteConstraintsUpdateTimer(int delayMs) {
2004         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2005         _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
2006             auto strong = weak.lock();
2007             if (!strong) {
2008                 return;
2009             }
2010 
2011             strong->maybeUpdateRemoteVideoConstraints();
2012 
2013             strong->beginRemoteConstraintsUpdateTimer(5000);
2014         }, delayMs);
2015     }
2016 
beginNetworkStatusTimer(int delayMs)2017     void beginNetworkStatusTimer(int delayMs) {
2018         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2019         _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak]() {
2020             auto strong = weak.lock();
2021             if (!strong) {
2022                 return;
2023             }
2024 
2025             if (strong->_connectionMode == GroupConnectionMode::GroupConnectionModeBroadcast || strong->_broadcastEnabledUntilRtcIsConnectedAtTimestamp) {
2026                 strong->updateBroadcastNetworkStatus();
2027             }
2028 
2029             strong->beginNetworkStatusTimer(500);
2030         }, delayMs);
2031     }
2032 
updateBroadcastNetworkStatus()2033     void updateBroadcastNetworkStatus() {
2034         bool isBroadcastConnected = true;
2035 
2036         if (isBroadcastConnected != _isBroadcastConnected) {
2037             _isBroadcastConnected = isBroadcastConnected;
2038             updateIsConnected();
2039         }
2040     }
2041 
configureVideoParams()2042     void configureVideoParams() {
2043         if (!_sharedVideoInformation) {
2044             return;
2045         }
2046         if (_selectedPayloadType) {
2047             // Already configured.
2048             return;
2049         }
2050 
2051         _availablePayloadTypes = assignPayloadTypes(_availableVideoFormats);
2052         if (_availablePayloadTypes.empty()) {
2053             return;
2054         }
2055 
2056         for (const auto &payloadType : _availablePayloadTypes) {
2057             GroupJoinPayloadVideoPayloadType payload;
2058             payload.id = payloadType.videoCodec.id;
2059             payload.name = payloadType.videoCodec.name;
2060             payload.clockrate = payloadType.videoCodec.clockrate;
2061             payload.channels = 0;
2062 
2063             std::vector<GroupJoinPayloadVideoPayloadType::FeedbackType> feedbackTypes;
2064 
2065             GroupJoinPayloadVideoPayloadType::FeedbackType fbGoogRemb;
2066             fbGoogRemb.type = "goog-remb";
2067             feedbackTypes.push_back(fbGoogRemb);
2068 
2069             GroupJoinPayloadVideoPayloadType::FeedbackType fbTransportCc;
2070             fbTransportCc.type = "transport-cc";
2071             feedbackTypes.push_back(fbTransportCc);
2072 
2073             GroupJoinPayloadVideoPayloadType::FeedbackType fbCcmFir;
2074             fbCcmFir.type = "ccm";
2075             fbCcmFir.subtype = "fir";
2076             feedbackTypes.push_back(fbCcmFir);
2077 
2078             GroupJoinPayloadVideoPayloadType::FeedbackType fbNack;
2079             fbNack.type = "nack";
2080             feedbackTypes.push_back(fbNack);
2081 
2082             GroupJoinPayloadVideoPayloadType::FeedbackType fbNackPli;
2083             fbNackPli.type = "nack";
2084             fbNackPli.subtype = "pli";
2085             feedbackTypes.push_back(fbNackPli);
2086 
2087             payload.feedbackTypes = feedbackTypes;
2088             payload.parameters = {};
2089 
2090             _videoPayloadTypes.push_back(std::move(payload));
2091 
2092             GroupJoinPayloadVideoPayloadType rtxPayload;
2093             rtxPayload.id = payloadType.rtxCodec.id;
2094             rtxPayload.name = payloadType.rtxCodec.name;
2095             rtxPayload.clockrate = payloadType.rtxCodec.clockrate;
2096             rtxPayload.parameters.push_back(std::make_pair("apt", intToString(payloadType.videoCodec.id)));
2097             _videoPayloadTypes.push_back(std::move(rtxPayload));
2098         }
2099 
2100         std::vector<std::string> codecPriorities;
2101         for (const auto name : _videoCodecPreferences) {
2102             std::string codecName;
2103             switch (name) {
2104             case VideoCodecName::VP8: {
2105                 codecName = cricket::kVp8CodecName;
2106                 break;
2107             }
2108             case VideoCodecName::VP9: {
2109                 codecName = cricket::kVp9CodecName;
2110                 break;
2111             }
2112             case VideoCodecName::H264: {
2113                 codecName = cricket::kH264CodecName;
2114                 break;
2115             }
2116             default: {
2117                 break;
2118             }
2119             }
2120             if (codecName.size() != 0) {
2121                 codecPriorities.push_back(std::move(codecName));
2122             }
2123         }
2124         std::vector<std::string> defaultCodecPriorities = {
2125             cricket::kVp8CodecName,
2126             cricket::kVp9CodecName
2127         };
2128 
2129         bool enableH264 = false;
2130         for (const auto &payloadType : _sharedVideoInformation->payloadTypes) {
2131             if (payloadType.name == cricket::kH264CodecName) {
2132                 enableH264 = true;
2133                 break;
2134             }
2135         }
2136         if (enableH264) {
2137             defaultCodecPriorities.insert(defaultCodecPriorities.begin(), cricket::kH264CodecName);
2138         }
2139 
2140         for (const auto &name : defaultCodecPriorities) {
2141             if (std::find(codecPriorities.begin(), codecPriorities.end(), name) == codecPriorities.end()) {
2142                 codecPriorities.push_back(name);
2143             }
2144         }
2145 
2146         for (const auto &codecName : codecPriorities) {
2147             if (_selectedPayloadType) {
2148                 break;
2149             }
2150             for (const auto &payloadType : _availablePayloadTypes) {
2151                 if (payloadType.videoCodec.name == codecName) {
2152                     _selectedPayloadType = payloadType;
2153                     break;
2154                 }
2155             }
2156         }
2157         if (!_selectedPayloadType) {
2158             return;
2159         }
2160 
2161         _videoExtensionMap.emplace_back(2, webrtc::RtpExtension::kAbsSendTimeUri);
2162         _videoExtensionMap.emplace_back(3, webrtc::RtpExtension::kTransportSequenceNumberUri);
2163         _videoExtensionMap.emplace_back(13, webrtc::RtpExtension::kVideoRotationUri);
2164     }
2165 
OnSentPacket_w(const rtc::SentPacket & sent_packet)2166     void OnSentPacket_w(const rtc::SentPacket& sent_packet) {
2167         _call->OnSentPacket(sent_packet);
2168     }
2169 
OnRtcpPacketReceived_n(rtc::CopyOnWriteBuffer * buffer,int64_t packet_time_us)2170     void OnRtcpPacketReceived_n(rtc::CopyOnWriteBuffer *buffer, int64_t packet_time_us) {
2171         rtc::CopyOnWriteBuffer packet = *buffer;
2172         _threads->getWorkerThread()->PostTask(ToQueuedTask(_workerThreadSafery, [this, packet, packet_time_us] {
2173             if (_call) {
2174                 _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, packet_time_us);
2175             }
2176         }));
2177     }
2178 
adjustBitratePreferences(bool resetStartBitrate)2179     void adjustBitratePreferences(bool resetStartBitrate) {
2180         webrtc::BitrateConstraints preferences;
2181         webrtc::BitrateSettings settings;
2182         if (_getVideoSource) {
2183             settings.min_bitrate_bps = _minOutgoingVideoBitrateKbit * 1024;
2184             if (resetStartBitrate) {
2185                 preferences.start_bitrate_bps = std::max(preferences.min_bitrate_bps, 400 * 1000);
2186             }
2187             if (_videoContentType == VideoContentType::Screencast) {
2188                 preferences.max_bitrate_bps = std::max(preferences.min_bitrate_bps, (1020 + 32) * 1000);
2189             } else {
2190                 preferences.max_bitrate_bps = std::max(preferences.min_bitrate_bps, (1020 + 32) * 1000);
2191             }
2192         } else {
2193             preferences.min_bitrate_bps = 32000;
2194             if (resetStartBitrate) {
2195                 preferences.start_bitrate_bps = 32000;
2196             }
2197             preferences.max_bitrate_bps = 32000;
2198         }
2199 
2200         settings.min_bitrate_bps = preferences.min_bitrate_bps;
2201         settings.start_bitrate_bps = preferences.start_bitrate_bps;
2202         settings.max_bitrate_bps = preferences.max_bitrate_bps;
2203 
2204         _call->GetTransportControllerSend()->SetSdpBitrateParameters(preferences);
2205 		_threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&]() {
2206 			_call->SetClientBitratePreferences(settings);
2207 		});
2208     }
2209 
setIsRtcConnected(bool isConnected)2210     void setIsRtcConnected(bool isConnected) {
2211         if (_isRtcConnected == isConnected) {
2212             return;
2213         }
2214         _isRtcConnected = isConnected;
2215 
2216         RTC_LOG(LS_INFO) << formatTimestampMillis(rtc::TimeMillis()) << ": " << "setIsRtcConnected: " << _isRtcConnected;
2217 
2218         if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp) {
2219             _broadcastEnabledUntilRtcIsConnectedAtTimestamp = absl::nullopt;
2220 
2221             if (_streamingContext) {
2222                 _streamingContext.reset();
2223                 _audioDeviceDataObserverShared->setStreamingContext(nullptr);
2224             }
2225         }
2226 
2227         updateIsConnected();
2228     }
2229 
updateIsConnected()2230     void updateIsConnected() {
2231         bool isEffectivelyConnected = false;
2232         bool isTransitioningFromBroadcastToRtc = false;
2233         switch (_connectionMode) {
2234             case GroupConnectionMode::GroupConnectionModeNone: {
2235                 isEffectivelyConnected = false;
2236                 if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp && _isBroadcastConnected) {
2237                     isEffectivelyConnected = true;
2238                     isTransitioningFromBroadcastToRtc = true;
2239                 }
2240                 break;
2241             }
2242             case GroupConnectionMode::GroupConnectionModeRtc: {
2243                 isEffectivelyConnected = _isRtcConnected;
2244                 if (_broadcastEnabledUntilRtcIsConnectedAtTimestamp && _isBroadcastConnected) {
2245                     isEffectivelyConnected = true;
2246                     isTransitioningFromBroadcastToRtc = true;
2247                 }
2248                 break;
2249             }
2250             case GroupConnectionMode::GroupConnectionModeBroadcast: {
2251                 isEffectivelyConnected = _isBroadcastConnected;
2252                 break;
2253             }
2254         }
2255 
2256         GroupNetworkState effectiveNetworkState;
2257         effectiveNetworkState.isConnected = isEffectivelyConnected;
2258         effectiveNetworkState.isTransitioningFromBroadcastToRtc = isTransitioningFromBroadcastToRtc;
2259 
2260         if (_effectiveNetworkState.isConnected != effectiveNetworkState.isConnected || _effectiveNetworkState.isTransitioningFromBroadcastToRtc != effectiveNetworkState.isTransitioningFromBroadcastToRtc) {
2261             _effectiveNetworkState = effectiveNetworkState;
2262 
2263             if (_networkStateUpdated) {
2264                 _networkStateUpdated(_effectiveNetworkState);
2265             }
2266         }
2267     }
2268 
updateIsDataChannelOpen(bool isDataChannelOpen)2269     void updateIsDataChannelOpen(bool isDataChannelOpen) {
2270         if (_isDataChannelOpen == isDataChannelOpen) {
2271             return;
2272         }
2273         _isDataChannelOpen = isDataChannelOpen;
2274 
2275         if (_isDataChannelOpen) {
2276             maybeUpdateRemoteVideoConstraints();
2277         }
2278     }
2279 
receivePacket(rtc::CopyOnWriteBuffer const & packet,bool isUnresolved)2280     void receivePacket(rtc::CopyOnWriteBuffer const &packet, bool isUnresolved) {
2281       if (packet.size() >= 4) {
2282             if (packet.data()[0] == 0x13 && packet.data()[1] == 0x88 && packet.data()[2] == 0x13 && packet.data()[3] == 0x88) {
2283                 // SCTP packet header (source port 5000, destination port 5000)
2284                 return;
2285             }
2286         }
2287 
2288         webrtc::RtpUtility::RtpHeaderParser rtpParser(packet.data(), packet.size());
2289 
2290         webrtc::RTPHeader header;
2291         if (rtpParser.RTCP()) {
2292             if (!rtpParser.ParseRtcp(&header)) {
2293                 RTC_LOG(LS_INFO) << "Could not parse rtcp header";
2294                 return;
2295             }
2296 
2297             _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this, packet]() {
2298                 _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, -1);
2299             });
2300         } else {
2301             if (!rtpParser.Parse(&header)) {
2302                 // Probably a data channel message
2303                 return;
2304             }
2305 
2306             if (header.ssrc == _outgoingAudioSsrc) {
2307                 return;
2308             }
2309 
2310             auto ssrcInfo = _channelBySsrc.find(header.ssrc);
2311             if (ssrcInfo == _channelBySsrc.end()) {
2312                 // opus
2313                 if (header.payloadType == 111) {
2314                     maybeRequestUnknownSsrc(header.ssrc);
2315                     _missingPacketBuffer.add(header.ssrc, packet);
2316                 }
2317             } else {
2318                 switch (ssrcInfo->second.type) {
2319                     case ChannelSsrcInfo::Type::Audio: {
2320                         const auto it = _incomingAudioChannels.find(ChannelId(header.ssrc));
2321                         if (it != _incomingAudioChannels.end()) {
2322                             it->second->updateActivity();
2323                         }
2324 
2325                         break;
2326                     }
2327                     case ChannelSsrcInfo::Type::Video: {
2328                         break;
2329                     }
2330                     default: {
2331                         break;
2332                     }
2333                 }
2334             }
2335         }
2336     }
2337 
receiveRtcpPacket(rtc::CopyOnWriteBuffer const & packet,int64_t timestamp)2338     void receiveRtcpPacket(rtc::CopyOnWriteBuffer const &packet, int64_t timestamp) {
2339         _threads->getWorkerThread()->PostTask(RTC_FROM_HERE, [this, packet, timestamp]() {
2340             _call->Receiver()->DeliverPacket(webrtc::MediaType::ANY, packet, timestamp);
2341         });
2342     }
2343 
receiveDataChannelMessage(std::string const & message)2344     void receiveDataChannelMessage(std::string const &message) {
2345         std::string parsingError;
2346         auto json = json11::Json::parse(message, parsingError);
2347         if (json.type() != json11::Json::OBJECT) {
2348             RTC_LOG(LS_WARNING) << "receiveDataChannelMessage: error parsing message: " << parsingError;
2349             return;
2350         }
2351 
2352         if (json.is_object()) {
2353             const auto colibriClass = json.object_items().find("colibriClass");
2354             if (colibriClass != json.object_items().end() && colibriClass->second.is_string()) {
2355                 const auto messageType = colibriClass->second.string_value();
2356                 if (messageType == "SenderVideoConstraints") {
2357                     const auto videoConstraints = json.object_items().find("videoConstraints");
2358                     if (videoConstraints != json.object_items().end() && videoConstraints->second.is_object()) {
2359                         const auto idealHeight = videoConstraints->second.object_items().find("idealHeight");
2360                         if (idealHeight != videoConstraints->second.object_items().end() && idealHeight->second.is_number()) {
2361                             int outgoingVideoConstraint = idealHeight->second.int_value();
2362                             if (_outgoingVideoConstraint != outgoingVideoConstraint) {
2363                                 if (_outgoingVideoConstraint > outgoingVideoConstraint) {
2364                                     _pendingOutgoingVideoConstraint = outgoingVideoConstraint;
2365 
2366                                     int requestId = _pendingOutgoingVideoConstraintRequestId;
2367                                     _pendingOutgoingVideoConstraintRequestId += 1;
2368 
2369                                     const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2370                                     _threads->getMediaThread()->PostDelayedTask(RTC_FROM_HERE, [weak, requestId]() {
2371                                         auto strong = weak.lock();
2372                                         if (!strong) {
2373                                             return;
2374                                         }
2375                                         if (strong->_pendingOutgoingVideoConstraint != -1 && strong->_pendingOutgoingVideoConstraintRequestId == requestId) {
2376                                             if (strong->_outgoingVideoConstraint != strong->_pendingOutgoingVideoConstraint) {
2377                                                 strong->_outgoingVideoConstraint = strong->_pendingOutgoingVideoConstraint;
2378                                                 strong->adjustVideoSendParams();
2379                                             }
2380                                             strong->_pendingOutgoingVideoConstraint = -1;
2381                                         }
2382                                     }, 2000);
2383                                 } else {
2384                                     _pendingOutgoingVideoConstraint = -1;
2385                                     _pendingOutgoingVideoConstraintRequestId += 1;
2386                                     _outgoingVideoConstraint = outgoingVideoConstraint;
2387                                     adjustVideoSendParams();
2388                                 }
2389                             }
2390                         }
2391                     }
2392                 } else if (messageType == "DebugMessage") {
2393                     const auto message = json.object_items().find("message");
2394                     if (message != json.object_items().end() && message->second.is_string()) {
2395                         std::vector<std::string> parts = splitString(message->second.string_value(), '\n');
2396                         for (const auto &part : parts) {
2397                             std::string cleanString = part;
2398                             std::size_t index = cleanString.find("=");
2399                             if (index == std::string::npos) {
2400                                 continue;
2401                             }
2402                             cleanString.erase(cleanString.begin(), cleanString.begin() + index + 1);
2403 
2404                             index = cleanString.find("target=");
2405                             if (index == std::string::npos) {
2406                                 continue;
2407                             }
2408 
2409                             std::string endpointId = cleanString.substr(0, index);
2410                             cleanString.erase(cleanString.begin(), cleanString.begin() + index + 7);
2411 
2412                             index = cleanString.find("p/");
2413                             if (index == std::string::npos) {
2414                                 continue;
2415                             }
2416 
2417                             std::string targetQuality = cleanString.substr(0, index);
2418                             cleanString.erase(cleanString.begin(), cleanString.begin() + index + 2);
2419 
2420                             index = cleanString.find("ideal=");
2421                             if (index == std::string::npos) {
2422                                 continue;
2423                             }
2424 
2425                             cleanString.erase(cleanString.begin(), cleanString.begin() + index + 6);
2426 
2427                             index = cleanString.find("p/");
2428                             if (index == std::string::npos) {
2429                                 continue;
2430                             }
2431 
2432                             std::string availableQuality = cleanString.substr(0, index);
2433 
2434                             for (const auto &it : _incomingVideoChannels) {
2435                                 if (it.second->endpointId() == endpointId) {
2436                                     GroupInstanceStats::IncomingVideoStats incomingVideoStats;
2437                                     incomingVideoStats.receivingQuality = stringToInt(targetQuality);
2438                                     incomingVideoStats.availableQuality = stringToInt(availableQuality);
2439                                     it.second->setStats(incomingVideoStats);
2440                                 }
2441                             }
2442                         }
2443                     }
2444                 }
2445             }
2446         }
2447     }
2448 
maybeRequestUnknownSsrc(uint32_t ssrc)2449     void maybeRequestUnknownSsrc(uint32_t ssrc) {
2450         if (!_requestMediaChannelDescriptions) {
2451             MediaChannelDescription description;
2452             description.audioSsrc = ssrc;
2453             processMediaChannelDescriptionsResponse(-1, {description});
2454             return;
2455         }
2456 
2457         for (const auto &it : _requestedMediaChannelDescriptions) {
2458             if (std::find(it.second.ssrcs.begin(), it.second.ssrcs.end(), ssrc) != it.second.ssrcs.end()) {
2459                 return;
2460             }
2461         }
2462 
2463         int requestId = _nextMediaChannelDescriptionsRequestId;
2464         _nextMediaChannelDescriptionsRequestId += 1;
2465 
2466         std::vector<uint32_t> requestSsrcs = { ssrc };
2467 
2468         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2469         auto task = _requestMediaChannelDescriptions(requestSsrcs, [weak, threads = _threads, requestId](std::vector<MediaChannelDescription> &&descriptions) {
2470             threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, requestId, descriptions = std::move(descriptions)]() mutable {
2471                 auto strong = weak.lock();
2472                 if (!strong) {
2473                     return;
2474                 }
2475 
2476                 strong->processMediaChannelDescriptionsResponse(requestId, descriptions);
2477             });
2478         });
2479         _requestedMediaChannelDescriptions.insert(std::make_pair(requestId, RequestedMediaChannelDescriptions(task, std::move(requestSsrcs))));
2480     }
2481 
processMediaChannelDescriptionsResponse(int requestId,std::vector<MediaChannelDescription> const & descriptions)2482     void processMediaChannelDescriptionsResponse(int requestId, std::vector<MediaChannelDescription> const &descriptions) {
2483         _requestedMediaChannelDescriptions.erase(requestId);
2484 
2485         if (_disableIncomingChannels) {
2486             return;
2487         }
2488 
2489         for (const auto &description : descriptions) {
2490             switch (description.type) {
2491                 case MediaChannelDescription::Type::Audio: {
2492                     if (description.audioSsrc != 0) {
2493                         addIncomingAudioChannel(ChannelId(description.audioSsrc));
2494                     }
2495                     break;
2496                 }
2497                 case MediaChannelDescription::Type::Video: {
2498                     break;
2499                 }
2500                 default: {
2501                     break;
2502                 }
2503             }
2504         }
2505     }
2506 
maybeDeliverBufferedPackets(uint32_t ssrc)2507     void maybeDeliverBufferedPackets(uint32_t ssrc) {
2508         // TODO: Re-enable after implementing custom transport
2509         /*auto packets = _missingPacketBuffer.get(ssrc);
2510         if (packets.size() != 0) {
2511             auto it = _ssrcMapping.find(ssrc);
2512             if (it != _ssrcMapping.end()) {
2513                 for (const auto &packet : packets) {
2514                     _threads->getNetworkThread()->Invoke<void>(RTC_FROM_HERE, [this, packet]() {
2515                         _rtpTransport->DemuxPacketInternal(packet, -1);
2516                     });
2517                 }
2518             }
2519         }*/
2520     }
2521 
maybeUpdateRemoteVideoConstraints()2522     void maybeUpdateRemoteVideoConstraints() {
2523         if (!_isDataChannelOpen) {
2524             return;
2525         }
2526 
2527         std::string pinnedEndpoint;
2528 
2529         json11::Json::object json;
2530         json.insert(std::make_pair("colibriClass", json11::Json("ReceiverVideoConstraints")));
2531 
2532         json11::Json::object defaultConstraints;
2533         defaultConstraints.insert(std::make_pair("maxHeight", json11::Json(0)));
2534         json.insert(std::make_pair("defaultConstraints", json11::Json(std::move(defaultConstraints))));
2535 
2536         json11::Json::array onStageEndpoints;
2537         json11::Json::object constraints;
2538 
2539         for (const auto &incomingVideoChannel : _incomingVideoChannels) {
2540             json11::Json::object selectedConstraint;
2541 
2542             switch (incomingVideoChannel.second->requestedMinQuality()) {
2543                 case VideoChannelDescription::Quality::Full: {
2544                     selectedConstraint.insert(std::make_pair("minHeight", json11::Json(720)));
2545                     break;
2546                 }
2547                 case VideoChannelDescription::Quality::Medium: {
2548                     selectedConstraint.insert(std::make_pair("minHeight", json11::Json(360)));
2549                     break;
2550                 }
2551                 case VideoChannelDescription::Quality::Thumbnail: {
2552                     selectedConstraint.insert(std::make_pair("minHeight", json11::Json(180)));
2553                     break;
2554                 }
2555                 default: {
2556                     break;
2557                 }
2558             }
2559             switch (incomingVideoChannel.second->requestedMaxQuality()) {
2560                 case VideoChannelDescription::Quality::Full: {
2561                     onStageEndpoints.push_back(json11::Json(incomingVideoChannel.first.endpointId));
2562                     selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(720)));
2563                     break;
2564                 }
2565                 case VideoChannelDescription::Quality::Medium: {
2566                     selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(360)));
2567                     break;
2568                 }
2569                 case VideoChannelDescription::Quality::Thumbnail: {
2570                     selectedConstraint.insert(std::make_pair("maxHeight", json11::Json(180)));
2571                     break;
2572                 }
2573                 default: {
2574                     break;
2575                 }
2576             }
2577 
2578             constraints.insert(std::make_pair(incomingVideoChannel.first.endpointId, json11::Json(std::move(selectedConstraint))));
2579         }
2580 
2581         json.insert(std::make_pair("onStageEndpoints", json11::Json(std::move(onStageEndpoints))));
2582         json.insert(std::make_pair("constraints", json11::Json(std::move(constraints))));
2583 
2584         std::string result = json11::Json(std::move(json)).dump();
2585         _networkManager->perform(RTC_FROM_HERE, [result = std::move(result)](GroupNetworkManager *networkManager) {
2586             networkManager->sendDataChannelMessage(result);
2587         });
2588     }
2589 
setConnectionMode(GroupConnectionMode connectionMode,bool keepBroadcastIfWasEnabled)2590     void setConnectionMode(GroupConnectionMode connectionMode, bool keepBroadcastIfWasEnabled) {
2591         if (_connectionMode != connectionMode || connectionMode == GroupConnectionMode::GroupConnectionModeNone) {
2592             GroupConnectionMode previousMode = _connectionMode;
2593             _connectionMode = connectionMode;
2594             onConnectionModeUpdated(previousMode, keepBroadcastIfWasEnabled);
2595         }
2596     }
2597 
onConnectionModeUpdated(GroupConnectionMode previousMode,bool keepBroadcastIfWasEnabled)2598     void onConnectionModeUpdated(GroupConnectionMode previousMode, bool keepBroadcastIfWasEnabled) {
2599         RTC_CHECK(_connectionMode != previousMode || _connectionMode == GroupConnectionMode::GroupConnectionModeNone);
2600 
2601         if (previousMode == GroupConnectionMode::GroupConnectionModeRtc) {
2602             _networkManager->perform(RTC_FROM_HERE, [](GroupNetworkManager *networkManager) {
2603                 networkManager->stop();
2604             });
2605         } else if (previousMode == GroupConnectionMode::GroupConnectionModeBroadcast) {
2606             if (keepBroadcastIfWasEnabled) {
2607                 _broadcastEnabledUntilRtcIsConnectedAtTimestamp = rtc::TimeMillis();
2608             } else {
2609                 if (_streamingContext) {
2610                     _streamingContext.reset();
2611                     _audioDeviceDataObserverShared->setStreamingContext(nullptr);
2612                 }
2613             }
2614         }
2615 
2616         if (_connectionMode == GroupConnectionMode::GroupConnectionModeNone) {
2617             destroyOutgoingAudioChannel();
2618             destroyOutgoingVideoChannel();
2619 
2620             // Regenerate and reconfigure.
2621             generateSsrcs();
2622 
2623             if (!_isMuted) {
2624                 createOutgoingAudioChannel();
2625             }
2626             createOutgoingVideoChannel();
2627         }
2628 
2629         switch (_connectionMode) {
2630             case GroupConnectionMode::GroupConnectionModeNone: {
2631                 break;
2632             }
2633             case GroupConnectionMode::GroupConnectionModeRtc: {
2634                 _networkManager->perform(RTC_FROM_HERE, [](GroupNetworkManager *networkManager) {
2635                     networkManager->start();
2636                 });
2637                 break;
2638             }
2639             case GroupConnectionMode::GroupConnectionModeBroadcast: {
2640                 _isBroadcastConnected = false;
2641 
2642                 if (!_streamingContext) {
2643                     StreamingMediaContext::StreamingMediaContextArguments arguments;
2644                     const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
2645                     arguments.threads = _threads;
2646                     arguments.requestCurrentTime = _requestCurrentTime;
2647                     arguments.requestAudioBroadcastPart = _requestAudioBroadcastPart;
2648                     arguments.requestVideoBroadcastPart = _requestVideoBroadcastPart;
2649                     arguments.updateAudioLevel = [weak, threads = _threads](uint32_t ssrc, float level, bool isSpeech) {
2650                         assert(threads->getMediaThread()->IsCurrent());
2651 
2652                         auto strong = weak.lock();
2653                         if (!strong) {
2654                             return;
2655                         }
2656 
2657                         InternalGroupLevelValue updated;
2658                         updated.value.level = level;
2659                         updated.value.voice = isSpeech;
2660                         updated.timestamp = rtc::TimeMillis();
2661                         strong->_audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
2662                     };
2663                     _streamingContext = std::make_shared<StreamingMediaContext>(std::move(arguments));
2664 
2665                     for (const auto &it : _pendingVideoSinks) {
2666                         for (const auto &sink : it.second) {
2667                             _streamingContext->addVideoSink(it.first.endpointId, sink);
2668                         }
2669                     }
2670 
2671                     for (const auto &it : _volumeBySsrc) {
2672                         _streamingContext->setVolume(it.first, it.second);
2673                     }
2674 
2675                     std::vector<StreamingMediaContext::VideoChannel> streamingVideoChannels;
2676                     for (const auto &it : _pendingRequestedVideo) {
2677                         streamingVideoChannels.emplace_back(it.maxQuality, it.endpointId);
2678                     }
2679                     _streamingContext->setActiveVideoChannels(streamingVideoChannels);
2680 
2681                     _audioDeviceDataObserverShared->setStreamingContext(_streamingContext);
2682                 }
2683 
2684                 break;
2685             }
2686             default: {
2687                 RTC_FATAL() << "Unknown connectionMode";
2688                 break;
2689             }
2690         }
2691 
2692         updateIsConnected();
2693     }
2694 
generateSsrcs()2695     void generateSsrcs() {
2696         auto generator = std::mt19937(std::random_device()());
2697         auto distribution = std::uniform_int_distribution<uint32_t>();
2698         do {
2699             _outgoingAudioSsrc = distribution(generator) & 0x7fffffffU;
2700         } while (!_outgoingAudioSsrc);
2701 
2702         uint32_t outgoingVideoSsrcBase = _outgoingAudioSsrc + 1;
2703         int numVideoSimulcastLayers = 3;
2704         if (_videoContentType == VideoContentType::Screencast) {
2705             numVideoSimulcastLayers = 2;
2706         }
2707         _outgoingVideoSsrcs.simulcastLayers.clear();
2708         for (int layerIndex = 0; layerIndex < numVideoSimulcastLayers; layerIndex++) {
2709             _outgoingVideoSsrcs.simulcastLayers.push_back(VideoSsrcs::SimulcastLayer(outgoingVideoSsrcBase + layerIndex * 2 + 0, outgoingVideoSsrcBase + layerIndex * 2 + 1));
2710         }
2711 
2712         _videoSourceGroups.clear();
2713 
2714         std::vector<uint32_t> simulcastGroupSsrcs;
2715         std::vector<cricket::SsrcGroup> fidGroups;
2716         for (const auto &layer : _outgoingVideoSsrcs.simulcastLayers) {
2717             simulcastGroupSsrcs.push_back(layer.ssrc);
2718 
2719             cricket::SsrcGroup fidGroup(cricket::kFidSsrcGroupSemantics, { layer.ssrc, layer.fidSsrc });
2720             fidGroups.push_back(fidGroup);
2721         }
2722         if (simulcastGroupSsrcs.size() > 1) {
2723             cricket::SsrcGroup simulcastGroup(cricket::kSimSsrcGroupSemantics, simulcastGroupSsrcs);
2724 
2725             GroupJoinPayloadVideoSourceGroup payloadSimulcastGroup;
2726             payloadSimulcastGroup.semantics = "SIM";
2727             payloadSimulcastGroup.ssrcs = simulcastGroupSsrcs;
2728             _videoSourceGroups.push_back(payloadSimulcastGroup);
2729         }
2730 
2731         for (auto fidGroup : fidGroups) {
2732             GroupJoinPayloadVideoSourceGroup payloadFidGroup;
2733             payloadFidGroup.semantics = "FID";
2734             payloadFidGroup.ssrcs = fidGroup.ssrcs;
2735             _videoSourceGroups.push_back(payloadFidGroup);
2736         }
2737     }
2738 
emitJoinPayload(std::function<void (GroupJoinPayload const &)> completion)2739     void emitJoinPayload(std::function<void(GroupJoinPayload const &)> completion) {
2740         _networkManager->perform(RTC_FROM_HERE, [outgoingAudioSsrc = _outgoingAudioSsrc, /*videoPayloadTypes = _videoPayloadTypes, videoExtensionMap = _videoExtensionMap, */videoSourceGroups = _videoSourceGroups, videoContentType = _videoContentType, completion](GroupNetworkManager *networkManager) {
2741             GroupJoinInternalPayload payload;
2742 
2743             payload.audioSsrc = outgoingAudioSsrc;
2744 
2745             if (videoContentType != VideoContentType::None) {
2746                 GroupParticipantVideoInformation videoInformation;
2747                 videoInformation.ssrcGroups = videoSourceGroups;
2748                 payload.videoInformation = std::move(videoInformation);
2749             }
2750 
2751             GroupJoinTransportDescription transportDescription;
2752 
2753             auto localIceParameters = networkManager->getLocalIceParameters();
2754             transportDescription.ufrag = localIceParameters.ufrag;
2755             transportDescription.pwd = localIceParameters.pwd;
2756 
2757             auto localFingerprint = networkManager->getLocalFingerprint();
2758             if (localFingerprint) {
2759                 GroupJoinTransportDescription::Fingerprint serializedFingerprint;
2760                 serializedFingerprint.hash = localFingerprint->algorithm;
2761                 serializedFingerprint.fingerprint = localFingerprint->GetRfc4572Fingerprint();
2762                 serializedFingerprint.setup = "passive";
2763                 transportDescription.fingerprints.push_back(std::move(serializedFingerprint));
2764             }
2765 
2766             payload.transport = std::move(transportDescription);
2767 
2768             GroupJoinPayload result;
2769             result.audioSsrc = payload.audioSsrc;
2770             result.json = payload.serialize();
2771             completion(result);
2772         });
2773     }
2774 
setVideoSource(std::function<webrtc::VideoTrackSourceInterface * ()> getVideoSource,bool isInitializing)2775     void setVideoSource(std::function<webrtc::VideoTrackSourceInterface*()> getVideoSource, bool isInitializing) {
2776         bool resetBitrate = (!_getVideoSource) != (!getVideoSource) && !isInitializing;
2777         if (!isInitializing && _getVideoSource && getVideoSource && getVideoSource() == _getVideoSource()) {
2778             return;
2779         }
2780 
2781         _getVideoSource = std::move(getVideoSource);
2782 		updateVideoSend();
2783         if (resetBitrate) {
2784             adjustBitratePreferences(true);
2785         }
2786     }
2787 
setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture,bool isInitializing)2788     void setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture, bool isInitializing) {
2789         _videoCapture = videoCapture;
2790         setVideoSource(videoCaptureToGetVideoSource(std::move(videoCapture)), isInitializing);
2791     }
2792 
setAudioOutputDevice(const std::string & id)2793     void setAudioOutputDevice(const std::string &id) {
2794 #ifndef WEBRTC_IOS
2795         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&] {
2796             SetAudioOutputDeviceById(_audioDeviceModule.get(), id);
2797         });
2798 #endif // WEBRTC_IOS
2799     }
2800 
setAudioInputDevice(const std::string & id)2801     void setAudioInputDevice(const std::string &id) {
2802 #ifndef WEBRTC_IOS
2803         _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [&] {
2804             SetAudioInputDeviceById(_audioDeviceModule.get(), id);
2805         });
2806 #endif // WEBRTC_IOS
2807     }
2808 
addExternalAudioSamples(std::vector<uint8_t> && samples)2809     void addExternalAudioSamples(std::vector<uint8_t> &&samples) {
2810         if (samples.size() % 2 != 0) {
2811             return;
2812         }
2813         _externalAudioSamplesMutex.Lock();
2814 
2815         size_t previousSize = _externalAudioSamples.size();
2816         _externalAudioSamples.resize(_externalAudioSamples.size() + samples.size() / 2);
2817         webrtc::S16ToFloatS16((const int16_t *)samples.data(), samples.size() / 2, _externalAudioSamples.data() + previousSize);
2818 
2819         if (_externalAudioSamples.size() > 2 * 48000) {
2820             _externalAudioSamples.erase(_externalAudioSamples.begin(), _externalAudioSamples.begin() + (_externalAudioSamples.size() - 2 * 48000));
2821         }
2822 
2823         _externalAudioSamplesMutex.Unlock();
2824     }
2825 
setJoinResponsePayload(std::string const & payload)2826     void setJoinResponsePayload(std::string const &payload) {
2827         RTC_LOG(LS_INFO) << formatTimestampMillis(rtc::TimeMillis()) << ": " << "setJoinResponsePayload";
2828 
2829         auto parsedPayload = GroupJoinResponsePayload::parse(payload);
2830         if (!parsedPayload) {
2831             RTC_LOG(LS_ERROR) << "Could not parse json response payload";
2832             return;
2833         }
2834 
2835         _sharedVideoInformation = parsedPayload->videoInformation;
2836 
2837         _serverBandwidthProbingVideoSsrc.reset();
2838 
2839         if (parsedPayload->videoInformation && parsedPayload->videoInformation->serverVideoBandwidthProbingSsrc) {
2840             setServerBandwidthProbingChannelSsrc(parsedPayload->videoInformation->serverVideoBandwidthProbingSsrc);
2841         }
2842 
2843         _networkManager->perform(RTC_FROM_HERE, [parsedTransport = parsedPayload->transport](GroupNetworkManager *networkManager) {
2844             PeerIceParameters remoteIceParameters;
2845             remoteIceParameters.ufrag = parsedTransport.ufrag;
2846             remoteIceParameters.pwd = parsedTransport.pwd;
2847 
2848             std::vector<cricket::Candidate> iceCandidates;
2849             for (auto const &candidate : parsedTransport.candidates) {
2850                 rtc::SocketAddress address(candidate.ip, stringToInt(candidate.port));
2851 
2852                 cricket::Candidate parsedCandidate(
2853                     /*component=*/stringToInt(candidate.component),
2854                     /*protocol=*/candidate.protocol,
2855                     /*address=*/address,
2856                     /*priority=*/stringToUInt32(candidate.priority),
2857                     /*username=*/parsedTransport.ufrag,
2858                     /*password=*/parsedTransport.pwd,
2859                     /*type=*/candidate.type,
2860                     /*generation=*/stringToUInt32(candidate.generation),
2861                     /*foundation=*/candidate.foundation,
2862                     /*network_id=*/stringToUInt16(candidate.network),
2863                     /*network_cost=*/0
2864                 );
2865                 iceCandidates.push_back(parsedCandidate);
2866             }
2867 
2868             std::unique_ptr<rtc::SSLFingerprint> fingerprint;
2869             if (parsedTransport.fingerprints.size() != 0) {
2870                 fingerprint = rtc::SSLFingerprint::CreateUniqueFromRfc4572(parsedTransport.fingerprints[0].hash, parsedTransport.fingerprints[0].fingerprint);
2871             }
2872 
2873             networkManager->setRemoteParams(remoteIceParameters, iceCandidates, fingerprint.get());
2874         });
2875 
2876         configureVideoParams();
2877         createOutgoingVideoChannel();
2878 
2879         adjustBitratePreferences(true);
2880 
2881         if (!_pendingRequestedVideo.empty()) {
2882             setRequestedVideoChannels(std::move(_pendingRequestedVideo));
2883             _pendingRequestedVideo.clear();
2884         }
2885     }
2886 
setServerBandwidthProbingChannelSsrc(uint32_t probingSsrc)2887     void setServerBandwidthProbingChannelSsrc(uint32_t probingSsrc) {
2888         RTC_CHECK(probingSsrc);
2889 
2890         if (!_sharedVideoInformation || _availablePayloadTypes.empty()) {
2891             return;
2892         }
2893 
2894         GroupParticipantVideoInformation videoInformation;
2895 
2896         GroupJoinPayloadVideoSourceGroup sourceGroup;
2897         sourceGroup.ssrcs.push_back(probingSsrc);
2898         sourceGroup.semantics = "SIM";
2899 
2900         videoInformation.ssrcGroups.push_back(std::move(sourceGroup));
2901 
2902         _serverBandwidthProbingVideoSsrc.reset(new IncomingVideoChannel(
2903             _channelManager.get(),
2904             _call.get(),
2905             _rtpTransport,
2906             _uniqueRandomIdGenerator.get(),
2907             _availableVideoFormats,
2908             _sharedVideoInformation.value(),
2909             123456,
2910             VideoChannelDescription::Quality::Thumbnail,
2911             VideoChannelDescription::Quality::Thumbnail,
2912             videoInformation,
2913             _threads
2914         ));
2915 
2916         ChannelSsrcInfo mapping;
2917         mapping.type = ChannelSsrcInfo::Type::Video;
2918         mapping.allSsrcs.push_back(probingSsrc);
2919         _channelBySsrc.insert(std::make_pair(probingSsrc, std::move(mapping)));
2920     }
2921 
removeSsrcs(std::vector<uint32_t> ssrcs)2922     void removeSsrcs(std::vector<uint32_t> ssrcs) {
2923     }
2924 
removeIncomingVideoSource(uint32_t ssrc)2925     void removeIncomingVideoSource(uint32_t ssrc) {
2926     }
2927 
setIsMuted(bool isMuted)2928     void setIsMuted(bool isMuted) {
2929         if (_isMuted == isMuted) {
2930             return;
2931         }
2932         _isMuted = isMuted;
2933 
2934         if (!_isMuted && !_outgoingAudioChannel) {
2935             createOutgoingAudioChannel();
2936         }
2937 
2938         onUpdatedIsMuted();
2939     }
2940 
onUpdatedIsMuted()2941     void onUpdatedIsMuted() {
2942         if (_outgoingAudioChannel) {
2943             _threads->getWorkerThread()->Invoke<void>(RTC_FROM_HERE, [this]() {
2944                 _outgoingAudioChannel->media_channel()->SetAudioSend(_outgoingAudioSsrc, !_isMuted, nullptr, &_audioSource);
2945                 _outgoingAudioChannel->Enable(!_isMuted);
2946             });
2947         }
2948     }
2949 
setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled)2950     void setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled) {
2951         _noiseSuppressionConfiguration->isEnabled = isNoiseSuppressionEnabled;
2952     }
2953 
addIncomingVideoOutput(std::string const & endpointId,std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink)2954     void addIncomingVideoOutput(std::string const &endpointId, std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
2955         if (_sharedVideoInformation && endpointId == _sharedVideoInformation->endpointId) {
2956             if (_videoCapture) {
2957                 _videoCaptureSink->addSink(sink);
2958                 _videoCapture->setOutput(_videoCaptureSink);
2959             }
2960         } else {
2961             auto it = _incomingVideoChannels.find(VideoChannelId(endpointId));
2962             if (it != _incomingVideoChannels.end()) {
2963                 it->second->addSink(sink);
2964             } else {
2965                 _pendingVideoSinks[VideoChannelId(endpointId)].push_back(sink);
2966             }
2967 
2968             if (_streamingContext) {
2969                 _streamingContext->addVideoSink(endpointId, sink);
2970             }
2971         }
2972     }
2973 
addIncomingAudioChannel(ChannelId ssrc,bool isRawPcm=false)2974     void addIncomingAudioChannel(ChannelId ssrc, bool isRawPcm = false) {
2975         if (_incomingAudioChannels.find(ssrc) != _incomingAudioChannels.end()) {
2976             return;
2977         }
2978 
2979         if (_incomingAudioChannels.size() > 5) {
2980             auto timestamp = rtc::TimeMillis();
2981 
2982             int64_t minActivity = INT64_MAX;
2983             ChannelId minActivityChannelId(0, 0);
2984 
2985             for (const auto &it : _incomingAudioChannels) {
2986                 if (it.first.networkSsrc == 1) {
2987                     continue;
2988                 }
2989                 auto activity = it.second->getActivity();
2990                 if (activity < minActivity && activity < timestamp - 1000) {
2991                     minActivity = activity;
2992                     minActivityChannelId = it.first;
2993                 }
2994             }
2995 
2996             if (minActivityChannelId.networkSsrc != 0) {
2997                 removeIncomingAudioChannel(minActivityChannelId);
2998             }
2999 
3000             if (_incomingAudioChannels.size() > 5) {
3001                 // Wait until there is a channel that hasn't been active in 1 second
3002                 return;
3003             }
3004         }
3005 
3006         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
3007 
3008         std::function<void(AudioSinkImpl::Update)> onAudioSinkUpdate;
3009         if (ssrc.actualSsrc != ssrc.networkSsrc) {
3010             if (_audioLevelsUpdated) {
3011                 onAudioSinkUpdate = [weak, ssrc = ssrc, threads = _threads](AudioSinkImpl::Update update) {
3012                     threads->getMediaThread()->PostTask(RTC_FROM_HERE, [weak, ssrc, update]() {
3013                         auto strong = weak.lock();
3014                         if (!strong) {
3015                             return;
3016                         }
3017 
3018                         auto it = strong->_audioLevels.find(ChannelId(ssrc));
3019                         if (it != strong->_audioLevels.end()) {
3020                             it->second.value.level = fmax(it->second.value.level, update.level);
3021                             if (update.hasSpeech) {
3022                                 it->second.value.voice = true;
3023                             }
3024                             it->second.timestamp = rtc::TimeMillis();
3025                         } else {
3026                             InternalGroupLevelValue updated;
3027                             updated.value.level = update.level;
3028                             updated.value.voice = update.hasSpeech;
3029                             updated.timestamp = rtc::TimeMillis();
3030                             strong->_audioLevels.insert(std::make_pair(ChannelId(ssrc), std::move(updated)));
3031                         }
3032                     });
3033                 };
3034             }
3035         }
3036 
3037         std::unique_ptr<IncomingAudioChannel> channel(new IncomingAudioChannel(
3038           _channelManager.get(),
3039             _call.get(),
3040             _rtpTransport,
3041             _uniqueRandomIdGenerator.get(),
3042             isRawPcm,
3043             ssrc,
3044             std::move(onAudioSinkUpdate),
3045             _onAudioFrame,
3046             _threads
3047         ));
3048 
3049         auto volume = _volumeBySsrc.find(ssrc.actualSsrc);
3050         if (volume != _volumeBySsrc.end()) {
3051             channel->setVolume(volume->second);
3052         }
3053 
3054         _incomingAudioChannels.insert(std::make_pair(ssrc, std::move(channel)));
3055 
3056         auto currentMapping = _channelBySsrc.find(ssrc.networkSsrc);
3057         if (currentMapping != _channelBySsrc.end()) {
3058             if (currentMapping->second.type == ChannelSsrcInfo::Type::Audio) {
3059                 if (std::find(currentMapping->second.allSsrcs.begin(), currentMapping->second.allSsrcs.end(), ssrc.networkSsrc) == currentMapping->second.allSsrcs.end()) {
3060                     currentMapping->second.allSsrcs.push_back(ssrc.networkSsrc);
3061                 }
3062             }
3063         } else {
3064             ChannelSsrcInfo mapping;
3065             mapping.type = ChannelSsrcInfo::Type::Audio;
3066             mapping.allSsrcs.push_back(ssrc.networkSsrc);
3067             _channelBySsrc.insert(std::make_pair(ssrc.networkSsrc, std::move(mapping)));
3068         }
3069 
3070         maybeDeliverBufferedPackets(ssrc.networkSsrc);
3071 
3072         adjustBitratePreferences(false);
3073     }
3074 
removeIncomingAudioChannel(ChannelId const & channelId)3075     void removeIncomingAudioChannel(ChannelId const &channelId) {
3076         const auto it = _incomingAudioChannels.find(channelId);
3077         if (it != _incomingAudioChannels.end()) {
3078             _incomingAudioChannels.erase(it);
3079         }
3080 
3081         auto currentMapping = _channelBySsrc.find(channelId.networkSsrc);
3082         if (currentMapping != _channelBySsrc.end()) {
3083             if (currentMapping->second.type == ChannelSsrcInfo::Type::Audio) {
3084                 auto ssrcs = currentMapping->second.allSsrcs;
3085                 for (auto ssrc : ssrcs) {
3086                     auto it = _channelBySsrc.find(ssrc);
3087                     if (it != _channelBySsrc.end()) {
3088                         _channelBySsrc.erase(it);
3089                     }
3090                 }
3091             }
3092         }
3093     }
3094 
addIncomingVideoChannel(uint32_t audioSsrc,GroupParticipantVideoInformation const & videoInformation,VideoChannelDescription::Quality minQuality,VideoChannelDescription::Quality maxQuality)3095     void addIncomingVideoChannel(uint32_t audioSsrc, GroupParticipantVideoInformation const &videoInformation, VideoChannelDescription::Quality minQuality, VideoChannelDescription::Quality maxQuality) {
3096         if (!_sharedVideoInformation) {
3097             return;
3098         }
3099         if (_incomingVideoChannels.find(VideoChannelId(videoInformation.endpointId)) != _incomingVideoChannels.end()) {
3100             return;
3101         }
3102 
3103         const auto weak = std::weak_ptr<GroupInstanceCustomInternal>(shared_from_this());
3104 
3105         std::unique_ptr<IncomingVideoChannel> channel(new IncomingVideoChannel(
3106             _channelManager.get(),
3107             _call.get(),
3108             _rtpTransport,
3109             _uniqueRandomIdGenerator.get(),
3110             _availableVideoFormats,
3111             _sharedVideoInformation.value(),
3112             audioSsrc,
3113             minQuality,
3114             maxQuality,
3115             videoInformation,
3116             _threads
3117         ));
3118 
3119         const auto pendingSinks = _pendingVideoSinks.find(VideoChannelId(videoInformation.endpointId));
3120         if (pendingSinks != _pendingVideoSinks.end()) {
3121             for (const auto &sink : pendingSinks->second) {
3122                 channel->addSink(sink);
3123             }
3124 
3125             _pendingVideoSinks.erase(pendingSinks);
3126         }
3127 
3128         _incomingVideoChannels.insert(std::make_pair(VideoChannelId(videoInformation.endpointId), std::move(channel)));
3129 
3130         std::vector<uint32_t> allSsrcs;
3131         for (const auto &group : videoInformation.ssrcGroups) {
3132             for (auto ssrc : group.ssrcs) {
3133                 if (std::find(allSsrcs.begin(), allSsrcs.end(), ssrc) == allSsrcs.end()) {
3134                     allSsrcs.push_back(ssrc);
3135                 }
3136             }
3137         }
3138 
3139         for (auto ssrc : allSsrcs) {
3140             ChannelSsrcInfo mapping;
3141             mapping.type = ChannelSsrcInfo::Type::Video;
3142             mapping.allSsrcs = allSsrcs;
3143             mapping.videoEndpointId = videoInformation.endpointId;
3144             _channelBySsrc.insert(std::make_pair(ssrc, std::move(mapping)));
3145         }
3146 
3147         for (auto ssrc : allSsrcs) {
3148             maybeDeliverBufferedPackets(ssrc);
3149         }
3150 
3151         adjustBitratePreferences(false);
3152     }
3153 
setVolume(uint32_t ssrc,double volume)3154     void setVolume(uint32_t ssrc, double volume) {
3155         auto current = _volumeBySsrc.find(ssrc);
3156         if (current != _volumeBySsrc.end() && std::abs(current->second - volume) < 0.0001) {
3157             return;
3158         }
3159 
3160         _volumeBySsrc[ssrc] = volume;
3161 
3162         auto it = _incomingAudioChannels.find(ChannelId(ssrc));
3163         if (it != _incomingAudioChannels.end()) {
3164             it->second->setVolume(volume);
3165         }
3166 
3167         it = _incomingAudioChannels.find(ChannelId(ssrc + 1000, ssrc));
3168         if (it != _incomingAudioChannels.end()) {
3169             it->second->setVolume(volume);
3170         }
3171 
3172         if (_streamingContext) {
3173             _streamingContext->setVolume(ssrc, volume);
3174         }
3175     }
3176 
setRequestedVideoChannels(std::vector<VideoChannelDescription> && requestedVideoChannels)3177     void setRequestedVideoChannels(std::vector<VideoChannelDescription> &&requestedVideoChannels) {
3178         if (_streamingContext) {
3179             std::vector<StreamingMediaContext::VideoChannel> streamingVideoChannels;
3180             for (const auto &it : requestedVideoChannels) {
3181                 streamingVideoChannels.emplace_back(it.maxQuality, it.endpointId);
3182             }
3183             _streamingContext->setActiveVideoChannels(streamingVideoChannels);
3184         }
3185 
3186         if (!_sharedVideoInformation) {
3187             _pendingRequestedVideo = std::move(requestedVideoChannels);
3188             return;
3189         }
3190         bool updated = false;
3191         std::vector<std::string> allEndpointIds;
3192 
3193         for (const auto &description : requestedVideoChannels) {
3194             if (_sharedVideoInformation && _sharedVideoInformation->endpointId == description.endpointId) {
3195                 continue;
3196             }
3197 
3198             GroupParticipantVideoInformation videoInformation;
3199             videoInformation.endpointId = description.endpointId;
3200             for (const auto &group : description.ssrcGroups) {
3201                 GroupJoinPayloadVideoSourceGroup parsedGroup;
3202                 parsedGroup.semantics = group.semantics;
3203                 parsedGroup.ssrcs = group.ssrcs;
3204                 videoInformation.ssrcGroups.push_back(std::move(parsedGroup));
3205             }
3206 
3207             allEndpointIds.push_back(videoInformation.endpointId);
3208 
3209             auto current = _incomingVideoChannels.find(VideoChannelId(videoInformation.endpointId));
3210             if (current != _incomingVideoChannels.end()) {
3211                 if (current->second->requestedMinQuality() != description.minQuality || current->second->requestedMaxQuality() != description.maxQuality) {
3212                     current->second->setRequstedMinQuality(description.minQuality);
3213                     current->second->setRequstedMaxQuality(description.maxQuality);
3214                     updated = true;
3215                 }
3216                 continue;
3217             }
3218 
3219             addIncomingVideoChannel(description.audioSsrc, videoInformation, description.minQuality, description.maxQuality);
3220             updated = true;
3221         }
3222 
3223         std::vector<std::string> removeEndpointIds;
3224         for (const auto &it : _incomingVideoChannels) {
3225             if (std::find(allEndpointIds.begin(), allEndpointIds.end(), it.first.endpointId) == allEndpointIds.end()) {
3226                 removeEndpointIds.push_back(it.first.endpointId);
3227                 updated = true;
3228             }
3229         }
3230 
3231         for (const auto &endpointId : removeEndpointIds) {
3232             const auto it = _incomingVideoChannels.find(VideoChannelId(endpointId));
3233             if (it != _incomingVideoChannels.end()) {
3234                 auto sinks = it->second->getSinks();
3235                 for (const auto &sink : sinks) {
3236                     _pendingVideoSinks[VideoChannelId(endpointId)].push_back(sink);
3237                 }
3238                 _incomingVideoChannels.erase(it);
3239             }
3240         }
3241 
3242         if (updated) {
3243             maybeUpdateRemoteVideoConstraints();
3244         }
3245     }
3246 
getStats(std::function<void (GroupInstanceStats)> completion)3247     void getStats(std::function<void(GroupInstanceStats)> completion) {
3248         GroupInstanceStats result;
3249 
3250         for (const auto &it : _incomingVideoChannels) {
3251             const auto videoStats = it.second->getStats();
3252             if (videoStats) {
3253                 result.incomingVideoStats.push_back(std::make_pair(it.second->endpointId(), videoStats.value()));
3254             }
3255         }
3256 
3257         completion(result);
3258     }
3259 
3260 private:
createAudioDeviceModule()3261     rtc::scoped_refptr<WrappedAudioDeviceModule> createAudioDeviceModule() {
3262         auto audioDeviceDataObserverShared = _audioDeviceDataObserverShared;
3263         const auto create = [&](webrtc::AudioDeviceModule::AudioLayer layer) {
3264             return webrtc::AudioDeviceModule::Create(
3265                 layer,
3266                 _taskQueueFactory.get());
3267         };
3268         const auto check = [&](const rtc::scoped_refptr<webrtc::AudioDeviceModule> &result) -> rtc::scoped_refptr<WrappedAudioDeviceModule> {
3269             if (!result) {
3270                 return nullptr;
3271             }
3272 
3273             auto audioDeviceObserver = std::make_unique<AudioDeviceDataObserverImpl>(audioDeviceDataObserverShared);
3274             auto module = webrtc::CreateAudioDeviceWithDataObserver(result, std::move(audioDeviceObserver));
3275 
3276             if (module->Init() == 0) {
3277                 return PlatformInterface::SharedInstance()->wrapAudioDeviceModule(module);
3278             } else {
3279                 return nullptr;
3280             }
3281         };
3282         if (_createAudioDeviceModule) {
3283             if (const auto result = check(_createAudioDeviceModule(_taskQueueFactory.get()))) {
3284                 return result;
3285             }
3286         } else if (_videoContentType == VideoContentType::Screencast) {
3287             FakeAudioDeviceModule::Options options;
3288             options.num_channels = 1;
3289             return check(FakeAudioDeviceModule::Creator(nullptr, _externalAudioRecorder, options)(_taskQueueFactory.get()));
3290         }
3291         return check(create(webrtc::AudioDeviceModule::kPlatformDefaultAudio));
3292     }
3293 
3294 private:
3295     std::shared_ptr<Threads> _threads;
3296     GroupConnectionMode _connectionMode = GroupConnectionMode::GroupConnectionModeNone;
3297 
3298     std::function<void(GroupNetworkState)> _networkStateUpdated;
3299     std::function<void(GroupLevelsUpdate const &)> _audioLevelsUpdated;
3300     std::function<void(uint32_t, const AudioFrame &)> _onAudioFrame;
3301     std::function<std::shared_ptr<RequestMediaChannelDescriptionTask>(std::vector<uint32_t> const &, std::function<void(std::vector<MediaChannelDescription> &&)>)> _requestMediaChannelDescriptions;
3302     std::function<std::shared_ptr<BroadcastPartTask>(std::function<void(int64_t)>)> _requestCurrentTime;
3303     std::function<std::shared_ptr<BroadcastPartTask>(int64_t, int64_t, std::function<void(BroadcastPart &&)>)> _requestAudioBroadcastPart;
3304     std::function<std::shared_ptr<BroadcastPartTask>(int64_t, int64_t, int32_t, VideoChannelDescription::Quality, std::function<void(BroadcastPart &&)>)> _requestVideoBroadcastPart;
3305     std::shared_ptr<VideoCaptureInterface> _videoCapture;
3306     std::shared_ptr<VideoSinkImpl> _videoCaptureSink;
3307     std::function<webrtc::VideoTrackSourceInterface*()> _getVideoSource;
3308     bool _disableIncomingChannels = false;
3309     bool _useDummyChannel{true};
3310     int _outgoingAudioBitrateKbit{32};
3311     bool _disableOutgoingAudioProcessing{false};
3312     int _minOutgoingVideoBitrateKbit{100};
3313     VideoContentType _videoContentType{VideoContentType::None};
3314     std::vector<VideoCodecName> _videoCodecPreferences;
3315 
3316     int _nextMediaChannelDescriptionsRequestId = 0;
3317     std::map<int, RequestedMediaChannelDescriptions> _requestedMediaChannelDescriptions;
3318 
3319     std::unique_ptr<ThreadLocalObject<GroupNetworkManager>> _networkManager;
3320 
3321     std::unique_ptr<webrtc::RtcEventLogNull> _eventLog;
3322     std::unique_ptr<webrtc::TaskQueueFactory> _taskQueueFactory;
3323     std::unique_ptr<cricket::MediaEngineInterface> _mediaEngine;
3324     std::unique_ptr<webrtc::Call> _call;
3325     webrtc::FieldTrialBasedConfig _fieldTrials;
3326     webrtc::LocalAudioSinkAdapter _audioSource;
3327     std::shared_ptr<AudioDeviceDataObserverShared> _audioDeviceDataObserverShared;
3328     rtc::scoped_refptr<WrappedAudioDeviceModule> _audioDeviceModule;
3329     std::function<rtc::scoped_refptr<webrtc::AudioDeviceModule>(webrtc::TaskQueueFactory*)> _createAudioDeviceModule;
3330     std::string _initialInputDeviceId;
3331     std::string _initialOutputDeviceId;
3332 
3333     // _outgoingAudioChannel memory is managed by _channelManager
3334     cricket::VoiceChannel *_outgoingAudioChannel = nullptr;
3335     uint32_t _outgoingAudioSsrc = 0;
3336 
3337     std::vector<webrtc::SdpVideoFormat> _availableVideoFormats;
3338     std::vector<OutgoingVideoFormat> _availablePayloadTypes;
3339     absl::optional<OutgoingVideoFormat> _selectedPayloadType;
3340 
3341     std::vector<GroupJoinPayloadVideoPayloadType> _videoPayloadTypes;
3342     std::vector<std::pair<uint32_t, std::string>> _videoExtensionMap;
3343     std::vector<GroupJoinPayloadVideoSourceGroup> _videoSourceGroups;
3344 
3345     std::unique_ptr<rtc::UniqueRandomIdGenerator> _uniqueRandomIdGenerator;
3346     webrtc::RtpTransport *_rtpTransport = nullptr;
3347     std::unique_ptr<cricket::ChannelManager> _channelManager;
3348 
3349     std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> _videoBitrateAllocatorFactory;
3350     // _outgoingVideoChannel memory is managed by _channelManager
3351     cricket::VideoChannel *_outgoingVideoChannel = nullptr;
3352     VideoSsrcs _outgoingVideoSsrcs;
3353     int _outgoingVideoConstraint = 720;
3354     int _pendingOutgoingVideoConstraint = -1;
3355     int _pendingOutgoingVideoConstraintRequestId = 0;
3356 
3357     std::map<ChannelId, InternalGroupLevelValue> _audioLevels;
3358     GroupLevelValue _myAudioLevel;
3359 
3360     bool _isMuted = true;
3361     std::shared_ptr<NoiseSuppressionConfiguration> _noiseSuppressionConfiguration;
3362 
3363     MissingSsrcPacketBuffer _missingPacketBuffer;
3364     std::map<uint32_t, ChannelSsrcInfo> _channelBySsrc;
3365     std::map<uint32_t, double> _volumeBySsrc;
3366     std::map<ChannelId, std::unique_ptr<IncomingAudioChannel>> _incomingAudioChannels;
3367     std::map<VideoChannelId, std::unique_ptr<IncomingVideoChannel>> _incomingVideoChannels;
3368 
3369     std::map<VideoChannelId, std::vector<std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>>>> _pendingVideoSinks;
3370     std::vector<VideoChannelDescription> _pendingRequestedVideo;
3371 
3372     std::unique_ptr<IncomingVideoChannel> _serverBandwidthProbingVideoSsrc;
3373 
3374     absl::optional<GroupJoinVideoInformation> _sharedVideoInformation;
3375 
3376     std::vector<float> _externalAudioSamples;
3377     webrtc::Mutex _externalAudioSamplesMutex;
3378     std::shared_ptr<ExternalAudioRecorder> _externalAudioRecorder;
3379 
3380     bool _isRtcConnected = false;
3381     bool _isBroadcastConnected = false;
3382     absl::optional<int64_t> _broadcastEnabledUntilRtcIsConnectedAtTimestamp;
3383     bool _isDataChannelOpen = false;
3384     GroupNetworkState _effectiveNetworkState;
3385 
3386     std::shared_ptr<StreamingMediaContext> _streamingContext;
3387 
3388     rtc::scoped_refptr<webrtc::PendingTaskSafetyFlag> _workerThreadSafery;
3389     rtc::scoped_refptr<webrtc::PendingTaskSafetyFlag> _networkThreadSafery;
3390 };
3391 
GroupInstanceCustomImpl(GroupInstanceDescriptor && descriptor)3392 GroupInstanceCustomImpl::GroupInstanceCustomImpl(GroupInstanceDescriptor &&descriptor) {
3393     if (descriptor.config.need_log) {
3394       _logSink = std::make_unique<LogSinkImpl>(descriptor.config.logPath);
3395       rtc::LogMessage::SetLogToStderr(true);
3396     } else {
3397         rtc::LogMessage::SetLogToStderr(false);
3398     }
3399     rtc::LogMessage::LogToDebug(rtc::LS_INFO);
3400     if (_logSink) {
3401         rtc::LogMessage::AddLogToStream(_logSink.get(), rtc::LS_INFO);
3402     }
3403 
3404     _threads = descriptor.threads;
3405     _internal.reset(new ThreadLocalObject<GroupInstanceCustomInternal>(_threads->getMediaThread(), [descriptor = std::move(descriptor), threads = _threads]() mutable {
3406         return new GroupInstanceCustomInternal(std::move(descriptor), threads);
3407     }));
3408     _internal->perform(RTC_FROM_HERE, [](GroupInstanceCustomInternal *internal) {
3409         internal->start();
3410     });
3411 }
3412 
~GroupInstanceCustomImpl()3413 GroupInstanceCustomImpl::~GroupInstanceCustomImpl() {
3414     if (_logSink) {
3415         rtc::LogMessage::RemoveLogToStream(_logSink.get());
3416     }
3417     _internal.reset();
3418 
3419     // Wait until _internal is destroyed
3420     _threads->getMediaThread()->Invoke<void>(RTC_FROM_HERE, [] {});
3421 }
3422 
stop()3423 void GroupInstanceCustomImpl::stop() {
3424     _internal->perform(RTC_FROM_HERE, [](GroupInstanceCustomInternal *internal) {
3425         internal->stop();
3426     });
3427 }
3428 
setConnectionMode(GroupConnectionMode connectionMode,bool keepBroadcastIfWasEnabled)3429 void GroupInstanceCustomImpl::setConnectionMode(GroupConnectionMode connectionMode, bool keepBroadcastIfWasEnabled) {
3430     _internal->perform(RTC_FROM_HERE, [connectionMode, keepBroadcastIfWasEnabled](GroupInstanceCustomInternal *internal) {
3431         internal->setConnectionMode(connectionMode, keepBroadcastIfWasEnabled);
3432     });
3433 }
3434 
emitJoinPayload(std::function<void (GroupJoinPayload const &)> completion)3435 void GroupInstanceCustomImpl::emitJoinPayload(std::function<void(GroupJoinPayload const &)> completion) {
3436     _internal->perform(RTC_FROM_HERE, [completion](GroupInstanceCustomInternal *internal) {
3437         internal->emitJoinPayload(completion);
3438     });
3439 }
3440 
setJoinResponsePayload(std::string const & payload)3441 void GroupInstanceCustomImpl::setJoinResponsePayload(std::string const &payload) {
3442     _internal->perform(RTC_FROM_HERE, [payload](GroupInstanceCustomInternal *internal) {
3443         internal->setJoinResponsePayload(payload);
3444     });
3445 }
3446 
removeSsrcs(std::vector<uint32_t> ssrcs)3447 void GroupInstanceCustomImpl::removeSsrcs(std::vector<uint32_t> ssrcs) {
3448     _internal->perform(RTC_FROM_HERE, [ssrcs = std::move(ssrcs)](GroupInstanceCustomInternal *internal) mutable {
3449         internal->removeSsrcs(ssrcs);
3450     });
3451 }
3452 
removeIncomingVideoSource(uint32_t ssrc)3453 void GroupInstanceCustomImpl::removeIncomingVideoSource(uint32_t ssrc) {
3454     _internal->perform(RTC_FROM_HERE, [ssrc](GroupInstanceCustomInternal *internal) mutable {
3455         internal->removeIncomingVideoSource(ssrc);
3456     });
3457 }
3458 
setIsMuted(bool isMuted)3459 void GroupInstanceCustomImpl::setIsMuted(bool isMuted) {
3460     _internal->perform(RTC_FROM_HERE, [isMuted](GroupInstanceCustomInternal *internal) {
3461         internal->setIsMuted(isMuted);
3462     });
3463 }
3464 
setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled)3465 void GroupInstanceCustomImpl::setIsNoiseSuppressionEnabled(bool isNoiseSuppressionEnabled) {
3466     _internal->perform(RTC_FROM_HERE, [isNoiseSuppressionEnabled](GroupInstanceCustomInternal *internal) {
3467         internal->setIsNoiseSuppressionEnabled(isNoiseSuppressionEnabled);
3468     });
3469 }
3470 
setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture)3471 void GroupInstanceCustomImpl::setVideoCapture(std::shared_ptr<VideoCaptureInterface> videoCapture) {
3472     _internal->perform(RTC_FROM_HERE, [videoCapture](GroupInstanceCustomInternal *internal) {
3473         internal->setVideoCapture(videoCapture, false);
3474     });
3475 }
3476 
setVideoSource(std::function<webrtc::VideoTrackSourceInterface * ()> getVideoSource)3477 void GroupInstanceCustomImpl::setVideoSource(std::function<webrtc::VideoTrackSourceInterface*()> getVideoSource) {
3478   _internal->perform(RTC_FROM_HERE, [getVideoSource](GroupInstanceCustomInternal *internal) {
3479     internal->setVideoSource(getVideoSource, false);
3480   });
3481 }
3482 
setAudioOutputDevice(std::string id)3483 void GroupInstanceCustomImpl::setAudioOutputDevice(std::string id) {
3484     _internal->perform(RTC_FROM_HERE, [id](GroupInstanceCustomInternal *internal) {
3485         internal->setAudioOutputDevice(id);
3486     });
3487 }
3488 
setAudioInputDevice(std::string id)3489 void GroupInstanceCustomImpl::setAudioInputDevice(std::string id) {
3490     _internal->perform(RTC_FROM_HERE, [id](GroupInstanceCustomInternal *internal) {
3491         internal->setAudioInputDevice(id);
3492     });
3493 }
3494 
addExternalAudioSamples(std::vector<uint8_t> && samples)3495 void GroupInstanceCustomImpl::addExternalAudioSamples(std::vector<uint8_t> &&samples) {
3496     _internal->perform(RTC_FROM_HERE, [samples = std::move(samples)](GroupInstanceCustomInternal *internal) mutable {
3497         internal->addExternalAudioSamples(std::move(samples));
3498     });
3499 }
3500 
addIncomingVideoOutput(std::string const & endpointId,std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink)3501 void GroupInstanceCustomImpl::addIncomingVideoOutput(std::string const &endpointId, std::weak_ptr<rtc::VideoSinkInterface<webrtc::VideoFrame>> sink) {
3502     _internal->perform(RTC_FROM_HERE, [endpointId, sink](GroupInstanceCustomInternal *internal) mutable {
3503         internal->addIncomingVideoOutput(endpointId, sink);
3504     });
3505 }
3506 
setVolume(uint32_t ssrc,double volume)3507 void GroupInstanceCustomImpl::setVolume(uint32_t ssrc, double volume) {
3508     _internal->perform(RTC_FROM_HERE, [ssrc, volume](GroupInstanceCustomInternal *internal) {
3509         internal->setVolume(ssrc, volume);
3510     });
3511 }
3512 
setRequestedVideoChannels(std::vector<VideoChannelDescription> && requestedVideoChannels)3513 void GroupInstanceCustomImpl::setRequestedVideoChannels(std::vector<VideoChannelDescription> &&requestedVideoChannels) {
3514     _internal->perform(RTC_FROM_HERE, [requestedVideoChannels = std::move(requestedVideoChannels)](GroupInstanceCustomInternal *internal) mutable {
3515         internal->setRequestedVideoChannels(std::move(requestedVideoChannels));
3516     });
3517 }
3518 
getStats(std::function<void (GroupInstanceStats)> completion)3519 void GroupInstanceCustomImpl::getStats(std::function<void(GroupInstanceStats)> completion) {
3520     _internal->perform(RTC_FROM_HERE, [completion = std::move(completion)](GroupInstanceCustomInternal *internal) mutable {
3521         internal->getStats(completion);
3522     });
3523 }
3524 
getAudioDevices(AudioDevice::Type type)3525 std::vector<GroupInstanceInterface::AudioDevice> GroupInstanceInterface::getAudioDevices(AudioDevice::Type type) {
3526   auto result = std::vector<AudioDevice>();
3527 #ifdef WEBRTC_LINUX //Not needed for ios, and some crl::sync stuff is needed for windows
3528   const auto resolve = [&] {
3529     const auto queueFactory = webrtc::CreateDefaultTaskQueueFactory();
3530     const auto info = webrtc::AudioDeviceModule::Create(
3531         webrtc::AudioDeviceModule::kPlatformDefaultAudio,
3532         queueFactory.get());
3533     if (!info || info->Init() < 0) {
3534       return;
3535     }
3536     const auto count = type == AudioDevice::Type::Input ? info->RecordingDevices() : info->PlayoutDevices();
3537     if (count <= 0) {
3538       return;
3539     }
3540     for (auto i = int16_t(); i != count; ++i) {
3541       char name[webrtc::kAdmMaxDeviceNameSize + 1] = { 0 };
3542       char id[webrtc::kAdmMaxGuidSize + 1] = { 0 };
3543       if (type == AudioDevice::Type::Input) {
3544         info->RecordingDeviceName(i, name, id);
3545       } else {
3546         info->PlayoutDeviceName(i, name, id);
3547       }
3548       result.push_back({ id, name });
3549     }
3550   };
3551   resolve();
3552 #endif
3553   return result;
3554 }
3555 
3556 } // namespace tgcalls
3557