1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "MediaEngineRemoteVideoSource.h"
7
8 #include "CamerasChild.h"
9 #include "Layers.h"
10 #include "MediaManager.h"
11 #include "MediaTrackConstraints.h"
12 #include "mozilla/ErrorNames.h"
13 #include "mozilla/gfx/Point.h"
14 #include "mozilla/RefPtr.h"
15 #include "Tracing.h"
16 #include "VideoFrameUtils.h"
17 #include "VideoUtils.h"
18 #include "webrtc/common_video/include/video_frame_buffer.h"
19 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
20
21 namespace mozilla {
22
23 extern LazyLogModule gMediaManagerLog;
24 #define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
25 #define LOG_FRAME(...) \
26 MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
27
28 using dom::ConstrainLongRange;
29 using dom::MediaSourceEnum;
30 using dom::MediaTrackConstraints;
31 using dom::MediaTrackConstraintSet;
32 using dom::MediaTrackSettings;
33 using dom::VideoFacingModeEnum;
34
GetFacingMode(const nsString & aDeviceName)35 static Maybe<VideoFacingModeEnum> GetFacingMode(const nsString& aDeviceName) {
36 // Set facing mode based on device name.
37 #if defined(ANDROID)
38 // Names are generated. Example: "Camera 0, Facing back, Orientation 90"
39 //
40 // See media/webrtc/trunk/webrtc/modules/video_capture/android/java/src/org/
41 // webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java
42
43 if (aDeviceName.Find(u"Facing back"_ns) != kNotFound) {
44 return Some(VideoFacingModeEnum::Environment);
45 }
46 if (aDeviceName.Find(u"Facing front"_ns) != kNotFound) {
47 return Some(VideoFacingModeEnum::User);
48 }
49 #endif // ANDROID
50 #ifdef XP_MACOSX
51 // Kludge to test user-facing cameras on OSX.
52 if (aDeviceName.Find(u"Face"_ns) != -1) {
53 return Some(VideoFacingModeEnum::User);
54 }
55 #endif
56 #ifdef XP_WIN
57 // The cameras' name of Surface book are "Microsoft Camera Front" and
58 // "Microsoft Camera Rear" respectively.
59
60 if (aDeviceName.Find(u"Front"_ns) != kNotFound) {
61 return Some(VideoFacingModeEnum::User);
62 }
63 if (aDeviceName.Find(u"Rear"_ns) != kNotFound) {
64 return Some(VideoFacingModeEnum::Environment);
65 }
66 #endif // WINDOWS
67
68 return Nothing();
69 }
70
MediaEngineRemoteVideoSource(int aIndex,camera::CaptureEngine aCapEngine,bool aScary)71 MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
72 int aIndex, camera::CaptureEngine aCapEngine, bool aScary)
73 : mCaptureIndex(aIndex),
74 mCapEngine(aCapEngine),
75 mScary(aScary),
76 mMutex("MediaEngineRemoteVideoSource::mMutex"),
77 mRescalingBufferPool(/* zero_initialize */ false,
78 /* max_number_of_buffers */ 1),
79 mSettingsUpdatedByFrame(MakeAndAddRef<media::Refcountable<AtomicBool>>()),
80 mSettings(MakeAndAddRef<media::Refcountable<MediaTrackSettings>>()),
81 mFirstFramePromise(mFirstFramePromiseHolder.Ensure(__func__)) {
82 mSettings->mWidth.Construct(0);
83 mSettings->mHeight.Construct(0);
84 mSettings->mFrameRate.Construct(0);
85 Init();
86 }
87
~MediaEngineRemoteVideoSource()88 MediaEngineRemoteVideoSource::~MediaEngineRemoteVideoSource() {
89 mFirstFramePromiseHolder.RejectIfExists(NS_ERROR_ABORT, __func__);
90 }
91
GetMediaSource() const92 dom::MediaSourceEnum MediaEngineRemoteVideoSource::GetMediaSource() const {
93 switch (mCapEngine) {
94 case camera::BrowserEngine:
95 return MediaSourceEnum::Browser;
96 case camera::CameraEngine:
97 return MediaSourceEnum::Camera;
98 case camera::ScreenEngine:
99 return MediaSourceEnum::Screen;
100 case camera::WinEngine:
101 return MediaSourceEnum::Window;
102 default:
103 MOZ_CRASH();
104 }
105 }
106
Init()107 void MediaEngineRemoteVideoSource::Init() {
108 LOG("%s", __PRETTY_FUNCTION__);
109 AssertIsOnOwningThread();
110
111 char deviceName[kMaxDeviceNameLength];
112 char uniqueId[kMaxUniqueIdLength];
113 if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
114 mCapEngine, mCaptureIndex, deviceName,
115 kMaxDeviceNameLength, uniqueId,
116 kMaxUniqueIdLength, nullptr)) {
117 LOG("Error initializing RemoteVideoSource (GetCaptureDevice)");
118 return;
119 }
120
121 SetName(NS_ConvertUTF8toUTF16(deviceName));
122 mUniqueId = uniqueId;
123 }
124
SetName(nsString aName)125 void MediaEngineRemoteVideoSource::SetName(nsString aName) {
126 LOG("%s", __PRETTY_FUNCTION__);
127 AssertIsOnOwningThread();
128
129 mDeviceName = std::move(aName);
130
131 Maybe<VideoFacingModeEnum> facingMode;
132 if (GetMediaSource() == MediaSourceEnum::Camera) {
133 // Only cameras can have a facing mode.
134 facingMode = GetFacingMode(mDeviceName);
135 }
136
137 mFacingMode = facingMode.map([](const auto& aFM) {
138 return NS_ConvertASCIItoUTF16(
139 dom::VideoFacingModeEnumValues::GetString(aFM));
140 });
141 NS_DispatchToMainThread(NS_NewRunnableFunction(
142 "MediaEngineRemoteVideoSource::SetName (facingMode updater)",
143 [settings = mSettings, mode = mFacingMode]() {
144 if (mode.isNothing()) {
145 settings->mFacingMode.Reset();
146 return;
147 }
148 settings->mFacingMode.Construct(*mode);
149 }));
150 }
151
GetName() const152 nsString MediaEngineRemoteVideoSource::GetName() const {
153 AssertIsOnOwningThread();
154
155 return mDeviceName;
156 }
157
GetUUID() const158 nsCString MediaEngineRemoteVideoSource::GetUUID() const {
159 AssertIsOnOwningThread();
160
161 return mUniqueId;
162 }
163
GetGroupId() const164 nsString MediaEngineRemoteVideoSource::GetGroupId() const {
165 AssertIsOnOwningThread();
166
167 // The remote video backend doesn't implement group id. We return the device
168 // name and higher layers will correlate this with the name of audio devices.
169 return mDeviceName;
170 }
171
Allocate(const MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,uint64_t aWindowID,const char ** aOutBadConstraint)172 nsresult MediaEngineRemoteVideoSource::Allocate(
173 const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
174 uint64_t aWindowID, const char** aOutBadConstraint) {
175 LOG("%s", __PRETTY_FUNCTION__);
176 AssertIsOnOwningThread();
177
178 MOZ_ASSERT(mState == kReleased);
179
180 NormalizedConstraints constraints(aConstraints);
181 webrtc::CaptureCapability newCapability;
182 LOG("ChooseCapability(kFitness) for mCapability (Allocate) ++");
183 if (!ChooseCapability(constraints, aPrefs, newCapability, kFitness)) {
184 *aOutBadConstraint =
185 MediaConstraintsHelper::FindBadConstraint(constraints, this);
186 return NS_ERROR_FAILURE;
187 }
188 LOG("ChooseCapability(kFitness) for mCapability (Allocate) --");
189
190 if (camera::GetChildAndCall(&camera::CamerasChild::AllocateCaptureDevice,
191 mCapEngine, mUniqueId.get(), kMaxUniqueIdLength,
192 mCaptureIndex, aWindowID)) {
193 return NS_ERROR_FAILURE;
194 }
195
196 {
197 MutexAutoLock lock(mMutex);
198 mState = kAllocated;
199 mCapability = newCapability;
200 }
201
202 LOG("Video device %d allocated", mCaptureIndex);
203 return NS_OK;
204 }
205
Deallocate()206 nsresult MediaEngineRemoteVideoSource::Deallocate() {
207 LOG("%s", __PRETTY_FUNCTION__);
208 AssertIsOnOwningThread();
209
210 MOZ_ASSERT(mState == kStopped || mState == kAllocated);
211
212 if (mTrack) {
213 mTrack->End();
214 }
215
216 {
217 MutexAutoLock lock(mMutex);
218
219 mTrack = nullptr;
220 mPrincipal = PRINCIPAL_HANDLE_NONE;
221 mState = kReleased;
222 }
223
224 // Stop() has stopped capture synchronously on the media thread before we get
225 // here, so there are no longer any callbacks on an IPC thread accessing
226 // mImageContainer or mRescalingBufferPool.
227 mImageContainer = nullptr;
228 mRescalingBufferPool.Release();
229
230 LOG("Video device %d deallocated", mCaptureIndex);
231
232 if (camera::GetChildAndCall(&camera::CamerasChild::ReleaseCaptureDevice,
233 mCapEngine, mCaptureIndex)) {
234 MOZ_ASSERT_UNREACHABLE("Couldn't release allocated device");
235 }
236 return NS_OK;
237 }
238
SetTrack(const RefPtr<MediaTrack> & aTrack,const PrincipalHandle & aPrincipal)239 void MediaEngineRemoteVideoSource::SetTrack(const RefPtr<MediaTrack>& aTrack,
240 const PrincipalHandle& aPrincipal) {
241 LOG("%s", __PRETTY_FUNCTION__);
242 AssertIsOnOwningThread();
243
244 MOZ_ASSERT(mState == kAllocated);
245 MOZ_ASSERT(!mTrack);
246 MOZ_ASSERT(aTrack);
247 MOZ_ASSERT(aTrack->AsSourceTrack());
248
249 if (!mImageContainer) {
250 mImageContainer = layers::LayerManager::CreateImageContainer(
251 layers::ImageContainer::ASYNCHRONOUS);
252 }
253
254 {
255 MutexAutoLock lock(mMutex);
256 mTrack = aTrack->AsSourceTrack();
257 mPrincipal = aPrincipal;
258 }
259 }
260
Start()261 nsresult MediaEngineRemoteVideoSource::Start() {
262 LOG("%s", __PRETTY_FUNCTION__);
263 AssertIsOnOwningThread();
264
265 MOZ_ASSERT(mState == kAllocated || mState == kStopped);
266 MOZ_ASSERT(mTrack);
267
268 {
269 MutexAutoLock lock(mMutex);
270 mState = kStarted;
271 }
272
273 mSettingsUpdatedByFrame->mValue = false;
274
275 if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture, mCapEngine,
276 mCaptureIndex, mCapability, this)) {
277 LOG("StartCapture failed");
278 MutexAutoLock lock(mMutex);
279 mState = kStopped;
280 return NS_ERROR_FAILURE;
281 }
282
283 NS_DispatchToMainThread(NS_NewRunnableFunction(
284 "MediaEngineRemoteVideoSource::SetLastCapability",
285 [settings = mSettings, updated = mSettingsUpdatedByFrame,
286 capEngine = mCapEngine, cap = mCapability]() mutable {
287 switch (capEngine) {
288 case camera::ScreenEngine:
289 case camera::WinEngine:
290 // Undo the hack where ideal and max constraints are crammed
291 // together in mCapability for consumption by low-level code. We
292 // don't actually know the real resolution yet, so report min(ideal,
293 // max) for now.
294 // TODO: This can be removed in bug 1453269.
295 cap.width = std::min(cap.width >> 16, cap.width & 0xffff);
296 cap.height = std::min(cap.height >> 16, cap.height & 0xffff);
297 break;
298 default:
299 break;
300 }
301
302 if (!updated->mValue) {
303 settings->mWidth.Value() = cap.width;
304 settings->mHeight.Value() = cap.height;
305 }
306 settings->mFrameRate.Value() = cap.maxFPS;
307 }));
308
309 return NS_OK;
310 }
311
FocusOnSelectedSource()312 nsresult MediaEngineRemoteVideoSource::FocusOnSelectedSource() {
313 LOG("%s", __PRETTY_FUNCTION__);
314 AssertIsOnOwningThread();
315
316 int result;
317 result = camera::GetChildAndCall(&camera::CamerasChild::FocusOnSelectedSource,
318 mCapEngine, mCaptureIndex);
319 return result == 0 ? NS_OK : NS_ERROR_FAILURE;
320 }
321
Stop()322 nsresult MediaEngineRemoteVideoSource::Stop() {
323 LOG("%s", __PRETTY_FUNCTION__);
324 AssertIsOnOwningThread();
325
326 if (mState == kStopped || mState == kAllocated) {
327 return NS_OK;
328 }
329
330 MOZ_ASSERT(mState == kStarted);
331
332 if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, mCapEngine,
333 mCaptureIndex)) {
334 MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed");
335 return NS_ERROR_FAILURE;
336 }
337
338 {
339 MutexAutoLock lock(mMutex);
340 mState = kStopped;
341 }
342
343 return NS_OK;
344 }
345
Reconfigure(const MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const char ** aOutBadConstraint)346 nsresult MediaEngineRemoteVideoSource::Reconfigure(
347 const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
348 const char** aOutBadConstraint) {
349 LOG("%s", __PRETTY_FUNCTION__);
350 AssertIsOnOwningThread();
351
352 NormalizedConstraints constraints(aConstraints);
353 webrtc::CaptureCapability newCapability;
354 LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) ++");
355 if (!ChooseCapability(constraints, aPrefs, newCapability, kFitness)) {
356 *aOutBadConstraint =
357 MediaConstraintsHelper::FindBadConstraint(constraints, this);
358 return NS_ERROR_INVALID_ARG;
359 }
360 LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) --");
361
362 if (mCapability == newCapability) {
363 return NS_OK;
364 }
365
366 bool started = mState == kStarted;
367 if (started) {
368 nsresult rv = Stop();
369 if (NS_WARN_IF(NS_FAILED(rv))) {
370 nsAutoCString name;
371 GetErrorName(rv, name);
372 LOG("Video source %p for video device %d Reconfigure() failed "
373 "unexpectedly in Stop(). rv=%s",
374 this, mCaptureIndex, name.Data());
375 return NS_ERROR_UNEXPECTED;
376 }
377 }
378
379 {
380 MutexAutoLock lock(mMutex);
381 // Start() applies mCapability on the device.
382 mCapability = newCapability;
383 }
384
385 if (started) {
386 nsresult rv = Start();
387 if (NS_WARN_IF(NS_FAILED(rv))) {
388 nsAutoCString name;
389 GetErrorName(rv, name);
390 LOG("Video source %p for video device %d Reconfigure() failed "
391 "unexpectedly in Start(). rv=%s",
392 this, mCaptureIndex, name.Data());
393 return NS_ERROR_UNEXPECTED;
394 }
395 }
396
397 return NS_OK;
398 }
399
NumCapabilities() const400 size_t MediaEngineRemoteVideoSource::NumCapabilities() const {
401 AssertIsOnOwningThread();
402
403 if (!mCapabilities.IsEmpty()) {
404 return mCapabilities.Length();
405 }
406
407 int num = camera::GetChildAndCall(&camera::CamerasChild::NumberOfCapabilities,
408 mCapEngine, mUniqueId.get());
409 if (num > 0) {
410 mCapabilities.SetLength(num);
411 } else {
412 // The default for devices that don't return discrete capabilities: treat
413 // them as supporting all capabilities orthogonally. E.g. screensharing.
414 // CaptureCapability defaults key values to 0, which means accept any value.
415 mCapabilities.AppendElement(MakeUnique<webrtc::CaptureCapability>());
416 mCapabilitiesAreHardcoded = true;
417 }
418
419 return mCapabilities.Length();
420 }
421
GetCapability(size_t aIndex) const422 webrtc::CaptureCapability& MediaEngineRemoteVideoSource::GetCapability(
423 size_t aIndex) const {
424 AssertIsOnOwningThread();
425 MOZ_RELEASE_ASSERT(aIndex < mCapabilities.Length());
426 if (!mCapabilities[aIndex]) {
427 mCapabilities[aIndex] = MakeUnique<webrtc::CaptureCapability>();
428 camera::GetChildAndCall(&camera::CamerasChild::GetCaptureCapability,
429 mCapEngine, mUniqueId.get(), aIndex,
430 *mCapabilities[aIndex]);
431 }
432 return *mCapabilities[aIndex];
433 }
434
DeliverFrame(uint8_t * aBuffer,const camera::VideoFrameProperties & aProps)435 int MediaEngineRemoteVideoSource::DeliverFrame(
436 uint8_t* aBuffer, const camera::VideoFrameProperties& aProps) {
437 // Cameras IPC thread - take great care with accessing members!
438
439 Maybe<int32_t> req_max_width;
440 Maybe<int32_t> req_max_height;
441 Maybe<int32_t> req_ideal_width;
442 Maybe<int32_t> req_ideal_height;
443 {
444 MutexAutoLock lock(mMutex);
445 MOZ_ASSERT(mState == kStarted);
446 // TODO: These can be removed in bug 1453269.
447 const int32_t max_width = mCapability.width & 0xffff;
448 const int32_t max_height = mCapability.height & 0xffff;
449 const int32_t ideal_width = (mCapability.width >> 16) & 0xffff;
450 const int32_t ideal_height = (mCapability.height >> 16) & 0xffff;
451
452 req_max_width = max_width ? Some(max_width) : Nothing();
453 req_max_height = max_height ? Some(max_height) : Nothing();
454 req_ideal_width = ideal_width ? Some(ideal_width) : Nothing();
455 req_ideal_height = ideal_height ? Some(ideal_height) : Nothing();
456 }
457
458 // This is only used in the case of screen sharing, see bug 1453269.
459
460 if (aProps.rotation() == 90 || aProps.rotation() == 270) {
461 // This frame is rotated, so what was negotiated as width is now height,
462 // and vice versa.
463 std::swap(req_max_width, req_max_height);
464 std::swap(req_ideal_width, req_ideal_height);
465 }
466
467 int32_t dst_max_width =
468 std::min(aProps.width(), req_max_width.valueOr(aProps.width()));
469 int32_t dst_max_height =
470 std::min(aProps.height(), req_max_height.valueOr(aProps.height()));
471 // This logic works for both camera and screen sharing case.
472 // for camera case, req_ideal_width and req_ideal_height are absent.
473 int32_t dst_width = req_ideal_width.valueOr(aProps.width());
474 int32_t dst_height = req_ideal_height.valueOr(aProps.height());
475
476 if (!req_ideal_width && req_ideal_height) {
477 dst_width = *req_ideal_height * aProps.width() / aProps.height();
478 } else if (!req_ideal_height && req_ideal_width) {
479 dst_height = *req_ideal_width * aProps.height() / aProps.width();
480 }
481 dst_width = std::min(dst_width, dst_max_width);
482 dst_height = std::min(dst_height, dst_max_height);
483
484 // Apply scaling for screen sharing, see bug 1453269.
485 switch (mCapEngine) {
486 case camera::ScreenEngine:
487 case camera::WinEngine: {
488 // scale to average of portrait and landscape
489 float scale_width = (float)dst_width / (float)aProps.width();
490 float scale_height = (float)dst_height / (float)aProps.height();
491 float scale = (scale_width + scale_height) / 2;
492 // If both req_ideal_width & req_ideal_height are absent, scale is 1, but
493 // if one is present and the other not, scale precisely to the one present
494 if (!req_ideal_width) {
495 scale = scale_height;
496 } else if (!req_ideal_height) {
497 scale = scale_width;
498 }
499 dst_width = int32_t(scale * (float)aProps.width());
500 dst_height = int32_t(scale * (float)aProps.height());
501
502 // if scaled rectangle exceeds max rectangle, scale to minimum of portrait
503 // and landscape
504 if (dst_width > dst_max_width || dst_height > dst_max_height) {
505 scale_width = (float)dst_max_width / (float)dst_width;
506 scale_height = (float)dst_max_height / (float)dst_height;
507 scale = std::min(scale_width, scale_height);
508 dst_width = int32_t(scale * dst_width);
509 dst_height = int32_t(scale * dst_height);
510 }
511 break;
512 }
513 default: {
514 break;
515 }
516 }
517
518 // Ensure width and height are at least two. Smaller frames can lead to
519 // problems with scaling and video encoding.
520 dst_width = std::max(2, dst_width);
521 dst_height = std::max(2, dst_height);
522
523 rtc::Callback0<void> callback_unused;
524 rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
525 new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
526 aProps.width(), aProps.height(), aBuffer, aProps.yStride(),
527 aBuffer + aProps.yAllocatedSize(), aProps.uStride(),
528 aBuffer + aProps.yAllocatedSize() + aProps.uAllocatedSize(),
529 aProps.vStride(), callback_unused);
530
531 if ((dst_width != aProps.width() || dst_height != aProps.height()) &&
532 dst_width <= aProps.width() && dst_height <= aProps.height()) {
533 // Destination resolution is smaller than source buffer. We'll rescale.
534 rtc::scoped_refptr<webrtc::I420Buffer> scaledBuffer =
535 mRescalingBufferPool.CreateBuffer(dst_width, dst_height);
536 if (!scaledBuffer) {
537 MOZ_ASSERT_UNREACHABLE(
538 "We might fail to allocate a buffer, but with this "
539 "being a recycling pool that shouldn't happen");
540 return 0;
541 }
542 scaledBuffer->CropAndScaleFrom(*buffer);
543 buffer = scaledBuffer;
544 }
545
546 layers::PlanarYCbCrData data;
547 data.mYChannel = const_cast<uint8_t*>(buffer->DataY());
548 data.mYSize = gfx::IntSize(buffer->width(), buffer->height());
549 data.mYStride = buffer->StrideY();
550 MOZ_ASSERT(buffer->StrideU() == buffer->StrideV());
551 data.mCbCrStride = buffer->StrideU();
552 data.mCbChannel = const_cast<uint8_t*>(buffer->DataU());
553 data.mCrChannel = const_cast<uint8_t*>(buffer->DataV());
554 data.mCbCrSize =
555 gfx::IntSize((buffer->width() + 1) / 2, (buffer->height() + 1) / 2);
556 data.mPicX = 0;
557 data.mPicY = 0;
558 data.mPicSize = gfx::IntSize(buffer->width(), buffer->height());
559 data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
560
561 RefPtr<layers::PlanarYCbCrImage> image =
562 mImageContainer->CreatePlanarYCbCrImage();
563 if (!image->CopyData(data)) {
564 MOZ_ASSERT_UNREACHABLE(
565 "We might fail to allocate a buffer, but with this "
566 "being a recycling container that shouldn't happen");
567 return 0;
568 }
569
570 #ifdef DEBUG
571 static uint32_t frame_num = 0;
572 LOG_FRAME(
573 "frame %d (%dx%d)->(%dx%d); rotation %d, timeStamp %u, ntpTimeMs %" PRIu64
574 ", renderTimeMs %" PRIu64,
575 frame_num++, aProps.width(), aProps.height(), dst_width, dst_height,
576 aProps.rotation(), aProps.timeStamp(), aProps.ntpTimeMs(),
577 aProps.renderTimeMs());
578 #endif
579
580 if (mImageSize.width != dst_width || mImageSize.height != dst_height) {
581 NS_DispatchToMainThread(NS_NewRunnableFunction(
582 "MediaEngineRemoteVideoSource::FrameSizeChange",
583 [settings = mSettings, updated = mSettingsUpdatedByFrame,
584 holder = std::move(mFirstFramePromiseHolder), dst_width,
585 dst_height]() mutable {
586 settings->mWidth.Value() = dst_width;
587 settings->mHeight.Value() = dst_height;
588 updated->mValue = true;
589 // Since mImageSize was initialized to (0,0), we end up here on the
590 // arrival of the first frame. We resolve the promise representing
591 // arrival of first frame, after correct settings values have been
592 // made available (Resolve() is idempotent if already resolved).
593 holder.ResolveIfExists(true, __func__);
594 }));
595 }
596
597 {
598 MutexAutoLock lock(mMutex);
599 MOZ_ASSERT(mState == kStarted);
600 VideoSegment segment;
601 mImageSize = image->GetSize();
602 segment.AppendFrame(image.forget(), mImageSize, mPrincipal);
603 mTrack->AppendData(&segment);
604 }
605
606 return 0;
607 }
608
GetDistance(const webrtc::CaptureCapability & aCandidate,const NormalizedConstraintSet & aConstraints,const DistanceCalculation aCalculate) const609 uint32_t MediaEngineRemoteVideoSource::GetDistance(
610 const webrtc::CaptureCapability& aCandidate,
611 const NormalizedConstraintSet& aConstraints,
612 const DistanceCalculation aCalculate) const {
613 if (aCalculate == kFeasibility) {
614 return GetFeasibilityDistance(aCandidate, aConstraints);
615 }
616 return GetFitnessDistance(aCandidate, aConstraints);
617 }
618
GetFitnessDistance(const webrtc::CaptureCapability & aCandidate,const NormalizedConstraintSet & aConstraints) const619 uint32_t MediaEngineRemoteVideoSource::GetFitnessDistance(
620 const webrtc::CaptureCapability& aCandidate,
621 const NormalizedConstraintSet& aConstraints) const {
622 AssertIsOnOwningThread();
623
624 // Treat width|height|frameRate == 0 on capability as "can do any".
625 // This allows for orthogonal capabilities that are not in discrete steps.
626
627 typedef MediaConstraintsHelper H;
628 uint64_t distance =
629 uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
630 uint64_t(aCandidate.width ? H::FitnessDistance(int32_t(aCandidate.width),
631 aConstraints.mWidth)
632 : 0) +
633 uint64_t(aCandidate.height
634 ? H::FitnessDistance(int32_t(aCandidate.height),
635 aConstraints.mHeight)
636 : 0) +
637 uint64_t(aCandidate.maxFPS ? H::FitnessDistance(double(aCandidate.maxFPS),
638 aConstraints.mFrameRate)
639 : 0);
640 return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
641 }
642
GetFeasibilityDistance(const webrtc::CaptureCapability & aCandidate,const NormalizedConstraintSet & aConstraints) const643 uint32_t MediaEngineRemoteVideoSource::GetFeasibilityDistance(
644 const webrtc::CaptureCapability& aCandidate,
645 const NormalizedConstraintSet& aConstraints) const {
646 AssertIsOnOwningThread();
647
648 // Treat width|height|frameRate == 0 on capability as "can do any".
649 // This allows for orthogonal capabilities that are not in discrete steps.
650
651 typedef MediaConstraintsHelper H;
652 uint64_t distance =
653 uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
654 uint64_t(aCandidate.width
655 ? H::FeasibilityDistance(int32_t(aCandidate.width),
656 aConstraints.mWidth)
657 : 0) +
658 uint64_t(aCandidate.height
659 ? H::FeasibilityDistance(int32_t(aCandidate.height),
660 aConstraints.mHeight)
661 : 0) +
662 uint64_t(aCandidate.maxFPS
663 ? H::FeasibilityDistance(double(aCandidate.maxFPS),
664 aConstraints.mFrameRate)
665 : 0);
666 return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
667 }
668
669 // Find best capability by removing inferiors. May leave >1 of equal distance
670
671 /* static */
TrimLessFitCandidates(nsTArray<CapabilityCandidate> & aSet)672 void MediaEngineRemoteVideoSource::TrimLessFitCandidates(
673 nsTArray<CapabilityCandidate>& aSet) {
674 uint32_t best = UINT32_MAX;
675 for (auto& candidate : aSet) {
676 if (best > candidate.mDistance) {
677 best = candidate.mDistance;
678 }
679 }
680 aSet.RemoveElementsBy(
681 [best](const auto& set) { return set.mDistance > best; });
682 MOZ_ASSERT(aSet.Length());
683 }
684
GetBestFitnessDistance(const nsTArray<const NormalizedConstraintSet * > & aConstraintSets) const685 uint32_t MediaEngineRemoteVideoSource::GetBestFitnessDistance(
686 const nsTArray<const NormalizedConstraintSet*>& aConstraintSets) const {
687 AssertIsOnOwningThread();
688
689 size_t num = NumCapabilities();
690 nsTArray<CapabilityCandidate> candidateSet;
691 for (size_t i = 0; i < num; i++) {
692 candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
693 }
694
695 bool first = true;
696 for (const NormalizedConstraintSet* ns : aConstraintSets) {
697 for (size_t i = 0; i < candidateSet.Length();) {
698 auto& candidate = candidateSet[i];
699 uint32_t distance = GetFitnessDistance(candidate.mCapability, *ns);
700 if (distance == UINT32_MAX) {
701 candidateSet.RemoveElementAt(i);
702 } else {
703 ++i;
704 if (first) {
705 candidate.mDistance = distance;
706 }
707 }
708 }
709 first = false;
710 }
711 if (!candidateSet.Length()) {
712 return UINT32_MAX;
713 }
714 TrimLessFitCandidates(candidateSet);
715 return candidateSet[0].mDistance;
716 }
717
LogCapability(const char * aHeader,const webrtc::CaptureCapability & aCapability,uint32_t aDistance)718 static void LogCapability(const char* aHeader,
719 const webrtc::CaptureCapability& aCapability,
720 uint32_t aDistance) {
721 static const char* const codec[] = {"VP8", "VP9", "H264",
722 "I420", "RED", "ULPFEC",
723 "Generic codec", "Unknown codec"};
724
725 LOG("%s: %4u x %4u x %2u maxFps, %s. Distance = %" PRIu32, aHeader,
726 aCapability.width, aCapability.height, aCapability.maxFPS,
727 codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.videoType)),
728 uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
729 aDistance);
730 }
731
ChooseCapability(const NormalizedConstraints & aConstraints,const MediaEnginePrefs & aPrefs,webrtc::CaptureCapability & aCapability,const DistanceCalculation aCalculate)732 bool MediaEngineRemoteVideoSource::ChooseCapability(
733 const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
734 webrtc::CaptureCapability& aCapability,
735 const DistanceCalculation aCalculate) {
736 LOG("%s", __PRETTY_FUNCTION__);
737 AssertIsOnOwningThread();
738
739 if (MOZ_LOG_TEST(gMediaManagerLog, LogLevel::Debug)) {
740 LOG("ChooseCapability: prefs: %dx%d @%dfps", aPrefs.GetWidth(),
741 aPrefs.GetHeight(), aPrefs.mFPS);
742 MediaConstraintsHelper::LogConstraints(aConstraints);
743 if (!aConstraints.mAdvanced.empty()) {
744 LOG("Advanced array[%zu]:", aConstraints.mAdvanced.size());
745 for (auto& advanced : aConstraints.mAdvanced) {
746 MediaConstraintsHelper::LogConstraints(advanced);
747 }
748 }
749 }
750
751 switch (mCapEngine) {
752 case camera::ScreenEngine:
753 case camera::WinEngine: {
754 FlattenedConstraints c(aConstraints);
755 // The actual resolution to constrain around is not easy to find ahead of
756 // time (and may in fact change over time), so as a hack, we push ideal
757 // and max constraints down to desktop_capture_impl.cc and finish the
758 // algorithm there.
759 // TODO: This can be removed in bug 1453269.
760 aCapability.width =
761 (std::min(0xffff, c.mWidth.mIdeal.valueOr(0)) & 0xffff) << 16 |
762 (std::min(0xffff, c.mWidth.mMax) & 0xffff);
763 aCapability.height =
764 (std::min(0xffff, c.mHeight.mIdeal.valueOr(0)) & 0xffff) << 16 |
765 (std::min(0xffff, c.mHeight.mMax) & 0xffff);
766 aCapability.maxFPS =
767 c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
768 return true;
769 }
770 default:
771 break;
772 }
773
774 nsTArray<CapabilityCandidate> candidateSet;
775 size_t num = NumCapabilities();
776 for (size_t i = 0; i < num; i++) {
777 candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
778 }
779
780 if (mCapabilitiesAreHardcoded && mCapEngine == camera::CameraEngine) {
781 // We have a hardcoded capability, which means this camera didn't report
782 // discrete capabilities. It might still allow a ranged capability, so we
783 // add a couple of default candidates based on prefs and constraints.
784 // The chosen candidate will be propagated to StartCapture() which will fail
785 // for an invalid candidate.
786 MOZ_DIAGNOSTIC_ASSERT(mCapabilities.Length() == 1);
787 MOZ_DIAGNOSTIC_ASSERT(candidateSet.Length() == 1);
788 candidateSet.Clear();
789
790 FlattenedConstraints c(aConstraints);
791 // Reuse the code across both the low-definition (`false`) pref and
792 // the high-definition (`true`) pref.
793 // If there are constraints we try to satisfy them but we default to prefs.
794 // Note that since constraints are from content and can literally be
795 // anything we put (rather generous) caps on them.
796 for (bool isHd : {false, true}) {
797 webrtc::CaptureCapability cap;
798 int32_t prefWidth = aPrefs.GetWidth(isHd);
799 int32_t prefHeight = aPrefs.GetHeight(isHd);
800
801 cap.width = c.mWidth.Get(prefWidth);
802 cap.width = std::max(0, std::min(cap.width, 7680));
803
804 cap.height = c.mHeight.Get(prefHeight);
805 cap.height = std::max(0, std::min(cap.height, 4320));
806
807 cap.maxFPS = c.mFrameRate.Get(aPrefs.mFPS);
808 cap.maxFPS = std::max(0, std::min(cap.maxFPS, 480));
809
810 if (cap.width != prefWidth) {
811 // Width was affected by constraints.
812 // We'll adjust the height too so the aspect ratio is retained.
813 cap.height = cap.width * prefHeight / prefWidth;
814 } else if (cap.height != prefHeight) {
815 // Height was affected by constraints but not width.
816 // We'll adjust the width too so the aspect ratio is retained.
817 cap.width = cap.height * prefWidth / prefHeight;
818 }
819
820 if (candidateSet.Contains(cap, CapabilityComparator())) {
821 continue;
822 }
823 LogCapability("Hardcoded capability", cap, 0);
824 candidateSet.AppendElement(cap);
825 }
826 }
827
828 // First, filter capabilities by required constraints (min, max, exact).
829
830 for (size_t i = 0; i < candidateSet.Length();) {
831 auto& candidate = candidateSet[i];
832 candidate.mDistance =
833 GetDistance(candidate.mCapability, aConstraints, aCalculate);
834 LogCapability("Capability", candidate.mCapability, candidate.mDistance);
835 if (candidate.mDistance == UINT32_MAX) {
836 candidateSet.RemoveElementAt(i);
837 } else {
838 ++i;
839 }
840 }
841
842 if (candidateSet.IsEmpty()) {
843 LOG("failed to find capability match from %zu choices",
844 candidateSet.Length());
845 return false;
846 }
847
848 // Filter further with all advanced constraints (that don't overconstrain).
849
850 for (const auto& cs : aConstraints.mAdvanced) {
851 nsTArray<CapabilityCandidate> rejects;
852 for (size_t i = 0; i < candidateSet.Length();) {
853 if (GetDistance(candidateSet[i].mCapability, cs, aCalculate) ==
854 UINT32_MAX) {
855 rejects.AppendElement(candidateSet[i]);
856 candidateSet.RemoveElementAt(i);
857 } else {
858 ++i;
859 }
860 }
861 if (!candidateSet.Length()) {
862 candidateSet.AppendElements(std::move(rejects));
863 }
864 }
865 MOZ_ASSERT(
866 candidateSet.Length(),
867 "advanced constraints filtering step can't reduce candidates to zero");
868
869 // Remaining algorithm is up to the UA.
870
871 TrimLessFitCandidates(candidateSet);
872
873 // Any remaining multiples all have the same distance. A common case of this
874 // occurs when no ideal is specified. Lean toward defaults.
875 uint32_t sameDistance = candidateSet[0].mDistance;
876 {
877 MediaTrackConstraintSet prefs;
878 prefs.mWidth.Construct().SetAsLong() = aPrefs.GetWidth();
879 prefs.mHeight.Construct().SetAsLong() = aPrefs.GetHeight();
880 prefs.mFrameRate.Construct().SetAsDouble() = aPrefs.mFPS;
881 NormalizedConstraintSet normPrefs(prefs, false);
882
883 for (auto& candidate : candidateSet) {
884 candidate.mDistance =
885 GetDistance(candidate.mCapability, normPrefs, aCalculate);
886 }
887 TrimLessFitCandidates(candidateSet);
888 }
889
890 aCapability = candidateSet[0].mCapability;
891
892 LogCapability("Chosen capability", aCapability, sameDistance);
893 return true;
894 }
895
GetSettings(MediaTrackSettings & aOutSettings) const896 void MediaEngineRemoteVideoSource::GetSettings(
897 MediaTrackSettings& aOutSettings) const {
898 aOutSettings = *mSettings;
899 }
900
Refresh(int aIndex)901 void MediaEngineRemoteVideoSource::Refresh(int aIndex) {
902 LOG("%s", __PRETTY_FUNCTION__);
903 AssertIsOnOwningThread();
904
905 // NOTE: mCaptureIndex might have changed when allocated!
906 // Use aIndex to update information, but don't change mCaptureIndex!!
907 // Caller looked up this source by uniqueId, so it shouldn't change
908 char deviceName[kMaxDeviceNameLength];
909 char uniqueId[kMaxUniqueIdLength];
910
911 if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
912 mCapEngine, aIndex, deviceName,
913 sizeof(deviceName), uniqueId, sizeof(uniqueId),
914 nullptr)) {
915 return;
916 }
917
918 SetName(NS_ConvertUTF8toUTF16(deviceName));
919 MOZ_DIAGNOSTIC_ASSERT(mUniqueId.Equals(uniqueId));
920 }
921
922 } // namespace mozilla
923