1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 * You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "MediaEngineRemoteVideoSource.h"
7
8 #include "mozilla/RefPtr.h"
9 #include "VideoUtils.h"
10 #include "nsIPrefService.h"
11 #include "MediaTrackConstraints.h"
12 #include "CamerasChild.h"
13
14 extern mozilla::LogModule* GetMediaManagerLog();
15 #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
16 #define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
17
18 namespace mozilla {
19
20 // These need a definition somewhere because template
21 // code is allowed to take their address, and they aren't
22 // guaranteed to have one without this.
23 const unsigned int MediaEngineSource::kMaxDeviceNameLength;
24 const unsigned int MediaEngineSource::kMaxUniqueIdLength;;
25
26 using dom::ConstrainLongRange;
27
NS_IMPL_ISUPPORTS0(MediaEngineRemoteVideoSource)28 NS_IMPL_ISUPPORTS0(MediaEngineRemoteVideoSource)
29
30 MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
31 int aIndex, mozilla::camera::CaptureEngine aCapEngine,
32 dom::MediaSourceEnum aMediaSource, bool aScary, const char* aMonitorName)
33 : MediaEngineCameraVideoSource(aIndex, aMonitorName),
34 mMediaSource(aMediaSource),
35 mCapEngine(aCapEngine),
36 mScary(aScary)
37 {
38 MOZ_ASSERT(aMediaSource != dom::MediaSourceEnum::Other);
39 mSettings.mWidth.Construct(0);
40 mSettings.mHeight.Construct(0);
41 mSettings.mFrameRate.Construct(0);
42 Init();
43 }
44
45 void
Init()46 MediaEngineRemoteVideoSource::Init()
47 {
48 LOG((__PRETTY_FUNCTION__));
49 char deviceName[kMaxDeviceNameLength];
50 char uniqueId[kMaxUniqueIdLength];
51 if (mozilla::camera::GetChildAndCall(
52 &mozilla::camera::CamerasChild::GetCaptureDevice,
53 mCapEngine, mCaptureIndex,
54 deviceName, kMaxDeviceNameLength,
55 uniqueId, kMaxUniqueIdLength, nullptr)) {
56 LOG(("Error initializing RemoteVideoSource (GetCaptureDevice)"));
57 return;
58 }
59
60 SetName(NS_ConvertUTF8toUTF16(deviceName));
61 SetUUID(uniqueId);
62
63 mInitDone = true;
64
65 return;
66 }
67
68 void
Shutdown()69 MediaEngineRemoteVideoSource::Shutdown()
70 {
71 LOG((__PRETTY_FUNCTION__));
72 if (!mInitDone) {
73 return;
74 }
75 Super::Shutdown();
76 if (mState == kStarted) {
77 SourceMediaStream *source;
78 bool empty;
79
80 while (1) {
81 {
82 MonitorAutoLock lock(mMonitor);
83 empty = mSources.IsEmpty();
84 if (empty) {
85 MOZ_ASSERT(mPrincipalHandles.IsEmpty());
86 break;
87 }
88 source = mSources[0];
89 }
90 Stop(source, kVideoTrack); // XXX change to support multiple tracks
91 }
92 MOZ_ASSERT(mState == kStopped);
93 }
94
95 for (auto& registered : mRegisteredHandles) {
96 MOZ_ASSERT(mState == kAllocated || mState == kStopped);
97 Deallocate(registered.get());
98 }
99
100 MOZ_ASSERT(mState == kReleased);
101 mInitDone = false;
102 return;
103 }
104
105 nsresult
Allocate(const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const nsACString & aOrigin,AllocationHandle ** aOutHandle,const char ** aOutBadConstraint)106 MediaEngineRemoteVideoSource::Allocate(
107 const dom::MediaTrackConstraints& aConstraints,
108 const MediaEnginePrefs& aPrefs,
109 const nsString& aDeviceId,
110 const nsACString& aOrigin,
111 AllocationHandle** aOutHandle,
112 const char** aOutBadConstraint)
113 {
114 LOG((__PRETTY_FUNCTION__));
115 AssertIsOnOwningThread();
116
117 if (!mInitDone) {
118 LOG(("Init not done"));
119 return NS_ERROR_FAILURE;
120 }
121
122 nsresult rv = Super::Allocate(aConstraints, aPrefs, aDeviceId, aOrigin,
123 aOutHandle, aOutBadConstraint);
124 if (NS_FAILED(rv)) {
125 return rv;
126 }
127 if (mState == kStarted &&
128 MOZ_LOG_TEST(GetMediaManagerLog(), mozilla::LogLevel::Debug)) {
129 MonitorAutoLock lock(mMonitor);
130 if (mSources.IsEmpty()) {
131 MOZ_ASSERT(mPrincipalHandles.IsEmpty());
132 LOG(("Video device %d reallocated", mCaptureIndex));
133 } else {
134 LOG(("Video device %d allocated shared", mCaptureIndex));
135 }
136 }
137 return NS_OK;
138 }
139
140 nsresult
Deallocate(AllocationHandle * aHandle)141 MediaEngineRemoteVideoSource::Deallocate(AllocationHandle* aHandle)
142 {
143 LOG((__PRETTY_FUNCTION__));
144 AssertIsOnOwningThread();
145
146 Super::Deallocate(aHandle);
147
148 if (!mRegisteredHandles.Length()) {
149 if (mState != kStopped && mState != kAllocated) {
150 return NS_ERROR_FAILURE;
151 }
152 mozilla::camera::GetChildAndCall(
153 &mozilla::camera::CamerasChild::ReleaseCaptureDevice,
154 mCapEngine, mCaptureIndex);
155 mState = kReleased;
156 LOG(("Video device %d deallocated", mCaptureIndex));
157 } else {
158 LOG(("Video device %d deallocated but still in use", mCaptureIndex));
159 }
160 return NS_OK;
161 }
162
163 nsresult
Start(SourceMediaStream * aStream,TrackID aID,const PrincipalHandle & aPrincipalHandle)164 MediaEngineRemoteVideoSource::Start(SourceMediaStream* aStream, TrackID aID,
165 const PrincipalHandle& aPrincipalHandle)
166 {
167 LOG((__PRETTY_FUNCTION__));
168 AssertIsOnOwningThread();
169 if (!mInitDone || !aStream) {
170 LOG(("No stream or init not done"));
171 return NS_ERROR_FAILURE;
172 }
173
174 {
175 MonitorAutoLock lock(mMonitor);
176 mSources.AppendElement(aStream);
177 mPrincipalHandles.AppendElement(aPrincipalHandle);
178 MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
179 }
180
181 aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED);
182
183 if (mState == kStarted) {
184 return NS_OK;
185 }
186 mImageContainer =
187 layers::LayerManager::CreateImageContainer(layers::ImageContainer::ASYNCHRONOUS);
188
189 mState = kStarted;
190 mTrackID = aID;
191
192 if (mozilla::camera::GetChildAndCall(
193 &mozilla::camera::CamerasChild::StartCapture,
194 mCapEngine, mCaptureIndex, mCapability, this)) {
195 LOG(("StartCapture failed"));
196 return NS_ERROR_FAILURE;
197 }
198
199 return NS_OK;
200 }
201
202 nsresult
Stop(mozilla::SourceMediaStream * aSource,mozilla::TrackID aID)203 MediaEngineRemoteVideoSource::Stop(mozilla::SourceMediaStream* aSource,
204 mozilla::TrackID aID)
205 {
206 LOG((__PRETTY_FUNCTION__));
207 AssertIsOnOwningThread();
208 {
209 MonitorAutoLock lock(mMonitor);
210
211 // Drop any cached image so we don't start with a stale image on next
212 // usage
213 mImage = nullptr;
214
215 size_t i = mSources.IndexOf(aSource);
216 if (i == mSources.NoIndex) {
217 // Already stopped - this is allowed
218 return NS_OK;
219 }
220
221 MOZ_ASSERT(mSources.Length() == mPrincipalHandles.Length());
222 mSources.RemoveElementAt(i);
223 mPrincipalHandles.RemoveElementAt(i);
224
225 aSource->EndTrack(aID);
226
227 if (!mSources.IsEmpty()) {
228 return NS_OK;
229 }
230 if (mState != kStarted) {
231 return NS_ERROR_FAILURE;
232 }
233
234 mState = kStopped;
235 }
236
237 mozilla::camera::GetChildAndCall(
238 &mozilla::camera::CamerasChild::StopCapture,
239 mCapEngine, mCaptureIndex);
240
241 return NS_OK;
242 }
243
244 nsresult
Restart(AllocationHandle * aHandle,const dom::MediaTrackConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)245 MediaEngineRemoteVideoSource::Restart(AllocationHandle* aHandle,
246 const dom::MediaTrackConstraints& aConstraints,
247 const MediaEnginePrefs& aPrefs,
248 const nsString& aDeviceId,
249 const char** aOutBadConstraint)
250 {
251 AssertIsOnOwningThread();
252 if (!mInitDone) {
253 LOG(("Init not done"));
254 return NS_ERROR_FAILURE;
255 }
256 MOZ_ASSERT(aHandle);
257 NormalizedConstraints constraints(aConstraints);
258 return ReevaluateAllocation(aHandle, &constraints, aPrefs, aDeviceId,
259 aOutBadConstraint);
260 }
261
262 nsresult
UpdateSingleSource(const AllocationHandle * aHandle,const NormalizedConstraints & aNetConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId,const char ** aOutBadConstraint)263 MediaEngineRemoteVideoSource::UpdateSingleSource(
264 const AllocationHandle* aHandle,
265 const NormalizedConstraints& aNetConstraints,
266 const MediaEnginePrefs& aPrefs,
267 const nsString& aDeviceId,
268 const char** aOutBadConstraint)
269 {
270 if (!ChooseCapability(aNetConstraints, aPrefs, aDeviceId)) {
271 *aOutBadConstraint = FindBadConstraint(aNetConstraints, *this, aDeviceId);
272 return NS_ERROR_FAILURE;
273 }
274
275 switch (mState) {
276 case kReleased:
277 MOZ_ASSERT(aHandle);
278 if (camera::GetChildAndCall(&camera::CamerasChild::AllocateCaptureDevice,
279 mCapEngine, GetUUID().get(),
280 kMaxUniqueIdLength, mCaptureIndex,
281 aHandle->mOrigin)) {
282 return NS_ERROR_FAILURE;
283 }
284 mState = kAllocated;
285 SetLastCapability(mCapability);
286 LOG(("Video device %d allocated for %s", mCaptureIndex,
287 aHandle->mOrigin.get()));
288 break;
289
290 case kStarted:
291 if (mCapability != mLastCapability) {
292 camera::GetChildAndCall(&camera::CamerasChild::StopCapture,
293 mCapEngine, mCaptureIndex);
294 if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture,
295 mCapEngine, mCaptureIndex, mCapability,
296 this)) {
297 LOG(("StartCapture failed"));
298 return NS_ERROR_FAILURE;
299 }
300 SetLastCapability(mCapability);
301 }
302 break;
303
304 default:
305 LOG(("Video device %d %s in ignored state %d", mCaptureIndex,
306 (aHandle? aHandle->mOrigin.get() : ""), mState));
307 break;
308 }
309 return NS_OK;
310 }
311
312 void
SetLastCapability(const webrtc::CaptureCapability & aCapability)313 MediaEngineRemoteVideoSource::SetLastCapability(
314 const webrtc::CaptureCapability& aCapability)
315 {
316 mLastCapability = mCapability;
317
318 webrtc::CaptureCapability cap = aCapability;
319 RefPtr<MediaEngineRemoteVideoSource> that = this;
320
321 NS_DispatchToMainThread(media::NewRunnableFrom([that, cap]() mutable {
322 that->mSettings.mWidth.Value() = cap.width;
323 that->mSettings.mHeight.Value() = cap.height;
324 that->mSettings.mFrameRate.Value() = cap.maxFPS;
325 return NS_OK;
326 }));
327 }
328
329 void
NotifyPull(MediaStreamGraph * aGraph,SourceMediaStream * aSource,TrackID aID,StreamTime aDesiredTime,const PrincipalHandle & aPrincipalHandle)330 MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph,
331 SourceMediaStream* aSource,
332 TrackID aID, StreamTime aDesiredTime,
333 const PrincipalHandle& aPrincipalHandle)
334 {
335 VideoSegment segment;
336
337 MonitorAutoLock lock(mMonitor);
338 if (mState != kStarted) {
339 return;
340 }
341
342 StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
343
344 if (delta > 0) {
345 // nullptr images are allowed
346 AppendToTrack(aSource, mImage, aID, delta, aPrincipalHandle);
347 }
348 }
349
350 int
FrameSizeChange(unsigned int w,unsigned int h,unsigned int streams)351 MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h,
352 unsigned int streams)
353 {
354 mWidth = w;
355 mHeight = h;
356 LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u", w, h));
357 return 0;
358 }
359
360 int
DeliverFrame(unsigned char * buffer,size_t size,uint32_t time_stamp,int64_t ntp_time,int64_t render_time,void * handle)361 MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer,
362 size_t size,
363 uint32_t time_stamp,
364 int64_t ntp_time,
365 int64_t render_time,
366 void *handle)
367 {
368 // Check for proper state.
369 if (mState != kStarted) {
370 LOG(("DeliverFrame: video not started"));
371 return 0;
372 }
373
374 if ((size_t) (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2))) != size) {
375 MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
376 return 0;
377 }
378
379 // Create a video frame and append it to the track.
380 RefPtr<layers::PlanarYCbCrImage> image = mImageContainer->CreatePlanarYCbCrImage();
381
382 uint8_t* frame = static_cast<uint8_t*> (buffer);
383 const uint8_t lumaBpp = 8;
384 const uint8_t chromaBpp = 4;
385
386 // Take lots of care to round up!
387 layers::PlanarYCbCrData data;
388 data.mYChannel = frame;
389 data.mYSize = IntSize(mWidth, mHeight);
390 data.mYStride = (mWidth * lumaBpp + 7)/ 8;
391 data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
392 data.mCbChannel = frame + mHeight * data.mYStride;
393 data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
394 data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
395 data.mPicX = 0;
396 data.mPicY = 0;
397 data.mPicSize = IntSize(mWidth, mHeight);
398 data.mStereoMode = StereoMode::MONO;
399
400 if (!image->CopyData(data)) {
401 MOZ_ASSERT(false);
402 return 0;
403 }
404
405 #ifdef DEBUG
406 static uint32_t frame_num = 0;
407 LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %" PRIu64 ", render_time %" PRIu64,
408 frame_num++, mWidth, mHeight, time_stamp, ntp_time, render_time));
409 #endif
410
411 // we don't touch anything in 'this' until here (except for snapshot,
412 // which has it's own lock)
413 MonitorAutoLock lock(mMonitor);
414
415 // implicitly releases last image
416 mImage = image.forget();
417
418 // We'll push the frame into the MSG on the next NotifyPull. This will avoid
419 // swamping the MSG with frames should it be taking longer than normal to run
420 // an iteration.
421
422 return 0;
423 }
424
425 size_t
NumCapabilities() const426 MediaEngineRemoteVideoSource::NumCapabilities() const
427 {
428 mHardcodedCapabilities.Clear();
429 int num = mozilla::camera::GetChildAndCall(
430 &mozilla::camera::CamerasChild::NumberOfCapabilities,
431 mCapEngine,
432 GetUUID().get());
433 if (num < 1) {
434 // The default for devices that don't return discrete capabilities: treat
435 // them as supporting all capabilities orthogonally. E.g. screensharing.
436 // CaptureCapability defaults key values to 0, which means accept any value.
437 mHardcodedCapabilities.AppendElement(webrtc::CaptureCapability());
438 num = mHardcodedCapabilities.Length(); // 1
439 }
440 return num;
441 }
442
443 bool
ChooseCapability(const NormalizedConstraints & aConstraints,const MediaEnginePrefs & aPrefs,const nsString & aDeviceId)444 MediaEngineRemoteVideoSource::ChooseCapability(
445 const NormalizedConstraints &aConstraints,
446 const MediaEnginePrefs &aPrefs,
447 const nsString& aDeviceId)
448 {
449 AssertIsOnOwningThread();
450
451 switch(mMediaSource) {
452 case dom::MediaSourceEnum::Screen:
453 case dom::MediaSourceEnum::Window:
454 case dom::MediaSourceEnum::Application: {
455 FlattenedConstraints c(aConstraints);
456 // The actual resolution to constrain around is not easy to find ahead of
457 // time (and may in fact change over time), so as a hack, we push ideal
458 // and max constraints down to desktop_capture_impl.cc and finish the
459 // algorithm there.
460 mCapability.width = (c.mWidth.mIdeal.valueOr(0) & 0xffff) << 16 |
461 (c.mWidth.mMax & 0xffff);
462 mCapability.height = (c.mHeight.mIdeal.valueOr(0) & 0xffff) << 16 |
463 (c.mHeight.mMax & 0xffff);
464 mCapability.maxFPS = c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
465 return true;
466 }
467 default:
468 return MediaEngineCameraVideoSource::ChooseCapability(aConstraints, aPrefs, aDeviceId);
469 }
470
471 }
472
473 void
GetCapability(size_t aIndex,webrtc::CaptureCapability & aOut) const474 MediaEngineRemoteVideoSource::GetCapability(size_t aIndex,
475 webrtc::CaptureCapability& aOut) const
476 {
477 if (!mHardcodedCapabilities.IsEmpty()) {
478 MediaEngineCameraVideoSource::GetCapability(aIndex, aOut);
479 }
480 mozilla::camera::GetChildAndCall(
481 &mozilla::camera::CamerasChild::GetCaptureCapability,
482 mCapEngine,
483 GetUUID().get(),
484 aIndex,
485 aOut);
486 }
487
Refresh(int aIndex)488 void MediaEngineRemoteVideoSource::Refresh(int aIndex) {
489 // NOTE: mCaptureIndex might have changed when allocated!
490 // Use aIndex to update information, but don't change mCaptureIndex!!
491 // Caller looked up this source by uniqueId, so it shouldn't change
492 char deviceName[kMaxDeviceNameLength];
493 char uniqueId[kMaxUniqueIdLength];
494
495 if (mozilla::camera::GetChildAndCall(
496 &mozilla::camera::CamerasChild::GetCaptureDevice,
497 mCapEngine, aIndex,
498 deviceName, sizeof(deviceName),
499 uniqueId, sizeof(uniqueId), nullptr)) {
500 return;
501 }
502
503 SetName(NS_ConvertUTF8toUTF16(deviceName));
504 #ifdef DEBUG
505 MOZ_ASSERT(GetUUID().Equals(uniqueId));
506 #endif
507 }
508
509 }
510