1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "media/capture/video/win/video_capture_device_mf_win.h"
6 
7 #include <mfapi.h>
8 #include <mferror.h>
9 #include <stddef.h>
10 #include <wincodec.h>
11 
12 #include <thread>
13 #include <utility>
14 
15 #include "base/bind.h"
16 #include "base/location.h"
17 #include "base/memory/ref_counted.h"
18 #include "base/strings/stringprintf.h"
19 #include "base/strings/sys_string_conversions.h"
20 #include "base/synchronization/waitable_event.h"
21 #include "base/win/scoped_co_mem.h"
22 #include "base/win/windows_version.h"
23 #include "media/capture/mojom/image_capture_types.h"
24 #include "media/capture/video/blob_utils.h"
25 #include "media/capture/video/win/capability_list_win.h"
26 #include "media/capture/video/win/sink_filter_win.h"
27 #include "media/capture/video/win/video_capture_device_utils_win.h"
28 
29 using base::Location;
30 using base::win::ScopedCoMem;
31 using Microsoft::WRL::ComPtr;
32 
33 namespace media {
34 
35 #if DCHECK_IS_ON()
36 #define DLOG_IF_FAILED_WITH_HRESULT(message, hr)                      \
37   {                                                                   \
38     DLOG_IF(ERROR, FAILED(hr))                                        \
39         << (message) << ": " << logging::SystemErrorCodeToString(hr); \
40   }
41 #else
42 #define DLOG_IF_FAILED_WITH_HRESULT(message, hr) \
43   {}
44 #endif
45 
46 namespace {
47 
48 class MFPhotoCallback final
49     : public base::RefCountedThreadSafe<MFPhotoCallback>,
50       public IMFCaptureEngineOnSampleCallback {
51  public:
MFPhotoCallback(VideoCaptureDevice::TakePhotoCallback callback,VideoCaptureFormat format)52   MFPhotoCallback(VideoCaptureDevice::TakePhotoCallback callback,
53                   VideoCaptureFormat format)
54       : callback_(std::move(callback)), format_(format) {}
55 
QueryInterface(REFIID riid,void ** object)56   IFACEMETHODIMP QueryInterface(REFIID riid, void** object) override {
57     if (riid == IID_IUnknown || riid == IID_IMFCaptureEngineOnSampleCallback) {
58       AddRef();
59       *object = static_cast<IMFCaptureEngineOnSampleCallback*>(this);
60       return S_OK;
61     }
62     return E_NOINTERFACE;
63   }
64 
AddRef()65   IFACEMETHODIMP_(ULONG) AddRef() override {
66     base::RefCountedThreadSafe<MFPhotoCallback>::AddRef();
67     return 1U;
68   }
69 
Release()70   IFACEMETHODIMP_(ULONG) Release() override {
71     base::RefCountedThreadSafe<MFPhotoCallback>::Release();
72     return 1U;
73   }
74 
OnSample(IMFSample * sample)75   IFACEMETHODIMP OnSample(IMFSample* sample) override {
76     if (!sample)
77       return S_OK;
78 
79     DWORD buffer_count = 0;
80     sample->GetBufferCount(&buffer_count);
81 
82     for (DWORD i = 0; i < buffer_count; ++i) {
83       ComPtr<IMFMediaBuffer> buffer;
84       sample->GetBufferByIndex(i, &buffer);
85       if (!buffer)
86         continue;
87 
88       BYTE* data = nullptr;
89       DWORD max_length = 0;
90       DWORD length = 0;
91       buffer->Lock(&data, &max_length, &length);
92       mojom::BlobPtr blob = RotateAndBlobify(data, length, format_, 0);
93       buffer->Unlock();
94       if (blob) {
95         std::move(callback_).Run(std::move(blob));
96         LogWindowsImageCaptureOutcome(
97             VideoCaptureWinBackend::kMediaFoundation,
98             ImageCaptureOutcome::kSucceededUsingPhotoStream,
99             IsHighResolution(format_));
100 
101         // What is it supposed to mean if there is more than one buffer sent to
102         // us as a response to requesting a single still image? Are we supposed
103         // to somehow concatenate the buffers? Or is it safe to ignore extra
104         // buffers? For now, we ignore extra buffers.
105         break;
106       }
107     }
108     return S_OK;
109   }
110 
111  private:
112   friend class base::RefCountedThreadSafe<MFPhotoCallback>;
~MFPhotoCallback()113   ~MFPhotoCallback() {
114     if (callback_) {
115       LogWindowsImageCaptureOutcome(
116           VideoCaptureWinBackend::kMediaFoundation,
117           ImageCaptureOutcome::kFailedUsingPhotoStream,
118           IsHighResolution(format_));
119     }
120   }
121 
122   VideoCaptureDevice::TakePhotoCallback callback_;
123   const VideoCaptureFormat format_;
124 
125   DISALLOW_COPY_AND_ASSIGN(MFPhotoCallback);
126 };
127 
128 // Locks the given buffer using the fastest supported method when constructed,
129 // and automatically unlocks the buffer when destroyed.
130 class ScopedBufferLock {
131  public:
ScopedBufferLock(ComPtr<IMFMediaBuffer> buffer)132   explicit ScopedBufferLock(ComPtr<IMFMediaBuffer> buffer)
133       : buffer_(std::move(buffer)) {
134     if (FAILED(buffer_.As(&buffer_2d_))) {
135       LockSlow();
136       return;
137     }
138     // Try lock methods from fastest to slowest: Lock2DSize(), then Lock2D(),
139     // then finally LockSlow().
140     if (Lock2DSize() || Lock2D()) {
141       if (IsContiguous())
142         return;
143       buffer_2d_->Unlock2D();
144     }
145     // Fall back to LockSlow() if 2D buffer was unsupported or noncontiguous.
146     buffer_2d_ = nullptr;
147     LockSlow();
148   }
149 
150   // Returns whether |buffer_2d_| is contiguous with positive pitch, i.e., the
151   // buffer format that the surrounding code expects.
IsContiguous()152   bool IsContiguous() {
153     BOOL is_contiguous;
154     return pitch_ > 0 &&
155            SUCCEEDED(buffer_2d_->IsContiguousFormat(&is_contiguous)) &&
156            is_contiguous &&
157            (length_ || SUCCEEDED(buffer_2d_->GetContiguousLength(&length_)));
158   }
159 
Lock2DSize()160   bool Lock2DSize() {
161     ComPtr<IMF2DBuffer2> buffer_2d_2;
162     if (FAILED(buffer_.As(&buffer_2d_2)))
163       return false;
164     BYTE* data_start;
165     return SUCCEEDED(buffer_2d_2->Lock2DSize(MF2DBuffer_LockFlags_Read, &data_,
166                                              &pitch_, &data_start, &length_));
167   }
168 
Lock2D()169   bool Lock2D() { return SUCCEEDED(buffer_2d_->Lock2D(&data_, &pitch_)); }
170 
LockSlow()171   void LockSlow() {
172     DWORD max_length = 0;
173     buffer_->Lock(&data_, &max_length, &length_);
174   }
175 
~ScopedBufferLock()176   ~ScopedBufferLock() {
177     if (buffer_2d_)
178       buffer_2d_->Unlock2D();
179     else
180       buffer_->Unlock();
181   }
182 
183   ScopedBufferLock(const ScopedBufferLock&) = delete;
184   ScopedBufferLock& operator=(const ScopedBufferLock&) = delete;
185 
data() const186   BYTE* data() const { return data_; }
length() const187   DWORD length() const { return length_; }
188 
189  private:
190   ComPtr<IMFMediaBuffer> buffer_;
191   ComPtr<IMF2DBuffer> buffer_2d_;
192   BYTE* data_ = nullptr;
193   DWORD length_ = 0;
194   LONG pitch_ = 0;
195 };
196 
CreateMFPhotoCallback(VideoCaptureDevice::TakePhotoCallback callback,VideoCaptureFormat format)197 scoped_refptr<IMFCaptureEngineOnSampleCallback> CreateMFPhotoCallback(
198     VideoCaptureDevice::TakePhotoCallback callback,
199     VideoCaptureFormat format) {
200   return scoped_refptr<IMFCaptureEngineOnSampleCallback>(
201       new MFPhotoCallback(std::move(callback), format));
202 }
203 
LogError(const Location & from_here,HRESULT hr)204 void LogError(const Location& from_here, HRESULT hr) {
205   DPLOG(ERROR) << from_here.ToString()
206                << " hr = " << logging::SystemErrorCodeToString(hr);
207 }
208 
GetFrameSizeFromMediaType(IMFMediaType * type,gfx::Size * frame_size)209 bool GetFrameSizeFromMediaType(IMFMediaType* type, gfx::Size* frame_size) {
210   UINT32 width32, height32;
211   if (FAILED(MFGetAttributeSize(type, MF_MT_FRAME_SIZE, &width32, &height32)))
212     return false;
213   frame_size->SetSize(width32, height32);
214   return true;
215 }
216 
GetFrameRateFromMediaType(IMFMediaType * type,float * frame_rate)217 bool GetFrameRateFromMediaType(IMFMediaType* type, float* frame_rate) {
218   UINT32 numerator, denominator;
219   if (FAILED(MFGetAttributeRatio(type, MF_MT_FRAME_RATE, &numerator,
220                                  &denominator)) ||
221       !denominator) {
222     return false;
223   }
224   *frame_rate = static_cast<float>(numerator) / denominator;
225   return true;
226 }
227 
GetFormatFromSourceMediaType(IMFMediaType * source_media_type,bool photo,bool use_hardware_format,VideoCaptureFormat * format)228 bool GetFormatFromSourceMediaType(IMFMediaType* source_media_type,
229                                   bool photo,
230                                   bool use_hardware_format,
231                                   VideoCaptureFormat* format) {
232   GUID major_type_guid;
233   if (FAILED(source_media_type->GetGUID(MF_MT_MAJOR_TYPE, &major_type_guid)) ||
234       (major_type_guid != MFMediaType_Image &&
235        (photo ||
236         !GetFrameRateFromMediaType(source_media_type, &format->frame_rate)))) {
237     return false;
238   }
239 
240   GUID sub_type_guid;
241   if (FAILED(source_media_type->GetGUID(MF_MT_SUBTYPE, &sub_type_guid)) ||
242       !GetFrameSizeFromMediaType(source_media_type, &format->frame_size) ||
243       !VideoCaptureDeviceMFWin::GetPixelFormatFromMFSourceMediaSubtype(
244           sub_type_guid, use_hardware_format, &format->pixel_format)) {
245     return false;
246   }
247 
248   return true;
249 }
250 
CopyAttribute(IMFAttributes * source_attributes,IMFAttributes * destination_attributes,const GUID & key)251 HRESULT CopyAttribute(IMFAttributes* source_attributes,
252                       IMFAttributes* destination_attributes,
253                       const GUID& key) {
254   PROPVARIANT var;
255   PropVariantInit(&var);
256   HRESULT hr = source_attributes->GetItem(key, &var);
257   if (FAILED(hr))
258     return hr;
259 
260   hr = destination_attributes->SetItem(key, var);
261   PropVariantClear(&var);
262   return hr;
263 }
264 
265 struct MediaFormatConfiguration {
266   GUID mf_source_media_subtype;
267   GUID mf_sink_media_subtype;
268   VideoPixelFormat pixel_format;
269 };
270 
GetMediaFormatConfigurationFromMFSourceMediaSubtype(const GUID & mf_source_media_subtype,bool use_hardware_format,MediaFormatConfiguration * media_format_configuration)271 bool GetMediaFormatConfigurationFromMFSourceMediaSubtype(
272     const GUID& mf_source_media_subtype,
273     bool use_hardware_format,
274     MediaFormatConfiguration* media_format_configuration) {
275   // Special case handling of the NV12 format when using hardware capture
276   // to ensure that captured buffers are passed through without copies
277   if (use_hardware_format && mf_source_media_subtype == MFVideoFormat_NV12) {
278     *media_format_configuration = {MFVideoFormat_NV12, MFVideoFormat_NV12,
279                                    PIXEL_FORMAT_NV12};
280     return true;
281   }
282   static const MediaFormatConfiguration kMediaFormatConfigurationMap[] = {
283       // IMFCaptureEngine inevitably performs the video frame decoding itself.
284       // This means that the sink must always be set to an uncompressed video
285       // format.
286 
287       // Since chromium uses I420 at the other end of the pipe, MF known video
288       // output formats are always set to I420.
289       {MFVideoFormat_I420, MFVideoFormat_I420, PIXEL_FORMAT_I420},
290       {MFVideoFormat_YUY2, MFVideoFormat_I420, PIXEL_FORMAT_I420},
291       {MFVideoFormat_UYVY, MFVideoFormat_I420, PIXEL_FORMAT_I420},
292       {MFVideoFormat_RGB24, MFVideoFormat_I420, PIXEL_FORMAT_I420},
293       {MFVideoFormat_RGB32, MFVideoFormat_I420, PIXEL_FORMAT_I420},
294       {MFVideoFormat_ARGB32, MFVideoFormat_I420, PIXEL_FORMAT_I420},
295       {MFVideoFormat_MJPG, MFVideoFormat_I420, PIXEL_FORMAT_I420},
296       {MFVideoFormat_NV12, MFVideoFormat_I420, PIXEL_FORMAT_I420},
297       {MFVideoFormat_YV12, MFVideoFormat_I420, PIXEL_FORMAT_I420},
298 
299       // Depth cameras use 16-bit uncompressed video formats.
300       // We ask IMFCaptureEngine to let the frame pass through, without
301       // transcoding, since transcoding would lead to precision loss.
302       {kMediaSubTypeY16, kMediaSubTypeY16, PIXEL_FORMAT_Y16},
303       {kMediaSubTypeZ16, kMediaSubTypeZ16, PIXEL_FORMAT_Y16},
304       {kMediaSubTypeINVZ, kMediaSubTypeINVZ, PIXEL_FORMAT_Y16},
305       {MFVideoFormat_D16, MFVideoFormat_D16, PIXEL_FORMAT_Y16},
306 
307       // Photo type
308       {GUID_ContainerFormatJpeg, GUID_ContainerFormatJpeg, PIXEL_FORMAT_MJPEG}};
309 
310   for (const auto& kMediaFormatConfiguration : kMediaFormatConfigurationMap) {
311     if (kMediaFormatConfiguration.mf_source_media_subtype ==
312         mf_source_media_subtype) {
313       *media_format_configuration = kMediaFormatConfiguration;
314       return true;
315     }
316   }
317 
318   return false;
319 }
320 
321 // Calculate sink subtype based on source subtype. |passthrough| is set when
322 // sink and source are the same and means that there should be no transcoding
323 // done by IMFCaptureEngine.
GetMFSinkMediaSubtype(IMFMediaType * source_media_type,bool use_hardware_format,GUID * mf_sink_media_subtype,bool * passthrough)324 HRESULT GetMFSinkMediaSubtype(IMFMediaType* source_media_type,
325                               bool use_hardware_format,
326                               GUID* mf_sink_media_subtype,
327                               bool* passthrough) {
328   GUID source_subtype;
329   HRESULT hr = source_media_type->GetGUID(MF_MT_SUBTYPE, &source_subtype);
330   if (FAILED(hr))
331     return hr;
332   MediaFormatConfiguration media_format_configuration;
333   if (!GetMediaFormatConfigurationFromMFSourceMediaSubtype(
334           source_subtype, use_hardware_format, &media_format_configuration))
335     return E_FAIL;
336   *mf_sink_media_subtype = media_format_configuration.mf_sink_media_subtype;
337   *passthrough =
338       (media_format_configuration.mf_sink_media_subtype == source_subtype);
339   return S_OK;
340 }
341 
ConvertToPhotoSinkMediaType(IMFMediaType * source_media_type,IMFMediaType * destination_media_type)342 HRESULT ConvertToPhotoSinkMediaType(IMFMediaType* source_media_type,
343                                     IMFMediaType* destination_media_type) {
344   HRESULT hr =
345       destination_media_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Image);
346   if (FAILED(hr))
347     return hr;
348 
349   bool passthrough = false;
350   GUID mf_sink_media_subtype;
351   hr = GetMFSinkMediaSubtype(source_media_type, /*use_hardware_format=*/false,
352                              &mf_sink_media_subtype, &passthrough);
353   if (FAILED(hr))
354     return hr;
355 
356   hr = destination_media_type->SetGUID(MF_MT_SUBTYPE, mf_sink_media_subtype);
357   if (FAILED(hr))
358     return hr;
359 
360   return CopyAttribute(source_media_type, destination_media_type,
361                        MF_MT_FRAME_SIZE);
362 }
363 
ConvertToVideoSinkMediaType(IMFMediaType * source_media_type,bool use_hardware_format,IMFMediaType * sink_media_type)364 HRESULT ConvertToVideoSinkMediaType(IMFMediaType* source_media_type,
365                                     bool use_hardware_format,
366                                     IMFMediaType* sink_media_type) {
367   HRESULT hr = sink_media_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
368   if (FAILED(hr))
369     return hr;
370 
371   bool passthrough = false;
372   GUID mf_sink_media_subtype;
373   hr = GetMFSinkMediaSubtype(source_media_type, use_hardware_format,
374                              &mf_sink_media_subtype, &passthrough);
375   if (FAILED(hr))
376     return hr;
377 
378   hr = sink_media_type->SetGUID(MF_MT_SUBTYPE, mf_sink_media_subtype);
379   // Copying attribute values for passthrough mode is redundant, since the
380   // format is kept unchanged, and causes AddStream error MF_E_INVALIDMEDIATYPE.
381   if (FAILED(hr) || passthrough)
382     return hr;
383 
384   hr = CopyAttribute(source_media_type, sink_media_type, MF_MT_FRAME_SIZE);
385   if (FAILED(hr))
386     return hr;
387 
388   hr = CopyAttribute(source_media_type, sink_media_type, MF_MT_FRAME_RATE);
389   if (FAILED(hr))
390     return hr;
391 
392   hr = CopyAttribute(source_media_type, sink_media_type,
393                      MF_MT_PIXEL_ASPECT_RATIO);
394   if (FAILED(hr))
395     return hr;
396 
397   return CopyAttribute(source_media_type, sink_media_type,
398                        MF_MT_INTERLACE_MODE);
399 }
400 
GetBestMatchedPhotoCapability(ComPtr<IMFMediaType> current_media_type,gfx::Size requested_size,const CapabilityList & capabilities)401 const CapabilityWin& GetBestMatchedPhotoCapability(
402     ComPtr<IMFMediaType> current_media_type,
403     gfx::Size requested_size,
404     const CapabilityList& capabilities) {
405   gfx::Size current_size;
406   GetFrameSizeFromMediaType(current_media_type.Get(), &current_size);
407 
408   int requested_height = requested_size.height() > 0 ? requested_size.height()
409                                                      : current_size.height();
410   int requested_width = requested_size.width() > 0 ? requested_size.width()
411                                                    : current_size.width();
412 
413   const CapabilityWin* best_match = &(*capabilities.begin());
414   for (const CapabilityWin& capability : capabilities) {
415     int height = capability.supported_format.frame_size.height();
416     int width = capability.supported_format.frame_size.width();
417     int best_height = best_match->supported_format.frame_size.height();
418     int best_width = best_match->supported_format.frame_size.width();
419 
420     if (std::abs(height - requested_height) <= std::abs(height - best_height) &&
421         std::abs(width - requested_width) <= std::abs(width - best_width)) {
422       best_match = &capability;
423     }
424   }
425   return *best_match;
426 }
427 
CreateCaptureEngine(IMFCaptureEngine ** engine)428 HRESULT CreateCaptureEngine(IMFCaptureEngine** engine) {
429   ComPtr<IMFCaptureEngineClassFactory> capture_engine_class_factory;
430   HRESULT hr = CoCreateInstance(CLSID_MFCaptureEngineClassFactory, nullptr,
431                                 CLSCTX_INPROC_SERVER,
432                                 IID_PPV_ARGS(&capture_engine_class_factory));
433   if (FAILED(hr))
434     return hr;
435 
436   return capture_engine_class_factory->CreateInstance(CLSID_MFCaptureEngine,
437                                                       IID_PPV_ARGS(engine));
438 }
439 
GetCameraControlSupport(ComPtr<IAMCameraControl> camera_control,CameraControlProperty control_property)440 bool GetCameraControlSupport(ComPtr<IAMCameraControl> camera_control,
441                              CameraControlProperty control_property) {
442   long min, max, step, default_value, flags;
443   HRESULT hr = camera_control->GetRange(control_property, &min, &max, &step,
444                                         &default_value, &flags);
445   return SUCCEEDED(hr) && min < max;
446 }
447 
448 // Retrieves the control range and value, and
449 // optionally returns the associated supported and current mode.
450 template <typename ControlInterface, typename ControlProperty>
RetrieveControlRangeAndCurrent(ComPtr<ControlInterface> & control_interface,ControlProperty control_property,std::vector<mojom::MeteringMode> * supported_modes=nullptr,mojom::MeteringMode * current_mode=nullptr,double (* value_converter)(long)=PlatformToCaptureValue,double (* step_converter)(long,double,double)=PlatformToCaptureStep)451 mojom::RangePtr RetrieveControlRangeAndCurrent(
452     ComPtr<ControlInterface>& control_interface,
453     ControlProperty control_property,
454     std::vector<mojom::MeteringMode>* supported_modes = nullptr,
455     mojom::MeteringMode* current_mode = nullptr,
456     double (*value_converter)(long) = PlatformToCaptureValue,
457     double (*step_converter)(long, double, double) = PlatformToCaptureStep) {
458   return media::RetrieveControlRangeAndCurrent(
459       [&control_interface, control_property](auto... args) {
460         return control_interface->GetRange(control_property, args...);
461       },
462       [&control_interface, control_property](auto... args) {
463         return control_interface->Get(control_property, args...);
464       },
465       supported_modes, current_mode, value_converter, step_converter);
466 }
467 }  // namespace
468 
469 class MFVideoCallback final
470     : public base::RefCountedThreadSafe<MFVideoCallback>,
471       public IMFCaptureEngineOnSampleCallback,
472       public IMFCaptureEngineOnEventCallback {
473  public:
MFVideoCallback(VideoCaptureDeviceMFWin * observer)474   MFVideoCallback(VideoCaptureDeviceMFWin* observer) : observer_(observer) {}
475 
QueryInterface(REFIID riid,void ** object)476   IFACEMETHODIMP QueryInterface(REFIID riid, void** object) override {
477     HRESULT hr = E_NOINTERFACE;
478     if (riid == IID_IUnknown) {
479       *object = this;
480       hr = S_OK;
481     } else if (riid == IID_IMFCaptureEngineOnSampleCallback) {
482       *object = static_cast<IMFCaptureEngineOnSampleCallback*>(this);
483       hr = S_OK;
484     } else if (riid == IID_IMFCaptureEngineOnEventCallback) {
485       *object = static_cast<IMFCaptureEngineOnEventCallback*>(this);
486       hr = S_OK;
487     }
488     if (SUCCEEDED(hr))
489       AddRef();
490 
491     return hr;
492   }
493 
AddRef()494   IFACEMETHODIMP_(ULONG) AddRef() override {
495     base::RefCountedThreadSafe<MFVideoCallback>::AddRef();
496     return 1U;
497   }
498 
Release()499   IFACEMETHODIMP_(ULONG) Release() override {
500     base::RefCountedThreadSafe<MFVideoCallback>::Release();
501     return 1U;
502   }
503 
OnEvent(IMFMediaEvent * media_event)504   IFACEMETHODIMP OnEvent(IMFMediaEvent* media_event) override {
505     base::AutoLock lock(lock_);
506     if (!observer_) {
507       return S_OK;
508     }
509     observer_->OnEvent(media_event);
510     return S_OK;
511   }
512 
OnSample(IMFSample * sample)513   IFACEMETHODIMP OnSample(IMFSample* sample) override {
514     base::AutoLock lock(lock_);
515     if (!observer_) {
516       return S_OK;
517     }
518     if (!sample) {
519       observer_->OnFrameDropped(
520           VideoCaptureFrameDropReason::kWinMediaFoundationReceivedSampleIsNull);
521       return S_OK;
522     }
523 
524     base::TimeTicks reference_time(base::TimeTicks::Now());
525     LONGLONG raw_time_stamp = 0;
526     sample->GetSampleTime(&raw_time_stamp);
527     base::TimeDelta timestamp =
528         base::TimeDelta::FromMicroseconds(raw_time_stamp / 10);
529 
530     DWORD count = 0;
531     sample->GetBufferCount(&count);
532 
533     for (DWORD i = 0; i < count; ++i) {
534       ComPtr<IMFMediaBuffer> buffer;
535       sample->GetBufferByIndex(i, &buffer);
536       if (buffer) {
537         ScopedBufferLock locked_buffer(buffer);
538         if (locked_buffer.data()) {
539           observer_->OnIncomingCapturedData(locked_buffer.data(),
540                                             locked_buffer.length(),
541                                             reference_time, timestamp);
542         } else {
543           observer_->OnFrameDropped(
544               VideoCaptureFrameDropReason::
545                   kWinMediaFoundationLockingBufferDelieveredNullptr);
546         }
547       } else {
548         observer_->OnFrameDropped(
549             VideoCaptureFrameDropReason::
550                 kWinMediaFoundationGetBufferByIndexReturnedNull);
551       }
552     }
553     return S_OK;
554   }
555 
Shutdown()556   void Shutdown() {
557     base::AutoLock lock(lock_);
558     observer_ = nullptr;
559   }
560 
561  private:
562   friend class base::RefCountedThreadSafe<MFVideoCallback>;
~MFVideoCallback()563   ~MFVideoCallback() {}
564 
565   // Protects access to |observer_|.
566   base::Lock lock_;
567   VideoCaptureDeviceMFWin* observer_ GUARDED_BY(lock_);
568 };
569 
570 // static
GetPixelFormatFromMFSourceMediaSubtype(const GUID & mf_source_media_subtype,bool use_hardware_format,VideoPixelFormat * pixel_format)571 bool VideoCaptureDeviceMFWin::GetPixelFormatFromMFSourceMediaSubtype(
572     const GUID& mf_source_media_subtype,
573     bool use_hardware_format,
574     VideoPixelFormat* pixel_format) {
575   MediaFormatConfiguration media_format_configuration;
576   if (!GetMediaFormatConfigurationFromMFSourceMediaSubtype(
577           mf_source_media_subtype, use_hardware_format,
578           &media_format_configuration))
579     return false;
580 
581   *pixel_format = media_format_configuration.pixel_format;
582   return true;
583 }
584 
585 // Check if the video capture device supports pan, tilt and zoom controls.
586 // static
GetControlSupport(ComPtr<IMFMediaSource> source)587 VideoCaptureControlSupport VideoCaptureDeviceMFWin::GetControlSupport(
588     ComPtr<IMFMediaSource> source) {
589   VideoCaptureControlSupport control_support;
590 
591   ComPtr<IAMCameraControl> camera_control;
592   HRESULT hr = source.As(&camera_control);
593   DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMCameraControl", hr);
594   ComPtr<IAMVideoProcAmp> video_control;
595   hr = source.As(&video_control);
596   DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMVideoProcAmp", hr);
597 
598   // On Windows platform, some Image Capture video constraints and settings are
599   // get or set using IAMCameraControl interface while the rest are get or set
600   // using IAMVideoProcAmp interface and most device drivers define both of
601   // them. So for simplicity GetPhotoState and SetPhotoState support Image
602   // Capture API constraints and settings only if both interfaces are available.
603   // Therefore, if either of these interface is missing, this backend does not
604   // really support pan, tilt nor zoom.
605   if (camera_control && video_control) {
606     control_support.pan =
607         GetCameraControlSupport(camera_control, CameraControl_Pan);
608     control_support.tilt =
609         GetCameraControlSupport(camera_control, CameraControl_Tilt);
610     control_support.zoom =
611         GetCameraControlSupport(camera_control, CameraControl_Zoom);
612   }
613 
614   return control_support;
615 }
616 
ExecuteHresultCallbackWithRetries(base::RepeatingCallback<HRESULT ()> callback,MediaFoundationFunctionRequiringRetry which_function)617 HRESULT VideoCaptureDeviceMFWin::ExecuteHresultCallbackWithRetries(
618     base::RepeatingCallback<HRESULT()> callback,
619     MediaFoundationFunctionRequiringRetry which_function) {
620   // Retry callback execution on MF_E_INVALIDREQUEST.
621   // MF_E_INVALIDREQUEST is not documented in MediaFoundation documentation.
622   // It could mean that MediaFoundation or the underlying device can be in a
623   // state that reject these calls. Since MediaFoundation gives no intel about
624   // that state beginning and ending (i.e. via some kind of event), we retry the
625   // call until it succeed.
626   HRESULT hr;
627   int retry_count = 0;
628   do {
629     hr = callback.Run();
630     if (FAILED(hr))
631       base::PlatformThread::Sleep(
632           base::TimeDelta::FromMilliseconds(retry_delay_in_ms_));
633 
634     // Give up after some amount of time
635   } while (hr == MF_E_INVALIDREQUEST && retry_count++ < max_retry_count_);
636   LogNumberOfRetriesNeededToWorkAroundMFInvalidRequest(which_function,
637                                                        retry_count);
638 
639   return hr;
640 }
641 
GetDeviceStreamCount(IMFCaptureSource * source,DWORD * count)642 HRESULT VideoCaptureDeviceMFWin::GetDeviceStreamCount(IMFCaptureSource* source,
643                                                       DWORD* count) {
644   // Sometimes, GetDeviceStreamCount returns an
645   // undocumented MF_E_INVALIDREQUEST. Retrying solves the issue.
646   return ExecuteHresultCallbackWithRetries(
647       base::BindRepeating(
648           [](IMFCaptureSource* source, DWORD* count) {
649             return source->GetDeviceStreamCount(count);
650           },
651           base::Unretained(source), count),
652       MediaFoundationFunctionRequiringRetry::kGetDeviceStreamCount);
653 }
654 
GetDeviceStreamCategory(IMFCaptureSource * source,DWORD stream_index,MF_CAPTURE_ENGINE_STREAM_CATEGORY * stream_category)655 HRESULT VideoCaptureDeviceMFWin::GetDeviceStreamCategory(
656     IMFCaptureSource* source,
657     DWORD stream_index,
658     MF_CAPTURE_ENGINE_STREAM_CATEGORY* stream_category) {
659   // We believe that GetDeviceStreamCategory could be affected by the same
660   // behaviour of GetDeviceStreamCount and GetAvailableDeviceMediaType
661   return ExecuteHresultCallbackWithRetries(
662       base::BindRepeating(
663           [](IMFCaptureSource* source, DWORD stream_index,
664              MF_CAPTURE_ENGINE_STREAM_CATEGORY* stream_category) {
665             return source->GetDeviceStreamCategory(stream_index,
666                                                    stream_category);
667           },
668           base::Unretained(source), stream_index, stream_category),
669       MediaFoundationFunctionRequiringRetry::kGetDeviceStreamCategory);
670 }
671 
GetAvailableDeviceMediaType(IMFCaptureSource * source,DWORD stream_index,DWORD media_type_index,IMFMediaType ** type)672 HRESULT VideoCaptureDeviceMFWin::GetAvailableDeviceMediaType(
673     IMFCaptureSource* source,
674     DWORD stream_index,
675     DWORD media_type_index,
676     IMFMediaType** type) {
677   // Rarely, for some unknown reason, GetAvailableDeviceMediaType returns an
678   // undocumented MF_E_INVALIDREQUEST. Retrying solves the issue.
679   return ExecuteHresultCallbackWithRetries(
680       base::BindRepeating(
681           [](IMFCaptureSource* source, DWORD stream_index,
682              DWORD media_type_index, IMFMediaType** type) {
683             return source->GetAvailableDeviceMediaType(stream_index,
684                                                        media_type_index, type);
685           },
686           base::Unretained(source), stream_index, media_type_index, type),
687       MediaFoundationFunctionRequiringRetry::kGetAvailableDeviceMediaType);
688 }
689 
FillCapabilities(IMFCaptureSource * source,bool photo,CapabilityList * capabilities)690 HRESULT VideoCaptureDeviceMFWin::FillCapabilities(
691     IMFCaptureSource* source,
692     bool photo,
693     CapabilityList* capabilities) {
694   DWORD stream_count = 0;
695   HRESULT hr = GetDeviceStreamCount(source, &stream_count);
696   if (FAILED(hr))
697     return hr;
698 
699   for (DWORD stream_index = 0; stream_index < stream_count; stream_index++) {
700     MF_CAPTURE_ENGINE_STREAM_CATEGORY stream_category;
701     hr = GetDeviceStreamCategory(source, stream_index, &stream_category);
702     if (FAILED(hr))
703       return hr;
704 
705     if ((photo && stream_category !=
706                       MF_CAPTURE_ENGINE_STREAM_CATEGORY_PHOTO_INDEPENDENT) ||
707         (!photo &&
708          stream_category != MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_PREVIEW &&
709          stream_category != MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_CAPTURE)) {
710       continue;
711     }
712 
713     DWORD media_type_index = 0;
714     ComPtr<IMFMediaType> type;
715     while (SUCCEEDED(hr = GetAvailableDeviceMediaType(
716                          source, stream_index, media_type_index, &type))) {
717       VideoCaptureFormat format;
718       if (GetFormatFromSourceMediaType(
719               type.Get(), photo,
720               /*use_hardware_format=*/!photo &&
721                   static_cast<bool>(dxgi_device_manager_),
722               &format))
723         capabilities->emplace_back(media_type_index, format, stream_index);
724       type.Reset();
725       ++media_type_index;
726     }
727     if (hr == MF_E_NO_MORE_TYPES) {
728       hr = S_OK;
729     }
730     if (FAILED(hr)) {
731       return hr;
732     }
733   }
734 
735   return hr;
736 }
737 
VideoCaptureDeviceMFWin(const VideoCaptureDeviceDescriptor & device_descriptor,ComPtr<IMFMediaSource> source,scoped_refptr<VideoCaptureDXGIDeviceManager> dxgi_device_manager)738 VideoCaptureDeviceMFWin::VideoCaptureDeviceMFWin(
739     const VideoCaptureDeviceDescriptor& device_descriptor,
740     ComPtr<IMFMediaSource> source,
741     scoped_refptr<VideoCaptureDXGIDeviceManager> dxgi_device_manager)
742     : VideoCaptureDeviceMFWin(device_descriptor,
743                               source,
744                               std::move(dxgi_device_manager),
745                               nullptr) {}
746 
VideoCaptureDeviceMFWin(const VideoCaptureDeviceDescriptor & device_descriptor,ComPtr<IMFMediaSource> source,scoped_refptr<VideoCaptureDXGIDeviceManager> dxgi_device_manager,ComPtr<IMFCaptureEngine> engine)747 VideoCaptureDeviceMFWin::VideoCaptureDeviceMFWin(
748     const VideoCaptureDeviceDescriptor& device_descriptor,
749     ComPtr<IMFMediaSource> source,
750     scoped_refptr<VideoCaptureDXGIDeviceManager> dxgi_device_manager,
751     ComPtr<IMFCaptureEngine> engine)
752     : facing_mode_(device_descriptor.facing),
753       create_mf_photo_callback_(base::BindRepeating(&CreateMFPhotoCallback)),
754       is_initialized_(false),
755       max_retry_count_(200),
756       retry_delay_in_ms_(50),
757       source_(source),
758       engine_(engine),
759       is_started_(false),
760       has_sent_on_started_to_client_(false),
761       exposure_mode_manual_(false),
762       focus_mode_manual_(false),
763       white_balance_mode_manual_(false),
764       capture_initialize_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
765                           base::WaitableEvent::InitialState::NOT_SIGNALED),
766       // We never want to reset |capture_error_|.
767       capture_error_(base::WaitableEvent::ResetPolicy::MANUAL,
768                      base::WaitableEvent::InitialState::NOT_SIGNALED),
769       dxgi_device_manager_(std::move(dxgi_device_manager)) {
770   DETACH_FROM_SEQUENCE(sequence_checker_);
771 }
772 
~VideoCaptureDeviceMFWin()773 VideoCaptureDeviceMFWin::~VideoCaptureDeviceMFWin() {
774   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
775   if (!video_stream_take_photo_callbacks_.empty()) {
776     for (size_t k = 0; k < video_stream_take_photo_callbacks_.size(); k++) {
777       LogWindowsImageCaptureOutcome(
778           VideoCaptureWinBackend::kMediaFoundation,
779           ImageCaptureOutcome::kFailedUsingVideoStream,
780           selected_video_capability_
781               ? IsHighResolution(selected_video_capability_->supported_format)
782               : false);
783     }
784   }
785   if (video_callback_) {
786     video_callback_->Shutdown();
787   }
788 }
789 
Init()790 bool VideoCaptureDeviceMFWin::Init() {
791   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
792   DCHECK(!is_initialized_);
793   HRESULT hr;
794 
795   hr = source_.As(&camera_control_);
796   DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMCameraControl", hr);
797 
798   hr = source_.As(&video_control_);
799   DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMVideoProcAmp", hr);
800 
801   if (!engine_) {
802     hr = CreateCaptureEngine(&engine_);
803     if (FAILED(hr)) {
804       LogError(FROM_HERE, hr);
805       return false;
806     }
807   }
808 
809   ComPtr<IMFAttributes> attributes;
810   hr = MFCreateAttributes(&attributes, 1);
811   if (FAILED(hr)) {
812     LogError(FROM_HERE, hr);
813     return false;
814   }
815 
816   hr = attributes->SetUINT32(MF_CAPTURE_ENGINE_USE_VIDEO_DEVICE_ONLY, TRUE);
817   if (FAILED(hr)) {
818     LogError(FROM_HERE, hr);
819     return false;
820   }
821 
822   if (dxgi_device_manager_) {
823     dxgi_device_manager_->RegisterInCaptureEngineAttributes(attributes.Get());
824   }
825 
826   video_callback_ = new MFVideoCallback(this);
827   hr = engine_->Initialize(video_callback_.get(), attributes.Get(), nullptr,
828                            source_.Get());
829   if (FAILED(hr)) {
830     LogError(FROM_HERE, hr);
831     return false;
832   }
833 
834   hr = WaitOnCaptureEvent(MF_CAPTURE_ENGINE_INITIALIZED);
835   if (FAILED(hr)) {
836     LogError(FROM_HERE, hr);
837     return false;
838   }
839 
840   is_initialized_ = true;
841   return true;
842 }
843 
AllocateAndStart(const VideoCaptureParams & params,std::unique_ptr<VideoCaptureDevice::Client> client)844 void VideoCaptureDeviceMFWin::AllocateAndStart(
845     const VideoCaptureParams& params,
846     std::unique_ptr<VideoCaptureDevice::Client> client) {
847   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
848 
849   base::AutoLock lock(lock_);
850 
851   client_ = std::move(client);
852   DCHECK_EQ(false, is_started_);
853 
854   if (!engine_) {
855     OnError(VideoCaptureError::kWinMediaFoundationEngineIsNull, FROM_HERE,
856             E_FAIL);
857     return;
858   }
859 
860   ComPtr<IMFCaptureSource> source;
861   HRESULT hr = engine_->GetSource(&source);
862   if (FAILED(hr)) {
863     OnError(VideoCaptureError::kWinMediaFoundationEngineGetSourceFailed,
864             FROM_HERE, hr);
865     return;
866   }
867 
868   hr = FillCapabilities(source.Get(), true, &photo_capabilities_);
869   if (FAILED(hr)) {
870     OnError(VideoCaptureError::kWinMediaFoundationFillPhotoCapabilitiesFailed,
871             FROM_HERE, hr);
872     return;
873   }
874 
875   if (!photo_capabilities_.empty()) {
876     selected_photo_capability_.reset(
877         new CapabilityWin(photo_capabilities_.front()));
878   }
879 
880   CapabilityList video_capabilities;
881   hr = FillCapabilities(source.Get(), false, &video_capabilities);
882   if (FAILED(hr)) {
883     OnError(VideoCaptureError::kWinMediaFoundationFillVideoCapabilitiesFailed,
884             FROM_HERE, hr);
885     return;
886   }
887 
888   if (video_capabilities.empty()) {
889     OnError(VideoCaptureError::kWinMediaFoundationNoVideoCapabilityFound,
890             FROM_HERE, "No video capability found");
891     return;
892   }
893 
894   const CapabilityWin best_match_video_capability =
895       GetBestMatchedCapability(params.requested_format, video_capabilities);
896   ComPtr<IMFMediaType> source_video_media_type;
897   hr = GetAvailableDeviceMediaType(
898       source.Get(), best_match_video_capability.stream_index,
899       best_match_video_capability.media_type_index, &source_video_media_type);
900   if (FAILED(hr)) {
901     OnError(
902         VideoCaptureError::kWinMediaFoundationGetAvailableDeviceMediaTypeFailed,
903         FROM_HERE, hr);
904     return;
905   }
906 
907   hr = source->SetCurrentDeviceMediaType(
908       best_match_video_capability.stream_index, source_video_media_type.Get());
909   if (FAILED(hr)) {
910     OnError(
911         VideoCaptureError::kWinMediaFoundationSetCurrentDeviceMediaTypeFailed,
912         FROM_HERE, hr);
913     return;
914   }
915 
916   ComPtr<IMFCaptureSink> sink;
917   hr = engine_->GetSink(MF_CAPTURE_ENGINE_SINK_TYPE_PREVIEW, &sink);
918   if (FAILED(hr)) {
919     OnError(VideoCaptureError::kWinMediaFoundationEngineGetSinkFailed,
920             FROM_HERE, hr);
921     return;
922   }
923 
924   ComPtr<IMFCapturePreviewSink> preview_sink;
925   hr = sink->QueryInterface(IID_PPV_ARGS(&preview_sink));
926   if (FAILED(hr)) {
927     OnError(VideoCaptureError::
928                 kWinMediaFoundationSinkQueryCapturePreviewInterfaceFailed,
929             FROM_HERE, hr);
930     return;
931   }
932 
933   hr = preview_sink->RemoveAllStreams();
934   if (FAILED(hr)) {
935     OnError(VideoCaptureError::kWinMediaFoundationSinkRemoveAllStreamsFailed,
936             FROM_HERE, hr);
937     return;
938   }
939 
940   ComPtr<IMFMediaType> sink_video_media_type;
941   hr = MFCreateMediaType(&sink_video_media_type);
942   if (FAILED(hr)) {
943     OnError(
944         VideoCaptureError::kWinMediaFoundationCreateSinkVideoMediaTypeFailed,
945         FROM_HERE, hr);
946     return;
947   }
948 
949   hr = ConvertToVideoSinkMediaType(
950       source_video_media_type.Get(),
951       /*use_hardware_format=*/static_cast<bool>(dxgi_device_manager_),
952       sink_video_media_type.Get());
953   if (FAILED(hr)) {
954     OnError(
955         VideoCaptureError::kWinMediaFoundationConvertToVideoSinkMediaTypeFailed,
956         FROM_HERE, hr);
957     return;
958   }
959 
960   DWORD dw_sink_stream_index = 0;
961   hr = preview_sink->AddStream(best_match_video_capability.stream_index,
962                                sink_video_media_type.Get(), nullptr,
963                                &dw_sink_stream_index);
964   if (FAILED(hr)) {
965     OnError(VideoCaptureError::kWinMediaFoundationSinkAddStreamFailed,
966             FROM_HERE, hr);
967     return;
968   }
969 
970   hr = preview_sink->SetSampleCallback(dw_sink_stream_index,
971                                        video_callback_.get());
972   if (FAILED(hr)) {
973     OnError(VideoCaptureError::kWinMediaFoundationSinkSetSampleCallbackFailed,
974             FROM_HERE, hr);
975     return;
976   }
977 
978   // Note, that it is not sufficient to wait for
979   // MF_CAPTURE_ENGINE_PREVIEW_STARTED as an indicator that starting capture has
980   // succeeded. If the capture device is already in use by a different
981   // application, MediaFoundation will still emit
982   // MF_CAPTURE_ENGINE_PREVIEW_STARTED, and only after that raise an error
983   // event. For the lack of any other events indicating success, we have to wait
984   // for the first video frame to arrive before sending our |OnStarted| event to
985   // |client_|.
986   has_sent_on_started_to_client_ = false;
987   hr = engine_->StartPreview();
988   if (FAILED(hr)) {
989     OnError(VideoCaptureError::kWinMediaFoundationEngineStartPreviewFailed,
990             FROM_HERE, hr);
991     return;
992   }
993 
994   selected_video_capability_.reset(
995       new CapabilityWin(best_match_video_capability));
996 
997   is_started_ = true;
998 }
999 
StopAndDeAllocate()1000 void VideoCaptureDeviceMFWin::StopAndDeAllocate() {
1001   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
1002   base::AutoLock lock(lock_);
1003 
1004   if (is_started_ && engine_)
1005     engine_->StopPreview();
1006   is_started_ = false;
1007 
1008   client_.reset();
1009 }
1010 
TakePhoto(TakePhotoCallback callback)1011 void VideoCaptureDeviceMFWin::TakePhoto(TakePhotoCallback callback) {
1012   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
1013   base::AutoLock lock(lock_);
1014 
1015   if (!is_started_)
1016     return;
1017 
1018   if (!selected_photo_capability_) {
1019     video_stream_take_photo_callbacks_.push(std::move(callback));
1020     return;
1021   }
1022 
1023   ComPtr<IMFCaptureSource> source;
1024   HRESULT hr = engine_->GetSource(&source);
1025   if (FAILED(hr)) {
1026     LogError(FROM_HERE, hr);
1027     return;
1028   }
1029 
1030   ComPtr<IMFMediaType> source_media_type;
1031   hr = GetAvailableDeviceMediaType(
1032       source.Get(), selected_photo_capability_->stream_index,
1033       selected_photo_capability_->media_type_index, &source_media_type);
1034   if (FAILED(hr)) {
1035     LogError(FROM_HERE, hr);
1036     return;
1037   }
1038 
1039   hr = source->SetCurrentDeviceMediaType(
1040       selected_photo_capability_->stream_index, source_media_type.Get());
1041   if (FAILED(hr)) {
1042     LogError(FROM_HERE, hr);
1043     return;
1044   }
1045 
1046   ComPtr<IMFMediaType> sink_media_type;
1047   hr = MFCreateMediaType(&sink_media_type);
1048   if (FAILED(hr)) {
1049     LogError(FROM_HERE, hr);
1050     return;
1051   }
1052 
1053   hr = ConvertToPhotoSinkMediaType(source_media_type.Get(),
1054                                    sink_media_type.Get());
1055   if (FAILED(hr)) {
1056     LogError(FROM_HERE, hr);
1057     return;
1058   }
1059 
1060   VideoCaptureFormat format;
1061   hr = GetFormatFromSourceMediaType(sink_media_type.Get(), true,
1062                                     /*use_hardware_format=*/false, &format)
1063            ? S_OK
1064            : E_FAIL;
1065   if (FAILED(hr)) {
1066     LogError(FROM_HERE, hr);
1067     return;
1068   }
1069 
1070   ComPtr<IMFCaptureSink> sink;
1071   hr = engine_->GetSink(MF_CAPTURE_ENGINE_SINK_TYPE_PHOTO, &sink);
1072   if (FAILED(hr)) {
1073     LogError(FROM_HERE, hr);
1074     return;
1075   }
1076 
1077   ComPtr<IMFCapturePhotoSink> photo_sink;
1078   hr = sink->QueryInterface(IID_PPV_ARGS(&photo_sink));
1079   if (FAILED(hr)) {
1080     LogError(FROM_HERE, hr);
1081     return;
1082   }
1083 
1084   hr = photo_sink->RemoveAllStreams();
1085   if (FAILED(hr)) {
1086     LogError(FROM_HERE, hr);
1087     return;
1088   }
1089 
1090   DWORD dw_sink_stream_index = 0;
1091   hr = photo_sink->AddStream(selected_photo_capability_->stream_index,
1092                              sink_media_type.Get(), nullptr,
1093                              &dw_sink_stream_index);
1094   if (FAILED(hr)) {
1095     LogError(FROM_HERE, hr);
1096     return;
1097   }
1098 
1099   scoped_refptr<IMFCaptureEngineOnSampleCallback> photo_callback =
1100       create_mf_photo_callback_.Run(std::move(callback), format);
1101   hr = photo_sink->SetSampleCallback(photo_callback.get());
1102   if (FAILED(hr)) {
1103     LogError(FROM_HERE, hr);
1104     return;
1105   }
1106 
1107   hr = engine_->TakePhoto();
1108   if (FAILED(hr))
1109     LogError(FROM_HERE, hr);
1110 }
1111 
GetPhotoState(GetPhotoStateCallback callback)1112 void VideoCaptureDeviceMFWin::GetPhotoState(GetPhotoStateCallback callback) {
1113   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
1114 
1115   if (!is_started_)
1116     return;
1117 
1118   ComPtr<IMFCaptureSource> source;
1119   HRESULT hr = engine_->GetSource(&source);
1120   if (FAILED(hr)) {
1121     LogError(FROM_HERE, hr);
1122     return;
1123   }
1124 
1125   ComPtr<IMFMediaType> current_media_type;
1126   hr = source->GetCurrentDeviceMediaType(
1127       selected_photo_capability_ ? selected_photo_capability_->stream_index
1128                                  : selected_video_capability_->stream_index,
1129       &current_media_type);
1130   if (FAILED(hr)) {
1131     LogError(FROM_HERE, hr);
1132     return;
1133   }
1134 
1135   auto photo_capabilities = mojo::CreateEmptyPhotoState();
1136   gfx::Size current_size;
1137   GetFrameSizeFromMediaType(current_media_type.Get(), &current_size);
1138 
1139   gfx::Size min_size = gfx::Size(current_size.width(), current_size.height());
1140   gfx::Size max_size = gfx::Size(current_size.width(), current_size.height());
1141   for (const CapabilityWin& capability : photo_capabilities_) {
1142     min_size.SetToMin(capability.supported_format.frame_size);
1143     max_size.SetToMax(capability.supported_format.frame_size);
1144   }
1145 
1146   photo_capabilities->height = mojom::Range::New(
1147       max_size.height(), min_size.height(), current_size.height(), 1);
1148   photo_capabilities->width = mojom::Range::New(
1149       max_size.width(), min_size.width(), current_size.width(), 1);
1150 
1151   if (camera_control_ && video_control_) {
1152     photo_capabilities->color_temperature = RetrieveControlRangeAndCurrent(
1153         video_control_, VideoProcAmp_WhiteBalance,
1154         &photo_capabilities->supported_white_balance_modes,
1155         &photo_capabilities->current_white_balance_mode);
1156 
1157     photo_capabilities->exposure_time = RetrieveControlRangeAndCurrent(
1158         camera_control_, CameraControl_Exposure,
1159         &photo_capabilities->supported_exposure_modes,
1160         &photo_capabilities->current_exposure_mode,
1161         PlatformExposureTimeToCaptureValue, PlatformExposureTimeToCaptureStep);
1162 
1163     photo_capabilities->focus_distance = RetrieveControlRangeAndCurrent(
1164         camera_control_, CameraControl_Focus,
1165         &photo_capabilities->supported_focus_modes,
1166         &photo_capabilities->current_focus_mode);
1167 
1168     photo_capabilities->brightness =
1169         RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Brightness);
1170     photo_capabilities->contrast =
1171         RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Contrast);
1172     photo_capabilities->exposure_compensation =
1173         RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Gain);
1174     // There is no ISO control property in IAMCameraControl or IAMVideoProcAmp
1175     // interfaces nor any other control property with direct mapping to ISO.
1176     photo_capabilities->iso = mojom::Range::New();
1177     photo_capabilities->red_eye_reduction = mojom::RedEyeReduction::NEVER;
1178     photo_capabilities->saturation =
1179         RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Saturation);
1180     photo_capabilities->sharpness =
1181         RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Sharpness);
1182     photo_capabilities->torch = false;
1183     photo_capabilities->pan = RetrieveControlRangeAndCurrent(
1184         camera_control_, CameraControl_Pan, nullptr, nullptr,
1185         PlatformAngleToCaptureValue, PlatformAngleToCaptureStep);
1186     photo_capabilities->tilt = RetrieveControlRangeAndCurrent(
1187         camera_control_, CameraControl_Tilt, nullptr, nullptr,
1188         PlatformAngleToCaptureValue, PlatformAngleToCaptureStep);
1189     photo_capabilities->zoom =
1190         RetrieveControlRangeAndCurrent(camera_control_, CameraControl_Zoom);
1191   }
1192 
1193   std::move(callback).Run(std::move(photo_capabilities));
1194 }
1195 
SetPhotoOptions(mojom::PhotoSettingsPtr settings,SetPhotoOptionsCallback callback)1196 void VideoCaptureDeviceMFWin::SetPhotoOptions(
1197     mojom::PhotoSettingsPtr settings,
1198     SetPhotoOptionsCallback callback) {
1199   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
1200 
1201   if (!is_started_)
1202     return;
1203 
1204   HRESULT hr = S_OK;
1205   ComPtr<IMFCaptureSource> source;
1206   hr = engine_->GetSource(&source);
1207 
1208   if (FAILED(hr)) {
1209     LogError(FROM_HERE, hr);
1210     return;
1211   }
1212 
1213   if (!photo_capabilities_.empty() &&
1214       (settings->has_height || settings->has_width)) {
1215     if (FAILED(hr)) {
1216       LogError(FROM_HERE, hr);
1217       return;
1218     }
1219 
1220     ComPtr<IMFMediaType> current_source_media_type;
1221     hr = source->GetCurrentDeviceMediaType(
1222         selected_photo_capability_->stream_index, &current_source_media_type);
1223 
1224     if (FAILED(hr)) {
1225       LogError(FROM_HERE, hr);
1226       return;
1227     }
1228 
1229     gfx::Size requested_size = gfx::Size();
1230     if (settings->has_height)
1231       requested_size.set_height(settings->height);
1232 
1233     if (settings->has_width)
1234       requested_size.set_width(settings->width);
1235 
1236     const CapabilityWin best_match = GetBestMatchedPhotoCapability(
1237         current_source_media_type, requested_size, photo_capabilities_);
1238     selected_photo_capability_.reset(new CapabilityWin(best_match));
1239   }
1240 
1241   if (camera_control_ && video_control_) {
1242     if (settings->has_white_balance_mode) {
1243       if (settings->white_balance_mode == mojom::MeteringMode::CONTINUOUS) {
1244         hr = video_control_->Set(VideoProcAmp_WhiteBalance, 0L,
1245                                  VideoProcAmp_Flags_Auto);
1246         DLOG_IF_FAILED_WITH_HRESULT("Auto white balance config failed", hr);
1247         if (FAILED(hr))
1248           return;
1249         white_balance_mode_manual_ = false;
1250       } else {
1251         white_balance_mode_manual_ = true;
1252       }
1253     }
1254     if (white_balance_mode_manual_ && settings->has_color_temperature) {
1255       hr = video_control_->Set(VideoProcAmp_WhiteBalance,
1256                                settings->color_temperature,
1257                                VideoProcAmp_Flags_Manual);
1258       DLOG_IF_FAILED_WITH_HRESULT("Color temperature config failed", hr);
1259       if (FAILED(hr))
1260         return;
1261     }
1262 
1263     if (settings->has_exposure_mode) {
1264       if (settings->exposure_mode == mojom::MeteringMode::CONTINUOUS) {
1265         hr = camera_control_->Set(CameraControl_Exposure, 0L,
1266                                   CameraControl_Flags_Auto);
1267         DLOG_IF_FAILED_WITH_HRESULT("Auto exposure config failed", hr);
1268         if (FAILED(hr))
1269           return;
1270         exposure_mode_manual_ = false;
1271       } else {
1272         exposure_mode_manual_ = true;
1273       }
1274     }
1275     if (exposure_mode_manual_ && settings->has_exposure_time) {
1276       hr = camera_control_->Set(
1277           CameraControl_Exposure,
1278           CaptureExposureTimeToPlatformValue(settings->exposure_time),
1279           CameraControl_Flags_Manual);
1280       DLOG_IF_FAILED_WITH_HRESULT("Exposure Time config failed", hr);
1281       if (FAILED(hr))
1282         return;
1283     }
1284 
1285     if (settings->has_focus_mode) {
1286       if (settings->focus_mode == mojom::MeteringMode::CONTINUOUS) {
1287         hr = camera_control_->Set(CameraControl_Focus, 0L,
1288                                   CameraControl_Flags_Auto);
1289         DLOG_IF_FAILED_WITH_HRESULT("Auto focus config failed", hr);
1290         if (FAILED(hr))
1291           return;
1292         focus_mode_manual_ = false;
1293       } else {
1294         focus_mode_manual_ = true;
1295       }
1296     }
1297     if (focus_mode_manual_ && settings->has_focus_distance) {
1298       hr = camera_control_->Set(CameraControl_Focus, settings->focus_distance,
1299                                 CameraControl_Flags_Manual);
1300       DLOG_IF_FAILED_WITH_HRESULT("Focus Distance config failed", hr);
1301       if (FAILED(hr))
1302         return;
1303     }
1304 
1305     if (settings->has_brightness) {
1306       hr = video_control_->Set(VideoProcAmp_Brightness, settings->brightness,
1307                                VideoProcAmp_Flags_Manual);
1308       DLOG_IF_FAILED_WITH_HRESULT("Brightness config failed", hr);
1309       if (FAILED(hr))
1310         return;
1311     }
1312     if (settings->has_contrast) {
1313       hr = video_control_->Set(VideoProcAmp_Contrast, settings->contrast,
1314                                VideoProcAmp_Flags_Manual);
1315       DLOG_IF_FAILED_WITH_HRESULT("Contrast config failed", hr);
1316       if (FAILED(hr))
1317         return;
1318     }
1319     if (settings->has_exposure_compensation) {
1320       hr = video_control_->Set(VideoProcAmp_Gain,
1321                                settings->exposure_compensation,
1322                                VideoProcAmp_Flags_Manual);
1323       DLOG_IF_FAILED_WITH_HRESULT("Exposure Compensation config failed", hr);
1324       if (FAILED(hr))
1325         return;
1326     }
1327     if (settings->has_saturation) {
1328       hr = video_control_->Set(VideoProcAmp_Saturation, settings->saturation,
1329                                VideoProcAmp_Flags_Manual);
1330       DLOG_IF_FAILED_WITH_HRESULT("Saturation config failed", hr);
1331       if (FAILED(hr))
1332         return;
1333     }
1334     if (settings->has_sharpness) {
1335       hr = video_control_->Set(VideoProcAmp_Sharpness, settings->sharpness,
1336                                VideoProcAmp_Flags_Manual);
1337       DLOG_IF_FAILED_WITH_HRESULT("Sharpness config failed", hr);
1338       if (FAILED(hr))
1339         return;
1340     }
1341     if (settings->has_pan) {
1342       hr = camera_control_->Set(CameraControl_Pan,
1343                                 CaptureAngleToPlatformValue(settings->pan),
1344                                 CameraControl_Flags_Manual);
1345       DLOG_IF_FAILED_WITH_HRESULT("Pan config failed", hr);
1346       if (FAILED(hr))
1347         return;
1348     }
1349     if (settings->has_tilt) {
1350       hr = camera_control_->Set(CameraControl_Tilt,
1351                                 CaptureAngleToPlatformValue(settings->tilt),
1352                                 CameraControl_Flags_Manual);
1353       DLOG_IF_FAILED_WITH_HRESULT("Tilt config failed", hr);
1354       if (FAILED(hr))
1355         return;
1356     }
1357     if (settings->has_zoom) {
1358       hr = camera_control_->Set(CameraControl_Zoom, settings->zoom,
1359                                 CameraControl_Flags_Manual);
1360       DLOG_IF_FAILED_WITH_HRESULT("Zoom config failed", hr);
1361       if (FAILED(hr))
1362         return;
1363     }
1364   }
1365 
1366   std::move(callback).Run(true);
1367 }
1368 
OnIncomingCapturedData(const uint8_t * data,int length,base::TimeTicks reference_time,base::TimeDelta timestamp)1369 void VideoCaptureDeviceMFWin::OnIncomingCapturedData(
1370     const uint8_t* data,
1371     int length,
1372     base::TimeTicks reference_time,
1373     base::TimeDelta timestamp) {
1374   base::AutoLock lock(lock_);
1375   DCHECK(data);
1376 
1377   SendOnStartedIfNotYetSent();
1378 
1379   if (client_.get()) {
1380     if (!has_sent_on_started_to_client_) {
1381       has_sent_on_started_to_client_ = true;
1382       client_->OnStarted();
1383     }
1384 
1385     // We always calculate camera rotation for the first frame. We also cache
1386     // the latest value to use when AutoRotation is turned off.
1387     if (!camera_rotation_.has_value() || IsAutoRotationEnabled())
1388       camera_rotation_ = GetCameraRotation(facing_mode_);
1389 
1390     // TODO(julien.isorce): retrieve the color space information using Media
1391     // Foundation api, MFGetAttributeSize/MF_MT_VIDEO_PRIMARIES,in order to
1392     // build a gfx::ColorSpace. See http://crbug.com/959988.
1393     client_->OnIncomingCapturedData(
1394         data, length, selected_video_capability_->supported_format,
1395         gfx::ColorSpace(), camera_rotation_.value(), false /* flip_y */,
1396         reference_time, timestamp);
1397   }
1398 
1399   while (!video_stream_take_photo_callbacks_.empty()) {
1400     TakePhotoCallback cb =
1401         std::move(video_stream_take_photo_callbacks_.front());
1402     video_stream_take_photo_callbacks_.pop();
1403 
1404     mojom::BlobPtr blob = RotateAndBlobify(
1405         data, length, selected_video_capability_->supported_format, 0);
1406     if (!blob) {
1407       LogWindowsImageCaptureOutcome(
1408           VideoCaptureWinBackend::kMediaFoundation,
1409           ImageCaptureOutcome::kFailedUsingVideoStream,
1410           IsHighResolution(selected_video_capability_->supported_format));
1411       continue;
1412     }
1413 
1414     std::move(cb).Run(std::move(blob));
1415     LogWindowsImageCaptureOutcome(
1416         VideoCaptureWinBackend::kMediaFoundation,
1417         ImageCaptureOutcome::kSucceededUsingVideoStream,
1418         IsHighResolution(selected_video_capability_->supported_format));
1419   }
1420 }
1421 
OnFrameDropped(VideoCaptureFrameDropReason reason)1422 void VideoCaptureDeviceMFWin::OnFrameDropped(
1423     VideoCaptureFrameDropReason reason) {
1424   base::AutoLock lock(lock_);
1425 
1426   SendOnStartedIfNotYetSent();
1427 
1428   if (client_.get()) {
1429     client_->OnFrameDropped(reason);
1430   }
1431 }
1432 
OnEvent(IMFMediaEvent * media_event)1433 void VideoCaptureDeviceMFWin::OnEvent(IMFMediaEvent* media_event) {
1434   base::AutoLock lock(lock_);
1435 
1436   HRESULT hr;
1437   GUID capture_event_guid = GUID_NULL;
1438 
1439   media_event->GetStatus(&hr);
1440   media_event->GetExtendedType(&capture_event_guid);
1441   // TODO(http://crbug.com/1093521): Add cases for Start
1442   // MF_CAPTURE_ENGINE_PREVIEW_STARTED and MF_CAPTURE_ENGINE_PREVIEW_STOPPED
1443   // When MF_CAPTURE_ENGINE_ERROR is returned the captureengine object is no
1444   // longer valid.
1445   if (capture_event_guid == MF_CAPTURE_ENGINE_ERROR || FAILED(hr)) {
1446     capture_error_.Signal();
1447     // There should always be a valid error
1448     hr = SUCCEEDED(hr) ? E_UNEXPECTED : hr;
1449   } else if (capture_event_guid == MF_CAPTURE_ENGINE_INITIALIZED) {
1450     capture_initialize_.Signal();
1451   }
1452 
1453   if (FAILED(hr))
1454     OnError(VideoCaptureError::kWinMediaFoundationGetMediaEventStatusFailed,
1455             FROM_HERE, hr);
1456 }
1457 
OnError(VideoCaptureError error,const Location & from_here,HRESULT hr)1458 void VideoCaptureDeviceMFWin::OnError(VideoCaptureError error,
1459                                       const Location& from_here,
1460                                       HRESULT hr) {
1461   OnError(error, from_here, logging::SystemErrorCodeToString(hr).c_str());
1462 }
1463 
OnError(VideoCaptureError error,const Location & from_here,const char * message)1464 void VideoCaptureDeviceMFWin::OnError(VideoCaptureError error,
1465                                       const Location& from_here,
1466                                       const char* message) {
1467   if (!client_.get())
1468     return;
1469 
1470   client_->OnError(error, from_here,
1471                    base::StringPrintf("VideoCaptureDeviceMFWin: %s", message));
1472 }
1473 
SendOnStartedIfNotYetSent()1474 void VideoCaptureDeviceMFWin::SendOnStartedIfNotYetSent() {
1475   if (!client_ || has_sent_on_started_to_client_)
1476     return;
1477   has_sent_on_started_to_client_ = true;
1478   client_->OnStarted();
1479 }
1480 
WaitOnCaptureEvent(GUID capture_event_guid)1481 HRESULT VideoCaptureDeviceMFWin::WaitOnCaptureEvent(GUID capture_event_guid) {
1482   HRESULT hr = S_OK;
1483   HANDLE events[] = {nullptr, capture_error_.handle()};
1484 
1485   // TODO(http://crbug.com/1093521): Add cases for Start
1486   // MF_CAPTURE_ENGINE_PREVIEW_STARTED and MF_CAPTURE_ENGINE_PREVIEW_STOPPED
1487   if (capture_event_guid == MF_CAPTURE_ENGINE_INITIALIZED) {
1488     events[0] = capture_initialize_.handle();
1489   } else {
1490     // no registered event handle for the event requested
1491     hr = E_NOTIMPL;
1492     LogError(FROM_HERE, hr);
1493     return hr;
1494   }
1495 
1496   DWORD wait_result =
1497       ::WaitForMultipleObjects(base::size(events), events, FALSE, INFINITE);
1498   switch (wait_result) {
1499     case WAIT_OBJECT_0:
1500       break;
1501     case WAIT_FAILED:
1502       hr = HRESULT_FROM_WIN32(::GetLastError());
1503       LogError(FROM_HERE, hr);
1504       break;
1505     default:
1506       hr = E_UNEXPECTED;
1507       LogError(FROM_HERE, hr);
1508       break;
1509   }
1510   return hr;
1511 }
1512 }  // namespace media
1513