1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "third_party/blink/renderer/modules/imagecapture/image_capture.h"
6 
7 #include <memory>
8 #include <utility>
9 
10 #include "third_party/blink/public/common/browser_interface_broker_proxy.h"
11 #include "third_party/blink/public/platform/platform.h"
12 #include "third_party/blink/public/platform/web_media_stream_track.h"
13 #include "third_party/blink/renderer/bindings/core/v8/callback_promise_adapter.h"
14 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
15 #include "third_party/blink/renderer/bindings/modules/v8/v8_media_track_capabilities.h"
16 #include "third_party/blink/renderer/bindings/modules/v8/v8_media_track_constraints.h"
17 #include "third_party/blink/renderer/core/dom/dom_exception.h"
18 #include "third_party/blink/renderer/core/fileapi/blob.h"
19 #include "third_party/blink/renderer/core/frame/local_frame.h"
20 #include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h"
21 #include "third_party/blink/renderer/modules/event_target_modules.h"
22 #include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
23 #include "third_party/blink/renderer/modules/imagecapture/media_settings_range.h"
24 #include "third_party/blink/renderer/modules/imagecapture/photo_capabilities.h"
25 #include "third_party/blink/renderer/modules/mediastream/media_stream_track.h"
26 #include "third_party/blink/renderer/platform/heap/heap.h"
27 #include "third_party/blink/renderer/platform/mojo/mojo_helper.h"
28 #include "third_party/blink/renderer/platform/wtf/functional.h"
29 
30 namespace blink {
31 
32 using FillLightMode = media::mojom::blink::FillLightMode;
33 using MeteringMode = media::mojom::blink::MeteringMode;
34 
35 namespace {
36 
37 const char kNoServiceError[] = "ImageCapture service unavailable.";
38 
TrackIsInactive(const MediaStreamTrack & track)39 bool TrackIsInactive(const MediaStreamTrack& track) {
40   // Spec instructs to return an exception if the Track's readyState() is not
41   // "live". Also reject if the track is disabled or muted.
42   return track.readyState() != "live" || !track.enabled() || track.muted();
43 }
44 
ParseMeteringMode(const String & blink_mode)45 MeteringMode ParseMeteringMode(const String& blink_mode) {
46   if (blink_mode == "manual")
47     return MeteringMode::MANUAL;
48   if (blink_mode == "single-shot")
49     return MeteringMode::SINGLE_SHOT;
50   if (blink_mode == "continuous")
51     return MeteringMode::CONTINUOUS;
52   if (blink_mode == "none")
53     return MeteringMode::NONE;
54   NOTREACHED();
55   return MeteringMode::NONE;
56 }
57 
ParseFillLightMode(const String & blink_mode)58 FillLightMode ParseFillLightMode(const String& blink_mode) {
59   if (blink_mode == "off")
60     return FillLightMode::OFF;
61   if (blink_mode == "auto")
62     return FillLightMode::AUTO;
63   if (blink_mode == "flash")
64     return FillLightMode::FLASH;
65   NOTREACHED();
66   return FillLightMode::OFF;
67 }
68 
ToString(MeteringMode value)69 WebString ToString(MeteringMode value) {
70   switch (value) {
71     case MeteringMode::NONE:
72       return WebString::FromUTF8("none");
73     case MeteringMode::MANUAL:
74       return WebString::FromUTF8("manual");
75     case MeteringMode::SINGLE_SHOT:
76       return WebString::FromUTF8("single-shot");
77     case MeteringMode::CONTINUOUS:
78       return WebString::FromUTF8("continuous");
79     default:
80       NOTREACHED() << "Unknown MeteringMode";
81   }
82   return WebString();
83 }
84 
85 }  // anonymous namespace
86 
Create(ExecutionContext * context,MediaStreamTrack * track,ExceptionState & exception_state)87 ImageCapture* ImageCapture::Create(ExecutionContext* context,
88                                    MediaStreamTrack* track,
89                                    ExceptionState& exception_state) {
90   if (track->kind() != "video") {
91     exception_state.ThrowDOMException(
92         DOMExceptionCode::kNotSupportedError,
93         "Cannot create an ImageCapturer from a non-video Track.");
94     return nullptr;
95   }
96 
97   return MakeGarbageCollected<ImageCapture>(context, track);
98 }
99 
~ImageCapture()100 ImageCapture::~ImageCapture() {
101   DCHECK(!HasEventListeners());
102   // There should be no more outstanding |m_serviceRequests| at this point
103   // since each of them holds a persistent handle to this object.
104   DCHECK(service_requests_.IsEmpty());
105 }
106 
InterfaceName() const107 const AtomicString& ImageCapture::InterfaceName() const {
108   return event_target_names::kImageCapture;
109 }
110 
GetExecutionContext() const111 ExecutionContext* ImageCapture::GetExecutionContext() const {
112   return ExecutionContextLifecycleObserver::GetExecutionContext();
113 }
114 
HasPendingActivity() const115 bool ImageCapture::HasPendingActivity() const {
116   return GetExecutionContext() && HasEventListeners();
117 }
118 
ContextDestroyed()119 void ImageCapture::ContextDestroyed() {
120   RemoveAllEventListeners();
121   service_requests_.clear();
122   DCHECK(!HasEventListeners());
123 }
124 
getPhotoCapabilities(ScriptState * script_state)125 ScriptPromise ImageCapture::getPhotoCapabilities(ScriptState* script_state) {
126   auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
127   ScriptPromise promise = resolver->Promise();
128 
129   if (!service_) {
130     resolver->Reject(MakeGarbageCollected<DOMException>(
131         DOMExceptionCode::kNotFoundError, kNoServiceError));
132     return promise;
133   }
134   service_requests_.insert(resolver);
135 
136   auto resolver_cb = WTF::Bind(&ImageCapture::ResolveWithPhotoCapabilities,
137                                WrapPersistent(this));
138 
139   // m_streamTrack->component()->source()->id() is the renderer "name" of the
140   // camera;
141   // TODO(mcasas) consider sending the security origin as well:
142   // scriptState->getExecutionContext()->getSecurityOrigin()->toString()
143   service_->GetPhotoState(
144       stream_track_->Component()->Source()->Id(),
145       WTF::Bind(&ImageCapture::OnMojoGetPhotoState, WrapPersistent(this),
146                 WrapPersistent(resolver), WTF::Passed(std::move(resolver_cb)),
147                 false /* trigger_take_photo */));
148   return promise;
149 }
150 
getPhotoSettings(ScriptState * script_state)151 ScriptPromise ImageCapture::getPhotoSettings(ScriptState* script_state) {
152   auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
153   ScriptPromise promise = resolver->Promise();
154 
155   if (!service_) {
156     resolver->Reject(MakeGarbageCollected<DOMException>(
157         DOMExceptionCode::kNotFoundError, kNoServiceError));
158     return promise;
159   }
160   service_requests_.insert(resolver);
161 
162   auto resolver_cb =
163       WTF::Bind(&ImageCapture::ResolveWithPhotoSettings, WrapPersistent(this));
164 
165   // m_streamTrack->component()->source()->id() is the renderer "name" of the
166   // camera;
167   // TODO(mcasas) consider sending the security origin as well:
168   // scriptState->getExecutionContext()->getSecurityOrigin()->toString()
169   service_->GetPhotoState(
170       stream_track_->Component()->Source()->Id(),
171       WTF::Bind(&ImageCapture::OnMojoGetPhotoState, WrapPersistent(this),
172                 WrapPersistent(resolver), WTF::Passed(std::move(resolver_cb)),
173                 false /* trigger_take_photo */));
174   return promise;
175 }
176 
setOptions(ScriptState * script_state,const PhotoSettings * photo_settings,bool trigger_take_photo)177 ScriptPromise ImageCapture::setOptions(ScriptState* script_state,
178                                        const PhotoSettings* photo_settings,
179                                        bool trigger_take_photo /* = false */) {
180   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
181                        "ImageCapture::setOptions", TRACE_EVENT_SCOPE_PROCESS);
182   auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
183   ScriptPromise promise = resolver->Promise();
184 
185   if (TrackIsInactive(*stream_track_)) {
186     resolver->Reject(MakeGarbageCollected<DOMException>(
187         DOMExceptionCode::kInvalidStateError,
188         "The associated Track is in an invalid state."));
189     return promise;
190   }
191 
192   if (!service_) {
193     resolver->Reject(MakeGarbageCollected<DOMException>(
194         DOMExceptionCode::kNotFoundError, kNoServiceError));
195     return promise;
196   }
197   service_requests_.insert(resolver);
198 
199   // TODO(mcasas): should be using a mojo::StructTraits instead.
200   auto settings = media::mojom::blink::PhotoSettings::New();
201 
202   settings->has_height = photo_settings->hasImageHeight();
203   if (settings->has_height) {
204     const double height = photo_settings->imageHeight();
205     if (photo_capabilities_ &&
206         (height < photo_capabilities_->imageHeight()->min() ||
207          height > photo_capabilities_->imageHeight()->max())) {
208       resolver->Reject(MakeGarbageCollected<DOMException>(
209           DOMExceptionCode::kNotSupportedError,
210           "imageHeight setting out of range"));
211       return promise;
212     }
213     settings->height = height;
214   }
215   settings->has_width = photo_settings->hasImageWidth();
216   if (settings->has_width) {
217     const double width = photo_settings->imageWidth();
218     if (photo_capabilities_ &&
219         (width < photo_capabilities_->imageWidth()->min() ||
220          width > photo_capabilities_->imageWidth()->max())) {
221       resolver->Reject(MakeGarbageCollected<DOMException>(
222           DOMExceptionCode::kNotSupportedError,
223           "imageWidth setting out of range"));
224       return promise;
225     }
226     settings->width = width;
227   }
228 
229   settings->has_red_eye_reduction = photo_settings->hasRedEyeReduction();
230   if (settings->has_red_eye_reduction) {
231     if (photo_capabilities_ &&
232         !photo_capabilities_->IsRedEyeReductionControllable()) {
233       resolver->Reject(MakeGarbageCollected<DOMException>(
234           DOMExceptionCode::kNotSupportedError,
235           "redEyeReduction is not controllable."));
236       return promise;
237     }
238     settings->red_eye_reduction = photo_settings->redEyeReduction();
239   }
240 
241   settings->has_fill_light_mode = photo_settings->hasFillLightMode();
242   if (settings->has_fill_light_mode) {
243     const String fill_light_mode = photo_settings->fillLightMode();
244     if (photo_capabilities_ && photo_capabilities_->fillLightMode().Find(
245                                    fill_light_mode) == kNotFound) {
246       resolver->Reject(MakeGarbageCollected<DOMException>(
247           DOMExceptionCode::kNotSupportedError, "Unsupported fillLightMode"));
248       return promise;
249     }
250     settings->fill_light_mode = ParseFillLightMode(fill_light_mode);
251   }
252 
253   service_->SetOptions(
254       stream_track_->Component()->Source()->Id(), std::move(settings),
255       WTF::Bind(&ImageCapture::OnMojoSetOptions, WrapPersistent(this),
256                 WrapPersistent(resolver), trigger_take_photo));
257   return promise;
258 }
259 
takePhoto(ScriptState * script_state)260 ScriptPromise ImageCapture::takePhoto(ScriptState* script_state) {
261   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
262                        "ImageCapture::takePhoto", TRACE_EVENT_SCOPE_PROCESS);
263   auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
264   ScriptPromise promise = resolver->Promise();
265 
266   if (TrackIsInactive(*stream_track_)) {
267     resolver->Reject(MakeGarbageCollected<DOMException>(
268         DOMExceptionCode::kInvalidStateError,
269         "The associated Track is in an invalid state."));
270     return promise;
271   }
272   if (!service_) {
273     resolver->Reject(MakeGarbageCollected<DOMException>(
274         DOMExceptionCode::kNotFoundError, kNoServiceError));
275     return promise;
276   }
277 
278   service_requests_.insert(resolver);
279 
280   // m_streamTrack->component()->source()->id() is the renderer "name" of the
281   // camera;
282   // TODO(mcasas) consider sending the security origin as well:
283   // scriptState->getExecutionContext()->getSecurityOrigin()->toString()
284   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
285                        "ImageCapture::takePhoto", TRACE_EVENT_SCOPE_PROCESS);
286   service_->TakePhoto(
287       stream_track_->Component()->Source()->Id(),
288       WTF::Bind(&ImageCapture::OnMojoTakePhoto, WrapPersistent(this),
289                 WrapPersistent(resolver)));
290   return promise;
291 }
292 
takePhoto(ScriptState * script_state,const PhotoSettings * photo_settings)293 ScriptPromise ImageCapture::takePhoto(ScriptState* script_state,
294                                       const PhotoSettings* photo_settings) {
295   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
296                        "ImageCapture::takePhoto (with settings)",
297                        TRACE_EVENT_SCOPE_PROCESS);
298 
299   return setOptions(script_state, photo_settings,
300                     true /* trigger_take_photo */);
301 }
302 
grabFrame(ScriptState * script_state)303 ScriptPromise ImageCapture::grabFrame(ScriptState* script_state) {
304   auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
305   ScriptPromise promise = resolver->Promise();
306 
307   if (TrackIsInactive(*stream_track_)) {
308     resolver->Reject(MakeGarbageCollected<DOMException>(
309         DOMExceptionCode::kInvalidStateError,
310         "The associated Track is in an invalid state."));
311     return promise;
312   }
313 
314   // Create |m_frameGrabber| the first time.
315   if (!frame_grabber_) {
316     frame_grabber_ = std::make_unique<ImageCaptureFrameGrabber>();
317   }
318 
319   if (!frame_grabber_) {
320     resolver->Reject(MakeGarbageCollected<DOMException>(
321         DOMExceptionCode::kUnknownError, "Couldn't create platform resources"));
322     return promise;
323   }
324 
325   // The platform does not know about MediaStreamTrack, so we wrap it up.
326   WebMediaStreamTrack track(stream_track_->Component());
327   auto resolver_callback_adapter =
328       std::make_unique<CallbackPromiseAdapter<ImageBitmap, void>>(resolver);
329   frame_grabber_->GrabFrame(&track, std::move(resolver_callback_adapter),
330                             ExecutionContext::From(script_state)
331                                 ->GetTaskRunner(TaskType::kDOMManipulation));
332 
333   return promise;
334 }
335 
GetMediaTrackCapabilities() const336 MediaTrackCapabilities* ImageCapture::GetMediaTrackCapabilities() const {
337   return capabilities_;
338 }
339 
340 // TODO(mcasas): make the implementation fully Spec compliant, see the TODOs
341 // inside the method, https://crbug.com/708723.
SetMediaTrackConstraints(ScriptPromiseResolver * resolver,const HeapVector<Member<MediaTrackConstraintSet>> & constraints_vector)342 void ImageCapture::SetMediaTrackConstraints(
343     ScriptPromiseResolver* resolver,
344     const HeapVector<Member<MediaTrackConstraintSet>>& constraints_vector) {
345   DCHECK_GT(constraints_vector.size(), 0u);
346   if (!service_) {
347     resolver->Reject(MakeGarbageCollected<DOMException>(
348         DOMExceptionCode::kNotFoundError, kNoServiceError));
349     return;
350   }
351   // TODO(mcasas): add support more than one single advanced constraint.
352   const MediaTrackConstraintSet* constraints = constraints_vector[0];
353 
354   if ((constraints->hasWhiteBalanceMode() &&
355        !capabilities_->hasWhiteBalanceMode()) ||
356       (constraints->hasExposureMode() && !capabilities_->hasExposureMode()) ||
357       (constraints->hasFocusMode() && !capabilities_->hasFocusMode()) ||
358       (constraints->hasExposureCompensation() &&
359        !capabilities_->hasExposureCompensation()) ||
360       (constraints->hasExposureTime() && !capabilities_->hasExposureTime()) ||
361       (constraints->hasColorTemperature() &&
362        !capabilities_->hasColorTemperature()) ||
363       (constraints->hasIso() && !capabilities_->hasIso()) ||
364       (constraints->hasBrightness() && !capabilities_->hasBrightness()) ||
365       (constraints->hasContrast() && !capabilities_->hasContrast()) ||
366       (constraints->hasSaturation() && !capabilities_->hasSaturation()) ||
367       (constraints->hasSharpness() && !capabilities_->hasSharpness()) ||
368       (constraints->hasFocusDistance() && !capabilities_->hasFocusDistance()) ||
369       (constraints->hasPan() && !capabilities_->hasPan()) ||
370       (constraints->hasTilt() && !capabilities_->hasTilt()) ||
371       (constraints->hasZoom() && !capabilities_->hasZoom()) ||
372       (constraints->hasTorch() && !capabilities_->hasTorch())) {
373     resolver->Reject(MakeGarbageCollected<DOMException>(
374         DOMExceptionCode::kNotSupportedError, "Unsupported constraint(s)"));
375     return;
376   }
377 
378   auto settings = media::mojom::blink::PhotoSettings::New();
379   MediaTrackConstraintSet* temp_constraints = current_constraints_;
380 
381   // TODO(mcasas): support other Mode types beyond simple string i.e. the
382   // equivalents of "sequence<DOMString>"" or "ConstrainDOMStringParameters".
383   settings->has_white_balance_mode = constraints->hasWhiteBalanceMode() &&
384                                      constraints->whiteBalanceMode().IsString();
385   if (settings->has_white_balance_mode) {
386     const auto white_balance_mode =
387         constraints->whiteBalanceMode().GetAsString();
388     if (capabilities_->whiteBalanceMode().Find(white_balance_mode) ==
389         kNotFound) {
390       resolver->Reject(MakeGarbageCollected<DOMException>(
391           DOMExceptionCode::kNotSupportedError,
392           "Unsupported whiteBalanceMode."));
393       return;
394     }
395     temp_constraints->setWhiteBalanceMode(constraints->whiteBalanceMode());
396     settings->white_balance_mode = ParseMeteringMode(white_balance_mode);
397   }
398   settings->has_exposure_mode =
399       constraints->hasExposureMode() && constraints->exposureMode().IsString();
400   if (settings->has_exposure_mode) {
401     const auto exposure_mode = constraints->exposureMode().GetAsString();
402     if (capabilities_->exposureMode().Find(exposure_mode) == kNotFound) {
403       resolver->Reject(MakeGarbageCollected<DOMException>(
404           DOMExceptionCode::kNotSupportedError, "Unsupported exposureMode."));
405       return;
406     }
407     temp_constraints->setExposureMode(constraints->exposureMode());
408     settings->exposure_mode = ParseMeteringMode(exposure_mode);
409   }
410 
411   settings->has_focus_mode =
412       constraints->hasFocusMode() && constraints->focusMode().IsString();
413   if (settings->has_focus_mode) {
414     const auto focus_mode = constraints->focusMode().GetAsString();
415     if (capabilities_->focusMode().Find(focus_mode) == kNotFound) {
416       resolver->Reject(MakeGarbageCollected<DOMException>(
417           DOMExceptionCode::kNotSupportedError, "Unsupported focusMode."));
418       return;
419     }
420     temp_constraints->setFocusMode(constraints->focusMode());
421     settings->focus_mode = ParseMeteringMode(focus_mode);
422   }
423 
424   // TODO(mcasas): support ConstrainPoint2DParameters.
425   if (constraints->hasPointsOfInterest() &&
426       constraints->pointsOfInterest().IsPoint2DSequence()) {
427     for (const auto& point :
428          constraints->pointsOfInterest().GetAsPoint2DSequence()) {
429       auto mojo_point = media::mojom::blink::Point2D::New();
430       mojo_point->x = point->x();
431       mojo_point->y = point->y();
432       settings->points_of_interest.push_back(std::move(mojo_point));
433     }
434     temp_constraints->setPointsOfInterest(constraints->pointsOfInterest());
435   }
436 
437   // TODO(mcasas): support ConstrainDoubleRange where applicable.
438   settings->has_exposure_compensation =
439       constraints->hasExposureCompensation() &&
440       constraints->exposureCompensation().IsDouble();
441   if (settings->has_exposure_compensation) {
442     const auto exposure_compensation =
443         constraints->exposureCompensation().GetAsDouble();
444     if (exposure_compensation < capabilities_->exposureCompensation()->min() ||
445         exposure_compensation > capabilities_->exposureCompensation()->max()) {
446       resolver->Reject(MakeGarbageCollected<DOMException>(
447           DOMExceptionCode::kNotSupportedError,
448           "exposureCompensation setting out of range"));
449       return;
450     }
451     temp_constraints->setExposureCompensation(
452         constraints->exposureCompensation());
453     settings->exposure_compensation = exposure_compensation;
454   }
455 
456   settings->has_exposure_time =
457       constraints->hasExposureTime() && constraints->exposureTime().IsDouble();
458   if (settings->has_exposure_time) {
459     const auto exposure_time = constraints->exposureTime().GetAsDouble();
460     if (exposure_time < capabilities_->exposureTime()->min() ||
461         exposure_time > capabilities_->exposureTime()->max()) {
462       resolver->Reject(MakeGarbageCollected<DOMException>(
463           DOMExceptionCode::kNotSupportedError,
464           "exposureTime setting out of range"));
465       return;
466     }
467     temp_constraints->setExposureTime(constraints->exposureTime());
468     settings->exposure_time = exposure_time;
469   }
470   settings->has_color_temperature = constraints->hasColorTemperature() &&
471                                     constraints->colorTemperature().IsDouble();
472   if (settings->has_color_temperature) {
473     const auto color_temperature =
474         constraints->colorTemperature().GetAsDouble();
475     if (color_temperature < capabilities_->colorTemperature()->min() ||
476         color_temperature > capabilities_->colorTemperature()->max()) {
477       resolver->Reject(MakeGarbageCollected<DOMException>(
478           DOMExceptionCode::kNotSupportedError,
479           "colorTemperature setting out of range"));
480       return;
481     }
482     temp_constraints->setColorTemperature(constraints->colorTemperature());
483     settings->color_temperature = color_temperature;
484   }
485   settings->has_iso = constraints->hasIso() && constraints->iso().IsDouble();
486   if (settings->has_iso) {
487     const auto iso = constraints->iso().GetAsDouble();
488     if (iso < capabilities_->iso()->min() ||
489         iso > capabilities_->iso()->max()) {
490       resolver->Reject(MakeGarbageCollected<DOMException>(
491           DOMExceptionCode::kNotSupportedError, "iso setting out of range"));
492       return;
493     }
494     temp_constraints->setIso(constraints->iso());
495     settings->iso = iso;
496   }
497 
498   settings->has_brightness =
499       constraints->hasBrightness() && constraints->brightness().IsDouble();
500   if (settings->has_brightness) {
501     const auto brightness = constraints->brightness().GetAsDouble();
502     if (brightness < capabilities_->brightness()->min() ||
503         brightness > capabilities_->brightness()->max()) {
504       resolver->Reject(MakeGarbageCollected<DOMException>(
505           DOMExceptionCode::kNotSupportedError,
506           "brightness setting out of range"));
507       return;
508     }
509     temp_constraints->setBrightness(constraints->brightness());
510     settings->brightness = brightness;
511   }
512   settings->has_contrast =
513       constraints->hasContrast() && constraints->contrast().IsDouble();
514   if (settings->has_contrast) {
515     const auto contrast = constraints->contrast().GetAsDouble();
516     if (contrast < capabilities_->contrast()->min() ||
517         contrast > capabilities_->contrast()->max()) {
518       resolver->Reject(MakeGarbageCollected<DOMException>(
519           DOMExceptionCode::kNotSupportedError,
520           "contrast setting out of range"));
521       return;
522     }
523     temp_constraints->setContrast(constraints->contrast());
524     settings->contrast = contrast;
525   }
526   settings->has_saturation =
527       constraints->hasSaturation() && constraints->saturation().IsDouble();
528   if (settings->has_saturation) {
529     const auto saturation = constraints->saturation().GetAsDouble();
530     if (saturation < capabilities_->saturation()->min() ||
531         saturation > capabilities_->saturation()->max()) {
532       resolver->Reject(MakeGarbageCollected<DOMException>(
533           DOMExceptionCode::kNotSupportedError,
534           "saturation setting out of range"));
535       return;
536     }
537     temp_constraints->setSaturation(constraints->saturation());
538     settings->saturation = saturation;
539   }
540   settings->has_sharpness =
541       constraints->hasSharpness() && constraints->sharpness().IsDouble();
542   if (settings->has_sharpness) {
543     const auto sharpness = constraints->sharpness().GetAsDouble();
544     if (sharpness < capabilities_->sharpness()->min() ||
545         sharpness > capabilities_->sharpness()->max()) {
546       resolver->Reject(MakeGarbageCollected<DOMException>(
547           DOMExceptionCode::kNotSupportedError,
548           "sharpness setting out of range"));
549       return;
550     }
551     temp_constraints->setSharpness(constraints->sharpness());
552     settings->sharpness = sharpness;
553   }
554 
555   settings->has_focus_distance = constraints->hasFocusDistance() &&
556                                  constraints->focusDistance().IsDouble();
557   if (settings->has_focus_distance) {
558     const auto focus_distance = constraints->focusDistance().GetAsDouble();
559     if (focus_distance < capabilities_->focusDistance()->min() ||
560         focus_distance > capabilities_->focusDistance()->max()) {
561       resolver->Reject(MakeGarbageCollected<DOMException>(
562           DOMExceptionCode::kNotSupportedError,
563           "focusDistance setting out of range"));
564       return;
565     }
566     temp_constraints->setFocusDistance(constraints->focusDistance());
567     settings->focus_distance = focus_distance;
568   }
569 
570   settings->has_pan = constraints->hasPan() && constraints->pan().IsDouble();
571   if (settings->has_pan) {
572     const auto pan = constraints->pan().GetAsDouble();
573     if (pan < capabilities_->pan()->min() ||
574         pan > capabilities_->pan()->max()) {
575       resolver->Reject(MakeGarbageCollected<DOMException>(
576           DOMExceptionCode::kNotSupportedError, "pan setting out of range"));
577       return;
578     }
579     temp_constraints->setPan(constraints->pan());
580     settings->pan = pan;
581   }
582 
583   settings->has_tilt = constraints->hasTilt() && constraints->tilt().IsDouble();
584   if (settings->has_tilt) {
585     const auto tilt = constraints->tilt().GetAsDouble();
586     if (tilt < capabilities_->tilt()->min() ||
587         tilt > capabilities_->tilt()->max()) {
588       resolver->Reject(MakeGarbageCollected<DOMException>(
589           DOMExceptionCode::kNotSupportedError, "tilt setting out of range"));
590       return;
591     }
592     temp_constraints->setTilt(constraints->tilt());
593     settings->tilt = tilt;
594   }
595 
596   settings->has_zoom = constraints->hasZoom() && constraints->zoom().IsDouble();
597   if (settings->has_zoom) {
598     const auto zoom = constraints->zoom().GetAsDouble();
599     if (zoom < capabilities_->zoom()->min() ||
600         zoom > capabilities_->zoom()->max()) {
601       resolver->Reject(MakeGarbageCollected<DOMException>(
602           DOMExceptionCode::kNotSupportedError, "zoom setting out of range"));
603       return;
604     }
605     temp_constraints->setZoom(constraints->zoom());
606     settings->zoom = zoom;
607   }
608 
609   // TODO(mcasas): support ConstrainBooleanParameters where applicable.
610   settings->has_torch =
611       constraints->hasTorch() && constraints->torch().IsBoolean();
612   if (settings->has_torch) {
613     const auto torch = constraints->torch().GetAsBoolean();
614     if (torch && !capabilities_->torch()) {
615       resolver->Reject(MakeGarbageCollected<DOMException>(
616           DOMExceptionCode::kNotSupportedError, "torch not supported"));
617       return;
618     }
619     temp_constraints->setTorch(constraints->torch());
620     settings->torch = torch;
621   }
622 
623   current_constraints_ = temp_constraints;
624 
625   service_requests_.insert(resolver);
626 
627   service_->SetOptions(
628       stream_track_->Component()->Source()->Id(), std::move(settings),
629       WTF::Bind(&ImageCapture::OnMojoSetOptions, WrapPersistent(this),
630                 WrapPersistent(resolver), false /* trigger_take_photo */));
631 }
632 
GetMediaTrackConstraints() const633 const MediaTrackConstraintSet* ImageCapture::GetMediaTrackConstraints() const {
634   return current_constraints_;
635 }
636 
ClearMediaTrackConstraints()637 void ImageCapture::ClearMediaTrackConstraints() {
638   current_constraints_ = MediaTrackConstraintSet::Create();
639 
640   // TODO(mcasas): Clear also any PhotoSettings that the device might have got
641   // configured, for that we need to know a "default" state of the device; take
642   // a snapshot upon first opening. https://crbug.com/700607.
643 }
644 
GetMediaTrackSettings(MediaTrackSettings * settings) const645 void ImageCapture::GetMediaTrackSettings(MediaTrackSettings* settings) const {
646   // Merge any present |settings_| members into |settings|.
647 
648   if (settings_->hasWhiteBalanceMode())
649     settings->setWhiteBalanceMode(settings_->whiteBalanceMode());
650   if (settings_->hasExposureMode())
651     settings->setExposureMode(settings_->exposureMode());
652   if (settings_->hasFocusMode())
653     settings->setFocusMode(settings_->focusMode());
654 
655   if (settings_->hasPointsOfInterest() &&
656       !settings_->pointsOfInterest().IsEmpty()) {
657     settings->setPointsOfInterest(settings_->pointsOfInterest());
658   }
659 
660   if (settings_->hasExposureCompensation())
661     settings->setExposureCompensation(settings_->exposureCompensation());
662   if (settings_->hasExposureTime())
663     settings->setExposureTime(settings_->exposureTime());
664   if (settings_->hasColorTemperature())
665     settings->setColorTemperature(settings_->colorTemperature());
666   if (settings_->hasIso())
667     settings->setIso(settings_->iso());
668 
669   if (settings_->hasBrightness())
670     settings->setBrightness(settings_->brightness());
671   if (settings_->hasContrast())
672     settings->setContrast(settings_->contrast());
673   if (settings_->hasSaturation())
674     settings->setSaturation(settings_->saturation());
675   if (settings_->hasSharpness())
676     settings->setSharpness(settings_->sharpness());
677 
678   if (settings_->hasFocusDistance())
679     settings->setFocusDistance(settings_->focusDistance());
680 
681   if (settings_->hasPan())
682     settings->setPan(settings_->pan());
683   if (settings_->hasTilt())
684     settings->setTilt(settings_->tilt());
685   if (settings_->hasZoom())
686     settings->setZoom(settings_->zoom());
687   if (settings_->hasTorch())
688     settings->setTorch(settings_->torch());
689 }
690 
ImageCapture(ExecutionContext * context,MediaStreamTrack * track)691 ImageCapture::ImageCapture(ExecutionContext* context, MediaStreamTrack* track)
692     : ExecutionContextLifecycleObserver(context),
693       stream_track_(track),
694       capabilities_(MediaTrackCapabilities::Create()),
695       settings_(MediaTrackSettings::Create()),
696       current_constraints_(MediaTrackConstraintSet::Create()),
697       photo_settings_(PhotoSettings::Create()) {
698   DCHECK(stream_track_);
699   DCHECK(!service_.is_bound());
700 
701   // This object may be constructed over an ExecutionContext that has already
702   // been detached. In this case the ImageCapture service will not be available.
703   if (!GetFrame())
704     return;
705 
706   GetFrame()->GetBrowserInterfaceBroker().GetInterface(
707       service_.BindNewPipeAndPassReceiver());
708 
709   service_.set_disconnect_handler(WTF::Bind(
710       &ImageCapture::OnServiceConnectionError, WrapWeakPersistent(this)));
711 
712   // Launch a retrieval of the current photo state, which arrive asynchronously
713   // to avoid blocking the main UI thread.
714   service_->GetPhotoState(stream_track_->Component()->Source()->Id(),
715                           WTF::Bind(&ImageCapture::UpdateMediaTrackCapabilities,
716                                     WrapPersistent(this)));
717 }
718 
OnMojoGetPhotoState(ScriptPromiseResolver * resolver,PromiseResolverFunction resolve_function,bool trigger_take_photo,media::mojom::blink::PhotoStatePtr photo_state)719 void ImageCapture::OnMojoGetPhotoState(
720     ScriptPromiseResolver* resolver,
721     PromiseResolverFunction resolve_function,
722     bool trigger_take_photo,
723     media::mojom::blink::PhotoStatePtr photo_state) {
724   DCHECK(service_requests_.Contains(resolver));
725 
726   if (photo_state.is_null()) {
727     resolver->Reject(MakeGarbageCollected<DOMException>(
728         DOMExceptionCode::kUnknownError, "platform error"));
729     service_requests_.erase(resolver);
730     return;
731   }
732 
733   photo_settings_ = PhotoSettings::Create();
734   photo_settings_->setImageHeight(photo_state->height->current);
735   photo_settings_->setImageWidth(photo_state->width->current);
736   // TODO(mcasas): collect the remaining two entries https://crbug.com/732521.
737 
738   photo_capabilities_ = MakeGarbageCollected<PhotoCapabilities>();
739   photo_capabilities_->SetRedEyeReduction(photo_state->red_eye_reduction);
740   // TODO(mcasas): Remove the explicit MediaSettingsRange::create() when
741   // mojo::StructTraits supports garbage-collected mappings,
742   // https://crbug.com/700180.
743   if (photo_state->height->min != 0 || photo_state->height->max != 0) {
744     photo_capabilities_->SetImageHeight(
745         MediaSettingsRange::Create(std::move(photo_state->height)));
746   }
747   if (photo_state->width->min != 0 || photo_state->width->max != 0) {
748     photo_capabilities_->SetImageWidth(
749         MediaSettingsRange::Create(std::move(photo_state->width)));
750   }
751   if (!photo_state->fill_light_mode.IsEmpty())
752     photo_capabilities_->SetFillLightMode(photo_state->fill_light_mode);
753 
754   // Update the local track photo_state cache.
755   UpdateMediaTrackCapabilities(std::move(photo_state));
756 
757   if (trigger_take_photo) {
758     service_->TakePhoto(
759         stream_track_->Component()->Source()->Id(),
760         WTF::Bind(&ImageCapture::OnMojoTakePhoto, WrapPersistent(this),
761                   WrapPersistent(resolver)));
762     return;
763   }
764 
765   std::move(resolve_function).Run(resolver);
766   service_requests_.erase(resolver);
767 }
768 
OnMojoSetOptions(ScriptPromiseResolver * resolver,bool trigger_take_photo,bool result)769 void ImageCapture::OnMojoSetOptions(ScriptPromiseResolver* resolver,
770                                     bool trigger_take_photo,
771                                     bool result) {
772   DCHECK(service_requests_.Contains(resolver));
773   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
774                        "ImageCapture::OnMojoSetOptions",
775                        TRACE_EVENT_SCOPE_PROCESS);
776 
777   if (!result) {
778     resolver->Reject(MakeGarbageCollected<DOMException>(
779         DOMExceptionCode::kUnknownError, "setOptions failed"));
780     service_requests_.erase(resolver);
781     return;
782   }
783 
784   auto resolver_cb =
785       WTF::Bind(&ImageCapture::ResolveWithNothing, WrapPersistent(this));
786 
787   // Retrieve the current device status after setting the options.
788   service_->GetPhotoState(
789       stream_track_->Component()->Source()->Id(),
790       WTF::Bind(&ImageCapture::OnMojoGetPhotoState, WrapPersistent(this),
791                 WrapPersistent(resolver), WTF::Passed(std::move(resolver_cb)),
792                 trigger_take_photo));
793 }
794 
OnMojoTakePhoto(ScriptPromiseResolver * resolver,media::mojom::blink::BlobPtr blob)795 void ImageCapture::OnMojoTakePhoto(ScriptPromiseResolver* resolver,
796                                    media::mojom::blink::BlobPtr blob) {
797   DCHECK(service_requests_.Contains(resolver));
798   TRACE_EVENT_INSTANT0(TRACE_DISABLED_BY_DEFAULT("video_and_image_capture"),
799                        "ImageCapture::OnMojoTakePhoto",
800                        TRACE_EVENT_SCOPE_PROCESS);
801 
802   // TODO(mcasas): Should be using a mojo::StructTraits.
803   if (blob->data.IsEmpty()) {
804     resolver->Reject(MakeGarbageCollected<DOMException>(
805         DOMExceptionCode::kUnknownError, "platform error"));
806   } else {
807     resolver->Resolve(
808         Blob::Create(blob->data.data(), blob->data.size(), blob->mime_type));
809   }
810   service_requests_.erase(resolver);
811 }
812 
UpdateMediaTrackCapabilities(media::mojom::blink::PhotoStatePtr photo_state)813 void ImageCapture::UpdateMediaTrackCapabilities(
814     media::mojom::blink::PhotoStatePtr photo_state) {
815   if (!photo_state)
816     return;
817 
818   WTF::Vector<WTF::String> supported_white_balance_modes;
819   supported_white_balance_modes.ReserveInitialCapacity(
820       photo_state->supported_white_balance_modes.size());
821   for (const auto& supported_mode : photo_state->supported_white_balance_modes)
822     supported_white_balance_modes.push_back(ToString(supported_mode));
823   if (!supported_white_balance_modes.IsEmpty()) {
824     capabilities_->setWhiteBalanceMode(
825         std::move(supported_white_balance_modes));
826     settings_->setWhiteBalanceMode(
827         ToString(photo_state->current_white_balance_mode));
828   }
829 
830   WTF::Vector<WTF::String> supported_exposure_modes;
831   supported_exposure_modes.ReserveInitialCapacity(
832       photo_state->supported_exposure_modes.size());
833   for (const auto& supported_mode : photo_state->supported_exposure_modes)
834     supported_exposure_modes.push_back(ToString(supported_mode));
835   if (!supported_exposure_modes.IsEmpty()) {
836     capabilities_->setExposureMode(std::move(supported_exposure_modes));
837     settings_->setExposureMode(ToString(photo_state->current_exposure_mode));
838   }
839 
840   WTF::Vector<WTF::String> supported_focus_modes;
841   supported_focus_modes.ReserveInitialCapacity(
842       photo_state->supported_focus_modes.size());
843   for (const auto& supported_mode : photo_state->supported_focus_modes)
844     supported_focus_modes.push_back(ToString(supported_mode));
845   if (!supported_focus_modes.IsEmpty()) {
846     capabilities_->setFocusMode(std::move(supported_focus_modes));
847     settings_->setFocusMode(ToString(photo_state->current_focus_mode));
848   }
849 
850   HeapVector<Member<Point2D>> current_points_of_interest;
851   if (!photo_state->points_of_interest.IsEmpty()) {
852     for (const auto& point : photo_state->points_of_interest) {
853       Point2D* web_point = Point2D::Create();
854       web_point->setX(point->x);
855       web_point->setY(point->y);
856       current_points_of_interest.push_back(web_point);
857     }
858   }
859   settings_->setPointsOfInterest(current_points_of_interest);
860 
861   // TODO(mcasas): Remove the explicit MediaSettingsRange::create() when
862   // mojo::StructTraits supports garbage-collected mappings,
863   // https://crbug.com/700180.
864   if (photo_state->exposure_compensation->max !=
865       photo_state->exposure_compensation->min) {
866     capabilities_->setExposureCompensation(
867         MediaSettingsRange::Create(*photo_state->exposure_compensation));
868     settings_->setExposureCompensation(
869         photo_state->exposure_compensation->current);
870   }
871   if (photo_state->exposure_time->max != photo_state->exposure_time->min) {
872     capabilities_->setExposureTime(
873         MediaSettingsRange::Create(*photo_state->exposure_time));
874     settings_->setExposureTime(photo_state->exposure_time->current);
875   }
876   if (photo_state->color_temperature->max !=
877       photo_state->color_temperature->min) {
878     capabilities_->setColorTemperature(
879         MediaSettingsRange::Create(*photo_state->color_temperature));
880     settings_->setColorTemperature(photo_state->color_temperature->current);
881   }
882   if (photo_state->iso->max != photo_state->iso->min) {
883     capabilities_->setIso(MediaSettingsRange::Create(*photo_state->iso));
884     settings_->setIso(photo_state->iso->current);
885   }
886 
887   if (photo_state->brightness->max != photo_state->brightness->min) {
888     capabilities_->setBrightness(
889         MediaSettingsRange::Create(*photo_state->brightness));
890     settings_->setBrightness(photo_state->brightness->current);
891   }
892   if (photo_state->contrast->max != photo_state->contrast->min) {
893     capabilities_->setContrast(
894         MediaSettingsRange::Create(*photo_state->contrast));
895     settings_->setContrast(photo_state->contrast->current);
896   }
897   if (photo_state->saturation->max != photo_state->saturation->min) {
898     capabilities_->setSaturation(
899         MediaSettingsRange::Create(*photo_state->saturation));
900     settings_->setSaturation(photo_state->saturation->current);
901   }
902   if (photo_state->sharpness->max != photo_state->sharpness->min) {
903     capabilities_->setSharpness(
904         MediaSettingsRange::Create(*photo_state->sharpness));
905     settings_->setSharpness(photo_state->sharpness->current);
906   }
907 
908   if (photo_state->focus_distance->max != photo_state->focus_distance->min) {
909     capabilities_->setFocusDistance(
910         MediaSettingsRange::Create(*photo_state->focus_distance));
911     settings_->setFocusDistance(photo_state->focus_distance->current);
912   }
913 
914   if (photo_state->pan->max != photo_state->pan->min) {
915     capabilities_->setPan(MediaSettingsRange::Create(*photo_state->pan));
916     settings_->setPan(photo_state->pan->current);
917   }
918   if (photo_state->tilt->max != photo_state->tilt->min) {
919     capabilities_->setTilt(MediaSettingsRange::Create(*photo_state->tilt));
920     settings_->setTilt(photo_state->tilt->current);
921   }
922   if (photo_state->zoom->max != photo_state->zoom->min) {
923     capabilities_->setZoom(MediaSettingsRange::Create(*photo_state->zoom));
924     settings_->setZoom(photo_state->zoom->current);
925   }
926 
927   if (photo_state->supports_torch)
928     capabilities_->setTorch(photo_state->supports_torch);
929   if (photo_state->supports_torch)
930     settings_->setTorch(photo_state->torch);
931 }
932 
OnServiceConnectionError()933 void ImageCapture::OnServiceConnectionError() {
934   service_.reset();
935   for (ScriptPromiseResolver* resolver : service_requests_) {
936     resolver->Reject(MakeGarbageCollected<DOMException>(
937         DOMExceptionCode::kNotFoundError, kNoServiceError));
938   }
939   service_requests_.clear();
940 }
941 
ResolveWithNothing(ScriptPromiseResolver * resolver)942 void ImageCapture::ResolveWithNothing(ScriptPromiseResolver* resolver) {
943   DCHECK(resolver);
944   resolver->Resolve();
945 }
946 
ResolveWithPhotoSettings(ScriptPromiseResolver * resolver)947 void ImageCapture::ResolveWithPhotoSettings(ScriptPromiseResolver* resolver) {
948   DCHECK(resolver);
949   resolver->Resolve(photo_settings_);
950 }
951 
ResolveWithPhotoCapabilities(ScriptPromiseResolver * resolver)952 void ImageCapture::ResolveWithPhotoCapabilities(
953     ScriptPromiseResolver* resolver) {
954   DCHECK(resolver);
955   resolver->Resolve(photo_capabilities_);
956 }
957 
Trace(Visitor * visitor)958 void ImageCapture::Trace(Visitor* visitor) {
959   visitor->Trace(stream_track_);
960   visitor->Trace(capabilities_);
961   visitor->Trace(settings_);
962   visitor->Trace(photo_settings_);
963   visitor->Trace(current_constraints_);
964   visitor->Trace(photo_capabilities_);
965   visitor->Trace(service_requests_);
966   EventTargetWithInlineData::Trace(visitor);
967   ExecutionContextLifecycleObserver::Trace(visitor);
968 }
969 
970 }  // namespace blink
971