1 // Copyright 2017 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_USER_MEDIA_PROCESSOR_H_ 6 #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_USER_MEDIA_PROCESSOR_H_ 7 8 #include <memory> 9 #include <utility> 10 11 #include "base/callback_forward.h" 12 #include "base/macros.h" 13 #include "base/memory/scoped_refptr.h" 14 #include "base/threading/thread_checker.h" 15 #include "third_party/blink/public/common/mediastream/media_stream_request.h" 16 #include "third_party/blink/public/mojom/mediastream/media_devices.mojom-blink.h" 17 #include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h" 18 #include "third_party/blink/public/platform/modules/mediastream/web_platform_media_stream_source.h" 19 #include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio.h" 20 #include "third_party/blink/renderer/modules/mediastream/user_media_request.h" 21 #include "third_party/blink/renderer/modules/modules_export.h" 22 #include "third_party/blink/renderer/platform/mojo/heap_mojo_remote.h" 23 #include "third_party/blink/renderer/platform/mojo/heap_mojo_wrapper_mode.h" 24 #include "third_party/blink/renderer/platform/wtf/text/wtf_string.h" 25 #include "third_party/blink/renderer/platform/wtf/vector.h" 26 27 namespace gfx { 28 class Size; 29 } 30 31 namespace blink { 32 class AudioCaptureSettings; 33 class LocalFrame; 34 class MediaStreamAudioSource; 35 class MediaStreamVideoSource; 36 class VideoCaptureSettings; 37 class WebMediaStreamDeviceObserver; 38 class WebMediaStreamSource; 39 class WebString; 40 41 // UserMediaProcessor is responsible for processing getUserMedia() requests. 42 // It also keeps tracks of all sources used by streams created with 43 // getUserMedia(). 44 // It communicates with the browser via MediaStreamDispatcherHost. 45 // Only one MediaStream at a time can be in the process of being created. 46 // UserMediaProcessor must be created, called and destroyed on the main 47 // render thread. There should be only one UserMediaProcessor per frame. 48 class MODULES_EXPORT UserMediaProcessor 49 : public GarbageCollected<UserMediaProcessor> { 50 public: 51 using MediaDevicesDispatcherCallback = base::RepeatingCallback< 52 blink::mojom::blink::MediaDevicesDispatcherHost*()>; 53 // |frame| must outlive this instance. 54 UserMediaProcessor(LocalFrame* frame, 55 MediaDevicesDispatcherCallback media_devices_dispatcher_cb, 56 scoped_refptr<base::SingleThreadTaskRunner> task_runner); 57 virtual ~UserMediaProcessor(); 58 59 // It can be assumed that the output of CurrentRequest() remains the same 60 // during the execution of a task on the main thread unless ProcessRequest or 61 // DeleteUserMediaRequest are invoked. 62 // TODO(guidou): Remove this method. https://crbug.com/764293 63 UserMediaRequest* CurrentRequest(); 64 65 // Starts processing |request| in order to create a new MediaStream. When 66 // processing of |request| is complete, it notifies by invoking |callback|. 67 // This method must be called only if there is no request currently being 68 // processed. 69 void ProcessRequest(UserMediaRequest* request, base::OnceClosure callback); 70 71 // If |user_media_request| is the request currently being processed, stops 72 // processing the request and returns true. Otherwise, performs no action and 73 // returns false. 74 // TODO(guidou): Make this method private and replace with a public 75 // CancelRequest() method that deletes the request only if it has not been 76 // generated yet. https://crbug.com/764293 77 bool DeleteUserMediaRequest(UserMediaRequest* user_media_request); 78 79 // Stops processing the current request, if any, and stops all sources 80 // currently being tracked, effectively stopping all tracks associated with 81 // those sources. 82 void StopAllProcessing(); 83 84 bool HasActiveSources() const; 85 86 void OnDeviceStopped(const blink::MediaStreamDevice& device); 87 void OnDeviceChanged(const blink::MediaStreamDevice& old_device, 88 const blink::MediaStreamDevice& new_device); 89 void OnDeviceRequestStateChange( 90 const MediaStreamDevice& device, 91 const mojom::blink::MediaStreamStateChange new_state); 92 set_media_stream_dispatcher_host_for_testing(mojo::PendingRemote<blink::mojom::blink::MediaStreamDispatcherHost> dispatcher_host)93 void set_media_stream_dispatcher_host_for_testing( 94 mojo::PendingRemote<blink::mojom::blink::MediaStreamDispatcherHost> 95 dispatcher_host) { 96 dispatcher_host_.Bind(std::move(dispatcher_host), task_runner_); 97 } 98 99 virtual void Trace(Visitor*) const; 100 101 protected: 102 // These methods are virtual for test purposes. A test can override them to 103 // test requesting local media streams. The function notifies WebKit that the 104 // |request| have completed. 105 virtual void GetUserMediaRequestSucceeded( 106 MediaStreamDescriptor* descriptor, 107 UserMediaRequest* user_media_request); 108 virtual void GetUserMediaRequestFailed( 109 blink::mojom::blink::MediaStreamRequestResult result, 110 const String& constraint_name = String()); 111 112 // Creates a MediaStreamAudioSource/MediaStreamVideoSource objects. 113 // These are virtual for test purposes. 114 virtual std::unique_ptr<blink::MediaStreamAudioSource> CreateAudioSource( 115 const blink::MediaStreamDevice& device, 116 blink::WebPlatformMediaStreamSource::ConstraintsRepeatingCallback 117 source_ready); 118 virtual std::unique_ptr<blink::MediaStreamVideoSource> CreateVideoSource( 119 const blink::MediaStreamDevice& device, 120 blink::WebPlatformMediaStreamSource::SourceStoppedCallback stop_callback); 121 122 // Intended to be used only for testing. 123 const blink::AudioCaptureSettings& AudioCaptureSettingsForTesting() const; 124 const blink::VideoCaptureSettings& VideoCaptureSettingsForTesting() const; 125 126 void SetMediaStreamDeviceObserverForTesting( 127 WebMediaStreamDeviceObserver* media_stream_device_observer); 128 129 private: 130 FRIEND_TEST_ALL_PREFIXES(UserMediaClientTest, 131 PanConstraintRequestPanTiltZoomPermission); 132 FRIEND_TEST_ALL_PREFIXES(UserMediaClientTest, 133 TiltConstraintRequestPanTiltZoomPermission); 134 FRIEND_TEST_ALL_PREFIXES(UserMediaClientTest, 135 ZoomConstraintRequestPanTiltZoomPermission); 136 class RequestInfo; 137 using LocalStreamSources = HeapVector<Member<MediaStreamSource>>; 138 139 void OnStreamGenerated(int request_id, 140 blink::mojom::blink::MediaStreamRequestResult result, 141 const String& label, 142 const Vector<blink::MediaStreamDevice>& audio_devices, 143 const Vector<blink::MediaStreamDevice>& video_devices, 144 bool pan_tilt_zoom_allowed); 145 146 void GotAllVideoInputFormatsForDevice( 147 UserMediaRequest* user_media_request, 148 const String& label, 149 const String& device_id, 150 const Vector<media::VideoCaptureFormat>& formats); 151 152 gfx::Size GetScreenSize(); 153 154 void OnStreamGenerationFailed( 155 int request_id, 156 blink::mojom::blink::MediaStreamRequestResult result); 157 158 bool IsCurrentRequestInfo(int request_id) const; 159 bool IsCurrentRequestInfo(UserMediaRequest* user_media_request) const; 160 void DelayedGetUserMediaRequestSucceeded( 161 int request_id, 162 MediaStreamDescriptor* descriptor, 163 UserMediaRequest* user_media_request); 164 void DelayedGetUserMediaRequestFailed( 165 int request_id, 166 UserMediaRequest* user_media_request, 167 blink::mojom::blink::MediaStreamRequestResult result, 168 const String& constraint_name); 169 170 // Called when |source| has been stopped from JavaScript. 171 void OnLocalSourceStopped(const blink::WebMediaStreamSource& source); 172 173 // Creates a WebKit representation of a stream source based on 174 // |device| from the MediaStreamDispatcherHost. 175 MediaStreamSource* InitializeVideoSourceObject( 176 const blink::MediaStreamDevice& device); 177 178 MediaStreamSource* InitializeAudioSourceObject( 179 const blink::MediaStreamDevice& device, 180 bool* is_pending); 181 182 void StartTracks(const String& label); 183 184 void CreateVideoTracks(const Vector<blink::MediaStreamDevice>& devices, 185 HeapVector<Member<MediaStreamComponent>>* components); 186 187 void CreateAudioTracks(const Vector<blink::MediaStreamDevice>& devices, 188 HeapVector<Member<MediaStreamComponent>>* components); 189 190 // Callback function triggered when all native versions of the 191 // underlying media sources and tracks have been created and started. 192 void OnCreateNativeTracksCompleted( 193 const String& label, 194 RequestInfo* request, 195 blink::mojom::blink::MediaStreamRequestResult result, 196 const String& result_name); 197 198 void OnStreamGeneratedForCancelledRequest( 199 const Vector<blink::MediaStreamDevice>& audio_devices, 200 const Vector<blink::MediaStreamDevice>& video_devices); 201 202 static void OnAudioSourceStartedOnAudioThread( 203 scoped_refptr<base::SingleThreadTaskRunner> task_runner, 204 UserMediaProcessor* weak_ptr, 205 blink::WebPlatformMediaStreamSource* source, 206 blink::mojom::blink::MediaStreamRequestResult result, 207 const blink::WebString& result_name); 208 209 void OnAudioSourceStarted( 210 blink::WebPlatformMediaStreamSource* source, 211 blink::mojom::blink::MediaStreamRequestResult result, 212 const String& result_name); 213 214 void NotifyCurrentRequestInfoOfAudioSourceStarted( 215 blink::WebPlatformMediaStreamSource* source, 216 blink::mojom::blink::MediaStreamRequestResult result, 217 const String& result_name); 218 219 void DeleteAllUserMediaRequests(); 220 221 // Returns the source that use a device with |device.session_id| 222 // and |device.device.id|. nullptr if such source doesn't exist. FindLocalSource(const MediaStreamDevice & device)223 MediaStreamSource* FindLocalSource(const MediaStreamDevice& device) const { 224 return FindLocalSource(local_sources_, device); 225 } FindPendingLocalSource(const MediaStreamDevice & device)226 MediaStreamSource* FindPendingLocalSource( 227 const MediaStreamDevice& device) const { 228 return FindLocalSource(pending_local_sources_, device); 229 } 230 MediaStreamSource* FindLocalSource( 231 const LocalStreamSources& sources, 232 const blink::MediaStreamDevice& device) const; 233 234 // Looks up a local source and returns it if found. If not found, prepares 235 // a new MediaStreamSource with a nullptr extraData pointer. 236 MediaStreamSource* FindOrInitializeSourceObject( 237 const MediaStreamDevice& device); 238 239 // Returns true if we do find and remove the |source|. 240 // Otherwise returns false. 241 bool RemoveLocalSource(MediaStreamSource* source); 242 243 void StopLocalSource(MediaStreamSource* source, bool notify_dispatcher); 244 245 blink::mojom::blink::MediaStreamDispatcherHost* 246 GetMediaStreamDispatcherHost(); 247 blink::mojom::blink::MediaDevicesDispatcherHost* GetMediaDevicesDispatcher(); 248 249 void SetupAudioInput(); 250 void SelectAudioDeviceSettings( 251 UserMediaRequest* user_media_request, 252 Vector<blink::mojom::blink::AudioInputDeviceCapabilitiesPtr> 253 audio_input_capabilities); 254 void SelectAudioSettings( 255 UserMediaRequest* user_media_request, 256 const blink::AudioDeviceCaptureCapabilities& capabilities); 257 258 void SetupVideoInput(); 259 // Exported for testing. 260 static bool IsPanTiltZoomPermissionRequested( 261 const MediaConstraints& constraints); 262 void SelectVideoDeviceSettings( 263 UserMediaRequest* user_media_request, 264 Vector<blink::mojom::blink::VideoInputDeviceCapabilitiesPtr> 265 video_input_capabilities); 266 void FinalizeSelectVideoDeviceSettings( 267 UserMediaRequest* user_media_request, 268 const blink::VideoCaptureSettings& settings); 269 void SelectVideoContentSettings(); 270 271 base::Optional<base::UnguessableToken> DetermineExistingAudioSessionId(); 272 273 void GenerateStreamForCurrentRequestInfo( 274 base::Optional<base::UnguessableToken> 275 requested_audio_capture_session_id = base::nullopt, 276 blink::mojom::StreamSelectionStrategy strategy = 277 blink::mojom::StreamSelectionStrategy::SEARCH_BY_DEVICE_ID); 278 279 WebMediaStreamDeviceObserver* GetMediaStreamDeviceObserver(); 280 281 // Owned by the test. 282 WebMediaStreamDeviceObserver* media_stream_device_observer_for_testing_ = 283 nullptr; 284 285 LocalStreamSources local_sources_; 286 LocalStreamSources pending_local_sources_; 287 288 HeapMojoRemote<blink::mojom::blink::MediaStreamDispatcherHost, 289 HeapMojoWrapperMode::kWithoutContextObserver> 290 dispatcher_host_; 291 292 // UserMedia requests are processed sequentially. |current_request_info_| 293 // contains the request currently being processed. 294 Member<RequestInfo> current_request_info_; 295 MediaDevicesDispatcherCallback media_devices_dispatcher_cb_; 296 base::OnceClosure request_completed_cb_; 297 298 Member<LocalFrame> frame_; 299 scoped_refptr<base::SingleThreadTaskRunner> task_runner_; 300 301 THREAD_CHECKER(thread_checker_); 302 303 DISALLOW_COPY_AND_ASSIGN(UserMediaProcessor); 304 }; 305 306 } // namespace blink 307 308 #endif // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_USER_MEDIA_PROCESSOR_H_ 309