1 // Copyright 2018 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef DEVICE_VR_ANDROID_ARCORE_ARCORE_GL_H_ 6 #define DEVICE_VR_ANDROID_ARCORE_ARCORE_GL_H_ 7 8 #include <memory> 9 #include <unordered_set> 10 #include <utility> 11 #include <vector> 12 13 #include "base/cancelable_callback.h" 14 #include "base/containers/queue.h" 15 #include "base/macros.h" 16 #include "base/memory/ref_counted.h" 17 #include "base/memory/weak_ptr.h" 18 #include "base/single_thread_task_runner.h" 19 #include "device/vr/public/mojom/isolated_xr_service.mojom.h" 20 #include "device/vr/public/mojom/vr_service.mojom.h" 21 #include "device/vr/util/fps_meter.h" 22 #include "device/vr/util/sliding_average.h" 23 #include "mojo/public/cpp/bindings/associated_receiver.h" 24 #include "mojo/public/cpp/bindings/pending_associated_receiver.h" 25 #include "mojo/public/cpp/bindings/pending_associated_remote.h" 26 #include "mojo/public/cpp/bindings/pending_remote.h" 27 #include "mojo/public/cpp/bindings/receiver.h" 28 #include "mojo/public/cpp/bindings/remote.h" 29 #include "ui/display/display.h" 30 #include "ui/gfx/geometry/point_f.h" 31 #include "ui/gfx/geometry/quaternion.h" 32 #include "ui/gfx/geometry/rect_f.h" 33 #include "ui/gfx/geometry/size_f.h" 34 #include "ui/gfx/native_widget_types.h" 35 36 namespace gfx { 37 class GpuFence; 38 } // namespace gfx 39 40 namespace gl { 41 class GLContext; 42 class GLSurface; 43 } // namespace gl 44 45 namespace vr { 46 class ArCoreSessionUtils; 47 class WebXrPresentationState; 48 } // namespace vr 49 50 namespace device { 51 52 class ArCore; 53 class ArCoreFactory; 54 class ArImageTransport; 55 56 using ArCoreGlCreateSessionCallback = base::OnceCallback<void( 57 mojo::PendingRemote<mojom::XRFrameDataProvider> frame_data_provider, 58 mojom::VRDisplayInfoPtr display_info, 59 mojo::PendingRemote<mojom::XRSessionController> session_controller, 60 mojom::XRPresentationConnectionPtr presentation_connection)>; 61 62 using ArCoreGlInitializeCallback = base::OnceCallback<void( 63 base::Optional<std::unordered_set<device::mojom::XRSessionFeature>>)>; 64 65 // All of this class's methods must be called on the same valid GL thread with 66 // the exception of GetGlThreadTaskRunner() and GetWeakPtr(). 67 class ArCoreGl : public mojom::XRFrameDataProvider, 68 public mojom::XRPresentationProvider, 69 public mojom::XREnvironmentIntegrationProvider, 70 public mojom::XRSessionController { 71 public: 72 explicit ArCoreGl(std::unique_ptr<ArImageTransport> ar_image_transport); 73 ~ArCoreGl() override; 74 75 void Initialize( 76 vr::ArCoreSessionUtils* session_utils, 77 ArCoreFactory* arcore_factory, 78 gfx::AcceleratedWidget drawing_widget, 79 const gfx::Size& frame_size, 80 display::Display::Rotation display_rotation, 81 const std::unordered_set<device::mojom::XRSessionFeature>& 82 required_features, 83 const std::unordered_set<device::mojom::XRSessionFeature>& 84 optional_features, 85 const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images, 86 ArCoreGlInitializeCallback callback); 87 88 void CreateSession(mojom::VRDisplayInfoPtr display_info, 89 ArCoreGlCreateSessionCallback create_callback, 90 base::OnceClosure shutdown_callback); 91 GetGlThreadTaskRunner()92 const scoped_refptr<base::SingleThreadTaskRunner>& GetGlThreadTaskRunner() { 93 return gl_thread_task_runner_; 94 } 95 96 // mojom::XRFrameDataProvider 97 void GetFrameData(mojom::XRFrameDataRequestOptionsPtr options, 98 GetFrameDataCallback callback) override; 99 100 void GetEnvironmentIntegrationProvider( 101 mojo::PendingAssociatedReceiver<mojom::XREnvironmentIntegrationProvider> 102 environment_provider) override; 103 void SetInputSourceButtonListener( 104 mojo::PendingAssociatedRemote<device::mojom::XRInputSourceButtonListener>) 105 override; 106 107 // XRPresentationProvider 108 void SubmitFrameMissing(int16_t frame_index, const gpu::SyncToken&) override; 109 void SubmitFrame(int16_t frame_index, 110 const gpu::MailboxHolder& mailbox, 111 base::TimeDelta time_waited) override; 112 void SubmitFrameWithTextureHandle( 113 int16_t frame_index, 114 mojo::PlatformHandle texture_handle) override; 115 void SubmitFrameDrawnIntoTexture(int16_t frame_index, 116 const gpu::SyncToken&, 117 base::TimeDelta time_waited) override; 118 void UpdateLayerBounds(int16_t frame_index, 119 const gfx::RectF& left_bounds, 120 const gfx::RectF& right_bounds, 121 const gfx::Size& source_size) override; 122 123 // XREnvironmentIntegrationProvider 124 void SubscribeToHitTest( 125 mojom::XRNativeOriginInformationPtr native_origin_information, 126 const std::vector<mojom::EntityTypeForHitTest>& entity_types, 127 mojom::XRRayPtr ray, 128 mojom::XREnvironmentIntegrationProvider::SubscribeToHitTestCallback 129 callback) override; 130 void SubscribeToHitTestForTransientInput( 131 const std::string& profile_name, 132 const std::vector<mojom::EntityTypeForHitTest>& entity_types, 133 mojom::XRRayPtr ray, 134 mojom::XREnvironmentIntegrationProvider:: 135 SubscribeToHitTestForTransientInputCallback callback) override; 136 137 void UnsubscribeFromHitTest(uint64_t subscription_id) override; 138 139 void CreateAnchor( 140 mojom::XRNativeOriginInformationPtr native_origin_information, 141 const device::Pose& native_origin_from_anchor, 142 CreateAnchorCallback callback) override; 143 void CreatePlaneAnchor( 144 mojom::XRNativeOriginInformationPtr native_origin_information, 145 const device::Pose& native_origin_from_anchor, 146 uint64_t plane_id, 147 CreatePlaneAnchorCallback callback) override; 148 149 void DetachAnchor(uint64_t anchor_id) override; 150 151 // mojom::XRSessionController 152 void SetFrameDataRestricted(bool restricted) override; 153 154 void ProcessFrameFromMailbox(int16_t frame_index, 155 const gpu::MailboxHolder& mailbox); 156 void ProcessFrameDrawnIntoTexture(int16_t frame_index, 157 const gpu::SyncToken& sync_token); 158 void OnWebXrTokenSignaled(int16_t frame_index, 159 std::unique_ptr<gfx::GpuFence> gpu_fence); 160 161 // Notifies that the screen was touched at |touch_point| using a pointer. 162 // |touching| will be set to true if the screen is still touched. |is_primary| 163 // signifies that the used pointer is considered primary. 164 void OnScreenTouch(bool is_primary, 165 bool touching, 166 int32_t pointer_id, 167 const gfx::PointF& touch_point); 168 std::vector<mojom::XRInputSourceStatePtr> GetInputSourceStates(); 169 170 base::WeakPtr<ArCoreGl> GetWeakPtr(); 171 172 private: 173 void Pause(); 174 void Resume(); 175 176 void FinishFrame(int16_t frame_index); 177 bool IsSubmitFrameExpected(int16_t frame_index); 178 void ProcessFrame(mojom::XRFrameDataRequestOptionsPtr options, 179 mojom::XRFrameDataPtr frame_data, 180 mojom::XRFrameDataProvider::GetFrameDataCallback callback); 181 182 bool InitializeGl(gfx::AcceleratedWidget drawing_widget); 183 void OnArImageTransportReady(ArCoreGlInitializeCallback callback); 184 bool IsOnGlThread() const; 185 void CopyCameraImageToFramebuffer(); 186 void OnTransportFrameAvailable(const gfx::Transform& uv_transform); 187 void TransitionProcessingFrameToRendering(); 188 189 void GetRenderedFrameStats(); 190 void FinishRenderingFrame(); 191 base::TimeDelta EstimatedArCoreFrameTime(); 192 base::TimeDelta WaitTimeForArCoreUpdate(); 193 base::TimeDelta WaitTimeForRenderCompletion(); 194 void ScheduleGetFrameData(); 195 void RunPendingGetFrameData(); 196 197 bool IsFeatureEnabled(mojom::XRSessionFeature feature); 198 199 // Set of features enabled on this session. Required to correctly configure 200 // the session and only send out necessary data related to reference spaces to 201 // blink. Valid after the call to |Initialize()| method. 202 std::unordered_set<device::mojom::XRSessionFeature> enabled_features_; 203 204 base::OnceClosure session_shutdown_callback_; 205 206 scoped_refptr<gl::GLSurface> surface_; 207 scoped_refptr<gl::GLContext> context_; 208 scoped_refptr<base::SingleThreadTaskRunner> gl_thread_task_runner_; 209 210 // Created on GL thread and should only be accessed on that thread. 211 std::unique_ptr<ArCore> arcore_; 212 std::unique_ptr<ArImageTransport> ar_image_transport_; 213 214 // This class uses the same overall presentation state logic 215 // as GvrGraphicsDelegate, with some difference due to drawing 216 // camera images even on frames with no pose and therefore 217 // no blink-generated rendered image. 218 // 219 // Rough sequence is: 220 // 221 // SubmitFrame N N animating->processing 222 // draw camera N 223 // waitForToken 224 // GetFrameData N+1 N+1 start animating 225 // update ARCore N to N+1 226 // OnToken N N processing->rendering 227 // draw rendered N 228 // swap N rendering done 229 // SubmitFrame N+1 N+1 animating->processing 230 // draw camera N+1 231 // waitForToken 232 std::unique_ptr<vr::WebXrPresentationState> webxr_; 233 234 // Default dummy values to ensure consistent behaviour. 235 236 // Transfer size is the size of the WebGL framebuffer, this may be 237 // smaller than the camera image if framebufferScaleFactor is < 1.0. 238 gfx::Size transfer_size_ = gfx::Size(0, 0); 239 240 // Viewport size to use for new animating frames. Currently in-flight 241 // processing/rendering frames continue using the viewport size stored 242 // in their WebXrFrame state. 243 gfx::RectF viewport_bounds_ = gfx::RectF(0.f, 0.f, 1.f, 1.f); 244 245 // The camera image size stays locked to the screen size even if 246 // framebufferScaleFactor changes. 247 gfx::Size camera_image_size_ = gfx::Size(0, 0); 248 display::Display::Rotation display_rotation_ = display::Display::ROTATE_0; 249 bool should_update_display_geometry_ = true; 250 251 // UV transform for drawing the camera texture, this is supplied by ARCore 252 // and can include 90 degree rotations or other nontrivial transforms. 253 gfx::Transform uv_transform_; 254 255 gfx::Transform projection_; 256 gfx::Transform inverse_projection_; 257 // The first run of ProduceFrame should set uv_transform_ and projection_ 258 // using the default settings in ArCore. 259 bool should_recalculate_uvs_ = true; 260 bool have_camera_image_ = false; 261 262 bool is_initialized_ = false; 263 bool is_paused_ = true; 264 265 bool restrict_frame_data_ = false; 266 267 base::TimeTicks last_arcore_update_time_; 268 base::TimeDelta last_arcore_frame_timestamp_; 269 270 device::SlidingTimeDeltaAverage average_camera_frametime_; 271 device::SlidingTimeDeltaAverage average_animate_time_; 272 device::SlidingTimeDeltaAverage average_process_time_; 273 device::SlidingTimeDeltaAverage average_render_time_; 274 275 float rendering_time_ratio_ = 0.0f; 276 277 FPSMeter fps_meter_; 278 279 mojo::Receiver<mojom::XRFrameDataProvider> frame_data_receiver_{this}; 280 mojo::Receiver<mojom::XRSessionController> session_controller_receiver_{this}; 281 mojo::AssociatedReceiver<mojom::XREnvironmentIntegrationProvider> 282 environment_receiver_{this}; 283 284 void OnBindingDisconnect(); 285 void CloseBindingsIfOpen(); 286 287 mojo::Receiver<device::mojom::XRPresentationProvider> presentation_receiver_{ 288 this}; 289 mojo::Remote<device::mojom::XRPresentationClient> submit_client_; 290 291 // This closure saves arguments for the next GetFrameData call, including a 292 // mojo callback. Must remain owned by ArCoreGl, don't pass it off to the task 293 // runner directly. Storing the mojo getframedata callback in a closure owned 294 // by the task runner would lead to inconsistent state on session shutdown. 295 // See https://crbug.com/1065572. 296 base::OnceClosure pending_getframedata_; 297 298 mojom::VRDisplayInfoPtr display_info_; 299 bool display_info_changed_ = false; 300 301 mojom::VRStageParametersPtr stage_parameters_; 302 uint32_t stage_parameters_id_; 303 304 // Currently estimated floor height. 305 base::Optional<float> floor_height_estimate_; 306 307 // Touch-related data. 308 // Android will report touch events via MotionEvent - see ArImmersiveOverlay 309 // for details. 310 struct ScreenTouchEvent { 311 gfx::PointF screen_last_touch; 312 313 // Screen touch start/end events get reported asynchronously. We want to 314 // report at least one "clicked" event even if start and end happen within a 315 // single frame. The "active" state corresponds to the current state and is 316 // updated asynchronously. The "pending" state is set to true whenever the 317 // screen is touched, but only gets cleared by the input source handler. 318 // 319 // active pending event 320 // 0 0 321 // 1 1 322 // 1 1 pressed=true (selectstart) 323 // 1 1 pressed=true 324 // 0 1->0 pressed=false clicked=true (selectend, click) 325 // 326 // 0 0 327 // 1 1 328 // 0 1 329 // 0 1->0 pressed=false clicked=true (selectend, click) 330 float screen_touch_pending = false; 331 float screen_touch_active = false; 332 333 // ID of the pointer that raised this event. 334 int32_t pointer_id; 335 bool is_primary; 336 }; 337 338 // Map from input source ID to its latest information. 339 std::unordered_map<uint32_t, ScreenTouchEvent> screen_touch_events_; 340 // Map from pointer ID to input source ID currently assigned to that pointer. 341 std::unordered_map<int32_t, uint32_t> pointer_id_to_input_source_id_; 342 343 uint32_t next_input_source_id_ = 1; 344 345 // Must be last. 346 base::WeakPtrFactory<ArCoreGl> weak_ptr_factory_{this}; 347 DISALLOW_COPY_AND_ASSIGN(ArCoreGl); 348 }; 349 350 } // namespace device 351 352 #endif // DEVICE_VR_ANDROID_ARCORE_ARCORE_GL_H_ 353