1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "device/vr/android/arcore/arcore_impl.h"
6 
7 #include "base/android/jni_android.h"
8 #include "base/bind.h"
9 #include "base/containers/span.h"
10 #include "base/numerics/checked_math.h"
11 #include "base/numerics/math_constants.h"
12 #include "base/optional.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "base/trace_event/trace_event.h"
15 #include "base/util/type_safety/pass_key.h"
16 #include "device/vr/android/arcore/arcore_math_utils.h"
17 #include "device/vr/android/arcore/arcore_plane_manager.h"
18 #include "device/vr/android/arcore/type_converters.h"
19 #include "device/vr/public/mojom/pose.h"
20 #include "device/vr/public/mojom/vr_service.mojom.h"
21 #include "third_party/skia/include/core/SkBitmap.h"
22 #include "third_party/skia/include/core/SkCanvas.h"
23 #include "third_party/skia/include/core/SkImage.h"
24 #include "third_party/skia/include/core/SkMatrix44.h"
25 #include "third_party/skia/include/core/SkPaint.h"
26 #include "third_party/skia/include/core/SkPixmap.h"
27 #include "ui/display/display.h"
28 #include "ui/gfx/geometry/point3_f.h"
29 #include "ui/gfx/geometry/point_f.h"
30 #include "ui/gfx/transform.h"
31 #include "ui/gfx/transform_util.h"
32 
33 using base::android::JavaRef;
34 
35 namespace {
36 
37 // Anchor creation requests that are older than 3 seconds are considered
38 // outdated and should be failed.
39 constexpr base::TimeDelta kOutdatedAnchorCreationRequestThreshold =
40     base::TimeDelta::FromSeconds(3);
41 
42 // Helper, returns new VRPosePtr with position and orientation set to match the
43 // position and orientation of passed in |pose|.
GetMojomVRPoseFromArPose(const ArSession * session,const ArPose * pose)44 device::mojom::VRPosePtr GetMojomVRPoseFromArPose(const ArSession* session,
45                                                   const ArPose* pose) {
46   device::mojom::VRPosePtr result = device::mojom::VRPose::New();
47   std::tie(result->orientation, result->position) =
48       device::GetPositionAndOrientationFromArPose(session, pose);
49 
50   return result;
51 }
52 
53 // Helper, creates new ArPose* with position and orientation set to match the
54 // position and orientation of passed in |pose|.
55 
GetArCoreEntityType(device::mojom::EntityTypeForHitTest entity_type)56 ArTrackableType GetArCoreEntityType(
57     device::mojom::EntityTypeForHitTest entity_type) {
58   switch (entity_type) {
59     case device::mojom::EntityTypeForHitTest::PLANE:
60       return AR_TRACKABLE_PLANE;
61     case device::mojom::EntityTypeForHitTest::POINT:
62       return AR_TRACKABLE_POINT;
63   }
64 }
65 
GetArCoreEntityTypes(const std::vector<device::mojom::EntityTypeForHitTest> & entity_types)66 std::set<ArTrackableType> GetArCoreEntityTypes(
67     const std::vector<device::mojom::EntityTypeForHitTest>& entity_types) {
68   std::set<ArTrackableType> result;
69 
70   std::transform(entity_types.begin(), entity_types.end(),
71                  std::inserter(result, result.end()), GetArCoreEntityType);
72 
73   return result;
74 }
75 
76 // Helper, computes mojo_from_input_source transform based on mojo_from_viever
77 // pose and input source state (containing input_from_pointer transform, which
78 // in case of input sources is equivalent to viewer_from_pointer).
79 // TODO(https://crbug.com/1043389): this currently assumes that the input source
80 // ray mode is "tapping", which is OK for input sources available for AR on
81 // Android, but is not true in the general case. This method should duplicate
82 // the logic found in XRTargetRaySpace::MojoFromNative().
GetMojoFromInputSource(const device::mojom::XRInputSourceStatePtr & input_source_state,const gfx::Transform & mojo_from_viewer)83 base::Optional<gfx::Transform> GetMojoFromInputSource(
84     const device::mojom::XRInputSourceStatePtr& input_source_state,
85     const gfx::Transform& mojo_from_viewer) {
86   if (!input_source_state->description ||
87       !input_source_state->description->input_from_pointer) {
88     return base::nullopt;
89   }
90 
91   gfx::Transform viewer_from_pointer =
92       *input_source_state->description->input_from_pointer;
93 
94   return mojo_from_viewer * viewer_from_pointer;
95 }
96 
ReleaseArCoreCubemap(ArImageCubemap * cube_map)97 void ReleaseArCoreCubemap(ArImageCubemap* cube_map) {
98   for (auto* image : *cube_map) {
99     ArImage_release(image);
100   }
101 
102   memset(cube_map, 0, sizeof(*cube_map));
103 }
104 
105 // Helper, copies ARCore image to the passed in buffer, assuming that the caller
106 // allocated the buffer to fit all the data.
107 template <typename T>
CopyArCoreImage(const ArSession * session,const ArImage * image,int32_t plane_index,base::span<T> out_pixels,uint32_t width,uint32_t height)108 void CopyArCoreImage(const ArSession* session,
109                      const ArImage* image,
110                      int32_t plane_index,
111                      base::span<T> out_pixels,
112                      uint32_t width,
113                      uint32_t height) {
114   DVLOG(3) << __func__ << ": width=" << width << ", height=" << height
115            << ", out_pixels.size()=" << out_pixels.size();
116 
117   DCHECK_GE(out_pixels.size(), width * height);
118 
119   int32_t src_row_stride = 0, src_pixel_stride = 0;
120   ArImage_getPlaneRowStride(session, image, plane_index, &src_row_stride);
121   ArImage_getPlanePixelStride(session, image, plane_index, &src_pixel_stride);
122 
123   // Naked pointer since ArImage_getPlaneData does not transfer ownership to us.
124   uint8_t const* src_buffer = nullptr;
125   int32_t src_buffer_length = 0;
126   ArImage_getPlaneData(session, image, plane_index, &src_buffer,
127                        &src_buffer_length);
128 
129   // Fast path: Source and destination have the same layout
130   bool const fast_path =
131       static_cast<size_t>(src_row_stride) == width * sizeof(T);
132   TRACE_EVENT1("xr", "CopyArCoreImage: memcpy", "fastPath", fast_path);
133 
134   DVLOG(3) << __func__ << ": plane_index=" << plane_index
135            << ", src_buffer_length=" << src_buffer_length
136            << ", src_row_stride=" << src_row_stride
137            << ", src_pixel_stride=" << src_pixel_stride
138            << ", fast_path=" << fast_path << ", sizeof(T)=" << sizeof(T);
139 
140   // If they have the same layout, we can copy the entire buffer at once
141   if (fast_path) {
142     CHECK_EQ(out_pixels.size() * sizeof(T),
143              static_cast<size_t>(src_buffer_length));
144     memcpy(out_pixels.data(), src_buffer, src_buffer_length);
145     return;
146   }
147 
148   CHECK_EQ(sizeof(T), static_cast<size_t>(src_pixel_stride));
149 
150   // Slow path: copy pixel by pixel, row by row
151   for (uint32_t row = 0; row < height; ++row) {
152     auto* src = src_buffer + src_row_stride * row;
153     auto* dest = out_pixels.data() + width * row;
154 
155     // For each pixel
156     for (uint32_t x = 0; x < width; ++x) {
157       memcpy(dest, src, sizeof(T));
158 
159       src += src_pixel_stride;
160       dest += 1;
161     }
162   }
163 }
164 
165 // Helper, copies ARCore image to the passed in vector, discovering the buffer
166 // size and resizing the vector first.
167 template <typename T>
CopyArCoreImage(const ArSession * session,const ArImage * image,int32_t plane_index,std::vector<T> * out_pixels,uint32_t * out_width,uint32_t * out_height)168 void CopyArCoreImage(const ArSession* session,
169                      const ArImage* image,
170                      int32_t plane_index,
171                      std::vector<T>* out_pixels,
172                      uint32_t* out_width,
173                      uint32_t* out_height) {
174   // Get source image information
175   int32_t width = 0, height = 0;
176   ArImage_getWidth(session, image, &width);
177   ArImage_getHeight(session, image, &height);
178 
179   *out_width = width;
180   *out_height = height;
181 
182   // Allocate memory for the output.
183   out_pixels->resize(width * height);
184 
185   CopyArCoreImage(session, image, plane_index, base::span<T>(*out_pixels),
186                   width, height);
187 }
188 
GetLightProbe(ArSession * arcore_session,ArLightEstimate * arcore_light_estimate)189 device::mojom::XRLightProbePtr GetLightProbe(
190     ArSession* arcore_session,
191     ArLightEstimate* arcore_light_estimate) {
192   // ArCore hands out 9 sets of RGB spherical harmonics coefficients
193   // https://developers.google.com/ar/reference/c/group/light#arlightestimate_getenvironmentalhdrambientsphericalharmonics
194   constexpr size_t kNumShCoefficients = 9;
195 
196   auto light_probe = device::mojom::XRLightProbe::New();
197 
198   light_probe->spherical_harmonics = device::mojom::XRSphericalHarmonics::New();
199   light_probe->spherical_harmonics->coefficients =
200       std::vector<device::RgbTupleF32>(kNumShCoefficients,
201                                        device::RgbTupleF32{});
202 
203   ArLightEstimate_getEnvironmentalHdrAmbientSphericalHarmonics(
204       arcore_session, arcore_light_estimate,
205       light_probe->spherical_harmonics->coefficients.data()->components);
206 
207   float main_light_direction[3] = {0};
208   ArLightEstimate_getEnvironmentalHdrMainLightDirection(
209       arcore_session, arcore_light_estimate, main_light_direction);
210   light_probe->main_light_direction.set_x(main_light_direction[0]);
211   light_probe->main_light_direction.set_y(main_light_direction[1]);
212   light_probe->main_light_direction.set_z(main_light_direction[2]);
213 
214   ArLightEstimate_getEnvironmentalHdrMainLightIntensity(
215       arcore_session, arcore_light_estimate,
216       light_probe->main_light_intensity.components);
217 
218   return light_probe;
219 }
220 
GetReflectionProbe(ArSession * arcore_session,ArLightEstimate * arcore_light_estimate)221 device::mojom::XRReflectionProbePtr GetReflectionProbe(
222     ArSession* arcore_session,
223     ArLightEstimate* arcore_light_estimate) {
224   ArImageCubemap arcore_cube_map = {nullptr};
225   ArLightEstimate_acquireEnvironmentalHdrCubemap(
226       arcore_session, arcore_light_estimate, arcore_cube_map);
227 
228   auto cube_map = device::mojom::XRCubeMap::New();
229   std::vector<device::RgbaTupleF16>* const cube_map_faces[] = {
230       &cube_map->positive_x, &cube_map->negative_x, &cube_map->positive_y,
231       &cube_map->negative_y, &cube_map->positive_z, &cube_map->negative_z};
232 
233   static_assert(
234       base::size(cube_map_faces) == base::size(arcore_cube_map),
235       "`ArImageCubemap` and `device::mojom::XRCubeMap` are expected to "
236       "have the same number of faces (6).");
237 
238   static_assert(device::mojom::XRCubeMap::kNumComponentsPerPixel == 4,
239                 "`device::mojom::XRCubeMap::kNumComponentsPerPixel` is "
240                 "expected to be 4 (RGBA)`, as that's the format ArCore uses.");
241 
242   for (size_t i = 0; i < base::size(arcore_cube_map); ++i) {
243     auto* arcore_cube_map_face = arcore_cube_map[i];
244     if (!arcore_cube_map_face) {
245       DVLOG(1) << "`ArLightEstimate_acquireEnvironmentalHdrCubemap` failed to "
246                   "return all faces";
247       ReleaseArCoreCubemap(&arcore_cube_map);
248       return nullptr;
249     }
250 
251     auto* cube_map_face = cube_map_faces[i];
252 
253     // Make sure we only have a single image plane
254     int32_t num_planes = 0;
255     ArImage_getNumberOfPlanes(arcore_session, arcore_cube_map_face,
256                               &num_planes);
257     if (num_planes != 1) {
258       DVLOG(1) << "ArCore cube map face " << i
259                << " does not have exactly 1 plane.";
260       ReleaseArCoreCubemap(&arcore_cube_map);
261       return nullptr;
262     }
263 
264     // Make sure the format for the image is in RGBA16F
265     ArImageFormat format = AR_IMAGE_FORMAT_INVALID;
266     ArImage_getFormat(arcore_session, arcore_cube_map_face, &format);
267     if (format != AR_IMAGE_FORMAT_RGBA_FP16) {
268       DVLOG(1) << "ArCore cube map face " << i
269                << " not in expected image format.";
270       ReleaseArCoreCubemap(&arcore_cube_map);
271       return nullptr;
272     }
273 
274     // Copy the cubemap
275     uint32_t face_width = 0, face_height = 0;
276     CopyArCoreImage(arcore_session, arcore_cube_map_face, 0, cube_map_face,
277                     &face_width, &face_height);
278 
279     // Make sure the cube map is square
280     if (face_width != face_height) {
281       DVLOG(1) << "ArCore cube map contains non-square image.";
282       ReleaseArCoreCubemap(&arcore_cube_map);
283       return nullptr;
284     }
285 
286     // Make sure all faces have the same dimensions
287     if (i == 0) {
288       cube_map->width_and_height = face_width;
289     } else if (face_width != cube_map->width_and_height ||
290                face_height != cube_map->width_and_height) {
291       DVLOG(1) << "ArCore cube map faces not all of the same dimensions.";
292       ReleaseArCoreCubemap(&arcore_cube_map);
293       return nullptr;
294     }
295   }
296 
297   ReleaseArCoreCubemap(&arcore_cube_map);
298 
299   auto reflection_probe = device::mojom::XRReflectionProbe::New();
300   reflection_probe->cube_map = std::move(cube_map);
301   return reflection_probe;
302 }
303 
304 constexpr float kDefaultFloorHeightEstimation = 1.2;
305 
306 }  // namespace
307 
308 namespace device {
309 
HitTestSubscriptionData(mojom::XRNativeOriginInformationPtr native_origin_information,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)310 HitTestSubscriptionData::HitTestSubscriptionData(
311     mojom::XRNativeOriginInformationPtr native_origin_information,
312     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
313     mojom::XRRayPtr ray)
314     : native_origin_information(std::move(native_origin_information)),
315       entity_types(entity_types),
316       ray(std::move(ray)) {}
317 
318 HitTestSubscriptionData::HitTestSubscriptionData(
319     HitTestSubscriptionData&& other) = default;
320 HitTestSubscriptionData::~HitTestSubscriptionData() = default;
321 
TransientInputHitTestSubscriptionData(const std::string & profile_name,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)322 TransientInputHitTestSubscriptionData::TransientInputHitTestSubscriptionData(
323     const std::string& profile_name,
324     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
325     mojom::XRRayPtr ray)
326     : profile_name(profile_name),
327       entity_types(entity_types),
328       ray(std::move(ray)) {}
329 
330 TransientInputHitTestSubscriptionData::TransientInputHitTestSubscriptionData(
331     TransientInputHitTestSubscriptionData&& other) = default;
332 TransientInputHitTestSubscriptionData::
333     ~TransientInputHitTestSubscriptionData() = default;
334 
ArCoreImpl()335 ArCoreImpl::ArCoreImpl()
336     : gl_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()) {}
337 
~ArCoreImpl()338 ArCoreImpl::~ArCoreImpl() {
339   for (auto& create_anchor : create_anchor_requests_) {
340     create_anchor.TakeCallback().Run(mojom::CreateAnchorResult::FAILURE, 0);
341   }
342 
343   for (auto& create_anchor : create_plane_attached_anchor_requests_) {
344     create_anchor.TakeCallback().Run(mojom::CreateAnchorResult::FAILURE, 0);
345   }
346 }
347 
Initialize(base::android::ScopedJavaLocalRef<jobject> context,const std::unordered_set<device::mojom::XRSessionFeature> & required_features,const std::unordered_set<device::mojom::XRSessionFeature> & optional_features,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)348 base::Optional<ArCore::InitializeResult> ArCoreImpl::Initialize(
349     base::android::ScopedJavaLocalRef<jobject> context,
350     const std::unordered_set<device::mojom::XRSessionFeature>&
351         required_features,
352     const std::unordered_set<device::mojom::XRSessionFeature>&
353         optional_features,
354     const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
355   DCHECK(IsOnGlThread());
356   DCHECK(!arcore_session_.is_valid());
357 
358   // TODO(https://crbug.com/837944): Notify error earlier if this will fail.
359 
360   JNIEnv* env = base::android::AttachCurrentThread();
361   if (!env) {
362     DLOG(ERROR) << "Unable to get JNIEnv for ArCore";
363     return base::nullopt;
364   }
365 
366   // Use a local scoped ArSession for the next steps, we want the
367   // arcore_session_ member to remain null until we complete successful
368   // initialization.
369   internal::ScopedArCoreObject<ArSession*> session;
370 
371   ArStatus status = ArSession_create(
372       env, context.obj(),
373       internal::ScopedArCoreObject<ArSession*>::Receiver(session).get());
374   if (status != AR_SUCCESS) {
375     DLOG(ERROR) << "ArSession_create failed: " << status;
376     return base::nullopt;
377   }
378 
379   // Set incognito mode for ARCore session - this is done unconditionally as we
380   // always want to limit the amount of logging done by ARCore.
381   ArSession_enableIncognitoMode_private(session.get());
382   DVLOG(1) << __func__ << ": ARCore incognito mode enabled";
383 
384   base::Optional<std::unordered_set<device::mojom::XRSessionFeature>>
385       maybe_enabled_features = ConfigureFeatures(
386           session.get(), required_features, optional_features, tracked_images);
387 
388   if (!maybe_enabled_features) {
389     DLOG(ERROR) << "Failed to configure session features";
390     return base::nullopt;
391   }
392 
393   if (!ConfigureCamera(session.get())) {
394     DLOG(ERROR) << "Failed to configure session camera";
395     return base::nullopt;
396   }
397 
398   internal::ScopedArCoreObject<ArFrame*> frame;
399   ArFrame_create(session.get(),
400                  internal::ScopedArCoreObject<ArFrame*>::Receiver(frame).get());
401   if (!frame.is_valid()) {
402     DLOG(ERROR) << "ArFrame_create failed";
403     return base::nullopt;
404   }
405 
406   internal::ScopedArCoreObject<ArLightEstimate*> light_estimate;
407   ArLightEstimate_create(
408       session.get(),
409       internal::ScopedArCoreObject<ArLightEstimate*>::Receiver(light_estimate)
410           .get());
411   if (!light_estimate.is_valid()) {
412     DVLOG(1) << "ArLightEstimate_create failed";
413     return base::nullopt;
414   }
415 
416   // Success, we now have a valid session and a valid frame.
417   arcore_frame_ = std::move(frame);
418   arcore_session_ = std::move(session);
419   arcore_light_estimate_ = std::move(light_estimate);
420   anchor_manager_ = std::make_unique<ArCoreAnchorManager>(
421       util::PassKey<ArCoreImpl>(), arcore_session_.get());
422   plane_manager_ = std::make_unique<ArCorePlaneManager>(
423       util::PassKey<ArCoreImpl>(), arcore_session_.get());
424   return ArCore::InitializeResult(*maybe_enabled_features);
425 }
426 
427 base::Optional<std::unordered_set<device::mojom::XRSessionFeature>>
ConfigureFeatures(ArSession * ar_session,const std::unordered_set<device::mojom::XRSessionFeature> & required_features,const std::unordered_set<device::mojom::XRSessionFeature> & optional_features,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)428 ArCoreImpl::ConfigureFeatures(
429     ArSession* ar_session,
430     const std::unordered_set<device::mojom::XRSessionFeature>&
431         required_features,
432     const std::unordered_set<device::mojom::XRSessionFeature>&
433         optional_features,
434     const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
435   // Let's assume we will be able to configure a session with all features -
436   // this will be adjusted if it turns out we can only create a session w/o some
437   // optional features. Currently, only depth sensing is not supported across
438   // all the ARCore-capable devices.
439   std::unordered_set<device::mojom::XRSessionFeature> enabled_features;
440   enabled_features.insert(required_features.begin(), required_features.end());
441   enabled_features.insert(optional_features.begin(), optional_features.end());
442 
443   internal::ScopedArCoreObject<ArConfig*> arcore_config;
444   ArConfig_create(
445       ar_session,
446       internal::ScopedArCoreObject<ArConfig*>::Receiver(arcore_config).get());
447   if (!arcore_config.is_valid()) {
448     DLOG(ERROR) << __func__ << ": ArConfig_create failed";
449     return base::nullopt;
450   }
451 
452   const bool light_estimation_requested =
453       base::Contains(required_features,
454                      device::mojom::XRSessionFeature::LIGHT_ESTIMATION) ||
455       base::Contains(optional_features,
456                      device::mojom::XRSessionFeature::LIGHT_ESTIMATION);
457 
458   if (light_estimation_requested) {
459     // Enable lighting estimation with spherical harmonics
460     ArConfig_setLightEstimationMode(ar_session, arcore_config.get(),
461                                     AR_LIGHT_ESTIMATION_MODE_ENVIRONMENTAL_HDR);
462   }
463 
464   const bool image_tracking_requested =
465       base::Contains(required_features,
466                      device::mojom::XRSessionFeature::IMAGE_TRACKING) ||
467       base::Contains(optional_features,
468                      device::mojom::XRSessionFeature::IMAGE_TRACKING);
469 
470   if (image_tracking_requested) {
471     internal::ScopedArCoreObject<ArAugmentedImageDatabase*> image_db;
472     ArAugmentedImageDatabase_create(
473         ar_session,
474         internal::ScopedArCoreObject<ArAugmentedImageDatabase*>::Receiver(
475             image_db)
476             .get());
477     if (!image_db.is_valid()) {
478       DLOG(ERROR) << "ArAugmentedImageDatabase creation failed";
479       return base::nullopt;
480     }
481 
482     // Populate the image tracking database and set up data structures,
483     // this doesn't modify the ArConfig or session yet.
484     BuildImageDatabase(ar_session, image_db.get(), tracked_images);
485 
486     if (!tracked_image_arcore_id_to_index_.empty()) {
487       // Image tracking with a non-empty image DB adds a few frames of
488       // synchronization delay internally in ARCore, has a high CPU cost, and
489       // reconfigures its graphics pipeline. Only activate it if we got images.
490       // (Apparently an empty image db is equivalent, but that seems fragile.)
491       ArConfig_setAugmentedImageDatabase(ar_session, arcore_config.get(),
492                                          image_db.get());
493       // Switch to autofocus mode when tracking images. The default fixed focus
494       // mode has trouble tracking close images since they end up blurry.
495       ArConfig_setFocusMode(ar_session, arcore_config.get(),
496                             AR_FOCUS_MODE_AUTO);
497     }
498   }
499 
500   const bool depth_api_optional =
501       base::Contains(optional_features, device::mojom::XRSessionFeature::DEPTH);
502   const bool depth_api_requested =
503       base::Contains(required_features,
504                      device::mojom::XRSessionFeature::DEPTH) ||
505       depth_api_optional;
506 
507   if (depth_api_requested) {
508     ArConfig_setDepthMode(ar_session, arcore_config.get(),
509                           AR_DEPTH_MODE_AUTOMATIC);
510   }
511 
512   ArStatus status = ArSession_configure(ar_session, arcore_config.get());
513   if (status != AR_SUCCESS && depth_api_optional) {
514     // Depth API is not available on some ARCore-capable devices - if it was
515     // requested optionally, let's try to request the session w/o it.
516 
517     DLOG(WARNING) << __func__
518                   << ": Depth API was optionally requested and the session "
519                      "creation failed, re-trying with depth API disabled";
520 
521     enabled_features.erase(device::mojom::XRSessionFeature::DEPTH);
522 
523     ArConfig_setDepthMode(ar_session, arcore_config.get(),
524                           AR_DEPTH_MODE_DISABLED);
525 
526     status = ArSession_configure(ar_session, arcore_config.get());
527   }
528 
529   if (status != AR_SUCCESS) {
530     DLOG(ERROR) << __func__ << ": ArSession_configure failed: " << status;
531     return base::nullopt;
532   }
533 
534   return enabled_features;
535 }
536 
ConfigureCamera(ArSession * ar_session)537 bool ArCoreImpl::ConfigureCamera(ArSession* ar_session) {
538   internal::ScopedArCoreObject<ArCameraConfigFilter*> camera_config_filter;
539   ArCameraConfigFilter_create(
540       ar_session, internal::ScopedArCoreObject<ArCameraConfigFilter*>::Receiver(
541                       camera_config_filter)
542                       .get());
543   if (!camera_config_filter.is_valid()) {
544     DLOG(ERROR) << "ArCameraConfigFilter_create failed";
545     return false;
546   }
547 
548   // We only want to work at 30fps for now.
549   ArCameraConfigFilter_setTargetFps(ar_session, camera_config_filter.get(),
550                                     AR_CAMERA_CONFIG_TARGET_FPS_30);
551   // We do not care if depth sensor is available or not for now.
552   // The default depth sensor usage of the newly created filter is not
553   // documented, so let's set the filter explicitly to accept both cameras with
554   // and without depth sensors.
555   ArCameraConfigFilter_setDepthSensorUsage(
556       ar_session, camera_config_filter.get(),
557       AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE |
558           AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_DO_NOT_USE);
559 
560   internal::ScopedArCoreObject<ArCameraConfigList*> camera_config_list;
561   ArCameraConfigList_create(
562       ar_session, internal::ScopedArCoreObject<ArCameraConfigList*>::Receiver(
563                       camera_config_list)
564                       .get());
565 
566   if (!camera_config_list.is_valid()) {
567     DLOG(ERROR) << "ArCameraConfigList_create failed";
568     return false;
569   }
570 
571   ArSession_getSupportedCameraConfigsWithFilter(
572       ar_session, camera_config_filter.get(), camera_config_list.get());
573   if (!camera_config_list.is_valid()) {
574     DLOG(ERROR) << "ArSession_getSupportedCameraConfigsWithFilter failed";
575     return false;
576   }
577 
578   int32_t available_configs_count;
579   ArCameraConfigList_getSize(ar_session, camera_config_list.get(),
580                              &available_configs_count);
581 
582   DVLOG(2) << __func__ << ": ARCore reported " << available_configs_count
583            << " available configurations";
584 
585   std::vector<internal::ScopedArCoreObject<ArCameraConfig*>> available_configs;
586   available_configs.reserve(available_configs_count);
587   for (int32_t i = 0; i < available_configs_count; ++i) {
588     internal::ScopedArCoreObject<ArCameraConfig*> camera_config;
589     ArCameraConfig_create(
590         ar_session,
591         internal::ScopedArCoreObject<ArCameraConfig*>::Receiver(camera_config)
592             .get());
593 
594     if (!camera_config.is_valid()) {
595       DVLOG(1) << __func__
596                << ": ArCameraConfig_create failed for camera config at index "
597                << i;
598       continue;
599     }
600 
601     ArCameraConfigList_getItem(ar_session, camera_config_list.get(), i,
602                                camera_config.get());
603 
604     ArCameraConfigFacingDirection facing_direction;
605     ArCameraConfig_getFacingDirection(ar_session, camera_config.get(),
606                                       &facing_direction);
607 
608     if (facing_direction != AR_CAMERA_CONFIG_FACING_DIRECTION_BACK) {
609       DVLOG(2)
610           << __func__
611           << ": camera config does not refer to back-facing camera, ignoring";
612       continue;
613     }
614 
615 #if DCHECK_IS_ON()
616     {
617       int32_t tex_width, tex_height;
618       ArCameraConfig_getTextureDimensions(ar_session, camera_config.get(),
619                                           &tex_width, &tex_height);
620 
621       int32_t img_width, img_height;
622       ArCameraConfig_getImageDimensions(ar_session, camera_config.get(),
623                                         &img_width, &img_height);
624 
625       uint32_t depth_sensor_usage;
626       ArCameraConfig_getDepthSensorUsage(ar_session, camera_config.get(),
627                                          &depth_sensor_usage);
628 
629       int32_t min_fps, max_fps;
630       ArCameraConfig_getFpsRange(ar_session, camera_config.get(), &min_fps,
631                                  &max_fps);
632 
633       DVLOG(3) << __func__
634                << ": matching camera config found, texture dimensions="
635                << tex_width << "x" << tex_height
636                << ", image dimensions= " << img_width << "x" << img_height
637                << ", depth sensor usage=" << depth_sensor_usage
638                << ", min_fps=" << min_fps << ", max_fps=" << max_fps;
639     }
640 #endif
641 
642     available_configs.push_back(std::move(camera_config));
643   }
644 
645   if (available_configs.empty()) {
646     DLOG(ERROR) << "No matching configs found";
647     return false;
648   }
649 
650   auto best_config = std::max_element(
651       available_configs.begin(), available_configs.end(),
652       [ar_session](const internal::ScopedArCoreObject<ArCameraConfig*>& lhs,
653                    const internal::ScopedArCoreObject<ArCameraConfig*>& rhs) {
654         // true means that lhs is less than rhs
655 
656         // We'll prefer the configs with higher total resolution (GPU first,
657         // then CPU), everything else does not matter for us now, but we will
658         // weakly prefer the cameras that support depth sensor (it will be used
659         // as a tie-breaker).
660 
661         {
662           int32_t lhs_tex_width, lhs_tex_height;
663           int32_t rhs_tex_width, rhs_tex_height;
664 
665           ArCameraConfig_getTextureDimensions(ar_session, lhs.get(),
666                                               &lhs_tex_width, &lhs_tex_height);
667           ArCameraConfig_getTextureDimensions(ar_session, rhs.get(),
668                                               &rhs_tex_width, &rhs_tex_height);
669 
670           if (lhs_tex_width * lhs_tex_height !=
671               rhs_tex_width * rhs_tex_height) {
672             return lhs_tex_width * lhs_tex_height <
673                    rhs_tex_width * rhs_tex_height;
674           }
675         }
676 
677         {
678           int32_t lhs_img_width, lhs_img_height;
679           int32_t rhs_img_width, rhs_img_height;
680 
681           ArCameraConfig_getImageDimensions(ar_session, lhs.get(),
682                                             &lhs_img_width, &lhs_img_height);
683           ArCameraConfig_getImageDimensions(ar_session, rhs.get(),
684                                             &rhs_img_width, &rhs_img_height);
685 
686           if (lhs_img_width * lhs_img_height !=
687               rhs_img_width * rhs_img_height) {
688             return lhs_img_width * lhs_img_height <
689                    rhs_img_width * rhs_img_height;
690           }
691         }
692 
693         {
694           uint32_t lhs_depth_sensor_usage;
695           uint32_t rhs_depth_sensor_usage;
696 
697           ArCameraConfig_getDepthSensorUsage(ar_session, lhs.get(),
698                                              &lhs_depth_sensor_usage);
699           ArCameraConfig_getDepthSensorUsage(ar_session, rhs.get(),
700                                              &rhs_depth_sensor_usage);
701 
702           bool lhs_has_depth =
703               lhs_depth_sensor_usage &
704               AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE;
705           bool rhs_has_depth =
706               rhs_depth_sensor_usage &
707               AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE;
708 
709           return lhs_has_depth < rhs_has_depth;
710         }
711       });
712 
713   int32_t fps_min, fps_max;
714   ArCameraConfig_getFpsRange(ar_session, best_config->get(), &fps_min,
715                              &fps_max);
716   target_framerate_range_ = {fps_min, fps_max};
717 
718 #if DCHECK_IS_ON()
719   {
720     int32_t tex_width, tex_height;
721     ArCameraConfig_getTextureDimensions(ar_session, best_config->get(),
722                                         &tex_width, &tex_height);
723 
724     int32_t img_width, img_height;
725     ArCameraConfig_getImageDimensions(ar_session, best_config->get(),
726                                       &img_width, &img_height);
727 
728     uint32_t depth_sensor_usage;
729     ArCameraConfig_getDepthSensorUsage(ar_session, best_config->get(),
730                                        &depth_sensor_usage);
731     DVLOG(3) << __func__
732              << ": selected camera config with texture dimensions=" << tex_width
733              << "x" << tex_height << ", image dimensions=" << img_width << "x"
734              << img_height << ", depth sensor usage=" << depth_sensor_usage
735              << ", min_fps=" << target_framerate_range_.min
736              << ", max_fps=" << target_framerate_range_.max;
737   }
738 #endif
739 
740   ArStatus status = ArSession_setCameraConfig(ar_session, best_config->get());
741   if (status != AR_SUCCESS) {
742     DLOG(ERROR) << "ArSession_setCameraConfig failed: " << status;
743     return false;
744   }
745 
746   return true;
747 }
748 
GetTargetFramerateRange()749 ArCore::MinMaxRange ArCoreImpl::GetTargetFramerateRange() {
750   return target_framerate_range_;
751 }
752 
BuildImageDatabase(const ArSession * session,ArAugmentedImageDatabase * image_db,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)753 void ArCoreImpl::BuildImageDatabase(
754     const ArSession* session,
755     ArAugmentedImageDatabase* image_db,
756     const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
757   for (std::size_t index = 0; index < tracked_images.size(); ++index) {
758     const device::mojom::XRTrackedImage* image = tracked_images[index].get();
759     gfx::Size size = image->size_in_pixels;
760 
761     // Use Skia to convert the image to grayscale.
762     const SkBitmap& src_bitmap = image->bitmap;
763     SkBitmap canvas_bitmap;
764     canvas_bitmap.allocPixelsFlags(
765         SkImageInfo::Make(size.width(), size.height(), kGray_8_SkColorType,
766                           kOpaque_SkAlphaType),
767         SkBitmap::kZeroPixels_AllocFlag);
768     SkCanvas gray_canvas(canvas_bitmap);
769     SkPaint paint;
770     sk_sp<SkImage> src_image = SkImage::MakeFromBitmap(src_bitmap);
771     gray_canvas.drawImage(src_image, 0, 0, &paint);
772     SkPixmap gray_pixmap;
773     if (!gray_canvas.peekPixels(&gray_pixmap)) {
774       DLOG(WARNING) << __func__ << ": failed to access grayscale bitmap";
775       image_trackable_scores_.push_back(false);
776       continue;
777     }
778 
779     const SkPixmap& pixmap = gray_pixmap;
780     float width_in_meters = image->width_in_meters;
781     DVLOG(3) << __func__ << " tracked image index=" << index
782              << " size=" << pixmap.width() << "x" << pixmap.height()
783              << " width_in_meters=" << width_in_meters;
784     int32_t arcore_id = -1;
785     std::string id_name = base::NumberToString(index);
786     ArStatus status = ArAugmentedImageDatabase_addImageWithPhysicalSize(
787         session, image_db, id_name.c_str(), pixmap.addr8(), pixmap.width(),
788         pixmap.height(), pixmap.rowBytesAsPixels(), width_in_meters,
789         &arcore_id);
790     if (status != AR_SUCCESS) {
791       DVLOG(2) << __func__ << ": add image failed";
792       image_trackable_scores_.push_back(false);
793       continue;
794     }
795 
796     // ARCore uses internal IDs for images, these only include the trackable
797     // images. The tracking results need to refer to the original image index
798     // corresponding to its position in the input tracked_images array.
799     tracked_image_arcore_id_to_index_[arcore_id] = index;
800     DVLOG(2) << __func__ << ": added image, index=" << index
801              << " arcore_id=" << arcore_id;
802     image_trackable_scores_.push_back(true);
803   }
804 }
805 
SetCameraTexture(uint32_t camera_texture_id)806 void ArCoreImpl::SetCameraTexture(uint32_t camera_texture_id) {
807   DCHECK(IsOnGlThread());
808   DCHECK(arcore_session_.is_valid());
809   ArSession_setCameraTextureName(arcore_session_.get(), camera_texture_id);
810 }
811 
SetDisplayGeometry(const gfx::Size & frame_size,display::Display::Rotation display_rotation)812 void ArCoreImpl::SetDisplayGeometry(
813     const gfx::Size& frame_size,
814     display::Display::Rotation display_rotation) {
815   DCHECK(IsOnGlThread());
816   DCHECK(arcore_session_.is_valid());
817   // Display::Rotation is the same as Android's rotation and is compatible with
818   // what ArCore is expecting.
819   ArSession_setDisplayGeometry(arcore_session_.get(), display_rotation,
820                                frame_size.width(), frame_size.height());
821 }
822 
TransformDisplayUvCoords(const base::span<const float> uvs) const823 std::vector<float> ArCoreImpl::TransformDisplayUvCoords(
824     const base::span<const float> uvs) const {
825   DCHECK(IsOnGlThread());
826   DCHECK(arcore_session_.is_valid());
827   DCHECK(arcore_frame_.is_valid());
828 
829   size_t num_elements = uvs.size();
830   DCHECK(num_elements % 2 == 0);
831   DCHECK_GE(num_elements, 6u);
832 
833   std::vector<float> uvs_out(num_elements);
834   ArFrame_transformCoordinates2d(
835       arcore_session_.get(), arcore_frame_.get(),
836       AR_COORDINATES_2D_VIEW_NORMALIZED, num_elements / 2, &uvs[0],
837       AR_COORDINATES_2D_TEXTURE_NORMALIZED, &uvs_out[0]);
838 
839   DVLOG(3) << __func__ << ": transformed uvs=[ " << uvs_out[0] << " , "
840            << uvs_out[1] << " , " << uvs_out[2] << " , " << uvs_out[3] << " , "
841            << uvs_out[4] << " , " << uvs_out[5] << " ]";
842 
843   return uvs_out;
844 }
845 
Update(bool * camera_updated)846 mojom::VRPosePtr ArCoreImpl::Update(bool* camera_updated) {
847   TRACE_EVENT0("gpu", "ArCoreImpl Update");
848 
849   DCHECK(IsOnGlThread());
850   DCHECK(arcore_session_.is_valid());
851   DCHECK(arcore_frame_.is_valid());
852   DCHECK(camera_updated);
853 
854   ArStatus status;
855 
856   TRACE_EVENT_BEGIN0("gpu", "ArCore Update");
857   status = ArSession_update(arcore_session_.get(), arcore_frame_.get());
858   TRACE_EVENT_END0("gpu", "ArCore Update");
859 
860   if (status != AR_SUCCESS) {
861     DLOG(ERROR) << "ArSession_update failed: " << status;
862     *camera_updated = false;
863     return nullptr;
864   }
865 
866   // If we get here, assume we have a valid camera image, but we don't know yet
867   // if tracking is working.
868   *camera_updated = true;
869   internal::ScopedArCoreObject<ArCamera*> arcore_camera;
870   ArFrame_acquireCamera(
871       arcore_session_.get(), arcore_frame_.get(),
872       internal::ScopedArCoreObject<ArCamera*>::Receiver(arcore_camera).get());
873   if (!arcore_camera.is_valid()) {
874     DLOG(ERROR) << "ArFrame_acquireCamera failed!";
875     return nullptr;
876   }
877 
878   ArTrackingState tracking_state;
879   ArCamera_getTrackingState(arcore_session_.get(), arcore_camera.get(),
880                             &tracking_state);
881   if (tracking_state != AR_TRACKING_STATE_TRACKING) {
882     DVLOG(1) << "Tracking state is not AR_TRACKING_STATE_TRACKING: "
883              << tracking_state;
884     return nullptr;
885   }
886 
887   internal::ScopedArCoreObject<ArPose*> arcore_pose;
888   ArPose_create(
889       arcore_session_.get(), nullptr,
890       internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
891   if (!arcore_pose.is_valid()) {
892     DLOG(ERROR) << "ArPose_create failed!";
893     return nullptr;
894   }
895 
896   ArCamera_getDisplayOrientedPose(arcore_session_.get(), arcore_camera.get(),
897                                   arcore_pose.get());
898 
899   auto mojo_from_viewer =
900       GetMojomVRPoseFromArPose(arcore_session_.get(), arcore_pose.get());
901 
902   TRACE_EVENT_BEGIN0("gpu", "ArCorePlaneManager Update");
903   plane_manager_->Update(arcore_frame_.get());
904   TRACE_EVENT_END0("gpu", "ArCorePlaneManager Update");
905 
906   TRACE_EVENT_BEGIN0("gpu", "ArCoreAnchorManager Update");
907   anchor_manager_->Update(arcore_frame_.get());
908   TRACE_EVENT_END0("gpu", "ArCoreAnchorManager Update");
909 
910   return mojo_from_viewer;
911 }
912 
GetTrackedImages()913 mojom::XRTrackedImagesDataPtr ArCoreImpl::GetTrackedImages() {
914   std::vector<mojom::XRTrackedImageDataPtr> images_data;
915 
916   internal::ScopedArCoreObject<ArTrackableList*> updated_images;
917   ArTrackableList_create(
918       arcore_session_.get(),
919       internal::ScopedArCoreObject<ArTrackableList*>::Receiver(updated_images)
920           .get());
921   ArFrame_getUpdatedTrackables(arcore_session_.get(), arcore_frame_.get(),
922                                AR_TRACKABLE_AUGMENTED_IMAGE,
923                                updated_images.get());
924 
925   int32_t images_count = 0;
926   ArTrackableList_getSize(arcore_session_.get(), updated_images.get(),
927                           &images_count);
928   DVLOG(2) << __func__ << ": trackable images count=" << images_count;
929 
930   for (int32_t i = 0; i < images_count; ++i) {
931     internal::ScopedArCoreObject<ArTrackable*> trackable;
932     ArTrackableList_acquireItem(
933         arcore_session_.get(), updated_images.get(), i,
934         internal::ScopedArCoreObject<ArTrackable*>::Receiver(trackable).get());
935     ArTrackingState tracking_state;
936     ArTrackable_getTrackingState(arcore_session_.get(), trackable.get(),
937                                  &tracking_state);
938     ArAugmentedImage* image = ArAsAugmentedImage(trackable.get());
939 
940     // Get the original image index from ARCore's internal ID for use in the
941     // returned results.
942     int32_t arcore_id;
943     ArAugmentedImage_getIndex(arcore_session_.get(), image, &arcore_id);
944     uint64_t index = tracked_image_arcore_id_to_index_[arcore_id];
945     DVLOG(3) << __func__ << ": #" << i << " tracked image index=" << index
946              << " arcore_id=" << arcore_id << " state=" << tracking_state;
947 
948     if (tracking_state == AR_TRACKING_STATE_TRACKING) {
949       internal::ScopedArCoreObject<ArPose*> arcore_pose;
950       ArPose_create(
951           arcore_session_.get(), nullptr,
952           internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
953       if (!arcore_pose.is_valid()) {
954         DLOG(ERROR) << "ArPose_create failed!";
955         continue;
956       }
957       ArAugmentedImage_getCenterPose(arcore_session_.get(), image,
958                                      arcore_pose.get());
959       float pose_raw[7];
960       ArPose_getPoseRaw(arcore_session_.get(), arcore_pose.get(), &pose_raw[0]);
961       DVLOG(3) << __func__ << ": tracked image pose_raw pos=(" << pose_raw[4]
962                << ", " << pose_raw[5] << ", " << pose_raw[6] << ")";
963 
964       device::Pose device_pose =
965           GetPoseFromArPose(arcore_session_.get(), arcore_pose.get());
966 
967       ArAugmentedImageTrackingMethod tracking_method;
968       ArAugmentedImage_getTrackingMethod(arcore_session_.get(), image,
969                                          &tracking_method);
970       bool actively_tracked =
971           tracking_method == AR_AUGMENTED_IMAGE_TRACKING_METHOD_FULL_TRACKING;
972 
973       float width_in_meters;
974       ArAugmentedImage_getExtentX(arcore_session_.get(), image,
975                                   &width_in_meters);
976 
977       images_data.push_back(mojom::XRTrackedImageData::New(
978           index, device_pose, actively_tracked, width_in_meters));
979     }
980   }
981 
982   // Include information about each image's trackability status in the first
983   // returned result list.
984   if (!image_trackable_scores_sent_) {
985     auto ret = mojom::XRTrackedImagesData::New(
986         std::move(images_data), std::move(image_trackable_scores_));
987     image_trackable_scores_sent_ = true;
988     image_trackable_scores_.clear();
989     return ret;
990   } else {
991     return mojom::XRTrackedImagesData::New(std::move(images_data),
992                                            base::nullopt);
993   }
994 }
995 
GetFrameTimestamp()996 base::TimeDelta ArCoreImpl::GetFrameTimestamp() {
997   DCHECK(arcore_session_.is_valid());
998   DCHECK(arcore_frame_.is_valid());
999   int64_t out_timestamp_ns;
1000   ArFrame_getTimestamp(arcore_session_.get(), arcore_frame_.get(),
1001                        &out_timestamp_ns);
1002   return base::TimeDelta::FromNanoseconds(out_timestamp_ns);
1003 }
1004 
GetDetectedPlanesData()1005 mojom::XRPlaneDetectionDataPtr ArCoreImpl::GetDetectedPlanesData() {
1006   DVLOG(2) << __func__;
1007 
1008   TRACE_EVENT0("gpu", __func__);
1009 
1010   return plane_manager_->GetDetectedPlanesData();
1011 }
1012 
GetAnchorsData()1013 mojom::XRAnchorsDataPtr ArCoreImpl::GetAnchorsData() {
1014   DVLOG(2) << __func__;
1015 
1016   TRACE_EVENT0("gpu", __func__);
1017 
1018   return anchor_manager_->GetAnchorsData();
1019 }
1020 
GetLightEstimationData()1021 mojom::XRLightEstimationDataPtr ArCoreImpl::GetLightEstimationData() {
1022   TRACE_EVENT0("gpu", __func__);
1023 
1024   ArFrame_getLightEstimate(arcore_session_.get(), arcore_frame_.get(),
1025                            arcore_light_estimate_.get());
1026 
1027   ArLightEstimateState light_estimate_state = AR_LIGHT_ESTIMATE_STATE_NOT_VALID;
1028   ArLightEstimate_getState(arcore_session_.get(), arcore_light_estimate_.get(),
1029                            &light_estimate_state);
1030 
1031   // The light estimate state is not guaranteed to be valid initially
1032   if (light_estimate_state != AR_LIGHT_ESTIMATE_STATE_VALID) {
1033     DVLOG(2) << "ArCore light estimation state invalid.";
1034     return nullptr;
1035   }
1036 
1037   auto light_estimation_data = mojom::XRLightEstimationData::New();
1038   light_estimation_data->light_probe =
1039       GetLightProbe(arcore_session_.get(), arcore_light_estimate_.get());
1040   if (!light_estimation_data->light_probe) {
1041     DVLOG(1) << "Failed to generate light probe.";
1042     return nullptr;
1043   }
1044   light_estimation_data->reflection_probe =
1045       GetReflectionProbe(arcore_session_.get(), arcore_light_estimate_.get());
1046   if (!light_estimation_data->reflection_probe) {
1047     DVLOG(1) << "Failed to generate reflection probe.";
1048     return nullptr;
1049   }
1050 
1051   return light_estimation_data;
1052 }
1053 
Pause()1054 void ArCoreImpl::Pause() {
1055   DVLOG(2) << __func__;
1056 
1057   DCHECK(IsOnGlThread());
1058   DCHECK(arcore_session_.is_valid());
1059 
1060   ArStatus status = ArSession_pause(arcore_session_.get());
1061   DLOG_IF(ERROR, status != AR_SUCCESS)
1062       << "ArSession_pause failed: status = " << status;
1063 }
1064 
Resume()1065 void ArCoreImpl::Resume() {
1066   DVLOG(2) << __func__;
1067 
1068   DCHECK(IsOnGlThread());
1069   DCHECK(arcore_session_.is_valid());
1070 
1071   ArStatus status = ArSession_resume(arcore_session_.get());
1072   DLOG_IF(ERROR, status != AR_SUCCESS)
1073       << "ArSession_resume failed: status = " << status;
1074 }
1075 
GetProjectionMatrix(float near,float far)1076 gfx::Transform ArCoreImpl::GetProjectionMatrix(float near, float far) {
1077   DCHECK(IsOnGlThread());
1078   DCHECK(arcore_session_.is_valid());
1079   DCHECK(arcore_frame_.is_valid());
1080 
1081   internal::ScopedArCoreObject<ArCamera*> arcore_camera;
1082   ArFrame_acquireCamera(
1083       arcore_session_.get(), arcore_frame_.get(),
1084       internal::ScopedArCoreObject<ArCamera*>::Receiver(arcore_camera).get());
1085   DCHECK(arcore_camera.is_valid())
1086       << "ArFrame_acquireCamera failed despite documentation saying it cannot";
1087 
1088   // ArCore's projection matrix is 16 floats in column-major order.
1089   float matrix_4x4[16];
1090   ArCamera_getProjectionMatrix(arcore_session_.get(), arcore_camera.get(), near,
1091                                far, matrix_4x4);
1092   gfx::Transform result;
1093   result.matrix().setColMajorf(matrix_4x4);
1094   return result;
1095 }
1096 
GetEstimatedFloorHeight()1097 float ArCoreImpl::GetEstimatedFloorHeight() {
1098   return kDefaultFloorHeightEstimation;
1099 }
1100 
SubscribeToHitTest(mojom::XRNativeOriginInformationPtr native_origin_information,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)1101 base::Optional<uint64_t> ArCoreImpl::SubscribeToHitTest(
1102     mojom::XRNativeOriginInformationPtr native_origin_information,
1103     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1104     mojom::XRRayPtr ray) {
1105   // First, check if we recognize the type of the native origin.
1106 
1107   if (native_origin_information->is_reference_space_type()) {
1108     // Reference spaces are implicitly recognized and don't carry an ID.
1109   } else if (native_origin_information->is_input_source_id()) {
1110     // Input source IDs are verified in the higher layer as ArCoreImpl does
1111     // not carry input source state.
1112   } else if (native_origin_information->is_plane_id()) {
1113     // Validate that we know which plane's space the hit test is interested in
1114     // tracking.
1115     if (!plane_manager_->PlaneExists(
1116             PlaneId(native_origin_information->get_plane_id()))) {
1117       return base::nullopt;
1118     }
1119   } else if (native_origin_information->is_anchor_id()) {
1120     // Validate that we know which anchor's space the hit test is interested
1121     // in tracking.
1122     if (!anchor_manager_->AnchorExists(
1123             AnchorId(native_origin_information->get_anchor_id()))) {
1124       return base::nullopt;
1125     }
1126   } else {
1127     NOTREACHED();
1128     return base::nullopt;
1129   }
1130 
1131   auto subscription_id = CreateHitTestSubscriptionId();
1132 
1133   hit_test_subscription_id_to_data_.emplace(
1134       subscription_id,
1135       HitTestSubscriptionData{std::move(native_origin_information),
1136                               entity_types, std::move(ray)});
1137 
1138   return subscription_id.GetUnsafeValue();
1139 }
1140 
SubscribeToHitTestForTransientInput(const std::string & profile_name,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)1141 base::Optional<uint64_t> ArCoreImpl::SubscribeToHitTestForTransientInput(
1142     const std::string& profile_name,
1143     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1144     mojom::XRRayPtr ray) {
1145   auto subscription_id = CreateHitTestSubscriptionId();
1146 
1147   hit_test_subscription_id_to_transient_hit_test_data_.emplace(
1148       subscription_id, TransientInputHitTestSubscriptionData{
1149                            profile_name, entity_types, std::move(ray)});
1150 
1151   return subscription_id.GetUnsafeValue();
1152 }
1153 
1154 mojom::XRHitTestSubscriptionResultsDataPtr
GetHitTestSubscriptionResults(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1155 ArCoreImpl::GetHitTestSubscriptionResults(
1156     const gfx::Transform& mojo_from_viewer,
1157     const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1158   mojom::XRHitTestSubscriptionResultsDataPtr result =
1159       mojom::XRHitTestSubscriptionResultsData::New();
1160 
1161   DVLOG(3) << __func__
1162            << ": calculating hit test subscription results, "
1163               "hit_test_subscription_id_to_data_.size()="
1164            << hit_test_subscription_id_to_data_.size();
1165 
1166   for (auto& subscription_id_and_data : hit_test_subscription_id_to_data_) {
1167     // First, check if we can find the current transformation for a ray. If not,
1168     // skip processing this subscription.
1169     auto maybe_mojo_from_native_origin = GetMojoFromNativeOrigin(
1170         *subscription_id_and_data.second.native_origin_information,
1171         mojo_from_viewer, input_state);
1172 
1173     if (!maybe_mojo_from_native_origin) {
1174       continue;
1175     }
1176 
1177     // Since we have a transform, let's use it to obtain hit test results.
1178     result->results.push_back(GetHitTestSubscriptionResult(
1179         HitTestSubscriptionId(subscription_id_and_data.first),
1180         *subscription_id_and_data.second.ray,
1181         subscription_id_and_data.second.entity_types,
1182         *maybe_mojo_from_native_origin));
1183   }
1184 
1185   DVLOG(3)
1186       << __func__
1187       << ": calculating hit test subscription results for transient input, "
1188          "hit_test_subscription_id_to_transient_hit_test_data_.size()="
1189       << hit_test_subscription_id_to_transient_hit_test_data_.size();
1190 
1191   for (const auto& subscription_id_and_data :
1192        hit_test_subscription_id_to_transient_hit_test_data_) {
1193     auto input_source_ids_and_transforms =
1194         GetMojoFromInputSources(subscription_id_and_data.second.profile_name,
1195                                 mojo_from_viewer, input_state);
1196 
1197     result->transient_input_results.push_back(
1198         GetTransientHitTestSubscriptionResult(
1199             HitTestSubscriptionId(subscription_id_and_data.first),
1200             *subscription_id_and_data.second.ray,
1201             subscription_id_and_data.second.entity_types,
1202             input_source_ids_and_transforms));
1203   }
1204 
1205   return result;
1206 }
1207 
1208 device::mojom::XRHitTestSubscriptionResultDataPtr
GetHitTestSubscriptionResult(HitTestSubscriptionId id,const mojom::XRRay & native_origin_ray,const std::vector<mojom::EntityTypeForHitTest> & entity_types,const gfx::Transform & mojo_from_native_origin)1209 ArCoreImpl::GetHitTestSubscriptionResult(
1210     HitTestSubscriptionId id,
1211     const mojom::XRRay& native_origin_ray,
1212     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1213     const gfx::Transform& mojo_from_native_origin) {
1214   DVLOG(3) << __func__ << ": id=" << id;
1215 
1216   // Transform the ray according to the latest transform based on the XRSpace
1217   // used in hit test subscription.
1218 
1219   gfx::Point3F origin = native_origin_ray.origin;
1220   mojo_from_native_origin.TransformPoint(&origin);
1221 
1222   gfx::Vector3dF direction = native_origin_ray.direction;
1223   mojo_from_native_origin.TransformVector(&direction);
1224 
1225   std::vector<mojom::XRHitResultPtr> hit_results;
1226   if (!RequestHitTest(origin, direction, entity_types, &hit_results)) {
1227     hit_results.clear();  // On failure, clear partial results.
1228   }
1229 
1230   return mojom::XRHitTestSubscriptionResultData::New(id.GetUnsafeValue(),
1231                                                      std::move(hit_results));
1232 }
1233 
1234 device::mojom::XRHitTestTransientInputSubscriptionResultDataPtr
GetTransientHitTestSubscriptionResult(HitTestSubscriptionId id,const mojom::XRRay & input_source_ray,const std::vector<mojom::EntityTypeForHitTest> & entity_types,const std::vector<std::pair<uint32_t,gfx::Transform>> & input_source_ids_and_mojo_from_input_sources)1235 ArCoreImpl::GetTransientHitTestSubscriptionResult(
1236     HitTestSubscriptionId id,
1237     const mojom::XRRay& input_source_ray,
1238     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1239     const std::vector<std::pair<uint32_t, gfx::Transform>>&
1240         input_source_ids_and_mojo_from_input_sources) {
1241   auto result =
1242       device::mojom::XRHitTestTransientInputSubscriptionResultData::New();
1243 
1244   result->subscription_id = id.GetUnsafeValue();
1245 
1246   for (const auto& input_source_id_and_mojo_from_input_source :
1247        input_source_ids_and_mojo_from_input_sources) {
1248     gfx::Point3F origin = input_source_ray.origin;
1249     input_source_id_and_mojo_from_input_source.second.TransformPoint(&origin);
1250 
1251     gfx::Vector3dF direction = input_source_ray.direction;
1252     input_source_id_and_mojo_from_input_source.second.TransformVector(
1253         &direction);
1254 
1255     std::vector<mojom::XRHitResultPtr> hit_results;
1256     if (!RequestHitTest(origin, direction, entity_types, &hit_results)) {
1257       hit_results.clear();  // On failure, clear partial results.
1258     }
1259 
1260     result->input_source_id_to_hit_test_results.insert(
1261         {input_source_id_and_mojo_from_input_source.first,
1262          std::move(hit_results)});
1263   }
1264 
1265   return result;
1266 }
1267 
1268 std::vector<std::pair<uint32_t, gfx::Transform>>
GetMojoFromInputSources(const std::string & profile_name,const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1269 ArCoreImpl::GetMojoFromInputSources(
1270     const std::string& profile_name,
1271     const gfx::Transform& mojo_from_viewer,
1272     const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1273   std::vector<std::pair<uint32_t, gfx::Transform>> result;
1274 
1275   for (const auto& input_source_state : input_state) {
1276     if (input_source_state && input_source_state->description) {
1277       if (base::Contains(input_source_state->description->profiles,
1278                          profile_name)) {
1279         // Input source represented by input_state matches the profile, find
1280         // the transform and grab input source id.
1281         base::Optional<gfx::Transform> maybe_mojo_from_input_source =
1282             GetMojoFromInputSource(input_source_state, mojo_from_viewer);
1283 
1284         if (!maybe_mojo_from_input_source)
1285           continue;
1286 
1287         result.push_back(
1288             {input_source_state->source_id, *maybe_mojo_from_input_source});
1289       }
1290     }
1291   }
1292 
1293   return result;
1294 }
1295 
GetMojoFromReferenceSpace(device::mojom::XRReferenceSpaceType type,const gfx::Transform & mojo_from_viewer)1296 base::Optional<gfx::Transform> ArCoreImpl::GetMojoFromReferenceSpace(
1297     device::mojom::XRReferenceSpaceType type,
1298     const gfx::Transform& mojo_from_viewer) {
1299   DVLOG(3) << __func__ << ": type=" << type;
1300 
1301   switch (type) {
1302     case device::mojom::XRReferenceSpaceType::kLocal:
1303       return gfx::Transform{};
1304     case device::mojom::XRReferenceSpaceType::kLocalFloor: {
1305       auto result = gfx::Transform{};
1306       result.Translate3d(0, -GetEstimatedFloorHeight(), 0);
1307       return result;
1308     }
1309     case device::mojom::XRReferenceSpaceType::kViewer:
1310       return mojo_from_viewer;
1311     case device::mojom::XRReferenceSpaceType::kBoundedFloor:
1312       return base::nullopt;
1313     case device::mojom::XRReferenceSpaceType::kUnbounded:
1314       return base::nullopt;
1315   }
1316 }
1317 
NativeOriginExists(const mojom::XRNativeOriginInformation & native_origin_information,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1318 bool ArCoreImpl::NativeOriginExists(
1319     const mojom::XRNativeOriginInformation& native_origin_information,
1320     const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1321   switch (native_origin_information.which()) {
1322     case mojom::XRNativeOriginInformation::Tag::INPUT_SOURCE_ID:
1323 
1324       // Linear search should be fine for ARCore device as it only has one input
1325       // source (for now).
1326       for (auto& input_source_state : input_state) {
1327         if (input_source_state->source_id ==
1328             native_origin_information.get_input_source_id()) {
1329           return true;
1330         }
1331       }
1332 
1333       return false;
1334     case mojom::XRNativeOriginInformation::Tag::REFERENCE_SPACE_TYPE:
1335       // All reference spaces are known to ARCore.
1336       return true;
1337 
1338     case mojom::XRNativeOriginInformation::Tag::PLANE_ID:
1339       return plane_manager_->PlaneExists(
1340           PlaneId(native_origin_information.get_plane_id()));
1341     case mojom::XRNativeOriginInformation::Tag::ANCHOR_ID:
1342 
1343       return anchor_manager_->AnchorExists(
1344           AnchorId(native_origin_information.get_anchor_id()));
1345   }
1346 }
1347 
GetMojoFromNativeOrigin(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1348 base::Optional<gfx::Transform> ArCoreImpl::GetMojoFromNativeOrigin(
1349     const mojom::XRNativeOriginInformation& native_origin_information,
1350     const gfx::Transform& mojo_from_viewer,
1351     const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1352   switch (native_origin_information.which()) {
1353     case mojom::XRNativeOriginInformation::Tag::INPUT_SOURCE_ID:
1354 
1355       // Linear search should be fine for ARCore device as it only has one input
1356       // source (for now).
1357       for (auto& input_source_state : input_state) {
1358         if (input_source_state->source_id ==
1359             native_origin_information.get_input_source_id()) {
1360           return GetMojoFromInputSource(input_source_state, mojo_from_viewer);
1361         }
1362       }
1363 
1364       return base::nullopt;
1365     case mojom::XRNativeOriginInformation::Tag::REFERENCE_SPACE_TYPE:
1366       return GetMojoFromReferenceSpace(
1367           native_origin_information.get_reference_space_type(),
1368           mojo_from_viewer);
1369     case mojom::XRNativeOriginInformation::Tag::PLANE_ID:
1370       return plane_manager_->GetMojoFromPlane(
1371           PlaneId(native_origin_information.get_plane_id()));
1372     case mojom::XRNativeOriginInformation::Tag::ANCHOR_ID:
1373       return anchor_manager_->GetMojoFromAnchor(
1374           AnchorId(native_origin_information.get_anchor_id()));
1375   }
1376 }
1377 
UnsubscribeFromHitTest(uint64_t subscription_id)1378 void ArCoreImpl::UnsubscribeFromHitTest(uint64_t subscription_id) {
1379   DVLOG(2) << __func__ << ": subscription_id=" << subscription_id;
1380 
1381   // Hit test subscription ID space is the same for transient and non-transient
1382   // hit test sources, so we can attempt to remove it from both collections (it
1383   // will succeed only for one of them anyway).
1384 
1385   hit_test_subscription_id_to_data_.erase(
1386       HitTestSubscriptionId(subscription_id));
1387   hit_test_subscription_id_to_transient_hit_test_data_.erase(
1388       HitTestSubscriptionId(subscription_id));
1389 }
1390 
CreateHitTestSubscriptionId()1391 HitTestSubscriptionId ArCoreImpl::CreateHitTestSubscriptionId() {
1392   CHECK(next_id_ != std::numeric_limits<uint64_t>::max())
1393       << "preventing ID overflow";
1394 
1395   uint64_t current_id = next_id_++;
1396 
1397   return HitTestSubscriptionId(current_id);
1398 }
1399 
RequestHitTest(const mojom::XRRayPtr & ray,std::vector<mojom::XRHitResultPtr> * hit_results)1400 bool ArCoreImpl::RequestHitTest(
1401     const mojom::XRRayPtr& ray,
1402     std::vector<mojom::XRHitResultPtr>* hit_results) {
1403   DCHECK(ray);
1404   return RequestHitTest(ray->origin, ray->direction,
1405                         {mojom::EntityTypeForHitTest::PLANE},
1406                         hit_results);  // "Plane" to maintain current behavior
1407                                        // of async hit test.
1408 }
1409 
RequestHitTest(const gfx::Point3F & origin,const gfx::Vector3dF & direction,const std::vector<mojom::EntityTypeForHitTest> & entity_types,std::vector<mojom::XRHitResultPtr> * hit_results)1410 bool ArCoreImpl::RequestHitTest(
1411     const gfx::Point3F& origin,
1412     const gfx::Vector3dF& direction,
1413     const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1414     std::vector<mojom::XRHitResultPtr>* hit_results) {
1415   DVLOG(2) << __func__ << ": origin=" << origin.ToString()
1416            << ", direction=" << direction.ToString();
1417 
1418   DCHECK(hit_results);
1419   DCHECK(IsOnGlThread());
1420   DCHECK(arcore_session_.is_valid());
1421   DCHECK(arcore_frame_.is_valid());
1422 
1423   auto arcore_entity_types = GetArCoreEntityTypes(entity_types);
1424 
1425   // ArCore returns hit-results in sorted order, thus providing the guarantee
1426   // of sorted results promised by the WebXR spec for requestHitTest().
1427   std::array<float, 3> origin_array = {origin.x(), origin.y(), origin.z()};
1428   std::array<float, 3> direction_array = {direction.x(), direction.y(),
1429                                           direction.z()};
1430 
1431   internal::ScopedArCoreObject<ArHitResultList*> arcore_hit_result_list;
1432   ArHitResultList_create(
1433       arcore_session_.get(),
1434       internal::ScopedArCoreObject<ArHitResultList*>::Receiver(
1435           arcore_hit_result_list)
1436           .get());
1437   if (!arcore_hit_result_list.is_valid()) {
1438     DLOG(ERROR) << "ArHitResultList_create failed!";
1439     return false;
1440   }
1441 
1442   ArFrame_hitTestRay(arcore_session_.get(), arcore_frame_.get(),
1443                      origin_array.data(), direction_array.data(),
1444                      arcore_hit_result_list.get());
1445 
1446   int arcore_hit_result_list_size = 0;
1447   ArHitResultList_getSize(arcore_session_.get(), arcore_hit_result_list.get(),
1448                           &arcore_hit_result_list_size);
1449   DVLOG(2) << __func__
1450            << ": arcore_hit_result_list_size=" << arcore_hit_result_list_size;
1451 
1452   // Go through the list in reverse order so the first hit we encounter is the
1453   // furthest.
1454   // We will accept the furthest hit and then for the rest require that the hit
1455   // be within the actual polygon detected by ArCore. This heuristic allows us
1456   // to get better results on floors w/o overestimating the size of tables etc.
1457   // See https://crbug.com/872855.
1458   for (int i = arcore_hit_result_list_size - 1; i >= 0; i--) {
1459     internal::ScopedArCoreObject<ArHitResult*> arcore_hit;
1460 
1461     ArHitResult_create(
1462         arcore_session_.get(),
1463         internal::ScopedArCoreObject<ArHitResult*>::Receiver(arcore_hit).get());
1464 
1465     if (!arcore_hit.is_valid()) {
1466       DLOG(ERROR) << "ArHitResult_create failed!";
1467       return false;
1468     }
1469 
1470     ArHitResultList_getItem(arcore_session_.get(), arcore_hit_result_list.get(),
1471                             i, arcore_hit.get());
1472 
1473     internal::ScopedArCoreObject<ArTrackable*> ar_trackable;
1474 
1475     ArHitResult_acquireTrackable(
1476         arcore_session_.get(), arcore_hit.get(),
1477         internal::ScopedArCoreObject<ArTrackable*>::Receiver(ar_trackable)
1478             .get());
1479     ArTrackableType ar_trackable_type = AR_TRACKABLE_NOT_VALID;
1480     ArTrackable_getType(arcore_session_.get(), ar_trackable.get(),
1481                         &ar_trackable_type);
1482 
1483     // Only consider trackables listed in arcore_entity_types.
1484     if (!base::Contains(arcore_entity_types, ar_trackable_type)) {
1485       DVLOG(2) << __func__
1486                << ": hit a trackable that is not in entity types set, ignoring "
1487                   "it. ar_trackable_type="
1488                << ar_trackable_type;
1489       continue;
1490     }
1491 
1492     internal::ScopedArCoreObject<ArPose*> arcore_pose;
1493     ArPose_create(
1494         arcore_session_.get(), nullptr,
1495         internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
1496     if (!arcore_pose.is_valid()) {
1497       DLOG(ERROR) << "ArPose_create failed!";
1498       return false;
1499     }
1500 
1501     ArHitResult_getHitPose(arcore_session_.get(), arcore_hit.get(),
1502                            arcore_pose.get());
1503 
1504     // After the first (furthest) hit, for planes, only return hits that are
1505     // within the actual detected polygon and not just within than the larger
1506     // plane.
1507     uint64_t plane_id = 0;
1508     if (ar_trackable_type == AR_TRACKABLE_PLANE) {
1509       ArPlane* ar_plane = ArAsPlane(ar_trackable.get());
1510 
1511       if (!hit_results->empty()) {
1512         int32_t in_polygon = 0;
1513         ArPlane_isPoseInPolygon(arcore_session_.get(), ar_plane,
1514                                 arcore_pose.get(), &in_polygon);
1515         if (!in_polygon) {
1516           DVLOG(2) << __func__
1517                    << ": hit a trackable that is not within detected polygon, "
1518                       "ignoring it";
1519           continue;
1520         }
1521       }
1522 
1523       base::Optional<PlaneId> maybe_plane_id =
1524           plane_manager_->GetPlaneId(ar_plane);
1525       if (maybe_plane_id) {
1526         plane_id = maybe_plane_id->GetUnsafeValue();
1527       }
1528     }
1529 
1530     mojom::XRHitResultPtr mojo_hit = mojom::XRHitResult::New();
1531 
1532     mojo_hit->plane_id = plane_id;
1533 
1534     {
1535       std::array<float, 7> raw_pose;
1536       ArPose_getPoseRaw(arcore_session_.get(), arcore_pose.get(),
1537                         raw_pose.data());
1538 
1539       gfx::Quaternion orientation(raw_pose[0], raw_pose[1], raw_pose[2],
1540                                   raw_pose[3]);
1541       gfx::Point3F position(raw_pose[4], raw_pose[5], raw_pose[6]);
1542 
1543       mojo_hit->mojo_from_result = device::Pose(position, orientation);
1544 
1545       DVLOG(3) << __func__
1546                << ": adding hit test result, position=" << position.ToString()
1547                << ", orientation=" << orientation.ToString()
1548                << ", plane_id=" << plane_id << " (0 means no plane)";
1549     }
1550 
1551     // Insert new results at head to preserver order from ArCore
1552     hit_results->insert(hit_results->begin(), std::move(mojo_hit));
1553   }
1554 
1555   DVLOG(2) << __func__ << ": hit_results->size()=" << hit_results->size();
1556   return true;
1557 }
1558 
CreateAnchor(const mojom::XRNativeOriginInformation & native_origin_information,const device::Pose & native_origin_from_anchor,CreateAnchorCallback callback)1559 void ArCoreImpl::CreateAnchor(
1560     const mojom::XRNativeOriginInformation& native_origin_information,
1561     const device::Pose& native_origin_from_anchor,
1562     CreateAnchorCallback callback) {
1563   DVLOG(2) << __func__ << ": native_origin_information.which()="
1564            << static_cast<uint32_t>(native_origin_information.which())
1565            << ", native_origin_from_anchor.position()="
1566            << native_origin_from_anchor.position().ToString()
1567            << ", native_origin_from_anchor.orientation()="
1568            << native_origin_from_anchor.orientation().ToString();
1569 
1570   create_anchor_requests_.emplace_back(native_origin_information,
1571                                        native_origin_from_anchor.ToTransform(),
1572                                        std::move(callback));
1573 }
1574 
CreatePlaneAttachedAnchor(const mojom::XRNativeOriginInformation & native_origin_information,const device::Pose & native_origin_from_anchor,uint64_t plane_id,CreateAnchorCallback callback)1575 void ArCoreImpl::CreatePlaneAttachedAnchor(
1576     const mojom::XRNativeOriginInformation& native_origin_information,
1577     const device::Pose& native_origin_from_anchor,
1578     uint64_t plane_id,
1579     CreateAnchorCallback callback) {
1580   DVLOG(2) << __func__ << ": native_origin_information.which()="
1581            << static_cast<uint32_t>(native_origin_information.which())
1582            << ", plane_id=" << plane_id
1583            << ", native_origin_from_anchor.position()="
1584            << native_origin_from_anchor.position().ToString()
1585            << ", native_origin_from_anchor.orientation()="
1586            << native_origin_from_anchor.orientation().ToString();
1587 
1588   create_plane_attached_anchor_requests_.emplace_back(
1589       native_origin_information, native_origin_from_anchor.ToTransform(),
1590       plane_id, std::move(callback));
1591 }
1592 
ProcessAnchorCreationRequests(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state,const base::TimeTicks & frame_time)1593 void ArCoreImpl::ProcessAnchorCreationRequests(
1594     const gfx::Transform& mojo_from_viewer,
1595     const std::vector<mojom::XRInputSourceStatePtr>& input_state,
1596     const base::TimeTicks& frame_time) {
1597   DVLOG(2) << __func__ << ": Processing free-floating anchor creation requests";
1598   ProcessAnchorCreationRequestsHelper(
1599       mojo_from_viewer, input_state, &create_anchor_requests_, frame_time,
1600       [this](const CreateAnchorRequest& create_anchor_request,
1601              const gfx::Point3F& position, const gfx::Quaternion& orientation) {
1602         return anchor_manager_->CreateAnchor(
1603             device::mojom::Pose(orientation, position));
1604       });
1605 
1606   DVLOG(2) << __func__
1607            << ": Processing plane-attached anchor creation requests";
1608   ProcessAnchorCreationRequestsHelper(
1609       mojo_from_viewer, input_state, &create_plane_attached_anchor_requests_,
1610       frame_time,
1611       [this](const CreatePlaneAttachedAnchorRequest& create_anchor_request,
1612              const gfx::Point3F& position, const gfx::Quaternion& orientation) {
1613         PlaneId plane_id = PlaneId(create_anchor_request.GetPlaneId());
1614         return anchor_manager_->CreateAnchor(
1615             plane_manager_.get(), device::mojom::Pose(orientation, position),
1616             plane_id);
1617       });
1618 }
1619 
1620 template <typename T, typename FunctionType>
ProcessAnchorCreationRequestsHelper(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state,std::vector<T> * anchor_creation_requests,const base::TimeTicks & frame_time,FunctionType && create_anchor_function)1621 void ArCoreImpl::ProcessAnchorCreationRequestsHelper(
1622     const gfx::Transform& mojo_from_viewer,
1623     const std::vector<mojom::XRInputSourceStatePtr>& input_state,
1624     std::vector<T>* anchor_creation_requests,
1625     const base::TimeTicks& frame_time,
1626     FunctionType&& create_anchor_function) {
1627   DCHECK(anchor_creation_requests);
1628 
1629   DVLOG(3) << __func__ << ": pre-call anchor_creation_requests->size()="
1630            << anchor_creation_requests->size();
1631 
1632   // If we are unable to create an anchor because position of the native origin
1633   // is unknown, keep deferring it. On the other hand, if the anchor creation
1634   // failed in ARCore SDK, notify blink - we are ensuring that anchor creation
1635   // requests are processed when ARCore is in correct state so any failures
1636   // coming from ARCore SDK are real failures we won't be able to recover from.
1637   std::vector<T> postponed_requests;
1638   for (auto& create_anchor : *anchor_creation_requests) {
1639     auto anchor_creation_age = frame_time - create_anchor.GetRequestStartTime();
1640 
1641     if (anchor_creation_age > kOutdatedAnchorCreationRequestThreshold) {
1642       DVLOG(3)
1643           << __func__
1644           << ": failing outdated anchor creation request, anchor_creation_age="
1645           << anchor_creation_age;
1646       create_anchor.TakeCallback().Run(
1647           device::mojom::CreateAnchorResult::FAILURE, 0);
1648       continue;
1649     }
1650 
1651     mojom::XRNativeOriginInformation native_origin_information =
1652         create_anchor.GetNativeOriginInformation();
1653 
1654     if (!NativeOriginExists(native_origin_information, input_state)) {
1655       DVLOG(3) << __func__
1656                << ": failing anchor creation request, native origin does not "
1657                   "exist";
1658       create_anchor.TakeCallback().Run(
1659           device::mojom::CreateAnchorResult::FAILURE, 0);
1660       continue;
1661     }
1662 
1663     base::Optional<gfx::Transform> maybe_mojo_from_native_origin =
1664         GetMojoFromNativeOrigin(native_origin_information, mojo_from_viewer,
1665                                 input_state);
1666 
1667     if (!maybe_mojo_from_native_origin) {
1668       // We don't know where the native origin currently is (but we know it is
1669       // still tracked), so let's postpone the request & try again later.
1670       DVLOG(3) << __func__
1671                << ": postponing anchor creation request, native origin is not "
1672                   "currently localizable";
1673       postponed_requests.emplace_back(std::move(create_anchor));
1674       continue;
1675     }
1676 
1677     base::Optional<device::Pose> mojo_from_anchor =
1678         device::Pose::Create(*maybe_mojo_from_native_origin *
1679                              create_anchor.GetNativeOriginFromAnchor());
1680 
1681     if (!mojo_from_anchor) {
1682       // Fail the call now, failure to decompose is unlikely to resolve itself.
1683       DVLOG(3)
1684           << __func__
1685           << ": failing anchor creation request, unable to decompose a matrix";
1686       create_anchor.TakeCallback().Run(
1687           device::mojom::CreateAnchorResult::FAILURE, 0);
1688       continue;
1689     }
1690 
1691     base::Optional<AnchorId> maybe_anchor_id = std::forward<FunctionType>(
1692         create_anchor_function)(create_anchor, mojo_from_anchor->position(),
1693                                 mojo_from_anchor->orientation());
1694 
1695     if (!maybe_anchor_id) {
1696       // Fail the call now, failure to create anchor in ARCore SDK is unlikely
1697       // to resolve itself.
1698       DVLOG(3) << __func__
1699                << ": failing anchor creation request, anchor creation "
1700                   "function did not return an anchor id";
1701       create_anchor.TakeCallback().Run(
1702           device::mojom::CreateAnchorResult::FAILURE, 0);
1703       continue;
1704     }
1705 
1706     DVLOG(3) << __func__ << ": anchor creation request succeeded, time taken: "
1707              << anchor_creation_age;
1708     create_anchor.TakeCallback().Run(device::mojom::CreateAnchorResult::SUCCESS,
1709                                      maybe_anchor_id->GetUnsafeValue());
1710   }
1711 
1712   // Return the postponed requests - all other requests should have their
1713   // status already reported to blink at this point:
1714   anchor_creation_requests->swap(postponed_requests);
1715   DVLOG(3) << __func__ << ": post-call anchor_creation_requests->size()="
1716            << anchor_creation_requests->size();
1717 }
1718 
DetachAnchor(uint64_t anchor_id)1719 void ArCoreImpl::DetachAnchor(uint64_t anchor_id) {
1720   anchor_manager_->DetachAnchor(AnchorId(anchor_id));
1721 }
1722 
GetDepthData()1723 mojom::XRDepthDataPtr ArCoreImpl::GetDepthData() {
1724   DVLOG(3) << __func__;
1725 
1726   internal::ScopedArCoreObject<ArImage*> ar_image;
1727   ArStatus status = ArFrame_acquireDepthImage(
1728       arcore_session_.get(), arcore_frame_.get(),
1729       internal::ScopedArCoreObject<ArImage*>::Receiver(ar_image).get());
1730 
1731   if (status != AR_SUCCESS) {
1732     DVLOG(2) << __func__
1733              << ": ArFrame_acquireDepthImage failed, status=" << status;
1734     return nullptr;
1735   }
1736 
1737   int64_t timestamp_ns;
1738   ArImage_getTimestamp(arcore_session_.get(), ar_image.get(), &timestamp_ns);
1739   base::TimeDelta time_delta = base::TimeDelta::FromNanoseconds(timestamp_ns);
1740   DVLOG(3) << __func__ << ": depth image time_delta=" << time_delta;
1741 
1742   // The image returned from ArFrame_acquireDepthImage() is documented to have
1743   // a single 16-bit plane at index 0. The ArImage format is documented to be
1744   // AR_IMAGE_FORMAT_DEPTH16 (equivalent to ImageFormat.DEPTH16). There should
1745   // be no need to validate this in non-debug builds.
1746   // https://developers.google.com/ar/reference/c/group/ar-frame#arframe_acquiredepthimage
1747   // https://developer.android.com/reference/android/graphics/ImageFormat#DEPTH16
1748 
1749   ArImageFormat image_format;
1750   ArImage_getFormat(arcore_session_.get(), ar_image.get(), &image_format);
1751 
1752   CHECK_EQ(image_format, AR_IMAGE_FORMAT_DEPTH16)
1753       << "Depth image format must be AR_IMAGE_FORMAT_DEPTH16, found: "
1754       << image_format;
1755 
1756   int32_t num_planes;
1757   ArImage_getNumberOfPlanes(arcore_session_.get(), ar_image.get(), &num_planes);
1758 
1759   CHECK_EQ(num_planes, 1) << "Depth image must have 1 plane, found: "
1760                           << num_planes;
1761 
1762   if (time_delta > previous_depth_data_time_) {
1763     mojom::XRDepthDataUpdatedPtr result = mojom::XRDepthDataUpdated::New();
1764 
1765     result->time_delta = time_delta;
1766 
1767     int32_t width = 0, height = 0;
1768     ArImage_getWidth(arcore_session_.get(), ar_image.get(), &width);
1769     ArImage_getHeight(arcore_session_.get(), ar_image.get(), &height);
1770 
1771     DVLOG(3) << __func__ << ": depth image dimensions=" << width << "x"
1772              << height;
1773 
1774     // Depth image is defined as a width by height array of 2-byte elements:
1775     auto checked_buffer_size = base::CheckMul<size_t>(2, width, height);
1776 
1777     size_t buffer_size;
1778     if (!checked_buffer_size.AssignIfValid(&buffer_size)) {
1779       DVLOG(2) << __func__
1780                << ": overflow in 2 * width * height expression, returning null "
1781                   "depth data";
1782       return nullptr;
1783     }
1784 
1785     mojo_base::BigBuffer pixels(buffer_size);
1786 
1787     // Interpret BigBuffer's data as a width by height array of uint16_t's and
1788     // copy image data into it:
1789     CopyArCoreImage(
1790         arcore_session_.get(), ar_image.get(), 0,
1791         base::span<uint16_t>(reinterpret_cast<uint16_t*>(pixels.data()),
1792                              pixels.size() / 2),
1793         width, height);
1794 
1795     result->pixel_data = std::move(pixels);
1796     // Transform needed to consume the data:
1797     result->norm_texture_from_norm_view = GetCameraUvFromScreenUvTransform();
1798     result->size = gfx::Size(width, height);
1799 
1800     DVLOG(3) << __func__ << ": norm_texture_from_norm_view=\n"
1801              << result->norm_texture_from_norm_view.ToString();
1802 
1803     previous_depth_data_time_ = time_delta;
1804 
1805     return mojom::XRDepthData::NewUpdatedDepthData(std::move(result));
1806   }
1807 
1808   return mojom::XRDepthData::NewDataStillValid(
1809       mojom::XRDepthDataStillValid::New());
1810 }
1811 
IsOnGlThread() const1812 bool ArCoreImpl::IsOnGlThread() const {
1813   return gl_thread_task_runner_->BelongsToCurrentThread();
1814 }
1815 
Create()1816 std::unique_ptr<ArCore> ArCoreImplFactory::Create() {
1817   return std::make_unique<ArCoreImpl>();
1818 }
1819 
CreateAnchorRequest(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & native_origin_from_anchor,ArCore::CreateAnchorCallback callback)1820 CreateAnchorRequest::CreateAnchorRequest(
1821     const mojom::XRNativeOriginInformation& native_origin_information,
1822     const gfx::Transform& native_origin_from_anchor,
1823     ArCore::CreateAnchorCallback callback)
1824     : native_origin_information_(native_origin_information),
1825       native_origin_from_anchor_(native_origin_from_anchor),
1826       request_start_time_(base::TimeTicks::Now()),
1827       callback_(std::move(callback)) {}
1828 CreateAnchorRequest::CreateAnchorRequest(CreateAnchorRequest&& other) = default;
1829 CreateAnchorRequest::~CreateAnchorRequest() = default;
1830 
1831 mojom::XRNativeOriginInformation
GetNativeOriginInformation() const1832 CreateAnchorRequest::GetNativeOriginInformation() const {
1833   return native_origin_information_;
1834 }
1835 
GetNativeOriginFromAnchor() const1836 gfx::Transform CreateAnchorRequest::GetNativeOriginFromAnchor() const {
1837   return native_origin_from_anchor_;
1838 }
1839 
GetRequestStartTime() const1840 base::TimeTicks CreateAnchorRequest::GetRequestStartTime() const {
1841   return request_start_time_;
1842 }
1843 
TakeCallback()1844 ArCore::CreateAnchorCallback CreateAnchorRequest::TakeCallback() {
1845   return std::move(callback_);
1846 }
1847 
CreatePlaneAttachedAnchorRequest(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & native_origin_from_anchor,uint64_t plane_id,ArCore::CreateAnchorCallback callback)1848 CreatePlaneAttachedAnchorRequest::CreatePlaneAttachedAnchorRequest(
1849     const mojom::XRNativeOriginInformation& native_origin_information,
1850     const gfx::Transform& native_origin_from_anchor,
1851     uint64_t plane_id,
1852     ArCore::CreateAnchorCallback callback)
1853     : native_origin_information_(native_origin_information),
1854       native_origin_from_anchor_(native_origin_from_anchor),
1855       plane_id_(plane_id),
1856       request_start_time_(base::TimeTicks::Now()),
1857       callback_(std::move(callback)) {}
1858 CreatePlaneAttachedAnchorRequest::CreatePlaneAttachedAnchorRequest(
1859     CreatePlaneAttachedAnchorRequest&& other) = default;
1860 CreatePlaneAttachedAnchorRequest::~CreatePlaneAttachedAnchorRequest() = default;
1861 
1862 mojom::XRNativeOriginInformation
GetNativeOriginInformation() const1863 CreatePlaneAttachedAnchorRequest::GetNativeOriginInformation() const {
1864   return native_origin_information_;
1865 }
1866 
GetPlaneId() const1867 uint64_t CreatePlaneAttachedAnchorRequest::GetPlaneId() const {
1868   return plane_id_;
1869 }
1870 
GetNativeOriginFromAnchor() const1871 gfx::Transform CreatePlaneAttachedAnchorRequest::GetNativeOriginFromAnchor()
1872     const {
1873   return native_origin_from_anchor_;
1874 }
1875 
GetRequestStartTime() const1876 base::TimeTicks CreatePlaneAttachedAnchorRequest::GetRequestStartTime() const {
1877   return request_start_time_;
1878 }
1879 
TakeCallback()1880 ArCore::CreateAnchorCallback CreatePlaneAttachedAnchorRequest::TakeCallback() {
1881   return std::move(callback_);
1882 }
1883 
1884 }  // namespace device
1885