1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "device/vr/android/arcore/arcore_impl.h"
6
7 #include "base/android/jni_android.h"
8 #include "base/bind.h"
9 #include "base/containers/span.h"
10 #include "base/numerics/checked_math.h"
11 #include "base/numerics/math_constants.h"
12 #include "base/optional.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "base/trace_event/trace_event.h"
15 #include "base/util/type_safety/pass_key.h"
16 #include "device/vr/android/arcore/arcore_math_utils.h"
17 #include "device/vr/android/arcore/arcore_plane_manager.h"
18 #include "device/vr/android/arcore/type_converters.h"
19 #include "device/vr/public/mojom/pose.h"
20 #include "device/vr/public/mojom/vr_service.mojom.h"
21 #include "third_party/skia/include/core/SkBitmap.h"
22 #include "third_party/skia/include/core/SkCanvas.h"
23 #include "third_party/skia/include/core/SkImage.h"
24 #include "third_party/skia/include/core/SkMatrix44.h"
25 #include "third_party/skia/include/core/SkPaint.h"
26 #include "third_party/skia/include/core/SkPixmap.h"
27 #include "ui/display/display.h"
28 #include "ui/gfx/geometry/point3_f.h"
29 #include "ui/gfx/geometry/point_f.h"
30 #include "ui/gfx/transform.h"
31 #include "ui/gfx/transform_util.h"
32
33 using base::android::JavaRef;
34
35 namespace {
36
37 // Anchor creation requests that are older than 3 seconds are considered
38 // outdated and should be failed.
39 constexpr base::TimeDelta kOutdatedAnchorCreationRequestThreshold =
40 base::TimeDelta::FromSeconds(3);
41
42 // Helper, returns new VRPosePtr with position and orientation set to match the
43 // position and orientation of passed in |pose|.
GetMojomVRPoseFromArPose(const ArSession * session,const ArPose * pose)44 device::mojom::VRPosePtr GetMojomVRPoseFromArPose(const ArSession* session,
45 const ArPose* pose) {
46 device::mojom::VRPosePtr result = device::mojom::VRPose::New();
47 std::tie(result->orientation, result->position) =
48 device::GetPositionAndOrientationFromArPose(session, pose);
49
50 return result;
51 }
52
53 // Helper, creates new ArPose* with position and orientation set to match the
54 // position and orientation of passed in |pose|.
55
GetArCoreEntityType(device::mojom::EntityTypeForHitTest entity_type)56 ArTrackableType GetArCoreEntityType(
57 device::mojom::EntityTypeForHitTest entity_type) {
58 switch (entity_type) {
59 case device::mojom::EntityTypeForHitTest::PLANE:
60 return AR_TRACKABLE_PLANE;
61 case device::mojom::EntityTypeForHitTest::POINT:
62 return AR_TRACKABLE_POINT;
63 }
64 }
65
GetArCoreEntityTypes(const std::vector<device::mojom::EntityTypeForHitTest> & entity_types)66 std::set<ArTrackableType> GetArCoreEntityTypes(
67 const std::vector<device::mojom::EntityTypeForHitTest>& entity_types) {
68 std::set<ArTrackableType> result;
69
70 std::transform(entity_types.begin(), entity_types.end(),
71 std::inserter(result, result.end()), GetArCoreEntityType);
72
73 return result;
74 }
75
76 // Helper, computes mojo_from_input_source transform based on mojo_from_viever
77 // pose and input source state (containing input_from_pointer transform, which
78 // in case of input sources is equivalent to viewer_from_pointer).
79 // TODO(https://crbug.com/1043389): this currently assumes that the input source
80 // ray mode is "tapping", which is OK for input sources available for AR on
81 // Android, but is not true in the general case. This method should duplicate
82 // the logic found in XRTargetRaySpace::MojoFromNative().
GetMojoFromInputSource(const device::mojom::XRInputSourceStatePtr & input_source_state,const gfx::Transform & mojo_from_viewer)83 base::Optional<gfx::Transform> GetMojoFromInputSource(
84 const device::mojom::XRInputSourceStatePtr& input_source_state,
85 const gfx::Transform& mojo_from_viewer) {
86 if (!input_source_state->description ||
87 !input_source_state->description->input_from_pointer) {
88 return base::nullopt;
89 }
90
91 gfx::Transform viewer_from_pointer =
92 *input_source_state->description->input_from_pointer;
93
94 return mojo_from_viewer * viewer_from_pointer;
95 }
96
ReleaseArCoreCubemap(ArImageCubemap * cube_map)97 void ReleaseArCoreCubemap(ArImageCubemap* cube_map) {
98 for (auto* image : *cube_map) {
99 ArImage_release(image);
100 }
101
102 memset(cube_map, 0, sizeof(*cube_map));
103 }
104
105 // Helper, copies ARCore image to the passed in buffer, assuming that the caller
106 // allocated the buffer to fit all the data.
107 template <typename T>
CopyArCoreImage(const ArSession * session,const ArImage * image,int32_t plane_index,base::span<T> out_pixels,uint32_t width,uint32_t height)108 void CopyArCoreImage(const ArSession* session,
109 const ArImage* image,
110 int32_t plane_index,
111 base::span<T> out_pixels,
112 uint32_t width,
113 uint32_t height) {
114 DVLOG(3) << __func__ << ": width=" << width << ", height=" << height
115 << ", out_pixels.size()=" << out_pixels.size();
116
117 DCHECK_GE(out_pixels.size(), width * height);
118
119 int32_t src_row_stride = 0, src_pixel_stride = 0;
120 ArImage_getPlaneRowStride(session, image, plane_index, &src_row_stride);
121 ArImage_getPlanePixelStride(session, image, plane_index, &src_pixel_stride);
122
123 // Naked pointer since ArImage_getPlaneData does not transfer ownership to us.
124 uint8_t const* src_buffer = nullptr;
125 int32_t src_buffer_length = 0;
126 ArImage_getPlaneData(session, image, plane_index, &src_buffer,
127 &src_buffer_length);
128
129 // Fast path: Source and destination have the same layout
130 bool const fast_path =
131 static_cast<size_t>(src_row_stride) == width * sizeof(T);
132 TRACE_EVENT1("xr", "CopyArCoreImage: memcpy", "fastPath", fast_path);
133
134 DVLOG(3) << __func__ << ": plane_index=" << plane_index
135 << ", src_buffer_length=" << src_buffer_length
136 << ", src_row_stride=" << src_row_stride
137 << ", src_pixel_stride=" << src_pixel_stride
138 << ", fast_path=" << fast_path << ", sizeof(T)=" << sizeof(T);
139
140 // If they have the same layout, we can copy the entire buffer at once
141 if (fast_path) {
142 CHECK_EQ(out_pixels.size() * sizeof(T),
143 static_cast<size_t>(src_buffer_length));
144 memcpy(out_pixels.data(), src_buffer, src_buffer_length);
145 return;
146 }
147
148 CHECK_EQ(sizeof(T), static_cast<size_t>(src_pixel_stride));
149
150 // Slow path: copy pixel by pixel, row by row
151 for (uint32_t row = 0; row < height; ++row) {
152 auto* src = src_buffer + src_row_stride * row;
153 auto* dest = out_pixels.data() + width * row;
154
155 // For each pixel
156 for (uint32_t x = 0; x < width; ++x) {
157 memcpy(dest, src, sizeof(T));
158
159 src += src_pixel_stride;
160 dest += 1;
161 }
162 }
163 }
164
165 // Helper, copies ARCore image to the passed in vector, discovering the buffer
166 // size and resizing the vector first.
167 template <typename T>
CopyArCoreImage(const ArSession * session,const ArImage * image,int32_t plane_index,std::vector<T> * out_pixels,uint32_t * out_width,uint32_t * out_height)168 void CopyArCoreImage(const ArSession* session,
169 const ArImage* image,
170 int32_t plane_index,
171 std::vector<T>* out_pixels,
172 uint32_t* out_width,
173 uint32_t* out_height) {
174 // Get source image information
175 int32_t width = 0, height = 0;
176 ArImage_getWidth(session, image, &width);
177 ArImage_getHeight(session, image, &height);
178
179 *out_width = width;
180 *out_height = height;
181
182 // Allocate memory for the output.
183 out_pixels->resize(width * height);
184
185 CopyArCoreImage(session, image, plane_index, base::span<T>(*out_pixels),
186 width, height);
187 }
188
GetLightProbe(ArSession * arcore_session,ArLightEstimate * arcore_light_estimate)189 device::mojom::XRLightProbePtr GetLightProbe(
190 ArSession* arcore_session,
191 ArLightEstimate* arcore_light_estimate) {
192 // ArCore hands out 9 sets of RGB spherical harmonics coefficients
193 // https://developers.google.com/ar/reference/c/group/light#arlightestimate_getenvironmentalhdrambientsphericalharmonics
194 constexpr size_t kNumShCoefficients = 9;
195
196 auto light_probe = device::mojom::XRLightProbe::New();
197
198 light_probe->spherical_harmonics = device::mojom::XRSphericalHarmonics::New();
199 light_probe->spherical_harmonics->coefficients =
200 std::vector<device::RgbTupleF32>(kNumShCoefficients,
201 device::RgbTupleF32{});
202
203 ArLightEstimate_getEnvironmentalHdrAmbientSphericalHarmonics(
204 arcore_session, arcore_light_estimate,
205 light_probe->spherical_harmonics->coefficients.data()->components);
206
207 float main_light_direction[3] = {0};
208 ArLightEstimate_getEnvironmentalHdrMainLightDirection(
209 arcore_session, arcore_light_estimate, main_light_direction);
210 light_probe->main_light_direction.set_x(main_light_direction[0]);
211 light_probe->main_light_direction.set_y(main_light_direction[1]);
212 light_probe->main_light_direction.set_z(main_light_direction[2]);
213
214 ArLightEstimate_getEnvironmentalHdrMainLightIntensity(
215 arcore_session, arcore_light_estimate,
216 light_probe->main_light_intensity.components);
217
218 return light_probe;
219 }
220
GetReflectionProbe(ArSession * arcore_session,ArLightEstimate * arcore_light_estimate)221 device::mojom::XRReflectionProbePtr GetReflectionProbe(
222 ArSession* arcore_session,
223 ArLightEstimate* arcore_light_estimate) {
224 ArImageCubemap arcore_cube_map = {nullptr};
225 ArLightEstimate_acquireEnvironmentalHdrCubemap(
226 arcore_session, arcore_light_estimate, arcore_cube_map);
227
228 auto cube_map = device::mojom::XRCubeMap::New();
229 std::vector<device::RgbaTupleF16>* const cube_map_faces[] = {
230 &cube_map->positive_x, &cube_map->negative_x, &cube_map->positive_y,
231 &cube_map->negative_y, &cube_map->positive_z, &cube_map->negative_z};
232
233 static_assert(
234 base::size(cube_map_faces) == base::size(arcore_cube_map),
235 "`ArImageCubemap` and `device::mojom::XRCubeMap` are expected to "
236 "have the same number of faces (6).");
237
238 static_assert(device::mojom::XRCubeMap::kNumComponentsPerPixel == 4,
239 "`device::mojom::XRCubeMap::kNumComponentsPerPixel` is "
240 "expected to be 4 (RGBA)`, as that's the format ArCore uses.");
241
242 for (size_t i = 0; i < base::size(arcore_cube_map); ++i) {
243 auto* arcore_cube_map_face = arcore_cube_map[i];
244 if (!arcore_cube_map_face) {
245 DVLOG(1) << "`ArLightEstimate_acquireEnvironmentalHdrCubemap` failed to "
246 "return all faces";
247 ReleaseArCoreCubemap(&arcore_cube_map);
248 return nullptr;
249 }
250
251 auto* cube_map_face = cube_map_faces[i];
252
253 // Make sure we only have a single image plane
254 int32_t num_planes = 0;
255 ArImage_getNumberOfPlanes(arcore_session, arcore_cube_map_face,
256 &num_planes);
257 if (num_planes != 1) {
258 DVLOG(1) << "ArCore cube map face " << i
259 << " does not have exactly 1 plane.";
260 ReleaseArCoreCubemap(&arcore_cube_map);
261 return nullptr;
262 }
263
264 // Make sure the format for the image is in RGBA16F
265 ArImageFormat format = AR_IMAGE_FORMAT_INVALID;
266 ArImage_getFormat(arcore_session, arcore_cube_map_face, &format);
267 if (format != AR_IMAGE_FORMAT_RGBA_FP16) {
268 DVLOG(1) << "ArCore cube map face " << i
269 << " not in expected image format.";
270 ReleaseArCoreCubemap(&arcore_cube_map);
271 return nullptr;
272 }
273
274 // Copy the cubemap
275 uint32_t face_width = 0, face_height = 0;
276 CopyArCoreImage(arcore_session, arcore_cube_map_face, 0, cube_map_face,
277 &face_width, &face_height);
278
279 // Make sure the cube map is square
280 if (face_width != face_height) {
281 DVLOG(1) << "ArCore cube map contains non-square image.";
282 ReleaseArCoreCubemap(&arcore_cube_map);
283 return nullptr;
284 }
285
286 // Make sure all faces have the same dimensions
287 if (i == 0) {
288 cube_map->width_and_height = face_width;
289 } else if (face_width != cube_map->width_and_height ||
290 face_height != cube_map->width_and_height) {
291 DVLOG(1) << "ArCore cube map faces not all of the same dimensions.";
292 ReleaseArCoreCubemap(&arcore_cube_map);
293 return nullptr;
294 }
295 }
296
297 ReleaseArCoreCubemap(&arcore_cube_map);
298
299 auto reflection_probe = device::mojom::XRReflectionProbe::New();
300 reflection_probe->cube_map = std::move(cube_map);
301 return reflection_probe;
302 }
303
304 constexpr float kDefaultFloorHeightEstimation = 1.2;
305
306 } // namespace
307
308 namespace device {
309
HitTestSubscriptionData(mojom::XRNativeOriginInformationPtr native_origin_information,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)310 HitTestSubscriptionData::HitTestSubscriptionData(
311 mojom::XRNativeOriginInformationPtr native_origin_information,
312 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
313 mojom::XRRayPtr ray)
314 : native_origin_information(std::move(native_origin_information)),
315 entity_types(entity_types),
316 ray(std::move(ray)) {}
317
318 HitTestSubscriptionData::HitTestSubscriptionData(
319 HitTestSubscriptionData&& other) = default;
320 HitTestSubscriptionData::~HitTestSubscriptionData() = default;
321
TransientInputHitTestSubscriptionData(const std::string & profile_name,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)322 TransientInputHitTestSubscriptionData::TransientInputHitTestSubscriptionData(
323 const std::string& profile_name,
324 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
325 mojom::XRRayPtr ray)
326 : profile_name(profile_name),
327 entity_types(entity_types),
328 ray(std::move(ray)) {}
329
330 TransientInputHitTestSubscriptionData::TransientInputHitTestSubscriptionData(
331 TransientInputHitTestSubscriptionData&& other) = default;
332 TransientInputHitTestSubscriptionData::
333 ~TransientInputHitTestSubscriptionData() = default;
334
ArCoreImpl()335 ArCoreImpl::ArCoreImpl()
336 : gl_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()) {}
337
~ArCoreImpl()338 ArCoreImpl::~ArCoreImpl() {
339 for (auto& create_anchor : create_anchor_requests_) {
340 create_anchor.TakeCallback().Run(mojom::CreateAnchorResult::FAILURE, 0);
341 }
342
343 for (auto& create_anchor : create_plane_attached_anchor_requests_) {
344 create_anchor.TakeCallback().Run(mojom::CreateAnchorResult::FAILURE, 0);
345 }
346 }
347
Initialize(base::android::ScopedJavaLocalRef<jobject> context,const std::unordered_set<device::mojom::XRSessionFeature> & required_features,const std::unordered_set<device::mojom::XRSessionFeature> & optional_features,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)348 base::Optional<ArCore::InitializeResult> ArCoreImpl::Initialize(
349 base::android::ScopedJavaLocalRef<jobject> context,
350 const std::unordered_set<device::mojom::XRSessionFeature>&
351 required_features,
352 const std::unordered_set<device::mojom::XRSessionFeature>&
353 optional_features,
354 const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
355 DCHECK(IsOnGlThread());
356 DCHECK(!arcore_session_.is_valid());
357
358 // TODO(https://crbug.com/837944): Notify error earlier if this will fail.
359
360 JNIEnv* env = base::android::AttachCurrentThread();
361 if (!env) {
362 DLOG(ERROR) << "Unable to get JNIEnv for ArCore";
363 return base::nullopt;
364 }
365
366 // Use a local scoped ArSession for the next steps, we want the
367 // arcore_session_ member to remain null until we complete successful
368 // initialization.
369 internal::ScopedArCoreObject<ArSession*> session;
370
371 ArStatus status = ArSession_create(
372 env, context.obj(),
373 internal::ScopedArCoreObject<ArSession*>::Receiver(session).get());
374 if (status != AR_SUCCESS) {
375 DLOG(ERROR) << "ArSession_create failed: " << status;
376 return base::nullopt;
377 }
378
379 // Set incognito mode for ARCore session - this is done unconditionally as we
380 // always want to limit the amount of logging done by ARCore.
381 ArSession_enableIncognitoMode_private(session.get());
382 DVLOG(1) << __func__ << ": ARCore incognito mode enabled";
383
384 base::Optional<std::unordered_set<device::mojom::XRSessionFeature>>
385 maybe_enabled_features = ConfigureFeatures(
386 session.get(), required_features, optional_features, tracked_images);
387
388 if (!maybe_enabled_features) {
389 DLOG(ERROR) << "Failed to configure session features";
390 return base::nullopt;
391 }
392
393 if (!ConfigureCamera(session.get())) {
394 DLOG(ERROR) << "Failed to configure session camera";
395 return base::nullopt;
396 }
397
398 internal::ScopedArCoreObject<ArFrame*> frame;
399 ArFrame_create(session.get(),
400 internal::ScopedArCoreObject<ArFrame*>::Receiver(frame).get());
401 if (!frame.is_valid()) {
402 DLOG(ERROR) << "ArFrame_create failed";
403 return base::nullopt;
404 }
405
406 internal::ScopedArCoreObject<ArLightEstimate*> light_estimate;
407 ArLightEstimate_create(
408 session.get(),
409 internal::ScopedArCoreObject<ArLightEstimate*>::Receiver(light_estimate)
410 .get());
411 if (!light_estimate.is_valid()) {
412 DVLOG(1) << "ArLightEstimate_create failed";
413 return base::nullopt;
414 }
415
416 // Success, we now have a valid session and a valid frame.
417 arcore_frame_ = std::move(frame);
418 arcore_session_ = std::move(session);
419 arcore_light_estimate_ = std::move(light_estimate);
420 anchor_manager_ = std::make_unique<ArCoreAnchorManager>(
421 util::PassKey<ArCoreImpl>(), arcore_session_.get());
422 plane_manager_ = std::make_unique<ArCorePlaneManager>(
423 util::PassKey<ArCoreImpl>(), arcore_session_.get());
424 return ArCore::InitializeResult(*maybe_enabled_features);
425 }
426
427 base::Optional<std::unordered_set<device::mojom::XRSessionFeature>>
ConfigureFeatures(ArSession * ar_session,const std::unordered_set<device::mojom::XRSessionFeature> & required_features,const std::unordered_set<device::mojom::XRSessionFeature> & optional_features,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)428 ArCoreImpl::ConfigureFeatures(
429 ArSession* ar_session,
430 const std::unordered_set<device::mojom::XRSessionFeature>&
431 required_features,
432 const std::unordered_set<device::mojom::XRSessionFeature>&
433 optional_features,
434 const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
435 // Let's assume we will be able to configure a session with all features -
436 // this will be adjusted if it turns out we can only create a session w/o some
437 // optional features. Currently, only depth sensing is not supported across
438 // all the ARCore-capable devices.
439 std::unordered_set<device::mojom::XRSessionFeature> enabled_features;
440 enabled_features.insert(required_features.begin(), required_features.end());
441 enabled_features.insert(optional_features.begin(), optional_features.end());
442
443 internal::ScopedArCoreObject<ArConfig*> arcore_config;
444 ArConfig_create(
445 ar_session,
446 internal::ScopedArCoreObject<ArConfig*>::Receiver(arcore_config).get());
447 if (!arcore_config.is_valid()) {
448 DLOG(ERROR) << __func__ << ": ArConfig_create failed";
449 return base::nullopt;
450 }
451
452 const bool light_estimation_requested =
453 base::Contains(required_features,
454 device::mojom::XRSessionFeature::LIGHT_ESTIMATION) ||
455 base::Contains(optional_features,
456 device::mojom::XRSessionFeature::LIGHT_ESTIMATION);
457
458 if (light_estimation_requested) {
459 // Enable lighting estimation with spherical harmonics
460 ArConfig_setLightEstimationMode(ar_session, arcore_config.get(),
461 AR_LIGHT_ESTIMATION_MODE_ENVIRONMENTAL_HDR);
462 }
463
464 const bool image_tracking_requested =
465 base::Contains(required_features,
466 device::mojom::XRSessionFeature::IMAGE_TRACKING) ||
467 base::Contains(optional_features,
468 device::mojom::XRSessionFeature::IMAGE_TRACKING);
469
470 if (image_tracking_requested) {
471 internal::ScopedArCoreObject<ArAugmentedImageDatabase*> image_db;
472 ArAugmentedImageDatabase_create(
473 ar_session,
474 internal::ScopedArCoreObject<ArAugmentedImageDatabase*>::Receiver(
475 image_db)
476 .get());
477 if (!image_db.is_valid()) {
478 DLOG(ERROR) << "ArAugmentedImageDatabase creation failed";
479 return base::nullopt;
480 }
481
482 // Populate the image tracking database and set up data structures,
483 // this doesn't modify the ArConfig or session yet.
484 BuildImageDatabase(ar_session, image_db.get(), tracked_images);
485
486 if (!tracked_image_arcore_id_to_index_.empty()) {
487 // Image tracking with a non-empty image DB adds a few frames of
488 // synchronization delay internally in ARCore, has a high CPU cost, and
489 // reconfigures its graphics pipeline. Only activate it if we got images.
490 // (Apparently an empty image db is equivalent, but that seems fragile.)
491 ArConfig_setAugmentedImageDatabase(ar_session, arcore_config.get(),
492 image_db.get());
493 // Switch to autofocus mode when tracking images. The default fixed focus
494 // mode has trouble tracking close images since they end up blurry.
495 ArConfig_setFocusMode(ar_session, arcore_config.get(),
496 AR_FOCUS_MODE_AUTO);
497 }
498 }
499
500 const bool depth_api_optional =
501 base::Contains(optional_features, device::mojom::XRSessionFeature::DEPTH);
502 const bool depth_api_requested =
503 base::Contains(required_features,
504 device::mojom::XRSessionFeature::DEPTH) ||
505 depth_api_optional;
506
507 if (depth_api_requested) {
508 ArConfig_setDepthMode(ar_session, arcore_config.get(),
509 AR_DEPTH_MODE_AUTOMATIC);
510 }
511
512 ArStatus status = ArSession_configure(ar_session, arcore_config.get());
513 if (status != AR_SUCCESS && depth_api_optional) {
514 // Depth API is not available on some ARCore-capable devices - if it was
515 // requested optionally, let's try to request the session w/o it.
516
517 DLOG(WARNING) << __func__
518 << ": Depth API was optionally requested and the session "
519 "creation failed, re-trying with depth API disabled";
520
521 enabled_features.erase(device::mojom::XRSessionFeature::DEPTH);
522
523 ArConfig_setDepthMode(ar_session, arcore_config.get(),
524 AR_DEPTH_MODE_DISABLED);
525
526 status = ArSession_configure(ar_session, arcore_config.get());
527 }
528
529 if (status != AR_SUCCESS) {
530 DLOG(ERROR) << __func__ << ": ArSession_configure failed: " << status;
531 return base::nullopt;
532 }
533
534 return enabled_features;
535 }
536
ConfigureCamera(ArSession * ar_session)537 bool ArCoreImpl::ConfigureCamera(ArSession* ar_session) {
538 internal::ScopedArCoreObject<ArCameraConfigFilter*> camera_config_filter;
539 ArCameraConfigFilter_create(
540 ar_session, internal::ScopedArCoreObject<ArCameraConfigFilter*>::Receiver(
541 camera_config_filter)
542 .get());
543 if (!camera_config_filter.is_valid()) {
544 DLOG(ERROR) << "ArCameraConfigFilter_create failed";
545 return false;
546 }
547
548 // We only want to work at 30fps for now.
549 ArCameraConfigFilter_setTargetFps(ar_session, camera_config_filter.get(),
550 AR_CAMERA_CONFIG_TARGET_FPS_30);
551 // We do not care if depth sensor is available or not for now.
552 // The default depth sensor usage of the newly created filter is not
553 // documented, so let's set the filter explicitly to accept both cameras with
554 // and without depth sensors.
555 ArCameraConfigFilter_setDepthSensorUsage(
556 ar_session, camera_config_filter.get(),
557 AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE |
558 AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_DO_NOT_USE);
559
560 internal::ScopedArCoreObject<ArCameraConfigList*> camera_config_list;
561 ArCameraConfigList_create(
562 ar_session, internal::ScopedArCoreObject<ArCameraConfigList*>::Receiver(
563 camera_config_list)
564 .get());
565
566 if (!camera_config_list.is_valid()) {
567 DLOG(ERROR) << "ArCameraConfigList_create failed";
568 return false;
569 }
570
571 ArSession_getSupportedCameraConfigsWithFilter(
572 ar_session, camera_config_filter.get(), camera_config_list.get());
573 if (!camera_config_list.is_valid()) {
574 DLOG(ERROR) << "ArSession_getSupportedCameraConfigsWithFilter failed";
575 return false;
576 }
577
578 int32_t available_configs_count;
579 ArCameraConfigList_getSize(ar_session, camera_config_list.get(),
580 &available_configs_count);
581
582 DVLOG(2) << __func__ << ": ARCore reported " << available_configs_count
583 << " available configurations";
584
585 std::vector<internal::ScopedArCoreObject<ArCameraConfig*>> available_configs;
586 available_configs.reserve(available_configs_count);
587 for (int32_t i = 0; i < available_configs_count; ++i) {
588 internal::ScopedArCoreObject<ArCameraConfig*> camera_config;
589 ArCameraConfig_create(
590 ar_session,
591 internal::ScopedArCoreObject<ArCameraConfig*>::Receiver(camera_config)
592 .get());
593
594 if (!camera_config.is_valid()) {
595 DVLOG(1) << __func__
596 << ": ArCameraConfig_create failed for camera config at index "
597 << i;
598 continue;
599 }
600
601 ArCameraConfigList_getItem(ar_session, camera_config_list.get(), i,
602 camera_config.get());
603
604 ArCameraConfigFacingDirection facing_direction;
605 ArCameraConfig_getFacingDirection(ar_session, camera_config.get(),
606 &facing_direction);
607
608 if (facing_direction != AR_CAMERA_CONFIG_FACING_DIRECTION_BACK) {
609 DVLOG(2)
610 << __func__
611 << ": camera config does not refer to back-facing camera, ignoring";
612 continue;
613 }
614
615 #if DCHECK_IS_ON()
616 {
617 int32_t tex_width, tex_height;
618 ArCameraConfig_getTextureDimensions(ar_session, camera_config.get(),
619 &tex_width, &tex_height);
620
621 int32_t img_width, img_height;
622 ArCameraConfig_getImageDimensions(ar_session, camera_config.get(),
623 &img_width, &img_height);
624
625 uint32_t depth_sensor_usage;
626 ArCameraConfig_getDepthSensorUsage(ar_session, camera_config.get(),
627 &depth_sensor_usage);
628
629 int32_t min_fps, max_fps;
630 ArCameraConfig_getFpsRange(ar_session, camera_config.get(), &min_fps,
631 &max_fps);
632
633 DVLOG(3) << __func__
634 << ": matching camera config found, texture dimensions="
635 << tex_width << "x" << tex_height
636 << ", image dimensions= " << img_width << "x" << img_height
637 << ", depth sensor usage=" << depth_sensor_usage
638 << ", min_fps=" << min_fps << ", max_fps=" << max_fps;
639 }
640 #endif
641
642 available_configs.push_back(std::move(camera_config));
643 }
644
645 if (available_configs.empty()) {
646 DLOG(ERROR) << "No matching configs found";
647 return false;
648 }
649
650 auto best_config = std::max_element(
651 available_configs.begin(), available_configs.end(),
652 [ar_session](const internal::ScopedArCoreObject<ArCameraConfig*>& lhs,
653 const internal::ScopedArCoreObject<ArCameraConfig*>& rhs) {
654 // true means that lhs is less than rhs
655
656 // We'll prefer the configs with higher total resolution (GPU first,
657 // then CPU), everything else does not matter for us now, but we will
658 // weakly prefer the cameras that support depth sensor (it will be used
659 // as a tie-breaker).
660
661 {
662 int32_t lhs_tex_width, lhs_tex_height;
663 int32_t rhs_tex_width, rhs_tex_height;
664
665 ArCameraConfig_getTextureDimensions(ar_session, lhs.get(),
666 &lhs_tex_width, &lhs_tex_height);
667 ArCameraConfig_getTextureDimensions(ar_session, rhs.get(),
668 &rhs_tex_width, &rhs_tex_height);
669
670 if (lhs_tex_width * lhs_tex_height !=
671 rhs_tex_width * rhs_tex_height) {
672 return lhs_tex_width * lhs_tex_height <
673 rhs_tex_width * rhs_tex_height;
674 }
675 }
676
677 {
678 int32_t lhs_img_width, lhs_img_height;
679 int32_t rhs_img_width, rhs_img_height;
680
681 ArCameraConfig_getImageDimensions(ar_session, lhs.get(),
682 &lhs_img_width, &lhs_img_height);
683 ArCameraConfig_getImageDimensions(ar_session, rhs.get(),
684 &rhs_img_width, &rhs_img_height);
685
686 if (lhs_img_width * lhs_img_height !=
687 rhs_img_width * rhs_img_height) {
688 return lhs_img_width * lhs_img_height <
689 rhs_img_width * rhs_img_height;
690 }
691 }
692
693 {
694 uint32_t lhs_depth_sensor_usage;
695 uint32_t rhs_depth_sensor_usage;
696
697 ArCameraConfig_getDepthSensorUsage(ar_session, lhs.get(),
698 &lhs_depth_sensor_usage);
699 ArCameraConfig_getDepthSensorUsage(ar_session, rhs.get(),
700 &rhs_depth_sensor_usage);
701
702 bool lhs_has_depth =
703 lhs_depth_sensor_usage &
704 AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE;
705 bool rhs_has_depth =
706 rhs_depth_sensor_usage &
707 AR_CAMERA_CONFIG_DEPTH_SENSOR_USAGE_REQUIRE_AND_USE;
708
709 return lhs_has_depth < rhs_has_depth;
710 }
711 });
712
713 int32_t fps_min, fps_max;
714 ArCameraConfig_getFpsRange(ar_session, best_config->get(), &fps_min,
715 &fps_max);
716 target_framerate_range_ = {fps_min, fps_max};
717
718 #if DCHECK_IS_ON()
719 {
720 int32_t tex_width, tex_height;
721 ArCameraConfig_getTextureDimensions(ar_session, best_config->get(),
722 &tex_width, &tex_height);
723
724 int32_t img_width, img_height;
725 ArCameraConfig_getImageDimensions(ar_session, best_config->get(),
726 &img_width, &img_height);
727
728 uint32_t depth_sensor_usage;
729 ArCameraConfig_getDepthSensorUsage(ar_session, best_config->get(),
730 &depth_sensor_usage);
731 DVLOG(3) << __func__
732 << ": selected camera config with texture dimensions=" << tex_width
733 << "x" << tex_height << ", image dimensions=" << img_width << "x"
734 << img_height << ", depth sensor usage=" << depth_sensor_usage
735 << ", min_fps=" << target_framerate_range_.min
736 << ", max_fps=" << target_framerate_range_.max;
737 }
738 #endif
739
740 ArStatus status = ArSession_setCameraConfig(ar_session, best_config->get());
741 if (status != AR_SUCCESS) {
742 DLOG(ERROR) << "ArSession_setCameraConfig failed: " << status;
743 return false;
744 }
745
746 return true;
747 }
748
GetTargetFramerateRange()749 ArCore::MinMaxRange ArCoreImpl::GetTargetFramerateRange() {
750 return target_framerate_range_;
751 }
752
BuildImageDatabase(const ArSession * session,ArAugmentedImageDatabase * image_db,const std::vector<device::mojom::XRTrackedImagePtr> & tracked_images)753 void ArCoreImpl::BuildImageDatabase(
754 const ArSession* session,
755 ArAugmentedImageDatabase* image_db,
756 const std::vector<device::mojom::XRTrackedImagePtr>& tracked_images) {
757 for (std::size_t index = 0; index < tracked_images.size(); ++index) {
758 const device::mojom::XRTrackedImage* image = tracked_images[index].get();
759 gfx::Size size = image->size_in_pixels;
760
761 // Use Skia to convert the image to grayscale.
762 const SkBitmap& src_bitmap = image->bitmap;
763 SkBitmap canvas_bitmap;
764 canvas_bitmap.allocPixelsFlags(
765 SkImageInfo::Make(size.width(), size.height(), kGray_8_SkColorType,
766 kOpaque_SkAlphaType),
767 SkBitmap::kZeroPixels_AllocFlag);
768 SkCanvas gray_canvas(canvas_bitmap);
769 SkPaint paint;
770 sk_sp<SkImage> src_image = SkImage::MakeFromBitmap(src_bitmap);
771 gray_canvas.drawImage(src_image, 0, 0, &paint);
772 SkPixmap gray_pixmap;
773 if (!gray_canvas.peekPixels(&gray_pixmap)) {
774 DLOG(WARNING) << __func__ << ": failed to access grayscale bitmap";
775 image_trackable_scores_.push_back(false);
776 continue;
777 }
778
779 const SkPixmap& pixmap = gray_pixmap;
780 float width_in_meters = image->width_in_meters;
781 DVLOG(3) << __func__ << " tracked image index=" << index
782 << " size=" << pixmap.width() << "x" << pixmap.height()
783 << " width_in_meters=" << width_in_meters;
784 int32_t arcore_id = -1;
785 std::string id_name = base::NumberToString(index);
786 ArStatus status = ArAugmentedImageDatabase_addImageWithPhysicalSize(
787 session, image_db, id_name.c_str(), pixmap.addr8(), pixmap.width(),
788 pixmap.height(), pixmap.rowBytesAsPixels(), width_in_meters,
789 &arcore_id);
790 if (status != AR_SUCCESS) {
791 DVLOG(2) << __func__ << ": add image failed";
792 image_trackable_scores_.push_back(false);
793 continue;
794 }
795
796 // ARCore uses internal IDs for images, these only include the trackable
797 // images. The tracking results need to refer to the original image index
798 // corresponding to its position in the input tracked_images array.
799 tracked_image_arcore_id_to_index_[arcore_id] = index;
800 DVLOG(2) << __func__ << ": added image, index=" << index
801 << " arcore_id=" << arcore_id;
802 image_trackable_scores_.push_back(true);
803 }
804 }
805
SetCameraTexture(uint32_t camera_texture_id)806 void ArCoreImpl::SetCameraTexture(uint32_t camera_texture_id) {
807 DCHECK(IsOnGlThread());
808 DCHECK(arcore_session_.is_valid());
809 ArSession_setCameraTextureName(arcore_session_.get(), camera_texture_id);
810 }
811
SetDisplayGeometry(const gfx::Size & frame_size,display::Display::Rotation display_rotation)812 void ArCoreImpl::SetDisplayGeometry(
813 const gfx::Size& frame_size,
814 display::Display::Rotation display_rotation) {
815 DCHECK(IsOnGlThread());
816 DCHECK(arcore_session_.is_valid());
817 // Display::Rotation is the same as Android's rotation and is compatible with
818 // what ArCore is expecting.
819 ArSession_setDisplayGeometry(arcore_session_.get(), display_rotation,
820 frame_size.width(), frame_size.height());
821 }
822
TransformDisplayUvCoords(const base::span<const float> uvs) const823 std::vector<float> ArCoreImpl::TransformDisplayUvCoords(
824 const base::span<const float> uvs) const {
825 DCHECK(IsOnGlThread());
826 DCHECK(arcore_session_.is_valid());
827 DCHECK(arcore_frame_.is_valid());
828
829 size_t num_elements = uvs.size();
830 DCHECK(num_elements % 2 == 0);
831 DCHECK_GE(num_elements, 6u);
832
833 std::vector<float> uvs_out(num_elements);
834 ArFrame_transformCoordinates2d(
835 arcore_session_.get(), arcore_frame_.get(),
836 AR_COORDINATES_2D_VIEW_NORMALIZED, num_elements / 2, &uvs[0],
837 AR_COORDINATES_2D_TEXTURE_NORMALIZED, &uvs_out[0]);
838
839 DVLOG(3) << __func__ << ": transformed uvs=[ " << uvs_out[0] << " , "
840 << uvs_out[1] << " , " << uvs_out[2] << " , " << uvs_out[3] << " , "
841 << uvs_out[4] << " , " << uvs_out[5] << " ]";
842
843 return uvs_out;
844 }
845
Update(bool * camera_updated)846 mojom::VRPosePtr ArCoreImpl::Update(bool* camera_updated) {
847 TRACE_EVENT0("gpu", "ArCoreImpl Update");
848
849 DCHECK(IsOnGlThread());
850 DCHECK(arcore_session_.is_valid());
851 DCHECK(arcore_frame_.is_valid());
852 DCHECK(camera_updated);
853
854 ArStatus status;
855
856 TRACE_EVENT_BEGIN0("gpu", "ArCore Update");
857 status = ArSession_update(arcore_session_.get(), arcore_frame_.get());
858 TRACE_EVENT_END0("gpu", "ArCore Update");
859
860 if (status != AR_SUCCESS) {
861 DLOG(ERROR) << "ArSession_update failed: " << status;
862 *camera_updated = false;
863 return nullptr;
864 }
865
866 // If we get here, assume we have a valid camera image, but we don't know yet
867 // if tracking is working.
868 *camera_updated = true;
869 internal::ScopedArCoreObject<ArCamera*> arcore_camera;
870 ArFrame_acquireCamera(
871 arcore_session_.get(), arcore_frame_.get(),
872 internal::ScopedArCoreObject<ArCamera*>::Receiver(arcore_camera).get());
873 if (!arcore_camera.is_valid()) {
874 DLOG(ERROR) << "ArFrame_acquireCamera failed!";
875 return nullptr;
876 }
877
878 ArTrackingState tracking_state;
879 ArCamera_getTrackingState(arcore_session_.get(), arcore_camera.get(),
880 &tracking_state);
881 if (tracking_state != AR_TRACKING_STATE_TRACKING) {
882 DVLOG(1) << "Tracking state is not AR_TRACKING_STATE_TRACKING: "
883 << tracking_state;
884 return nullptr;
885 }
886
887 internal::ScopedArCoreObject<ArPose*> arcore_pose;
888 ArPose_create(
889 arcore_session_.get(), nullptr,
890 internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
891 if (!arcore_pose.is_valid()) {
892 DLOG(ERROR) << "ArPose_create failed!";
893 return nullptr;
894 }
895
896 ArCamera_getDisplayOrientedPose(arcore_session_.get(), arcore_camera.get(),
897 arcore_pose.get());
898
899 auto mojo_from_viewer =
900 GetMojomVRPoseFromArPose(arcore_session_.get(), arcore_pose.get());
901
902 TRACE_EVENT_BEGIN0("gpu", "ArCorePlaneManager Update");
903 plane_manager_->Update(arcore_frame_.get());
904 TRACE_EVENT_END0("gpu", "ArCorePlaneManager Update");
905
906 TRACE_EVENT_BEGIN0("gpu", "ArCoreAnchorManager Update");
907 anchor_manager_->Update(arcore_frame_.get());
908 TRACE_EVENT_END0("gpu", "ArCoreAnchorManager Update");
909
910 return mojo_from_viewer;
911 }
912
GetTrackedImages()913 mojom::XRTrackedImagesDataPtr ArCoreImpl::GetTrackedImages() {
914 std::vector<mojom::XRTrackedImageDataPtr> images_data;
915
916 internal::ScopedArCoreObject<ArTrackableList*> updated_images;
917 ArTrackableList_create(
918 arcore_session_.get(),
919 internal::ScopedArCoreObject<ArTrackableList*>::Receiver(updated_images)
920 .get());
921 ArFrame_getUpdatedTrackables(arcore_session_.get(), arcore_frame_.get(),
922 AR_TRACKABLE_AUGMENTED_IMAGE,
923 updated_images.get());
924
925 int32_t images_count = 0;
926 ArTrackableList_getSize(arcore_session_.get(), updated_images.get(),
927 &images_count);
928 DVLOG(2) << __func__ << ": trackable images count=" << images_count;
929
930 for (int32_t i = 0; i < images_count; ++i) {
931 internal::ScopedArCoreObject<ArTrackable*> trackable;
932 ArTrackableList_acquireItem(
933 arcore_session_.get(), updated_images.get(), i,
934 internal::ScopedArCoreObject<ArTrackable*>::Receiver(trackable).get());
935 ArTrackingState tracking_state;
936 ArTrackable_getTrackingState(arcore_session_.get(), trackable.get(),
937 &tracking_state);
938 ArAugmentedImage* image = ArAsAugmentedImage(trackable.get());
939
940 // Get the original image index from ARCore's internal ID for use in the
941 // returned results.
942 int32_t arcore_id;
943 ArAugmentedImage_getIndex(arcore_session_.get(), image, &arcore_id);
944 uint64_t index = tracked_image_arcore_id_to_index_[arcore_id];
945 DVLOG(3) << __func__ << ": #" << i << " tracked image index=" << index
946 << " arcore_id=" << arcore_id << " state=" << tracking_state;
947
948 if (tracking_state == AR_TRACKING_STATE_TRACKING) {
949 internal::ScopedArCoreObject<ArPose*> arcore_pose;
950 ArPose_create(
951 arcore_session_.get(), nullptr,
952 internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
953 if (!arcore_pose.is_valid()) {
954 DLOG(ERROR) << "ArPose_create failed!";
955 continue;
956 }
957 ArAugmentedImage_getCenterPose(arcore_session_.get(), image,
958 arcore_pose.get());
959 float pose_raw[7];
960 ArPose_getPoseRaw(arcore_session_.get(), arcore_pose.get(), &pose_raw[0]);
961 DVLOG(3) << __func__ << ": tracked image pose_raw pos=(" << pose_raw[4]
962 << ", " << pose_raw[5] << ", " << pose_raw[6] << ")";
963
964 device::Pose device_pose =
965 GetPoseFromArPose(arcore_session_.get(), arcore_pose.get());
966
967 ArAugmentedImageTrackingMethod tracking_method;
968 ArAugmentedImage_getTrackingMethod(arcore_session_.get(), image,
969 &tracking_method);
970 bool actively_tracked =
971 tracking_method == AR_AUGMENTED_IMAGE_TRACKING_METHOD_FULL_TRACKING;
972
973 float width_in_meters;
974 ArAugmentedImage_getExtentX(arcore_session_.get(), image,
975 &width_in_meters);
976
977 images_data.push_back(mojom::XRTrackedImageData::New(
978 index, device_pose, actively_tracked, width_in_meters));
979 }
980 }
981
982 // Include information about each image's trackability status in the first
983 // returned result list.
984 if (!image_trackable_scores_sent_) {
985 auto ret = mojom::XRTrackedImagesData::New(
986 std::move(images_data), std::move(image_trackable_scores_));
987 image_trackable_scores_sent_ = true;
988 image_trackable_scores_.clear();
989 return ret;
990 } else {
991 return mojom::XRTrackedImagesData::New(std::move(images_data),
992 base::nullopt);
993 }
994 }
995
GetFrameTimestamp()996 base::TimeDelta ArCoreImpl::GetFrameTimestamp() {
997 DCHECK(arcore_session_.is_valid());
998 DCHECK(arcore_frame_.is_valid());
999 int64_t out_timestamp_ns;
1000 ArFrame_getTimestamp(arcore_session_.get(), arcore_frame_.get(),
1001 &out_timestamp_ns);
1002 return base::TimeDelta::FromNanoseconds(out_timestamp_ns);
1003 }
1004
GetDetectedPlanesData()1005 mojom::XRPlaneDetectionDataPtr ArCoreImpl::GetDetectedPlanesData() {
1006 DVLOG(2) << __func__;
1007
1008 TRACE_EVENT0("gpu", __func__);
1009
1010 return plane_manager_->GetDetectedPlanesData();
1011 }
1012
GetAnchorsData()1013 mojom::XRAnchorsDataPtr ArCoreImpl::GetAnchorsData() {
1014 DVLOG(2) << __func__;
1015
1016 TRACE_EVENT0("gpu", __func__);
1017
1018 return anchor_manager_->GetAnchorsData();
1019 }
1020
GetLightEstimationData()1021 mojom::XRLightEstimationDataPtr ArCoreImpl::GetLightEstimationData() {
1022 TRACE_EVENT0("gpu", __func__);
1023
1024 ArFrame_getLightEstimate(arcore_session_.get(), arcore_frame_.get(),
1025 arcore_light_estimate_.get());
1026
1027 ArLightEstimateState light_estimate_state = AR_LIGHT_ESTIMATE_STATE_NOT_VALID;
1028 ArLightEstimate_getState(arcore_session_.get(), arcore_light_estimate_.get(),
1029 &light_estimate_state);
1030
1031 // The light estimate state is not guaranteed to be valid initially
1032 if (light_estimate_state != AR_LIGHT_ESTIMATE_STATE_VALID) {
1033 DVLOG(2) << "ArCore light estimation state invalid.";
1034 return nullptr;
1035 }
1036
1037 auto light_estimation_data = mojom::XRLightEstimationData::New();
1038 light_estimation_data->light_probe =
1039 GetLightProbe(arcore_session_.get(), arcore_light_estimate_.get());
1040 if (!light_estimation_data->light_probe) {
1041 DVLOG(1) << "Failed to generate light probe.";
1042 return nullptr;
1043 }
1044 light_estimation_data->reflection_probe =
1045 GetReflectionProbe(arcore_session_.get(), arcore_light_estimate_.get());
1046 if (!light_estimation_data->reflection_probe) {
1047 DVLOG(1) << "Failed to generate reflection probe.";
1048 return nullptr;
1049 }
1050
1051 return light_estimation_data;
1052 }
1053
Pause()1054 void ArCoreImpl::Pause() {
1055 DVLOG(2) << __func__;
1056
1057 DCHECK(IsOnGlThread());
1058 DCHECK(arcore_session_.is_valid());
1059
1060 ArStatus status = ArSession_pause(arcore_session_.get());
1061 DLOG_IF(ERROR, status != AR_SUCCESS)
1062 << "ArSession_pause failed: status = " << status;
1063 }
1064
Resume()1065 void ArCoreImpl::Resume() {
1066 DVLOG(2) << __func__;
1067
1068 DCHECK(IsOnGlThread());
1069 DCHECK(arcore_session_.is_valid());
1070
1071 ArStatus status = ArSession_resume(arcore_session_.get());
1072 DLOG_IF(ERROR, status != AR_SUCCESS)
1073 << "ArSession_resume failed: status = " << status;
1074 }
1075
GetProjectionMatrix(float near,float far)1076 gfx::Transform ArCoreImpl::GetProjectionMatrix(float near, float far) {
1077 DCHECK(IsOnGlThread());
1078 DCHECK(arcore_session_.is_valid());
1079 DCHECK(arcore_frame_.is_valid());
1080
1081 internal::ScopedArCoreObject<ArCamera*> arcore_camera;
1082 ArFrame_acquireCamera(
1083 arcore_session_.get(), arcore_frame_.get(),
1084 internal::ScopedArCoreObject<ArCamera*>::Receiver(arcore_camera).get());
1085 DCHECK(arcore_camera.is_valid())
1086 << "ArFrame_acquireCamera failed despite documentation saying it cannot";
1087
1088 // ArCore's projection matrix is 16 floats in column-major order.
1089 float matrix_4x4[16];
1090 ArCamera_getProjectionMatrix(arcore_session_.get(), arcore_camera.get(), near,
1091 far, matrix_4x4);
1092 gfx::Transform result;
1093 result.matrix().setColMajorf(matrix_4x4);
1094 return result;
1095 }
1096
GetEstimatedFloorHeight()1097 float ArCoreImpl::GetEstimatedFloorHeight() {
1098 return kDefaultFloorHeightEstimation;
1099 }
1100
SubscribeToHitTest(mojom::XRNativeOriginInformationPtr native_origin_information,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)1101 base::Optional<uint64_t> ArCoreImpl::SubscribeToHitTest(
1102 mojom::XRNativeOriginInformationPtr native_origin_information,
1103 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1104 mojom::XRRayPtr ray) {
1105 // First, check if we recognize the type of the native origin.
1106
1107 if (native_origin_information->is_reference_space_type()) {
1108 // Reference spaces are implicitly recognized and don't carry an ID.
1109 } else if (native_origin_information->is_input_source_id()) {
1110 // Input source IDs are verified in the higher layer as ArCoreImpl does
1111 // not carry input source state.
1112 } else if (native_origin_information->is_plane_id()) {
1113 // Validate that we know which plane's space the hit test is interested in
1114 // tracking.
1115 if (!plane_manager_->PlaneExists(
1116 PlaneId(native_origin_information->get_plane_id()))) {
1117 return base::nullopt;
1118 }
1119 } else if (native_origin_information->is_anchor_id()) {
1120 // Validate that we know which anchor's space the hit test is interested
1121 // in tracking.
1122 if (!anchor_manager_->AnchorExists(
1123 AnchorId(native_origin_information->get_anchor_id()))) {
1124 return base::nullopt;
1125 }
1126 } else {
1127 NOTREACHED();
1128 return base::nullopt;
1129 }
1130
1131 auto subscription_id = CreateHitTestSubscriptionId();
1132
1133 hit_test_subscription_id_to_data_.emplace(
1134 subscription_id,
1135 HitTestSubscriptionData{std::move(native_origin_information),
1136 entity_types, std::move(ray)});
1137
1138 return subscription_id.GetUnsafeValue();
1139 }
1140
SubscribeToHitTestForTransientInput(const std::string & profile_name,const std::vector<mojom::EntityTypeForHitTest> & entity_types,mojom::XRRayPtr ray)1141 base::Optional<uint64_t> ArCoreImpl::SubscribeToHitTestForTransientInput(
1142 const std::string& profile_name,
1143 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1144 mojom::XRRayPtr ray) {
1145 auto subscription_id = CreateHitTestSubscriptionId();
1146
1147 hit_test_subscription_id_to_transient_hit_test_data_.emplace(
1148 subscription_id, TransientInputHitTestSubscriptionData{
1149 profile_name, entity_types, std::move(ray)});
1150
1151 return subscription_id.GetUnsafeValue();
1152 }
1153
1154 mojom::XRHitTestSubscriptionResultsDataPtr
GetHitTestSubscriptionResults(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1155 ArCoreImpl::GetHitTestSubscriptionResults(
1156 const gfx::Transform& mojo_from_viewer,
1157 const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1158 mojom::XRHitTestSubscriptionResultsDataPtr result =
1159 mojom::XRHitTestSubscriptionResultsData::New();
1160
1161 DVLOG(3) << __func__
1162 << ": calculating hit test subscription results, "
1163 "hit_test_subscription_id_to_data_.size()="
1164 << hit_test_subscription_id_to_data_.size();
1165
1166 for (auto& subscription_id_and_data : hit_test_subscription_id_to_data_) {
1167 // First, check if we can find the current transformation for a ray. If not,
1168 // skip processing this subscription.
1169 auto maybe_mojo_from_native_origin = GetMojoFromNativeOrigin(
1170 *subscription_id_and_data.second.native_origin_information,
1171 mojo_from_viewer, input_state);
1172
1173 if (!maybe_mojo_from_native_origin) {
1174 continue;
1175 }
1176
1177 // Since we have a transform, let's use it to obtain hit test results.
1178 result->results.push_back(GetHitTestSubscriptionResult(
1179 HitTestSubscriptionId(subscription_id_and_data.first),
1180 *subscription_id_and_data.second.ray,
1181 subscription_id_and_data.second.entity_types,
1182 *maybe_mojo_from_native_origin));
1183 }
1184
1185 DVLOG(3)
1186 << __func__
1187 << ": calculating hit test subscription results for transient input, "
1188 "hit_test_subscription_id_to_transient_hit_test_data_.size()="
1189 << hit_test_subscription_id_to_transient_hit_test_data_.size();
1190
1191 for (const auto& subscription_id_and_data :
1192 hit_test_subscription_id_to_transient_hit_test_data_) {
1193 auto input_source_ids_and_transforms =
1194 GetMojoFromInputSources(subscription_id_and_data.second.profile_name,
1195 mojo_from_viewer, input_state);
1196
1197 result->transient_input_results.push_back(
1198 GetTransientHitTestSubscriptionResult(
1199 HitTestSubscriptionId(subscription_id_and_data.first),
1200 *subscription_id_and_data.second.ray,
1201 subscription_id_and_data.second.entity_types,
1202 input_source_ids_and_transforms));
1203 }
1204
1205 return result;
1206 }
1207
1208 device::mojom::XRHitTestSubscriptionResultDataPtr
GetHitTestSubscriptionResult(HitTestSubscriptionId id,const mojom::XRRay & native_origin_ray,const std::vector<mojom::EntityTypeForHitTest> & entity_types,const gfx::Transform & mojo_from_native_origin)1209 ArCoreImpl::GetHitTestSubscriptionResult(
1210 HitTestSubscriptionId id,
1211 const mojom::XRRay& native_origin_ray,
1212 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1213 const gfx::Transform& mojo_from_native_origin) {
1214 DVLOG(3) << __func__ << ": id=" << id;
1215
1216 // Transform the ray according to the latest transform based on the XRSpace
1217 // used in hit test subscription.
1218
1219 gfx::Point3F origin = native_origin_ray.origin;
1220 mojo_from_native_origin.TransformPoint(&origin);
1221
1222 gfx::Vector3dF direction = native_origin_ray.direction;
1223 mojo_from_native_origin.TransformVector(&direction);
1224
1225 std::vector<mojom::XRHitResultPtr> hit_results;
1226 if (!RequestHitTest(origin, direction, entity_types, &hit_results)) {
1227 hit_results.clear(); // On failure, clear partial results.
1228 }
1229
1230 return mojom::XRHitTestSubscriptionResultData::New(id.GetUnsafeValue(),
1231 std::move(hit_results));
1232 }
1233
1234 device::mojom::XRHitTestTransientInputSubscriptionResultDataPtr
GetTransientHitTestSubscriptionResult(HitTestSubscriptionId id,const mojom::XRRay & input_source_ray,const std::vector<mojom::EntityTypeForHitTest> & entity_types,const std::vector<std::pair<uint32_t,gfx::Transform>> & input_source_ids_and_mojo_from_input_sources)1235 ArCoreImpl::GetTransientHitTestSubscriptionResult(
1236 HitTestSubscriptionId id,
1237 const mojom::XRRay& input_source_ray,
1238 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1239 const std::vector<std::pair<uint32_t, gfx::Transform>>&
1240 input_source_ids_and_mojo_from_input_sources) {
1241 auto result =
1242 device::mojom::XRHitTestTransientInputSubscriptionResultData::New();
1243
1244 result->subscription_id = id.GetUnsafeValue();
1245
1246 for (const auto& input_source_id_and_mojo_from_input_source :
1247 input_source_ids_and_mojo_from_input_sources) {
1248 gfx::Point3F origin = input_source_ray.origin;
1249 input_source_id_and_mojo_from_input_source.second.TransformPoint(&origin);
1250
1251 gfx::Vector3dF direction = input_source_ray.direction;
1252 input_source_id_and_mojo_from_input_source.second.TransformVector(
1253 &direction);
1254
1255 std::vector<mojom::XRHitResultPtr> hit_results;
1256 if (!RequestHitTest(origin, direction, entity_types, &hit_results)) {
1257 hit_results.clear(); // On failure, clear partial results.
1258 }
1259
1260 result->input_source_id_to_hit_test_results.insert(
1261 {input_source_id_and_mojo_from_input_source.first,
1262 std::move(hit_results)});
1263 }
1264
1265 return result;
1266 }
1267
1268 std::vector<std::pair<uint32_t, gfx::Transform>>
GetMojoFromInputSources(const std::string & profile_name,const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1269 ArCoreImpl::GetMojoFromInputSources(
1270 const std::string& profile_name,
1271 const gfx::Transform& mojo_from_viewer,
1272 const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1273 std::vector<std::pair<uint32_t, gfx::Transform>> result;
1274
1275 for (const auto& input_source_state : input_state) {
1276 if (input_source_state && input_source_state->description) {
1277 if (base::Contains(input_source_state->description->profiles,
1278 profile_name)) {
1279 // Input source represented by input_state matches the profile, find
1280 // the transform and grab input source id.
1281 base::Optional<gfx::Transform> maybe_mojo_from_input_source =
1282 GetMojoFromInputSource(input_source_state, mojo_from_viewer);
1283
1284 if (!maybe_mojo_from_input_source)
1285 continue;
1286
1287 result.push_back(
1288 {input_source_state->source_id, *maybe_mojo_from_input_source});
1289 }
1290 }
1291 }
1292
1293 return result;
1294 }
1295
GetMojoFromReferenceSpace(device::mojom::XRReferenceSpaceType type,const gfx::Transform & mojo_from_viewer)1296 base::Optional<gfx::Transform> ArCoreImpl::GetMojoFromReferenceSpace(
1297 device::mojom::XRReferenceSpaceType type,
1298 const gfx::Transform& mojo_from_viewer) {
1299 DVLOG(3) << __func__ << ": type=" << type;
1300
1301 switch (type) {
1302 case device::mojom::XRReferenceSpaceType::kLocal:
1303 return gfx::Transform{};
1304 case device::mojom::XRReferenceSpaceType::kLocalFloor: {
1305 auto result = gfx::Transform{};
1306 result.Translate3d(0, -GetEstimatedFloorHeight(), 0);
1307 return result;
1308 }
1309 case device::mojom::XRReferenceSpaceType::kViewer:
1310 return mojo_from_viewer;
1311 case device::mojom::XRReferenceSpaceType::kBoundedFloor:
1312 return base::nullopt;
1313 case device::mojom::XRReferenceSpaceType::kUnbounded:
1314 return base::nullopt;
1315 }
1316 }
1317
NativeOriginExists(const mojom::XRNativeOriginInformation & native_origin_information,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1318 bool ArCoreImpl::NativeOriginExists(
1319 const mojom::XRNativeOriginInformation& native_origin_information,
1320 const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1321 switch (native_origin_information.which()) {
1322 case mojom::XRNativeOriginInformation::Tag::INPUT_SOURCE_ID:
1323
1324 // Linear search should be fine for ARCore device as it only has one input
1325 // source (for now).
1326 for (auto& input_source_state : input_state) {
1327 if (input_source_state->source_id ==
1328 native_origin_information.get_input_source_id()) {
1329 return true;
1330 }
1331 }
1332
1333 return false;
1334 case mojom::XRNativeOriginInformation::Tag::REFERENCE_SPACE_TYPE:
1335 // All reference spaces are known to ARCore.
1336 return true;
1337
1338 case mojom::XRNativeOriginInformation::Tag::PLANE_ID:
1339 return plane_manager_->PlaneExists(
1340 PlaneId(native_origin_information.get_plane_id()));
1341 case mojom::XRNativeOriginInformation::Tag::ANCHOR_ID:
1342
1343 return anchor_manager_->AnchorExists(
1344 AnchorId(native_origin_information.get_anchor_id()));
1345 }
1346 }
1347
GetMojoFromNativeOrigin(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state)1348 base::Optional<gfx::Transform> ArCoreImpl::GetMojoFromNativeOrigin(
1349 const mojom::XRNativeOriginInformation& native_origin_information,
1350 const gfx::Transform& mojo_from_viewer,
1351 const std::vector<mojom::XRInputSourceStatePtr>& input_state) {
1352 switch (native_origin_information.which()) {
1353 case mojom::XRNativeOriginInformation::Tag::INPUT_SOURCE_ID:
1354
1355 // Linear search should be fine for ARCore device as it only has one input
1356 // source (for now).
1357 for (auto& input_source_state : input_state) {
1358 if (input_source_state->source_id ==
1359 native_origin_information.get_input_source_id()) {
1360 return GetMojoFromInputSource(input_source_state, mojo_from_viewer);
1361 }
1362 }
1363
1364 return base::nullopt;
1365 case mojom::XRNativeOriginInformation::Tag::REFERENCE_SPACE_TYPE:
1366 return GetMojoFromReferenceSpace(
1367 native_origin_information.get_reference_space_type(),
1368 mojo_from_viewer);
1369 case mojom::XRNativeOriginInformation::Tag::PLANE_ID:
1370 return plane_manager_->GetMojoFromPlane(
1371 PlaneId(native_origin_information.get_plane_id()));
1372 case mojom::XRNativeOriginInformation::Tag::ANCHOR_ID:
1373 return anchor_manager_->GetMojoFromAnchor(
1374 AnchorId(native_origin_information.get_anchor_id()));
1375 }
1376 }
1377
UnsubscribeFromHitTest(uint64_t subscription_id)1378 void ArCoreImpl::UnsubscribeFromHitTest(uint64_t subscription_id) {
1379 DVLOG(2) << __func__ << ": subscription_id=" << subscription_id;
1380
1381 // Hit test subscription ID space is the same for transient and non-transient
1382 // hit test sources, so we can attempt to remove it from both collections (it
1383 // will succeed only for one of them anyway).
1384
1385 hit_test_subscription_id_to_data_.erase(
1386 HitTestSubscriptionId(subscription_id));
1387 hit_test_subscription_id_to_transient_hit_test_data_.erase(
1388 HitTestSubscriptionId(subscription_id));
1389 }
1390
CreateHitTestSubscriptionId()1391 HitTestSubscriptionId ArCoreImpl::CreateHitTestSubscriptionId() {
1392 CHECK(next_id_ != std::numeric_limits<uint64_t>::max())
1393 << "preventing ID overflow";
1394
1395 uint64_t current_id = next_id_++;
1396
1397 return HitTestSubscriptionId(current_id);
1398 }
1399
RequestHitTest(const mojom::XRRayPtr & ray,std::vector<mojom::XRHitResultPtr> * hit_results)1400 bool ArCoreImpl::RequestHitTest(
1401 const mojom::XRRayPtr& ray,
1402 std::vector<mojom::XRHitResultPtr>* hit_results) {
1403 DCHECK(ray);
1404 return RequestHitTest(ray->origin, ray->direction,
1405 {mojom::EntityTypeForHitTest::PLANE},
1406 hit_results); // "Plane" to maintain current behavior
1407 // of async hit test.
1408 }
1409
RequestHitTest(const gfx::Point3F & origin,const gfx::Vector3dF & direction,const std::vector<mojom::EntityTypeForHitTest> & entity_types,std::vector<mojom::XRHitResultPtr> * hit_results)1410 bool ArCoreImpl::RequestHitTest(
1411 const gfx::Point3F& origin,
1412 const gfx::Vector3dF& direction,
1413 const std::vector<mojom::EntityTypeForHitTest>& entity_types,
1414 std::vector<mojom::XRHitResultPtr>* hit_results) {
1415 DVLOG(2) << __func__ << ": origin=" << origin.ToString()
1416 << ", direction=" << direction.ToString();
1417
1418 DCHECK(hit_results);
1419 DCHECK(IsOnGlThread());
1420 DCHECK(arcore_session_.is_valid());
1421 DCHECK(arcore_frame_.is_valid());
1422
1423 auto arcore_entity_types = GetArCoreEntityTypes(entity_types);
1424
1425 // ArCore returns hit-results in sorted order, thus providing the guarantee
1426 // of sorted results promised by the WebXR spec for requestHitTest().
1427 std::array<float, 3> origin_array = {origin.x(), origin.y(), origin.z()};
1428 std::array<float, 3> direction_array = {direction.x(), direction.y(),
1429 direction.z()};
1430
1431 internal::ScopedArCoreObject<ArHitResultList*> arcore_hit_result_list;
1432 ArHitResultList_create(
1433 arcore_session_.get(),
1434 internal::ScopedArCoreObject<ArHitResultList*>::Receiver(
1435 arcore_hit_result_list)
1436 .get());
1437 if (!arcore_hit_result_list.is_valid()) {
1438 DLOG(ERROR) << "ArHitResultList_create failed!";
1439 return false;
1440 }
1441
1442 ArFrame_hitTestRay(arcore_session_.get(), arcore_frame_.get(),
1443 origin_array.data(), direction_array.data(),
1444 arcore_hit_result_list.get());
1445
1446 int arcore_hit_result_list_size = 0;
1447 ArHitResultList_getSize(arcore_session_.get(), arcore_hit_result_list.get(),
1448 &arcore_hit_result_list_size);
1449 DVLOG(2) << __func__
1450 << ": arcore_hit_result_list_size=" << arcore_hit_result_list_size;
1451
1452 // Go through the list in reverse order so the first hit we encounter is the
1453 // furthest.
1454 // We will accept the furthest hit and then for the rest require that the hit
1455 // be within the actual polygon detected by ArCore. This heuristic allows us
1456 // to get better results on floors w/o overestimating the size of tables etc.
1457 // See https://crbug.com/872855.
1458 for (int i = arcore_hit_result_list_size - 1; i >= 0; i--) {
1459 internal::ScopedArCoreObject<ArHitResult*> arcore_hit;
1460
1461 ArHitResult_create(
1462 arcore_session_.get(),
1463 internal::ScopedArCoreObject<ArHitResult*>::Receiver(arcore_hit).get());
1464
1465 if (!arcore_hit.is_valid()) {
1466 DLOG(ERROR) << "ArHitResult_create failed!";
1467 return false;
1468 }
1469
1470 ArHitResultList_getItem(arcore_session_.get(), arcore_hit_result_list.get(),
1471 i, arcore_hit.get());
1472
1473 internal::ScopedArCoreObject<ArTrackable*> ar_trackable;
1474
1475 ArHitResult_acquireTrackable(
1476 arcore_session_.get(), arcore_hit.get(),
1477 internal::ScopedArCoreObject<ArTrackable*>::Receiver(ar_trackable)
1478 .get());
1479 ArTrackableType ar_trackable_type = AR_TRACKABLE_NOT_VALID;
1480 ArTrackable_getType(arcore_session_.get(), ar_trackable.get(),
1481 &ar_trackable_type);
1482
1483 // Only consider trackables listed in arcore_entity_types.
1484 if (!base::Contains(arcore_entity_types, ar_trackable_type)) {
1485 DVLOG(2) << __func__
1486 << ": hit a trackable that is not in entity types set, ignoring "
1487 "it. ar_trackable_type="
1488 << ar_trackable_type;
1489 continue;
1490 }
1491
1492 internal::ScopedArCoreObject<ArPose*> arcore_pose;
1493 ArPose_create(
1494 arcore_session_.get(), nullptr,
1495 internal::ScopedArCoreObject<ArPose*>::Receiver(arcore_pose).get());
1496 if (!arcore_pose.is_valid()) {
1497 DLOG(ERROR) << "ArPose_create failed!";
1498 return false;
1499 }
1500
1501 ArHitResult_getHitPose(arcore_session_.get(), arcore_hit.get(),
1502 arcore_pose.get());
1503
1504 // After the first (furthest) hit, for planes, only return hits that are
1505 // within the actual detected polygon and not just within than the larger
1506 // plane.
1507 uint64_t plane_id = 0;
1508 if (ar_trackable_type == AR_TRACKABLE_PLANE) {
1509 ArPlane* ar_plane = ArAsPlane(ar_trackable.get());
1510
1511 if (!hit_results->empty()) {
1512 int32_t in_polygon = 0;
1513 ArPlane_isPoseInPolygon(arcore_session_.get(), ar_plane,
1514 arcore_pose.get(), &in_polygon);
1515 if (!in_polygon) {
1516 DVLOG(2) << __func__
1517 << ": hit a trackable that is not within detected polygon, "
1518 "ignoring it";
1519 continue;
1520 }
1521 }
1522
1523 base::Optional<PlaneId> maybe_plane_id =
1524 plane_manager_->GetPlaneId(ar_plane);
1525 if (maybe_plane_id) {
1526 plane_id = maybe_plane_id->GetUnsafeValue();
1527 }
1528 }
1529
1530 mojom::XRHitResultPtr mojo_hit = mojom::XRHitResult::New();
1531
1532 mojo_hit->plane_id = plane_id;
1533
1534 {
1535 std::array<float, 7> raw_pose;
1536 ArPose_getPoseRaw(arcore_session_.get(), arcore_pose.get(),
1537 raw_pose.data());
1538
1539 gfx::Quaternion orientation(raw_pose[0], raw_pose[1], raw_pose[2],
1540 raw_pose[3]);
1541 gfx::Point3F position(raw_pose[4], raw_pose[5], raw_pose[6]);
1542
1543 mojo_hit->mojo_from_result = device::Pose(position, orientation);
1544
1545 DVLOG(3) << __func__
1546 << ": adding hit test result, position=" << position.ToString()
1547 << ", orientation=" << orientation.ToString()
1548 << ", plane_id=" << plane_id << " (0 means no plane)";
1549 }
1550
1551 // Insert new results at head to preserver order from ArCore
1552 hit_results->insert(hit_results->begin(), std::move(mojo_hit));
1553 }
1554
1555 DVLOG(2) << __func__ << ": hit_results->size()=" << hit_results->size();
1556 return true;
1557 }
1558
CreateAnchor(const mojom::XRNativeOriginInformation & native_origin_information,const device::Pose & native_origin_from_anchor,CreateAnchorCallback callback)1559 void ArCoreImpl::CreateAnchor(
1560 const mojom::XRNativeOriginInformation& native_origin_information,
1561 const device::Pose& native_origin_from_anchor,
1562 CreateAnchorCallback callback) {
1563 DVLOG(2) << __func__ << ": native_origin_information.which()="
1564 << static_cast<uint32_t>(native_origin_information.which())
1565 << ", native_origin_from_anchor.position()="
1566 << native_origin_from_anchor.position().ToString()
1567 << ", native_origin_from_anchor.orientation()="
1568 << native_origin_from_anchor.orientation().ToString();
1569
1570 create_anchor_requests_.emplace_back(native_origin_information,
1571 native_origin_from_anchor.ToTransform(),
1572 std::move(callback));
1573 }
1574
CreatePlaneAttachedAnchor(const mojom::XRNativeOriginInformation & native_origin_information,const device::Pose & native_origin_from_anchor,uint64_t plane_id,CreateAnchorCallback callback)1575 void ArCoreImpl::CreatePlaneAttachedAnchor(
1576 const mojom::XRNativeOriginInformation& native_origin_information,
1577 const device::Pose& native_origin_from_anchor,
1578 uint64_t plane_id,
1579 CreateAnchorCallback callback) {
1580 DVLOG(2) << __func__ << ": native_origin_information.which()="
1581 << static_cast<uint32_t>(native_origin_information.which())
1582 << ", plane_id=" << plane_id
1583 << ", native_origin_from_anchor.position()="
1584 << native_origin_from_anchor.position().ToString()
1585 << ", native_origin_from_anchor.orientation()="
1586 << native_origin_from_anchor.orientation().ToString();
1587
1588 create_plane_attached_anchor_requests_.emplace_back(
1589 native_origin_information, native_origin_from_anchor.ToTransform(),
1590 plane_id, std::move(callback));
1591 }
1592
ProcessAnchorCreationRequests(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state,const base::TimeTicks & frame_time)1593 void ArCoreImpl::ProcessAnchorCreationRequests(
1594 const gfx::Transform& mojo_from_viewer,
1595 const std::vector<mojom::XRInputSourceStatePtr>& input_state,
1596 const base::TimeTicks& frame_time) {
1597 DVLOG(2) << __func__ << ": Processing free-floating anchor creation requests";
1598 ProcessAnchorCreationRequestsHelper(
1599 mojo_from_viewer, input_state, &create_anchor_requests_, frame_time,
1600 [this](const CreateAnchorRequest& create_anchor_request,
1601 const gfx::Point3F& position, const gfx::Quaternion& orientation) {
1602 return anchor_manager_->CreateAnchor(
1603 device::mojom::Pose(orientation, position));
1604 });
1605
1606 DVLOG(2) << __func__
1607 << ": Processing plane-attached anchor creation requests";
1608 ProcessAnchorCreationRequestsHelper(
1609 mojo_from_viewer, input_state, &create_plane_attached_anchor_requests_,
1610 frame_time,
1611 [this](const CreatePlaneAttachedAnchorRequest& create_anchor_request,
1612 const gfx::Point3F& position, const gfx::Quaternion& orientation) {
1613 PlaneId plane_id = PlaneId(create_anchor_request.GetPlaneId());
1614 return anchor_manager_->CreateAnchor(
1615 plane_manager_.get(), device::mojom::Pose(orientation, position),
1616 plane_id);
1617 });
1618 }
1619
1620 template <typename T, typename FunctionType>
ProcessAnchorCreationRequestsHelper(const gfx::Transform & mojo_from_viewer,const std::vector<mojom::XRInputSourceStatePtr> & input_state,std::vector<T> * anchor_creation_requests,const base::TimeTicks & frame_time,FunctionType && create_anchor_function)1621 void ArCoreImpl::ProcessAnchorCreationRequestsHelper(
1622 const gfx::Transform& mojo_from_viewer,
1623 const std::vector<mojom::XRInputSourceStatePtr>& input_state,
1624 std::vector<T>* anchor_creation_requests,
1625 const base::TimeTicks& frame_time,
1626 FunctionType&& create_anchor_function) {
1627 DCHECK(anchor_creation_requests);
1628
1629 DVLOG(3) << __func__ << ": pre-call anchor_creation_requests->size()="
1630 << anchor_creation_requests->size();
1631
1632 // If we are unable to create an anchor because position of the native origin
1633 // is unknown, keep deferring it. On the other hand, if the anchor creation
1634 // failed in ARCore SDK, notify blink - we are ensuring that anchor creation
1635 // requests are processed when ARCore is in correct state so any failures
1636 // coming from ARCore SDK are real failures we won't be able to recover from.
1637 std::vector<T> postponed_requests;
1638 for (auto& create_anchor : *anchor_creation_requests) {
1639 auto anchor_creation_age = frame_time - create_anchor.GetRequestStartTime();
1640
1641 if (anchor_creation_age > kOutdatedAnchorCreationRequestThreshold) {
1642 DVLOG(3)
1643 << __func__
1644 << ": failing outdated anchor creation request, anchor_creation_age="
1645 << anchor_creation_age;
1646 create_anchor.TakeCallback().Run(
1647 device::mojom::CreateAnchorResult::FAILURE, 0);
1648 continue;
1649 }
1650
1651 mojom::XRNativeOriginInformation native_origin_information =
1652 create_anchor.GetNativeOriginInformation();
1653
1654 if (!NativeOriginExists(native_origin_information, input_state)) {
1655 DVLOG(3) << __func__
1656 << ": failing anchor creation request, native origin does not "
1657 "exist";
1658 create_anchor.TakeCallback().Run(
1659 device::mojom::CreateAnchorResult::FAILURE, 0);
1660 continue;
1661 }
1662
1663 base::Optional<gfx::Transform> maybe_mojo_from_native_origin =
1664 GetMojoFromNativeOrigin(native_origin_information, mojo_from_viewer,
1665 input_state);
1666
1667 if (!maybe_mojo_from_native_origin) {
1668 // We don't know where the native origin currently is (but we know it is
1669 // still tracked), so let's postpone the request & try again later.
1670 DVLOG(3) << __func__
1671 << ": postponing anchor creation request, native origin is not "
1672 "currently localizable";
1673 postponed_requests.emplace_back(std::move(create_anchor));
1674 continue;
1675 }
1676
1677 base::Optional<device::Pose> mojo_from_anchor =
1678 device::Pose::Create(*maybe_mojo_from_native_origin *
1679 create_anchor.GetNativeOriginFromAnchor());
1680
1681 if (!mojo_from_anchor) {
1682 // Fail the call now, failure to decompose is unlikely to resolve itself.
1683 DVLOG(3)
1684 << __func__
1685 << ": failing anchor creation request, unable to decompose a matrix";
1686 create_anchor.TakeCallback().Run(
1687 device::mojom::CreateAnchorResult::FAILURE, 0);
1688 continue;
1689 }
1690
1691 base::Optional<AnchorId> maybe_anchor_id = std::forward<FunctionType>(
1692 create_anchor_function)(create_anchor, mojo_from_anchor->position(),
1693 mojo_from_anchor->orientation());
1694
1695 if (!maybe_anchor_id) {
1696 // Fail the call now, failure to create anchor in ARCore SDK is unlikely
1697 // to resolve itself.
1698 DVLOG(3) << __func__
1699 << ": failing anchor creation request, anchor creation "
1700 "function did not return an anchor id";
1701 create_anchor.TakeCallback().Run(
1702 device::mojom::CreateAnchorResult::FAILURE, 0);
1703 continue;
1704 }
1705
1706 DVLOG(3) << __func__ << ": anchor creation request succeeded, time taken: "
1707 << anchor_creation_age;
1708 create_anchor.TakeCallback().Run(device::mojom::CreateAnchorResult::SUCCESS,
1709 maybe_anchor_id->GetUnsafeValue());
1710 }
1711
1712 // Return the postponed requests - all other requests should have their
1713 // status already reported to blink at this point:
1714 anchor_creation_requests->swap(postponed_requests);
1715 DVLOG(3) << __func__ << ": post-call anchor_creation_requests->size()="
1716 << anchor_creation_requests->size();
1717 }
1718
DetachAnchor(uint64_t anchor_id)1719 void ArCoreImpl::DetachAnchor(uint64_t anchor_id) {
1720 anchor_manager_->DetachAnchor(AnchorId(anchor_id));
1721 }
1722
GetDepthData()1723 mojom::XRDepthDataPtr ArCoreImpl::GetDepthData() {
1724 DVLOG(3) << __func__;
1725
1726 internal::ScopedArCoreObject<ArImage*> ar_image;
1727 ArStatus status = ArFrame_acquireDepthImage(
1728 arcore_session_.get(), arcore_frame_.get(),
1729 internal::ScopedArCoreObject<ArImage*>::Receiver(ar_image).get());
1730
1731 if (status != AR_SUCCESS) {
1732 DVLOG(2) << __func__
1733 << ": ArFrame_acquireDepthImage failed, status=" << status;
1734 return nullptr;
1735 }
1736
1737 int64_t timestamp_ns;
1738 ArImage_getTimestamp(arcore_session_.get(), ar_image.get(), ×tamp_ns);
1739 base::TimeDelta time_delta = base::TimeDelta::FromNanoseconds(timestamp_ns);
1740 DVLOG(3) << __func__ << ": depth image time_delta=" << time_delta;
1741
1742 // The image returned from ArFrame_acquireDepthImage() is documented to have
1743 // a single 16-bit plane at index 0. The ArImage format is documented to be
1744 // AR_IMAGE_FORMAT_DEPTH16 (equivalent to ImageFormat.DEPTH16). There should
1745 // be no need to validate this in non-debug builds.
1746 // https://developers.google.com/ar/reference/c/group/ar-frame#arframe_acquiredepthimage
1747 // https://developer.android.com/reference/android/graphics/ImageFormat#DEPTH16
1748
1749 ArImageFormat image_format;
1750 ArImage_getFormat(arcore_session_.get(), ar_image.get(), &image_format);
1751
1752 CHECK_EQ(image_format, AR_IMAGE_FORMAT_DEPTH16)
1753 << "Depth image format must be AR_IMAGE_FORMAT_DEPTH16, found: "
1754 << image_format;
1755
1756 int32_t num_planes;
1757 ArImage_getNumberOfPlanes(arcore_session_.get(), ar_image.get(), &num_planes);
1758
1759 CHECK_EQ(num_planes, 1) << "Depth image must have 1 plane, found: "
1760 << num_planes;
1761
1762 if (time_delta > previous_depth_data_time_) {
1763 mojom::XRDepthDataUpdatedPtr result = mojom::XRDepthDataUpdated::New();
1764
1765 result->time_delta = time_delta;
1766
1767 int32_t width = 0, height = 0;
1768 ArImage_getWidth(arcore_session_.get(), ar_image.get(), &width);
1769 ArImage_getHeight(arcore_session_.get(), ar_image.get(), &height);
1770
1771 DVLOG(3) << __func__ << ": depth image dimensions=" << width << "x"
1772 << height;
1773
1774 // Depth image is defined as a width by height array of 2-byte elements:
1775 auto checked_buffer_size = base::CheckMul<size_t>(2, width, height);
1776
1777 size_t buffer_size;
1778 if (!checked_buffer_size.AssignIfValid(&buffer_size)) {
1779 DVLOG(2) << __func__
1780 << ": overflow in 2 * width * height expression, returning null "
1781 "depth data";
1782 return nullptr;
1783 }
1784
1785 mojo_base::BigBuffer pixels(buffer_size);
1786
1787 // Interpret BigBuffer's data as a width by height array of uint16_t's and
1788 // copy image data into it:
1789 CopyArCoreImage(
1790 arcore_session_.get(), ar_image.get(), 0,
1791 base::span<uint16_t>(reinterpret_cast<uint16_t*>(pixels.data()),
1792 pixels.size() / 2),
1793 width, height);
1794
1795 result->pixel_data = std::move(pixels);
1796 // Transform needed to consume the data:
1797 result->norm_texture_from_norm_view = GetCameraUvFromScreenUvTransform();
1798 result->size = gfx::Size(width, height);
1799
1800 DVLOG(3) << __func__ << ": norm_texture_from_norm_view=\n"
1801 << result->norm_texture_from_norm_view.ToString();
1802
1803 previous_depth_data_time_ = time_delta;
1804
1805 return mojom::XRDepthData::NewUpdatedDepthData(std::move(result));
1806 }
1807
1808 return mojom::XRDepthData::NewDataStillValid(
1809 mojom::XRDepthDataStillValid::New());
1810 }
1811
IsOnGlThread() const1812 bool ArCoreImpl::IsOnGlThread() const {
1813 return gl_thread_task_runner_->BelongsToCurrentThread();
1814 }
1815
Create()1816 std::unique_ptr<ArCore> ArCoreImplFactory::Create() {
1817 return std::make_unique<ArCoreImpl>();
1818 }
1819
CreateAnchorRequest(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & native_origin_from_anchor,ArCore::CreateAnchorCallback callback)1820 CreateAnchorRequest::CreateAnchorRequest(
1821 const mojom::XRNativeOriginInformation& native_origin_information,
1822 const gfx::Transform& native_origin_from_anchor,
1823 ArCore::CreateAnchorCallback callback)
1824 : native_origin_information_(native_origin_information),
1825 native_origin_from_anchor_(native_origin_from_anchor),
1826 request_start_time_(base::TimeTicks::Now()),
1827 callback_(std::move(callback)) {}
1828 CreateAnchorRequest::CreateAnchorRequest(CreateAnchorRequest&& other) = default;
1829 CreateAnchorRequest::~CreateAnchorRequest() = default;
1830
1831 mojom::XRNativeOriginInformation
GetNativeOriginInformation() const1832 CreateAnchorRequest::GetNativeOriginInformation() const {
1833 return native_origin_information_;
1834 }
1835
GetNativeOriginFromAnchor() const1836 gfx::Transform CreateAnchorRequest::GetNativeOriginFromAnchor() const {
1837 return native_origin_from_anchor_;
1838 }
1839
GetRequestStartTime() const1840 base::TimeTicks CreateAnchorRequest::GetRequestStartTime() const {
1841 return request_start_time_;
1842 }
1843
TakeCallback()1844 ArCore::CreateAnchorCallback CreateAnchorRequest::TakeCallback() {
1845 return std::move(callback_);
1846 }
1847
CreatePlaneAttachedAnchorRequest(const mojom::XRNativeOriginInformation & native_origin_information,const gfx::Transform & native_origin_from_anchor,uint64_t plane_id,ArCore::CreateAnchorCallback callback)1848 CreatePlaneAttachedAnchorRequest::CreatePlaneAttachedAnchorRequest(
1849 const mojom::XRNativeOriginInformation& native_origin_information,
1850 const gfx::Transform& native_origin_from_anchor,
1851 uint64_t plane_id,
1852 ArCore::CreateAnchorCallback callback)
1853 : native_origin_information_(native_origin_information),
1854 native_origin_from_anchor_(native_origin_from_anchor),
1855 plane_id_(plane_id),
1856 request_start_time_(base::TimeTicks::Now()),
1857 callback_(std::move(callback)) {}
1858 CreatePlaneAttachedAnchorRequest::CreatePlaneAttachedAnchorRequest(
1859 CreatePlaneAttachedAnchorRequest&& other) = default;
1860 CreatePlaneAttachedAnchorRequest::~CreatePlaneAttachedAnchorRequest() = default;
1861
1862 mojom::XRNativeOriginInformation
GetNativeOriginInformation() const1863 CreatePlaneAttachedAnchorRequest::GetNativeOriginInformation() const {
1864 return native_origin_information_;
1865 }
1866
GetPlaneId() const1867 uint64_t CreatePlaneAttachedAnchorRequest::GetPlaneId() const {
1868 return plane_id_;
1869 }
1870
GetNativeOriginFromAnchor() const1871 gfx::Transform CreatePlaneAttachedAnchorRequest::GetNativeOriginFromAnchor()
1872 const {
1873 return native_origin_from_anchor_;
1874 }
1875
GetRequestStartTime() const1876 base::TimeTicks CreatePlaneAttachedAnchorRequest::GetRequestStartTime() const {
1877 return request_start_time_;
1878 }
1879
TakeCallback()1880 ArCore::CreateAnchorCallback CreatePlaneAttachedAnchorRequest::TakeCallback() {
1881 return std::move(callback_);
1882 }
1883
1884 } // namespace device
1885