vulkan/runtime/vk_sync.c

/*
 * Copyright © 2021 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "vk_sync.h"

#include <assert.h>
#include <string.h>

#include "util/debug.h"
#include "util/macros.h"
#include "util/os_time.h"

#include "vk_alloc.h"
#include "vk_device.h"
#include "vk_log.h"

static void
vk_sync_type_validate(const struct vk_sync_type *type)
{
   assert(type->init);
   assert(type->finish);

   assert(type->features & (VK_SYNC_FEATURE_BINARY |
                            VK_SYNC_FEATURE_TIMELINE));

   if (type->features & VK_SYNC_FEATURE_TIMELINE) {
      assert(type->features & VK_SYNC_FEATURE_GPU_WAIT);
      assert(type->features & VK_SYNC_FEATURE_CPU_WAIT);
      assert(type->features & VK_SYNC_FEATURE_CPU_SIGNAL);
      assert(type->features & (VK_SYNC_FEATURE_WAIT_BEFORE_SIGNAL |
                               VK_SYNC_FEATURE_WAIT_PENDING));
      assert(type->signal);
      assert(type->get_value);
   }

   if (!(type->features & VK_SYNC_FEATURE_BINARY)) {
      assert(!(type->features & (VK_SYNC_FEATURE_GPU_MULTI_WAIT |
                                 VK_SYNC_FEATURE_CPU_RESET)));
      assert(!type->import_sync_file);
      assert(!type->export_sync_file);
   }

   if (type->features & VK_SYNC_FEATURE_CPU_WAIT) {
      assert(type->wait || type->wait_many);
   } else {
      assert(!(type->features & (VK_SYNC_FEATURE_WAIT_ANY |
                                 VK_SYNC_FEATURE_WAIT_PENDING)));
   }

   if (type->features & VK_SYNC_FEATURE_GPU_MULTI_WAIT)
      assert(type->features & VK_SYNC_FEATURE_GPU_WAIT);

   if (type->features & VK_SYNC_FEATURE_CPU_RESET)
      assert(type->reset);

   if (type->features & VK_SYNC_FEATURE_CPU_SIGNAL)
      assert(type->signal);
}

VkResult
vk_sync_init(struct vk_device *device,
             struct vk_sync *sync,
             const struct vk_sync_type *type,
             enum vk_sync_flags flags,
             uint64_t initial_value)
{
   vk_sync_type_validate(type);

   if (flags & VK_SYNC_IS_TIMELINE)
      assert(type->features & VK_SYNC_FEATURE_TIMELINE);
   else
      assert(type->features & VK_SYNC_FEATURE_BINARY);

   assert(type->size >= sizeof(*sync));
   memset(sync, 0, type->size);
   sync->type = type;
   sync->flags = flags;

   return type->init(device, sync, initial_value);
}

void
vk_sync_finish(struct vk_device *device,
               struct vk_sync *sync)
{
   sync->type->finish(device, sync);
}

VkResult
vk_sync_create(struct vk_device *device,
               const struct vk_sync_type *type,
               enum vk_sync_flags flags,
               uint64_t initial_value,
               struct vk_sync **sync_out)
{
   struct vk_sync *sync;

   sync = vk_alloc(&device->alloc, type->size, 8,
                   VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
   if (sync == NULL)
      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

   VkResult result = vk_sync_init(device, sync, type, flags, initial_value);
   if (result != VK_SUCCESS) {
      vk_free(&device->alloc, sync);
      return result;
   }

   *sync_out = sync;

   return VK_SUCCESS;
}

void
vk_sync_destroy(struct vk_device *device,
                struct vk_sync *sync)
{
   vk_sync_finish(device, sync);
   vk_free(&device->alloc, sync);
}

VkResult
vk_sync_signal(struct vk_device *device,
               struct vk_sync *sync,
               uint64_t value)
{
   assert(sync->type->features & VK_SYNC_FEATURE_CPU_SIGNAL);

   if (sync->flags & VK_SYNC_IS_TIMELINE)
      assert(value > 0);
   else
      assert(value == 0);

   return sync->type->signal(device, sync, value);
}

VkResult
vk_sync_get_value(struct vk_device *device,
                  struct vk_sync *sync,
                  uint64_t *value)
{
   assert(sync->flags & VK_SYNC_IS_TIMELINE);
   return sync->type->get_value(device, sync, value);
}

VkResult
vk_sync_reset(struct vk_device *device,
              struct vk_sync *sync)
{
   assert(sync->type->features & VK_SYNC_FEATURE_CPU_RESET);
   assert(!(sync->flags & VK_SYNC_IS_TIMELINE));
   return sync->type->reset(device, sync);
}

VkResult vk_sync_move(struct vk_device *device,
                      struct vk_sync *dst,
                      struct vk_sync *src)
{
   assert(!(dst->flags & VK_SYNC_IS_TIMELINE));
   assert(!(src->flags & VK_SYNC_IS_TIMELINE));
   assert(dst->type == src->type);

   return src->type->move(device, dst, src);
}

static void
assert_valid_wait(struct vk_sync *sync,
                  uint64_t wait_value,
                  enum vk_sync_wait_flags wait_flags)
{
   assert(sync->type->features & VK_SYNC_FEATURE_CPU_WAIT);

   if (!(sync->flags & VK_SYNC_IS_TIMELINE))
      assert(wait_value == 0);

   if (wait_flags & VK_SYNC_WAIT_PENDING)
      assert(sync->type->features & VK_SYNC_FEATURE_WAIT_PENDING);
}

static uint64_t
get_max_abs_timeout_ns(void)
{
   static int max_timeout_ms = -1;
   if (max_timeout_ms < 0)
      max_timeout_ms = env_var_as_unsigned("MESA_VK_MAX_TIMEOUT", 0);

   if (max_timeout_ms == 0)
      return UINT64_MAX;
   else
      return os_time_get_absolute_timeout(max_timeout_ms * 1000000ull);
}

static VkResult
__vk_sync_wait(struct vk_device *device,
               struct vk_sync *sync,
               uint64_t wait_value,
               enum vk_sync_wait_flags wait_flags,
               uint64_t abs_timeout_ns)
{
   assert_valid_wait(sync, wait_value, wait_flags);

   /* This doesn't make sense for a single wait */
   assert(!(wait_flags & VK_SYNC_WAIT_ANY));

   if (sync->type->wait) {
      return sync->type->wait(device, sync, wait_value,
                              wait_flags, abs_timeout_ns);
   } else {
      struct vk_sync_wait wait = {
         .sync = sync,
         .stage_mask = ~(VkPipelineStageFlags2KHR)0,
         .wait_value = wait_value,
      };
      return sync->type->wait_many(device, 1, &wait, wait_flags,
                                   abs_timeout_ns);
   }
}

VkResult
vk_sync_wait(struct vk_device *device,
             struct vk_sync *sync,
             uint64_t wait_value,
             enum vk_sync_wait_flags wait_flags,
             uint64_t abs_timeout_ns)
{
   uint64_t max_abs_timeout_ns = get_max_abs_timeout_ns();
   if (abs_timeout_ns > max_abs_timeout_ns) {
      VkResult result =
         __vk_sync_wait(device, sync, wait_value, wait_flags,
                        max_abs_timeout_ns);
      if (unlikely(result == VK_TIMEOUT))
         return vk_device_set_lost(device, "Maximum timeout exceeded!");
      return result;
   } else {
      return __vk_sync_wait(device, sync, wait_value, wait_flags,
                            abs_timeout_ns);
   }
}

static bool
can_wait_many(uint32_t wait_count,
              const struct vk_sync_wait *waits,
              enum vk_sync_wait_flags wait_flags)
{
   if (waits[0].sync->type->wait_many == NULL)
      return false;

   if ((wait_flags & VK_SYNC_WAIT_ANY) &&
       !(waits[0].sync->type->features & VK_SYNC_FEATURE_WAIT_ANY))
      return false;

   for (uint32_t i = 0; i < wait_count; i++) {
      assert_valid_wait(waits[i].sync, waits[i].wait_value, wait_flags);
      if (waits[i].sync->type != waits[0].sync->type)
         return false;
   }

   return true;
}

static VkResult
__vk_sync_wait_many(struct vk_device *device,
                    uint32_t wait_count,
                    const struct vk_sync_wait *waits,
                    enum vk_sync_wait_flags wait_flags,
                    uint64_t abs_timeout_ns)
{
   if (wait_count == 0)
      return VK_SUCCESS;

   if (wait_count == 1) {
      return __vk_sync_wait(device, waits[0].sync, waits[0].wait_value,
                            wait_flags & ~VK_SYNC_WAIT_ANY, abs_timeout_ns);
   }

   if (can_wait_many(wait_count, waits, wait_flags)) {
      return waits[0].sync->type->wait_many(device, wait_count, waits,
                                            wait_flags, abs_timeout_ns);
   } else if (wait_flags & VK_SYNC_WAIT_ANY) {
      /* If we have multiple syncs and they don't support wait_any or they're
       * not all the same type, there's nothing better we can do than spin.
       */
      do {
         for (uint32_t i = 0; i < wait_count; i++) {
            VkResult result = __vk_sync_wait(device, waits[i].sync,
                                             waits[i].wait_value,
                                             wait_flags & ~VK_SYNC_WAIT_ANY,
                                             0 /* abs_timeout_ns */);
            if (result != VK_TIMEOUT)
               return result;
         }
      } while (os_time_get_nano() < abs_timeout_ns);

      return VK_TIMEOUT;
   } else {
      for (uint32_t i = 0; i < wait_count; i++) {
         VkResult result = __vk_sync_wait(device, waits[i].sync,
                                          waits[i].wait_value,
                                          wait_flags, abs_timeout_ns);
         if (result != VK_SUCCESS)
            return result;
      }
      return VK_SUCCESS;
   }
}

VkResult
vk_sync_wait_many(struct vk_device *device,
                  uint32_t wait_count,
                  const struct vk_sync_wait *waits,
                  enum vk_sync_wait_flags wait_flags,
                  uint64_t abs_timeout_ns)
{
   uint64_t max_abs_timeout_ns = get_max_abs_timeout_ns();
   if (abs_timeout_ns > max_abs_timeout_ns) {
      VkResult result =
         __vk_sync_wait_many(device, wait_count, waits, wait_flags,
                             max_abs_timeout_ns);
      if (unlikely(result == VK_TIMEOUT))
         return vk_device_set_lost(device, "Maximum timeout exceeded!");
      return result;
   } else {
      return __vk_sync_wait_many(device, wait_count, waits, wait_flags,
                                 abs_timeout_ns);
   }
}

VkResult
vk_sync_import_opaque_fd(struct vk_device *device,
                         struct vk_sync *sync,
                         int fd)
{
   VkResult result = sync->type->import_opaque_fd(device, sync, fd);
   if (unlikely(result != VK_SUCCESS))
      return result;

   sync->flags |= VK_SYNC_IS_SHAREABLE |
                  VK_SYNC_IS_SHARED;

   return VK_SUCCESS;
}

VkResult
vk_sync_export_opaque_fd(struct vk_device *device,
                         struct vk_sync *sync,
                         int *fd)
{
   assert(sync->flags & VK_SYNC_IS_SHAREABLE);

   VkResult result = sync->type->export_opaque_fd(device, sync, fd);
   if (unlikely(result != VK_SUCCESS))
      return result;

   sync->flags |= VK_SYNC_IS_SHARED;

   return VK_SUCCESS;
}

VkResult
vk_sync_import_sync_file(struct vk_device *device,
                         struct vk_sync *sync,
                         int sync_file)
{
   assert(!(sync->flags & VK_SYNC_IS_TIMELINE));

   /* Silently handle negative file descriptors in case the driver doesn't
    * want to bother.
    */
   if (sync_file < 0 && sync->type->signal)
      return sync->type->signal(device, sync, 0);

   return sync->type->import_sync_file(device, sync, sync_file);
}

VkResult
vk_sync_export_sync_file(struct vk_device *device,
                         struct vk_sync *sync,
                         int *sync_file)
{
   assert(!(sync->flags & VK_SYNC_IS_TIMELINE));
   return sync->type->export_sync_file(device, sync, sync_file);
}