lib/iomgr/lockfree_event.cc

/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <grpc/support/port_platform.h>

#include "src/core/lib/iomgr/lockfree_event.h"

#include <grpc/support/log.h>

#include "src/core/lib/debug/trace.h"
#include "src/core/lib/iomgr/exec_ctx.h"

extern grpc_core::DebugOnlyTraceFlag grpc_polling_trace;

/* 'state' holds the to call when the fd is readable or writable respectively.
   It can contain one of the following values:
     kClosureReady     : The fd has an I/O event of interest but there is no
                         closure yet to execute

     kClosureNotReady : The fd has no I/O event of interest

     closure ptr       : The closure to be executed when the fd has an I/O
                         event of interest

     shutdown_error | kShutdownBit :
                        'shutdown_error' field ORed with kShutdownBit.
                         This indicates that the fd is shutdown. Since all
                         memory allocations are word-aligned, the lower two
                         bits of the shutdown_error pointer are always 0. So
                         it is safe to OR these with kShutdownBit

   Valid state transitions:

     <closure ptr> <-----3------ kClosureNotReady -----1------->  kClosureReady
       |  |                         ^   |    ^                         |  |
       |  |                         |   |    |                         |  |
       |  +--------------4----------+   6    +---------2---------------+  |
       |                                |                                 |
       |                                v                                 |
       +-----5------->  [shutdown_error | kShutdownBit] <-------7---------+

    For 1, 4 : See SetReady() function
    For 2, 3 : See NotifyOn() function
    For 5,6,7: See SetShutdown() function */

namespace grpc_core {

LockfreeEvent::LockfreeEvent() { InitEvent(); }

void LockfreeEvent::InitEvent() {
  /* Perform an atomic store to start the state machine.

     Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
     state, while a file descriptor is on a freelist. In such a state it may
     be SetReady'd, and so we need to perform an atomic operation here to
     ensure no races */
  gpr_atm_no_barrier_store(&state_, kClosureNotReady);
}

void LockfreeEvent::DestroyEvent() {
  gpr_atm curr;
  do {
    curr = gpr_atm_no_barrier_load(&state_);
    if (curr & kShutdownBit) {
#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
      internal::StatusFreeHeapPtr(curr & ~kShutdownBit);
#else
      GRPC_ERROR_UNREF((grpc_error_handle)(curr & ~kShutdownBit));
#endif
    } else {
      GPR_ASSERT(curr == kClosureNotReady || curr == kClosureReady);
    }
    /* we CAS in a shutdown, no error value here. If this event is interacted
       with post-deletion (see the note in the constructor) we want the bit
       pattern to prevent error retention in a deleted object */
  } while (!gpr_atm_no_barrier_cas(&state_, curr,
                                   kShutdownBit /* shutdown, no error */));
}

void LockfreeEvent::NotifyOn(grpc_closure* closure) {
  while (true) {
    /* This load needs to be an acquire load because this can be a shutdown
     * error that we might need to reference. Adding acquire semantics makes
     * sure that the shutdown error has been initialized properly before us
     * referencing it. */
    gpr_atm curr = gpr_atm_acq_load(&state_);
    if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
      gpr_log(GPR_DEBUG,
              "LockfreeEvent::NotifyOn: %p curr=%" PRIxPTR " closure=%p", this,
              curr, closure);
    }
    switch (curr) {
      case kClosureNotReady: {
        /* kClosureNotReady -> <closure>.

           We're guaranteed by API that there's an acquire barrier before here,
           so there's no need to double-dip and this can be a release-only.

           The release itself pairs with the acquire half of a set_ready full
           barrier. */
        if (gpr_atm_rel_cas(&state_, kClosureNotReady,
                            reinterpret_cast<gpr_atm>(closure))) {
          return; /* Successful. Return */
        }

        break; /* retry */
      }

      case kClosureReady: {
        /* Change the state to kClosureNotReady. Schedule the closure if
           successful. If not, the state most likely transitioned to shutdown.
           We should retry.

           This can be a no-barrier cas since the state is being transitioned to
           kClosureNotReady; set_ready and set_shutdown do not schedule any
           closure when transitioning out of CLOSURE_NO_READY state (i.e there
           is no other code that needs to 'happen-after' this) */
        if (gpr_atm_no_barrier_cas(&state_, kClosureReady, kClosureNotReady)) {
          ExecCtx::Run(DEBUG_LOCATION, closure, GRPC_ERROR_NONE);
          return; /* Successful. Return */
        }

        break; /* retry */
      }

      default: {
        /* 'curr' is either a closure or the fd is shutdown(in which case 'curr'
           contains a pointer to the shutdown-error). If the fd is shutdown,
           schedule the closure with the shutdown error */
        if ((curr & kShutdownBit) > 0) {
#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
          grpc_error_handle shutdown_err =
              internal::StatusGetFromHeapPtr(curr & ~kShutdownBit);
#else
          grpc_error_handle shutdown_err =
              reinterpret_cast<grpc_error_handle>(curr & ~kShutdownBit);
#endif
          ExecCtx::Run(DEBUG_LOCATION, closure,
                       GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                           "FD Shutdown", &shutdown_err, 1));
          return;
        }

        /* There is already a closure!. This indicates a bug in the code */
        gpr_log(GPR_ERROR,
                "LockfreeEvent::NotifyOn: notify_on called with a previous "
                "callback still pending");
        abort();
      }
    }
  }

  GPR_UNREACHABLE_CODE(return );
}

bool LockfreeEvent::SetShutdown(grpc_error_handle shutdown_error) {
#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
  intptr_t status_ptr = internal::StatusAllocHeapPtr(shutdown_error);
  gpr_atm new_state = status_ptr | kShutdownBit;
#else
  gpr_atm new_state = reinterpret_cast<gpr_atm>(shutdown_error) | kShutdownBit;
#endif

  while (true) {
    gpr_atm curr = gpr_atm_no_barrier_load(&state_);
    if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
      gpr_log(GPR_DEBUG,
              "LockfreeEvent::SetShutdown: %p curr=%" PRIxPTR " err=%s",
              &state_, curr, grpc_error_std_string(shutdown_error).c_str());
    }
    switch (curr) {
      case kClosureReady:
      case kClosureNotReady:
        /* Need a full barrier here so that the initial load in notify_on
           doesn't need a barrier */
        if (gpr_atm_full_cas(&state_, curr, new_state)) {
          return true; /* early out */
        }
        break; /* retry */

      default: {
        /* 'curr' is either a closure or the fd is already shutdown */

        /* If fd is already shutdown, we are done */
        if ((curr & kShutdownBit) > 0) {
#ifdef GRPC_ERROR_IS_ABSEIL_STATUS
          internal::StatusFreeHeapPtr(status_ptr);
#else
          GRPC_ERROR_UNREF(shutdown_error);
#endif
          return false;
        }

        /* Fd is not shutdown. Schedule the closure and move the state to
           shutdown state.
           Needs an acquire to pair with setting the closure (and get a
           happens-after on that edge), and a release to pair with anything
           loading the shutdown state. */
        if (gpr_atm_full_cas(&state_, curr, new_state)) {
          ExecCtx::Run(DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
                       GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                           "FD Shutdown", &shutdown_error, 1));
          return true;
        }

        /* 'curr' was a closure but now changed to a different state. We will
          have to retry */
        break;
      }
    }
  }

  GPR_UNREACHABLE_CODE(return false);
}

void LockfreeEvent::SetReady() {
  while (true) {
    gpr_atm curr = gpr_atm_no_barrier_load(&state_);

    if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
      gpr_log(GPR_DEBUG, "LockfreeEvent::SetReady: %p curr=%" PRIxPTR, &state_,
              curr);
    }

    switch (curr) {
      case kClosureReady: {
        /* Already ready. We are done here */
        return;
      }

      case kClosureNotReady: {
        /* No barrier required as we're transitioning to a state that does not
           involve a closure */
        if (gpr_atm_no_barrier_cas(&state_, kClosureNotReady, kClosureReady)) {
          return; /* early out */
        }
        break; /* retry */
      }

      default: {
        /* 'curr' is either a closure or the fd is shutdown */
        if ((curr & kShutdownBit) > 0) {
          /* The fd is shutdown. Do nothing */
          return;
        }
        /* Full cas: acquire pairs with this cas' release in the event of a
           spurious set_ready; release pairs with this or the acquire in
           notify_on (or set_shutdown) */
        else if (gpr_atm_full_cas(&state_, curr, kClosureNotReady)) {
          ExecCtx::Run(DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
                       GRPC_ERROR_NONE);
          return;
        }
        /* else the state changed again (only possible by either a racing
           set_ready or set_shutdown functions. In both these cases, the closure
           would have been scheduled for execution. So we are done here */
        return;
      }
    }
  }
}

}  // namespace grpc_core