// Copyright (C) 2019-2021 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <config.h>

#include <exceptions/exceptions.h>
#include <util/multi_threading_mgr.h>
#include <testutils/gtest_utils.h>

#include <gtest/gtest.h>

using namespace isc::util;
using namespace isc;

/// @brief Verifies that the default mode is false (MT disabled).
TEST(MultiThreadingMgrTest, defaultMode) {
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
}

/// @brief Verifies that the mode setter works.
TEST(MultiThreadingMgrTest, setMode) {
    // enable MT
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setMode(true));
    // MT should be enabled
    EXPECT_TRUE(MultiThreadingMgr::instance().getMode());
    // disable MT
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setMode(false));
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
}

/// @brief Verifies that accessing the thread pool works.
TEST(MultiThreadingMgrTest, threadPool) {
    // get the thread pool
    EXPECT_NO_THROW(MultiThreadingMgr::instance().getThreadPool());
}

/// @brief Verifies that the thread pool size setter works.
TEST(MultiThreadingMgrTest, threadPoolSize) {
    // default thread count is 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // set thread count to 16
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setThreadPoolSize(16));
    // thread count should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
    // set thread count to 0
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setThreadPoolSize(0));
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
}

/// @brief Verifies that the packet queue size setter works.
TEST(MultiThreadingMgrTest, packetQueueSize) {
    // default queue size is 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPool().getMaxQueueSize(), 0);
    // set queue size to 16
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setPacketQueueSize(16));
    // queue size should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 16);
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPool().getMaxQueueSize(), 16);
    // set queue size to 0
    EXPECT_NO_THROW(MultiThreadingMgr::instance().setPacketQueueSize(0));
    // queue size should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPool().getMaxQueueSize(), 0);
}

/// @brief Verifies that detecting thread count works.
TEST(MultiThreadingMgrTest, detectThreadCount) {
    // detecting thread count should work
    EXPECT_NE(MultiThreadingMgr::detectThreadCount(), 0);
}

/// @brief Verifies that apply settings works.
TEST(MultiThreadingMgrTest, applyConfig) {
    // get the thread pool
    auto& thread_pool = MultiThreadingMgr::instance().getThreadPool();
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
    // default thread count is 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
    // enable MT with 16 threads and queue size 256
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(true, 16, 256));
    // MT should be enabled
    EXPECT_TRUE(MultiThreadingMgr::instance().getMode());
    // thread count should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
    // queue size should be 256
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    // thread pool should be started
    EXPECT_EQ(thread_pool.size(), 16);
    // disable MT
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(false, 16, 256));
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // queue size should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
    // enable MT with auto scaling
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(true, 0, 0));
    // MT should be enabled
    EXPECT_TRUE(MultiThreadingMgr::instance().getMode());
    // thread count should be detected automatically
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), MultiThreadingMgr::detectThreadCount());
    // thread pool should be started
    EXPECT_EQ(thread_pool.size(), MultiThreadingMgr::detectThreadCount());
    // disable MT
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(false, 0, 0));
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
}

/// @brief Verifies that the critical section flag works.
TEST(MultiThreadingMgrTest, criticalSectionFlag) {
    // get the thread pool
    auto& thread_pool = MultiThreadingMgr::instance().getThreadPool();
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
    // critical section should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().isInCriticalSection());
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
    // exit critical section
    EXPECT_THROW(MultiThreadingMgr::instance().exitCriticalSection(), InvalidOperation);
    // critical section should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().isInCriticalSection());
    // enter critical section
    EXPECT_NO_THROW(MultiThreadingMgr::instance().enterCriticalSection());
    // critical section should be enabled
    EXPECT_TRUE(MultiThreadingMgr::instance().isInCriticalSection());
    // enable MT with 16 threads and queue size 256
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(true, 16, 256));
    // MT should be enabled
    EXPECT_TRUE(MultiThreadingMgr::instance().getMode());
    // thread count should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
    // queue size should be 256
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
    // exit critical section
    EXPECT_NO_THROW(MultiThreadingMgr::instance().exitCriticalSection());
    // critical section should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().isInCriticalSection());
    // exit critical section
    EXPECT_THROW(MultiThreadingMgr::instance().exitCriticalSection(), InvalidOperation);
    // critical section should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().isInCriticalSection());
    // disable MT
    EXPECT_NO_THROW(MultiThreadingMgr::instance().apply(false, 0, 0));
    // MT should be disabled
    EXPECT_FALSE(MultiThreadingMgr::instance().getMode());
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // queue size should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
}

/// @brief Verifies that the critical section works.
TEST(MultiThreadingMgrTest, criticalSection) {
    // get the thread pool instance
    auto& thread_pool = MultiThreadingMgr::instance().getThreadPool();
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);
    // apply multi-threading configuration with 16 threads and queue size 256
    MultiThreadingMgr::instance().apply(true, 16, 256);
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 16);
    // thread count should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
    // queue size should be 256
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    // use scope to test constructor and destructor
    {
        MultiThreadingCriticalSection cs;
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 16
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
        // queue size should be 256
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
        // use scope to test constructor and destructor
        {
            MultiThreadingCriticalSection inner_cs;
            // thread pool should be stopped
            EXPECT_EQ(thread_pool.size(), 0);
            // thread count should be 16
            EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
            // queue size should be 256
            EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
        }
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 16
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
        // queue size should be 256
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    }
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 16);
    // thread count should be 16
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
    // queue size should be 256
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    // use scope to test constructor and destructor
    {
        MultiThreadingCriticalSection cs;
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 16
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 16);
        // queue size should be 256
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
        // apply multi-threading configuration with 64 threads and queue size 4
        MultiThreadingMgr::instance().apply(true, 64, 4);
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 64
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 64);
        // queue size should be 4
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 4);
    }
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 64);
    // thread count should be 64
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 64);
    // queue size should be 4
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 4);
    // use scope to test constructor and destructor
    {
        MultiThreadingCriticalSection cs;
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 64
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 64);
        // queue size should be 4
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 4);
        // apply multi-threading configuration with 0 threads
        MultiThreadingMgr::instance().apply(false, 64, 256);
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
        // queue size should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    }
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 0);
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // queue size should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    // use scope to test constructor and destructor
    {
        MultiThreadingCriticalSection cs;
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
        // queue size should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
        // use scope to test constructor and destructor
        {
            MultiThreadingCriticalSection inner_cs;
            // thread pool should be stopped
            EXPECT_EQ(thread_pool.size(), 0);
            // thread count should be 0
            EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
            // queue size should be 0
            EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
        }
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
        // queue size should be 0
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    }
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 0);
    // thread count should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 0);
    // queue size should be 0
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 0);
    // use scope to test constructor and destructor
    {
        MultiThreadingCriticalSection cs;
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // apply multi-threading configuration with 64 threads
        MultiThreadingMgr::instance().apply(true, 64, 256);
        // thread pool should be stopped
        EXPECT_EQ(thread_pool.size(), 0);
        // thread count should be 64
        EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 64);
        // queue size should be 256
        EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    }
    // thread count should match
    EXPECT_EQ(thread_pool.size(), 64);
    // thread count should be 64
    EXPECT_EQ(MultiThreadingMgr::instance().getThreadPoolSize(), 64);
    // queue size should be 256
    EXPECT_EQ(MultiThreadingMgr::instance().getPacketQueueSize(), 256);
    // apply multi-threading configuration with 0 threads
    MultiThreadingMgr::instance().apply(false, 0, 0);
}

/// @brief Test fixture for exercised CriticalSection callbacks.
class CriticalSectionCallbackTest : public ::testing::Test {
public:
    /// @brief Constructor.
    CriticalSectionCallbackTest() {
        MultiThreadingMgr::instance().apply(false, 0, 0);
    }

    /// @brief Destructor.
    ~CriticalSectionCallbackTest() {
        MultiThreadingMgr::instance().apply(false, 0, 0);
    }

    /// @brief A callback that adds the value 1 to invocations lists.
    void one() {
        invocations_.push_back(1);
    }

    /// @brief A callback that adds the value 2 to invocations lists.
    void two() {
        invocations_.push_back(2);
    }

    /// @brief A callback that adds the value 3 to invocations lists.
    void three() {
        invocations_.push_back(3);
    }

    /// @brief A callback that adds the value 4 to invocations lists.
    void four() {
        invocations_.push_back(4);
    }

    /// @brief A callback that throws @ref isc::Exception which is ignored.
    void ignoredException() {
        isc_throw(isc::Exception, "ignored");
    }

    /// @brief A callback that throws @ref isc::MultiThreadingInvalidOperation
    /// which is propagated to the scope of the
    /// @ref MultiThreadingCriticalSection constructor.
    void observedException() {
        isc_throw(isc::MultiThreadingInvalidOperation, "observed");
    }

    /// @brief Indicates whether or not the DHCP thread pool is running.
    ///
    /// @return True if the pool is running, false otherwise.
    bool isThreadPoolRunning() {
        return (MultiThreadingMgr::instance().getThreadPool().size());
    }

    /// @brief Checks callback invocations over a series of nested
    /// CriticalSections.
    ///
    /// @param entries A vector of the invocation values that should
    /// be present after entry into the outermost CriticalSection.  The
    /// expected values should be in the order the callbacks were added
    /// to the MultiThreadingMgr's list of callbacks.
    /// @param exits A vector of the invocation values that should
    /// be present after exiting the outermost CriticalSection.  The
    /// expected values should be in the order the callbacks were added
    /// to the MultiThreadingMgr's list of callbacks.
    /// @param should_throw The flag indicating if the CriticalSection should
    /// throw, simulating a dead-lock scenario when a processing thread tries
    /// to stop the thread pool.
    void runCriticalSections(std::vector<int> entries, std::vector<int>exits,
                             bool should_throw = false) {
        // Pool must be running.
        ASSERT_TRUE(isThreadPoolRunning());

        // Clear the invocations list.
        invocations_.clear();

        // Use scope to create nested CriticalSections.
        if (!should_throw) {
            // Enter a critical section.
            MultiThreadingCriticalSection cs;

            // Thread pool should be stopped.
            ASSERT_FALSE(isThreadPoolRunning());

            if (entries.size()) {
                // We expect entry invocations.
                ASSERT_EQ(invocations_.size(), entries.size());
                ASSERT_EQ(invocations_, entries);
            } else {
                // We do not expect entry invocations.
                ASSERT_FALSE(invocations_.size());
            }

            // Clear the invocations list.
            invocations_.clear();

            {
                // Enter another CriticalSection.
                MultiThreadingCriticalSection inner_cs;

                // Thread pool should still be stopped.
                ASSERT_FALSE(isThreadPoolRunning());

                // We should not have had any callback invocations.
                ASSERT_FALSE(invocations_.size());
            }

            // After exiting inner section, the thread pool should
            // still be stopped.
            ASSERT_FALSE(isThreadPoolRunning());

            // We should not have had more callback invocations.
            ASSERT_FALSE(invocations_.size());
        } else {
            ASSERT_THROW(MultiThreadingCriticalSection cs, MultiThreadingInvalidOperation);

            if (entries.size()) {
                // We expect entry invocations.
                ASSERT_EQ(invocations_.size(), entries.size());
                ASSERT_EQ(invocations_, entries);
            } else {
                // We do not expect entry invocations.
                ASSERT_FALSE(invocations_.size());
            }

            // Clear the invocations list.
            invocations_.clear();
        }

        // After exiting the outer section, the thread pool should
        // match the thread count.
        ASSERT_TRUE(isThreadPoolRunning());

        if (exits.size()) {
            // We expect exit invocations.
            ASSERT_EQ(invocations_, exits);
        } else {
            // We do not expect exit invocations.
            ASSERT_FALSE(invocations_.size());
        }
    }

    /// @brief A list of values set by callback invocations.
    std::vector<int> invocations_;
};

/// @brief Verifies critical section callback maintenance:
/// catch invalid pairs, add pairs, remove pairs.
TEST_F(CriticalSectionCallbackTest, addAndRemove) {
    auto& mgr = MultiThreadingMgr::instance();

    // Cannot add with a blank name.
    ASSERT_THROW_MSG(mgr.addCriticalSectionCallbacks("", [](){}, [](){}, [](){}),
                     BadValue, "CSCallbackSetList - name cannot be empty");

    // Cannot add with an empty check callback.
    ASSERT_THROW_MSG(mgr.addCriticalSectionCallbacks("bad", nullptr, [](){}, [](){}),
                     BadValue, "CSCallbackSetList - check callback for bad cannot be empty");

    // Cannot add with an empty exit callback.
    ASSERT_THROW_MSG(mgr.addCriticalSectionCallbacks("bad", [](){}, nullptr, [](){}),
                     BadValue, "CSCallbackSetList - entry callback for bad cannot be empty");

    // Cannot add with an empty exit callback.
    ASSERT_THROW_MSG(mgr.addCriticalSectionCallbacks("bad", [](){}, [](){}, nullptr),
                     BadValue, "CSCallbackSetList - exit callback for bad cannot be empty");

    // Should be able to add foo.
    ASSERT_NO_THROW_LOG(mgr.addCriticalSectionCallbacks("foo", [](){}, [](){}, [](){}));

    // Should not be able to add foo twice.
    ASSERT_THROW_MSG(mgr.addCriticalSectionCallbacks("foo", [](){}, [](){}, [](){}),
                     BadValue, "CSCallbackSetList - callbacks for foo already exist");

    // Should be able to add bar.
    ASSERT_NO_THROW_LOG(mgr.addCriticalSectionCallbacks("bar", [](){}, [](){}, [](){}));

    // Should be able to remove foo.
    ASSERT_NO_THROW_LOG(mgr.removeCriticalSectionCallbacks("foo"));

    // Should be able to remove foo twice without issue.
    ASSERT_NO_THROW_LOG(mgr.removeCriticalSectionCallbacks("foo"));

    // Should be able to remove all without issue.
    ASSERT_NO_THROW_LOG(mgr.removeAllCriticalSectionCallbacks());
}

/// @brief Verifies that the critical section callbacks work.
TEST_F(CriticalSectionCallbackTest, invocations) {
    // get the thread pool instance
    auto& thread_pool = MultiThreadingMgr::instance().getThreadPool();
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);

    // Add two sets of CriticalSection call backs.
    MultiThreadingMgr::instance().addCriticalSectionCallbacks("oneAndTwo",
         std::bind(&CriticalSectionCallbackTest::ignoredException, this),
         std::bind(&CriticalSectionCallbackTest::one, this),
         std::bind(&CriticalSectionCallbackTest::two, this));

    MultiThreadingMgr::instance().addCriticalSectionCallbacks("threeAndFour",
         std::bind(&CriticalSectionCallbackTest::ignoredException, this),
         std::bind(&CriticalSectionCallbackTest::three, this),
         std::bind(&CriticalSectionCallbackTest::four, this));

    // Apply multi-threading configuration with 16 threads and queue size 256.
    MultiThreadingMgr::instance().apply(true, 16, 256);

    // Make three passes over nested CriticalSections to ensure
    // callbacks execute at the appropriate times and we can do
    // so repeatedly.
    for (int i = 0; i < 3; ++i) {
        runCriticalSections({1 ,3}, {4, 2});
    }

    // Now remove the first set of callbacks.
    MultiThreadingMgr::instance().removeCriticalSectionCallbacks("oneAndTwo");

    // Retest CriticalSections.
    runCriticalSections({3}, {4});

    // Now remove the remaining callbacks.
    MultiThreadingMgr::instance().removeAllCriticalSectionCallbacks();

    // Retest CriticalSections.
    runCriticalSections({}, {});
}

/// @brief Verifies that the critical section callbacks work.
TEST_F(CriticalSectionCallbackTest, invocationsWithExceptions) {
    // get the thread pool instance
    auto& thread_pool = MultiThreadingMgr::instance().getThreadPool();
    // thread pool should be stopped
    EXPECT_EQ(thread_pool.size(), 0);

    // Apply multi-threading configuration with 16 threads and queue size 256.
    MultiThreadingMgr::instance().apply(true, 16, 256);

    // Add two sets of CriticalSection call backs.
    MultiThreadingMgr::instance().addCriticalSectionCallbacks("observed",
         std::bind(&CriticalSectionCallbackTest::observedException, this),
         std::bind(&CriticalSectionCallbackTest::one, this),
         std::bind(&CriticalSectionCallbackTest::two, this));

    MultiThreadingMgr::instance().addCriticalSectionCallbacks("ignored",
         std::bind(&CriticalSectionCallbackTest::ignoredException, this),
         std::bind(&CriticalSectionCallbackTest::three, this),
         std::bind(&CriticalSectionCallbackTest::four, this));

    // Make three passes over nested CriticalSections to ensure
    // callbacks execute at the appropriate times and we can do
    // so repeatedly.
    for (int i = 0; i < 3; ++i) {
        runCriticalSections({}, {}, true);
    }

    // Now remove the first set of callbacks.
    MultiThreadingMgr::instance().removeCriticalSectionCallbacks("observed");

    // Retest CriticalSections.
    runCriticalSections({3}, {4});

    // Now remove the remaining callbacks.
    MultiThreadingMgr::instance().removeAllCriticalSectionCallbacks();

    // Retest CriticalSections.
    runCriticalSections({}, {});
}