1e974f91cSConrad Meyer /*- 2e974f91cSConrad Meyer * Copyright (C) 2012 Intel Corporation 3e974f91cSConrad Meyer * All rights reserved. 4e974f91cSConrad Meyer * 5e974f91cSConrad Meyer * Redistribution and use in source and binary forms, with or without 6e974f91cSConrad Meyer * modification, are permitted provided that the following conditions 7e974f91cSConrad Meyer * are met: 8e974f91cSConrad Meyer * 1. Redistributions of source code must retain the above copyright 9e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer. 10e974f91cSConrad Meyer * 2. Redistributions in binary form must reproduce the above copyright 11e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer in the 12e974f91cSConrad Meyer * documentation and/or other materials provided with the distribution. 13e974f91cSConrad Meyer * 14e974f91cSConrad Meyer * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e974f91cSConrad Meyer * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e974f91cSConrad Meyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e974f91cSConrad Meyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e974f91cSConrad Meyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e974f91cSConrad Meyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e974f91cSConrad Meyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e974f91cSConrad Meyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e974f91cSConrad Meyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e974f91cSConrad Meyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e974f91cSConrad Meyer * SUCH DAMAGE. 25e974f91cSConrad Meyer */ 26e974f91cSConrad Meyer 27e974f91cSConrad Meyer #include <sys/cdefs.h> 28e974f91cSConrad Meyer __FBSDID("$FreeBSD$"); 29e974f91cSConrad Meyer 30e974f91cSConrad Meyer #include <sys/param.h> 31e974f91cSConrad Meyer #include <sys/systm.h> 32e974f91cSConrad Meyer #include <sys/bus.h> 33e974f91cSConrad Meyer #include <sys/conf.h> 34e974f91cSConrad Meyer #include <sys/ioccom.h> 35e974f91cSConrad Meyer #include <sys/kernel.h> 36e974f91cSConrad Meyer #include <sys/lock.h> 37e974f91cSConrad Meyer #include <sys/malloc.h> 38e974f91cSConrad Meyer #include <sys/module.h> 39e974f91cSConrad Meyer #include <sys/mutex.h> 40e974f91cSConrad Meyer #include <sys/rman.h> 41e974f91cSConrad Meyer #include <sys/sysctl.h> 42e974f91cSConrad Meyer #include <dev/pci/pcireg.h> 43e974f91cSConrad Meyer #include <dev/pci/pcivar.h> 44e974f91cSConrad Meyer #include <machine/bus.h> 45e974f91cSConrad Meyer #include <machine/resource.h> 461c25420eSConrad Meyer #include <machine/stdarg.h> 47e974f91cSConrad Meyer #include <vm/vm.h> 48e974f91cSConrad Meyer #include <vm/pmap.h> 49e974f91cSConrad Meyer 50e974f91cSConrad Meyer #include "ioat.h" 51e974f91cSConrad Meyer #include "ioat_hw.h" 52e974f91cSConrad Meyer #include "ioat_internal.h" 53e974f91cSConrad Meyer #include "ioat_test.h" 54e974f91cSConrad Meyer 557c69db50SConrad Meyer #ifndef time_after 567c69db50SConrad Meyer #define time_after(a,b) ((long)(b) - (long)(a) < 0) 577c69db50SConrad Meyer #endif 587c69db50SConrad Meyer 59e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations"); 60e974f91cSConrad Meyer 617c69db50SConrad Meyer #define IOAT_MAX_BUFS 256 62e974f91cSConrad Meyer 63e974f91cSConrad Meyer struct test_transaction { 64e974f91cSConrad Meyer void *buf[IOAT_MAX_BUFS]; 65e974f91cSConrad Meyer uint32_t length; 667c69db50SConrad Meyer uint32_t depth; 67e974f91cSConrad Meyer struct ioat_test *test; 687c69db50SConrad Meyer TAILQ_ENTRY(test_transaction) entry; 69e974f91cSConrad Meyer }; 70e974f91cSConrad Meyer 717c69db50SConrad Meyer #define IT_LOCK() mtx_lock(&ioat_test_lk) 727c69db50SConrad Meyer #define IT_UNLOCK() mtx_unlock(&ioat_test_lk) 737c69db50SConrad Meyer #define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED) 747c69db50SConrad Meyer static struct mtx ioat_test_lk; 757c69db50SConrad Meyer MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF); 767c69db50SConrad Meyer 77e974f91cSConrad Meyer static int g_thread_index = 1; 78e974f91cSConrad Meyer static struct cdev *g_ioat_cdev = NULL; 79e974f91cSConrad Meyer 80592fe72dSConrad Meyer #define ioat_test_log(v, ...) _ioat_test_log((v), "ioat_test: " __VA_ARGS__) 811c25420eSConrad Meyer static inline void _ioat_test_log(int verbosity, const char *fmt, ...); 821c25420eSConrad Meyer 83e974f91cSConrad Meyer static void 84e974f91cSConrad Meyer ioat_test_transaction_destroy(struct test_transaction *tx) 85e974f91cSConrad Meyer { 86e974f91cSConrad Meyer int i; 87e974f91cSConrad Meyer 88e974f91cSConrad Meyer for (i = 0; i < IOAT_MAX_BUFS; i++) { 89e974f91cSConrad Meyer if (tx->buf[i] != NULL) { 907c69db50SConrad Meyer contigfree(tx->buf[i], tx->length, M_IOAT_TEST); 91e974f91cSConrad Meyer tx->buf[i] = NULL; 92e974f91cSConrad Meyer } 93e974f91cSConrad Meyer } 94e974f91cSConrad Meyer 95e974f91cSConrad Meyer free(tx, M_IOAT_TEST); 96e974f91cSConrad Meyer } 97e974f91cSConrad Meyer 98e974f91cSConrad Meyer static struct 997c69db50SConrad Meyer test_transaction *ioat_test_transaction_create(unsigned num_buffers, 100e974f91cSConrad Meyer uint32_t buffer_size) 101e974f91cSConrad Meyer { 102e974f91cSConrad Meyer struct test_transaction *tx; 1037c69db50SConrad Meyer unsigned i; 104e974f91cSConrad Meyer 1057c69db50SConrad Meyer tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO); 106e974f91cSConrad Meyer if (tx == NULL) 107e974f91cSConrad Meyer return (NULL); 108e974f91cSConrad Meyer 109e974f91cSConrad Meyer tx->length = buffer_size; 110e974f91cSConrad Meyer 111e974f91cSConrad Meyer for (i = 0; i < num_buffers; i++) { 112e974f91cSConrad Meyer tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT, 113e974f91cSConrad Meyer 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 114e974f91cSConrad Meyer 115e974f91cSConrad Meyer if (tx->buf[i] == NULL) { 116e974f91cSConrad Meyer ioat_test_transaction_destroy(tx); 117e974f91cSConrad Meyer return (NULL); 118e974f91cSConrad Meyer } 119e974f91cSConrad Meyer } 120e974f91cSConrad Meyer return (tx); 121e974f91cSConrad Meyer } 122e974f91cSConrad Meyer 1237c69db50SConrad Meyer static bool 1247c69db50SConrad Meyer ioat_compare_ok(struct test_transaction *tx) 1257c69db50SConrad Meyer { 1262a4fd6b1SConrad Meyer struct ioat_test *test; 1272a4fd6b1SConrad Meyer char *dst, *src; 1282a4fd6b1SConrad Meyer uint32_t i, j; 1292a4fd6b1SConrad Meyer 1302a4fd6b1SConrad Meyer test = tx->test; 1317c69db50SConrad Meyer 1327c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) { 1332a4fd6b1SConrad Meyer dst = tx->buf[2 * i + 1]; 1342a4fd6b1SConrad Meyer src = tx->buf[2 * i]; 1352a4fd6b1SConrad Meyer 1362a4fd6b1SConrad Meyer if (test->testkind == IOAT_TEST_FILL) { 1372a4fd6b1SConrad Meyer for (j = 0; j < tx->length; j += sizeof(uint64_t)) { 1382a4fd6b1SConrad Meyer if (memcmp(src, &dst[j], 1392a4fd6b1SConrad Meyer MIN(sizeof(uint64_t), tx->length - j)) 1402a4fd6b1SConrad Meyer != 0) 1412a4fd6b1SConrad Meyer return (false); 1422a4fd6b1SConrad Meyer } 1432a4fd6b1SConrad Meyer } else if (test->testkind == IOAT_TEST_DMA) 1442a4fd6b1SConrad Meyer if (memcmp(src, dst, tx->length) != 0) 1457c69db50SConrad Meyer return (false); 1467c69db50SConrad Meyer } 1477c69db50SConrad Meyer return (true); 1487c69db50SConrad Meyer } 1497c69db50SConrad Meyer 150e974f91cSConrad Meyer static void 151e974f91cSConrad Meyer ioat_dma_test_callback(void *arg) 152e974f91cSConrad Meyer { 153e974f91cSConrad Meyer struct test_transaction *tx; 154e974f91cSConrad Meyer struct ioat_test *test; 155e974f91cSConrad Meyer 156e974f91cSConrad Meyer tx = arg; 157e974f91cSConrad Meyer test = tx->test; 158e974f91cSConrad Meyer 1597c69db50SConrad Meyer if (test->verify && !ioat_compare_ok(tx)) { 1601c25420eSConrad Meyer ioat_test_log(0, "miscompare found\n"); 1617c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth); 1627c69db50SConrad Meyer } else if (!test->too_late) 1637c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth); 1647c69db50SConrad Meyer 1657c69db50SConrad Meyer IT_LOCK(); 1667c69db50SConrad Meyer TAILQ_REMOVE(&test->pend_q, tx, entry); 1677c69db50SConrad Meyer TAILQ_INSERT_TAIL(&test->free_q, tx, entry); 1687c69db50SConrad Meyer wakeup(&test->free_q); 1697c69db50SConrad Meyer IT_UNLOCK(); 170e974f91cSConrad Meyer } 1717c69db50SConrad Meyer 1727c69db50SConrad Meyer static int 1737c69db50SConrad Meyer ioat_test_prealloc_memory(struct ioat_test *test, int index) 1747c69db50SConrad Meyer { 1757c69db50SConrad Meyer uint32_t i, j, k; 1767c69db50SConrad Meyer struct test_transaction *tx; 1777c69db50SConrad Meyer 1787c69db50SConrad Meyer for (i = 0; i < test->transactions; i++) { 1797c69db50SConrad Meyer tx = ioat_test_transaction_create(test->chain_depth * 2, 1807c69db50SConrad Meyer test->buffer_size); 1817c69db50SConrad Meyer if (tx == NULL) { 1821c25420eSConrad Meyer ioat_test_log(0, "tx == NULL - memory exhausted\n"); 1837c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 1847c69db50SConrad Meyer return (ENOMEM); 1857c69db50SConrad Meyer } 1867c69db50SConrad Meyer 1877c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 1887c69db50SConrad Meyer 1897c69db50SConrad Meyer tx->test = test; 1907c69db50SConrad Meyer tx->depth = test->chain_depth; 1917c69db50SConrad Meyer 1927c69db50SConrad Meyer /* fill in source buffers */ 1937c69db50SConrad Meyer for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) { 1947c69db50SConrad Meyer uint32_t val = j + (index << 28); 1957c69db50SConrad Meyer 1967c69db50SConrad Meyer for (k = 0; k < test->chain_depth; k++) { 1977c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k])[j] = ~val; 1987c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k+1])[j] = val; 1997c69db50SConrad Meyer } 2007c69db50SConrad Meyer } 2017c69db50SConrad Meyer } 2027c69db50SConrad Meyer return (0); 2037c69db50SConrad Meyer } 2047c69db50SConrad Meyer 2057c69db50SConrad Meyer static void 2067c69db50SConrad Meyer ioat_test_release_memory(struct ioat_test *test) 2077c69db50SConrad Meyer { 2087c69db50SConrad Meyer struct test_transaction *tx, *s; 2097c69db50SConrad Meyer 2107c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s) 211e974f91cSConrad Meyer ioat_test_transaction_destroy(tx); 2127c69db50SConrad Meyer TAILQ_INIT(&test->free_q); 2137c69db50SConrad Meyer 2147c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s) 2157c69db50SConrad Meyer ioat_test_transaction_destroy(tx); 2167c69db50SConrad Meyer TAILQ_INIT(&test->pend_q); 2177c69db50SConrad Meyer } 2187c69db50SConrad Meyer 2197c69db50SConrad Meyer static void 2207c69db50SConrad Meyer ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma) 2217c69db50SConrad Meyer { 2227c69db50SConrad Meyer struct test_transaction *tx; 2237c69db50SConrad Meyer struct bus_dmadesc *desc; 2247c69db50SConrad Meyer bus_dmaengine_callback_t cb; 2257c69db50SConrad Meyer bus_addr_t src, dest; 2262a4fd6b1SConrad Meyer uint64_t fillpattern; 2277c69db50SConrad Meyer uint32_t i, flags; 2287c69db50SConrad Meyer 2292a4fd6b1SConrad Meyer desc = NULL; 2302a4fd6b1SConrad Meyer 2317c69db50SConrad Meyer IT_LOCK(); 2327c69db50SConrad Meyer while (TAILQ_EMPTY(&test->free_q)) 2337c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0); 2347c69db50SConrad Meyer 2357c69db50SConrad Meyer tx = TAILQ_FIRST(&test->free_q); 2367c69db50SConrad Meyer TAILQ_REMOVE(&test->free_q, tx, entry); 2377c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->pend_q, tx, entry); 2387c69db50SConrad Meyer IT_UNLOCK(); 2397c69db50SConrad Meyer 2407c69db50SConrad Meyer ioat_acquire(dma); 2417c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) { 2427c69db50SConrad Meyer src = vtophys((vm_offset_t)tx->buf[2*i]); 2437c69db50SConrad Meyer dest = vtophys((vm_offset_t)tx->buf[2*i+1]); 2447c69db50SConrad Meyer 2457c69db50SConrad Meyer if (i == tx->depth - 1) { 2467c69db50SConrad Meyer cb = ioat_dma_test_callback; 2477c69db50SConrad Meyer flags = DMA_INT_EN; 2487c69db50SConrad Meyer } else { 2497c69db50SConrad Meyer cb = NULL; 2507c69db50SConrad Meyer flags = 0; 2517c69db50SConrad Meyer } 2527c69db50SConrad Meyer 2532a4fd6b1SConrad Meyer if (test->testkind == IOAT_TEST_DMA) 2542a4fd6b1SConrad Meyer desc = ioat_copy(dma, dest, src, tx->length, cb, tx, 2552a4fd6b1SConrad Meyer flags); 2562a4fd6b1SConrad Meyer else if (test->testkind == IOAT_TEST_FILL) { 2572a4fd6b1SConrad Meyer fillpattern = *(uint64_t *)tx->buf[2*i]; 2582a4fd6b1SConrad Meyer desc = ioat_blockfill(dma, dest, fillpattern, 2592a4fd6b1SConrad Meyer tx->length, cb, tx, flags); 2602a4fd6b1SConrad Meyer } 2612a4fd6b1SConrad Meyer 2627c69db50SConrad Meyer if (desc == NULL) 2637c69db50SConrad Meyer panic("Failed to allocate a ring slot " 2647c69db50SConrad Meyer "-- this shouldn't happen!"); 2657c69db50SConrad Meyer } 2667c69db50SConrad Meyer ioat_release(dma); 267e974f91cSConrad Meyer } 268e974f91cSConrad Meyer 269e974f91cSConrad Meyer static void 270e974f91cSConrad Meyer ioat_dma_test(void *arg) 271e974f91cSConrad Meyer { 272e974f91cSConrad Meyer struct ioat_test *test; 273e974f91cSConrad Meyer bus_dmaengine_t dmaengine; 274e974f91cSConrad Meyer uint32_t loops; 2757c69db50SConrad Meyer int index, rc, start, end; 276e974f91cSConrad Meyer 277e974f91cSConrad Meyer test = arg; 2787c69db50SConrad Meyer memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status)); 279e974f91cSConrad Meyer 2807c69db50SConrad Meyer if (test->buffer_size > 1024 * 1024) { 2811c25420eSConrad Meyer ioat_test_log(0, "Buffer size too large >1MB\n"); 2827c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 283e974f91cSConrad Meyer return; 284e974f91cSConrad Meyer } 285e974f91cSConrad Meyer 2867c69db50SConrad Meyer if (test->chain_depth * 2 > IOAT_MAX_BUFS) { 2871c25420eSConrad Meyer ioat_test_log(0, "Depth too large (> %u)\n", 2887c69db50SConrad Meyer (unsigned)IOAT_MAX_BUFS / 2); 2897c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 2907c69db50SConrad Meyer return; 291e974f91cSConrad Meyer } 292e974f91cSConrad Meyer 2937c69db50SConrad Meyer if (btoc((uint64_t)test->buffer_size * test->chain_depth * 2947c69db50SConrad Meyer test->transactions) > (physmem / 4)) { 2951c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would " 2967c69db50SConrad Meyer "use more than 1/4 of phys mem.\n"); 2977c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 2987c69db50SConrad Meyer return; 299e974f91cSConrad Meyer } 300e974f91cSConrad Meyer 3017c69db50SConrad Meyer if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) { 3021c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would " 3037c69db50SConrad Meyer "use more than available IOAT ring space.\n"); 3047c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 3057c69db50SConrad Meyer return; 3067c69db50SConrad Meyer } 3077c69db50SConrad Meyer 3082a4fd6b1SConrad Meyer if (test->testkind >= IOAT_NUM_TESTKINDS) { 3092a4fd6b1SConrad Meyer ioat_test_log(0, "Invalid kind %u\n", 3102a4fd6b1SConrad Meyer (unsigned)test->testkind); 3112a4fd6b1SConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 3122a4fd6b1SConrad Meyer return; 3132a4fd6b1SConrad Meyer } 3142a4fd6b1SConrad Meyer 3157c69db50SConrad Meyer dmaengine = ioat_get_dmaengine(test->channel_index); 3167c69db50SConrad Meyer if (dmaengine == NULL) { 3171c25420eSConrad Meyer ioat_test_log(0, "Couldn't acquire dmaengine\n"); 3187c69db50SConrad Meyer test->status[IOAT_TEST_NO_DMA_ENGINE]++; 3197c69db50SConrad Meyer return; 3207c69db50SConrad Meyer } 3217c69db50SConrad Meyer 3227c69db50SConrad Meyer index = g_thread_index++; 3237c69db50SConrad Meyer TAILQ_INIT(&test->free_q); 3247c69db50SConrad Meyer TAILQ_INIT(&test->pend_q); 3257c69db50SConrad Meyer 3267c69db50SConrad Meyer if (test->duration == 0) 3271c25420eSConrad Meyer ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n", 3287c69db50SConrad Meyer index, test->transactions); 3297c69db50SConrad Meyer else 3301c25420eSConrad Meyer ioat_test_log(1, "Thread %d: starting\n", index); 3317c69db50SConrad Meyer 3327c69db50SConrad Meyer rc = ioat_test_prealloc_memory(test, index); 3337c69db50SConrad Meyer if (rc != 0) { 3341c25420eSConrad Meyer ioat_test_log(0, "prealloc_memory: %d\n", rc); 335466b3540SConrad Meyer goto out; 3367c69db50SConrad Meyer } 337e974f91cSConrad Meyer wmb(); 338e974f91cSConrad Meyer 3397c69db50SConrad Meyer test->too_late = false; 3407c69db50SConrad Meyer start = ticks; 3417c69db50SConrad Meyer end = start + (((sbintime_t)test->duration * hz) / 1000); 3427c69db50SConrad Meyer 3437c69db50SConrad Meyer for (loops = 0;; loops++) { 3447c69db50SConrad Meyer if (test->duration == 0 && loops >= test->transactions) 3457c69db50SConrad Meyer break; 3467c69db50SConrad Meyer else if (test->duration != 0 && time_after(ticks, end)) { 3477c69db50SConrad Meyer test->too_late = true; 3487c69db50SConrad Meyer break; 349e974f91cSConrad Meyer } 350e974f91cSConrad Meyer 3517c69db50SConrad Meyer ioat_test_submit_1_tx(test, dmaengine); 352e974f91cSConrad Meyer } 353e974f91cSConrad Meyer 3541c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n", 3557c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz); 356e974f91cSConrad Meyer 3577c69db50SConrad Meyer IT_LOCK(); 3587c69db50SConrad Meyer while (!TAILQ_EMPTY(&test->pend_q)) 3597c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz); 3607c69db50SConrad Meyer IT_UNLOCK(); 3617c69db50SConrad Meyer 3621c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n", 3637c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz); 3647c69db50SConrad Meyer 3657c69db50SConrad Meyer ioat_test_release_memory(test); 366466b3540SConrad Meyer out: 367466b3540SConrad Meyer ioat_put_dmaengine(dmaengine); 368e974f91cSConrad Meyer } 369e974f91cSConrad Meyer 370e974f91cSConrad Meyer static int 371e974f91cSConrad Meyer ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td) 372e974f91cSConrad Meyer { 373e974f91cSConrad Meyer 374e974f91cSConrad Meyer return (0); 375e974f91cSConrad Meyer } 376e974f91cSConrad Meyer 377e974f91cSConrad Meyer static int 378e974f91cSConrad Meyer ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td) 379e974f91cSConrad Meyer { 380e974f91cSConrad Meyer 381e974f91cSConrad Meyer return (0); 382e974f91cSConrad Meyer } 383e974f91cSConrad Meyer 384e974f91cSConrad Meyer static int 385e974f91cSConrad Meyer ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag, 386e974f91cSConrad Meyer struct thread *td) 387e974f91cSConrad Meyer { 388e974f91cSConrad Meyer 389e974f91cSConrad Meyer switch (cmd) { 390e974f91cSConrad Meyer case IOAT_DMATEST: 391e974f91cSConrad Meyer ioat_dma_test(arg); 392e974f91cSConrad Meyer break; 393e974f91cSConrad Meyer default: 394e974f91cSConrad Meyer return (EINVAL); 395e974f91cSConrad Meyer } 396e974f91cSConrad Meyer return (0); 397e974f91cSConrad Meyer } 398e974f91cSConrad Meyer 399e974f91cSConrad Meyer static struct cdevsw ioat_cdevsw = { 400e974f91cSConrad Meyer .d_version = D_VERSION, 401e974f91cSConrad Meyer .d_flags = 0, 402e974f91cSConrad Meyer .d_open = ioat_test_open, 403e974f91cSConrad Meyer .d_close = ioat_test_close, 404e974f91cSConrad Meyer .d_ioctl = ioat_test_ioctl, 405e974f91cSConrad Meyer .d_name = "ioat_test", 406e974f91cSConrad Meyer }; 407e974f91cSConrad Meyer 408e974f91cSConrad Meyer static int 4097afbb263SConrad Meyer enable_ioat_test(bool enable) 4107afbb263SConrad Meyer { 4117afbb263SConrad Meyer 4127afbb263SConrad Meyer mtx_assert(&Giant, MA_OWNED); 4137afbb263SConrad Meyer 4147afbb263SConrad Meyer if (enable && g_ioat_cdev == NULL) { 4157afbb263SConrad Meyer g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL, 4167afbb263SConrad Meyer 0600, "ioat_test"); 4177afbb263SConrad Meyer } else if (!enable && g_ioat_cdev != NULL) { 4187afbb263SConrad Meyer destroy_dev(g_ioat_cdev); 4197afbb263SConrad Meyer g_ioat_cdev = NULL; 4207afbb263SConrad Meyer } 4217afbb263SConrad Meyer return (0); 4227afbb263SConrad Meyer } 4237afbb263SConrad Meyer 4247afbb263SConrad Meyer static int 425e974f91cSConrad Meyer sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS) 426e974f91cSConrad Meyer { 427e974f91cSConrad Meyer int error, enabled; 428e974f91cSConrad Meyer 429e974f91cSConrad Meyer enabled = (g_ioat_cdev != NULL); 430e974f91cSConrad Meyer error = sysctl_handle_int(oidp, &enabled, 0, req); 431e974f91cSConrad Meyer if (error != 0 || req->newptr == NULL) 432e974f91cSConrad Meyer return (error); 433e974f91cSConrad Meyer 4347afbb263SConrad Meyer enable_ioat_test(enabled); 435e974f91cSConrad Meyer return (0); 436e974f91cSConrad Meyer } 437e974f91cSConrad Meyer SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW, 438e974f91cSConrad Meyer 0, 0, sysctl_enable_ioat_test, "I", 439e974f91cSConrad Meyer "Non-zero: Enable the /dev/ioat_test device"); 4407afbb263SConrad Meyer 4417afbb263SConrad Meyer void 4427afbb263SConrad Meyer ioat_test_attach(void) 4437afbb263SConrad Meyer { 4447afbb263SConrad Meyer char *val; 4457afbb263SConrad Meyer 4467afbb263SConrad Meyer val = kern_getenv("hw.ioat.enable_ioat_test"); 4477afbb263SConrad Meyer if (val != NULL && strcmp(val, "0") != 0) { 4487afbb263SConrad Meyer mtx_lock(&Giant); 4497afbb263SConrad Meyer enable_ioat_test(true); 4507afbb263SConrad Meyer mtx_unlock(&Giant); 4517afbb263SConrad Meyer } 4527afbb263SConrad Meyer freeenv(val); 4537afbb263SConrad Meyer } 4547afbb263SConrad Meyer 4557afbb263SConrad Meyer void 4567afbb263SConrad Meyer ioat_test_detach(void) 4577afbb263SConrad Meyer { 4587afbb263SConrad Meyer 4597afbb263SConrad Meyer mtx_lock(&Giant); 4607afbb263SConrad Meyer enable_ioat_test(false); 4617afbb263SConrad Meyer mtx_unlock(&Giant); 4627afbb263SConrad Meyer } 4631c25420eSConrad Meyer 4641c25420eSConrad Meyer static inline void 4651c25420eSConrad Meyer _ioat_test_log(int verbosity, const char *fmt, ...) 4661c25420eSConrad Meyer { 4671c25420eSConrad Meyer va_list argp; 4681c25420eSConrad Meyer 4691c25420eSConrad Meyer if (verbosity > g_ioat_debug_level) 4701c25420eSConrad Meyer return; 4711c25420eSConrad Meyer 4721c25420eSConrad Meyer va_start(argp, fmt); 4731c25420eSConrad Meyer vprintf(fmt, argp); 4741c25420eSConrad Meyer va_end(argp); 4751c25420eSConrad Meyer } 476