1e974f91cSConrad Meyer /*- 2e974f91cSConrad Meyer * Copyright (C) 2012 Intel Corporation 3e974f91cSConrad Meyer * All rights reserved. 4e974f91cSConrad Meyer * 5e974f91cSConrad Meyer * Redistribution and use in source and binary forms, with or without 6e974f91cSConrad Meyer * modification, are permitted provided that the following conditions 7e974f91cSConrad Meyer * are met: 8e974f91cSConrad Meyer * 1. Redistributions of source code must retain the above copyright 9e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer. 10e974f91cSConrad Meyer * 2. Redistributions in binary form must reproduce the above copyright 11e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer in the 12e974f91cSConrad Meyer * documentation and/or other materials provided with the distribution. 13e974f91cSConrad Meyer * 14e974f91cSConrad Meyer * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e974f91cSConrad Meyer * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e974f91cSConrad Meyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e974f91cSConrad Meyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e974f91cSConrad Meyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e974f91cSConrad Meyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e974f91cSConrad Meyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e974f91cSConrad Meyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e974f91cSConrad Meyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e974f91cSConrad Meyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e974f91cSConrad Meyer * SUCH DAMAGE. 25e974f91cSConrad Meyer */ 26e974f91cSConrad Meyer 27e974f91cSConrad Meyer #include <sys/cdefs.h> 28e974f91cSConrad Meyer __FBSDID("$FreeBSD$"); 29e974f91cSConrad Meyer 30e974f91cSConrad Meyer #include <sys/param.h> 31e974f91cSConrad Meyer #include <sys/systm.h> 32e974f91cSConrad Meyer #include <sys/bus.h> 33e974f91cSConrad Meyer #include <sys/conf.h> 34e974f91cSConrad Meyer #include <sys/ioccom.h> 35e974f91cSConrad Meyer #include <sys/kernel.h> 36e974f91cSConrad Meyer #include <sys/lock.h> 37e974f91cSConrad Meyer #include <sys/malloc.h> 38e974f91cSConrad Meyer #include <sys/module.h> 39e974f91cSConrad Meyer #include <sys/mutex.h> 40e974f91cSConrad Meyer #include <sys/rman.h> 41e974f91cSConrad Meyer #include <sys/sysctl.h> 42e974f91cSConrad Meyer #include <dev/pci/pcireg.h> 43e974f91cSConrad Meyer #include <dev/pci/pcivar.h> 44e974f91cSConrad Meyer #include <machine/bus.h> 45e974f91cSConrad Meyer #include <machine/resource.h> 461c25420eSConrad Meyer #include <machine/stdarg.h> 47e974f91cSConrad Meyer #include <vm/vm.h> 48e9497f9bSConrad Meyer #include <vm/vm_param.h> 49e974f91cSConrad Meyer #include <vm/pmap.h> 50e974f91cSConrad Meyer 51e974f91cSConrad Meyer #include "ioat.h" 52e974f91cSConrad Meyer #include "ioat_hw.h" 53e974f91cSConrad Meyer #include "ioat_internal.h" 54e974f91cSConrad Meyer #include "ioat_test.h" 55e974f91cSConrad Meyer 567c69db50SConrad Meyer #ifndef time_after 577c69db50SConrad Meyer #define time_after(a,b) ((long)(b) - (long)(a) < 0) 587c69db50SConrad Meyer #endif 597c69db50SConrad Meyer 60e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations"); 61e974f91cSConrad Meyer 627c69db50SConrad Meyer #define IOAT_MAX_BUFS 256 63e974f91cSConrad Meyer 64e974f91cSConrad Meyer struct test_transaction { 65e974f91cSConrad Meyer void *buf[IOAT_MAX_BUFS]; 66e974f91cSConrad Meyer uint32_t length; 677c69db50SConrad Meyer uint32_t depth; 68a8d9ee9cSTycho Nightingale uint32_t crc[IOAT_MAX_BUFS]; 69e974f91cSConrad Meyer struct ioat_test *test; 707c69db50SConrad Meyer TAILQ_ENTRY(test_transaction) entry; 71e974f91cSConrad Meyer }; 72e974f91cSConrad Meyer 737c69db50SConrad Meyer #define IT_LOCK() mtx_lock(&ioat_test_lk) 747c69db50SConrad Meyer #define IT_UNLOCK() mtx_unlock(&ioat_test_lk) 757c69db50SConrad Meyer #define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED) 767c69db50SConrad Meyer static struct mtx ioat_test_lk; 777c69db50SConrad Meyer MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF); 787c69db50SConrad Meyer 79e974f91cSConrad Meyer static int g_thread_index = 1; 80e974f91cSConrad Meyer static struct cdev *g_ioat_cdev = NULL; 81e974f91cSConrad Meyer 82592fe72dSConrad Meyer #define ioat_test_log(v, ...) _ioat_test_log((v), "ioat_test: " __VA_ARGS__) 831a140621SAndriy Gapon static void _ioat_test_log(int verbosity, const char *fmt, ...); 841c25420eSConrad Meyer 85e974f91cSConrad Meyer static void 86e974f91cSConrad Meyer ioat_test_transaction_destroy(struct test_transaction *tx) 87e974f91cSConrad Meyer { 886a301ac8SConrad Meyer struct ioat_test *test; 89e974f91cSConrad Meyer int i; 90e974f91cSConrad Meyer 916a301ac8SConrad Meyer test = tx->test; 926a301ac8SConrad Meyer 93e974f91cSConrad Meyer for (i = 0; i < IOAT_MAX_BUFS; i++) { 94e974f91cSConrad Meyer if (tx->buf[i] != NULL) { 956a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_DMA_8K) 966a301ac8SConrad Meyer free(tx->buf[i], M_IOAT_TEST); 976a301ac8SConrad Meyer else 987c69db50SConrad Meyer contigfree(tx->buf[i], tx->length, M_IOAT_TEST); 99e974f91cSConrad Meyer tx->buf[i] = NULL; 100e974f91cSConrad Meyer } 101e974f91cSConrad Meyer } 102e974f91cSConrad Meyer 103e974f91cSConrad Meyer free(tx, M_IOAT_TEST); 104e974f91cSConrad Meyer } 105e974f91cSConrad Meyer 106e974f91cSConrad Meyer static struct 1076a301ac8SConrad Meyer test_transaction *ioat_test_transaction_create(struct ioat_test *test, 1086a301ac8SConrad Meyer unsigned num_buffers) 109e974f91cSConrad Meyer { 110e974f91cSConrad Meyer struct test_transaction *tx; 1117c69db50SConrad Meyer unsigned i; 112e974f91cSConrad Meyer 1137c69db50SConrad Meyer tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO); 114e974f91cSConrad Meyer if (tx == NULL) 115e974f91cSConrad Meyer return (NULL); 116e974f91cSConrad Meyer 1176a301ac8SConrad Meyer tx->length = test->buffer_size; 118e974f91cSConrad Meyer 119e974f91cSConrad Meyer for (i = 0; i < num_buffers; i++) { 1206a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_DMA_8K) 1216a301ac8SConrad Meyer tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST, 1226a301ac8SConrad Meyer M_NOWAIT); 1236a301ac8SConrad Meyer else 1246a301ac8SConrad Meyer tx->buf[i] = contigmalloc(test->buffer_size, 1256a301ac8SConrad Meyer M_IOAT_TEST, M_NOWAIT, 0, BUS_SPACE_MAXADDR, 1266a301ac8SConrad Meyer PAGE_SIZE, 0); 127e974f91cSConrad Meyer 128e974f91cSConrad Meyer if (tx->buf[i] == NULL) { 129e974f91cSConrad Meyer ioat_test_transaction_destroy(tx); 130e974f91cSConrad Meyer return (NULL); 131e974f91cSConrad Meyer } 132e974f91cSConrad Meyer } 133e974f91cSConrad Meyer return (tx); 134e974f91cSConrad Meyer } 135e974f91cSConrad Meyer 136e9497f9bSConrad Meyer static void 137e9497f9bSConrad Meyer dump_hex(void *p, size_t chunks) 138e9497f9bSConrad Meyer { 139e9497f9bSConrad Meyer size_t i, j; 140e9497f9bSConrad Meyer 141e9497f9bSConrad Meyer for (i = 0; i < chunks; i++) { 142e9497f9bSConrad Meyer for (j = 0; j < 8; j++) 143e9497f9bSConrad Meyer printf("%08x ", ((uint32_t *)p)[i * 8 + j]); 144e9497f9bSConrad Meyer printf("\n"); 145e9497f9bSConrad Meyer } 146e9497f9bSConrad Meyer } 147e9497f9bSConrad Meyer 1487c69db50SConrad Meyer static bool 1497c69db50SConrad Meyer ioat_compare_ok(struct test_transaction *tx) 1507c69db50SConrad Meyer { 1512a4fd6b1SConrad Meyer struct ioat_test *test; 1522a4fd6b1SConrad Meyer char *dst, *src; 1532a4fd6b1SConrad Meyer uint32_t i, j; 1542a4fd6b1SConrad Meyer 1552a4fd6b1SConrad Meyer test = tx->test; 1567c69db50SConrad Meyer 1577c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) { 1582a4fd6b1SConrad Meyer dst = tx->buf[2 * i + 1]; 1592a4fd6b1SConrad Meyer src = tx->buf[2 * i]; 1602a4fd6b1SConrad Meyer 1612a4fd6b1SConrad Meyer if (test->testkind == IOAT_TEST_FILL) { 1622a4fd6b1SConrad Meyer for (j = 0; j < tx->length; j += sizeof(uint64_t)) { 1632a4fd6b1SConrad Meyer if (memcmp(src, &dst[j], 1642a4fd6b1SConrad Meyer MIN(sizeof(uint64_t), tx->length - j)) 1652a4fd6b1SConrad Meyer != 0) 1662a4fd6b1SConrad Meyer return (false); 1672a4fd6b1SConrad Meyer } 168e9497f9bSConrad Meyer } else if (test->testkind == IOAT_TEST_DMA) { 1692a4fd6b1SConrad Meyer if (memcmp(src, dst, tx->length) != 0) 1707c69db50SConrad Meyer return (false); 171e9497f9bSConrad Meyer } else if (test->testkind == IOAT_TEST_RAW_DMA) { 172e9497f9bSConrad Meyer if (test->raw_write) 173e9497f9bSConrad Meyer dst = test->raw_vtarget; 174e9497f9bSConrad Meyer dump_hex(dst, tx->length / 32); 175e9497f9bSConrad Meyer } 1767c69db50SConrad Meyer } 1777c69db50SConrad Meyer return (true); 1787c69db50SConrad Meyer } 1797c69db50SConrad Meyer 180e974f91cSConrad Meyer static void 181faefad9cSConrad Meyer ioat_dma_test_callback(void *arg, int error) 182e974f91cSConrad Meyer { 183e974f91cSConrad Meyer struct test_transaction *tx; 184e974f91cSConrad Meyer struct ioat_test *test; 185e974f91cSConrad Meyer 186faefad9cSConrad Meyer if (error != 0) 187faefad9cSConrad Meyer ioat_test_log(0, "%s: Got error: %d\n", __func__, error); 188faefad9cSConrad Meyer 189e974f91cSConrad Meyer tx = arg; 190e974f91cSConrad Meyer test = tx->test; 191e974f91cSConrad Meyer 1927c69db50SConrad Meyer if (test->verify && !ioat_compare_ok(tx)) { 1931c25420eSConrad Meyer ioat_test_log(0, "miscompare found\n"); 1947c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth); 1957c69db50SConrad Meyer } else if (!test->too_late) 1967c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth); 1977c69db50SConrad Meyer 1987c69db50SConrad Meyer IT_LOCK(); 1997c69db50SConrad Meyer TAILQ_REMOVE(&test->pend_q, tx, entry); 2007c69db50SConrad Meyer TAILQ_INSERT_TAIL(&test->free_q, tx, entry); 2017c69db50SConrad Meyer wakeup(&test->free_q); 2027c69db50SConrad Meyer IT_UNLOCK(); 203e974f91cSConrad Meyer } 2047c69db50SConrad Meyer 2057c69db50SConrad Meyer static int 2067c69db50SConrad Meyer ioat_test_prealloc_memory(struct ioat_test *test, int index) 2077c69db50SConrad Meyer { 2087c69db50SConrad Meyer uint32_t i, j, k; 2097c69db50SConrad Meyer struct test_transaction *tx; 2107c69db50SConrad Meyer 2117c69db50SConrad Meyer for (i = 0; i < test->transactions; i++) { 2126a301ac8SConrad Meyer tx = ioat_test_transaction_create(test, test->chain_depth * 2); 2137c69db50SConrad Meyer if (tx == NULL) { 2141c25420eSConrad Meyer ioat_test_log(0, "tx == NULL - memory exhausted\n"); 2157c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 2167c69db50SConrad Meyer return (ENOMEM); 2177c69db50SConrad Meyer } 2187c69db50SConrad Meyer 2197c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 2207c69db50SConrad Meyer 2217c69db50SConrad Meyer tx->test = test; 2227c69db50SConrad Meyer tx->depth = test->chain_depth; 2237c69db50SConrad Meyer 2247c69db50SConrad Meyer /* fill in source buffers */ 2257c69db50SConrad Meyer for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) { 2267c69db50SConrad Meyer uint32_t val = j + (index << 28); 2277c69db50SConrad Meyer 2287c69db50SConrad Meyer for (k = 0; k < test->chain_depth; k++) { 2297c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k])[j] = ~val; 2307c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k+1])[j] = val; 2317c69db50SConrad Meyer } 2327c69db50SConrad Meyer } 2337c69db50SConrad Meyer } 2347c69db50SConrad Meyer return (0); 2357c69db50SConrad Meyer } 2367c69db50SConrad Meyer 2377c69db50SConrad Meyer static void 2387c69db50SConrad Meyer ioat_test_release_memory(struct ioat_test *test) 2397c69db50SConrad Meyer { 2407c69db50SConrad Meyer struct test_transaction *tx, *s; 2417c69db50SConrad Meyer 2427c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s) 243e974f91cSConrad Meyer ioat_test_transaction_destroy(tx); 2447c69db50SConrad Meyer TAILQ_INIT(&test->free_q); 2457c69db50SConrad Meyer 2467c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s) 2477c69db50SConrad Meyer ioat_test_transaction_destroy(tx); 2487c69db50SConrad Meyer TAILQ_INIT(&test->pend_q); 2497c69db50SConrad Meyer } 2507c69db50SConrad Meyer 2517c69db50SConrad Meyer static void 2527c69db50SConrad Meyer ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma) 2537c69db50SConrad Meyer { 2547c69db50SConrad Meyer struct test_transaction *tx; 2557c69db50SConrad Meyer struct bus_dmadesc *desc; 2567c69db50SConrad Meyer bus_dmaengine_callback_t cb; 2577c69db50SConrad Meyer bus_addr_t src, dest; 2582a4fd6b1SConrad Meyer uint64_t fillpattern; 2597c69db50SConrad Meyer uint32_t i, flags; 2607c69db50SConrad Meyer 2612a4fd6b1SConrad Meyer desc = NULL; 2622a4fd6b1SConrad Meyer 2637c69db50SConrad Meyer IT_LOCK(); 2647c69db50SConrad Meyer while (TAILQ_EMPTY(&test->free_q)) 2657c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0); 2667c69db50SConrad Meyer 2677c69db50SConrad Meyer tx = TAILQ_FIRST(&test->free_q); 2687c69db50SConrad Meyer TAILQ_REMOVE(&test->free_q, tx, entry); 2697c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->pend_q, tx, entry); 2707c69db50SConrad Meyer IT_UNLOCK(); 2717c69db50SConrad Meyer 2726a301ac8SConrad Meyer if (test->testkind != IOAT_TEST_MEMCPY) 2737c69db50SConrad Meyer ioat_acquire(dma); 2747c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) { 2756a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_MEMCPY) { 2766a301ac8SConrad Meyer memcpy(tx->buf[2 * i + 1], tx->buf[2 * i], tx->length); 2776a301ac8SConrad Meyer if (i == tx->depth - 1) 2786a301ac8SConrad Meyer ioat_dma_test_callback(tx, 0); 2796a301ac8SConrad Meyer continue; 2806a301ac8SConrad Meyer } 2816a301ac8SConrad Meyer 2827c69db50SConrad Meyer src = vtophys((vm_offset_t)tx->buf[2*i]); 2837c69db50SConrad Meyer dest = vtophys((vm_offset_t)tx->buf[2*i+1]); 2847c69db50SConrad Meyer 285e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA) { 286e9497f9bSConrad Meyer if (test->raw_write) 287e9497f9bSConrad Meyer dest = test->raw_target; 288e9497f9bSConrad Meyer else 289e9497f9bSConrad Meyer src = test->raw_target; 290e9497f9bSConrad Meyer } 291e9497f9bSConrad Meyer 2927c69db50SConrad Meyer if (i == tx->depth - 1) { 2937c69db50SConrad Meyer cb = ioat_dma_test_callback; 2947c69db50SConrad Meyer flags = DMA_INT_EN; 2957c69db50SConrad Meyer } else { 2967c69db50SConrad Meyer cb = NULL; 2977c69db50SConrad Meyer flags = 0; 2987c69db50SConrad Meyer } 2997c69db50SConrad Meyer 300e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_DMA || 301e9497f9bSConrad Meyer test->testkind == IOAT_TEST_RAW_DMA) 3022a4fd6b1SConrad Meyer desc = ioat_copy(dma, dest, src, tx->length, cb, tx, 3032a4fd6b1SConrad Meyer flags); 3042a4fd6b1SConrad Meyer else if (test->testkind == IOAT_TEST_FILL) { 3052a4fd6b1SConrad Meyer fillpattern = *(uint64_t *)tx->buf[2*i]; 3062a4fd6b1SConrad Meyer desc = ioat_blockfill(dma, dest, fillpattern, 3072a4fd6b1SConrad Meyer tx->length, cb, tx, flags); 3086a301ac8SConrad Meyer } else if (test->testkind == IOAT_TEST_DMA_8K) { 3096a301ac8SConrad Meyer bus_addr_t src2, dst2; 3106a301ac8SConrad Meyer 3116a301ac8SConrad Meyer src2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE); 3126a301ac8SConrad Meyer dst2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE); 3136a301ac8SConrad Meyer 3146a301ac8SConrad Meyer desc = ioat_copy_8k_aligned(dma, dest, dst2, src, src2, 3156a301ac8SConrad Meyer cb, tx, flags); 316a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_8K_PB) { 317a8d9ee9cSTycho Nightingale bus_addr_t src2, dst2; 318a8d9ee9cSTycho Nightingale 319a8d9ee9cSTycho Nightingale src2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE); 320a8d9ee9cSTycho Nightingale dst2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE); 321a8d9ee9cSTycho Nightingale 322a8d9ee9cSTycho Nightingale desc = ioat_copy_8k_aligned(dma, dest, dst2, src, src2, 323a8d9ee9cSTycho Nightingale cb, tx, flags); 324a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_CRC) { 325a8d9ee9cSTycho Nightingale bus_addr_t crc; 326a8d9ee9cSTycho Nightingale 327a8d9ee9cSTycho Nightingale tx->crc[i] = 0; 328a8d9ee9cSTycho Nightingale crc = vtophys((vm_offset_t)&tx->crc[i]); 329a8d9ee9cSTycho Nightingale desc = ioat_crc(dma, src, tx->length, 330a8d9ee9cSTycho Nightingale NULL, crc, cb, tx, flags | DMA_CRC_STORE); 331a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_CRC_COPY) { 332a8d9ee9cSTycho Nightingale bus_addr_t crc; 333a8d9ee9cSTycho Nightingale 334a8d9ee9cSTycho Nightingale tx->crc[i] = 0; 335a8d9ee9cSTycho Nightingale crc = vtophys((vm_offset_t)&tx->crc[i]); 336a8d9ee9cSTycho Nightingale desc = ioat_copy_crc(dma, dest, src, tx->length, 337a8d9ee9cSTycho Nightingale NULL, crc, cb, tx, flags | DMA_CRC_STORE); 3382a4fd6b1SConrad Meyer } 3397c69db50SConrad Meyer if (desc == NULL) 3401ffae6e8SConrad Meyer break; 3417c69db50SConrad Meyer } 3426a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_MEMCPY) 3436a301ac8SConrad Meyer return; 3447c69db50SConrad Meyer ioat_release(dma); 3451ffae6e8SConrad Meyer 3461ffae6e8SConrad Meyer /* 3471ffae6e8SConrad Meyer * We couldn't issue an IO -- either the device is being detached or 3481ffae6e8SConrad Meyer * the HW reset. Essentially spin until the device comes back up or 3491ffae6e8SConrad Meyer * our timer expires. 3501ffae6e8SConrad Meyer */ 3511ffae6e8SConrad Meyer if (desc == NULL && tx->depth > 0) { 3521ffae6e8SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth); 3531ffae6e8SConrad Meyer IT_LOCK(); 3541ffae6e8SConrad Meyer TAILQ_REMOVE(&test->pend_q, tx, entry); 3551ffae6e8SConrad Meyer TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 3561ffae6e8SConrad Meyer IT_UNLOCK(); 3571ffae6e8SConrad Meyer } 358e974f91cSConrad Meyer } 359e974f91cSConrad Meyer 360e974f91cSConrad Meyer static void 361e974f91cSConrad Meyer ioat_dma_test(void *arg) 362e974f91cSConrad Meyer { 363d37872daSConrad Meyer struct ioat_softc *ioat; 364e974f91cSConrad Meyer struct ioat_test *test; 365e974f91cSConrad Meyer bus_dmaengine_t dmaengine; 366e974f91cSConrad Meyer uint32_t loops; 367d37872daSConrad Meyer int index, rc, start, end, error; 368e974f91cSConrad Meyer 369e974f91cSConrad Meyer test = arg; 3707c69db50SConrad Meyer memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status)); 371e974f91cSConrad Meyer 372a8d9ee9cSTycho Nightingale if ((test->testkind == IOAT_TEST_DMA_8K || 373a8d9ee9cSTycho Nightingale test->testkind == IOAT_TEST_DMA_8K_PB) && 3746a301ac8SConrad Meyer test->buffer_size != 2 * PAGE_SIZE) { 3756a301ac8SConrad Meyer ioat_test_log(0, "Asked for 8k test and buffer size isn't 8k\n"); 3766a301ac8SConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 3776a301ac8SConrad Meyer return; 3786a301ac8SConrad Meyer } 3796a301ac8SConrad Meyer 3807c69db50SConrad Meyer if (test->buffer_size > 1024 * 1024) { 3811c25420eSConrad Meyer ioat_test_log(0, "Buffer size too large >1MB\n"); 3827c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 383e974f91cSConrad Meyer return; 384e974f91cSConrad Meyer } 385e974f91cSConrad Meyer 3867c69db50SConrad Meyer if (test->chain_depth * 2 > IOAT_MAX_BUFS) { 3871c25420eSConrad Meyer ioat_test_log(0, "Depth too large (> %u)\n", 3887c69db50SConrad Meyer (unsigned)IOAT_MAX_BUFS / 2); 3897c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 3907c69db50SConrad Meyer return; 391e974f91cSConrad Meyer } 392e974f91cSConrad Meyer 3937c69db50SConrad Meyer if (btoc((uint64_t)test->buffer_size * test->chain_depth * 3947c69db50SConrad Meyer test->transactions) > (physmem / 4)) { 3951c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would " 3967c69db50SConrad Meyer "use more than 1/4 of phys mem.\n"); 3977c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 3987c69db50SConrad Meyer return; 399e974f91cSConrad Meyer } 400e974f91cSConrad Meyer 4017c69db50SConrad Meyer if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) { 4021c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would " 4037c69db50SConrad Meyer "use more than available IOAT ring space.\n"); 4047c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++; 4057c69db50SConrad Meyer return; 4067c69db50SConrad Meyer } 4077c69db50SConrad Meyer 4082a4fd6b1SConrad Meyer if (test->testkind >= IOAT_NUM_TESTKINDS) { 4092a4fd6b1SConrad Meyer ioat_test_log(0, "Invalid kind %u\n", 4102a4fd6b1SConrad Meyer (unsigned)test->testkind); 4112a4fd6b1SConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 4122a4fd6b1SConrad Meyer return; 4132a4fd6b1SConrad Meyer } 4142a4fd6b1SConrad Meyer 4150ff814e8SConrad Meyer dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT); 4167c69db50SConrad Meyer if (dmaengine == NULL) { 4171c25420eSConrad Meyer ioat_test_log(0, "Couldn't acquire dmaengine\n"); 4187c69db50SConrad Meyer test->status[IOAT_TEST_NO_DMA_ENGINE]++; 4197c69db50SConrad Meyer return; 4207c69db50SConrad Meyer } 421d37872daSConrad Meyer ioat = to_ioat_softc(dmaengine); 4227c69db50SConrad Meyer 4231693d27bSConrad Meyer if (test->testkind == IOAT_TEST_FILL && 424d37872daSConrad Meyer (ioat->capabilities & IOAT_DMACAP_BFILL) == 0) 4251693d27bSConrad Meyer { 4261693d27bSConrad Meyer ioat_test_log(0, 4271693d27bSConrad Meyer "Hardware doesn't support block fill, aborting test\n"); 4281693d27bSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 4291693d27bSConrad Meyer goto out; 4301693d27bSConrad Meyer } 4311693d27bSConrad Meyer 432d37872daSConrad Meyer if (test->coalesce_period > ioat->intrdelay_max) { 433d37872daSConrad Meyer ioat_test_log(0, 434d37872daSConrad Meyer "Hardware doesn't support intrdelay of %u us.\n", 435d37872daSConrad Meyer (unsigned)test->coalesce_period); 436d37872daSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 437d37872daSConrad Meyer goto out; 438d37872daSConrad Meyer } 439d37872daSConrad Meyer error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period); 440d37872daSConrad Meyer if (error == ENODEV && test->coalesce_period == 0) 441d37872daSConrad Meyer error = 0; 442d37872daSConrad Meyer if (error != 0) { 443d37872daSConrad Meyer ioat_test_log(0, "ioat_set_interrupt_coalesce: %d\n", error); 444d37872daSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++; 445d37872daSConrad Meyer goto out; 446d37872daSConrad Meyer } 447d37872daSConrad Meyer 448d37872daSConrad Meyer if (test->zero_stats) 449d37872daSConrad Meyer memset(&ioat->stats, 0, sizeof(ioat->stats)); 450d37872daSConrad Meyer 451e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA) { 452e9497f9bSConrad Meyer if (test->raw_is_virtual) { 453e9497f9bSConrad Meyer test->raw_vtarget = (void *)test->raw_target; 454e9497f9bSConrad Meyer test->raw_target = vtophys(test->raw_vtarget); 455e9497f9bSConrad Meyer } else { 456e9497f9bSConrad Meyer test->raw_vtarget = pmap_mapdev(test->raw_target, 457e9497f9bSConrad Meyer test->buffer_size); 458e9497f9bSConrad Meyer } 459e9497f9bSConrad Meyer } 460e9497f9bSConrad Meyer 4617c69db50SConrad Meyer index = g_thread_index++; 4627c69db50SConrad Meyer TAILQ_INIT(&test->free_q); 4637c69db50SConrad Meyer TAILQ_INIT(&test->pend_q); 4647c69db50SConrad Meyer 4657c69db50SConrad Meyer if (test->duration == 0) 4661c25420eSConrad Meyer ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n", 4677c69db50SConrad Meyer index, test->transactions); 4687c69db50SConrad Meyer else 4691c25420eSConrad Meyer ioat_test_log(1, "Thread %d: starting\n", index); 4707c69db50SConrad Meyer 4717c69db50SConrad Meyer rc = ioat_test_prealloc_memory(test, index); 4727c69db50SConrad Meyer if (rc != 0) { 4731c25420eSConrad Meyer ioat_test_log(0, "prealloc_memory: %d\n", rc); 474466b3540SConrad Meyer goto out; 4757c69db50SConrad Meyer } 476e974f91cSConrad Meyer wmb(); 477e974f91cSConrad Meyer 4787c69db50SConrad Meyer test->too_late = false; 4797c69db50SConrad Meyer start = ticks; 4807c69db50SConrad Meyer end = start + (((sbintime_t)test->duration * hz) / 1000); 4817c69db50SConrad Meyer 4827c69db50SConrad Meyer for (loops = 0;; loops++) { 4837c69db50SConrad Meyer if (test->duration == 0 && loops >= test->transactions) 4847c69db50SConrad Meyer break; 4857c69db50SConrad Meyer else if (test->duration != 0 && time_after(ticks, end)) { 4867c69db50SConrad Meyer test->too_late = true; 4877c69db50SConrad Meyer break; 488e974f91cSConrad Meyer } 489e974f91cSConrad Meyer 4907c69db50SConrad Meyer ioat_test_submit_1_tx(test, dmaengine); 491e974f91cSConrad Meyer } 492e974f91cSConrad Meyer 4931c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n", 4947c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz); 495e974f91cSConrad Meyer 4967c69db50SConrad Meyer IT_LOCK(); 4977c69db50SConrad Meyer while (!TAILQ_EMPTY(&test->pend_q)) 4987c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz); 4997c69db50SConrad Meyer IT_UNLOCK(); 5007c69db50SConrad Meyer 5011c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n", 5027c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz); 5037c69db50SConrad Meyer 5047c69db50SConrad Meyer ioat_test_release_memory(test); 505466b3540SConrad Meyer out: 506e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual) 5077ae99f80SJohn Baldwin pmap_unmapdev(test->raw_vtarget, test->buffer_size); 508466b3540SConrad Meyer ioat_put_dmaengine(dmaengine); 509e974f91cSConrad Meyer } 510e974f91cSConrad Meyer 511e974f91cSConrad Meyer static int 512e974f91cSConrad Meyer ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td) 513e974f91cSConrad Meyer { 514e974f91cSConrad Meyer 515e974f91cSConrad Meyer return (0); 516e974f91cSConrad Meyer } 517e974f91cSConrad Meyer 518e974f91cSConrad Meyer static int 519e974f91cSConrad Meyer ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td) 520e974f91cSConrad Meyer { 521e974f91cSConrad Meyer 522e974f91cSConrad Meyer return (0); 523e974f91cSConrad Meyer } 524e974f91cSConrad Meyer 525e974f91cSConrad Meyer static int 526e974f91cSConrad Meyer ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag, 527e974f91cSConrad Meyer struct thread *td) 528e974f91cSConrad Meyer { 529e974f91cSConrad Meyer 530e974f91cSConrad Meyer switch (cmd) { 531e974f91cSConrad Meyer case IOAT_DMATEST: 532e974f91cSConrad Meyer ioat_dma_test(arg); 533e974f91cSConrad Meyer break; 534e974f91cSConrad Meyer default: 535e974f91cSConrad Meyer return (EINVAL); 536e974f91cSConrad Meyer } 537e974f91cSConrad Meyer return (0); 538e974f91cSConrad Meyer } 539e974f91cSConrad Meyer 540e974f91cSConrad Meyer static struct cdevsw ioat_cdevsw = { 541e974f91cSConrad Meyer .d_version = D_VERSION, 542e974f91cSConrad Meyer .d_flags = 0, 543e974f91cSConrad Meyer .d_open = ioat_test_open, 544e974f91cSConrad Meyer .d_close = ioat_test_close, 545e974f91cSConrad Meyer .d_ioctl = ioat_test_ioctl, 546e974f91cSConrad Meyer .d_name = "ioat_test", 547e974f91cSConrad Meyer }; 548e974f91cSConrad Meyer 549e974f91cSConrad Meyer static int 5507afbb263SConrad Meyer enable_ioat_test(bool enable) 5517afbb263SConrad Meyer { 55271bf3900SAlexander Motin struct make_dev_args devargs; 55371bf3900SAlexander Motin int error = 0; 5547afbb263SConrad Meyer 5557afbb263SConrad Meyer if (enable && g_ioat_cdev == NULL) { 55671bf3900SAlexander Motin make_dev_args_init(&devargs); 55771bf3900SAlexander Motin devargs.mda_devsw = &ioat_cdevsw; 55871bf3900SAlexander Motin devargs.mda_uid = UID_ROOT; 55971bf3900SAlexander Motin devargs.mda_gid = GID_WHEEL; 56071bf3900SAlexander Motin devargs.mda_mode = 0600; 56171bf3900SAlexander Motin error = make_dev_s(&devargs, &g_ioat_cdev, "ioat_test"); 5627afbb263SConrad Meyer } else if (!enable && g_ioat_cdev != NULL) { 5637afbb263SConrad Meyer destroy_dev(g_ioat_cdev); 5647afbb263SConrad Meyer g_ioat_cdev = NULL; 5657afbb263SConrad Meyer } 56671bf3900SAlexander Motin return (error); 5677afbb263SConrad Meyer } 5687afbb263SConrad Meyer 5697afbb263SConrad Meyer static int 570e974f91cSConrad Meyer sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS) 571e974f91cSConrad Meyer { 572e974f91cSConrad Meyer int error, enabled; 573e974f91cSConrad Meyer 574e974f91cSConrad Meyer enabled = (g_ioat_cdev != NULL); 575e974f91cSConrad Meyer error = sysctl_handle_int(oidp, &enabled, 0, req); 576e974f91cSConrad Meyer if (error != 0 || req->newptr == NULL) 577e974f91cSConrad Meyer return (error); 578e974f91cSConrad Meyer 57971bf3900SAlexander Motin return (enable_ioat_test(enabled)); 580e974f91cSConrad Meyer } 5817029da5cSPawel Biernacki SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, 58271bf3900SAlexander Motin CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, 5837029da5cSPawel Biernacki sysctl_enable_ioat_test, "I", 584e974f91cSConrad Meyer "Non-zero: Enable the /dev/ioat_test device"); 5857afbb263SConrad Meyer 5867afbb263SConrad Meyer void 5877afbb263SConrad Meyer ioat_test_attach(void) 5887afbb263SConrad Meyer { 5897afbb263SConrad Meyer char *val; 5907afbb263SConrad Meyer 5917afbb263SConrad Meyer val = kern_getenv("hw.ioat.enable_ioat_test"); 59271bf3900SAlexander Motin if (val != NULL && strcmp(val, "0") != 0) 5937afbb263SConrad Meyer enable_ioat_test(true); 5947afbb263SConrad Meyer freeenv(val); 5957afbb263SConrad Meyer } 5967afbb263SConrad Meyer 5977afbb263SConrad Meyer void 5987afbb263SConrad Meyer ioat_test_detach(void) 5997afbb263SConrad Meyer { 6007afbb263SConrad Meyer 6017afbb263SConrad Meyer enable_ioat_test(false); 6027afbb263SConrad Meyer } 6031c25420eSConrad Meyer 6041a140621SAndriy Gapon static void 6051c25420eSConrad Meyer _ioat_test_log(int verbosity, const char *fmt, ...) 6061c25420eSConrad Meyer { 6071c25420eSConrad Meyer va_list argp; 6081c25420eSConrad Meyer 6091c25420eSConrad Meyer if (verbosity > g_ioat_debug_level) 6101c25420eSConrad Meyer return; 6111c25420eSConrad Meyer 6121c25420eSConrad Meyer va_start(argp, fmt); 6131c25420eSConrad Meyer vprintf(fmt, argp); 6141c25420eSConrad Meyer va_end(argp); 6151c25420eSConrad Meyer } 616