1e974f91cSConrad Meyer /*-
2e974f91cSConrad Meyer * Copyright (C) 2012 Intel Corporation
3e974f91cSConrad Meyer * All rights reserved.
4e974f91cSConrad Meyer *
5e974f91cSConrad Meyer * Redistribution and use in source and binary forms, with or without
6e974f91cSConrad Meyer * modification, are permitted provided that the following conditions
7e974f91cSConrad Meyer * are met:
8e974f91cSConrad Meyer * 1. Redistributions of source code must retain the above copyright
9e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer.
10e974f91cSConrad Meyer * 2. Redistributions in binary form must reproduce the above copyright
11e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer in the
12e974f91cSConrad Meyer * documentation and/or other materials provided with the distribution.
13e974f91cSConrad Meyer *
14e974f91cSConrad Meyer * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15e974f91cSConrad Meyer * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16e974f91cSConrad Meyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17e974f91cSConrad Meyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18e974f91cSConrad Meyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19e974f91cSConrad Meyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20e974f91cSConrad Meyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21e974f91cSConrad Meyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22e974f91cSConrad Meyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23e974f91cSConrad Meyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24e974f91cSConrad Meyer * SUCH DAMAGE.
25e974f91cSConrad Meyer */
26e974f91cSConrad Meyer
27e974f91cSConrad Meyer #include <sys/param.h>
28e974f91cSConrad Meyer #include <sys/systm.h>
29e974f91cSConrad Meyer #include <sys/bus.h>
30e974f91cSConrad Meyer #include <sys/conf.h>
31e974f91cSConrad Meyer #include <sys/ioccom.h>
32e974f91cSConrad Meyer #include <sys/kernel.h>
33e974f91cSConrad Meyer #include <sys/lock.h>
34e974f91cSConrad Meyer #include <sys/malloc.h>
35e974f91cSConrad Meyer #include <sys/module.h>
36e974f91cSConrad Meyer #include <sys/mutex.h>
37e974f91cSConrad Meyer #include <sys/rman.h>
38e974f91cSConrad Meyer #include <sys/sysctl.h>
39e974f91cSConrad Meyer #include <dev/pci/pcireg.h>
40e974f91cSConrad Meyer #include <dev/pci/pcivar.h>
41e974f91cSConrad Meyer #include <machine/bus.h>
42e974f91cSConrad Meyer #include <machine/resource.h>
431c25420eSConrad Meyer #include <machine/stdarg.h>
44e974f91cSConrad Meyer #include <vm/vm.h>
45e9497f9bSConrad Meyer #include <vm/vm_param.h>
46e974f91cSConrad Meyer #include <vm/pmap.h>
47e974f91cSConrad Meyer
48e974f91cSConrad Meyer #include "ioat.h"
49e974f91cSConrad Meyer #include "ioat_hw.h"
50e974f91cSConrad Meyer #include "ioat_internal.h"
51e974f91cSConrad Meyer #include "ioat_test.h"
52e974f91cSConrad Meyer
537c69db50SConrad Meyer #ifndef time_after
547c69db50SConrad Meyer #define time_after(a,b) ((long)(b) - (long)(a) < 0)
557c69db50SConrad Meyer #endif
567c69db50SConrad Meyer
57e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
58e974f91cSConrad Meyer
597c69db50SConrad Meyer #define IOAT_MAX_BUFS 256
60e974f91cSConrad Meyer
61e974f91cSConrad Meyer struct test_transaction {
62e974f91cSConrad Meyer void *buf[IOAT_MAX_BUFS];
63e974f91cSConrad Meyer uint32_t length;
647c69db50SConrad Meyer uint32_t depth;
65a8d9ee9cSTycho Nightingale uint32_t crc[IOAT_MAX_BUFS];
66e974f91cSConrad Meyer struct ioat_test *test;
677c69db50SConrad Meyer TAILQ_ENTRY(test_transaction) entry;
68e974f91cSConrad Meyer };
69e974f91cSConrad Meyer
707c69db50SConrad Meyer #define IT_LOCK() mtx_lock(&ioat_test_lk)
717c69db50SConrad Meyer #define IT_UNLOCK() mtx_unlock(&ioat_test_lk)
727c69db50SConrad Meyer #define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED)
737c69db50SConrad Meyer static struct mtx ioat_test_lk;
747c69db50SConrad Meyer MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF);
757c69db50SConrad Meyer
76e974f91cSConrad Meyer static int g_thread_index = 1;
77e974f91cSConrad Meyer static struct cdev *g_ioat_cdev = NULL;
78e974f91cSConrad Meyer
79592fe72dSConrad Meyer #define ioat_test_log(v, ...) _ioat_test_log((v), "ioat_test: " __VA_ARGS__)
801a140621SAndriy Gapon static void _ioat_test_log(int verbosity, const char *fmt, ...);
811c25420eSConrad Meyer
82e974f91cSConrad Meyer static void
ioat_test_transaction_destroy(struct test_transaction * tx)83e974f91cSConrad Meyer ioat_test_transaction_destroy(struct test_transaction *tx)
84e974f91cSConrad Meyer {
856a301ac8SConrad Meyer struct ioat_test *test;
86e974f91cSConrad Meyer int i;
87e974f91cSConrad Meyer
886a301ac8SConrad Meyer test = tx->test;
896a301ac8SConrad Meyer
90e974f91cSConrad Meyer for (i = 0; i < IOAT_MAX_BUFS; i++) {
91e974f91cSConrad Meyer if (tx->buf[i] != NULL) {
926a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_DMA_8K)
936a301ac8SConrad Meyer free(tx->buf[i], M_IOAT_TEST);
946a301ac8SConrad Meyer else
957c69db50SConrad Meyer contigfree(tx->buf[i], tx->length, M_IOAT_TEST);
96e974f91cSConrad Meyer tx->buf[i] = NULL;
97e974f91cSConrad Meyer }
98e974f91cSConrad Meyer }
99e974f91cSConrad Meyer
100e974f91cSConrad Meyer free(tx, M_IOAT_TEST);
101e974f91cSConrad Meyer }
102e974f91cSConrad Meyer
103e974f91cSConrad Meyer static struct
ioat_test_transaction_create(struct ioat_test * test,unsigned num_buffers)1046a301ac8SConrad Meyer test_transaction *ioat_test_transaction_create(struct ioat_test *test,
1056a301ac8SConrad Meyer unsigned num_buffers)
106e974f91cSConrad Meyer {
107e974f91cSConrad Meyer struct test_transaction *tx;
1087c69db50SConrad Meyer unsigned i;
109e974f91cSConrad Meyer
1107c69db50SConrad Meyer tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO);
111e974f91cSConrad Meyer if (tx == NULL)
112e974f91cSConrad Meyer return (NULL);
113e974f91cSConrad Meyer
1146a301ac8SConrad Meyer tx->length = test->buffer_size;
115e974f91cSConrad Meyer
116e974f91cSConrad Meyer for (i = 0; i < num_buffers; i++) {
1176a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_DMA_8K)
1186a301ac8SConrad Meyer tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,
1196a301ac8SConrad Meyer M_NOWAIT);
1206a301ac8SConrad Meyer else
1216a301ac8SConrad Meyer tx->buf[i] = contigmalloc(test->buffer_size,
1226a301ac8SConrad Meyer M_IOAT_TEST, M_NOWAIT, 0, BUS_SPACE_MAXADDR,
1236a301ac8SConrad Meyer PAGE_SIZE, 0);
124e974f91cSConrad Meyer
125e974f91cSConrad Meyer if (tx->buf[i] == NULL) {
126e974f91cSConrad Meyer ioat_test_transaction_destroy(tx);
127e974f91cSConrad Meyer return (NULL);
128e974f91cSConrad Meyer }
129e974f91cSConrad Meyer }
130e974f91cSConrad Meyer return (tx);
131e974f91cSConrad Meyer }
132e974f91cSConrad Meyer
133e9497f9bSConrad Meyer static void
dump_hex(void * p,size_t chunks)134e9497f9bSConrad Meyer dump_hex(void *p, size_t chunks)
135e9497f9bSConrad Meyer {
136e9497f9bSConrad Meyer size_t i, j;
137e9497f9bSConrad Meyer
138e9497f9bSConrad Meyer for (i = 0; i < chunks; i++) {
139e9497f9bSConrad Meyer for (j = 0; j < 8; j++)
140e9497f9bSConrad Meyer printf("%08x ", ((uint32_t *)p)[i * 8 + j]);
141e9497f9bSConrad Meyer printf("\n");
142e9497f9bSConrad Meyer }
143e9497f9bSConrad Meyer }
144e9497f9bSConrad Meyer
1457c69db50SConrad Meyer static bool
ioat_compare_ok(struct test_transaction * tx)1467c69db50SConrad Meyer ioat_compare_ok(struct test_transaction *tx)
1477c69db50SConrad Meyer {
1482a4fd6b1SConrad Meyer struct ioat_test *test;
1492a4fd6b1SConrad Meyer char *dst, *src;
1502a4fd6b1SConrad Meyer uint32_t i, j;
1512a4fd6b1SConrad Meyer
1522a4fd6b1SConrad Meyer test = tx->test;
1537c69db50SConrad Meyer
1547c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) {
1552a4fd6b1SConrad Meyer dst = tx->buf[2 * i + 1];
1562a4fd6b1SConrad Meyer src = tx->buf[2 * i];
1572a4fd6b1SConrad Meyer
1582a4fd6b1SConrad Meyer if (test->testkind == IOAT_TEST_FILL) {
1592a4fd6b1SConrad Meyer for (j = 0; j < tx->length; j += sizeof(uint64_t)) {
1602a4fd6b1SConrad Meyer if (memcmp(src, &dst[j],
1612a4fd6b1SConrad Meyer MIN(sizeof(uint64_t), tx->length - j))
1622a4fd6b1SConrad Meyer != 0)
1632a4fd6b1SConrad Meyer return (false);
1642a4fd6b1SConrad Meyer }
165e9497f9bSConrad Meyer } else if (test->testkind == IOAT_TEST_DMA) {
1662a4fd6b1SConrad Meyer if (memcmp(src, dst, tx->length) != 0)
1677c69db50SConrad Meyer return (false);
168e9497f9bSConrad Meyer } else if (test->testkind == IOAT_TEST_RAW_DMA) {
169e9497f9bSConrad Meyer if (test->raw_write)
170e9497f9bSConrad Meyer dst = test->raw_vtarget;
171e9497f9bSConrad Meyer dump_hex(dst, tx->length / 32);
172e9497f9bSConrad Meyer }
1737c69db50SConrad Meyer }
1747c69db50SConrad Meyer return (true);
1757c69db50SConrad Meyer }
1767c69db50SConrad Meyer
177e974f91cSConrad Meyer static void
ioat_dma_test_callback(void * arg,int error)178faefad9cSConrad Meyer ioat_dma_test_callback(void *arg, int error)
179e974f91cSConrad Meyer {
180e974f91cSConrad Meyer struct test_transaction *tx;
181e974f91cSConrad Meyer struct ioat_test *test;
182e974f91cSConrad Meyer
183faefad9cSConrad Meyer if (error != 0)
184faefad9cSConrad Meyer ioat_test_log(0, "%s: Got error: %d\n", __func__, error);
185faefad9cSConrad Meyer
186e974f91cSConrad Meyer tx = arg;
187e974f91cSConrad Meyer test = tx->test;
188e974f91cSConrad Meyer
1897c69db50SConrad Meyer if (test->verify && !ioat_compare_ok(tx)) {
1901c25420eSConrad Meyer ioat_test_log(0, "miscompare found\n");
1917c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);
1927c69db50SConrad Meyer } else if (!test->too_late)
1937c69db50SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);
1947c69db50SConrad Meyer
1957c69db50SConrad Meyer IT_LOCK();
1967c69db50SConrad Meyer TAILQ_REMOVE(&test->pend_q, tx, entry);
1977c69db50SConrad Meyer TAILQ_INSERT_TAIL(&test->free_q, tx, entry);
1987c69db50SConrad Meyer wakeup(&test->free_q);
1997c69db50SConrad Meyer IT_UNLOCK();
200e974f91cSConrad Meyer }
2017c69db50SConrad Meyer
2027c69db50SConrad Meyer static int
ioat_test_prealloc_memory(struct ioat_test * test,int index)2037c69db50SConrad Meyer ioat_test_prealloc_memory(struct ioat_test *test, int index)
2047c69db50SConrad Meyer {
2057c69db50SConrad Meyer uint32_t i, j, k;
2067c69db50SConrad Meyer struct test_transaction *tx;
2077c69db50SConrad Meyer
2087c69db50SConrad Meyer for (i = 0; i < test->transactions; i++) {
2096a301ac8SConrad Meyer tx = ioat_test_transaction_create(test, test->chain_depth * 2);
2107c69db50SConrad Meyer if (tx == NULL) {
2111c25420eSConrad Meyer ioat_test_log(0, "tx == NULL - memory exhausted\n");
2127c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++;
2137c69db50SConrad Meyer return (ENOMEM);
2147c69db50SConrad Meyer }
2157c69db50SConrad Meyer
2167c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->free_q, tx, entry);
2177c69db50SConrad Meyer
2187c69db50SConrad Meyer tx->test = test;
2197c69db50SConrad Meyer tx->depth = test->chain_depth;
2207c69db50SConrad Meyer
2217c69db50SConrad Meyer /* fill in source buffers */
2227c69db50SConrad Meyer for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) {
2237c69db50SConrad Meyer uint32_t val = j + (index << 28);
2247c69db50SConrad Meyer
2257c69db50SConrad Meyer for (k = 0; k < test->chain_depth; k++) {
2267c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k])[j] = ~val;
2277c69db50SConrad Meyer ((uint32_t *)tx->buf[2*k+1])[j] = val;
2287c69db50SConrad Meyer }
2297c69db50SConrad Meyer }
2307c69db50SConrad Meyer }
2317c69db50SConrad Meyer return (0);
2327c69db50SConrad Meyer }
2337c69db50SConrad Meyer
2347c69db50SConrad Meyer static void
ioat_test_release_memory(struct ioat_test * test)2357c69db50SConrad Meyer ioat_test_release_memory(struct ioat_test *test)
2367c69db50SConrad Meyer {
2377c69db50SConrad Meyer struct test_transaction *tx, *s;
2387c69db50SConrad Meyer
2397c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)
240e974f91cSConrad Meyer ioat_test_transaction_destroy(tx);
2417c69db50SConrad Meyer TAILQ_INIT(&test->free_q);
2427c69db50SConrad Meyer
2437c69db50SConrad Meyer TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)
2447c69db50SConrad Meyer ioat_test_transaction_destroy(tx);
2457c69db50SConrad Meyer TAILQ_INIT(&test->pend_q);
2467c69db50SConrad Meyer }
2477c69db50SConrad Meyer
2487c69db50SConrad Meyer static void
ioat_test_submit_1_tx(struct ioat_test * test,bus_dmaengine_t dma)2497c69db50SConrad Meyer ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)
2507c69db50SConrad Meyer {
2517c69db50SConrad Meyer struct test_transaction *tx;
2527c69db50SConrad Meyer struct bus_dmadesc *desc;
2537c69db50SConrad Meyer bus_dmaengine_callback_t cb;
2547c69db50SConrad Meyer bus_addr_t src, dest;
2552a4fd6b1SConrad Meyer uint64_t fillpattern;
2567c69db50SConrad Meyer uint32_t i, flags;
2577c69db50SConrad Meyer
2582a4fd6b1SConrad Meyer desc = NULL;
2592a4fd6b1SConrad Meyer
2607c69db50SConrad Meyer IT_LOCK();
2617c69db50SConrad Meyer while (TAILQ_EMPTY(&test->free_q))
2627c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);
2637c69db50SConrad Meyer
2647c69db50SConrad Meyer tx = TAILQ_FIRST(&test->free_q);
2657c69db50SConrad Meyer TAILQ_REMOVE(&test->free_q, tx, entry);
2667c69db50SConrad Meyer TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);
2677c69db50SConrad Meyer IT_UNLOCK();
2687c69db50SConrad Meyer
2696a301ac8SConrad Meyer if (test->testkind != IOAT_TEST_MEMCPY)
2707c69db50SConrad Meyer ioat_acquire(dma);
2717c69db50SConrad Meyer for (i = 0; i < tx->depth; i++) {
2726a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_MEMCPY) {
2736a301ac8SConrad Meyer memcpy(tx->buf[2 * i + 1], tx->buf[2 * i], tx->length);
2746a301ac8SConrad Meyer if (i == tx->depth - 1)
2756a301ac8SConrad Meyer ioat_dma_test_callback(tx, 0);
2766a301ac8SConrad Meyer continue;
2776a301ac8SConrad Meyer }
2786a301ac8SConrad Meyer
2797c69db50SConrad Meyer src = vtophys((vm_offset_t)tx->buf[2*i]);
2807c69db50SConrad Meyer dest = vtophys((vm_offset_t)tx->buf[2*i+1]);
2817c69db50SConrad Meyer
282e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA) {
283e9497f9bSConrad Meyer if (test->raw_write)
284e9497f9bSConrad Meyer dest = test->raw_target;
285e9497f9bSConrad Meyer else
286e9497f9bSConrad Meyer src = test->raw_target;
287e9497f9bSConrad Meyer }
288e9497f9bSConrad Meyer
2897c69db50SConrad Meyer if (i == tx->depth - 1) {
2907c69db50SConrad Meyer cb = ioat_dma_test_callback;
2917c69db50SConrad Meyer flags = DMA_INT_EN;
2927c69db50SConrad Meyer } else {
2937c69db50SConrad Meyer cb = NULL;
2947c69db50SConrad Meyer flags = 0;
2957c69db50SConrad Meyer }
2967c69db50SConrad Meyer
297e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_DMA ||
298e9497f9bSConrad Meyer test->testkind == IOAT_TEST_RAW_DMA)
2992a4fd6b1SConrad Meyer desc = ioat_copy(dma, dest, src, tx->length, cb, tx,
3002a4fd6b1SConrad Meyer flags);
3012a4fd6b1SConrad Meyer else if (test->testkind == IOAT_TEST_FILL) {
3022a4fd6b1SConrad Meyer fillpattern = *(uint64_t *)tx->buf[2*i];
3032a4fd6b1SConrad Meyer desc = ioat_blockfill(dma, dest, fillpattern,
3042a4fd6b1SConrad Meyer tx->length, cb, tx, flags);
3056a301ac8SConrad Meyer } else if (test->testkind == IOAT_TEST_DMA_8K) {
3066a301ac8SConrad Meyer bus_addr_t src2, dst2;
3076a301ac8SConrad Meyer
3086a301ac8SConrad Meyer src2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);
3096a301ac8SConrad Meyer dst2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);
3106a301ac8SConrad Meyer
3116a301ac8SConrad Meyer desc = ioat_copy_8k_aligned(dma, dest, dst2, src, src2,
3126a301ac8SConrad Meyer cb, tx, flags);
313a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_8K_PB) {
314a8d9ee9cSTycho Nightingale bus_addr_t src2, dst2;
315a8d9ee9cSTycho Nightingale
316a8d9ee9cSTycho Nightingale src2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);
317a8d9ee9cSTycho Nightingale dst2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);
318a8d9ee9cSTycho Nightingale
319a8d9ee9cSTycho Nightingale desc = ioat_copy_8k_aligned(dma, dest, dst2, src, src2,
320a8d9ee9cSTycho Nightingale cb, tx, flags);
321a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_CRC) {
322a8d9ee9cSTycho Nightingale bus_addr_t crc;
323a8d9ee9cSTycho Nightingale
324a8d9ee9cSTycho Nightingale tx->crc[i] = 0;
325a8d9ee9cSTycho Nightingale crc = vtophys((vm_offset_t)&tx->crc[i]);
326a8d9ee9cSTycho Nightingale desc = ioat_crc(dma, src, tx->length,
327a8d9ee9cSTycho Nightingale NULL, crc, cb, tx, flags | DMA_CRC_STORE);
328a8d9ee9cSTycho Nightingale } else if (test->testkind == IOAT_TEST_DMA_CRC_COPY) {
329a8d9ee9cSTycho Nightingale bus_addr_t crc;
330a8d9ee9cSTycho Nightingale
331a8d9ee9cSTycho Nightingale tx->crc[i] = 0;
332a8d9ee9cSTycho Nightingale crc = vtophys((vm_offset_t)&tx->crc[i]);
333a8d9ee9cSTycho Nightingale desc = ioat_copy_crc(dma, dest, src, tx->length,
334a8d9ee9cSTycho Nightingale NULL, crc, cb, tx, flags | DMA_CRC_STORE);
3352a4fd6b1SConrad Meyer }
3367c69db50SConrad Meyer if (desc == NULL)
3371ffae6e8SConrad Meyer break;
3387c69db50SConrad Meyer }
3396a301ac8SConrad Meyer if (test->testkind == IOAT_TEST_MEMCPY)
3406a301ac8SConrad Meyer return;
3417c69db50SConrad Meyer ioat_release(dma);
3421ffae6e8SConrad Meyer
3431ffae6e8SConrad Meyer /*
3441ffae6e8SConrad Meyer * We couldn't issue an IO -- either the device is being detached or
3451ffae6e8SConrad Meyer * the HW reset. Essentially spin until the device comes back up or
3461ffae6e8SConrad Meyer * our timer expires.
3471ffae6e8SConrad Meyer */
3481ffae6e8SConrad Meyer if (desc == NULL && tx->depth > 0) {
3491ffae6e8SConrad Meyer atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);
3501ffae6e8SConrad Meyer IT_LOCK();
3511ffae6e8SConrad Meyer TAILQ_REMOVE(&test->pend_q, tx, entry);
3521ffae6e8SConrad Meyer TAILQ_INSERT_HEAD(&test->free_q, tx, entry);
3531ffae6e8SConrad Meyer IT_UNLOCK();
3541ffae6e8SConrad Meyer }
355e974f91cSConrad Meyer }
356e974f91cSConrad Meyer
357e974f91cSConrad Meyer static void
ioat_dma_test(void * arg)358e974f91cSConrad Meyer ioat_dma_test(void *arg)
359e974f91cSConrad Meyer {
360d37872daSConrad Meyer struct ioat_softc *ioat;
361e974f91cSConrad Meyer struct ioat_test *test;
362e974f91cSConrad Meyer bus_dmaengine_t dmaengine;
363e974f91cSConrad Meyer uint32_t loops;
364d37872daSConrad Meyer int index, rc, start, end, error;
365e974f91cSConrad Meyer
366e974f91cSConrad Meyer test = arg;
3677c69db50SConrad Meyer memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));
368e974f91cSConrad Meyer
369a8d9ee9cSTycho Nightingale if ((test->testkind == IOAT_TEST_DMA_8K ||
370a8d9ee9cSTycho Nightingale test->testkind == IOAT_TEST_DMA_8K_PB) &&
3716a301ac8SConrad Meyer test->buffer_size != 2 * PAGE_SIZE) {
3726a301ac8SConrad Meyer ioat_test_log(0, "Asked for 8k test and buffer size isn't 8k\n");
3736a301ac8SConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++;
3746a301ac8SConrad Meyer return;
3756a301ac8SConrad Meyer }
3766a301ac8SConrad Meyer
3777c69db50SConrad Meyer if (test->buffer_size > 1024 * 1024) {
3781c25420eSConrad Meyer ioat_test_log(0, "Buffer size too large >1MB\n");
3797c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++;
380e974f91cSConrad Meyer return;
381e974f91cSConrad Meyer }
382e974f91cSConrad Meyer
3837c69db50SConrad Meyer if (test->chain_depth * 2 > IOAT_MAX_BUFS) {
3841c25420eSConrad Meyer ioat_test_log(0, "Depth too large (> %u)\n",
3857c69db50SConrad Meyer (unsigned)IOAT_MAX_BUFS / 2);
3867c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++;
3877c69db50SConrad Meyer return;
388e974f91cSConrad Meyer }
389e974f91cSConrad Meyer
3907c69db50SConrad Meyer if (btoc((uint64_t)test->buffer_size * test->chain_depth *
3917c69db50SConrad Meyer test->transactions) > (physmem / 4)) {
3921c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would "
3937c69db50SConrad Meyer "use more than 1/4 of phys mem.\n");
3947c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++;
3957c69db50SConrad Meyer return;
396e974f91cSConrad Meyer }
397e974f91cSConrad Meyer
3987c69db50SConrad Meyer if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {
3991c25420eSConrad Meyer ioat_test_log(0, "Sanity check failed -- test would "
4007c69db50SConrad Meyer "use more than available IOAT ring space.\n");
4017c69db50SConrad Meyer test->status[IOAT_TEST_NO_MEMORY]++;
4027c69db50SConrad Meyer return;
4037c69db50SConrad Meyer }
4047c69db50SConrad Meyer
4052a4fd6b1SConrad Meyer if (test->testkind >= IOAT_NUM_TESTKINDS) {
4062a4fd6b1SConrad Meyer ioat_test_log(0, "Invalid kind %u\n",
4072a4fd6b1SConrad Meyer (unsigned)test->testkind);
4082a4fd6b1SConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++;
4092a4fd6b1SConrad Meyer return;
4102a4fd6b1SConrad Meyer }
4112a4fd6b1SConrad Meyer
4120ff814e8SConrad Meyer dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT);
4137c69db50SConrad Meyer if (dmaengine == NULL) {
4141c25420eSConrad Meyer ioat_test_log(0, "Couldn't acquire dmaengine\n");
4157c69db50SConrad Meyer test->status[IOAT_TEST_NO_DMA_ENGINE]++;
4167c69db50SConrad Meyer return;
4177c69db50SConrad Meyer }
418d37872daSConrad Meyer ioat = to_ioat_softc(dmaengine);
4197c69db50SConrad Meyer
4201693d27bSConrad Meyer if (test->testkind == IOAT_TEST_FILL &&
421d37872daSConrad Meyer (ioat->capabilities & IOAT_DMACAP_BFILL) == 0)
4221693d27bSConrad Meyer {
4231693d27bSConrad Meyer ioat_test_log(0,
4241693d27bSConrad Meyer "Hardware doesn't support block fill, aborting test\n");
4251693d27bSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++;
4261693d27bSConrad Meyer goto out;
4271693d27bSConrad Meyer }
4281693d27bSConrad Meyer
429d37872daSConrad Meyer if (test->coalesce_period > ioat->intrdelay_max) {
430d37872daSConrad Meyer ioat_test_log(0,
431d37872daSConrad Meyer "Hardware doesn't support intrdelay of %u us.\n",
432d37872daSConrad Meyer (unsigned)test->coalesce_period);
433d37872daSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++;
434d37872daSConrad Meyer goto out;
435d37872daSConrad Meyer }
436d37872daSConrad Meyer error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period);
437d37872daSConrad Meyer if (error == ENODEV && test->coalesce_period == 0)
438d37872daSConrad Meyer error = 0;
439d37872daSConrad Meyer if (error != 0) {
440d37872daSConrad Meyer ioat_test_log(0, "ioat_set_interrupt_coalesce: %d\n", error);
441d37872daSConrad Meyer test->status[IOAT_TEST_INVALID_INPUT]++;
442d37872daSConrad Meyer goto out;
443d37872daSConrad Meyer }
444d37872daSConrad Meyer
445d37872daSConrad Meyer if (test->zero_stats)
446d37872daSConrad Meyer memset(&ioat->stats, 0, sizeof(ioat->stats));
447d37872daSConrad Meyer
448e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA) {
449e9497f9bSConrad Meyer if (test->raw_is_virtual) {
450e9497f9bSConrad Meyer test->raw_vtarget = (void *)test->raw_target;
451e9497f9bSConrad Meyer test->raw_target = vtophys(test->raw_vtarget);
452e9497f9bSConrad Meyer } else {
453e9497f9bSConrad Meyer test->raw_vtarget = pmap_mapdev(test->raw_target,
454e9497f9bSConrad Meyer test->buffer_size);
455e9497f9bSConrad Meyer }
456e9497f9bSConrad Meyer }
457e9497f9bSConrad Meyer
4587c69db50SConrad Meyer index = g_thread_index++;
4597c69db50SConrad Meyer TAILQ_INIT(&test->free_q);
4607c69db50SConrad Meyer TAILQ_INIT(&test->pend_q);
4617c69db50SConrad Meyer
4627c69db50SConrad Meyer if (test->duration == 0)
4631c25420eSConrad Meyer ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n",
4647c69db50SConrad Meyer index, test->transactions);
4657c69db50SConrad Meyer else
4661c25420eSConrad Meyer ioat_test_log(1, "Thread %d: starting\n", index);
4677c69db50SConrad Meyer
4687c69db50SConrad Meyer rc = ioat_test_prealloc_memory(test, index);
4697c69db50SConrad Meyer if (rc != 0) {
4701c25420eSConrad Meyer ioat_test_log(0, "prealloc_memory: %d\n", rc);
471466b3540SConrad Meyer goto out;
4727c69db50SConrad Meyer }
473e974f91cSConrad Meyer wmb();
474e974f91cSConrad Meyer
4757c69db50SConrad Meyer test->too_late = false;
4767c69db50SConrad Meyer start = ticks;
4777c69db50SConrad Meyer end = start + (((sbintime_t)test->duration * hz) / 1000);
4787c69db50SConrad Meyer
4797c69db50SConrad Meyer for (loops = 0;; loops++) {
4807c69db50SConrad Meyer if (test->duration == 0 && loops >= test->transactions)
4817c69db50SConrad Meyer break;
4827c69db50SConrad Meyer else if (test->duration != 0 && time_after(ticks, end)) {
4837c69db50SConrad Meyer test->too_late = true;
4847c69db50SConrad Meyer break;
485e974f91cSConrad Meyer }
486e974f91cSConrad Meyer
4877c69db50SConrad Meyer ioat_test_submit_1_tx(test, dmaengine);
488e974f91cSConrad Meyer }
489e974f91cSConrad Meyer
4901c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n",
4917c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz);
492e974f91cSConrad Meyer
4937c69db50SConrad Meyer IT_LOCK();
4947c69db50SConrad Meyer while (!TAILQ_EMPTY(&test->pend_q))
4957c69db50SConrad Meyer msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);
4967c69db50SConrad Meyer IT_UNLOCK();
4977c69db50SConrad Meyer
4981c25420eSConrad Meyer ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n",
4997c69db50SConrad Meyer ticks - start, ticks - end, (ticks - start) / hz);
5007c69db50SConrad Meyer
5017c69db50SConrad Meyer ioat_test_release_memory(test);
502466b3540SConrad Meyer out:
503e9497f9bSConrad Meyer if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual)
5047ae99f80SJohn Baldwin pmap_unmapdev(test->raw_vtarget, test->buffer_size);
505466b3540SConrad Meyer ioat_put_dmaengine(dmaengine);
506e974f91cSConrad Meyer }
507e974f91cSConrad Meyer
508e974f91cSConrad Meyer static int
ioat_test_open(struct cdev * dev,int flags,int fmt,struct thread * td)509e974f91cSConrad Meyer ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td)
510e974f91cSConrad Meyer {
511e974f91cSConrad Meyer
512e974f91cSConrad Meyer return (0);
513e974f91cSConrad Meyer }
514e974f91cSConrad Meyer
515e974f91cSConrad Meyer static int
ioat_test_close(struct cdev * dev,int flags,int fmt,struct thread * td)516e974f91cSConrad Meyer ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td)
517e974f91cSConrad Meyer {
518e974f91cSConrad Meyer
519e974f91cSConrad Meyer return (0);
520e974f91cSConrad Meyer }
521e974f91cSConrad Meyer
522e974f91cSConrad Meyer static int
ioat_test_ioctl(struct cdev * dev,unsigned long cmd,caddr_t arg,int flag,struct thread * td)523e974f91cSConrad Meyer ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag,
524e974f91cSConrad Meyer struct thread *td)
525e974f91cSConrad Meyer {
526e974f91cSConrad Meyer
527e974f91cSConrad Meyer switch (cmd) {
528e974f91cSConrad Meyer case IOAT_DMATEST:
529e974f91cSConrad Meyer ioat_dma_test(arg);
530e974f91cSConrad Meyer break;
531e974f91cSConrad Meyer default:
532e974f91cSConrad Meyer return (EINVAL);
533e974f91cSConrad Meyer }
534e974f91cSConrad Meyer return (0);
535e974f91cSConrad Meyer }
536e974f91cSConrad Meyer
537e974f91cSConrad Meyer static struct cdevsw ioat_cdevsw = {
538e974f91cSConrad Meyer .d_version = D_VERSION,
539e974f91cSConrad Meyer .d_flags = 0,
540e974f91cSConrad Meyer .d_open = ioat_test_open,
541e974f91cSConrad Meyer .d_close = ioat_test_close,
542e974f91cSConrad Meyer .d_ioctl = ioat_test_ioctl,
543e974f91cSConrad Meyer .d_name = "ioat_test",
544e974f91cSConrad Meyer };
545e974f91cSConrad Meyer
546e974f91cSConrad Meyer static int
enable_ioat_test(bool enable)5477afbb263SConrad Meyer enable_ioat_test(bool enable)
5487afbb263SConrad Meyer {
54971bf3900SAlexander Motin struct make_dev_args devargs;
55071bf3900SAlexander Motin int error = 0;
5517afbb263SConrad Meyer
5527afbb263SConrad Meyer if (enable && g_ioat_cdev == NULL) {
55371bf3900SAlexander Motin make_dev_args_init(&devargs);
55471bf3900SAlexander Motin devargs.mda_devsw = &ioat_cdevsw;
55571bf3900SAlexander Motin devargs.mda_uid = UID_ROOT;
55671bf3900SAlexander Motin devargs.mda_gid = GID_WHEEL;
55771bf3900SAlexander Motin devargs.mda_mode = 0600;
55871bf3900SAlexander Motin error = make_dev_s(&devargs, &g_ioat_cdev, "ioat_test");
5597afbb263SConrad Meyer } else if (!enable && g_ioat_cdev != NULL) {
5607afbb263SConrad Meyer destroy_dev(g_ioat_cdev);
5617afbb263SConrad Meyer g_ioat_cdev = NULL;
5627afbb263SConrad Meyer }
56371bf3900SAlexander Motin return (error);
5647afbb263SConrad Meyer }
5657afbb263SConrad Meyer
5667afbb263SConrad Meyer static int
sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)567e974f91cSConrad Meyer sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS)
568e974f91cSConrad Meyer {
569e974f91cSConrad Meyer int error, enabled;
570e974f91cSConrad Meyer
571e974f91cSConrad Meyer enabled = (g_ioat_cdev != NULL);
572e974f91cSConrad Meyer error = sysctl_handle_int(oidp, &enabled, 0, req);
573e974f91cSConrad Meyer if (error != 0 || req->newptr == NULL)
574e974f91cSConrad Meyer return (error);
575e974f91cSConrad Meyer
57671bf3900SAlexander Motin return (enable_ioat_test(enabled));
577e974f91cSConrad Meyer }
5787029da5cSPawel Biernacki SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test,
57971bf3900SAlexander Motin CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
5807029da5cSPawel Biernacki sysctl_enable_ioat_test, "I",
581e974f91cSConrad Meyer "Non-zero: Enable the /dev/ioat_test device");
5827afbb263SConrad Meyer
5837afbb263SConrad Meyer void
ioat_test_attach(void)5847afbb263SConrad Meyer ioat_test_attach(void)
5857afbb263SConrad Meyer {
5867afbb263SConrad Meyer char *val;
5877afbb263SConrad Meyer
5887afbb263SConrad Meyer val = kern_getenv("hw.ioat.enable_ioat_test");
58971bf3900SAlexander Motin if (val != NULL && strcmp(val, "0") != 0)
5907afbb263SConrad Meyer enable_ioat_test(true);
5917afbb263SConrad Meyer freeenv(val);
5927afbb263SConrad Meyer }
5937afbb263SConrad Meyer
5947afbb263SConrad Meyer void
ioat_test_detach(void)5957afbb263SConrad Meyer ioat_test_detach(void)
5967afbb263SConrad Meyer {
5977afbb263SConrad Meyer
5987afbb263SConrad Meyer enable_ioat_test(false);
5997afbb263SConrad Meyer }
6001c25420eSConrad Meyer
6011a140621SAndriy Gapon static void
_ioat_test_log(int verbosity,const char * fmt,...)6021c25420eSConrad Meyer _ioat_test_log(int verbosity, const char *fmt, ...)
6031c25420eSConrad Meyer {
6041c25420eSConrad Meyer va_list argp;
6051c25420eSConrad Meyer
6061c25420eSConrad Meyer if (verbosity > g_ioat_debug_level)
6071c25420eSConrad Meyer return;
6081c25420eSConrad Meyer
6091c25420eSConrad Meyer va_start(argp, fmt);
6101c25420eSConrad Meyer vprintf(fmt, argp);
6111c25420eSConrad Meyer va_end(argp);
6121c25420eSConrad Meyer }
613