1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 __FBSDID("$FreeBSD$"); 28 29 #ifndef __IOAT_INTERNAL_H__ 30 #define __IOAT_INTERNAL_H__ 31 32 #include <sys/_task.h> 33 34 #define DEVICE2SOFTC(dev) ((struct ioat_softc *) device_get_softc(dev)) 35 #define KTR_IOAT KTR_SPARE3 36 37 #define ioat_read_chancnt(ioat) \ 38 ioat_read_1((ioat), IOAT_CHANCNT_OFFSET) 39 40 #define ioat_read_xfercap(ioat) \ 41 (ioat_read_1((ioat), IOAT_XFERCAP_OFFSET) & IOAT_XFERCAP_VALID_MASK) 42 43 #define ioat_write_intrctrl(ioat, value) \ 44 ioat_write_1((ioat), IOAT_INTRCTRL_OFFSET, (value)) 45 46 #define ioat_read_cbver(ioat) \ 47 (ioat_read_1((ioat), IOAT_CBVER_OFFSET) & 0xFF) 48 49 #define ioat_read_dmacapability(ioat) \ 50 ioat_read_4((ioat), IOAT_DMACAPABILITY_OFFSET) 51 52 #define ioat_write_chanctrl(ioat, value) \ 53 ioat_write_2((ioat), IOAT_CHANCTRL_OFFSET, (value)) 54 55 static __inline uint64_t 56 ioat_bus_space_read_8_lower_first(bus_space_tag_t tag, 57 bus_space_handle_t handle, bus_size_t offset) 58 { 59 return (bus_space_read_4(tag, handle, offset) | 60 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 61 } 62 63 static __inline void 64 ioat_bus_space_write_8_lower_first(bus_space_tag_t tag, 65 bus_space_handle_t handle, bus_size_t offset, uint64_t val) 66 { 67 bus_space_write_4(tag, handle, offset, val); 68 bus_space_write_4(tag, handle, offset + 4, val >> 32); 69 } 70 71 #ifdef __i386__ 72 #define ioat_bus_space_read_8 ioat_bus_space_read_8_lower_first 73 #define ioat_bus_space_write_8 ioat_bus_space_write_8_lower_first 74 #else 75 #define ioat_bus_space_read_8(tag, handle, offset) \ 76 bus_space_read_8((tag), (handle), (offset)) 77 #define ioat_bus_space_write_8(tag, handle, offset, val) \ 78 bus_space_write_8((tag), (handle), (offset), (val)) 79 #endif 80 81 #define ioat_read_1(ioat, offset) \ 82 bus_space_read_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 83 (offset)) 84 85 #define ioat_read_2(ioat, offset) \ 86 bus_space_read_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 87 (offset)) 88 89 #define ioat_read_4(ioat, offset) \ 90 bus_space_read_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 91 (offset)) 92 93 #define ioat_read_8(ioat, offset) \ 94 ioat_bus_space_read_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 95 (offset)) 96 97 #define ioat_read_double_4(ioat, offset) \ 98 ioat_bus_space_read_8_lower_first((ioat)->pci_bus_tag, \ 99 (ioat)->pci_bus_handle, (offset)) 100 101 #define ioat_write_1(ioat, offset, value) \ 102 bus_space_write_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 103 (offset), (value)) 104 105 #define ioat_write_2(ioat, offset, value) \ 106 bus_space_write_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 107 (offset), (value)) 108 109 #define ioat_write_4(ioat, offset, value) \ 110 bus_space_write_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 111 (offset), (value)) 112 113 #define ioat_write_8(ioat, offset, value) \ 114 ioat_bus_space_write_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 115 (offset), (value)) 116 117 #define ioat_write_double_4(ioat, offset, value) \ 118 ioat_bus_space_write_8_lower_first((ioat)->pci_bus_tag, \ 119 (ioat)->pci_bus_handle, (offset), (value)) 120 121 MALLOC_DECLARE(M_IOAT); 122 123 SYSCTL_DECL(_hw_ioat); 124 125 extern int g_ioat_debug_level; 126 127 struct generic_dma_control { 128 uint32_t int_enable:1; 129 uint32_t src_snoop_disable:1; 130 uint32_t dest_snoop_disable:1; 131 uint32_t completion_update:1; 132 uint32_t fence:1; 133 uint32_t reserved1:1; 134 uint32_t src_page_break:1; 135 uint32_t dest_page_break:1; 136 uint32_t bundle:1; 137 uint32_t dest_dca:1; 138 uint32_t hint:1; 139 uint32_t reserved2:13; 140 uint32_t op:8; 141 }; 142 143 struct ioat_generic_hw_descriptor { 144 uint32_t size; 145 union { 146 uint32_t control_raw; 147 struct generic_dma_control control_generic; 148 } u; 149 uint64_t src_addr; 150 uint64_t dest_addr; 151 uint64_t next; 152 uint64_t reserved[4]; 153 }; 154 155 struct ioat_dma_hw_descriptor { 156 uint32_t size; 157 union { 158 uint32_t control_raw; 159 struct generic_dma_control control_generic; 160 struct { 161 uint32_t int_enable:1; 162 uint32_t src_snoop_disable:1; 163 uint32_t dest_snoop_disable:1; 164 uint32_t completion_update:1; 165 uint32_t fence:1; 166 uint32_t null:1; 167 uint32_t src_page_break:1; 168 uint32_t dest_page_break:1; 169 uint32_t bundle:1; 170 uint32_t dest_dca:1; 171 uint32_t hint:1; 172 uint32_t reserved:13; 173 #define IOAT_OP_COPY 0x00 174 uint32_t op:8; 175 } control; 176 } u; 177 uint64_t src_addr; 178 uint64_t dest_addr; 179 uint64_t next; 180 uint64_t next_src_addr; 181 uint64_t next_dest_addr; 182 uint64_t user1; 183 uint64_t user2; 184 }; 185 186 struct ioat_fill_hw_descriptor { 187 uint32_t size; 188 union { 189 uint32_t control_raw; 190 struct generic_dma_control control_generic; 191 struct { 192 uint32_t int_enable:1; 193 uint32_t reserved:1; 194 uint32_t dest_snoop_disable:1; 195 uint32_t completion_update:1; 196 uint32_t fence:1; 197 uint32_t reserved2:2; 198 uint32_t dest_page_break:1; 199 uint32_t bundle:1; 200 uint32_t reserved3:15; 201 #define IOAT_OP_FILL 0x01 202 uint32_t op:8; 203 } control; 204 } u; 205 uint64_t src_data; 206 uint64_t dest_addr; 207 uint64_t next; 208 uint64_t reserved; 209 uint64_t next_dest_addr; 210 uint64_t user1; 211 uint64_t user2; 212 }; 213 214 struct ioat_xor_hw_descriptor { 215 uint32_t size; 216 union { 217 uint32_t control_raw; 218 struct generic_dma_control control_generic; 219 struct { 220 uint32_t int_enable:1; 221 uint32_t src_snoop_disable:1; 222 uint32_t dest_snoop_disable:1; 223 uint32_t completion_update:1; 224 uint32_t fence:1; 225 uint32_t src_count:3; 226 uint32_t bundle:1; 227 uint32_t dest_dca:1; 228 uint32_t hint:1; 229 uint32_t reserved:13; 230 #define IOAT_OP_XOR 0x87 231 #define IOAT_OP_XOR_VAL 0x88 232 uint32_t op:8; 233 } control; 234 } u; 235 uint64_t src_addr; 236 uint64_t dest_addr; 237 uint64_t next; 238 uint64_t src_addr2; 239 uint64_t src_addr3; 240 uint64_t src_addr4; 241 uint64_t src_addr5; 242 }; 243 244 struct ioat_xor_ext_hw_descriptor { 245 uint64_t src_addr6; 246 uint64_t src_addr7; 247 uint64_t src_addr8; 248 uint64_t next; 249 uint64_t reserved[4]; 250 }; 251 252 struct ioat_pq_hw_descriptor { 253 uint32_t size; 254 union { 255 uint32_t control_raw; 256 struct generic_dma_control control_generic; 257 struct { 258 uint32_t int_enable:1; 259 uint32_t src_snoop_disable:1; 260 uint32_t dest_snoop_disable:1; 261 uint32_t completion_update:1; 262 uint32_t fence:1; 263 uint32_t src_count:3; 264 uint32_t bundle:1; 265 uint32_t dest_dca:1; 266 uint32_t hint:1; 267 uint32_t p_disable:1; 268 uint32_t q_disable:1; 269 uint32_t reserved:11; 270 #define IOAT_OP_PQ 0x89 271 #define IOAT_OP_PQ_VAL 0x8a 272 uint32_t op:8; 273 } control; 274 } u; 275 uint64_t src_addr; 276 uint64_t p_addr; 277 uint64_t next; 278 uint64_t src_addr2; 279 uint64_t src_addr3; 280 uint8_t coef[8]; 281 uint64_t q_addr; 282 }; 283 284 struct ioat_pq_ext_hw_descriptor { 285 uint64_t src_addr4; 286 uint64_t src_addr5; 287 uint64_t src_addr6; 288 uint64_t next; 289 uint64_t src_addr7; 290 uint64_t src_addr8; 291 uint64_t reserved[2]; 292 }; 293 294 struct ioat_pq_update_hw_descriptor { 295 uint32_t size; 296 union { 297 uint32_t control_raw; 298 struct generic_dma_control control_generic; 299 struct { 300 uint32_t int_enable:1; 301 uint32_t src_snoop_disable:1; 302 uint32_t dest_snoop_disable:1; 303 uint32_t completion_update:1; 304 uint32_t fence:1; 305 uint32_t src_cnt:3; 306 uint32_t bundle:1; 307 uint32_t dest_dca:1; 308 uint32_t hint:1; 309 uint32_t p_disable:1; 310 uint32_t q_disable:1; 311 uint32_t reserved:3; 312 uint32_t coef:8; 313 #define IOAT_OP_PQ_UP 0x8b 314 uint32_t op:8; 315 } control; 316 } u; 317 uint64_t src_addr; 318 uint64_t p_addr; 319 uint64_t next; 320 uint64_t src_addr2; 321 uint64_t p_src; 322 uint64_t q_src; 323 uint64_t q_addr; 324 }; 325 326 struct ioat_raw_hw_descriptor { 327 uint64_t field[8]; 328 }; 329 330 struct bus_dmadesc { 331 bus_dmaengine_callback_t callback_fn; 332 void *callback_arg; 333 }; 334 335 struct ioat_descriptor { 336 struct bus_dmadesc bus_dmadesc; 337 union { 338 struct ioat_generic_hw_descriptor *generic; 339 struct ioat_dma_hw_descriptor *dma; 340 struct ioat_fill_hw_descriptor *fill; 341 struct ioat_xor_hw_descriptor *xor; 342 struct ioat_xor_ext_hw_descriptor *xor_ext; 343 struct ioat_pq_hw_descriptor *pq; 344 struct ioat_pq_ext_hw_descriptor *pq_ext; 345 struct ioat_raw_hw_descriptor *raw; 346 } u; 347 uint32_t id; 348 bus_addr_t hw_desc_bus_addr; 349 }; 350 351 /* Unsupported by this driver at this time. */ 352 #define IOAT_OP_MOVECRC 0x41 353 #define IOAT_OP_MOVECRC_TEST 0x42 354 #define IOAT_OP_MOVECRC_STORE 0x43 355 #define IOAT_OP_CRC 0x81 356 #define IOAT_OP_CRC_TEST 0x82 357 #define IOAT_OP_CRC_STORE 0x83 358 #define IOAT_OP_MARKER 0x84 359 360 /* 361 * Deprecated OPs -- v3 DMA generates an abort if given these. And this driver 362 * doesn't support anything older than v3. 363 */ 364 #define IOAT_OP_OLD_XOR 0x85 365 #define IOAT_OP_OLD_XOR_VAL 0x86 366 367 enum ioat_ref_kind { 368 IOAT_DMAENGINE_REF = 0, 369 IOAT_ACTIVE_DESCR_REF, 370 IOAT_NUM_REF_KINDS 371 }; 372 373 /* One of these per allocated PCI device. */ 374 struct ioat_softc { 375 bus_dmaengine_t dmaengine; 376 #define to_ioat_softc(_dmaeng) \ 377 ({ \ 378 bus_dmaengine_t *_p = (_dmaeng); \ 379 (struct ioat_softc *)((char *)_p - \ 380 offsetof(struct ioat_softc, dmaengine)); \ 381 }) 382 383 int version; 384 int chan_idx; 385 386 struct mtx submit_lock; 387 device_t device; 388 bus_space_tag_t pci_bus_tag; 389 bus_space_handle_t pci_bus_handle; 390 int pci_resource_id; 391 struct resource *pci_resource; 392 uint32_t max_xfer_size; 393 uint32_t capabilities; 394 uint16_t intrdelay_max; 395 uint16_t cached_intrdelay; 396 397 struct resource *res; 398 int rid; 399 void *tag; 400 401 bus_dma_tag_t hw_desc_tag; 402 bus_dmamap_t hw_desc_map; 403 404 bus_dma_tag_t comp_update_tag; 405 bus_dmamap_t comp_update_map; 406 uint64_t *comp_update; 407 bus_addr_t comp_update_bus_addr; 408 409 struct callout timer; 410 struct task reset_task; 411 412 boolean_t quiescing; 413 boolean_t is_resize_pending; 414 boolean_t is_completion_pending; 415 boolean_t is_reset_pending; 416 boolean_t is_channel_running; 417 boolean_t intrdelay_supported; 418 419 uint32_t head; 420 uint32_t tail; 421 uint32_t hw_head; 422 uint32_t ring_size_order; 423 bus_addr_t last_seen; 424 425 struct ioat_descriptor **ring; 426 427 struct mtx cleanup_lock; 428 volatile uint32_t refcnt; 429 #ifdef INVARIANTS 430 volatile uint32_t refkinds[IOAT_NUM_REF_KINDS]; 431 #endif 432 433 struct { 434 uint64_t interrupts; 435 uint64_t descriptors_processed; 436 uint64_t descriptors_error; 437 uint64_t descriptors_submitted; 438 439 uint32_t channel_halts; 440 uint32_t last_halt_chanerr; 441 } stats; 442 }; 443 444 void ioat_test_attach(void); 445 void ioat_test_detach(void); 446 447 static inline uint64_t 448 ioat_get_chansts(struct ioat_softc *ioat) 449 { 450 uint64_t status; 451 452 if (ioat->version >= IOAT_VER_3_3) 453 status = ioat_read_8(ioat, IOAT_CHANSTS_OFFSET); 454 else 455 /* Must read lower 4 bytes before upper 4 bytes. */ 456 status = ioat_read_double_4(ioat, IOAT_CHANSTS_OFFSET); 457 return (status); 458 } 459 460 static inline void 461 ioat_write_chancmp(struct ioat_softc *ioat, uint64_t addr) 462 { 463 464 if (ioat->version >= IOAT_VER_3_3) 465 ioat_write_8(ioat, IOAT_CHANCMP_OFFSET_LOW, addr); 466 else 467 ioat_write_double_4(ioat, IOAT_CHANCMP_OFFSET_LOW, addr); 468 } 469 470 static inline void 471 ioat_write_chainaddr(struct ioat_softc *ioat, uint64_t addr) 472 { 473 474 if (ioat->version >= IOAT_VER_3_3) 475 ioat_write_8(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr); 476 else 477 ioat_write_double_4(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr); 478 } 479 480 static inline boolean_t 481 is_ioat_active(uint64_t status) 482 { 483 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE); 484 } 485 486 static inline boolean_t 487 is_ioat_idle(uint64_t status) 488 { 489 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_IDLE); 490 } 491 492 static inline boolean_t 493 is_ioat_halted(uint64_t status) 494 { 495 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED); 496 } 497 498 static inline boolean_t 499 is_ioat_suspended(uint64_t status) 500 { 501 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED); 502 } 503 504 static inline void 505 ioat_suspend(struct ioat_softc *ioat) 506 { 507 ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_SUSPEND); 508 } 509 510 static inline void 511 ioat_reset(struct ioat_softc *ioat) 512 { 513 ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); 514 } 515 516 static inline boolean_t 517 ioat_reset_pending(struct ioat_softc *ioat) 518 { 519 uint8_t cmd; 520 521 cmd = ioat_read_1(ioat, IOAT_CHANCMD_OFFSET); 522 return ((cmd & IOAT_CHANCMD_RESET) != 0); 523 } 524 525 #endif /* __IOAT_INTERNAL_H__ */ 526