1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <hxge_impl.h> 27 #include <hxge_rxdma.h> 28 29 /* 30 * Number of blocks to accumulate before re-enabling DMA 31 * when we get RBR empty. 32 */ 33 #define HXGE_RBR_EMPTY_THRESHOLD 64 34 35 /* 36 * Globals: tunable parameters (/etc/system or adb) 37 * 38 */ 39 extern uint32_t hxge_rbr_size; 40 extern uint32_t hxge_rcr_size; 41 extern uint32_t hxge_rbr_spare_size; 42 extern uint32_t hxge_mblks_pending; 43 44 /* 45 * Tunable to reduce the amount of time spent in the 46 * ISR doing Rx Processing. 47 */ 48 extern uint32_t hxge_max_rx_pkts; 49 50 /* 51 * Tunables to manage the receive buffer blocks. 52 * 53 * hxge_rx_threshold_hi: copy all buffers. 54 * hxge_rx_bcopy_size_type: receive buffer block size type. 55 * hxge_rx_threshold_lo: copy only up to tunable block size type. 56 */ 57 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi; 58 extern hxge_rxbuf_type_t hxge_rx_buf_size_type; 59 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo; 60 61 /* 62 * Static local functions. 63 */ 64 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep); 65 static void hxge_unmap_rxdma(p_hxge_t hxgep); 66 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep); 67 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep); 68 static void hxge_rxdma_hw_stop(p_hxge_t hxgep); 69 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 70 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 71 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 72 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 73 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 74 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 75 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 76 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, 77 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p, 78 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 79 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p); 80 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 81 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p); 82 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, 83 uint16_t channel, p_hxge_dma_common_t *dma_buf_p, 84 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks); 85 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 86 p_rx_rbr_ring_t rbr_p); 87 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 88 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 89 int n_init_kick); 90 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel); 91 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 92 p_rx_rcr_ring_t *rcr_p, rdc_stat_t cs); 93 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p, 94 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, 95 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry); 96 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep, 97 uint16_t channel); 98 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t); 99 static void hxge_freeb(p_rx_msg_t); 100 static void hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, 101 p_hxge_ldv_t ldvp, rdc_stat_t cs); 102 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, 103 p_hxge_ldv_t ldvp, rdc_stat_t cs); 104 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep, 105 p_rx_rbr_ring_t rx_dmap); 106 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, 107 uint16_t channel); 108 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep); 109 110 hxge_status_t 111 hxge_init_rxdma_channels(p_hxge_t hxgep) 112 { 113 hxge_status_t status = HXGE_OK; 114 block_reset_t reset_reg; 115 116 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels")); 117 118 /* Reset RDC block from PEU to clear any previous state */ 119 reset_reg.value = 0; 120 reset_reg.bits.rdc_rst = 1; 121 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 122 HXGE_DELAY(1000); 123 124 status = hxge_map_rxdma(hxgep); 125 if (status != HXGE_OK) { 126 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 127 "<== hxge_init_rxdma: status 0x%x", status)); 128 return (status); 129 } 130 131 status = hxge_rxdma_hw_start_common(hxgep); 132 if (status != HXGE_OK) { 133 hxge_unmap_rxdma(hxgep); 134 } 135 136 status = hxge_rxdma_hw_start(hxgep); 137 if (status != HXGE_OK) { 138 hxge_unmap_rxdma(hxgep); 139 } 140 141 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 142 "<== hxge_init_rxdma_channels: status 0x%x", status)); 143 return (status); 144 } 145 146 void 147 hxge_uninit_rxdma_channels(p_hxge_t hxgep) 148 { 149 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels")); 150 151 hxge_rxdma_hw_stop(hxgep); 152 hxge_unmap_rxdma(hxgep); 153 154 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels")); 155 } 156 157 hxge_status_t 158 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel, 159 rdc_stat_t *cs_p) 160 { 161 hpi_handle_t handle; 162 hpi_status_t rs = HPI_SUCCESS; 163 hxge_status_t status = HXGE_OK; 164 165 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 166 "<== hxge_init_rxdma_channel_cntl_stat")); 167 168 handle = HXGE_DEV_HPI_HANDLE(hxgep); 169 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p); 170 171 if (rs != HPI_SUCCESS) { 172 status = HXGE_ERROR | rs; 173 } 174 return (status); 175 } 176 177 178 hxge_status_t 179 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 180 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 181 int n_init_kick) 182 { 183 hpi_handle_t handle; 184 rdc_desc_cfg_t rdc_desc; 185 rdc_rcr_cfg_b_t *cfgb_p; 186 hpi_status_t rs = HPI_SUCCESS; 187 188 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel")); 189 handle = HXGE_DEV_HPI_HANDLE(hxgep); 190 191 /* 192 * Use configuration data composed at init time. Write to hardware the 193 * receive ring configurations. 194 */ 195 rdc_desc.mbox_enable = 1; 196 rdc_desc.mbox_addr = mbox_p->mbox_addr; 197 HXGE_DEBUG_MSG((hxgep, RX_CTL, 198 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)", 199 mbox_p->mbox_addr, rdc_desc.mbox_addr)); 200 201 rdc_desc.rbr_len = rbr_p->rbb_max; 202 rdc_desc.rbr_addr = rbr_p->rbr_addr; 203 204 switch (hxgep->rx_bksize_code) { 205 case RBR_BKSIZE_4K: 206 rdc_desc.page_size = SIZE_4KB; 207 break; 208 case RBR_BKSIZE_8K: 209 rdc_desc.page_size = SIZE_8KB; 210 break; 211 } 212 213 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0; 214 rdc_desc.valid0 = 1; 215 216 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1; 217 rdc_desc.valid1 = 1; 218 219 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2; 220 rdc_desc.valid2 = 1; 221 222 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 223 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 224 225 rdc_desc.rcr_len = rcr_p->comp_size; 226 rdc_desc.rcr_addr = rcr_p->rcr_addr; 227 228 cfgb_p = &(rcr_p->rcr_cfgb); 229 rdc_desc.rcr_threshold = cfgb_p->bits.pthres; 230 rdc_desc.rcr_timeout = cfgb_p->bits.timeout; 231 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout; 232 233 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 234 "rbr_len qlen %d pagesize code %d rcr_len %d", 235 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 236 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: " 237 "size 0 %d size 1 %d size 2 %d", 238 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1, 239 rbr_p->hpi_pkt_buf_size2)); 240 241 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 242 if (rs != HPI_SUCCESS) { 243 return (HXGE_ERROR | rs); 244 } 245 246 /* 247 * Enable the timeout and threshold. 248 */ 249 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 250 rdc_desc.rcr_threshold); 251 if (rs != HPI_SUCCESS) { 252 return (HXGE_ERROR | rs); 253 } 254 255 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 256 rdc_desc.rcr_timeout); 257 if (rs != HPI_SUCCESS) { 258 return (HXGE_ERROR | rs); 259 } 260 261 /* Enable the DMA */ 262 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 263 if (rs != HPI_SUCCESS) { 264 return (HXGE_ERROR | rs); 265 } 266 267 /* Kick the DMA engine */ 268 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick); 269 270 /* Clear the rbr empty bit */ 271 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel); 272 273 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel")); 274 275 return (HXGE_OK); 276 } 277 278 static hxge_status_t 279 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel) 280 { 281 hpi_handle_t handle; 282 hpi_status_t rs = HPI_SUCCESS; 283 284 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel")); 285 286 handle = HXGE_DEV_HPI_HANDLE(hxgep); 287 288 /* disable the DMA */ 289 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 290 if (rs != HPI_SUCCESS) { 291 HXGE_DEBUG_MSG((hxgep, RX_CTL, 292 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs)); 293 return (HXGE_ERROR | rs); 294 } 295 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel")); 296 return (HXGE_OK); 297 } 298 299 hxge_status_t 300 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel) 301 { 302 hpi_handle_t handle; 303 hxge_status_t status = HXGE_OK; 304 305 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 306 "==> hxge_rxdma_channel_rcrflush")); 307 308 handle = HXGE_DEV_HPI_HANDLE(hxgep); 309 hpi_rxdma_rdc_rcr_flush(handle, channel); 310 311 HXGE_DEBUG_MSG((hxgep, DMA_CTL, 312 "<== hxge_rxdma_channel_rcrflush")); 313 return (status); 314 315 } 316 317 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 318 319 #define TO_LEFT -1 320 #define TO_RIGHT 1 321 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 322 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 323 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 324 #define NO_HINT 0xffffffff 325 326 /*ARGSUSED*/ 327 hxge_status_t 328 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p, 329 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 330 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 331 { 332 int bufsize; 333 uint64_t pktbuf_pp; 334 uint64_t dvma_addr; 335 rxring_info_t *ring_info; 336 int base_side, end_side; 337 int r_index, l_index, anchor_index; 338 int found, search_done; 339 uint32_t offset, chunk_size, block_size, page_size_mask; 340 uint32_t chunk_index, block_index, total_index; 341 int max_iterations, iteration; 342 rxbuf_index_info_t *bufinfo; 343 344 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp")); 345 346 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 347 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 348 pkt_buf_addr_pp, pktbufsz_type)); 349 350 #if defined(__i386) 351 pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 352 #else 353 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 354 #endif 355 356 switch (pktbufsz_type) { 357 case 0: 358 bufsize = rbr_p->pkt_buf_size0; 359 break; 360 case 1: 361 bufsize = rbr_p->pkt_buf_size1; 362 break; 363 case 2: 364 bufsize = rbr_p->pkt_buf_size2; 365 break; 366 case RCR_SINGLE_BLOCK: 367 bufsize = 0; 368 anchor_index = 0; 369 break; 370 default: 371 return (HXGE_ERROR); 372 } 373 374 if (rbr_p->num_blocks == 1) { 375 anchor_index = 0; 376 ring_info = rbr_p->ring_info; 377 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 378 379 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 380 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) " 381 "buf_pp $%p btype %d anchor_index %d bufinfo $%p", 382 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo)); 383 384 goto found_index; 385 } 386 387 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 388 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d", 389 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 390 391 ring_info = rbr_p->ring_info; 392 found = B_FALSE; 393 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 394 iteration = 0; 395 max_iterations = ring_info->max_iterations; 396 397 /* 398 * First check if this block have been seen recently. This is indicated 399 * by a hint which is initialized when the first buffer of the block is 400 * seen. The hint is reset when the last buffer of the block has been 401 * processed. As three block sizes are supported, three hints are kept. 402 * The idea behind the hints is that once the hardware uses a block 403 * for a buffer of that size, it will use it exclusively for that size 404 * and will use it until it is exhausted. It is assumed that there 405 * would a single block being used for the same buffer sizes at any 406 * given time. 407 */ 408 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 409 anchor_index = ring_info->hint[pktbufsz_type]; 410 dvma_addr = bufinfo[anchor_index].dvma_addr; 411 chunk_size = bufinfo[anchor_index].buf_size; 412 if ((pktbuf_pp >= dvma_addr) && 413 (pktbuf_pp < (dvma_addr + chunk_size))) { 414 found = B_TRUE; 415 /* 416 * check if this is the last buffer in the block If so, 417 * then reset the hint for the size; 418 */ 419 420 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 421 ring_info->hint[pktbufsz_type] = NO_HINT; 422 } 423 } 424 425 if (found == B_FALSE) { 426 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 427 "==> hxge_rxbuf_pp_to_vp: (!found)" 428 "buf_pp $%p btype %d anchor_index %d", 429 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 430 431 /* 432 * This is the first buffer of the block of this size. Need to 433 * search the whole information array. the search algorithm 434 * uses a binary tree search algorithm. It assumes that the 435 * information is already sorted with increasing order info[0] 436 * < info[1] < info[2] .... < info[n-1] where n is the size of 437 * the information array 438 */ 439 r_index = rbr_p->num_blocks - 1; 440 l_index = 0; 441 search_done = B_FALSE; 442 anchor_index = MID_INDEX(r_index, l_index); 443 while (search_done == B_FALSE) { 444 if ((r_index == l_index) || 445 (iteration >= max_iterations)) 446 search_done = B_TRUE; 447 448 end_side = TO_RIGHT; /* to the right */ 449 base_side = TO_LEFT; /* to the left */ 450 /* read the DVMA address information and sort it */ 451 dvma_addr = bufinfo[anchor_index].dvma_addr; 452 chunk_size = bufinfo[anchor_index].buf_size; 453 454 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 455 "==> hxge_rxbuf_pp_to_vp: (searching)" 456 "buf_pp $%p btype %d " 457 "anchor_index %d chunk_size %d dvmaaddr $%p", 458 pkt_buf_addr_pp, pktbufsz_type, anchor_index, 459 chunk_size, dvma_addr)); 460 461 if (pktbuf_pp >= dvma_addr) 462 base_side = TO_RIGHT; /* to the right */ 463 if (pktbuf_pp < (dvma_addr + chunk_size)) 464 end_side = TO_LEFT; /* to the left */ 465 466 switch (base_side + end_side) { 467 case IN_MIDDLE: 468 /* found */ 469 found = B_TRUE; 470 search_done = B_TRUE; 471 if ((pktbuf_pp + bufsize) < 472 (dvma_addr + chunk_size)) 473 ring_info->hint[pktbufsz_type] = 474 bufinfo[anchor_index].buf_index; 475 break; 476 case BOTH_RIGHT: 477 /* not found: go to the right */ 478 l_index = anchor_index + 1; 479 anchor_index = MID_INDEX(r_index, l_index); 480 break; 481 482 case BOTH_LEFT: 483 /* not found: go to the left */ 484 r_index = anchor_index - 1; 485 anchor_index = MID_INDEX(r_index, l_index); 486 break; 487 default: /* should not come here */ 488 return (HXGE_ERROR); 489 } 490 iteration++; 491 } 492 493 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 494 "==> hxge_rxbuf_pp_to_vp: (search done)" 495 "buf_pp $%p btype %d anchor_index %d", 496 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 497 } 498 499 if (found == B_FALSE) { 500 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 501 "==> hxge_rxbuf_pp_to_vp: (search failed)" 502 "buf_pp $%p btype %d anchor_index %d", 503 pkt_buf_addr_pp, pktbufsz_type, anchor_index)); 504 return (HXGE_ERROR); 505 } 506 507 found_index: 508 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 509 "==> hxge_rxbuf_pp_to_vp: (FOUND1)" 510 "buf_pp $%p btype %d bufsize %d anchor_index %d", 511 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index)); 512 513 /* index of the first block in this chunk */ 514 chunk_index = bufinfo[anchor_index].start_index; 515 dvma_addr = bufinfo[anchor_index].dvma_addr; 516 page_size_mask = ring_info->block_size_mask; 517 518 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 519 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 520 "buf_pp $%p btype %d bufsize %d " 521 "anchor_index %d chunk_index %d dvma $%p", 522 pkt_buf_addr_pp, pktbufsz_type, bufsize, 523 anchor_index, chunk_index, dvma_addr)); 524 525 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 526 block_size = rbr_p->block_size; /* System block(page) size */ 527 528 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 529 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 530 "buf_pp $%p btype %d bufsize %d " 531 "anchor_index %d chunk_index %d dvma $%p " 532 "offset %d block_size %d", 533 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index, 534 chunk_index, dvma_addr, offset, block_size)); 535 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index")); 536 537 block_index = (offset / block_size); /* index within chunk */ 538 total_index = chunk_index + block_index; 539 540 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 541 "==> hxge_rxbuf_pp_to_vp: " 542 "total_index %d dvma_addr $%p " 543 "offset %d block_size %d " 544 "block_index %d ", 545 total_index, dvma_addr, offset, block_size, block_index)); 546 547 #if defined(__i386) 548 *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 549 (uint32_t)offset); 550 #else 551 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 552 offset); 553 #endif 554 555 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 556 "==> hxge_rxbuf_pp_to_vp: " 557 "total_index %d dvma_addr $%p " 558 "offset %d block_size %d " 559 "block_index %d " 560 "*pkt_buf_addr_p $%p", 561 total_index, dvma_addr, offset, block_size, 562 block_index, *pkt_buf_addr_p)); 563 564 *msg_index = total_index; 565 *bufoffset = (offset & page_size_mask); 566 567 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 568 "==> hxge_rxbuf_pp_to_vp: get msg index: " 569 "msg_index %d bufoffset_index %d", 570 *msg_index, *bufoffset)); 571 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp")); 572 573 return (HXGE_OK); 574 } 575 576 577 /* 578 * used by quick sort (qsort) function 579 * to perform comparison 580 */ 581 static int 582 hxge_sort_compare(const void *p1, const void *p2) 583 { 584 585 rxbuf_index_info_t *a, *b; 586 587 a = (rxbuf_index_info_t *)p1; 588 b = (rxbuf_index_info_t *)p2; 589 590 if (a->dvma_addr > b->dvma_addr) 591 return (1); 592 if (a->dvma_addr < b->dvma_addr) 593 return (-1); 594 return (0); 595 } 596 597 /* 598 * Grabbed this sort implementation from common/syscall/avl.c 599 * 600 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 601 * v = Ptr to array/vector of objs 602 * n = # objs in the array 603 * s = size of each obj (must be multiples of a word size) 604 * f = ptr to function to compare two objs 605 * returns (-1 = less than, 0 = equal, 1 = greater than 606 */ 607 void 608 hxge_ksort(caddr_t v, int n, int s, int (*f) ()) 609 { 610 int g, i, j, ii; 611 unsigned int *p1, *p2; 612 unsigned int tmp; 613 614 /* No work to do */ 615 if (v == NULL || n <= 1) 616 return; 617 /* Sanity check on arguments */ 618 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 619 ASSERT(s > 0); 620 621 for (g = n / 2; g > 0; g /= 2) { 622 for (i = g; i < n; i++) { 623 for (j = i - g; j >= 0 && 624 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) { 625 p1 = (unsigned *)(v + j * s); 626 p2 = (unsigned *)(v + (j + g) * s); 627 for (ii = 0; ii < s / 4; ii++) { 628 tmp = *p1; 629 *p1++ = *p2; 630 *p2++ = tmp; 631 } 632 } 633 } 634 } 635 } 636 637 /* 638 * Initialize data structures required for rxdma 639 * buffer dvma->vmem address lookup 640 */ 641 /*ARGSUSED*/ 642 static hxge_status_t 643 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp) 644 { 645 int index; 646 rxring_info_t *ring_info; 647 int max_iteration = 0, max_index = 0; 648 649 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init")); 650 651 ring_info = rbrp->ring_info; 652 ring_info->hint[0] = NO_HINT; 653 ring_info->hint[1] = NO_HINT; 654 ring_info->hint[2] = NO_HINT; 655 max_index = rbrp->num_blocks; 656 657 /* read the DVMA address information and sort it */ 658 /* do init of the information array */ 659 660 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 661 " hxge_rxbuf_index_info_init Sort ptrs")); 662 663 /* sort the array */ 664 hxge_ksort((void *) ring_info->buffer, max_index, 665 sizeof (rxbuf_index_info_t), hxge_sort_compare); 666 667 for (index = 0; index < max_index; index++) { 668 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 669 " hxge_rxbuf_index_info_init: sorted chunk %d " 670 " ioaddr $%p kaddr $%p size %x", 671 index, ring_info->buffer[index].dvma_addr, 672 ring_info->buffer[index].kaddr, 673 ring_info->buffer[index].buf_size)); 674 } 675 676 max_iteration = 0; 677 while (max_index >= (1ULL << max_iteration)) 678 max_iteration++; 679 ring_info->max_iterations = max_iteration + 1; 680 681 HXGE_DEBUG_MSG((hxgep, DMA2_CTL, 682 " hxge_rxbuf_index_info_init Find max iter %d", 683 ring_info->max_iterations)); 684 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init")); 685 686 return (HXGE_OK); 687 } 688 689 /*ARGSUSED*/ 690 void 691 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p) 692 { 693 #ifdef HXGE_DEBUG 694 695 uint32_t bptr; 696 uint64_t pp; 697 698 bptr = entry_p->bits.pkt_buf_addr; 699 700 HXGE_DEBUG_MSG((hxgep, RX_CTL, 701 "\trcr entry $%p " 702 "\trcr entry 0x%0llx " 703 "\trcr entry 0x%08x " 704 "\trcr entry 0x%08x " 705 "\tvalue 0x%0llx\n" 706 "\tmulti = %d\n" 707 "\tpkt_type = 0x%x\n" 708 "\terror = 0x%04x\n" 709 "\tl2_len = %d\n" 710 "\tpktbufsize = %d\n" 711 "\tpkt_buf_addr = $%p\n" 712 "\tpkt_buf_addr (<< 6) = $%p\n", 713 entry_p, 714 *(int64_t *)entry_p, 715 *(int32_t *)entry_p, 716 *(int32_t *)((char *)entry_p + 32), 717 entry_p->value, 718 entry_p->bits.multi, 719 entry_p->bits.pkt_type, 720 entry_p->bits.error, 721 entry_p->bits.l2_len, 722 entry_p->bits.pktbufsz, 723 bptr, 724 entry_p->bits.pkt_buf_addr_l)); 725 726 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 727 RCR_PKT_BUF_ADDR_SHIFT; 728 729 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 730 pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 731 #endif 732 } 733 734 /*ARGSUSED*/ 735 void 736 hxge_rxdma_stop(p_hxge_t hxgep) 737 { 738 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop")); 739 740 (void) hxge_rx_vmac_disable(hxgep); 741 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP); 742 743 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop")); 744 } 745 746 void 747 hxge_rxdma_stop_reinit(p_hxge_t hxgep) 748 { 749 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit")); 750 751 (void) hxge_rxdma_stop(hxgep); 752 (void) hxge_uninit_rxdma_channels(hxgep); 753 (void) hxge_init_rxdma_channels(hxgep); 754 755 (void) hxge_rx_vmac_enable(hxgep); 756 757 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit")); 758 } 759 760 hxge_status_t 761 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable) 762 { 763 int i, ndmas; 764 uint16_t channel; 765 p_rx_rbr_rings_t rx_rbr_rings; 766 p_rx_rbr_ring_t *rbr_rings; 767 hpi_handle_t handle; 768 hpi_status_t rs = HPI_SUCCESS; 769 hxge_status_t status = HXGE_OK; 770 771 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 772 "==> hxge_rxdma_hw_mode: mode %d", enable)); 773 774 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 775 HXGE_DEBUG_MSG((hxgep, RX_CTL, 776 "<== hxge_rxdma_mode: not initialized")); 777 return (HXGE_ERROR); 778 } 779 780 rx_rbr_rings = hxgep->rx_rbr_rings; 781 if (rx_rbr_rings == NULL) { 782 HXGE_DEBUG_MSG((hxgep, RX_CTL, 783 "<== hxge_rxdma_mode: NULL ring pointer")); 784 return (HXGE_ERROR); 785 } 786 787 if (rx_rbr_rings->rbr_rings == NULL) { 788 HXGE_DEBUG_MSG((hxgep, RX_CTL, 789 "<== hxge_rxdma_mode: NULL rbr rings pointer")); 790 return (HXGE_ERROR); 791 } 792 793 ndmas = rx_rbr_rings->ndmas; 794 if (!ndmas) { 795 HXGE_DEBUG_MSG((hxgep, RX_CTL, 796 "<== hxge_rxdma_mode: no channel")); 797 return (HXGE_ERROR); 798 } 799 800 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 801 "==> hxge_rxdma_mode (ndmas %d)", ndmas)); 802 803 rbr_rings = rx_rbr_rings->rbr_rings; 804 805 handle = HXGE_DEV_HPI_HANDLE(hxgep); 806 807 for (i = 0; i < ndmas; i++) { 808 if (rbr_rings == NULL || rbr_rings[i] == NULL) { 809 continue; 810 } 811 channel = rbr_rings[i]->rdc; 812 if (enable) { 813 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 814 "==> hxge_rxdma_hw_mode: channel %d (enable)", 815 channel)); 816 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 817 } else { 818 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 819 "==> hxge_rxdma_hw_mode: channel %d (disable)", 820 channel)); 821 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 822 } 823 } 824 825 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs); 826 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 827 "<== hxge_rxdma_hw_mode: status 0x%x", status)); 828 829 return (status); 830 } 831 832 int 833 hxge_rxdma_get_ring_index(p_hxge_t hxgep, uint16_t channel) 834 { 835 int i, ndmas; 836 uint16_t rdc; 837 p_rx_rbr_rings_t rx_rbr_rings; 838 p_rx_rbr_ring_t *rbr_rings; 839 840 HXGE_DEBUG_MSG((hxgep, RX_CTL, 841 "==> hxge_rxdma_get_ring_index: channel %d", channel)); 842 843 rx_rbr_rings = hxgep->rx_rbr_rings; 844 if (rx_rbr_rings == NULL) { 845 HXGE_DEBUG_MSG((hxgep, RX_CTL, 846 "<== hxge_rxdma_get_ring_index: NULL ring pointer")); 847 return (-1); 848 } 849 850 ndmas = rx_rbr_rings->ndmas; 851 if (!ndmas) { 852 HXGE_DEBUG_MSG((hxgep, RX_CTL, 853 "<== hxge_rxdma_get_ring_index: no channel")); 854 return (-1); 855 } 856 857 HXGE_DEBUG_MSG((hxgep, RX_CTL, 858 "==> hxge_rxdma_get_ring_index (ndmas %d)", ndmas)); 859 860 rbr_rings = rx_rbr_rings->rbr_rings; 861 for (i = 0; i < ndmas; i++) { 862 rdc = rbr_rings[i]->rdc; 863 if (channel == rdc) { 864 HXGE_DEBUG_MSG((hxgep, RX_CTL, 865 "==> hxge_rxdma_get_rbr_ring: " 866 "channel %d (index %d) " 867 "ring %d", channel, i, rbr_rings[i])); 868 869 return (i); 870 } 871 } 872 873 HXGE_DEBUG_MSG((hxgep, RX_CTL, 874 "<== hxge_rxdma_get_rbr_ring_index: not found")); 875 876 return (-1); 877 } 878 879 /* 880 * Static functions start here. 881 */ 882 static p_rx_msg_t 883 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p) 884 { 885 p_rx_msg_t hxge_mp = NULL; 886 p_hxge_dma_common_t dmamsg_p; 887 uchar_t *buffer; 888 889 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 890 if (hxge_mp == NULL) { 891 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 892 "Allocation of a rx msg failed.")); 893 goto hxge_allocb_exit; 894 } 895 896 hxge_mp->use_buf_pool = B_FALSE; 897 if (dmabuf_p) { 898 hxge_mp->use_buf_pool = B_TRUE; 899 900 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma; 901 *dmamsg_p = *dmabuf_p; 902 dmamsg_p->nblocks = 1; 903 dmamsg_p->block_size = size; 904 dmamsg_p->alength = size; 905 buffer = (uchar_t *)dmabuf_p->kaddrp; 906 907 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size); 908 dmabuf_p->ioaddr_pp = (void *) 909 ((char *)dmabuf_p->ioaddr_pp + size); 910 911 dmabuf_p->alength -= size; 912 dmabuf_p->offset += size; 913 dmabuf_p->dma_cookie.dmac_laddress += size; 914 dmabuf_p->dma_cookie.dmac_size -= size; 915 } else { 916 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 917 if (buffer == NULL) { 918 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, 919 "Allocation of a receive page failed.")); 920 goto hxge_allocb_fail1; 921 } 922 } 923 924 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb); 925 if (hxge_mp->rx_mblk_p == NULL) { 926 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed.")); 927 goto hxge_allocb_fail2; 928 } 929 hxge_mp->buffer = buffer; 930 hxge_mp->block_size = size; 931 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb; 932 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp; 933 hxge_mp->ref_cnt = 1; 934 hxge_mp->free = B_TRUE; 935 hxge_mp->rx_use_bcopy = B_FALSE; 936 937 atomic_inc_32(&hxge_mblks_pending); 938 939 goto hxge_allocb_exit; 940 941 hxge_allocb_fail2: 942 if (!hxge_mp->use_buf_pool) { 943 KMEM_FREE(buffer, size); 944 } 945 hxge_allocb_fail1: 946 KMEM_FREE(hxge_mp, sizeof (rx_msg_t)); 947 hxge_mp = NULL; 948 949 hxge_allocb_exit: 950 return (hxge_mp); 951 } 952 953 p_mblk_t 954 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 955 { 956 p_mblk_t mp; 957 958 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb")); 959 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p " 960 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size)); 961 962 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb); 963 if (mp == NULL) { 964 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 965 goto hxge_dupb_exit; 966 } 967 968 atomic_inc_32(&hxge_mp->ref_cnt); 969 970 hxge_dupb_exit: 971 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 972 return (mp); 973 } 974 975 p_mblk_t 976 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size) 977 { 978 p_mblk_t mp; 979 uchar_t *dp; 980 981 mp = allocb(size + HXGE_RXBUF_EXTRA, 0); 982 if (mp == NULL) { 983 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 984 goto hxge_dupb_bcopy_exit; 985 } 986 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA; 987 bcopy((void *) &hxge_mp->buffer[offset], dp, size); 988 mp->b_wptr = dp + size; 989 990 hxge_dupb_bcopy_exit: 991 992 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp)); 993 994 return (mp); 995 } 996 997 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, 998 p_rx_msg_t rx_msg_p); 999 1000 void 1001 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 1002 { 1003 hpi_status_t hpi_status; 1004 hxge_status_t status; 1005 int i; 1006 1007 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page")); 1008 1009 /* Reuse this buffer */ 1010 rx_msg_p->free = B_FALSE; 1011 rx_msg_p->cur_usage_cnt = 0; 1012 rx_msg_p->max_usage_cnt = 0; 1013 rx_msg_p->pkt_buf_size = 0; 1014 1015 if (rx_rbr_p->rbr_use_bcopy) { 1016 rx_msg_p->rx_use_bcopy = B_FALSE; 1017 atomic_dec_32(&rx_rbr_p->rbr_consumed); 1018 } 1019 1020 /* 1021 * Get the rbr header pointer and its offset index. 1022 */ 1023 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 1024 rx_rbr_p->rbr_wrap_mask); 1025 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 1026 1027 /* 1028 * Accumulate some buffers in the ring before re-enabling the 1029 * DMA channel, if rbr empty was signaled. 1030 */ 1031 if (!rx_rbr_p->rbr_is_empty) { 1032 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), 1033 rx_rbr_p->rdc, 1); 1034 } else { 1035 rx_rbr_p->accumulate++; 1036 if (rx_rbr_p->accumulate >= HXGE_RBR_EMPTY_THRESHOLD) { 1037 rx_rbr_p->rbr_is_empty = B_FALSE; 1038 rx_rbr_p->accumulate = 0; 1039 1040 /* 1041 * Complete the processing for the RBR Empty by: 1042 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD 1043 * packets. 1044 * 1) Disable the RX vmac. 1045 * 2) Re-enable the affected DMA channel. 1046 * 3) Re-enable the RX vmac. 1047 */ 1048 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), 1049 rx_rbr_p->rdc, HXGE_RBR_EMPTY_THRESHOLD); 1050 1051 /* 1052 * Disable the RX VMAC, but setting the framelength 1053 * to 0, since there is a hardware bug when disabling 1054 * the vmac. 1055 */ 1056 MUTEX_ENTER(hxgep->genlock); 1057 (void) hpi_vmac_rx_set_framesize( 1058 HXGE_DEV_HPI_HANDLE(hxgep), (uint16_t)0); 1059 1060 hpi_status = hpi_rxdma_cfg_rdc_enable( 1061 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc); 1062 if (hpi_status != HPI_SUCCESS) { 1063 p_hxge_rx_ring_stats_t rdc_stats; 1064 1065 rdc_stats = 1066 &hxgep->statsp->rdc_stats[rx_rbr_p->rdc]; 1067 rdc_stats->rbr_empty_fail++; 1068 1069 status = hxge_rxdma_fatal_err_recover(hxgep, 1070 rx_rbr_p->rdc); 1071 if (status != HXGE_OK) { 1072 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1073 "hxge(%d): channel(%d) is empty.", 1074 hxgep->instance, rx_rbr_p->rdc)); 1075 } 1076 } 1077 1078 for (i = 0; i < 1024; i++) { 1079 uint64_t value; 1080 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep), 1081 RDC_STAT, i & 3, &value); 1082 } 1083 1084 /* 1085 * Re-enable the RX VMAC. 1086 */ 1087 (void) hpi_vmac_rx_set_framesize( 1088 HXGE_DEV_HPI_HANDLE(hxgep), 1089 (uint16_t)hxgep->vmac.maxframesize); 1090 MUTEX_EXIT(hxgep->genlock); 1091 } 1092 } 1093 1094 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1095 "<== hxge_post_page (channel %d post_next_index %d)", 1096 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 1097 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page")); 1098 } 1099 1100 void 1101 hxge_freeb(p_rx_msg_t rx_msg_p) 1102 { 1103 size_t size; 1104 uchar_t *buffer = NULL; 1105 int ref_cnt; 1106 boolean_t free_state = B_FALSE; 1107 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 1108 1109 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb")); 1110 HXGE_DEBUG_MSG((NULL, MEM2_CTL, 1111 "hxge_freeb:rx_msg_p = $%p (block pending %d)", 1112 rx_msg_p, hxge_mblks_pending)); 1113 1114 if (ring == NULL) 1115 return; 1116 1117 /* 1118 * This is to prevent posting activities while we are recovering 1119 * from fatal errors. This should not be a performance drag since 1120 * ref_cnt != 0 most times. 1121 */ 1122 MUTEX_ENTER(&ring->post_lock); 1123 1124 /* 1125 * First we need to get the free state, then 1126 * atomic decrement the reference count to prevent 1127 * the race condition with the interrupt thread that 1128 * is processing a loaned up buffer block. 1129 */ 1130 free_state = rx_msg_p->free; 1131 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 1132 if (!ref_cnt) { 1133 atomic_dec_32(&hxge_mblks_pending); 1134 1135 buffer = rx_msg_p->buffer; 1136 size = rx_msg_p->block_size; 1137 1138 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: " 1139 "will free: rx_msg_p = $%p (block pending %d)", 1140 rx_msg_p, hxge_mblks_pending)); 1141 1142 if (!rx_msg_p->use_buf_pool) { 1143 KMEM_FREE(buffer, size); 1144 } 1145 1146 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 1147 /* 1148 * Decrement the receive buffer ring's reference 1149 * count, too. 1150 */ 1151 atomic_dec_32(&ring->rbr_ref_cnt); 1152 1153 /* 1154 * Free the receive buffer ring, iff 1155 * 1. all the receive buffers have been freed 1156 * 2. and we are in the proper state (that is, 1157 * we are not UNMAPPING). 1158 */ 1159 if (ring->rbr_ref_cnt == 0 && 1160 ring->rbr_state == RBR_UNMAPPED) { 1161 KMEM_FREE(ring, sizeof (*ring)); 1162 } 1163 } 1164 1165 /* 1166 * Repost buffer. 1167 */ 1168 if (free_state && (ref_cnt == 1)) { 1169 HXGE_DEBUG_MSG((NULL, RX_CTL, 1170 "hxge_freeb: post page $%p:", rx_msg_p)); 1171 if (ring->rbr_state == RBR_POSTING) 1172 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p); 1173 } 1174 1175 MUTEX_EXIT(&ring->post_lock); 1176 1177 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb")); 1178 } 1179 1180 uint_t 1181 hxge_rx_intr(caddr_t arg1, caddr_t arg2) 1182 { 1183 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1; 1184 p_hxge_t hxgep = (p_hxge_t)arg2; 1185 p_hxge_ldg_t ldgp; 1186 uint8_t channel; 1187 hpi_handle_t handle; 1188 rdc_stat_t cs; 1189 uint_t serviced = DDI_INTR_UNCLAIMED; 1190 1191 if (ldvp == NULL) { 1192 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, 1193 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1194 return (DDI_INTR_UNCLAIMED); 1195 } 1196 1197 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) { 1198 hxgep = ldvp->hxgep; 1199 } 1200 1201 /* 1202 * If the interface is not started, just swallow the interrupt 1203 * for the logical device and don't rearm it. 1204 */ 1205 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED) 1206 return (DDI_INTR_CLAIMED); 1207 1208 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1209 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp)); 1210 1211 /* 1212 * This interrupt handler is for a specific receive dma channel. 1213 */ 1214 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1215 1216 /* 1217 * Get the control and status for this channel. 1218 */ 1219 channel = ldvp->channel; 1220 ldgp = ldvp->ldgp; 1221 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value); 1222 1223 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_intr:channel %d " 1224 "cs 0x%016llx rcrto 0x%x rcrthres %x", 1225 channel, cs.value, cs.bits.rcr_to, cs.bits.rcr_thres)); 1226 1227 hxge_rx_pkts_vring(hxgep, ldvp->vdma_index, ldvp, cs); 1228 serviced = DDI_INTR_CLAIMED; 1229 1230 /* error events. */ 1231 if (cs.value & RDC_STAT_ERROR) { 1232 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs); 1233 } 1234 1235 hxge_intr_exit: 1236 /* 1237 * Enable the mailbox update interrupt if we want to use mailbox. We 1238 * probably don't need to use mailbox as it only saves us one pio read. 1239 * Also write 1 to rcrthres and rcrto to clear these two edge triggered 1240 * bits. 1241 */ 1242 cs.value &= RDC_STAT_WR1C; 1243 cs.bits.mex = 1; 1244 cs.bits.ptrread = 0; 1245 cs.bits.pktread = 0; 1246 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1247 1248 /* 1249 * Rearm this logical group if this is a single device group. 1250 */ 1251 if (ldgp->nldvs == 1) { 1252 ld_intr_mgmt_t mgm; 1253 1254 mgm.value = 0; 1255 mgm.bits.arm = 1; 1256 mgm.bits.timer = ldgp->ldg_timer; 1257 HXGE_REG_WR32(handle, 1258 LD_INTR_MGMT + LDSV_OFFSET(ldgp->ldg), mgm.value); 1259 } 1260 1261 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1262 "<== hxge_rx_intr: serviced %d", serviced)); 1263 1264 return (serviced); 1265 } 1266 1267 static void 1268 hxge_rx_pkts_vring(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1269 rdc_stat_t cs) 1270 { 1271 p_mblk_t mp; 1272 p_rx_rcr_ring_t rcrp; 1273 1274 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts_vring")); 1275 if ((mp = hxge_rx_pkts(hxgep, vindex, ldvp, &rcrp, cs)) == NULL) { 1276 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1277 "<== hxge_rx_pkts_vring: no mp")); 1278 return; 1279 } 1280 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts_vring: $%p", mp)); 1281 1282 #ifdef HXGE_DEBUG 1283 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1284 "==> hxge_rx_pkts_vring:calling mac_rx (NEMO) " 1285 "LEN %d mp $%p mp->b_next $%p rcrp $%p " 1286 "mac_handle $%p", 1287 (mp->b_wptr - mp->b_rptr), mp, mp->b_next, 1288 rcrp, rcrp->rcr_mac_handle)); 1289 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1290 "==> hxge_rx_pkts_vring: dump packets " 1291 "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 1292 mp, mp->b_rptr, mp->b_wptr, 1293 hxge_dump_packet((char *)mp->b_rptr, 64))); 1294 1295 if (mp->b_cont) { 1296 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1297 "==> hxge_rx_pkts_vring: dump b_cont packets " 1298 "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 1299 mp->b_cont, mp->b_cont->b_rptr, mp->b_cont->b_wptr, 1300 hxge_dump_packet((char *)mp->b_cont->b_rptr, 1301 mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 1302 } 1303 if (mp->b_next) { 1304 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1305 "==> hxge_rx_pkts_vring: dump next packets " 1306 "(b_rptr $%p): %s", 1307 mp->b_next->b_rptr, 1308 hxge_dump_packet((char *)mp->b_next->b_rptr, 64))); 1309 } 1310 #endif 1311 1312 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1313 "==> hxge_rx_pkts_vring: send packet to stack")); 1314 mac_rx(hxgep->mach, rcrp->rcr_mac_handle, mp); 1315 1316 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_pkts_vring")); 1317 } 1318 1319 /*ARGSUSED*/ 1320 mblk_t * 1321 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp, 1322 p_rx_rcr_ring_t *rcrp, rdc_stat_t cs) 1323 { 1324 hpi_handle_t handle; 1325 uint8_t channel; 1326 p_rx_rcr_rings_t rx_rcr_rings; 1327 p_rx_rcr_ring_t rcr_p; 1328 uint32_t comp_rd_index; 1329 p_rcr_entry_t rcr_desc_rd_head_p; 1330 p_rcr_entry_t rcr_desc_rd_head_pp; 1331 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 1332 uint16_t qlen, nrcr_read, npkt_read; 1333 uint32_t qlen_hw, qlen_sw; 1334 uint32_t invalid_rcr_entry; 1335 boolean_t multi; 1336 rdc_rcr_cfg_b_t rcr_cfg_b; 1337 p_rx_mbox_t rx_mboxp; 1338 p_rxdma_mailbox_t mboxp; 1339 uint64_t rcr_head_index, rcr_tail_index; 1340 uint64_t rcr_tail; 1341 uint64_t value; 1342 rdc_rcr_tail_t rcr_tail_reg; 1343 p_hxge_rx_ring_stats_t rdc_stats; 1344 1345 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d " 1346 "channel %d", vindex, ldvp->channel)); 1347 1348 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) { 1349 return (NULL); 1350 } 1351 1352 handle = HXGE_DEV_HPI_HANDLE(hxgep); 1353 rx_rcr_rings = hxgep->rx_rcr_rings; 1354 rcr_p = rx_rcr_rings->rcr_rings[vindex]; 1355 channel = rcr_p->rdc; 1356 if (channel != ldvp->channel) { 1357 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d " 1358 "channel %d, and rcr channel %d not matched.", 1359 vindex, ldvp->channel, channel)); 1360 return (NULL); 1361 } 1362 1363 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1364 "==> hxge_rx_pkts: START: rcr channel %d " 1365 "head_p $%p head_pp $%p index %d ", 1366 channel, rcr_p->rcr_desc_rd_head_p, 1367 rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1368 1369 rx_mboxp = hxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 1370 mboxp = (p_rxdma_mailbox_t)rx_mboxp->rx_mbox.kaddrp; 1371 1372 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 1373 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, &rcr_tail_reg.value); 1374 rcr_tail = rcr_tail_reg.bits.tail; 1375 1376 if (!qlen) { 1377 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1378 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)", 1379 channel, qlen)); 1380 return (NULL); 1381 } 1382 1383 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d " 1384 "qlen %d", channel, qlen)); 1385 1386 comp_rd_index = rcr_p->comp_rd_index; 1387 1388 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 1389 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 1390 nrcr_read = npkt_read = 0; 1391 1392 /* 1393 * Number of packets queued (The jumbo or multi packet will be counted 1394 * as only one paccket and it may take up more than one completion 1395 * entry). 1396 */ 1397 qlen_hw = (qlen < hxge_max_rx_pkts) ? qlen : hxge_max_rx_pkts; 1398 head_mp = NULL; 1399 tail_mp = &head_mp; 1400 nmp = mp_cont = NULL; 1401 multi = B_FALSE; 1402 1403 rcr_head_index = rcr_p->rcr_desc_rd_head_p - rcr_p->rcr_desc_first_p; 1404 rcr_tail_index = rcr_tail - rcr_p->rcr_tail_begin; 1405 1406 if (rcr_tail_index >= rcr_head_index) { 1407 qlen_sw = rcr_tail_index - rcr_head_index; 1408 } else { 1409 /* rcr_tail has wrapped around */ 1410 qlen_sw = (rcr_p->comp_size - rcr_head_index) + rcr_tail_index; 1411 } 1412 1413 if (qlen_hw > qlen_sw) { 1414 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1415 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n", 1416 channel, qlen_hw, qlen_sw)); 1417 qlen_hw = qlen_sw; 1418 } 1419 1420 while (qlen_hw) { 1421 #ifdef HXGE_DEBUG 1422 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p); 1423 #endif 1424 /* 1425 * Process one completion ring entry. 1426 */ 1427 invalid_rcr_entry = 0; 1428 hxge_receive_packet(hxgep, 1429 rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont, 1430 &invalid_rcr_entry); 1431 if (invalid_rcr_entry != 0) { 1432 rdc_stats = rcr_p->rdc_stats; 1433 rdc_stats->rcr_invalids++; 1434 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1435 "Channel %d could only read 0x%x packets, " 1436 "but 0x%x pending\n", channel, npkt_read, qlen_hw)); 1437 break; 1438 } 1439 1440 /* 1441 * message chaining modes (nemo msg chaining) 1442 */ 1443 if (nmp) { 1444 nmp->b_next = NULL; 1445 if (!multi && !mp_cont) { /* frame fits a partition */ 1446 *tail_mp = nmp; 1447 tail_mp = &nmp->b_next; 1448 nmp = NULL; 1449 } else if (multi && !mp_cont) { /* first segment */ 1450 *tail_mp = nmp; 1451 tail_mp = &nmp->b_cont; 1452 } else if (multi && mp_cont) { /* mid of multi segs */ 1453 *tail_mp = mp_cont; 1454 tail_mp = &mp_cont->b_cont; 1455 } else if (!multi && mp_cont) { /* last segment */ 1456 *tail_mp = mp_cont; 1457 tail_mp = &nmp->b_next; 1458 nmp = NULL; 1459 } 1460 } 1461 1462 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1463 "==> hxge_rx_pkts: loop: rcr channel %d " 1464 "before updating: multi %d " 1465 "nrcr_read %d " 1466 "npk read %d " 1467 "head_pp $%p index %d ", 1468 channel, multi, 1469 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index)); 1470 1471 if (!multi) { 1472 qlen_hw--; 1473 npkt_read++; 1474 } 1475 1476 /* 1477 * Update the next read entry. 1478 */ 1479 comp_rd_index = NEXT_ENTRY(comp_rd_index, 1480 rcr_p->comp_wrap_mask); 1481 1482 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 1483 rcr_p->rcr_desc_first_p, rcr_p->rcr_desc_last_p); 1484 1485 nrcr_read++; 1486 1487 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1488 "<== hxge_rx_pkts: (SAM, process one packet) " 1489 "nrcr_read %d", nrcr_read)); 1490 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1491 "==> hxge_rx_pkts: loop: rcr channel %d " 1492 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ", 1493 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp, 1494 comp_rd_index)); 1495 } 1496 1497 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 1498 rcr_p->comp_rd_index = comp_rd_index; 1499 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 1500 1501 /* Adjust the mailbox queue length for a hardware bug workaround */ 1502 mboxp->rcrstat_a.bits.qlen -= npkt_read; 1503 1504 if ((hxgep->intr_timeout != rcr_p->intr_timeout) || 1505 (hxgep->intr_threshold != rcr_p->intr_threshold)) { 1506 rcr_p->intr_timeout = hxgep->intr_timeout; 1507 rcr_p->intr_threshold = hxgep->intr_threshold; 1508 rcr_cfg_b.value = 0x0ULL; 1509 if (rcr_p->intr_timeout) 1510 rcr_cfg_b.bits.entout = 1; 1511 rcr_cfg_b.bits.timeout = rcr_p->intr_timeout; 1512 rcr_cfg_b.bits.pthres = rcr_p->intr_threshold; 1513 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B, 1514 channel, rcr_cfg_b.value); 1515 } 1516 1517 cs.bits.pktread = npkt_read; 1518 cs.bits.ptrread = nrcr_read; 1519 value = cs.value; 1520 cs.value &= 0xffffffffULL; 1521 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1522 1523 cs.value = value & ~0xffffffffULL; 1524 cs.bits.pktread = 0; 1525 cs.bits.ptrread = 0; 1526 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value); 1527 1528 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, 1529 "==> hxge_rx_pkts: EXIT: rcr channel %d " 1530 "head_pp $%p index %016llx ", 1531 channel, rcr_p->rcr_desc_rd_head_pp, rcr_p->comp_rd_index)); 1532 1533 /* 1534 * Update RCR buffer pointer read and number of packets read. 1535 */ 1536 1537 *rcrp = rcr_p; 1538 1539 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts")); 1540 1541 return (head_mp); 1542 } 1543 1544 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL 1545 1546 /*ARGSUSED*/ 1547 void 1548 hxge_receive_packet(p_hxge_t hxgep, 1549 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 1550 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont, 1551 uint32_t *invalid_rcr_entry) 1552 { 1553 p_mblk_t nmp = NULL; 1554 uint64_t multi; 1555 uint8_t channel; 1556 1557 boolean_t first_entry = B_TRUE; 1558 boolean_t is_tcp_udp = B_FALSE; 1559 boolean_t buffer_free = B_FALSE; 1560 boolean_t error_send_up = B_FALSE; 1561 uint8_t error_type; 1562 uint16_t l2_len; 1563 uint16_t skip_len; 1564 uint8_t pktbufsz_type; 1565 uint64_t rcr_entry; 1566 uint64_t *pkt_buf_addr_pp; 1567 uint64_t *pkt_buf_addr_p; 1568 uint32_t buf_offset; 1569 uint32_t bsize; 1570 uint32_t msg_index; 1571 p_rx_rbr_ring_t rx_rbr_p; 1572 p_rx_msg_t *rx_msg_ring_p; 1573 p_rx_msg_t rx_msg_p; 1574 1575 uint16_t sw_offset_bytes = 0, hdr_size = 0; 1576 hxge_status_t status = HXGE_OK; 1577 boolean_t is_valid = B_FALSE; 1578 p_hxge_rx_ring_stats_t rdc_stats; 1579 uint32_t bytes_read; 1580 1581 uint64_t pkt_type; 1582 1583 channel = rcr_p->rdc; 1584 1585 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet")); 1586 1587 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 1588 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 1589 1590 /* Verify the content of the rcr_entry for a hardware bug workaround */ 1591 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) { 1592 *invalid_rcr_entry = 1; 1593 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet " 1594 "Channel %d invalid RCR entry 0x%llx found, returning\n", 1595 channel, (long long) rcr_entry)); 1596 return; 1597 } 1598 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN; 1599 1600 multi = (rcr_entry & RCR_MULTI_MASK); 1601 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 1602 1603 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 1604 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 1605 1606 /* 1607 * Hardware does not strip the CRC due bug ID 11451 where 1608 * the hardware mis handles minimum size packets. 1609 */ 1610 l2_len -= ETHERFCSL; 1611 1612 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 1613 RCR_PKTBUFSZ_SHIFT); 1614 #if defined(__i386) 1615 pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 1616 RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 1617 #else 1618 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 1619 RCR_PKT_BUF_ADDR_SHIFT); 1620 #endif 1621 1622 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1623 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1624 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1625 "error_type 0x%x pkt_type 0x%x " 1626 "pktbufsz_type %d ", 1627 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len, 1628 multi, error_type, pkt_type, pktbufsz_type)); 1629 1630 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1631 "==> hxge_receive_packet: entryp $%p entry 0x%0llx " 1632 "pkt_buf_addr_pp $%p l2_len %d multi %d " 1633 "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p, 1634 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type, pkt_type)); 1635 1636 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1637 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1638 "full pkt_buf_addr_pp $%p l2_len %d", 1639 rcr_entry, pkt_buf_addr_pp, l2_len)); 1640 1641 /* get the stats ptr */ 1642 rdc_stats = rcr_p->rdc_stats; 1643 1644 if (!l2_len) { 1645 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1646 "<== hxge_receive_packet: failed: l2 length is 0.")); 1647 return; 1648 } 1649 1650 /* shift 6 bits to get the full io address */ 1651 #if defined(__i386) 1652 pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 1653 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1654 #else 1655 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 1656 RCR_PKT_BUF_ADDR_SHIFT_FULL); 1657 #endif 1658 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1659 "==> (rbr) hxge_receive_packet: entry 0x%0llx " 1660 "full pkt_buf_addr_pp $%p l2_len %d", 1661 rcr_entry, pkt_buf_addr_pp, l2_len)); 1662 1663 rx_rbr_p = rcr_p->rx_rbr_p; 1664 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 1665 1666 if (first_entry) { 1667 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 1668 RXDMA_HDR_SIZE_DEFAULT); 1669 1670 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1671 "==> hxge_receive_packet: first entry 0x%016llx " 1672 "pkt_buf_addr_pp $%p l2_len %d hdr %d", 1673 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size)); 1674 } 1675 1676 MUTEX_ENTER(&rcr_p->lock); 1677 MUTEX_ENTER(&rx_rbr_p->lock); 1678 1679 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1680 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx " 1681 "full pkt_buf_addr_pp $%p l2_len %d", 1682 rcr_entry, pkt_buf_addr_pp, l2_len)); 1683 1684 /* 1685 * Packet buffer address in the completion entry points to the starting 1686 * buffer address (offset 0). Use the starting buffer address to locate 1687 * the corresponding kernel address. 1688 */ 1689 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p, 1690 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 1691 &buf_offset, &msg_index); 1692 1693 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1694 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx " 1695 "full pkt_buf_addr_pp $%p l2_len %d", 1696 rcr_entry, pkt_buf_addr_pp, l2_len)); 1697 1698 if (status != HXGE_OK) { 1699 MUTEX_EXIT(&rx_rbr_p->lock); 1700 MUTEX_EXIT(&rcr_p->lock); 1701 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1702 "<== hxge_receive_packet: found vaddr failed %d", status)); 1703 return; 1704 } 1705 1706 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1707 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx " 1708 "full pkt_buf_addr_pp $%p l2_len %d", 1709 rcr_entry, pkt_buf_addr_pp, l2_len)); 1710 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1711 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1712 "full pkt_buf_addr_pp $%p l2_len %d", 1713 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1714 1715 if (msg_index >= rx_rbr_p->tnblocks) { 1716 MUTEX_EXIT(&rx_rbr_p->lock); 1717 MUTEX_EXIT(&rcr_p->lock); 1718 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1719 "==> hxge_receive_packet: FATAL msg_index (%d) " 1720 "should be smaller than tnblocks (%d)\n", 1721 msg_index, rx_rbr_p->tnblocks)); 1722 return; 1723 } 1724 1725 rx_msg_p = rx_msg_ring_p[msg_index]; 1726 1727 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1728 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx " 1729 "full pkt_buf_addr_pp $%p l2_len %d", 1730 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 1731 1732 switch (pktbufsz_type) { 1733 case RCR_PKTBUFSZ_0: 1734 bsize = rx_rbr_p->pkt_buf_size0_bytes; 1735 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1736 "==> hxge_receive_packet: 0 buf %d", bsize)); 1737 break; 1738 case RCR_PKTBUFSZ_1: 1739 bsize = rx_rbr_p->pkt_buf_size1_bytes; 1740 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1741 "==> hxge_receive_packet: 1 buf %d", bsize)); 1742 break; 1743 case RCR_PKTBUFSZ_2: 1744 bsize = rx_rbr_p->pkt_buf_size2_bytes; 1745 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1746 "==> hxge_receive_packet: 2 buf %d", bsize)); 1747 break; 1748 case RCR_SINGLE_BLOCK: 1749 bsize = rx_msg_p->block_size; 1750 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1751 "==> hxge_receive_packet: single %d", bsize)); 1752 1753 break; 1754 default: 1755 MUTEX_EXIT(&rx_rbr_p->lock); 1756 MUTEX_EXIT(&rcr_p->lock); 1757 return; 1758 } 1759 1760 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 1761 (buf_offset + sw_offset_bytes), (hdr_size + l2_len), 1762 DDI_DMA_SYNC_FORCPU); 1763 1764 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1765 "==> hxge_receive_packet: after first dump:usage count")); 1766 1767 if (rx_msg_p->cur_usage_cnt == 0) { 1768 if (rx_rbr_p->rbr_use_bcopy) { 1769 atomic_inc_32(&rx_rbr_p->rbr_consumed); 1770 if (rx_rbr_p->rbr_consumed > 1771 rx_rbr_p->rbr_threshold_hi) { 1772 rx_msg_p->rx_use_bcopy = B_TRUE; 1773 } 1774 } 1775 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1776 "==> hxge_receive_packet: buf %d (new block) ", bsize)); 1777 1778 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 1779 rx_msg_p->pkt_buf_size = bsize; 1780 rx_msg_p->cur_usage_cnt = 1; 1781 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 1782 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1783 "==> hxge_receive_packet: buf %d (single block) ", 1784 bsize)); 1785 /* 1786 * Buffer can be reused once the free function is 1787 * called. 1788 */ 1789 rx_msg_p->max_usage_cnt = 1; 1790 buffer_free = B_TRUE; 1791 } else { 1792 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize; 1793 if (rx_msg_p->max_usage_cnt == 1) { 1794 buffer_free = B_TRUE; 1795 } 1796 } 1797 } else { 1798 rx_msg_p->cur_usage_cnt++; 1799 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 1800 buffer_free = B_TRUE; 1801 } 1802 } 1803 1804 if (rx_msg_p->rx_use_bcopy) { 1805 rdc_stats->pkt_drop++; 1806 atomic_inc_32(&rx_msg_p->ref_cnt); 1807 if (buffer_free == B_TRUE) { 1808 rx_msg_p->free = B_TRUE; 1809 } 1810 1811 MUTEX_EXIT(&rx_rbr_p->lock); 1812 MUTEX_EXIT(&rcr_p->lock); 1813 hxge_freeb(rx_msg_p); 1814 return; 1815 } 1816 1817 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1818 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 1819 msg_index, l2_len, 1820 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 1821 1822 if (error_type) { 1823 rdc_stats->ierrors++; 1824 /* Update error stats */ 1825 rdc_stats->errlog.compl_err_type = error_type; 1826 HXGE_FM_REPORT_ERROR(hxgep, NULL, HXGE_FM_EREPORT_RDMC_RCR_ERR); 1827 1828 if (error_type & RCR_CTRL_FIFO_DED) { 1829 rdc_stats->ctrl_fifo_ecc_err++; 1830 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1831 " hxge_receive_packet: " 1832 " channel %d RCR ctrl_fifo_ded error", channel)); 1833 } else if (error_type & RCR_DATA_FIFO_DED) { 1834 rdc_stats->data_fifo_ecc_err++; 1835 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 1836 " hxge_receive_packet: channel %d" 1837 " RCR data_fifo_ded error", channel)); 1838 } 1839 1840 /* 1841 * Update and repost buffer block if max usage count is 1842 * reached. 1843 */ 1844 if (error_send_up == B_FALSE) { 1845 atomic_inc_32(&rx_msg_p->ref_cnt); 1846 if (buffer_free == B_TRUE) { 1847 rx_msg_p->free = B_TRUE; 1848 } 1849 1850 MUTEX_EXIT(&rx_rbr_p->lock); 1851 MUTEX_EXIT(&rcr_p->lock); 1852 hxge_freeb(rx_msg_p); 1853 return; 1854 } 1855 } 1856 1857 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1858 "==> hxge_receive_packet: DMA sync second ")); 1859 1860 bytes_read = rcr_p->rcvd_pkt_bytes; 1861 skip_len = sw_offset_bytes + hdr_size; 1862 if (!rx_msg_p->rx_use_bcopy) { 1863 /* 1864 * For loaned up buffers, the driver reference count 1865 * will be incremented first and then the free state. 1866 */ 1867 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 1868 if (first_entry) { 1869 nmp->b_rptr = &nmp->b_rptr[skip_len]; 1870 if (l2_len < bsize - skip_len) { 1871 nmp->b_wptr = &nmp->b_rptr[l2_len]; 1872 } else { 1873 nmp->b_wptr = &nmp->b_rptr[bsize 1874 - skip_len]; 1875 } 1876 } else { 1877 if (l2_len - bytes_read < bsize) { 1878 nmp->b_wptr = 1879 &nmp->b_rptr[l2_len - bytes_read]; 1880 } else { 1881 nmp->b_wptr = &nmp->b_rptr[bsize]; 1882 } 1883 } 1884 } 1885 } else { 1886 if (first_entry) { 1887 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 1888 l2_len < bsize - skip_len ? 1889 l2_len : bsize - skip_len); 1890 } else { 1891 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset, 1892 l2_len - bytes_read < bsize ? 1893 l2_len - bytes_read : bsize); 1894 } 1895 } 1896 1897 if (nmp != NULL) { 1898 if (first_entry) 1899 bytes_read = nmp->b_wptr - nmp->b_rptr; 1900 else 1901 bytes_read += nmp->b_wptr - nmp->b_rptr; 1902 1903 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1904 "==> hxge_receive_packet after dupb: " 1905 "rbr consumed %d " 1906 "pktbufsz_type %d " 1907 "nmp $%p rptr $%p wptr $%p " 1908 "buf_offset %d bzise %d l2_len %d skip_len %d", 1909 rx_rbr_p->rbr_consumed, 1910 pktbufsz_type, 1911 nmp, nmp->b_rptr, nmp->b_wptr, 1912 buf_offset, bsize, l2_len, skip_len)); 1913 } else { 1914 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)"); 1915 1916 atomic_inc_32(&rx_msg_p->ref_cnt); 1917 if (buffer_free == B_TRUE) { 1918 rx_msg_p->free = B_TRUE; 1919 } 1920 1921 MUTEX_EXIT(&rx_rbr_p->lock); 1922 MUTEX_EXIT(&rcr_p->lock); 1923 hxge_freeb(rx_msg_p); 1924 return; 1925 } 1926 1927 if (buffer_free == B_TRUE) { 1928 rx_msg_p->free = B_TRUE; 1929 } 1930 1931 /* 1932 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a 1933 * packet is not fragmented and no error bit is set, then L4 checksum 1934 * is OK. 1935 */ 1936 is_valid = (nmp != NULL); 1937 if (first_entry) { 1938 rdc_stats->ipackets++; /* count only 1st seg for jumbo */ 1939 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL)) 1940 rdc_stats->jumbo_pkts++; 1941 rdc_stats->ibytes += skip_len + l2_len < bsize ? 1942 l2_len : bsize; 1943 } else { 1944 /* 1945 * Add the current portion of the packet to the kstats. 1946 * The current portion of the packet is calculated by using 1947 * length of the packet and the previously received portion. 1948 */ 1949 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ? 1950 l2_len - rcr_p->rcvd_pkt_bytes : bsize; 1951 } 1952 1953 rcr_p->rcvd_pkt_bytes = bytes_read; 1954 1955 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 1956 atomic_inc_32(&rx_msg_p->ref_cnt); 1957 MUTEX_EXIT(&rx_rbr_p->lock); 1958 MUTEX_EXIT(&rcr_p->lock); 1959 hxge_freeb(rx_msg_p); 1960 } else { 1961 MUTEX_EXIT(&rx_rbr_p->lock); 1962 MUTEX_EXIT(&rcr_p->lock); 1963 } 1964 1965 if (is_valid) { 1966 nmp->b_cont = NULL; 1967 if (first_entry) { 1968 *mp = nmp; 1969 *mp_cont = NULL; 1970 } else { 1971 *mp_cont = nmp; 1972 } 1973 } 1974 1975 /* 1976 * Update stats and hardware checksuming. 1977 */ 1978 if (is_valid && !multi) { 1979 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 1980 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE); 1981 1982 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_receive_packet: " 1983 "is_valid 0x%x multi %d pkt %d d error %d", 1984 is_valid, multi, is_tcp_udp, error_type)); 1985 1986 if (is_tcp_udp && !error_type) { 1987 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 1988 HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 1989 1990 HXGE_DEBUG_MSG((hxgep, RX_CTL, 1991 "==> hxge_receive_packet: Full tcp/udp cksum " 1992 "is_valid 0x%x multi %d pkt %d " 1993 "error %d", 1994 is_valid, multi, is_tcp_udp, error_type)); 1995 } 1996 } 1997 1998 HXGE_DEBUG_MSG((hxgep, RX2_CTL, 1999 "==> hxge_receive_packet: *mp 0x%016llx", *mp)); 2000 2001 *multi_p = (multi == RCR_MULTI_MASK); 2002 2003 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: " 2004 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 2005 *multi_p, nmp, *mp, *mp_cont)); 2006 } 2007 2008 static void 2009 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel) 2010 { 2011 hpi_handle_t handle; 2012 p_rx_rcr_ring_t rcrp; 2013 p_rx_rbr_ring_t rbrp; 2014 2015 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel]; 2016 rbrp = rcrp->rx_rbr_p; 2017 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2018 2019 /* 2020 * Wait for the channel to be quiet 2021 */ 2022 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel); 2023 2024 /* 2025 * Post page will accumulate some buffers before re-enabling 2026 * the DMA channel. 2027 */ 2028 MUTEX_ENTER(&rbrp->post_lock); 2029 rbrp->rbr_is_empty = B_TRUE; 2030 MUTEX_EXIT(&rbrp->post_lock); 2031 } 2032 2033 /*ARGSUSED*/ 2034 static hxge_status_t 2035 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp, 2036 rdc_stat_t cs) 2037 { 2038 p_hxge_rx_ring_stats_t rdc_stats; 2039 hpi_handle_t handle; 2040 boolean_t rxchan_fatal = B_FALSE; 2041 uint8_t channel; 2042 hxge_status_t status = HXGE_OK; 2043 uint64_t cs_val; 2044 2045 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts")); 2046 2047 handle = HXGE_DEV_HPI_HANDLE(hxgep); 2048 channel = ldvp->channel; 2049 2050 /* Clear the interrupts */ 2051 cs.bits.pktread = 0; 2052 cs.bits.ptrread = 0; 2053 cs_val = cs.value & RDC_STAT_WR1C; 2054 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs_val); 2055 2056 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index]; 2057 2058 if (cs.bits.rbr_cpl_to) { 2059 rdc_stats->rbr_tmout++; 2060 HXGE_FM_REPORT_ERROR(hxgep, channel, 2061 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO); 2062 rxchan_fatal = B_TRUE; 2063 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2064 "==> hxge_rx_err_evnts(channel %d): " 2065 "fatal error: rx_rbr_timeout", channel)); 2066 } 2067 2068 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) { 2069 (void) hpi_rxdma_ring_perr_stat_get(handle, 2070 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par); 2071 } 2072 2073 if (cs.bits.rcr_shadow_par_err) { 2074 rdc_stats->rcr_sha_par++; 2075 HXGE_FM_REPORT_ERROR(hxgep, channel, 2076 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 2077 rxchan_fatal = B_TRUE; 2078 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2079 "==> hxge_rx_err_evnts(channel %d): " 2080 "fatal error: rcr_shadow_par_err", channel)); 2081 } 2082 2083 if (cs.bits.rbr_prefetch_par_err) { 2084 rdc_stats->rbr_pre_par++; 2085 HXGE_FM_REPORT_ERROR(hxgep, channel, 2086 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 2087 rxchan_fatal = B_TRUE; 2088 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2089 "==> hxge_rx_err_evnts(channel %d): " 2090 "fatal error: rbr_prefetch_par_err", channel)); 2091 } 2092 2093 if (cs.bits.rbr_pre_empty) { 2094 rdc_stats->rbr_pre_empty++; 2095 HXGE_FM_REPORT_ERROR(hxgep, channel, 2096 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY); 2097 rxchan_fatal = B_TRUE; 2098 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2099 "==> hxge_rx_err_evnts(channel %d): " 2100 "fatal error: rbr_pre_empty", channel)); 2101 } 2102 2103 if (cs.bits.peu_resp_err) { 2104 rdc_stats->peu_resp_err++; 2105 HXGE_FM_REPORT_ERROR(hxgep, channel, 2106 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR); 2107 rxchan_fatal = B_TRUE; 2108 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2109 "==> hxge_rx_err_evnts(channel %d): " 2110 "fatal error: peu_resp_err", channel)); 2111 } 2112 2113 if (cs.bits.rcr_thres) { 2114 rdc_stats->rcr_thres++; 2115 } 2116 2117 if (cs.bits.rcr_to) { 2118 rdc_stats->rcr_to++; 2119 } 2120 2121 if (cs.bits.rcr_shadow_full) { 2122 rdc_stats->rcr_shadow_full++; 2123 HXGE_FM_REPORT_ERROR(hxgep, channel, 2124 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL); 2125 rxchan_fatal = B_TRUE; 2126 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2127 "==> hxge_rx_err_evnts(channel %d): " 2128 "fatal error: rcr_shadow_full", channel)); 2129 } 2130 2131 if (cs.bits.rcr_full) { 2132 rdc_stats->rcrfull++; 2133 HXGE_FM_REPORT_ERROR(hxgep, channel, 2134 HXGE_FM_EREPORT_RDMC_RCRFULL); 2135 rxchan_fatal = B_TRUE; 2136 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2137 "==> hxge_rx_err_evnts(channel %d): " 2138 "fatal error: rcrfull error", channel)); 2139 } 2140 2141 if (cs.bits.rbr_empty) { 2142 rdc_stats->rbr_empty++; 2143 hxge_rx_rbr_empty_recover(hxgep, channel); 2144 } 2145 2146 if (cs.bits.rbr_full) { 2147 rdc_stats->rbrfull++; 2148 HXGE_FM_REPORT_ERROR(hxgep, channel, 2149 HXGE_FM_EREPORT_RDMC_RBRFULL); 2150 rxchan_fatal = B_TRUE; 2151 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2152 "==> hxge_rx_err_evnts(channel %d): " 2153 "fatal error: rbr_full error", channel)); 2154 } 2155 2156 if (rxchan_fatal) { 2157 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2158 " hxge_rx_err_evnts: fatal error on Channel #%d\n", 2159 channel)); 2160 status = hxge_rxdma_fatal_err_recover(hxgep, channel); 2161 if (status == HXGE_OK) { 2162 FM_SERVICE_RESTORED(hxgep); 2163 } 2164 } 2165 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts")); 2166 2167 return (status); 2168 } 2169 2170 static hxge_status_t 2171 hxge_map_rxdma(p_hxge_t hxgep) 2172 { 2173 int i, ndmas; 2174 uint16_t channel; 2175 p_rx_rbr_rings_t rx_rbr_rings; 2176 p_rx_rbr_ring_t *rbr_rings; 2177 p_rx_rcr_rings_t rx_rcr_rings; 2178 p_rx_rcr_ring_t *rcr_rings; 2179 p_rx_mbox_areas_t rx_mbox_areas_p; 2180 p_rx_mbox_t *rx_mbox_p; 2181 p_hxge_dma_pool_t dma_buf_poolp; 2182 p_hxge_dma_common_t *dma_buf_p; 2183 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2184 p_hxge_dma_common_t *dma_rbr_cntl_p; 2185 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2186 p_hxge_dma_common_t *dma_rcr_cntl_p; 2187 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2188 p_hxge_dma_common_t *dma_mbox_cntl_p; 2189 uint32_t *num_chunks; 2190 hxge_status_t status = HXGE_OK; 2191 2192 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma")); 2193 2194 dma_buf_poolp = hxgep->rx_buf_pool_p; 2195 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2196 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2197 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2198 2199 if (!dma_buf_poolp->buf_allocated || 2200 !dma_rbr_cntl_poolp->buf_allocated || 2201 !dma_rcr_cntl_poolp->buf_allocated || 2202 !dma_mbox_cntl_poolp->buf_allocated) { 2203 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2204 "<== hxge_map_rxdma: buf not allocated")); 2205 return (HXGE_ERROR); 2206 } 2207 2208 ndmas = dma_buf_poolp->ndmas; 2209 if (!ndmas) { 2210 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2211 "<== hxge_map_rxdma: no dma allocated")); 2212 return (HXGE_ERROR); 2213 } 2214 2215 num_chunks = dma_buf_poolp->num_chunks; 2216 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2217 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p; 2218 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p; 2219 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p; 2220 2221 rx_rbr_rings = (p_rx_rbr_rings_t) 2222 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP); 2223 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC( 2224 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP); 2225 2226 rx_rcr_rings = (p_rx_rcr_rings_t) 2227 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP); 2228 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC( 2229 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP); 2230 2231 rx_mbox_areas_p = (p_rx_mbox_areas_t) 2232 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP); 2233 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC( 2234 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP); 2235 2236 /* 2237 * Timeout should be set based on the system clock divider. 2238 * The following timeout value of 1 assumes that the 2239 * granularity (1000) is 3 microseconds running at 300MHz. 2240 */ 2241 2242 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 2243 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 2244 2245 /* 2246 * Map descriptors from the buffer polls for each dam channel. 2247 */ 2248 for (i = 0; i < ndmas; i++) { 2249 /* 2250 * Set up and prepare buffer blocks, descriptors and mailbox. 2251 */ 2252 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2253 status = hxge_map_rxdma_channel(hxgep, channel, 2254 (p_hxge_dma_common_t *)&dma_buf_p[i], 2255 (p_rx_rbr_ring_t *)&rbr_rings[i], 2256 num_chunks[i], 2257 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i], 2258 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i], 2259 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i], 2260 (p_rx_rcr_ring_t *)&rcr_rings[i], 2261 (p_rx_mbox_t *)&rx_mbox_p[i]); 2262 if (status != HXGE_OK) { 2263 goto hxge_map_rxdma_fail1; 2264 } 2265 rbr_rings[i]->index = (uint16_t)i; 2266 rcr_rings[i]->index = (uint16_t)i; 2267 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i]; 2268 } 2269 2270 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas; 2271 rx_rbr_rings->rbr_rings = rbr_rings; 2272 hxgep->rx_rbr_rings = rx_rbr_rings; 2273 rx_rcr_rings->rcr_rings = rcr_rings; 2274 hxgep->rx_rcr_rings = rx_rcr_rings; 2275 2276 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p; 2277 hxgep->rx_mbox_areas_p = rx_mbox_areas_p; 2278 2279 goto hxge_map_rxdma_exit; 2280 2281 hxge_map_rxdma_fail1: 2282 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2283 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)", 2284 status, channel, i)); 2285 i--; 2286 for (; i >= 0; i--) { 2287 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2288 hxge_unmap_rxdma_channel(hxgep, channel, 2289 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]); 2290 } 2291 2292 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2293 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2294 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2295 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2296 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2297 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2298 2299 hxge_map_rxdma_exit: 2300 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2301 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 2302 2303 return (status); 2304 } 2305 2306 static void 2307 hxge_unmap_rxdma(p_hxge_t hxgep) 2308 { 2309 int i, ndmas; 2310 uint16_t channel; 2311 p_rx_rbr_rings_t rx_rbr_rings; 2312 p_rx_rbr_ring_t *rbr_rings; 2313 p_rx_rcr_rings_t rx_rcr_rings; 2314 p_rx_rcr_ring_t *rcr_rings; 2315 p_rx_mbox_areas_t rx_mbox_areas_p; 2316 p_rx_mbox_t *rx_mbox_p; 2317 p_hxge_dma_pool_t dma_buf_poolp; 2318 p_hxge_dma_pool_t dma_rbr_cntl_poolp; 2319 p_hxge_dma_pool_t dma_rcr_cntl_poolp; 2320 p_hxge_dma_pool_t dma_mbox_cntl_poolp; 2321 p_hxge_dma_common_t *dma_buf_p; 2322 2323 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma")); 2324 2325 dma_buf_poolp = hxgep->rx_buf_pool_p; 2326 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p; 2327 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p; 2328 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p; 2329 2330 if (!dma_buf_poolp->buf_allocated || 2331 !dma_rbr_cntl_poolp->buf_allocated || 2332 !dma_rcr_cntl_poolp->buf_allocated || 2333 !dma_mbox_cntl_poolp->buf_allocated) { 2334 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2335 "<== hxge_unmap_rxdma: NULL buf pointers")); 2336 return; 2337 } 2338 2339 rx_rbr_rings = hxgep->rx_rbr_rings; 2340 rx_rcr_rings = hxgep->rx_rcr_rings; 2341 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 2342 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2343 "<== hxge_unmap_rxdma: NULL pointers")); 2344 return; 2345 } 2346 2347 ndmas = rx_rbr_rings->ndmas; 2348 if (!ndmas) { 2349 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2350 "<== hxge_unmap_rxdma: no channel")); 2351 return; 2352 } 2353 2354 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2355 "==> hxge_unmap_rxdma (ndmas %d)", ndmas)); 2356 2357 rbr_rings = rx_rbr_rings->rbr_rings; 2358 rcr_rings = rx_rcr_rings->rcr_rings; 2359 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 2360 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 2361 dma_buf_p = dma_buf_poolp->dma_buf_pool_p; 2362 2363 for (i = 0; i < ndmas; i++) { 2364 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 2365 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2366 "==> hxge_unmap_rxdma (ndmas %d) channel %d", 2367 ndmas, channel)); 2368 (void) hxge_unmap_rxdma_channel(hxgep, channel, 2369 (p_rx_rbr_ring_t)rbr_rings[i], 2370 (p_rx_rcr_ring_t)rcr_rings[i], 2371 (p_rx_mbox_t)rx_mbox_p[i]); 2372 } 2373 2374 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t)); 2375 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas); 2376 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t)); 2377 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas); 2378 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t)); 2379 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas); 2380 2381 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma")); 2382 } 2383 2384 hxge_status_t 2385 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2386 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 2387 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p, 2388 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p, 2389 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2390 { 2391 int status = HXGE_OK; 2392 2393 /* 2394 * Set up and prepare buffer blocks, descriptors and mailbox. 2395 */ 2396 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2397 "==> hxge_map_rxdma_channel (channel %d)", channel)); 2398 2399 /* 2400 * Receive buffer blocks 2401 */ 2402 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel, 2403 dma_buf_p, rbr_p, num_chunks); 2404 if (status != HXGE_OK) { 2405 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2406 "==> hxge_map_rxdma_channel (channel %d): " 2407 "map buffer failed 0x%x", channel, status)); 2408 goto hxge_map_rxdma_channel_exit; 2409 } 2410 2411 /* 2412 * Receive block ring, completion ring and mailbox. 2413 */ 2414 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel, 2415 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p, 2416 rbr_p, rcr_p, rx_mbox_p); 2417 if (status != HXGE_OK) { 2418 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2419 "==> hxge_map_rxdma_channel (channel %d): " 2420 "map config failed 0x%x", channel, status)); 2421 goto hxge_map_rxdma_channel_fail2; 2422 } 2423 goto hxge_map_rxdma_channel_exit; 2424 2425 hxge_map_rxdma_channel_fail3: 2426 /* Free rbr, rcr */ 2427 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2428 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)", 2429 status, channel)); 2430 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p); 2431 2432 hxge_map_rxdma_channel_fail2: 2433 /* Free buffer blocks */ 2434 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2435 "==> hxge_map_rxdma_channel: free rx buffers" 2436 "(hxgep 0x%x status 0x%x channel %d)", 2437 hxgep, status, channel)); 2438 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p); 2439 2440 status = HXGE_ERROR; 2441 2442 hxge_map_rxdma_channel_exit: 2443 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2444 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)", 2445 hxgep, status, channel)); 2446 2447 return (status); 2448 } 2449 2450 /*ARGSUSED*/ 2451 static void 2452 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel, 2453 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2454 { 2455 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2456 "==> hxge_unmap_rxdma_channel (channel %d)", channel)); 2457 2458 /* 2459 * unmap receive block ring, completion ring and mailbox. 2460 */ 2461 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p); 2462 2463 /* unmap buffer blocks */ 2464 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p); 2465 2466 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel")); 2467 } 2468 2469 /*ARGSUSED*/ 2470 static hxge_status_t 2471 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel, 2472 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p, 2473 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p, 2474 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 2475 { 2476 p_rx_rbr_ring_t rbrp; 2477 p_rx_rcr_ring_t rcrp; 2478 p_rx_mbox_t mboxp; 2479 p_hxge_dma_common_t cntl_dmap; 2480 p_hxge_dma_common_t dmap; 2481 p_rx_msg_t *rx_msg_ring; 2482 p_rx_msg_t rx_msg_p; 2483 rdc_rbr_cfg_a_t *rcfga_p; 2484 rdc_rbr_cfg_b_t *rcfgb_p; 2485 rdc_rcr_cfg_a_t *cfga_p; 2486 rdc_rcr_cfg_b_t *cfgb_p; 2487 rdc_rx_cfg1_t *cfig1_p; 2488 rdc_rx_cfg2_t *cfig2_p; 2489 rdc_rbr_kick_t *kick_p; 2490 uint32_t dmaaddrp; 2491 uint32_t *rbr_vaddrp; 2492 uint32_t bkaddr; 2493 hxge_status_t status = HXGE_OK; 2494 int i; 2495 uint32_t hxge_port_rcr_size; 2496 2497 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2498 "==> hxge_map_rxdma_channel_cfg_ring")); 2499 2500 cntl_dmap = *dma_rbr_cntl_p; 2501 2502 /* 2503 * Map in the receive block ring 2504 */ 2505 rbrp = *rbr_p; 2506 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc; 2507 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 2508 2509 /* 2510 * Zero out buffer block ring descriptors. 2511 */ 2512 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2513 2514 rcfga_p = &(rbrp->rbr_cfga); 2515 rcfgb_p = &(rbrp->rbr_cfgb); 2516 kick_p = &(rbrp->rbr_kick); 2517 rcfga_p->value = 0; 2518 rcfgb_p->value = 0; 2519 kick_p->value = 0; 2520 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 2521 rcfga_p->value = (rbrp->rbr_addr & 2522 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK)); 2523 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 2524 2525 /* XXXX: how to choose packet buffer sizes */ 2526 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0; 2527 rcfgb_p->bits.vld0 = 1; 2528 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1; 2529 rcfgb_p->bits.vld1 = 1; 2530 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2; 2531 rcfgb_p->bits.vld2 = 1; 2532 rcfgb_p->bits.bksize = hxgep->rx_bksize_code; 2533 2534 /* 2535 * For each buffer block, enter receive block address to the ring. 2536 */ 2537 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 2538 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 2539 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2540 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2541 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 2542 2543 rx_msg_ring = rbrp->rx_msg_ring; 2544 for (i = 0; i < rbrp->tnblocks; i++) { 2545 rx_msg_p = rx_msg_ring[i]; 2546 rx_msg_p->hxgep = hxgep; 2547 rx_msg_p->rx_rbr_p = rbrp; 2548 bkaddr = (uint32_t) 2549 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2550 RBR_BKADDR_SHIFT)); 2551 rx_msg_p->free = B_FALSE; 2552 rx_msg_p->max_usage_cnt = 0xbaddcafe; 2553 2554 *rbr_vaddrp++ = bkaddr; 2555 } 2556 2557 kick_p->bits.bkadd = rbrp->rbb_max; 2558 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 2559 2560 rbrp->rbr_rd_index = 0; 2561 2562 rbrp->rbr_consumed = 0; 2563 rbrp->rbr_use_bcopy = B_TRUE; 2564 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 2565 2566 /* 2567 * Do bcopy on packets greater than bcopy size once the lo threshold is 2568 * reached. This lo threshold should be less than the hi threshold. 2569 * 2570 * Do bcopy on every packet once the hi threshold is reached. 2571 */ 2572 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) { 2573 /* default it to use hi */ 2574 hxge_rx_threshold_lo = hxge_rx_threshold_hi; 2575 } 2576 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) { 2577 hxge_rx_buf_size_type = HXGE_RBR_TYPE2; 2578 } 2579 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type; 2580 2581 switch (hxge_rx_threshold_hi) { 2582 default: 2583 case HXGE_RX_COPY_NONE: 2584 /* Do not do bcopy at all */ 2585 rbrp->rbr_use_bcopy = B_FALSE; 2586 rbrp->rbr_threshold_hi = rbrp->rbb_max; 2587 break; 2588 2589 case HXGE_RX_COPY_1: 2590 case HXGE_RX_COPY_2: 2591 case HXGE_RX_COPY_3: 2592 case HXGE_RX_COPY_4: 2593 case HXGE_RX_COPY_5: 2594 case HXGE_RX_COPY_6: 2595 case HXGE_RX_COPY_7: 2596 rbrp->rbr_threshold_hi = 2597 rbrp->rbb_max * (hxge_rx_threshold_hi) / 2598 HXGE_RX_BCOPY_SCALE; 2599 break; 2600 2601 case HXGE_RX_COPY_ALL: 2602 rbrp->rbr_threshold_hi = 0; 2603 break; 2604 } 2605 2606 switch (hxge_rx_threshold_lo) { 2607 default: 2608 case HXGE_RX_COPY_NONE: 2609 /* Do not do bcopy at all */ 2610 if (rbrp->rbr_use_bcopy) { 2611 rbrp->rbr_use_bcopy = B_FALSE; 2612 } 2613 rbrp->rbr_threshold_lo = rbrp->rbb_max; 2614 break; 2615 2616 case HXGE_RX_COPY_1: 2617 case HXGE_RX_COPY_2: 2618 case HXGE_RX_COPY_3: 2619 case HXGE_RX_COPY_4: 2620 case HXGE_RX_COPY_5: 2621 case HXGE_RX_COPY_6: 2622 case HXGE_RX_COPY_7: 2623 rbrp->rbr_threshold_lo = 2624 rbrp->rbb_max * (hxge_rx_threshold_lo) / 2625 HXGE_RX_BCOPY_SCALE; 2626 break; 2627 2628 case HXGE_RX_COPY_ALL: 2629 rbrp->rbr_threshold_lo = 0; 2630 break; 2631 } 2632 2633 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2634 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d " 2635 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d " 2636 "rbb_threshold_lo %d", 2637 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type, 2638 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo)); 2639 2640 /* Map in the receive completion ring */ 2641 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 2642 rcrp->rdc = dma_channel; 2643 rcrp->hxgep = hxgep; 2644 2645 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 2646 rcrp->comp_size = hxge_port_rcr_size; 2647 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1; 2648 2649 rcrp->max_receive_pkts = hxge_max_rx_pkts; 2650 2651 cntl_dmap = *dma_rcr_cntl_p; 2652 2653 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 2654 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 2655 sizeof (rcr_entry_t)); 2656 rcrp->comp_rd_index = 0; 2657 rcrp->comp_wt_index = 0; 2658 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 2659 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 2660 #if defined(__i386) 2661 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2662 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2663 #else 2664 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 2665 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 2666 #endif 2667 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 2668 (hxge_port_rcr_size - 1); 2669 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 2670 (hxge_port_rcr_size - 1); 2671 2672 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 2673 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 2674 2675 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2676 "==> hxge_map_rxdma_channel_cfg_ring: channel %d " 2677 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p " 2678 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p " 2679 "rcr_desc_rd_last_pp $%p ", 2680 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p, 2681 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p, 2682 rcrp->rcr_desc_last_pp)); 2683 2684 /* 2685 * Zero out buffer block ring descriptors. 2686 */ 2687 bzero((caddr_t)dmap->kaddrp, dmap->alength); 2688 rcrp->intr_timeout = hxgep->intr_timeout; 2689 rcrp->intr_threshold = hxgep->intr_threshold; 2690 rcrp->full_hdr_flag = B_FALSE; 2691 rcrp->sw_priv_hdr_len = 0; 2692 2693 cfga_p = &(rcrp->rcr_cfga); 2694 cfgb_p = &(rcrp->rcr_cfgb); 2695 cfga_p->value = 0; 2696 cfgb_p->value = 0; 2697 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 2698 2699 cfga_p->value = (rcrp->rcr_addr & 2700 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK)); 2701 2702 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF); 2703 2704 /* 2705 * Timeout should be set based on the system clock divider. The 2706 * following timeout value of 1 assumes that the granularity (1000) is 2707 * 3 microseconds running at 300MHz. 2708 */ 2709 cfgb_p->bits.pthres = rcrp->intr_threshold; 2710 cfgb_p->bits.timeout = rcrp->intr_timeout; 2711 cfgb_p->bits.entout = 1; 2712 2713 /* Map in the mailbox */ 2714 cntl_dmap = *dma_mbox_cntl_p; 2715 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 2716 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox; 2717 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 2718 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1; 2719 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2; 2720 cfig1_p->value = cfig2_p->value = 0; 2721 2722 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 2723 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2724 "==> hxge_map_rxdma_channel_cfg_ring: " 2725 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 2726 dma_channel, cfig1_p->value, cfig2_p->value, 2727 mboxp->mbox_addr)); 2728 2729 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff); 2730 cfig1_p->bits.mbaddr_h = dmaaddrp; 2731 2732 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 2733 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 2734 RXDMA_CFIG2_MBADDR_L_MASK); 2735 2736 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 2737 2738 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2739 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p " 2740 "cfg1 0x%016llx cfig2 0x%016llx", 2741 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value)); 2742 2743 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag; 2744 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len; 2745 2746 rbrp->rx_rcr_p = rcrp; 2747 rcrp->rx_rbr_p = rbrp; 2748 *rcr_p = rcrp; 2749 *rx_mbox_p = mboxp; 2750 2751 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2752 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 2753 return (status); 2754 } 2755 2756 /*ARGSUSED*/ 2757 static void 2758 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep, 2759 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 2760 { 2761 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2762 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc)); 2763 2764 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 2765 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 2766 2767 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2768 "<== hxge_unmap_rxdma_channel_cfg_ring")); 2769 } 2770 2771 static hxge_status_t 2772 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel, 2773 p_hxge_dma_common_t *dma_buf_p, 2774 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 2775 { 2776 p_rx_rbr_ring_t rbrp; 2777 p_hxge_dma_common_t dma_bufp, tmp_bufp; 2778 p_rx_msg_t *rx_msg_ring; 2779 p_rx_msg_t rx_msg_p; 2780 p_mblk_t mblk_p; 2781 2782 rxring_info_t *ring_info; 2783 hxge_status_t status = HXGE_OK; 2784 int i, j, index; 2785 uint32_t size, bsize, nblocks, nmsgs; 2786 2787 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2788 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel)); 2789 2790 dma_bufp = tmp_bufp = *dma_buf_p; 2791 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2792 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d " 2793 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp)); 2794 2795 nmsgs = 0; 2796 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 2797 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2798 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2799 "bufp 0x%016llx nblocks %d nmsgs %d", 2800 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 2801 nmsgs += tmp_bufp->nblocks; 2802 } 2803 if (!nmsgs) { 2804 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2805 "<== hxge_map_rxdma_channel_buf_ring: channel %d " 2806 "no msg blocks", channel)); 2807 status = HXGE_ERROR; 2808 goto hxge_map_rxdma_channel_buf_ring_exit; 2809 } 2810 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP); 2811 2812 size = nmsgs * sizeof (p_rx_msg_t); 2813 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 2814 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 2815 KM_SLEEP); 2816 2817 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 2818 (void *) hxgep->interrupt_cookie); 2819 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 2820 (void *) hxgep->interrupt_cookie); 2821 2822 rbrp->rdc = channel; 2823 rbrp->num_blocks = num_chunks; 2824 rbrp->tnblocks = nmsgs; 2825 rbrp->rbb_max = nmsgs; 2826 rbrp->rbr_max_size = nmsgs; 2827 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 2828 2829 /* 2830 * Buffer sizes suggested by NIU architect. 256, 512 and 2K. 2831 */ 2832 2833 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 2834 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 2835 rbrp->hpi_pkt_buf_size0 = SIZE_256B; 2836 2837 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 2838 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 2839 rbrp->hpi_pkt_buf_size1 = SIZE_1KB; 2840 2841 rbrp->block_size = hxgep->rx_default_block_size; 2842 2843 if (!hxgep->param_arr[param_accept_jumbo].value) { 2844 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 2845 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 2846 rbrp->hpi_pkt_buf_size2 = SIZE_2KB; 2847 } else { 2848 rbrp->hpi_pkt_buf_size2 = SIZE_4KB; 2849 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 2850 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 2851 } 2852 2853 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2854 "==> hxge_map_rxdma_channel_buf_ring: channel %d " 2855 "actual rbr max %d rbb_max %d nmsgs %d " 2856 "rbrp->block_size %d default_block_size %d " 2857 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)", 2858 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 2859 rbrp->block_size, hxgep->rx_default_block_size, 2860 hxge_rbr_size, hxge_rbr_spare_size)); 2861 2862 /* 2863 * Map in buffers from the buffer pool. 2864 * Note that num_blocks is the num_chunks. For Sparc, there is likely 2865 * only one chunk. For x86, there will be many chunks. 2866 * Loop over chunks. 2867 */ 2868 index = 0; 2869 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 2870 bsize = dma_bufp->block_size; 2871 nblocks = dma_bufp->nblocks; 2872 #if defined(__i386) 2873 ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 2874 #else 2875 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 2876 #endif 2877 ring_info->buffer[i].buf_index = i; 2878 ring_info->buffer[i].buf_size = dma_bufp->alength; 2879 ring_info->buffer[i].start_index = index; 2880 #if defined(__i386) 2881 ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 2882 #else 2883 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 2884 #endif 2885 2886 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2887 " hxge_map_rxdma_channel_buf_ring: map channel %d " 2888 "chunk %d nblocks %d chunk_size %x block_size 0x%x " 2889 "dma_bufp $%p dvma_addr $%p", channel, i, 2890 dma_bufp->nblocks, 2891 ring_info->buffer[i].buf_size, bsize, dma_bufp, 2892 ring_info->buffer[i].dvma_addr)); 2893 2894 /* loop over blocks within a chunk */ 2895 for (j = 0; j < nblocks; j++) { 2896 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO, 2897 dma_bufp)) == NULL) { 2898 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 2899 "allocb failed (index %d i %d j %d)", 2900 index, i, j)); 2901 goto hxge_map_rxdma_channel_buf_ring_fail1; 2902 } 2903 rx_msg_ring[index] = rx_msg_p; 2904 rx_msg_p->block_index = index; 2905 rx_msg_p->shifted_addr = (uint32_t) 2906 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 2907 RBR_BKADDR_SHIFT)); 2908 /* 2909 * Too much output 2910 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2911 * "index %d j %d rx_msg_p $%p mblk %p", 2912 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 2913 */ 2914 mblk_p = rx_msg_p->rx_mblk_p; 2915 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 2916 2917 rbrp->rbr_ref_cnt++; 2918 index++; 2919 rx_msg_p->buf_dma.dma_channel = channel; 2920 } 2921 } 2922 if (i < rbrp->num_blocks) { 2923 goto hxge_map_rxdma_channel_buf_ring_fail1; 2924 } 2925 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2926 "hxge_map_rxdma_channel_buf_ring: done buf init " 2927 "channel %d msg block entries %d", channel, index)); 2928 ring_info->block_size_mask = bsize - 1; 2929 rbrp->rx_msg_ring = rx_msg_ring; 2930 rbrp->dma_bufp = dma_buf_p; 2931 rbrp->ring_info = ring_info; 2932 2933 status = hxge_rxbuf_index_info_init(hxgep, rbrp); 2934 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: " 2935 "channel %d done buf info init", channel)); 2936 2937 /* 2938 * Finally, permit hxge_freeb() to call hxge_post_page(). 2939 */ 2940 rbrp->rbr_state = RBR_POSTING; 2941 2942 *rbr_p = rbrp; 2943 2944 goto hxge_map_rxdma_channel_buf_ring_exit; 2945 2946 hxge_map_rxdma_channel_buf_ring_fail1: 2947 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2948 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 2949 channel, status)); 2950 2951 index--; 2952 for (; index >= 0; index--) { 2953 rx_msg_p = rx_msg_ring[index]; 2954 if (rx_msg_p != NULL) { 2955 hxge_freeb(rx_msg_p); 2956 rx_msg_ring[index] = NULL; 2957 } 2958 } 2959 2960 hxge_map_rxdma_channel_buf_ring_fail: 2961 MUTEX_DESTROY(&rbrp->post_lock); 2962 MUTEX_DESTROY(&rbrp->lock); 2963 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 2964 KMEM_FREE(rx_msg_ring, size); 2965 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 2966 2967 status = HXGE_ERROR; 2968 2969 hxge_map_rxdma_channel_buf_ring_exit: 2970 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2971 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 2972 2973 return (status); 2974 } 2975 2976 /*ARGSUSED*/ 2977 static void 2978 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep, 2979 p_rx_rbr_ring_t rbr_p) 2980 { 2981 p_rx_msg_t *rx_msg_ring; 2982 p_rx_msg_t rx_msg_p; 2983 rxring_info_t *ring_info; 2984 int i; 2985 uint32_t size; 2986 2987 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2988 "==> hxge_unmap_rxdma_channel_buf_ring")); 2989 if (rbr_p == NULL) { 2990 HXGE_DEBUG_MSG((hxgep, RX_CTL, 2991 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 2992 return; 2993 } 2994 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 2995 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc)); 2996 2997 rx_msg_ring = rbr_p->rx_msg_ring; 2998 ring_info = rbr_p->ring_info; 2999 3000 if (rx_msg_ring == NULL || ring_info == NULL) { 3001 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3002 "<== hxge_unmap_rxdma_channel_buf_ring: " 3003 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info)); 3004 return; 3005 } 3006 3007 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 3008 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3009 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 3010 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks, 3011 rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 3012 3013 for (i = 0; i < rbr_p->tnblocks; i++) { 3014 rx_msg_p = rx_msg_ring[i]; 3015 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3016 " hxge_unmap_rxdma_channel_buf_ring: " 3017 "rx_msg_p $%p", rx_msg_p)); 3018 if (rx_msg_p != NULL) { 3019 hxge_freeb(rx_msg_p); 3020 rx_msg_ring[i] = NULL; 3021 } 3022 } 3023 3024 /* 3025 * We no longer may use the mutex <post_lock>. By setting 3026 * <rbr_state> to anything but POSTING, we prevent 3027 * hxge_post_page() from accessing a dead mutex. 3028 */ 3029 rbr_p->rbr_state = RBR_UNMAPPING; 3030 MUTEX_DESTROY(&rbr_p->post_lock); 3031 3032 MUTEX_DESTROY(&rbr_p->lock); 3033 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 3034 KMEM_FREE(rx_msg_ring, size); 3035 3036 if (rbr_p->rbr_ref_cnt == 0) { 3037 /* This is the normal state of affairs. */ 3038 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 3039 } else { 3040 /* 3041 * Some of our buffers are still being used. 3042 * Therefore, tell hxge_freeb() this ring is 3043 * unmapped, so it may free <rbr_p> for us. 3044 */ 3045 rbr_p->rbr_state = RBR_UNMAPPED; 3046 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3047 "unmap_rxdma_buf_ring: %d %s outstanding.", 3048 rbr_p->rbr_ref_cnt, 3049 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 3050 } 3051 3052 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3053 "<== hxge_unmap_rxdma_channel_buf_ring")); 3054 } 3055 3056 static hxge_status_t 3057 hxge_rxdma_hw_start_common(p_hxge_t hxgep) 3058 { 3059 hxge_status_t status = HXGE_OK; 3060 3061 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3062 3063 /* 3064 * Load the sharable parameters by writing to the function zero control 3065 * registers. These FZC registers should be initialized only once for 3066 * the entire chip. 3067 */ 3068 (void) hxge_init_fzc_rx_common(hxgep); 3069 3070 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common")); 3071 3072 return (status); 3073 } 3074 3075 static hxge_status_t 3076 hxge_rxdma_hw_start(p_hxge_t hxgep) 3077 { 3078 int i, ndmas; 3079 uint16_t channel; 3080 p_rx_rbr_rings_t rx_rbr_rings; 3081 p_rx_rbr_ring_t *rbr_rings; 3082 p_rx_rcr_rings_t rx_rcr_rings; 3083 p_rx_rcr_ring_t *rcr_rings; 3084 p_rx_mbox_areas_t rx_mbox_areas_p; 3085 p_rx_mbox_t *rx_mbox_p; 3086 hxge_status_t status = HXGE_OK; 3087 3088 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start")); 3089 3090 rx_rbr_rings = hxgep->rx_rbr_rings; 3091 rx_rcr_rings = hxgep->rx_rcr_rings; 3092 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3093 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3094 "<== hxge_rxdma_hw_start: NULL ring pointers")); 3095 return (HXGE_ERROR); 3096 } 3097 3098 ndmas = rx_rbr_rings->ndmas; 3099 if (ndmas == 0) { 3100 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3101 "<== hxge_rxdma_hw_start: no dma channel allocated")); 3102 return (HXGE_ERROR); 3103 } 3104 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3105 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas)); 3106 3107 /* 3108 * Scrub the RDC Rx DMA Prefetch Buffer Command. 3109 */ 3110 for (i = 0; i < 128; i++) { 3111 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i); 3112 } 3113 3114 /* 3115 * Scrub Rx DMA Shadow Tail Command. 3116 */ 3117 for (i = 0; i < 64; i++) { 3118 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i); 3119 } 3120 3121 /* 3122 * Scrub Rx DMA Control Fifo Command. 3123 */ 3124 for (i = 0; i < 512; i++) { 3125 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i); 3126 } 3127 3128 /* 3129 * Scrub Rx DMA Data Fifo Command. 3130 */ 3131 for (i = 0; i < 1536; i++) { 3132 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i); 3133 } 3134 3135 /* 3136 * Reset the FIFO Error Stat. 3137 */ 3138 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF); 3139 3140 /* Set the error mask to receive interrupts */ 3141 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3142 3143 rbr_rings = rx_rbr_rings->rbr_rings; 3144 rcr_rings = rx_rcr_rings->rcr_rings; 3145 rx_mbox_areas_p = hxgep->rx_mbox_areas_p; 3146 if (rx_mbox_areas_p) { 3147 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 3148 } 3149 3150 for (i = 0; i < ndmas; i++) { 3151 channel = rbr_rings[i]->rdc; 3152 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3153 "==> hxge_rxdma_hw_start (ndmas %d) channel %d", 3154 ndmas, channel)); 3155 status = hxge_rxdma_start_channel(hxgep, channel, 3156 (p_rx_rbr_ring_t)rbr_rings[i], 3157 (p_rx_rcr_ring_t)rcr_rings[i], 3158 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max); 3159 if (status != HXGE_OK) { 3160 goto hxge_rxdma_hw_start_fail1; 3161 } 3162 } 3163 3164 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: " 3165 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3166 rx_rbr_rings, rx_rcr_rings)); 3167 goto hxge_rxdma_hw_start_exit; 3168 3169 hxge_rxdma_hw_start_fail1: 3170 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3171 "==> hxge_rxdma_hw_start: disable " 3172 "(status 0x%x channel %d i %d)", status, channel, i)); 3173 for (; i >= 0; i--) { 3174 channel = rbr_rings[i]->rdc; 3175 (void) hxge_rxdma_stop_channel(hxgep, channel); 3176 } 3177 3178 hxge_rxdma_hw_start_exit: 3179 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3180 "==> hxge_rxdma_hw_start: (status 0x%x)", status)); 3181 return (status); 3182 } 3183 3184 static void 3185 hxge_rxdma_hw_stop(p_hxge_t hxgep) 3186 { 3187 int i, ndmas; 3188 uint16_t channel; 3189 p_rx_rbr_rings_t rx_rbr_rings; 3190 p_rx_rbr_ring_t *rbr_rings; 3191 p_rx_rcr_rings_t rx_rcr_rings; 3192 3193 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop")); 3194 3195 rx_rbr_rings = hxgep->rx_rbr_rings; 3196 rx_rcr_rings = hxgep->rx_rcr_rings; 3197 3198 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 3199 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3200 "<== hxge_rxdma_hw_stop: NULL ring pointers")); 3201 return; 3202 } 3203 3204 ndmas = rx_rbr_rings->ndmas; 3205 if (!ndmas) { 3206 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3207 "<== hxge_rxdma_hw_stop: no dma channel allocated")); 3208 return; 3209 } 3210 3211 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3212 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas)); 3213 3214 rbr_rings = rx_rbr_rings->rbr_rings; 3215 for (i = 0; i < ndmas; i++) { 3216 channel = rbr_rings[i]->rdc; 3217 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3218 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d", 3219 ndmas, channel)); 3220 (void) hxge_rxdma_stop_channel(hxgep, channel); 3221 } 3222 3223 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: " 3224 "rx_rbr_rings 0x%016llx rings 0x%016llx", 3225 rx_rbr_rings, rx_rcr_rings)); 3226 3227 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop")); 3228 } 3229 3230 static hxge_status_t 3231 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel, 3232 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p, 3233 int n_init_kick) 3234 { 3235 hpi_handle_t handle; 3236 hpi_status_t rs = HPI_SUCCESS; 3237 rdc_stat_t cs; 3238 rdc_int_mask_t ent_mask; 3239 hxge_status_t status = HXGE_OK; 3240 3241 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel")); 3242 3243 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3244 3245 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: " 3246 "hpi handle addr $%p acc $%p", 3247 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3248 3249 /* Reset RXDMA channel */ 3250 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3251 if (rs != HPI_SUCCESS) { 3252 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3253 "==> hxge_rxdma_start_channel: " 3254 "reset rxdma failed (0x%08x channel %d)", 3255 status, channel)); 3256 return (HXGE_ERROR | rs); 3257 } 3258 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3259 "==> hxge_rxdma_start_channel: reset done: channel %d", channel)); 3260 3261 /* 3262 * Initialize the RXDMA channel specific FZC control configurations. 3263 * These FZC registers are pertaining to each RX channel (logical 3264 * pages). 3265 */ 3266 status = hxge_init_fzc_rxdma_channel(hxgep, 3267 channel, rbr_p, rcr_p, mbox_p); 3268 if (status != HXGE_OK) { 3269 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3270 "==> hxge_rxdma_start_channel: " 3271 "init fzc rxdma failed (0x%08x channel %d)", 3272 status, channel)); 3273 return (status); 3274 } 3275 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3276 "==> hxge_rxdma_start_channel: fzc done")); 3277 3278 /* 3279 * Zero out the shadow and prefetch ram. 3280 */ 3281 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3282 "==> hxge_rxdma_start_channel: ram done")); 3283 3284 /* Set up the interrupt event masks. */ 3285 ent_mask.value = 0; 3286 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3287 if (rs != HPI_SUCCESS) { 3288 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3289 "==> hxge_rxdma_start_channel: " 3290 "init rxdma event masks failed (0x%08x channel %d)", 3291 status, channel)); 3292 return (HXGE_ERROR | rs); 3293 } 3294 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3295 "event done: channel %d (mask 0x%016llx)", 3296 channel, ent_mask.value)); 3297 3298 /* 3299 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA 3300 * channels and enable each DMA channel. 3301 */ 3302 status = hxge_enable_rxdma_channel(hxgep, 3303 channel, rbr_p, rcr_p, mbox_p, n_init_kick); 3304 if (status != HXGE_OK) { 3305 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3306 " hxge_rxdma_start_channel: " 3307 " init enable rxdma failed (0x%08x channel %d)", 3308 status, channel)); 3309 return (status); 3310 } 3311 3312 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3313 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3314 3315 /* 3316 * Initialize the receive DMA control and status register 3317 * Note that rdc_stat HAS to be set after RBR and RCR rings are set 3318 */ 3319 cs.value = 0; 3320 cs.bits.mex = 1; 3321 cs.bits.rcr_thres = 1; 3322 cs.bits.rcr_to = 1; 3323 cs.bits.rbr_empty = 1; 3324 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3325 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3326 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 3327 if (status != HXGE_OK) { 3328 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3329 "==> hxge_rxdma_start_channel: " 3330 "init rxdma control register failed (0x%08x channel %d", 3331 status, channel)); 3332 return (status); 3333 } 3334 3335 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: " 3336 "control done - channel %d cs 0x%016llx", channel, cs.value)); 3337 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, 3338 "==> hxge_rxdma_start_channel: enable done")); 3339 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel")); 3340 3341 return (HXGE_OK); 3342 } 3343 3344 static hxge_status_t 3345 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel) 3346 { 3347 hpi_handle_t handle; 3348 hpi_status_t rs = HPI_SUCCESS; 3349 rdc_stat_t cs; 3350 rdc_int_mask_t ent_mask; 3351 hxge_status_t status = HXGE_OK; 3352 3353 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel")); 3354 3355 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3356 3357 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: " 3358 "hpi handle addr $%p acc $%p", 3359 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh)); 3360 3361 /* Reset RXDMA channel */ 3362 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3363 if (rs != HPI_SUCCESS) { 3364 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3365 " hxge_rxdma_stop_channel: " 3366 " reset rxdma failed (0x%08x channel %d)", 3367 rs, channel)); 3368 return (HXGE_ERROR | rs); 3369 } 3370 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3371 "==> hxge_rxdma_stop_channel: reset done")); 3372 3373 /* Set up the interrupt event masks. */ 3374 ent_mask.value = RDC_INT_MASK_ALL; 3375 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3376 if (rs != HPI_SUCCESS) { 3377 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3378 "==> hxge_rxdma_stop_channel: " 3379 "set rxdma event masks failed (0x%08x channel %d)", 3380 rs, channel)); 3381 return (HXGE_ERROR | rs); 3382 } 3383 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3384 "==> hxge_rxdma_stop_channel: event done")); 3385 3386 /* Initialize the receive DMA control and status register */ 3387 cs.value = 0; 3388 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs); 3389 3390 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control " 3391 " to default (all 0s) 0x%08x", cs.value)); 3392 3393 if (status != HXGE_OK) { 3394 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3395 " hxge_rxdma_stop_channel: init rxdma" 3396 " control register failed (0x%08x channel %d", 3397 status, channel)); 3398 return (status); 3399 } 3400 3401 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3402 "==> hxge_rxdma_stop_channel: control done")); 3403 3404 /* disable dma channel */ 3405 status = hxge_disable_rxdma_channel(hxgep, channel); 3406 3407 if (status != HXGE_OK) { 3408 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3409 " hxge_rxdma_stop_channel: " 3410 " init enable rxdma failed (0x%08x channel %d)", 3411 status, channel)); 3412 return (status); 3413 } 3414 3415 HXGE_DEBUG_MSG((hxgep, RX_CTL, 3416 "==> hxge_rxdma_stop_channel: disable done")); 3417 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel")); 3418 3419 return (HXGE_OK); 3420 } 3421 3422 hxge_status_t 3423 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep) 3424 { 3425 hpi_handle_t handle; 3426 p_hxge_rdc_sys_stats_t statsp; 3427 rdc_fifo_err_stat_t stat; 3428 hxge_status_t status = HXGE_OK; 3429 3430 handle = hxgep->hpi_handle; 3431 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats; 3432 3433 /* Clear the int_dbg register in case it is an injected err */ 3434 HXGE_REG_WR64(handle, RDC_FIFO_ERR_INT_DBG, 0x0); 3435 3436 /* Get the error status and clear the register */ 3437 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value); 3438 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value); 3439 3440 if (stat.bits.rx_ctrl_fifo_sec) { 3441 statsp->ctrl_fifo_sec++; 3442 if (statsp->ctrl_fifo_sec == 1) 3443 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3444 "==> hxge_rxdma_handle_sys_errors: " 3445 "rx_ctrl_fifo_sec")); 3446 } 3447 3448 if (stat.bits.rx_ctrl_fifo_ded) { 3449 /* Global fatal error encountered */ 3450 statsp->ctrl_fifo_ded++; 3451 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3452 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED); 3453 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3454 "==> hxge_rxdma_handle_sys_errors: " 3455 "fatal error: rx_ctrl_fifo_ded error")); 3456 } 3457 3458 if (stat.bits.rx_data_fifo_sec) { 3459 statsp->data_fifo_sec++; 3460 if (statsp->data_fifo_sec == 1) 3461 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3462 "==> hxge_rxdma_handle_sys_errors: " 3463 "rx_data_fifo_sec")); 3464 } 3465 3466 if (stat.bits.rx_data_fifo_ded) { 3467 /* Global fatal error encountered */ 3468 statsp->data_fifo_ded++; 3469 HXGE_FM_REPORT_ERROR(hxgep, NULL, 3470 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED); 3471 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3472 "==> hxge_rxdma_handle_sys_errors: " 3473 "fatal error: rx_data_fifo_ded error")); 3474 } 3475 3476 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) { 3477 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3478 " hxge_rxdma_handle_sys_errors: fatal error\n")); 3479 status = hxge_rx_port_fatal_err_recover(hxgep); 3480 if (status == HXGE_OK) { 3481 FM_SERVICE_RESTORED(hxgep); 3482 } 3483 } 3484 3485 return (HXGE_OK); 3486 } 3487 3488 static hxge_status_t 3489 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel) 3490 { 3491 hpi_handle_t handle; 3492 hpi_status_t rs = HPI_SUCCESS; 3493 hxge_status_t status = HXGE_OK; 3494 p_rx_rbr_ring_t rbrp; 3495 p_rx_rcr_ring_t rcrp; 3496 p_rx_mbox_t mboxp; 3497 rdc_int_mask_t ent_mask; 3498 p_hxge_dma_common_t dmap; 3499 int ring_idx; 3500 p_rx_msg_t rx_msg_p; 3501 int i; 3502 uint32_t hxge_port_rcr_size; 3503 uint64_t tmp; 3504 int n_init_kick = 0; 3505 3506 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover")); 3507 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3508 "Recovering from RxDMAChannel#%d error...", channel)); 3509 3510 /* 3511 * Stop the dma channel waits for the stop done. If the stop done bit 3512 * is not set, then create an error. 3513 */ 3514 3515 handle = HXGE_DEV_HPI_HANDLE(hxgep); 3516 3517 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop...")); 3518 3519 ring_idx = hxge_rxdma_get_ring_index(hxgep, channel); 3520 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[ring_idx]; 3521 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[ring_idx]; 3522 3523 MUTEX_ENTER(&rcrp->lock); 3524 MUTEX_ENTER(&rbrp->lock); 3525 MUTEX_ENTER(&rbrp->post_lock); 3526 3527 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel...")); 3528 3529 rs = hpi_rxdma_cfg_rdc_disable(handle, channel); 3530 if (rs != HPI_SUCCESS) { 3531 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3532 "hxge_disable_rxdma_channel:failed")); 3533 goto fail; 3534 } 3535 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt...")); 3536 3537 /* Disable interrupt */ 3538 ent_mask.value = RDC_INT_MASK_ALL; 3539 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 3540 if (rs != HPI_SUCCESS) { 3541 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3542 "Set rxdma event masks failed (channel %d)", channel)); 3543 } 3544 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset...")); 3545 3546 /* Reset RXDMA channel */ 3547 rs = hpi_rxdma_cfg_rdc_reset(handle, channel); 3548 if (rs != HPI_SUCCESS) { 3549 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3550 "Reset rxdma failed (channel %d)", channel)); 3551 goto fail; 3552 } 3553 hxge_port_rcr_size = hxgep->hxge_port_rcr_size; 3554 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 3555 3556 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 3557 rbrp->rbr_rd_index = 0; 3558 3559 rcrp->comp_rd_index = 0; 3560 rcrp->comp_wt_index = 0; 3561 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 3562 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 3563 #if defined(__i386) 3564 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3565 (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3566 #else 3567 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 3568 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 3569 #endif 3570 3571 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 3572 (hxge_port_rcr_size - 1); 3573 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 3574 (hxge_port_rcr_size - 1); 3575 3576 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc); 3577 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3; 3578 3579 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc; 3580 bzero((caddr_t)dmap->kaddrp, dmap->alength); 3581 3582 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n", 3583 rbrp->rbr_max_size)); 3584 3585 /* Count the number of buffers owned by the hardware at this moment */ 3586 for (i = 0; i < rbrp->rbr_max_size; i++) { 3587 rx_msg_p = rbrp->rx_msg_ring[i]; 3588 if (rx_msg_p->ref_cnt == 1) { 3589 n_init_kick++; 3590 } 3591 } 3592 3593 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start...")); 3594 3595 /* 3596 * This is error recover! Some buffers are owned by the hardware and 3597 * the rest are owned by the apps. We should only kick in those 3598 * owned by the hardware initially. The apps will post theirs 3599 * eventually. 3600 */ 3601 status = hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp, 3602 n_init_kick); 3603 if (status != HXGE_OK) { 3604 goto fail; 3605 } 3606 3607 /* 3608 * The DMA channel may disable itself automatically. 3609 * The following is a work-around. 3610 */ 3611 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp); 3612 rs = hpi_rxdma_cfg_rdc_enable(handle, channel); 3613 if (rs != HPI_SUCCESS) { 3614 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3615 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel)); 3616 } 3617 3618 MUTEX_EXIT(&rbrp->post_lock); 3619 MUTEX_EXIT(&rbrp->lock); 3620 MUTEX_EXIT(&rcrp->lock); 3621 3622 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3623 "Recovery Successful, RxDMAChannel#%d Restored", channel)); 3624 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover")); 3625 3626 return (HXGE_OK); 3627 3628 fail: 3629 MUTEX_EXIT(&rbrp->post_lock); 3630 MUTEX_EXIT(&rbrp->lock); 3631 MUTEX_EXIT(&rcrp->lock); 3632 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3633 3634 return (HXGE_ERROR | rs); 3635 } 3636 3637 static hxge_status_t 3638 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep) 3639 { 3640 hxge_status_t status = HXGE_OK; 3641 p_hxge_dma_common_t *dma_buf_p; 3642 uint16_t channel; 3643 int ndmas; 3644 int i; 3645 block_reset_t reset_reg; 3646 3647 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover")); 3648 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ...")); 3649 3650 /* Reset RDC block from PEU for this fatal error */ 3651 reset_reg.value = 0; 3652 reset_reg.bits.rdc_rst = 1; 3653 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value); 3654 3655 /* Disable RxMAC */ 3656 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n")); 3657 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK) 3658 goto fail; 3659 3660 HXGE_DELAY(1000); 3661 3662 /* Restore any common settings after PEU reset */ 3663 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK) 3664 goto fail; 3665 3666 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels...")); 3667 3668 ndmas = hxgep->rx_buf_pool_p->ndmas; 3669 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p; 3670 3671 for (i = 0; i < ndmas; i++) { 3672 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel; 3673 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) { 3674 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3675 "Could not recover channel %d", channel)); 3676 } 3677 } 3678 3679 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC...")); 3680 3681 /* Reset RxMAC */ 3682 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) { 3683 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3684 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3685 goto fail; 3686 } 3687 3688 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC...")); 3689 3690 /* Re-Initialize RxMAC */ 3691 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) { 3692 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3693 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC")); 3694 goto fail; 3695 } 3696 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC...")); 3697 3698 /* Re-enable RxMAC */ 3699 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) { 3700 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3701 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC")); 3702 goto fail; 3703 } 3704 3705 /* Reset the error mask since PEU reset cleared it */ 3706 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0); 3707 3708 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, 3709 "Recovery Successful, RxPort Restored")); 3710 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover")); 3711 3712 return (HXGE_OK); 3713 fail: 3714 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed")); 3715 return (status); 3716 } 3717