1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/kernel.h> 34 #include <linux/slab.h> 35 #include <linux/mm.h> 36 #include <linux/scatterlist.h> 37 #include <linux/kfifo.h> 38 #include <scsi/scsi_cmnd.h> 39 #include <scsi/scsi_host.h> 40 41 #include "iscsi_iser.h" 42 43 /* Register user buffer memory and initialize passive rdma 44 * dto descriptor. Data size is stored in 45 * task->data[ISER_DIR_IN].data_len, Protection size 46 * os stored in task->prot[ISER_DIR_IN].data_len 47 */ 48 static int iser_prepare_read_cmd(struct iscsi_task *task) 49 50 { 51 struct iscsi_iser_task *iser_task = task->dd_data; 52 struct iser_mem_reg *mem_reg; 53 int err; 54 struct iser_ctrl *hdr = &iser_task->desc.iser_header; 55 56 err = iser_dma_map_task_data(iser_task, 57 ISER_DIR_IN, 58 DMA_FROM_DEVICE); 59 if (err) 60 return err; 61 62 err = iser_reg_mem_fastreg(iser_task, ISER_DIR_IN, false); 63 if (err) { 64 iser_err("Failed to set up Data-IN RDMA\n"); 65 goto out_err; 66 } 67 mem_reg = &iser_task->rdma_reg[ISER_DIR_IN]; 68 69 hdr->flags |= ISER_RSV; 70 hdr->read_stag = cpu_to_be32(mem_reg->rkey); 71 hdr->read_va = cpu_to_be64(mem_reg->sge.addr); 72 73 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", 74 task->itt, mem_reg->rkey, 75 (unsigned long long)mem_reg->sge.addr); 76 77 return 0; 78 79 out_err: 80 iser_dma_unmap_task_data(iser_task, ISER_DIR_IN, DMA_FROM_DEVICE); 81 return err; 82 } 83 84 /* Register user buffer memory and initialize passive rdma 85 * dto descriptor. Data size is stored in 86 * task->data[ISER_DIR_OUT].data_len, Protection size 87 * is stored at task->prot[ISER_DIR_OUT].data_len 88 */ 89 static int iser_prepare_write_cmd(struct iscsi_task *task, unsigned int imm_sz, 90 unsigned int unsol_sz, unsigned int edtl) 91 { 92 struct iscsi_iser_task *iser_task = task->dd_data; 93 struct iser_mem_reg *mem_reg; 94 int err; 95 struct iser_ctrl *hdr = &iser_task->desc.iser_header; 96 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; 97 struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; 98 99 err = iser_dma_map_task_data(iser_task, 100 ISER_DIR_OUT, 101 DMA_TO_DEVICE); 102 if (err) 103 return err; 104 105 err = iser_reg_mem_fastreg(iser_task, ISER_DIR_OUT, 106 buf_out->data_len == imm_sz); 107 if (err) { 108 iser_err("Failed to register write cmd RDMA mem\n"); 109 goto out_err; 110 } 111 112 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; 113 114 if (unsol_sz < edtl) { 115 hdr->flags |= ISER_WSV; 116 if (buf_out->data_len > imm_sz) { 117 hdr->write_stag = cpu_to_be32(mem_reg->rkey); 118 hdr->write_va = cpu_to_be64(mem_reg->sge.addr + unsol_sz); 119 } 120 121 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X VA:%#llX + unsol:%d\n", 122 task->itt, mem_reg->rkey, 123 (unsigned long long)mem_reg->sge.addr, unsol_sz); 124 } 125 126 if (imm_sz > 0) { 127 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", 128 task->itt, imm_sz); 129 tx_dsg->addr = mem_reg->sge.addr; 130 tx_dsg->length = imm_sz; 131 tx_dsg->lkey = mem_reg->sge.lkey; 132 iser_task->desc.num_sge = 2; 133 } 134 135 return 0; 136 137 out_err: 138 iser_dma_unmap_task_data(iser_task, ISER_DIR_OUT, DMA_TO_DEVICE); 139 return err; 140 } 141 142 /* creates a new tx descriptor and adds header regd buffer */ 143 static void iser_create_send_desc(struct iser_conn *iser_conn, 144 struct iser_tx_desc *tx_desc) 145 { 146 struct iser_device *device = iser_conn->ib_conn.device; 147 148 ib_dma_sync_single_for_cpu(device->ib_device, 149 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 150 151 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 152 tx_desc->iser_header.flags = ISER_VER; 153 tx_desc->num_sge = 1; 154 } 155 156 static void iser_free_login_buf(struct iser_conn *iser_conn) 157 { 158 struct iser_device *device = iser_conn->ib_conn.device; 159 struct iser_login_desc *desc = &iser_conn->login_desc; 160 161 if (!desc->req) 162 return; 163 164 ib_dma_unmap_single(device->ib_device, desc->req_dma, 165 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 166 167 ib_dma_unmap_single(device->ib_device, desc->rsp_dma, 168 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 169 170 kfree(desc->req); 171 kfree(desc->rsp); 172 173 /* make sure we never redo any unmapping */ 174 desc->req = NULL; 175 desc->rsp = NULL; 176 } 177 178 static int iser_alloc_login_buf(struct iser_conn *iser_conn) 179 { 180 struct iser_device *device = iser_conn->ib_conn.device; 181 struct iser_login_desc *desc = &iser_conn->login_desc; 182 183 desc->req = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN, GFP_KERNEL); 184 if (!desc->req) 185 return -ENOMEM; 186 187 desc->req_dma = ib_dma_map_single(device->ib_device, desc->req, 188 ISCSI_DEF_MAX_RECV_SEG_LEN, 189 DMA_TO_DEVICE); 190 if (ib_dma_mapping_error(device->ib_device, 191 desc->req_dma)) 192 goto free_req; 193 194 desc->rsp = kmalloc(ISER_RX_LOGIN_SIZE, GFP_KERNEL); 195 if (!desc->rsp) 196 goto unmap_req; 197 198 desc->rsp_dma = ib_dma_map_single(device->ib_device, desc->rsp, 199 ISER_RX_LOGIN_SIZE, 200 DMA_FROM_DEVICE); 201 if (ib_dma_mapping_error(device->ib_device, 202 desc->rsp_dma)) 203 goto free_rsp; 204 205 return 0; 206 207 free_rsp: 208 kfree(desc->rsp); 209 unmap_req: 210 ib_dma_unmap_single(device->ib_device, desc->req_dma, 211 ISCSI_DEF_MAX_RECV_SEG_LEN, 212 DMA_TO_DEVICE); 213 free_req: 214 kfree(desc->req); 215 216 return -ENOMEM; 217 } 218 219 int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, 220 struct iscsi_session *session) 221 { 222 int i, j; 223 u64 dma_addr; 224 struct iser_rx_desc *rx_desc; 225 struct ib_sge *rx_sg; 226 struct ib_conn *ib_conn = &iser_conn->ib_conn; 227 struct iser_device *device = ib_conn->device; 228 229 iser_conn->qp_max_recv_dtos = session->cmds_max; 230 231 if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max, 232 iser_conn->pages_per_mr)) 233 goto create_rdma_reg_res_failed; 234 235 if (iser_alloc_login_buf(iser_conn)) 236 goto alloc_login_buf_fail; 237 238 iser_conn->num_rx_descs = session->cmds_max; 239 iser_conn->rx_descs = kmalloc_array(iser_conn->num_rx_descs, 240 sizeof(struct iser_rx_desc), 241 GFP_KERNEL); 242 if (!iser_conn->rx_descs) 243 goto rx_desc_alloc_fail; 244 245 rx_desc = iser_conn->rx_descs; 246 247 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) { 248 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 249 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 250 if (ib_dma_mapping_error(device->ib_device, dma_addr)) 251 goto rx_desc_dma_map_failed; 252 253 rx_desc->dma_addr = dma_addr; 254 rx_desc->cqe.done = iser_task_rsp; 255 rx_sg = &rx_desc->rx_sg; 256 rx_sg->addr = rx_desc->dma_addr; 257 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 258 rx_sg->lkey = device->pd->local_dma_lkey; 259 } 260 261 return 0; 262 263 rx_desc_dma_map_failed: 264 rx_desc = iser_conn->rx_descs; 265 for (j = 0; j < i; j++, rx_desc++) 266 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 267 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 268 kfree(iser_conn->rx_descs); 269 iser_conn->rx_descs = NULL; 270 rx_desc_alloc_fail: 271 iser_free_login_buf(iser_conn); 272 alloc_login_buf_fail: 273 iser_free_fastreg_pool(ib_conn); 274 create_rdma_reg_res_failed: 275 iser_err("failed allocating rx descriptors / data buffers\n"); 276 return -ENOMEM; 277 } 278 279 void iser_free_rx_descriptors(struct iser_conn *iser_conn) 280 { 281 int i; 282 struct iser_rx_desc *rx_desc; 283 struct ib_conn *ib_conn = &iser_conn->ib_conn; 284 struct iser_device *device = ib_conn->device; 285 286 iser_free_fastreg_pool(ib_conn); 287 288 rx_desc = iser_conn->rx_descs; 289 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) 290 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 291 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 292 kfree(iser_conn->rx_descs); 293 /* make sure we never redo any unmapping */ 294 iser_conn->rx_descs = NULL; 295 296 iser_free_login_buf(iser_conn); 297 } 298 299 static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 300 { 301 struct iser_conn *iser_conn = conn->dd_data; 302 struct iscsi_session *session = conn->session; 303 int err = 0; 304 int i; 305 306 iser_dbg("req op %x flags %x\n", req->opcode, req->flags); 307 /* check if this is the last login - going to full feature phase */ 308 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) 309 goto out; 310 311 if (session->discovery_sess) { 312 iser_info("Discovery session, re-using login RX buffer\n"); 313 goto out; 314 } 315 316 iser_info("Normal session, posting batch of RX %d buffers\n", 317 iser_conn->qp_max_recv_dtos - 1); 318 319 /* 320 * Initial post receive buffers. 321 * There is one already posted recv buffer (for the last login 322 * response). Therefore, the first recv buffer is skipped here. 323 */ 324 for (i = 1; i < iser_conn->qp_max_recv_dtos; i++) { 325 err = iser_post_recvm(iser_conn, &iser_conn->rx_descs[i]); 326 if (err) 327 goto out; 328 } 329 out: 330 return err; 331 } 332 333 /** 334 * iser_send_command - send command PDU 335 * @conn: link to matching iscsi connection 336 * @task: SCSI command task 337 */ 338 int iser_send_command(struct iscsi_conn *conn, struct iscsi_task *task) 339 { 340 struct iser_conn *iser_conn = conn->dd_data; 341 struct iscsi_iser_task *iser_task = task->dd_data; 342 unsigned long edtl; 343 int err; 344 struct iser_data_buf *data_buf, *prot_buf; 345 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 346 struct scsi_cmnd *sc = task->sc; 347 struct iser_tx_desc *tx_desc = &iser_task->desc; 348 349 edtl = ntohl(hdr->data_length); 350 351 /* build the tx desc regd header and add it to the tx desc dto */ 352 tx_desc->type = ISCSI_TX_SCSI_COMMAND; 353 tx_desc->cqe.done = iser_cmd_comp; 354 iser_create_send_desc(iser_conn, tx_desc); 355 356 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 357 data_buf = &iser_task->data[ISER_DIR_IN]; 358 prot_buf = &iser_task->prot[ISER_DIR_IN]; 359 } else { 360 data_buf = &iser_task->data[ISER_DIR_OUT]; 361 prot_buf = &iser_task->prot[ISER_DIR_OUT]; 362 } 363 364 if (scsi_sg_count(sc)) { /* using a scatter list */ 365 data_buf->sg = scsi_sglist(sc); 366 data_buf->size = scsi_sg_count(sc); 367 } 368 data_buf->data_len = scsi_bufflen(sc); 369 370 if (scsi_prot_sg_count(sc)) { 371 prot_buf->sg = scsi_prot_sglist(sc); 372 prot_buf->size = scsi_prot_sg_count(sc); 373 prot_buf->data_len = (data_buf->data_len >> 374 ilog2(sc->device->sector_size)) * 8; 375 } 376 377 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 378 err = iser_prepare_read_cmd(task); 379 if (err) 380 goto send_command_error; 381 } 382 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { 383 err = iser_prepare_write_cmd(task, 384 task->imm_count, 385 task->imm_count + 386 task->unsol_r2t.data_length, 387 edtl); 388 if (err) 389 goto send_command_error; 390 } 391 392 iser_task->status = ISER_TASK_STATUS_STARTED; 393 394 err = iser_post_send(&iser_conn->ib_conn, tx_desc); 395 if (!err) 396 return 0; 397 398 send_command_error: 399 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); 400 return err; 401 } 402 403 /** 404 * iser_send_data_out - send data out PDU 405 * @conn: link to matching iscsi connection 406 * @task: SCSI command task 407 * @hdr: pointer to the LLD's iSCSI message header 408 */ 409 int iser_send_data_out(struct iscsi_conn *conn, struct iscsi_task *task, 410 struct iscsi_data *hdr) 411 { 412 struct iser_conn *iser_conn = conn->dd_data; 413 struct iscsi_iser_task *iser_task = task->dd_data; 414 struct iser_tx_desc *tx_desc; 415 struct iser_mem_reg *mem_reg; 416 unsigned long buf_offset; 417 unsigned long data_seg_len; 418 uint32_t itt; 419 int err; 420 struct ib_sge *tx_dsg; 421 422 itt = (__force uint32_t)hdr->itt; 423 data_seg_len = ntoh24(hdr->dlength); 424 buf_offset = ntohl(hdr->offset); 425 426 iser_dbg("%s itt %d dseg_len %d offset %d\n", 427 __func__,(int)itt,(int)data_seg_len,(int)buf_offset); 428 429 tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); 430 if (!tx_desc) 431 return -ENOMEM; 432 433 tx_desc->type = ISCSI_TX_DATAOUT; 434 tx_desc->cqe.done = iser_dataout_comp; 435 tx_desc->iser_header.flags = ISER_VER; 436 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); 437 438 /* build the tx desc */ 439 err = iser_initialize_task_headers(task, tx_desc); 440 if (err) 441 goto send_data_out_error; 442 443 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; 444 tx_dsg = &tx_desc->tx_sg[1]; 445 tx_dsg->addr = mem_reg->sge.addr + buf_offset; 446 tx_dsg->length = data_seg_len; 447 tx_dsg->lkey = mem_reg->sge.lkey; 448 tx_desc->num_sge = 2; 449 450 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { 451 iser_err("Offset:%ld & DSL:%ld in Data-Out inconsistent with total len:%ld, itt:%d\n", 452 buf_offset, data_seg_len, 453 iser_task->data[ISER_DIR_OUT].data_len, itt); 454 err = -EINVAL; 455 goto send_data_out_error; 456 } 457 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", 458 itt, buf_offset, data_seg_len); 459 460 461 err = iser_post_send(&iser_conn->ib_conn, tx_desc); 462 if (!err) 463 return 0; 464 465 send_data_out_error: 466 kmem_cache_free(ig.desc_cache, tx_desc); 467 iser_err("conn %p failed err %d\n", conn, err); 468 return err; 469 } 470 471 int iser_send_control(struct iscsi_conn *conn, struct iscsi_task *task) 472 { 473 struct iser_conn *iser_conn = conn->dd_data; 474 struct iscsi_iser_task *iser_task = task->dd_data; 475 struct iser_tx_desc *mdesc = &iser_task->desc; 476 unsigned long data_seg_len; 477 int err = 0; 478 struct iser_device *device; 479 480 /* build the tx desc regd header and add it to the tx desc dto */ 481 mdesc->type = ISCSI_TX_CONTROL; 482 mdesc->cqe.done = iser_ctrl_comp; 483 iser_create_send_desc(iser_conn, mdesc); 484 485 device = iser_conn->ib_conn.device; 486 487 data_seg_len = ntoh24(task->hdr->dlength); 488 489 if (data_seg_len > 0) { 490 struct iser_login_desc *desc = &iser_conn->login_desc; 491 struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; 492 493 if (task != conn->login_task) { 494 iser_err("data present on non login task!!!\n"); 495 goto send_control_error; 496 } 497 498 ib_dma_sync_single_for_cpu(device->ib_device, desc->req_dma, 499 task->data_count, DMA_TO_DEVICE); 500 501 memcpy(desc->req, task->data, task->data_count); 502 503 ib_dma_sync_single_for_device(device->ib_device, desc->req_dma, 504 task->data_count, DMA_TO_DEVICE); 505 506 tx_dsg->addr = desc->req_dma; 507 tx_dsg->length = task->data_count; 508 tx_dsg->lkey = device->pd->local_dma_lkey; 509 mdesc->num_sge = 2; 510 } 511 512 if (task == conn->login_task) { 513 iser_dbg("op %x dsl %lx, posting login rx buffer\n", 514 task->hdr->opcode, data_seg_len); 515 err = iser_post_recvl(iser_conn); 516 if (err) 517 goto send_control_error; 518 err = iser_post_rx_bufs(conn, task->hdr); 519 if (err) 520 goto send_control_error; 521 } 522 523 err = iser_post_send(&iser_conn->ib_conn, mdesc); 524 if (!err) 525 return 0; 526 527 send_control_error: 528 iser_err("conn %p failed err %d\n",conn, err); 529 return err; 530 } 531 532 void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc) 533 { 534 struct ib_conn *ib_conn = wc->qp->qp_context; 535 struct iser_conn *iser_conn = to_iser_conn(ib_conn); 536 struct iser_login_desc *desc = iser_login(wc->wr_cqe); 537 struct iscsi_hdr *hdr; 538 char *data; 539 int length; 540 541 if (unlikely(wc->status != IB_WC_SUCCESS)) { 542 iser_err_comp(wc, "login_rsp"); 543 return; 544 } 545 546 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, 547 desc->rsp_dma, ISER_RX_LOGIN_SIZE, 548 DMA_FROM_DEVICE); 549 550 hdr = desc->rsp + sizeof(struct iser_ctrl); 551 data = desc->rsp + ISER_HEADERS_LEN; 552 length = wc->byte_len - ISER_HEADERS_LEN; 553 554 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 555 hdr->itt, length); 556 557 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, data, length); 558 559 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 560 desc->rsp_dma, ISER_RX_LOGIN_SIZE, 561 DMA_FROM_DEVICE); 562 563 if (iser_conn->iscsi_conn->session->discovery_sess) 564 return; 565 566 /* Post the first RX buffer that is skipped in iser_post_rx_bufs() */ 567 iser_post_recvm(iser_conn, iser_conn->rx_descs); 568 } 569 570 static inline int iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) 571 { 572 if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) || 573 (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) { 574 iser_err("Bogus remote invalidation for rkey %#x\n", rkey); 575 return -EINVAL; 576 } 577 578 desc->rsc.mr_valid = 0; 579 580 return 0; 581 } 582 583 static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, 584 struct iscsi_hdr *hdr) 585 { 586 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) { 587 struct iscsi_task *task; 588 u32 rkey = wc->ex.invalidate_rkey; 589 590 iser_dbg("conn %p: remote invalidation for rkey %#x\n", 591 iser_conn, rkey); 592 593 if (unlikely(!iser_conn->snd_w_inv)) { 594 iser_err("conn %p: unexpected remote invalidation, terminating connection\n", 595 iser_conn); 596 return -EPROTO; 597 } 598 599 task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt); 600 if (likely(task)) { 601 struct iscsi_iser_task *iser_task = task->dd_data; 602 struct iser_fr_desc *desc; 603 604 if (iser_task->dir[ISER_DIR_IN]) { 605 desc = iser_task->rdma_reg[ISER_DIR_IN].desc; 606 if (unlikely(iser_inv_desc(desc, rkey))) 607 return -EINVAL; 608 } 609 610 if (iser_task->dir[ISER_DIR_OUT]) { 611 desc = iser_task->rdma_reg[ISER_DIR_OUT].desc; 612 if (unlikely(iser_inv_desc(desc, rkey))) 613 return -EINVAL; 614 } 615 } else { 616 iser_err("failed to get task for itt=%d\n", hdr->itt); 617 return -EINVAL; 618 } 619 } 620 621 return 0; 622 } 623 624 625 void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) 626 { 627 struct ib_conn *ib_conn = wc->qp->qp_context; 628 struct iser_conn *iser_conn = to_iser_conn(ib_conn); 629 struct iser_rx_desc *desc = iser_rx(wc->wr_cqe); 630 struct iscsi_hdr *hdr; 631 int length, err; 632 633 if (unlikely(wc->status != IB_WC_SUCCESS)) { 634 iser_err_comp(wc, "task_rsp"); 635 return; 636 } 637 638 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, 639 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, 640 DMA_FROM_DEVICE); 641 642 hdr = &desc->iscsi_header; 643 length = wc->byte_len - ISER_HEADERS_LEN; 644 645 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 646 hdr->itt, length); 647 648 if (iser_check_remote_inv(iser_conn, wc, hdr)) { 649 iscsi_conn_failure(iser_conn->iscsi_conn, 650 ISCSI_ERR_CONN_FAILED); 651 return; 652 } 653 654 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, desc->data, length); 655 656 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 657 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, 658 DMA_FROM_DEVICE); 659 660 err = iser_post_recvm(iser_conn, desc); 661 if (err) 662 iser_err("posting rx buffer err %d\n", err); 663 } 664 665 void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc) 666 { 667 if (unlikely(wc->status != IB_WC_SUCCESS)) 668 iser_err_comp(wc, "command"); 669 } 670 671 void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc) 672 { 673 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); 674 struct iscsi_task *task; 675 676 if (unlikely(wc->status != IB_WC_SUCCESS)) { 677 iser_err_comp(wc, "control"); 678 return; 679 } 680 681 /* this arithmetic is legal by libiscsi dd_data allocation */ 682 task = (void *)desc - sizeof(struct iscsi_task); 683 if (task->hdr->itt == RESERVED_ITT) 684 iscsi_put_task(task); 685 } 686 687 void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc) 688 { 689 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); 690 struct ib_conn *ib_conn = wc->qp->qp_context; 691 struct iser_device *device = ib_conn->device; 692 693 if (unlikely(wc->status != IB_WC_SUCCESS)) 694 iser_err_comp(wc, "dataout"); 695 696 ib_dma_unmap_single(device->ib_device, desc->dma_addr, 697 ISER_HEADERS_LEN, DMA_TO_DEVICE); 698 kmem_cache_free(ig.desc_cache, desc); 699 } 700 701 void iser_task_rdma_init(struct iscsi_iser_task *iser_task) 702 703 { 704 iser_task->status = ISER_TASK_STATUS_INIT; 705 706 iser_task->dir[ISER_DIR_IN] = 0; 707 iser_task->dir[ISER_DIR_OUT] = 0; 708 709 iser_task->data[ISER_DIR_IN].data_len = 0; 710 iser_task->data[ISER_DIR_OUT].data_len = 0; 711 712 iser_task->prot[ISER_DIR_IN].data_len = 0; 713 iser_task->prot[ISER_DIR_OUT].data_len = 0; 714 715 iser_task->prot[ISER_DIR_IN].dma_nents = 0; 716 iser_task->prot[ISER_DIR_OUT].dma_nents = 0; 717 718 memset(&iser_task->rdma_reg[ISER_DIR_IN], 0, 719 sizeof(struct iser_mem_reg)); 720 memset(&iser_task->rdma_reg[ISER_DIR_OUT], 0, 721 sizeof(struct iser_mem_reg)); 722 } 723 724 void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 725 { 726 727 if (iser_task->dir[ISER_DIR_IN]) { 728 iser_unreg_mem_fastreg(iser_task, ISER_DIR_IN); 729 iser_dma_unmap_task_data(iser_task, ISER_DIR_IN, 730 DMA_FROM_DEVICE); 731 } 732 733 if (iser_task->dir[ISER_DIR_OUT]) { 734 iser_unreg_mem_fastreg(iser_task, ISER_DIR_OUT); 735 iser_dma_unmap_task_data(iser_task, ISER_DIR_OUT, 736 DMA_TO_DEVICE); 737 } 738 } 739