1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/err.h> 11 #include <linux/nvme-tcp.h> 12 #include <net/sock.h> 13 #include <net/tcp.h> 14 #include <linux/blk-mq.h> 15 #include <crypto/hash.h> 16 #include <net/busy_poll.h> 17 18 #include "nvme.h" 19 #include "fabrics.h" 20 21 struct nvme_tcp_queue; 22 23 /* Define the socket priority to use for connections were it is desirable 24 * that the NIC consider performing optimized packet processing or filtering. 25 * A non-zero value being sufficient to indicate general consideration of any 26 * possible optimization. Making it a module param allows for alternative 27 * values that may be unique for some NIC implementations. 28 */ 29 static int so_priority; 30 module_param(so_priority, int, 0644); 31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 32 33 #ifdef CONFIG_DEBUG_LOCK_ALLOC 34 /* lockdep can detect a circular dependency of the form 35 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 36 * because dependencies are tracked for both nvme-tcp and user contexts. Using 37 * a separate class prevents lockdep from conflating nvme-tcp socket use with 38 * user-space socket API use. 39 */ 40 static struct lock_class_key nvme_tcp_sk_key[2]; 41 static struct lock_class_key nvme_tcp_slock_key[2]; 42 43 static void nvme_tcp_reclassify_socket(struct socket *sock) 44 { 45 struct sock *sk = sock->sk; 46 47 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 48 return; 49 50 switch (sk->sk_family) { 51 case AF_INET: 52 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 53 &nvme_tcp_slock_key[0], 54 "sk_lock-AF_INET-NVME", 55 &nvme_tcp_sk_key[0]); 56 break; 57 case AF_INET6: 58 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 59 &nvme_tcp_slock_key[1], 60 "sk_lock-AF_INET6-NVME", 61 &nvme_tcp_sk_key[1]); 62 break; 63 default: 64 WARN_ON_ONCE(1); 65 } 66 } 67 #else 68 static void nvme_tcp_reclassify_socket(struct socket *sock) { } 69 #endif 70 71 enum nvme_tcp_send_state { 72 NVME_TCP_SEND_CMD_PDU = 0, 73 NVME_TCP_SEND_H2C_PDU, 74 NVME_TCP_SEND_DATA, 75 NVME_TCP_SEND_DDGST, 76 }; 77 78 struct nvme_tcp_request { 79 struct nvme_request req; 80 void *pdu; 81 struct nvme_tcp_queue *queue; 82 u32 data_len; 83 u32 pdu_len; 84 u32 pdu_sent; 85 u32 h2cdata_left; 86 u32 h2cdata_offset; 87 u16 ttag; 88 __le16 status; 89 struct list_head entry; 90 struct llist_node lentry; 91 __le32 ddgst; 92 93 struct bio *curr_bio; 94 struct iov_iter iter; 95 96 /* send state */ 97 size_t offset; 98 size_t data_sent; 99 enum nvme_tcp_send_state state; 100 }; 101 102 enum nvme_tcp_queue_flags { 103 NVME_TCP_Q_ALLOCATED = 0, 104 NVME_TCP_Q_LIVE = 1, 105 NVME_TCP_Q_POLLING = 2, 106 }; 107 108 enum nvme_tcp_recv_state { 109 NVME_TCP_RECV_PDU = 0, 110 NVME_TCP_RECV_DATA, 111 NVME_TCP_RECV_DDGST, 112 }; 113 114 struct nvme_tcp_ctrl; 115 struct nvme_tcp_queue { 116 struct socket *sock; 117 struct work_struct io_work; 118 int io_cpu; 119 120 struct mutex queue_lock; 121 struct mutex send_mutex; 122 struct llist_head req_list; 123 struct list_head send_list; 124 125 /* recv state */ 126 void *pdu; 127 int pdu_remaining; 128 int pdu_offset; 129 size_t data_remaining; 130 size_t ddgst_remaining; 131 unsigned int nr_cqe; 132 133 /* send state */ 134 struct nvme_tcp_request *request; 135 136 u32 maxh2cdata; 137 size_t cmnd_capsule_len; 138 struct nvme_tcp_ctrl *ctrl; 139 unsigned long flags; 140 bool rd_enabled; 141 142 bool hdr_digest; 143 bool data_digest; 144 struct ahash_request *rcv_hash; 145 struct ahash_request *snd_hash; 146 __le32 exp_ddgst; 147 __le32 recv_ddgst; 148 149 struct page_frag_cache pf_cache; 150 151 void (*state_change)(struct sock *); 152 void (*data_ready)(struct sock *); 153 void (*write_space)(struct sock *); 154 }; 155 156 struct nvme_tcp_ctrl { 157 /* read only in the hot path */ 158 struct nvme_tcp_queue *queues; 159 struct blk_mq_tag_set tag_set; 160 161 /* other member variables */ 162 struct list_head list; 163 struct blk_mq_tag_set admin_tag_set; 164 struct sockaddr_storage addr; 165 struct sockaddr_storage src_addr; 166 struct nvme_ctrl ctrl; 167 168 struct work_struct err_work; 169 struct delayed_work connect_work; 170 struct nvme_tcp_request async_req; 171 u32 io_queues[HCTX_MAX_TYPES]; 172 }; 173 174 static LIST_HEAD(nvme_tcp_ctrl_list); 175 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 176 static struct workqueue_struct *nvme_tcp_wq; 177 static const struct blk_mq_ops nvme_tcp_mq_ops; 178 static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 179 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 180 181 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 182 { 183 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 184 } 185 186 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 187 { 188 return queue - queue->ctrl->queues; 189 } 190 191 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 192 { 193 u32 queue_idx = nvme_tcp_queue_id(queue); 194 195 if (queue_idx == 0) 196 return queue->ctrl->admin_tag_set.tags[queue_idx]; 197 return queue->ctrl->tag_set.tags[queue_idx - 1]; 198 } 199 200 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 201 { 202 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 203 } 204 205 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 206 { 207 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 208 } 209 210 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req) 211 { 212 if (nvme_is_fabrics(req->req.cmd)) 213 return NVME_TCP_ADMIN_CCSZ; 214 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command); 215 } 216 217 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 218 { 219 return req == &req->queue->ctrl->async_req; 220 } 221 222 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 223 { 224 struct request *rq; 225 226 if (unlikely(nvme_tcp_async_req(req))) 227 return false; /* async events don't have a request */ 228 229 rq = blk_mq_rq_from_pdu(req); 230 231 return rq_data_dir(rq) == WRITE && req->data_len && 232 req->data_len <= nvme_tcp_inline_data_size(req); 233 } 234 235 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 236 { 237 return req->iter.bvec->bv_page; 238 } 239 240 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 241 { 242 return req->iter.bvec->bv_offset + req->iter.iov_offset; 243 } 244 245 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 246 { 247 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 248 req->pdu_len - req->pdu_sent); 249 } 250 251 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 252 { 253 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 254 req->pdu_len - req->pdu_sent : 0; 255 } 256 257 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 258 int len) 259 { 260 return nvme_tcp_pdu_data_left(req) <= len; 261 } 262 263 static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 264 unsigned int dir) 265 { 266 struct request *rq = blk_mq_rq_from_pdu(req); 267 struct bio_vec *vec; 268 unsigned int size; 269 int nr_bvec; 270 size_t offset; 271 272 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 273 vec = &rq->special_vec; 274 nr_bvec = 1; 275 size = blk_rq_payload_bytes(rq); 276 offset = 0; 277 } else { 278 struct bio *bio = req->curr_bio; 279 struct bvec_iter bi; 280 struct bio_vec bv; 281 282 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 283 nr_bvec = 0; 284 bio_for_each_bvec(bv, bio, bi) { 285 nr_bvec++; 286 } 287 size = bio->bi_iter.bi_size; 288 offset = bio->bi_iter.bi_bvec_done; 289 } 290 291 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 292 req->iter.iov_offset = offset; 293 } 294 295 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 296 int len) 297 { 298 req->data_sent += len; 299 req->pdu_sent += len; 300 iov_iter_advance(&req->iter, len); 301 if (!iov_iter_count(&req->iter) && 302 req->data_sent < req->data_len) { 303 req->curr_bio = req->curr_bio->bi_next; 304 nvme_tcp_init_iter(req, ITER_SOURCE); 305 } 306 } 307 308 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 309 { 310 int ret; 311 312 /* drain the send queue as much as we can... */ 313 do { 314 ret = nvme_tcp_try_send(queue); 315 } while (ret > 0); 316 } 317 318 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 319 { 320 return !list_empty(&queue->send_list) || 321 !llist_empty(&queue->req_list); 322 } 323 324 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 325 bool sync, bool last) 326 { 327 struct nvme_tcp_queue *queue = req->queue; 328 bool empty; 329 330 empty = llist_add(&req->lentry, &queue->req_list) && 331 list_empty(&queue->send_list) && !queue->request; 332 333 /* 334 * if we're the first on the send_list and we can try to send 335 * directly, otherwise queue io_work. Also, only do that if we 336 * are on the same cpu, so we don't introduce contention. 337 */ 338 if (queue->io_cpu == raw_smp_processor_id() && 339 sync && empty && mutex_trylock(&queue->send_mutex)) { 340 nvme_tcp_send_all(queue); 341 mutex_unlock(&queue->send_mutex); 342 } 343 344 if (last && nvme_tcp_queue_more(queue)) 345 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 346 } 347 348 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 349 { 350 struct nvme_tcp_request *req; 351 struct llist_node *node; 352 353 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 354 req = llist_entry(node, struct nvme_tcp_request, lentry); 355 list_add(&req->entry, &queue->send_list); 356 } 357 } 358 359 static inline struct nvme_tcp_request * 360 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 361 { 362 struct nvme_tcp_request *req; 363 364 req = list_first_entry_or_null(&queue->send_list, 365 struct nvme_tcp_request, entry); 366 if (!req) { 367 nvme_tcp_process_req_list(queue); 368 req = list_first_entry_or_null(&queue->send_list, 369 struct nvme_tcp_request, entry); 370 if (unlikely(!req)) 371 return NULL; 372 } 373 374 list_del(&req->entry); 375 return req; 376 } 377 378 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 379 __le32 *dgst) 380 { 381 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 382 crypto_ahash_final(hash); 383 } 384 385 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 386 struct page *page, off_t off, size_t len) 387 { 388 struct scatterlist sg; 389 390 sg_init_table(&sg, 1); 391 sg_set_page(&sg, page, len, off); 392 ahash_request_set_crypt(hash, &sg, NULL, len); 393 crypto_ahash_update(hash); 394 } 395 396 static inline void nvme_tcp_hdgst(struct ahash_request *hash, 397 void *pdu, size_t len) 398 { 399 struct scatterlist sg; 400 401 sg_init_one(&sg, pdu, len); 402 ahash_request_set_crypt(hash, &sg, pdu + len, len); 403 crypto_ahash_digest(hash); 404 } 405 406 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 407 void *pdu, size_t pdu_len) 408 { 409 struct nvme_tcp_hdr *hdr = pdu; 410 __le32 recv_digest; 411 __le32 exp_digest; 412 413 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 414 dev_err(queue->ctrl->ctrl.device, 415 "queue %d: header digest flag is cleared\n", 416 nvme_tcp_queue_id(queue)); 417 return -EPROTO; 418 } 419 420 recv_digest = *(__le32 *)(pdu + hdr->hlen); 421 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 422 exp_digest = *(__le32 *)(pdu + hdr->hlen); 423 if (recv_digest != exp_digest) { 424 dev_err(queue->ctrl->ctrl.device, 425 "header digest error: recv %#x expected %#x\n", 426 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 427 return -EIO; 428 } 429 430 return 0; 431 } 432 433 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 434 { 435 struct nvme_tcp_hdr *hdr = pdu; 436 u8 digest_len = nvme_tcp_hdgst_len(queue); 437 u32 len; 438 439 len = le32_to_cpu(hdr->plen) - hdr->hlen - 440 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 441 442 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 443 dev_err(queue->ctrl->ctrl.device, 444 "queue %d: data digest flag is cleared\n", 445 nvme_tcp_queue_id(queue)); 446 return -EPROTO; 447 } 448 crypto_ahash_init(queue->rcv_hash); 449 450 return 0; 451 } 452 453 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 454 struct request *rq, unsigned int hctx_idx) 455 { 456 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 457 458 page_frag_free(req->pdu); 459 } 460 461 static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 462 struct request *rq, unsigned int hctx_idx, 463 unsigned int numa_node) 464 { 465 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 466 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 467 struct nvme_tcp_cmd_pdu *pdu; 468 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 469 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 470 u8 hdgst = nvme_tcp_hdgst_len(queue); 471 472 req->pdu = page_frag_alloc(&queue->pf_cache, 473 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 474 GFP_KERNEL | __GFP_ZERO); 475 if (!req->pdu) 476 return -ENOMEM; 477 478 pdu = req->pdu; 479 req->queue = queue; 480 nvme_req(rq)->ctrl = &ctrl->ctrl; 481 nvme_req(rq)->cmd = &pdu->cmd; 482 483 return 0; 484 } 485 486 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 487 unsigned int hctx_idx) 488 { 489 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 490 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 491 492 hctx->driver_data = queue; 493 return 0; 494 } 495 496 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 497 unsigned int hctx_idx) 498 { 499 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 500 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 501 502 hctx->driver_data = queue; 503 return 0; 504 } 505 506 static enum nvme_tcp_recv_state 507 nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 508 { 509 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 510 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 511 NVME_TCP_RECV_DATA; 512 } 513 514 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 515 { 516 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 517 nvme_tcp_hdgst_len(queue); 518 queue->pdu_offset = 0; 519 queue->data_remaining = -1; 520 queue->ddgst_remaining = 0; 521 } 522 523 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 524 { 525 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 526 return; 527 528 dev_warn(ctrl->device, "starting error recovery\n"); 529 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 530 } 531 532 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 533 struct nvme_completion *cqe) 534 { 535 struct nvme_tcp_request *req; 536 struct request *rq; 537 538 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 539 if (!rq) { 540 dev_err(queue->ctrl->ctrl.device, 541 "got bad cqe.command_id %#x on queue %d\n", 542 cqe->command_id, nvme_tcp_queue_id(queue)); 543 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 544 return -EINVAL; 545 } 546 547 req = blk_mq_rq_to_pdu(rq); 548 if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 549 req->status = cqe->status; 550 551 if (!nvme_try_complete_req(rq, req->status, cqe->result)) 552 nvme_complete_rq(rq); 553 queue->nr_cqe++; 554 555 return 0; 556 } 557 558 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 559 struct nvme_tcp_data_pdu *pdu) 560 { 561 struct request *rq; 562 563 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 564 if (!rq) { 565 dev_err(queue->ctrl->ctrl.device, 566 "got bad c2hdata.command_id %#x on queue %d\n", 567 pdu->command_id, nvme_tcp_queue_id(queue)); 568 return -ENOENT; 569 } 570 571 if (!blk_rq_payload_bytes(rq)) { 572 dev_err(queue->ctrl->ctrl.device, 573 "queue %d tag %#x unexpected data\n", 574 nvme_tcp_queue_id(queue), rq->tag); 575 return -EIO; 576 } 577 578 queue->data_remaining = le32_to_cpu(pdu->data_length); 579 580 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 581 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 582 dev_err(queue->ctrl->ctrl.device, 583 "queue %d tag %#x SUCCESS set but not last PDU\n", 584 nvme_tcp_queue_id(queue), rq->tag); 585 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 586 return -EPROTO; 587 } 588 589 return 0; 590 } 591 592 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 593 struct nvme_tcp_rsp_pdu *pdu) 594 { 595 struct nvme_completion *cqe = &pdu->cqe; 596 int ret = 0; 597 598 /* 599 * AEN requests are special as they don't time out and can 600 * survive any kind of queue freeze and often don't respond to 601 * aborts. We don't even bother to allocate a struct request 602 * for them but rather special case them here. 603 */ 604 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 605 cqe->command_id))) 606 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 607 &cqe->result); 608 else 609 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 610 611 return ret; 612 } 613 614 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req) 615 { 616 struct nvme_tcp_data_pdu *data = req->pdu; 617 struct nvme_tcp_queue *queue = req->queue; 618 struct request *rq = blk_mq_rq_from_pdu(req); 619 u32 h2cdata_sent = req->pdu_len; 620 u8 hdgst = nvme_tcp_hdgst_len(queue); 621 u8 ddgst = nvme_tcp_ddgst_len(queue); 622 623 req->state = NVME_TCP_SEND_H2C_PDU; 624 req->offset = 0; 625 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata); 626 req->pdu_sent = 0; 627 req->h2cdata_left -= req->pdu_len; 628 req->h2cdata_offset += h2cdata_sent; 629 630 memset(data, 0, sizeof(*data)); 631 data->hdr.type = nvme_tcp_h2c_data; 632 if (!req->h2cdata_left) 633 data->hdr.flags = NVME_TCP_F_DATA_LAST; 634 if (queue->hdr_digest) 635 data->hdr.flags |= NVME_TCP_F_HDGST; 636 if (queue->data_digest) 637 data->hdr.flags |= NVME_TCP_F_DDGST; 638 data->hdr.hlen = sizeof(*data); 639 data->hdr.pdo = data->hdr.hlen + hdgst; 640 data->hdr.plen = 641 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 642 data->ttag = req->ttag; 643 data->command_id = nvme_cid(rq); 644 data->data_offset = cpu_to_le32(req->h2cdata_offset); 645 data->data_length = cpu_to_le32(req->pdu_len); 646 } 647 648 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 649 struct nvme_tcp_r2t_pdu *pdu) 650 { 651 struct nvme_tcp_request *req; 652 struct request *rq; 653 u32 r2t_length = le32_to_cpu(pdu->r2t_length); 654 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset); 655 656 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 657 if (!rq) { 658 dev_err(queue->ctrl->ctrl.device, 659 "got bad r2t.command_id %#x on queue %d\n", 660 pdu->command_id, nvme_tcp_queue_id(queue)); 661 return -ENOENT; 662 } 663 req = blk_mq_rq_to_pdu(rq); 664 665 if (unlikely(!r2t_length)) { 666 dev_err(queue->ctrl->ctrl.device, 667 "req %d r2t len is %u, probably a bug...\n", 668 rq->tag, r2t_length); 669 return -EPROTO; 670 } 671 672 if (unlikely(req->data_sent + r2t_length > req->data_len)) { 673 dev_err(queue->ctrl->ctrl.device, 674 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 675 rq->tag, r2t_length, req->data_len, req->data_sent); 676 return -EPROTO; 677 } 678 679 if (unlikely(r2t_offset < req->data_sent)) { 680 dev_err(queue->ctrl->ctrl.device, 681 "req %d unexpected r2t offset %u (expected %zu)\n", 682 rq->tag, r2t_offset, req->data_sent); 683 return -EPROTO; 684 } 685 686 req->pdu_len = 0; 687 req->h2cdata_left = r2t_length; 688 req->h2cdata_offset = r2t_offset; 689 req->ttag = pdu->ttag; 690 691 nvme_tcp_setup_h2c_data_pdu(req); 692 nvme_tcp_queue_request(req, false, true); 693 694 return 0; 695 } 696 697 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 698 unsigned int *offset, size_t *len) 699 { 700 struct nvme_tcp_hdr *hdr; 701 char *pdu = queue->pdu; 702 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 703 int ret; 704 705 ret = skb_copy_bits(skb, *offset, 706 &pdu[queue->pdu_offset], rcv_len); 707 if (unlikely(ret)) 708 return ret; 709 710 queue->pdu_remaining -= rcv_len; 711 queue->pdu_offset += rcv_len; 712 *offset += rcv_len; 713 *len -= rcv_len; 714 if (queue->pdu_remaining) 715 return 0; 716 717 hdr = queue->pdu; 718 if (queue->hdr_digest) { 719 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 720 if (unlikely(ret)) 721 return ret; 722 } 723 724 725 if (queue->data_digest) { 726 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 727 if (unlikely(ret)) 728 return ret; 729 } 730 731 switch (hdr->type) { 732 case nvme_tcp_c2h_data: 733 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 734 case nvme_tcp_rsp: 735 nvme_tcp_init_recv_ctx(queue); 736 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 737 case nvme_tcp_r2t: 738 nvme_tcp_init_recv_ctx(queue); 739 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 740 default: 741 dev_err(queue->ctrl->ctrl.device, 742 "unsupported pdu type (%d)\n", hdr->type); 743 return -EINVAL; 744 } 745 } 746 747 static inline void nvme_tcp_end_request(struct request *rq, u16 status) 748 { 749 union nvme_result res = {}; 750 751 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 752 nvme_complete_rq(rq); 753 } 754 755 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 756 unsigned int *offset, size_t *len) 757 { 758 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 759 struct request *rq = 760 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 761 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 762 763 while (true) { 764 int recv_len, ret; 765 766 recv_len = min_t(size_t, *len, queue->data_remaining); 767 if (!recv_len) 768 break; 769 770 if (!iov_iter_count(&req->iter)) { 771 req->curr_bio = req->curr_bio->bi_next; 772 773 /* 774 * If we don`t have any bios it means that controller 775 * sent more data than we requested, hence error 776 */ 777 if (!req->curr_bio) { 778 dev_err(queue->ctrl->ctrl.device, 779 "queue %d no space in request %#x", 780 nvme_tcp_queue_id(queue), rq->tag); 781 nvme_tcp_init_recv_ctx(queue); 782 return -EIO; 783 } 784 nvme_tcp_init_iter(req, ITER_DEST); 785 } 786 787 /* we can read only from what is left in this bio */ 788 recv_len = min_t(size_t, recv_len, 789 iov_iter_count(&req->iter)); 790 791 if (queue->data_digest) 792 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 793 &req->iter, recv_len, queue->rcv_hash); 794 else 795 ret = skb_copy_datagram_iter(skb, *offset, 796 &req->iter, recv_len); 797 if (ret) { 798 dev_err(queue->ctrl->ctrl.device, 799 "queue %d failed to copy request %#x data", 800 nvme_tcp_queue_id(queue), rq->tag); 801 return ret; 802 } 803 804 *len -= recv_len; 805 *offset += recv_len; 806 queue->data_remaining -= recv_len; 807 } 808 809 if (!queue->data_remaining) { 810 if (queue->data_digest) { 811 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 812 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 813 } else { 814 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 815 nvme_tcp_end_request(rq, 816 le16_to_cpu(req->status)); 817 queue->nr_cqe++; 818 } 819 nvme_tcp_init_recv_ctx(queue); 820 } 821 } 822 823 return 0; 824 } 825 826 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 827 struct sk_buff *skb, unsigned int *offset, size_t *len) 828 { 829 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 830 char *ddgst = (char *)&queue->recv_ddgst; 831 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 832 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 833 int ret; 834 835 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 836 if (unlikely(ret)) 837 return ret; 838 839 queue->ddgst_remaining -= recv_len; 840 *offset += recv_len; 841 *len -= recv_len; 842 if (queue->ddgst_remaining) 843 return 0; 844 845 if (queue->recv_ddgst != queue->exp_ddgst) { 846 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 847 pdu->command_id); 848 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 849 850 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 851 852 dev_err(queue->ctrl->ctrl.device, 853 "data digest error: recv %#x expected %#x\n", 854 le32_to_cpu(queue->recv_ddgst), 855 le32_to_cpu(queue->exp_ddgst)); 856 } 857 858 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 859 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 860 pdu->command_id); 861 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 862 863 nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 864 queue->nr_cqe++; 865 } 866 867 nvme_tcp_init_recv_ctx(queue); 868 return 0; 869 } 870 871 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 872 unsigned int offset, size_t len) 873 { 874 struct nvme_tcp_queue *queue = desc->arg.data; 875 size_t consumed = len; 876 int result; 877 878 while (len) { 879 switch (nvme_tcp_recv_state(queue)) { 880 case NVME_TCP_RECV_PDU: 881 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 882 break; 883 case NVME_TCP_RECV_DATA: 884 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 885 break; 886 case NVME_TCP_RECV_DDGST: 887 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 888 break; 889 default: 890 result = -EFAULT; 891 } 892 if (result) { 893 dev_err(queue->ctrl->ctrl.device, 894 "receive failed: %d\n", result); 895 queue->rd_enabled = false; 896 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 897 return result; 898 } 899 } 900 901 return consumed; 902 } 903 904 static void nvme_tcp_data_ready(struct sock *sk) 905 { 906 struct nvme_tcp_queue *queue; 907 908 read_lock_bh(&sk->sk_callback_lock); 909 queue = sk->sk_user_data; 910 if (likely(queue && queue->rd_enabled) && 911 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 912 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 913 read_unlock_bh(&sk->sk_callback_lock); 914 } 915 916 static void nvme_tcp_write_space(struct sock *sk) 917 { 918 struct nvme_tcp_queue *queue; 919 920 read_lock_bh(&sk->sk_callback_lock); 921 queue = sk->sk_user_data; 922 if (likely(queue && sk_stream_is_writeable(sk))) { 923 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 924 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 925 } 926 read_unlock_bh(&sk->sk_callback_lock); 927 } 928 929 static void nvme_tcp_state_change(struct sock *sk) 930 { 931 struct nvme_tcp_queue *queue; 932 933 read_lock_bh(&sk->sk_callback_lock); 934 queue = sk->sk_user_data; 935 if (!queue) 936 goto done; 937 938 switch (sk->sk_state) { 939 case TCP_CLOSE: 940 case TCP_CLOSE_WAIT: 941 case TCP_LAST_ACK: 942 case TCP_FIN_WAIT1: 943 case TCP_FIN_WAIT2: 944 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 945 break; 946 default: 947 dev_info(queue->ctrl->ctrl.device, 948 "queue %d socket state %d\n", 949 nvme_tcp_queue_id(queue), sk->sk_state); 950 } 951 952 queue->state_change(sk); 953 done: 954 read_unlock_bh(&sk->sk_callback_lock); 955 } 956 957 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 958 { 959 queue->request = NULL; 960 } 961 962 static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 963 { 964 if (nvme_tcp_async_req(req)) { 965 union nvme_result res = {}; 966 967 nvme_complete_async_event(&req->queue->ctrl->ctrl, 968 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 969 } else { 970 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 971 NVME_SC_HOST_PATH_ERROR); 972 } 973 } 974 975 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 976 { 977 struct nvme_tcp_queue *queue = req->queue; 978 int req_data_len = req->data_len; 979 u32 h2cdata_left = req->h2cdata_left; 980 981 while (true) { 982 struct page *page = nvme_tcp_req_cur_page(req); 983 size_t offset = nvme_tcp_req_cur_offset(req); 984 size_t len = nvme_tcp_req_cur_length(req); 985 bool last = nvme_tcp_pdu_last_send(req, len); 986 int req_data_sent = req->data_sent; 987 int ret, flags = MSG_DONTWAIT; 988 989 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 990 flags |= MSG_EOR; 991 else 992 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 993 994 if (sendpage_ok(page)) { 995 ret = kernel_sendpage(queue->sock, page, offset, len, 996 flags); 997 } else { 998 ret = sock_no_sendpage(queue->sock, page, offset, len, 999 flags); 1000 } 1001 if (ret <= 0) 1002 return ret; 1003 1004 if (queue->data_digest) 1005 nvme_tcp_ddgst_update(queue->snd_hash, page, 1006 offset, ret); 1007 1008 /* 1009 * update the request iterator except for the last payload send 1010 * in the request where we don't want to modify it as we may 1011 * compete with the RX path completing the request. 1012 */ 1013 if (req_data_sent + ret < req_data_len) 1014 nvme_tcp_advance_req(req, ret); 1015 1016 /* fully successful last send in current PDU */ 1017 if (last && ret == len) { 1018 if (queue->data_digest) { 1019 nvme_tcp_ddgst_final(queue->snd_hash, 1020 &req->ddgst); 1021 req->state = NVME_TCP_SEND_DDGST; 1022 req->offset = 0; 1023 } else { 1024 if (h2cdata_left) 1025 nvme_tcp_setup_h2c_data_pdu(req); 1026 else 1027 nvme_tcp_done_send_req(queue); 1028 } 1029 return 1; 1030 } 1031 } 1032 return -EAGAIN; 1033 } 1034 1035 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 1036 { 1037 struct nvme_tcp_queue *queue = req->queue; 1038 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 1039 bool inline_data = nvme_tcp_has_inline_data(req); 1040 u8 hdgst = nvme_tcp_hdgst_len(queue); 1041 int len = sizeof(*pdu) + hdgst - req->offset; 1042 int flags = MSG_DONTWAIT; 1043 int ret; 1044 1045 if (inline_data || nvme_tcp_queue_more(queue)) 1046 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 1047 else 1048 flags |= MSG_EOR; 1049 1050 if (queue->hdr_digest && !req->offset) 1051 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1052 1053 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1054 offset_in_page(pdu) + req->offset, len, flags); 1055 if (unlikely(ret <= 0)) 1056 return ret; 1057 1058 len -= ret; 1059 if (!len) { 1060 if (inline_data) { 1061 req->state = NVME_TCP_SEND_DATA; 1062 if (queue->data_digest) 1063 crypto_ahash_init(queue->snd_hash); 1064 } else { 1065 nvme_tcp_done_send_req(queue); 1066 } 1067 return 1; 1068 } 1069 req->offset += ret; 1070 1071 return -EAGAIN; 1072 } 1073 1074 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1075 { 1076 struct nvme_tcp_queue *queue = req->queue; 1077 struct nvme_tcp_data_pdu *pdu = req->pdu; 1078 u8 hdgst = nvme_tcp_hdgst_len(queue); 1079 int len = sizeof(*pdu) - req->offset + hdgst; 1080 int ret; 1081 1082 if (queue->hdr_digest && !req->offset) 1083 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1084 1085 if (!req->h2cdata_left) 1086 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1087 offset_in_page(pdu) + req->offset, len, 1088 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); 1089 else 1090 ret = sock_no_sendpage(queue->sock, virt_to_page(pdu), 1091 offset_in_page(pdu) + req->offset, len, 1092 MSG_DONTWAIT | MSG_MORE); 1093 if (unlikely(ret <= 0)) 1094 return ret; 1095 1096 len -= ret; 1097 if (!len) { 1098 req->state = NVME_TCP_SEND_DATA; 1099 if (queue->data_digest) 1100 crypto_ahash_init(queue->snd_hash); 1101 return 1; 1102 } 1103 req->offset += ret; 1104 1105 return -EAGAIN; 1106 } 1107 1108 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1109 { 1110 struct nvme_tcp_queue *queue = req->queue; 1111 size_t offset = req->offset; 1112 u32 h2cdata_left = req->h2cdata_left; 1113 int ret; 1114 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1115 struct kvec iov = { 1116 .iov_base = (u8 *)&req->ddgst + req->offset, 1117 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1118 }; 1119 1120 if (nvme_tcp_queue_more(queue)) 1121 msg.msg_flags |= MSG_MORE; 1122 else 1123 msg.msg_flags |= MSG_EOR; 1124 1125 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1126 if (unlikely(ret <= 0)) 1127 return ret; 1128 1129 if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1130 if (h2cdata_left) 1131 nvme_tcp_setup_h2c_data_pdu(req); 1132 else 1133 nvme_tcp_done_send_req(queue); 1134 return 1; 1135 } 1136 1137 req->offset += ret; 1138 return -EAGAIN; 1139 } 1140 1141 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1142 { 1143 struct nvme_tcp_request *req; 1144 unsigned int noreclaim_flag; 1145 int ret = 1; 1146 1147 if (!queue->request) { 1148 queue->request = nvme_tcp_fetch_request(queue); 1149 if (!queue->request) 1150 return 0; 1151 } 1152 req = queue->request; 1153 1154 noreclaim_flag = memalloc_noreclaim_save(); 1155 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1156 ret = nvme_tcp_try_send_cmd_pdu(req); 1157 if (ret <= 0) 1158 goto done; 1159 if (!nvme_tcp_has_inline_data(req)) 1160 goto out; 1161 } 1162 1163 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1164 ret = nvme_tcp_try_send_data_pdu(req); 1165 if (ret <= 0) 1166 goto done; 1167 } 1168 1169 if (req->state == NVME_TCP_SEND_DATA) { 1170 ret = nvme_tcp_try_send_data(req); 1171 if (ret <= 0) 1172 goto done; 1173 } 1174 1175 if (req->state == NVME_TCP_SEND_DDGST) 1176 ret = nvme_tcp_try_send_ddgst(req); 1177 done: 1178 if (ret == -EAGAIN) { 1179 ret = 0; 1180 } else if (ret < 0) { 1181 dev_err(queue->ctrl->ctrl.device, 1182 "failed to send request %d\n", ret); 1183 nvme_tcp_fail_request(queue->request); 1184 nvme_tcp_done_send_req(queue); 1185 } 1186 out: 1187 memalloc_noreclaim_restore(noreclaim_flag); 1188 return ret; 1189 } 1190 1191 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1192 { 1193 struct socket *sock = queue->sock; 1194 struct sock *sk = sock->sk; 1195 read_descriptor_t rd_desc; 1196 int consumed; 1197 1198 rd_desc.arg.data = queue; 1199 rd_desc.count = 1; 1200 lock_sock(sk); 1201 queue->nr_cqe = 0; 1202 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1203 release_sock(sk); 1204 return consumed; 1205 } 1206 1207 static void nvme_tcp_io_work(struct work_struct *w) 1208 { 1209 struct nvme_tcp_queue *queue = 1210 container_of(w, struct nvme_tcp_queue, io_work); 1211 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1212 1213 do { 1214 bool pending = false; 1215 int result; 1216 1217 if (mutex_trylock(&queue->send_mutex)) { 1218 result = nvme_tcp_try_send(queue); 1219 mutex_unlock(&queue->send_mutex); 1220 if (result > 0) 1221 pending = true; 1222 else if (unlikely(result < 0)) 1223 break; 1224 } 1225 1226 result = nvme_tcp_try_recv(queue); 1227 if (result > 0) 1228 pending = true; 1229 else if (unlikely(result < 0)) 1230 return; 1231 1232 if (!pending || !queue->rd_enabled) 1233 return; 1234 1235 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1236 1237 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1238 } 1239 1240 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1241 { 1242 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1243 1244 ahash_request_free(queue->rcv_hash); 1245 ahash_request_free(queue->snd_hash); 1246 crypto_free_ahash(tfm); 1247 } 1248 1249 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1250 { 1251 struct crypto_ahash *tfm; 1252 1253 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1254 if (IS_ERR(tfm)) 1255 return PTR_ERR(tfm); 1256 1257 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1258 if (!queue->snd_hash) 1259 goto free_tfm; 1260 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1261 1262 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1263 if (!queue->rcv_hash) 1264 goto free_snd_hash; 1265 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1266 1267 return 0; 1268 free_snd_hash: 1269 ahash_request_free(queue->snd_hash); 1270 free_tfm: 1271 crypto_free_ahash(tfm); 1272 return -ENOMEM; 1273 } 1274 1275 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1276 { 1277 struct nvme_tcp_request *async = &ctrl->async_req; 1278 1279 page_frag_free(async->pdu); 1280 } 1281 1282 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1283 { 1284 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1285 struct nvme_tcp_request *async = &ctrl->async_req; 1286 u8 hdgst = nvme_tcp_hdgst_len(queue); 1287 1288 async->pdu = page_frag_alloc(&queue->pf_cache, 1289 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1290 GFP_KERNEL | __GFP_ZERO); 1291 if (!async->pdu) 1292 return -ENOMEM; 1293 1294 async->queue = &ctrl->queues[0]; 1295 return 0; 1296 } 1297 1298 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1299 { 1300 struct page *page; 1301 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1302 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1303 unsigned int noreclaim_flag; 1304 1305 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1306 return; 1307 1308 if (queue->hdr_digest || queue->data_digest) 1309 nvme_tcp_free_crypto(queue); 1310 1311 if (queue->pf_cache.va) { 1312 page = virt_to_head_page(queue->pf_cache.va); 1313 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); 1314 queue->pf_cache.va = NULL; 1315 } 1316 1317 noreclaim_flag = memalloc_noreclaim_save(); 1318 sock_release(queue->sock); 1319 memalloc_noreclaim_restore(noreclaim_flag); 1320 1321 kfree(queue->pdu); 1322 mutex_destroy(&queue->send_mutex); 1323 mutex_destroy(&queue->queue_lock); 1324 } 1325 1326 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1327 { 1328 struct nvme_tcp_icreq_pdu *icreq; 1329 struct nvme_tcp_icresp_pdu *icresp; 1330 struct msghdr msg = {}; 1331 struct kvec iov; 1332 bool ctrl_hdgst, ctrl_ddgst; 1333 u32 maxh2cdata; 1334 int ret; 1335 1336 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1337 if (!icreq) 1338 return -ENOMEM; 1339 1340 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1341 if (!icresp) { 1342 ret = -ENOMEM; 1343 goto free_icreq; 1344 } 1345 1346 icreq->hdr.type = nvme_tcp_icreq; 1347 icreq->hdr.hlen = sizeof(*icreq); 1348 icreq->hdr.pdo = 0; 1349 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1350 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1351 icreq->maxr2t = 0; /* single inflight r2t supported */ 1352 icreq->hpda = 0; /* no alignment constraint */ 1353 if (queue->hdr_digest) 1354 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1355 if (queue->data_digest) 1356 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1357 1358 iov.iov_base = icreq; 1359 iov.iov_len = sizeof(*icreq); 1360 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1361 if (ret < 0) 1362 goto free_icresp; 1363 1364 memset(&msg, 0, sizeof(msg)); 1365 iov.iov_base = icresp; 1366 iov.iov_len = sizeof(*icresp); 1367 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1368 iov.iov_len, msg.msg_flags); 1369 if (ret < 0) 1370 goto free_icresp; 1371 1372 ret = -EINVAL; 1373 if (icresp->hdr.type != nvme_tcp_icresp) { 1374 pr_err("queue %d: bad type returned %d\n", 1375 nvme_tcp_queue_id(queue), icresp->hdr.type); 1376 goto free_icresp; 1377 } 1378 1379 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1380 pr_err("queue %d: bad pdu length returned %d\n", 1381 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1382 goto free_icresp; 1383 } 1384 1385 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1386 pr_err("queue %d: bad pfv returned %d\n", 1387 nvme_tcp_queue_id(queue), icresp->pfv); 1388 goto free_icresp; 1389 } 1390 1391 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1392 if ((queue->data_digest && !ctrl_ddgst) || 1393 (!queue->data_digest && ctrl_ddgst)) { 1394 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1395 nvme_tcp_queue_id(queue), 1396 queue->data_digest ? "enabled" : "disabled", 1397 ctrl_ddgst ? "enabled" : "disabled"); 1398 goto free_icresp; 1399 } 1400 1401 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1402 if ((queue->hdr_digest && !ctrl_hdgst) || 1403 (!queue->hdr_digest && ctrl_hdgst)) { 1404 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1405 nvme_tcp_queue_id(queue), 1406 queue->hdr_digest ? "enabled" : "disabled", 1407 ctrl_hdgst ? "enabled" : "disabled"); 1408 goto free_icresp; 1409 } 1410 1411 if (icresp->cpda != 0) { 1412 pr_err("queue %d: unsupported cpda returned %d\n", 1413 nvme_tcp_queue_id(queue), icresp->cpda); 1414 goto free_icresp; 1415 } 1416 1417 maxh2cdata = le32_to_cpu(icresp->maxdata); 1418 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) { 1419 pr_err("queue %d: invalid maxh2cdata returned %u\n", 1420 nvme_tcp_queue_id(queue), maxh2cdata); 1421 goto free_icresp; 1422 } 1423 queue->maxh2cdata = maxh2cdata; 1424 1425 ret = 0; 1426 free_icresp: 1427 kfree(icresp); 1428 free_icreq: 1429 kfree(icreq); 1430 return ret; 1431 } 1432 1433 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1434 { 1435 return nvme_tcp_queue_id(queue) == 0; 1436 } 1437 1438 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1439 { 1440 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1441 int qid = nvme_tcp_queue_id(queue); 1442 1443 return !nvme_tcp_admin_queue(queue) && 1444 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1445 } 1446 1447 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1448 { 1449 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1450 int qid = nvme_tcp_queue_id(queue); 1451 1452 return !nvme_tcp_admin_queue(queue) && 1453 !nvme_tcp_default_queue(queue) && 1454 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1455 ctrl->io_queues[HCTX_TYPE_READ]; 1456 } 1457 1458 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1459 { 1460 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1461 int qid = nvme_tcp_queue_id(queue); 1462 1463 return !nvme_tcp_admin_queue(queue) && 1464 !nvme_tcp_default_queue(queue) && 1465 !nvme_tcp_read_queue(queue) && 1466 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1467 ctrl->io_queues[HCTX_TYPE_READ] + 1468 ctrl->io_queues[HCTX_TYPE_POLL]; 1469 } 1470 1471 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1472 { 1473 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1474 int qid = nvme_tcp_queue_id(queue); 1475 int n = 0; 1476 1477 if (nvme_tcp_default_queue(queue)) 1478 n = qid - 1; 1479 else if (nvme_tcp_read_queue(queue)) 1480 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 1481 else if (nvme_tcp_poll_queue(queue)) 1482 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1483 ctrl->io_queues[HCTX_TYPE_READ] - 1; 1484 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 1485 } 1486 1487 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) 1488 { 1489 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1490 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1491 int ret, rcv_pdu_size; 1492 1493 mutex_init(&queue->queue_lock); 1494 queue->ctrl = ctrl; 1495 init_llist_head(&queue->req_list); 1496 INIT_LIST_HEAD(&queue->send_list); 1497 mutex_init(&queue->send_mutex); 1498 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1499 1500 if (qid > 0) 1501 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1502 else 1503 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1504 NVME_TCP_ADMIN_CCSZ; 1505 1506 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 1507 IPPROTO_TCP, &queue->sock); 1508 if (ret) { 1509 dev_err(nctrl->device, 1510 "failed to create socket: %d\n", ret); 1511 goto err_destroy_mutex; 1512 } 1513 1514 nvme_tcp_reclassify_socket(queue->sock); 1515 1516 /* Single syn retry */ 1517 tcp_sock_set_syncnt(queue->sock->sk, 1); 1518 1519 /* Set TCP no delay */ 1520 tcp_sock_set_nodelay(queue->sock->sk); 1521 1522 /* 1523 * Cleanup whatever is sitting in the TCP transmit queue on socket 1524 * close. This is done to prevent stale data from being sent should 1525 * the network connection be restored before TCP times out. 1526 */ 1527 sock_no_linger(queue->sock->sk); 1528 1529 if (so_priority > 0) 1530 sock_set_priority(queue->sock->sk, so_priority); 1531 1532 /* Set socket type of service */ 1533 if (nctrl->opts->tos >= 0) 1534 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1535 1536 /* Set 10 seconds timeout for icresp recvmsg */ 1537 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1538 1539 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1540 queue->sock->sk->sk_use_task_frag = false; 1541 nvme_tcp_set_queue_io_cpu(queue); 1542 queue->request = NULL; 1543 queue->data_remaining = 0; 1544 queue->ddgst_remaining = 0; 1545 queue->pdu_remaining = 0; 1546 queue->pdu_offset = 0; 1547 sk_set_memalloc(queue->sock->sk); 1548 1549 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1550 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1551 sizeof(ctrl->src_addr)); 1552 if (ret) { 1553 dev_err(nctrl->device, 1554 "failed to bind queue %d socket %d\n", 1555 qid, ret); 1556 goto err_sock; 1557 } 1558 } 1559 1560 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 1561 char *iface = nctrl->opts->host_iface; 1562 sockptr_t optval = KERNEL_SOCKPTR(iface); 1563 1564 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 1565 optval, strlen(iface)); 1566 if (ret) { 1567 dev_err(nctrl->device, 1568 "failed to bind to interface %s queue %d err %d\n", 1569 iface, qid, ret); 1570 goto err_sock; 1571 } 1572 } 1573 1574 queue->hdr_digest = nctrl->opts->hdr_digest; 1575 queue->data_digest = nctrl->opts->data_digest; 1576 if (queue->hdr_digest || queue->data_digest) { 1577 ret = nvme_tcp_alloc_crypto(queue); 1578 if (ret) { 1579 dev_err(nctrl->device, 1580 "failed to allocate queue %d crypto\n", qid); 1581 goto err_sock; 1582 } 1583 } 1584 1585 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1586 nvme_tcp_hdgst_len(queue); 1587 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1588 if (!queue->pdu) { 1589 ret = -ENOMEM; 1590 goto err_crypto; 1591 } 1592 1593 dev_dbg(nctrl->device, "connecting queue %d\n", 1594 nvme_tcp_queue_id(queue)); 1595 1596 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1597 sizeof(ctrl->addr), 0); 1598 if (ret) { 1599 dev_err(nctrl->device, 1600 "failed to connect socket: %d\n", ret); 1601 goto err_rcv_pdu; 1602 } 1603 1604 ret = nvme_tcp_init_connection(queue); 1605 if (ret) 1606 goto err_init_connect; 1607 1608 queue->rd_enabled = true; 1609 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1610 nvme_tcp_init_recv_ctx(queue); 1611 1612 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1613 queue->sock->sk->sk_user_data = queue; 1614 queue->state_change = queue->sock->sk->sk_state_change; 1615 queue->data_ready = queue->sock->sk->sk_data_ready; 1616 queue->write_space = queue->sock->sk->sk_write_space; 1617 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1618 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1619 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1620 #ifdef CONFIG_NET_RX_BUSY_POLL 1621 queue->sock->sk->sk_ll_usec = 1; 1622 #endif 1623 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1624 1625 return 0; 1626 1627 err_init_connect: 1628 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1629 err_rcv_pdu: 1630 kfree(queue->pdu); 1631 err_crypto: 1632 if (queue->hdr_digest || queue->data_digest) 1633 nvme_tcp_free_crypto(queue); 1634 err_sock: 1635 sock_release(queue->sock); 1636 queue->sock = NULL; 1637 err_destroy_mutex: 1638 mutex_destroy(&queue->send_mutex); 1639 mutex_destroy(&queue->queue_lock); 1640 return ret; 1641 } 1642 1643 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue) 1644 { 1645 struct socket *sock = queue->sock; 1646 1647 write_lock_bh(&sock->sk->sk_callback_lock); 1648 sock->sk->sk_user_data = NULL; 1649 sock->sk->sk_data_ready = queue->data_ready; 1650 sock->sk->sk_state_change = queue->state_change; 1651 sock->sk->sk_write_space = queue->write_space; 1652 write_unlock_bh(&sock->sk->sk_callback_lock); 1653 } 1654 1655 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1656 { 1657 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1658 nvme_tcp_restore_sock_calls(queue); 1659 cancel_work_sync(&queue->io_work); 1660 } 1661 1662 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1663 { 1664 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1665 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1666 1667 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1668 return; 1669 1670 mutex_lock(&queue->queue_lock); 1671 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1672 __nvme_tcp_stop_queue(queue); 1673 mutex_unlock(&queue->queue_lock); 1674 } 1675 1676 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1677 { 1678 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1679 int ret; 1680 1681 if (idx) 1682 ret = nvmf_connect_io_queue(nctrl, idx); 1683 else 1684 ret = nvmf_connect_admin_queue(nctrl); 1685 1686 if (!ret) { 1687 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags); 1688 } else { 1689 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags)) 1690 __nvme_tcp_stop_queue(&ctrl->queues[idx]); 1691 dev_err(nctrl->device, 1692 "failed to connect queue: %d ret=%d\n", idx, ret); 1693 } 1694 return ret; 1695 } 1696 1697 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 1698 { 1699 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1700 cancel_work_sync(&ctrl->async_event_work); 1701 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 1702 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 1703 } 1704 1705 nvme_tcp_free_queue(ctrl, 0); 1706 } 1707 1708 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 1709 { 1710 int i; 1711 1712 for (i = 1; i < ctrl->queue_count; i++) 1713 nvme_tcp_free_queue(ctrl, i); 1714 } 1715 1716 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 1717 { 1718 int i; 1719 1720 for (i = 1; i < ctrl->queue_count; i++) 1721 nvme_tcp_stop_queue(ctrl, i); 1722 } 1723 1724 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, 1725 int first, int last) 1726 { 1727 int i, ret; 1728 1729 for (i = first; i < last; i++) { 1730 ret = nvme_tcp_start_queue(ctrl, i); 1731 if (ret) 1732 goto out_stop_queues; 1733 } 1734 1735 return 0; 1736 1737 out_stop_queues: 1738 for (i--; i >= first; i--) 1739 nvme_tcp_stop_queue(ctrl, i); 1740 return ret; 1741 } 1742 1743 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 1744 { 1745 int ret; 1746 1747 ret = nvme_tcp_alloc_queue(ctrl, 0); 1748 if (ret) 1749 return ret; 1750 1751 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 1752 if (ret) 1753 goto out_free_queue; 1754 1755 return 0; 1756 1757 out_free_queue: 1758 nvme_tcp_free_queue(ctrl, 0); 1759 return ret; 1760 } 1761 1762 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1763 { 1764 int i, ret; 1765 1766 for (i = 1; i < ctrl->queue_count; i++) { 1767 ret = nvme_tcp_alloc_queue(ctrl, i); 1768 if (ret) 1769 goto out_free_queues; 1770 } 1771 1772 return 0; 1773 1774 out_free_queues: 1775 for (i--; i >= 1; i--) 1776 nvme_tcp_free_queue(ctrl, i); 1777 1778 return ret; 1779 } 1780 1781 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 1782 { 1783 unsigned int nr_io_queues; 1784 1785 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1786 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 1787 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1788 1789 return nr_io_queues; 1790 } 1791 1792 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 1793 unsigned int nr_io_queues) 1794 { 1795 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1796 struct nvmf_ctrl_options *opts = nctrl->opts; 1797 1798 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 1799 /* 1800 * separate read/write queues 1801 * hand out dedicated default queues only after we have 1802 * sufficient read queues. 1803 */ 1804 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 1805 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 1806 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1807 min(opts->nr_write_queues, nr_io_queues); 1808 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1809 } else { 1810 /* 1811 * shared read/write queues 1812 * either no write queues were requested, or we don't have 1813 * sufficient queue count to have dedicated default queues. 1814 */ 1815 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1816 min(opts->nr_io_queues, nr_io_queues); 1817 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1818 } 1819 1820 if (opts->nr_poll_queues && nr_io_queues) { 1821 /* map dedicated poll queues only if we have queues left */ 1822 ctrl->io_queues[HCTX_TYPE_POLL] = 1823 min(opts->nr_poll_queues, nr_io_queues); 1824 } 1825 } 1826 1827 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1828 { 1829 unsigned int nr_io_queues; 1830 int ret; 1831 1832 nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 1833 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 1834 if (ret) 1835 return ret; 1836 1837 if (nr_io_queues == 0) { 1838 dev_err(ctrl->device, 1839 "unable to set any I/O queues\n"); 1840 return -ENOMEM; 1841 } 1842 1843 ctrl->queue_count = nr_io_queues + 1; 1844 dev_info(ctrl->device, 1845 "creating %d I/O queues.\n", nr_io_queues); 1846 1847 nvme_tcp_set_io_queues(ctrl, nr_io_queues); 1848 1849 return __nvme_tcp_alloc_io_queues(ctrl); 1850 } 1851 1852 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 1853 { 1854 nvme_tcp_stop_io_queues(ctrl); 1855 if (remove) 1856 nvme_remove_io_tag_set(ctrl); 1857 nvme_tcp_free_io_queues(ctrl); 1858 } 1859 1860 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 1861 { 1862 int ret, nr_queues; 1863 1864 ret = nvme_tcp_alloc_io_queues(ctrl); 1865 if (ret) 1866 return ret; 1867 1868 if (new) { 1869 ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set, 1870 &nvme_tcp_mq_ops, 1871 ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2, 1872 sizeof(struct nvme_tcp_request)); 1873 if (ret) 1874 goto out_free_io_queues; 1875 } 1876 1877 /* 1878 * Only start IO queues for which we have allocated the tagset 1879 * and limitted it to the available queues. On reconnects, the 1880 * queue number might have changed. 1881 */ 1882 nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); 1883 ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues); 1884 if (ret) 1885 goto out_cleanup_connect_q; 1886 1887 if (!new) { 1888 nvme_unquiesce_io_queues(ctrl); 1889 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 1890 /* 1891 * If we timed out waiting for freeze we are likely to 1892 * be stuck. Fail the controller initialization just 1893 * to be safe. 1894 */ 1895 ret = -ENODEV; 1896 goto out_wait_freeze_timed_out; 1897 } 1898 blk_mq_update_nr_hw_queues(ctrl->tagset, 1899 ctrl->queue_count - 1); 1900 nvme_unfreeze(ctrl); 1901 } 1902 1903 /* 1904 * If the number of queues has increased (reconnect case) 1905 * start all new queues now. 1906 */ 1907 ret = nvme_tcp_start_io_queues(ctrl, nr_queues, 1908 ctrl->tagset->nr_hw_queues + 1); 1909 if (ret) 1910 goto out_wait_freeze_timed_out; 1911 1912 return 0; 1913 1914 out_wait_freeze_timed_out: 1915 nvme_quiesce_io_queues(ctrl); 1916 nvme_sync_io_queues(ctrl); 1917 nvme_tcp_stop_io_queues(ctrl); 1918 out_cleanup_connect_q: 1919 nvme_cancel_tagset(ctrl); 1920 if (new) 1921 nvme_remove_io_tag_set(ctrl); 1922 out_free_io_queues: 1923 nvme_tcp_free_io_queues(ctrl); 1924 return ret; 1925 } 1926 1927 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 1928 { 1929 nvme_tcp_stop_queue(ctrl, 0); 1930 if (remove) 1931 nvme_remove_admin_tag_set(ctrl); 1932 nvme_tcp_free_admin_queue(ctrl); 1933 } 1934 1935 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 1936 { 1937 int error; 1938 1939 error = nvme_tcp_alloc_admin_queue(ctrl); 1940 if (error) 1941 return error; 1942 1943 if (new) { 1944 error = nvme_alloc_admin_tag_set(ctrl, 1945 &to_tcp_ctrl(ctrl)->admin_tag_set, 1946 &nvme_tcp_admin_mq_ops, 1947 sizeof(struct nvme_tcp_request)); 1948 if (error) 1949 goto out_free_queue; 1950 } 1951 1952 error = nvme_tcp_start_queue(ctrl, 0); 1953 if (error) 1954 goto out_cleanup_tagset; 1955 1956 error = nvme_enable_ctrl(ctrl); 1957 if (error) 1958 goto out_stop_queue; 1959 1960 nvme_unquiesce_admin_queue(ctrl); 1961 1962 error = nvme_init_ctrl_finish(ctrl, false); 1963 if (error) 1964 goto out_quiesce_queue; 1965 1966 return 0; 1967 1968 out_quiesce_queue: 1969 nvme_quiesce_admin_queue(ctrl); 1970 blk_sync_queue(ctrl->admin_q); 1971 out_stop_queue: 1972 nvme_tcp_stop_queue(ctrl, 0); 1973 nvme_cancel_admin_tagset(ctrl); 1974 out_cleanup_tagset: 1975 if (new) 1976 nvme_remove_admin_tag_set(ctrl); 1977 out_free_queue: 1978 nvme_tcp_free_admin_queue(ctrl); 1979 return error; 1980 } 1981 1982 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 1983 bool remove) 1984 { 1985 nvme_quiesce_admin_queue(ctrl); 1986 blk_sync_queue(ctrl->admin_q); 1987 nvme_tcp_stop_queue(ctrl, 0); 1988 nvme_cancel_admin_tagset(ctrl); 1989 if (remove) 1990 nvme_unquiesce_admin_queue(ctrl); 1991 nvme_tcp_destroy_admin_queue(ctrl, remove); 1992 } 1993 1994 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 1995 bool remove) 1996 { 1997 if (ctrl->queue_count <= 1) 1998 return; 1999 nvme_quiesce_admin_queue(ctrl); 2000 nvme_start_freeze(ctrl); 2001 nvme_quiesce_io_queues(ctrl); 2002 nvme_sync_io_queues(ctrl); 2003 nvme_tcp_stop_io_queues(ctrl); 2004 nvme_cancel_tagset(ctrl); 2005 if (remove) 2006 nvme_unquiesce_io_queues(ctrl); 2007 nvme_tcp_destroy_io_queues(ctrl, remove); 2008 } 2009 2010 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 2011 { 2012 /* If we are resetting/deleting then do nothing */ 2013 if (ctrl->state != NVME_CTRL_CONNECTING) { 2014 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 2015 ctrl->state == NVME_CTRL_LIVE); 2016 return; 2017 } 2018 2019 if (nvmf_should_reconnect(ctrl)) { 2020 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 2021 ctrl->opts->reconnect_delay); 2022 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 2023 ctrl->opts->reconnect_delay * HZ); 2024 } else { 2025 dev_info(ctrl->device, "Removing controller...\n"); 2026 nvme_delete_ctrl(ctrl); 2027 } 2028 } 2029 2030 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 2031 { 2032 struct nvmf_ctrl_options *opts = ctrl->opts; 2033 int ret; 2034 2035 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2036 if (ret) 2037 return ret; 2038 2039 if (ctrl->icdoff) { 2040 ret = -EOPNOTSUPP; 2041 dev_err(ctrl->device, "icdoff is not supported!\n"); 2042 goto destroy_admin; 2043 } 2044 2045 if (!nvme_ctrl_sgl_supported(ctrl)) { 2046 ret = -EOPNOTSUPP; 2047 dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 2048 goto destroy_admin; 2049 } 2050 2051 if (opts->queue_size > ctrl->sqsize + 1) 2052 dev_warn(ctrl->device, 2053 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2054 opts->queue_size, ctrl->sqsize + 1); 2055 2056 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2057 dev_warn(ctrl->device, 2058 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2059 ctrl->sqsize + 1, ctrl->maxcmd); 2060 ctrl->sqsize = ctrl->maxcmd - 1; 2061 } 2062 2063 if (ctrl->queue_count > 1) { 2064 ret = nvme_tcp_configure_io_queues(ctrl, new); 2065 if (ret) 2066 goto destroy_admin; 2067 } 2068 2069 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2070 /* 2071 * state change failure is ok if we started ctrl delete, 2072 * unless we're during creation of a new controller to 2073 * avoid races with teardown flow. 2074 */ 2075 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2076 ctrl->state != NVME_CTRL_DELETING_NOIO); 2077 WARN_ON_ONCE(new); 2078 ret = -EINVAL; 2079 goto destroy_io; 2080 } 2081 2082 nvme_start_ctrl(ctrl); 2083 return 0; 2084 2085 destroy_io: 2086 if (ctrl->queue_count > 1) { 2087 nvme_quiesce_io_queues(ctrl); 2088 nvme_sync_io_queues(ctrl); 2089 nvme_tcp_stop_io_queues(ctrl); 2090 nvme_cancel_tagset(ctrl); 2091 nvme_tcp_destroy_io_queues(ctrl, new); 2092 } 2093 destroy_admin: 2094 nvme_quiesce_admin_queue(ctrl); 2095 blk_sync_queue(ctrl->admin_q); 2096 nvme_tcp_stop_queue(ctrl, 0); 2097 nvme_cancel_admin_tagset(ctrl); 2098 nvme_tcp_destroy_admin_queue(ctrl, new); 2099 return ret; 2100 } 2101 2102 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2103 { 2104 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2105 struct nvme_tcp_ctrl, connect_work); 2106 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2107 2108 ++ctrl->nr_reconnects; 2109 2110 if (nvme_tcp_setup_ctrl(ctrl, false)) 2111 goto requeue; 2112 2113 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 2114 ctrl->nr_reconnects); 2115 2116 ctrl->nr_reconnects = 0; 2117 2118 return; 2119 2120 requeue: 2121 dev_info(ctrl->device, "Failed reconnect attempt %d\n", 2122 ctrl->nr_reconnects); 2123 nvme_tcp_reconnect_or_remove(ctrl); 2124 } 2125 2126 static void nvme_tcp_error_recovery_work(struct work_struct *work) 2127 { 2128 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2129 struct nvme_tcp_ctrl, err_work); 2130 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2131 2132 nvme_stop_keep_alive(ctrl); 2133 flush_work(&ctrl->async_event_work); 2134 nvme_tcp_teardown_io_queues(ctrl, false); 2135 /* unquiesce to fail fast pending requests */ 2136 nvme_unquiesce_io_queues(ctrl); 2137 nvme_tcp_teardown_admin_queue(ctrl, false); 2138 nvme_unquiesce_admin_queue(ctrl); 2139 nvme_auth_stop(ctrl); 2140 2141 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2142 /* state change failure is ok if we started ctrl delete */ 2143 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2144 ctrl->state != NVME_CTRL_DELETING_NOIO); 2145 return; 2146 } 2147 2148 nvme_tcp_reconnect_or_remove(ctrl); 2149 } 2150 2151 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2152 { 2153 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2154 nvme_quiesce_admin_queue(ctrl); 2155 nvme_disable_ctrl(ctrl, shutdown); 2156 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2157 } 2158 2159 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2160 { 2161 nvme_tcp_teardown_ctrl(ctrl, true); 2162 } 2163 2164 static void nvme_reset_ctrl_work(struct work_struct *work) 2165 { 2166 struct nvme_ctrl *ctrl = 2167 container_of(work, struct nvme_ctrl, reset_work); 2168 2169 nvme_stop_ctrl(ctrl); 2170 nvme_tcp_teardown_ctrl(ctrl, false); 2171 2172 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2173 /* state change failure is ok if we started ctrl delete */ 2174 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2175 ctrl->state != NVME_CTRL_DELETING_NOIO); 2176 return; 2177 } 2178 2179 if (nvme_tcp_setup_ctrl(ctrl, false)) 2180 goto out_fail; 2181 2182 return; 2183 2184 out_fail: 2185 ++ctrl->nr_reconnects; 2186 nvme_tcp_reconnect_or_remove(ctrl); 2187 } 2188 2189 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2190 { 2191 flush_work(&to_tcp_ctrl(ctrl)->err_work); 2192 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2193 } 2194 2195 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2196 { 2197 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2198 2199 if (list_empty(&ctrl->list)) 2200 goto free_ctrl; 2201 2202 mutex_lock(&nvme_tcp_ctrl_mutex); 2203 list_del(&ctrl->list); 2204 mutex_unlock(&nvme_tcp_ctrl_mutex); 2205 2206 nvmf_free_options(nctrl->opts); 2207 free_ctrl: 2208 kfree(ctrl->queues); 2209 kfree(ctrl); 2210 } 2211 2212 static void nvme_tcp_set_sg_null(struct nvme_command *c) 2213 { 2214 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2215 2216 sg->addr = 0; 2217 sg->length = 0; 2218 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2219 NVME_SGL_FMT_TRANSPORT_A; 2220 } 2221 2222 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2223 struct nvme_command *c, u32 data_len) 2224 { 2225 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2226 2227 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2228 sg->length = cpu_to_le32(data_len); 2229 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2230 } 2231 2232 static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2233 u32 data_len) 2234 { 2235 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2236 2237 sg->addr = 0; 2238 sg->length = cpu_to_le32(data_len); 2239 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2240 NVME_SGL_FMT_TRANSPORT_A; 2241 } 2242 2243 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2244 { 2245 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2246 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2247 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2248 struct nvme_command *cmd = &pdu->cmd; 2249 u8 hdgst = nvme_tcp_hdgst_len(queue); 2250 2251 memset(pdu, 0, sizeof(*pdu)); 2252 pdu->hdr.type = nvme_tcp_cmd; 2253 if (queue->hdr_digest) 2254 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2255 pdu->hdr.hlen = sizeof(*pdu); 2256 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2257 2258 cmd->common.opcode = nvme_admin_async_event; 2259 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2260 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2261 nvme_tcp_set_sg_null(cmd); 2262 2263 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2264 ctrl->async_req.offset = 0; 2265 ctrl->async_req.curr_bio = NULL; 2266 ctrl->async_req.data_len = 0; 2267 2268 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2269 } 2270 2271 static void nvme_tcp_complete_timed_out(struct request *rq) 2272 { 2273 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2274 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2275 2276 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2277 nvmf_complete_timed_out_request(rq); 2278 } 2279 2280 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) 2281 { 2282 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2283 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2284 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2285 u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype; 2286 int qid = nvme_tcp_queue_id(req->queue); 2287 2288 dev_warn(ctrl->device, 2289 "queue %d: timeout cid %#x type %d opcode %#x (%s)\n", 2290 nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type, 2291 opc, nvme_opcode_str(qid, opc, fctype)); 2292 2293 if (ctrl->state != NVME_CTRL_LIVE) { 2294 /* 2295 * If we are resetting, connecting or deleting we should 2296 * complete immediately because we may block controller 2297 * teardown or setup sequence 2298 * - ctrl disable/shutdown fabrics requests 2299 * - connect requests 2300 * - initialization admin requests 2301 * - I/O requests that entered after unquiescing and 2302 * the controller stopped responding 2303 * 2304 * All other requests should be cancelled by the error 2305 * recovery work, so it's fine that we fail it here. 2306 */ 2307 nvme_tcp_complete_timed_out(rq); 2308 return BLK_EH_DONE; 2309 } 2310 2311 /* 2312 * LIVE state should trigger the normal error recovery which will 2313 * handle completing this request. 2314 */ 2315 nvme_tcp_error_recovery(ctrl); 2316 return BLK_EH_RESET_TIMER; 2317 } 2318 2319 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2320 struct request *rq) 2321 { 2322 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2323 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2324 struct nvme_command *c = &pdu->cmd; 2325 2326 c->common.flags |= NVME_CMD_SGL_METABUF; 2327 2328 if (!blk_rq_nr_phys_segments(rq)) 2329 nvme_tcp_set_sg_null(c); 2330 else if (rq_data_dir(rq) == WRITE && 2331 req->data_len <= nvme_tcp_inline_data_size(req)) 2332 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2333 else 2334 nvme_tcp_set_sg_host_data(c, req->data_len); 2335 2336 return 0; 2337 } 2338 2339 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2340 struct request *rq) 2341 { 2342 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2343 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2344 struct nvme_tcp_queue *queue = req->queue; 2345 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2346 blk_status_t ret; 2347 2348 ret = nvme_setup_cmd(ns, rq); 2349 if (ret) 2350 return ret; 2351 2352 req->state = NVME_TCP_SEND_CMD_PDU; 2353 req->status = cpu_to_le16(NVME_SC_SUCCESS); 2354 req->offset = 0; 2355 req->data_sent = 0; 2356 req->pdu_len = 0; 2357 req->pdu_sent = 0; 2358 req->h2cdata_left = 0; 2359 req->data_len = blk_rq_nr_phys_segments(rq) ? 2360 blk_rq_payload_bytes(rq) : 0; 2361 req->curr_bio = rq->bio; 2362 if (req->curr_bio && req->data_len) 2363 nvme_tcp_init_iter(req, rq_data_dir(rq)); 2364 2365 if (rq_data_dir(rq) == WRITE && 2366 req->data_len <= nvme_tcp_inline_data_size(req)) 2367 req->pdu_len = req->data_len; 2368 2369 pdu->hdr.type = nvme_tcp_cmd; 2370 pdu->hdr.flags = 0; 2371 if (queue->hdr_digest) 2372 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2373 if (queue->data_digest && req->pdu_len) { 2374 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2375 ddgst = nvme_tcp_ddgst_len(queue); 2376 } 2377 pdu->hdr.hlen = sizeof(*pdu); 2378 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2379 pdu->hdr.plen = 2380 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2381 2382 ret = nvme_tcp_map_data(queue, rq); 2383 if (unlikely(ret)) { 2384 nvme_cleanup_cmd(rq); 2385 dev_err(queue->ctrl->ctrl.device, 2386 "Failed to map data (%d)\n", ret); 2387 return ret; 2388 } 2389 2390 return 0; 2391 } 2392 2393 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2394 { 2395 struct nvme_tcp_queue *queue = hctx->driver_data; 2396 2397 if (!llist_empty(&queue->req_list)) 2398 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2399 } 2400 2401 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2402 const struct blk_mq_queue_data *bd) 2403 { 2404 struct nvme_ns *ns = hctx->queue->queuedata; 2405 struct nvme_tcp_queue *queue = hctx->driver_data; 2406 struct request *rq = bd->rq; 2407 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2408 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2409 blk_status_t ret; 2410 2411 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2412 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2413 2414 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2415 if (unlikely(ret)) 2416 return ret; 2417 2418 nvme_start_request(rq); 2419 2420 nvme_tcp_queue_request(req, true, bd->last); 2421 2422 return BLK_STS_OK; 2423 } 2424 2425 static void nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2426 { 2427 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 2428 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2429 2430 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2431 /* separate read/write queues */ 2432 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2433 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2434 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2435 set->map[HCTX_TYPE_READ].nr_queues = 2436 ctrl->io_queues[HCTX_TYPE_READ]; 2437 set->map[HCTX_TYPE_READ].queue_offset = 2438 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2439 } else { 2440 /* shared read/write queues */ 2441 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2442 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2443 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2444 set->map[HCTX_TYPE_READ].nr_queues = 2445 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2446 set->map[HCTX_TYPE_READ].queue_offset = 0; 2447 } 2448 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2449 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 2450 2451 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 2452 /* map dedicated poll queues only if we have queues left */ 2453 set->map[HCTX_TYPE_POLL].nr_queues = 2454 ctrl->io_queues[HCTX_TYPE_POLL]; 2455 set->map[HCTX_TYPE_POLL].queue_offset = 2456 ctrl->io_queues[HCTX_TYPE_DEFAULT] + 2457 ctrl->io_queues[HCTX_TYPE_READ]; 2458 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 2459 } 2460 2461 dev_info(ctrl->ctrl.device, 2462 "mapped %d/%d/%d default/read/poll queues.\n", 2463 ctrl->io_queues[HCTX_TYPE_DEFAULT], 2464 ctrl->io_queues[HCTX_TYPE_READ], 2465 ctrl->io_queues[HCTX_TYPE_POLL]); 2466 } 2467 2468 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 2469 { 2470 struct nvme_tcp_queue *queue = hctx->driver_data; 2471 struct sock *sk = queue->sock->sk; 2472 2473 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2474 return 0; 2475 2476 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2477 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2478 sk_busy_loop(sk, true); 2479 nvme_tcp_try_recv(queue); 2480 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2481 return queue->nr_cqe; 2482 } 2483 2484 static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) 2485 { 2486 struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0]; 2487 struct sockaddr_storage src_addr; 2488 int ret, len; 2489 2490 len = nvmf_get_address(ctrl, buf, size); 2491 2492 ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr); 2493 if (ret > 0) { 2494 if (len > 0) 2495 len--; /* strip trailing newline */ 2496 len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n", 2497 (len) ? "," : "", &src_addr); 2498 } 2499 2500 return len; 2501 } 2502 2503 static const struct blk_mq_ops nvme_tcp_mq_ops = { 2504 .queue_rq = nvme_tcp_queue_rq, 2505 .commit_rqs = nvme_tcp_commit_rqs, 2506 .complete = nvme_complete_rq, 2507 .init_request = nvme_tcp_init_request, 2508 .exit_request = nvme_tcp_exit_request, 2509 .init_hctx = nvme_tcp_init_hctx, 2510 .timeout = nvme_tcp_timeout, 2511 .map_queues = nvme_tcp_map_queues, 2512 .poll = nvme_tcp_poll, 2513 }; 2514 2515 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2516 .queue_rq = nvme_tcp_queue_rq, 2517 .complete = nvme_complete_rq, 2518 .init_request = nvme_tcp_init_request, 2519 .exit_request = nvme_tcp_exit_request, 2520 .init_hctx = nvme_tcp_init_admin_hctx, 2521 .timeout = nvme_tcp_timeout, 2522 }; 2523 2524 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2525 .name = "tcp", 2526 .module = THIS_MODULE, 2527 .flags = NVME_F_FABRICS | NVME_F_BLOCKING, 2528 .reg_read32 = nvmf_reg_read32, 2529 .reg_read64 = nvmf_reg_read64, 2530 .reg_write32 = nvmf_reg_write32, 2531 .free_ctrl = nvme_tcp_free_ctrl, 2532 .submit_async_event = nvme_tcp_submit_async_event, 2533 .delete_ctrl = nvme_tcp_delete_ctrl, 2534 .get_address = nvme_tcp_get_address, 2535 .stop_ctrl = nvme_tcp_stop_ctrl, 2536 }; 2537 2538 static bool 2539 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2540 { 2541 struct nvme_tcp_ctrl *ctrl; 2542 bool found = false; 2543 2544 mutex_lock(&nvme_tcp_ctrl_mutex); 2545 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2546 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2547 if (found) 2548 break; 2549 } 2550 mutex_unlock(&nvme_tcp_ctrl_mutex); 2551 2552 return found; 2553 } 2554 2555 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2556 struct nvmf_ctrl_options *opts) 2557 { 2558 struct nvme_tcp_ctrl *ctrl; 2559 int ret; 2560 2561 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2562 if (!ctrl) 2563 return ERR_PTR(-ENOMEM); 2564 2565 INIT_LIST_HEAD(&ctrl->list); 2566 ctrl->ctrl.opts = opts; 2567 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2568 opts->nr_poll_queues + 1; 2569 ctrl->ctrl.sqsize = opts->queue_size - 1; 2570 ctrl->ctrl.kato = opts->kato; 2571 2572 INIT_DELAYED_WORK(&ctrl->connect_work, 2573 nvme_tcp_reconnect_ctrl_work); 2574 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2575 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2576 2577 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2578 opts->trsvcid = 2579 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2580 if (!opts->trsvcid) { 2581 ret = -ENOMEM; 2582 goto out_free_ctrl; 2583 } 2584 opts->mask |= NVMF_OPT_TRSVCID; 2585 } 2586 2587 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2588 opts->traddr, opts->trsvcid, &ctrl->addr); 2589 if (ret) { 2590 pr_err("malformed address passed: %s:%s\n", 2591 opts->traddr, opts->trsvcid); 2592 goto out_free_ctrl; 2593 } 2594 2595 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2596 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2597 opts->host_traddr, NULL, &ctrl->src_addr); 2598 if (ret) { 2599 pr_err("malformed src address passed: %s\n", 2600 opts->host_traddr); 2601 goto out_free_ctrl; 2602 } 2603 } 2604 2605 if (opts->mask & NVMF_OPT_HOST_IFACE) { 2606 if (!__dev_get_by_name(&init_net, opts->host_iface)) { 2607 pr_err("invalid interface passed: %s\n", 2608 opts->host_iface); 2609 ret = -ENODEV; 2610 goto out_free_ctrl; 2611 } 2612 } 2613 2614 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2615 ret = -EALREADY; 2616 goto out_free_ctrl; 2617 } 2618 2619 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2620 GFP_KERNEL); 2621 if (!ctrl->queues) { 2622 ret = -ENOMEM; 2623 goto out_free_ctrl; 2624 } 2625 2626 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2627 if (ret) 2628 goto out_kfree_queues; 2629 2630 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2631 WARN_ON_ONCE(1); 2632 ret = -EINTR; 2633 goto out_uninit_ctrl; 2634 } 2635 2636 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2637 if (ret) 2638 goto out_uninit_ctrl; 2639 2640 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2641 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); 2642 2643 mutex_lock(&nvme_tcp_ctrl_mutex); 2644 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2645 mutex_unlock(&nvme_tcp_ctrl_mutex); 2646 2647 return &ctrl->ctrl; 2648 2649 out_uninit_ctrl: 2650 nvme_uninit_ctrl(&ctrl->ctrl); 2651 nvme_put_ctrl(&ctrl->ctrl); 2652 if (ret > 0) 2653 ret = -EIO; 2654 return ERR_PTR(ret); 2655 out_kfree_queues: 2656 kfree(ctrl->queues); 2657 out_free_ctrl: 2658 kfree(ctrl); 2659 return ERR_PTR(ret); 2660 } 2661 2662 static struct nvmf_transport_ops nvme_tcp_transport = { 2663 .name = "tcp", 2664 .module = THIS_MODULE, 2665 .required_opts = NVMF_OPT_TRADDR, 2666 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 2667 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2668 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2669 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2670 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, 2671 .create_ctrl = nvme_tcp_create_ctrl, 2672 }; 2673 2674 static int __init nvme_tcp_init_module(void) 2675 { 2676 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 2677 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 2678 if (!nvme_tcp_wq) 2679 return -ENOMEM; 2680 2681 nvmf_register_transport(&nvme_tcp_transport); 2682 return 0; 2683 } 2684 2685 static void __exit nvme_tcp_cleanup_module(void) 2686 { 2687 struct nvme_tcp_ctrl *ctrl; 2688 2689 nvmf_unregister_transport(&nvme_tcp_transport); 2690 2691 mutex_lock(&nvme_tcp_ctrl_mutex); 2692 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 2693 nvme_delete_ctrl(&ctrl->ctrl); 2694 mutex_unlock(&nvme_tcp_ctrl_mutex); 2695 flush_workqueue(nvme_delete_wq); 2696 2697 destroy_workqueue(nvme_tcp_wq); 2698 } 2699 2700 module_init(nvme_tcp_init_module); 2701 module_exit(nvme_tcp_cleanup_module); 2702 2703 MODULE_LICENSE("GPL v2"); 2704