1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2016 Chelsio Communications, Inc.
4 */
5
6 #include <linux/workqueue.h>
7 #include <linux/kthread.h>
8 #include <linux/sched/signal.h>
9
10 #include <asm/unaligned.h>
11 #include <net/tcp.h>
12 #include <target/target_core_base.h>
13 #include <target/target_core_fabric.h>
14 #include "cxgbit.h"
15
16 struct sge_opaque_hdr {
17 void *dev;
18 dma_addr_t addr[MAX_SKB_FRAGS + 1];
19 };
20
21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
22
23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
24 sizeof(struct fw_ofld_tx_data_wr))
25
26 static struct sk_buff *
__cxgbit_alloc_skb(struct cxgbit_sock * csk,u32 len,bool iso)27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
28 {
29 struct sk_buff *skb = NULL;
30 u8 submode = 0;
31 int errcode;
32 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
33
34 if (len) {
35 skb = alloc_skb_with_frags(hdr_len, len,
36 0, &errcode,
37 GFP_KERNEL);
38 if (!skb)
39 return NULL;
40
41 skb_reserve(skb, TX_HDR_LEN);
42 skb_reset_transport_header(skb);
43 __skb_put(skb, ISCSI_HDR_LEN);
44 skb->data_len = len;
45 skb->len += len;
46 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
47
48 } else {
49 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
50
51 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
52 if (!skb)
53 return NULL;
54
55 skb_reserve(skb, TX_HDR_LEN + iso_len);
56 skb_reset_transport_header(skb);
57 __skb_put(skb, ISCSI_HDR_LEN);
58 }
59
60 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
61 cxgbit_skcb_submode(skb) = submode;
62 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
63 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
64 return skb;
65 }
66
cxgbit_alloc_skb(struct cxgbit_sock * csk,u32 len)67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
68 {
69 return __cxgbit_alloc_skb(csk, len, false);
70 }
71
72 /*
73 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
74 * @skb: the packet
75 *
76 * Returns true if a packet can be sent as an offload WR with immediate
77 * data. We currently use the same limit as for Ethernet packets.
78 */
cxgbit_is_ofld_imm(const struct sk_buff * skb)79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
80 {
81 int length = skb->len;
82
83 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
84 length += sizeof(struct fw_ofld_tx_data_wr);
85
86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
87 length += sizeof(struct cpl_tx_data_iso);
88
89 return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
90 }
91
92 /*
93 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
94 * @n: the number of SGL entries
95 * Calculates the number of flits needed for a scatter/gather list that
96 * can hold the given number of entries.
97 */
cxgbit_sgl_len(unsigned int n)98 static inline unsigned int cxgbit_sgl_len(unsigned int n)
99 {
100 n--;
101 return (3 * n) / 2 + (n & 1) + 2;
102 }
103
104 /*
105 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
106 * @skb: the packet
107 *
108 * Returns the number of flits needed for the given offload packet.
109 * These packets are already fully constructed and no additional headers
110 * will be added.
111 */
cxgbit_calc_tx_flits_ofld(const struct sk_buff * skb)112 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
113 {
114 unsigned int flits, cnt;
115
116 if (cxgbit_is_ofld_imm(skb))
117 return DIV_ROUND_UP(skb->len, 8);
118 flits = skb_transport_offset(skb) / 8;
119 cnt = skb_shinfo(skb)->nr_frags;
120 if (skb_tail_pointer(skb) != skb_transport_header(skb))
121 cnt++;
122 return flits + cxgbit_sgl_len(cnt);
123 }
124
125 #define CXGBIT_ISO_FSLICE 0x1
126 #define CXGBIT_ISO_LSLICE 0x2
127 static void
cxgbit_cpl_tx_data_iso(struct sk_buff * skb,struct cxgbit_iso_info * iso_info)128 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
129 {
130 struct cpl_tx_data_iso *cpl;
131 unsigned int submode = cxgbit_skcb_submode(skb);
132 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
133 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
134
135 cpl = __skb_push(skb, sizeof(*cpl));
136
137 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
138 CPL_TX_DATA_ISO_FIRST_V(fslice) |
139 CPL_TX_DATA_ISO_LAST_V(lslice) |
140 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
141 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
142 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
143 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
144 CPL_TX_DATA_ISO_SCSI_V(2));
145
146 cpl->ahs_len = 0;
147 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
148 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
149 cpl->len = htonl(iso_info->len);
150 cpl->reserved2_seglen_offset = htonl(0);
151 cpl->datasn_offset = htonl(0);
152 cpl->buffer_offset = htonl(0);
153 cpl->reserved3 = 0;
154
155 __skb_pull(skb, sizeof(*cpl));
156 }
157
158 static void
cxgbit_tx_data_wr(struct cxgbit_sock * csk,struct sk_buff * skb,u32 dlen,u32 len,u32 credits,u32 compl)159 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
160 u32 len, u32 credits, u32 compl)
161 {
162 struct fw_ofld_tx_data_wr *req;
163 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
164 u32 submode = cxgbit_skcb_submode(skb);
165 u32 wr_ulp_mode = 0;
166 u32 hdr_size = sizeof(*req);
167 u32 opcode = FW_OFLD_TX_DATA_WR;
168 u32 immlen = 0;
169 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
170 T6_TX_FORCE_F;
171
172 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
173 opcode = FW_ISCSI_TX_DATA_WR;
174 immlen += sizeof(struct cpl_tx_data_iso);
175 hdr_size += sizeof(struct cpl_tx_data_iso);
176 submode |= 8;
177 }
178
179 if (cxgbit_is_ofld_imm(skb))
180 immlen += dlen;
181
182 req = __skb_push(skb, hdr_size);
183 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
184 FW_WR_COMPL_V(compl) |
185 FW_WR_IMMDLEN_V(immlen));
186 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
187 FW_WR_LEN16_V(credits));
188 req->plen = htonl(len);
189 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
190 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
191
192 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
193 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
194 }
195
cxgbit_arp_failure_skb_discard(void * handle,struct sk_buff * skb)196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
197 {
198 kfree_skb(skb);
199 }
200
cxgbit_push_tx_frames(struct cxgbit_sock * csk)201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
202 {
203 struct sk_buff *skb;
204
205 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
206 u32 dlen = skb->len;
207 u32 len = skb->len;
208 u32 credits_needed;
209 u32 compl = 0;
210 u32 flowclen16 = 0;
211 u32 iso_cpl_len = 0;
212
213 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
214 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
215
216 if (cxgbit_is_ofld_imm(skb))
217 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
218 else
219 credits_needed = DIV_ROUND_UP((8 *
220 cxgbit_calc_tx_flits_ofld(skb)) +
221 iso_cpl_len, 16);
222
223 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
224 credits_needed += DIV_ROUND_UP(
225 sizeof(struct fw_ofld_tx_data_wr), 16);
226 /*
227 * Assumes the initial credits is large enough to support
228 * fw_flowc_wr plus largest possible first payload
229 */
230
231 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
232 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
233 csk->wr_cred -= flowclen16;
234 csk->wr_una_cred += flowclen16;
235 }
236
237 if (csk->wr_cred < credits_needed) {
238 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
239 csk, skb->len, skb->data_len,
240 credits_needed, csk->wr_cred);
241 break;
242 }
243 __skb_unlink(skb, &csk->txq);
244 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
245 skb->csum = (__force __wsum)(credits_needed + flowclen16);
246 csk->wr_cred -= credits_needed;
247 csk->wr_una_cred += credits_needed;
248
249 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
250 csk, skb->len, skb->data_len, credits_needed,
251 csk->wr_cred, csk->wr_una_cred);
252
253 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
254 len += cxgbit_skcb_tx_extralen(skb);
255
256 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
257 (!before(csk->write_seq,
258 csk->snd_una + csk->snd_win))) {
259 compl = 1;
260 csk->wr_una_cred = 0;
261 }
262
263 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
264 compl);
265 csk->snd_nxt += len;
266
267 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
268 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
269 struct cpl_close_con_req *req =
270 (struct cpl_close_con_req *)skb->data;
271 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
272 csk->wr_una_cred = 0;
273 }
274
275 cxgbit_sock_enqueue_wr(csk, skb);
276 t4_set_arp_err_handler(skb, csk,
277 cxgbit_arp_failure_skb_discard);
278
279 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
280 csk, csk->tid, skb, len);
281
282 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
283 }
284 }
285
cxgbit_unlock_sock(struct cxgbit_sock * csk)286 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
287 {
288 struct sk_buff_head backlogq;
289 struct sk_buff *skb;
290 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
291
292 skb_queue_head_init(&backlogq);
293
294 spin_lock_bh(&csk->lock);
295 while (skb_queue_len(&csk->backlogq)) {
296 skb_queue_splice_init(&csk->backlogq, &backlogq);
297 spin_unlock_bh(&csk->lock);
298
299 while ((skb = __skb_dequeue(&backlogq))) {
300 fn = cxgbit_skcb_rx_backlog_fn(skb);
301 fn(csk, skb);
302 }
303
304 spin_lock_bh(&csk->lock);
305 }
306
307 csk->lock_owner = false;
308 spin_unlock_bh(&csk->lock);
309 }
310
cxgbit_queue_skb(struct cxgbit_sock * csk,struct sk_buff * skb)311 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
312 {
313 int ret = 0;
314
315 spin_lock_bh(&csk->lock);
316 csk->lock_owner = true;
317 spin_unlock_bh(&csk->lock);
318
319 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
320 signal_pending(current))) {
321 __kfree_skb(skb);
322 __skb_queue_purge(&csk->ppodq);
323 ret = -1;
324 goto unlock;
325 }
326
327 csk->write_seq += skb->len +
328 cxgbit_skcb_tx_extralen(skb);
329
330 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
331 __skb_queue_tail(&csk->txq, skb);
332 cxgbit_push_tx_frames(csk);
333
334 unlock:
335 cxgbit_unlock_sock(csk);
336 return ret;
337 }
338
339 static int
cxgbit_map_skb(struct iscsi_cmd * cmd,struct sk_buff * skb,u32 data_offset,u32 data_length)340 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
341 u32 data_length)
342 {
343 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
344 u32 padding = ((-data_length) & 3);
345 struct scatterlist *sg;
346 struct page *page;
347 unsigned int page_off;
348
349 if (padding)
350 nr_frags--;
351
352 /*
353 * We know each entry in t_data_sg contains a page.
354 */
355 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
356 page_off = (data_offset % PAGE_SIZE);
357
358 while (data_length && (i < nr_frags)) {
359 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
360
361 page = sg_page(sg);
362
363 get_page(page);
364 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
365 cur_len);
366 skb->data_len += cur_len;
367 skb->len += cur_len;
368 skb->truesize += cur_len;
369
370 data_length -= cur_len;
371 page_off = 0;
372 sg = sg_next(sg);
373 i++;
374 }
375
376 if (data_length)
377 return -1;
378
379 if (padding) {
380 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
381 if (!page)
382 return -1;
383 skb_fill_page_desc(skb, i, page, 0, padding);
384 skb->data_len += padding;
385 skb->len += padding;
386 skb->truesize += padding;
387 }
388
389 return 0;
390 }
391
392 static int
cxgbit_tx_datain_iso(struct cxgbit_sock * csk,struct iscsi_cmd * cmd,struct iscsi_datain_req * dr)393 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
394 struct iscsi_datain_req *dr)
395 {
396 struct iscsi_conn *conn = csk->conn;
397 struct sk_buff *skb;
398 struct iscsi_datain datain;
399 struct cxgbit_iso_info iso_info;
400 u32 data_length = cmd->se_cmd.data_length;
401 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
402 u32 num_pdu, plen, tx_data = 0;
403 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
404 SCF_TRANSPORT_TASK_SENSE);
405 bool set_statsn = false;
406 int ret = -1;
407
408 while (data_length) {
409 num_pdu = (data_length + mrdsl - 1) / mrdsl;
410 if (num_pdu > csk->max_iso_npdu)
411 num_pdu = csk->max_iso_npdu;
412
413 plen = num_pdu * mrdsl;
414 if (plen > data_length)
415 plen = data_length;
416
417 skb = __cxgbit_alloc_skb(csk, 0, true);
418 if (unlikely(!skb))
419 return -ENOMEM;
420
421 memset(skb->data, 0, ISCSI_HDR_LEN);
422 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
423 cxgbit_skcb_submode(skb) |= (csk->submode &
424 CXGBIT_SUBMODE_DCRC);
425 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
426 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
427 ((num_pdu - 1) * ISCSI_HDR_LEN);
428
429 memset(&datain, 0, sizeof(struct iscsi_datain));
430 memset(&iso_info, 0, sizeof(iso_info));
431
432 if (!tx_data)
433 iso_info.flags |= CXGBIT_ISO_FSLICE;
434
435 if (!(data_length - plen)) {
436 iso_info.flags |= CXGBIT_ISO_LSLICE;
437 if (!task_sense) {
438 datain.flags = ISCSI_FLAG_DATA_STATUS;
439 iscsit_increment_maxcmdsn(cmd, conn->sess);
440 cmd->stat_sn = conn->stat_sn++;
441 set_statsn = true;
442 }
443 }
444
445 iso_info.burst_len = num_pdu * mrdsl;
446 iso_info.mpdu = mrdsl;
447 iso_info.len = ISCSI_HDR_LEN + plen;
448
449 cxgbit_cpl_tx_data_iso(skb, &iso_info);
450
451 datain.offset = tx_data;
452 datain.data_sn = cmd->data_sn - 1;
453
454 iscsit_build_datain_pdu(cmd, conn, &datain,
455 (struct iscsi_data_rsp *)skb->data,
456 set_statsn);
457
458 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
459 if (unlikely(ret)) {
460 __kfree_skb(skb);
461 goto out;
462 }
463
464 ret = cxgbit_queue_skb(csk, skb);
465 if (unlikely(ret))
466 goto out;
467
468 tx_data += plen;
469 data_length -= plen;
470
471 cmd->read_data_done += plen;
472 cmd->data_sn += num_pdu;
473 }
474
475 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
476
477 return 0;
478
479 out:
480 return ret;
481 }
482
483 static int
cxgbit_tx_datain(struct cxgbit_sock * csk,struct iscsi_cmd * cmd,const struct iscsi_datain * datain)484 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
485 const struct iscsi_datain *datain)
486 {
487 struct sk_buff *skb;
488 int ret = 0;
489
490 skb = cxgbit_alloc_skb(csk, 0);
491 if (unlikely(!skb))
492 return -ENOMEM;
493
494 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
495
496 if (datain->length) {
497 cxgbit_skcb_submode(skb) |= (csk->submode &
498 CXGBIT_SUBMODE_DCRC);
499 cxgbit_skcb_tx_extralen(skb) =
500 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
501 }
502
503 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
504 if (ret < 0) {
505 __kfree_skb(skb);
506 return ret;
507 }
508
509 return cxgbit_queue_skb(csk, skb);
510 }
511
512 static int
cxgbit_xmit_datain_pdu(struct iscsi_conn * conn,struct iscsi_cmd * cmd,struct iscsi_datain_req * dr,const struct iscsi_datain * datain)513 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
514 struct iscsi_datain_req *dr,
515 const struct iscsi_datain *datain)
516 {
517 struct cxgbit_sock *csk = conn->context;
518 u32 data_length = cmd->se_cmd.data_length;
519 u32 padding = ((-data_length) & 3);
520 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
521
522 if ((data_length > mrdsl) && (!dr->recovery) &&
523 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
524 atomic_long_add(data_length - datain->length,
525 &conn->sess->tx_data_octets);
526 return cxgbit_tx_datain_iso(csk, cmd, dr);
527 }
528
529 return cxgbit_tx_datain(csk, cmd, datain);
530 }
531
532 static int
cxgbit_xmit_nondatain_pdu(struct iscsi_conn * conn,struct iscsi_cmd * cmd,const void * data_buf,u32 data_buf_len)533 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
534 const void *data_buf, u32 data_buf_len)
535 {
536 struct cxgbit_sock *csk = conn->context;
537 struct sk_buff *skb;
538 u32 padding = ((-data_buf_len) & 3);
539
540 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
541 if (unlikely(!skb))
542 return -ENOMEM;
543
544 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
545
546 if (data_buf_len) {
547 u32 pad_bytes = 0;
548
549 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
550
551 if (padding)
552 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
553 &pad_bytes, padding);
554 }
555
556 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
557 cxgbit_skcb_submode(skb)];
558
559 return cxgbit_queue_skb(csk, skb);
560 }
561
562 int
cxgbit_xmit_pdu(struct iscsi_conn * conn,struct iscsi_cmd * cmd,struct iscsi_datain_req * dr,const void * buf,u32 buf_len)563 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
564 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
565 {
566 if (dr)
567 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
568 else
569 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
570 }
571
cxgbit_validate_params(struct iscsi_conn * conn)572 int cxgbit_validate_params(struct iscsi_conn *conn)
573 {
574 struct cxgbit_sock *csk = conn->context;
575 struct cxgbit_device *cdev = csk->com.cdev;
576 struct iscsi_param *param;
577 u32 max_xmitdsl;
578
579 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
580 conn->param_list);
581 if (!param)
582 return -1;
583
584 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
585 return -1;
586
587 if (max_xmitdsl > cdev->mdsl) {
588 if (iscsi_change_param_sprintf(
589 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
590 return -1;
591 }
592
593 return 0;
594 }
595
cxgbit_set_digest(struct cxgbit_sock * csk)596 static int cxgbit_set_digest(struct cxgbit_sock *csk)
597 {
598 struct iscsi_conn *conn = csk->conn;
599 struct iscsi_param *param;
600
601 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
602 if (!param) {
603 pr_err("param not found key %s\n", HEADERDIGEST);
604 return -1;
605 }
606
607 if (!strcmp(param->value, CRC32C))
608 csk->submode |= CXGBIT_SUBMODE_HCRC;
609
610 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
611 if (!param) {
612 csk->submode = 0;
613 pr_err("param not found key %s\n", DATADIGEST);
614 return -1;
615 }
616
617 if (!strcmp(param->value, CRC32C))
618 csk->submode |= CXGBIT_SUBMODE_DCRC;
619
620 if (cxgbit_setup_conn_digest(csk)) {
621 csk->submode = 0;
622 return -1;
623 }
624
625 return 0;
626 }
627
cxgbit_set_iso_npdu(struct cxgbit_sock * csk)628 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
629 {
630 struct iscsi_conn *conn = csk->conn;
631 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
632 struct iscsi_param *param;
633 u32 mrdsl, mbl;
634 u32 max_npdu, max_iso_npdu;
635 u32 max_iso_payload;
636
637 if (conn->login->leading_connection) {
638 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
639 conn->param_list);
640 if (!param) {
641 pr_err("param not found key %s\n", MAXBURSTLENGTH);
642 return -1;
643 }
644
645 if (kstrtou32(param->value, 0, &mbl) < 0)
646 return -1;
647 } else {
648 mbl = conn->sess->sess_ops->MaxBurstLength;
649 }
650
651 mrdsl = conn_ops->MaxRecvDataSegmentLength;
652 max_npdu = mbl / mrdsl;
653
654 max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
655
656 max_iso_npdu = max_iso_payload /
657 (ISCSI_HDR_LEN + mrdsl +
658 cxgbit_digest_len[csk->submode]);
659
660 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
661
662 if (csk->max_iso_npdu <= 1)
663 csk->max_iso_npdu = 0;
664
665 return 0;
666 }
667
668 /*
669 * cxgbit_seq_pdu_inorder()
670 * @csk: pointer to cxgbit socket structure
671 *
672 * This function checks whether data sequence and data
673 * pdu are in order.
674 *
675 * Return: returns -1 on error, 0 if data sequence and
676 * data pdu are in order, 1 if data sequence or data pdu
677 * is not in order.
678 */
cxgbit_seq_pdu_inorder(struct cxgbit_sock * csk)679 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
680 {
681 struct iscsi_conn *conn = csk->conn;
682 struct iscsi_param *param;
683
684 if (conn->login->leading_connection) {
685 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
686 conn->param_list);
687 if (!param) {
688 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
689 return -1;
690 }
691
692 if (strcmp(param->value, YES))
693 return 1;
694
695 param = iscsi_find_param_from_key(DATAPDUINORDER,
696 conn->param_list);
697 if (!param) {
698 pr_err("param not found key %s\n", DATAPDUINORDER);
699 return -1;
700 }
701
702 if (strcmp(param->value, YES))
703 return 1;
704
705 } else {
706 if (!conn->sess->sess_ops->DataSequenceInOrder)
707 return 1;
708 if (!conn->sess->sess_ops->DataPDUInOrder)
709 return 1;
710 }
711
712 return 0;
713 }
714
cxgbit_set_params(struct iscsi_conn * conn)715 static int cxgbit_set_params(struct iscsi_conn *conn)
716 {
717 struct cxgbit_sock *csk = conn->context;
718 struct cxgbit_device *cdev = csk->com.cdev;
719 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
720 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
721 struct iscsi_param *param;
722 u8 erl;
723
724 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
725 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
726
727 if (cxgbit_set_digest(csk))
728 return -1;
729
730 if (conn->login->leading_connection) {
731 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
732 conn->param_list);
733 if (!param) {
734 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
735 return -1;
736 }
737 if (kstrtou8(param->value, 0, &erl) < 0)
738 return -1;
739 } else {
740 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
741 }
742
743 if (!erl) {
744 int ret;
745
746 ret = cxgbit_seq_pdu_inorder(csk);
747 if (ret < 0) {
748 return -1;
749 } else if (ret > 0) {
750 if (is_t5(cdev->lldi.adapter_type))
751 goto enable_ddp;
752 else
753 return 0;
754 }
755
756 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
757 if (cxgbit_set_iso_npdu(csk))
758 return -1;
759 }
760
761 enable_ddp:
762 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
763 if (cxgbit_setup_conn_pgidx(csk,
764 ppm->tformat.pgsz_idx_dflt))
765 return -1;
766 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
767 }
768 }
769
770 return 0;
771 }
772
773 int
cxgbit_put_login_tx(struct iscsi_conn * conn,struct iscsi_login * login,u32 length)774 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
775 u32 length)
776 {
777 struct cxgbit_sock *csk = conn->context;
778 struct sk_buff *skb;
779 u32 padding_buf = 0;
780 u8 padding = ((-length) & 3);
781
782 skb = cxgbit_alloc_skb(csk, length + padding);
783 if (!skb)
784 return -ENOMEM;
785 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
786 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
787
788 if (padding)
789 skb_store_bits(skb, ISCSI_HDR_LEN + length,
790 &padding_buf, padding);
791
792 if (login->login_complete) {
793 if (cxgbit_set_params(conn)) {
794 kfree_skb(skb);
795 return -1;
796 }
797
798 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
799 }
800
801 if (cxgbit_queue_skb(csk, skb))
802 return -1;
803
804 if ((!login->login_complete) && (!login->login_failed))
805 schedule_delayed_work(&conn->login_work, 0);
806
807 return 0;
808 }
809
810 static void
cxgbit_skb_copy_to_sg(struct sk_buff * skb,struct scatterlist * sg,unsigned int nents,u32 skip)811 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
812 unsigned int nents, u32 skip)
813 {
814 struct skb_seq_state st;
815 const u8 *buf;
816 unsigned int consumed = 0, buf_len;
817 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
818
819 skb_prepare_seq_read(skb, pdu_cb->doffset,
820 pdu_cb->doffset + pdu_cb->dlen,
821 &st);
822
823 while (true) {
824 buf_len = skb_seq_read(consumed, &buf, &st);
825 if (!buf_len) {
826 skb_abort_seq_read(&st);
827 break;
828 }
829
830 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
831 buf_len, skip + consumed);
832 }
833 }
834
cxgbit_allocate_cmd(struct cxgbit_sock * csk)835 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
836 {
837 struct iscsi_conn *conn = csk->conn;
838 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
839 struct cxgbit_cmd *ccmd;
840 struct iscsi_cmd *cmd;
841
842 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
843 if (!cmd) {
844 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
845 return NULL;
846 }
847
848 ccmd = iscsit_priv_cmd(cmd);
849 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
850 ccmd->setup_ddp = true;
851
852 return cmd;
853 }
854
855 static int
cxgbit_handle_immediate_data(struct iscsi_cmd * cmd,struct iscsi_scsi_req * hdr,u32 length)856 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
857 u32 length)
858 {
859 struct iscsi_conn *conn = cmd->conn;
860 struct cxgbit_sock *csk = conn->context;
861 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
862
863 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
864 pr_err("ImmediateData CRC32C DataDigest error\n");
865 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
866 pr_err("Unable to recover from"
867 " Immediate Data digest failure while"
868 " in ERL=0.\n");
869 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
870 (unsigned char *)hdr);
871 return IMMEDIATE_DATA_CANNOT_RECOVER;
872 }
873
874 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
875 (unsigned char *)hdr);
876 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
877 }
878
879 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
880 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
881 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
882 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
883
884 sg_init_table(&ccmd->sg, 1);
885 sg_set_page(&ccmd->sg, skb_frag_page(dfrag),
886 skb_frag_size(dfrag), skb_frag_off(dfrag));
887 get_page(skb_frag_page(dfrag));
888
889 cmd->se_cmd.t_data_sg = &ccmd->sg;
890 cmd->se_cmd.t_data_nents = 1;
891
892 ccmd->release = true;
893 } else {
894 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
895 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
896
897 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
898 }
899
900 cmd->write_data_done += pdu_cb->dlen;
901
902 if (cmd->write_data_done == cmd->se_cmd.data_length) {
903 spin_lock_bh(&cmd->istate_lock);
904 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
905 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
906 spin_unlock_bh(&cmd->istate_lock);
907 }
908
909 return IMMEDIATE_DATA_NORMAL_OPERATION;
910 }
911
912 static int
cxgbit_get_immediate_data(struct iscsi_cmd * cmd,struct iscsi_scsi_req * hdr,bool dump_payload)913 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
914 bool dump_payload)
915 {
916 struct iscsi_conn *conn = cmd->conn;
917 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
918 /*
919 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
920 */
921 if (dump_payload)
922 goto after_immediate_data;
923
924 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
925 cmd->first_burst_len);
926 after_immediate_data:
927 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
928 /*
929 * A PDU/CmdSN carrying Immediate Data passed
930 * DataCRC, check against ExpCmdSN/MaxCmdSN if
931 * Immediate Bit is not set.
932 */
933 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
934 (unsigned char *)hdr,
935 hdr->cmdsn);
936 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
937 return -1;
938
939 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
940 target_put_sess_cmd(&cmd->se_cmd);
941 return 0;
942 } else if (cmd->unsolicited_data) {
943 iscsit_set_unsolicited_dataout(cmd);
944 }
945
946 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
947 /*
948 * Immediate Data failed DataCRC and ERL>=1,
949 * silently drop this PDU and let the initiator
950 * plug the CmdSN gap.
951 *
952 * FIXME: Send Unsolicited NOPIN with reserved
953 * TTT here to help the initiator figure out
954 * the missing CmdSN, although they should be
955 * intelligent enough to determine the missing
956 * CmdSN and issue a retry to plug the sequence.
957 */
958 cmd->i_state = ISTATE_REMOVE;
959 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
960 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
961 return -1;
962
963 return 0;
964 }
965
966 static int
cxgbit_handle_scsi_cmd(struct cxgbit_sock * csk,struct iscsi_cmd * cmd)967 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
968 {
969 struct iscsi_conn *conn = csk->conn;
970 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
971 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
972 int rc;
973 bool dump_payload = false;
974
975 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
976 if (rc < 0)
977 return rc;
978
979 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
980 (pdu_cb->nr_dfrags == 1))
981 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
982
983 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
984 if (rc < 0)
985 return 0;
986 else if (rc > 0)
987 dump_payload = true;
988
989 if (!pdu_cb->dlen)
990 return 0;
991
992 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
993 }
994
cxgbit_handle_iscsi_dataout(struct cxgbit_sock * csk)995 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
996 {
997 struct scatterlist *sg_start;
998 struct iscsi_conn *conn = csk->conn;
999 struct iscsi_cmd *cmd = NULL;
1000 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1001 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1002 u32 data_offset = be32_to_cpu(hdr->offset);
1003 u32 data_len = pdu_cb->dlen;
1004 int rc, sg_nents, sg_off;
1005 bool dcrc_err = false;
1006
1007 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1008 u32 offset = be32_to_cpu(hdr->offset);
1009 u32 ddp_data_len;
1010 u32 payload_length = ntoh24(hdr->dlength);
1011 bool success = false;
1012
1013 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1014 if (!cmd)
1015 return 0;
1016
1017 ddp_data_len = offset - cmd->write_data_done;
1018 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1019
1020 cmd->write_data_done = offset;
1021 cmd->next_burst_len = ddp_data_len;
1022 cmd->data_sn = be32_to_cpu(hdr->datasn);
1023
1024 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1025 cmd, payload_length, &success);
1026 if (rc < 0)
1027 return rc;
1028 else if (!success)
1029 return 0;
1030 } else {
1031 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1032 if (rc < 0)
1033 return rc;
1034 else if (!cmd)
1035 return 0;
1036 }
1037
1038 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1039 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1040 " DataSN: 0x%08x\n",
1041 hdr->itt, hdr->offset, data_len,
1042 hdr->datasn);
1043
1044 dcrc_err = true;
1045 goto check_payload;
1046 }
1047
1048 pr_debug("DataOut data_len: %u, "
1049 "write_data_done: %u, data_length: %u\n",
1050 data_len, cmd->write_data_done,
1051 cmd->se_cmd.data_length);
1052
1053 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1054 u32 skip = data_offset % PAGE_SIZE;
1055
1056 sg_off = data_offset / PAGE_SIZE;
1057 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1058 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1059
1060 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1061 }
1062
1063 check_payload:
1064
1065 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1066 if (rc < 0)
1067 return rc;
1068
1069 return 0;
1070 }
1071
cxgbit_handle_nop_out(struct cxgbit_sock * csk,struct iscsi_cmd * cmd)1072 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1073 {
1074 struct iscsi_conn *conn = csk->conn;
1075 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1076 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1077 unsigned char *ping_data = NULL;
1078 u32 payload_length = pdu_cb->dlen;
1079 int ret;
1080
1081 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1082 if (ret < 0)
1083 return 0;
1084
1085 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1086 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1087 pr_err("Unable to recover from"
1088 " NOPOUT Ping DataCRC failure while in"
1089 " ERL=0.\n");
1090 ret = -1;
1091 goto out;
1092 } else {
1093 /*
1094 * drop this PDU and let the
1095 * initiator plug the CmdSN gap.
1096 */
1097 pr_info("Dropping NOPOUT"
1098 " Command CmdSN: 0x%08x due to"
1099 " DataCRC error.\n", hdr->cmdsn);
1100 ret = 0;
1101 goto out;
1102 }
1103 }
1104
1105 /*
1106 * Handle NOP-OUT payload for traditional iSCSI sockets
1107 */
1108 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1109 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1110 if (!ping_data) {
1111 pr_err("Unable to allocate memory for"
1112 " NOPOUT ping data.\n");
1113 ret = -1;
1114 goto out;
1115 }
1116
1117 skb_copy_bits(csk->skb, pdu_cb->doffset,
1118 ping_data, payload_length);
1119
1120 ping_data[payload_length] = '\0';
1121 /*
1122 * Attach ping data to struct iscsi_cmd->buf_ptr.
1123 */
1124 cmd->buf_ptr = ping_data;
1125 cmd->buf_ptr_size = payload_length;
1126
1127 pr_debug("Got %u bytes of NOPOUT ping"
1128 " data.\n", payload_length);
1129 pr_debug("Ping Data: \"%s\"\n", ping_data);
1130 }
1131
1132 return iscsit_process_nop_out(conn, cmd, hdr);
1133 out:
1134 if (cmd)
1135 iscsit_free_cmd(cmd, false);
1136 return ret;
1137 }
1138
1139 static int
cxgbit_handle_text_cmd(struct cxgbit_sock * csk,struct iscsi_cmd * cmd)1140 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1141 {
1142 struct iscsi_conn *conn = csk->conn;
1143 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1144 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1145 u32 payload_length = pdu_cb->dlen;
1146 int rc;
1147 unsigned char *text_in = NULL;
1148
1149 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1150 if (rc < 0)
1151 return rc;
1152
1153 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1154 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1155 pr_err("Unable to recover from"
1156 " Text Data digest failure while in"
1157 " ERL=0.\n");
1158 goto reject;
1159 } else {
1160 /*
1161 * drop this PDU and let the
1162 * initiator plug the CmdSN gap.
1163 */
1164 pr_info("Dropping Text"
1165 " Command CmdSN: 0x%08x due to"
1166 " DataCRC error.\n", hdr->cmdsn);
1167 return 0;
1168 }
1169 }
1170
1171 if (payload_length) {
1172 text_in = kzalloc(payload_length, GFP_KERNEL);
1173 if (!text_in) {
1174 pr_err("Unable to allocate text_in of payload_length: %u\n",
1175 payload_length);
1176 return -ENOMEM;
1177 }
1178 skb_copy_bits(csk->skb, pdu_cb->doffset,
1179 text_in, payload_length);
1180
1181 text_in[payload_length - 1] = '\0';
1182
1183 cmd->text_in_ptr = text_in;
1184 }
1185
1186 return iscsit_process_text_cmd(conn, cmd, hdr);
1187
1188 reject:
1189 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1190 pdu_cb->hdr);
1191 }
1192
cxgbit_target_rx_opcode(struct cxgbit_sock * csk)1193 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1194 {
1195 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1196 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1197 struct iscsi_conn *conn = csk->conn;
1198 struct iscsi_cmd *cmd = NULL;
1199 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1200 int ret = -EINVAL;
1201
1202 switch (opcode) {
1203 case ISCSI_OP_SCSI_CMD:
1204 cmd = cxgbit_allocate_cmd(csk);
1205 if (!cmd)
1206 goto reject;
1207
1208 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1209 break;
1210 case ISCSI_OP_SCSI_DATA_OUT:
1211 ret = cxgbit_handle_iscsi_dataout(csk);
1212 break;
1213 case ISCSI_OP_NOOP_OUT:
1214 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1215 cmd = cxgbit_allocate_cmd(csk);
1216 if (!cmd)
1217 goto reject;
1218 }
1219
1220 ret = cxgbit_handle_nop_out(csk, cmd);
1221 break;
1222 case ISCSI_OP_SCSI_TMFUNC:
1223 cmd = cxgbit_allocate_cmd(csk);
1224 if (!cmd)
1225 goto reject;
1226
1227 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1228 (unsigned char *)hdr);
1229 break;
1230 case ISCSI_OP_TEXT:
1231 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1232 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1233 if (!cmd)
1234 goto reject;
1235 } else {
1236 cmd = cxgbit_allocate_cmd(csk);
1237 if (!cmd)
1238 goto reject;
1239 }
1240
1241 ret = cxgbit_handle_text_cmd(csk, cmd);
1242 break;
1243 case ISCSI_OP_LOGOUT:
1244 cmd = cxgbit_allocate_cmd(csk);
1245 if (!cmd)
1246 goto reject;
1247
1248 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1249 if (ret > 0)
1250 wait_for_completion_timeout(&conn->conn_logout_comp,
1251 SECONDS_FOR_LOGOUT_COMP
1252 * HZ);
1253 break;
1254 case ISCSI_OP_SNACK:
1255 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1256 break;
1257 default:
1258 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1259 dump_stack();
1260 break;
1261 }
1262
1263 return ret;
1264
1265 reject:
1266 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1267 (unsigned char *)hdr);
1268 return ret;
1269 }
1270
cxgbit_rx_opcode(struct cxgbit_sock * csk)1271 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1272 {
1273 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1274 struct iscsi_conn *conn = csk->conn;
1275 struct iscsi_hdr *hdr = pdu_cb->hdr;
1276 u8 opcode;
1277
1278 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1279 atomic_long_inc(&conn->sess->conn_digest_errors);
1280 goto transport_err;
1281 }
1282
1283 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1284 goto transport_err;
1285
1286 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1287
1288 if (conn->sess->sess_ops->SessionType &&
1289 ((!(opcode & ISCSI_OP_TEXT)) ||
1290 (!(opcode & ISCSI_OP_LOGOUT)))) {
1291 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1292 " while in Discovery Session, rejecting.\n", opcode);
1293 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1294 (unsigned char *)hdr);
1295 goto transport_err;
1296 }
1297
1298 if (cxgbit_target_rx_opcode(csk) < 0)
1299 goto transport_err;
1300
1301 return 0;
1302
1303 transport_err:
1304 return -1;
1305 }
1306
cxgbit_rx_login_pdu(struct cxgbit_sock * csk)1307 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1308 {
1309 struct iscsi_conn *conn = csk->conn;
1310 struct iscsi_login *login = conn->login;
1311 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1312 struct iscsi_login_req *login_req;
1313
1314 login_req = (struct iscsi_login_req *)login->req;
1315 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1316
1317 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1318 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1319 login_req->flags, login_req->itt, login_req->cmdsn,
1320 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1321 /*
1322 * Setup the initial iscsi_login values from the leading
1323 * login request PDU.
1324 */
1325 if (login->first_request) {
1326 login_req = (struct iscsi_login_req *)login->req;
1327 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1328 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1329 login_req->flags);
1330 login->version_min = login_req->min_version;
1331 login->version_max = login_req->max_version;
1332 memcpy(login->isid, login_req->isid, 6);
1333 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1334 login->init_task_tag = login_req->itt;
1335 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1336 login->cid = be16_to_cpu(login_req->cid);
1337 login->tsih = be16_to_cpu(login_req->tsih);
1338 }
1339
1340 if (iscsi_target_check_login_request(conn, login) < 0)
1341 return -1;
1342
1343 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1344 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1345
1346 return 0;
1347 }
1348
1349 static int
cxgbit_process_iscsi_pdu(struct cxgbit_sock * csk,struct sk_buff * skb,int idx)1350 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1351 {
1352 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1353 int ret;
1354
1355 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1356
1357 csk->skb = skb;
1358
1359 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1360 ret = cxgbit_rx_login_pdu(csk);
1361 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1362 } else {
1363 ret = cxgbit_rx_opcode(csk);
1364 }
1365
1366 return ret;
1367 }
1368
cxgbit_lro_skb_dump(struct sk_buff * skb)1369 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1370 {
1371 struct skb_shared_info *ssi = skb_shinfo(skb);
1372 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1373 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1374 u8 i;
1375
1376 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1377 skb, skb->head, skb->data, skb->len, skb->data_len,
1378 ssi->nr_frags);
1379 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1380 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1381
1382 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1383 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1384 "frags %u.\n",
1385 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1386 pdu_cb->ddigest, pdu_cb->frags);
1387 for (i = 0; i < ssi->nr_frags; i++)
1388 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1389 skb, i, skb_frag_off(&ssi->frags[i]),
1390 skb_frag_size(&ssi->frags[i]));
1391 }
1392
cxgbit_lro_hskb_reset(struct cxgbit_sock * csk)1393 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1394 {
1395 struct sk_buff *skb = csk->lro_hskb;
1396 struct skb_shared_info *ssi = skb_shinfo(skb);
1397 u8 i;
1398
1399 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1400 for (i = 0; i < ssi->nr_frags; i++)
1401 put_page(skb_frag_page(&ssi->frags[i]));
1402 ssi->nr_frags = 0;
1403 skb->data_len = 0;
1404 skb->truesize -= skb->len;
1405 skb->len = 0;
1406 }
1407
1408 static void
cxgbit_lro_skb_merge(struct cxgbit_sock * csk,struct sk_buff * skb,u8 pdu_idx)1409 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1410 {
1411 struct sk_buff *hskb = csk->lro_hskb;
1412 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1413 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1414 struct skb_shared_info *hssi = skb_shinfo(hskb);
1415 struct skb_shared_info *ssi = skb_shinfo(skb);
1416 unsigned int len = 0;
1417
1418 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1419 u8 hfrag_idx = hssi->nr_frags;
1420
1421 hpdu_cb->flags |= pdu_cb->flags;
1422 hpdu_cb->seq = pdu_cb->seq;
1423 hpdu_cb->hdr = pdu_cb->hdr;
1424 hpdu_cb->hlen = pdu_cb->hlen;
1425
1426 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1427 sizeof(skb_frag_t));
1428
1429 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1430 hssi->nr_frags++;
1431 hpdu_cb->frags++;
1432 hpdu_cb->hfrag_idx = hfrag_idx;
1433
1434 len = skb_frag_size(&hssi->frags[hfrag_idx]);
1435 hskb->len += len;
1436 hskb->data_len += len;
1437 hskb->truesize += len;
1438 }
1439
1440 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1441 u8 dfrag_idx = hssi->nr_frags, i;
1442
1443 hpdu_cb->flags |= pdu_cb->flags;
1444 hpdu_cb->dfrag_idx = dfrag_idx;
1445
1446 len = 0;
1447 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1448 memcpy(&hssi->frags[dfrag_idx],
1449 &ssi->frags[pdu_cb->dfrag_idx + i],
1450 sizeof(skb_frag_t));
1451
1452 get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1453
1454 len += skb_frag_size(&hssi->frags[dfrag_idx]);
1455
1456 hssi->nr_frags++;
1457 hpdu_cb->frags++;
1458 }
1459
1460 hpdu_cb->dlen = pdu_cb->dlen;
1461 hpdu_cb->doffset = hpdu_cb->hlen;
1462 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1463 hskb->len += len;
1464 hskb->data_len += len;
1465 hskb->truesize += len;
1466 }
1467
1468 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1469 hpdu_cb->flags |= pdu_cb->flags;
1470
1471 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1472 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1473
1474 hpdu_cb->ddigest = pdu_cb->ddigest;
1475 hpdu_cb->pdulen = pdu_cb->pdulen;
1476 }
1477 }
1478
cxgbit_process_lro_skb(struct cxgbit_sock * csk,struct sk_buff * skb)1479 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1480 {
1481 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1482 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1483 u8 pdu_idx = 0, last_idx = 0;
1484 int ret = 0;
1485
1486 if (!pdu_cb->complete) {
1487 cxgbit_lro_skb_merge(csk, skb, 0);
1488
1489 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1490 struct sk_buff *hskb = csk->lro_hskb;
1491
1492 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1493
1494 cxgbit_lro_hskb_reset(csk);
1495
1496 if (ret < 0)
1497 goto out;
1498 }
1499
1500 pdu_idx = 1;
1501 }
1502
1503 if (lro_cb->pdu_idx)
1504 last_idx = lro_cb->pdu_idx - 1;
1505
1506 for (; pdu_idx <= last_idx; pdu_idx++) {
1507 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1508 if (ret < 0)
1509 goto out;
1510 }
1511
1512 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1513 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1514
1515 out:
1516 return ret;
1517 }
1518
cxgbit_rx_lro_skb(struct cxgbit_sock * csk,struct sk_buff * skb)1519 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1520 {
1521 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1522 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1523 int ret = -1;
1524
1525 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1526 (pdu_cb->seq != csk->rcv_nxt)) {
1527 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1528 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1529 cxgbit_lro_skb_dump(skb);
1530 return ret;
1531 }
1532
1533 csk->rcv_nxt += lro_cb->pdu_totallen;
1534
1535 ret = cxgbit_process_lro_skb(csk, skb);
1536
1537 csk->rx_credits += lro_cb->pdu_totallen;
1538
1539 if (csk->rx_credits >= (csk->rcv_win / 4))
1540 cxgbit_rx_data_ack(csk);
1541
1542 return ret;
1543 }
1544
cxgbit_rx_skb(struct cxgbit_sock * csk,struct sk_buff * skb)1545 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1546 {
1547 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1548 int ret = -1;
1549
1550 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1551 if (is_t5(lldi->adapter_type))
1552 ret = cxgbit_rx_lro_skb(csk, skb);
1553 else
1554 ret = cxgbit_process_lro_skb(csk, skb);
1555 }
1556
1557 __kfree_skb(skb);
1558 return ret;
1559 }
1560
cxgbit_rxq_len(struct cxgbit_sock * csk,struct sk_buff_head * rxq)1561 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1562 {
1563 spin_lock_bh(&csk->rxq.lock);
1564 if (skb_queue_len(&csk->rxq)) {
1565 skb_queue_splice_init(&csk->rxq, rxq);
1566 spin_unlock_bh(&csk->rxq.lock);
1567 return true;
1568 }
1569 spin_unlock_bh(&csk->rxq.lock);
1570 return false;
1571 }
1572
cxgbit_wait_rxq(struct cxgbit_sock * csk)1573 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1574 {
1575 struct sk_buff *skb;
1576 struct sk_buff_head rxq;
1577
1578 skb_queue_head_init(&rxq);
1579
1580 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1581
1582 if (signal_pending(current))
1583 goto out;
1584
1585 while ((skb = __skb_dequeue(&rxq))) {
1586 if (cxgbit_rx_skb(csk, skb))
1587 goto out;
1588 }
1589
1590 return 0;
1591 out:
1592 __skb_queue_purge(&rxq);
1593 return -1;
1594 }
1595
cxgbit_get_login_rx(struct iscsi_conn * conn,struct iscsi_login * login)1596 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1597 {
1598 struct cxgbit_sock *csk = conn->context;
1599 int ret = -1;
1600
1601 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1602 ret = cxgbit_wait_rxq(csk);
1603 if (ret) {
1604 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1605 break;
1606 }
1607 }
1608
1609 return ret;
1610 }
1611
cxgbit_get_rx_pdu(struct iscsi_conn * conn)1612 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1613 {
1614 struct cxgbit_sock *csk = conn->context;
1615
1616 while (!kthread_should_stop()) {
1617 iscsit_thread_check_cpumask(conn, current, 0);
1618 if (cxgbit_wait_rxq(csk))
1619 return;
1620 }
1621 }
1622