1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2016 Chelsio Communications, Inc.
4 */
5
6 #include "cxgbit.h"
7
8 static void
cxgbit_set_one_ppod(struct cxgbi_pagepod * ppod,struct cxgbi_task_tag_info * ttinfo,struct scatterlist ** sg_pp,unsigned int * sg_off)9 cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod,
10 struct cxgbi_task_tag_info *ttinfo,
11 struct scatterlist **sg_pp, unsigned int *sg_off)
12 {
13 struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
14 unsigned int offset = sg_off ? *sg_off : 0;
15 dma_addr_t addr = 0UL;
16 unsigned int len = 0;
17 int i;
18
19 memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
20
21 if (sg) {
22 addr = sg_dma_address(sg);
23 len = sg_dma_len(sg);
24 }
25
26 for (i = 0; i < PPOD_PAGES_MAX; i++) {
27 if (sg) {
28 ppod->addr[i] = cpu_to_be64(addr + offset);
29 offset += PAGE_SIZE;
30 if (offset == (len + sg->offset)) {
31 offset = 0;
32 sg = sg_next(sg);
33 if (sg) {
34 addr = sg_dma_address(sg);
35 len = sg_dma_len(sg);
36 }
37 }
38 } else {
39 ppod->addr[i] = 0ULL;
40 }
41 }
42
43 /*
44 * the fifth address needs to be repeated in the next ppod, so do
45 * not move sg
46 */
47 if (sg_pp) {
48 *sg_pp = sg;
49 *sg_off = offset;
50 }
51
52 if (offset == len) {
53 offset = 0;
54 if (sg) {
55 sg = sg_next(sg);
56 if (sg)
57 addr = sg_dma_address(sg);
58 }
59 }
60 ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
61 }
62
63 static struct sk_buff *
cxgbit_ppod_init_idata(struct cxgbit_device * cdev,struct cxgbi_ppm * ppm,unsigned int idx,unsigned int npods,unsigned int tid)64 cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm,
65 unsigned int idx, unsigned int npods, unsigned int tid)
66 {
67 struct ulp_mem_io *req;
68 struct ulptx_idata *idata;
69 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
70 unsigned int dlen = npods << PPOD_SIZE_SHIFT;
71 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
72 sizeof(struct ulptx_idata) + dlen, 16);
73 struct sk_buff *skb;
74
75 skb = alloc_skb(wr_len, GFP_KERNEL);
76 if (!skb)
77 return NULL;
78
79 req = __skb_put(skb, wr_len);
80 INIT_ULPTX_WR(req, wr_len, 0, tid);
81 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
82 FW_WR_ATOMIC_V(0));
83 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
84 ULP_MEMIO_ORDER_V(0) |
85 T5_ULP_MEMIO_IMM_V(1));
86 req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
87 req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
88 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
89
90 idata = (struct ulptx_idata *)(req + 1);
91 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
92 idata->len = htonl(dlen);
93
94 return skb;
95 }
96
97 static int
cxgbit_ppod_write_idata(struct cxgbi_ppm * ppm,struct cxgbit_sock * csk,struct cxgbi_task_tag_info * ttinfo,unsigned int idx,unsigned int npods,struct scatterlist ** sg_pp,unsigned int * sg_off)98 cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
99 struct cxgbi_task_tag_info *ttinfo, unsigned int idx,
100 unsigned int npods, struct scatterlist **sg_pp,
101 unsigned int *sg_off)
102 {
103 struct cxgbit_device *cdev = csk->com.cdev;
104 struct sk_buff *skb;
105 struct ulp_mem_io *req;
106 struct ulptx_idata *idata;
107 struct cxgbi_pagepod *ppod;
108 unsigned int i;
109
110 skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid);
111 if (!skb)
112 return -ENOMEM;
113
114 req = (struct ulp_mem_io *)skb->data;
115 idata = (struct ulptx_idata *)(req + 1);
116 ppod = (struct cxgbi_pagepod *)(idata + 1);
117
118 for (i = 0; i < npods; i++, ppod++)
119 cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
120
121 __skb_queue_tail(&csk->ppodq, skb);
122
123 return 0;
124 }
125
126 static int
cxgbit_ddp_set_map(struct cxgbi_ppm * ppm,struct cxgbit_sock * csk,struct cxgbi_task_tag_info * ttinfo)127 cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
128 struct cxgbi_task_tag_info *ttinfo)
129 {
130 unsigned int pidx = ttinfo->idx;
131 unsigned int npods = ttinfo->npods;
132 unsigned int i, cnt;
133 struct scatterlist *sg = ttinfo->sgl;
134 unsigned int offset = 0;
135 int ret = 0;
136
137 for (i = 0; i < npods; i += cnt, pidx += cnt) {
138 cnt = npods - i;
139
140 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
141 cnt = ULPMEM_IDATA_MAX_NPPODS;
142
143 ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
144 &sg, &offset);
145 if (ret < 0)
146 break;
147 }
148
149 return ret;
150 }
151
cxgbit_ddp_sgl_check(struct scatterlist * sg,unsigned int nents)152 static int cxgbit_ddp_sgl_check(struct scatterlist *sg,
153 unsigned int nents)
154 {
155 unsigned int last_sgidx = nents - 1;
156 unsigned int i;
157
158 for (i = 0; i < nents; i++, sg = sg_next(sg)) {
159 unsigned int len = sg->length + sg->offset;
160
161 if ((sg->offset & 0x3) || (i && sg->offset) ||
162 ((i != last_sgidx) && (len != PAGE_SIZE))) {
163 return -EINVAL;
164 }
165 }
166
167 return 0;
168 }
169
170 static int
cxgbit_ddp_reserve(struct cxgbit_sock * csk,struct cxgbi_task_tag_info * ttinfo,unsigned int xferlen)171 cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo,
172 unsigned int xferlen)
173 {
174 struct cxgbit_device *cdev = csk->com.cdev;
175 struct cxgbi_ppm *ppm = cdev2ppm(cdev);
176 struct scatterlist *sgl = ttinfo->sgl;
177 unsigned int sgcnt = ttinfo->nents;
178 unsigned int sg_offset = sgl->offset;
179 int ret;
180
181 if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) {
182 pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
183 ppm, ppm->tformat.pgsz_idx_dflt,
184 xferlen, ttinfo->nents);
185 return -EINVAL;
186 }
187
188 if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0)
189 return -EINVAL;
190
191 ttinfo->nr_pages = (xferlen + sgl->offset +
192 (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT;
193
194 /*
195 * the ddp tag will be used for the ttt in the outgoing r2t pdu
196 */
197 ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
198 &ttinfo->tag, 0);
199 if (ret < 0)
200 return ret;
201 ttinfo->npods = ret;
202
203 sgl->offset = 0;
204 ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
205 sgl->offset = sg_offset;
206 if (!ret) {
207 pr_debug("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
208 __func__, 0, xferlen, sgcnt);
209 goto rel_ppods;
210 }
211
212 cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
213 xferlen, &ttinfo->hdr);
214
215 ret = cxgbit_ddp_set_map(ppm, csk, ttinfo);
216 if (ret < 0) {
217 __skb_queue_purge(&csk->ppodq);
218 dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
219 goto rel_ppods;
220 }
221
222 return 0;
223
224 rel_ppods:
225 cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
226 return -EINVAL;
227 }
228
229 void
cxgbit_get_r2t_ttt(struct iscsit_conn * conn,struct iscsit_cmd * cmd,struct iscsi_r2t * r2t)230 cxgbit_get_r2t_ttt(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
231 struct iscsi_r2t *r2t)
232 {
233 struct cxgbit_sock *csk = conn->context;
234 struct cxgbit_device *cdev = csk->com.cdev;
235 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
236 struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
237 int ret;
238
239 if ((!ccmd->setup_ddp) ||
240 (!test_bit(CSK_DDP_ENABLE, &csk->com.flags)))
241 goto out;
242
243 ccmd->setup_ddp = false;
244
245 ttinfo->sgl = cmd->se_cmd.t_data_sg;
246 ttinfo->nents = cmd->se_cmd.t_data_nents;
247
248 ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length);
249 if (ret < 0) {
250 pr_debug("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n",
251 csk, cmd, cmd->se_cmd.data_length, ttinfo->nents);
252
253 ttinfo->sgl = NULL;
254 ttinfo->nents = 0;
255 } else {
256 ccmd->release = true;
257 }
258 out:
259 pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag);
260 r2t->targ_xfer_tag = ttinfo->tag;
261 }
262
cxgbit_unmap_cmd(struct iscsit_conn * conn,struct iscsit_cmd * cmd)263 void cxgbit_unmap_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
264 {
265 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
266
267 if (ccmd->release) {
268 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
269 put_page(sg_page(&ccmd->sg));
270 } else {
271 struct cxgbit_sock *csk = conn->context;
272 struct cxgbit_device *cdev = csk->com.cdev;
273 struct cxgbi_ppm *ppm = cdev2ppm(cdev);
274 struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
275
276 /* Abort the TCP conn if DDP is not complete to
277 * avoid any possibility of DDP after freeing
278 * the cmd.
279 */
280 if (unlikely(cmd->write_data_done !=
281 cmd->se_cmd.data_length))
282 cxgbit_abort_conn(csk);
283
284 if (unlikely(ttinfo->sgl)) {
285 dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl,
286 ttinfo->nents, DMA_FROM_DEVICE);
287 ttinfo->nents = 0;
288 ttinfo->sgl = NULL;
289 }
290 cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
291 }
292 ccmd->release = false;
293 }
294 }
295
cxgbit_ddp_init(struct cxgbit_device * cdev)296 int cxgbit_ddp_init(struct cxgbit_device *cdev)
297 {
298 struct cxgb4_lld_info *lldi = &cdev->lldi;
299 struct net_device *ndev = cdev->lldi.ports[0];
300 struct cxgbi_tag_format tformat;
301 int ret, i;
302
303 if (!lldi->vr->iscsi.size) {
304 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
305 return -EACCES;
306 }
307
308 memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
309 for (i = 0; i < 4; i++)
310 tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
311 & 0xF;
312 cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
313
314 ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0],
315 cdev->lldi.pdev, &cdev->lldi, &tformat,
316 lldi->vr->iscsi.size, lldi->iscsi_llimit,
317 lldi->vr->iscsi.start, 2,
318 lldi->vr->ppod_edram.start,
319 lldi->vr->ppod_edram.size);
320 if (ret >= 0) {
321 struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm);
322
323 if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) &&
324 (ppm->ppmax >= 1024))
325 set_bit(CDEV_DDP_ENABLE, &cdev->flags);
326 ret = 0;
327 }
328
329 return ret;
330 }
331