1 /*
2  * Copyright (c) 2007, 2008, 2009 QLogic Corporation. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/mm.h>
33 #include <linux/types.h>
34 #include <linux/device.h>
35 #include <linux/dmapool.h>
36 #include <linux/slab.h>
37 #include <linux/list.h>
38 #include <linux/highmem.h>
39 #include <linux/io.h>
40 #include <linux/uio.h>
41 #include <linux/rbtree.h>
42 #include <linux/spinlock.h>
43 #include <linux/delay.h>
44 
45 #include "qib.h"
46 #include "qib_user_sdma.h"
47 
48 /* minimum size of header */
49 #define QIB_USER_SDMA_MIN_HEADER_LENGTH 64
50 /* expected size of headers (for dma_pool) */
51 #define QIB_USER_SDMA_EXP_HEADER_LENGTH 64
52 /* attempt to drain the queue for 5secs */
53 #define QIB_USER_SDMA_DRAIN_TIMEOUT 250
54 
55 /*
56  * track how many times a process open this driver.
57  */
58 static struct rb_root qib_user_sdma_rb_root = RB_ROOT;
59 
60 struct qib_user_sdma_rb_node {
61 	struct rb_node node;
62 	int refcount;
63 	pid_t pid;
64 };
65 
66 struct qib_user_sdma_pkt {
67 	struct list_head list;  /* list element */
68 
69 	u8  tiddma;		/* if this is NEW tid-sdma */
70 	u8  largepkt;		/* this is large pkt from kmalloc */
71 	u16 frag_size;		/* frag size used by PSM */
72 	u16 index;              /* last header index or push index */
73 	u16 naddr;              /* dimension of addr (1..3) ... */
74 	u16 addrlimit;		/* addr array size */
75 	u16 tidsmidx;		/* current tidsm index */
76 	u16 tidsmcount;		/* tidsm array item count */
77 	u16 payload_size;	/* payload size so far for header */
78 	u32 bytes_togo;		/* bytes for processing */
79 	u32 counter;            /* sdma pkts queued counter for this entry */
80 	struct qib_tid_session_member *tidsm;	/* tid session member array */
81 	struct qib_user_sdma_queue *pq;	/* which pq this pkt belongs to */
82 	u64 added;              /* global descq number of entries */
83 
84 	struct {
85 		u16 offset;                     /* offset for kvaddr, addr */
86 		u16 length;                     /* length in page */
87 		u16 first_desc;			/* first desc */
88 		u16 last_desc;			/* last desc */
89 		u16 put_page;                   /* should we put_page? */
90 		u16 dma_mapped;                 /* is page dma_mapped? */
91 		u16 dma_length;			/* for dma_unmap_page() */
92 		u16 padding;
93 		struct page *page;              /* may be NULL (coherent mem) */
94 		void *kvaddr;                   /* FIXME: only for pio hack */
95 		dma_addr_t addr;
96 	} addr[4];   /* max pages, any more and we coalesce */
97 };
98 
99 struct qib_user_sdma_queue {
100 	/*
101 	 * pkts sent to dma engine are queued on this
102 	 * list head.  the type of the elements of this
103 	 * list are struct qib_user_sdma_pkt...
104 	 */
105 	struct list_head sent;
106 
107 	/*
108 	 * Because above list will be accessed by both process and
109 	 * signal handler, we need a spinlock for it.
110 	 */
111 	spinlock_t sent_lock ____cacheline_aligned_in_smp;
112 
113 	/* headers with expected length are allocated from here... */
114 	char header_cache_name[64];
115 	struct dma_pool *header_cache;
116 
117 	/* packets are allocated from the slab cache... */
118 	char pkt_slab_name[64];
119 	struct kmem_cache *pkt_slab;
120 
121 	/* as packets go on the queued queue, they are counted... */
122 	u32 counter;
123 	u32 sent_counter;
124 	/* pending packets, not sending yet */
125 	u32 num_pending;
126 	/* sending packets, not complete yet */
127 	u32 num_sending;
128 	/* global descq number of entry of last sending packet */
129 	u64 added;
130 
131 	/* dma page table */
132 	struct rb_root dma_pages_root;
133 
134 	struct qib_user_sdma_rb_node *sdma_rb_node;
135 
136 	/* protect everything above... */
137 	struct mutex lock;
138 };
139 
140 static struct qib_user_sdma_rb_node *
141 qib_user_sdma_rb_search(struct rb_root *root, pid_t pid)
142 {
143 	struct qib_user_sdma_rb_node *sdma_rb_node;
144 	struct rb_node *node = root->rb_node;
145 
146 	while (node) {
147 		sdma_rb_node = rb_entry(node, struct qib_user_sdma_rb_node,
148 					node);
149 		if (pid < sdma_rb_node->pid)
150 			node = node->rb_left;
151 		else if (pid > sdma_rb_node->pid)
152 			node = node->rb_right;
153 		else
154 			return sdma_rb_node;
155 	}
156 	return NULL;
157 }
158 
159 static int
160 qib_user_sdma_rb_insert(struct rb_root *root, struct qib_user_sdma_rb_node *new)
161 {
162 	struct rb_node **node = &(root->rb_node);
163 	struct rb_node *parent = NULL;
164 	struct qib_user_sdma_rb_node *got;
165 
166 	while (*node) {
167 		got = rb_entry(*node, struct qib_user_sdma_rb_node, node);
168 		parent = *node;
169 		if (new->pid < got->pid)
170 			node = &((*node)->rb_left);
171 		else if (new->pid > got->pid)
172 			node = &((*node)->rb_right);
173 		else
174 			return 0;
175 	}
176 
177 	rb_link_node(&new->node, parent, node);
178 	rb_insert_color(&new->node, root);
179 	return 1;
180 }
181 
182 struct qib_user_sdma_queue *
183 qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
184 {
185 	struct qib_user_sdma_queue *pq =
186 		kmalloc(sizeof(struct qib_user_sdma_queue), GFP_KERNEL);
187 	struct qib_user_sdma_rb_node *sdma_rb_node;
188 
189 	if (!pq)
190 		goto done;
191 
192 	pq->counter = 0;
193 	pq->sent_counter = 0;
194 	pq->num_pending = 0;
195 	pq->num_sending = 0;
196 	pq->added = 0;
197 	pq->sdma_rb_node = NULL;
198 
199 	INIT_LIST_HEAD(&pq->sent);
200 	spin_lock_init(&pq->sent_lock);
201 	mutex_init(&pq->lock);
202 
203 	snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
204 		 "qib-user-sdma-pkts-%u-%02u.%02u", unit, ctxt, sctxt);
205 	pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name,
206 					 sizeof(struct qib_user_sdma_pkt),
207 					 0, 0, NULL);
208 
209 	if (!pq->pkt_slab)
210 		goto err_kfree;
211 
212 	snprintf(pq->header_cache_name, sizeof(pq->header_cache_name),
213 		 "qib-user-sdma-headers-%u-%02u.%02u", unit, ctxt, sctxt);
214 	pq->header_cache = dma_pool_create(pq->header_cache_name,
215 					   dev,
216 					   QIB_USER_SDMA_EXP_HEADER_LENGTH,
217 					   4, 0);
218 	if (!pq->header_cache)
219 		goto err_slab;
220 
221 	pq->dma_pages_root = RB_ROOT;
222 
223 	sdma_rb_node = qib_user_sdma_rb_search(&qib_user_sdma_rb_root,
224 					current->pid);
225 	if (sdma_rb_node) {
226 		sdma_rb_node->refcount++;
227 	} else {
228 		sdma_rb_node = kmalloc(sizeof(
229 			struct qib_user_sdma_rb_node), GFP_KERNEL);
230 		if (!sdma_rb_node)
231 			goto err_rb;
232 
233 		sdma_rb_node->refcount = 1;
234 		sdma_rb_node->pid = current->pid;
235 
236 		qib_user_sdma_rb_insert(&qib_user_sdma_rb_root, sdma_rb_node);
237 	}
238 	pq->sdma_rb_node = sdma_rb_node;
239 
240 	goto done;
241 
242 err_rb:
243 	dma_pool_destroy(pq->header_cache);
244 err_slab:
245 	kmem_cache_destroy(pq->pkt_slab);
246 err_kfree:
247 	kfree(pq);
248 	pq = NULL;
249 
250 done:
251 	return pq;
252 }
253 
254 static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt,
255 				    int i, u16 offset, u16 len,
256 				    u16 first_desc, u16 last_desc,
257 				    u16 put_page, u16 dma_mapped,
258 				    struct page *page, void *kvaddr,
259 				    dma_addr_t dma_addr, u16 dma_length)
260 {
261 	pkt->addr[i].offset = offset;
262 	pkt->addr[i].length = len;
263 	pkt->addr[i].first_desc = first_desc;
264 	pkt->addr[i].last_desc = last_desc;
265 	pkt->addr[i].put_page = put_page;
266 	pkt->addr[i].dma_mapped = dma_mapped;
267 	pkt->addr[i].page = page;
268 	pkt->addr[i].kvaddr = kvaddr;
269 	pkt->addr[i].addr = dma_addr;
270 	pkt->addr[i].dma_length = dma_length;
271 }
272 
273 static void *qib_user_sdma_alloc_header(struct qib_user_sdma_queue *pq,
274 				size_t len, dma_addr_t *dma_addr)
275 {
276 	void *hdr;
277 
278 	if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH)
279 		hdr = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
280 					     dma_addr);
281 	else
282 		hdr = NULL;
283 
284 	if (!hdr) {
285 		hdr = kmalloc(len, GFP_KERNEL);
286 		if (!hdr)
287 			return NULL;
288 
289 		*dma_addr = 0;
290 	}
291 
292 	return hdr;
293 }
294 
295 static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
296 				       struct qib_user_sdma_queue *pq,
297 				       struct qib_user_sdma_pkt *pkt,
298 				       struct page *page, u16 put,
299 				       u16 offset, u16 len, void *kvaddr)
300 {
301 	__le16 *pbc16;
302 	void *pbcvaddr;
303 	struct qib_message_header *hdr;
304 	u16 newlen, pbclen, lastdesc, dma_mapped;
305 	u32 vcto;
306 	union qib_seqnum seqnum;
307 	dma_addr_t pbcdaddr;
308 	dma_addr_t dma_addr =
309 		dma_map_page(&dd->pcidev->dev,
310 			page, offset, len, DMA_TO_DEVICE);
311 	int ret = 0;
312 
313 	if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
314 		/*
315 		 * dma mapping error, pkt has not managed
316 		 * this page yet, return the page here so
317 		 * the caller can ignore this page.
318 		 */
319 		if (put) {
320 			unpin_user_page(page);
321 		} else {
322 			/* coalesce case */
323 			kunmap(page);
324 			__free_page(page);
325 		}
326 		ret = -ENOMEM;
327 		goto done;
328 	}
329 	offset = 0;
330 	dma_mapped = 1;
331 
332 
333 next_fragment:
334 
335 	/*
336 	 * In tid-sdma, the transfer length is restricted by
337 	 * receiver side current tid page length.
338 	 */
339 	if (pkt->tiddma && len > pkt->tidsm[pkt->tidsmidx].length)
340 		newlen = pkt->tidsm[pkt->tidsmidx].length;
341 	else
342 		newlen = len;
343 
344 	/*
345 	 * Then the transfer length is restricted by MTU.
346 	 * the last descriptor flag is determined by:
347 	 * 1. the current packet is at frag size length.
348 	 * 2. the current tid page is done if tid-sdma.
349 	 * 3. there is no more byte togo if sdma.
350 	 */
351 	lastdesc = 0;
352 	if ((pkt->payload_size + newlen) >= pkt->frag_size) {
353 		newlen = pkt->frag_size - pkt->payload_size;
354 		lastdesc = 1;
355 	} else if (pkt->tiddma) {
356 		if (newlen == pkt->tidsm[pkt->tidsmidx].length)
357 			lastdesc = 1;
358 	} else {
359 		if (newlen == pkt->bytes_togo)
360 			lastdesc = 1;
361 	}
362 
363 	/* fill the next fragment in this page */
364 	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
365 		offset, newlen,		/* offset, len */
366 		0, lastdesc,		/* first last desc */
367 		put, dma_mapped,	/* put page, dma mapped */
368 		page, kvaddr,		/* struct page, virt addr */
369 		dma_addr, len);		/* dma addr, dma length */
370 	pkt->bytes_togo -= newlen;
371 	pkt->payload_size += newlen;
372 	pkt->naddr++;
373 	if (pkt->naddr == pkt->addrlimit) {
374 		ret = -EFAULT;
375 		goto done;
376 	}
377 
378 	/* If there is no more byte togo. (lastdesc==1) */
379 	if (pkt->bytes_togo == 0) {
380 		/* The packet is done, header is not dma mapped yet.
381 		 * it should be from kmalloc */
382 		if (!pkt->addr[pkt->index].addr) {
383 			pkt->addr[pkt->index].addr =
384 				dma_map_single(&dd->pcidev->dev,
385 					pkt->addr[pkt->index].kvaddr,
386 					pkt->addr[pkt->index].dma_length,
387 					DMA_TO_DEVICE);
388 			if (dma_mapping_error(&dd->pcidev->dev,
389 					pkt->addr[pkt->index].addr)) {
390 				ret = -ENOMEM;
391 				goto done;
392 			}
393 			pkt->addr[pkt->index].dma_mapped = 1;
394 		}
395 
396 		goto done;
397 	}
398 
399 	/* If tid-sdma, advance tid info. */
400 	if (pkt->tiddma) {
401 		pkt->tidsm[pkt->tidsmidx].length -= newlen;
402 		if (pkt->tidsm[pkt->tidsmidx].length) {
403 			pkt->tidsm[pkt->tidsmidx].offset += newlen;
404 		} else {
405 			pkt->tidsmidx++;
406 			if (pkt->tidsmidx == pkt->tidsmcount) {
407 				ret = -EFAULT;
408 				goto done;
409 			}
410 		}
411 	}
412 
413 	/*
414 	 * If this is NOT the last descriptor. (newlen==len)
415 	 * the current packet is not done yet, but the current
416 	 * send side page is done.
417 	 */
418 	if (lastdesc == 0)
419 		goto done;
420 
421 	/*
422 	 * If running this driver under PSM with message size
423 	 * fitting into one transfer unit, it is not possible
424 	 * to pass this line. otherwise, it is a buggggg.
425 	 */
426 
427 	/*
428 	 * Since the current packet is done, and there are more
429 	 * bytes togo, we need to create a new sdma header, copying
430 	 * from previous sdma header and modify both.
431 	 */
432 	pbclen = pkt->addr[pkt->index].length;
433 	pbcvaddr = qib_user_sdma_alloc_header(pq, pbclen, &pbcdaddr);
434 	if (!pbcvaddr) {
435 		ret = -ENOMEM;
436 		goto done;
437 	}
438 	/* Copy the previous sdma header to new sdma header */
439 	pbc16 = (__le16 *)pkt->addr[pkt->index].kvaddr;
440 	memcpy(pbcvaddr, pbc16, pbclen);
441 
442 	/* Modify the previous sdma header */
443 	hdr = (struct qib_message_header *)&pbc16[4];
444 
445 	/* New pbc length */
446 	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->bytes_togo>>2));
447 
448 	/* New packet length */
449 	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
450 
451 	if (pkt->tiddma) {
452 		/* turn on the header suppression */
453 		hdr->iph.pkt_flags =
454 			cpu_to_le16(le16_to_cpu(hdr->iph.pkt_flags)|0x2);
455 		/* turn off ACK_REQ: 0x04 and EXPECTED_DONE: 0x20 */
456 		hdr->flags &= ~(0x04|0x20);
457 	} else {
458 		/* turn off extra bytes: 20-21 bits */
459 		hdr->bth[0] = cpu_to_be32(be32_to_cpu(hdr->bth[0])&0xFFCFFFFF);
460 		/* turn off ACK_REQ: 0x04 */
461 		hdr->flags &= ~(0x04);
462 	}
463 
464 	/* New kdeth checksum */
465 	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
466 	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
467 		be16_to_cpu(hdr->lrh[2]) -
468 		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
469 		le16_to_cpu(hdr->iph.pkt_flags));
470 
471 	/* The packet is done, header is not dma mapped yet.
472 	 * it should be from kmalloc */
473 	if (!pkt->addr[pkt->index].addr) {
474 		pkt->addr[pkt->index].addr =
475 			dma_map_single(&dd->pcidev->dev,
476 				pkt->addr[pkt->index].kvaddr,
477 				pkt->addr[pkt->index].dma_length,
478 				DMA_TO_DEVICE);
479 		if (dma_mapping_error(&dd->pcidev->dev,
480 				pkt->addr[pkt->index].addr)) {
481 			ret = -ENOMEM;
482 			goto done;
483 		}
484 		pkt->addr[pkt->index].dma_mapped = 1;
485 	}
486 
487 	/* Modify the new sdma header */
488 	pbc16 = (__le16 *)pbcvaddr;
489 	hdr = (struct qib_message_header *)&pbc16[4];
490 
491 	/* New pbc length */
492 	pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->payload_size>>2));
493 
494 	/* New packet length */
495 	hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
496 
497 	if (pkt->tiddma) {
498 		/* Set new tid and offset for new sdma header */
499 		hdr->iph.ver_ctxt_tid_offset = cpu_to_le32(
500 			(le32_to_cpu(hdr->iph.ver_ctxt_tid_offset)&0xFF000000) +
501 			(pkt->tidsm[pkt->tidsmidx].tid<<QLOGIC_IB_I_TID_SHIFT) +
502 			(pkt->tidsm[pkt->tidsmidx].offset>>2));
503 	} else {
504 		/* Middle protocol new packet offset */
505 		hdr->uwords[2] += pkt->payload_size;
506 	}
507 
508 	/* New kdeth checksum */
509 	vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
510 	hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
511 		be16_to_cpu(hdr->lrh[2]) -
512 		((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
513 		le16_to_cpu(hdr->iph.pkt_flags));
514 
515 	/* Next sequence number in new sdma header */
516 	seqnum.val = be32_to_cpu(hdr->bth[2]);
517 	if (pkt->tiddma)
518 		seqnum.seq++;
519 	else
520 		seqnum.pkt++;
521 	hdr->bth[2] = cpu_to_be32(seqnum.val);
522 
523 	/* Init new sdma header. */
524 	qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
525 		0, pbclen,		/* offset, len */
526 		1, 0,			/* first last desc */
527 		0, 0,			/* put page, dma mapped */
528 		NULL, pbcvaddr,		/* struct page, virt addr */
529 		pbcdaddr, pbclen);	/* dma addr, dma length */
530 	pkt->index = pkt->naddr;
531 	pkt->payload_size = 0;
532 	pkt->naddr++;
533 	if (pkt->naddr == pkt->addrlimit) {
534 		ret = -EFAULT;
535 		goto done;
536 	}
537 
538 	/* Prepare for next fragment in this page */
539 	if (newlen != len) {
540 		if (dma_mapped) {
541 			put = 0;
542 			dma_mapped = 0;
543 			page = NULL;
544 			kvaddr = NULL;
545 		}
546 		len -= newlen;
547 		offset += newlen;
548 
549 		goto next_fragment;
550 	}
551 
552 done:
553 	return ret;
554 }
555 
556 /* we've too many pages in the iovec, coalesce to a single page */
557 static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
558 				  struct qib_user_sdma_queue *pq,
559 				  struct qib_user_sdma_pkt *pkt,
560 				  const struct iovec *iov,
561 				  unsigned long niov)
562 {
563 	int ret = 0;
564 	struct page *page = alloc_page(GFP_KERNEL);
565 	void *mpage_save;
566 	char *mpage;
567 	int i;
568 	int len = 0;
569 
570 	if (!page) {
571 		ret = -ENOMEM;
572 		goto done;
573 	}
574 
575 	mpage = kmap(page);
576 	mpage_save = mpage;
577 	for (i = 0; i < niov; i++) {
578 		int cfur;
579 
580 		cfur = copy_from_user(mpage,
581 				      iov[i].iov_base, iov[i].iov_len);
582 		if (cfur) {
583 			ret = -EFAULT;
584 			goto free_unmap;
585 		}
586 
587 		mpage += iov[i].iov_len;
588 		len += iov[i].iov_len;
589 	}
590 
591 	ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
592 			page, 0, 0, len, mpage_save);
593 	goto done;
594 
595 free_unmap:
596 	kunmap(page);
597 	__free_page(page);
598 done:
599 	return ret;
600 }
601 
602 /*
603  * How many pages in this iovec element?
604  */
605 static size_t qib_user_sdma_num_pages(const struct iovec *iov)
606 {
607 	const unsigned long addr  = (unsigned long) iov->iov_base;
608 	const unsigned long  len  = iov->iov_len;
609 	const unsigned long spage = addr & PAGE_MASK;
610 	const unsigned long epage = (addr + len - 1) & PAGE_MASK;
611 
612 	return 1 + ((epage - spage) >> PAGE_SHIFT);
613 }
614 
615 static void qib_user_sdma_free_pkt_frag(struct device *dev,
616 					struct qib_user_sdma_queue *pq,
617 					struct qib_user_sdma_pkt *pkt,
618 					int frag)
619 {
620 	const int i = frag;
621 
622 	if (pkt->addr[i].page) {
623 		/* only user data has page */
624 		if (pkt->addr[i].dma_mapped)
625 			dma_unmap_page(dev,
626 				       pkt->addr[i].addr,
627 				       pkt->addr[i].dma_length,
628 				       DMA_TO_DEVICE);
629 
630 		if (pkt->addr[i].kvaddr)
631 			kunmap(pkt->addr[i].page);
632 
633 		if (pkt->addr[i].put_page)
634 			unpin_user_page(pkt->addr[i].page);
635 		else
636 			__free_page(pkt->addr[i].page);
637 	} else if (pkt->addr[i].kvaddr) {
638 		/* for headers */
639 		if (pkt->addr[i].dma_mapped) {
640 			/* from kmalloc & dma mapped */
641 			dma_unmap_single(dev,
642 				       pkt->addr[i].addr,
643 				       pkt->addr[i].dma_length,
644 				       DMA_TO_DEVICE);
645 			kfree(pkt->addr[i].kvaddr);
646 		} else if (pkt->addr[i].addr) {
647 			/* free coherent mem from cache... */
648 			dma_pool_free(pq->header_cache,
649 			      pkt->addr[i].kvaddr, pkt->addr[i].addr);
650 		} else {
651 			/* from kmalloc but not dma mapped */
652 			kfree(pkt->addr[i].kvaddr);
653 		}
654 	}
655 }
656 
657 /* return number of pages pinned... */
658 static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
659 				   struct qib_user_sdma_queue *pq,
660 				   struct qib_user_sdma_pkt *pkt,
661 				   unsigned long addr, int tlen, size_t npages)
662 {
663 	struct page *pages[8];
664 	int i, j;
665 	int ret = 0;
666 
667 	while (npages) {
668 		if (npages > 8)
669 			j = 8;
670 		else
671 			j = npages;
672 
673 		ret = pin_user_pages_fast(addr, j, FOLL_LONGTERM, pages);
674 		if (ret != j) {
675 			i = 0;
676 			j = ret;
677 			ret = -ENOMEM;
678 			goto free_pages;
679 		}
680 
681 		for (i = 0; i < j; i++) {
682 			/* map the pages... */
683 			unsigned long fofs = addr & ~PAGE_MASK;
684 			int flen = ((fofs + tlen) > PAGE_SIZE) ?
685 				(PAGE_SIZE - fofs) : tlen;
686 
687 			ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
688 				pages[i], 1, fofs, flen, NULL);
689 			if (ret < 0) {
690 				/* current page has beed taken
691 				 * care of inside above call.
692 				 */
693 				i++;
694 				goto free_pages;
695 			}
696 
697 			addr += flen;
698 			tlen -= flen;
699 		}
700 
701 		npages -= j;
702 	}
703 
704 	goto done;
705 
706 	/* if error, return all pages not managed by pkt */
707 free_pages:
708 	while (i < j)
709 		unpin_user_page(pages[i++]);
710 
711 done:
712 	return ret;
713 }
714 
715 static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
716 				 struct qib_user_sdma_queue *pq,
717 				 struct qib_user_sdma_pkt *pkt,
718 				 const struct iovec *iov,
719 				 unsigned long niov)
720 {
721 	int ret = 0;
722 	unsigned long idx;
723 
724 	for (idx = 0; idx < niov; idx++) {
725 		const size_t npages = qib_user_sdma_num_pages(iov + idx);
726 		const unsigned long addr = (unsigned long) iov[idx].iov_base;
727 
728 		ret = qib_user_sdma_pin_pages(dd, pq, pkt, addr,
729 					      iov[idx].iov_len, npages);
730 		if (ret < 0)
731 			goto free_pkt;
732 	}
733 
734 	goto done;
735 
736 free_pkt:
737 	/* we need to ignore the first entry here */
738 	for (idx = 1; idx < pkt->naddr; idx++)
739 		qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
740 
741 	/* need to dma unmap the first entry, this is to restore to
742 	 * the original state so that caller can free the memory in
743 	 * error condition. Caller does not know if dma mapped or not*/
744 	if (pkt->addr[0].dma_mapped) {
745 		dma_unmap_single(&dd->pcidev->dev,
746 		       pkt->addr[0].addr,
747 		       pkt->addr[0].dma_length,
748 		       DMA_TO_DEVICE);
749 		pkt->addr[0].addr = 0;
750 		pkt->addr[0].dma_mapped = 0;
751 	}
752 
753 done:
754 	return ret;
755 }
756 
757 static int qib_user_sdma_init_payload(const struct qib_devdata *dd,
758 				      struct qib_user_sdma_queue *pq,
759 				      struct qib_user_sdma_pkt *pkt,
760 				      const struct iovec *iov,
761 				      unsigned long niov, int npages)
762 {
763 	int ret = 0;
764 
765 	if (pkt->frag_size == pkt->bytes_togo &&
766 			npages >= ARRAY_SIZE(pkt->addr))
767 		ret = qib_user_sdma_coalesce(dd, pq, pkt, iov, niov);
768 	else
769 		ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
770 
771 	return ret;
772 }
773 
774 /* free a packet list -- return counter value of last packet */
775 static void qib_user_sdma_free_pkt_list(struct device *dev,
776 					struct qib_user_sdma_queue *pq,
777 					struct list_head *list)
778 {
779 	struct qib_user_sdma_pkt *pkt, *pkt_next;
780 
781 	list_for_each_entry_safe(pkt, pkt_next, list, list) {
782 		int i;
783 
784 		for (i = 0; i < pkt->naddr; i++)
785 			qib_user_sdma_free_pkt_frag(dev, pq, pkt, i);
786 
787 		if (pkt->largepkt)
788 			kfree(pkt);
789 		else
790 			kmem_cache_free(pq->pkt_slab, pkt);
791 	}
792 	INIT_LIST_HEAD(list);
793 }
794 
795 /*
796  * copy headers, coalesce etc -- pq->lock must be held
797  *
798  * we queue all the packets to list, returning the
799  * number of bytes total.  list must be empty initially,
800  * as, if there is an error we clean it...
801  */
802 static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
803 				    struct qib_pportdata *ppd,
804 				    struct qib_user_sdma_queue *pq,
805 				    const struct iovec *iov,
806 				    unsigned long niov,
807 				    struct list_head *list,
808 				    int *maxpkts, int *ndesc)
809 {
810 	unsigned long idx = 0;
811 	int ret = 0;
812 	int npkts = 0;
813 	__le32 *pbc;
814 	dma_addr_t dma_addr;
815 	struct qib_user_sdma_pkt *pkt = NULL;
816 	size_t len;
817 	size_t nw;
818 	u32 counter = pq->counter;
819 	u16 frag_size;
820 
821 	while (idx < niov && npkts < *maxpkts) {
822 		const unsigned long addr = (unsigned long) iov[idx].iov_base;
823 		const unsigned long idx_save = idx;
824 		unsigned pktnw;
825 		unsigned pktnwc;
826 		int nfrags = 0;
827 		size_t npages = 0;
828 		size_t bytes_togo = 0;
829 		int tiddma = 0;
830 		int cfur;
831 
832 		len = iov[idx].iov_len;
833 		nw = len >> 2;
834 
835 		if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH ||
836 		    len > PAGE_SIZE || len & 3 || addr & 3) {
837 			ret = -EINVAL;
838 			goto free_list;
839 		}
840 
841 		pbc = qib_user_sdma_alloc_header(pq, len, &dma_addr);
842 		if (!pbc) {
843 			ret = -ENOMEM;
844 			goto free_list;
845 		}
846 
847 		cfur = copy_from_user(pbc, iov[idx].iov_base, len);
848 		if (cfur) {
849 			ret = -EFAULT;
850 			goto free_pbc;
851 		}
852 
853 		/*
854 		 * This assignment is a bit strange.  it's because the
855 		 * the pbc counts the number of 32 bit words in the full
856 		 * packet _except_ the first word of the pbc itself...
857 		 */
858 		pktnwc = nw - 1;
859 
860 		/*
861 		 * pktnw computation yields the number of 32 bit words
862 		 * that the caller has indicated in the PBC.  note that
863 		 * this is one less than the total number of words that
864 		 * goes to the send DMA engine as the first 32 bit word
865 		 * of the PBC itself is not counted.  Armed with this count,
866 		 * we can verify that the packet is consistent with the
867 		 * iovec lengths.
868 		 */
869 		pktnw = le32_to_cpu(*pbc) & 0xFFFF;
870 		if (pktnw < pktnwc) {
871 			ret = -EINVAL;
872 			goto free_pbc;
873 		}
874 
875 		idx++;
876 		while (pktnwc < pktnw && idx < niov) {
877 			const size_t slen = iov[idx].iov_len;
878 			const unsigned long faddr =
879 				(unsigned long) iov[idx].iov_base;
880 
881 			if (slen & 3 || faddr & 3 || !slen) {
882 				ret = -EINVAL;
883 				goto free_pbc;
884 			}
885 
886 			npages += qib_user_sdma_num_pages(&iov[idx]);
887 
888 			if (check_add_overflow(bytes_togo, slen, &bytes_togo) ||
889 			    bytes_togo > type_max(typeof(pkt->bytes_togo))) {
890 				ret = -EINVAL;
891 				goto free_pbc;
892 			}
893 			pktnwc += slen >> 2;
894 			idx++;
895 			nfrags++;
896 		}
897 
898 		if (pktnwc != pktnw) {
899 			ret = -EINVAL;
900 			goto free_pbc;
901 		}
902 
903 		frag_size = ((le32_to_cpu(*pbc))>>16) & 0xFFFF;
904 		if (((frag_size ? frag_size : bytes_togo) + len) >
905 						ppd->ibmaxlen) {
906 			ret = -EINVAL;
907 			goto free_pbc;
908 		}
909 
910 		if (frag_size) {
911 			size_t tidsmsize, n, pktsize, sz, addrlimit;
912 
913 			n = npages*((2*PAGE_SIZE/frag_size)+1);
914 			pktsize = struct_size(pkt, addr, n);
915 
916 			/*
917 			 * Determine if this is tid-sdma or just sdma.
918 			 */
919 			tiddma = (((le32_to_cpu(pbc[7])>>
920 				QLOGIC_IB_I_TID_SHIFT)&
921 				QLOGIC_IB_I_TID_MASK) !=
922 				QLOGIC_IB_I_TID_MASK);
923 
924 			if (tiddma)
925 				tidsmsize = iov[idx].iov_len;
926 			else
927 				tidsmsize = 0;
928 
929 			if (check_add_overflow(pktsize, tidsmsize, &sz)) {
930 				ret = -EINVAL;
931 				goto free_pbc;
932 			}
933 			pkt = kmalloc(sz, GFP_KERNEL);
934 			if (!pkt) {
935 				ret = -ENOMEM;
936 				goto free_pbc;
937 			}
938 			pkt->largepkt = 1;
939 			pkt->frag_size = frag_size;
940 			if (check_add_overflow(n, ARRAY_SIZE(pkt->addr),
941 					       &addrlimit) ||
942 			    addrlimit > type_max(typeof(pkt->addrlimit))) {
943 				ret = -EINVAL;
944 				goto free_pkt;
945 			}
946 			pkt->addrlimit = addrlimit;
947 
948 			if (tiddma) {
949 				char *tidsm = (char *)pkt + pktsize;
950 
951 				cfur = copy_from_user(tidsm,
952 					iov[idx].iov_base, tidsmsize);
953 				if (cfur) {
954 					ret = -EFAULT;
955 					goto free_pkt;
956 				}
957 				pkt->tidsm =
958 					(struct qib_tid_session_member *)tidsm;
959 				pkt->tidsmcount = tidsmsize/
960 					sizeof(struct qib_tid_session_member);
961 				pkt->tidsmidx = 0;
962 				idx++;
963 			}
964 
965 			/*
966 			 * pbc 'fill1' field is borrowed to pass frag size,
967 			 * we need to clear it after picking frag size, the
968 			 * hardware requires this field to be zero.
969 			 */
970 			*pbc = cpu_to_le32(le32_to_cpu(*pbc) & 0x0000FFFF);
971 		} else {
972 			pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
973 			if (!pkt) {
974 				ret = -ENOMEM;
975 				goto free_pbc;
976 			}
977 			pkt->largepkt = 0;
978 			pkt->frag_size = bytes_togo;
979 			pkt->addrlimit = ARRAY_SIZE(pkt->addr);
980 		}
981 		pkt->bytes_togo = bytes_togo;
982 		pkt->payload_size = 0;
983 		pkt->counter = counter;
984 		pkt->tiddma = tiddma;
985 
986 		/* setup the first header */
987 		qib_user_sdma_init_frag(pkt, 0, /* index */
988 			0, len,		/* offset, len */
989 			1, 0,		/* first last desc */
990 			0, 0,		/* put page, dma mapped */
991 			NULL, pbc,	/* struct page, virt addr */
992 			dma_addr, len);	/* dma addr, dma length */
993 		pkt->index = 0;
994 		pkt->naddr = 1;
995 
996 		if (nfrags) {
997 			ret = qib_user_sdma_init_payload(dd, pq, pkt,
998 							 iov + idx_save + 1,
999 							 nfrags, npages);
1000 			if (ret < 0)
1001 				goto free_pkt;
1002 		} else {
1003 			/* since there is no payload, mark the
1004 			 * header as the last desc. */
1005 			pkt->addr[0].last_desc = 1;
1006 
1007 			if (dma_addr == 0) {
1008 				/*
1009 				 * the header is not dma mapped yet.
1010 				 * it should be from kmalloc.
1011 				 */
1012 				dma_addr = dma_map_single(&dd->pcidev->dev,
1013 					pbc, len, DMA_TO_DEVICE);
1014 				if (dma_mapping_error(&dd->pcidev->dev,
1015 								dma_addr)) {
1016 					ret = -ENOMEM;
1017 					goto free_pkt;
1018 				}
1019 				pkt->addr[0].addr = dma_addr;
1020 				pkt->addr[0].dma_mapped = 1;
1021 			}
1022 		}
1023 
1024 		counter++;
1025 		npkts++;
1026 		pkt->pq = pq;
1027 		pkt->index = 0; /* reset index for push on hw */
1028 		*ndesc += pkt->naddr;
1029 
1030 		list_add_tail(&pkt->list, list);
1031 	}
1032 
1033 	*maxpkts = npkts;
1034 	ret = idx;
1035 	goto done;
1036 
1037 free_pkt:
1038 	if (pkt->largepkt)
1039 		kfree(pkt);
1040 	else
1041 		kmem_cache_free(pq->pkt_slab, pkt);
1042 free_pbc:
1043 	if (dma_addr)
1044 		dma_pool_free(pq->header_cache, pbc, dma_addr);
1045 	else
1046 		kfree(pbc);
1047 free_list:
1048 	qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
1049 done:
1050 	return ret;
1051 }
1052 
1053 static void qib_user_sdma_set_complete_counter(struct qib_user_sdma_queue *pq,
1054 					       u32 c)
1055 {
1056 	pq->sent_counter = c;
1057 }
1058 
1059 /* try to clean out queue -- needs pq->lock */
1060 static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
1061 				     struct qib_user_sdma_queue *pq)
1062 {
1063 	struct qib_devdata *dd = ppd->dd;
1064 	struct list_head free_list;
1065 	struct qib_user_sdma_pkt *pkt;
1066 	struct qib_user_sdma_pkt *pkt_prev;
1067 	unsigned long flags;
1068 	int ret = 0;
1069 
1070 	if (!pq->num_sending)
1071 		return 0;
1072 
1073 	INIT_LIST_HEAD(&free_list);
1074 
1075 	/*
1076 	 * We need this spin lock here because interrupt handler
1077 	 * might modify this list in qib_user_sdma_send_desc(), also
1078 	 * we can not get interrupted, otherwise it is a deadlock.
1079 	 */
1080 	spin_lock_irqsave(&pq->sent_lock, flags);
1081 	list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
1082 		s64 descd = ppd->sdma_descq_removed - pkt->added;
1083 
1084 		if (descd < 0)
1085 			break;
1086 
1087 		list_move_tail(&pkt->list, &free_list);
1088 
1089 		/* one more packet cleaned */
1090 		ret++;
1091 		pq->num_sending--;
1092 	}
1093 	spin_unlock_irqrestore(&pq->sent_lock, flags);
1094 
1095 	if (!list_empty(&free_list)) {
1096 		u32 counter;
1097 
1098 		pkt = list_entry(free_list.prev,
1099 				 struct qib_user_sdma_pkt, list);
1100 		counter = pkt->counter;
1101 
1102 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1103 		qib_user_sdma_set_complete_counter(pq, counter);
1104 	}
1105 
1106 	return ret;
1107 }
1108 
1109 void qib_user_sdma_queue_destroy(struct qib_user_sdma_queue *pq)
1110 {
1111 	if (!pq)
1112 		return;
1113 
1114 	pq->sdma_rb_node->refcount--;
1115 	if (pq->sdma_rb_node->refcount == 0) {
1116 		rb_erase(&pq->sdma_rb_node->node, &qib_user_sdma_rb_root);
1117 		kfree(pq->sdma_rb_node);
1118 	}
1119 	dma_pool_destroy(pq->header_cache);
1120 	kmem_cache_destroy(pq->pkt_slab);
1121 	kfree(pq);
1122 }
1123 
1124 /* clean descriptor queue, returns > 0 if some elements cleaned */
1125 static int qib_user_sdma_hwqueue_clean(struct qib_pportdata *ppd)
1126 {
1127 	int ret;
1128 	unsigned long flags;
1129 
1130 	spin_lock_irqsave(&ppd->sdma_lock, flags);
1131 	ret = qib_sdma_make_progress(ppd);
1132 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1133 
1134 	return ret;
1135 }
1136 
1137 /* we're in close, drain packets so that we can cleanup successfully... */
1138 void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
1139 			       struct qib_user_sdma_queue *pq)
1140 {
1141 	struct qib_devdata *dd = ppd->dd;
1142 	unsigned long flags;
1143 	int i;
1144 
1145 	if (!pq)
1146 		return;
1147 
1148 	for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) {
1149 		mutex_lock(&pq->lock);
1150 		if (!pq->num_pending && !pq->num_sending) {
1151 			mutex_unlock(&pq->lock);
1152 			break;
1153 		}
1154 		qib_user_sdma_hwqueue_clean(ppd);
1155 		qib_user_sdma_queue_clean(ppd, pq);
1156 		mutex_unlock(&pq->lock);
1157 		msleep(20);
1158 	}
1159 
1160 	if (pq->num_pending || pq->num_sending) {
1161 		struct qib_user_sdma_pkt *pkt;
1162 		struct qib_user_sdma_pkt *pkt_prev;
1163 		struct list_head free_list;
1164 
1165 		mutex_lock(&pq->lock);
1166 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1167 		/*
1168 		 * Since we hold sdma_lock, it is safe without sent_lock.
1169 		 */
1170 		if (pq->num_pending) {
1171 			list_for_each_entry_safe(pkt, pkt_prev,
1172 					&ppd->sdma_userpending, list) {
1173 				if (pkt->pq == pq) {
1174 					list_move_tail(&pkt->list, &pq->sent);
1175 					pq->num_pending--;
1176 					pq->num_sending++;
1177 				}
1178 			}
1179 		}
1180 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1181 
1182 		qib_dev_err(dd, "user sdma lists not empty: forcing!\n");
1183 		INIT_LIST_HEAD(&free_list);
1184 		list_splice_init(&pq->sent, &free_list);
1185 		pq->num_sending = 0;
1186 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
1187 		mutex_unlock(&pq->lock);
1188 	}
1189 }
1190 
1191 static inline __le64 qib_sdma_make_desc0(u8 gen,
1192 					 u64 addr, u64 dwlen, u64 dwoffset)
1193 {
1194 	return cpu_to_le64(/* SDmaPhyAddr[31:0] */
1195 			   ((addr & 0xfffffffcULL) << 32) |
1196 			   /* SDmaGeneration[1:0] */
1197 			   ((gen & 3ULL) << 30) |
1198 			   /* SDmaDwordCount[10:0] */
1199 			   ((dwlen & 0x7ffULL) << 16) |
1200 			   /* SDmaBufOffset[12:2] */
1201 			   (dwoffset & 0x7ffULL));
1202 }
1203 
1204 static inline __le64 qib_sdma_make_first_desc0(__le64 descq)
1205 {
1206 	return descq | cpu_to_le64(1ULL << 12);
1207 }
1208 
1209 static inline __le64 qib_sdma_make_last_desc0(__le64 descq)
1210 {
1211 					      /* last */  /* dma head */
1212 	return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13);
1213 }
1214 
1215 static inline __le64 qib_sdma_make_desc1(u64 addr)
1216 {
1217 	/* SDmaPhyAddr[47:32] */
1218 	return cpu_to_le64(addr >> 32);
1219 }
1220 
1221 static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
1222 				    struct qib_user_sdma_pkt *pkt, int idx,
1223 				    unsigned ofs, u16 tail, u8 gen)
1224 {
1225 	const u64 addr = (u64) pkt->addr[idx].addr +
1226 		(u64) pkt->addr[idx].offset;
1227 	const u64 dwlen = (u64) pkt->addr[idx].length / 4;
1228 	__le64 *descqp;
1229 	__le64 descq0;
1230 
1231 	descqp = &ppd->sdma_descq[tail].qw[0];
1232 
1233 	descq0 = qib_sdma_make_desc0(gen, addr, dwlen, ofs);
1234 	if (pkt->addr[idx].first_desc)
1235 		descq0 = qib_sdma_make_first_desc0(descq0);
1236 	if (pkt->addr[idx].last_desc) {
1237 		descq0 = qib_sdma_make_last_desc0(descq0);
1238 		if (ppd->sdma_intrequest) {
1239 			descq0 |= cpu_to_le64(1ULL << 15);
1240 			ppd->sdma_intrequest = 0;
1241 		}
1242 	}
1243 
1244 	descqp[0] = descq0;
1245 	descqp[1] = qib_sdma_make_desc1(addr);
1246 }
1247 
1248 void qib_user_sdma_send_desc(struct qib_pportdata *ppd,
1249 				struct list_head *pktlist)
1250 {
1251 	struct qib_devdata *dd = ppd->dd;
1252 	u16 nfree, nsent;
1253 	u16 tail, tail_c;
1254 	u8 gen, gen_c;
1255 
1256 	nfree = qib_sdma_descq_freecnt(ppd);
1257 	if (!nfree)
1258 		return;
1259 
1260 retry:
1261 	nsent = 0;
1262 	tail_c = tail = ppd->sdma_descq_tail;
1263 	gen_c = gen = ppd->sdma_generation;
1264 	while (!list_empty(pktlist)) {
1265 		struct qib_user_sdma_pkt *pkt =
1266 			list_entry(pktlist->next, struct qib_user_sdma_pkt,
1267 				   list);
1268 		int i, j, c = 0;
1269 		unsigned ofs = 0;
1270 		u16 dtail = tail;
1271 
1272 		for (i = pkt->index; i < pkt->naddr && nfree; i++) {
1273 			qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);
1274 			ofs += pkt->addr[i].length >> 2;
1275 
1276 			if (++tail == ppd->sdma_descq_cnt) {
1277 				tail = 0;
1278 				++gen;
1279 				ppd->sdma_intrequest = 1;
1280 			} else if (tail == (ppd->sdma_descq_cnt>>1)) {
1281 				ppd->sdma_intrequest = 1;
1282 			}
1283 			nfree--;
1284 			if (pkt->addr[i].last_desc == 0)
1285 				continue;
1286 
1287 			/*
1288 			 * If the packet is >= 2KB mtu equivalent, we
1289 			 * have to use the large buffers, and have to
1290 			 * mark each descriptor as part of a large
1291 			 * buffer packet.
1292 			 */
1293 			if (ofs > dd->piosize2kmax_dwords) {
1294 				for (j = pkt->index; j <= i; j++) {
1295 					ppd->sdma_descq[dtail].qw[0] |=
1296 						cpu_to_le64(1ULL << 14);
1297 					if (++dtail == ppd->sdma_descq_cnt)
1298 						dtail = 0;
1299 				}
1300 			}
1301 			c += i + 1 - pkt->index;
1302 			pkt->index = i + 1; /* index for next first */
1303 			tail_c = dtail = tail;
1304 			gen_c = gen;
1305 			ofs = 0;  /* reset for next packet */
1306 		}
1307 
1308 		ppd->sdma_descq_added += c;
1309 		nsent += c;
1310 		if (pkt->index == pkt->naddr) {
1311 			pkt->added = ppd->sdma_descq_added;
1312 			pkt->pq->added = pkt->added;
1313 			pkt->pq->num_pending--;
1314 			spin_lock(&pkt->pq->sent_lock);
1315 			pkt->pq->num_sending++;
1316 			list_move_tail(&pkt->list, &pkt->pq->sent);
1317 			spin_unlock(&pkt->pq->sent_lock);
1318 		}
1319 		if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)
1320 			break;
1321 	}
1322 
1323 	/* advance the tail on the chip if necessary */
1324 	if (ppd->sdma_descq_tail != tail_c) {
1325 		ppd->sdma_generation = gen_c;
1326 		dd->f_sdma_update_tail(ppd, tail_c);
1327 	}
1328 
1329 	if (nfree && !list_empty(pktlist))
1330 		goto retry;
1331 }
1332 
1333 /* pq->lock must be held, get packets on the wire... */
1334 static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,
1335 				 struct qib_user_sdma_queue *pq,
1336 				 struct list_head *pktlist, int count)
1337 {
1338 	unsigned long flags;
1339 
1340 	if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))
1341 		return -ECOMM;
1342 
1343 	/* non-blocking mode */
1344 	if (pq->sdma_rb_node->refcount > 1) {
1345 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1346 		if (unlikely(!__qib_sdma_running(ppd))) {
1347 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1348 			return -ECOMM;
1349 		}
1350 		pq->num_pending += count;
1351 		list_splice_tail_init(pktlist, &ppd->sdma_userpending);
1352 		qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
1353 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1354 		return 0;
1355 	}
1356 
1357 	/* In this case, descriptors from this process are not
1358 	 * linked to ppd pending queue, interrupt handler
1359 	 * won't update this process, it is OK to directly
1360 	 * modify without sdma lock.
1361 	 */
1362 
1363 
1364 	pq->num_pending += count;
1365 	/*
1366 	 * Blocking mode for single rail process, we must
1367 	 * release/regain sdma_lock to give other process
1368 	 * chance to make progress. This is important for
1369 	 * performance.
1370 	 */
1371 	do {
1372 		spin_lock_irqsave(&ppd->sdma_lock, flags);
1373 		if (unlikely(!__qib_sdma_running(ppd))) {
1374 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1375 			return -ECOMM;
1376 		}
1377 		qib_user_sdma_send_desc(ppd, pktlist);
1378 		if (!list_empty(pktlist))
1379 			qib_sdma_make_progress(ppd);
1380 		spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1381 	} while (!list_empty(pktlist));
1382 
1383 	return 0;
1384 }
1385 
1386 int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
1387 			 struct qib_user_sdma_queue *pq,
1388 			 const struct iovec *iov,
1389 			 unsigned long dim)
1390 {
1391 	struct qib_devdata *dd = rcd->dd;
1392 	struct qib_pportdata *ppd = rcd->ppd;
1393 	int ret = 0;
1394 	struct list_head list;
1395 	int npkts = 0;
1396 
1397 	INIT_LIST_HEAD(&list);
1398 
1399 	mutex_lock(&pq->lock);
1400 
1401 	/* why not -ECOMM like qib_user_sdma_push_pkts() below? */
1402 	if (!qib_sdma_running(ppd))
1403 		goto done_unlock;
1404 
1405 	/* if I have packets not complete yet */
1406 	if (pq->added > ppd->sdma_descq_removed)
1407 		qib_user_sdma_hwqueue_clean(ppd);
1408 	/* if I have complete packets to be freed */
1409 	if (pq->num_sending)
1410 		qib_user_sdma_queue_clean(ppd, pq);
1411 
1412 	while (dim) {
1413 		int mxp = 1;
1414 		int ndesc = 0;
1415 
1416 		ret = qib_user_sdma_queue_pkts(dd, ppd, pq,
1417 				iov, dim, &list, &mxp, &ndesc);
1418 		if (ret < 0)
1419 			goto done_unlock;
1420 		else {
1421 			dim -= ret;
1422 			iov += ret;
1423 		}
1424 
1425 		/* force packets onto the sdma hw queue... */
1426 		if (!list_empty(&list)) {
1427 			/*
1428 			 * Lazily clean hw queue.
1429 			 */
1430 			if (qib_sdma_descq_freecnt(ppd) < ndesc) {
1431 				qib_user_sdma_hwqueue_clean(ppd);
1432 				if (pq->num_sending)
1433 					qib_user_sdma_queue_clean(ppd, pq);
1434 			}
1435 
1436 			ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);
1437 			if (ret < 0)
1438 				goto done_unlock;
1439 			else {
1440 				npkts += mxp;
1441 				pq->counter += mxp;
1442 			}
1443 		}
1444 	}
1445 
1446 done_unlock:
1447 	if (!list_empty(&list))
1448 		qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list);
1449 	mutex_unlock(&pq->lock);
1450 
1451 	return (ret < 0) ? ret : npkts;
1452 }
1453 
1454 int qib_user_sdma_make_progress(struct qib_pportdata *ppd,
1455 				struct qib_user_sdma_queue *pq)
1456 {
1457 	int ret = 0;
1458 
1459 	mutex_lock(&pq->lock);
1460 	qib_user_sdma_hwqueue_clean(ppd);
1461 	ret = qib_user_sdma_queue_clean(ppd, pq);
1462 	mutex_unlock(&pq->lock);
1463 
1464 	return ret;
1465 }
1466 
1467 u32 qib_user_sdma_complete_counter(const struct qib_user_sdma_queue *pq)
1468 {
1469 	return pq ? pq->sent_counter : 0;
1470 }
1471 
1472 u32 qib_user_sdma_inflight_counter(struct qib_user_sdma_queue *pq)
1473 {
1474 	return pq ? pq->counter : 0;
1475 }
1476