1 /*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
6 * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
7 * Copyright (c) 2016 Matthew Macy
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice unmodified, this list of conditions, and the following
15 * disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #ifndef _LINUXKPI_LINUX_SCATTERLIST_H_
32 #define _LINUXKPI_LINUX_SCATTERLIST_H_
33
34 #include <sys/types.h>
35 #include <sys/proc.h>
36 #include <sys/sched.h>
37 #include <sys/sf_buf.h>
38
39 #include <linux/err.h>
40 #include <linux/page.h>
41 #include <linux/slab.h>
42 #include <linux/mm.h>
43
44 struct bus_dmamap;
45 struct scatterlist {
46 unsigned long page_link;
47 #define SG_PAGE_LINK_CHAIN 0x1UL
48 #define SG_PAGE_LINK_LAST 0x2UL
49 #define SG_PAGE_LINK_MASK 0x3UL
50 unsigned int offset;
51 unsigned int length;
52 dma_addr_t dma_address;
53 struct bus_dmamap *dma_map; /* FreeBSD specific */
54 };
55
56 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0);
57
58 struct sg_table {
59 struct scatterlist *sgl;
60 unsigned int nents;
61 unsigned int orig_nents;
62 };
63
64 struct sg_page_iter {
65 struct scatterlist *sg;
66 unsigned int sg_pgoffset;
67 unsigned int maxents;
68 struct {
69 unsigned int nents;
70 int pg_advance;
71 } internal;
72 };
73
74 struct sg_dma_page_iter {
75 struct sg_page_iter base;
76 };
77
78 #define SCATTERLIST_MAX_SEGMENT (-1U & ~(PAGE_SIZE - 1))
79
80 #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist))
81
82 #define SG_MAGIC 0x87654321UL
83 #define SG_CHAIN SG_PAGE_LINK_CHAIN
84 #define SG_END SG_PAGE_LINK_LAST
85
86 #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN)
87 #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST)
88 #define sg_chain_ptr(sg) \
89 ((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK))
90
91 #define sg_dma_address(sg) (sg)->dma_address
92 #define sg_dma_len(sg) (sg)->length
93
94 #define for_each_sg_page(sgl, iter, nents, pgoffset) \
95 for (_sg_iter_init(sgl, iter, nents, pgoffset); \
96 (iter)->sg; _sg_iter_next(iter))
97 #define for_each_sg_dma_page(sgl, iter, nents, pgoffset) \
98 for_each_sg_page(sgl, &(iter)->base, nents, pgoffset)
99
100 #define for_each_sg(sglist, sg, sgmax, iter) \
101 for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg))
102
103 #define for_each_sgtable_sg(sgt, sg, i) \
104 for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
105
106 #define for_each_sgtable_page(sgt, iter, pgoffset) \
107 for_each_sg_page((sgt)->sgl, iter, (sgt)->orig_nents, pgoffset)
108
109 #define for_each_sgtable_dma_sg(sgt, sg, iter) \
110 for_each_sg((sgt)->sgl, sg, (sgt)->nents, iter)
111
112 #define for_each_sgtable_dma_page(sgt, iter, pgoffset) \
113 for_each_sg_dma_page((sgt)->sgl, iter, (sgt)->nents, pgoffset)
114
115 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t);
116 typedef void (sg_free_fn) (struct scatterlist *, unsigned int);
117
118 static inline void
sg_assign_page(struct scatterlist * sg,struct page * page)119 sg_assign_page(struct scatterlist *sg, struct page *page)
120 {
121 unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK;
122
123 sg->page_link = page_link | (unsigned long)page;
124 }
125
126 static inline void
sg_set_page(struct scatterlist * sg,struct page * page,unsigned int len,unsigned int offset)127 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
128 unsigned int offset)
129 {
130 sg_assign_page(sg, page);
131 sg->offset = offset;
132 sg->length = len;
133 }
134
135 static inline struct page *
sg_page(struct scatterlist * sg)136 sg_page(struct scatterlist *sg)
137 {
138 return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK));
139 }
140
141 static inline void
sg_set_buf(struct scatterlist * sg,const void * buf,unsigned int buflen)142 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)
143 {
144 sg_set_page(sg, virt_to_page(buf), buflen,
145 ((uintptr_t)buf) & (PAGE_SIZE - 1));
146 }
147
148 static inline struct scatterlist *
sg_next(struct scatterlist * sg)149 sg_next(struct scatterlist *sg)
150 {
151 if (sg_is_last(sg))
152 return (NULL);
153 sg++;
154 if (sg_is_chain(sg))
155 sg = sg_chain_ptr(sg);
156 return (sg);
157 }
158
159 static inline vm_paddr_t
sg_phys(struct scatterlist * sg)160 sg_phys(struct scatterlist *sg)
161 {
162 return (page_to_phys(sg_page(sg)) + sg->offset);
163 }
164
165 static inline void *
sg_virt(struct scatterlist * sg)166 sg_virt(struct scatterlist *sg)
167 {
168
169 return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset));
170 }
171
172 static inline void
sg_chain(struct scatterlist * prv,unsigned int prv_nents,struct scatterlist * sgl)173 sg_chain(struct scatterlist *prv, unsigned int prv_nents,
174 struct scatterlist *sgl)
175 {
176 struct scatterlist *sg = &prv[prv_nents - 1];
177
178 sg->offset = 0;
179 sg->length = 0;
180 sg->page_link = ((unsigned long)sgl |
181 SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST;
182 }
183
184 static inline void
sg_mark_end(struct scatterlist * sg)185 sg_mark_end(struct scatterlist *sg)
186 {
187 sg->page_link |= SG_PAGE_LINK_LAST;
188 sg->page_link &= ~SG_PAGE_LINK_CHAIN;
189 }
190
191 static inline void
sg_init_table(struct scatterlist * sg,unsigned int nents)192 sg_init_table(struct scatterlist *sg, unsigned int nents)
193 {
194 bzero(sg, sizeof(*sg) * nents);
195 sg_mark_end(&sg[nents - 1]);
196 }
197
198 static inline void
sg_init_one(struct scatterlist * sg,const void * buf,unsigned int buflen)199 sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)
200 {
201 sg_init_table(sg, 1);
202 sg_set_buf(sg, buf, buflen);
203 }
204
205 static struct scatterlist *
sg_kmalloc(unsigned int nents,gfp_t gfp_mask)206 sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
207 {
208 if (nents == SG_MAX_SINGLE_ALLOC) {
209 return ((void *)__get_free_page(gfp_mask));
210 } else
211 return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask));
212 }
213
214 static inline void
sg_kfree(struct scatterlist * sg,unsigned int nents)215 sg_kfree(struct scatterlist *sg, unsigned int nents)
216 {
217 if (nents == SG_MAX_SINGLE_ALLOC) {
218 free_page((unsigned long)sg);
219 } else
220 kfree(sg);
221 }
222
223 static inline void
__sg_free_table(struct sg_table * table,unsigned int max_ents,bool skip_first_chunk,sg_free_fn * free_fn)224 __sg_free_table(struct sg_table *table, unsigned int max_ents,
225 bool skip_first_chunk, sg_free_fn * free_fn)
226 {
227 struct scatterlist *sgl, *next;
228
229 if (unlikely(!table->sgl))
230 return;
231
232 sgl = table->sgl;
233 while (table->orig_nents) {
234 unsigned int alloc_size = table->orig_nents;
235 unsigned int sg_size;
236
237 if (alloc_size > max_ents) {
238 next = sg_chain_ptr(&sgl[max_ents - 1]);
239 alloc_size = max_ents;
240 sg_size = alloc_size - 1;
241 } else {
242 sg_size = alloc_size;
243 next = NULL;
244 }
245
246 table->orig_nents -= sg_size;
247 if (skip_first_chunk)
248 skip_first_chunk = 0;
249 else
250 free_fn(sgl, alloc_size);
251 sgl = next;
252 }
253
254 table->sgl = NULL;
255 }
256
257 static inline void
sg_free_table(struct sg_table * table)258 sg_free_table(struct sg_table *table)
259 {
260 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
261 }
262
263 static inline int
__sg_alloc_table(struct sg_table * table,unsigned int nents,unsigned int max_ents,struct scatterlist * first_chunk,gfp_t gfp_mask,sg_alloc_fn * alloc_fn)264 __sg_alloc_table(struct sg_table *table, unsigned int nents,
265 unsigned int max_ents, struct scatterlist *first_chunk,
266 gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
267 {
268 struct scatterlist *sg, *prv;
269 unsigned int left;
270
271 memset(table, 0, sizeof(*table));
272
273 if (nents == 0)
274 return (-EINVAL);
275 left = nents;
276 prv = NULL;
277 do {
278 unsigned int sg_size;
279 unsigned int alloc_size = left;
280
281 if (alloc_size > max_ents) {
282 alloc_size = max_ents;
283 sg_size = alloc_size - 1;
284 } else
285 sg_size = alloc_size;
286
287 left -= sg_size;
288
289 if (first_chunk) {
290 sg = first_chunk;
291 first_chunk = NULL;
292 } else {
293 sg = alloc_fn(alloc_size, gfp_mask);
294 }
295 if (unlikely(!sg)) {
296 if (prv)
297 table->nents = ++table->orig_nents;
298
299 return (-ENOMEM);
300 }
301 sg_init_table(sg, alloc_size);
302 table->nents = table->orig_nents += sg_size;
303
304 if (prv)
305 sg_chain(prv, max_ents, sg);
306 else
307 table->sgl = sg;
308
309 if (!left)
310 sg_mark_end(&sg[sg_size - 1]);
311
312 prv = sg;
313 } while (left);
314
315 return (0);
316 }
317
318 static inline int
sg_alloc_table(struct sg_table * table,unsigned int nents,gfp_t gfp_mask)319 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
320 {
321 int ret;
322
323 ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
324 NULL, gfp_mask, sg_kmalloc);
325 if (unlikely(ret))
326 __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
327
328 return (ret);
329 }
330
331 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
332 static inline struct scatterlist *
__sg_alloc_table_from_pages(struct sg_table * sgt,struct page ** pages,unsigned int count,unsigned long off,unsigned long size,unsigned int max_segment,struct scatterlist * prv,unsigned int left_pages,gfp_t gfp_mask)333 __sg_alloc_table_from_pages(struct sg_table *sgt,
334 struct page **pages, unsigned int count,
335 unsigned long off, unsigned long size,
336 unsigned int max_segment,
337 struct scatterlist *prv, unsigned int left_pages,
338 gfp_t gfp_mask)
339 #else
340 static inline int
341 __sg_alloc_table_from_pages(struct sg_table *sgt,
342 struct page **pages, unsigned int count,
343 unsigned long off, unsigned long size,
344 unsigned int max_segment, gfp_t gfp_mask)
345 #endif
346 {
347 unsigned int i, segs, cur, len;
348 int rc;
349 struct scatterlist *s, *sg_iter;
350
351 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
352 if (prv != NULL) {
353 panic(
354 "Support for prv != NULL not implemented in "
355 "__sg_alloc_table_from_pages()");
356 }
357 #endif
358
359 if (__predict_false(!max_segment || offset_in_page(max_segment)))
360 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
361 return (ERR_PTR(-EINVAL));
362 #else
363 return (-EINVAL);
364 #endif
365
366 len = 0;
367 for (segs = i = 1; i < count; ++i) {
368 len += PAGE_SIZE;
369 if (len >= max_segment ||
370 page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) {
371 ++segs;
372 len = 0;
373 }
374 }
375 if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask))))
376 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
377 return (ERR_PTR(rc));
378 #else
379 return (rc);
380 #endif
381
382 cur = 0;
383 for_each_sg(sgt->sgl, sg_iter, sgt->orig_nents, i) {
384 unsigned long seg_size;
385 unsigned int j;
386
387 /*
388 * We need to make sure that when we exit this loop "s" has the
389 * last sg in the chain so we can call sg_mark_end() on it.
390 * Only set this inside the loop since sg_iter will be iterated
391 * until it is NULL.
392 */
393 s = sg_iter;
394
395 len = 0;
396 for (j = cur + 1; j < count; ++j) {
397 len += PAGE_SIZE;
398 if (len >= max_segment || page_to_pfn(pages[j]) !=
399 page_to_pfn(pages[j - 1]) + 1)
400 break;
401 }
402
403 seg_size = ((j - cur) << PAGE_SHIFT) - off;
404 sg_set_page(s, pages[cur], MIN(size, seg_size), off);
405 size -= seg_size;
406 off = 0;
407 cur = j;
408 }
409 KASSERT(s != NULL, ("s is NULL after loop in __sg_alloc_table_from_pages()"));
410
411 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
412 if (left_pages == 0)
413 sg_mark_end(s);
414
415 return (s);
416 #else
417 return (0);
418 #endif
419 }
420
421 static inline int
sg_alloc_table_from_pages(struct sg_table * sgt,struct page ** pages,unsigned int count,unsigned long off,unsigned long size,gfp_t gfp_mask)422 sg_alloc_table_from_pages(struct sg_table *sgt,
423 struct page **pages, unsigned int count,
424 unsigned long off, unsigned long size,
425 gfp_t gfp_mask)
426 {
427
428 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
429 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size,
430 SCATTERLIST_MAX_SEGMENT, NULL, 0, gfp_mask)));
431 #else
432 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size,
433 SCATTERLIST_MAX_SEGMENT, gfp_mask));
434 #endif
435 }
436
437 static inline int
sg_alloc_table_from_pages_segment(struct sg_table * sgt,struct page ** pages,unsigned int count,unsigned int off,unsigned long size,unsigned int max_segment,gfp_t gfp_mask)438 sg_alloc_table_from_pages_segment(struct sg_table *sgt,
439 struct page **pages, unsigned int count, unsigned int off,
440 unsigned long size, unsigned int max_segment, gfp_t gfp_mask)
441 {
442 #if defined(LINUXKPI_VERSION) && LINUXKPI_VERSION >= 51300
443 return (PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, count, off, size,
444 max_segment, NULL, 0, gfp_mask)));
445 #else
446 return (__sg_alloc_table_from_pages(sgt, pages, count, off, size,
447 max_segment, gfp_mask));
448 #endif
449 }
450
451 static inline int
sg_nents(struct scatterlist * sg)452 sg_nents(struct scatterlist *sg)
453 {
454 int nents;
455
456 for (nents = 0; sg; sg = sg_next(sg))
457 nents++;
458 return (nents);
459 }
460
461 static inline void
__sg_page_iter_start(struct sg_page_iter * piter,struct scatterlist * sglist,unsigned int nents,unsigned long pgoffset)462 __sg_page_iter_start(struct sg_page_iter *piter,
463 struct scatterlist *sglist, unsigned int nents,
464 unsigned long pgoffset)
465 {
466 piter->internal.pg_advance = 0;
467 piter->internal.nents = nents;
468
469 piter->sg = sglist;
470 piter->sg_pgoffset = pgoffset;
471 }
472
473 static inline void
_sg_iter_next(struct sg_page_iter * iter)474 _sg_iter_next(struct sg_page_iter *iter)
475 {
476 struct scatterlist *sg;
477 unsigned int pgcount;
478
479 sg = iter->sg;
480 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
481
482 ++iter->sg_pgoffset;
483 while (iter->sg_pgoffset >= pgcount) {
484 iter->sg_pgoffset -= pgcount;
485 sg = sg_next(sg);
486 --iter->maxents;
487 if (sg == NULL || iter->maxents == 0)
488 break;
489 pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
490 }
491 iter->sg = sg;
492 }
493
494 static inline int
sg_page_count(struct scatterlist * sg)495 sg_page_count(struct scatterlist *sg)
496 {
497 return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT);
498 }
499 #define sg_dma_page_count(sg) \
500 sg_page_count(sg)
501
502 static inline bool
__sg_page_iter_next(struct sg_page_iter * piter)503 __sg_page_iter_next(struct sg_page_iter *piter)
504 {
505 unsigned int pgcount;
506
507 if (piter->internal.nents == 0)
508 return (0);
509 if (piter->sg == NULL)
510 return (0);
511
512 piter->sg_pgoffset += piter->internal.pg_advance;
513 piter->internal.pg_advance = 1;
514
515 while (1) {
516 pgcount = sg_page_count(piter->sg);
517 if (likely(piter->sg_pgoffset < pgcount))
518 break;
519 piter->sg_pgoffset -= pgcount;
520 piter->sg = sg_next(piter->sg);
521 if (--piter->internal.nents == 0)
522 return (0);
523 if (piter->sg == NULL)
524 return (0);
525 }
526 return (1);
527 }
528 #define __sg_page_iter_dma_next(itr) \
529 __sg_page_iter_next(&(itr)->base)
530
531 static inline void
_sg_iter_init(struct scatterlist * sgl,struct sg_page_iter * iter,unsigned int nents,unsigned long pgoffset)532 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter,
533 unsigned int nents, unsigned long pgoffset)
534 {
535 if (nents) {
536 iter->sg = sgl;
537 iter->sg_pgoffset = pgoffset - 1;
538 iter->maxents = nents;
539 _sg_iter_next(iter);
540 } else {
541 iter->sg = NULL;
542 iter->sg_pgoffset = 0;
543 iter->maxents = 0;
544 }
545 }
546
547 /*
548 * sg_page_iter_dma_address() is implemented as a macro because it
549 * needs to accept two different and identical structure types. This
550 * allows both old and new code to co-exist. The compile time assert
551 * adds some safety, that the structure sizes match.
552 */
553 #define sg_page_iter_dma_address(spi) ({ \
554 struct sg_page_iter *__spi = (void *)(spi); \
555 dma_addr_t __dma_address; \
556 CTASSERT(sizeof(*(spi)) == sizeof(*__spi)); \
557 __dma_address = __spi->sg->dma_address + \
558 (__spi->sg_pgoffset << PAGE_SHIFT); \
559 __dma_address; \
560 })
561
562 static inline struct page *
sg_page_iter_page(struct sg_page_iter * piter)563 sg_page_iter_page(struct sg_page_iter *piter)
564 {
565 return (nth_page(sg_page(piter->sg), piter->sg_pgoffset));
566 }
567
568 static __inline size_t
sg_pcopy_from_buffer(struct scatterlist * sgl,unsigned int nents,const void * buf,size_t buflen,off_t skip)569 sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
570 const void *buf, size_t buflen, off_t skip)
571 {
572 struct sg_page_iter piter;
573 struct page *page;
574 struct sf_buf *sf;
575 size_t len, copied;
576 char *p, *b;
577
578 if (buflen == 0)
579 return (0);
580
581 b = __DECONST(char *, buf);
582 copied = 0;
583 sched_pin();
584 for_each_sg_page(sgl, &piter, nents, 0) {
585
586 /* Skip to the start. */
587 if (piter.sg->length <= skip) {
588 skip -= piter.sg->length;
589 continue;
590 }
591
592 /* See how much to copy. */
593 KASSERT(((piter.sg->length - skip) != 0 && (buflen != 0)),
594 ("%s: sg len %u - skip %ju || buflen %zu is 0\n",
595 __func__, piter.sg->length, (uintmax_t)skip, buflen));
596 len = min(piter.sg->length - skip, buflen);
597
598 page = sg_page_iter_page(&piter);
599 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT);
600 if (sf == NULL)
601 break;
602 p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip;
603 memcpy(p, b, len);
604 sf_buf_free(sf);
605
606 /* We copied so nothing more to skip. */
607 skip = 0;
608 copied += len;
609 /* Either we exactly filled the page, or we are done. */
610 buflen -= len;
611 if (buflen == 0)
612 break;
613 b += len;
614 }
615 sched_unpin();
616
617 return (copied);
618 }
619
620 static inline size_t
sg_copy_from_buffer(struct scatterlist * sgl,unsigned int nents,const void * buf,size_t buflen)621 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
622 const void *buf, size_t buflen)
623 {
624 return (sg_pcopy_from_buffer(sgl, nents, buf, buflen, 0));
625 }
626
627 static inline size_t
sg_pcopy_to_buffer(struct scatterlist * sgl,unsigned int nents,void * buf,size_t buflen,off_t offset)628 sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
629 void *buf, size_t buflen, off_t offset)
630 {
631 struct sg_page_iter iter;
632 struct scatterlist *sg;
633 struct page *page;
634 struct sf_buf *sf;
635 char *vaddr;
636 size_t total = 0;
637 size_t len;
638
639 if (!PMAP_HAS_DMAP)
640 sched_pin();
641 for_each_sg_page(sgl, &iter, nents, 0) {
642 sg = iter.sg;
643
644 if (offset >= sg->length) {
645 offset -= sg->length;
646 continue;
647 }
648 len = ulmin(buflen, sg->length - offset);
649 if (len == 0)
650 break;
651
652 page = sg_page_iter_page(&iter);
653 if (!PMAP_HAS_DMAP) {
654 sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT);
655 if (sf == NULL)
656 break;
657 vaddr = (char *)sf_buf_kva(sf);
658 } else
659 vaddr = (char *)PHYS_TO_DMAP(page_to_phys(page));
660 memcpy(buf, vaddr + sg->offset + offset, len);
661 if (!PMAP_HAS_DMAP)
662 sf_buf_free(sf);
663
664 /* start at beginning of next page */
665 offset = 0;
666
667 /* advance buffer */
668 buf = (char *)buf + len;
669 buflen -= len;
670 total += len;
671 }
672 if (!PMAP_HAS_DMAP)
673 sched_unpin();
674 return (total);
675 }
676
677 #endif /* _LINUXKPI_LINUX_SCATTERLIST_H_ */
678