1b36cfff7SJohn Baldwin /*-
251369649SPedro F. Giffuni * SPDX-License-Identifier: BSD-3-Clause
351369649SPedro F. Giffuni *
4b36cfff7SJohn Baldwin * Copyright (c) 2008 Yahoo!, Inc.
5b36cfff7SJohn Baldwin * All rights reserved.
6b36cfff7SJohn Baldwin * Written by: John Baldwin <jhb@FreeBSD.org>
7b36cfff7SJohn Baldwin *
8b36cfff7SJohn Baldwin * Redistribution and use in source and binary forms, with or without
9b36cfff7SJohn Baldwin * modification, are permitted provided that the following conditions
10b36cfff7SJohn Baldwin * are met:
11b36cfff7SJohn Baldwin * 1. Redistributions of source code must retain the above copyright
12b36cfff7SJohn Baldwin * notice, this list of conditions and the following disclaimer.
13b36cfff7SJohn Baldwin * 2. Redistributions in binary form must reproduce the above copyright
14b36cfff7SJohn Baldwin * notice, this list of conditions and the following disclaimer in the
15b36cfff7SJohn Baldwin * documentation and/or other materials provided with the distribution.
16b36cfff7SJohn Baldwin * 3. Neither the name of the author nor the names of any co-contributors
17b36cfff7SJohn Baldwin * may be used to endorse or promote products derived from this software
18b36cfff7SJohn Baldwin * without specific prior written permission.
19b36cfff7SJohn Baldwin *
20b36cfff7SJohn Baldwin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21b36cfff7SJohn Baldwin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22b36cfff7SJohn Baldwin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23b36cfff7SJohn Baldwin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24b36cfff7SJohn Baldwin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25b36cfff7SJohn Baldwin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26b36cfff7SJohn Baldwin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27b36cfff7SJohn Baldwin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28b36cfff7SJohn Baldwin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29b36cfff7SJohn Baldwin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30b36cfff7SJohn Baldwin * SUCH DAMAGE.
31b36cfff7SJohn Baldwin */
32b36cfff7SJohn Baldwin
33b36cfff7SJohn Baldwin #include <sys/param.h>
34b36cfff7SJohn Baldwin #include <sys/kernel.h>
35fb6c2518SBryan Venteicher #include <sys/bio.h>
36b36cfff7SJohn Baldwin #include <sys/malloc.h>
37b36cfff7SJohn Baldwin #include <sys/mbuf.h>
38b36cfff7SJohn Baldwin #include <sys/proc.h>
39b36cfff7SJohn Baldwin #include <sys/sglist.h>
40b36cfff7SJohn Baldwin #include <sys/uio.h>
41b36cfff7SJohn Baldwin
42b36cfff7SJohn Baldwin #include <vm/vm.h>
43fb6c2518SBryan Venteicher #include <vm/vm_page.h>
44b36cfff7SJohn Baldwin #include <vm/pmap.h>
45b36cfff7SJohn Baldwin #include <vm/vm_map.h>
46b36cfff7SJohn Baldwin
47b36cfff7SJohn Baldwin #include <sys/ktr.h>
48b36cfff7SJohn Baldwin
49b36cfff7SJohn Baldwin static MALLOC_DEFINE(M_SGLIST, "sglist", "scatter/gather lists");
50b36cfff7SJohn Baldwin
51b36cfff7SJohn Baldwin /*
52eb5a1e8fSJohn Baldwin * Convenience macros to save the state of an sglist so it can be restored
53eb5a1e8fSJohn Baldwin * if an append attempt fails. Since sglist's only grow we only need to
54eb5a1e8fSJohn Baldwin * save the current count of segments and the length of the ending segment.
55eb5a1e8fSJohn Baldwin * Earlier segments will not be changed by an append, and the only change
56eb5a1e8fSJohn Baldwin * that can occur to the ending segment is that it can be extended.
57eb5a1e8fSJohn Baldwin */
58eb5a1e8fSJohn Baldwin struct sgsave {
59eb5a1e8fSJohn Baldwin u_short sg_nseg;
60eb5a1e8fSJohn Baldwin size_t ss_len;
61eb5a1e8fSJohn Baldwin };
62eb5a1e8fSJohn Baldwin
63eb5a1e8fSJohn Baldwin #define SGLIST_SAVE(sg, sgsave) do { \
64eb5a1e8fSJohn Baldwin (sgsave).sg_nseg = (sg)->sg_nseg; \
65eb5a1e8fSJohn Baldwin if ((sgsave).sg_nseg > 0) \
66eb5a1e8fSJohn Baldwin (sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \
67eb5a1e8fSJohn Baldwin else \
68eb5a1e8fSJohn Baldwin (sgsave).ss_len = 0; \
69eb5a1e8fSJohn Baldwin } while (0)
70eb5a1e8fSJohn Baldwin
71eb5a1e8fSJohn Baldwin #define SGLIST_RESTORE(sg, sgsave) do { \
72eb5a1e8fSJohn Baldwin (sg)->sg_nseg = (sgsave).sg_nseg; \
73eb5a1e8fSJohn Baldwin if ((sgsave).sg_nseg > 0) \
74eb5a1e8fSJohn Baldwin (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \
75eb5a1e8fSJohn Baldwin } while (0)
76eb5a1e8fSJohn Baldwin
77eb5a1e8fSJohn Baldwin /*
78b36cfff7SJohn Baldwin * Append a single (paddr, len) to a sglist. sg is the list and ss is
79b36cfff7SJohn Baldwin * the current segment in the list. If we run out of segments then
80b36cfff7SJohn Baldwin * EFBIG will be returned.
81b36cfff7SJohn Baldwin */
82b36cfff7SJohn Baldwin static __inline int
_sglist_append_range(struct sglist * sg,struct sglist_seg ** ssp,vm_paddr_t paddr,size_t len)83b36cfff7SJohn Baldwin _sglist_append_range(struct sglist *sg, struct sglist_seg **ssp,
84b36cfff7SJohn Baldwin vm_paddr_t paddr, size_t len)
85b36cfff7SJohn Baldwin {
86b36cfff7SJohn Baldwin struct sglist_seg *ss;
87b36cfff7SJohn Baldwin
88b36cfff7SJohn Baldwin ss = *ssp;
89b36cfff7SJohn Baldwin if (ss->ss_paddr + ss->ss_len == paddr)
90b36cfff7SJohn Baldwin ss->ss_len += len;
91b36cfff7SJohn Baldwin else {
92eb5a1e8fSJohn Baldwin if (sg->sg_nseg == sg->sg_maxseg)
93b36cfff7SJohn Baldwin return (EFBIG);
94b36cfff7SJohn Baldwin ss++;
95b36cfff7SJohn Baldwin ss->ss_paddr = paddr;
96b36cfff7SJohn Baldwin ss->ss_len = len;
97b36cfff7SJohn Baldwin sg->sg_nseg++;
98b36cfff7SJohn Baldwin *ssp = ss;
99b36cfff7SJohn Baldwin }
100b36cfff7SJohn Baldwin return (0);
101b36cfff7SJohn Baldwin }
102b36cfff7SJohn Baldwin
103b36cfff7SJohn Baldwin /*
104b36cfff7SJohn Baldwin * Worker routine to append a virtual address range (either kernel or
105b36cfff7SJohn Baldwin * user) to a scatter/gather list.
106b36cfff7SJohn Baldwin */
107b36cfff7SJohn Baldwin static __inline int
_sglist_append_buf(struct sglist * sg,void * buf,size_t len,pmap_t pmap,size_t * donep)108b36cfff7SJohn Baldwin _sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap,
109b36cfff7SJohn Baldwin size_t *donep)
110b36cfff7SJohn Baldwin {
111b36cfff7SJohn Baldwin struct sglist_seg *ss;
112b36cfff7SJohn Baldwin vm_offset_t vaddr, offset;
113b36cfff7SJohn Baldwin vm_paddr_t paddr;
114b36cfff7SJohn Baldwin size_t seglen;
115b36cfff7SJohn Baldwin int error;
116b36cfff7SJohn Baldwin
117b36cfff7SJohn Baldwin if (donep)
118b36cfff7SJohn Baldwin *donep = 0;
119b36cfff7SJohn Baldwin if (len == 0)
120b36cfff7SJohn Baldwin return (0);
121b36cfff7SJohn Baldwin
122b36cfff7SJohn Baldwin /* Do the first page. It may have an offset. */
123b36cfff7SJohn Baldwin vaddr = (vm_offset_t)buf;
124b36cfff7SJohn Baldwin offset = vaddr & PAGE_MASK;
125b36cfff7SJohn Baldwin if (pmap != NULL)
126b36cfff7SJohn Baldwin paddr = pmap_extract(pmap, vaddr);
127b36cfff7SJohn Baldwin else
128b36cfff7SJohn Baldwin paddr = pmap_kextract(vaddr);
129b36cfff7SJohn Baldwin seglen = MIN(len, PAGE_SIZE - offset);
130b36cfff7SJohn Baldwin if (sg->sg_nseg == 0) {
131b36cfff7SJohn Baldwin ss = sg->sg_segs;
132b36cfff7SJohn Baldwin ss->ss_paddr = paddr;
133b36cfff7SJohn Baldwin ss->ss_len = seglen;
134b36cfff7SJohn Baldwin sg->sg_nseg = 1;
135b36cfff7SJohn Baldwin } else {
136b36cfff7SJohn Baldwin ss = &sg->sg_segs[sg->sg_nseg - 1];
137b36cfff7SJohn Baldwin error = _sglist_append_range(sg, &ss, paddr, seglen);
138eb5a1e8fSJohn Baldwin if (error)
139eb5a1e8fSJohn Baldwin return (error);
140b36cfff7SJohn Baldwin }
141b36cfff7SJohn Baldwin vaddr += seglen;
142b36cfff7SJohn Baldwin len -= seglen;
143b36cfff7SJohn Baldwin if (donep)
144b36cfff7SJohn Baldwin *donep += seglen;
145eb5a1e8fSJohn Baldwin
146eb5a1e8fSJohn Baldwin while (len > 0) {
147b36cfff7SJohn Baldwin seglen = MIN(len, PAGE_SIZE);
148b36cfff7SJohn Baldwin if (pmap != NULL)
149b36cfff7SJohn Baldwin paddr = pmap_extract(pmap, vaddr);
150b36cfff7SJohn Baldwin else
151b36cfff7SJohn Baldwin paddr = pmap_kextract(vaddr);
152b36cfff7SJohn Baldwin error = _sglist_append_range(sg, &ss, paddr, seglen);
153eb5a1e8fSJohn Baldwin if (error)
154eb5a1e8fSJohn Baldwin return (error);
155eb5a1e8fSJohn Baldwin vaddr += seglen;
156eb5a1e8fSJohn Baldwin len -= seglen;
157eb5a1e8fSJohn Baldwin if (donep)
158eb5a1e8fSJohn Baldwin *donep += seglen;
159b36cfff7SJohn Baldwin }
160b36cfff7SJohn Baldwin
161eb5a1e8fSJohn Baldwin return (0);
162b36cfff7SJohn Baldwin }
163b36cfff7SJohn Baldwin
164b36cfff7SJohn Baldwin /*
165b36cfff7SJohn Baldwin * Determine the number of scatter/gather list elements needed to
166b36cfff7SJohn Baldwin * describe a kernel virtual address range.
167b36cfff7SJohn Baldwin */
168b36cfff7SJohn Baldwin int
sglist_count(void * buf,size_t len)169b36cfff7SJohn Baldwin sglist_count(void *buf, size_t len)
170b36cfff7SJohn Baldwin {
171b36cfff7SJohn Baldwin vm_offset_t vaddr, vendaddr;
172b36cfff7SJohn Baldwin vm_paddr_t lastaddr, paddr;
173b36cfff7SJohn Baldwin int nsegs;
174b36cfff7SJohn Baldwin
175b36cfff7SJohn Baldwin if (len == 0)
176b36cfff7SJohn Baldwin return (0);
177b36cfff7SJohn Baldwin
178b36cfff7SJohn Baldwin vaddr = trunc_page((vm_offset_t)buf);
179b36cfff7SJohn Baldwin vendaddr = (vm_offset_t)buf + len;
180b36cfff7SJohn Baldwin nsegs = 1;
181b36cfff7SJohn Baldwin lastaddr = pmap_kextract(vaddr);
182b36cfff7SJohn Baldwin vaddr += PAGE_SIZE;
183b36cfff7SJohn Baldwin while (vaddr < vendaddr) {
184b36cfff7SJohn Baldwin paddr = pmap_kextract(vaddr);
185b36cfff7SJohn Baldwin if (lastaddr + PAGE_SIZE != paddr)
186b36cfff7SJohn Baldwin nsegs++;
187b36cfff7SJohn Baldwin lastaddr = paddr;
188b36cfff7SJohn Baldwin vaddr += PAGE_SIZE;
189b36cfff7SJohn Baldwin }
190b36cfff7SJohn Baldwin return (nsegs);
191b36cfff7SJohn Baldwin }
192b36cfff7SJohn Baldwin
193b36cfff7SJohn Baldwin /*
19420fee109SJohn Baldwin * Determine the number of scatter/gather list elements needed to
19520fee109SJohn Baldwin * describe a buffer backed by an array of VM pages.
19620fee109SJohn Baldwin */
19720fee109SJohn Baldwin int
sglist_count_vmpages(vm_page_t * m,size_t pgoff,size_t len)19820fee109SJohn Baldwin sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len)
19920fee109SJohn Baldwin {
20020fee109SJohn Baldwin vm_paddr_t lastaddr, paddr;
20120fee109SJohn Baldwin int i, nsegs;
20220fee109SJohn Baldwin
20320fee109SJohn Baldwin if (len == 0)
20420fee109SJohn Baldwin return (0);
20520fee109SJohn Baldwin
20620fee109SJohn Baldwin len += pgoff;
20720fee109SJohn Baldwin nsegs = 1;
20820fee109SJohn Baldwin lastaddr = VM_PAGE_TO_PHYS(m[0]);
20920fee109SJohn Baldwin for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) {
21020fee109SJohn Baldwin paddr = VM_PAGE_TO_PHYS(m[i]);
21120fee109SJohn Baldwin if (lastaddr + PAGE_SIZE != paddr)
21220fee109SJohn Baldwin nsegs++;
21320fee109SJohn Baldwin lastaddr = paddr;
21420fee109SJohn Baldwin }
21520fee109SJohn Baldwin return (nsegs);
21620fee109SJohn Baldwin }
21720fee109SJohn Baldwin
21820fee109SJohn Baldwin /*
21982334850SJohn Baldwin * Determine the number of scatter/gather list elements needed to
22061664ee7SGleb Smirnoff * describe an M_EXTPG mbuf.
22182334850SJohn Baldwin */
22282334850SJohn Baldwin int
sglist_count_mbuf_epg(struct mbuf * m,size_t off,size_t len)22349b6b60eSGleb Smirnoff sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len)
22482334850SJohn Baldwin {
22582334850SJohn Baldwin vm_paddr_t nextaddr, paddr;
22682334850SJohn Baldwin size_t seglen, segoff;
22782334850SJohn Baldwin int i, nsegs, pglen, pgoff;
22882334850SJohn Baldwin
22982334850SJohn Baldwin if (len == 0)
23082334850SJohn Baldwin return (0);
23182334850SJohn Baldwin
23282334850SJohn Baldwin nsegs = 0;
2337b6c99d0SGleb Smirnoff if (m->m_epg_hdrlen != 0) {
2347b6c99d0SGleb Smirnoff if (off >= m->m_epg_hdrlen) {
2357b6c99d0SGleb Smirnoff off -= m->m_epg_hdrlen;
23682334850SJohn Baldwin } else {
2377b6c99d0SGleb Smirnoff seglen = m->m_epg_hdrlen - off;
23882334850SJohn Baldwin segoff = off;
23982334850SJohn Baldwin seglen = MIN(seglen, len);
24082334850SJohn Baldwin off = 0;
24182334850SJohn Baldwin len -= seglen;
2420c103266SGleb Smirnoff nsegs += sglist_count(&m->m_epg_hdr[segoff],
24323feb563SAndrew Gallatin seglen);
24482334850SJohn Baldwin }
24582334850SJohn Baldwin }
24682334850SJohn Baldwin nextaddr = 0;
2477b6c99d0SGleb Smirnoff pgoff = m->m_epg_1st_off;
2487b6c99d0SGleb Smirnoff for (i = 0; i < m->m_epg_npgs && len > 0; i++) {
249c4ee38f8SGleb Smirnoff pglen = m_epg_pagelen(m, i, pgoff);
25082334850SJohn Baldwin if (off >= pglen) {
25182334850SJohn Baldwin off -= pglen;
25282334850SJohn Baldwin pgoff = 0;
25382334850SJohn Baldwin continue;
25482334850SJohn Baldwin }
25582334850SJohn Baldwin seglen = pglen - off;
25682334850SJohn Baldwin segoff = pgoff + off;
25782334850SJohn Baldwin off = 0;
25882334850SJohn Baldwin seglen = MIN(seglen, len);
25982334850SJohn Baldwin len -= seglen;
2600c103266SGleb Smirnoff paddr = m->m_epg_pa[i] + segoff;
26182334850SJohn Baldwin if (paddr != nextaddr)
26282334850SJohn Baldwin nsegs++;
26382334850SJohn Baldwin nextaddr = paddr + seglen;
26482334850SJohn Baldwin pgoff = 0;
26582334850SJohn Baldwin };
26682334850SJohn Baldwin if (len != 0) {
2677b6c99d0SGleb Smirnoff seglen = MIN(len, m->m_epg_trllen - off);
26882334850SJohn Baldwin len -= seglen;
2690c103266SGleb Smirnoff nsegs += sglist_count(&m->m_epg_trail[off], seglen);
27082334850SJohn Baldwin }
27182334850SJohn Baldwin KASSERT(len == 0, ("len != 0"));
27282334850SJohn Baldwin return (nsegs);
27382334850SJohn Baldwin }
27482334850SJohn Baldwin
27582334850SJohn Baldwin /*
276b36cfff7SJohn Baldwin * Allocate a scatter/gather list along with 'nsegs' segments. The
277b36cfff7SJohn Baldwin * 'mflags' parameters are the same as passed to malloc(9). The caller
278b36cfff7SJohn Baldwin * should use sglist_free() to free this list.
279b36cfff7SJohn Baldwin */
280b36cfff7SJohn Baldwin struct sglist *
sglist_alloc(int nsegs,int mflags)281b36cfff7SJohn Baldwin sglist_alloc(int nsegs, int mflags)
282b36cfff7SJohn Baldwin {
283b36cfff7SJohn Baldwin struct sglist *sg;
284b36cfff7SJohn Baldwin
285b36cfff7SJohn Baldwin sg = malloc(sizeof(struct sglist) + nsegs * sizeof(struct sglist_seg),
286b36cfff7SJohn Baldwin M_SGLIST, mflags);
287b36cfff7SJohn Baldwin if (sg == NULL)
288b36cfff7SJohn Baldwin return (NULL);
289b36cfff7SJohn Baldwin sglist_init(sg, nsegs, (struct sglist_seg *)(sg + 1));
290b36cfff7SJohn Baldwin return (sg);
291b36cfff7SJohn Baldwin }
292b36cfff7SJohn Baldwin
293b36cfff7SJohn Baldwin /*
294b36cfff7SJohn Baldwin * Free a scatter/gather list allocated via sglist_allc().
295b36cfff7SJohn Baldwin */
296b36cfff7SJohn Baldwin void
sglist_free(struct sglist * sg)297b36cfff7SJohn Baldwin sglist_free(struct sglist *sg)
298b36cfff7SJohn Baldwin {
299b36cfff7SJohn Baldwin
3004f621933SJohn Baldwin if (sg == NULL)
3014f621933SJohn Baldwin return;
3024f621933SJohn Baldwin
303b36cfff7SJohn Baldwin if (refcount_release(&sg->sg_refs))
304b36cfff7SJohn Baldwin free(sg, M_SGLIST);
305b36cfff7SJohn Baldwin }
306b36cfff7SJohn Baldwin
307b36cfff7SJohn Baldwin /*
308b36cfff7SJohn Baldwin * Append the segments to describe a single kernel virtual address
309b36cfff7SJohn Baldwin * range to a scatter/gather list. If there are insufficient
310b36cfff7SJohn Baldwin * segments, then this fails with EFBIG.
311b36cfff7SJohn Baldwin */
312b36cfff7SJohn Baldwin int
sglist_append(struct sglist * sg,void * buf,size_t len)313b36cfff7SJohn Baldwin sglist_append(struct sglist *sg, void *buf, size_t len)
314b36cfff7SJohn Baldwin {
315eb5a1e8fSJohn Baldwin struct sgsave save;
316eb5a1e8fSJohn Baldwin int error;
317b36cfff7SJohn Baldwin
318b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
319b36cfff7SJohn Baldwin return (EINVAL);
320eb5a1e8fSJohn Baldwin SGLIST_SAVE(sg, save);
321eb5a1e8fSJohn Baldwin error = _sglist_append_buf(sg, buf, len, NULL, NULL);
322eb5a1e8fSJohn Baldwin if (error)
323eb5a1e8fSJohn Baldwin SGLIST_RESTORE(sg, save);
324eb5a1e8fSJohn Baldwin return (error);
325b36cfff7SJohn Baldwin }
326b36cfff7SJohn Baldwin
327b36cfff7SJohn Baldwin /*
328fb6c2518SBryan Venteicher * Append the segments to describe a bio's data to a scatter/gather list.
329fb6c2518SBryan Venteicher * If there are insufficient segments, then this fails with EFBIG.
330fb6c2518SBryan Venteicher *
331fb6c2518SBryan Venteicher * NOTE: This function expects bio_bcount to be initialized.
332fb6c2518SBryan Venteicher */
333fb6c2518SBryan Venteicher int
sglist_append_bio(struct sglist * sg,struct bio * bp)334fb6c2518SBryan Venteicher sglist_append_bio(struct sglist *sg, struct bio *bp)
335fb6c2518SBryan Venteicher {
33620fee109SJohn Baldwin int error;
337fb6c2518SBryan Venteicher
33820fee109SJohn Baldwin if ((bp->bio_flags & BIO_UNMAPPED) == 0)
339fb6c2518SBryan Venteicher error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
34020fee109SJohn Baldwin else
34120fee109SJohn Baldwin error = sglist_append_vmpages(sg, bp->bio_ma,
34220fee109SJohn Baldwin bp->bio_ma_offset, bp->bio_bcount);
343fb6c2518SBryan Venteicher return (error);
344fb6c2518SBryan Venteicher }
345fb6c2518SBryan Venteicher
346fb6c2518SBryan Venteicher /*
347b36cfff7SJohn Baldwin * Append a single physical address range to a scatter/gather list.
348b36cfff7SJohn Baldwin * If there are insufficient segments, then this fails with EFBIG.
349b36cfff7SJohn Baldwin */
350b36cfff7SJohn Baldwin int
sglist_append_phys(struct sglist * sg,vm_paddr_t paddr,size_t len)351b36cfff7SJohn Baldwin sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len)
352b36cfff7SJohn Baldwin {
353b36cfff7SJohn Baldwin struct sglist_seg *ss;
354eb5a1e8fSJohn Baldwin struct sgsave save;
355eb5a1e8fSJohn Baldwin int error;
356b36cfff7SJohn Baldwin
357b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
358b36cfff7SJohn Baldwin return (EINVAL);
359b36cfff7SJohn Baldwin if (len == 0)
360b36cfff7SJohn Baldwin return (0);
361b36cfff7SJohn Baldwin
362b36cfff7SJohn Baldwin if (sg->sg_nseg == 0) {
363b36cfff7SJohn Baldwin sg->sg_segs[0].ss_paddr = paddr;
364b36cfff7SJohn Baldwin sg->sg_segs[0].ss_len = len;
365b36cfff7SJohn Baldwin sg->sg_nseg = 1;
366b36cfff7SJohn Baldwin return (0);
367b36cfff7SJohn Baldwin }
368b36cfff7SJohn Baldwin ss = &sg->sg_segs[sg->sg_nseg - 1];
369eb5a1e8fSJohn Baldwin SGLIST_SAVE(sg, save);
370eb5a1e8fSJohn Baldwin error = _sglist_append_range(sg, &ss, paddr, len);
371eb5a1e8fSJohn Baldwin if (error)
372eb5a1e8fSJohn Baldwin SGLIST_RESTORE(sg, save);
373eb5a1e8fSJohn Baldwin return (error);
374b36cfff7SJohn Baldwin }
375b36cfff7SJohn Baldwin
376b36cfff7SJohn Baldwin /*
37749b6b60eSGleb Smirnoff * Append the segments of single multi-page mbuf.
37849b6b60eSGleb Smirnoff * If there are insufficient segments, then this fails with EFBIG.
37982334850SJohn Baldwin */
38082334850SJohn Baldwin int
sglist_append_mbuf_epg(struct sglist * sg,struct mbuf * m,size_t off,size_t len)38149b6b60eSGleb Smirnoff sglist_append_mbuf_epg(struct sglist *sg, struct mbuf *m, size_t off,
38249b6b60eSGleb Smirnoff size_t len)
38382334850SJohn Baldwin {
38482334850SJohn Baldwin size_t seglen, segoff;
38582334850SJohn Baldwin vm_paddr_t paddr;
38682334850SJohn Baldwin int error, i, pglen, pgoff;
38782334850SJohn Baldwin
388365e8da4SGleb Smirnoff M_ASSERTEXTPG(m);
38949b6b60eSGleb Smirnoff
39082334850SJohn Baldwin error = 0;
3917b6c99d0SGleb Smirnoff if (m->m_epg_hdrlen != 0) {
3927b6c99d0SGleb Smirnoff if (off >= m->m_epg_hdrlen) {
3937b6c99d0SGleb Smirnoff off -= m->m_epg_hdrlen;
39482334850SJohn Baldwin } else {
3957b6c99d0SGleb Smirnoff seglen = m->m_epg_hdrlen - off;
39682334850SJohn Baldwin segoff = off;
39782334850SJohn Baldwin seglen = MIN(seglen, len);
39882334850SJohn Baldwin off = 0;
39982334850SJohn Baldwin len -= seglen;
40082334850SJohn Baldwin error = sglist_append(sg,
4010c103266SGleb Smirnoff &m->m_epg_hdr[segoff], seglen);
40282334850SJohn Baldwin }
40382334850SJohn Baldwin }
4047b6c99d0SGleb Smirnoff pgoff = m->m_epg_1st_off;
4057b6c99d0SGleb Smirnoff for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) {
406c4ee38f8SGleb Smirnoff pglen = m_epg_pagelen(m, i, pgoff);
40782334850SJohn Baldwin if (off >= pglen) {
40882334850SJohn Baldwin off -= pglen;
40982334850SJohn Baldwin pgoff = 0;
41082334850SJohn Baldwin continue;
41182334850SJohn Baldwin }
41282334850SJohn Baldwin seglen = pglen - off;
41382334850SJohn Baldwin segoff = pgoff + off;
41482334850SJohn Baldwin off = 0;
41582334850SJohn Baldwin seglen = MIN(seglen, len);
41682334850SJohn Baldwin len -= seglen;
4170c103266SGleb Smirnoff paddr = m->m_epg_pa[i] + segoff;
41882334850SJohn Baldwin error = sglist_append_phys(sg, paddr, seglen);
41982334850SJohn Baldwin pgoff = 0;
42082334850SJohn Baldwin };
42182334850SJohn Baldwin if (error == 0 && len > 0) {
4227b6c99d0SGleb Smirnoff seglen = MIN(len, m->m_epg_trllen - off);
42382334850SJohn Baldwin len -= seglen;
42482334850SJohn Baldwin error = sglist_append(sg,
4250c103266SGleb Smirnoff &m->m_epg_trail[off], seglen);
42682334850SJohn Baldwin }
42782334850SJohn Baldwin if (error == 0)
42882334850SJohn Baldwin KASSERT(len == 0, ("len != 0"));
42982334850SJohn Baldwin return (error);
43082334850SJohn Baldwin }
43182334850SJohn Baldwin
43282334850SJohn Baldwin /*
433b36cfff7SJohn Baldwin * Append the segments that describe a single mbuf chain to a
434b36cfff7SJohn Baldwin * scatter/gather list. If there are insufficient segments, then this
435b36cfff7SJohn Baldwin * fails with EFBIG.
436b36cfff7SJohn Baldwin */
437b36cfff7SJohn Baldwin int
sglist_append_mbuf(struct sglist * sg,struct mbuf * m0)438b36cfff7SJohn Baldwin sglist_append_mbuf(struct sglist *sg, struct mbuf *m0)
439b36cfff7SJohn Baldwin {
440eb5a1e8fSJohn Baldwin struct sgsave save;
441b36cfff7SJohn Baldwin struct mbuf *m;
442b36cfff7SJohn Baldwin int error;
443b36cfff7SJohn Baldwin
444b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
445b36cfff7SJohn Baldwin return (EINVAL);
446b36cfff7SJohn Baldwin
447b36cfff7SJohn Baldwin error = 0;
448eb5a1e8fSJohn Baldwin SGLIST_SAVE(sg, save);
449b36cfff7SJohn Baldwin for (m = m0; m != NULL; m = m->m_next) {
450b36cfff7SJohn Baldwin if (m->m_len > 0) {
4516edfd179SGleb Smirnoff if ((m->m_flags & M_EXTPG) != 0)
45249b6b60eSGleb Smirnoff error = sglist_append_mbuf_epg(sg, m,
45349b6b60eSGleb Smirnoff mtod(m, vm_offset_t), m->m_len);
45482334850SJohn Baldwin else
45582334850SJohn Baldwin error = sglist_append(sg, m->m_data,
45682334850SJohn Baldwin m->m_len);
457eb5a1e8fSJohn Baldwin if (error) {
458eb5a1e8fSJohn Baldwin SGLIST_RESTORE(sg, save);
459b36cfff7SJohn Baldwin return (error);
460b36cfff7SJohn Baldwin }
461b36cfff7SJohn Baldwin }
462eb5a1e8fSJohn Baldwin }
463b36cfff7SJohn Baldwin return (0);
464b36cfff7SJohn Baldwin }
465b36cfff7SJohn Baldwin
466b36cfff7SJohn Baldwin /*
4676663f8a2SJohn Baldwin * Append the segments that describe a single mbuf to a scatter/gather
4686663f8a2SJohn Baldwin * list. If there are insufficient segments, then this fails with
4696663f8a2SJohn Baldwin * EFBIG.
4706663f8a2SJohn Baldwin */
4716663f8a2SJohn Baldwin int
sglist_append_single_mbuf(struct sglist * sg,struct mbuf * m)4726663f8a2SJohn Baldwin sglist_append_single_mbuf(struct sglist *sg, struct mbuf *m)
4736663f8a2SJohn Baldwin {
4746663f8a2SJohn Baldwin if ((m->m_flags & M_EXTPG) != 0)
4756663f8a2SJohn Baldwin return (sglist_append_mbuf_epg(sg, m,
4766663f8a2SJohn Baldwin mtod(m, vm_offset_t), m->m_len));
4776663f8a2SJohn Baldwin else
4786663f8a2SJohn Baldwin return (sglist_append(sg, m->m_data, m->m_len));
4796663f8a2SJohn Baldwin }
4806663f8a2SJohn Baldwin
4816663f8a2SJohn Baldwin /*
48220fee109SJohn Baldwin * Append the segments that describe a buffer spanning an array of VM
48320fee109SJohn Baldwin * pages. The buffer begins at an offset of 'pgoff' in the first
48420fee109SJohn Baldwin * page.
48520fee109SJohn Baldwin */
48620fee109SJohn Baldwin int
sglist_append_vmpages(struct sglist * sg,vm_page_t * m,size_t pgoff,size_t len)48720fee109SJohn Baldwin sglist_append_vmpages(struct sglist *sg, vm_page_t *m, size_t pgoff,
48820fee109SJohn Baldwin size_t len)
48920fee109SJohn Baldwin {
49020fee109SJohn Baldwin struct sgsave save;
49120fee109SJohn Baldwin struct sglist_seg *ss;
49220fee109SJohn Baldwin vm_paddr_t paddr;
49320fee109SJohn Baldwin size_t seglen;
49420fee109SJohn Baldwin int error, i;
49520fee109SJohn Baldwin
49620fee109SJohn Baldwin if (sg->sg_maxseg == 0)
49720fee109SJohn Baldwin return (EINVAL);
49820fee109SJohn Baldwin if (len == 0)
49920fee109SJohn Baldwin return (0);
50020fee109SJohn Baldwin
50120fee109SJohn Baldwin SGLIST_SAVE(sg, save);
50220fee109SJohn Baldwin i = 0;
50320fee109SJohn Baldwin if (sg->sg_nseg == 0) {
50420fee109SJohn Baldwin seglen = min(PAGE_SIZE - pgoff, len);
50520fee109SJohn Baldwin sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff;
50620fee109SJohn Baldwin sg->sg_segs[0].ss_len = seglen;
50720fee109SJohn Baldwin sg->sg_nseg = 1;
50820fee109SJohn Baldwin pgoff = 0;
50920fee109SJohn Baldwin len -= seglen;
51020fee109SJohn Baldwin i++;
51120fee109SJohn Baldwin }
51220fee109SJohn Baldwin ss = &sg->sg_segs[sg->sg_nseg - 1];
51320fee109SJohn Baldwin for (; len > 0; i++, len -= seglen) {
51420fee109SJohn Baldwin seglen = min(PAGE_SIZE - pgoff, len);
51520fee109SJohn Baldwin paddr = VM_PAGE_TO_PHYS(m[i]) + pgoff;
51620fee109SJohn Baldwin error = _sglist_append_range(sg, &ss, paddr, seglen);
51720fee109SJohn Baldwin if (error) {
51820fee109SJohn Baldwin SGLIST_RESTORE(sg, save);
51920fee109SJohn Baldwin return (error);
52020fee109SJohn Baldwin }
52120fee109SJohn Baldwin pgoff = 0;
52220fee109SJohn Baldwin }
52320fee109SJohn Baldwin return (0);
52420fee109SJohn Baldwin }
52520fee109SJohn Baldwin
52620fee109SJohn Baldwin /*
527b36cfff7SJohn Baldwin * Append the segments that describe a single user address range to a
528b36cfff7SJohn Baldwin * scatter/gather list. If there are insufficient segments, then this
529b36cfff7SJohn Baldwin * fails with EFBIG.
530b36cfff7SJohn Baldwin */
531b36cfff7SJohn Baldwin int
sglist_append_user(struct sglist * sg,void * buf,size_t len,struct thread * td)532b36cfff7SJohn Baldwin sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td)
533b36cfff7SJohn Baldwin {
534eb5a1e8fSJohn Baldwin struct sgsave save;
535eb5a1e8fSJohn Baldwin int error;
536b36cfff7SJohn Baldwin
537b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
538b36cfff7SJohn Baldwin return (EINVAL);
539eb5a1e8fSJohn Baldwin SGLIST_SAVE(sg, save);
540eb5a1e8fSJohn Baldwin error = _sglist_append_buf(sg, buf, len,
541eb5a1e8fSJohn Baldwin vmspace_pmap(td->td_proc->p_vmspace), NULL);
542eb5a1e8fSJohn Baldwin if (error)
543eb5a1e8fSJohn Baldwin SGLIST_RESTORE(sg, save);
544eb5a1e8fSJohn Baldwin return (error);
545b36cfff7SJohn Baldwin }
546b36cfff7SJohn Baldwin
547b36cfff7SJohn Baldwin /*
54800f6cd3fSJohn Baldwin * Append a subset of an existing scatter/gather list 'source' to a
54900f6cd3fSJohn Baldwin * the scatter/gather list 'sg'. If there are insufficient segments,
55000f6cd3fSJohn Baldwin * then this fails with EFBIG.
55100f6cd3fSJohn Baldwin */
55200f6cd3fSJohn Baldwin int
sglist_append_sglist(struct sglist * sg,struct sglist * source,size_t offset,size_t length)55300f6cd3fSJohn Baldwin sglist_append_sglist(struct sglist *sg, struct sglist *source, size_t offset,
55400f6cd3fSJohn Baldwin size_t length)
55500f6cd3fSJohn Baldwin {
55600f6cd3fSJohn Baldwin struct sgsave save;
55700f6cd3fSJohn Baldwin struct sglist_seg *ss;
55800f6cd3fSJohn Baldwin size_t seglen;
55900f6cd3fSJohn Baldwin int error, i;
56000f6cd3fSJohn Baldwin
56100f6cd3fSJohn Baldwin if (sg->sg_maxseg == 0 || length == 0)
56200f6cd3fSJohn Baldwin return (EINVAL);
56300f6cd3fSJohn Baldwin SGLIST_SAVE(sg, save);
56400f6cd3fSJohn Baldwin error = EINVAL;
56500f6cd3fSJohn Baldwin ss = &sg->sg_segs[sg->sg_nseg - 1];
56600f6cd3fSJohn Baldwin for (i = 0; i < source->sg_nseg; i++) {
56700f6cd3fSJohn Baldwin if (offset >= source->sg_segs[i].ss_len) {
56800f6cd3fSJohn Baldwin offset -= source->sg_segs[i].ss_len;
56900f6cd3fSJohn Baldwin continue;
57000f6cd3fSJohn Baldwin }
57100f6cd3fSJohn Baldwin seglen = source->sg_segs[i].ss_len - offset;
57200f6cd3fSJohn Baldwin if (seglen > length)
57300f6cd3fSJohn Baldwin seglen = length;
57400f6cd3fSJohn Baldwin error = _sglist_append_range(sg, &ss,
57500f6cd3fSJohn Baldwin source->sg_segs[i].ss_paddr + offset, seglen);
57600f6cd3fSJohn Baldwin if (error)
57700f6cd3fSJohn Baldwin break;
57800f6cd3fSJohn Baldwin offset = 0;
57900f6cd3fSJohn Baldwin length -= seglen;
58000f6cd3fSJohn Baldwin if (length == 0)
58100f6cd3fSJohn Baldwin break;
58200f6cd3fSJohn Baldwin }
58300f6cd3fSJohn Baldwin if (length != 0)
58400f6cd3fSJohn Baldwin error = EINVAL;
58500f6cd3fSJohn Baldwin if (error)
58600f6cd3fSJohn Baldwin SGLIST_RESTORE(sg, save);
58700f6cd3fSJohn Baldwin return (error);
58800f6cd3fSJohn Baldwin }
58900f6cd3fSJohn Baldwin
59000f6cd3fSJohn Baldwin /*
591b36cfff7SJohn Baldwin * Append the segments that describe a single uio to a scatter/gather
592b36cfff7SJohn Baldwin * list. If there are insufficient segments, then this fails with
593b36cfff7SJohn Baldwin * EFBIG.
594b36cfff7SJohn Baldwin */
595b36cfff7SJohn Baldwin int
sglist_append_uio(struct sglist * sg,struct uio * uio)596b36cfff7SJohn Baldwin sglist_append_uio(struct sglist *sg, struct uio *uio)
597b36cfff7SJohn Baldwin {
598b36cfff7SJohn Baldwin struct iovec *iov;
599eb5a1e8fSJohn Baldwin struct sgsave save;
600b36cfff7SJohn Baldwin size_t resid, minlen;
601b36cfff7SJohn Baldwin pmap_t pmap;
602b36cfff7SJohn Baldwin int error, i;
603b36cfff7SJohn Baldwin
604b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
605b36cfff7SJohn Baldwin return (EINVAL);
606b36cfff7SJohn Baldwin
607b36cfff7SJohn Baldwin resid = uio->uio_resid;
608b36cfff7SJohn Baldwin iov = uio->uio_iov;
609b36cfff7SJohn Baldwin
610b36cfff7SJohn Baldwin if (uio->uio_segflg == UIO_USERSPACE) {
611b36cfff7SJohn Baldwin KASSERT(uio->uio_td != NULL,
612b36cfff7SJohn Baldwin ("sglist_append_uio: USERSPACE but no thread"));
613b36cfff7SJohn Baldwin pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
614b36cfff7SJohn Baldwin } else
615b36cfff7SJohn Baldwin pmap = NULL;
616b36cfff7SJohn Baldwin
617b36cfff7SJohn Baldwin error = 0;
618eb5a1e8fSJohn Baldwin SGLIST_SAVE(sg, save);
619b36cfff7SJohn Baldwin for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
620b36cfff7SJohn Baldwin /*
621b36cfff7SJohn Baldwin * Now at the first iovec to load. Load each iovec
622b36cfff7SJohn Baldwin * until we have exhausted the residual count.
623b36cfff7SJohn Baldwin */
624b36cfff7SJohn Baldwin minlen = MIN(resid, iov[i].iov_len);
625b36cfff7SJohn Baldwin if (minlen > 0) {
626b36cfff7SJohn Baldwin error = _sglist_append_buf(sg, iov[i].iov_base, minlen,
627b36cfff7SJohn Baldwin pmap, NULL);
628eb5a1e8fSJohn Baldwin if (error) {
629eb5a1e8fSJohn Baldwin SGLIST_RESTORE(sg, save);
630b36cfff7SJohn Baldwin return (error);
631eb5a1e8fSJohn Baldwin }
632b36cfff7SJohn Baldwin resid -= minlen;
633b36cfff7SJohn Baldwin }
634b36cfff7SJohn Baldwin }
635b36cfff7SJohn Baldwin return (0);
636b36cfff7SJohn Baldwin }
637b36cfff7SJohn Baldwin
638b36cfff7SJohn Baldwin /*
639b36cfff7SJohn Baldwin * Append the segments that describe at most 'resid' bytes from a
640b36cfff7SJohn Baldwin * single uio to a scatter/gather list. If there are insufficient
641b36cfff7SJohn Baldwin * segments, then only the amount that fits is appended.
642b36cfff7SJohn Baldwin */
643b36cfff7SJohn Baldwin int
sglist_consume_uio(struct sglist * sg,struct uio * uio,size_t resid)6440cef25aeSJohn Baldwin sglist_consume_uio(struct sglist *sg, struct uio *uio, size_t resid)
645b36cfff7SJohn Baldwin {
646b36cfff7SJohn Baldwin struct iovec *iov;
647b36cfff7SJohn Baldwin size_t done;
648b36cfff7SJohn Baldwin pmap_t pmap;
649b36cfff7SJohn Baldwin int error, len;
650b36cfff7SJohn Baldwin
651b36cfff7SJohn Baldwin if (sg->sg_maxseg == 0)
652b36cfff7SJohn Baldwin return (EINVAL);
653b36cfff7SJohn Baldwin
654b36cfff7SJohn Baldwin if (uio->uio_segflg == UIO_USERSPACE) {
655b36cfff7SJohn Baldwin KASSERT(uio->uio_td != NULL,
656b36cfff7SJohn Baldwin ("sglist_consume_uio: USERSPACE but no thread"));
657b36cfff7SJohn Baldwin pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
658b36cfff7SJohn Baldwin } else
659b36cfff7SJohn Baldwin pmap = NULL;
660b36cfff7SJohn Baldwin
661b36cfff7SJohn Baldwin error = 0;
662b36cfff7SJohn Baldwin while (resid > 0 && uio->uio_resid) {
663b36cfff7SJohn Baldwin iov = uio->uio_iov;
664b36cfff7SJohn Baldwin len = iov->iov_len;
665b36cfff7SJohn Baldwin if (len == 0) {
666b36cfff7SJohn Baldwin uio->uio_iov++;
667b36cfff7SJohn Baldwin uio->uio_iovcnt--;
668b36cfff7SJohn Baldwin continue;
669b36cfff7SJohn Baldwin }
670b36cfff7SJohn Baldwin if (len > resid)
671b36cfff7SJohn Baldwin len = resid;
672b36cfff7SJohn Baldwin
673b36cfff7SJohn Baldwin /*
674b36cfff7SJohn Baldwin * Try to append this iovec. If we run out of room,
675b36cfff7SJohn Baldwin * then break out of the loop.
676b36cfff7SJohn Baldwin */
677b36cfff7SJohn Baldwin error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done);
678b36cfff7SJohn Baldwin iov->iov_base = (char *)iov->iov_base + done;
679b36cfff7SJohn Baldwin iov->iov_len -= done;
680b36cfff7SJohn Baldwin uio->uio_resid -= done;
681b36cfff7SJohn Baldwin uio->uio_offset += done;
682b36cfff7SJohn Baldwin resid -= done;
683b36cfff7SJohn Baldwin if (error)
684b36cfff7SJohn Baldwin break;
685b36cfff7SJohn Baldwin }
686b36cfff7SJohn Baldwin return (0);
687b36cfff7SJohn Baldwin }
688b36cfff7SJohn Baldwin
689b36cfff7SJohn Baldwin /*
690b36cfff7SJohn Baldwin * Allocate and populate a scatter/gather list to describe a single
691b36cfff7SJohn Baldwin * kernel virtual address range.
692b36cfff7SJohn Baldwin */
693b36cfff7SJohn Baldwin struct sglist *
sglist_build(void * buf,size_t len,int mflags)694b36cfff7SJohn Baldwin sglist_build(void *buf, size_t len, int mflags)
695b36cfff7SJohn Baldwin {
696b36cfff7SJohn Baldwin struct sglist *sg;
697b36cfff7SJohn Baldwin int nsegs;
698b36cfff7SJohn Baldwin
699b36cfff7SJohn Baldwin if (len == 0)
700b36cfff7SJohn Baldwin return (NULL);
701b36cfff7SJohn Baldwin
702b36cfff7SJohn Baldwin nsegs = sglist_count(buf, len);
703b36cfff7SJohn Baldwin sg = sglist_alloc(nsegs, mflags);
704b36cfff7SJohn Baldwin if (sg == NULL)
705b36cfff7SJohn Baldwin return (NULL);
706b36cfff7SJohn Baldwin if (sglist_append(sg, buf, len) != 0) {
707b36cfff7SJohn Baldwin sglist_free(sg);
708b36cfff7SJohn Baldwin return (NULL);
709b36cfff7SJohn Baldwin }
710b36cfff7SJohn Baldwin return (sg);
711b36cfff7SJohn Baldwin }
712b36cfff7SJohn Baldwin
713b36cfff7SJohn Baldwin /*
714b36cfff7SJohn Baldwin * Clone a new copy of a scatter/gather list.
715b36cfff7SJohn Baldwin */
716b36cfff7SJohn Baldwin struct sglist *
sglist_clone(struct sglist * sg,int mflags)717b36cfff7SJohn Baldwin sglist_clone(struct sglist *sg, int mflags)
718b36cfff7SJohn Baldwin {
719b36cfff7SJohn Baldwin struct sglist *new;
720b36cfff7SJohn Baldwin
721b36cfff7SJohn Baldwin if (sg == NULL)
722b36cfff7SJohn Baldwin return (NULL);
723b36cfff7SJohn Baldwin new = sglist_alloc(sg->sg_maxseg, mflags);
724b36cfff7SJohn Baldwin if (new == NULL)
725b36cfff7SJohn Baldwin return (NULL);
726eb5a1e8fSJohn Baldwin new->sg_nseg = sg->sg_nseg;
727b36cfff7SJohn Baldwin bcopy(sg->sg_segs, new->sg_segs, sizeof(struct sglist_seg) *
728b36cfff7SJohn Baldwin sg->sg_nseg);
729b36cfff7SJohn Baldwin return (new);
730b36cfff7SJohn Baldwin }
731b36cfff7SJohn Baldwin
732b36cfff7SJohn Baldwin /*
733b36cfff7SJohn Baldwin * Calculate the total length of the segments described in a
734b36cfff7SJohn Baldwin * scatter/gather list.
735b36cfff7SJohn Baldwin */
736b36cfff7SJohn Baldwin size_t
sglist_length(struct sglist * sg)737b36cfff7SJohn Baldwin sglist_length(struct sglist *sg)
738b36cfff7SJohn Baldwin {
739b36cfff7SJohn Baldwin size_t space;
740b36cfff7SJohn Baldwin int i;
741b36cfff7SJohn Baldwin
742b36cfff7SJohn Baldwin space = 0;
743b36cfff7SJohn Baldwin for (i = 0; i < sg->sg_nseg; i++)
744b36cfff7SJohn Baldwin space += sg->sg_segs[i].ss_len;
745b36cfff7SJohn Baldwin return (space);
746b36cfff7SJohn Baldwin }
747b36cfff7SJohn Baldwin
748b36cfff7SJohn Baldwin /*
749b36cfff7SJohn Baldwin * Split a scatter/gather list into two lists. The scatter/gather
750b36cfff7SJohn Baldwin * entries for the first 'length' bytes of the 'original' list are
751b36cfff7SJohn Baldwin * stored in the '*head' list and are removed from 'original'.
752b36cfff7SJohn Baldwin *
753b36cfff7SJohn Baldwin * If '*head' is NULL, then a new list will be allocated using
754b36cfff7SJohn Baldwin * 'mflags'. If M_NOWAIT is specified and the allocation fails,
755b36cfff7SJohn Baldwin * ENOMEM will be returned.
756b36cfff7SJohn Baldwin *
757b36cfff7SJohn Baldwin * If '*head' is not NULL, it should point to an empty sglist. If it
758b36cfff7SJohn Baldwin * does not have enough room for the remaining space, then EFBIG will
759b36cfff7SJohn Baldwin * be returned. If '*head' is not empty, then EINVAL will be
760b36cfff7SJohn Baldwin * returned.
761b36cfff7SJohn Baldwin *
762b36cfff7SJohn Baldwin * If 'original' is shared (refcount > 1), then EDOOFUS will be
763b36cfff7SJohn Baldwin * returned.
764b36cfff7SJohn Baldwin */
765b36cfff7SJohn Baldwin int
sglist_split(struct sglist * original,struct sglist ** head,size_t length,int mflags)766b36cfff7SJohn Baldwin sglist_split(struct sglist *original, struct sglist **head, size_t length,
767b36cfff7SJohn Baldwin int mflags)
768b36cfff7SJohn Baldwin {
769b36cfff7SJohn Baldwin struct sglist *sg;
770b36cfff7SJohn Baldwin size_t space, split;
771b36cfff7SJohn Baldwin int count, i;
772b36cfff7SJohn Baldwin
773b36cfff7SJohn Baldwin if (original->sg_refs > 1)
774b36cfff7SJohn Baldwin return (EDOOFUS);
775b36cfff7SJohn Baldwin
776b36cfff7SJohn Baldwin /* Figure out how big of a sglist '*head' has to hold. */
777b36cfff7SJohn Baldwin count = 0;
778b36cfff7SJohn Baldwin space = 0;
779b36cfff7SJohn Baldwin split = 0;
780b36cfff7SJohn Baldwin for (i = 0; i < original->sg_nseg; i++) {
781b36cfff7SJohn Baldwin space += original->sg_segs[i].ss_len;
782b36cfff7SJohn Baldwin count++;
783b36cfff7SJohn Baldwin if (space >= length) {
784b36cfff7SJohn Baldwin /*
785b36cfff7SJohn Baldwin * If 'length' falls in the middle of a
786b36cfff7SJohn Baldwin * scatter/gather list entry, then 'split'
787b36cfff7SJohn Baldwin * holds how much of that entry will remain in
788b36cfff7SJohn Baldwin * 'original'.
789b36cfff7SJohn Baldwin */
790b36cfff7SJohn Baldwin split = space - length;
791b36cfff7SJohn Baldwin break;
792b36cfff7SJohn Baldwin }
793b36cfff7SJohn Baldwin }
794b36cfff7SJohn Baldwin
795b36cfff7SJohn Baldwin /* Nothing to do, so leave head empty. */
796b36cfff7SJohn Baldwin if (count == 0)
797b36cfff7SJohn Baldwin return (0);
798b36cfff7SJohn Baldwin
799b36cfff7SJohn Baldwin if (*head == NULL) {
800b36cfff7SJohn Baldwin sg = sglist_alloc(count, mflags);
801b36cfff7SJohn Baldwin if (sg == NULL)
802b36cfff7SJohn Baldwin return (ENOMEM);
803b36cfff7SJohn Baldwin *head = sg;
804b36cfff7SJohn Baldwin } else {
805b36cfff7SJohn Baldwin sg = *head;
806b36cfff7SJohn Baldwin if (sg->sg_maxseg < count)
807b36cfff7SJohn Baldwin return (EFBIG);
808b36cfff7SJohn Baldwin if (sg->sg_nseg != 0)
809b36cfff7SJohn Baldwin return (EINVAL);
810b36cfff7SJohn Baldwin }
811b36cfff7SJohn Baldwin
812b36cfff7SJohn Baldwin /* Copy 'count' entries to 'sg' from 'original'. */
813b36cfff7SJohn Baldwin bcopy(original->sg_segs, sg->sg_segs, count *
814b36cfff7SJohn Baldwin sizeof(struct sglist_seg));
815b36cfff7SJohn Baldwin sg->sg_nseg = count;
816b36cfff7SJohn Baldwin
817b36cfff7SJohn Baldwin /*
818b36cfff7SJohn Baldwin * If we had to split a list entry, fixup the last entry in
819b36cfff7SJohn Baldwin * 'sg' and the new first entry in 'original'. We also
820b36cfff7SJohn Baldwin * decrement 'count' by 1 since we will only be removing
821b36cfff7SJohn Baldwin * 'count - 1' segments from 'original' now.
822b36cfff7SJohn Baldwin */
823b36cfff7SJohn Baldwin if (split != 0) {
824b36cfff7SJohn Baldwin count--;
825b36cfff7SJohn Baldwin sg->sg_segs[count].ss_len -= split;
826b36cfff7SJohn Baldwin original->sg_segs[count].ss_paddr =
827b36cfff7SJohn Baldwin sg->sg_segs[count].ss_paddr + split;
828b36cfff7SJohn Baldwin original->sg_segs[count].ss_len = split;
829b36cfff7SJohn Baldwin }
830b36cfff7SJohn Baldwin
831b36cfff7SJohn Baldwin /* Trim 'count' entries from the front of 'original'. */
832b36cfff7SJohn Baldwin original->sg_nseg -= count;
833b36cfff7SJohn Baldwin bcopy(original->sg_segs + count, original->sg_segs, count *
834b36cfff7SJohn Baldwin sizeof(struct sglist_seg));
835b36cfff7SJohn Baldwin return (0);
836b36cfff7SJohn Baldwin }
837b36cfff7SJohn Baldwin
838b36cfff7SJohn Baldwin /*
839b36cfff7SJohn Baldwin * Append the scatter/gather list elements in 'second' to the
840b36cfff7SJohn Baldwin * scatter/gather list 'first'. If there is not enough space in
841b36cfff7SJohn Baldwin * 'first', EFBIG is returned.
842b36cfff7SJohn Baldwin */
843b36cfff7SJohn Baldwin int
sglist_join(struct sglist * first,struct sglist * second)844b36cfff7SJohn Baldwin sglist_join(struct sglist *first, struct sglist *second)
845b36cfff7SJohn Baldwin {
846b36cfff7SJohn Baldwin struct sglist_seg *flast, *sfirst;
847b36cfff7SJohn Baldwin int append;
848b36cfff7SJohn Baldwin
849b36cfff7SJohn Baldwin /* If 'second' is empty, there is nothing to do. */
850b36cfff7SJohn Baldwin if (second->sg_nseg == 0)
851b36cfff7SJohn Baldwin return (0);
852b36cfff7SJohn Baldwin
853b36cfff7SJohn Baldwin /*
854b36cfff7SJohn Baldwin * If the first entry in 'second' can be appended to the last entry
855b36cfff7SJohn Baldwin * in 'first' then set append to '1'.
856b36cfff7SJohn Baldwin */
857b36cfff7SJohn Baldwin append = 0;
858b36cfff7SJohn Baldwin flast = &first->sg_segs[first->sg_nseg - 1];
859b36cfff7SJohn Baldwin sfirst = &second->sg_segs[0];
860b36cfff7SJohn Baldwin if (first->sg_nseg != 0 &&
861b36cfff7SJohn Baldwin flast->ss_paddr + flast->ss_len == sfirst->ss_paddr)
862b36cfff7SJohn Baldwin append = 1;
863b36cfff7SJohn Baldwin
864b36cfff7SJohn Baldwin /* Make sure 'first' has enough room. */
865b36cfff7SJohn Baldwin if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg)
866b36cfff7SJohn Baldwin return (EFBIG);
867b36cfff7SJohn Baldwin
868b36cfff7SJohn Baldwin /* Merge last in 'first' and first in 'second' if needed. */
869b36cfff7SJohn Baldwin if (append)
870b36cfff7SJohn Baldwin flast->ss_len += sfirst->ss_len;
871b36cfff7SJohn Baldwin
872b36cfff7SJohn Baldwin /* Append new segments from 'second' to 'first'. */
873b36cfff7SJohn Baldwin bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append,
874b36cfff7SJohn Baldwin (second->sg_nseg - append) * sizeof(struct sglist_seg));
875b36cfff7SJohn Baldwin first->sg_nseg += second->sg_nseg - append;
876b36cfff7SJohn Baldwin sglist_reset(second);
877b36cfff7SJohn Baldwin return (0);
878b36cfff7SJohn Baldwin }
879b36cfff7SJohn Baldwin
880b36cfff7SJohn Baldwin /*
881b36cfff7SJohn Baldwin * Generate a new scatter/gather list from a range of an existing
882b36cfff7SJohn Baldwin * scatter/gather list. The 'offset' and 'length' parameters specify
883b36cfff7SJohn Baldwin * the logical range of the 'original' list to extract. If that range
884b36cfff7SJohn Baldwin * is not a subset of the length of 'original', then EINVAL is
885b36cfff7SJohn Baldwin * returned. The new scatter/gather list is stored in '*slice'.
886b36cfff7SJohn Baldwin *
887b36cfff7SJohn Baldwin * If '*slice' is NULL, then a new list will be allocated using
888b36cfff7SJohn Baldwin * 'mflags'. If M_NOWAIT is specified and the allocation fails,
889b36cfff7SJohn Baldwin * ENOMEM will be returned.
890b36cfff7SJohn Baldwin *
891b36cfff7SJohn Baldwin * If '*slice' is not NULL, it should point to an empty sglist. If it
892b36cfff7SJohn Baldwin * does not have enough room for the remaining space, then EFBIG will
893b36cfff7SJohn Baldwin * be returned. If '*slice' is not empty, then EINVAL will be
894b36cfff7SJohn Baldwin * returned.
895b36cfff7SJohn Baldwin */
896b36cfff7SJohn Baldwin int
sglist_slice(struct sglist * original,struct sglist ** slice,size_t offset,size_t length,int mflags)897b36cfff7SJohn Baldwin sglist_slice(struct sglist *original, struct sglist **slice, size_t offset,
898b36cfff7SJohn Baldwin size_t length, int mflags)
899b36cfff7SJohn Baldwin {
900b36cfff7SJohn Baldwin struct sglist *sg;
901b36cfff7SJohn Baldwin size_t space, end, foffs, loffs;
902b36cfff7SJohn Baldwin int count, i, fseg;
903b36cfff7SJohn Baldwin
904b36cfff7SJohn Baldwin /* Nothing to do. */
905b36cfff7SJohn Baldwin if (length == 0)
906b36cfff7SJohn Baldwin return (0);
907b36cfff7SJohn Baldwin
908b36cfff7SJohn Baldwin /* Figure out how many segments '*slice' needs to have. */
909b36cfff7SJohn Baldwin end = offset + length;
910b36cfff7SJohn Baldwin space = 0;
911b36cfff7SJohn Baldwin count = 0;
912b36cfff7SJohn Baldwin fseg = 0;
913b36cfff7SJohn Baldwin foffs = loffs = 0;
914b36cfff7SJohn Baldwin for (i = 0; i < original->sg_nseg; i++) {
915b36cfff7SJohn Baldwin space += original->sg_segs[i].ss_len;
916b36cfff7SJohn Baldwin if (space > offset) {
917b36cfff7SJohn Baldwin /*
918b36cfff7SJohn Baldwin * When we hit the first segment, store its index
919b36cfff7SJohn Baldwin * in 'fseg' and the offset into the first segment
920b36cfff7SJohn Baldwin * of 'offset' in 'foffs'.
921b36cfff7SJohn Baldwin */
922b36cfff7SJohn Baldwin if (count == 0) {
923b36cfff7SJohn Baldwin fseg = i;
924b36cfff7SJohn Baldwin foffs = offset - (space -
925b36cfff7SJohn Baldwin original->sg_segs[i].ss_len);
926b36cfff7SJohn Baldwin CTR1(KTR_DEV, "sglist_slice: foffs = %08lx",
927b36cfff7SJohn Baldwin foffs);
928b36cfff7SJohn Baldwin }
929b36cfff7SJohn Baldwin count++;
930b36cfff7SJohn Baldwin
931b36cfff7SJohn Baldwin /*
932b36cfff7SJohn Baldwin * When we hit the last segment, break out of
933b36cfff7SJohn Baldwin * the loop. Store the amount of extra space
934b36cfff7SJohn Baldwin * at the end of this segment in 'loffs'.
935b36cfff7SJohn Baldwin */
936b36cfff7SJohn Baldwin if (space >= end) {
937b36cfff7SJohn Baldwin loffs = space - end;
938b36cfff7SJohn Baldwin CTR1(KTR_DEV, "sglist_slice: loffs = %08lx",
939b36cfff7SJohn Baldwin loffs);
940b36cfff7SJohn Baldwin break;
941b36cfff7SJohn Baldwin }
942b36cfff7SJohn Baldwin }
943b36cfff7SJohn Baldwin }
944b36cfff7SJohn Baldwin
945b36cfff7SJohn Baldwin /* If we never hit 'end', then 'length' ran off the end, so fail. */
946b36cfff7SJohn Baldwin if (space < end)
947b36cfff7SJohn Baldwin return (EINVAL);
948b36cfff7SJohn Baldwin
949b36cfff7SJohn Baldwin if (*slice == NULL) {
950b36cfff7SJohn Baldwin sg = sglist_alloc(count, mflags);
951b36cfff7SJohn Baldwin if (sg == NULL)
952b36cfff7SJohn Baldwin return (ENOMEM);
953b36cfff7SJohn Baldwin *slice = sg;
954b36cfff7SJohn Baldwin } else {
955b36cfff7SJohn Baldwin sg = *slice;
956b36cfff7SJohn Baldwin if (sg->sg_maxseg < count)
957b36cfff7SJohn Baldwin return (EFBIG);
958b36cfff7SJohn Baldwin if (sg->sg_nseg != 0)
959b36cfff7SJohn Baldwin return (EINVAL);
960b36cfff7SJohn Baldwin }
961b36cfff7SJohn Baldwin
962b36cfff7SJohn Baldwin /*
963b36cfff7SJohn Baldwin * Copy over 'count' segments from 'original' starting at
964b36cfff7SJohn Baldwin * 'fseg' to 'sg'.
965b36cfff7SJohn Baldwin */
966b36cfff7SJohn Baldwin bcopy(original->sg_segs + fseg, sg->sg_segs,
967b36cfff7SJohn Baldwin count * sizeof(struct sglist_seg));
968b36cfff7SJohn Baldwin sg->sg_nseg = count;
969b36cfff7SJohn Baldwin
970b36cfff7SJohn Baldwin /* Fixup first and last segments if needed. */
971b36cfff7SJohn Baldwin if (foffs != 0) {
972b36cfff7SJohn Baldwin sg->sg_segs[0].ss_paddr += foffs;
973b36cfff7SJohn Baldwin sg->sg_segs[0].ss_len -= foffs;
974b36cfff7SJohn Baldwin CTR2(KTR_DEV, "sglist_slice seg[0]: %08lx:%08lx",
975b36cfff7SJohn Baldwin (long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len);
976b36cfff7SJohn Baldwin }
977b36cfff7SJohn Baldwin if (loffs != 0) {
978b36cfff7SJohn Baldwin sg->sg_segs[count - 1].ss_len -= loffs;
979b36cfff7SJohn Baldwin CTR2(KTR_DEV, "sglist_slice seg[%d]: len %08x", count - 1,
980b36cfff7SJohn Baldwin sg->sg_segs[count - 1].ss_len);
981b36cfff7SJohn Baldwin }
982b36cfff7SJohn Baldwin return (0);
983b36cfff7SJohn Baldwin }
984