1 /*-
2  * Copyright (c) 2010 Isilon Systems, Inc.
3  * Copyright (c) 2010 iX Systems, Inc.
4  * Copyright (c) 2010 Panasas, Inc.
5  * Copyright (c) 2013-2017 Mellanox Technologies, Ltd.
6  * Copyright (c) 2015 Matthew Dillon <dillon@backplane.com>
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice unmodified, this list of conditions, and the following
14  *    disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  *
30  * $FreeBSD$
31  */
32 #ifndef	_LINUX_SCATTERLIST_H_
33 #define	_LINUX_SCATTERLIST_H_
34 
35 #include <linux/page.h>
36 #include <linux/slab.h>
37 #include <linux/mm.h>
38 
39 struct scatterlist {
40 	unsigned long page_link;
41 #define	SG_PAGE_LINK_CHAIN	0x1UL
42 #define	SG_PAGE_LINK_LAST	0x2UL
43 #define	SG_PAGE_LINK_MASK	0x3UL
44 	unsigned int offset;
45 	unsigned int length;
46 	dma_addr_t address;
47 };
48 
49 CTASSERT((sizeof(struct scatterlist) & SG_PAGE_LINK_MASK) == 0);
50 
51 struct sg_table {
52 	struct scatterlist *sgl;
53 	unsigned int nents;
54 	unsigned int orig_nents;
55 };
56 
57 struct sg_page_iter {
58 	struct scatterlist *sg;
59 	unsigned int sg_pgoffset;
60 	unsigned int maxents;
61 	struct {
62 		unsigned int nents;
63 		int	pg_advance;
64 	} internal;
65 };
66 
67 #define	SG_MAX_SINGLE_ALLOC	(PAGE_SIZE / sizeof(struct scatterlist))
68 
69 #define	SG_MAGIC		0x87654321UL
70 
71 #define	sg_is_chain(sg)		((sg)->page_link & SG_PAGE_LINK_CHAIN)
72 #define	sg_is_last(sg)		((sg)->page_link & SG_PAGE_LINK_LAST)
73 #define	sg_chain_ptr(sg)	\
74 	((struct scatterlist *) ((sg)->page_link & ~SG_PAGE_LINK_MASK))
75 
76 #define	sg_dma_address(sg)	(sg)->address
77 #define	sg_dma_len(sg)		(sg)->length
78 
79 #define	for_each_sg_page(sgl, iter, nents, pgoffset)			\
80 	for (_sg_iter_init(sgl, iter, nents, pgoffset);			\
81 	     (iter)->sg; _sg_iter_next(iter))
82 
83 #define	for_each_sg(sglist, sg, sgmax, iter)				\
84 	for (iter = 0, sg = (sglist); iter < (sgmax); iter++, sg = sg_next(sg))
85 
86 typedef struct scatterlist *(sg_alloc_fn) (unsigned int, gfp_t);
87 typedef void (sg_free_fn) (struct scatterlist *, unsigned int);
88 
89 static inline void
90 sg_assign_page(struct scatterlist *sg, struct page *page)
91 {
92 	unsigned long page_link = sg->page_link & SG_PAGE_LINK_MASK;
93 
94 	sg->page_link = page_link | (unsigned long)page;
95 }
96 
97 static inline void
98 sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len,
99     unsigned int offset)
100 {
101 	sg_assign_page(sg, page);
102 	sg->offset = offset;
103 	sg->length = len;
104 }
105 
106 static inline struct page *
107 sg_page(struct scatterlist *sg)
108 {
109 	return ((struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK));
110 }
111 
112 static inline void
113 sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen)
114 {
115 	sg_set_page(sg, virt_to_page(buf), buflen,
116 	    ((uintptr_t)buf) & (PAGE_SIZE - 1));
117 }
118 
119 static inline struct scatterlist *
120 sg_next(struct scatterlist *sg)
121 {
122 	if (sg_is_last(sg))
123 		return (NULL);
124 	sg++;
125 	if (sg_is_chain(sg))
126 		sg = sg_chain_ptr(sg);
127 	return (sg);
128 }
129 
130 static inline vm_paddr_t
131 sg_phys(struct scatterlist *sg)
132 {
133 	return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset);
134 }
135 
136 static inline void
137 sg_chain(struct scatterlist *prv, unsigned int prv_nents,
138     struct scatterlist *sgl)
139 {
140 	struct scatterlist *sg = &prv[prv_nents - 1];
141 
142 	sg->offset = 0;
143 	sg->length = 0;
144 	sg->page_link = ((unsigned long)sgl |
145 	    SG_PAGE_LINK_CHAIN) & ~SG_PAGE_LINK_LAST;
146 }
147 
148 static inline void
149 sg_mark_end(struct scatterlist *sg)
150 {
151 	sg->page_link |= SG_PAGE_LINK_LAST;
152 	sg->page_link &= ~SG_PAGE_LINK_CHAIN;
153 }
154 
155 static inline void
156 sg_init_table(struct scatterlist *sg, unsigned int nents)
157 {
158 	bzero(sg, sizeof(*sg) * nents);
159 	sg_mark_end(&sg[nents - 1]);
160 }
161 
162 static struct scatterlist *
163 sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
164 {
165 	if (nents == SG_MAX_SINGLE_ALLOC) {
166 		return ((void *)__get_free_page(gfp_mask));
167 	} else
168 		return (kmalloc(nents * sizeof(struct scatterlist), gfp_mask));
169 }
170 
171 static inline void
172 sg_kfree(struct scatterlist *sg, unsigned int nents)
173 {
174 	if (nents == SG_MAX_SINGLE_ALLOC) {
175 		free_page((unsigned long)sg);
176 	} else
177 		kfree(sg);
178 }
179 
180 static inline void
181 __sg_free_table(struct sg_table *table, unsigned int max_ents,
182     bool skip_first_chunk, sg_free_fn * free_fn)
183 {
184 	struct scatterlist *sgl, *next;
185 
186 	if (unlikely(!table->sgl))
187 		return;
188 
189 	sgl = table->sgl;
190 	while (table->orig_nents) {
191 		unsigned int alloc_size = table->orig_nents;
192 		unsigned int sg_size;
193 
194 		if (alloc_size > max_ents) {
195 			next = sg_chain_ptr(&sgl[max_ents - 1]);
196 			alloc_size = max_ents;
197 			sg_size = alloc_size - 1;
198 		} else {
199 			sg_size = alloc_size;
200 			next = NULL;
201 		}
202 
203 		table->orig_nents -= sg_size;
204 		if (skip_first_chunk)
205 			skip_first_chunk = 0;
206 		else
207 			free_fn(sgl, alloc_size);
208 		sgl = next;
209 	}
210 
211 	table->sgl = NULL;
212 }
213 
214 static inline void
215 sg_free_table(struct sg_table *table)
216 {
217 	__sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
218 }
219 
220 static inline int
221 __sg_alloc_table(struct sg_table *table, unsigned int nents,
222     unsigned int max_ents, struct scatterlist *first_chunk,
223     gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
224 {
225 	struct scatterlist *sg, *prv;
226 	unsigned int left;
227 
228 	memset(table, 0, sizeof(*table));
229 
230 	if (nents == 0)
231 		return (-EINVAL);
232 	left = nents;
233 	prv = NULL;
234 	do {
235 		unsigned int sg_size;
236 		unsigned int alloc_size = left;
237 
238 		if (alloc_size > max_ents) {
239 			alloc_size = max_ents;
240 			sg_size = alloc_size - 1;
241 		} else
242 			sg_size = alloc_size;
243 
244 		left -= sg_size;
245 
246 		if (first_chunk) {
247 			sg = first_chunk;
248 			first_chunk = NULL;
249 		} else {
250 			sg = alloc_fn(alloc_size, gfp_mask);
251 		}
252 		if (unlikely(!sg)) {
253 			if (prv)
254 				table->nents = ++table->orig_nents;
255 
256 			return (-ENOMEM);
257 		}
258 		sg_init_table(sg, alloc_size);
259 		table->nents = table->orig_nents += sg_size;
260 
261 		if (prv)
262 			sg_chain(prv, max_ents, sg);
263 		else
264 			table->sgl = sg;
265 
266 		if (!left)
267 			sg_mark_end(&sg[sg_size - 1]);
268 
269 		prv = sg;
270 	} while (left);
271 
272 	return (0);
273 }
274 
275 static inline int
276 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
277 {
278 	int ret;
279 
280 	ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
281 	    NULL, gfp_mask, sg_kmalloc);
282 	if (unlikely(ret))
283 		__sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
284 
285 	return (ret);
286 }
287 
288 static inline int
289 sg_alloc_table_from_pages(struct sg_table *sgt,
290     struct page **pages, unsigned int count,
291     unsigned long off, unsigned long size,
292     gfp_t gfp_mask)
293 {
294 	unsigned int i, segs, cur;
295 	int rc;
296 	struct scatterlist *s;
297 
298 	for (segs = i = 1; i < count; ++i) {
299 		if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1)
300 			++segs;
301 	}
302 	if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask))))
303 		return (rc);
304 
305 	cur = 0;
306 	for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
307 		unsigned long seg_size;
308 		unsigned int j;
309 
310 		for (j = cur + 1; j < count; ++j)
311 			if (page_to_pfn(pages[j]) !=
312 			    page_to_pfn(pages[j - 1]) + 1)
313 				break;
314 
315 		seg_size = ((j - cur) << PAGE_SHIFT) - off;
316 		sg_set_page(s, pages[cur], min(size, seg_size), off);
317 		size -= seg_size;
318 		off = 0;
319 		cur = j;
320 	}
321 	return (0);
322 }
323 
324 
325 static inline int
326 sg_nents(struct scatterlist *sg)
327 {
328 	int nents;
329 
330 	for (nents = 0; sg; sg = sg_next(sg))
331 		nents++;
332 	return (nents);
333 }
334 
335 static inline void
336 __sg_page_iter_start(struct sg_page_iter *piter,
337     struct scatterlist *sglist, unsigned int nents,
338     unsigned long pgoffset)
339 {
340 	piter->internal.pg_advance = 0;
341 	piter->internal.nents = nents;
342 
343 	piter->sg = sglist;
344 	piter->sg_pgoffset = pgoffset;
345 }
346 
347 static inline void
348 _sg_iter_next(struct sg_page_iter *iter)
349 {
350 	struct scatterlist *sg;
351 	unsigned int pgcount;
352 
353 	sg = iter->sg;
354 	pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
355 
356 	++iter->sg_pgoffset;
357 	while (iter->sg_pgoffset >= pgcount) {
358 		iter->sg_pgoffset -= pgcount;
359 		sg = sg_next(sg);
360 		--iter->maxents;
361 		if (sg == NULL || iter->maxents == 0)
362 			break;
363 		pgcount = (sg->offset + sg->length + PAGE_SIZE - 1) >> PAGE_SHIFT;
364 	}
365 	iter->sg = sg;
366 }
367 
368 static inline int
369 sg_page_count(struct scatterlist *sg)
370 {
371 	return (PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT);
372 }
373 
374 static inline bool
375 __sg_page_iter_next(struct sg_page_iter *piter)
376 {
377 	if (piter->internal.nents == 0)
378 		return (0);
379 	if (piter->sg == NULL)
380 		return (0);
381 
382 	piter->sg_pgoffset += piter->internal.pg_advance;
383 	piter->internal.pg_advance = 1;
384 
385 	while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
386 		piter->sg_pgoffset -= sg_page_count(piter->sg);
387 		piter->sg = sg_next(piter->sg);
388 		if (--piter->internal.nents == 0)
389 			return (0);
390 		if (piter->sg == NULL)
391 			return (0);
392 	}
393 	return (1);
394 }
395 
396 static inline void
397 _sg_iter_init(struct scatterlist *sgl, struct sg_page_iter *iter,
398     unsigned int nents, unsigned long pgoffset)
399 {
400 	if (nents) {
401 		iter->sg = sgl;
402 		iter->sg_pgoffset = pgoffset - 1;
403 		iter->maxents = nents;
404 		_sg_iter_next(iter);
405 	} else {
406 		iter->sg = NULL;
407 		iter->sg_pgoffset = 0;
408 		iter->maxents = 0;
409 	}
410 }
411 
412 static inline dma_addr_t
413 sg_page_iter_dma_address(struct sg_page_iter *spi)
414 {
415 	return (spi->sg->address + (spi->sg_pgoffset << PAGE_SHIFT));
416 }
417 
418 static inline struct page *
419 sg_page_iter_page(struct sg_page_iter *piter)
420 {
421 	return (nth_page(sg_page(piter->sg), piter->sg_pgoffset));
422 }
423 
424 
425 #endif					/* _LINUX_SCATTERLIST_H_ */
426