1 /*
2  * videobuf2-dma-sg.c - dma scatter/gather memory allocator for videobuf2
3  *
4  * Copyright (C) 2010 Samsung Electronics
5  *
6  * Author: Andrzej Pietrasiewicz <andrzejtp2010@gmail.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation.
11  */
12 
13 #include <linux/module.h>
14 #include <linux/mm.h>
15 #include <linux/refcount.h>
16 #include <linux/scatterlist.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 
21 #include <media/videobuf2-v4l2.h>
22 #include <media/videobuf2-memops.h>
23 #include <media/videobuf2-dma-sg.h>
24 
25 static int debug;
26 module_param(debug, int, 0644);
27 
28 #define dprintk(level, fmt, arg...)					\
29 	do {								\
30 		if (debug >= level)					\
31 			printk(KERN_DEBUG "vb2-dma-sg: " fmt, ## arg);	\
32 	} while (0)
33 
34 struct vb2_dma_sg_buf {
35 	struct device			*dev;
36 	void				*vaddr;
37 	struct page			**pages;
38 	struct frame_vector		*vec;
39 	int				offset;
40 	enum dma_data_direction		dma_dir;
41 	struct sg_table			sg_table;
42 	/*
43 	 * This will point to sg_table when used with the MMAP or USERPTR
44 	 * memory model, and to the dma_buf sglist when used with the
45 	 * DMABUF memory model.
46 	 */
47 	struct sg_table			*dma_sgt;
48 	size_t				size;
49 	unsigned int			num_pages;
50 	refcount_t			refcount;
51 	struct vb2_vmarea_handler	handler;
52 
53 	struct dma_buf_attachment	*db_attach;
54 
55 	struct vb2_buffer		*vb;
56 };
57 
58 static void vb2_dma_sg_put(void *buf_priv);
59 
60 static int vb2_dma_sg_alloc_compacted(struct vb2_dma_sg_buf *buf,
61 		gfp_t gfp_flags)
62 {
63 	unsigned int last_page = 0;
64 	unsigned long size = buf->size;
65 
66 	while (size > 0) {
67 		struct page *pages;
68 		int order;
69 		int i;
70 
71 		order = get_order(size);
72 		/* Don't over allocate*/
73 		if ((PAGE_SIZE << order) > size)
74 			order--;
75 
76 		pages = NULL;
77 		while (!pages) {
78 			pages = alloc_pages(GFP_KERNEL | __GFP_ZERO |
79 					__GFP_NOWARN | gfp_flags, order);
80 			if (pages)
81 				break;
82 
83 			if (order == 0) {
84 				while (last_page--)
85 					__free_page(buf->pages[last_page]);
86 				return -ENOMEM;
87 			}
88 			order--;
89 		}
90 
91 		split_page(pages, order);
92 		for (i = 0; i < (1 << order); i++)
93 			buf->pages[last_page++] = &pages[i];
94 
95 		size -= PAGE_SIZE << order;
96 	}
97 
98 	return 0;
99 }
100 
101 static void *vb2_dma_sg_alloc(struct vb2_buffer *vb, struct device *dev,
102 			      unsigned long size)
103 {
104 	struct vb2_dma_sg_buf *buf;
105 	struct sg_table *sgt;
106 	int ret;
107 	int num_pages;
108 
109 	if (WARN_ON(!dev) || WARN_ON(!size))
110 		return ERR_PTR(-EINVAL);
111 
112 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
113 	if (!buf)
114 		return ERR_PTR(-ENOMEM);
115 
116 	buf->vaddr = NULL;
117 	buf->dma_dir = vb->vb2_queue->dma_dir;
118 	buf->offset = 0;
119 	buf->size = size;
120 	/* size is already page aligned */
121 	buf->num_pages = size >> PAGE_SHIFT;
122 	buf->dma_sgt = &buf->sg_table;
123 
124 	/*
125 	 * NOTE: dma-sg allocates memory using the page allocator directly, so
126 	 * there is no memory consistency guarantee, hence dma-sg ignores DMA
127 	 * attributes passed from the upper layer.
128 	 */
129 	buf->pages = kvcalloc(buf->num_pages, sizeof(struct page *), GFP_KERNEL);
130 	if (!buf->pages)
131 		goto fail_pages_array_alloc;
132 
133 	ret = vb2_dma_sg_alloc_compacted(buf, vb->vb2_queue->gfp_flags);
134 	if (ret)
135 		goto fail_pages_alloc;
136 
137 	ret = sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
138 			buf->num_pages, 0, size, GFP_KERNEL);
139 	if (ret)
140 		goto fail_table_alloc;
141 
142 	/* Prevent the device from being released while the buffer is used */
143 	buf->dev = get_device(dev);
144 
145 	sgt = &buf->sg_table;
146 	/*
147 	 * No need to sync to the device, this will happen later when the
148 	 * prepare() memop is called.
149 	 */
150 	if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
151 			    DMA_ATTR_SKIP_CPU_SYNC))
152 		goto fail_map;
153 
154 	buf->handler.refcount = &buf->refcount;
155 	buf->handler.put = vb2_dma_sg_put;
156 	buf->handler.arg = buf;
157 	buf->vb = vb;
158 
159 	refcount_set(&buf->refcount, 1);
160 
161 	dprintk(1, "%s: Allocated buffer of %d pages\n",
162 		__func__, buf->num_pages);
163 	return buf;
164 
165 fail_map:
166 	put_device(buf->dev);
167 	sg_free_table(buf->dma_sgt);
168 fail_table_alloc:
169 	num_pages = buf->num_pages;
170 	while (num_pages--)
171 		__free_page(buf->pages[num_pages]);
172 fail_pages_alloc:
173 	kvfree(buf->pages);
174 fail_pages_array_alloc:
175 	kfree(buf);
176 	return ERR_PTR(-ENOMEM);
177 }
178 
179 static void vb2_dma_sg_put(void *buf_priv)
180 {
181 	struct vb2_dma_sg_buf *buf = buf_priv;
182 	struct sg_table *sgt = &buf->sg_table;
183 	int i = buf->num_pages;
184 
185 	if (refcount_dec_and_test(&buf->refcount)) {
186 		dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
187 			buf->num_pages);
188 		dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,
189 				  DMA_ATTR_SKIP_CPU_SYNC);
190 		if (buf->vaddr)
191 			vm_unmap_ram(buf->vaddr, buf->num_pages);
192 		sg_free_table(buf->dma_sgt);
193 		while (--i >= 0)
194 			__free_page(buf->pages[i]);
195 		kvfree(buf->pages);
196 		put_device(buf->dev);
197 		kfree(buf);
198 	}
199 }
200 
201 static void vb2_dma_sg_prepare(void *buf_priv)
202 {
203 	struct vb2_dma_sg_buf *buf = buf_priv;
204 	struct sg_table *sgt = buf->dma_sgt;
205 
206 	if (buf->vb->skip_cache_sync_on_prepare)
207 		return;
208 
209 	dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);
210 }
211 
212 static void vb2_dma_sg_finish(void *buf_priv)
213 {
214 	struct vb2_dma_sg_buf *buf = buf_priv;
215 	struct sg_table *sgt = buf->dma_sgt;
216 
217 	if (buf->vb->skip_cache_sync_on_finish)
218 		return;
219 
220 	dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
221 }
222 
223 static void *vb2_dma_sg_get_userptr(struct vb2_buffer *vb, struct device *dev,
224 				    unsigned long vaddr, unsigned long size)
225 {
226 	struct vb2_dma_sg_buf *buf;
227 	struct sg_table *sgt;
228 	struct frame_vector *vec;
229 
230 	if (WARN_ON(!dev))
231 		return ERR_PTR(-EINVAL);
232 
233 	buf = kzalloc(sizeof *buf, GFP_KERNEL);
234 	if (!buf)
235 		return ERR_PTR(-ENOMEM);
236 
237 	buf->vaddr = NULL;
238 	buf->dev = dev;
239 	buf->dma_dir = vb->vb2_queue->dma_dir;
240 	buf->offset = vaddr & ~PAGE_MASK;
241 	buf->size = size;
242 	buf->dma_sgt = &buf->sg_table;
243 	buf->vb = vb;
244 	vec = vb2_create_framevec(vaddr, size,
245 				  buf->dma_dir == DMA_FROM_DEVICE ||
246 				  buf->dma_dir == DMA_BIDIRECTIONAL);
247 	if (IS_ERR(vec))
248 		goto userptr_fail_pfnvec;
249 	buf->vec = vec;
250 
251 	buf->pages = frame_vector_pages(vec);
252 	if (IS_ERR(buf->pages))
253 		goto userptr_fail_sgtable;
254 	buf->num_pages = frame_vector_count(vec);
255 
256 	if (sg_alloc_table_from_pages(buf->dma_sgt, buf->pages,
257 			buf->num_pages, buf->offset, size, 0))
258 		goto userptr_fail_sgtable;
259 
260 	sgt = &buf->sg_table;
261 	/*
262 	 * No need to sync to the device, this will happen later when the
263 	 * prepare() memop is called.
264 	 */
265 	if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
266 			    DMA_ATTR_SKIP_CPU_SYNC))
267 		goto userptr_fail_map;
268 
269 	return buf;
270 
271 userptr_fail_map:
272 	sg_free_table(&buf->sg_table);
273 userptr_fail_sgtable:
274 	vb2_destroy_framevec(vec);
275 userptr_fail_pfnvec:
276 	kfree(buf);
277 	return ERR_PTR(-ENOMEM);
278 }
279 
280 /*
281  * @put_userptr: inform the allocator that a USERPTR buffer will no longer
282  *		 be used
283  */
284 static void vb2_dma_sg_put_userptr(void *buf_priv)
285 {
286 	struct vb2_dma_sg_buf *buf = buf_priv;
287 	struct sg_table *sgt = &buf->sg_table;
288 	int i = buf->num_pages;
289 
290 	dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
291 	       __func__, buf->num_pages);
292 	dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
293 	if (buf->vaddr)
294 		vm_unmap_ram(buf->vaddr, buf->num_pages);
295 	sg_free_table(buf->dma_sgt);
296 	if (buf->dma_dir == DMA_FROM_DEVICE ||
297 	    buf->dma_dir == DMA_BIDIRECTIONAL)
298 		while (--i >= 0)
299 			set_page_dirty_lock(buf->pages[i]);
300 	vb2_destroy_framevec(buf->vec);
301 	kfree(buf);
302 }
303 
304 static void *vb2_dma_sg_vaddr(struct vb2_buffer *vb, void *buf_priv)
305 {
306 	struct vb2_dma_sg_buf *buf = buf_priv;
307 	struct iosys_map map;
308 	int ret;
309 
310 	BUG_ON(!buf);
311 
312 	if (!buf->vaddr) {
313 		if (buf->db_attach) {
314 			ret = dma_buf_vmap_unlocked(buf->db_attach->dmabuf, &map);
315 			buf->vaddr = ret ? NULL : map.vaddr;
316 		} else {
317 			buf->vaddr = vm_map_ram(buf->pages, buf->num_pages, -1);
318 		}
319 	}
320 
321 	/* add offset in case userptr is not page-aligned */
322 	return buf->vaddr ? buf->vaddr + buf->offset : NULL;
323 }
324 
325 static unsigned int vb2_dma_sg_num_users(void *buf_priv)
326 {
327 	struct vb2_dma_sg_buf *buf = buf_priv;
328 
329 	return refcount_read(&buf->refcount);
330 }
331 
332 static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma)
333 {
334 	struct vb2_dma_sg_buf *buf = buf_priv;
335 	int err;
336 
337 	if (!buf) {
338 		printk(KERN_ERR "No memory to map\n");
339 		return -EINVAL;
340 	}
341 
342 	err = vm_map_pages(vma, buf->pages, buf->num_pages);
343 	if (err) {
344 		printk(KERN_ERR "Remapping memory, error: %d\n", err);
345 		return err;
346 	}
347 
348 	/*
349 	 * Use common vm_area operations to track buffer refcount.
350 	 */
351 	vma->vm_private_data	= &buf->handler;
352 	vma->vm_ops		= &vb2_common_vm_ops;
353 
354 	vma->vm_ops->open(vma);
355 
356 	return 0;
357 }
358 
359 /*********************************************/
360 /*         DMABUF ops for exporters          */
361 /*********************************************/
362 
363 struct vb2_dma_sg_attachment {
364 	struct sg_table sgt;
365 	enum dma_data_direction dma_dir;
366 };
367 
368 static int vb2_dma_sg_dmabuf_ops_attach(struct dma_buf *dbuf,
369 	struct dma_buf_attachment *dbuf_attach)
370 {
371 	struct vb2_dma_sg_attachment *attach;
372 	unsigned int i;
373 	struct scatterlist *rd, *wr;
374 	struct sg_table *sgt;
375 	struct vb2_dma_sg_buf *buf = dbuf->priv;
376 	int ret;
377 
378 	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
379 	if (!attach)
380 		return -ENOMEM;
381 
382 	sgt = &attach->sgt;
383 	/* Copy the buf->base_sgt scatter list to the attachment, as we can't
384 	 * map the same scatter list to multiple attachments at the same time.
385 	 */
386 	ret = sg_alloc_table(sgt, buf->dma_sgt->orig_nents, GFP_KERNEL);
387 	if (ret) {
388 		kfree(attach);
389 		return -ENOMEM;
390 	}
391 
392 	rd = buf->dma_sgt->sgl;
393 	wr = sgt->sgl;
394 	for (i = 0; i < sgt->orig_nents; ++i) {
395 		sg_set_page(wr, sg_page(rd), rd->length, rd->offset);
396 		rd = sg_next(rd);
397 		wr = sg_next(wr);
398 	}
399 
400 	attach->dma_dir = DMA_NONE;
401 	dbuf_attach->priv = attach;
402 
403 	return 0;
404 }
405 
406 static void vb2_dma_sg_dmabuf_ops_detach(struct dma_buf *dbuf,
407 	struct dma_buf_attachment *db_attach)
408 {
409 	struct vb2_dma_sg_attachment *attach = db_attach->priv;
410 	struct sg_table *sgt;
411 
412 	if (!attach)
413 		return;
414 
415 	sgt = &attach->sgt;
416 
417 	/* release the scatterlist cache */
418 	if (attach->dma_dir != DMA_NONE)
419 		dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
420 	sg_free_table(sgt);
421 	kfree(attach);
422 	db_attach->priv = NULL;
423 }
424 
425 static struct sg_table *vb2_dma_sg_dmabuf_ops_map(
426 	struct dma_buf_attachment *db_attach, enum dma_data_direction dma_dir)
427 {
428 	struct vb2_dma_sg_attachment *attach = db_attach->priv;
429 	struct sg_table *sgt;
430 
431 	sgt = &attach->sgt;
432 	/* return previously mapped sg table */
433 	if (attach->dma_dir == dma_dir)
434 		return sgt;
435 
436 	/* release any previous cache */
437 	if (attach->dma_dir != DMA_NONE) {
438 		dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
439 		attach->dma_dir = DMA_NONE;
440 	}
441 
442 	/* mapping to the client with new direction */
443 	if (dma_map_sgtable(db_attach->dev, sgt, dma_dir, 0)) {
444 		pr_err("failed to map scatterlist\n");
445 		return ERR_PTR(-EIO);
446 	}
447 
448 	attach->dma_dir = dma_dir;
449 
450 	return sgt;
451 }
452 
453 static void vb2_dma_sg_dmabuf_ops_unmap(struct dma_buf_attachment *db_attach,
454 	struct sg_table *sgt, enum dma_data_direction dma_dir)
455 {
456 	/* nothing to be done here */
457 }
458 
459 static void vb2_dma_sg_dmabuf_ops_release(struct dma_buf *dbuf)
460 {
461 	/* drop reference obtained in vb2_dma_sg_get_dmabuf */
462 	vb2_dma_sg_put(dbuf->priv);
463 }
464 
465 static int
466 vb2_dma_sg_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf,
467 				       enum dma_data_direction direction)
468 {
469 	struct vb2_dma_sg_buf *buf = dbuf->priv;
470 	struct sg_table *sgt = buf->dma_sgt;
471 
472 	dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
473 	return 0;
474 }
475 
476 static int
477 vb2_dma_sg_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf,
478 				     enum dma_data_direction direction)
479 {
480 	struct vb2_dma_sg_buf *buf = dbuf->priv;
481 	struct sg_table *sgt = buf->dma_sgt;
482 
483 	dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
484 	return 0;
485 }
486 
487 static int vb2_dma_sg_dmabuf_ops_vmap(struct dma_buf *dbuf,
488 				      struct iosys_map *map)
489 {
490 	struct vb2_dma_sg_buf *buf = dbuf->priv;
491 
492 	iosys_map_set_vaddr(map, buf->vaddr);
493 
494 	return 0;
495 }
496 
497 static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf,
498 	struct vm_area_struct *vma)
499 {
500 	return vb2_dma_sg_mmap(dbuf->priv, vma);
501 }
502 
503 static const struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
504 	.attach = vb2_dma_sg_dmabuf_ops_attach,
505 	.detach = vb2_dma_sg_dmabuf_ops_detach,
506 	.map_dma_buf = vb2_dma_sg_dmabuf_ops_map,
507 	.unmap_dma_buf = vb2_dma_sg_dmabuf_ops_unmap,
508 	.begin_cpu_access = vb2_dma_sg_dmabuf_ops_begin_cpu_access,
509 	.end_cpu_access = vb2_dma_sg_dmabuf_ops_end_cpu_access,
510 	.vmap = vb2_dma_sg_dmabuf_ops_vmap,
511 	.mmap = vb2_dma_sg_dmabuf_ops_mmap,
512 	.release = vb2_dma_sg_dmabuf_ops_release,
513 };
514 
515 static struct dma_buf *vb2_dma_sg_get_dmabuf(struct vb2_buffer *vb,
516 					     void *buf_priv,
517 					     unsigned long flags)
518 {
519 	struct vb2_dma_sg_buf *buf = buf_priv;
520 	struct dma_buf *dbuf;
521 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
522 
523 	exp_info.ops = &vb2_dma_sg_dmabuf_ops;
524 	exp_info.size = buf->size;
525 	exp_info.flags = flags;
526 	exp_info.priv = buf;
527 
528 	if (WARN_ON(!buf->dma_sgt))
529 		return NULL;
530 
531 	dbuf = dma_buf_export(&exp_info);
532 	if (IS_ERR(dbuf))
533 		return NULL;
534 
535 	/* dmabuf keeps reference to vb2 buffer */
536 	refcount_inc(&buf->refcount);
537 
538 	return dbuf;
539 }
540 
541 /*********************************************/
542 /*       callbacks for DMABUF buffers        */
543 /*********************************************/
544 
545 static int vb2_dma_sg_map_dmabuf(void *mem_priv)
546 {
547 	struct vb2_dma_sg_buf *buf = mem_priv;
548 	struct sg_table *sgt;
549 
550 	if (WARN_ON(!buf->db_attach)) {
551 		pr_err("trying to pin a non attached buffer\n");
552 		return -EINVAL;
553 	}
554 
555 	if (WARN_ON(buf->dma_sgt)) {
556 		pr_err("dmabuf buffer is already pinned\n");
557 		return 0;
558 	}
559 
560 	/* get the associated scatterlist for this buffer */
561 	sgt = dma_buf_map_attachment_unlocked(buf->db_attach, buf->dma_dir);
562 	if (IS_ERR(sgt)) {
563 		pr_err("Error getting dmabuf scatterlist\n");
564 		return -EINVAL;
565 	}
566 
567 	buf->dma_sgt = sgt;
568 	buf->vaddr = NULL;
569 
570 	return 0;
571 }
572 
573 static void vb2_dma_sg_unmap_dmabuf(void *mem_priv)
574 {
575 	struct vb2_dma_sg_buf *buf = mem_priv;
576 	struct sg_table *sgt = buf->dma_sgt;
577 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(buf->vaddr);
578 
579 	if (WARN_ON(!buf->db_attach)) {
580 		pr_err("trying to unpin a not attached buffer\n");
581 		return;
582 	}
583 
584 	if (WARN_ON(!sgt)) {
585 		pr_err("dmabuf buffer is already unpinned\n");
586 		return;
587 	}
588 
589 	if (buf->vaddr) {
590 		dma_buf_vunmap_unlocked(buf->db_attach->dmabuf, &map);
591 		buf->vaddr = NULL;
592 	}
593 	dma_buf_unmap_attachment_unlocked(buf->db_attach, sgt, buf->dma_dir);
594 
595 	buf->dma_sgt = NULL;
596 }
597 
598 static void vb2_dma_sg_detach_dmabuf(void *mem_priv)
599 {
600 	struct vb2_dma_sg_buf *buf = mem_priv;
601 
602 	/* if vb2 works correctly you should never detach mapped buffer */
603 	if (WARN_ON(buf->dma_sgt))
604 		vb2_dma_sg_unmap_dmabuf(buf);
605 
606 	/* detach this attachment */
607 	dma_buf_detach(buf->db_attach->dmabuf, buf->db_attach);
608 	kfree(buf);
609 }
610 
611 static void *vb2_dma_sg_attach_dmabuf(struct vb2_buffer *vb, struct device *dev,
612 				      struct dma_buf *dbuf, unsigned long size)
613 {
614 	struct vb2_dma_sg_buf *buf;
615 	struct dma_buf_attachment *dba;
616 
617 	if (WARN_ON(!dev))
618 		return ERR_PTR(-EINVAL);
619 
620 	if (dbuf->size < size)
621 		return ERR_PTR(-EFAULT);
622 
623 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
624 	if (!buf)
625 		return ERR_PTR(-ENOMEM);
626 
627 	buf->dev = dev;
628 	/* create attachment for the dmabuf with the user device */
629 	dba = dma_buf_attach(dbuf, buf->dev);
630 	if (IS_ERR(dba)) {
631 		pr_err("failed to attach dmabuf\n");
632 		kfree(buf);
633 		return dba;
634 	}
635 
636 	buf->dma_dir = vb->vb2_queue->dma_dir;
637 	buf->size = size;
638 	buf->db_attach = dba;
639 	buf->vb = vb;
640 
641 	return buf;
642 }
643 
644 static void *vb2_dma_sg_cookie(struct vb2_buffer *vb, void *buf_priv)
645 {
646 	struct vb2_dma_sg_buf *buf = buf_priv;
647 
648 	return buf->dma_sgt;
649 }
650 
651 const struct vb2_mem_ops vb2_dma_sg_memops = {
652 	.alloc		= vb2_dma_sg_alloc,
653 	.put		= vb2_dma_sg_put,
654 	.get_userptr	= vb2_dma_sg_get_userptr,
655 	.put_userptr	= vb2_dma_sg_put_userptr,
656 	.prepare	= vb2_dma_sg_prepare,
657 	.finish		= vb2_dma_sg_finish,
658 	.vaddr		= vb2_dma_sg_vaddr,
659 	.mmap		= vb2_dma_sg_mmap,
660 	.num_users	= vb2_dma_sg_num_users,
661 	.get_dmabuf	= vb2_dma_sg_get_dmabuf,
662 	.map_dmabuf	= vb2_dma_sg_map_dmabuf,
663 	.unmap_dmabuf	= vb2_dma_sg_unmap_dmabuf,
664 	.attach_dmabuf	= vb2_dma_sg_attach_dmabuf,
665 	.detach_dmabuf	= vb2_dma_sg_detach_dmabuf,
666 	.cookie		= vb2_dma_sg_cookie,
667 };
668 EXPORT_SYMBOL_GPL(vb2_dma_sg_memops);
669 
670 MODULE_DESCRIPTION("dma scatter/gather memory handling routines for videobuf2");
671 MODULE_AUTHOR("Andrzej Pietrasiewicz");
672 MODULE_LICENSE("GPL");
673 MODULE_IMPORT_NS(DMA_BUF);
674