1	/*
2	* videobuf2-core.c - video buffer 2 core framework
3	*
4	* Copyright (C) 2010 Samsung Electronics
5	*
6	* Author: Pawel Osciak <pawel@osciak.com>
7	* Marek Szyprowski <m.szyprowski@samsung.com>
8	*
9	* The vb2_thread implementation was based on code from videobuf-dvb.c:
10	* (c) 2004 Gerd Knorr <kraxel@bytesex.org> [SUSE Labs]
11	*
12	* This program is free software; you can redistribute it and/or modify
13	* it under the terms of the GNU General Public License as published by
14	* the Free Software Foundation.
15	*/
16
17	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19	#include <linux/err.h>
20	#include <linux/kernel.h>
21	#include <linux/module.h>
22	#include <linux/mm.h>
23	#include <linux/poll.h>
24	#include <linux/slab.h>
25	#include <linux/sched.h>
26	#include <linux/freezer.h>
27	#include <linux/kthread.h>
28
29	#include <media/videobuf2-core.h>
30	#include <media/v4l2-mc.h>
31
32	#include <trace/events/vb2.h>
33
34	#define PLANE_INDEX_BITS 3
35	#define PLANE_INDEX_SHIFT (PAGE_SHIFT + PLANE_INDEX_BITS)
36	#define PLANE_INDEX_MASK (BIT_MASK(PLANE_INDEX_BITS) - 1)
37	#define MAX_BUFFER_INDEX BIT_MASK(30 - PLANE_INDEX_SHIFT)
38	#define BUFFER_INDEX_MASK (MAX_BUFFER_INDEX - 1)
39
40	#if BIT(PLANE_INDEX_BITS) != VIDEO_MAX_PLANES
41	#error PLANE_INDEX_BITS order must be equal to VIDEO_MAX_PLANES
42	#endif
43
44	static int debug;
45	module_param(debug, int, 0644);
46
47	#define dprintk(q, level, fmt, arg...) \
48	do { \
49	if (debug >= level) \
50	pr_info("[%s] %s: " fmt, (q)->name, __func__, \
51	## arg); \
52	} while (0)
53
54	#ifdef CONFIG_VIDEO_ADV_DEBUG
55
56	/*
57	* If advanced debugging is on, then count how often each op is called
58	* successfully, which can either be per-buffer or per-queue.
59	*
60	* This makes it easy to check that the 'init' and 'cleanup'
61	* (and variations thereof) stay balanced.
62	*/
63
64	#define log_memop(vb, op) \
65	dprintk((vb)->vb2_queue, 2, "call_memop(%d, %s)%s\n", \
66	(vb)->index, #op, \
67	(vb)->vb2_queue->mem_ops->op ? "" : " (nop)")
68
69	#define call_memop(vb, op, args...) \
70	({ \
71	struct vb2_queue *_q = (vb)->vb2_queue; \
72	int err; \
73	\
74	log_memop(vb, op); \
75	err = _q->mem_ops->op ? _q->mem_ops->op(args) : 0; \
76	if (!err) \
77	(vb)->cnt_mem_ ## op++; \
78	err; \
79	})
80
81	#define call_ptr_memop(op, vb, args...) \
82	({ \
83	struct vb2_queue *_q = (vb)->vb2_queue; \
84	void *ptr; \
85	\
86	log_memop(vb, op); \
87	ptr = _q->mem_ops->op ? _q->mem_ops->op(vb, args) : NULL; \
88	if (!IS_ERR_OR_NULL(ptr)) \
89	(vb)->cnt_mem_ ## op++; \
90	ptr; \
91	})
92
93	#define call_void_memop(vb, op, args...) \
94	({ \
95	struct vb2_queue *_q = (vb)->vb2_queue; \
96	\
97	log_memop(vb, op); \
98	if (_q->mem_ops->op) \
99	_q->mem_ops->op(args); \
100	(vb)->cnt_mem_ ## op++; \
101	})
102
103	#define log_qop(q, op) \
104	dprintk(q, 2, "call_qop(%s)%s\n", #op, \
105	(q)->ops->op ? "" : " (nop)")
106
107	#define call_qop(q, op, args...) \
108	({ \
109	int err; \
110	\
111	log_qop(q, op); \
112	err = (q)->ops->op ? (q)->ops->op(args) : 0; \
113	if (!err) \
114	(q)->cnt_ ## op++; \
115	err; \
116	})
117
118	#define call_void_qop(q, op, args...) \
119	({ \
120	log_qop(q, op); \
121	if ((q)->ops->op) \
122	(q)->ops->op(args); \
123	(q)->cnt_ ## op++; \
124	})
125
126	#define log_vb_qop(vb, op, args...) \
127	dprintk((vb)->vb2_queue, 2, "call_vb_qop(%d, %s)%s\n", \
128	(vb)->index, #op, \
129	(vb)->vb2_queue->ops->op ? "" : " (nop)")
130
131	#define call_vb_qop(vb, op, args...) \
132	({ \
133	int err; \
134	\
135	log_vb_qop(vb, op); \
136	err = (vb)->vb2_queue->ops->op ? \
137	(vb)->vb2_queue->ops->op(args) : 0; \
138	if (!err) \
139	(vb)->cnt_ ## op++; \
140	err; \
141	})
142
143	#define call_void_vb_qop(vb, op, args...) \
144	({ \
145	log_vb_qop(vb, op); \
146	if ((vb)->vb2_queue->ops->op) \
147	(vb)->vb2_queue->ops->op(args); \
148	(vb)->cnt_ ## op++; \
149	})
150
151	#else
152
153	#define call_memop(vb, op, args...) \
154	((vb)->vb2_queue->mem_ops->op ? \
155	(vb)->vb2_queue->mem_ops->op(args) : 0)
156
157	#define call_ptr_memop(op, vb, args...) \
158	((vb)->vb2_queue->mem_ops->op ? \
159	(vb)->vb2_queue->mem_ops->op(vb, args) : NULL)
160
161	#define call_void_memop(vb, op, args...) \
162	do { \
163	if ((vb)->vb2_queue->mem_ops->op) \
164	(vb)->vb2_queue->mem_ops->op(args); \
165	} while (0)
166
167	#define call_qop(q, op, args...) \
168	((q)->ops->op ? (q)->ops->op(args) : 0)
169
170	#define call_void_qop(q, op, args...) \
171	do { \
172	if ((q)->ops->op) \
173	(q)->ops->op(args); \
174	} while (0)
175
176	#define call_vb_qop(vb, op, args...) \
177	((vb)->vb2_queue->ops->op ? (vb)->vb2_queue->ops->op(args) : 0)
178
179	#define call_void_vb_qop(vb, op, args...) \
180	do { \
181	if ((vb)->vb2_queue->ops->op) \
182	(vb)->vb2_queue->ops->op(args); \
183	} while (0)
184
185	#endif
186
187	#define call_bufop(q, op, args...) \
188	({ \
189	int ret = 0; \
190	if (q && q->buf_ops && q->buf_ops->op) \
191	ret = q->buf_ops->op(args); \
192	ret; \
193	})
194
195	#define call_void_bufop(q, op, args...) \
196	({ \
197	if (q && q->buf_ops && q->buf_ops->op) \
198	q->buf_ops->op(args); \
199	})
200
201	static void __vb2_queue_cancel(struct vb2_queue *q);
202
203	static const char *vb2_state_name(enum vb2_buffer_state s)
204	{
205	static const char * const state_names[] = {
206	[VB2_BUF_STATE_DEQUEUED] = "dequeued",
207	[VB2_BUF_STATE_IN_REQUEST] = "in request",
208	[VB2_BUF_STATE_PREPARING] = "preparing",
209	[VB2_BUF_STATE_QUEUED] = "queued",
210	[VB2_BUF_STATE_ACTIVE] = "active",
211	[VB2_BUF_STATE_DONE] = "done",
212	[VB2_BUF_STATE_ERROR] = "error",
213	};
214
215	if ((unsigned int)(s) < ARRAY_SIZE(state_names))
216	return state_names[s];
217	return "unknown";
218	}
219
220	/*
221	* __vb2_buf_mem_alloc() - allocate video memory for the given buffer
222	*/
223	static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
224	{
225	struct vb2_queue *q = vb->vb2_queue;
226	void *mem_priv;
227	int plane;
228	int ret = -ENOMEM;
229
230	/*
231	* Allocate memory for all planes in this buffer
232	* NOTE: mmapped areas should be page aligned
233	*/
234	for (plane = 0; plane < vb->num_planes; ++plane) {
235	/* Memops alloc requires size to be page aligned. */
236	unsigned long size = PAGE_ALIGN(vb->planes[plane].length);
237
238	/* Did it wrap around? */
239	if (size < vb->planes[plane].length)
240	goto free;
241
242	mem_priv = call_ptr_memop(alloc,
243	vb,
244	q->alloc_devs[plane] ? : q->dev,
245	size);
246	if (IS_ERR_OR_NULL(ptr: mem_priv)) {
247	if (mem_priv)
248	ret = PTR_ERR(ptr: mem_priv);
249	goto free;
250	}
251
252	/* Associate allocator private data with this plane */
253	vb->planes[plane].mem_priv = mem_priv;
254	}
255
256	return 0;
257	free:
258	/* Free already allocated memory if one of the allocations failed */
259	for (; plane > 0; --plane) {
260	call_void_memop(vb, put, vb->planes[plane - 1].mem_priv);
261	vb->planes[plane - 1].mem_priv = NULL;
262	}
263
264	return ret;
265	}
266
267	/*
268	* __vb2_buf_mem_free() - free memory of the given buffer
269	*/
270	static void __vb2_buf_mem_free(struct vb2_buffer *vb)
271	{
272	unsigned int plane;
273
274	for (plane = 0; plane < vb->num_planes; ++plane) {
275	call_void_memop(vb, put, vb->planes[plane].mem_priv);
276	vb->planes[plane].mem_priv = NULL;
277	dprintk(vb->vb2_queue, 3, "freed plane %d of buffer %d\n",
278	plane, vb->index);
279	}
280	}
281
282	/*
283	* __vb2_buf_userptr_put() - release userspace memory associated with
284	* a USERPTR buffer
285	*/
286	static void __vb2_buf_userptr_put(struct vb2_buffer *vb)
287	{
288	unsigned int plane;
289
290	for (plane = 0; plane < vb->num_planes; ++plane) {
291	if (vb->planes[plane].mem_priv)
292	call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
293	vb->planes[plane].mem_priv = NULL;
294	}
295	}
296
297	/*
298	* __vb2_plane_dmabuf_put() - release memory associated with
299	* a DMABUF shared plane
300	*/
301	static void __vb2_plane_dmabuf_put(struct vb2_buffer *vb, struct vb2_plane *p)
302	{
303	if (!p->mem_priv)
304	return;
305
306	if (!p->dbuf_duplicated) {
307	if (p->dbuf_mapped)
308	call_void_memop(vb, unmap_dmabuf, p->mem_priv);
309
310	call_void_memop(vb, detach_dmabuf, p->mem_priv);
311	}
312
313	dma_buf_put(dmabuf: p->dbuf);
314	p->mem_priv = NULL;
315	p->dbuf = NULL;
316	p->dbuf_mapped = 0;
317	p->bytesused = 0;
318	p->length = 0;
319	p->m.fd = 0;
320	p->data_offset = 0;
321	p->dbuf_duplicated = false;
322	}
323
324	/*
325	* __vb2_buf_dmabuf_put() - release memory associated with
326	* a DMABUF shared buffer
327	*/
328	static void __vb2_buf_dmabuf_put(struct vb2_buffer *vb)
329	{
330	int plane;
331
332	/*
333	* When multiple planes share the same DMA buffer attachment, the plane
334	* with the lowest index owns the mem_priv.
335	* Put planes in the reversed order so that we don't leave invalid
336	* mem_priv behind.
337	*/
338	for (plane = vb->num_planes - 1; plane >= 0; --plane)
339	__vb2_plane_dmabuf_put(vb, p: &vb->planes[plane]);
340	}
341
342	/*
343	* __vb2_buf_mem_prepare() - call ->prepare() on buffer's private memory
344	* to sync caches
345	*/
346	static void __vb2_buf_mem_prepare(struct vb2_buffer *vb)
347	{
348	unsigned int plane;
349
350	if (vb->synced)
351	return;
352
353	vb->synced = 1;
354	for (plane = 0; plane < vb->num_planes; ++plane)
355	call_void_memop(vb, prepare, vb->planes[plane].mem_priv);
356	}
357
358	/*
359	* __vb2_buf_mem_finish() - call ->finish on buffer's private memory
360	* to sync caches
361	*/
362	static void __vb2_buf_mem_finish(struct vb2_buffer *vb)
363	{
364	unsigned int plane;
365
366	if (!vb->synced)
367	return;
368
369	vb->synced = 0;
370	for (plane = 0; plane < vb->num_planes; ++plane)
371	call_void_memop(vb, finish, vb->planes[plane].mem_priv);
372	}
373
374	/*
375	* __setup_offsets() - setup unique offsets ("cookies") for every plane in
376	* the buffer.
377	*/
378	static void __setup_offsets(struct vb2_buffer *vb)
379	{
380	struct vb2_queue *q = vb->vb2_queue;
381	unsigned int plane;
382	unsigned long offset = 0;
383
384	/*
385	* The offset "cookie" value has the following constraints:
386	* - a buffer can have up to 8 planes.
387	* - v4l2 mem2mem uses bit 30 to distinguish between
388	* OUTPUT (aka "source", bit 30 is 0) and
389	* CAPTURE (aka "destination", bit 30 is 1) buffers.
390	* - must be page aligned
391	* That led to this bit mapping when PAGE_SHIFT = 12:
392	* |30 |29 15|14 12|11 0|
393	* |DST_QUEUE_OFF_BASE|buffer index|plane index| 0 |
394	* where there are 15 bits to store the buffer index.
395	* Depending on PAGE_SHIFT value we can have fewer bits
396	* to store the buffer index.
397	*/
398	offset = vb->index << PLANE_INDEX_SHIFT;
399
400	for (plane = 0; plane < vb->num_planes; ++plane) {
401	vb->planes[plane].m.offset = offset + (plane << PAGE_SHIFT);
402
403	dprintk(q, 3, "buffer %d, plane %d offset 0x%08lx\n",
404	vb->index, plane, offset);
405	}
406	}
407
408	static void init_buffer_cache_hints(struct vb2_queue *q, struct vb2_buffer *vb)
409	{
410	/*
411	* DMA exporter should take care of cache syncs, so we can avoid
412	* explicit ->prepare()/->finish() syncs. For other ->memory types
413	* we always need ->prepare() or/and ->finish() cache sync.
414	*/
415	if (q->memory == VB2_MEMORY_DMABUF) {
416	vb->skip_cache_sync_on_finish = 1;
417	vb->skip_cache_sync_on_prepare = 1;
418	return;
419	}
420
421	/*
422	* ->finish() cache sync can be avoided when queue direction is
423	* TO_DEVICE.
424	*/
425	if (q->dma_dir == DMA_TO_DEVICE)
426	vb->skip_cache_sync_on_finish = 1;
427	}
428
429	/**
430	* vb2_queue_add_buffer() - add a buffer to a queue
431	* @q: pointer to &struct vb2_queue with videobuf2 queue.
432	* @vb: pointer to &struct vb2_buffer to be added to the queue.
433	* @index: index where add vb2_buffer in the queue
434	*/
435	static void vb2_queue_add_buffer(struct vb2_queue *q, struct vb2_buffer *vb, unsigned int index)
436	{
437	WARN_ON(index >= q->max_num_buffers || test_bit(index, q->bufs_bitmap) || vb->vb2_queue);
438
439	q->bufs[index] = vb;
440	vb->index = index;
441	vb->vb2_queue = q;
442	set_bit(nr: index, addr: q->bufs_bitmap);
443	}
444
445	/**
446	* vb2_queue_remove_buffer() - remove a buffer from a queue
447	* @vb: pointer to &struct vb2_buffer to be removed from the queue.
448	*/
449	static void vb2_queue_remove_buffer(struct vb2_buffer *vb)
450	{
451	clear_bit(nr: vb->index, addr: vb->vb2_queue->bufs_bitmap);
452	vb->vb2_queue->bufs[vb->index] = NULL;
453	vb->vb2_queue = NULL;
454	}
455
456	/*
457	* __vb2_queue_alloc() - allocate vb2 buffer structures and (for MMAP type)
458	* video buffer memory for all buffers/planes on the queue and initializes the
459	* queue
460	* @first_index: index of the first created buffer, all newly allocated buffers
461	* have indices in the range [first_index..first_index+count-1]
462	*
463	* Returns the number of buffers successfully allocated.
464	*/
465	static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory,
466	unsigned int num_buffers, unsigned int num_planes,
467	const unsigned int plane_sizes[VB2_MAX_PLANES],
468	unsigned int *first_index)
469	{
470	unsigned int buffer, plane;
471	struct vb2_buffer *vb;
472	unsigned long index = q->max_num_buffers;
473	int ret;
474
475	/*
476	* Ensure that the number of already queue + the number of buffers already
477	* in the queue is below q->max_num_buffers
478	*/
479	num_buffers = min_t(unsigned int, num_buffers,
480	q->max_num_buffers - vb2_get_num_buffers(q));
481
482	while (num_buffers) {
483	index = bitmap_find_next_zero_area(map: q->bufs_bitmap, size: q->max_num_buffers,
484	start: 0, nr: num_buffers, align_mask: 0);
485
486	if (index < q->max_num_buffers)
487	break;
488	/* Try to find free space for less buffers */
489	num_buffers--;
490	}
491
492	/* If there is no space left to allocate buffers return 0 to indicate the error */
493	if (!num_buffers) {
494	*first_index = 0;
495	return 0;
496	}
497
498	*first_index = index;
499
500	for (buffer = 0; buffer < num_buffers; ++buffer) {
501	/* Allocate vb2 buffer structures */
502	vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
503	if (!vb) {
504	dprintk(q, 1, "memory alloc for buffer struct failed\n");
505	break;
506	}
507
508	vb->state = VB2_BUF_STATE_DEQUEUED;
509	vb->num_planes = num_planes;
510	vb->type = q->type;
511	vb->memory = memory;
512	init_buffer_cache_hints(q, vb);
513	for (plane = 0; plane < num_planes; ++plane) {
514	vb->planes[plane].length = plane_sizes[plane];
515	vb->planes[plane].min_length = plane_sizes[plane];
516	}
517
518	vb2_queue_add_buffer(q, vb, index: index++);
519	call_void_bufop(q, init_buffer, vb);
520
521	/* Allocate video buffer memory for the MMAP type */
522	if (memory == VB2_MEMORY_MMAP) {
523	ret = __vb2_buf_mem_alloc(vb);
524	if (ret) {
525	dprintk(q, 1, "failed allocating memory for buffer %d\n",
526	buffer);
527	vb2_queue_remove_buffer(vb);
528	kfree(objp: vb);
529	break;
530	}
531	__setup_offsets(vb);
532	/*
533	* Call the driver-provided buffer initialization
534	* callback, if given. An error in initialization
535	* results in queue setup failure.
536	*/
537	ret = call_vb_qop(vb, buf_init, vb);
538	if (ret) {
539	dprintk(q, 1, "buffer %d %p initialization failed\n",
540	buffer, vb);
541	__vb2_buf_mem_free(vb);
542	vb2_queue_remove_buffer(vb);
543	kfree(objp: vb);
544	break;
545	}
546	}
547	}
548
549	dprintk(q, 3, "allocated %d buffers, %d plane(s) each\n",
550	buffer, num_planes);
551
552	return buffer;
553	}
554
555	/*
556	* __vb2_free_mem() - release video buffer memory for a given range of
557	* buffers in a given queue
558	*/
559	static void __vb2_free_mem(struct vb2_queue *q, unsigned int start, unsigned int count)
560	{
561	unsigned int i;
562	struct vb2_buffer *vb;
563
564	for (i = start; i < start + count; i++) {
565	vb = vb2_get_buffer(q, index: i);
566	if (!vb)
567	continue;
568
569	/* Free MMAP buffers or release USERPTR buffers */
570	if (q->memory == VB2_MEMORY_MMAP)
571	__vb2_buf_mem_free(vb);
572	else if (q->memory == VB2_MEMORY_DMABUF)
573	__vb2_buf_dmabuf_put(vb);
574	else
575	__vb2_buf_userptr_put(vb);
576	}
577	}
578
579	/*
580	* __vb2_queue_free() - free @count buffers from @start index of the queue - video memory and
581	* related information, if no buffers are left return the queue to an
582	* uninitialized state. Might be called even if the queue has already been freed.
583	*/
584	static void __vb2_queue_free(struct vb2_queue *q, unsigned int start, unsigned int count)
585	{
586	unsigned int i;
587
588	lockdep_assert_held(&q->mmap_lock);
589
590	/* Call driver-provided cleanup function for each buffer, if provided */
591	for (i = start; i < start + count; i++) {
592	struct vb2_buffer *vb = vb2_get_buffer(q, index: i);
593
594	if (vb && vb->planes[0].mem_priv)
595	call_void_vb_qop(vb, buf_cleanup, vb);
596	}
597
598	/* Release video buffer memory */
599	__vb2_free_mem(q, start, count);
600
601	#ifdef CONFIG_VIDEO_ADV_DEBUG
602	/*
603	* Check that all the calls were balanced during the life-time of this
604	* queue. If not then dump the counters to the kernel log.
605	*/
606	if (vb2_get_num_buffers(q)) {
607	bool unbalanced = q->cnt_start_streaming != q->cnt_stop_streaming ||
608	q->cnt_prepare_streaming != q->cnt_unprepare_streaming ||
609	q->cnt_wait_prepare != q->cnt_wait_finish;
610
611	if (unbalanced) {
612	pr_info("unbalanced counters for queue %p:\n", q);
613	if (q->cnt_start_streaming != q->cnt_stop_streaming)
614	pr_info(" setup: %u start_streaming: %u stop_streaming: %u\n",
615	q->cnt_queue_setup, q->cnt_start_streaming,
616	q->cnt_stop_streaming);
617	if (q->cnt_prepare_streaming != q->cnt_unprepare_streaming)
618	pr_info(" prepare_streaming: %u unprepare_streaming: %u\n",
619	q->cnt_prepare_streaming, q->cnt_unprepare_streaming);
620	if (q->cnt_wait_prepare != q->cnt_wait_finish)
621	pr_info(" wait_prepare: %u wait_finish: %u\n",
622	q->cnt_wait_prepare, q->cnt_wait_finish);
623	}
624	q->cnt_queue_setup = 0;
625	q->cnt_wait_prepare = 0;
626	q->cnt_wait_finish = 0;
627	q->cnt_prepare_streaming = 0;
628	q->cnt_start_streaming = 0;
629	q->cnt_stop_streaming = 0;
630	q->cnt_unprepare_streaming = 0;
631	}
632	for (i = start; i < start + count; i++) {
633	struct vb2_buffer *vb = vb2_get_buffer(q, index: i);
634	bool unbalanced;
635
636	if (!vb)
637	continue;
638
639	unbalanced = vb->cnt_mem_alloc != vb->cnt_mem_put ||
640	vb->cnt_mem_prepare != vb->cnt_mem_finish ||
641	vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr ||
642	vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf ||
643	vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf ||
644	vb->cnt_buf_queue != vb->cnt_buf_done ||
645	vb->cnt_buf_prepare != vb->cnt_buf_finish ||
646	vb->cnt_buf_init != vb->cnt_buf_cleanup;
647
648	if (unbalanced) {
649	pr_info("unbalanced counters for queue %p, buffer %d:\n",
650	q, i);
651	if (vb->cnt_buf_init != vb->cnt_buf_cleanup)
652	pr_info(" buf_init: %u buf_cleanup: %u\n",
653	vb->cnt_buf_init, vb->cnt_buf_cleanup);
654	if (vb->cnt_buf_prepare != vb->cnt_buf_finish)
655	pr_info(" buf_prepare: %u buf_finish: %u\n",
656	vb->cnt_buf_prepare, vb->cnt_buf_finish);
657	if (vb->cnt_buf_queue != vb->cnt_buf_done)
658	pr_info(" buf_out_validate: %u buf_queue: %u buf_done: %u buf_request_complete: %u\n",
659	vb->cnt_buf_out_validate, vb->cnt_buf_queue,
660	vb->cnt_buf_done, vb->cnt_buf_request_complete);
661	if (vb->cnt_mem_alloc != vb->cnt_mem_put)
662	pr_info(" alloc: %u put: %u\n",
663	vb->cnt_mem_alloc, vb->cnt_mem_put);
664	if (vb->cnt_mem_prepare != vb->cnt_mem_finish)
665	pr_info(" prepare: %u finish: %u\n",
666	vb->cnt_mem_prepare, vb->cnt_mem_finish);
667	if (vb->cnt_mem_get_userptr != vb->cnt_mem_put_userptr)
668	pr_info(" get_userptr: %u put_userptr: %u\n",
669	vb->cnt_mem_get_userptr, vb->cnt_mem_put_userptr);
670	if (vb->cnt_mem_attach_dmabuf != vb->cnt_mem_detach_dmabuf)
671	pr_info(" attach_dmabuf: %u detach_dmabuf: %u\n",
672	vb->cnt_mem_attach_dmabuf, vb->cnt_mem_detach_dmabuf);
673	if (vb->cnt_mem_map_dmabuf != vb->cnt_mem_unmap_dmabuf)
674	pr_info(" map_dmabuf: %u unmap_dmabuf: %u\n",
675	vb->cnt_mem_map_dmabuf, vb->cnt_mem_unmap_dmabuf);
676	pr_info(" get_dmabuf: %u num_users: %u\n",
677	vb->cnt_mem_get_dmabuf,
678	vb->cnt_mem_num_users);
679	}
680	}
681	#endif
682
683	/* Free vb2 buffers */
684	for (i = start; i < start + count; i++) {
685	struct vb2_buffer *vb = vb2_get_buffer(q, index: i);
686
687	if (!vb)
688	continue;
689
690	vb2_queue_remove_buffer(vb);
691	kfree(objp: vb);
692	}
693
694	if (!vb2_get_num_buffers(q)) {
695	q->memory = VB2_MEMORY_UNKNOWN;
696	INIT_LIST_HEAD(list: &q->queued_list);
697	}
698	}
699
700	bool vb2_buffer_in_use(struct vb2_queue *q, struct vb2_buffer *vb)
701	{
702	unsigned int plane;
703	for (plane = 0; plane < vb->num_planes; ++plane) {
704	void *mem_priv = vb->planes[plane].mem_priv;
705	/*
706	* If num_users() has not been provided, call_memop
707	* will return 0, apparently nobody cares about this
708	* case anyway. If num_users() returns more than 1,
709	* we are not the only user of the plane's memory.
710	*/
711	if (mem_priv && call_memop(vb, num_users, mem_priv) > 1)
712	return true;
713	}
714	return false;
715	}
716	EXPORT_SYMBOL(vb2_buffer_in_use);
717
718	/*
719	* __buffers_in_use() - return true if any buffers on the queue are in use and
720	* the queue cannot be freed (by the means of REQBUFS(0)) call
721	*/
722	static bool __buffers_in_use(struct vb2_queue *q)
723	{
724	unsigned int buffer;
725	for (buffer = 0; buffer < q->max_num_buffers; ++buffer) {
726	struct vb2_buffer *vb = vb2_get_buffer(q, index: buffer);
727
728	if (!vb)
729	continue;
730
731	if (vb2_buffer_in_use(q, vb))
732	return true;
733	}
734	return false;
735	}
736
737	void vb2_core_querybuf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb)
738	{
739	call_void_bufop(q, fill_user_buffer, vb, pb);
740	}
741	EXPORT_SYMBOL_GPL(vb2_core_querybuf);
742
743	/*
744	* __verify_userptr_ops() - verify that all memory operations required for
745	* USERPTR queue type have been provided
746	*/
747	static int __verify_userptr_ops(struct vb2_queue *q)
748	{
749	if (!(q->io_modes & VB2_USERPTR) || !q->mem_ops->get_userptr ||
750	!q->mem_ops->put_userptr)
751	return -EINVAL;
752
753	return 0;
754	}
755
756	/*
757	* __verify_mmap_ops() - verify that all memory operations required for
758	* MMAP queue type have been provided
759	*/
760	static int __verify_mmap_ops(struct vb2_queue *q)
761	{
762	if (!(q->io_modes & VB2_MMAP) || !q->mem_ops->alloc ||
763	!q->mem_ops->put || !q->mem_ops->mmap)
764	return -EINVAL;
765
766	return 0;
767	}
768
769	/*
770	* __verify_dmabuf_ops() - verify that all memory operations required for
771	* DMABUF queue type have been provided
772	*/
773	static int __verify_dmabuf_ops(struct vb2_queue *q)
774	{
775	if (!(q->io_modes & VB2_DMABUF) || !q->mem_ops->attach_dmabuf ||
776	!q->mem_ops->detach_dmabuf || !q->mem_ops->map_dmabuf ||
777	!q->mem_ops->unmap_dmabuf)
778	return -EINVAL;
779
780	return 0;
781	}
782
783	int vb2_verify_memory_type(struct vb2_queue *q,
784	enum vb2_memory memory, unsigned int type)
785	{
786	if (memory != VB2_MEMORY_MMAP && memory != VB2_MEMORY_USERPTR &&
787	memory != VB2_MEMORY_DMABUF) {
788	dprintk(q, 1, "unsupported memory type\n");
789	return -EINVAL;
790	}
791
792	if (type != q->type) {
793	dprintk(q, 1, "requested type is incorrect\n");
794	return -EINVAL;
795	}
796
797	/*
798	* Make sure all the required memory ops for given memory type
799	* are available.
800	*/
801	if (memory == VB2_MEMORY_MMAP && __verify_mmap_ops(q)) {
802	dprintk(q, 1, "MMAP for current setup unsupported\n");
803	return -EINVAL;
804	}
805
806	if (memory == VB2_MEMORY_USERPTR && __verify_userptr_ops(q)) {
807	dprintk(q, 1, "USERPTR for current setup unsupported\n");
808	return -EINVAL;
809	}
810
811	if (memory == VB2_MEMORY_DMABUF && __verify_dmabuf_ops(q)) {
812	dprintk(q, 1, "DMABUF for current setup unsupported\n");
813	return -EINVAL;
814	}
815
816	/*
817	* Place the busy tests at the end: -EBUSY can be ignored when
818	* create_bufs is called with count == 0, but count == 0 should still
819	* do the memory and type validation.
820	*/
821	if (vb2_fileio_is_active(q)) {
822	dprintk(q, 1, "file io in progress\n");
823	return -EBUSY;
824	}
825	return 0;
826	}
827	EXPORT_SYMBOL(vb2_verify_memory_type);
828
829	static void set_queue_coherency(struct vb2_queue *q, bool non_coherent_mem)
830	{
831	q->non_coherent_mem = 0;
832
833	if (!vb2_queue_allows_cache_hints(q))
834	return;
835	q->non_coherent_mem = non_coherent_mem;
836	}
837
838	static bool verify_coherency_flags(struct vb2_queue *q, bool non_coherent_mem)
839	{
840	if (non_coherent_mem != q->non_coherent_mem) {
841	dprintk(q, 1, "memory coherency model mismatch\n");
842	return false;
843	}
844	return true;
845	}
846
847	static int vb2_core_allocated_buffers_storage(struct vb2_queue *q)
848	{
849	if (!q->bufs)
850	q->bufs = kcalloc(q->max_num_buffers, sizeof(*q->bufs), GFP_KERNEL);
851	if (!q->bufs)
852	return -ENOMEM;
853
854	if (!q->bufs_bitmap)
855	q->bufs_bitmap = bitmap_zalloc(nbits: q->max_num_buffers, GFP_KERNEL);
856	if (!q->bufs_bitmap) {
857	kfree(objp: q->bufs);
858	q->bufs = NULL;
859	return -ENOMEM;
860	}
861
862	return 0;
863	}
864
865	static void vb2_core_free_buffers_storage(struct vb2_queue *q)
866	{
867	kfree(objp: q->bufs);
868	q->bufs = NULL;
869	bitmap_free(bitmap: q->bufs_bitmap);
870	q->bufs_bitmap = NULL;
871	}
872
873	int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
874	unsigned int flags, unsigned int *count)
875	{
876	unsigned int num_buffers, allocated_buffers, num_planes = 0;
877	unsigned int q_num_bufs = vb2_get_num_buffers(q);
878	unsigned plane_sizes[VB2_MAX_PLANES] = { };
879	bool non_coherent_mem = flags & V4L2_MEMORY_FLAG_NON_COHERENT;
880	unsigned int i, first_index;
881	int ret = 0;
882
883	if (q->streaming) {
884	dprintk(q, 1, "streaming active\n");
885	return -EBUSY;
886	}
887
888	if (q->waiting_in_dqbuf && *count) {
889	dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
890	return -EBUSY;
891	}
892
893	if (*count == 0 || q_num_bufs != 0 ||
894	(q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory) ||
895	!verify_coherency_flags(q, non_coherent_mem)) {
896	/*
897	* We already have buffers allocated, so first check if they
898	* are not in use and can be freed.
899	*/
900	mutex_lock(&q->mmap_lock);
901	if (debug && q->memory == VB2_MEMORY_MMAP &&
902	__buffers_in_use(q))
903	dprintk(q, 1, "memory in use, orphaning buffers\n");
904
905	/*
906	* Call queue_cancel to clean up any buffers in the
907	* QUEUED state which is possible if buffers were prepared or
908	* queued without ever calling STREAMON.
909	*/
910	__vb2_queue_cancel(q);
911	__vb2_queue_free(q, start: 0, count: q->max_num_buffers);
912	mutex_unlock(lock: &q->mmap_lock);
913
914	q->is_busy = 0;
915	/*
916	* In case of REQBUFS(0) return immediately without calling
917	* driver's queue_setup() callback and allocating resources.
918	*/
919	if (*count == 0)
920	return 0;
921	}
922
923	/*
924	* Make sure the requested values and current defaults are sane.
925	*/
926	num_buffers = max_t(unsigned int, *count, q->min_reqbufs_allocation);
927	num_buffers = min_t(unsigned int, num_buffers, q->max_num_buffers);
928	memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
929	/*
930	* Set this now to ensure that drivers see the correct q->memory value
931	* in the queue_setup op.
932	*/
933	mutex_lock(&q->mmap_lock);
934	ret = vb2_core_allocated_buffers_storage(q);
935	q->memory = memory;
936	mutex_unlock(lock: &q->mmap_lock);
937	if (ret)
938	return ret;
939	set_queue_coherency(q, non_coherent_mem);
940
941	/*
942	* Ask the driver how many buffers and planes per buffer it requires.
943	* Driver also sets the size and allocator context for each plane.
944	*/
945	ret = call_qop(q, queue_setup, q, &num_buffers, &num_planes,
946	plane_sizes, q->alloc_devs);
947	if (ret)
948	goto error;
949
950	/* Check that driver has set sane values */
951	if (WARN_ON(!num_planes)) {
952	ret = -EINVAL;
953	goto error;
954	}
955
956	for (i = 0; i < num_planes; i++)
957	if (WARN_ON(!plane_sizes[i])) {
958	ret = -EINVAL;
959	goto error;
960	}
961
962	/* Finally, allocate buffers and video memory */
963	allocated_buffers =
964	__vb2_queue_alloc(q, memory, num_buffers, num_planes, plane_sizes, first_index: &first_index);
965	if (allocated_buffers == 0) {
966	/* There shouldn't be any buffers allocated, so first_index == 0 */
967	WARN_ON(first_index);
968	dprintk(q, 1, "memory allocation failed\n");
969	ret = -ENOMEM;
970	goto error;
971	}
972
973	/*
974	* There is no point in continuing if we can't allocate the minimum
975	* number of buffers needed by this vb2_queue.
976	*/
977	if (allocated_buffers < q->min_reqbufs_allocation)
978	ret = -ENOMEM;
979
980	/*
981	* Check if driver can handle the allocated number of buffers.
982	*/
983	if (!ret && allocated_buffers < num_buffers) {
984	num_buffers = allocated_buffers;
985	/*
986	* num_planes is set by the previous queue_setup(), but since it
987	* signals to queue_setup() whether it is called from create_bufs()
988	* vs reqbufs() we zero it here to signal that queue_setup() is
989	* called for the reqbufs() case.
990	*/
991	num_planes = 0;
992
993	ret = call_qop(q, queue_setup, q, &num_buffers,
994	&num_planes, plane_sizes, q->alloc_devs);
995
996	if (!ret && allocated_buffers < num_buffers)
997	ret = -ENOMEM;
998
999	/*
1000	* Either the driver has accepted a smaller number of buffers,
1001	* or .queue_setup() returned an error
1002	*/
1003	}
1004
1005	mutex_lock(&q->mmap_lock);
1006
1007	if (ret < 0) {
1008	/*
1009	* Note: __vb2_queue_free() will subtract 'allocated_buffers'
1010	* from already queued buffers and it will reset q->memory to
1011	* VB2_MEMORY_UNKNOWN.
1012	*/
1013	__vb2_queue_free(q, start: first_index, count: allocated_buffers);
1014	mutex_unlock(lock: &q->mmap_lock);
1015	return ret;
1016	}
1017	mutex_unlock(lock: &q->mmap_lock);
1018
1019	/*
1020	* Return the number of successfully allocated buffers
1021	* to the userspace.
1022	*/
1023	*count = allocated_buffers;
1024	q->waiting_for_buffers = !q->is_output;
1025	q->is_busy = 1;
1026
1027	return 0;
1028
1029	error:
1030	mutex_lock(&q->mmap_lock);
1031	q->memory = VB2_MEMORY_UNKNOWN;
1032	mutex_unlock(lock: &q->mmap_lock);
1033	vb2_core_free_buffers_storage(q);
1034	return ret;
1035	}
1036	EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
1037
1038	int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
1039	unsigned int flags, unsigned int *count,
1040	unsigned int requested_planes,
1041	const unsigned int requested_sizes[],
1042	unsigned int *first_index)
1043	{
1044	unsigned int num_planes = 0, num_buffers, allocated_buffers;
1045	unsigned plane_sizes[VB2_MAX_PLANES] = { };
1046	bool non_coherent_mem = flags & V4L2_MEMORY_FLAG_NON_COHERENT;
1047	unsigned int q_num_bufs = vb2_get_num_buffers(q);
1048	bool no_previous_buffers = !q_num_bufs;
1049	int ret = 0;
1050
1051	if (q_num_bufs == q->max_num_buffers) {
1052	dprintk(q, 1, "maximum number of buffers already allocated\n");
1053	return -ENOBUFS;
1054	}
1055
1056	if (no_previous_buffers) {
1057	if (q->waiting_in_dqbuf && *count) {
1058	dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
1059	return -EBUSY;
1060	}
1061	memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
1062	/*
1063	* Set this now to ensure that drivers see the correct q->memory
1064	* value in the queue_setup op.
1065	*/
1066	mutex_lock(&q->mmap_lock);
1067	ret = vb2_core_allocated_buffers_storage(q);
1068	q->memory = memory;
1069	mutex_unlock(lock: &q->mmap_lock);
1070	if (ret)
1071	return ret;
1072	q->waiting_for_buffers = !q->is_output;
1073	set_queue_coherency(q, non_coherent_mem);
1074	} else {
1075	if (q->memory != memory) {
1076	dprintk(q, 1, "memory model mismatch\n");
1077	return -EINVAL;
1078	}
1079	if (!verify_coherency_flags(q, non_coherent_mem))
1080	return -EINVAL;
1081	}
1082
1083	num_buffers = min(*count, q->max_num_buffers - q_num_bufs);
1084
1085	if (requested_planes && requested_sizes) {
1086	num_planes = requested_planes;
1087	memcpy(plane_sizes, requested_sizes, sizeof(plane_sizes));
1088	}
1089
1090	/*
1091	* Ask the driver, whether the requested number of buffers, planes per
1092	* buffer and their sizes are acceptable
1093	*/
1094	ret = call_qop(q, queue_setup, q, &num_buffers,
1095	&num_planes, plane_sizes, q->alloc_devs);
1096	if (ret)
1097	goto error;
1098
1099	/* Finally, allocate buffers and video memory */
1100	allocated_buffers = __vb2_queue_alloc(q, memory, num_buffers,
1101	num_planes, plane_sizes, first_index);
1102	if (allocated_buffers == 0) {
1103	dprintk(q, 1, "memory allocation failed\n");
1104	ret = -ENOMEM;
1105	goto error;
1106	}
1107
1108	/*
1109	* Check if driver can handle the so far allocated number of buffers.
1110	*/
1111	if (allocated_buffers < num_buffers) {
1112	num_buffers = allocated_buffers;
1113
1114	/*
1115	* num_buffers contains the total number of buffers, that the
1116	* queue driver has set up
1117	*/
1118	ret = call_qop(q, queue_setup, q, &num_buffers,
1119	&num_planes, plane_sizes, q->alloc_devs);
1120
1121	if (!ret && allocated_buffers < num_buffers)
1122	ret = -ENOMEM;
1123
1124	/*
1125	* Either the driver has accepted a smaller number of buffers,
1126	* or .queue_setup() returned an error
1127	*/
1128	}
1129
1130	mutex_lock(&q->mmap_lock);
1131
1132	if (ret < 0) {
1133	/*
1134	* Note: __vb2_queue_free() will subtract 'allocated_buffers'
1135	* from already queued buffers and it will reset q->memory to
1136	* VB2_MEMORY_UNKNOWN.
1137	*/
1138	__vb2_queue_free(q, start: *first_index, count: allocated_buffers);
1139	mutex_unlock(lock: &q->mmap_lock);
1140	return -ENOMEM;
1141	}
1142	mutex_unlock(lock: &q->mmap_lock);
1143
1144	/*
1145	* Return the number of successfully allocated buffers
1146	* to the userspace.
1147	*/
1148	*count = allocated_buffers;
1149	q->is_busy = 1;
1150
1151	return 0;
1152
1153	error:
1154	if (no_previous_buffers) {
1155	mutex_lock(&q->mmap_lock);
1156	q->memory = VB2_MEMORY_UNKNOWN;
1157	mutex_unlock(lock: &q->mmap_lock);
1158	}
1159	return ret;
1160	}
1161	EXPORT_SYMBOL_GPL(vb2_core_create_bufs);
1162
1163	void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no)
1164	{
1165	if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
1166	return NULL;
1167
1168	return call_ptr_memop(vaddr, vb, vb->planes[plane_no].mem_priv);
1169
1170	}
1171	EXPORT_SYMBOL_GPL(vb2_plane_vaddr);
1172
1173	void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no)
1174	{
1175	if (plane_no >= vb->num_planes || !vb->planes[plane_no].mem_priv)
1176	return NULL;
1177
1178	return call_ptr_memop(cookie, vb, vb->planes[plane_no].mem_priv);
1179	}
1180	EXPORT_SYMBOL_GPL(vb2_plane_cookie);
1181
1182	void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
1183	{
1184	struct vb2_queue *q = vb->vb2_queue;
1185	unsigned long flags;
1186
1187	if (WARN_ON(vb->state != VB2_BUF_STATE_ACTIVE))
1188	return;
1189
1190	if (WARN_ON(state != VB2_BUF_STATE_DONE &&
1191	state != VB2_BUF_STATE_ERROR &&
1192	state != VB2_BUF_STATE_QUEUED))
1193	state = VB2_BUF_STATE_ERROR;
1194
1195	#ifdef CONFIG_VIDEO_ADV_DEBUG
1196	/*
1197	* Although this is not a callback, it still does have to balance
1198	* with the buf_queue op. So update this counter manually.
1199	*/
1200	vb->cnt_buf_done++;
1201	#endif
1202	dprintk(q, 4, "done processing on buffer %d, state: %s\n",
1203	vb->index, vb2_state_name(state));
1204
1205	if (state != VB2_BUF_STATE_QUEUED)
1206	__vb2_buf_mem_finish(vb);
1207
1208	spin_lock_irqsave(&q->done_lock, flags);
1209	if (state == VB2_BUF_STATE_QUEUED) {
1210	vb->state = VB2_BUF_STATE_QUEUED;
1211	} else {
1212	/* Add the buffer to the done buffers list */
1213	list_add_tail(new: &vb->done_entry, head: &q->done_list);
1214	vb->state = state;
1215	}
1216	atomic_dec(v: &q->owned_by_drv_count);
1217
1218	if (state != VB2_BUF_STATE_QUEUED && vb->req_obj.req) {
1219	media_request_object_unbind(obj: &vb->req_obj);
1220	media_request_object_put(obj: &vb->req_obj);
1221	}
1222
1223	spin_unlock_irqrestore(lock: &q->done_lock, flags);
1224
1225	trace_vb2_buf_done(q, vb);
1226
1227	switch (state) {
1228	case VB2_BUF_STATE_QUEUED:
1229	return;
1230	default:
1231	/* Inform any processes that may be waiting for buffers */
1232	wake_up(&q->done_wq);
1233	break;
1234	}
1235	}
1236	EXPORT_SYMBOL_GPL(vb2_buffer_done);
1237
1238	void vb2_discard_done(struct vb2_queue *q)
1239	{
1240	struct vb2_buffer *vb;
1241	unsigned long flags;
1242
1243	spin_lock_irqsave(&q->done_lock, flags);
1244	list_for_each_entry(vb, &q->done_list, done_entry)
1245	vb->state = VB2_BUF_STATE_ERROR;
1246	spin_unlock_irqrestore(lock: &q->done_lock, flags);
1247	}
1248	EXPORT_SYMBOL_GPL(vb2_discard_done);
1249
1250	/*
1251	* __prepare_mmap() - prepare an MMAP buffer
1252	*/
1253	static int __prepare_mmap(struct vb2_buffer *vb)
1254	{
1255	int ret = 0;
1256
1257	ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1258	vb, vb->planes);
1259	return ret ? ret : call_vb_qop(vb, buf_prepare, vb);
1260	}
1261
1262	/*
1263	* __prepare_userptr() - prepare a USERPTR buffer
1264	*/
1265	static int __prepare_userptr(struct vb2_buffer *vb)
1266	{
1267	struct vb2_plane planes[VB2_MAX_PLANES];
1268	struct vb2_queue *q = vb->vb2_queue;
1269	void *mem_priv;
1270	unsigned int plane;
1271	int ret = 0;
1272	bool reacquired = vb->planes[0].mem_priv == NULL;
1273
1274	memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1275	/* Copy relevant information provided by the userspace */
1276	ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1277	vb, planes);
1278	if (ret)
1279	return ret;
1280
1281	for (plane = 0; plane < vb->num_planes; ++plane) {
1282	/* Skip the plane if already verified */
1283	if (vb->planes[plane].m.userptr &&
1284	vb->planes[plane].m.userptr == planes[plane].m.userptr
1285	&& vb->planes[plane].length == planes[plane].length)
1286	continue;
1287
1288	dprintk(q, 3, "userspace address for plane %d changed, reacquiring memory\n",
1289	plane);
1290
1291	/* Check if the provided plane buffer is large enough */
1292	if (planes[plane].length < vb->planes[plane].min_length) {
1293	dprintk(q, 1, "provided buffer size %u is less than setup size %u for plane %d\n",
1294	planes[plane].length,
1295	vb->planes[plane].min_length,
1296	plane);
1297	ret = -EINVAL;
1298	goto err;
1299	}
1300
1301	/* Release previously acquired memory if present */
1302	if (vb->planes[plane].mem_priv) {
1303	if (!reacquired) {
1304	reacquired = true;
1305	vb->copied_timestamp = 0;
1306	call_void_vb_qop(vb, buf_cleanup, vb);
1307	}
1308	call_void_memop(vb, put_userptr, vb->planes[plane].mem_priv);
1309	}
1310
1311	vb->planes[plane].mem_priv = NULL;
1312	vb->planes[plane].bytesused = 0;
1313	vb->planes[plane].length = 0;
1314	vb->planes[plane].m.userptr = 0;
1315	vb->planes[plane].data_offset = 0;
1316
1317	/* Acquire each plane's memory */
1318	mem_priv = call_ptr_memop(get_userptr,
1319	vb,
1320	q->alloc_devs[plane] ? : q->dev,
1321	planes[plane].m.userptr,
1322	planes[plane].length);
1323	if (IS_ERR(ptr: mem_priv)) {
1324	dprintk(q, 1, "failed acquiring userspace memory for plane %d\n",
1325	plane);
1326	ret = PTR_ERR(ptr: mem_priv);
1327	goto err;
1328	}
1329	vb->planes[plane].mem_priv = mem_priv;
1330	}
1331
1332	/*
1333	* Now that everything is in order, copy relevant information
1334	* provided by userspace.
1335	*/
1336	for (plane = 0; plane < vb->num_planes; ++plane) {
1337	vb->planes[plane].bytesused = planes[plane].bytesused;
1338	vb->planes[plane].length = planes[plane].length;
1339	vb->planes[plane].m.userptr = planes[plane].m.userptr;
1340	vb->planes[plane].data_offset = planes[plane].data_offset;
1341	}
1342
1343	if (reacquired) {
1344	/*
1345	* One or more planes changed, so we must call buf_init to do
1346	* the driver-specific initialization on the newly acquired
1347	* buffer, if provided.
1348	*/
1349	ret = call_vb_qop(vb, buf_init, vb);
1350	if (ret) {
1351	dprintk(q, 1, "buffer initialization failed\n");
1352	goto err;
1353	}
1354	}
1355
1356	ret = call_vb_qop(vb, buf_prepare, vb);
1357	if (ret) {
1358	dprintk(q, 1, "buffer preparation failed\n");
1359	call_void_vb_qop(vb, buf_cleanup, vb);
1360	goto err;
1361	}
1362
1363	return 0;
1364	err:
1365	/* In case of errors, release planes that were already acquired */
1366	for (plane = 0; plane < vb->num_planes; ++plane) {
1367	if (vb->planes[plane].mem_priv)
1368	call_void_memop(vb, put_userptr,
1369	vb->planes[plane].mem_priv);
1370	vb->planes[plane].mem_priv = NULL;
1371	vb->planes[plane].m.userptr = 0;
1372	vb->planes[plane].length = 0;
1373	}
1374
1375	return ret;
1376	}
1377
1378	/*
1379	* __prepare_dmabuf() - prepare a DMABUF buffer
1380	*/
1381	static int __prepare_dmabuf(struct vb2_buffer *vb)
1382	{
1383	struct vb2_plane planes[VB2_MAX_PLANES];
1384	struct vb2_queue *q = vb->vb2_queue;
1385	void *mem_priv;
1386	unsigned int plane, i;
1387	int ret = 0;
1388	bool reacquired = vb->planes[0].mem_priv == NULL;
1389
1390	memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
1391	/* Copy relevant information provided by the userspace */
1392	ret = call_bufop(vb->vb2_queue, fill_vb2_buffer,
1393	vb, planes);
1394	if (ret)
1395	return ret;
1396
1397	for (plane = 0; plane < vb->num_planes; ++plane) {
1398	struct dma_buf *dbuf = dma_buf_get(fd: planes[plane].m.fd);
1399
1400	planes[plane].dbuf = dbuf;
1401
1402	if (IS_ERR_OR_NULL(ptr: dbuf)) {
1403	dprintk(q, 1, "invalid dmabuf fd for plane %d\n",
1404	plane);
1405	ret = -EINVAL;
1406	goto err_put_planes;
1407	}
1408
1409	/* use DMABUF size if length is not provided */
1410	if (planes[plane].length == 0)
1411	planes[plane].length = dbuf->size;
1412
1413	if (planes[plane].length < vb->planes[plane].min_length) {
1414	dprintk(q, 1, "invalid dmabuf length %u for plane %d, minimum length %u\n",
1415	planes[plane].length, plane,
1416	vb->planes[plane].min_length);
1417	ret = -EINVAL;
1418	goto err_put_planes;
1419	}
1420
1421	/* Skip the plane if already verified */
1422	if (dbuf == vb->planes[plane].dbuf &&
1423	vb->planes[plane].length == planes[plane].length)
1424	continue;
1425
1426	dprintk(q, 3, "buffer for plane %d changed\n", plane);
1427
1428	reacquired = true;
1429	}
1430
1431	if (reacquired) {
1432	if (vb->planes[0].mem_priv) {
1433	vb->copied_timestamp = 0;
1434	call_void_vb_qop(vb, buf_cleanup, vb);
1435	__vb2_buf_dmabuf_put(vb);
1436	}
1437
1438	for (plane = 0; plane < vb->num_planes; ++plane) {
1439	/*
1440	* This is an optimization to reduce dma_buf attachment/mapping.
1441	* When the same dma_buf is used for multiple planes, there is no need
1442	* to create duplicated attachments.
1443	*/
1444	for (i = 0; i < plane; ++i) {
1445	if (planes[plane].dbuf == vb->planes[i].dbuf &&
1446	q->alloc_devs[plane] == q->alloc_devs[i]) {
1447	vb->planes[plane].dbuf_duplicated = true;
1448	vb->planes[plane].dbuf = vb->planes[i].dbuf;
1449	vb->planes[plane].mem_priv = vb->planes[i].mem_priv;
1450	break;
1451	}
1452	}
1453
1454	if (vb->planes[plane].dbuf_duplicated)
1455	continue;
1456
1457	/* Acquire each plane's memory */
1458	mem_priv = call_ptr_memop(attach_dmabuf,
1459	vb,
1460	q->alloc_devs[plane] ? : q->dev,
1461	planes[plane].dbuf,
1462	planes[plane].length);
1463	if (IS_ERR(ptr: mem_priv)) {
1464	dprintk(q, 1, "failed to attach dmabuf\n");
1465	ret = PTR_ERR(ptr: mem_priv);
1466	goto err_put_vb2_buf;
1467	}
1468
1469	vb->planes[plane].dbuf = planes[plane].dbuf;
1470	vb->planes[plane].mem_priv = mem_priv;
1471
1472	/*
1473	* This pins the buffer(s) with dma_buf_map_attachment()). It's done
1474	* here instead just before the DMA, while queueing the buffer(s) so
1475	* userspace knows sooner rather than later if the dma-buf map fails.
1476	*/
1477	ret = call_memop(vb, map_dmabuf, vb->planes[plane].mem_priv);
1478	if (ret) {
1479	dprintk(q, 1, "failed to map dmabuf for plane %d\n",
1480	plane);
1481	goto err_put_vb2_buf;
1482	}
1483	vb->planes[plane].dbuf_mapped = 1;
1484	}
1485	} else {
1486	for (plane = 0; plane < vb->num_planes; ++plane)
1487	dma_buf_put(dmabuf: planes[plane].dbuf);
1488	}
1489
1490	/*
1491	* Now that everything is in order, copy relevant information
1492	* provided by userspace.
1493	*/
1494	for (plane = 0; plane < vb->num_planes; ++plane) {
1495	vb->planes[plane].bytesused = planes[plane].bytesused;
1496	vb->planes[plane].length = planes[plane].length;
1497	vb->planes[plane].m.fd = planes[plane].m.fd;
1498	vb->planes[plane].data_offset = planes[plane].data_offset;
1499	}
1500
1501	if (reacquired) {
1502	/*
1503	* Call driver-specific initialization on the newly acquired buffer,
1504	* if provided.
1505	*/
1506	ret = call_vb_qop(vb, buf_init, vb);
1507	if (ret) {
1508	dprintk(q, 1, "buffer initialization failed\n");
1509	goto err_put_vb2_buf;
1510	}
1511	}
1512
1513	ret = call_vb_qop(vb, buf_prepare, vb);
1514	if (ret) {
1515	dprintk(q, 1, "buffer preparation failed\n");
1516	call_void_vb_qop(vb, buf_cleanup, vb);
1517	goto err_put_vb2_buf;
1518	}
1519
1520	return 0;
1521
1522	err_put_planes:
1523	for (plane = 0; plane < vb->num_planes; ++plane) {
1524	if (!IS_ERR_OR_NULL(ptr: planes[plane].dbuf))
1525	dma_buf_put(dmabuf: planes[plane].dbuf);
1526	}
1527	err_put_vb2_buf:
1528	/* In case of errors, release planes that were already acquired */
1529	__vb2_buf_dmabuf_put(vb);
1530
1531	return ret;
1532	}
1533
1534	/*
1535	* __enqueue_in_driver() - enqueue a vb2_buffer in driver for processing
1536	*/
1537	static void __enqueue_in_driver(struct vb2_buffer *vb)
1538	{
1539	struct vb2_queue *q = vb->vb2_queue;
1540
1541	vb->state = VB2_BUF_STATE_ACTIVE;
1542	atomic_inc(v: &q->owned_by_drv_count);
1543
1544	trace_vb2_buf_queue(q, vb);
1545
1546	call_void_vb_qop(vb, buf_queue, vb);
1547	}
1548
1549	static int __buf_prepare(struct vb2_buffer *vb)
1550	{
1551	struct vb2_queue *q = vb->vb2_queue;
1552	enum vb2_buffer_state orig_state = vb->state;
1553	int ret;
1554
1555	if (q->error) {
1556	dprintk(q, 1, "fatal error occurred on queue\n");
1557	return -EIO;
1558	}
1559
1560	if (vb->prepared)
1561	return 0;
1562	WARN_ON(vb->synced);
1563
1564	if (q->is_output) {
1565	ret = call_vb_qop(vb, buf_out_validate, vb);
1566	if (ret) {
1567	dprintk(q, 1, "buffer validation failed\n");
1568	return ret;
1569	}
1570	}
1571
1572	vb->state = VB2_BUF_STATE_PREPARING;
1573
1574	switch (q->memory) {
1575	case VB2_MEMORY_MMAP:
1576	ret = __prepare_mmap(vb);
1577	break;
1578	case VB2_MEMORY_USERPTR:
1579	ret = __prepare_userptr(vb);
1580	break;
1581	case VB2_MEMORY_DMABUF:
1582	ret = __prepare_dmabuf(vb);
1583	break;
1584	default:
1585	WARN(1, "Invalid queue type\n");
1586	ret = -EINVAL;
1587	break;
1588	}
1589
1590	if (ret) {
1591	dprintk(q, 1, "buffer preparation failed: %d\n", ret);
1592	vb->state = orig_state;
1593	return ret;
1594	}
1595
1596	__vb2_buf_mem_prepare(vb);
1597	vb->prepared = 1;
1598	vb->state = orig_state;
1599
1600	return 0;
1601	}
1602
1603	static int vb2_req_prepare(struct media_request_object *obj)
1604	{
1605	struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1606	int ret;
1607
1608	if (WARN_ON(vb->state != VB2_BUF_STATE_IN_REQUEST))
1609	return -EINVAL;
1610
1611	mutex_lock(vb->vb2_queue->lock);
1612	ret = __buf_prepare(vb);
1613	mutex_unlock(lock: vb->vb2_queue->lock);
1614	return ret;
1615	}
1616
1617	static void __vb2_dqbuf(struct vb2_buffer *vb);
1618
1619	static void vb2_req_unprepare(struct media_request_object *obj)
1620	{
1621	struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1622
1623	mutex_lock(vb->vb2_queue->lock);
1624	__vb2_dqbuf(vb);
1625	vb->state = VB2_BUF_STATE_IN_REQUEST;
1626	mutex_unlock(lock: vb->vb2_queue->lock);
1627	WARN_ON(!vb->req_obj.req);
1628	}
1629
1630	static void vb2_req_queue(struct media_request_object *obj)
1631	{
1632	struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1633	int err;
1634
1635	mutex_lock(vb->vb2_queue->lock);
1636	/*
1637	* There is no method to propagate an error from vb2_core_qbuf(),
1638	* so if this returns a non-0 value, then WARN.
1639	*
1640	* The only exception is -EIO which is returned if q->error is
1641	* set. We just ignore that, and expect this will be caught the
1642	* next time vb2_req_prepare() is called.
1643	*/
1644	err = vb2_core_qbuf(q: vb->vb2_queue, vb, NULL, NULL);
1645	WARN_ON_ONCE(err && err != -EIO);
1646	mutex_unlock(lock: vb->vb2_queue->lock);
1647	}
1648
1649	static void vb2_req_unbind(struct media_request_object *obj)
1650	{
1651	struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1652
1653	if (vb->state == VB2_BUF_STATE_IN_REQUEST)
1654	call_void_bufop(vb->vb2_queue, init_buffer, vb);
1655	}
1656
1657	static void vb2_req_release(struct media_request_object *obj)
1658	{
1659	struct vb2_buffer *vb = container_of(obj, struct vb2_buffer, req_obj);
1660
1661	if (vb->state == VB2_BUF_STATE_IN_REQUEST) {
1662	vb->state = VB2_BUF_STATE_DEQUEUED;
1663	if (vb->request)
1664	media_request_put(req: vb->request);
1665	vb->request = NULL;
1666	}
1667	}
1668
1669	static const struct media_request_object_ops vb2_core_req_ops = {
1670	.prepare = vb2_req_prepare,
1671	.unprepare = vb2_req_unprepare,
1672	.queue = vb2_req_queue,
1673	.unbind = vb2_req_unbind,
1674	.release = vb2_req_release,
1675	};
1676
1677	bool vb2_request_object_is_buffer(struct media_request_object *obj)
1678	{
1679	return obj->ops == &vb2_core_req_ops;
1680	}
1681	EXPORT_SYMBOL_GPL(vb2_request_object_is_buffer);
1682
1683	unsigned int vb2_request_buffer_cnt(struct media_request *req)
1684	{
1685	struct media_request_object *obj;
1686	unsigned long flags;
1687	unsigned int buffer_cnt = 0;
1688
1689	spin_lock_irqsave(&req->lock, flags);
1690	list_for_each_entry(obj, &req->objects, list)
1691	if (vb2_request_object_is_buffer(obj))
1692	buffer_cnt++;
1693	spin_unlock_irqrestore(lock: &req->lock, flags);
1694
1695	return buffer_cnt;
1696	}
1697	EXPORT_SYMBOL_GPL(vb2_request_buffer_cnt);
1698
1699	int vb2_core_prepare_buf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb)
1700	{
1701	int ret;
1702
1703	if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1704	dprintk(q, 1, "invalid buffer state %s\n",
1705	vb2_state_name(vb->state));
1706	return -EINVAL;
1707	}
1708	if (vb->prepared) {
1709	dprintk(q, 1, "buffer already prepared\n");
1710	return -EINVAL;
1711	}
1712
1713	ret = __buf_prepare(vb);
1714	if (ret)
1715	return ret;
1716
1717	/* Fill buffer information for the userspace */
1718	call_void_bufop(q, fill_user_buffer, vb, pb);
1719
1720	dprintk(q, 2, "prepare of buffer %d succeeded\n", vb->index);
1721
1722	return 0;
1723	}
1724	EXPORT_SYMBOL_GPL(vb2_core_prepare_buf);
1725
1726	int vb2_core_remove_bufs(struct vb2_queue *q, unsigned int start, unsigned int count)
1727	{
1728	unsigned int i, ret = 0;
1729	unsigned int q_num_bufs = vb2_get_num_buffers(q);
1730
1731	if (count == 0)
1732	return 0;
1733
1734	if (count > q_num_bufs)
1735	return -EINVAL;
1736
1737	if (start > q->max_num_buffers - count)
1738	return -EINVAL;
1739
1740	mutex_lock(&q->mmap_lock);
1741
1742	/* Check that all buffers in the range exist */
1743	for (i = start; i < start + count; i++) {
1744	struct vb2_buffer *vb = vb2_get_buffer(q, index: i);
1745
1746	if (!vb) {
1747	ret = -EINVAL;
1748	goto unlock;
1749	}
1750	if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1751	ret = -EBUSY;
1752	goto unlock;
1753	}
1754	}
1755	__vb2_queue_free(q, start, count);
1756	dprintk(q, 2, "%u buffers removed\n", count);
1757
1758	unlock:
1759	mutex_unlock(lock: &q->mmap_lock);
1760	return ret;
1761	}
1762	EXPORT_SYMBOL_GPL(vb2_core_remove_bufs);
1763
1764	/*
1765	* vb2_start_streaming() - Attempt to start streaming.
1766	* @q: videobuf2 queue
1767	*
1768	* Attempt to start streaming. When this function is called there must be
1769	* at least q->min_queued_buffers queued up (i.e. the minimum
1770	* number of buffers required for the DMA engine to function). If the
1771	* @start_streaming op fails it is supposed to return all the driver-owned
1772	* buffers back to vb2 in state QUEUED. Check if that happened and if
1773	* not warn and reclaim them forcefully.
1774	*/
1775	static int vb2_start_streaming(struct vb2_queue *q)
1776	{
1777	struct vb2_buffer *vb;
1778	int ret;
1779
1780	/*
1781	* If any buffers were queued before streamon,
1782	* we can now pass them to driver for processing.
1783	*/
1784	list_for_each_entry(vb, &q->queued_list, queued_entry)
1785	__enqueue_in_driver(vb);
1786
1787	/* Tell the driver to start streaming */
1788	q->start_streaming_called = 1;
1789	ret = call_qop(q, start_streaming, q,
1790	atomic_read(&q->owned_by_drv_count));
1791	if (!ret)
1792	return 0;
1793
1794	q->start_streaming_called = 0;
1795
1796	dprintk(q, 1, "driver refused to start streaming\n");
1797	/*
1798	* If you see this warning, then the driver isn't cleaning up properly
1799	* after a failed start_streaming(). See the start_streaming()
1800	* documentation in videobuf2-core.h for more information how buffers
1801	* should be returned to vb2 in start_streaming().
1802	*/
1803	if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
1804	unsigned i;
1805
1806	/*
1807	* Forcefully reclaim buffers if the driver did not
1808	* correctly return them to vb2.
1809	*/
1810	for (i = 0; i < q->max_num_buffers; ++i) {
1811	vb = vb2_get_buffer(q, index: i);
1812
1813	if (!vb)
1814	continue;
1815
1816	if (vb->state == VB2_BUF_STATE_ACTIVE)
1817	vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED);
1818	}
1819	/* Must be zero now */
1820	WARN_ON(atomic_read(&q->owned_by_drv_count));
1821	}
1822	/*
1823	* If done_list is not empty, then start_streaming() didn't call
1824	* vb2_buffer_done(vb, VB2_BUF_STATE_QUEUED) but STATE_ERROR or
1825	* STATE_DONE.
1826	*/
1827	WARN_ON(!list_empty(&q->done_list));
1828	return ret;
1829	}
1830
1831	int vb2_core_qbuf(struct vb2_queue *q, struct vb2_buffer *vb, void *pb,
1832	struct media_request *req)
1833	{
1834	enum vb2_buffer_state orig_state;
1835	int ret;
1836
1837	if (q->error) {
1838	dprintk(q, 1, "fatal error occurred on queue\n");
1839	return -EIO;
1840	}
1841
1842	if (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
1843	q->requires_requests) {
1844	dprintk(q, 1, "qbuf requires a request\n");
1845	return -EBADR;
1846	}
1847
1848	if ((req && q->uses_qbuf) ||
1849	(!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
1850	q->uses_requests)) {
1851	dprintk(q, 1, "queue in wrong mode (qbuf vs requests)\n");
1852	return -EBUSY;
1853	}
1854
1855	if (req) {
1856	int ret;
1857
1858	q->uses_requests = 1;
1859	if (vb->state != VB2_BUF_STATE_DEQUEUED) {
1860	dprintk(q, 1, "buffer %d not in dequeued state\n",
1861	vb->index);
1862	return -EINVAL;
1863	}
1864
1865	if (q->is_output && !vb->prepared) {
1866	ret = call_vb_qop(vb, buf_out_validate, vb);
1867	if (ret) {
1868	dprintk(q, 1, "buffer validation failed\n");
1869	return ret;
1870	}
1871	}
1872
1873	media_request_object_init(obj: &vb->req_obj);
1874
1875	/* Make sure the request is in a safe state for updating. */
1876	ret = media_request_lock_for_update(req);
1877	if (ret)
1878	return ret;
1879	ret = media_request_object_bind(req, ops: &vb2_core_req_ops,
1880	priv: q, is_buffer: true, obj: &vb->req_obj);
1881	media_request_unlock_for_update(req);
1882	if (ret)
1883	return ret;
1884
1885	vb->state = VB2_BUF_STATE_IN_REQUEST;
1886
1887	/*
1888	* Increment the refcount and store the request.
1889	* The request refcount is decremented again when the
1890	* buffer is dequeued. This is to prevent vb2_buffer_done()
1891	* from freeing the request from interrupt context, which can
1892	* happen if the application closed the request fd after
1893	* queueing the request.
1894	*/
1895	media_request_get(req);
1896	vb->request = req;
1897
1898	/* Fill buffer information for the userspace */
1899	if (pb) {
1900	call_void_bufop(q, copy_timestamp, vb, pb);
1901	call_void_bufop(q, fill_user_buffer, vb, pb);
1902	}
1903
1904	dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
1905	return 0;
1906	}
1907
1908	if (vb->state != VB2_BUF_STATE_IN_REQUEST)
1909	q->uses_qbuf = 1;
1910
1911	switch (vb->state) {
1912	case VB2_BUF_STATE_DEQUEUED:
1913	case VB2_BUF_STATE_IN_REQUEST:
1914	if (!vb->prepared) {
1915	ret = __buf_prepare(vb);
1916	if (ret)
1917	return ret;
1918	}
1919	break;
1920	case VB2_BUF_STATE_PREPARING:
1921	dprintk(q, 1, "buffer still being prepared\n");
1922	return -EINVAL;
1923	default:
1924	dprintk(q, 1, "invalid buffer state %s\n",
1925	vb2_state_name(vb->state));
1926	return -EINVAL;
1927	}
1928
1929	/*
1930	* Add to the queued buffers list, a buffer will stay on it until
1931	* dequeued in dqbuf.
1932	*/
1933	orig_state = vb->state;
1934	list_add_tail(new: &vb->queued_entry, head: &q->queued_list);
1935	q->queued_count++;
1936	q->waiting_for_buffers = false;
1937	vb->state = VB2_BUF_STATE_QUEUED;
1938
1939	if (pb)
1940	call_void_bufop(q, copy_timestamp, vb, pb);
1941
1942	trace_vb2_qbuf(q, vb);
1943
1944	/*
1945	* If already streaming, give the buffer to driver for processing.
1946	* If not, the buffer will be given to driver on next streamon.
1947	*/
1948	if (q->start_streaming_called)
1949	__enqueue_in_driver(vb);
1950
1951	/* Fill buffer information for the userspace */
1952	if (pb)
1953	call_void_bufop(q, fill_user_buffer, vb, pb);
1954
1955	/*
1956	* If streamon has been called, and we haven't yet called
1957	* start_streaming() since not enough buffers were queued, and
1958	* we now have reached the minimum number of queued buffers,
1959	* then we can finally call start_streaming().
1960	*/
1961	if (q->streaming && !q->start_streaming_called &&
1962	q->queued_count >= q->min_queued_buffers) {
1963	ret = vb2_start_streaming(q);
1964	if (ret) {
1965	/*
1966	* Since vb2_core_qbuf will return with an error,
1967	* we should return it to state DEQUEUED since
1968	* the error indicates that the buffer wasn't queued.
1969	*/
1970	list_del(entry: &vb->queued_entry);
1971	q->queued_count--;
1972	vb->state = orig_state;
1973	return ret;
1974	}
1975	}
1976
1977	dprintk(q, 2, "qbuf of buffer %d succeeded\n", vb->index);
1978	return 0;
1979	}
1980	EXPORT_SYMBOL_GPL(vb2_core_qbuf);
1981
1982	/*
1983	* __vb2_wait_for_done_vb() - wait for a buffer to become available
1984	* for dequeuing
1985	*
1986	* Will sleep if required for nonblocking == false.
1987	*/
1988	static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
1989	{
1990	/*
1991	* All operations on vb_done_list are performed under done_lock
1992	* spinlock protection. However, buffers may be removed from
1993	* it and returned to userspace only while holding both driver's
1994	* lock and the done_lock spinlock. Thus we can be sure that as
1995	* long as we hold the driver's lock, the list will remain not
1996	* empty if list_empty() check succeeds.
1997	*/
1998
1999	for (;;) {
2000	int ret;
2001
2002	if (q->waiting_in_dqbuf) {
2003	dprintk(q, 1, "another dup()ped fd is waiting for a buffer\n");
2004	return -EBUSY;
2005	}
2006
2007	if (!q->streaming) {
2008	dprintk(q, 1, "streaming off, will not wait for buffers\n");
2009	return -EINVAL;
2010	}
2011
2012	if (q->error) {
2013	dprintk(q, 1, "Queue in error state, will not wait for buffers\n");
2014	return -EIO;
2015	}
2016
2017	if (q->last_buffer_dequeued) {
2018	dprintk(q, 3, "last buffer dequeued already, will not wait for buffers\n");
2019	return -EPIPE;
2020	}
2021
2022	if (!list_empty(head: &q->done_list)) {
2023	/*
2024	* Found a buffer that we were waiting for.
2025	*/
2026	break;
2027	}
2028
2029	if (nonblocking) {
2030	dprintk(q, 3, "nonblocking and no buffers to dequeue, will not wait\n");
2031	return -EAGAIN;
2032	}
2033
2034	q->waiting_in_dqbuf = 1;
2035	/*
2036	* We are streaming and blocking, wait for another buffer to
2037	* become ready or for streamoff. Driver's lock is released to
2038	* allow streamoff or qbuf to be called while waiting.
2039	*/
2040	if (q->ops->wait_prepare)
2041	call_void_qop(q, wait_prepare, q);
2042	else if (q->lock)
2043	mutex_unlock(lock: q->lock);
2044
2045	/*
2046	* All locks have been released, it is safe to sleep now.
2047	*/
2048	dprintk(q, 3, "will sleep waiting for buffers\n");
2049	ret = wait_event_interruptible(q->done_wq,
2050	!list_empty(&q->done_list) || !q->streaming ||
2051	q->error);
2052
2053	if (q->ops->wait_finish)
2054	call_void_qop(q, wait_finish, q);
2055	else if (q->lock)
2056	mutex_lock(q->lock);
2057
2058	q->waiting_in_dqbuf = 0;
2059	/*
2060	* We need to reevaluate both conditions again after reacquiring
2061	* the locks or return an error if one occurred.
2062	*/
2063	if (ret) {
2064	dprintk(q, 1, "sleep was interrupted\n");
2065	return ret;
2066	}
2067	}
2068	return 0;
2069	}
2070
2071	/*
2072	* __vb2_get_done_vb() - get a buffer ready for dequeuing
2073	*
2074	* Will sleep if required for nonblocking == false.
2075	*/
2076	static int __vb2_get_done_vb(struct vb2_queue *q, struct vb2_buffer **vb,
2077	void *pb, int nonblocking)
2078	{
2079	unsigned long flags;
2080	int ret = 0;
2081
2082	/*
2083	* Wait for at least one buffer to become available on the done_list.
2084	*/
2085	ret = __vb2_wait_for_done_vb(q, nonblocking);
2086	if (ret)
2087	return ret;
2088
2089	/*
2090	* Driver's lock has been held since we last verified that done_list
2091	* is not empty, so no need for another list_empty(done_list) check.
2092	*/
2093	spin_lock_irqsave(&q->done_lock, flags);
2094	*vb = list_first_entry(&q->done_list, struct vb2_buffer, done_entry);
2095	/*
2096	* Only remove the buffer from done_list if all planes can be
2097	* handled. Some cases such as V4L2 file I/O and DVB have pb
2098	* == NULL; skip the check then as there's nothing to verify.
2099	*/
2100	if (pb)
2101	ret = call_bufop(q, verify_planes_array, *vb, pb);
2102	if (!ret)
2103	list_del(entry: &(*vb)->done_entry);
2104	spin_unlock_irqrestore(lock: &q->done_lock, flags);
2105
2106	return ret;
2107	}
2108
2109	int vb2_wait_for_all_buffers(struct vb2_queue *q)
2110	{
2111	if (!q->streaming) {
2112	dprintk(q, 1, "streaming off, will not wait for buffers\n");
2113	return -EINVAL;
2114	}
2115
2116	if (q->start_streaming_called)
2117	wait_event(q->done_wq, !atomic_read(&q->owned_by_drv_count));
2118	return 0;
2119	}
2120	EXPORT_SYMBOL_GPL(vb2_wait_for_all_buffers);
2121
2122	/*
2123	* __vb2_dqbuf() - bring back the buffer to the DEQUEUED state
2124	*/
2125	static void __vb2_dqbuf(struct vb2_buffer *vb)
2126	{
2127	struct vb2_queue *q = vb->vb2_queue;
2128
2129	/* nothing to do if the buffer is already dequeued */
2130	if (vb->state == VB2_BUF_STATE_DEQUEUED)
2131	return;
2132
2133	vb->state = VB2_BUF_STATE_DEQUEUED;
2134
2135	call_void_bufop(q, init_buffer, vb);
2136	}
2137
2138	int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
2139	bool nonblocking)
2140	{
2141	struct vb2_buffer *vb = NULL;
2142	int ret;
2143
2144	ret = __vb2_get_done_vb(q, vb: &vb, pb, nonblocking);
2145	if (ret < 0)
2146	return ret;
2147
2148	switch (vb->state) {
2149	case VB2_BUF_STATE_DONE:
2150	dprintk(q, 3, "returning done buffer\n");
2151	break;
2152	case VB2_BUF_STATE_ERROR:
2153	dprintk(q, 3, "returning done buffer with errors\n");
2154	break;
2155	default:
2156	dprintk(q, 1, "invalid buffer state %s\n",
2157	vb2_state_name(vb->state));
2158	return -EINVAL;
2159	}
2160
2161	call_void_vb_qop(vb, buf_finish, vb);
2162	vb->prepared = 0;
2163
2164	if (pindex)
2165	*pindex = vb->index;
2166
2167	/* Fill buffer information for the userspace */
2168	if (pb)
2169	call_void_bufop(q, fill_user_buffer, vb, pb);
2170
2171	/* Remove from vb2 queue */
2172	list_del(entry: &vb->queued_entry);
2173	q->queued_count--;
2174
2175	trace_vb2_dqbuf(q, vb);
2176
2177	/* go back to dequeued state */
2178	__vb2_dqbuf(vb);
2179
2180	if (WARN_ON(vb->req_obj.req)) {
2181	media_request_object_unbind(obj: &vb->req_obj);
2182	media_request_object_put(obj: &vb->req_obj);
2183	}
2184	if (vb->request)
2185	media_request_put(req: vb->request);
2186	vb->request = NULL;
2187
2188	dprintk(q, 2, "dqbuf of buffer %d, state: %s\n",
2189	vb->index, vb2_state_name(vb->state));
2190
2191	return 0;
2192
2193	}
2194	EXPORT_SYMBOL_GPL(vb2_core_dqbuf);
2195
2196	/*
2197	* __vb2_queue_cancel() - cancel and stop (pause) streaming
2198	*
2199	* Removes all queued buffers from driver's queue and all buffers queued by
2200	* userspace from vb2's queue. Returns to state after reqbufs.
2201	*/
2202	static void __vb2_queue_cancel(struct vb2_queue *q)
2203	{
2204	unsigned int i;
2205
2206	/*
2207	* Tell driver to stop all transactions and release all queued
2208	* buffers.
2209	*/
2210	if (q->start_streaming_called)
2211	call_void_qop(q, stop_streaming, q);
2212
2213	if (q->streaming)
2214	call_void_qop(q, unprepare_streaming, q);
2215
2216	/*
2217	* If you see this warning, then the driver isn't cleaning up properly
2218	* in stop_streaming(). See the stop_streaming() documentation in
2219	* videobuf2-core.h for more information how buffers should be returned
2220	* to vb2 in stop_streaming().
2221	*/
2222	if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
2223	for (i = 0; i < q->max_num_buffers; i++) {
2224	struct vb2_buffer *vb = vb2_get_buffer(q, index: i);
2225
2226	if (!vb)
2227	continue;
2228
2229	if (vb->state == VB2_BUF_STATE_ACTIVE) {
2230	pr_warn("driver bug: stop_streaming operation is leaving buffer %u in active state\n",
2231	vb->index);
2232	vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
2233	}
2234	}
2235	/* Must be zero now */
2236	WARN_ON(atomic_read(&q->owned_by_drv_count));
2237	}
2238
2239	q->streaming = 0;
2240	q->start_streaming_called = 0;
2241	q->queued_count = 0;
2242	q->error = 0;
2243	q->uses_requests = 0;
2244	q->uses_qbuf = 0;
2245
2246	/*
2247	* Remove all buffers from vb2's list...
2248	*/
2249	INIT_LIST_HEAD(list: &q->queued_list);
2250	/*
2251	* ...and done list; userspace will not receive any buffers it
2252	* has not already dequeued before initiating cancel.
2253	*/
2254	INIT_LIST_HEAD(list: &q->done_list);
2255	atomic_set(v: &q->owned_by_drv_count, i: 0);
2256	wake_up_all(&q->done_wq);
2257
2258	/*
2259	* Reinitialize all buffers for next use.
2260	* Make sure to call buf_finish for any queued buffers. Normally
2261	* that's done in dqbuf, but that's not going to happen when we
2262	* cancel the whole queue. Note: this code belongs here, not in
2263	* __vb2_dqbuf() since in vb2_core_dqbuf() there is a critical
2264	* call to __fill_user_buffer() after buf_finish(). That order can't
2265	* be changed, so we can't move the buf_finish() to __vb2_dqbuf().
2266	*/
2267	for (i = 0; i < q->max_num_buffers; i++) {
2268	struct vb2_buffer *vb;
2269	struct media_request *req;
2270
2271	vb = vb2_get_buffer(q, index: i);
2272	if (!vb)
2273	continue;
2274
2275	req = vb->req_obj.req;
2276	/*
2277	* If a request is associated with this buffer, then
2278	* call buf_request_cancel() to give the driver to complete()
2279	* related request objects. Otherwise those objects would
2280	* never complete.
2281	*/
2282	if (req) {
2283	enum media_request_state state;
2284	unsigned long flags;
2285
2286	spin_lock_irqsave(&req->lock, flags);
2287	state = req->state;
2288	spin_unlock_irqrestore(lock: &req->lock, flags);
2289
2290	if (state == MEDIA_REQUEST_STATE_QUEUED)
2291	call_void_vb_qop(vb, buf_request_complete, vb);
2292	}
2293
2294	__vb2_buf_mem_finish(vb);
2295
2296	if (vb->prepared) {
2297	call_void_vb_qop(vb, buf_finish, vb);
2298	vb->prepared = 0;
2299	}
2300	__vb2_dqbuf(vb);
2301
2302	if (vb->req_obj.req) {
2303	media_request_object_unbind(obj: &vb->req_obj);
2304	media_request_object_put(obj: &vb->req_obj);
2305	}
2306	if (vb->request)
2307	media_request_put(req: vb->request);
2308	vb->request = NULL;
2309	vb->copied_timestamp = 0;
2310	}
2311	}
2312
2313	int vb2_core_streamon(struct vb2_queue *q, unsigned int type)
2314	{
2315	unsigned int q_num_bufs = vb2_get_num_buffers(q);
2316	int ret;
2317
2318	if (type != q->type) {
2319	dprintk(q, 1, "invalid stream type\n");
2320	return -EINVAL;
2321	}
2322
2323	if (q->streaming) {
2324	dprintk(q, 3, "already streaming\n");
2325	return 0;
2326	}
2327
2328	if (!q_num_bufs) {
2329	dprintk(q, 1, "no buffers have been allocated\n");
2330	return -EINVAL;
2331	}
2332
2333	if (q_num_bufs < q->min_queued_buffers) {
2334	dprintk(q, 1, "need at least %u allocated buffers\n",
2335	q->min_queued_buffers);
2336	return -EINVAL;
2337	}
2338
2339	ret = call_qop(q, prepare_streaming, q);
2340	if (ret)
2341	return ret;
2342
2343	/*
2344	* Tell driver to start streaming provided sufficient buffers
2345	* are available.
2346	*/
2347	if (q->queued_count >= q->min_queued_buffers) {
2348	ret = vb2_start_streaming(q);
2349	if (ret)
2350	goto unprepare;
2351	}
2352
2353	q->streaming = 1;
2354
2355	dprintk(q, 3, "successful\n");
2356	return 0;
2357
2358	unprepare:
2359	call_void_qop(q, unprepare_streaming, q);
2360	return ret;
2361	}
2362	EXPORT_SYMBOL_GPL(vb2_core_streamon);
2363
2364	void vb2_queue_error(struct vb2_queue *q)
2365	{
2366	q->error = 1;
2367
2368	wake_up_all(&q->done_wq);
2369	}
2370	EXPORT_SYMBOL_GPL(vb2_queue_error);
2371
2372	int vb2_core_streamoff(struct vb2_queue *q, unsigned int type)
2373	{
2374	if (type != q->type) {
2375	dprintk(q, 1, "invalid stream type\n");
2376	return -EINVAL;
2377	}
2378
2379	/*
2380	* Cancel will pause streaming and remove all buffers from the driver
2381	* and vb2, effectively returning control over them to userspace.
2382	*
2383	* Note that we do this even if q->streaming == 0: if you prepare or
2384	* queue buffers, and then call streamoff without ever having called
2385	* streamon, you would still expect those buffers to be returned to
2386	* their normal dequeued state.
2387	*/
2388	__vb2_queue_cancel(q);
2389	q->waiting_for_buffers = !q->is_output;
2390	q->last_buffer_dequeued = false;
2391
2392	dprintk(q, 3, "successful\n");
2393	return 0;
2394	}
2395	EXPORT_SYMBOL_GPL(vb2_core_streamoff);
2396
2397	/*
2398	* __find_plane_by_offset() - find plane associated with the given offset
2399	*/
2400	static int __find_plane_by_offset(struct vb2_queue *q, unsigned long offset,
2401	struct vb2_buffer **vb, unsigned int *plane)
2402	{
2403	unsigned int buffer;
2404
2405	/*
2406	* Sanity checks to ensure the lock is held, MEMORY_MMAP is
2407	* used and fileio isn't active.
2408	*/
2409	lockdep_assert_held(&q->mmap_lock);
2410
2411	if (q->memory != VB2_MEMORY_MMAP) {
2412	dprintk(q, 1, "queue is not currently set up for mmap\n");
2413	return -EINVAL;
2414	}
2415
2416	if (vb2_fileio_is_active(q)) {
2417	dprintk(q, 1, "file io in progress\n");
2418	return -EBUSY;
2419	}
2420
2421	/* Get buffer and plane from the offset */
2422	buffer = (offset >> PLANE_INDEX_SHIFT) & BUFFER_INDEX_MASK;
2423	*plane = (offset >> PAGE_SHIFT) & PLANE_INDEX_MASK;
2424
2425	*vb = vb2_get_buffer(q, index: buffer);
2426	if (!*vb)
2427	return -EINVAL;
2428	if (*plane >= (*vb)->num_planes)
2429	return -EINVAL;
2430
2431	return 0;
2432	}
2433
2434	int vb2_core_expbuf(struct vb2_queue *q, int *fd, unsigned int type,
2435	struct vb2_buffer *vb, unsigned int plane, unsigned int flags)
2436	{
2437	struct vb2_plane *vb_plane;
2438	int ret;
2439	struct dma_buf *dbuf;
2440
2441	if (q->memory != VB2_MEMORY_MMAP) {
2442	dprintk(q, 1, "queue is not currently set up for mmap\n");
2443	return -EINVAL;
2444	}
2445
2446	if (!q->mem_ops->get_dmabuf) {
2447	dprintk(q, 1, "queue does not support DMA buffer exporting\n");
2448	return -EINVAL;
2449	}
2450
2451	if (flags & ~(O_CLOEXEC | O_ACCMODE)) {
2452	dprintk(q, 1, "queue does support only O_CLOEXEC and access mode flags\n");
2453	return -EINVAL;
2454	}
2455
2456	if (type != q->type) {
2457	dprintk(q, 1, "invalid buffer type\n");
2458	return -EINVAL;
2459	}
2460
2461	if (plane >= vb->num_planes) {
2462	dprintk(q, 1, "buffer plane out of range\n");
2463	return -EINVAL;
2464	}
2465
2466	if (vb2_fileio_is_active(q)) {
2467	dprintk(q, 1, "expbuf: file io in progress\n");
2468	return -EBUSY;
2469	}
2470
2471	vb_plane = &vb->planes[plane];
2472
2473	dbuf = call_ptr_memop(get_dmabuf,
2474	vb,
2475	vb_plane->mem_priv,
2476	flags & O_ACCMODE);
2477	if (IS_ERR_OR_NULL(ptr: dbuf)) {
2478	dprintk(q, 1, "failed to export buffer %d, plane %d\n",
2479	vb->index, plane);
2480	return -EINVAL;
2481	}
2482
2483	ret = dma_buf_fd(dmabuf: dbuf, flags: flags & ~O_ACCMODE);
2484	if (ret < 0) {
2485	dprintk(q, 3, "buffer %d, plane %d failed to export (%d)\n",
2486	vb->index, plane, ret);
2487	dma_buf_put(dmabuf: dbuf);
2488	return ret;
2489	}
2490
2491	dprintk(q, 3, "buffer %d, plane %d exported as %d descriptor\n",
2492	vb->index, plane, ret);
2493	*fd = ret;
2494
2495	return 0;
2496	}
2497	EXPORT_SYMBOL_GPL(vb2_core_expbuf);
2498
2499	int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
2500	{
2501	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
2502	struct vb2_buffer *vb;
2503	unsigned int plane = 0;
2504	int ret;
2505	unsigned long length;
2506
2507	/*
2508	* Check memory area access mode.
2509	*/
2510	if (!(vma->vm_flags & VM_SHARED)) {
2511	dprintk(q, 1, "invalid vma flags, VM_SHARED needed\n");
2512	return -EINVAL;
2513	}
2514	if (q->is_output) {
2515	if (!(vma->vm_flags & VM_WRITE)) {
2516	dprintk(q, 1, "invalid vma flags, VM_WRITE needed\n");
2517	return -EINVAL;
2518	}
2519	} else {
2520	if (!(vma->vm_flags & VM_READ)) {
2521	dprintk(q, 1, "invalid vma flags, VM_READ needed\n");
2522	return -EINVAL;
2523	}
2524	}
2525
2526	mutex_lock(&q->mmap_lock);
2527
2528	/*
2529	* Find the plane corresponding to the offset passed by userspace. This
2530	* will return an error if not MEMORY_MMAP or file I/O is in progress.
2531	*/
2532	ret = __find_plane_by_offset(q, offset, vb: &vb, plane: &plane);
2533	if (ret)
2534	goto unlock;
2535
2536	/*
2537	* MMAP requires page_aligned buffers.
2538	* The buffer length was page_aligned at __vb2_buf_mem_alloc(),
2539	* so, we need to do the same here.
2540	*/
2541	length = PAGE_ALIGN(vb->planes[plane].length);
2542	if (length < (vma->vm_end - vma->vm_start)) {
2543	dprintk(q, 1,
2544	"MMAP invalid, as it would overflow buffer length\n");
2545	ret = -EINVAL;
2546	goto unlock;
2547	}
2548
2549	/*
2550	* vm_pgoff is treated in V4L2 API as a 'cookie' to select a buffer,
2551	* not as a in-buffer offset. We always want to mmap a whole buffer
2552	* from its beginning.
2553	*/
2554	vma->vm_pgoff = 0;
2555
2556	ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma);
2557
2558	unlock:
2559	mutex_unlock(lock: &q->mmap_lock);
2560	if (ret)
2561	return ret;
2562
2563	dprintk(q, 3, "buffer %u, plane %d successfully mapped\n", vb->index, plane);
2564	return 0;
2565	}
2566	EXPORT_SYMBOL_GPL(vb2_mmap);
2567
2568	#ifndef CONFIG_MMU
2569	unsigned long vb2_get_unmapped_area(struct vb2_queue *q,
2570	unsigned long addr,
2571	unsigned long len,
2572	unsigned long pgoff,
2573	unsigned long flags)
2574	{
2575	unsigned long offset = pgoff << PAGE_SHIFT;
2576	struct vb2_buffer *vb;
2577	unsigned int plane;
2578	void *vaddr;
2579	int ret;
2580
2581	mutex_lock(&q->mmap_lock);
2582
2583	/*
2584	* Find the plane corresponding to the offset passed by userspace. This
2585	* will return an error if not MEMORY_MMAP or file I/O is in progress.
2586	*/
2587	ret = __find_plane_by_offset(q, offset, &vb, &plane);
2588	if (ret)
2589	goto unlock;
2590
2591	vaddr = vb2_plane_vaddr(vb, plane);
2592	mutex_unlock(&q->mmap_lock);
2593	return vaddr ? (unsigned long)vaddr : -EINVAL;
2594
2595	unlock:
2596	mutex_unlock(&q->mmap_lock);
2597	return ret;
2598	}
2599	EXPORT_SYMBOL_GPL(vb2_get_unmapped_area);
2600	#endif
2601
2602	int vb2_core_queue_init(struct vb2_queue *q)
2603	{
2604	/*
2605	* Sanity check
2606	*/
2607	/*
2608	* For drivers who don't support max_num_buffers ensure
2609	* a backward compatibility.
2610	*/
2611	if (!q->max_num_buffers)
2612	q->max_num_buffers = VB2_MAX_FRAME;
2613
2614	/* The maximum is limited by offset cookie encoding pattern */
2615	q->max_num_buffers = min_t(unsigned int, q->max_num_buffers, MAX_BUFFER_INDEX);
2616
2617	if (WARN_ON(!q) ||
2618	WARN_ON(!q->ops) ||
2619	WARN_ON(!q->mem_ops) ||
2620	WARN_ON(!q->type) ||
2621	WARN_ON(!q->io_modes) ||
2622	WARN_ON(!q->ops->queue_setup) ||
2623	WARN_ON(!q->ops->buf_queue))
2624	return -EINVAL;
2625
2626	if (WARN_ON(q->max_num_buffers < VB2_MAX_FRAME) ||
2627	WARN_ON(q->min_queued_buffers > q->max_num_buffers))
2628	return -EINVAL;
2629
2630	if (WARN_ON(q->requires_requests && !q->supports_requests))
2631	return -EINVAL;
2632
2633	/*
2634	* This combination is not allowed since a non-zero value of
2635	* q->min_queued_buffers can cause vb2_core_qbuf() to fail if
2636	* it has to call start_streaming(), and the Request API expects
2637	* that queueing a request (and thus queueing a buffer contained
2638	* in that request) will always succeed. There is no method of
2639	* propagating an error back to userspace.
2640	*/
2641	if (WARN_ON(q->supports_requests && q->min_queued_buffers))
2642	return -EINVAL;
2643
2644	/*
2645	* If the driver needs 'min_queued_buffers' in the queue before
2646	* calling start_streaming() then the minimum requirement is
2647	* 'min_queued_buffers + 1' to keep at least one buffer available
2648	* for userspace.
2649	*/
2650	if (q->min_reqbufs_allocation < q->min_queued_buffers + 1)
2651	q->min_reqbufs_allocation = q->min_queued_buffers + 1;
2652
2653	if (WARN_ON(q->min_reqbufs_allocation > q->max_num_buffers))
2654	return -EINVAL;
2655
2656	/* Either both or none are set */
2657	if (WARN_ON(!q->ops->wait_prepare ^ !q->ops->wait_finish))
2658	return -EINVAL;
2659
2660	/* Warn if q->lock is NULL and no custom wait_prepare is provided */
2661	if (WARN_ON(!q->lock && !q->ops->wait_prepare))
2662	return -EINVAL;
2663
2664	INIT_LIST_HEAD(list: &q->queued_list);
2665	INIT_LIST_HEAD(list: &q->done_list);
2666	spin_lock_init(&q->done_lock);
2667	mutex_init(&q->mmap_lock);
2668	init_waitqueue_head(&q->done_wq);
2669
2670	q->memory = VB2_MEMORY_UNKNOWN;
2671
2672	if (q->buf_struct_size == 0)
2673	q->buf_struct_size = sizeof(struct vb2_buffer);
2674
2675	if (q->bidirectional)
2676	q->dma_dir = DMA_BIDIRECTIONAL;
2677	else
2678	q->dma_dir = q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
2679
2680	if (q->name[0] == '\0')
2681	snprintf(buf: q->name, size: sizeof(q->name), fmt: "%s-%p",
2682	q->is_output ? "out" : "cap", q);
2683
2684	return 0;
2685	}
2686	EXPORT_SYMBOL_GPL(vb2_core_queue_init);
2687
2688	static int __vb2_init_fileio(struct vb2_queue *q, int read);
2689	static int __vb2_cleanup_fileio(struct vb2_queue *q);
2690	void vb2_core_queue_release(struct vb2_queue *q)
2691	{
2692	__vb2_cleanup_fileio(q);
2693	__vb2_queue_cancel(q);
2694	mutex_lock(&q->mmap_lock);
2695	__vb2_queue_free(q, start: 0, count: q->max_num_buffers);
2696	vb2_core_free_buffers_storage(q);
2697	q->is_busy = 0;
2698	mutex_unlock(lock: &q->mmap_lock);
2699	}
2700	EXPORT_SYMBOL_GPL(vb2_core_queue_release);
2701
2702	__poll_t vb2_core_poll(struct vb2_queue *q, struct file *file,
2703	poll_table *wait)
2704	{
2705	__poll_t req_events = poll_requested_events(p: wait);
2706	struct vb2_buffer *vb = NULL;
2707	unsigned long flags;
2708
2709	/*
2710	* poll_wait() MUST be called on the first invocation on all the
2711	* potential queues of interest, even if we are not interested in their
2712	* events during this first call. Failure to do so will result in
2713	* queue's events to be ignored because the poll_table won't be capable
2714	* of adding new wait queues thereafter.
2715	*/
2716	poll_wait(filp: file, wait_address: &q->done_wq, p: wait);
2717
2718	if (!q->is_output && !(req_events & (EPOLLIN | EPOLLRDNORM)))
2719	return 0;
2720	if (q->is_output && !(req_events & (EPOLLOUT | EPOLLWRNORM)))
2721	return 0;
2722
2723	/*
2724	* Start file I/O emulator only if streaming API has not been used yet.
2725	*/
2726	if (vb2_get_num_buffers(q) == 0 && !vb2_fileio_is_active(q)) {
2727	if (!q->is_output && (q->io_modes & VB2_READ) &&
2728	(req_events & (EPOLLIN | EPOLLRDNORM))) {
2729	if (__vb2_init_fileio(q, read: 1))
2730	return EPOLLERR;
2731	}
2732	if (q->is_output && (q->io_modes & VB2_WRITE) &&
2733	(req_events & (EPOLLOUT | EPOLLWRNORM))) {
2734	if (__vb2_init_fileio(q, read: 0))
2735	return EPOLLERR;
2736	/*
2737	* Write to OUTPUT queue can be done immediately.
2738	*/
2739	return EPOLLOUT | EPOLLWRNORM;
2740	}
2741	}
2742
2743	/*
2744	* There is nothing to wait for if the queue isn't streaming, or if the
2745	* error flag is set.
2746	*/
2747	if (!vb2_is_streaming(q) || q->error)
2748	return EPOLLERR;
2749
2750	/*
2751	* If this quirk is set and QBUF hasn't been called yet then
2752	* return EPOLLERR as well. This only affects capture queues, output
2753	* queues will always initialize waiting_for_buffers to false.
2754	* This quirk is set by V4L2 for backwards compatibility reasons.
2755	*/
2756	if (q->quirk_poll_must_check_waiting_for_buffers &&
2757	q->waiting_for_buffers && (req_events & (EPOLLIN | EPOLLRDNORM)))
2758	return EPOLLERR;
2759
2760	/*
2761	* For output streams you can call write() as long as there are fewer
2762	* buffers queued than there are buffers available.
2763	*/
2764	if (q->is_output && q->fileio && q->queued_count < vb2_get_num_buffers(q))
2765	return EPOLLOUT | EPOLLWRNORM;
2766
2767	if (list_empty(head: &q->done_list)) {
2768	/*
2769	* If the last buffer was dequeued from a capture queue,
2770	* return immediately. DQBUF will return -EPIPE.
2771	*/
2772	if (q->last_buffer_dequeued)
2773	return EPOLLIN | EPOLLRDNORM;
2774	}
2775
2776	/*
2777	* Take first buffer available for dequeuing.
2778	*/
2779	spin_lock_irqsave(&q->done_lock, flags);
2780	if (!list_empty(head: &q->done_list))
2781	vb = list_first_entry(&q->done_list, struct vb2_buffer,
2782	done_entry);
2783	spin_unlock_irqrestore(lock: &q->done_lock, flags);
2784
2785	if (vb && (vb->state == VB2_BUF_STATE_DONE
2786	|| vb->state == VB2_BUF_STATE_ERROR)) {
2787	return (q->is_output) ?
2788	EPOLLOUT | EPOLLWRNORM :
2789	EPOLLIN | EPOLLRDNORM;
2790	}
2791	return 0;
2792	}
2793	EXPORT_SYMBOL_GPL(vb2_core_poll);
2794
2795	/*
2796	* struct vb2_fileio_buf - buffer context used by file io emulator
2797	*
2798	* vb2 provides a compatibility layer and emulator of file io (read and
2799	* write) calls on top of streaming API. This structure is used for
2800	* tracking context related to the buffers.
2801	*/
2802	struct vb2_fileio_buf {
2803	void *vaddr;
2804	unsigned int size;
2805	unsigned int pos;
2806	unsigned int queued:1;
2807	};
2808
2809	/*
2810	* struct vb2_fileio_data - queue context used by file io emulator
2811	*
2812	* @cur_index: the index of the buffer currently being read from or
2813	* written to. If equal to number of buffers in the vb2_queue
2814	* then a new buffer must be dequeued.
2815	* @initial_index: in the read() case all buffers are queued up immediately
2816	* in __vb2_init_fileio() and __vb2_perform_fileio() just cycles
2817	* buffers. However, in the write() case no buffers are initially
2818	* queued, instead whenever a buffer is full it is queued up by
2819	* __vb2_perform_fileio(). Only once all available buffers have
2820	* been queued up will __vb2_perform_fileio() start to dequeue
2821	* buffers. This means that initially __vb2_perform_fileio()
2822	* needs to know what buffer index to use when it is queuing up
2823	* the buffers for the first time. That initial index is stored
2824	* in this field. Once it is equal to number of buffers in the
2825	* vb2_queue all available buffers have been queued and
2826	* __vb2_perform_fileio() should start the normal dequeue/queue cycle.
2827	*
2828	* vb2 provides a compatibility layer and emulator of file io (read and
2829	* write) calls on top of streaming API. For proper operation it required
2830	* this structure to save the driver state between each call of the read
2831	* or write function.
2832	*/
2833	struct vb2_fileio_data {
2834	unsigned int count;
2835	unsigned int type;
2836	unsigned int memory;
2837	struct vb2_fileio_buf bufs[VB2_MAX_FRAME];
2838	unsigned int cur_index;
2839	unsigned int initial_index;
2840	unsigned int q_count;
2841	unsigned int dq_count;
2842	unsigned read_once:1;
2843	unsigned write_immediately:1;
2844	};
2845
2846	/*
2847	* __vb2_init_fileio() - initialize file io emulator
2848	* @q: videobuf2 queue
2849	* @read: mode selector (1 means read, 0 means write)
2850	*/
2851	static int __vb2_init_fileio(struct vb2_queue *q, int read)
2852	{
2853	struct vb2_fileio_data *fileio;
2854	struct vb2_buffer *vb;
2855	int i, ret;
2856
2857	/*
2858	* Sanity check
2859	*/
2860	if (WARN_ON((read && !(q->io_modes & VB2_READ)) ||
2861	(!read && !(q->io_modes & VB2_WRITE))))
2862	return -EINVAL;
2863
2864	/*
2865	* Check if device supports mapping buffers to kernel virtual space.
2866	*/
2867	if (!q->mem_ops->vaddr)
2868	return -EBUSY;
2869
2870	/*
2871	* Check if streaming api has not been already activated.
2872	*/
2873	if (q->streaming || vb2_get_num_buffers(q) > 0)
2874	return -EBUSY;
2875
2876	dprintk(q, 3, "setting up file io: mode %s, count %d, read_once %d, write_immediately %d\n",
2877	(read) ? "read" : "write", q->min_reqbufs_allocation, q->fileio_read_once,
2878	q->fileio_write_immediately);
2879
2880	fileio = kzalloc(sizeof(*fileio), GFP_KERNEL);
2881	if (fileio == NULL)
2882	return -ENOMEM;
2883
2884	fileio->read_once = q->fileio_read_once;
2885	fileio->write_immediately = q->fileio_write_immediately;
2886
2887	/*
2888	* Request buffers and use MMAP type to force driver
2889	* to allocate buffers by itself.
2890	*/
2891	fileio->count = q->min_reqbufs_allocation;
2892	fileio->memory = VB2_MEMORY_MMAP;
2893	fileio->type = q->type;
2894	q->fileio = fileio;
2895	ret = vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
2896	if (ret)
2897	goto err_kfree;
2898	/* vb2_fileio_data supports max VB2_MAX_FRAME buffers */
2899	if (fileio->count > VB2_MAX_FRAME) {
2900	dprintk(q, 1, "fileio: more than VB2_MAX_FRAME buffers requested\n");
2901	ret = -ENOSPC;
2902	goto err_reqbufs;
2903	}
2904
2905	/*
2906	* Userspace can never add or delete buffers later, so there
2907	* will never be holes. It is safe to assume that vb2_get_buffer(q, 0)
2908	* will always return a valid vb pointer
2909	*/
2910	vb = vb2_get_buffer(q, index: 0);
2911
2912	/*
2913	* Check if plane_count is correct
2914	* (multiplane buffers are not supported).
2915	*/
2916	if (vb->num_planes != 1) {
2917	ret = -EBUSY;
2918	goto err_reqbufs;
2919	}
2920
2921	/*
2922	* Get kernel address of each buffer.
2923	*/
2924	for (i = 0; i < vb2_get_num_buffers(q); i++) {
2925	/* vb can never be NULL when using fileio. */
2926	vb = vb2_get_buffer(q, index: i);
2927
2928	fileio->bufs[i].vaddr = vb2_plane_vaddr(vb, 0);
2929	if (fileio->bufs[i].vaddr == NULL) {
2930	ret = -EINVAL;
2931	goto err_reqbufs;
2932	}
2933	fileio->bufs[i].size = vb2_plane_size(vb, plane_no: 0);
2934	}
2935
2936	/*
2937	* Read mode requires pre queuing of all buffers.
2938	*/
2939	if (read) {
2940	/*
2941	* Queue all buffers.
2942	*/
2943	for (i = 0; i < vb2_get_num_buffers(q); i++) {
2944	struct vb2_buffer *vb2 = vb2_get_buffer(q, index: i);
2945
2946	if (!vb2)
2947	continue;
2948
2949	ret = vb2_core_qbuf(q, vb2, NULL, NULL);
2950	if (ret)
2951	goto err_reqbufs;
2952	fileio->bufs[i].queued = 1;
2953	}
2954	/*
2955	* All buffers have been queued, so mark that by setting
2956	* initial_index to the number of buffers in the vb2_queue
2957	*/
2958	fileio->initial_index = vb2_get_num_buffers(q);
2959	fileio->cur_index = fileio->initial_index;
2960	}
2961
2962	/*
2963	* Start streaming.
2964	*/
2965	ret = vb2_core_streamon(q, q->type);
2966	if (ret)
2967	goto err_reqbufs;
2968
2969	return ret;
2970
2971	err_reqbufs:
2972	fileio->count = 0;
2973	vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
2974
2975	err_kfree:
2976	q->fileio = NULL;
2977	kfree(objp: fileio);
2978	return ret;
2979	}
2980
2981	/*
2982	* __vb2_cleanup_fileio() - free resourced used by file io emulator
2983	* @q: videobuf2 queue
2984	*/
2985	static int __vb2_cleanup_fileio(struct vb2_queue *q)
2986	{
2987	struct vb2_fileio_data *fileio = q->fileio;
2988
2989	if (fileio) {
2990	vb2_core_streamoff(q, q->type);
2991	q->fileio = NULL;
2992	fileio->count = 0;
2993	vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
2994	kfree(objp: fileio);
2995	dprintk(q, 3, "file io emulator closed\n");
2996	}
2997	return 0;
2998	}
2999
3000	/*
3001	* __vb2_perform_fileio() - perform a single file io (read or write) operation
3002	* @q: videobuf2 queue
3003	* @data: pointed to target userspace buffer
3004	* @count: number of bytes to read or write
3005	* @ppos: file handle position tracking pointer
3006	* @nonblock: mode selector (1 means blocking calls, 0 means nonblocking)
3007	* @read: access mode selector (1 means read, 0 means write)
3008	*/
3009	static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
3010	loff_t *ppos, int nonblock, int read)
3011	{
3012	struct vb2_fileio_data *fileio;
3013	struct vb2_fileio_buf *buf;
3014	bool is_multiplanar = q->is_multiplanar;
3015	/*
3016	* When using write() to write data to an output video node the vb2 core
3017	* should copy timestamps if V4L2_BUF_FLAG_TIMESTAMP_COPY is set. Nobody
3018	* else is able to provide this information with the write() operation.
3019	*/
3020	bool copy_timestamp = !read && q->copy_timestamp;
3021	unsigned index;
3022	int ret;
3023
3024	dprintk(q, 3, "mode %s, offset %ld, count %zd, %sblocking\n",
3025	read ? "read" : "write", (long)*ppos, count,
3026	nonblock ? "non" : "");
3027
3028	if (!data)
3029	return -EINVAL;
3030
3031	if (q->waiting_in_dqbuf) {
3032	dprintk(q, 3, "another dup()ped fd is %s\n",
3033	read ? "reading" : "writing");
3034	return -EBUSY;
3035	}
3036
3037	/*
3038	* Initialize emulator on first call.
3039	*/
3040	if (!vb2_fileio_is_active(q)) {
3041	ret = __vb2_init_fileio(q, read);
3042	dprintk(q, 3, "vb2_init_fileio result: %d\n", ret);
3043	if (ret)
3044	return ret;
3045	}
3046	fileio = q->fileio;
3047
3048	/*
3049	* Check if we need to dequeue the buffer.
3050	*/
3051	index = fileio->cur_index;
3052	if (index >= vb2_get_num_buffers(q)) {
3053	struct vb2_buffer *b;
3054
3055	/*
3056	* Call vb2_dqbuf to get buffer back.
3057	*/
3058	ret = vb2_core_dqbuf(q, &index, NULL, nonblock);
3059	dprintk(q, 5, "vb2_dqbuf result: %d\n", ret);
3060	if (ret)
3061	return ret;
3062	fileio->dq_count += 1;
3063
3064	fileio->cur_index = index;
3065	buf = &fileio->bufs[index];
3066
3067	/* b can never be NULL when using fileio. */
3068	b = vb2_get_buffer(q, index);
3069
3070	/*
3071	* Get number of bytes filled by the driver
3072	*/
3073	buf->pos = 0;
3074	buf->queued = 0;
3075	buf->size = read ? vb2_get_plane_payload(vb: b, plane_no: 0)
3076	: vb2_plane_size(vb: b, plane_no: 0);
3077	/* Compensate for data_offset on read in the multiplanar case. */
3078	if (is_multiplanar && read &&
3079	b->planes[0].data_offset < buf->size) {
3080	buf->pos = b->planes[0].data_offset;
3081	buf->size -= buf->pos;
3082	}
3083	} else {
3084	buf = &fileio->bufs[index];
3085	}
3086
3087	/*
3088	* Limit count on last few bytes of the buffer.
3089	*/
3090	if (buf->pos + count > buf->size) {
3091	count = buf->size - buf->pos;
3092	dprintk(q, 5, "reducing read count: %zd\n", count);
3093	}
3094
3095	/*
3096	* Transfer data to userspace.
3097	*/
3098	dprintk(q, 3, "copying %zd bytes - buffer %d, offset %u\n",
3099	count, index, buf->pos);
3100	if (read)
3101	ret = copy_to_user(to: data, from: buf->vaddr + buf->pos, n: count);
3102	else
3103	ret = copy_from_user(to: buf->vaddr + buf->pos, from: data, n: count);
3104	if (ret) {
3105	dprintk(q, 3, "error copying data\n");
3106	return -EFAULT;
3107	}
3108
3109	/*
3110	* Update counters.
3111	*/
3112	buf->pos += count;
3113	*ppos += count;
3114
3115	/*
3116	* Queue next buffer if required.
3117	*/
3118	if (buf->pos == buf->size || (!read && fileio->write_immediately)) {
3119	/* b can never be NULL when using fileio. */
3120	struct vb2_buffer *b = vb2_get_buffer(q, index);
3121
3122	/*
3123	* Check if this is the last buffer to read.
3124	*/
3125	if (read && fileio->read_once && fileio->dq_count == 1) {
3126	dprintk(q, 3, "read limit reached\n");
3127	return __vb2_cleanup_fileio(q);
3128	}
3129
3130	/*
3131	* Call vb2_qbuf and give buffer to the driver.
3132	*/
3133	b->planes[0].bytesused = buf->pos;
3134
3135	if (copy_timestamp)
3136	b->timestamp = ktime_get_ns();
3137	ret = vb2_core_qbuf(q, b, NULL, NULL);
3138	dprintk(q, 5, "vb2_qbuf result: %d\n", ret);
3139	if (ret)
3140	return ret;
3141
3142	/*
3143	* Buffer has been queued, update the status
3144	*/
3145	buf->pos = 0;
3146	buf->queued = 1;
3147	buf->size = vb2_plane_size(vb: b, plane_no: 0);
3148	fileio->q_count += 1;
3149	/*
3150	* If we are queuing up buffers for the first time, then
3151	* increase initial_index by one.
3152	*/
3153	if (fileio->initial_index < vb2_get_num_buffers(q))
3154	fileio->initial_index++;
3155	/*
3156	* The next buffer to use is either a buffer that's going to be
3157	* queued for the first time (initial_index < number of buffers in the vb2_queue)
3158	* or it is equal to the number of buffers in the vb2_queue,
3159	* meaning that the next time we need to dequeue a buffer since
3160	* we've now queued up all the 'first time' buffers.
3161	*/
3162	fileio->cur_index = fileio->initial_index;
3163	}
3164
3165	/*
3166	* Return proper number of bytes processed.
3167	*/
3168	if (ret == 0)
3169	ret = count;
3170	return ret;
3171	}
3172
3173	size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
3174	loff_t *ppos, int nonblocking)
3175	{
3176	return __vb2_perform_fileio(q, data, count, ppos, nonblock: nonblocking, read: 1);
3177	}
3178	EXPORT_SYMBOL_GPL(vb2_read);
3179
3180	size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
3181	loff_t *ppos, int nonblocking)
3182	{
3183	return __vb2_perform_fileio(q, data: (char __user *) data, count,
3184	ppos, nonblock: nonblocking, read: 0);
3185	}
3186	EXPORT_SYMBOL_GPL(vb2_write);
3187
3188	struct vb2_threadio_data {
3189	struct task_struct *thread;
3190	vb2_thread_fnc fnc;
3191	void *priv;
3192	bool stop;
3193	};
3194
3195	static int vb2_thread(void *data)
3196	{
3197	struct vb2_queue *q = data;
3198	struct vb2_threadio_data *threadio = q->threadio;
3199	bool copy_timestamp = false;
3200	unsigned prequeue = 0;
3201	unsigned index = 0;
3202	int ret = 0;
3203
3204	if (q->is_output) {
3205	prequeue = vb2_get_num_buffers(q);
3206	copy_timestamp = q->copy_timestamp;
3207	}
3208
3209	set_freezable();
3210
3211	for (;;) {
3212	struct vb2_buffer *vb;
3213
3214	/*
3215	* Call vb2_dqbuf to get buffer back.
3216	*/
3217	if (prequeue) {
3218	vb = vb2_get_buffer(q, index: index++);
3219	if (!vb)
3220	continue;
3221	prequeue--;
3222	} else {
3223	if (!threadio->stop) {
3224	if (q->ops->wait_finish)
3225	call_void_qop(q, wait_finish, q);
3226	else if (q->lock)
3227	mutex_lock(q->lock);
3228	ret = vb2_core_dqbuf(q, &index, NULL, 0);
3229	if (q->ops->wait_prepare)
3230	call_void_qop(q, wait_prepare, q);
3231	else if (q->lock)
3232	mutex_unlock(lock: q->lock);
3233	}
3234	dprintk(q, 5, "file io: vb2_dqbuf result: %d\n", ret);
3235	if (!ret)
3236	vb = vb2_get_buffer(q, index);
3237	}
3238	if (ret || threadio->stop)
3239	break;
3240	try_to_freeze();
3241
3242	if (vb->state != VB2_BUF_STATE_ERROR)
3243	if (threadio->fnc(vb, threadio->priv))
3244	break;
3245	if (copy_timestamp)
3246	vb->timestamp = ktime_get_ns();
3247	if (!threadio->stop) {
3248	if (q->ops->wait_finish)
3249	call_void_qop(q, wait_finish, q);
3250	else if (q->lock)
3251	mutex_lock(q->lock);
3252	ret = vb2_core_qbuf(q, vb, NULL, NULL);
3253	if (q->ops->wait_prepare)
3254	call_void_qop(q, wait_prepare, q);
3255	else if (q->lock)
3256	mutex_unlock(lock: q->lock);
3257	}
3258	if (ret || threadio->stop)
3259	break;
3260	}
3261
3262	/* Hmm, linux becomes *very* unhappy without this ... */
3263	while (!kthread_should_stop()) {
3264	set_current_state(TASK_INTERRUPTIBLE);
3265	schedule();
3266	}
3267	return 0;
3268	}
3269
3270	/*
3271	* This function should not be used for anything else but the videobuf2-dvb
3272	* support. If you think you have another good use-case for this, then please
3273	* contact the linux-media mailinglist first.
3274	*/
3275	int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
3276	const char *thread_name)
3277	{
3278	struct vb2_threadio_data *threadio;
3279	int ret = 0;
3280
3281	if (q->threadio)
3282	return -EBUSY;
3283	if (vb2_is_busy(q))
3284	return -EBUSY;
3285	if (WARN_ON(q->fileio))
3286	return -EBUSY;
3287
3288	threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
3289	if (threadio == NULL)
3290	return -ENOMEM;
3291	threadio->fnc = fnc;
3292	threadio->priv = priv;
3293
3294	ret = __vb2_init_fileio(q, read: !q->is_output);
3295	dprintk(q, 3, "file io: vb2_init_fileio result: %d\n", ret);
3296	if (ret)
3297	goto nomem;
3298	q->threadio = threadio;
3299	threadio->thread = kthread_run(vb2_thread, q, "vb2-%s", thread_name);
3300	if (IS_ERR(ptr: threadio->thread)) {
3301	ret = PTR_ERR(ptr: threadio->thread);
3302	threadio->thread = NULL;
3303	goto nothread;
3304	}
3305	return 0;
3306
3307	nothread:
3308	__vb2_cleanup_fileio(q);
3309	nomem:
3310	kfree(objp: threadio);
3311	return ret;
3312	}
3313	EXPORT_SYMBOL_GPL(vb2_thread_start);
3314
3315	int vb2_thread_stop(struct vb2_queue *q)
3316	{
3317	struct vb2_threadio_data *threadio = q->threadio;
3318	int err;
3319
3320	if (threadio == NULL)
3321	return 0;
3322	threadio->stop = true;
3323	/* Wake up all pending sleeps in the thread */
3324	vb2_queue_error(q);
3325	err = kthread_stop(k: threadio->thread);
3326	__vb2_cleanup_fileio(q);
3327	threadio->thread = NULL;
3328	kfree(objp: threadio);
3329	q->threadio = NULL;
3330	return err;
3331	}
3332	EXPORT_SYMBOL_GPL(vb2_thread_stop);
3333
3334	MODULE_DESCRIPTION("Media buffer core framework");
3335	MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>, Marek Szyprowski");
3336	MODULE_LICENSE("GPL");
3337	MODULE_IMPORT_NS("DMA_BUF");
3338