1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2012 Samsung Electronics Co.Ltd
4	* Authors: Joonyoung Shim <jy0922.shim@samsung.com>
5	*/
6
7	#include <linux/refcount.h>
8	#include <linux/clk.h>
9	#include <linux/component.h>
10	#include <linux/delay.h>
11	#include <linux/dma-mapping.h>
12	#include <linux/err.h>
13	#include <linux/interrupt.h>
14	#include <linux/io.h>
15	#include <linux/kernel.h>
16	#include <linux/of.h>
17	#include <linux/platform_device.h>
18	#include <linux/pm_runtime.h>
19	#include <linux/slab.h>
20	#include <linux/uaccess.h>
21	#include <linux/workqueue.h>
22
23	#include <drm/drm_file.h>
24	#include <drm/exynos_drm.h>
25
26	#include "exynos_drm_drv.h"
27	#include "exynos_drm_g2d.h"
28	#include "exynos_drm_gem.h"
29
30	#define G2D_HW_MAJOR_VER 4
31	#define G2D_HW_MINOR_VER 1
32
33	/* vaild register range set from user: 0x0104 ~ 0x0880 */
34	#define G2D_VALID_START 0x0104
35	#define G2D_VALID_END 0x0880
36
37	/* general registers */
38	#define G2D_SOFT_RESET 0x0000
39	#define G2D_INTEN 0x0004
40	#define G2D_INTC_PEND 0x000C
41	#define G2D_DMA_SFR_BASE_ADDR 0x0080
42	#define G2D_DMA_COMMAND 0x0084
43	#define G2D_DMA_STATUS 0x008C
44	#define G2D_DMA_HOLD_CMD 0x0090
45
46	/* command registers */
47	#define G2D_BITBLT_START 0x0100
48
49	/* registers for base address */
50	#define G2D_SRC_BASE_ADDR 0x0304
51	#define G2D_SRC_STRIDE 0x0308
52	#define G2D_SRC_COLOR_MODE 0x030C
53	#define G2D_SRC_LEFT_TOP 0x0310
54	#define G2D_SRC_RIGHT_BOTTOM 0x0314
55	#define G2D_SRC_PLANE2_BASE_ADDR 0x0318
56	#define G2D_DST_BASE_ADDR 0x0404
57	#define G2D_DST_STRIDE 0x0408
58	#define G2D_DST_COLOR_MODE 0x040C
59	#define G2D_DST_LEFT_TOP 0x0410
60	#define G2D_DST_RIGHT_BOTTOM 0x0414
61	#define G2D_DST_PLANE2_BASE_ADDR 0x0418
62	#define G2D_PAT_BASE_ADDR 0x0500
63	#define G2D_MSK_BASE_ADDR 0x0520
64
65	/* G2D_SOFT_RESET */
66	#define G2D_SFRCLEAR (1 << 1)
67	#define G2D_R (1 << 0)
68
69	/* G2D_INTEN */
70	#define G2D_INTEN_ACF (1 << 3)
71	#define G2D_INTEN_UCF (1 << 2)
72	#define G2D_INTEN_GCF (1 << 1)
73	#define G2D_INTEN_SCF (1 << 0)
74
75	/* G2D_INTC_PEND */
76	#define G2D_INTP_ACMD_FIN (1 << 3)
77	#define G2D_INTP_UCMD_FIN (1 << 2)
78	#define G2D_INTP_GCMD_FIN (1 << 1)
79	#define G2D_INTP_SCMD_FIN (1 << 0)
80
81	/* G2D_DMA_COMMAND */
82	#define G2D_DMA_HALT (1 << 2)
83	#define G2D_DMA_CONTINUE (1 << 1)
84	#define G2D_DMA_START (1 << 0)
85
86	/* G2D_DMA_STATUS */
87	#define G2D_DMA_LIST_DONE_COUNT (0xFF << 17)
88	#define G2D_DMA_BITBLT_DONE_COUNT (0xFFFF << 1)
89	#define G2D_DMA_DONE (1 << 0)
90	#define G2D_DMA_LIST_DONE_COUNT_OFFSET 17
91
92	/* G2D_DMA_HOLD_CMD */
93	#define G2D_USER_HOLD (1 << 2)
94	#define G2D_LIST_HOLD (1 << 1)
95	#define G2D_BITBLT_HOLD (1 << 0)
96
97	/* G2D_BITBLT_START */
98	#define G2D_START_CASESEL (1 << 2)
99	#define G2D_START_NHOLT (1 << 1)
100	#define G2D_START_BITBLT (1 << 0)
101
102	/* buffer color format */
103	#define G2D_FMT_XRGB8888 0
104	#define G2D_FMT_ARGB8888 1
105	#define G2D_FMT_RGB565 2
106	#define G2D_FMT_XRGB1555 3
107	#define G2D_FMT_ARGB1555 4
108	#define G2D_FMT_XRGB4444 5
109	#define G2D_FMT_ARGB4444 6
110	#define G2D_FMT_PACKED_RGB888 7
111	#define G2D_FMT_A8 11
112	#define G2D_FMT_L8 12
113
114	/* buffer valid length */
115	#define G2D_LEN_MIN 1
116	#define G2D_LEN_MAX 8000
117
118	#define G2D_CMDLIST_SIZE (PAGE_SIZE / 4)
119	#define G2D_CMDLIST_NUM 64
120	#define G2D_CMDLIST_POOL_SIZE (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
121	#define G2D_CMDLIST_DATA_NUM (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
122
123	/* maximum buffer pool size of userptr is 64MB as default */
124	#define MAX_POOL (64 * 1024 * 1024)
125
126	enum {
127	BUF_TYPE_GEM = 1,
128	BUF_TYPE_USERPTR,
129	};
130
131	enum g2d_reg_type {
132	REG_TYPE_NONE = -1,
133	REG_TYPE_SRC,
134	REG_TYPE_SRC_PLANE2,
135	REG_TYPE_DST,
136	REG_TYPE_DST_PLANE2,
137	REG_TYPE_PAT,
138	REG_TYPE_MSK,
139	MAX_REG_TYPE_NR
140	};
141
142	enum g2d_flag_bits {
143	/*
144	* If set, suspends the runqueue worker after the currently
145	* processed node is finished.
146	*/
147	G2D_BIT_SUSPEND_RUNQUEUE,
148	/*
149	* If set, indicates that the engine is currently busy.
150	*/
151	G2D_BIT_ENGINE_BUSY,
152	};
153
154	/* cmdlist data structure */
155	struct g2d_cmdlist {
156	u32 head;
157	unsigned long data[G2D_CMDLIST_DATA_NUM];
158	u32 last; /* last data offset */
159	};
160
161	/*
162	* A structure of buffer description
163	*
164	* @format: color format
165	* @stride: buffer stride/pitch in bytes
166	* @left_x: the x coordinates of left top corner
167	* @top_y: the y coordinates of left top corner
168	* @right_x: the x coordinates of right bottom corner
169	* @bottom_y: the y coordinates of right bottom corner
170	*
171	*/
172	struct g2d_buf_desc {
173	unsigned int format;
174	unsigned int stride;
175	unsigned int left_x;
176	unsigned int top_y;
177	unsigned int right_x;
178	unsigned int bottom_y;
179	};
180
181	/*
182	* A structure of buffer information
183	*
184	* @map_nr: manages the number of mapped buffers
185	* @reg_types: stores regitster type in the order of requested command
186	* @handles: stores buffer handle in its reg_type position
187	* @types: stores buffer type in its reg_type position
188	* @descs: stores buffer description in its reg_type position
189	*
190	*/
191	struct g2d_buf_info {
192	unsigned int map_nr;
193	enum g2d_reg_type reg_types[MAX_REG_TYPE_NR];
194	void *obj[MAX_REG_TYPE_NR];
195	unsigned int types[MAX_REG_TYPE_NR];
196	struct g2d_buf_desc descs[MAX_REG_TYPE_NR];
197	};
198
199	struct drm_exynos_pending_g2d_event {
200	struct drm_pending_event base;
201	struct drm_exynos_g2d_event event;
202	};
203
204	struct g2d_cmdlist_userptr {
205	struct list_head list;
206	dma_addr_t dma_addr;
207	unsigned long userptr;
208	unsigned long size;
209	struct page **pages;
210	unsigned int npages;
211	struct sg_table *sgt;
212	refcount_t refcount;
213	bool in_pool;
214	bool out_of_list;
215	};
216	struct g2d_cmdlist_node {
217	struct list_head list;
218	struct g2d_cmdlist *cmdlist;
219	dma_addr_t dma_addr;
220	struct g2d_buf_info buf_info;
221
222	struct drm_exynos_pending_g2d_event *event;
223	};
224
225	struct g2d_runqueue_node {
226	struct list_head list;
227	struct list_head run_cmdlist;
228	struct list_head event_list;
229	struct drm_file *filp;
230	pid_t pid;
231	struct completion complete;
232	int async;
233	};
234
235	struct g2d_data {
236	struct device *dev;
237	void *dma_priv;
238	struct clk *gate_clk;
239	void __iomem *regs;
240	int irq;
241	struct workqueue_struct *g2d_workq;
242	struct work_struct runqueue_work;
243	struct drm_device *drm_dev;
244	unsigned long flags;
245
246	/* cmdlist */
247	struct g2d_cmdlist_node *cmdlist_node;
248	struct list_head free_cmdlist;
249	struct mutex cmdlist_mutex;
250	dma_addr_t cmdlist_pool;
251	void *cmdlist_pool_virt;
252	unsigned long cmdlist_dma_attrs;
253
254	/* runqueue*/
255	struct g2d_runqueue_node *runqueue_node;
256	struct list_head runqueue;
257	struct mutex runqueue_mutex;
258	struct kmem_cache *runqueue_slab;
259
260	unsigned long current_pool;
261	unsigned long max_pool;
262	};
263
264	static inline void g2d_hw_reset(struct g2d_data *g2d)
265	{
266	writel(G2D_R | G2D_SFRCLEAR, addr: g2d->regs + G2D_SOFT_RESET);
267	clear_bit(nr: G2D_BIT_ENGINE_BUSY, addr: &g2d->flags);
268	}
269
270	static int g2d_init_cmdlist(struct g2d_data *g2d)
271	{
272	struct device *dev = g2d->dev;
273	struct g2d_cmdlist_node *node;
274	int nr;
275	int ret;
276	struct g2d_buf_info *buf_info;
277
278	g2d->cmdlist_dma_attrs = DMA_ATTR_WRITE_COMBINE;
279
280	g2d->cmdlist_pool_virt = dma_alloc_attrs(dev: to_dma_dev(dev: g2d->drm_dev),
281	G2D_CMDLIST_POOL_SIZE,
282	dma_handle: &g2d->cmdlist_pool, GFP_KERNEL,
283	attrs: g2d->cmdlist_dma_attrs);
284	if (!g2d->cmdlist_pool_virt) {
285	dev_err(dev, "failed to allocate dma memory\n");
286	return -ENOMEM;
287	}
288
289	node = kcalloc(G2D_CMDLIST_NUM, size: sizeof(*node), GFP_KERNEL);
290	if (!node) {
291	ret = -ENOMEM;
292	goto err;
293	}
294
295	for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
296	unsigned int i;
297
298	node[nr].cmdlist =
299	g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
300	node[nr].dma_addr =
301	g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
302
303	buf_info = &node[nr].buf_info;
304	for (i = 0; i < MAX_REG_TYPE_NR; i++)
305	buf_info->reg_types[i] = REG_TYPE_NONE;
306
307	list_add_tail(new: &node[nr].list, head: &g2d->free_cmdlist);
308	}
309
310	return 0;
311
312	err:
313	dma_free_attrs(dev: to_dma_dev(dev: g2d->drm_dev), G2D_CMDLIST_POOL_SIZE,
314	cpu_addr: g2d->cmdlist_pool_virt,
315	dma_handle: g2d->cmdlist_pool, attrs: g2d->cmdlist_dma_attrs);
316	return ret;
317	}
318
319	static void g2d_fini_cmdlist(struct g2d_data *g2d)
320	{
321	kfree(objp: g2d->cmdlist_node);
322
323	if (g2d->cmdlist_pool_virt && g2d->cmdlist_pool) {
324	dma_free_attrs(dev: to_dma_dev(dev: g2d->drm_dev),
325	G2D_CMDLIST_POOL_SIZE,
326	cpu_addr: g2d->cmdlist_pool_virt,
327	dma_handle: g2d->cmdlist_pool, attrs: g2d->cmdlist_dma_attrs);
328	}
329	}
330
331	static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
332	{
333	struct device *dev = g2d->dev;
334	struct g2d_cmdlist_node *node;
335
336	mutex_lock(&g2d->cmdlist_mutex);
337	if (list_empty(head: &g2d->free_cmdlist)) {
338	dev_err(dev, "there is no free cmdlist\n");
339	mutex_unlock(lock: &g2d->cmdlist_mutex);
340	return NULL;
341	}
342
343	node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
344	list);
345	list_del_init(entry: &node->list);
346	mutex_unlock(lock: &g2d->cmdlist_mutex);
347
348	return node;
349	}
350
351	static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
352	{
353	mutex_lock(&g2d->cmdlist_mutex);
354	list_move_tail(list: &node->list, head: &g2d->free_cmdlist);
355	mutex_unlock(lock: &g2d->cmdlist_mutex);
356	}
357
358	static void g2d_add_cmdlist_to_inuse(struct drm_exynos_file_private *file_priv,
359	struct g2d_cmdlist_node *node)
360	{
361	struct g2d_cmdlist_node *lnode;
362
363	if (list_empty(head: &file_priv->inuse_cmdlist))
364	goto add_to_list;
365
366	/* this links to base address of new cmdlist */
367	lnode = list_entry(file_priv->inuse_cmdlist.prev,
368	struct g2d_cmdlist_node, list);
369	lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
370
371	add_to_list:
372	list_add_tail(new: &node->list, head: &file_priv->inuse_cmdlist);
373
374	if (node->event)
375	list_add_tail(new: &node->event->base.link, head: &file_priv->event_list);
376	}
377
378	static void g2d_userptr_put_dma_addr(struct g2d_data *g2d,
379	void *obj,
380	bool force)
381	{
382	struct g2d_cmdlist_userptr *g2d_userptr = obj;
383
384	if (!obj)
385	return;
386
387	if (force)
388	goto out;
389
390	refcount_dec(r: &g2d_userptr->refcount);
391
392	if (refcount_read(r: &g2d_userptr->refcount) > 0)
393	return;
394
395	if (g2d_userptr->in_pool)
396	return;
397
398	out:
399	dma_unmap_sgtable(dev: to_dma_dev(dev: g2d->drm_dev), sgt: g2d_userptr->sgt,
400	dir: DMA_BIDIRECTIONAL, attrs: 0);
401
402	unpin_user_pages_dirty_lock(pages: g2d_userptr->pages, npages: g2d_userptr->npages,
403	make_dirty: true);
404	kvfree(addr: g2d_userptr->pages);
405
406	if (!g2d_userptr->out_of_list)
407	list_del_init(entry: &g2d_userptr->list);
408
409	sg_free_table(g2d_userptr->sgt);
410	kfree(objp: g2d_userptr->sgt);
411	kfree(objp: g2d_userptr);
412	}
413
414	static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
415	unsigned long userptr,
416	unsigned long size,
417	struct drm_file *filp,
418	void **obj)
419	{
420	struct drm_exynos_file_private *file_priv = filp->driver_priv;
421	struct g2d_cmdlist_userptr *g2d_userptr;
422	struct sg_table *sgt;
423	unsigned long start, end;
424	unsigned int npages, offset;
425	int ret;
426
427	if (!size) {
428	DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
429	return ERR_PTR(error: -EINVAL);
430	}
431
432	/* check if userptr already exists in userptr_list. */
433	list_for_each_entry(g2d_userptr, &file_priv->userptr_list, list) {
434	if (g2d_userptr->userptr == userptr) {
435	/*
436	* also check size because there could be same address
437	* and different size.
438	*/
439	if (g2d_userptr->size == size) {
440	refcount_inc(r: &g2d_userptr->refcount);
441	*obj = g2d_userptr;
442
443	return &g2d_userptr->dma_addr;
444	}
445
446	/*
447	* at this moment, maybe g2d dma is accessing this
448	* g2d_userptr memory region so just remove this
449	* g2d_userptr object from userptr_list not to be
450	* referred again and also except it the userptr
451	* pool to be released after the dma access completion.
452	*/
453	g2d_userptr->out_of_list = true;
454	g2d_userptr->in_pool = false;
455	list_del_init(entry: &g2d_userptr->list);
456
457	break;
458	}
459	}
460
461	g2d_userptr = kzalloc(size: sizeof(*g2d_userptr), GFP_KERNEL);
462	if (!g2d_userptr)
463	return ERR_PTR(error: -ENOMEM);
464
465	refcount_set(r: &g2d_userptr->refcount, n: 1);
466	g2d_userptr->size = size;
467
468	start = userptr & PAGE_MASK;
469	offset = userptr & ~PAGE_MASK;
470	end = PAGE_ALIGN(userptr + size);
471	npages = (end - start) >> PAGE_SHIFT;
472	g2d_userptr->pages = kvmalloc_array(n: npages, size: sizeof(*g2d_userptr->pages),
473	GFP_KERNEL);
474	if (!g2d_userptr->pages) {
475	ret = -ENOMEM;
476	goto err_free;
477	}
478
479	ret = pin_user_pages_fast(start, nr_pages: npages,
480	gup_flags: FOLL_WRITE | FOLL_LONGTERM,
481	pages: g2d_userptr->pages);
482	if (ret != npages) {
483	DRM_DEV_ERROR(g2d->dev,
484	"failed to get user pages from userptr.\n");
485	if (ret < 0)
486	goto err_destroy_pages;
487	npages = ret;
488	ret = -EFAULT;
489	goto err_unpin_pages;
490	}
491	g2d_userptr->npages = npages;
492
493	sgt = kzalloc(size: sizeof(*sgt), GFP_KERNEL);
494	if (!sgt) {
495	ret = -ENOMEM;
496	goto err_unpin_pages;
497	}
498
499	ret = sg_alloc_table_from_pages(sgt,
500	pages: g2d_userptr->pages,
501	n_pages: npages, offset, size, GFP_KERNEL);
502	if (ret < 0) {
503	DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
504	goto err_free_sgt;
505	}
506
507	g2d_userptr->sgt = sgt;
508
509	ret = dma_map_sgtable(dev: to_dma_dev(dev: g2d->drm_dev), sgt,
510	dir: DMA_BIDIRECTIONAL, attrs: 0);
511	if (ret) {
512	DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
513	goto err_sg_free_table;
514	}
515
516	g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
517	g2d_userptr->userptr = userptr;
518
519	list_add_tail(new: &g2d_userptr->list, head: &file_priv->userptr_list);
520
521	if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
522	g2d->current_pool += npages << PAGE_SHIFT;
523	g2d_userptr->in_pool = true;
524	}
525
526	*obj = g2d_userptr;
527
528	return &g2d_userptr->dma_addr;
529
530	err_sg_free_table:
531	sg_free_table(sgt);
532
533	err_free_sgt:
534	kfree(objp: sgt);
535
536	err_unpin_pages:
537	unpin_user_pages(pages: g2d_userptr->pages, npages);
538
539	err_destroy_pages:
540	kvfree(addr: g2d_userptr->pages);
541
542	err_free:
543	kfree(objp: g2d_userptr);
544
545	return ERR_PTR(error: ret);
546	}
547
548	static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
549	{
550	struct drm_exynos_file_private *file_priv = filp->driver_priv;
551	struct g2d_cmdlist_userptr *g2d_userptr, *n;
552
553	list_for_each_entry_safe(g2d_userptr, n, &file_priv->userptr_list, list)
554	if (g2d_userptr->in_pool)
555	g2d_userptr_put_dma_addr(g2d, obj: g2d_userptr, force: true);
556
557	g2d->current_pool = 0;
558	}
559
560	static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
561	{
562	enum g2d_reg_type reg_type;
563
564	switch (reg_offset) {
565	case G2D_SRC_BASE_ADDR:
566	case G2D_SRC_STRIDE:
567	case G2D_SRC_COLOR_MODE:
568	case G2D_SRC_LEFT_TOP:
569	case G2D_SRC_RIGHT_BOTTOM:
570	reg_type = REG_TYPE_SRC;
571	break;
572	case G2D_SRC_PLANE2_BASE_ADDR:
573	reg_type = REG_TYPE_SRC_PLANE2;
574	break;
575	case G2D_DST_BASE_ADDR:
576	case G2D_DST_STRIDE:
577	case G2D_DST_COLOR_MODE:
578	case G2D_DST_LEFT_TOP:
579	case G2D_DST_RIGHT_BOTTOM:
580	reg_type = REG_TYPE_DST;
581	break;
582	case G2D_DST_PLANE2_BASE_ADDR:
583	reg_type = REG_TYPE_DST_PLANE2;
584	break;
585	case G2D_PAT_BASE_ADDR:
586	reg_type = REG_TYPE_PAT;
587	break;
588	case G2D_MSK_BASE_ADDR:
589	reg_type = REG_TYPE_MSK;
590	break;
591	default:
592	reg_type = REG_TYPE_NONE;
593	DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
594	reg_offset);
595	break;
596	}
597
598	return reg_type;
599	}
600
601	static unsigned long g2d_get_buf_bpp(unsigned int format)
602	{
603	unsigned long bpp;
604
605	switch (format) {
606	case G2D_FMT_XRGB8888:
607	case G2D_FMT_ARGB8888:
608	bpp = 4;
609	break;
610	case G2D_FMT_RGB565:
611	case G2D_FMT_XRGB1555:
612	case G2D_FMT_ARGB1555:
613	case G2D_FMT_XRGB4444:
614	case G2D_FMT_ARGB4444:
615	bpp = 2;
616	break;
617	case G2D_FMT_PACKED_RGB888:
618	bpp = 3;
619	break;
620	default:
621	bpp = 1;
622	break;
623	}
624
625	return bpp;
626	}
627
628	static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
629	struct g2d_buf_desc *buf_desc,
630	enum g2d_reg_type reg_type,
631	unsigned long size)
632	{
633	int width, height;
634	unsigned long bpp, last_pos;
635
636	/*
637	* check source and destination buffers only.
638	* so the others are always valid.
639	*/
640	if (reg_type != REG_TYPE_SRC && reg_type != REG_TYPE_DST)
641	return true;
642
643	/* This check also makes sure that right_x > left_x. */
644	width = (int)buf_desc->right_x - (int)buf_desc->left_x;
645	if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
646	DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
647	return false;
648	}
649
650	/* This check also makes sure that bottom_y > top_y. */
651	height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
652	if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
653	DRM_DEV_ERROR(g2d->dev,
654	"height[%d] is out of range!\n", height);
655	return false;
656	}
657
658	bpp = g2d_get_buf_bpp(format: buf_desc->format);
659
660	/* Compute the position of the last byte that the engine accesses. */
661	last_pos = ((unsigned long)buf_desc->bottom_y - 1) *
662	(unsigned long)buf_desc->stride +
663	(unsigned long)buf_desc->right_x * bpp - 1;
664
665	/*
666	* Since right_x > left_x and bottom_y > top_y we already know
667	* that the first_pos < last_pos (first_pos being the position
668	* of the first byte the engine accesses), it just remains to
669	* check if last_pos is smaller then the buffer size.
670	*/
671
672	if (last_pos >= size) {
673	DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
674	"is out of range [%lu]!\n", last_pos, size);
675	return false;
676	}
677
678	return true;
679	}
680
681	static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
682	struct g2d_cmdlist_node *node,
683	struct drm_device *drm_dev,
684	struct drm_file *file)
685	{
686	struct g2d_cmdlist *cmdlist = node->cmdlist;
687	struct g2d_buf_info *buf_info = &node->buf_info;
688	int offset;
689	int ret;
690	int i;
691
692	for (i = 0; i < buf_info->map_nr; i++) {
693	struct g2d_buf_desc *buf_desc;
694	enum g2d_reg_type reg_type;
695	int reg_pos;
696	unsigned long handle;
697	dma_addr_t *addr;
698
699	reg_pos = cmdlist->last - 2 * (i + 1);
700
701	offset = cmdlist->data[reg_pos];
702	handle = cmdlist->data[reg_pos + 1];
703
704	reg_type = g2d_get_reg_type(g2d, reg_offset: offset);
705	if (reg_type == REG_TYPE_NONE) {
706	ret = -EFAULT;
707	goto err;
708	}
709
710	buf_desc = &buf_info->descs[reg_type];
711
712	if (buf_info->types[reg_type] == BUF_TYPE_GEM) {
713	struct exynos_drm_gem *exynos_gem;
714
715	exynos_gem = exynos_drm_gem_get(filp: file, gem_handle: handle);
716	if (!exynos_gem) {
717	ret = -EFAULT;
718	goto err;
719	}
720
721	if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
722	reg_type, size: exynos_gem->size)) {
723	exynos_drm_gem_put(exynos_gem);
724	ret = -EFAULT;
725	goto err;
726	}
727
728	addr = &exynos_gem->dma_addr;
729	buf_info->obj[reg_type] = exynos_gem;
730	} else {
731	struct drm_exynos_g2d_userptr g2d_userptr;
732
733	if (copy_from_user(to: &g2d_userptr, from: (void __user *)handle,
734	n: sizeof(struct drm_exynos_g2d_userptr))) {
735	ret = -EFAULT;
736	goto err;
737	}
738
739	if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
740	reg_type,
741	size: g2d_userptr.size)) {
742	ret = -EFAULT;
743	goto err;
744	}
745
746	addr = g2d_userptr_get_dma_addr(g2d,
747	userptr: g2d_userptr.userptr,
748	size: g2d_userptr.size,
749	filp: file,
750	obj: &buf_info->obj[reg_type]);
751	if (IS_ERR(ptr: addr)) {
752	ret = -EFAULT;
753	goto err;
754	}
755	}
756
757	cmdlist->data[reg_pos + 1] = *addr;
758	buf_info->reg_types[i] = reg_type;
759	}
760
761	return 0;
762
763	err:
764	buf_info->map_nr = i;
765	return ret;
766	}
767
768	static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
769	struct g2d_cmdlist_node *node,
770	struct drm_file *filp)
771	{
772	struct g2d_buf_info *buf_info = &node->buf_info;
773	int i;
774
775	for (i = 0; i < buf_info->map_nr; i++) {
776	struct g2d_buf_desc *buf_desc;
777	enum g2d_reg_type reg_type;
778	void *obj;
779
780	reg_type = buf_info->reg_types[i];
781
782	buf_desc = &buf_info->descs[reg_type];
783	obj = buf_info->obj[reg_type];
784
785	if (buf_info->types[reg_type] == BUF_TYPE_GEM)
786	exynos_drm_gem_put(exynos_gem: obj);
787	else
788	g2d_userptr_put_dma_addr(g2d, obj, force: false);
789
790	buf_info->reg_types[i] = REG_TYPE_NONE;
791	buf_info->obj[reg_type] = NULL;
792	buf_info->types[reg_type] = 0;
793	memset(buf_desc, 0x00, sizeof(*buf_desc));
794	}
795
796	buf_info->map_nr = 0;
797	}
798
799	static void g2d_dma_start(struct g2d_data *g2d,
800	struct g2d_runqueue_node *runqueue_node)
801	{
802	struct g2d_cmdlist_node *node =
803	list_first_entry(&runqueue_node->run_cmdlist,
804	struct g2d_cmdlist_node, list);
805
806	set_bit(nr: G2D_BIT_ENGINE_BUSY, addr: &g2d->flags);
807	writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
808	writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
809	}
810
811	static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
812	{
813	struct g2d_runqueue_node *runqueue_node;
814
815	if (list_empty(head: &g2d->runqueue))
816	return NULL;
817
818	runqueue_node = list_first_entry(&g2d->runqueue,
819	struct g2d_runqueue_node, list);
820	list_del_init(entry: &runqueue_node->list);
821	return runqueue_node;
822	}
823
824	static void g2d_free_runqueue_node(struct g2d_data *g2d,
825	struct g2d_runqueue_node *runqueue_node)
826	{
827	struct g2d_cmdlist_node *node;
828
829	mutex_lock(&g2d->cmdlist_mutex);
830	/*
831	* commands in run_cmdlist have been completed so unmap all gem
832	* objects in each command node so that they are unreferenced.
833	*/
834	list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
835	g2d_unmap_cmdlist_gem(g2d, node, filp: runqueue_node->filp);
836	list_splice_tail_init(list: &runqueue_node->run_cmdlist, head: &g2d->free_cmdlist);
837	mutex_unlock(lock: &g2d->cmdlist_mutex);
838
839	kmem_cache_free(s: g2d->runqueue_slab, objp: runqueue_node);
840	}
841
842	/**
843	* g2d_remove_runqueue_nodes - remove items from the list of runqueue nodes
844	* @g2d: G2D state object
845	* @file: if not zero, only remove items with this DRM file
846	*
847	* Has to be called under runqueue lock.
848	*/
849	static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
850	{
851	struct g2d_runqueue_node *node, *n;
852
853	if (list_empty(head: &g2d->runqueue))
854	return;
855
856	list_for_each_entry_safe(node, n, &g2d->runqueue, list) {
857	if (file && node->filp != file)
858	continue;
859
860	list_del_init(entry: &node->list);
861	g2d_free_runqueue_node(g2d, runqueue_node: node);
862	}
863	}
864
865	static void g2d_runqueue_worker(struct work_struct *work)
866	{
867	struct g2d_data *g2d = container_of(work, struct g2d_data,
868	runqueue_work);
869	struct g2d_runqueue_node *runqueue_node;
870
871	/*
872	* The engine is busy and the completion of the current node is going
873	* to poke the runqueue worker, so nothing to do here.
874	*/
875	if (test_bit(G2D_BIT_ENGINE_BUSY, &g2d->flags))
876	return;
877
878	mutex_lock(&g2d->runqueue_mutex);
879
880	runqueue_node = g2d->runqueue_node;
881	g2d->runqueue_node = NULL;
882
883	if (runqueue_node) {
884	pm_runtime_mark_last_busy(dev: g2d->dev);
885	pm_runtime_put_autosuspend(dev: g2d->dev);
886
887	complete(&runqueue_node->complete);
888	if (runqueue_node->async)
889	g2d_free_runqueue_node(g2d, runqueue_node);
890	}
891
892	if (!test_bit(G2D_BIT_SUSPEND_RUNQUEUE, &g2d->flags)) {
893	g2d->runqueue_node = g2d_get_runqueue_node(g2d);
894
895	if (g2d->runqueue_node) {
896	int ret;
897
898	ret = pm_runtime_resume_and_get(dev: g2d->dev);
899	if (ret < 0) {
900	dev_err(g2d->dev, "failed to enable G2D device.\n");
901	goto out;
902	}
903
904	g2d_dma_start(g2d, runqueue_node: g2d->runqueue_node);
905	}
906	}
907
908	out:
909	mutex_unlock(lock: &g2d->runqueue_mutex);
910	}
911
912	static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
913	{
914	struct drm_device *drm_dev = g2d->drm_dev;
915	struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
916	struct drm_exynos_pending_g2d_event *e;
917	struct timespec64 now;
918
919	if (list_empty(head: &runqueue_node->event_list))
920	return;
921
922	e = list_first_entry(&runqueue_node->event_list,
923	struct drm_exynos_pending_g2d_event, base.link);
924
925	ktime_get_ts64(ts: &now);
926	e->event.tv_sec = now.tv_sec;
927	e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
928	e->event.cmdlist_no = cmdlist_no;
929
930	drm_send_event(dev: drm_dev, e: &e->base);
931	}
932
933	static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
934	{
935	struct g2d_data *g2d = dev_id;
936	u32 pending;
937
938	pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
939	if (pending)
940	writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
941
942	if (pending & G2D_INTP_GCMD_FIN) {
943	u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
944
945	cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
946	G2D_DMA_LIST_DONE_COUNT_OFFSET;
947
948	g2d_finish_event(g2d, cmdlist_no);
949
950	writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
951	if (!(pending & G2D_INTP_ACMD_FIN)) {
952	writel_relaxed(G2D_DMA_CONTINUE,
953	g2d->regs + G2D_DMA_COMMAND);
954	}
955	}
956
957	if (pending & G2D_INTP_ACMD_FIN) {
958	clear_bit(nr: G2D_BIT_ENGINE_BUSY, addr: &g2d->flags);
959	queue_work(wq: g2d->g2d_workq, work: &g2d->runqueue_work);
960	}
961
962	return IRQ_HANDLED;
963	}
964
965	/**
966	* g2d_wait_finish - wait for the G2D engine to finish the current runqueue node
967	* @g2d: G2D state object
968	* @file: if not zero, only wait if the current runqueue node belongs
969	* to the DRM file
970	*
971	* Should the engine not become idle after a 100ms timeout, a hardware
972	* reset is issued.
973	*/
974	static void g2d_wait_finish(struct g2d_data *g2d, struct drm_file *file)
975	{
976	struct device *dev = g2d->dev;
977
978	struct g2d_runqueue_node *runqueue_node = NULL;
979	unsigned int tries = 10;
980
981	mutex_lock(&g2d->runqueue_mutex);
982
983	/* If no node is currently processed, we have nothing to do. */
984	if (!g2d->runqueue_node)
985	goto out;
986
987	runqueue_node = g2d->runqueue_node;
988
989	/* Check if the currently processed item belongs to us. */
990	if (file && runqueue_node->filp != file)
991	goto out;
992
993	mutex_unlock(lock: &g2d->runqueue_mutex);
994
995	/* Wait for the G2D engine to finish. */
996	while (tries-- && (g2d->runqueue_node == runqueue_node))
997	mdelay(10);
998
999	mutex_lock(&g2d->runqueue_mutex);
1000
1001	if (g2d->runqueue_node != runqueue_node)
1002	goto out;
1003
1004	dev_err(dev, "wait timed out, resetting engine...\n");
1005	g2d_hw_reset(g2d);
1006
1007	/*
1008	* After the hardware reset of the engine we are going to loose
1009	* the IRQ which triggers the PM runtime put().
1010	* So do this manually here.
1011	*/
1012	pm_runtime_mark_last_busy(dev);
1013	pm_runtime_put_autosuspend(dev);
1014
1015	complete(&runqueue_node->complete);
1016	if (runqueue_node->async)
1017	g2d_free_runqueue_node(g2d, runqueue_node);
1018
1019	out:
1020	mutex_unlock(lock: &g2d->runqueue_mutex);
1021	}
1022
1023	static int g2d_check_reg_offset(struct g2d_data *g2d,
1024	struct g2d_cmdlist_node *node,
1025	int nr, bool for_addr)
1026	{
1027	struct g2d_cmdlist *cmdlist = node->cmdlist;
1028	int reg_offset;
1029	int index;
1030	int i;
1031
1032	for (i = 0; i < nr; i++) {
1033	struct g2d_buf_info *buf_info = &node->buf_info;
1034	struct g2d_buf_desc *buf_desc;
1035	enum g2d_reg_type reg_type;
1036	unsigned long value;
1037
1038	index = cmdlist->last - 2 * (i + 1);
1039
1040	reg_offset = cmdlist->data[index] & ~0xfffff000;
1041	if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
1042	goto err;
1043	if (reg_offset % 4)
1044	goto err;
1045
1046	switch (reg_offset) {
1047	case G2D_SRC_BASE_ADDR:
1048	case G2D_SRC_PLANE2_BASE_ADDR:
1049	case G2D_DST_BASE_ADDR:
1050	case G2D_DST_PLANE2_BASE_ADDR:
1051	case G2D_PAT_BASE_ADDR:
1052	case G2D_MSK_BASE_ADDR:
1053	if (!for_addr)
1054	goto err;
1055
1056	reg_type = g2d_get_reg_type(g2d, reg_offset);
1057
1058	/* check userptr buffer type. */
1059	if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
1060	buf_info->types[reg_type] = BUF_TYPE_USERPTR;
1061	cmdlist->data[index] &= ~G2D_BUF_USERPTR;
1062	} else
1063	buf_info->types[reg_type] = BUF_TYPE_GEM;
1064	break;
1065	case G2D_SRC_STRIDE:
1066	case G2D_DST_STRIDE:
1067	if (for_addr)
1068	goto err;
1069
1070	reg_type = g2d_get_reg_type(g2d, reg_offset);
1071
1072	buf_desc = &buf_info->descs[reg_type];
1073	buf_desc->stride = cmdlist->data[index + 1];
1074	break;
1075	case G2D_SRC_COLOR_MODE:
1076	case G2D_DST_COLOR_MODE:
1077	if (for_addr)
1078	goto err;
1079
1080	reg_type = g2d_get_reg_type(g2d, reg_offset);
1081
1082	buf_desc = &buf_info->descs[reg_type];
1083	value = cmdlist->data[index + 1];
1084
1085	buf_desc->format = value & 0xf;
1086	break;
1087	case G2D_SRC_LEFT_TOP:
1088	case G2D_DST_LEFT_TOP:
1089	if (for_addr)
1090	goto err;
1091
1092	reg_type = g2d_get_reg_type(g2d, reg_offset);
1093
1094	buf_desc = &buf_info->descs[reg_type];
1095	value = cmdlist->data[index + 1];
1096
1097	buf_desc->left_x = value & 0x1fff;
1098	buf_desc->top_y = (value & 0x1fff0000) >> 16;
1099	break;
1100	case G2D_SRC_RIGHT_BOTTOM:
1101	case G2D_DST_RIGHT_BOTTOM:
1102	if (for_addr)
1103	goto err;
1104
1105	reg_type = g2d_get_reg_type(g2d, reg_offset);
1106
1107	buf_desc = &buf_info->descs[reg_type];
1108	value = cmdlist->data[index + 1];
1109
1110	buf_desc->right_x = value & 0x1fff;
1111	buf_desc->bottom_y = (value & 0x1fff0000) >> 16;
1112	break;
1113	default:
1114	if (for_addr)
1115	goto err;
1116	break;
1117	}
1118	}
1119
1120	return 0;
1121
1122	err:
1123	dev_err(g2d->dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
1124	return -EINVAL;
1125	}
1126
1127	/* ioctl functions */
1128	int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
1129	struct drm_file *file)
1130	{
1131	struct drm_exynos_g2d_get_ver *ver = data;
1132
1133	ver->major = G2D_HW_MAJOR_VER;
1134	ver->minor = G2D_HW_MINOR_VER;
1135
1136	return 0;
1137	}
1138
1139	int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
1140	struct drm_file *file)
1141	{
1142	struct drm_exynos_file_private *file_priv = file->driver_priv;
1143	struct exynos_drm_private *priv = drm_dev->dev_private;
1144	struct g2d_data *g2d = dev_get_drvdata(dev: priv->g2d_dev);
1145	struct drm_exynos_g2d_set_cmdlist *req = data;
1146	struct drm_exynos_g2d_cmd *cmd;
1147	struct drm_exynos_pending_g2d_event *e;
1148	struct g2d_cmdlist_node *node;
1149	struct g2d_cmdlist *cmdlist;
1150	int size;
1151	int ret;
1152
1153	node = g2d_get_cmdlist(g2d);
1154	if (!node)
1155	return -ENOMEM;
1156
1157	/*
1158	* To avoid an integer overflow for the later size computations, we
1159	* enforce a maximum number of submitted commands here. This limit is
1160	* sufficient for all conceivable usage cases of the G2D.
1161	*/
1162	if (req->cmd_nr > G2D_CMDLIST_DATA_NUM ||
1163	req->cmd_buf_nr > G2D_CMDLIST_DATA_NUM) {
1164	dev_err(g2d->dev, "number of submitted G2D commands exceeds limit\n");
1165	return -EINVAL;
1166	}
1167
1168	node->event = NULL;
1169
1170	if (req->event_type != G2D_EVENT_NOT) {
1171	e = kzalloc(size: sizeof(*node->event), GFP_KERNEL);
1172	if (!e) {
1173	ret = -ENOMEM;
1174	goto err;
1175	}
1176
1177	e->event.base.type = DRM_EXYNOS_G2D_EVENT;
1178	e->event.base.length = sizeof(e->event);
1179	e->event.user_data = req->user_data;
1180
1181	ret = drm_event_reserve_init(dev: drm_dev, file_priv: file, p: &e->base, e: &e->event.base);
1182	if (ret) {
1183	kfree(objp: e);
1184	goto err;
1185	}
1186
1187	node->event = e;
1188	}
1189
1190	cmdlist = node->cmdlist;
1191
1192	cmdlist->last = 0;
1193
1194	/*
1195	* If don't clear SFR registers, the cmdlist is affected by register
1196	* values of previous cmdlist. G2D hw executes SFR clear command and
1197	* a next command at the same time then the next command is ignored and
1198	* is executed rightly from next next command, so needs a dummy command
1199	* to next command of SFR clear command.
1200	*/
1201	cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
1202	cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
1203	cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
1204	cmdlist->data[cmdlist->last++] = 0;
1205
1206	/*
1207	* 'LIST_HOLD' command should be set to the DMA_HOLD_CMD_REG
1208	* and GCF bit should be set to INTEN register if user wants
1209	* G2D interrupt event once current command list execution is
1210	* finished.
1211	* Otherwise only ACF bit should be set to INTEN register so
1212	* that one interrupt is occurred after all command lists
1213	* have been completed.
1214	*/
1215	if (node->event) {
1216	cmdlist->data[cmdlist->last++] = G2D_INTEN;
1217	cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF | G2D_INTEN_GCF;
1218	cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
1219	cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
1220	} else {
1221	cmdlist->data[cmdlist->last++] = G2D_INTEN;
1222	cmdlist->data[cmdlist->last++] = G2D_INTEN_ACF;
1223	}
1224
1225	/*
1226	* Check the size of cmdlist. The 2 that is added last comes from
1227	* the implicit G2D_BITBLT_START that is appended once we have
1228	* checked all the submitted commands.
1229	*/
1230	size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
1231	if (size > G2D_CMDLIST_DATA_NUM) {
1232	dev_err(g2d->dev, "cmdlist size is too big\n");
1233	ret = -EINVAL;
1234	goto err_free_event;
1235	}
1236
1237	cmd = (struct drm_exynos_g2d_cmd *)(unsigned long)req->cmd;
1238
1239	if (copy_from_user(to: cmdlist->data + cmdlist->last,
1240	from: (void __user *)cmd,
1241	n: sizeof(*cmd) * req->cmd_nr)) {
1242	ret = -EFAULT;
1243	goto err_free_event;
1244	}
1245	cmdlist->last += req->cmd_nr * 2;
1246
1247	ret = g2d_check_reg_offset(g2d, node, nr: req->cmd_nr, for_addr: false);
1248	if (ret < 0)
1249	goto err_free_event;
1250
1251	node->buf_info.map_nr = req->cmd_buf_nr;
1252	if (req->cmd_buf_nr) {
1253	struct drm_exynos_g2d_cmd *cmd_buf;
1254
1255	cmd_buf = (struct drm_exynos_g2d_cmd *)
1256	(unsigned long)req->cmd_buf;
1257
1258	if (copy_from_user(to: cmdlist->data + cmdlist->last,
1259	from: (void __user *)cmd_buf,
1260	n: sizeof(*cmd_buf) * req->cmd_buf_nr)) {
1261	ret = -EFAULT;
1262	goto err_free_event;
1263	}
1264	cmdlist->last += req->cmd_buf_nr * 2;
1265
1266	ret = g2d_check_reg_offset(g2d, node, nr: req->cmd_buf_nr, for_addr: true);
1267	if (ret < 0)
1268	goto err_free_event;
1269
1270	ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
1271	if (ret < 0)
1272	goto err_unmap;
1273	}
1274
1275	cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
1276	cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
1277
1278	/* head */
1279	cmdlist->head = cmdlist->last / 2;
1280
1281	/* tail */
1282	cmdlist->data[cmdlist->last] = 0;
1283
1284	g2d_add_cmdlist_to_inuse(file_priv, node);
1285
1286	return 0;
1287
1288	err_unmap:
1289	g2d_unmap_cmdlist_gem(g2d, node, filp: file);
1290	err_free_event:
1291	if (node->event)
1292	drm_event_cancel_free(dev: drm_dev, p: &node->event->base);
1293	err:
1294	g2d_put_cmdlist(g2d, node);
1295	return ret;
1296	}
1297
1298	int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
1299	struct drm_file *file)
1300	{
1301	struct drm_exynos_file_private *file_priv = file->driver_priv;
1302	struct exynos_drm_private *priv = drm_dev->dev_private;
1303	struct g2d_data *g2d = dev_get_drvdata(dev: priv->g2d_dev);
1304	struct drm_exynos_g2d_exec *req = data;
1305	struct g2d_runqueue_node *runqueue_node;
1306	struct list_head *run_cmdlist;
1307	struct list_head *event_list;
1308
1309	runqueue_node = kmem_cache_alloc(cachep: g2d->runqueue_slab, GFP_KERNEL);
1310	if (!runqueue_node)
1311	return -ENOMEM;
1312
1313	run_cmdlist = &runqueue_node->run_cmdlist;
1314	event_list = &runqueue_node->event_list;
1315	INIT_LIST_HEAD(list: run_cmdlist);
1316	INIT_LIST_HEAD(list: event_list);
1317	init_completion(x: &runqueue_node->complete);
1318	runqueue_node->async = req->async;
1319
1320	list_splice_init(list: &file_priv->inuse_cmdlist, head: run_cmdlist);
1321	list_splice_init(list: &file_priv->event_list, head: event_list);
1322
1323	if (list_empty(head: run_cmdlist)) {
1324	dev_err(g2d->dev, "there is no inuse cmdlist\n");
1325	kmem_cache_free(s: g2d->runqueue_slab, objp: runqueue_node);
1326	return -EPERM;
1327	}
1328
1329	mutex_lock(&g2d->runqueue_mutex);
1330	runqueue_node->pid = current->pid;
1331	runqueue_node->filp = file;
1332	list_add_tail(new: &runqueue_node->list, head: &g2d->runqueue);
1333	mutex_unlock(lock: &g2d->runqueue_mutex);
1334
1335	/* Let the runqueue know that there is work to do. */
1336	queue_work(wq: g2d->g2d_workq, work: &g2d->runqueue_work);
1337
1338	if (req->async)
1339	goto out;
1340
1341	wait_for_completion(&runqueue_node->complete);
1342	g2d_free_runqueue_node(g2d, runqueue_node);
1343
1344	out:
1345	return 0;
1346	}
1347
1348	int g2d_open(struct drm_device *drm_dev, struct drm_file *file)
1349	{
1350	struct drm_exynos_file_private *file_priv = file->driver_priv;
1351
1352	INIT_LIST_HEAD(list: &file_priv->inuse_cmdlist);
1353	INIT_LIST_HEAD(list: &file_priv->event_list);
1354	INIT_LIST_HEAD(list: &file_priv->userptr_list);
1355
1356	return 0;
1357	}
1358
1359	void g2d_close(struct drm_device *drm_dev, struct drm_file *file)
1360	{
1361	struct drm_exynos_file_private *file_priv = file->driver_priv;
1362	struct exynos_drm_private *priv = drm_dev->dev_private;
1363	struct g2d_data *g2d;
1364	struct g2d_cmdlist_node *node, *n;
1365
1366	if (!priv->g2d_dev)
1367	return;
1368
1369	g2d = dev_get_drvdata(dev: priv->g2d_dev);
1370
1371	/* Remove the runqueue nodes that belong to us. */
1372	mutex_lock(&g2d->runqueue_mutex);
1373	g2d_remove_runqueue_nodes(g2d, file);
1374	mutex_unlock(lock: &g2d->runqueue_mutex);
1375
1376	/*
1377	* Wait for the runqueue worker to finish its current node.
1378	* After this the engine should no longer be accessing any
1379	* memory belonging to us.
1380	*/
1381	g2d_wait_finish(g2d, file);
1382
1383	/*
1384	* Even after the engine is idle, there might still be stale cmdlists
1385	* (i.e. cmdlisst which we submitted but never executed) around, with
1386	* their corresponding GEM/userptr buffers.
1387	* Properly unmap these buffers here.
1388	*/
1389	mutex_lock(&g2d->cmdlist_mutex);
1390	list_for_each_entry_safe(node, n, &file_priv->inuse_cmdlist, list) {
1391	g2d_unmap_cmdlist_gem(g2d, node, filp: file);
1392	list_move_tail(list: &node->list, head: &g2d->free_cmdlist);
1393	}
1394	mutex_unlock(lock: &g2d->cmdlist_mutex);
1395
1396	/* release all g2d_userptr in pool. */
1397	g2d_userptr_free_all(g2d, filp: file);
1398	}
1399
1400	static int g2d_bind(struct device *dev, struct device *master, void *data)
1401	{
1402	struct g2d_data *g2d = dev_get_drvdata(dev);
1403	struct drm_device *drm_dev = data;
1404	struct exynos_drm_private *priv = drm_dev->dev_private;
1405	int ret;
1406
1407	g2d->drm_dev = drm_dev;
1408
1409	/* allocate dma-aware cmdlist buffer. */
1410	ret = g2d_init_cmdlist(g2d);
1411	if (ret < 0) {
1412	dev_err(dev, "cmdlist init failed\n");
1413	return ret;
1414	}
1415
1416	ret = exynos_drm_register_dma(drm: drm_dev, dev, dma_priv: &g2d->dma_priv);
1417	if (ret < 0) {
1418	dev_err(dev, "failed to enable iommu.\n");
1419	g2d_fini_cmdlist(g2d);
1420	return ret;
1421	}
1422	priv->g2d_dev = dev;
1423
1424	dev_info(dev, "The Exynos G2D (ver %d.%d) successfully registered.\n",
1425	G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1426	return 0;
1427	}
1428
1429	static void g2d_unbind(struct device *dev, struct device *master, void *data)
1430	{
1431	struct g2d_data *g2d = dev_get_drvdata(dev);
1432	struct drm_device *drm_dev = data;
1433	struct exynos_drm_private *priv = drm_dev->dev_private;
1434
1435	/* Suspend operation and wait for engine idle. */
1436	set_bit(nr: G2D_BIT_SUSPEND_RUNQUEUE, addr: &g2d->flags);
1437	g2d_wait_finish(g2d, NULL);
1438	priv->g2d_dev = NULL;
1439
1440	cancel_work_sync(work: &g2d->runqueue_work);
1441	exynos_drm_unregister_dma(drm: g2d->drm_dev, dev, dma_priv: &g2d->dma_priv);
1442	}
1443
1444	static const struct component_ops g2d_component_ops = {
1445	.bind = g2d_bind,
1446	.unbind = g2d_unbind,
1447	};
1448
1449	static int g2d_probe(struct platform_device *pdev)
1450	{
1451	struct device *dev = &pdev->dev;
1452	struct g2d_data *g2d;
1453	int ret;
1454
1455	g2d = devm_kzalloc(dev, size: sizeof(*g2d), GFP_KERNEL);
1456	if (!g2d)
1457	return -ENOMEM;
1458
1459	g2d->runqueue_slab = kmem_cache_create(name: "g2d_runqueue_slab",
1460	size: sizeof(struct g2d_runqueue_node), align: 0, flags: 0, NULL);
1461	if (!g2d->runqueue_slab)
1462	return -ENOMEM;
1463
1464	g2d->dev = dev;
1465
1466	g2d->g2d_workq = create_singlethread_workqueue("g2d");
1467	if (!g2d->g2d_workq) {
1468	dev_err(dev, "failed to create workqueue\n");
1469	ret = -EINVAL;
1470	goto err_destroy_slab;
1471	}
1472
1473	INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1474	INIT_LIST_HEAD(list: &g2d->free_cmdlist);
1475	INIT_LIST_HEAD(list: &g2d->runqueue);
1476
1477	mutex_init(&g2d->cmdlist_mutex);
1478	mutex_init(&g2d->runqueue_mutex);
1479
1480	g2d->gate_clk = devm_clk_get(dev, id: "fimg2d");
1481	if (IS_ERR(ptr: g2d->gate_clk)) {
1482	dev_err(dev, "failed to get gate clock\n");
1483	ret = PTR_ERR(ptr: g2d->gate_clk);
1484	goto err_destroy_workqueue;
1485	}
1486
1487	pm_runtime_use_autosuspend(dev);
1488	pm_runtime_set_autosuspend_delay(dev, delay: 2000);
1489	pm_runtime_enable(dev);
1490	clear_bit(nr: G2D_BIT_SUSPEND_RUNQUEUE, addr: &g2d->flags);
1491	clear_bit(nr: G2D_BIT_ENGINE_BUSY, addr: &g2d->flags);
1492
1493	g2d->regs = devm_platform_ioremap_resource(pdev, index: 0);
1494	if (IS_ERR(ptr: g2d->regs)) {
1495	ret = PTR_ERR(ptr: g2d->regs);
1496	goto err_put_clk;
1497	}
1498
1499	g2d->irq = platform_get_irq(pdev, 0);
1500	if (g2d->irq < 0) {
1501	ret = g2d->irq;
1502	goto err_put_clk;
1503	}
1504
1505	ret = devm_request_irq(dev, irq: g2d->irq, handler: g2d_irq_handler, irqflags: 0,
1506	devname: "drm_g2d", dev_id: g2d);
1507	if (ret < 0) {
1508	dev_err(dev, "irq request failed\n");
1509	goto err_put_clk;
1510	}
1511
1512	g2d->max_pool = MAX_POOL;
1513
1514	platform_set_drvdata(pdev, data: g2d);
1515
1516	ret = component_add(dev, &g2d_component_ops);
1517	if (ret < 0) {
1518	dev_err(dev, "failed to register drm g2d device\n");
1519	goto err_put_clk;
1520	}
1521
1522	return 0;
1523
1524	err_put_clk:
1525	pm_runtime_disable(dev);
1526	err_destroy_workqueue:
1527	destroy_workqueue(wq: g2d->g2d_workq);
1528	err_destroy_slab:
1529	kmem_cache_destroy(s: g2d->runqueue_slab);
1530	return ret;
1531	}
1532
1533	static void g2d_remove(struct platform_device *pdev)
1534	{
1535	struct g2d_data *g2d = platform_get_drvdata(pdev);
1536
1537	component_del(&pdev->dev, &g2d_component_ops);
1538
1539	/* There should be no locking needed here. */
1540	g2d_remove_runqueue_nodes(g2d, NULL);
1541
1542	pm_runtime_dont_use_autosuspend(dev: &pdev->dev);
1543	pm_runtime_disable(dev: &pdev->dev);
1544
1545	g2d_fini_cmdlist(g2d);
1546	destroy_workqueue(wq: g2d->g2d_workq);
1547	kmem_cache_destroy(s: g2d->runqueue_slab);
1548	}
1549
1550	static int g2d_suspend(struct device *dev)
1551	{
1552	struct g2d_data *g2d = dev_get_drvdata(dev);
1553
1554	/*
1555	* Suspend the runqueue worker operation and wait until the G2D
1556	* engine is idle.
1557	*/
1558	set_bit(nr: G2D_BIT_SUSPEND_RUNQUEUE, addr: &g2d->flags);
1559	g2d_wait_finish(g2d, NULL);
1560	flush_work(work: &g2d->runqueue_work);
1561
1562	return 0;
1563	}
1564
1565	static int g2d_resume(struct device *dev)
1566	{
1567	struct g2d_data *g2d = dev_get_drvdata(dev);
1568
1569	clear_bit(nr: G2D_BIT_SUSPEND_RUNQUEUE, addr: &g2d->flags);
1570	queue_work(wq: g2d->g2d_workq, work: &g2d->runqueue_work);
1571
1572	return 0;
1573	}
1574
1575	static int g2d_runtime_suspend(struct device *dev)
1576	{
1577	struct g2d_data *g2d = dev_get_drvdata(dev);
1578
1579	clk_disable_unprepare(clk: g2d->gate_clk);
1580
1581	return 0;
1582	}
1583
1584	static int g2d_runtime_resume(struct device *dev)
1585	{
1586	struct g2d_data *g2d = dev_get_drvdata(dev);
1587	int ret;
1588
1589	ret = clk_prepare_enable(clk: g2d->gate_clk);
1590	if (ret < 0)
1591	dev_warn(dev, "failed to enable clock.\n");
1592
1593	return ret;
1594	}
1595
1596	static const struct dev_pm_ops g2d_pm_ops = {
1597	SYSTEM_SLEEP_PM_OPS(g2d_suspend, g2d_resume)
1598	RUNTIME_PM_OPS(g2d_runtime_suspend, g2d_runtime_resume, NULL)
1599	};
1600
1601	static const struct of_device_id exynos_g2d_match[] = {
1602	{ .compatible = "samsung,exynos5250-g2d" },
1603	{ .compatible = "samsung,exynos4212-g2d" },
1604	{},
1605	};
1606	MODULE_DEVICE_TABLE(of, exynos_g2d_match);
1607
1608	struct platform_driver g2d_driver = {
1609	.probe = g2d_probe,
1610	.remove_new = g2d_remove,
1611	.driver = {
1612	.name = "exynos-drm-g2d",
1613	.owner = THIS_MODULE,
1614	.pm = pm_ptr(&g2d_pm_ops),
1615	.of_match_table = exynos_g2d_match,
1616	},
1617	};
1618