1	/*
2	* Copyright © 2016 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*
23	*/
24
25	#include <linux/sched/mm.h>
26	#include <linux/dma-fence-array.h>
27	#include <drm/drm_gem.h>
28
29	#include "display/intel_display.h"
30	#include "display/intel_frontbuffer.h"
31	#include "gem/i915_gem_lmem.h"
32	#include "gem/i915_gem_object_frontbuffer.h"
33	#include "gem/i915_gem_tiling.h"
34	#include "gt/intel_engine.h"
35	#include "gt/intel_engine_heartbeat.h"
36	#include "gt/intel_gt.h"
37	#include "gt/intel_gt_requests.h"
38	#include "gt/intel_tlb.h"
39
40	#include "i915_drv.h"
41	#include "i915_gem_evict.h"
42	#include "i915_sw_fence_work.h"
43	#include "i915_trace.h"
44	#include "i915_vma.h"
45	#include "i915_vma_resource.h"
46
47	static inline void assert_vma_held_evict(const struct i915_vma *vma)
48	{
49	/*
50	* We may be forced to unbind when the vm is dead, to clean it up.
51	* This is the only exception to the requirement of the object lock
52	* being held.
53	*/
54	if (kref_read(kref: &vma->vm->ref))
55	assert_object_held_shared(obj: vma->obj);
56	}
57
58	static struct kmem_cache *slab_vmas;
59
60	static struct i915_vma *i915_vma_alloc(void)
61	{
62	return kmem_cache_zalloc(k: slab_vmas, GFP_KERNEL);
63	}
64
65	static void i915_vma_free(struct i915_vma *vma)
66	{
67	return kmem_cache_free(s: slab_vmas, objp: vma);
68	}
69
70	#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
71
72	#include <linux/stackdepot.h>
73
74	static void vma_print_allocator(struct i915_vma *vma, const char *reason)
75	{
76	char buf[512];
77
78	if (!vma->node.stack) {
79	drm_dbg(vma->obj->base.dev,
80	"vma.node [%08llx + %08llx] %s: unknown owner\n",
81	vma->node.start, vma->node.size, reason);
82	return;
83	}
84
85	stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
86	drm_dbg(vma->obj->base.dev,
87	"vma.node [%08llx + %08llx] %s: inserted at %s\n",
88	vma->node.start, vma->node.size, reason, buf);
89	}
90
91	#else
92
93	static void vma_print_allocator(struct i915_vma *vma, const char *reason)
94	{
95	}
96
97	#endif
98
99	static inline struct i915_vma *active_to_vma(struct i915_active *ref)
100	{
101	return container_of(ref, typeof(struct i915_vma), active);
102	}
103
104	static int __i915_vma_active(struct i915_active *ref)
105	{
106	return i915_vma_tryget(vma: active_to_vma(ref)) ? 0 : -ENOENT;
107	}
108
109	static void __i915_vma_retire(struct i915_active *ref)
110	{
111	i915_vma_put(vma: active_to_vma(ref));
112	}
113
114	static struct i915_vma *
115	vma_create(struct drm_i915_gem_object *obj,
116	struct i915_address_space *vm,
117	const struct i915_gtt_view *view)
118	{
119	struct i915_vma *pos = ERR_PTR(error: -E2BIG);
120	struct i915_vma *vma;
121	struct rb_node *rb, **p;
122	int err;
123
124	/* The aliasing_ppgtt should never be used directly! */
125	GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
126
127	vma = i915_vma_alloc();
128	if (vma == NULL)
129	return ERR_PTR(error: -ENOMEM);
130
131	vma->ops = &vm->vma_ops;
132	vma->obj = obj;
133	vma->size = obj->base.size;
134	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
135
136	i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
137
138	/* Declare ourselves safe for use inside shrinkers */
139	if (IS_ENABLED(CONFIG_LOCKDEP)) {
140	fs_reclaim_acquire(GFP_KERNEL);
141	might_lock(&vma->active.mutex);
142	fs_reclaim_release(GFP_KERNEL);
143	}
144
145	INIT_LIST_HEAD(list: &vma->closed_link);
146	INIT_LIST_HEAD(list: &vma->obj_link);
147	RB_CLEAR_NODE(&vma->obj_node);
148
149	if (view && view->type != I915_GTT_VIEW_NORMAL) {
150	vma->gtt_view = *view;
151	if (view->type == I915_GTT_VIEW_PARTIAL) {
152	GEM_BUG_ON(range_overflows_t(u64,
153	view->partial.offset,
154	view->partial.size,
155	obj->base.size >> PAGE_SHIFT));
156	vma->size = view->partial.size;
157	vma->size <<= PAGE_SHIFT;
158	GEM_BUG_ON(vma->size > obj->base.size);
159	} else if (view->type == I915_GTT_VIEW_ROTATED) {
160	vma->size = intel_rotation_info_size(rot_info: &view->rotated);
161	vma->size <<= PAGE_SHIFT;
162	} else if (view->type == I915_GTT_VIEW_REMAPPED) {
163	vma->size = intel_remapped_info_size(rem_info: &view->remapped);
164	vma->size <<= PAGE_SHIFT;
165	}
166	}
167
168	if (unlikely(vma->size > vm->total))
169	goto err_vma;
170
171	GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
172
173	err = mutex_lock_interruptible(&vm->mutex);
174	if (err) {
175	pos = ERR_PTR(error: err);
176	goto err_vma;
177	}
178
179	vma->vm = vm;
180	list_add_tail(new: &vma->vm_link, head: &vm->unbound_list);
181
182	spin_lock(lock: &obj->vma.lock);
183	if (i915_is_ggtt(vm)) {
184	if (unlikely(overflows_type(vma->size, u32)))
185	goto err_unlock;
186
187	vma->fence_size = i915_gem_fence_size(i915: vm->i915, size: vma->size,
188	tiling: i915_gem_object_get_tiling(obj),
189	stride: i915_gem_object_get_stride(obj));
190	if (unlikely(vma->fence_size < vma->size || /* overflow */
191	vma->fence_size > vm->total))
192	goto err_unlock;
193
194	GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
195
196	vma->fence_alignment = i915_gem_fence_alignment(i915: vm->i915, size: vma->size,
197	tiling: i915_gem_object_get_tiling(obj),
198	stride: i915_gem_object_get_stride(obj));
199	GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
200
201	__set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
202	}
203
204	rb = NULL;
205	p = &obj->vma.tree.rb_node;
206	while (*p) {
207	long cmp;
208
209	rb = *p;
210	pos = rb_entry(rb, struct i915_vma, obj_node);
211
212	/*
213	* If the view already exists in the tree, another thread
214	* already created a matching vma, so return the older instance
215	* and dispose of ours.
216	*/
217	cmp = i915_vma_compare(vma: pos, vm, view);
218	if (cmp < 0)
219	p = &rb->rb_right;
220	else if (cmp > 0)
221	p = &rb->rb_left;
222	else
223	goto err_unlock;
224	}
225	rb_link_node(node: &vma->obj_node, parent: rb, rb_link: p);
226	rb_insert_color(&vma->obj_node, &obj->vma.tree);
227
228	if (i915_vma_is_ggtt(vma))
229	/*
230	* We put the GGTT vma at the start of the vma-list, followed
231	* by the ppGGTT vma. This allows us to break early when
232	* iterating over only the GGTT vma for an object, see
233	* for_each_ggtt_vma()
234	*/
235	list_add(new: &vma->obj_link, head: &obj->vma.list);
236	else
237	list_add_tail(new: &vma->obj_link, head: &obj->vma.list);
238
239	spin_unlock(lock: &obj->vma.lock);
240	mutex_unlock(lock: &vm->mutex);
241
242	return vma;
243
244	err_unlock:
245	spin_unlock(lock: &obj->vma.lock);
246	list_del_init(entry: &vma->vm_link);
247	mutex_unlock(lock: &vm->mutex);
248	err_vma:
249	i915_vma_free(vma);
250	return pos;
251	}
252
253	static struct i915_vma *
254	i915_vma_lookup(struct drm_i915_gem_object *obj,
255	struct i915_address_space *vm,
256	const struct i915_gtt_view *view)
257	{
258	struct rb_node *rb;
259
260	rb = obj->vma.tree.rb_node;
261	while (rb) {
262	struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
263	long cmp;
264
265	cmp = i915_vma_compare(vma, vm, view);
266	if (cmp == 0)
267	return vma;
268
269	if (cmp < 0)
270	rb = rb->rb_right;
271	else
272	rb = rb->rb_left;
273	}
274
275	return NULL;
276	}
277
278	/**
279	* i915_vma_instance - return the singleton instance of the VMA
280	* @obj: parent &struct drm_i915_gem_object to be mapped
281	* @vm: address space in which the mapping is located
282	* @view: additional mapping requirements
283	*
284	* i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
285	* the same @view characteristics. If a match is not found, one is created.
286	* Once created, the VMA is kept until either the object is freed, or the
287	* address space is closed.
288	*
289	* Returns the vma, or an error pointer.
290	*/
291	struct i915_vma *
292	i915_vma_instance(struct drm_i915_gem_object *obj,
293	struct i915_address_space *vm,
294	const struct i915_gtt_view *view)
295	{
296	struct i915_vma *vma;
297
298	GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
299	GEM_BUG_ON(!kref_read(&vm->ref));
300
301	spin_lock(lock: &obj->vma.lock);
302	vma = i915_vma_lookup(obj, vm, view);
303	spin_unlock(lock: &obj->vma.lock);
304
305	/* vma_create() will resolve the race if another creates the vma */
306	if (unlikely(!vma))
307	vma = vma_create(obj, vm, view);
308
309	GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
310	return vma;
311	}
312
313	struct i915_vma_work {
314	struct dma_fence_work base;
315	struct i915_address_space *vm;
316	struct i915_vm_pt_stash stash;
317	struct i915_vma_resource *vma_res;
318	struct drm_i915_gem_object *obj;
319	struct i915_sw_dma_fence_cb cb;
320	unsigned int pat_index;
321	unsigned int flags;
322	};
323
324	static void __vma_bind(struct dma_fence_work *work)
325	{
326	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
327	struct i915_vma_resource *vma_res = vw->vma_res;
328
329	/*
330	* We are about the bind the object, which must mean we have already
331	* signaled the work to potentially clear/move the pages underneath. If
332	* something went wrong at that stage then the object should have
333	* unknown_state set, in which case we need to skip the bind.
334	*/
335	if (i915_gem_object_has_unknown_state(obj: vw->obj))
336	return;
337
338	vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
339	vma_res, vw->pat_index, vw->flags);
340	}
341
342	static void __vma_release(struct dma_fence_work *work)
343	{
344	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
345
346	if (vw->obj)
347	i915_gem_object_put(obj: vw->obj);
348
349	i915_vm_free_pt_stash(vm: vw->vm, stash: &vw->stash);
350	if (vw->vma_res)
351	i915_vma_resource_put(vma_res: vw->vma_res);
352	}
353
354	static const struct dma_fence_work_ops bind_ops = {
355	.name = "bind",
356	.work = __vma_bind,
357	.release = __vma_release,
358	};
359
360	struct i915_vma_work *i915_vma_work(void)
361	{
362	struct i915_vma_work *vw;
363
364	vw = kzalloc(size: sizeof(*vw), GFP_KERNEL);
365	if (!vw)
366	return NULL;
367
368	dma_fence_work_init(f: &vw->base, ops: &bind_ops);
369	vw->base.dma.error = -EAGAIN; /* disable the worker by default */
370
371	return vw;
372	}
373
374	int i915_vma_wait_for_bind(struct i915_vma *vma)
375	{
376	int err = 0;
377
378	if (rcu_access_pointer(vma->active.excl.fence)) {
379	struct dma_fence *fence;
380
381	rcu_read_lock();
382	fence = dma_fence_get_rcu_safe(fencep: &vma->active.excl.fence);
383	rcu_read_unlock();
384	if (fence) {
385	err = dma_fence_wait(fence, intr: true);
386	dma_fence_put(fence);
387	}
388	}
389
390	return err;
391	}
392
393	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
394	static int i915_vma_verify_bind_complete(struct i915_vma *vma)
395	{
396	struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
397	int err;
398
399	if (!fence)
400	return 0;
401
402	if (dma_fence_is_signaled(fence))
403	err = fence->error;
404	else
405	err = -EBUSY;
406
407	dma_fence_put(fence);
408
409	return err;
410	}
411	#else
412	#define i915_vma_verify_bind_complete(_vma) 0
413	#endif
414
415	I915_SELFTEST_EXPORT void
416	i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
417	struct i915_vma *vma)
418	{
419	struct drm_i915_gem_object *obj = vma->obj;
420
421	i915_vma_resource_init(vma_res, vm: vma->vm, pages: vma->pages, page_sizes: &vma->page_sizes,
422	pages_rsgt: obj->mm.rsgt, readonly: i915_gem_object_is_readonly(obj),
423	lmem: i915_gem_object_is_lmem(obj), mr: obj->mm.region,
424	ops: vma->ops, private: vma->private, start: __i915_vma_offset(vma),
425	node_size: __i915_vma_size(vma), size: vma->size, guard: vma->guard);
426	}
427
428	/**
429	* i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
430	* @vma: VMA to map
431	* @pat_index: PAT index to set in PTE
432	* @flags: flags like global or local mapping
433	* @work: preallocated worker for allocating and binding the PTE
434	* @vma_res: pointer to a preallocated vma resource. The resource is either
435	* consumed or freed.
436	*
437	* DMA addresses are taken from the scatter-gather table of this object (or of
438	* this VMA in case of non-default GGTT views) and PTE entries set up.
439	* Note that DMA addresses are also the only part of the SG table we care about.
440	*/
441	int i915_vma_bind(struct i915_vma *vma,
442	unsigned int pat_index,
443	u32 flags,
444	struct i915_vma_work *work,
445	struct i915_vma_resource *vma_res)
446	{
447	u32 bind_flags;
448	u32 vma_flags;
449	int ret;
450
451	lockdep_assert_held(&vma->vm->mutex);
452	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
453	GEM_BUG_ON(vma->size > i915_vma_size(vma));
454
455	if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
456	vma->node.size,
457	vma->vm->total))) {
458	i915_vma_resource_free(vma_res);
459	return -ENODEV;
460	}
461
462	if (GEM_DEBUG_WARN_ON(!flags)) {
463	i915_vma_resource_free(vma_res);
464	return -EINVAL;
465	}
466
467	bind_flags = flags;
468	bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
469
470	vma_flags = atomic_read(v: &vma->flags);
471	vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
472
473	bind_flags &= ~vma_flags;
474	if (bind_flags == 0) {
475	i915_vma_resource_free(vma_res);
476	return 0;
477	}
478
479	GEM_BUG_ON(!atomic_read(&vma->pages_count));
480
481	/* Wait for or await async unbinds touching our range */
482	if (work && bind_flags & vma->vm->bind_async_flags)
483	ret = i915_vma_resource_bind_dep_await(vm: vma->vm,
484	sw_fence: &work->base.chain,
485	first: vma->node.start,
486	last: vma->node.size,
487	intr: true,
488	GFP_NOWAIT |
489	__GFP_RETRY_MAYFAIL |
490	__GFP_NOWARN);
491	else
492	ret = i915_vma_resource_bind_dep_sync(vm: vma->vm, first: vma->node.start,
493	last: vma->node.size, intr: true);
494	if (ret) {
495	i915_vma_resource_free(vma_res);
496	return ret;
497	}
498
499	if (vma->resource || !vma_res) {
500	/* Rebinding with an additional I915_VMA_*_BIND */
501	GEM_WARN_ON(!vma_flags);
502	i915_vma_resource_free(vma_res);
503	} else {
504	i915_vma_resource_init_from_vma(vma_res, vma);
505	vma->resource = vma_res;
506	}
507	trace_i915_vma_bind(vma, flags: bind_flags);
508	if (work && bind_flags & vma->vm->bind_async_flags) {
509	struct dma_fence *prev;
510
511	work->vma_res = i915_vma_resource_get(vma_res: vma->resource);
512	work->pat_index = pat_index;
513	work->flags = bind_flags;
514
515	/*
516	* Note we only want to chain up to the migration fence on
517	* the pages (not the object itself). As we don't track that,
518	* yet, we have to use the exclusive fence instead.
519	*
520	* Also note that we do not want to track the async vma as
521	* part of the obj->resv->excl_fence as it only affects
522	* execution and not content or object's backing store lifetime.
523	*/
524	prev = i915_active_set_exclusive(ref: &vma->active, f: &work->base.dma);
525	if (prev) {
526	__i915_sw_fence_await_dma_fence(fence: &work->base.chain,
527	dma: prev,
528	cb: &work->cb);
529	dma_fence_put(fence: prev);
530	}
531
532	work->base.dma.error = 0; /* enable the queue_work() */
533	work->obj = i915_gem_object_get(obj: vma->obj);
534	} else {
535	ret = i915_gem_object_wait_moving_fence(obj: vma->obj, intr: true);
536	if (ret) {
537	i915_vma_resource_free(vma_res: vma->resource);
538	vma->resource = NULL;
539
540	return ret;
541	}
542	vma->ops->bind_vma(vma->vm, NULL, vma->resource, pat_index,
543	bind_flags);
544	}
545
546	atomic_or(i: bind_flags, v: &vma->flags);
547	return 0;
548	}
549
550	void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
551	{
552	void __iomem *ptr;
553	int err;
554
555	if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY))
556	return IOMEM_ERR_PTR(-EINVAL);
557
558	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
559	GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
560	GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
561
562	ptr = READ_ONCE(vma->iomap);
563	if (ptr == NULL) {
564	/*
565	* TODO: consider just using i915_gem_object_pin_map() for lmem
566	* instead, which already supports mapping non-contiguous chunks
567	* of pages, that way we can also drop the
568	* I915_BO_ALLOC_CONTIGUOUS when allocating the object.
569	*/
570	if (i915_gem_object_is_lmem(obj: vma->obj)) {
571	ptr = i915_gem_object_lmem_io_map(obj: vma->obj, n: 0,
572	size: vma->obj->base.size);
573	} else if (i915_vma_is_map_and_fenceable(vma)) {
574	ptr = io_mapping_map_wc(mapping: &i915_vm_to_ggtt(vm: vma->vm)->iomap,
575	offset: i915_vma_offset(vma),
576	size: i915_vma_size(vma));
577	} else {
578	ptr = (void __iomem *)
579	i915_gem_object_pin_map(obj: vma->obj, type: I915_MAP_WC);
580	if (IS_ERR(ptr)) {
581	err = PTR_ERR(ptr);
582	goto err;
583	}
584	ptr = page_pack_bits(ptr, 1);
585	}
586
587	if (ptr == NULL) {
588	err = -ENOMEM;
589	goto err;
590	}
591
592	if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
593	if (page_unmask_bits(ptr))
594	__i915_gem_object_release_map(obj: vma->obj);
595	else
596	io_mapping_unmap(vaddr: ptr);
597	ptr = vma->iomap;
598	}
599	}
600
601	__i915_vma_pin(vma);
602
603	err = i915_vma_pin_fence(vma);
604	if (err)
605	goto err_unpin;
606
607	i915_vma_set_ggtt_write(vma);
608
609	/* NB Access through the GTT requires the device to be awake. */
610	return page_mask_bits(ptr);
611
612	err_unpin:
613	__i915_vma_unpin(vma);
614	err:
615	return IOMEM_ERR_PTR(err);
616	}
617
618	void i915_vma_flush_writes(struct i915_vma *vma)
619	{
620	if (i915_vma_unset_ggtt_write(vma))
621	intel_gt_flush_ggtt_writes(gt: vma->vm->gt);
622	}
623
624	void i915_vma_unpin_iomap(struct i915_vma *vma)
625	{
626	GEM_BUG_ON(vma->iomap == NULL);
627
628	/* XXX We keep the mapping until __i915_vma_unbind()/evict() */
629
630	i915_vma_flush_writes(vma);
631
632	i915_vma_unpin_fence(vma);
633	i915_vma_unpin(vma);
634	}
635
636	void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
637	{
638	struct i915_vma *vma;
639	struct drm_i915_gem_object *obj;
640
641	vma = fetch_and_zero(p_vma);
642	if (!vma)
643	return;
644
645	obj = vma->obj;
646	GEM_BUG_ON(!obj);
647
648	i915_vma_unpin(vma);
649
650	if (flags & I915_VMA_RELEASE_MAP)
651	i915_gem_object_unpin_map(obj);
652
653	i915_gem_object_put(obj);
654	}
655
656	bool i915_vma_misplaced(const struct i915_vma *vma,
657	u64 size, u64 alignment, u64 flags)
658	{
659	if (!drm_mm_node_allocated(node: &vma->node))
660	return false;
661
662	if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
663	return true;
664
665	if (i915_vma_size(vma) < size)
666	return true;
667
668	GEM_BUG_ON(alignment && !is_power_of_2(alignment));
669	if (alignment && !IS_ALIGNED(i915_vma_offset(vma), alignment))
670	return true;
671
672	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
673	return true;
674
675	if (flags & PIN_OFFSET_BIAS &&
676	i915_vma_offset(vma) < (flags & PIN_OFFSET_MASK))
677	return true;
678
679	if (flags & PIN_OFFSET_FIXED &&
680	i915_vma_offset(vma) != (flags & PIN_OFFSET_MASK))
681	return true;
682
683	if (flags & PIN_OFFSET_GUARD &&
684	vma->guard < (flags & PIN_OFFSET_MASK))
685	return true;
686
687	return false;
688	}
689
690	void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
691	{
692	bool mappable, fenceable;
693
694	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
695	GEM_BUG_ON(!vma->fence_size);
696
697	fenceable = (i915_vma_size(vma) >= vma->fence_size &&
698	IS_ALIGNED(i915_vma_offset(vma), vma->fence_alignment));
699
700	mappable = i915_ggtt_offset(vma) + vma->fence_size <=
701	i915_vm_to_ggtt(vm: vma->vm)->mappable_end;
702
703	if (mappable && fenceable)
704	set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
705	else
706	clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
707	}
708
709	bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
710	{
711	struct drm_mm_node *node = &vma->node;
712	struct drm_mm_node *other;
713
714	/*
715	* On some machines we have to be careful when putting differing types
716	* of snoopable memory together to avoid the prefetcher crossing memory
717	* domains and dying. During vm initialisation, we decide whether or not
718	* these constraints apply and set the drm_mm.color_adjust
719	* appropriately.
720	*/
721	if (!i915_vm_has_cache_coloring(vm: vma->vm))
722	return true;
723
724	/* Only valid to be called on an already inserted vma */
725	GEM_BUG_ON(!drm_mm_node_allocated(node));
726	GEM_BUG_ON(list_empty(&node->node_list));
727
728	other = list_prev_entry(node, node_list);
729	if (i915_node_color_differs(node: other, color) &&
730	!drm_mm_hole_follows(node: other))
731	return false;
732
733	other = list_next_entry(node, node_list);
734	if (i915_node_color_differs(node: other, color) &&
735	!drm_mm_hole_follows(node))
736	return false;
737
738	return true;
739	}
740
741	/**
742	* i915_vma_insert - finds a slot for the vma in its address space
743	* @vma: the vma
744	* @ww: An optional struct i915_gem_ww_ctx
745	* @size: requested size in bytes (can be larger than the VMA)
746	* @alignment: required alignment
747	* @flags: mask of PIN_* flags to use
748	*
749	* First we try to allocate some free space that meets the requirements for
750	* the VMA. Failiing that, if the flags permit, it will evict an old VMA,
751	* preferrably the oldest idle entry to make room for the new VMA.
752	*
753	* Returns:
754	* 0 on success, negative error code otherwise.
755	*/
756	static int
757	i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
758	u64 size, u64 alignment, u64 flags)
759	{
760	unsigned long color, guard;
761	u64 start, end;
762	int ret;
763
764	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
765	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
766	GEM_BUG_ON(hweight64(flags & (PIN_OFFSET_GUARD | PIN_OFFSET_FIXED | PIN_OFFSET_BIAS)) > 1);
767
768	size = max(size, vma->size);
769	alignment = max_t(typeof(alignment), alignment, vma->display_alignment);
770	if (flags & PIN_MAPPABLE) {
771	size = max_t(typeof(size), size, vma->fence_size);
772	alignment = max_t(typeof(alignment),
773	alignment, vma->fence_alignment);
774	}
775
776	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
777	GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
778	GEM_BUG_ON(!is_power_of_2(alignment));
779
780	guard = vma->guard; /* retain guard across rebinds */
781	if (flags & PIN_OFFSET_GUARD) {
782	GEM_BUG_ON(overflows_type(flags & PIN_OFFSET_MASK, u32));
783	guard = max_t(u32, guard, flags & PIN_OFFSET_MASK);
784	}
785	/*
786	* As we align the node upon insertion, but the hardware gets
787	* node.start + guard, the easiest way to make that work is
788	* to make the guard a multiple of the alignment size.
789	*/
790	guard = ALIGN(guard, alignment);
791
792	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
793	GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
794
795	end = vma->vm->total;
796	if (flags & PIN_MAPPABLE)
797	end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
798	if (flags & PIN_ZONE_4G)
799	end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
800	GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
801
802	alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj));
803
804	/*
805	* If binding the object/GGTT view requires more space than the entire
806	* aperture has, reject it early before evicting everything in a vain
807	* attempt to find space.
808	*/
809	if (size > end - 2 * guard) {
810	drm_dbg(vma->obj->base.dev,
811	"Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
812	size, flags & PIN_MAPPABLE ? "mappable" : "total", end);
813	return -ENOSPC;
814	}
815
816	color = 0;
817
818	if (i915_vm_has_cache_coloring(vm: vma->vm))
819	color = vma->obj->pat_index;
820
821	if (flags & PIN_OFFSET_FIXED) {
822	u64 offset = flags & PIN_OFFSET_MASK;
823	if (!IS_ALIGNED(offset, alignment) ||
824	range_overflows(offset, size, end))
825	return -EINVAL;
826	/*
827	* The caller knows not of the guard added by others and
828	* requests for the offset of the start of its buffer
829	* to be fixed, which may not be the same as the position
830	* of the vma->node due to the guard pages.
831	*/
832	if (offset < guard || offset + size > end - guard)
833	return -ENOSPC;
834
835	ret = i915_gem_gtt_reserve(vm: vma->vm, ww, node: &vma->node,
836	size: size + 2 * guard,
837	offset: offset - guard,
838	color, flags);
839	if (ret)
840	return ret;
841	} else {
842	size += 2 * guard;
843	/*
844	* We only support huge gtt pages through the 48b PPGTT,
845	* however we also don't want to force any alignment for
846	* objects which need to be tightly packed into the low 32bits.
847	*
848	* Note that we assume that GGTT are limited to 4GiB for the
849	* forseeable future. See also i915_ggtt_offset().
850	*/
851	if (upper_32_bits(end - 1) &&
852	vma->page_sizes.sg > I915_GTT_PAGE_SIZE &&
853	!HAS_64K_PAGES(vma->vm->i915)) {
854	/*
855	* We can't mix 64K and 4K PTEs in the same page-table
856	* (2M block), and so to avoid the ugliness and
857	* complexity of coloring we opt for just aligning 64K
858	* objects to 2M.
859	*/
860	u64 page_alignment =
861	rounddown_pow_of_two(vma->page_sizes.sg |
862	I915_GTT_PAGE_SIZE_2M);
863
864	/*
865	* Check we don't expand for the limited Global GTT
866	* (mappable aperture is even more precious!). This
867	* also checks that we exclude the aliasing-ppgtt.
868	*/
869	GEM_BUG_ON(i915_vma_is_ggtt(vma));
870
871	alignment = max(alignment, page_alignment);
872
873	if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
874	size = round_up(size, I915_GTT_PAGE_SIZE_2M);
875	}
876
877	ret = i915_gem_gtt_insert(vm: vma->vm, ww, node: &vma->node,
878	size, alignment, color,
879	start, end, flags);
880	if (ret)
881	return ret;
882
883	GEM_BUG_ON(vma->node.start < start);
884	GEM_BUG_ON(vma->node.start + vma->node.size > end);
885	}
886	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
887	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
888
889	list_move_tail(list: &vma->vm_link, head: &vma->vm->bound_list);
890	vma->guard = guard;
891
892	return 0;
893	}
894
895	static void
896	i915_vma_detach(struct i915_vma *vma)
897	{
898	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
899	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
900
901	/*
902	* And finally now the object is completely decoupled from this
903	* vma, we can drop its hold on the backing storage and allow
904	* it to be reaped by the shrinker.
905	*/
906	list_move_tail(list: &vma->vm_link, head: &vma->vm->unbound_list);
907	}
908
909	static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
910	{
911	unsigned int bound;
912
913	bound = atomic_read(v: &vma->flags);
914
915	if (flags & PIN_VALIDATE) {
916	flags &= I915_VMA_BIND_MASK;
917
918	return (flags & bound) == flags;
919	}
920
921	/* with the lock mandatory for unbind, we don't race here */
922	flags &= I915_VMA_BIND_MASK;
923	do {
924	if (unlikely(flags & ~bound))
925	return false;
926
927	if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
928	return false;
929
930	GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
931	} while (!atomic_try_cmpxchg(v: &vma->flags, old: &bound, new: bound + 1));
932
933	return true;
934	}
935
936	static struct scatterlist *
937	rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
938	unsigned int width, unsigned int height,
939	unsigned int src_stride, unsigned int dst_stride,
940	struct sg_table *st, struct scatterlist *sg)
941	{
942	unsigned int column, row;
943	pgoff_t src_idx;
944
945	for (column = 0; column < width; column++) {
946	unsigned int left;
947
948	src_idx = src_stride * (height - 1) + column + offset;
949	for (row = 0; row < height; row++) {
950	st->nents++;
951	/*
952	* We don't need the pages, but need to initialize
953	* the entries so the sg list can be happily traversed.
954	* The only thing we need are DMA addresses.
955	*/
956	sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, offset: 0);
957	sg_dma_address(sg) =
958	i915_gem_object_get_dma_address(obj, src_idx);
959	sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
960	sg = sg_next(sg);
961	src_idx -= src_stride;
962	}
963
964	left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
965
966	if (!left)
967	continue;
968
969	st->nents++;
970
971	/*
972	* The DE ignores the PTEs for the padding tiles, the sg entry
973	* here is just a conenience to indicate how many padding PTEs
974	* to insert at this spot.
975	*/
976	sg_set_page(sg, NULL, len: left, offset: 0);
977	sg_dma_address(sg) = 0;
978	sg_dma_len(sg) = left;
979	sg = sg_next(sg);
980	}
981
982	return sg;
983	}
984
985	static noinline struct sg_table *
986	intel_rotate_pages(struct intel_rotation_info *rot_info,
987	struct drm_i915_gem_object *obj)
988	{
989	unsigned int size = intel_rotation_info_size(rot_info);
990	struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
991	struct sg_table *st;
992	struct scatterlist *sg;
993	int ret = -ENOMEM;
994	int i;
995
996	/* Allocate target SG list. */
997	st = kmalloc(size: sizeof(*st), GFP_KERNEL);
998	if (!st)
999	goto err_st_alloc;
1000
1001	ret = sg_alloc_table(st, size, GFP_KERNEL);
1002	if (ret)
1003	goto err_sg_alloc;
1004
1005	st->nents = 0;
1006	sg = st->sgl;
1007
1008	for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1009	sg = rotate_pages(obj, offset: rot_info->plane[i].offset,
1010	width: rot_info->plane[i].width, height: rot_info->plane[i].height,
1011	src_stride: rot_info->plane[i].src_stride,
1012	dst_stride: rot_info->plane[i].dst_stride,
1013	st, sg);
1014
1015	return st;
1016
1017	err_sg_alloc:
1018	kfree(objp: st);
1019	err_st_alloc:
1020
1021	drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1022	obj->base.size, rot_info->plane[0].width,
1023	rot_info->plane[0].height, size);
1024
1025	return ERR_PTR(error: ret);
1026	}
1027
1028	static struct scatterlist *
1029	add_padding_pages(unsigned int count,
1030	struct sg_table *st, struct scatterlist *sg)
1031	{
1032	st->nents++;
1033
1034	/*
1035	* The DE ignores the PTEs for the padding tiles, the sg entry
1036	* here is just a convenience to indicate how many padding PTEs
1037	* to insert at this spot.
1038	*/
1039	sg_set_page(sg, NULL, len: count * I915_GTT_PAGE_SIZE, offset: 0);
1040	sg_dma_address(sg) = 0;
1041	sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE;
1042	sg = sg_next(sg);
1043
1044	return sg;
1045	}
1046
1047	static struct scatterlist *
1048	remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj,
1049	unsigned long offset, unsigned int alignment_pad,
1050	unsigned int width, unsigned int height,
1051	unsigned int src_stride, unsigned int dst_stride,
1052	struct sg_table *st, struct scatterlist *sg,
1053	unsigned int *gtt_offset)
1054	{
1055	unsigned int row;
1056
1057	if (!width || !height)
1058	return sg;
1059
1060	if (alignment_pad)
1061	sg = add_padding_pages(count: alignment_pad, st, sg);
1062
1063	for (row = 0; row < height; row++) {
1064	unsigned int left = width * I915_GTT_PAGE_SIZE;
1065
1066	while (left) {
1067	dma_addr_t addr;
1068	unsigned int length;
1069
1070	/*
1071	* We don't need the pages, but need to initialize
1072	* the entries so the sg list can be happily traversed.
1073	* The only thing we need are DMA addresses.
1074	*/
1075
1076	addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1077
1078	length = min(left, length);
1079
1080	st->nents++;
1081
1082	sg_set_page(sg, NULL, len: length, offset: 0);
1083	sg_dma_address(sg) = addr;
1084	sg_dma_len(sg) = length;
1085	sg = sg_next(sg);
1086
1087	offset += length / I915_GTT_PAGE_SIZE;
1088	left -= length;
1089	}
1090
1091	offset += src_stride - width;
1092
1093	left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1094
1095	if (!left)
1096	continue;
1097
1098	sg = add_padding_pages(count: left >> PAGE_SHIFT, st, sg);
1099	}
1100
1101	*gtt_offset += alignment_pad + dst_stride * height;
1102
1103	return sg;
1104	}
1105
1106	static struct scatterlist *
1107	remap_contiguous_pages(struct drm_i915_gem_object *obj,
1108	pgoff_t obj_offset,
1109	unsigned int count,
1110	struct sg_table *st, struct scatterlist *sg)
1111	{
1112	struct scatterlist *iter;
1113	unsigned int offset;
1114
1115	iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset);
1116	GEM_BUG_ON(!iter);
1117
1118	do {
1119	unsigned int len;
1120
1121	len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1122	count << PAGE_SHIFT);
1123	sg_set_page(sg, NULL, len, offset: 0);
1124	sg_dma_address(sg) =
1125	sg_dma_address(iter) + (offset << PAGE_SHIFT);
1126	sg_dma_len(sg) = len;
1127
1128	st->nents++;
1129	count -= len >> PAGE_SHIFT;
1130	if (count == 0)
1131	return sg;
1132
1133	sg = __sg_next(sg);
1134	iter = __sg_next(sg: iter);
1135	offset = 0;
1136	} while (1);
1137	}
1138
1139	static struct scatterlist *
1140	remap_linear_color_plane_pages(struct drm_i915_gem_object *obj,
1141	pgoff_t obj_offset, unsigned int alignment_pad,
1142	unsigned int size,
1143	struct sg_table *st, struct scatterlist *sg,
1144	unsigned int *gtt_offset)
1145	{
1146	if (!size)
1147	return sg;
1148
1149	if (alignment_pad)
1150	sg = add_padding_pages(count: alignment_pad, st, sg);
1151
1152	sg = remap_contiguous_pages(obj, obj_offset, count: size, st, sg);
1153	sg = sg_next(sg);
1154
1155	*gtt_offset += alignment_pad + size;
1156
1157	return sg;
1158	}
1159
1160	static struct scatterlist *
1161	remap_color_plane_pages(const struct intel_remapped_info *rem_info,
1162	struct drm_i915_gem_object *obj,
1163	int color_plane,
1164	struct sg_table *st, struct scatterlist *sg,
1165	unsigned int *gtt_offset)
1166	{
1167	unsigned int alignment_pad = 0;
1168
1169	if (rem_info->plane_alignment)
1170	alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset;
1171
1172	if (rem_info->plane[color_plane].linear)
1173	sg = remap_linear_color_plane_pages(obj,
1174	obj_offset: rem_info->plane[color_plane].offset,
1175	alignment_pad,
1176	size: rem_info->plane[color_plane].size,
1177	st, sg,
1178	gtt_offset);
1179
1180	else
1181	sg = remap_tiled_color_plane_pages(obj,
1182	offset: rem_info->plane[color_plane].offset,
1183	alignment_pad,
1184	width: rem_info->plane[color_plane].width,
1185	height: rem_info->plane[color_plane].height,
1186	src_stride: rem_info->plane[color_plane].src_stride,
1187	dst_stride: rem_info->plane[color_plane].dst_stride,
1188	st, sg,
1189	gtt_offset);
1190
1191	return sg;
1192	}
1193
1194	static noinline struct sg_table *
1195	intel_remap_pages(struct intel_remapped_info *rem_info,
1196	struct drm_i915_gem_object *obj)
1197	{
1198	unsigned int size = intel_remapped_info_size(rem_info);
1199	struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
1200	struct sg_table *st;
1201	struct scatterlist *sg;
1202	unsigned int gtt_offset = 0;
1203	int ret = -ENOMEM;
1204	int i;
1205
1206	/* Allocate target SG list. */
1207	st = kmalloc(size: sizeof(*st), GFP_KERNEL);
1208	if (!st)
1209	goto err_st_alloc;
1210
1211	ret = sg_alloc_table(st, size, GFP_KERNEL);
1212	if (ret)
1213	goto err_sg_alloc;
1214
1215	st->nents = 0;
1216	sg = st->sgl;
1217
1218	for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1219	sg = remap_color_plane_pages(rem_info, obj, color_plane: i, st, sg, gtt_offset: &gtt_offset);
1220
1221	i915_sg_trim(orig_st: st);
1222
1223	return st;
1224
1225	err_sg_alloc:
1226	kfree(objp: st);
1227	err_st_alloc:
1228
1229	drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1230	obj->base.size, rem_info->plane[0].width,
1231	rem_info->plane[0].height, size);
1232
1233	return ERR_PTR(error: ret);
1234	}
1235
1236	static noinline struct sg_table *
1237	intel_partial_pages(const struct i915_gtt_view *view,
1238	struct drm_i915_gem_object *obj)
1239	{
1240	struct sg_table *st;
1241	struct scatterlist *sg;
1242	unsigned int count = view->partial.size;
1243	int ret = -ENOMEM;
1244
1245	st = kmalloc(size: sizeof(*st), GFP_KERNEL);
1246	if (!st)
1247	goto err_st_alloc;
1248
1249	ret = sg_alloc_table(st, count, GFP_KERNEL);
1250	if (ret)
1251	goto err_sg_alloc;
1252
1253	st->nents = 0;
1254
1255	sg = remap_contiguous_pages(obj, obj_offset: view->partial.offset, count, st, sg: st->sgl);
1256
1257	sg_mark_end(sg);
1258	i915_sg_trim(orig_st: st); /* Drop any unused tail entries. */
1259
1260	return st;
1261
1262	err_sg_alloc:
1263	kfree(objp: st);
1264	err_st_alloc:
1265	return ERR_PTR(error: ret);
1266	}
1267
1268	static int
1269	__i915_vma_get_pages(struct i915_vma *vma)
1270	{
1271	struct sg_table *pages;
1272
1273	/*
1274	* The vma->pages are only valid within the lifespan of the borrowed
1275	* obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1276	* must be the vma->pages. A simple rule is that vma->pages must only
1277	* be accessed when the obj->mm.pages are pinned.
1278	*/
1279	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1280
1281	switch (vma->gtt_view.type) {
1282	default:
1283	GEM_BUG_ON(vma->gtt_view.type);
1284	fallthrough;
1285	case I915_GTT_VIEW_NORMAL:
1286	pages = vma->obj->mm.pages;
1287	break;
1288
1289	case I915_GTT_VIEW_ROTATED:
1290	pages =
1291	intel_rotate_pages(rot_info: &vma->gtt_view.rotated, obj: vma->obj);
1292	break;
1293
1294	case I915_GTT_VIEW_REMAPPED:
1295	pages =
1296	intel_remap_pages(rem_info: &vma->gtt_view.remapped, obj: vma->obj);
1297	break;
1298
1299	case I915_GTT_VIEW_PARTIAL:
1300	pages = intel_partial_pages(view: &vma->gtt_view, obj: vma->obj);
1301	break;
1302	}
1303
1304	if (IS_ERR(ptr: pages)) {
1305	drm_err(&vma->vm->i915->drm,
1306	"Failed to get pages for VMA view type %u (%ld)!\n",
1307	vma->gtt_view.type, PTR_ERR(pages));
1308	return PTR_ERR(ptr: pages);
1309	}
1310
1311	vma->pages = pages;
1312
1313	return 0;
1314	}
1315
1316	I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1317	{
1318	int err;
1319
1320	if (atomic_add_unless(v: &vma->pages_count, a: 1, u: 0))
1321	return 0;
1322
1323	err = i915_gem_object_pin_pages(obj: vma->obj);
1324	if (err)
1325	return err;
1326
1327	err = __i915_vma_get_pages(vma);
1328	if (err)
1329	goto err_unpin;
1330
1331	vma->page_sizes = vma->obj->mm.page_sizes;
1332	atomic_inc(v: &vma->pages_count);
1333
1334	return 0;
1335
1336	err_unpin:
1337	__i915_gem_object_unpin_pages(obj: vma->obj);
1338
1339	return err;
1340	}
1341
1342	void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
1343	{
1344	struct intel_gt *gt;
1345	int id;
1346
1347	if (!tlb)
1348	return;
1349
1350	/*
1351	* Before we release the pages that were bound by this vma, we
1352	* must invalidate all the TLBs that may still have a reference
1353	* back to our physical address. It only needs to be done once,
1354	* so after updating the PTE to point away from the pages, record
1355	* the most recent TLB invalidation seqno, and if we have not yet
1356	* flushed the TLBs upon release, perform a full invalidation.
1357	*/
1358	for_each_gt(gt, vm->i915, id)
1359	WRITE_ONCE(tlb[id],
1360	intel_gt_next_invalidate_tlb_full(gt));
1361	}
1362
1363	static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1364	{
1365	/* We allocate under vma_get_pages, so beware the shrinker */
1366	GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1367
1368	if (atomic_sub_return(i: count, v: &vma->pages_count) == 0) {
1369	if (vma->pages != vma->obj->mm.pages) {
1370	sg_free_table(vma->pages);
1371	kfree(objp: vma->pages);
1372	}
1373	vma->pages = NULL;
1374
1375	i915_gem_object_unpin_pages(obj: vma->obj);
1376	}
1377	}
1378
1379	I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1380	{
1381	if (atomic_add_unless(v: &vma->pages_count, a: -1, u: 1))
1382	return;
1383
1384	__vma_put_pages(vma, count: 1);
1385	}
1386
1387	static void vma_unbind_pages(struct i915_vma *vma)
1388	{
1389	unsigned int count;
1390
1391	lockdep_assert_held(&vma->vm->mutex);
1392
1393	/* The upper portion of pages_count is the number of bindings */
1394	count = atomic_read(v: &vma->pages_count);
1395	count >>= I915_VMA_PAGES_BIAS;
1396	GEM_BUG_ON(!count);
1397
1398	__vma_put_pages(vma, count: count | count << I915_VMA_PAGES_BIAS);
1399	}
1400
1401	int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1402	u64 size, u64 alignment, u64 flags)
1403	{
1404	struct i915_vma_work *work = NULL;
1405	struct dma_fence *moving = NULL;
1406	struct i915_vma_resource *vma_res = NULL;
1407	intel_wakeref_t wakeref = 0;
1408	unsigned int bound;
1409	int err;
1410
1411	assert_vma_held(vma);
1412	GEM_BUG_ON(!ww);
1413
1414	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1415	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1416
1417	GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1418
1419	/* First try and grab the pin without rebinding the vma */
1420	if (try_qad_pin(vma, flags))
1421	return 0;
1422
1423	err = i915_vma_get_pages(vma);
1424	if (err)
1425	return err;
1426
1427	if (flags & PIN_GLOBAL)
1428	wakeref = intel_runtime_pm_get(rpm: &vma->vm->i915->runtime_pm);
1429
1430	if (flags & vma->vm->bind_async_flags) {
1431	/* lock VM */
1432	err = i915_vm_lock_objects(vm: vma->vm, ww);
1433	if (err)
1434	goto err_rpm;
1435
1436	work = i915_vma_work();
1437	if (!work) {
1438	err = -ENOMEM;
1439	goto err_rpm;
1440	}
1441
1442	work->vm = vma->vm;
1443
1444	err = i915_gem_object_get_moving_fence(obj: vma->obj, fence: &moving);
1445	if (err)
1446	goto err_rpm;
1447
1448	dma_fence_work_chain(f: &work->base, signal: moving);
1449
1450	/* Allocate enough page directories to used PTE */
1451	if (vma->vm->allocate_va_range) {
1452	err = i915_vm_alloc_pt_stash(vm: vma->vm,
1453	stash: &work->stash,
1454	size: vma->size);
1455	if (err)
1456	goto err_fence;
1457
1458	err = i915_vm_map_pt_stash(vm: vma->vm, stash: &work->stash);
1459	if (err)
1460	goto err_fence;
1461	}
1462	}
1463
1464	vma_res = i915_vma_resource_alloc();
1465	if (IS_ERR(ptr: vma_res)) {
1466	err = PTR_ERR(ptr: vma_res);
1467	goto err_fence;
1468	}
1469
1470	/*
1471	* Differentiate between user/kernel vma inside the aliasing-ppgtt.
1472	*
1473	* We conflate the Global GTT with the user's vma when using the
1474	* aliasing-ppgtt, but it is still vitally important to try and
1475	* keep the use cases distinct. For example, userptr objects are
1476	* not allowed inside the Global GTT as that will cause lock
1477	* inversions when we have to evict them the mmu_notifier callbacks -
1478	* but they are allowed to be part of the user ppGTT which can never
1479	* be mapped. As such we try to give the distinct users of the same
1480	* mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1481	* and i915_ppgtt separate].
1482	*
1483	* NB this may cause us to mask real lock inversions -- while the
1484	* code is safe today, lockdep may not be able to spot future
1485	* transgressions.
1486	*/
1487	err = mutex_lock_interruptible_nested(lock: &vma->vm->mutex,
1488	subclass: !(flags & PIN_GLOBAL));
1489	if (err)
1490	goto err_vma_res;
1491
1492	/* No more allocations allowed now we hold vm->mutex */
1493
1494	if (unlikely(i915_vma_is_closed(vma))) {
1495	err = -ENOENT;
1496	goto err_unlock;
1497	}
1498
1499	bound = atomic_read(v: &vma->flags);
1500	if (unlikely(bound & I915_VMA_ERROR)) {
1501	err = -ENOMEM;
1502	goto err_unlock;
1503	}
1504
1505	if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1506	err = -EAGAIN; /* pins are meant to be fairly temporary */
1507	goto err_unlock;
1508	}
1509
1510	if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1511	if (!(flags & PIN_VALIDATE))
1512	__i915_vma_pin(vma);
1513	goto err_unlock;
1514	}
1515
1516	err = i915_active_acquire(ref: &vma->active);
1517	if (err)
1518	goto err_unlock;
1519
1520	if (!(bound & I915_VMA_BIND_MASK)) {
1521	err = i915_vma_insert(vma, ww, size, alignment, flags);
1522	if (err)
1523	goto err_active;
1524
1525	if (i915_is_ggtt(vma->vm))
1526	__i915_vma_set_map_and_fenceable(vma);
1527	}
1528
1529	GEM_BUG_ON(!vma->pages);
1530	err = i915_vma_bind(vma,
1531	pat_index: vma->obj->pat_index,
1532	flags, work, vma_res);
1533	vma_res = NULL;
1534	if (err)
1535	goto err_remove;
1536
1537	/* There should only be at most 2 active bindings (user, global) */
1538	GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1539	atomic_add(I915_VMA_PAGES_ACTIVE, v: &vma->pages_count);
1540	list_move_tail(list: &vma->vm_link, head: &vma->vm->bound_list);
1541
1542	if (!(flags & PIN_VALIDATE)) {
1543	__i915_vma_pin(vma);
1544	GEM_BUG_ON(!i915_vma_is_pinned(vma));
1545	}
1546	GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1547	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1548
1549	err_remove:
1550	if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1551	i915_vma_detach(vma);
1552	drm_mm_remove_node(node: &vma->node);
1553	}
1554	err_active:
1555	i915_active_release(ref: &vma->active);
1556	err_unlock:
1557	mutex_unlock(lock: &vma->vm->mutex);
1558	err_vma_res:
1559	i915_vma_resource_free(vma_res);
1560	err_fence:
1561	if (work)
1562	dma_fence_work_commit_imm(f: &work->base);
1563	err_rpm:
1564	if (wakeref)
1565	intel_runtime_pm_put(rpm: &vma->vm->i915->runtime_pm, wref: wakeref);
1566
1567	if (moving)
1568	dma_fence_put(fence: moving);
1569
1570	i915_vma_put_pages(vma);
1571	return err;
1572	}
1573
1574	static void flush_idle_contexts(struct intel_gt *gt)
1575	{
1576	struct intel_engine_cs *engine;
1577	enum intel_engine_id id;
1578
1579	for_each_engine(engine, gt, id)
1580	intel_engine_flush_barriers(engine);
1581
1582	intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1583	}
1584
1585	static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1586	u32 align, unsigned int flags)
1587	{
1588	struct i915_address_space *vm = vma->vm;
1589	struct intel_gt *gt;
1590	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
1591	int err;
1592
1593	do {
1594	err = i915_vma_pin_ww(vma, ww, size: 0, alignment: align, flags: flags | PIN_GLOBAL);
1595
1596	if (err != -ENOSPC) {
1597	if (!err) {
1598	err = i915_vma_wait_for_bind(vma);
1599	if (err)
1600	i915_vma_unpin(vma);
1601	}
1602	return err;
1603	}
1604
1605	/* Unlike i915_vma_pin, we don't take no for an answer! */
1606	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1607	flush_idle_contexts(gt);
1608	if (mutex_lock_interruptible(&vm->mutex) == 0) {
1609	/*
1610	* We pass NULL ww here, as we don't want to unbind
1611	* locked objects when called from execbuf when pinning
1612	* is removed. This would probably regress badly.
1613	*/
1614	i915_gem_evict_vm(vm, NULL, NULL);
1615	mutex_unlock(lock: &vm->mutex);
1616	}
1617	} while (1);
1618	}
1619
1620	int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1621	u32 align, unsigned int flags)
1622	{
1623	struct i915_gem_ww_ctx _ww;
1624	int err;
1625
1626	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1627
1628	if (ww)
1629	return __i915_ggtt_pin(vma, ww, align, flags);
1630
1631	lockdep_assert_not_held(&vma->obj->base.resv->lock.base);
1632
1633	for_i915_gem_ww(&_ww, err, true) {
1634	err = i915_gem_object_lock(obj: vma->obj, ww: &_ww);
1635	if (!err)
1636	err = __i915_ggtt_pin(vma, ww: &_ww, align, flags);
1637	}
1638
1639	return err;
1640	}
1641
1642	/**
1643	* i915_ggtt_clear_scanout - Clear scanout flag for all objects ggtt vmas
1644	* @obj: i915 GEM object
1645	* This function clears scanout flags for objects ggtt vmas. These flags are set
1646	* when object is pinned for display use and this function to clear them all is
1647	* targeted to be called by frontbuffer tracking code when the frontbuffer is
1648	* about to be released.
1649	*/
1650	void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj)
1651	{
1652	struct i915_vma *vma;
1653
1654	spin_lock(lock: &obj->vma.lock);
1655	for_each_ggtt_vma(vma, obj) {
1656	i915_vma_clear_scanout(vma);
1657	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
1658	}
1659	spin_unlock(lock: &obj->vma.lock);
1660	}
1661
1662	static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1663	{
1664	/*
1665	* We defer actually closing, unbinding and destroying the VMA until
1666	* the next idle point, or if the object is freed in the meantime. By
1667	* postponing the unbind, we allow for it to be resurrected by the
1668	* client, avoiding the work required to rebind the VMA. This is
1669	* advantageous for DRI, where the client/server pass objects
1670	* between themselves, temporarily opening a local VMA to the
1671	* object, and then closing it again. The same object is then reused
1672	* on the next frame (or two, depending on the depth of the swap queue)
1673	* causing us to rebind the VMA once more. This ends up being a lot
1674	* of wasted work for the steady state.
1675	*/
1676	GEM_BUG_ON(i915_vma_is_closed(vma));
1677	list_add(new: &vma->closed_link, head: &gt->closed_vma);
1678	}
1679
1680	void i915_vma_close(struct i915_vma *vma)
1681	{
1682	struct intel_gt *gt = vma->vm->gt;
1683	unsigned long flags;
1684
1685	if (i915_vma_is_ggtt(vma))
1686	return;
1687
1688	GEM_BUG_ON(!atomic_read(&vma->open_count));
1689	if (atomic_dec_and_lock_irqsave(&vma->open_count,
1690	&gt->closed_lock,
1691	flags)) {
1692	__vma_close(vma, gt);
1693	spin_unlock_irqrestore(lock: &gt->closed_lock, flags);
1694	}
1695	}
1696
1697	static void __i915_vma_remove_closed(struct i915_vma *vma)
1698	{
1699	list_del_init(entry: &vma->closed_link);
1700	}
1701
1702	void i915_vma_reopen(struct i915_vma *vma)
1703	{
1704	struct intel_gt *gt = vma->vm->gt;
1705
1706	spin_lock_irq(lock: &gt->closed_lock);
1707	if (i915_vma_is_closed(vma))
1708	__i915_vma_remove_closed(vma);
1709	spin_unlock_irq(lock: &gt->closed_lock);
1710	}
1711
1712	static void force_unbind(struct i915_vma *vma)
1713	{
1714	if (!drm_mm_node_allocated(node: &vma->node))
1715	return;
1716
1717	atomic_and(i: ~I915_VMA_PIN_MASK, v: &vma->flags);
1718	WARN_ON(__i915_vma_unbind(vma));
1719	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1720	}
1721
1722	static void release_references(struct i915_vma *vma, struct intel_gt *gt,
1723	bool vm_ddestroy)
1724	{
1725	struct drm_i915_gem_object *obj = vma->obj;
1726
1727	GEM_BUG_ON(i915_vma_is_active(vma));
1728
1729	spin_lock(lock: &obj->vma.lock);
1730	list_del(entry: &vma->obj_link);
1731	if (!RB_EMPTY_NODE(&vma->obj_node))
1732	rb_erase(&vma->obj_node, &obj->vma.tree);
1733
1734	spin_unlock(lock: &obj->vma.lock);
1735
1736	spin_lock_irq(lock: &gt->closed_lock);
1737	__i915_vma_remove_closed(vma);
1738	spin_unlock_irq(lock: &gt->closed_lock);
1739
1740	if (vm_ddestroy)
1741	i915_vm_resv_put(vm: vma->vm);
1742
1743	/* Wait for async active retire */
1744	i915_active_wait(ref: &vma->active);
1745	i915_active_fini(ref: &vma->active);
1746	GEM_WARN_ON(vma->resource);
1747	i915_vma_free(vma);
1748	}
1749
1750	/*
1751	* i915_vma_destroy_locked - Remove all weak reference to the vma and put
1752	* the initial reference.
1753	*
1754	* This function should be called when it's decided the vma isn't needed
1755	* anymore. The caller must assure that it doesn't race with another lookup
1756	* plus destroy, typically by taking an appropriate reference.
1757	*
1758	* Current callsites are
1759	* - __i915_gem_object_pages_fini()
1760	* - __i915_vm_close() - Blocks the above function by taking a reference on
1761	* the object.
1762	* - __i915_vma_parked() - Blocks the above functions by taking a reference
1763	* on the vm and a reference on the object. Also takes the object lock so
1764	* destruction from __i915_vma_parked() can be blocked by holding the
1765	* object lock. Since the object lock is only allowed from within i915 with
1766	* an object refcount, holding the object lock also implicitly blocks the
1767	* vma freeing from __i915_gem_object_pages_fini().
1768	*
1769	* Because of locks taken during destruction, a vma is also guaranteed to
1770	* stay alive while the following locks are held if it was looked up while
1771	* holding one of the locks:
1772	* - vm->mutex
1773	* - obj->vma.lock
1774	* - gt->closed_lock
1775	*/
1776	void i915_vma_destroy_locked(struct i915_vma *vma)
1777	{
1778	lockdep_assert_held(&vma->vm->mutex);
1779
1780	force_unbind(vma);
1781	list_del_init(entry: &vma->vm_link);
1782	release_references(vma, gt: vma->vm->gt, vm_ddestroy: false);
1783	}
1784
1785	void i915_vma_destroy(struct i915_vma *vma)
1786	{
1787	struct intel_gt *gt;
1788	bool vm_ddestroy;
1789
1790	mutex_lock(&vma->vm->mutex);
1791	force_unbind(vma);
1792	list_del_init(entry: &vma->vm_link);
1793	vm_ddestroy = vma->vm_ddestroy;
1794	vma->vm_ddestroy = false;
1795
1796	/* vma->vm may be freed when releasing vma->vm->mutex. */
1797	gt = vma->vm->gt;
1798	mutex_unlock(lock: &vma->vm->mutex);
1799	release_references(vma, gt, vm_ddestroy);
1800	}
1801
1802	void i915_vma_parked(struct intel_gt *gt)
1803	{
1804	struct i915_vma *vma, *next;
1805	LIST_HEAD(closed);
1806
1807	spin_lock_irq(lock: &gt->closed_lock);
1808	list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link) {
1809	struct drm_i915_gem_object *obj = vma->obj;
1810	struct i915_address_space *vm = vma->vm;
1811
1812	/* XXX All to avoid keeping a reference on i915_vma itself */
1813
1814	if (!kref_get_unless_zero(kref: &obj->base.refcount))
1815	continue;
1816
1817	if (!i915_vm_tryget(vm)) {
1818	i915_gem_object_put(obj);
1819	continue;
1820	}
1821
1822	list_move(list: &vma->closed_link, head: &closed);
1823	}
1824	spin_unlock_irq(lock: &gt->closed_lock);
1825
1826	/* As the GT is held idle, no vma can be reopened as we destroy them */
1827	list_for_each_entry_safe(vma, next, &closed, closed_link) {
1828	struct drm_i915_gem_object *obj = vma->obj;
1829	struct i915_address_space *vm = vma->vm;
1830
1831	if (i915_gem_object_trylock(obj, NULL)) {
1832	INIT_LIST_HEAD(list: &vma->closed_link);
1833	i915_vma_destroy(vma);
1834	i915_gem_object_unlock(obj);
1835	} else {
1836	/* back you go.. */
1837	spin_lock_irq(lock: &gt->closed_lock);
1838	list_add(new: &vma->closed_link, head: &gt->closed_vma);
1839	spin_unlock_irq(lock: &gt->closed_lock);
1840	}
1841
1842	i915_gem_object_put(obj);
1843	i915_vm_put(vm);
1844	}
1845	}
1846
1847	static void __i915_vma_iounmap(struct i915_vma *vma)
1848	{
1849	GEM_BUG_ON(i915_vma_is_pinned(vma));
1850
1851	if (vma->iomap == NULL)
1852	return;
1853
1854	if (page_unmask_bits(vma->iomap))
1855	__i915_gem_object_release_map(obj: vma->obj);
1856	else
1857	io_mapping_unmap(vaddr: vma->iomap);
1858	vma->iomap = NULL;
1859	}
1860
1861	void i915_vma_revoke_mmap(struct i915_vma *vma)
1862	{
1863	struct drm_vma_offset_node *node;
1864	u64 vma_offset;
1865
1866	if (!i915_vma_has_userfault(vma))
1867	return;
1868
1869	GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1870	GEM_BUG_ON(!vma->obj->userfault_count);
1871
1872	node = &vma->mmo->vma_node;
1873	vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT;
1874	unmap_mapping_range(mapping: vma->vm->i915->drm.anon_inode->i_mapping,
1875	holebegin: drm_vma_node_offset_addr(node) + vma_offset,
1876	holelen: vma->size,
1877	even_cows: 1);
1878
1879	i915_vma_unset_userfault(vma);
1880	if (!--vma->obj->userfault_count)
1881	list_del(entry: &vma->obj->userfault_link);
1882	}
1883
1884	static int
1885	__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1886	{
1887	return __i915_request_await_exclusive(rq, active: &vma->active);
1888	}
1889
1890	static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1891	{
1892	int err;
1893
1894	/* Wait for the vma to be bound before we start! */
1895	err = __i915_request_await_bind(rq, vma);
1896	if (err)
1897	return err;
1898
1899	return i915_active_add_request(ref: &vma->active, rq);
1900	}
1901
1902	int _i915_vma_move_to_active(struct i915_vma *vma,
1903	struct i915_request *rq,
1904	struct dma_fence *fence,
1905	unsigned int flags)
1906	{
1907	struct drm_i915_gem_object *obj = vma->obj;
1908	int err;
1909
1910	assert_object_held(obj);
1911
1912	GEM_BUG_ON(!vma->pages);
1913
1914	if (!(flags & __EXEC_OBJECT_NO_REQUEST_AWAIT)) {
1915	err = i915_request_await_object(to: rq, obj: vma->obj, write: flags & EXEC_OBJECT_WRITE);
1916	if (unlikely(err))
1917	return err;
1918	}
1919	err = __i915_vma_move_to_active(vma, rq);
1920	if (unlikely(err))
1921	return err;
1922
1923	/*
1924	* Reserve fences slot early to prevent an allocation after preparing
1925	* the workload and associating fences with dma_resv.
1926	*/
1927	if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) {
1928	struct dma_fence *curr;
1929	int idx;
1930
1931	dma_fence_array_for_each(curr, idx, fence)
1932	;
1933	err = dma_resv_reserve_fences(obj: vma->obj->base.resv, num_fences: idx);
1934	if (unlikely(err))
1935	return err;
1936	}
1937
1938	if (flags & EXEC_OBJECT_WRITE) {
1939	struct intel_frontbuffer *front;
1940
1941	front = i915_gem_object_get_frontbuffer(obj);
1942	if (unlikely(front)) {
1943	if (intel_frontbuffer_invalidate(front, origin: ORIGIN_CS))
1944	i915_active_add_request(ref: &front->write, rq);
1945	intel_frontbuffer_put(front);
1946	}
1947	}
1948
1949	if (fence) {
1950	struct dma_fence *curr;
1951	enum dma_resv_usage usage;
1952	int idx;
1953
1954	if (flags & EXEC_OBJECT_WRITE) {
1955	usage = DMA_RESV_USAGE_WRITE;
1956	obj->write_domain = I915_GEM_DOMAIN_RENDER;
1957	obj->read_domains = 0;
1958	} else {
1959	usage = DMA_RESV_USAGE_READ;
1960	obj->write_domain = 0;
1961	}
1962
1963	dma_fence_array_for_each(curr, idx, fence)
1964	dma_resv_add_fence(obj: vma->obj->base.resv, fence: curr, usage);
1965	}
1966
1967	if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
1968	i915_active_add_request(ref: &vma->fence->active, rq);
1969
1970	obj->read_domains |= I915_GEM_GPU_DOMAINS;
1971	obj->mm.dirty = true;
1972
1973	GEM_BUG_ON(!i915_vma_is_active(vma));
1974	return 0;
1975	}
1976
1977	struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
1978	{
1979	struct i915_vma_resource *vma_res = vma->resource;
1980	struct dma_fence *unbind_fence;
1981
1982	GEM_BUG_ON(i915_vma_is_pinned(vma));
1983	assert_vma_held_evict(vma);
1984
1985	if (i915_vma_is_map_and_fenceable(vma)) {
1986	/* Force a pagefault for domain tracking on next user access */
1987	i915_vma_revoke_mmap(vma);
1988
1989	/*
1990	* Check that we have flushed all writes through the GGTT
1991	* before the unbind, other due to non-strict nature of those
1992	* indirect writes they may end up referencing the GGTT PTE
1993	* after the unbind.
1994	*
1995	* Note that we may be concurrently poking at the GGTT_WRITE
1996	* bit from set-domain, as we mark all GGTT vma associated
1997	* with an object. We know this is for another vma, as we
1998	* are currently unbinding this one -- so if this vma will be
1999	* reused, it will be refaulted and have its dirty bit set
2000	* before the next write.
2001	*/
2002	i915_vma_flush_writes(vma);
2003
2004	/* release the fence reg _after_ flushing */
2005	i915_vma_revoke_fence(vma);
2006
2007	clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
2008	}
2009
2010	__i915_vma_iounmap(vma);
2011
2012	GEM_BUG_ON(vma->fence);
2013	GEM_BUG_ON(i915_vma_has_userfault(vma));
2014
2015	/* Object backend must be async capable. */
2016	GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
2017
2018	/* If vm is not open, unbind is a nop. */
2019	vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
2020	kref_read(kref: &vma->vm->ref);
2021	vma_res->skip_pte_rewrite = !kref_read(kref: &vma->vm->ref) ||
2022	vma->vm->skip_pte_rewrite;
2023	trace_i915_vma_unbind(vma);
2024
2025	if (async)
2026	unbind_fence = i915_vma_resource_unbind(vma_res,
2027	tlb: vma->obj->mm.tlb);
2028	else
2029	unbind_fence = i915_vma_resource_unbind(vma_res, NULL);
2030
2031	vma->resource = NULL;
2032
2033	atomic_and(i: ~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
2034	v: &vma->flags);
2035
2036	i915_vma_detach(vma);
2037
2038	if (!async) {
2039	if (unbind_fence) {
2040	dma_fence_wait(fence: unbind_fence, intr: false);
2041	dma_fence_put(fence: unbind_fence);
2042	unbind_fence = NULL;
2043	}
2044	vma_invalidate_tlb(vm: vma->vm, tlb: vma->obj->mm.tlb);
2045	}
2046
2047	/*
2048	* Binding itself may not have completed until the unbind fence signals,
2049	* so don't drop the pages until that happens, unless the resource is
2050	* async_capable.
2051	*/
2052
2053	vma_unbind_pages(vma);
2054	return unbind_fence;
2055	}
2056
2057	int __i915_vma_unbind(struct i915_vma *vma)
2058	{
2059	int ret;
2060
2061	lockdep_assert_held(&vma->vm->mutex);
2062	assert_vma_held_evict(vma);
2063
2064	if (!drm_mm_node_allocated(node: &vma->node))
2065	return 0;
2066
2067	if (i915_vma_is_pinned(vma)) {
2068	vma_print_allocator(vma, reason: "is pinned");
2069	return -EAGAIN;
2070	}
2071
2072	/*
2073	* After confirming that no one else is pinning this vma, wait for
2074	* any laggards who may have crept in during the wait (through
2075	* a residual pin skipping the vm->mutex) to complete.
2076	*/
2077	ret = i915_vma_sync(vma);
2078	if (ret)
2079	return ret;
2080
2081	GEM_BUG_ON(i915_vma_is_active(vma));
2082	__i915_vma_evict(vma, async: false);
2083
2084	drm_mm_remove_node(node: &vma->node); /* pairs with i915_vma_release() */
2085	return 0;
2086	}
2087
2088	static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
2089	{
2090	struct dma_fence *fence;
2091
2092	lockdep_assert_held(&vma->vm->mutex);
2093
2094	if (!drm_mm_node_allocated(node: &vma->node))
2095	return NULL;
2096
2097	if (i915_vma_is_pinned(vma) ||
2098	&vma->obj->mm.rsgt->table != vma->resource->bi.pages)
2099	return ERR_PTR(error: -EAGAIN);
2100
2101	/*
2102	* We probably need to replace this with awaiting the fences of the
2103	* object's dma_resv when the vma active goes away. When doing that
2104	* we need to be careful to not add the vma_resource unbind fence
2105	* immediately to the object's dma_resv, because then unbinding
2106	* the next vma from the object, in case there are many, will
2107	* actually await the unbinding of the previous vmas, which is
2108	* undesirable.
2109	*/
2110	if (i915_sw_fence_await_active(fence: &vma->resource->chain, ref: &vma->active,
2111	I915_ACTIVE_AWAIT_EXCL |
2112	I915_ACTIVE_AWAIT_ACTIVE) < 0) {
2113	return ERR_PTR(error: -EBUSY);
2114	}
2115
2116	fence = __i915_vma_evict(vma, async: true);
2117
2118	drm_mm_remove_node(node: &vma->node); /* pairs with i915_vma_release() */
2119
2120	return fence;
2121	}
2122
2123	int i915_vma_unbind(struct i915_vma *vma)
2124	{
2125	struct i915_address_space *vm = vma->vm;
2126	intel_wakeref_t wakeref = 0;
2127	int err;
2128
2129	assert_object_held_shared(obj: vma->obj);
2130
2131	/* Optimistic wait before taking the mutex */
2132	err = i915_vma_sync(vma);
2133	if (err)
2134	return err;
2135
2136	if (!drm_mm_node_allocated(node: &vma->node))
2137	return 0;
2138
2139	if (i915_vma_is_pinned(vma)) {
2140	vma_print_allocator(vma, reason: "is pinned");
2141	return -EAGAIN;
2142	}
2143
2144	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2145	/* XXX not always required: nop_clear_range */
2146	wakeref = intel_runtime_pm_get(rpm: &vm->i915->runtime_pm);
2147
2148	err = mutex_lock_interruptible_nested(lock: &vma->vm->mutex, subclass: !wakeref);
2149	if (err)
2150	goto out_rpm;
2151
2152	err = __i915_vma_unbind(vma);
2153	mutex_unlock(lock: &vm->mutex);
2154
2155	out_rpm:
2156	if (wakeref)
2157	intel_runtime_pm_put(rpm: &vm->i915->runtime_pm, wref: wakeref);
2158	return err;
2159	}
2160
2161	int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
2162	{
2163	struct drm_i915_gem_object *obj = vma->obj;
2164	struct i915_address_space *vm = vma->vm;
2165	intel_wakeref_t wakeref = 0;
2166	struct dma_fence *fence;
2167	int err;
2168
2169	/*
2170	* We need the dma-resv lock since we add the
2171	* unbind fence to the dma-resv object.
2172	*/
2173	assert_object_held(obj);
2174
2175	if (!drm_mm_node_allocated(node: &vma->node))
2176	return 0;
2177
2178	if (i915_vma_is_pinned(vma)) {
2179	vma_print_allocator(vma, reason: "is pinned");
2180	return -EAGAIN;
2181	}
2182
2183	if (!obj->mm.rsgt)
2184	return -EBUSY;
2185
2186	err = dma_resv_reserve_fences(obj: obj->base.resv, num_fences: 2);
2187	if (err)
2188	return -EBUSY;
2189
2190	/*
2191	* It would be great if we could grab this wakeref from the
2192	* async unbind work if needed, but we can't because it uses
2193	* kmalloc and it's in the dma-fence signalling critical path.
2194	*/
2195	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2196	wakeref = intel_runtime_pm_get(rpm: &vm->i915->runtime_pm);
2197
2198	if (trylock_vm && !mutex_trylock(lock: &vm->mutex)) {
2199	err = -EBUSY;
2200	goto out_rpm;
2201	} else if (!trylock_vm) {
2202	err = mutex_lock_interruptible_nested(lock: &vm->mutex, subclass: !wakeref);
2203	if (err)
2204	goto out_rpm;
2205	}
2206
2207	fence = __i915_vma_unbind_async(vma);
2208	mutex_unlock(lock: &vm->mutex);
2209	if (IS_ERR_OR_NULL(ptr: fence)) {
2210	err = PTR_ERR_OR_ZERO(ptr: fence);
2211	goto out_rpm;
2212	}
2213
2214	dma_resv_add_fence(obj: obj->base.resv, fence, usage: DMA_RESV_USAGE_READ);
2215	dma_fence_put(fence);
2216
2217	out_rpm:
2218	if (wakeref)
2219	intel_runtime_pm_put(rpm: &vm->i915->runtime_pm, wref: wakeref);
2220	return err;
2221	}
2222
2223	int i915_vma_unbind_unlocked(struct i915_vma *vma)
2224	{
2225	int err;
2226
2227	i915_gem_object_lock(obj: vma->obj, NULL);
2228	err = i915_vma_unbind(vma);
2229	i915_gem_object_unlock(obj: vma->obj);
2230
2231	return err;
2232	}
2233
2234	struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
2235	{
2236	i915_gem_object_make_unshrinkable(obj: vma->obj);
2237	return vma;
2238	}
2239
2240	void i915_vma_make_shrinkable(struct i915_vma *vma)
2241	{
2242	i915_gem_object_make_shrinkable(obj: vma->obj);
2243	}
2244
2245	void i915_vma_make_purgeable(struct i915_vma *vma)
2246	{
2247	i915_gem_object_make_purgeable(obj: vma->obj);
2248	}
2249
2250	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2251	#include "selftests/i915_vma.c"
2252	#endif
2253
2254	void i915_vma_module_exit(void)
2255	{
2256	kmem_cache_destroy(s: slab_vmas);
2257	}
2258
2259	int __init i915_vma_module_init(void)
2260	{
2261	slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2262	if (!slab_vmas)
2263	return -ENOMEM;
2264
2265	return 0;
2266	}
2267