1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* GICv3 ITS emulation
4	*
5	* Copyright (C) 2015,2016 ARM Ltd.
6	* Author: Andre Przywara <andre.przywara@arm.com>
7	*/
8
9	#include <linux/cpu.h>
10	#include <linux/kvm.h>
11	#include <linux/kvm_host.h>
12	#include <linux/interrupt.h>
13	#include <linux/list.h>
14	#include <linux/uaccess.h>
15	#include <linux/list_sort.h>
16
17	#include <linux/irqchip/arm-gic-v3.h>
18
19	#include <asm/kvm_emulate.h>
20	#include <asm/kvm_arm.h>
21	#include <asm/kvm_mmu.h>
22
23	#include "vgic.h"
24	#include "vgic-mmio.h"
25
26	static int vgic_its_save_tables_v0(struct vgic_its *its);
27	static int vgic_its_restore_tables_v0(struct vgic_its *its);
28	static int vgic_its_commit_v0(struct vgic_its *its);
29	static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
30	struct kvm_vcpu *filter_vcpu, bool needs_inv);
31
32	/*
33	* Creates a new (reference to a) struct vgic_irq for a given LPI.
34	* If this LPI is already mapped on another ITS, we increase its refcount
35	* and return a pointer to the existing structure.
36	* If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
37	* This function returns a pointer to the _unlocked_ structure.
38	*/
39	static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
40	struct kvm_vcpu *vcpu)
41	{
42	struct vgic_dist *dist = &kvm->arch.vgic;
43	struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
44	unsigned long flags;
45	int ret;
46
47	/* In this case there is no put, since we keep the reference. */
48	if (irq)
49	return irq;
50
51	irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT);
52	if (!irq)
53	return ERR_PTR(error: -ENOMEM);
54
55	ret = xa_reserve_irq(xa: &dist->lpi_xa, index: intid, GFP_KERNEL_ACCOUNT);
56	if (ret) {
57	kfree(objp: irq);
58	return ERR_PTR(error: ret);
59	}
60
61	INIT_LIST_HEAD(list: &irq->ap_list);
62	raw_spin_lock_init(&irq->irq_lock);
63
64	irq->config = VGIC_CONFIG_EDGE;
65	kref_init(kref: &irq->refcount);
66	irq->intid = intid;
67	irq->target_vcpu = vcpu;
68	irq->group = 1;
69
70	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
71
72	/*
73	* There could be a race with another vgic_add_lpi(), so we need to
74	* check that we don't add a second list entry with the same LPI.
75	*/
76	oldirq = xa_load(&dist->lpi_xa, index: intid);
77	if (vgic_try_get_irq_kref(irq: oldirq)) {
78	/* Someone was faster with adding this LPI, lets use that. */
79	kfree(objp: irq);
80	irq = oldirq;
81
82	goto out_unlock;
83	}
84
85	ret = xa_err(entry: xa_store(&dist->lpi_xa, index: intid, entry: irq, 0));
86	if (ret) {
87	xa_release(xa: &dist->lpi_xa, index: intid);
88	kfree(objp: irq);
89	goto out_unlock;
90	}
91
92	atomic_inc(v: &dist->lpi_count);
93
94	out_unlock:
95	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
96
97	if (ret)
98	return ERR_PTR(error: ret);
99
100	/*
101	* We "cache" the configuration table entries in our struct vgic_irq's.
102	* However we only have those structs for mapped IRQs, so we read in
103	* the respective config data from memory here upon mapping the LPI.
104	*
105	* Should any of these fail, behave as if we couldn't create the LPI
106	* by dropping the refcount and returning the error.
107	*/
108	ret = update_lpi_config(kvm, irq, NULL, needs_inv: false);
109	if (ret) {
110	vgic_put_irq(kvm, irq);
111	return ERR_PTR(error: ret);
112	}
113
114	ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
115	if (ret) {
116	vgic_put_irq(kvm, irq);
117	return ERR_PTR(error: ret);
118	}
119
120	return irq;
121	}
122
123	struct its_device {
124	struct list_head dev_list;
125
126	/* the head for the list of ITTEs */
127	struct list_head itt_head;
128	u32 num_eventid_bits;
129	gpa_t itt_addr;
130	u32 device_id;
131	};
132
133	#define COLLECTION_NOT_MAPPED ((u32)~0)
134
135	struct its_collection {
136	struct list_head coll_list;
137
138	u32 collection_id;
139	u32 target_addr;
140	};
141
142	#define its_is_collection_mapped(coll) ((coll) && \
143	((coll)->target_addr != COLLECTION_NOT_MAPPED))
144
145	struct its_ite {
146	struct list_head ite_list;
147
148	struct vgic_irq *irq;
149	struct its_collection *collection;
150	u32 event_id;
151	};
152
153	struct vgic_translation_cache_entry {
154	struct list_head entry;
155	phys_addr_t db;
156	u32 devid;
157	u32 eventid;
158	struct vgic_irq *irq;
159	};
160
161	/**
162	* struct vgic_its_abi - ITS abi ops and settings
163	* @cte_esz: collection table entry size
164	* @dte_esz: device table entry size
165	* @ite_esz: interrupt translation table entry size
166	* @save_tables: save the ITS tables into guest RAM
167	* @restore_tables: restore the ITS internal structs from tables
168	* stored in guest RAM
169	* @commit: initialize the registers which expose the ABI settings,
170	* especially the entry sizes
171	*/
172	struct vgic_its_abi {
173	int cte_esz;
174	int dte_esz;
175	int ite_esz;
176	int (*save_tables)(struct vgic_its *its);
177	int (*restore_tables)(struct vgic_its *its);
178	int (*commit)(struct vgic_its *its);
179	};
180
181	#define ABI_0_ESZ 8
182	#define ESZ_MAX ABI_0_ESZ
183
184	static const struct vgic_its_abi its_table_abi_versions[] = {
185	[0] = {
186	.cte_esz = ABI_0_ESZ,
187	.dte_esz = ABI_0_ESZ,
188	.ite_esz = ABI_0_ESZ,
189	.save_tables = vgic_its_save_tables_v0,
190	.restore_tables = vgic_its_restore_tables_v0,
191	.commit = vgic_its_commit_v0,
192	},
193	};
194
195	#define NR_ITS_ABIS ARRAY_SIZE(its_table_abi_versions)
196
197	inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
198	{
199	return &its_table_abi_versions[its->abi_rev];
200	}
201
202	static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
203	{
204	const struct vgic_its_abi *abi;
205
206	its->abi_rev = rev;
207	abi = vgic_its_get_abi(its);
208	return abi->commit(its);
209	}
210
211	/*
212	* Find and returns a device in the device table for an ITS.
213	* Must be called with the its_lock mutex held.
214	*/
215	static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
216	{
217	struct its_device *device;
218
219	list_for_each_entry(device, &its->device_list, dev_list)
220	if (device_id == device->device_id)
221	return device;
222
223	return NULL;
224	}
225
226	/*
227	* Find and returns an interrupt translation table entry (ITTE) for a given
228	* Device ID/Event ID pair on an ITS.
229	* Must be called with the its_lock mutex held.
230	*/
231	static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
232	u32 event_id)
233	{
234	struct its_device *device;
235	struct its_ite *ite;
236
237	device = find_its_device(its, device_id);
238	if (device == NULL)
239	return NULL;
240
241	list_for_each_entry(ite, &device->itt_head, ite_list)
242	if (ite->event_id == event_id)
243	return ite;
244
245	return NULL;
246	}
247
248	/* To be used as an iterator this macro misses the enclosing parentheses */
249	#define for_each_lpi_its(dev, ite, its) \
250	list_for_each_entry(dev, &(its)->device_list, dev_list) \
251	list_for_each_entry(ite, &(dev)->itt_head, ite_list)
252
253	#define GIC_LPI_OFFSET 8192
254
255	#define VITS_TYPER_IDBITS 16
256	#define VITS_TYPER_DEVBITS 16
257	#define VITS_DTE_MAX_DEVID_OFFSET (BIT(14) - 1)
258	#define VITS_ITE_MAX_EVENTID_OFFSET (BIT(16) - 1)
259
260	/*
261	* Finds and returns a collection in the ITS collection table.
262	* Must be called with the its_lock mutex held.
263	*/
264	static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
265	{
266	struct its_collection *collection;
267
268	list_for_each_entry(collection, &its->collection_list, coll_list) {
269	if (coll_id == collection->collection_id)
270	return collection;
271	}
272
273	return NULL;
274	}
275
276	#define LPI_PROP_ENABLE_BIT(p) ((p) & LPI_PROP_ENABLED)
277	#define LPI_PROP_PRIORITY(p) ((p) & 0xfc)
278
279	/*
280	* Reads the configuration data for a given LPI from guest memory and
281	* updates the fields in struct vgic_irq.
282	* If filter_vcpu is not NULL, applies only if the IRQ is targeting this
283	* VCPU. Unconditionally applies if filter_vcpu is NULL.
284	*/
285	static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
286	struct kvm_vcpu *filter_vcpu, bool needs_inv)
287	{
288	u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
289	u8 prop;
290	int ret;
291	unsigned long flags;
292
293	ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
294	&prop, 1);
295
296	if (ret)
297	return ret;
298
299	raw_spin_lock_irqsave(&irq->irq_lock, flags);
300
301	if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
302	irq->priority = LPI_PROP_PRIORITY(prop);
303	irq->enabled = LPI_PROP_ENABLE_BIT(prop);
304
305	if (!irq->hw) {
306	vgic_queue_irq_unlock(kvm, irq, flags);
307	return 0;
308	}
309	}
310
311	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
312
313	if (irq->hw)
314	return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
315
316	return 0;
317	}
318
319	#define GIC_LPI_MAX_INTID ((1 << INTERRUPT_ID_BITS_ITS) - 1)
320
321	/*
322	* Create a snapshot of the current LPIs targeting @vcpu, so that we can
323	* enumerate those LPIs without holding any lock.
324	* Returns their number and puts the kmalloc'ed array into intid_ptr.
325	*/
326	int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)
327	{
328	struct vgic_dist *dist = &kvm->arch.vgic;
329	XA_STATE(xas, &dist->lpi_xa, GIC_LPI_OFFSET);
330	struct vgic_irq *irq;
331	unsigned long flags;
332	u32 *intids;
333	int irq_count, i = 0;
334
335	/*
336	* There is an obvious race between allocating the array and LPIs
337	* being mapped/unmapped. If we ended up here as a result of a
338	* command, we're safe (locks are held, preventing another
339	* command). If coming from another path (such as enabling LPIs),
340	* we must be careful not to overrun the array.
341	*/
342	irq_count = atomic_read(v: &dist->lpi_count);
343	intids = kmalloc_array(n: irq_count, size: sizeof(intids[0]), GFP_KERNEL_ACCOUNT);
344	if (!intids)
345	return -ENOMEM;
346
347	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
348	rcu_read_lock();
349
350	xas_for_each(&xas, irq, GIC_LPI_MAX_INTID) {
351	if (i == irq_count)
352	break;
353	/* We don't need to "get" the IRQ, as we hold the list lock. */
354	if (vcpu && irq->target_vcpu != vcpu)
355	continue;
356	intids[i++] = irq->intid;
357	}
358
359	rcu_read_unlock();
360	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
361
362	*intid_ptr = intids;
363	return i;
364	}
365
366	static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
367	{
368	int ret = 0;
369	unsigned long flags;
370
371	raw_spin_lock_irqsave(&irq->irq_lock, flags);
372	irq->target_vcpu = vcpu;
373	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
374
375	if (irq->hw) {
376	struct its_vlpi_map map;
377
378	ret = its_get_vlpi(irq->host_irq, &map);
379	if (ret)
380	return ret;
381
382	if (map.vpe)
383	atomic_dec(v: &map.vpe->vlpi_count);
384	map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
385	atomic_inc(v: &map.vpe->vlpi_count);
386
387	ret = its_map_vlpi(irq->host_irq, &map);
388	}
389
390	return ret;
391	}
392
393	static struct kvm_vcpu *collection_to_vcpu(struct kvm *kvm,
394	struct its_collection *col)
395	{
396	return kvm_get_vcpu_by_id(kvm, id: col->target_addr);
397	}
398
399	/*
400	* Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
401	* is targeting) to the VGIC's view, which deals with target VCPUs.
402	* Needs to be called whenever either the collection for a LPIs has
403	* changed or the collection itself got retargeted.
404	*/
405	static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
406	{
407	struct kvm_vcpu *vcpu;
408
409	if (!its_is_collection_mapped(ite->collection))
410	return;
411
412	vcpu = collection_to_vcpu(kvm, col: ite->collection);
413	update_affinity(irq: ite->irq, vcpu);
414	}
415
416	/*
417	* Updates the target VCPU for every LPI targeting this collection.
418	* Must be called with the its_lock mutex held.
419	*/
420	static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
421	struct its_collection *coll)
422	{
423	struct its_device *device;
424	struct its_ite *ite;
425
426	for_each_lpi_its(device, ite, its) {
427	if (ite->collection != coll)
428	continue;
429
430	update_affinity_ite(kvm, ite);
431	}
432	}
433
434	static u32 max_lpis_propbaser(u64 propbaser)
435	{
436	int nr_idbits = (propbaser & 0x1f) + 1;
437
438	return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
439	}
440
441	/*
442	* Sync the pending table pending bit of LPIs targeting @vcpu
443	* with our own data structures. This relies on the LPI being
444	* mapped before.
445	*/
446	static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
447	{
448	gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
449	struct vgic_irq *irq;
450	int last_byte_offset = -1;
451	int ret = 0;
452	u32 *intids;
453	int nr_irqs, i;
454	unsigned long flags;
455	u8 pendmask;
456
457	nr_irqs = vgic_copy_lpi_list(kvm: vcpu->kvm, vcpu, intid_ptr: &intids);
458	if (nr_irqs < 0)
459	return nr_irqs;
460
461	for (i = 0; i < nr_irqs; i++) {
462	int byte_offset, bit_nr;
463
464	byte_offset = intids[i] / BITS_PER_BYTE;
465	bit_nr = intids[i] % BITS_PER_BYTE;
466
467	/*
468	* For contiguously allocated LPIs chances are we just read
469	* this very same byte in the last iteration. Reuse that.
470	*/
471	if (byte_offset != last_byte_offset) {
472	ret = kvm_read_guest_lock(vcpu->kvm,
473	pendbase + byte_offset,
474	&pendmask, 1);
475	if (ret) {
476	kfree(objp: intids);
477	return ret;
478	}
479	last_byte_offset = byte_offset;
480	}
481
482	irq = vgic_get_irq(kvm: vcpu->kvm, NULL, intid: intids[i]);
483	if (!irq)
484	continue;
485
486	raw_spin_lock_irqsave(&irq->irq_lock, flags);
487	irq->pending_latch = pendmask & (1U << bit_nr);
488	vgic_queue_irq_unlock(kvm: vcpu->kvm, irq, flags);
489	vgic_put_irq(kvm: vcpu->kvm, irq);
490	}
491
492	kfree(objp: intids);
493
494	return ret;
495	}
496
497	static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
498	struct vgic_its *its,
499	gpa_t addr, unsigned int len)
500	{
501	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
502	u64 reg = GITS_TYPER_PLPIS;
503
504	/*
505	* We use linear CPU numbers for redistributor addressing,
506	* so GITS_TYPER.PTA is 0.
507	* Also we force all PROPBASER registers to be the same, so
508	* CommonLPIAff is 0 as well.
509	* To avoid memory waste in the guest, we keep the number of IDBits and
510	* DevBits low - as least for the time being.
511	*/
512	reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
513	reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
514	reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
515
516	return extract_bytes(data: reg, offset: addr & 7, num: len);
517	}
518
519	static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
520	struct vgic_its *its,
521	gpa_t addr, unsigned int len)
522	{
523	u32 val;
524
525	val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
526	val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
527	return val;
528	}
529
530	static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
531	struct vgic_its *its,
532	gpa_t addr, unsigned int len,
533	unsigned long val)
534	{
535	u32 rev = GITS_IIDR_REV(val);
536
537	if (rev >= NR_ITS_ABIS)
538	return -EINVAL;
539	return vgic_its_set_abi(its, rev);
540	}
541
542	static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
543	struct vgic_its *its,
544	gpa_t addr, unsigned int len)
545	{
546	switch (addr & 0xffff) {
547	case GITS_PIDR0:
548	return 0x92; /* part number, bits[7:0] */
549	case GITS_PIDR1:
550	return 0xb4; /* part number, bits[11:8] */
551	case GITS_PIDR2:
552	return GIC_PIDR2_ARCH_GICv3 | 0x0b;
553	case GITS_PIDR4:
554	return 0x40; /* This is a 64K software visible page */
555	/* The following are the ID registers for (any) GIC. */
556	case GITS_CIDR0:
557	return 0x0d;
558	case GITS_CIDR1:
559	return 0xf0;
560	case GITS_CIDR2:
561	return 0x05;
562	case GITS_CIDR3:
563	return 0xb1;
564	}
565
566	return 0;
567	}
568
569	static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,
570	phys_addr_t db,
571	u32 devid, u32 eventid)
572	{
573	struct vgic_translation_cache_entry *cte;
574
575	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
576	/*
577	* If we hit a NULL entry, there is nothing after this
578	* point.
579	*/
580	if (!cte->irq)
581	break;
582
583	if (cte->db != db || cte->devid != devid ||
584	cte->eventid != eventid)
585	continue;
586
587	/*
588	* Move this entry to the head, as it is the most
589	* recently used.
590	*/
591	if (!list_is_first(list: &cte->entry, head: &dist->lpi_translation_cache))
592	list_move(list: &cte->entry, head: &dist->lpi_translation_cache);
593
594	return cte->irq;
595	}
596
597	return NULL;
598	}
599
600	static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,
601	u32 devid, u32 eventid)
602	{
603	struct vgic_dist *dist = &kvm->arch.vgic;
604	struct vgic_irq *irq;
605	unsigned long flags;
606
607	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
608
609	irq = __vgic_its_check_cache(dist, db, devid, eventid);
610	if (!vgic_try_get_irq_kref(irq))
611	irq = NULL;
612
613	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
614
615	return irq;
616	}
617
618	static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
619	u32 devid, u32 eventid,
620	struct vgic_irq *irq)
621	{
622	struct vgic_dist *dist = &kvm->arch.vgic;
623	struct vgic_translation_cache_entry *cte;
624	unsigned long flags;
625	phys_addr_t db;
626
627	/* Do not cache a directly injected interrupt */
628	if (irq->hw)
629	return;
630
631	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
632
633	if (unlikely(list_empty(&dist->lpi_translation_cache)))
634	goto out;
635
636	/*
637	* We could have raced with another CPU caching the same
638	* translation behind our back, so let's check it is not in
639	* already
640	*/
641	db = its->vgic_its_base + GITS_TRANSLATER;
642	if (__vgic_its_check_cache(dist, db, devid, eventid))
643	goto out;
644
645	/* Always reuse the last entry (LRU policy) */
646	cte = list_last_entry(&dist->lpi_translation_cache,
647	typeof(*cte), entry);
648
649	/*
650	* Caching the translation implies having an extra reference
651	* to the interrupt, so drop the potential reference on what
652	* was in the cache, and increment it on the new interrupt.
653	*/
654	if (cte->irq)
655	vgic_put_irq(kvm, irq: cte->irq);
656
657	/*
658	* The irq refcount is guaranteed to be nonzero while holding the
659	* its_lock, as the ITE (and the reference it holds) cannot be freed.
660	*/
661	lockdep_assert_held(&its->its_lock);
662	vgic_get_irq_kref(irq);
663
664	cte->db = db;
665	cte->devid = devid;
666	cte->eventid = eventid;
667	cte->irq = irq;
668
669	/* Move the new translation to the head of the list */
670	list_move(list: &cte->entry, head: &dist->lpi_translation_cache);
671
672	out:
673	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
674	}
675
676	void vgic_its_invalidate_cache(struct kvm *kvm)
677	{
678	struct vgic_dist *dist = &kvm->arch.vgic;
679	struct vgic_translation_cache_entry *cte;
680	unsigned long flags;
681
682	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
683
684	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
685	/*
686	* If we hit a NULL entry, there is nothing after this
687	* point.
688	*/
689	if (!cte->irq)
690	break;
691
692	vgic_put_irq(kvm, irq: cte->irq);
693	cte->irq = NULL;
694	}
695
696	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
697	}
698
699	int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
700	u32 devid, u32 eventid, struct vgic_irq **irq)
701	{
702	struct kvm_vcpu *vcpu;
703	struct its_ite *ite;
704
705	if (!its->enabled)
706	return -EBUSY;
707
708	ite = find_ite(its, device_id: devid, event_id: eventid);
709	if (!ite || !its_is_collection_mapped(ite->collection))
710	return E_ITS_INT_UNMAPPED_INTERRUPT;
711
712	vcpu = collection_to_vcpu(kvm, col: ite->collection);
713	if (!vcpu)
714	return E_ITS_INT_UNMAPPED_INTERRUPT;
715
716	if (!vgic_lpis_enabled(vcpu))
717	return -EBUSY;
718
719	vgic_its_cache_translation(kvm, its, devid, eventid, irq: ite->irq);
720
721	*irq = ite->irq;
722	return 0;
723	}
724
725	struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
726	{
727	u64 address;
728	struct kvm_io_device *kvm_io_dev;
729	struct vgic_io_device *iodev;
730
731	if (!vgic_has_its(kvm))
732	return ERR_PTR(error: -ENODEV);
733
734	if (!(msi->flags & KVM_MSI_VALID_DEVID))
735	return ERR_PTR(error: -EINVAL);
736
737	address = (u64)msi->address_hi << 32 | msi->address_lo;
738
739	kvm_io_dev = kvm_io_bus_get_dev(kvm, bus_idx: KVM_MMIO_BUS, addr: address);
740	if (!kvm_io_dev)
741	return ERR_PTR(error: -EINVAL);
742
743	if (kvm_io_dev->ops != &kvm_io_gic_ops)
744	return ERR_PTR(error: -EINVAL);
745
746	iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
747	if (iodev->iodev_type != IODEV_ITS)
748	return ERR_PTR(error: -EINVAL);
749
750	return iodev->its;
751	}
752
753	/*
754	* Find the target VCPU and the LPI number for a given devid/eventid pair
755	* and make this IRQ pending, possibly injecting it.
756	* Must be called with the its_lock mutex held.
757	* Returns 0 on success, a positive error value for any ITS mapping
758	* related errors and negative error values for generic errors.
759	*/
760	static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
761	u32 devid, u32 eventid)
762	{
763	struct vgic_irq *irq = NULL;
764	unsigned long flags;
765	int err;
766
767	err = vgic_its_resolve_lpi(kvm, its, devid, eventid, irq: &irq);
768	if (err)
769	return err;
770
771	if (irq->hw)
772	return irq_set_irqchip_state(irq: irq->host_irq,
773	which: IRQCHIP_STATE_PENDING, state: true);
774
775	raw_spin_lock_irqsave(&irq->irq_lock, flags);
776	irq->pending_latch = true;
777	vgic_queue_irq_unlock(kvm, irq, flags);
778
779	return 0;
780	}
781
782	int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi)
783	{
784	struct vgic_irq *irq;
785	unsigned long flags;
786	phys_addr_t db;
787
788	db = (u64)msi->address_hi << 32 | msi->address_lo;
789	irq = vgic_its_check_cache(kvm, db, devid: msi->devid, eventid: msi->data);
790	if (!irq)
791	return -EWOULDBLOCK;
792
793	raw_spin_lock_irqsave(&irq->irq_lock, flags);
794	irq->pending_latch = true;
795	vgic_queue_irq_unlock(kvm, irq, flags);
796	vgic_put_irq(kvm, irq);
797
798	return 0;
799	}
800
801	/*
802	* Queries the KVM IO bus framework to get the ITS pointer from the given
803	* doorbell address.
804	* We then call vgic_its_trigger_msi() with the decoded data.
805	* According to the KVM_SIGNAL_MSI API description returns 1 on success.
806	*/
807	int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
808	{
809	struct vgic_its *its;
810	int ret;
811
812	if (!vgic_its_inject_cached_translation(kvm, msi))
813	return 1;
814
815	its = vgic_msi_to_its(kvm, msi);
816	if (IS_ERR(ptr: its))
817	return PTR_ERR(ptr: its);
818
819	mutex_lock(&its->its_lock);
820	ret = vgic_its_trigger_msi(kvm, its, devid: msi->devid, eventid: msi->data);
821	mutex_unlock(lock: &its->its_lock);
822
823	if (ret < 0)
824	return ret;
825
826	/*
827	* KVM_SIGNAL_MSI demands a return value > 0 for success and 0
828	* if the guest has blocked the MSI. So we map any LPI mapping
829	* related error to that.
830	*/
831	if (ret)
832	return 0;
833	else
834	return 1;
835	}
836
837	/* Requires the its_lock to be held. */
838	static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
839	{
840	list_del(entry: &ite->ite_list);
841
842	/* This put matches the get in vgic_add_lpi. */
843	if (ite->irq) {
844	if (ite->irq->hw)
845	WARN_ON(its_unmap_vlpi(ite->irq->host_irq));
846
847	vgic_put_irq(kvm, irq: ite->irq);
848	}
849
850	kfree(objp: ite);
851	}
852
853	static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
854	{
855	return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
856	}
857
858	#define its_cmd_get_command(cmd) its_cmd_mask_field(cmd, 0, 0, 8)
859	#define its_cmd_get_deviceid(cmd) its_cmd_mask_field(cmd, 0, 32, 32)
860	#define its_cmd_get_size(cmd) (its_cmd_mask_field(cmd, 1, 0, 5) + 1)
861	#define its_cmd_get_id(cmd) its_cmd_mask_field(cmd, 1, 0, 32)
862	#define its_cmd_get_physical_id(cmd) its_cmd_mask_field(cmd, 1, 32, 32)
863	#define its_cmd_get_collection(cmd) its_cmd_mask_field(cmd, 2, 0, 16)
864	#define its_cmd_get_ittaddr(cmd) (its_cmd_mask_field(cmd, 2, 8, 44) << 8)
865	#define its_cmd_get_target_addr(cmd) its_cmd_mask_field(cmd, 2, 16, 32)
866	#define its_cmd_get_validbit(cmd) its_cmd_mask_field(cmd, 2, 63, 1)
867
868	/*
869	* The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
870	* Must be called with the its_lock mutex held.
871	*/
872	static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
873	u64 *its_cmd)
874	{
875	u32 device_id = its_cmd_get_deviceid(its_cmd);
876	u32 event_id = its_cmd_get_id(its_cmd);
877	struct its_ite *ite;
878
879	ite = find_ite(its, device_id, event_id);
880	if (ite && its_is_collection_mapped(ite->collection)) {
881	/*
882	* Though the spec talks about removing the pending state, we
883	* don't bother here since we clear the ITTE anyway and the
884	* pending state is a property of the ITTE struct.
885	*/
886	vgic_its_invalidate_cache(kvm);
887
888	its_free_ite(kvm, ite);
889	return 0;
890	}
891
892	return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
893	}
894
895	/*
896	* The MOVI command moves an ITTE to a different collection.
897	* Must be called with the its_lock mutex held.
898	*/
899	static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
900	u64 *its_cmd)
901	{
902	u32 device_id = its_cmd_get_deviceid(its_cmd);
903	u32 event_id = its_cmd_get_id(its_cmd);
904	u32 coll_id = its_cmd_get_collection(its_cmd);
905	struct kvm_vcpu *vcpu;
906	struct its_ite *ite;
907	struct its_collection *collection;
908
909	ite = find_ite(its, device_id, event_id);
910	if (!ite)
911	return E_ITS_MOVI_UNMAPPED_INTERRUPT;
912
913	if (!its_is_collection_mapped(ite->collection))
914	return E_ITS_MOVI_UNMAPPED_COLLECTION;
915
916	collection = find_collection(its, coll_id);
917	if (!its_is_collection_mapped(collection))
918	return E_ITS_MOVI_UNMAPPED_COLLECTION;
919
920	ite->collection = collection;
921	vcpu = collection_to_vcpu(kvm, col: collection);
922
923	vgic_its_invalidate_cache(kvm);
924
925	return update_affinity(irq: ite->irq, vcpu);
926	}
927
928	static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
929	{
930	gfn_t gfn = gpa >> PAGE_SHIFT;
931	int idx;
932	bool ret;
933
934	idx = srcu_read_lock(ssp: &its->dev->kvm->srcu);
935	ret = kvm_is_visible_gfn(kvm: its->dev->kvm, gfn);
936	srcu_read_unlock(ssp: &its->dev->kvm->srcu, idx);
937	return ret;
938	}
939
940	/*
941	* Check whether an ID can be stored into the corresponding guest table.
942	* For a direct table this is pretty easy, but gets a bit nasty for
943	* indirect tables. We check whether the resulting guest physical address
944	* is actually valid (covered by a memslot and guest accessible).
945	* For this we have to read the respective first level entry.
946	*/
947	static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
948	gpa_t *eaddr)
949	{
950	int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
951	u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
952	phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser);
953	int esz = GITS_BASER_ENTRY_SIZE(baser);
954	int index;
955
956	switch (type) {
957	case GITS_BASER_TYPE_DEVICE:
958	if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
959	return false;
960	break;
961	case GITS_BASER_TYPE_COLLECTION:
962	/* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
963	if (id >= BIT_ULL(16))
964	return false;
965	break;
966	default:
967	return false;
968	}
969
970	if (!(baser & GITS_BASER_INDIRECT)) {
971	phys_addr_t addr;
972
973	if (id >= (l1_tbl_size / esz))
974	return false;
975
976	addr = base + id * esz;
977
978	if (eaddr)
979	*eaddr = addr;
980
981	return __is_visible_gfn_locked(its, gpa: addr);
982	}
983
984	/* calculate and check the index into the 1st level */
985	index = id / (SZ_64K / esz);
986	if (index >= (l1_tbl_size / sizeof(u64)))
987	return false;
988
989	/* Each 1st level entry is represented by a 64-bit value. */
990	if (kvm_read_guest_lock(its->dev->kvm,
991	base + index * sizeof(indirect_ptr),
992	&indirect_ptr, sizeof(indirect_ptr)))
993	return false;
994
995	indirect_ptr = le64_to_cpu(indirect_ptr);
996
997	/* check the valid bit of the first level entry */
998	if (!(indirect_ptr & BIT_ULL(63)))
999	return false;
1000
1001	/* Mask the guest physical address and calculate the frame number. */
1002	indirect_ptr &= GENMASK_ULL(51, 16);
1003
1004	/* Find the address of the actual entry */
1005	index = id % (SZ_64K / esz);
1006	indirect_ptr += index * esz;
1007
1008	if (eaddr)
1009	*eaddr = indirect_ptr;
1010
1011	return __is_visible_gfn_locked(its, gpa: indirect_ptr);
1012	}
1013
1014	/*
1015	* Check whether an event ID can be stored in the corresponding Interrupt
1016	* Translation Table, which starts at device->itt_addr.
1017	*/
1018	static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,
1019	u32 event_id)
1020	{
1021	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1022	int ite_esz = abi->ite_esz;
1023	gpa_t gpa;
1024
1025	/* max table size is: BIT_ULL(device->num_eventid_bits) * ite_esz */
1026	if (event_id >= BIT_ULL(device->num_eventid_bits))
1027	return false;
1028
1029	gpa = device->itt_addr + event_id * ite_esz;
1030	return __is_visible_gfn_locked(its, gpa);
1031	}
1032
1033	/*
1034	* Add a new collection into the ITS collection table.
1035	* Returns 0 on success, and a negative error value for generic errors.
1036	*/
1037	static int vgic_its_alloc_collection(struct vgic_its *its,
1038	struct its_collection **colp,
1039	u32 coll_id)
1040	{
1041	struct its_collection *collection;
1042
1043	collection = kzalloc(size: sizeof(*collection), GFP_KERNEL_ACCOUNT);
1044	if (!collection)
1045	return -ENOMEM;
1046
1047	collection->collection_id = coll_id;
1048	collection->target_addr = COLLECTION_NOT_MAPPED;
1049
1050	list_add_tail(new: &collection->coll_list, head: &its->collection_list);
1051	*colp = collection;
1052
1053	return 0;
1054	}
1055
1056	static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
1057	{
1058	struct its_collection *collection;
1059	struct its_device *device;
1060	struct its_ite *ite;
1061
1062	/*
1063	* Clearing the mapping for that collection ID removes the
1064	* entry from the list. If there wasn't any before, we can
1065	* go home early.
1066	*/
1067	collection = find_collection(its, coll_id);
1068	if (!collection)
1069	return;
1070
1071	for_each_lpi_its(device, ite, its)
1072	if (ite->collection &&
1073	ite->collection->collection_id == coll_id)
1074	ite->collection = NULL;
1075
1076	list_del(entry: &collection->coll_list);
1077	kfree(objp: collection);
1078	}
1079
1080	/* Must be called with its_lock mutex held */
1081	static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
1082	struct its_collection *collection,
1083	u32 event_id)
1084	{
1085	struct its_ite *ite;
1086
1087	ite = kzalloc(size: sizeof(*ite), GFP_KERNEL_ACCOUNT);
1088	if (!ite)
1089	return ERR_PTR(error: -ENOMEM);
1090
1091	ite->event_id = event_id;
1092	ite->collection = collection;
1093
1094	list_add_tail(new: &ite->ite_list, head: &device->itt_head);
1095	return ite;
1096	}
1097
1098	/*
1099	* The MAPTI and MAPI commands map LPIs to ITTEs.
1100	* Must be called with its_lock mutex held.
1101	*/
1102	static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1103	u64 *its_cmd)
1104	{
1105	u32 device_id = its_cmd_get_deviceid(its_cmd);
1106	u32 event_id = its_cmd_get_id(its_cmd);
1107	u32 coll_id = its_cmd_get_collection(its_cmd);
1108	struct its_ite *ite;
1109	struct kvm_vcpu *vcpu = NULL;
1110	struct its_device *device;
1111	struct its_collection *collection, *new_coll = NULL;
1112	struct vgic_irq *irq;
1113	int lpi_nr;
1114
1115	device = find_its_device(its, device_id);
1116	if (!device)
1117	return E_ITS_MAPTI_UNMAPPED_DEVICE;
1118
1119	if (!vgic_its_check_event_id(its, device, event_id))
1120	return E_ITS_MAPTI_ID_OOR;
1121
1122	if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
1123	lpi_nr = its_cmd_get_physical_id(its_cmd);
1124	else
1125	lpi_nr = event_id;
1126	if (lpi_nr < GIC_LPI_OFFSET ||
1127	lpi_nr >= max_lpis_propbaser(kvm->arch.propbaser: vgic.propbaser))
1128	return E_ITS_MAPTI_PHYSICALID_OOR;
1129
1130	/* If there is an existing mapping, behavior is UNPREDICTABLE. */
1131	if (find_ite(its, device_id, event_id))
1132	return 0;
1133
1134	collection = find_collection(its, coll_id);
1135	if (!collection) {
1136	int ret;
1137
1138	if (!vgic_its_check_id(its, baser: its->baser_coll_table, id: coll_id, NULL))
1139	return E_ITS_MAPC_COLLECTION_OOR;
1140
1141	ret = vgic_its_alloc_collection(its, colp: &collection, coll_id);
1142	if (ret)
1143	return ret;
1144	new_coll = collection;
1145	}
1146
1147	ite = vgic_its_alloc_ite(device, collection, event_id);
1148	if (IS_ERR(ptr: ite)) {
1149	if (new_coll)
1150	vgic_its_free_collection(its, coll_id);
1151	return PTR_ERR(ptr: ite);
1152	}
1153
1154	if (its_is_collection_mapped(collection))
1155	vcpu = collection_to_vcpu(kvm, col: collection);
1156
1157	irq = vgic_add_lpi(kvm, intid: lpi_nr, vcpu);
1158	if (IS_ERR(ptr: irq)) {
1159	if (new_coll)
1160	vgic_its_free_collection(its, coll_id);
1161	its_free_ite(kvm, ite);
1162	return PTR_ERR(ptr: irq);
1163	}
1164	ite->irq = irq;
1165
1166	return 0;
1167	}
1168
1169	/* Requires the its_lock to be held. */
1170	static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)
1171	{
1172	struct its_ite *ite, *temp;
1173
1174	/*
1175	* The spec says that unmapping a device with still valid
1176	* ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
1177	* since we cannot leave the memory unreferenced.
1178	*/
1179	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
1180	its_free_ite(kvm, ite);
1181
1182	vgic_its_invalidate_cache(kvm);
1183
1184	list_del(entry: &device->dev_list);
1185	kfree(objp: device);
1186	}
1187
1188	/* its lock must be held */
1189	static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1190	{
1191	struct its_device *cur, *temp;
1192
1193	list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1194	vgic_its_free_device(kvm, device: cur);
1195	}
1196
1197	/* its lock must be held */
1198	static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1199	{
1200	struct its_collection *cur, *temp;
1201
1202	list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1203	vgic_its_free_collection(its, coll_id: cur->collection_id);
1204	}
1205
1206	/* Must be called with its_lock mutex held */
1207	static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1208	u32 device_id, gpa_t itt_addr,
1209	u8 num_eventid_bits)
1210	{
1211	struct its_device *device;
1212
1213	device = kzalloc(size: sizeof(*device), GFP_KERNEL_ACCOUNT);
1214	if (!device)
1215	return ERR_PTR(error: -ENOMEM);
1216
1217	device->device_id = device_id;
1218	device->itt_addr = itt_addr;
1219	device->num_eventid_bits = num_eventid_bits;
1220	INIT_LIST_HEAD(list: &device->itt_head);
1221
1222	list_add_tail(new: &device->dev_list, head: &its->device_list);
1223	return device;
1224	}
1225
1226	/*
1227	* MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1228	* Must be called with the its_lock mutex held.
1229	*/
1230	static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1231	u64 *its_cmd)
1232	{
1233	u32 device_id = its_cmd_get_deviceid(its_cmd);
1234	bool valid = its_cmd_get_validbit(its_cmd);
1235	u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1236	gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1237	struct its_device *device;
1238
1239	if (!vgic_its_check_id(its, baser: its->baser_device_table, id: device_id, NULL))
1240	return E_ITS_MAPD_DEVICE_OOR;
1241
1242	if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1243	return E_ITS_MAPD_ITTSIZE_OOR;
1244
1245	device = find_its_device(its, device_id);
1246
1247	/*
1248	* The spec says that calling MAPD on an already mapped device
1249	* invalidates all cached data for this device. We implement this
1250	* by removing the mapping and re-establishing it.
1251	*/
1252	if (device)
1253	vgic_its_free_device(kvm, device);
1254
1255	/*
1256	* The spec does not say whether unmapping a not-mapped device
1257	* is an error, so we are done in any case.
1258	*/
1259	if (!valid)
1260	return 0;
1261
1262	device = vgic_its_alloc_device(its, device_id, itt_addr,
1263	num_eventid_bits);
1264
1265	return PTR_ERR_OR_ZERO(ptr: device);
1266	}
1267
1268	/*
1269	* The MAPC command maps collection IDs to redistributors.
1270	* Must be called with the its_lock mutex held.
1271	*/
1272	static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1273	u64 *its_cmd)
1274	{
1275	u16 coll_id;
1276	struct its_collection *collection;
1277	bool valid;
1278
1279	valid = its_cmd_get_validbit(its_cmd);
1280	coll_id = its_cmd_get_collection(its_cmd);
1281
1282	if (!valid) {
1283	vgic_its_free_collection(its, coll_id);
1284	vgic_its_invalidate_cache(kvm);
1285	} else {
1286	struct kvm_vcpu *vcpu;
1287
1288	vcpu = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
1289	if (!vcpu)
1290	return E_ITS_MAPC_PROCNUM_OOR;
1291
1292	collection = find_collection(its, coll_id);
1293
1294	if (!collection) {
1295	int ret;
1296
1297	if (!vgic_its_check_id(its, baser: its->baser_coll_table,
1298	id: coll_id, NULL))
1299	return E_ITS_MAPC_COLLECTION_OOR;
1300
1301	ret = vgic_its_alloc_collection(its, colp: &collection,
1302	coll_id);
1303	if (ret)
1304	return ret;
1305	collection->target_addr = vcpu->vcpu_id;
1306	} else {
1307	collection->target_addr = vcpu->vcpu_id;
1308	update_affinity_collection(kvm, its, coll: collection);
1309	}
1310	}
1311
1312	return 0;
1313	}
1314
1315	/*
1316	* The CLEAR command removes the pending state for a particular LPI.
1317	* Must be called with the its_lock mutex held.
1318	*/
1319	static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1320	u64 *its_cmd)
1321	{
1322	u32 device_id = its_cmd_get_deviceid(its_cmd);
1323	u32 event_id = its_cmd_get_id(its_cmd);
1324	struct its_ite *ite;
1325
1326
1327	ite = find_ite(its, device_id, event_id);
1328	if (!ite)
1329	return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1330
1331	ite->irq->pending_latch = false;
1332
1333	if (ite->irq->hw)
1334	return irq_set_irqchip_state(irq: ite->irq->host_irq,
1335	which: IRQCHIP_STATE_PENDING, state: false);
1336
1337	return 0;
1338	}
1339
1340	int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq)
1341	{
1342	return update_lpi_config(kvm, irq, NULL, needs_inv: true);
1343	}
1344
1345	/*
1346	* The INV command syncs the configuration bits from the memory table.
1347	* Must be called with the its_lock mutex held.
1348	*/
1349	static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1350	u64 *its_cmd)
1351	{
1352	u32 device_id = its_cmd_get_deviceid(its_cmd);
1353	u32 event_id = its_cmd_get_id(its_cmd);
1354	struct its_ite *ite;
1355
1356
1357	ite = find_ite(its, device_id, event_id);
1358	if (!ite)
1359	return E_ITS_INV_UNMAPPED_INTERRUPT;
1360
1361	return vgic_its_inv_lpi(kvm, irq: ite->irq);
1362	}
1363
1364	/**
1365	* vgic_its_invall - invalidate all LPIs targeting a given vcpu
1366	* @vcpu: the vcpu for which the RD is targeted by an invalidation
1367	*
1368	* Contrary to the INVALL command, this targets a RD instead of a
1369	* collection, and we don't need to hold the its_lock, since no ITS is
1370	* involved here.
1371	*/
1372	int vgic_its_invall(struct kvm_vcpu *vcpu)
1373	{
1374	struct kvm *kvm = vcpu->kvm;
1375	int irq_count, i = 0;
1376	u32 *intids;
1377
1378	irq_count = vgic_copy_lpi_list(kvm, vcpu, intid_ptr: &intids);
1379	if (irq_count < 0)
1380	return irq_count;
1381
1382	for (i = 0; i < irq_count; i++) {
1383	struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid: intids[i]);
1384	if (!irq)
1385	continue;
1386	update_lpi_config(kvm, irq, filter_vcpu: vcpu, needs_inv: false);
1387	vgic_put_irq(kvm, irq);
1388	}
1389
1390	kfree(objp: intids);
1391
1392	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1393	its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1394
1395	return 0;
1396	}
1397
1398	/*
1399	* The INVALL command requests flushing of all IRQ data in this collection.
1400	* Find the VCPU mapped to that collection, then iterate over the VM's list
1401	* of mapped LPIs and update the configuration for each IRQ which targets
1402	* the specified vcpu. The configuration will be read from the in-memory
1403	* configuration table.
1404	* Must be called with the its_lock mutex held.
1405	*/
1406	static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1407	u64 *its_cmd)
1408	{
1409	u32 coll_id = its_cmd_get_collection(its_cmd);
1410	struct its_collection *collection;
1411	struct kvm_vcpu *vcpu;
1412
1413	collection = find_collection(its, coll_id);
1414	if (!its_is_collection_mapped(collection))
1415	return E_ITS_INVALL_UNMAPPED_COLLECTION;
1416
1417	vcpu = collection_to_vcpu(kvm, col: collection);
1418	vgic_its_invall(vcpu);
1419
1420	return 0;
1421	}
1422
1423	/*
1424	* The MOVALL command moves the pending state of all IRQs targeting one
1425	* redistributor to another. We don't hold the pending state in the VCPUs,
1426	* but in the IRQs instead, so there is really not much to do for us here.
1427	* However the spec says that no IRQ must target the old redistributor
1428	* afterwards, so we make sure that no LPI is using the associated target_vcpu.
1429	* This command affects all LPIs in the system that target that redistributor.
1430	*/
1431	static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1432	u64 *its_cmd)
1433	{
1434	struct kvm_vcpu *vcpu1, *vcpu2;
1435	struct vgic_irq *irq;
1436	u32 *intids;
1437	int irq_count, i;
1438
1439	/* We advertise GITS_TYPER.PTA==0, making the address the vcpu ID */
1440	vcpu1 = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
1441	vcpu2 = kvm_get_vcpu_by_id(kvm, id: its_cmd_mask_field(its_cmd, word: 3, shift: 16, size: 32));
1442
1443	if (!vcpu1 || !vcpu2)
1444	return E_ITS_MOVALL_PROCNUM_OOR;
1445
1446	if (vcpu1 == vcpu2)
1447	return 0;
1448
1449	irq_count = vgic_copy_lpi_list(kvm, vcpu: vcpu1, intid_ptr: &intids);
1450	if (irq_count < 0)
1451	return irq_count;
1452
1453	for (i = 0; i < irq_count; i++) {
1454	irq = vgic_get_irq(kvm, NULL, intid: intids[i]);
1455	if (!irq)
1456	continue;
1457
1458	update_affinity(irq, vcpu: vcpu2);
1459
1460	vgic_put_irq(kvm, irq);
1461	}
1462
1463	vgic_its_invalidate_cache(kvm);
1464
1465	kfree(objp: intids);
1466	return 0;
1467	}
1468
1469	/*
1470	* The INT command injects the LPI associated with that DevID/EvID pair.
1471	* Must be called with the its_lock mutex held.
1472	*/
1473	static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1474	u64 *its_cmd)
1475	{
1476	u32 msi_data = its_cmd_get_id(its_cmd);
1477	u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1478
1479	return vgic_its_trigger_msi(kvm, its, devid: msi_devid, eventid: msi_data);
1480	}
1481
1482	/*
1483	* This function is called with the its_cmd lock held, but the ITS data
1484	* structure lock dropped.
1485	*/
1486	static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1487	u64 *its_cmd)
1488	{
1489	int ret = -ENODEV;
1490
1491	mutex_lock(&its->its_lock);
1492	switch (its_cmd_get_command(its_cmd)) {
1493	case GITS_CMD_MAPD:
1494	ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1495	break;
1496	case GITS_CMD_MAPC:
1497	ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1498	break;
1499	case GITS_CMD_MAPI:
1500	ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1501	break;
1502	case GITS_CMD_MAPTI:
1503	ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1504	break;
1505	case GITS_CMD_MOVI:
1506	ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1507	break;
1508	case GITS_CMD_DISCARD:
1509	ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1510	break;
1511	case GITS_CMD_CLEAR:
1512	ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1513	break;
1514	case GITS_CMD_MOVALL:
1515	ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1516	break;
1517	case GITS_CMD_INT:
1518	ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1519	break;
1520	case GITS_CMD_INV:
1521	ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1522	break;
1523	case GITS_CMD_INVALL:
1524	ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1525	break;
1526	case GITS_CMD_SYNC:
1527	/* we ignore this command: we are in sync all of the time */
1528	ret = 0;
1529	break;
1530	}
1531	mutex_unlock(lock: &its->its_lock);
1532
1533	return ret;
1534	}
1535
1536	static u64 vgic_sanitise_its_baser(u64 reg)
1537	{
1538	reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1539	GITS_BASER_SHAREABILITY_SHIFT,
1540	sanitise_fn: vgic_sanitise_shareability);
1541	reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1542	GITS_BASER_INNER_CACHEABILITY_SHIFT,
1543	sanitise_fn: vgic_sanitise_inner_cacheability);
1544	reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1545	GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1546	sanitise_fn: vgic_sanitise_outer_cacheability);
1547
1548	/* We support only one (ITS) page size: 64K */
1549	reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1550
1551	return reg;
1552	}
1553
1554	static u64 vgic_sanitise_its_cbaser(u64 reg)
1555	{
1556	reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1557	GITS_CBASER_SHAREABILITY_SHIFT,
1558	sanitise_fn: vgic_sanitise_shareability);
1559	reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1560	GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1561	sanitise_fn: vgic_sanitise_inner_cacheability);
1562	reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1563	GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1564	sanitise_fn: vgic_sanitise_outer_cacheability);
1565
1566	/* Sanitise the physical address to be 64k aligned. */
1567	reg &= ~GENMASK_ULL(15, 12);
1568
1569	return reg;
1570	}
1571
1572	static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1573	struct vgic_its *its,
1574	gpa_t addr, unsigned int len)
1575	{
1576	return extract_bytes(data: its->cbaser, offset: addr & 7, num: len);
1577	}
1578
1579	static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1580	gpa_t addr, unsigned int len,
1581	unsigned long val)
1582	{
1583	/* When GITS_CTLR.Enable is 1, this register is RO. */
1584	if (its->enabled)
1585	return;
1586
1587	mutex_lock(&its->cmd_lock);
1588	its->cbaser = update_64bit_reg(reg: its->cbaser, offset: addr & 7, len, val);
1589	its->cbaser = vgic_sanitise_its_cbaser(reg: its->cbaser);
1590	its->creadr = 0;
1591	/*
1592	* CWRITER is architecturally UNKNOWN on reset, but we need to reset
1593	* it to CREADR to make sure we start with an empty command buffer.
1594	*/
1595	its->cwriter = its->creadr;
1596	mutex_unlock(lock: &its->cmd_lock);
1597	}
1598
1599	#define ITS_CMD_BUFFER_SIZE(baser) ((((baser) & 0xff) + 1) << 12)
1600	#define ITS_CMD_SIZE 32
1601	#define ITS_CMD_OFFSET(reg) ((reg) & GENMASK(19, 5))
1602
1603	/* Must be called with the cmd_lock held. */
1604	static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1605	{
1606	gpa_t cbaser;
1607	u64 cmd_buf[4];
1608
1609	/* Commands are only processed when the ITS is enabled. */
1610	if (!its->enabled)
1611	return;
1612
1613	cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1614
1615	while (its->cwriter != its->creadr) {
1616	int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1617	cmd_buf, ITS_CMD_SIZE);
1618	/*
1619	* If kvm_read_guest() fails, this could be due to the guest
1620	* programming a bogus value in CBASER or something else going
1621	* wrong from which we cannot easily recover.
1622	* According to section 6.3.2 in the GICv3 spec we can just
1623	* ignore that command then.
1624	*/
1625	if (!ret)
1626	vgic_its_handle_command(kvm, its, its_cmd: cmd_buf);
1627
1628	its->creadr += ITS_CMD_SIZE;
1629	if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1630	its->creadr = 0;
1631	}
1632	}
1633
1634	/*
1635	* By writing to CWRITER the guest announces new commands to be processed.
1636	* To avoid any races in the first place, we take the its_cmd lock, which
1637	* protects our ring buffer variables, so that there is only one user
1638	* per ITS handling commands at a given time.
1639	*/
1640	static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1641	gpa_t addr, unsigned int len,
1642	unsigned long val)
1643	{
1644	u64 reg;
1645
1646	if (!its)
1647	return;
1648
1649	mutex_lock(&its->cmd_lock);
1650
1651	reg = update_64bit_reg(reg: its->cwriter, offset: addr & 7, len, val);
1652	reg = ITS_CMD_OFFSET(reg);
1653	if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1654	mutex_unlock(lock: &its->cmd_lock);
1655	return;
1656	}
1657	its->cwriter = reg;
1658
1659	vgic_its_process_commands(kvm, its);
1660
1661	mutex_unlock(lock: &its->cmd_lock);
1662	}
1663
1664	static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1665	struct vgic_its *its,
1666	gpa_t addr, unsigned int len)
1667	{
1668	return extract_bytes(data: its->cwriter, offset: addr & 0x7, num: len);
1669	}
1670
1671	static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1672	struct vgic_its *its,
1673	gpa_t addr, unsigned int len)
1674	{
1675	return extract_bytes(data: its->creadr, offset: addr & 0x7, num: len);
1676	}
1677
1678	static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1679	struct vgic_its *its,
1680	gpa_t addr, unsigned int len,
1681	unsigned long val)
1682	{
1683	u32 cmd_offset;
1684	int ret = 0;
1685
1686	mutex_lock(&its->cmd_lock);
1687
1688	if (its->enabled) {
1689	ret = -EBUSY;
1690	goto out;
1691	}
1692
1693	cmd_offset = ITS_CMD_OFFSET(val);
1694	if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1695	ret = -EINVAL;
1696	goto out;
1697	}
1698
1699	its->creadr = cmd_offset;
1700	out:
1701	mutex_unlock(lock: &its->cmd_lock);
1702	return ret;
1703	}
1704
1705	#define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1706	static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1707	struct vgic_its *its,
1708	gpa_t addr, unsigned int len)
1709	{
1710	u64 reg;
1711
1712	switch (BASER_INDEX(addr)) {
1713	case 0:
1714	reg = its->baser_device_table;
1715	break;
1716	case 1:
1717	reg = its->baser_coll_table;
1718	break;
1719	default:
1720	reg = 0;
1721	break;
1722	}
1723
1724	return extract_bytes(data: reg, offset: addr & 7, num: len);
1725	}
1726
1727	#define GITS_BASER_RO_MASK (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1728	static void vgic_mmio_write_its_baser(struct kvm *kvm,
1729	struct vgic_its *its,
1730	gpa_t addr, unsigned int len,
1731	unsigned long val)
1732	{
1733	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1734	u64 entry_size, table_type;
1735	u64 reg, *regptr, clearbits = 0;
1736
1737	/* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1738	if (its->enabled)
1739	return;
1740
1741	switch (BASER_INDEX(addr)) {
1742	case 0:
1743	regptr = &its->baser_device_table;
1744	entry_size = abi->dte_esz;
1745	table_type = GITS_BASER_TYPE_DEVICE;
1746	break;
1747	case 1:
1748	regptr = &its->baser_coll_table;
1749	entry_size = abi->cte_esz;
1750	table_type = GITS_BASER_TYPE_COLLECTION;
1751	clearbits = GITS_BASER_INDIRECT;
1752	break;
1753	default:
1754	return;
1755	}
1756
1757	reg = update_64bit_reg(reg: *regptr, offset: addr & 7, len, val);
1758	reg &= ~GITS_BASER_RO_MASK;
1759	reg &= ~clearbits;
1760
1761	reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1762	reg |= table_type << GITS_BASER_TYPE_SHIFT;
1763	reg = vgic_sanitise_its_baser(reg);
1764
1765	*regptr = reg;
1766
1767	if (!(reg & GITS_BASER_VALID)) {
1768	/* Take the its_lock to prevent a race with a save/restore */
1769	mutex_lock(&its->its_lock);
1770	switch (table_type) {
1771	case GITS_BASER_TYPE_DEVICE:
1772	vgic_its_free_device_list(kvm, its);
1773	break;
1774	case GITS_BASER_TYPE_COLLECTION:
1775	vgic_its_free_collection_list(kvm, its);
1776	break;
1777	}
1778	mutex_unlock(lock: &its->its_lock);
1779	}
1780	}
1781
1782	static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1783	struct vgic_its *its,
1784	gpa_t addr, unsigned int len)
1785	{
1786	u32 reg = 0;
1787
1788	mutex_lock(&its->cmd_lock);
1789	if (its->creadr == its->cwriter)
1790	reg |= GITS_CTLR_QUIESCENT;
1791	if (its->enabled)
1792	reg |= GITS_CTLR_ENABLE;
1793	mutex_unlock(lock: &its->cmd_lock);
1794
1795	return reg;
1796	}
1797
1798	static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1799	gpa_t addr, unsigned int len,
1800	unsigned long val)
1801	{
1802	mutex_lock(&its->cmd_lock);
1803
1804	/*
1805	* It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1806	* device/collection BASER are invalid
1807	*/
1808	if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1809	(!(its->baser_device_table & GITS_BASER_VALID) ||
1810	!(its->baser_coll_table & GITS_BASER_VALID) ||
1811	!(its->cbaser & GITS_CBASER_VALID)))
1812	goto out;
1813
1814	its->enabled = !!(val & GITS_CTLR_ENABLE);
1815	if (!its->enabled)
1816	vgic_its_invalidate_cache(kvm);
1817
1818	/*
1819	* Try to process any pending commands. This function bails out early
1820	* if the ITS is disabled or no commands have been queued.
1821	*/
1822	vgic_its_process_commands(kvm, its);
1823
1824	out:
1825	mutex_unlock(lock: &its->cmd_lock);
1826	}
1827
1828	#define REGISTER_ITS_DESC(off, rd, wr, length, acc) \
1829	{ \
1830	.reg_offset = off, \
1831	.len = length, \
1832	.access_flags = acc, \
1833	.its_read = rd, \
1834	.its_write = wr, \
1835	}
1836
1837	#define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1838	{ \
1839	.reg_offset = off, \
1840	.len = length, \
1841	.access_flags = acc, \
1842	.its_read = rd, \
1843	.its_write = wr, \
1844	.uaccess_its_write = uwr, \
1845	}
1846
1847	static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1848	gpa_t addr, unsigned int len, unsigned long val)
1849	{
1850	/* Ignore */
1851	}
1852
1853	static struct vgic_register_region its_registers[] = {
1854	REGISTER_ITS_DESC(GITS_CTLR,
1855	vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1856	VGIC_ACCESS_32bit),
1857	REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1858	vgic_mmio_read_its_iidr, its_mmio_write_wi,
1859	vgic_mmio_uaccess_write_its_iidr, 4,
1860	VGIC_ACCESS_32bit),
1861	REGISTER_ITS_DESC(GITS_TYPER,
1862	vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1863	VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1864	REGISTER_ITS_DESC(GITS_CBASER,
1865	vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1866	VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1867	REGISTER_ITS_DESC(GITS_CWRITER,
1868	vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1869	VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1870	REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1871	vgic_mmio_read_its_creadr, its_mmio_write_wi,
1872	vgic_mmio_uaccess_write_its_creadr, 8,
1873	VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1874	REGISTER_ITS_DESC(GITS_BASER,
1875	vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1876	VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1877	REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1878	vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1879	VGIC_ACCESS_32bit),
1880	};
1881
1882	/* This is called on setting the LPI enable bit in the redistributor. */
1883	void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1884	{
1885	if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1886	its_sync_lpi_pending_table(vcpu);
1887	}
1888
1889	static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1890	u64 addr)
1891	{
1892	struct vgic_io_device *iodev = &its->iodev;
1893	int ret;
1894
1895	mutex_lock(&kvm->slots_lock);
1896	if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1897	ret = -EBUSY;
1898	goto out;
1899	}
1900
1901	its->vgic_its_base = addr;
1902	iodev->regions = its_registers;
1903	iodev->nr_regions = ARRAY_SIZE(its_registers);
1904	kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1905
1906	iodev->base_addr = its->vgic_its_base;
1907	iodev->iodev_type = IODEV_ITS;
1908	iodev->its = its;
1909	ret = kvm_io_bus_register_dev(kvm, bus_idx: KVM_MMIO_BUS, addr: iodev->base_addr,
1910	len: KVM_VGIC_V3_ITS_SIZE, dev: &iodev->dev);
1911	out:
1912	mutex_unlock(lock: &kvm->slots_lock);
1913
1914	return ret;
1915	}
1916
1917	/* Default is 16 cached LPIs per vcpu */
1918	#define LPI_DEFAULT_PCPU_CACHE_SIZE 16
1919
1920	void vgic_lpi_translation_cache_init(struct kvm *kvm)
1921	{
1922	struct vgic_dist *dist = &kvm->arch.vgic;
1923	unsigned int sz;
1924	int i;
1925
1926	if (!list_empty(head: &dist->lpi_translation_cache))
1927	return;
1928
1929	sz = atomic_read(v: &kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;
1930
1931	for (i = 0; i < sz; i++) {
1932	struct vgic_translation_cache_entry *cte;
1933
1934	/* An allocation failure is not fatal */
1935	cte = kzalloc(size: sizeof(*cte), GFP_KERNEL_ACCOUNT);
1936	if (WARN_ON(!cte))
1937	break;
1938
1939	INIT_LIST_HEAD(list: &cte->entry);
1940	list_add(new: &cte->entry, head: &dist->lpi_translation_cache);
1941	}
1942	}
1943
1944	void vgic_lpi_translation_cache_destroy(struct kvm *kvm)
1945	{
1946	struct vgic_dist *dist = &kvm->arch.vgic;
1947	struct vgic_translation_cache_entry *cte, *tmp;
1948
1949	vgic_its_invalidate_cache(kvm);
1950
1951	list_for_each_entry_safe(cte, tmp,
1952	&dist->lpi_translation_cache, entry) {
1953	list_del(entry: &cte->entry);
1954	kfree(objp: cte);
1955	}
1956	}
1957
1958	#define INITIAL_BASER_VALUE \
1959	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) | \
1960	GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) | \
1961	GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) | \
1962	GITS_BASER_PAGE_SIZE_64K)
1963
1964	#define INITIAL_PROPBASER_VALUE \
1965	(GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb) | \
1966	GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner) | \
1967	GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1968
1969	static int vgic_its_create(struct kvm_device *dev, u32 type)
1970	{
1971	int ret;
1972	struct vgic_its *its;
1973
1974	if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1975	return -ENODEV;
1976
1977	its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1978	if (!its)
1979	return -ENOMEM;
1980
1981	mutex_lock(&dev->kvm->arch.config_lock);
1982
1983	if (vgic_initialized(dev->kvm)) {
1984	ret = vgic_v4_init(kvm: dev->kvm);
1985	if (ret < 0) {
1986	mutex_unlock(lock: &dev->kvm->arch.config_lock);
1987	kfree(objp: its);
1988	return ret;
1989	}
1990
1991	vgic_lpi_translation_cache_init(kvm: dev->kvm);
1992	}
1993
1994	mutex_init(&its->its_lock);
1995	mutex_init(&its->cmd_lock);
1996
1997	/* Yep, even more trickery for lock ordering... */
1998	#ifdef CONFIG_LOCKDEP
1999	mutex_lock(&its->cmd_lock);
2000	mutex_lock(&its->its_lock);
2001	mutex_unlock(lock: &its->its_lock);
2002	mutex_unlock(lock: &its->cmd_lock);
2003	#endif
2004
2005	its->vgic_its_base = VGIC_ADDR_UNDEF;
2006
2007	INIT_LIST_HEAD(list: &its->device_list);
2008	INIT_LIST_HEAD(list: &its->collection_list);
2009
2010	dev->kvm->arch.vgic.msis_require_devid = true;
2011	dev->kvm->arch.vgic.has_its = true;
2012	its->enabled = false;
2013	its->dev = dev;
2014
2015	its->baser_device_table = INITIAL_BASER_VALUE |
2016	((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
2017	its->baser_coll_table = INITIAL_BASER_VALUE |
2018	((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
2019	dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
2020
2021	dev->private = its;
2022
2023	ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);
2024
2025	mutex_unlock(lock: &dev->kvm->arch.config_lock);
2026
2027	return ret;
2028	}
2029
2030	static void vgic_its_destroy(struct kvm_device *kvm_dev)
2031	{
2032	struct kvm *kvm = kvm_dev->kvm;
2033	struct vgic_its *its = kvm_dev->private;
2034
2035	mutex_lock(&its->its_lock);
2036
2037	vgic_its_free_device_list(kvm, its);
2038	vgic_its_free_collection_list(kvm, its);
2039
2040	mutex_unlock(lock: &its->its_lock);
2041	kfree(objp: its);
2042	kfree(objp: kvm_dev);/* alloc by kvm_ioctl_create_device, free by .destroy */
2043	}
2044
2045	static int vgic_its_has_attr_regs(struct kvm_device *dev,
2046	struct kvm_device_attr *attr)
2047	{
2048	const struct vgic_register_region *region;
2049	gpa_t offset = attr->attr;
2050	int align;
2051
2052	align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
2053
2054	if (offset & align)
2055	return -EINVAL;
2056
2057	region = vgic_find_mmio_region(regions: its_registers,
2058	ARRAY_SIZE(its_registers),
2059	offset);
2060	if (!region)
2061	return -ENXIO;
2062
2063	return 0;
2064	}
2065
2066	static int vgic_its_attr_regs_access(struct kvm_device *dev,
2067	struct kvm_device_attr *attr,
2068	u64 *reg, bool is_write)
2069	{
2070	const struct vgic_register_region *region;
2071	struct vgic_its *its;
2072	gpa_t addr, offset;
2073	unsigned int len;
2074	int align, ret = 0;
2075
2076	its = dev->private;
2077	offset = attr->attr;
2078
2079	/*
2080	* Although the spec supports upper/lower 32-bit accesses to
2081	* 64-bit ITS registers, the userspace ABI requires 64-bit
2082	* accesses to all 64-bit wide registers. We therefore only
2083	* support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
2084	* registers
2085	*/
2086	if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
2087	align = 0x3;
2088	else
2089	align = 0x7;
2090
2091	if (offset & align)
2092	return -EINVAL;
2093
2094	mutex_lock(&dev->kvm->lock);
2095
2096	if (!lock_all_vcpus(dev->kvm)) {
2097	mutex_unlock(lock: &dev->kvm->lock);
2098	return -EBUSY;
2099	}
2100
2101	mutex_lock(&dev->kvm->arch.config_lock);
2102
2103	if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
2104	ret = -ENXIO;
2105	goto out;
2106	}
2107
2108	region = vgic_find_mmio_region(regions: its_registers,
2109	ARRAY_SIZE(its_registers),
2110	offset);
2111	if (!region) {
2112	ret = -ENXIO;
2113	goto out;
2114	}
2115
2116	addr = its->vgic_its_base + offset;
2117
2118	len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
2119
2120	if (is_write) {
2121	if (region->uaccess_its_write)
2122	ret = region->uaccess_its_write(dev->kvm, its, addr,
2123	len, *reg);
2124	else
2125	region->its_write(dev->kvm, its, addr, len, *reg);
2126	} else {
2127	*reg = region->its_read(dev->kvm, its, addr, len);
2128	}
2129	out:
2130	mutex_unlock(lock: &dev->kvm->arch.config_lock);
2131	unlock_all_vcpus(dev->kvm);
2132	mutex_unlock(lock: &dev->kvm->lock);
2133	return ret;
2134	}
2135
2136	static u32 compute_next_devid_offset(struct list_head *h,
2137	struct its_device *dev)
2138	{
2139	struct its_device *next;
2140	u32 next_offset;
2141
2142	if (list_is_last(list: &dev->dev_list, head: h))
2143	return 0;
2144	next = list_next_entry(dev, dev_list);
2145	next_offset = next->device_id - dev->device_id;
2146
2147	return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
2148	}
2149
2150	static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
2151	{
2152	struct its_ite *next;
2153	u32 next_offset;
2154
2155	if (list_is_last(list: &ite->ite_list, head: h))
2156	return 0;
2157	next = list_next_entry(ite, ite_list);
2158	next_offset = next->event_id - ite->event_id;
2159
2160	return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
2161	}
2162
2163	/**
2164	* typedef entry_fn_t - Callback called on a table entry restore path
2165	* @its: its handle
2166	* @id: id of the entry
2167	* @entry: pointer to the entry
2168	* @opaque: pointer to an opaque data
2169	*
2170	* Return: < 0 on error, 0 if last element was identified, id offset to next
2171	* element otherwise
2172	*/
2173	typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2174	void *opaque);
2175
2176	/**
2177	* scan_its_table - Scan a contiguous table in guest RAM and applies a function
2178	* to each entry
2179	*
2180	* @its: its handle
2181	* @base: base gpa of the table
2182	* @size: size of the table in bytes
2183	* @esz: entry size in bytes
2184	* @start_id: the ID of the first entry in the table
2185	* (non zero for 2d level tables)
2186	* @fn: function to apply on each entry
2187	*
2188	* Return: < 0 on error, 0 if last element was identified, 1 otherwise
2189	* (the last element may not be found on second level tables)
2190	*/
2191	static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2192	int start_id, entry_fn_t fn, void *opaque)
2193	{
2194	struct kvm *kvm = its->dev->kvm;
2195	unsigned long len = size;
2196	int id = start_id;
2197	gpa_t gpa = base;
2198	char entry[ESZ_MAX];
2199	int ret;
2200
2201	memset(entry, 0, esz);
2202
2203	while (true) {
2204	int next_offset;
2205	size_t byte_offset;
2206
2207	ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
2208	if (ret)
2209	return ret;
2210
2211	next_offset = fn(its, id, entry, opaque);
2212	if (next_offset <= 0)
2213	return next_offset;
2214
2215	byte_offset = next_offset * esz;
2216	if (byte_offset >= len)
2217	break;
2218
2219	id += next_offset;
2220	gpa += byte_offset;
2221	len -= byte_offset;
2222	}
2223	return 1;
2224	}
2225
2226	/**
2227	* vgic_its_save_ite - Save an interrupt translation entry at @gpa
2228	*/
2229	static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2230	struct its_ite *ite, gpa_t gpa, int ite_esz)
2231	{
2232	struct kvm *kvm = its->dev->kvm;
2233	u32 next_offset;
2234	u64 val;
2235
2236	next_offset = compute_next_eventid_offset(h: &dev->itt_head, ite);
2237	val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
2238	((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
2239	ite->collection->collection_id;
2240	val = cpu_to_le64(val);
2241	return vgic_write_guest_lock(kvm, gpa, data: &val, len: ite_esz);
2242	}
2243
2244	/**
2245	* vgic_its_restore_ite - restore an interrupt translation entry
2246	* @event_id: id used for indexing
2247	* @ptr: pointer to the ITE entry
2248	* @opaque: pointer to the its_device
2249	*/
2250	static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2251	void *ptr, void *opaque)
2252	{
2253	struct its_device *dev = opaque;
2254	struct its_collection *collection;
2255	struct kvm *kvm = its->dev->kvm;
2256	struct kvm_vcpu *vcpu = NULL;
2257	u64 val;
2258	u64 *p = (u64 *)ptr;
2259	struct vgic_irq *irq;
2260	u32 coll_id, lpi_id;
2261	struct its_ite *ite;
2262	u32 offset;
2263
2264	val = *p;
2265
2266	val = le64_to_cpu(val);
2267
2268	coll_id = val & KVM_ITS_ITE_ICID_MASK;
2269	lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
2270
2271	if (!lpi_id)
2272	return 1; /* invalid entry, no choice but to scan next entry */
2273
2274	if (lpi_id < VGIC_MIN_LPI)
2275	return -EINVAL;
2276
2277	offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
2278	if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
2279	return -EINVAL;
2280
2281	collection = find_collection(its, coll_id);
2282	if (!collection)
2283	return -EINVAL;
2284
2285	if (!vgic_its_check_event_id(its, device: dev, event_id))
2286	return -EINVAL;
2287
2288	ite = vgic_its_alloc_ite(device: dev, collection, event_id);
2289	if (IS_ERR(ptr: ite))
2290	return PTR_ERR(ptr: ite);
2291
2292	if (its_is_collection_mapped(collection))
2293	vcpu = kvm_get_vcpu_by_id(kvm, id: collection->target_addr);
2294
2295	irq = vgic_add_lpi(kvm, intid: lpi_id, vcpu);
2296	if (IS_ERR(ptr: irq)) {
2297	its_free_ite(kvm, ite);
2298	return PTR_ERR(ptr: irq);
2299	}
2300	ite->irq = irq;
2301
2302	return offset;
2303	}
2304
2305	static int vgic_its_ite_cmp(void *priv, const struct list_head *a,
2306	const struct list_head *b)
2307	{
2308	struct its_ite *itea = container_of(a, struct its_ite, ite_list);
2309	struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
2310
2311	if (itea->event_id < iteb->event_id)
2312	return -1;
2313	else
2314	return 1;
2315	}
2316
2317	static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2318	{
2319	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2320	gpa_t base = device->itt_addr;
2321	struct its_ite *ite;
2322	int ret;
2323	int ite_esz = abi->ite_esz;
2324
2325	list_sort(NULL, head: &device->itt_head, cmp: vgic_its_ite_cmp);
2326
2327	list_for_each_entry(ite, &device->itt_head, ite_list) {
2328	gpa_t gpa = base + ite->event_id * ite_esz;
2329
2330	/*
2331	* If an LPI carries the HW bit, this means that this
2332	* interrupt is controlled by GICv4, and we do not
2333	* have direct access to that state without GICv4.1.
2334	* Let's simply fail the save operation...
2335	*/
2336	if (ite->irq->hw && !kvm_vgic_global_state.has_gicv4_1)
2337	return -EACCES;
2338
2339	ret = vgic_its_save_ite(its, dev: device, ite, gpa, ite_esz);
2340	if (ret)
2341	return ret;
2342	}
2343	return 0;
2344	}
2345
2346	/**
2347	* vgic_its_restore_itt - restore the ITT of a device
2348	*
2349	* @its: its handle
2350	* @dev: device handle
2351	*
2352	* Return 0 on success, < 0 on error
2353	*/
2354	static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2355	{
2356	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2357	gpa_t base = dev->itt_addr;
2358	int ret;
2359	int ite_esz = abi->ite_esz;
2360	size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2361
2362	ret = scan_its_table(its, base, size: max_size, esz: ite_esz, start_id: 0,
2363	fn: vgic_its_restore_ite, opaque: dev);
2364
2365	/* scan_its_table returns +1 if all ITEs are invalid */
2366	if (ret > 0)
2367	ret = 0;
2368
2369	return ret;
2370	}
2371
2372	/**
2373	* vgic_its_save_dte - Save a device table entry at a given GPA
2374	*
2375	* @its: ITS handle
2376	* @dev: ITS device
2377	* @ptr: GPA
2378	*/
2379	static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2380	gpa_t ptr, int dte_esz)
2381	{
2382	struct kvm *kvm = its->dev->kvm;
2383	u64 val, itt_addr_field;
2384	u32 next_offset;
2385
2386	itt_addr_field = dev->itt_addr >> 8;
2387	next_offset = compute_next_devid_offset(h: &its->device_list, dev);
2388	val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2389	((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2390	(itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2391	(dev->num_eventid_bits - 1));
2392	val = cpu_to_le64(val);
2393	return vgic_write_guest_lock(kvm, gpa: ptr, data: &val, len: dte_esz);
2394	}
2395
2396	/**
2397	* vgic_its_restore_dte - restore a device table entry
2398	*
2399	* @its: its handle
2400	* @id: device id the DTE corresponds to
2401	* @ptr: kernel VA where the 8 byte DTE is located
2402	* @opaque: unused
2403	*
2404	* Return: < 0 on error, 0 if the dte is the last one, id offset to the
2405	* next dte otherwise
2406	*/
2407	static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2408	void *ptr, void *opaque)
2409	{
2410	struct its_device *dev;
2411	u64 baser = its->baser_device_table;
2412	gpa_t itt_addr;
2413	u8 num_eventid_bits;
2414	u64 entry = *(u64 *)ptr;
2415	bool valid;
2416	u32 offset;
2417	int ret;
2418
2419	entry = le64_to_cpu(entry);
2420
2421	valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2422	num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2423	itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2424	>> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2425
2426	if (!valid)
2427	return 1;
2428
2429	/* dte entry is valid */
2430	offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2431
2432	if (!vgic_its_check_id(its, baser, id, NULL))
2433	return -EINVAL;
2434
2435	dev = vgic_its_alloc_device(its, device_id: id, itt_addr, num_eventid_bits);
2436	if (IS_ERR(ptr: dev))
2437	return PTR_ERR(ptr: dev);
2438
2439	ret = vgic_its_restore_itt(its, dev);
2440	if (ret) {
2441	vgic_its_free_device(kvm: its->dev->kvm, device: dev);
2442	return ret;
2443	}
2444
2445	return offset;
2446	}
2447
2448	static int vgic_its_device_cmp(void *priv, const struct list_head *a,
2449	const struct list_head *b)
2450	{
2451	struct its_device *deva = container_of(a, struct its_device, dev_list);
2452	struct its_device *devb = container_of(b, struct its_device, dev_list);
2453
2454	if (deva->device_id < devb->device_id)
2455	return -1;
2456	else
2457	return 1;
2458	}
2459
2460	/**
2461	* vgic_its_save_device_tables - Save the device table and all ITT
2462	* into guest RAM
2463	*
2464	* L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2465	* returns the GPA of the device entry
2466	*/
2467	static int vgic_its_save_device_tables(struct vgic_its *its)
2468	{
2469	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2470	u64 baser = its->baser_device_table;
2471	struct its_device *dev;
2472	int dte_esz = abi->dte_esz;
2473
2474	if (!(baser & GITS_BASER_VALID))
2475	return 0;
2476
2477	list_sort(NULL, head: &its->device_list, cmp: vgic_its_device_cmp);
2478
2479	list_for_each_entry(dev, &its->device_list, dev_list) {
2480	int ret;
2481	gpa_t eaddr;
2482
2483	if (!vgic_its_check_id(its, baser,
2484	id: dev->device_id, eaddr: &eaddr))
2485	return -EINVAL;
2486
2487	ret = vgic_its_save_itt(its, device: dev);
2488	if (ret)
2489	return ret;
2490
2491	ret = vgic_its_save_dte(its, dev, ptr: eaddr, dte_esz);
2492	if (ret)
2493	return ret;
2494	}
2495	return 0;
2496	}
2497
2498	/**
2499	* handle_l1_dte - callback used for L1 device table entries (2 stage case)
2500	*
2501	* @its: its handle
2502	* @id: index of the entry in the L1 table
2503	* @addr: kernel VA
2504	* @opaque: unused
2505	*
2506	* L1 table entries are scanned by steps of 1 entry
2507	* Return < 0 if error, 0 if last dte was found when scanning the L2
2508	* table, +1 otherwise (meaning next L1 entry must be scanned)
2509	*/
2510	static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2511	void *opaque)
2512	{
2513	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2514	int l2_start_id = id * (SZ_64K / abi->dte_esz);
2515	u64 entry = *(u64 *)addr;
2516	int dte_esz = abi->dte_esz;
2517	gpa_t gpa;
2518	int ret;
2519
2520	entry = le64_to_cpu(entry);
2521
2522	if (!(entry & KVM_ITS_L1E_VALID_MASK))
2523	return 1;
2524
2525	gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2526
2527	ret = scan_its_table(its, base: gpa, SZ_64K, esz: dte_esz,
2528	start_id: l2_start_id, fn: vgic_its_restore_dte, NULL);
2529
2530	return ret;
2531	}
2532
2533	/**
2534	* vgic_its_restore_device_tables - Restore the device table and all ITT
2535	* from guest RAM to internal data structs
2536	*/
2537	static int vgic_its_restore_device_tables(struct vgic_its *its)
2538	{
2539	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2540	u64 baser = its->baser_device_table;
2541	int l1_esz, ret;
2542	int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2543	gpa_t l1_gpa;
2544
2545	if (!(baser & GITS_BASER_VALID))
2546	return 0;
2547
2548	l1_gpa = GITS_BASER_ADDR_48_to_52(baser);
2549
2550	if (baser & GITS_BASER_INDIRECT) {
2551	l1_esz = GITS_LVL1_ENTRY_SIZE;
2552	ret = scan_its_table(its, base: l1_gpa, size: l1_tbl_size, esz: l1_esz, start_id: 0,
2553	fn: handle_l1_dte, NULL);
2554	} else {
2555	l1_esz = abi->dte_esz;
2556	ret = scan_its_table(its, base: l1_gpa, size: l1_tbl_size, esz: l1_esz, start_id: 0,
2557	fn: vgic_its_restore_dte, NULL);
2558	}
2559
2560	/* scan_its_table returns +1 if all entries are invalid */
2561	if (ret > 0)
2562	ret = 0;
2563
2564	if (ret < 0)
2565	vgic_its_free_device_list(kvm: its->dev->kvm, its);
2566
2567	return ret;
2568	}
2569
2570	static int vgic_its_save_cte(struct vgic_its *its,
2571	struct its_collection *collection,
2572	gpa_t gpa, int esz)
2573	{
2574	u64 val;
2575
2576	val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2577	((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2578	collection->collection_id);
2579	val = cpu_to_le64(val);
2580	return vgic_write_guest_lock(kvm: its->dev->kvm, gpa, data: &val, len: esz);
2581	}
2582
2583	/*
2584	* Restore a collection entry into the ITS collection table.
2585	* Return +1 on success, 0 if the entry was invalid (which should be
2586	* interpreted as end-of-table), and a negative error value for generic errors.
2587	*/
2588	static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2589	{
2590	struct its_collection *collection;
2591	struct kvm *kvm = its->dev->kvm;
2592	u32 target_addr, coll_id;
2593	u64 val;
2594	int ret;
2595
2596	BUG_ON(esz > sizeof(val));
2597	ret = kvm_read_guest_lock(kvm, gpa, &val, esz);
2598	if (ret)
2599	return ret;
2600	val = le64_to_cpu(val);
2601	if (!(val & KVM_ITS_CTE_VALID_MASK))
2602	return 0;
2603
2604	target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2605	coll_id = val & KVM_ITS_CTE_ICID_MASK;
2606
2607	if (target_addr != COLLECTION_NOT_MAPPED &&
2608	!kvm_get_vcpu_by_id(kvm, id: target_addr))
2609	return -EINVAL;
2610
2611	collection = find_collection(its, coll_id);
2612	if (collection)
2613	return -EEXIST;
2614
2615	if (!vgic_its_check_id(its, baser: its->baser_coll_table, id: coll_id, NULL))
2616	return -EINVAL;
2617
2618	ret = vgic_its_alloc_collection(its, colp: &collection, coll_id);
2619	if (ret)
2620	return ret;
2621	collection->target_addr = target_addr;
2622	return 1;
2623	}
2624
2625	/**
2626	* vgic_its_save_collection_table - Save the collection table into
2627	* guest RAM
2628	*/
2629	static int vgic_its_save_collection_table(struct vgic_its *its)
2630	{
2631	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2632	u64 baser = its->baser_coll_table;
2633	gpa_t gpa = GITS_BASER_ADDR_48_to_52(baser);
2634	struct its_collection *collection;
2635	u64 val;
2636	size_t max_size, filled = 0;
2637	int ret, cte_esz = abi->cte_esz;
2638
2639	if (!(baser & GITS_BASER_VALID))
2640	return 0;
2641
2642	max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2643
2644	list_for_each_entry(collection, &its->collection_list, coll_list) {
2645	ret = vgic_its_save_cte(its, collection, gpa, esz: cte_esz);
2646	if (ret)
2647	return ret;
2648	gpa += cte_esz;
2649	filled += cte_esz;
2650	}
2651
2652	if (filled == max_size)
2653	return 0;
2654
2655	/*
2656	* table is not fully filled, add a last dummy element
2657	* with valid bit unset
2658	*/
2659	val = 0;
2660	BUG_ON(cte_esz > sizeof(val));
2661	ret = vgic_write_guest_lock(kvm: its->dev->kvm, gpa, data: &val, len: cte_esz);
2662	return ret;
2663	}
2664
2665	/**
2666	* vgic_its_restore_collection_table - reads the collection table
2667	* in guest memory and restores the ITS internal state. Requires the
2668	* BASER registers to be restored before.
2669	*/
2670	static int vgic_its_restore_collection_table(struct vgic_its *its)
2671	{
2672	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2673	u64 baser = its->baser_coll_table;
2674	int cte_esz = abi->cte_esz;
2675	size_t max_size, read = 0;
2676	gpa_t gpa;
2677	int ret;
2678
2679	if (!(baser & GITS_BASER_VALID))
2680	return 0;
2681
2682	gpa = GITS_BASER_ADDR_48_to_52(baser);
2683
2684	max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2685
2686	while (read < max_size) {
2687	ret = vgic_its_restore_cte(its, gpa, esz: cte_esz);
2688	if (ret <= 0)
2689	break;
2690	gpa += cte_esz;
2691	read += cte_esz;
2692	}
2693
2694	if (ret > 0)
2695	return 0;
2696
2697	if (ret < 0)
2698	vgic_its_free_collection_list(kvm: its->dev->kvm, its);
2699
2700	return ret;
2701	}
2702
2703	/**
2704	* vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2705	* according to v0 ABI
2706	*/
2707	static int vgic_its_save_tables_v0(struct vgic_its *its)
2708	{
2709	int ret;
2710
2711	ret = vgic_its_save_device_tables(its);
2712	if (ret)
2713	return ret;
2714
2715	return vgic_its_save_collection_table(its);
2716	}
2717
2718	/**
2719	* vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2720	* to internal data structs according to V0 ABI
2721	*
2722	*/
2723	static int vgic_its_restore_tables_v0(struct vgic_its *its)
2724	{
2725	int ret;
2726
2727	ret = vgic_its_restore_collection_table(its);
2728	if (ret)
2729	return ret;
2730
2731	ret = vgic_its_restore_device_tables(its);
2732	if (ret)
2733	vgic_its_free_collection_list(kvm: its->dev->kvm, its);
2734	return ret;
2735	}
2736
2737	static int vgic_its_commit_v0(struct vgic_its *its)
2738	{
2739	const struct vgic_its_abi *abi;
2740
2741	abi = vgic_its_get_abi(its);
2742	its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2743	its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2744
2745	its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2746	<< GITS_BASER_ENTRY_SIZE_SHIFT);
2747
2748	its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2749	<< GITS_BASER_ENTRY_SIZE_SHIFT);
2750	return 0;
2751	}
2752
2753	static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2754	{
2755	/* We need to keep the ABI specific field values */
2756	its->baser_coll_table &= ~GITS_BASER_VALID;
2757	its->baser_device_table &= ~GITS_BASER_VALID;
2758	its->cbaser = 0;
2759	its->creadr = 0;
2760	its->cwriter = 0;
2761	its->enabled = 0;
2762	vgic_its_free_device_list(kvm, its);
2763	vgic_its_free_collection_list(kvm, its);
2764	}
2765
2766	static int vgic_its_has_attr(struct kvm_device *dev,
2767	struct kvm_device_attr *attr)
2768	{
2769	switch (attr->group) {
2770	case KVM_DEV_ARM_VGIC_GRP_ADDR:
2771	switch (attr->attr) {
2772	case KVM_VGIC_ITS_ADDR_TYPE:
2773	return 0;
2774	}
2775	break;
2776	case KVM_DEV_ARM_VGIC_GRP_CTRL:
2777	switch (attr->attr) {
2778	case KVM_DEV_ARM_VGIC_CTRL_INIT:
2779	return 0;
2780	case KVM_DEV_ARM_ITS_CTRL_RESET:
2781	return 0;
2782	case KVM_DEV_ARM_ITS_SAVE_TABLES:
2783	return 0;
2784	case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2785	return 0;
2786	}
2787	break;
2788	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2789	return vgic_its_has_attr_regs(dev, attr);
2790	}
2791	return -ENXIO;
2792	}
2793
2794	static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2795	{
2796	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2797	int ret = 0;
2798
2799	if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2800	return 0;
2801
2802	mutex_lock(&kvm->lock);
2803
2804	if (!lock_all_vcpus(kvm)) {
2805	mutex_unlock(lock: &kvm->lock);
2806	return -EBUSY;
2807	}
2808
2809	mutex_lock(&kvm->arch.config_lock);
2810	mutex_lock(&its->its_lock);
2811
2812	switch (attr) {
2813	case KVM_DEV_ARM_ITS_CTRL_RESET:
2814	vgic_its_reset(kvm, its);
2815	break;
2816	case KVM_DEV_ARM_ITS_SAVE_TABLES:
2817	ret = abi->save_tables(its);
2818	break;
2819	case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2820	ret = abi->restore_tables(its);
2821	break;
2822	}
2823
2824	mutex_unlock(lock: &its->its_lock);
2825	mutex_unlock(lock: &kvm->arch.config_lock);
2826	unlock_all_vcpus(kvm);
2827	mutex_unlock(lock: &kvm->lock);
2828	return ret;
2829	}
2830
2831	/*
2832	* kvm_arch_allow_write_without_running_vcpu - allow writing guest memory
2833	* without the running VCPU when dirty ring is enabled.
2834	*
2835	* The running VCPU is required to track dirty guest pages when dirty ring
2836	* is enabled. Otherwise, the backup bitmap should be used to track the
2837	* dirty guest pages. When vgic/its tables are being saved, the backup
2838	* bitmap is used to track the dirty guest pages due to the missed running
2839	* VCPU in the period.
2840	*/
2841	bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
2842	{
2843	struct vgic_dist *dist = &kvm->arch.vgic;
2844
2845	return dist->table_write_in_progress;
2846	}
2847
2848	static int vgic_its_set_attr(struct kvm_device *dev,
2849	struct kvm_device_attr *attr)
2850	{
2851	struct vgic_its *its = dev->private;
2852	int ret;
2853
2854	switch (attr->group) {
2855	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2856	u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2857	unsigned long type = (unsigned long)attr->attr;
2858	u64 addr;
2859
2860	if (type != KVM_VGIC_ITS_ADDR_TYPE)
2861	return -ENODEV;
2862
2863	if (copy_from_user(to: &addr, from: uaddr, n: sizeof(addr)))
2864	return -EFAULT;
2865
2866	ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2867	addr, SZ_64K, KVM_VGIC_V3_ITS_SIZE);
2868	if (ret)
2869	return ret;
2870
2871	return vgic_register_its_iodev(kvm: dev->kvm, its, addr);
2872	}
2873	case KVM_DEV_ARM_VGIC_GRP_CTRL:
2874	return vgic_its_ctrl(kvm: dev->kvm, its, attr: attr->attr);
2875	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2876	u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2877	u64 reg;
2878
2879	if (get_user(reg, uaddr))
2880	return -EFAULT;
2881
2882	return vgic_its_attr_regs_access(dev, attr, reg: &reg, is_write: true);
2883	}
2884	}
2885	return -ENXIO;
2886	}
2887
2888	static int vgic_its_get_attr(struct kvm_device *dev,
2889	struct kvm_device_attr *attr)
2890	{
2891	switch (attr->group) {
2892	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2893	struct vgic_its *its = dev->private;
2894	u64 addr = its->vgic_its_base;
2895	u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2896	unsigned long type = (unsigned long)attr->attr;
2897
2898	if (type != KVM_VGIC_ITS_ADDR_TYPE)
2899	return -ENODEV;
2900
2901	if (copy_to_user(to: uaddr, from: &addr, n: sizeof(addr)))
2902	return -EFAULT;
2903	break;
2904	}
2905	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2906	u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2907	u64 reg;
2908	int ret;
2909
2910	ret = vgic_its_attr_regs_access(dev, attr, reg: &reg, is_write: false);
2911	if (ret)
2912	return ret;
2913	return put_user(reg, uaddr);
2914	}
2915	default:
2916	return -ENXIO;
2917	}
2918
2919	return 0;
2920	}
2921
2922	static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2923	.name = "kvm-arm-vgic-its",
2924	.create = vgic_its_create,
2925	.destroy = vgic_its_destroy,
2926	.set_attr = vgic_its_set_attr,
2927	.get_attr = vgic_its_get_attr,
2928	.has_attr = vgic_its_has_attr,
2929	};
2930
2931	int kvm_vgic_register_its_device(void)
2932	{
2933	return kvm_register_device_ops(ops: &kvm_arm_vgic_its_ops,
2934	KVM_DEV_TYPE_ARM_VGIC_ITS);
2935	}
2936