1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved. |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> |
5 | */ |
6 | |
7 | #include <linux/interrupt.h> |
8 | #include <linux/irq.h> |
9 | #include <linux/irqdomain.h> |
10 | #include <linux/msi.h> |
11 | #include <linux/pid.h> |
12 | #include <linux/sched.h> |
13 | |
14 | #include <linux/irqchip/arm-gic-v4.h> |
15 | |
16 | /* |
17 | * WARNING: The blurb below assumes that you understand the |
18 | * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets |
19 | * translated into GICv4 commands. So it effectively targets at most |
20 | * two individuals. You know who you are. |
21 | * |
22 | * The core GICv4 code is designed to *avoid* exposing too much of the |
23 | * core GIC code (that would in turn leak into the hypervisor code), |
24 | * and instead provide a hypervisor agnostic interface to the HW (of |
25 | * course, the astute reader will quickly realize that hypervisor |
26 | * agnostic actually means KVM-specific - what were you thinking?). |
27 | * |
28 | * In order to achieve a modicum of isolation, we try to hide most of |
29 | * the GICv4 "stuff" behind normal irqchip operations: |
30 | * |
31 | * - Any guest-visible VLPI is backed by a Linux interrupt (and a |
32 | * physical LPI which gets unmapped when the guest maps the |
33 | * VLPI). This allows the same DevID/EventID pair to be either |
34 | * mapped to the LPI (host) or the VLPI (guest). Note that this is |
35 | * exclusive, and you cannot have both. |
36 | * |
37 | * - Enabling/disabling a VLPI is done by issuing mask/unmask calls. |
38 | * |
39 | * - Guest INT/CLEAR commands are implemented through |
40 | * irq_set_irqchip_state(). |
41 | * |
42 | * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or |
43 | * issuing an INV after changing a priority) gets shoved into the |
44 | * irq_set_vcpu_affinity() method. While this is quite horrible |
45 | * (let's face it, this is the irqchip version of an ioctl), it |
46 | * confines the crap to a single location. And map/unmap really is |
47 | * about setting the affinity of a VLPI to a vcpu, so only INV is |
48 | * majorly out of place. So there. |
49 | * |
50 | * A number of commands are simply not provided by this interface, as |
51 | * they do not make direct sense. For example, MAPD is purely local to |
52 | * the virtual ITS (because it references a virtual device, and the |
53 | * physical ITS is still very much in charge of the physical |
54 | * device). Same goes for things like MAPC (the physical ITS deals |
55 | * with the actual vPE affinity, and not the braindead concept of |
56 | * collection). SYNC is not provided either, as each and every command |
57 | * is followed by a VSYNC. This could be relaxed in the future, should |
58 | * this be seen as a bottleneck (yes, this means *never*). |
59 | * |
60 | * But handling VLPIs is only one side of the job of the GICv4 |
61 | * code. The other (darker) side is to take care of the doorbell |
62 | * interrupts which are delivered when a VLPI targeting a non-running |
63 | * vcpu is being made pending. |
64 | * |
65 | * The choice made here is that each vcpu (VPE in old northern GICv4 |
66 | * dialect) gets a single doorbell LPI, no matter how many interrupts |
67 | * are targeting it. This has a nice property, which is that the |
68 | * interrupt becomes a handle for the VPE, and that the hypervisor |
69 | * code can manipulate it through the normal interrupt API: |
70 | * |
71 | * - VMs (or rather the VM abstraction that matters to the GIC) |
72 | * contain an irq domain where each interrupt maps to a VPE. In |
73 | * turn, this domain sits on top of the normal LPI allocator, and a |
74 | * specially crafted irq_chip implementation. |
75 | * |
76 | * - mask/unmask do what is expected on the doorbell interrupt. |
77 | * |
78 | * - irq_set_affinity is used to move a VPE from one redistributor to |
79 | * another. |
80 | * |
81 | * - irq_set_vcpu_affinity once again gets hijacked for the purpose of |
82 | * creating a new sub-API, namely scheduling/descheduling a VPE |
83 | * (which involves programming GICR_V{PROP,PEND}BASER) and |
84 | * performing INVALL operations. |
85 | */ |
86 | |
87 | static struct irq_domain *gic_domain; |
88 | static const struct irq_domain_ops *vpe_domain_ops; |
89 | static const struct irq_domain_ops *sgi_domain_ops; |
90 | |
91 | #ifdef CONFIG_ARM64 |
92 | #include <asm/cpufeature.h> |
93 | |
94 | bool gic_cpuif_has_vsgi(void) |
95 | { |
96 | unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); |
97 | |
98 | fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); |
99 | |
100 | return fld >= 0x3; |
101 | } |
102 | #else |
103 | bool gic_cpuif_has_vsgi(void) |
104 | { |
105 | return false; |
106 | } |
107 | #endif |
108 | |
109 | static bool has_v4_1(void) |
110 | { |
111 | return !!sgi_domain_ops; |
112 | } |
113 | |
114 | static bool has_v4_1_sgi(void) |
115 | { |
116 | return has_v4_1() && gic_cpuif_has_vsgi(); |
117 | } |
118 | |
119 | static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) |
120 | { |
121 | char *name; |
122 | int sgi_base; |
123 | |
124 | if (!has_v4_1_sgi()) |
125 | return 0; |
126 | |
127 | name = kasprintf(GFP_KERNEL, fmt: "GICv4-sgi-%d" , task_pid_nr(current)); |
128 | if (!name) |
129 | goto err; |
130 | |
131 | vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, id: idx); |
132 | if (!vpe->fwnode) |
133 | goto err; |
134 | |
135 | kfree(objp: name); |
136 | name = NULL; |
137 | |
138 | vpe->sgi_domain = irq_domain_create_linear(fwnode: vpe->fwnode, size: 16, |
139 | ops: sgi_domain_ops, host_data: vpe); |
140 | if (!vpe->sgi_domain) |
141 | goto err; |
142 | |
143 | sgi_base = irq_domain_alloc_irqs(domain: vpe->sgi_domain, nr_irqs: 16, NUMA_NO_NODE, arg: vpe); |
144 | if (sgi_base <= 0) |
145 | goto err; |
146 | |
147 | return 0; |
148 | |
149 | err: |
150 | if (vpe->sgi_domain) |
151 | irq_domain_remove(host: vpe->sgi_domain); |
152 | if (vpe->fwnode) |
153 | irq_domain_free_fwnode(fwnode: vpe->fwnode); |
154 | kfree(objp: name); |
155 | return -ENOMEM; |
156 | } |
157 | |
158 | int its_alloc_vcpu_irqs(struct its_vm *vm) |
159 | { |
160 | int vpe_base_irq, i; |
161 | |
162 | vm->fwnode = irq_domain_alloc_named_id_fwnode(name: "GICv4-vpe" , |
163 | id: task_pid_nr(current)); |
164 | if (!vm->fwnode) |
165 | goto err; |
166 | |
167 | vm->domain = irq_domain_create_hierarchy(parent: gic_domain, flags: 0, size: vm->nr_vpes, |
168 | fwnode: vm->fwnode, ops: vpe_domain_ops, |
169 | host_data: vm); |
170 | if (!vm->domain) |
171 | goto err; |
172 | |
173 | for (i = 0; i < vm->nr_vpes; i++) { |
174 | vm->vpes[i]->its_vm = vm; |
175 | vm->vpes[i]->idai = true; |
176 | } |
177 | |
178 | vpe_base_irq = irq_domain_alloc_irqs(domain: vm->domain, nr_irqs: vm->nr_vpes, |
179 | NUMA_NO_NODE, arg: vm); |
180 | if (vpe_base_irq <= 0) |
181 | goto err; |
182 | |
183 | for (i = 0; i < vm->nr_vpes; i++) { |
184 | int ret; |
185 | vm->vpes[i]->irq = vpe_base_irq + i; |
186 | ret = its_alloc_vcpu_sgis(vpe: vm->vpes[i], idx: i); |
187 | if (ret) |
188 | goto err; |
189 | } |
190 | |
191 | return 0; |
192 | |
193 | err: |
194 | if (vm->domain) |
195 | irq_domain_remove(host: vm->domain); |
196 | if (vm->fwnode) |
197 | irq_domain_free_fwnode(fwnode: vm->fwnode); |
198 | |
199 | return -ENOMEM; |
200 | } |
201 | |
202 | static void its_free_sgi_irqs(struct its_vm *vm) |
203 | { |
204 | int i; |
205 | |
206 | if (!has_v4_1_sgi()) |
207 | return; |
208 | |
209 | for (i = 0; i < vm->nr_vpes; i++) { |
210 | unsigned int irq = irq_find_mapping(domain: vm->vpes[i]->sgi_domain, hwirq: 0); |
211 | |
212 | if (WARN_ON(!irq)) |
213 | continue; |
214 | |
215 | irq_domain_free_irqs(virq: irq, nr_irqs: 16); |
216 | irq_domain_remove(host: vm->vpes[i]->sgi_domain); |
217 | irq_domain_free_fwnode(fwnode: vm->vpes[i]->fwnode); |
218 | } |
219 | } |
220 | |
221 | void its_free_vcpu_irqs(struct its_vm *vm) |
222 | { |
223 | its_free_sgi_irqs(vm); |
224 | irq_domain_free_irqs(virq: vm->vpes[0]->irq, nr_irqs: vm->nr_vpes); |
225 | irq_domain_remove(host: vm->domain); |
226 | irq_domain_free_fwnode(fwnode: vm->fwnode); |
227 | } |
228 | |
229 | static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) |
230 | { |
231 | return irq_set_vcpu_affinity(irq: vpe->irq, vcpu_info: info); |
232 | } |
233 | |
234 | int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) |
235 | { |
236 | struct irq_desc *desc = irq_to_desc(irq: vpe->irq); |
237 | struct its_cmd_info info = { }; |
238 | int ret; |
239 | |
240 | WARN_ON(preemptible()); |
241 | |
242 | info.cmd_type = DESCHEDULE_VPE; |
243 | if (has_v4_1()) { |
244 | /* GICv4.1 can directly deal with doorbells */ |
245 | info.req_db = db; |
246 | } else { |
247 | /* Undo the nested disable_irq() calls... */ |
248 | while (db && irqd_irq_disabled(d: &desc->irq_data)) |
249 | enable_irq(irq: vpe->irq); |
250 | } |
251 | |
252 | ret = its_send_vpe_cmd(vpe, info: &info); |
253 | if (!ret) |
254 | vpe->resident = false; |
255 | |
256 | vpe->ready = false; |
257 | |
258 | return ret; |
259 | } |
260 | |
261 | int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) |
262 | { |
263 | struct its_cmd_info info = { }; |
264 | int ret; |
265 | |
266 | WARN_ON(preemptible()); |
267 | |
268 | info.cmd_type = SCHEDULE_VPE; |
269 | if (has_v4_1()) { |
270 | info.g0en = g0en; |
271 | info.g1en = g1en; |
272 | } else { |
273 | /* Disabled the doorbell, as we're about to enter the guest */ |
274 | disable_irq_nosync(irq: vpe->irq); |
275 | } |
276 | |
277 | ret = its_send_vpe_cmd(vpe, info: &info); |
278 | if (!ret) |
279 | vpe->resident = true; |
280 | |
281 | return ret; |
282 | } |
283 | |
284 | int its_commit_vpe(struct its_vpe *vpe) |
285 | { |
286 | struct its_cmd_info info = { |
287 | .cmd_type = COMMIT_VPE, |
288 | }; |
289 | int ret; |
290 | |
291 | WARN_ON(preemptible()); |
292 | |
293 | ret = its_send_vpe_cmd(vpe, info: &info); |
294 | if (!ret) |
295 | vpe->ready = true; |
296 | |
297 | return ret; |
298 | } |
299 | |
300 | |
301 | int its_invall_vpe(struct its_vpe *vpe) |
302 | { |
303 | struct its_cmd_info info = { |
304 | .cmd_type = INVALL_VPE, |
305 | }; |
306 | |
307 | return its_send_vpe_cmd(vpe, info: &info); |
308 | } |
309 | |
310 | int its_map_vlpi(int irq, struct its_vlpi_map *map) |
311 | { |
312 | struct its_cmd_info info = { |
313 | .cmd_type = MAP_VLPI, |
314 | { |
315 | .map = map, |
316 | }, |
317 | }; |
318 | int ret; |
319 | |
320 | /* |
321 | * The host will never see that interrupt firing again, so it |
322 | * is vital that we don't do any lazy masking. |
323 | */ |
324 | irq_set_status_flags(irq, set: IRQ_DISABLE_UNLAZY); |
325 | |
326 | ret = irq_set_vcpu_affinity(irq, vcpu_info: &info); |
327 | if (ret) |
328 | irq_clear_status_flags(irq, clr: IRQ_DISABLE_UNLAZY); |
329 | |
330 | return ret; |
331 | } |
332 | |
333 | int its_get_vlpi(int irq, struct its_vlpi_map *map) |
334 | { |
335 | struct its_cmd_info info = { |
336 | .cmd_type = GET_VLPI, |
337 | { |
338 | .map = map, |
339 | }, |
340 | }; |
341 | |
342 | return irq_set_vcpu_affinity(irq, vcpu_info: &info); |
343 | } |
344 | |
345 | int its_unmap_vlpi(int irq) |
346 | { |
347 | irq_clear_status_flags(irq, clr: IRQ_DISABLE_UNLAZY); |
348 | return irq_set_vcpu_affinity(irq, NULL); |
349 | } |
350 | |
351 | int its_prop_update_vlpi(int irq, u8 config, bool inv) |
352 | { |
353 | struct its_cmd_info info = { |
354 | .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI, |
355 | { |
356 | .config = config, |
357 | }, |
358 | }; |
359 | |
360 | return irq_set_vcpu_affinity(irq, vcpu_info: &info); |
361 | } |
362 | |
363 | int its_prop_update_vsgi(int irq, u8 priority, bool group) |
364 | { |
365 | struct its_cmd_info info = { |
366 | .cmd_type = PROP_UPDATE_VSGI, |
367 | { |
368 | .priority = priority, |
369 | .group = group, |
370 | }, |
371 | }; |
372 | |
373 | return irq_set_vcpu_affinity(irq, vcpu_info: &info); |
374 | } |
375 | |
376 | int its_init_v4(struct irq_domain *domain, |
377 | const struct irq_domain_ops *vpe_ops, |
378 | const struct irq_domain_ops *sgi_ops) |
379 | { |
380 | if (domain) { |
381 | pr_info("ITS: Enabling GICv4 support\n" ); |
382 | gic_domain = domain; |
383 | vpe_domain_ops = vpe_ops; |
384 | sgi_domain_ops = sgi_ops; |
385 | return 0; |
386 | } |
387 | |
388 | pr_err("ITS: No GICv4 VPE domain allocated\n" ); |
389 | return -ENODEV; |
390 | } |
391 | |