1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2021 Western Digital Corporation or its affiliates. |
4 | * Copyright (C) 2022 Ventana Micro Systems Inc. |
5 | * |
6 | * Authors: |
7 | * Anup Patel <apatel@ventanamicro.com> |
8 | */ |
9 | |
10 | #include <linux/kernel.h> |
11 | #include <linux/bitops.h> |
12 | #include <linux/irq.h> |
13 | #include <linux/irqdomain.h> |
14 | #include <linux/kvm_host.h> |
15 | #include <linux/percpu.h> |
16 | #include <linux/spinlock.h> |
17 | #include <asm/cpufeature.h> |
18 | #include <asm/kvm_aia_imsic.h> |
19 | |
20 | struct aia_hgei_control { |
21 | raw_spinlock_t lock; |
22 | unsigned long free_bitmap; |
23 | struct kvm_vcpu *owners[BITS_PER_LONG]; |
24 | }; |
25 | static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei); |
26 | static int hgei_parent_irq; |
27 | |
28 | unsigned int kvm_riscv_aia_nr_hgei; |
29 | unsigned int kvm_riscv_aia_max_ids; |
30 | DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available); |
31 | |
32 | static int aia_find_hgei(struct kvm_vcpu *owner) |
33 | { |
34 | int i, hgei; |
35 | unsigned long flags; |
36 | struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); |
37 | |
38 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
39 | |
40 | hgei = -1; |
41 | for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) { |
42 | if (hgctrl->owners[i] == owner) { |
43 | hgei = i; |
44 | break; |
45 | } |
46 | } |
47 | |
48 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
49 | |
50 | put_cpu_ptr(&aia_hgei); |
51 | return hgei; |
52 | } |
53 | |
54 | static void aia_set_hvictl(bool ext_irq_pending) |
55 | { |
56 | unsigned long hvictl; |
57 | |
58 | /* |
59 | * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents |
60 | * no interrupt in HVICTL. |
61 | */ |
62 | |
63 | hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID; |
64 | hvictl |= ext_irq_pending; |
65 | csr_write(CSR_HVICTL, hvictl); |
66 | } |
67 | |
68 | #ifdef CONFIG_32BIT |
69 | void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) |
70 | { |
71 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
72 | unsigned long mask, val; |
73 | |
74 | if (!kvm_riscv_aia_available()) |
75 | return; |
76 | |
77 | if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { |
78 | mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); |
79 | val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; |
80 | |
81 | csr->hviph &= ~mask; |
82 | csr->hviph |= val; |
83 | } |
84 | } |
85 | |
86 | void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) |
87 | { |
88 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
89 | |
90 | if (kvm_riscv_aia_available()) |
91 | csr->vsieh = csr_read(CSR_VSIEH); |
92 | } |
93 | #endif |
94 | |
95 | bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) |
96 | { |
97 | int hgei; |
98 | unsigned long seip; |
99 | |
100 | if (!kvm_riscv_aia_available()) |
101 | return false; |
102 | |
103 | #ifdef CONFIG_32BIT |
104 | if (READ_ONCE(vcpu->arch.irqs_pending[1]) & |
105 | (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) |
106 | return true; |
107 | #endif |
108 | |
109 | seip = vcpu->arch.guest_csr.vsie; |
110 | seip &= (unsigned long)mask; |
111 | seip &= BIT(IRQ_S_EXT); |
112 | |
113 | if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip) |
114 | return false; |
115 | |
116 | hgei = aia_find_hgei(owner: vcpu); |
117 | if (hgei > 0) |
118 | return !!(csr_read(CSR_HGEIP) & BIT(hgei)); |
119 | |
120 | return false; |
121 | } |
122 | |
123 | void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu) |
124 | { |
125 | struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; |
126 | |
127 | if (!kvm_riscv_aia_available()) |
128 | return; |
129 | |
130 | #ifdef CONFIG_32BIT |
131 | csr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph); |
132 | #endif |
133 | aia_set_hvictl(!!(csr->hvip & BIT(IRQ_VS_EXT))); |
134 | } |
135 | |
136 | void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu) |
137 | { |
138 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
139 | |
140 | if (!kvm_riscv_aia_available()) |
141 | return; |
142 | |
143 | csr_write(CSR_VSISELECT, csr->vsiselect); |
144 | csr_write(CSR_HVIPRIO1, csr->hviprio1); |
145 | csr_write(CSR_HVIPRIO2, csr->hviprio2); |
146 | #ifdef CONFIG_32BIT |
147 | csr_write(CSR_VSIEH, csr->vsieh); |
148 | csr_write(CSR_HVIPH, csr->hviph); |
149 | csr_write(CSR_HVIPRIO1H, csr->hviprio1h); |
150 | csr_write(CSR_HVIPRIO2H, csr->hviprio2h); |
151 | #endif |
152 | } |
153 | |
154 | void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu) |
155 | { |
156 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
157 | |
158 | if (!kvm_riscv_aia_available()) |
159 | return; |
160 | |
161 | csr->vsiselect = csr_read(CSR_VSISELECT); |
162 | csr->hviprio1 = csr_read(CSR_HVIPRIO1); |
163 | csr->hviprio2 = csr_read(CSR_HVIPRIO2); |
164 | #ifdef CONFIG_32BIT |
165 | csr->vsieh = csr_read(CSR_VSIEH); |
166 | csr->hviph = csr_read(CSR_HVIPH); |
167 | csr->hviprio1h = csr_read(CSR_HVIPRIO1H); |
168 | csr->hviprio2h = csr_read(CSR_HVIPRIO2H); |
169 | #endif |
170 | } |
171 | |
172 | int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu, |
173 | unsigned long reg_num, |
174 | unsigned long *out_val) |
175 | { |
176 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
177 | |
178 | if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) |
179 | return -ENOENT; |
180 | |
181 | *out_val = 0; |
182 | if (kvm_riscv_aia_available()) |
183 | *out_val = ((unsigned long *)csr)[reg_num]; |
184 | |
185 | return 0; |
186 | } |
187 | |
188 | int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, |
189 | unsigned long reg_num, |
190 | unsigned long val) |
191 | { |
192 | struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; |
193 | |
194 | if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) |
195 | return -ENOENT; |
196 | |
197 | if (kvm_riscv_aia_available()) { |
198 | ((unsigned long *)csr)[reg_num] = val; |
199 | |
200 | #ifdef CONFIG_32BIT |
201 | if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) |
202 | WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); |
203 | #endif |
204 | } |
205 | |
206 | return 0; |
207 | } |
208 | |
209 | int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu, |
210 | unsigned int csr_num, |
211 | unsigned long *val, |
212 | unsigned long new_val, |
213 | unsigned long wr_mask) |
214 | { |
215 | /* If AIA not available then redirect trap */ |
216 | if (!kvm_riscv_aia_available()) |
217 | return KVM_INSN_ILLEGAL_TRAP; |
218 | |
219 | /* If AIA not initialized then forward to user space */ |
220 | if (!kvm_riscv_aia_initialized(vcpu->kvm)) |
221 | return KVM_INSN_EXIT_TO_USER_SPACE; |
222 | |
223 | return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI, |
224 | val, new_val, wr_mask); |
225 | } |
226 | |
227 | /* |
228 | * External IRQ priority always read-only zero. This means default |
229 | * priority order is always preferred for external IRQs unless |
230 | * HVICTL.IID == 9 and HVICTL.IPRIO != 0 |
231 | */ |
232 | static int aia_irq2bitpos[] = { |
233 | 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */ |
234 | 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */ |
235 | 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */ |
236 | -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */ |
237 | -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */ |
238 | -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */ |
239 | -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */ |
240 | -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */ |
241 | }; |
242 | |
243 | static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq) |
244 | { |
245 | unsigned long hviprio; |
246 | int bitpos = aia_irq2bitpos[irq]; |
247 | |
248 | if (bitpos < 0) |
249 | return 0; |
250 | |
251 | switch (bitpos / BITS_PER_LONG) { |
252 | case 0: |
253 | hviprio = csr_read(CSR_HVIPRIO1); |
254 | break; |
255 | case 1: |
256 | #ifndef CONFIG_32BIT |
257 | hviprio = csr_read(CSR_HVIPRIO2); |
258 | break; |
259 | #else |
260 | hviprio = csr_read(CSR_HVIPRIO1H); |
261 | break; |
262 | case 2: |
263 | hviprio = csr_read(CSR_HVIPRIO2); |
264 | break; |
265 | case 3: |
266 | hviprio = csr_read(CSR_HVIPRIO2H); |
267 | break; |
268 | #endif |
269 | default: |
270 | return 0; |
271 | } |
272 | |
273 | return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK; |
274 | } |
275 | |
276 | static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio) |
277 | { |
278 | unsigned long hviprio; |
279 | int bitpos = aia_irq2bitpos[irq]; |
280 | |
281 | if (bitpos < 0) |
282 | return; |
283 | |
284 | switch (bitpos / BITS_PER_LONG) { |
285 | case 0: |
286 | hviprio = csr_read(CSR_HVIPRIO1); |
287 | break; |
288 | case 1: |
289 | #ifndef CONFIG_32BIT |
290 | hviprio = csr_read(CSR_HVIPRIO2); |
291 | break; |
292 | #else |
293 | hviprio = csr_read(CSR_HVIPRIO1H); |
294 | break; |
295 | case 2: |
296 | hviprio = csr_read(CSR_HVIPRIO2); |
297 | break; |
298 | case 3: |
299 | hviprio = csr_read(CSR_HVIPRIO2H); |
300 | break; |
301 | #endif |
302 | default: |
303 | return; |
304 | } |
305 | |
306 | hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG)); |
307 | hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG); |
308 | |
309 | switch (bitpos / BITS_PER_LONG) { |
310 | case 0: |
311 | csr_write(CSR_HVIPRIO1, hviprio); |
312 | break; |
313 | case 1: |
314 | #ifndef CONFIG_32BIT |
315 | csr_write(CSR_HVIPRIO2, hviprio); |
316 | break; |
317 | #else |
318 | csr_write(CSR_HVIPRIO1H, hviprio); |
319 | break; |
320 | case 2: |
321 | csr_write(CSR_HVIPRIO2, hviprio); |
322 | break; |
323 | case 3: |
324 | csr_write(CSR_HVIPRIO2H, hviprio); |
325 | break; |
326 | #endif |
327 | default: |
328 | return; |
329 | } |
330 | } |
331 | |
332 | static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel, |
333 | unsigned long *val, unsigned long new_val, |
334 | unsigned long wr_mask) |
335 | { |
336 | int i, first_irq, nirqs; |
337 | unsigned long old_val; |
338 | u8 prio; |
339 | |
340 | #ifndef CONFIG_32BIT |
341 | if (isel & 0x1) |
342 | return KVM_INSN_ILLEGAL_TRAP; |
343 | #endif |
344 | |
345 | nirqs = 4 * (BITS_PER_LONG / 32); |
346 | first_irq = (isel - ISELECT_IPRIO0) * 4; |
347 | |
348 | old_val = 0; |
349 | for (i = 0; i < nirqs; i++) { |
350 | prio = aia_get_iprio8(vcpu, irq: first_irq + i); |
351 | old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i); |
352 | } |
353 | |
354 | if (val) |
355 | *val = old_val; |
356 | |
357 | if (wr_mask) { |
358 | new_val = (old_val & ~wr_mask) | (new_val & wr_mask); |
359 | for (i = 0; i < nirqs; i++) { |
360 | prio = (new_val >> (TOPI_IPRIO_BITS * i)) & |
361 | TOPI_IPRIO_MASK; |
362 | aia_set_iprio8(vcpu, irq: first_irq + i, prio); |
363 | } |
364 | } |
365 | |
366 | return KVM_INSN_CONTINUE_NEXT_SEPC; |
367 | } |
368 | |
369 | int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num, |
370 | unsigned long *val, unsigned long new_val, |
371 | unsigned long wr_mask) |
372 | { |
373 | unsigned int isel; |
374 | |
375 | /* If AIA not available then redirect trap */ |
376 | if (!kvm_riscv_aia_available()) |
377 | return KVM_INSN_ILLEGAL_TRAP; |
378 | |
379 | /* First try to emulate in kernel space */ |
380 | isel = csr_read(CSR_VSISELECT) & ISELECT_MASK; |
381 | if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15) |
382 | return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask); |
383 | else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST && |
384 | kvm_riscv_aia_initialized(vcpu->kvm)) |
385 | return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val, |
386 | wr_mask); |
387 | |
388 | /* We can't handle it here so redirect to user space */ |
389 | return KVM_INSN_EXIT_TO_USER_SPACE; |
390 | } |
391 | |
392 | int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner, |
393 | void __iomem **hgei_va, phys_addr_t *hgei_pa) |
394 | { |
395 | int ret = -ENOENT; |
396 | unsigned long flags; |
397 | struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); |
398 | |
399 | if (!kvm_riscv_aia_available() || !hgctrl) |
400 | return -ENODEV; |
401 | |
402 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
403 | |
404 | if (hgctrl->free_bitmap) { |
405 | ret = __ffs(hgctrl->free_bitmap); |
406 | hgctrl->free_bitmap &= ~BIT(ret); |
407 | hgctrl->owners[ret] = owner; |
408 | } |
409 | |
410 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
411 | |
412 | /* TODO: To be updated later by AIA IMSIC HW guest file support */ |
413 | if (hgei_va) |
414 | *hgei_va = NULL; |
415 | if (hgei_pa) |
416 | *hgei_pa = 0; |
417 | |
418 | return ret; |
419 | } |
420 | |
421 | void kvm_riscv_aia_free_hgei(int cpu, int hgei) |
422 | { |
423 | unsigned long flags; |
424 | struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu); |
425 | |
426 | if (!kvm_riscv_aia_available() || !hgctrl) |
427 | return; |
428 | |
429 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
430 | |
431 | if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) { |
432 | if (!(hgctrl->free_bitmap & BIT(hgei))) { |
433 | hgctrl->free_bitmap |= BIT(hgei); |
434 | hgctrl->owners[hgei] = NULL; |
435 | } |
436 | } |
437 | |
438 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
439 | } |
440 | |
441 | void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable) |
442 | { |
443 | int hgei; |
444 | |
445 | if (!kvm_riscv_aia_available()) |
446 | return; |
447 | |
448 | hgei = aia_find_hgei(owner); |
449 | if (hgei > 0) { |
450 | if (enable) |
451 | csr_set(CSR_HGEIE, BIT(hgei)); |
452 | else |
453 | csr_clear(CSR_HGEIE, BIT(hgei)); |
454 | } |
455 | } |
456 | |
457 | static irqreturn_t hgei_interrupt(int irq, void *dev_id) |
458 | { |
459 | int i; |
460 | unsigned long hgei_mask, flags; |
461 | struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei); |
462 | |
463 | hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE); |
464 | csr_clear(CSR_HGEIE, hgei_mask); |
465 | |
466 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
467 | |
468 | for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) { |
469 | if (hgctrl->owners[i]) |
470 | kvm_vcpu_kick(vcpu: hgctrl->owners[i]); |
471 | } |
472 | |
473 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
474 | |
475 | put_cpu_ptr(&aia_hgei); |
476 | return IRQ_HANDLED; |
477 | } |
478 | |
479 | static int aia_hgei_init(void) |
480 | { |
481 | int cpu, rc; |
482 | struct irq_domain *domain; |
483 | struct aia_hgei_control *hgctrl; |
484 | |
485 | /* Initialize per-CPU guest external interrupt line management */ |
486 | for_each_possible_cpu(cpu) { |
487 | hgctrl = per_cpu_ptr(&aia_hgei, cpu); |
488 | raw_spin_lock_init(&hgctrl->lock); |
489 | if (kvm_riscv_aia_nr_hgei) { |
490 | hgctrl->free_bitmap = |
491 | BIT(kvm_riscv_aia_nr_hgei + 1) - 1; |
492 | hgctrl->free_bitmap &= ~BIT(0); |
493 | } else |
494 | hgctrl->free_bitmap = 0; |
495 | } |
496 | |
497 | /* Find INTC irq domain */ |
498 | domain = irq_find_matching_fwnode(fwnode: riscv_get_intc_hwnode(), |
499 | bus_token: DOMAIN_BUS_ANY); |
500 | if (!domain) { |
501 | kvm_err("unable to find INTC domain\n" ); |
502 | return -ENOENT; |
503 | } |
504 | |
505 | /* Map per-CPU SGEI interrupt from INTC domain */ |
506 | hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT); |
507 | if (!hgei_parent_irq) { |
508 | kvm_err("unable to map SGEI IRQ\n" ); |
509 | return -ENOMEM; |
510 | } |
511 | |
512 | /* Request per-CPU SGEI interrupt */ |
513 | rc = request_percpu_irq(irq: hgei_parent_irq, handler: hgei_interrupt, |
514 | devname: "riscv-kvm" , percpu_dev_id: &aia_hgei); |
515 | if (rc) { |
516 | kvm_err("failed to request SGEI IRQ\n" ); |
517 | return rc; |
518 | } |
519 | |
520 | return 0; |
521 | } |
522 | |
523 | static void aia_hgei_exit(void) |
524 | { |
525 | /* Free per-CPU SGEI interrupt */ |
526 | free_percpu_irq(hgei_parent_irq, &aia_hgei); |
527 | } |
528 | |
529 | void kvm_riscv_aia_enable(void) |
530 | { |
531 | if (!kvm_riscv_aia_available()) |
532 | return; |
533 | |
534 | aia_set_hvictl(ext_irq_pending: false); |
535 | csr_write(CSR_HVIPRIO1, 0x0); |
536 | csr_write(CSR_HVIPRIO2, 0x0); |
537 | #ifdef CONFIG_32BIT |
538 | csr_write(CSR_HVIPH, 0x0); |
539 | csr_write(CSR_HIDELEGH, 0x0); |
540 | csr_write(CSR_HVIPRIO1H, 0x0); |
541 | csr_write(CSR_HVIPRIO2H, 0x0); |
542 | #endif |
543 | |
544 | /* Enable per-CPU SGEI interrupt */ |
545 | enable_percpu_irq(irq: hgei_parent_irq, |
546 | type: irq_get_trigger_type(irq: hgei_parent_irq)); |
547 | csr_set(CSR_HIE, BIT(IRQ_S_GEXT)); |
548 | } |
549 | |
550 | void kvm_riscv_aia_disable(void) |
551 | { |
552 | int i; |
553 | unsigned long flags; |
554 | struct kvm_vcpu *vcpu; |
555 | struct aia_hgei_control *hgctrl; |
556 | |
557 | if (!kvm_riscv_aia_available()) |
558 | return; |
559 | hgctrl = get_cpu_ptr(&aia_hgei); |
560 | |
561 | /* Disable per-CPU SGEI interrupt */ |
562 | csr_clear(CSR_HIE, BIT(IRQ_S_GEXT)); |
563 | disable_percpu_irq(irq: hgei_parent_irq); |
564 | |
565 | aia_set_hvictl(ext_irq_pending: false); |
566 | |
567 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
568 | |
569 | for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) { |
570 | vcpu = hgctrl->owners[i]; |
571 | if (!vcpu) |
572 | continue; |
573 | |
574 | /* |
575 | * We release hgctrl->lock before notifying IMSIC |
576 | * so that we don't have lock ordering issues. |
577 | */ |
578 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
579 | |
580 | /* Notify IMSIC */ |
581 | kvm_riscv_vcpu_aia_imsic_release(vcpu); |
582 | |
583 | /* |
584 | * Wakeup VCPU if it was blocked so that it can |
585 | * run on other HARTs |
586 | */ |
587 | if (csr_read(CSR_HGEIE) & BIT(i)) { |
588 | csr_clear(CSR_HGEIE, BIT(i)); |
589 | kvm_vcpu_kick(vcpu); |
590 | } |
591 | |
592 | raw_spin_lock_irqsave(&hgctrl->lock, flags); |
593 | } |
594 | |
595 | raw_spin_unlock_irqrestore(&hgctrl->lock, flags); |
596 | |
597 | put_cpu_ptr(&aia_hgei); |
598 | } |
599 | |
600 | int kvm_riscv_aia_init(void) |
601 | { |
602 | int rc; |
603 | |
604 | if (!riscv_isa_extension_available(NULL, SxAIA)) |
605 | return -ENODEV; |
606 | |
607 | /* Figure-out number of bits in HGEIE */ |
608 | csr_write(CSR_HGEIE, -1UL); |
609 | kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE)); |
610 | csr_write(CSR_HGEIE, 0); |
611 | if (kvm_riscv_aia_nr_hgei) |
612 | kvm_riscv_aia_nr_hgei--; |
613 | |
614 | /* |
615 | * Number of usable HGEI lines should be minimum of per-HART |
616 | * IMSIC guest files and number of bits in HGEIE |
617 | * |
618 | * TODO: To be updated later by AIA IMSIC HW guest file support |
619 | */ |
620 | kvm_riscv_aia_nr_hgei = 0; |
621 | |
622 | /* |
623 | * Find number of guest MSI IDs |
624 | * |
625 | * TODO: To be updated later by AIA IMSIC HW guest file support |
626 | */ |
627 | kvm_riscv_aia_max_ids = IMSIC_MAX_ID; |
628 | |
629 | /* Initialize guest external interrupt line management */ |
630 | rc = aia_hgei_init(); |
631 | if (rc) |
632 | return rc; |
633 | |
634 | /* Register device operations */ |
635 | rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops, |
636 | KVM_DEV_TYPE_RISCV_AIA); |
637 | if (rc) { |
638 | aia_hgei_exit(); |
639 | return rc; |
640 | } |
641 | |
642 | /* Enable KVM AIA support */ |
643 | static_branch_enable(&kvm_riscv_aia_available); |
644 | |
645 | return 0; |
646 | } |
647 | |
648 | void kvm_riscv_aia_exit(void) |
649 | { |
650 | if (!kvm_riscv_aia_available()) |
651 | return; |
652 | |
653 | /* Unregister device operations */ |
654 | kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA); |
655 | |
656 | /* Cleanup the HGEI state */ |
657 | aia_hgei_exit(); |
658 | } |
659 | |