1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright 2015 IBM Corp. |
4 | */ |
5 | |
6 | #include <linux/spinlock.h> |
7 | #include <linux/uaccess.h> |
8 | #include <linux/delay.h> |
9 | #include <linux/irqdomain.h> |
10 | #include <linux/platform_device.h> |
11 | |
12 | #include "cxl.h" |
13 | #include "hcalls.h" |
14 | #include "trace.h" |
15 | |
16 | #define CXL_ERROR_DETECTED_EVENT 1 |
17 | #define CXL_SLOT_RESET_EVENT 2 |
18 | #define CXL_RESUME_EVENT 3 |
19 | |
20 | static void pci_error_handlers(struct cxl_afu *afu, |
21 | int bus_error_event, |
22 | pci_channel_state_t state) |
23 | { |
24 | struct pci_dev *afu_dev; |
25 | struct pci_driver *afu_drv; |
26 | const struct pci_error_handlers *err_handler; |
27 | |
28 | if (afu->phb == NULL) |
29 | return; |
30 | |
31 | list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) { |
32 | afu_drv = to_pci_driver(drv: afu_dev->dev.driver); |
33 | if (!afu_drv) |
34 | continue; |
35 | |
36 | err_handler = afu_drv->err_handler; |
37 | switch (bus_error_event) { |
38 | case CXL_ERROR_DETECTED_EVENT: |
39 | afu_dev->error_state = state; |
40 | |
41 | if (err_handler && |
42 | err_handler->error_detected) |
43 | err_handler->error_detected(afu_dev, state); |
44 | break; |
45 | case CXL_SLOT_RESET_EVENT: |
46 | afu_dev->error_state = state; |
47 | |
48 | if (err_handler && |
49 | err_handler->slot_reset) |
50 | err_handler->slot_reset(afu_dev); |
51 | break; |
52 | case CXL_RESUME_EVENT: |
53 | if (err_handler && |
54 | err_handler->resume) |
55 | err_handler->resume(afu_dev); |
56 | break; |
57 | } |
58 | } |
59 | } |
60 | |
61 | static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr, |
62 | u64 errstat) |
63 | { |
64 | pr_devel("in %s\n" , __func__); |
65 | dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n" , errstat); |
66 | |
67 | return cxl_ops->ack_irq(ctx, 0, errstat); |
68 | } |
69 | |
70 | static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu, |
71 | void *buf, size_t len) |
72 | { |
73 | unsigned int entries, mod; |
74 | unsigned long **vpd_buf = NULL; |
75 | struct sg_list *le; |
76 | int rc = 0, i, tocopy; |
77 | u64 out = 0; |
78 | |
79 | if (buf == NULL) |
80 | return -EINVAL; |
81 | |
82 | /* number of entries in the list */ |
83 | entries = len / SG_BUFFER_SIZE; |
84 | mod = len % SG_BUFFER_SIZE; |
85 | if (mod) |
86 | entries++; |
87 | |
88 | if (entries > SG_MAX_ENTRIES) { |
89 | entries = SG_MAX_ENTRIES; |
90 | len = SG_MAX_ENTRIES * SG_BUFFER_SIZE; |
91 | mod = 0; |
92 | } |
93 | |
94 | vpd_buf = kcalloc(n: entries, size: sizeof(unsigned long *), GFP_KERNEL); |
95 | if (!vpd_buf) |
96 | return -ENOMEM; |
97 | |
98 | le = (struct sg_list *)get_zeroed_page(GFP_KERNEL); |
99 | if (!le) { |
100 | rc = -ENOMEM; |
101 | goto err1; |
102 | } |
103 | |
104 | for (i = 0; i < entries; i++) { |
105 | vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL); |
106 | if (!vpd_buf[i]) { |
107 | rc = -ENOMEM; |
108 | goto err2; |
109 | } |
110 | le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i])); |
111 | le[i].len = cpu_to_be64(SG_BUFFER_SIZE); |
112 | if ((i == (entries - 1)) && mod) |
113 | le[i].len = cpu_to_be64(mod); |
114 | } |
115 | |
116 | if (adapter) |
117 | rc = cxl_h_collect_vpd_adapter(unit_address: adapter->guest->handle, |
118 | virt_to_phys(address: le), num: entries, out: &out); |
119 | else |
120 | rc = cxl_h_collect_vpd(unit_address: afu->guest->handle, record: 0, |
121 | virt_to_phys(address: le), num: entries, out: &out); |
122 | pr_devel("length of available (entries: %i), vpd: %#llx\n" , |
123 | entries, out); |
124 | |
125 | if (!rc) { |
126 | /* |
127 | * hcall returns in 'out' the size of available VPDs. |
128 | * It fills the buffer with as much data as possible. |
129 | */ |
130 | if (out < len) |
131 | len = out; |
132 | rc = len; |
133 | if (out) { |
134 | for (i = 0; i < entries; i++) { |
135 | if (len < SG_BUFFER_SIZE) |
136 | tocopy = len; |
137 | else |
138 | tocopy = SG_BUFFER_SIZE; |
139 | memcpy(buf, vpd_buf[i], tocopy); |
140 | buf += tocopy; |
141 | len -= tocopy; |
142 | } |
143 | } |
144 | } |
145 | err2: |
146 | for (i = 0; i < entries; i++) { |
147 | if (vpd_buf[i]) |
148 | free_page((unsigned long) vpd_buf[i]); |
149 | } |
150 | free_page((unsigned long) le); |
151 | err1: |
152 | kfree(objp: vpd_buf); |
153 | return rc; |
154 | } |
155 | |
156 | static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info) |
157 | { |
158 | return cxl_h_collect_int_info(unit_address: ctx->afu->guest->handle, process_token: ctx->process_token, info); |
159 | } |
160 | |
161 | static irqreturn_t guest_psl_irq(int irq, void *data) |
162 | { |
163 | struct cxl_context *ctx = data; |
164 | struct cxl_irq_info irq_info; |
165 | int rc; |
166 | |
167 | pr_devel("%d: received PSL interrupt %i\n" , ctx->pe, irq); |
168 | rc = guest_get_irq_info(ctx, info: &irq_info); |
169 | if (rc) { |
170 | WARN(1, "Unable to get IRQ info: %i\n" , rc); |
171 | return IRQ_HANDLED; |
172 | } |
173 | |
174 | rc = cxl_irq_psl8(irq, ctx, irq_info: &irq_info); |
175 | return rc; |
176 | } |
177 | |
178 | static int afu_read_error_state(struct cxl_afu *afu, int *state_out) |
179 | { |
180 | u64 state; |
181 | int rc = 0; |
182 | |
183 | if (!afu) |
184 | return -EIO; |
185 | |
186 | rc = cxl_h_read_error_state(unit_address: afu->guest->handle, state: &state); |
187 | if (!rc) { |
188 | WARN_ON(state != H_STATE_NORMAL && |
189 | state != H_STATE_DISABLE && |
190 | state != H_STATE_TEMP_UNAVAILABLE && |
191 | state != H_STATE_PERM_UNAVAILABLE); |
192 | *state_out = state & 0xffffffff; |
193 | } |
194 | return rc; |
195 | } |
196 | |
197 | static irqreturn_t guest_slice_irq_err(int irq, void *data) |
198 | { |
199 | struct cxl_afu *afu = data; |
200 | int rc; |
201 | u64 serr, afu_error, dsisr; |
202 | |
203 | rc = cxl_h_get_fn_error_interrupt(unit_address: afu->guest->handle, reg: &serr); |
204 | if (rc) { |
205 | dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n" , rc); |
206 | return IRQ_HANDLED; |
207 | } |
208 | afu_error = cxl_p2n_read(afu, reg: CXL_AFU_ERR_An); |
209 | dsisr = cxl_p2n_read(afu, reg: CXL_PSL_DSISR_An); |
210 | cxl_afu_decode_psl_serr(afu, serr); |
211 | dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n" , afu_error); |
212 | dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n" , dsisr); |
213 | |
214 | rc = cxl_h_ack_fn_error_interrupt(unit_address: afu->guest->handle, value: serr); |
215 | if (rc) |
216 | dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n" , |
217 | rc); |
218 | |
219 | return IRQ_HANDLED; |
220 | } |
221 | |
222 | |
223 | static int irq_alloc_range(struct cxl *adapter, int len, int *irq) |
224 | { |
225 | int i, n; |
226 | struct irq_avail *cur; |
227 | |
228 | for (i = 0; i < adapter->guest->irq_nranges; i++) { |
229 | cur = &adapter->guest->irq_avail[i]; |
230 | n = bitmap_find_next_zero_area(map: cur->bitmap, size: cur->range, |
231 | start: 0, nr: len, align_mask: 0); |
232 | if (n < cur->range) { |
233 | bitmap_set(map: cur->bitmap, start: n, nbits: len); |
234 | *irq = cur->offset + n; |
235 | pr_devel("guest: allocate IRQs %#x->%#x\n" , |
236 | *irq, *irq + len - 1); |
237 | |
238 | return 0; |
239 | } |
240 | } |
241 | return -ENOSPC; |
242 | } |
243 | |
244 | static int irq_free_range(struct cxl *adapter, int irq, int len) |
245 | { |
246 | int i, n; |
247 | struct irq_avail *cur; |
248 | |
249 | if (len == 0) |
250 | return -ENOENT; |
251 | |
252 | for (i = 0; i < adapter->guest->irq_nranges; i++) { |
253 | cur = &adapter->guest->irq_avail[i]; |
254 | if (irq >= cur->offset && |
255 | (irq + len) <= (cur->offset + cur->range)) { |
256 | n = irq - cur->offset; |
257 | bitmap_clear(map: cur->bitmap, start: n, nbits: len); |
258 | pr_devel("guest: release IRQs %#x->%#x\n" , |
259 | irq, irq + len - 1); |
260 | return 0; |
261 | } |
262 | } |
263 | return -ENOENT; |
264 | } |
265 | |
266 | static int guest_reset(struct cxl *adapter) |
267 | { |
268 | struct cxl_afu *afu = NULL; |
269 | int i, rc; |
270 | |
271 | pr_devel("Adapter reset request\n" ); |
272 | spin_lock(lock: &adapter->afu_list_lock); |
273 | for (i = 0; i < adapter->slices; i++) { |
274 | if ((afu = adapter->afu[i])) { |
275 | pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT, |
276 | state: pci_channel_io_frozen); |
277 | cxl_context_detach_all(afu); |
278 | } |
279 | } |
280 | |
281 | rc = cxl_h_reset_adapter(unit_address: adapter->guest->handle); |
282 | for (i = 0; i < adapter->slices; i++) { |
283 | if (!rc && (afu = adapter->afu[i])) { |
284 | pci_error_handlers(afu, CXL_SLOT_RESET_EVENT, |
285 | state: pci_channel_io_normal); |
286 | pci_error_handlers(afu, CXL_RESUME_EVENT, state: 0); |
287 | } |
288 | } |
289 | spin_unlock(lock: &adapter->afu_list_lock); |
290 | return rc; |
291 | } |
292 | |
293 | static int guest_alloc_one_irq(struct cxl *adapter) |
294 | { |
295 | int irq; |
296 | |
297 | spin_lock(lock: &adapter->guest->irq_alloc_lock); |
298 | if (irq_alloc_range(adapter, len: 1, irq: &irq)) |
299 | irq = -ENOSPC; |
300 | spin_unlock(lock: &adapter->guest->irq_alloc_lock); |
301 | return irq; |
302 | } |
303 | |
304 | static void guest_release_one_irq(struct cxl *adapter, int irq) |
305 | { |
306 | spin_lock(lock: &adapter->guest->irq_alloc_lock); |
307 | irq_free_range(adapter, irq, len: 1); |
308 | spin_unlock(lock: &adapter->guest->irq_alloc_lock); |
309 | } |
310 | |
311 | static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs, |
312 | struct cxl *adapter, unsigned int num) |
313 | { |
314 | int i, try, irq; |
315 | |
316 | memset(irqs, 0, sizeof(struct cxl_irq_ranges)); |
317 | |
318 | spin_lock(lock: &adapter->guest->irq_alloc_lock); |
319 | for (i = 0; i < CXL_IRQ_RANGES && num; i++) { |
320 | try = num; |
321 | while (try) { |
322 | if (irq_alloc_range(adapter, len: try, irq: &irq) == 0) |
323 | break; |
324 | try /= 2; |
325 | } |
326 | if (!try) |
327 | goto error; |
328 | irqs->offset[i] = irq; |
329 | irqs->range[i] = try; |
330 | num -= try; |
331 | } |
332 | if (num) |
333 | goto error; |
334 | spin_unlock(lock: &adapter->guest->irq_alloc_lock); |
335 | return 0; |
336 | |
337 | error: |
338 | for (i = 0; i < CXL_IRQ_RANGES; i++) |
339 | irq_free_range(adapter, irq: irqs->offset[i], len: irqs->range[i]); |
340 | spin_unlock(lock: &adapter->guest->irq_alloc_lock); |
341 | return -ENOSPC; |
342 | } |
343 | |
344 | static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs, |
345 | struct cxl *adapter) |
346 | { |
347 | int i; |
348 | |
349 | spin_lock(lock: &adapter->guest->irq_alloc_lock); |
350 | for (i = 0; i < CXL_IRQ_RANGES; i++) |
351 | irq_free_range(adapter, irq: irqs->offset[i], len: irqs->range[i]); |
352 | spin_unlock(lock: &adapter->guest->irq_alloc_lock); |
353 | } |
354 | |
355 | static int guest_register_serr_irq(struct cxl_afu *afu) |
356 | { |
357 | afu->err_irq_name = kasprintf(GFP_KERNEL, fmt: "cxl-%s-err" , |
358 | dev_name(dev: &afu->dev)); |
359 | if (!afu->err_irq_name) |
360 | return -ENOMEM; |
361 | |
362 | if (!(afu->serr_virq = cxl_map_irq(adapter: afu->adapter, hwirq: afu->serr_hwirq, |
363 | handler: guest_slice_irq_err, cookie: afu, name: afu->err_irq_name))) { |
364 | kfree(objp: afu->err_irq_name); |
365 | afu->err_irq_name = NULL; |
366 | return -ENOMEM; |
367 | } |
368 | |
369 | return 0; |
370 | } |
371 | |
372 | static void guest_release_serr_irq(struct cxl_afu *afu) |
373 | { |
374 | cxl_unmap_irq(virq: afu->serr_virq, cookie: afu); |
375 | cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq); |
376 | kfree(objp: afu->err_irq_name); |
377 | } |
378 | |
379 | static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask) |
380 | { |
381 | return cxl_h_control_faults(unit_address: ctx->afu->guest->handle, process_token: ctx->process_token, |
382 | control_mask: tfc >> 32, reset_mask: (psl_reset_mask != 0)); |
383 | } |
384 | |
385 | static void disable_afu_irqs(struct cxl_context *ctx) |
386 | { |
387 | irq_hw_number_t hwirq; |
388 | unsigned int virq; |
389 | int r, i; |
390 | |
391 | pr_devel("Disabling AFU(%d) interrupts\n" , ctx->afu->slice); |
392 | for (r = 0; r < CXL_IRQ_RANGES; r++) { |
393 | hwirq = ctx->irqs.offset[r]; |
394 | for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) { |
395 | virq = irq_find_mapping(NULL, hwirq); |
396 | disable_irq(irq: virq); |
397 | } |
398 | } |
399 | } |
400 | |
401 | static void enable_afu_irqs(struct cxl_context *ctx) |
402 | { |
403 | irq_hw_number_t hwirq; |
404 | unsigned int virq; |
405 | int r, i; |
406 | |
407 | pr_devel("Enabling AFU(%d) interrupts\n" , ctx->afu->slice); |
408 | for (r = 0; r < CXL_IRQ_RANGES; r++) { |
409 | hwirq = ctx->irqs.offset[r]; |
410 | for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) { |
411 | virq = irq_find_mapping(NULL, hwirq); |
412 | enable_irq(irq: virq); |
413 | } |
414 | } |
415 | } |
416 | |
417 | static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx, |
418 | u64 offset, u64 *val) |
419 | { |
420 | unsigned long cr; |
421 | char c; |
422 | int rc = 0; |
423 | |
424 | if (afu->crs_len < sz) |
425 | return -ENOENT; |
426 | |
427 | if (unlikely(offset >= afu->crs_len)) |
428 | return -ERANGE; |
429 | |
430 | cr = get_zeroed_page(GFP_KERNEL); |
431 | if (!cr) |
432 | return -ENOMEM; |
433 | |
434 | rc = cxl_h_get_config(unit_address: afu->guest->handle, cr_num: cr_idx, offset, |
435 | virt_to_phys(address: (void *)cr), len: sz); |
436 | if (rc) |
437 | goto err; |
438 | |
439 | switch (sz) { |
440 | case 1: |
441 | c = *((char *) cr); |
442 | *val = c; |
443 | break; |
444 | case 2: |
445 | *val = in_le16((u16 *)cr); |
446 | break; |
447 | case 4: |
448 | *val = in_le32((unsigned *)cr); |
449 | break; |
450 | case 8: |
451 | *val = in_le64((u64 *)cr); |
452 | break; |
453 | default: |
454 | WARN_ON(1); |
455 | } |
456 | err: |
457 | free_page(cr); |
458 | return rc; |
459 | } |
460 | |
461 | static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset, |
462 | u32 *out) |
463 | { |
464 | int rc; |
465 | u64 val; |
466 | |
467 | rc = _guest_afu_cr_readXX(sz: 4, afu, cr_idx, offset, val: &val); |
468 | if (!rc) |
469 | *out = (u32) val; |
470 | return rc; |
471 | } |
472 | |
473 | static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset, |
474 | u16 *out) |
475 | { |
476 | int rc; |
477 | u64 val; |
478 | |
479 | rc = _guest_afu_cr_readXX(sz: 2, afu, cr_idx, offset, val: &val); |
480 | if (!rc) |
481 | *out = (u16) val; |
482 | return rc; |
483 | } |
484 | |
485 | static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset, |
486 | u8 *out) |
487 | { |
488 | int rc; |
489 | u64 val; |
490 | |
491 | rc = _guest_afu_cr_readXX(sz: 1, afu, cr_idx, offset, val: &val); |
492 | if (!rc) |
493 | *out = (u8) val; |
494 | return rc; |
495 | } |
496 | |
497 | static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset, |
498 | u64 *out) |
499 | { |
500 | return _guest_afu_cr_readXX(sz: 8, afu, cr_idx, offset, val: out); |
501 | } |
502 | |
503 | static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in) |
504 | { |
505 | /* config record is not writable from guest */ |
506 | return -EPERM; |
507 | } |
508 | |
509 | static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in) |
510 | { |
511 | /* config record is not writable from guest */ |
512 | return -EPERM; |
513 | } |
514 | |
515 | static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in) |
516 | { |
517 | /* config record is not writable from guest */ |
518 | return -EPERM; |
519 | } |
520 | |
521 | static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) |
522 | { |
523 | struct cxl_process_element_hcall *elem; |
524 | struct cxl *adapter = ctx->afu->adapter; |
525 | const struct cred *cred; |
526 | u32 pid, idx; |
527 | int rc, r, i; |
528 | u64 mmio_addr, mmio_size; |
529 | __be64 flags = 0; |
530 | |
531 | /* Must be 8 byte aligned and cannot cross a 4096 byte boundary */ |
532 | if (!(elem = (struct cxl_process_element_hcall *) |
533 | get_zeroed_page(GFP_KERNEL))) |
534 | return -ENOMEM; |
535 | |
536 | elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION); |
537 | if (ctx->kernel) { |
538 | pid = 0; |
539 | flags |= CXL_PE_TRANSLATION_ENABLED; |
540 | flags |= CXL_PE_PRIVILEGED_PROCESS; |
541 | if (mfmsr() & MSR_SF) |
542 | flags |= CXL_PE_64_BIT; |
543 | } else { |
544 | pid = current->pid; |
545 | flags |= CXL_PE_PROBLEM_STATE; |
546 | flags |= CXL_PE_TRANSLATION_ENABLED; |
547 | if (!test_tsk_thread_flag(current, flag: TIF_32BIT)) |
548 | flags |= CXL_PE_64_BIT; |
549 | cred = get_current_cred(); |
550 | if (uid_eq(left: cred->euid, GLOBAL_ROOT_UID)) |
551 | flags |= CXL_PE_PRIVILEGED_PROCESS; |
552 | put_cred(cred); |
553 | } |
554 | elem->flags = cpu_to_be64(flags); |
555 | elem->common.tid = cpu_to_be32(0); /* Unused */ |
556 | elem->common.pid = cpu_to_be32(pid); |
557 | elem->common.csrp = cpu_to_be64(0); /* disable */ |
558 | elem->common.u.psl8.aurp0 = cpu_to_be64(0); /* disable */ |
559 | elem->common.u.psl8.aurp1 = cpu_to_be64(0); /* disable */ |
560 | |
561 | cxl_prefault(ctx, wed); |
562 | |
563 | elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0); |
564 | elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1); |
565 | |
566 | /* |
567 | * Ensure we have at least one interrupt allocated to take faults for |
568 | * kernel contexts that may not have allocated any AFU IRQs at all: |
569 | */ |
570 | if (ctx->irqs.range[0] == 0) { |
571 | rc = afu_register_irqs(ctx, 0); |
572 | if (rc) |
573 | goto out_free; |
574 | } |
575 | |
576 | for (r = 0; r < CXL_IRQ_RANGES; r++) { |
577 | for (i = 0; i < ctx->irqs.range[r]; i++) { |
578 | if (r == 0 && i == 0) { |
579 | elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]); |
580 | } else { |
581 | idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset; |
582 | elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8); |
583 | } |
584 | } |
585 | } |
586 | elem->common.amr = cpu_to_be64(amr); |
587 | elem->common.wed = cpu_to_be64(wed); |
588 | |
589 | disable_afu_irqs(ctx); |
590 | |
591 | rc = cxl_h_attach_process(ctx->afu->guest->handle, elem, |
592 | &ctx->process_token, &mmio_addr, &mmio_size); |
593 | if (rc == H_SUCCESS) { |
594 | if (ctx->master || !ctx->afu->pp_psa) { |
595 | ctx->psn_phys = ctx->afu->psn_phys; |
596 | ctx->psn_size = ctx->afu->adapter->ps_size; |
597 | } else { |
598 | ctx->psn_phys = mmio_addr; |
599 | ctx->psn_size = mmio_size; |
600 | } |
601 | if (ctx->afu->pp_psa && mmio_size && |
602 | ctx->afu->pp_size == 0) { |
603 | /* |
604 | * There's no property in the device tree to read the |
605 | * pp_size. We only find out at the 1st attach. |
606 | * Compared to bare-metal, it is too late and we |
607 | * should really lock here. However, on powerVM, |
608 | * pp_size is really only used to display in /sys. |
609 | * Being discussed with pHyp for their next release. |
610 | */ |
611 | ctx->afu->pp_size = mmio_size; |
612 | } |
613 | /* from PAPR: process element is bytes 4-7 of process token */ |
614 | ctx->external_pe = ctx->process_token & 0xFFFFFFFF; |
615 | pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx" , |
616 | ctx->pe, ctx->external_pe, ctx->psn_size); |
617 | ctx->pe_inserted = true; |
618 | enable_afu_irqs(ctx); |
619 | } |
620 | |
621 | out_free: |
622 | free_page((u64)elem); |
623 | return rc; |
624 | } |
625 | |
626 | static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr) |
627 | { |
628 | pr_devel("in %s\n" , __func__); |
629 | |
630 | ctx->kernel = kernel; |
631 | if (ctx->afu->current_mode == CXL_MODE_DIRECTED) |
632 | return attach_afu_directed(ctx, wed, amr); |
633 | |
634 | /* dedicated mode not supported on FW840 */ |
635 | |
636 | return -EINVAL; |
637 | } |
638 | |
639 | static int detach_afu_directed(struct cxl_context *ctx) |
640 | { |
641 | if (!ctx->pe_inserted) |
642 | return 0; |
643 | if (cxl_h_detach_process(unit_address: ctx->afu->guest->handle, process_token: ctx->process_token)) |
644 | return -1; |
645 | return 0; |
646 | } |
647 | |
648 | static int guest_detach_process(struct cxl_context *ctx) |
649 | { |
650 | pr_devel("in %s\n" , __func__); |
651 | trace_cxl_detach(ctx); |
652 | |
653 | if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) |
654 | return -EIO; |
655 | |
656 | if (ctx->afu->current_mode == CXL_MODE_DIRECTED) |
657 | return detach_afu_directed(ctx); |
658 | |
659 | return -EINVAL; |
660 | } |
661 | |
662 | static void guest_release_afu(struct device *dev) |
663 | { |
664 | struct cxl_afu *afu = to_cxl_afu(dev); |
665 | |
666 | pr_devel("%s\n" , __func__); |
667 | |
668 | idr_destroy(&afu->contexts_idr); |
669 | |
670 | kfree(objp: afu->guest); |
671 | kfree(objp: afu); |
672 | } |
673 | |
674 | ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len) |
675 | { |
676 | return guest_collect_vpd(NULL, afu, buf, len); |
677 | } |
678 | |
679 | #define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE |
680 | static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf, |
681 | loff_t off, size_t count) |
682 | { |
683 | void *tbuf = NULL; |
684 | int rc = 0; |
685 | |
686 | tbuf = (void *) get_zeroed_page(GFP_KERNEL); |
687 | if (!tbuf) |
688 | return -ENOMEM; |
689 | |
690 | rc = cxl_h_get_afu_err(unit_address: afu->guest->handle, |
691 | offset: off & 0x7, |
692 | virt_to_phys(address: tbuf), |
693 | len: count); |
694 | if (rc) |
695 | goto err; |
696 | |
697 | if (count > ERR_BUFF_MAX_COPY_SIZE) |
698 | count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7); |
699 | memcpy(buf, tbuf, count); |
700 | err: |
701 | free_page((u64)tbuf); |
702 | |
703 | return rc; |
704 | } |
705 | |
706 | static int guest_afu_check_and_enable(struct cxl_afu *afu) |
707 | { |
708 | return 0; |
709 | } |
710 | |
711 | static bool guest_support_attributes(const char *attr_name, |
712 | enum cxl_attrs type) |
713 | { |
714 | switch (type) { |
715 | case CXL_ADAPTER_ATTRS: |
716 | if ((strcmp(attr_name, "base_image" ) == 0) || |
717 | (strcmp(attr_name, "load_image_on_perst" ) == 0) || |
718 | (strcmp(attr_name, "perst_reloads_same_image" ) == 0) || |
719 | (strcmp(attr_name, "image_loaded" ) == 0)) |
720 | return false; |
721 | break; |
722 | case CXL_AFU_MASTER_ATTRS: |
723 | if ((strcmp(attr_name, "pp_mmio_off" ) == 0)) |
724 | return false; |
725 | break; |
726 | case CXL_AFU_ATTRS: |
727 | break; |
728 | default: |
729 | break; |
730 | } |
731 | |
732 | return true; |
733 | } |
734 | |
735 | static int activate_afu_directed(struct cxl_afu *afu) |
736 | { |
737 | int rc; |
738 | |
739 | dev_info(&afu->dev, "Activating AFU(%d) directed mode\n" , afu->slice); |
740 | |
741 | afu->current_mode = CXL_MODE_DIRECTED; |
742 | |
743 | afu->num_procs = afu->max_procs_virtualised; |
744 | |
745 | if ((rc = cxl_chardev_m_afu_add(afu))) |
746 | return rc; |
747 | |
748 | if ((rc = cxl_sysfs_afu_m_add(afu))) |
749 | goto err; |
750 | |
751 | if ((rc = cxl_chardev_s_afu_add(afu))) |
752 | goto err1; |
753 | |
754 | return 0; |
755 | err1: |
756 | cxl_sysfs_afu_m_remove(afu); |
757 | err: |
758 | cxl_chardev_afu_remove(afu); |
759 | return rc; |
760 | } |
761 | |
762 | static int guest_afu_activate_mode(struct cxl_afu *afu, int mode) |
763 | { |
764 | if (!mode) |
765 | return 0; |
766 | if (!(mode & afu->modes_supported)) |
767 | return -EINVAL; |
768 | |
769 | if (mode == CXL_MODE_DIRECTED) |
770 | return activate_afu_directed(afu); |
771 | |
772 | if (mode == CXL_MODE_DEDICATED) |
773 | dev_err(&afu->dev, "Dedicated mode not supported\n" ); |
774 | |
775 | return -EINVAL; |
776 | } |
777 | |
778 | static int deactivate_afu_directed(struct cxl_afu *afu) |
779 | { |
780 | dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n" , afu->slice); |
781 | |
782 | afu->current_mode = 0; |
783 | afu->num_procs = 0; |
784 | |
785 | cxl_sysfs_afu_m_remove(afu); |
786 | cxl_chardev_afu_remove(afu); |
787 | |
788 | cxl_ops->afu_reset(afu); |
789 | |
790 | return 0; |
791 | } |
792 | |
793 | static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode) |
794 | { |
795 | if (!mode) |
796 | return 0; |
797 | if (!(mode & afu->modes_supported)) |
798 | return -EINVAL; |
799 | |
800 | if (mode == CXL_MODE_DIRECTED) |
801 | return deactivate_afu_directed(afu); |
802 | return 0; |
803 | } |
804 | |
805 | static int guest_afu_reset(struct cxl_afu *afu) |
806 | { |
807 | pr_devel("AFU(%d) reset request\n" , afu->slice); |
808 | return cxl_h_reset_afu(unit_address: afu->guest->handle); |
809 | } |
810 | |
811 | static int guest_map_slice_regs(struct cxl_afu *afu) |
812 | { |
813 | if (!(afu->p2n_mmio = ioremap(offset: afu->guest->p2n_phys, size: afu->guest->p2n_size))) { |
814 | dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n" , |
815 | afu->slice); |
816 | return -ENOMEM; |
817 | } |
818 | return 0; |
819 | } |
820 | |
821 | static void guest_unmap_slice_regs(struct cxl_afu *afu) |
822 | { |
823 | if (afu->p2n_mmio) |
824 | iounmap(addr: afu->p2n_mmio); |
825 | } |
826 | |
827 | static int afu_update_state(struct cxl_afu *afu) |
828 | { |
829 | int rc, cur_state; |
830 | |
831 | rc = afu_read_error_state(afu, state_out: &cur_state); |
832 | if (rc) |
833 | return rc; |
834 | |
835 | if (afu->guest->previous_state == cur_state) |
836 | return 0; |
837 | |
838 | pr_devel("AFU(%d) update state to %#x\n" , afu->slice, cur_state); |
839 | |
840 | switch (cur_state) { |
841 | case H_STATE_NORMAL: |
842 | afu->guest->previous_state = cur_state; |
843 | break; |
844 | |
845 | case H_STATE_DISABLE: |
846 | pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT, |
847 | state: pci_channel_io_frozen); |
848 | |
849 | cxl_context_detach_all(afu); |
850 | if ((rc = cxl_ops->afu_reset(afu))) |
851 | pr_devel("reset hcall failed %d\n" , rc); |
852 | |
853 | rc = afu_read_error_state(afu, state_out: &cur_state); |
854 | if (!rc && cur_state == H_STATE_NORMAL) { |
855 | pci_error_handlers(afu, CXL_SLOT_RESET_EVENT, |
856 | state: pci_channel_io_normal); |
857 | pci_error_handlers(afu, CXL_RESUME_EVENT, state: 0); |
858 | } |
859 | afu->guest->previous_state = 0; |
860 | break; |
861 | |
862 | case H_STATE_TEMP_UNAVAILABLE: |
863 | afu->guest->previous_state = cur_state; |
864 | break; |
865 | |
866 | case H_STATE_PERM_UNAVAILABLE: |
867 | dev_err(&afu->dev, "AFU is in permanent error state\n" ); |
868 | pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT, |
869 | state: pci_channel_io_perm_failure); |
870 | afu->guest->previous_state = cur_state; |
871 | break; |
872 | |
873 | default: |
874 | pr_err("Unexpected AFU(%d) error state: %#x\n" , |
875 | afu->slice, cur_state); |
876 | return -EINVAL; |
877 | } |
878 | |
879 | return rc; |
880 | } |
881 | |
882 | static void afu_handle_errstate(struct work_struct *work) |
883 | { |
884 | struct cxl_afu_guest *afu_guest = |
885 | container_of(to_delayed_work(work), struct cxl_afu_guest, work_err); |
886 | |
887 | if (!afu_update_state(afu: afu_guest->parent) && |
888 | afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE) |
889 | return; |
890 | |
891 | if (afu_guest->handle_err) |
892 | schedule_delayed_work(dwork: &afu_guest->work_err, |
893 | delay: msecs_to_jiffies(m: 3000)); |
894 | } |
895 | |
896 | static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu) |
897 | { |
898 | int state; |
899 | |
900 | if (afu && (!afu_read_error_state(afu, state_out: &state))) { |
901 | if (state == H_STATE_NORMAL) |
902 | return true; |
903 | } |
904 | |
905 | return false; |
906 | } |
907 | |
908 | static int afu_properties_look_ok(struct cxl_afu *afu) |
909 | { |
910 | if (afu->pp_irqs < 0) { |
911 | dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n" ); |
912 | return -EINVAL; |
913 | } |
914 | |
915 | if (afu->max_procs_virtualised < 1) { |
916 | dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n" ); |
917 | return -EINVAL; |
918 | } |
919 | |
920 | return 0; |
921 | } |
922 | |
923 | int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np) |
924 | { |
925 | struct cxl_afu *afu; |
926 | bool free = true; |
927 | int rc; |
928 | |
929 | pr_devel("in %s - AFU(%d)\n" , __func__, slice); |
930 | if (!(afu = cxl_alloc_afu(adapter, slice))) |
931 | return -ENOMEM; |
932 | |
933 | if (!(afu->guest = kzalloc(size: sizeof(struct cxl_afu_guest), GFP_KERNEL))) { |
934 | kfree(objp: afu); |
935 | return -ENOMEM; |
936 | } |
937 | |
938 | if ((rc = dev_set_name(dev: &afu->dev, name: "afu%i.%i" , |
939 | adapter->adapter_num, |
940 | slice))) |
941 | goto err1; |
942 | |
943 | adapter->slices++; |
944 | |
945 | if ((rc = cxl_of_read_afu_handle(afu, afu_np))) |
946 | goto err1; |
947 | |
948 | if ((rc = cxl_ops->afu_reset(afu))) |
949 | goto err1; |
950 | |
951 | if ((rc = cxl_of_read_afu_properties(afu, afu_np))) |
952 | goto err1; |
953 | |
954 | if ((rc = afu_properties_look_ok(afu))) |
955 | goto err1; |
956 | |
957 | if ((rc = guest_map_slice_regs(afu))) |
958 | goto err1; |
959 | |
960 | if ((rc = guest_register_serr_irq(afu))) |
961 | goto err2; |
962 | |
963 | /* |
964 | * After we call this function we must not free the afu directly, even |
965 | * if it returns an error! |
966 | */ |
967 | if ((rc = cxl_register_afu(afu))) |
968 | goto err_put_dev; |
969 | |
970 | if ((rc = cxl_sysfs_afu_add(afu))) |
971 | goto err_del_dev; |
972 | |
973 | /* |
974 | * pHyp doesn't expose the programming models supported by the |
975 | * AFU. pHyp currently only supports directed mode. If it adds |
976 | * dedicated mode later, this version of cxl has no way to |
977 | * detect it. So we'll initialize the driver, but the first |
978 | * attach will fail. |
979 | * Being discussed with pHyp to do better (likely new property) |
980 | */ |
981 | if (afu->max_procs_virtualised == 1) |
982 | afu->modes_supported = CXL_MODE_DEDICATED; |
983 | else |
984 | afu->modes_supported = CXL_MODE_DIRECTED; |
985 | |
986 | if ((rc = cxl_afu_select_best_mode(afu))) |
987 | goto err_remove_sysfs; |
988 | |
989 | adapter->afu[afu->slice] = afu; |
990 | |
991 | afu->enabled = true; |
992 | |
993 | /* |
994 | * wake up the cpu periodically to check the state |
995 | * of the AFU using "afu" stored in the guest structure. |
996 | */ |
997 | afu->guest->parent = afu; |
998 | afu->guest->handle_err = true; |
999 | INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate); |
1000 | schedule_delayed_work(dwork: &afu->guest->work_err, delay: msecs_to_jiffies(m: 1000)); |
1001 | |
1002 | if ((rc = cxl_pci_vphb_add(afu))) |
1003 | dev_info(&afu->dev, "Can't register vPHB\n" ); |
1004 | |
1005 | return 0; |
1006 | |
1007 | err_remove_sysfs: |
1008 | cxl_sysfs_afu_remove(afu); |
1009 | err_del_dev: |
1010 | device_del(dev: &afu->dev); |
1011 | err_put_dev: |
1012 | put_device(dev: &afu->dev); |
1013 | free = false; |
1014 | guest_release_serr_irq(afu); |
1015 | err2: |
1016 | guest_unmap_slice_regs(afu); |
1017 | err1: |
1018 | if (free) { |
1019 | kfree(objp: afu->guest); |
1020 | kfree(objp: afu); |
1021 | } |
1022 | return rc; |
1023 | } |
1024 | |
1025 | void cxl_guest_remove_afu(struct cxl_afu *afu) |
1026 | { |
1027 | if (!afu) |
1028 | return; |
1029 | |
1030 | /* flush and stop pending job */ |
1031 | afu->guest->handle_err = false; |
1032 | flush_delayed_work(dwork: &afu->guest->work_err); |
1033 | |
1034 | cxl_pci_vphb_remove(afu); |
1035 | cxl_sysfs_afu_remove(afu); |
1036 | |
1037 | spin_lock(lock: &afu->adapter->afu_list_lock); |
1038 | afu->adapter->afu[afu->slice] = NULL; |
1039 | spin_unlock(lock: &afu->adapter->afu_list_lock); |
1040 | |
1041 | cxl_context_detach_all(afu); |
1042 | cxl_ops->afu_deactivate_mode(afu, afu->current_mode); |
1043 | guest_release_serr_irq(afu); |
1044 | guest_unmap_slice_regs(afu); |
1045 | |
1046 | device_unregister(dev: &afu->dev); |
1047 | } |
1048 | |
1049 | static void free_adapter(struct cxl *adapter) |
1050 | { |
1051 | struct irq_avail *cur; |
1052 | int i; |
1053 | |
1054 | if (adapter->guest) { |
1055 | if (adapter->guest->irq_avail) { |
1056 | for (i = 0; i < adapter->guest->irq_nranges; i++) { |
1057 | cur = &adapter->guest->irq_avail[i]; |
1058 | bitmap_free(bitmap: cur->bitmap); |
1059 | } |
1060 | kfree(objp: adapter->guest->irq_avail); |
1061 | } |
1062 | kfree(objp: adapter->guest->status); |
1063 | kfree(objp: adapter->guest); |
1064 | } |
1065 | cxl_remove_adapter_nr(adapter); |
1066 | kfree(objp: adapter); |
1067 | } |
1068 | |
1069 | static int properties_look_ok(struct cxl *adapter) |
1070 | { |
1071 | /* The absence of this property means that the operational |
1072 | * status is unknown or okay |
1073 | */ |
1074 | if (strlen(adapter->guest->status) && |
1075 | strcmp(adapter->guest->status, "okay" )) { |
1076 | pr_err("ABORTING:Bad operational status of the device\n" ); |
1077 | return -EINVAL; |
1078 | } |
1079 | |
1080 | return 0; |
1081 | } |
1082 | |
1083 | ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len) |
1084 | { |
1085 | return guest_collect_vpd(adapter, NULL, buf, len); |
1086 | } |
1087 | |
1088 | void cxl_guest_remove_adapter(struct cxl *adapter) |
1089 | { |
1090 | pr_devel("in %s\n" , __func__); |
1091 | |
1092 | cxl_sysfs_adapter_remove(adapter); |
1093 | |
1094 | cxl_guest_remove_chardev(adapter); |
1095 | device_unregister(dev: &adapter->dev); |
1096 | } |
1097 | |
1098 | static void release_adapter(struct device *dev) |
1099 | { |
1100 | free_adapter(to_cxl_adapter(dev)); |
1101 | } |
1102 | |
1103 | struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev) |
1104 | { |
1105 | struct cxl *adapter; |
1106 | bool free = true; |
1107 | int rc; |
1108 | |
1109 | if (!(adapter = cxl_alloc_adapter())) |
1110 | return ERR_PTR(error: -ENOMEM); |
1111 | |
1112 | if (!(adapter->guest = kzalloc(size: sizeof(struct cxl_guest), GFP_KERNEL))) { |
1113 | free_adapter(adapter); |
1114 | return ERR_PTR(error: -ENOMEM); |
1115 | } |
1116 | |
1117 | adapter->slices = 0; |
1118 | adapter->guest->pdev = pdev; |
1119 | adapter->dev.parent = &pdev->dev; |
1120 | adapter->dev.release = release_adapter; |
1121 | dev_set_drvdata(dev: &pdev->dev, data: adapter); |
1122 | |
1123 | /* |
1124 | * Hypervisor controls PSL timebase initialization (p1 register). |
1125 | * On FW840, PSL is initialized. |
1126 | */ |
1127 | adapter->psl_timebase_synced = true; |
1128 | |
1129 | if ((rc = cxl_of_read_adapter_handle(adapter, np))) |
1130 | goto err1; |
1131 | |
1132 | if ((rc = cxl_of_read_adapter_properties(adapter, np))) |
1133 | goto err1; |
1134 | |
1135 | if ((rc = properties_look_ok(adapter))) |
1136 | goto err1; |
1137 | |
1138 | if ((rc = cxl_guest_add_chardev(adapter))) |
1139 | goto err1; |
1140 | |
1141 | /* |
1142 | * After we call this function we must not free the adapter directly, |
1143 | * even if it returns an error! |
1144 | */ |
1145 | if ((rc = cxl_register_adapter(adapter))) |
1146 | goto err_put_dev; |
1147 | |
1148 | if ((rc = cxl_sysfs_adapter_add(adapter))) |
1149 | goto err_del_dev; |
1150 | |
1151 | /* release the context lock as the adapter is configured */ |
1152 | cxl_adapter_context_unlock(adapter); |
1153 | |
1154 | return adapter; |
1155 | |
1156 | err_del_dev: |
1157 | device_del(dev: &adapter->dev); |
1158 | err_put_dev: |
1159 | put_device(dev: &adapter->dev); |
1160 | free = false; |
1161 | cxl_guest_remove_chardev(adapter); |
1162 | err1: |
1163 | if (free) |
1164 | free_adapter(adapter); |
1165 | return ERR_PTR(error: rc); |
1166 | } |
1167 | |
1168 | void cxl_guest_reload_module(struct cxl *adapter) |
1169 | { |
1170 | struct platform_device *pdev; |
1171 | |
1172 | pdev = adapter->guest->pdev; |
1173 | cxl_guest_remove_adapter(adapter); |
1174 | |
1175 | cxl_of_probe(pdev); |
1176 | } |
1177 | |
1178 | const struct cxl_backend_ops cxl_guest_ops = { |
1179 | .module = THIS_MODULE, |
1180 | .adapter_reset = guest_reset, |
1181 | .alloc_one_irq = guest_alloc_one_irq, |
1182 | .release_one_irq = guest_release_one_irq, |
1183 | .alloc_irq_ranges = guest_alloc_irq_ranges, |
1184 | .release_irq_ranges = guest_release_irq_ranges, |
1185 | .setup_irq = NULL, |
1186 | .handle_psl_slice_error = guest_handle_psl_slice_error, |
1187 | .psl_interrupt = guest_psl_irq, |
1188 | .ack_irq = guest_ack_irq, |
1189 | .attach_process = guest_attach_process, |
1190 | .detach_process = guest_detach_process, |
1191 | .update_ivtes = NULL, |
1192 | .support_attributes = guest_support_attributes, |
1193 | .link_ok = guest_link_ok, |
1194 | .release_afu = guest_release_afu, |
1195 | .afu_read_err_buffer = guest_afu_read_err_buffer, |
1196 | .afu_check_and_enable = guest_afu_check_and_enable, |
1197 | .afu_activate_mode = guest_afu_activate_mode, |
1198 | .afu_deactivate_mode = guest_afu_deactivate_mode, |
1199 | .afu_reset = guest_afu_reset, |
1200 | .afu_cr_read8 = guest_afu_cr_read8, |
1201 | .afu_cr_read16 = guest_afu_cr_read16, |
1202 | .afu_cr_read32 = guest_afu_cr_read32, |
1203 | .afu_cr_read64 = guest_afu_cr_read64, |
1204 | .afu_cr_write8 = guest_afu_cr_write8, |
1205 | .afu_cr_write16 = guest_afu_cr_write16, |
1206 | .afu_cr_write32 = guest_afu_cr_write32, |
1207 | .read_adapter_vpd = cxl_guest_read_adapter_vpd, |
1208 | }; |
1209 | |