1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // Copyright 2017 IBM Corp. |
3 | #include <linux/pci.h> |
4 | #include <asm/pnv-ocxl.h> |
5 | #include <misc/ocxl-config.h> |
6 | #include "ocxl_internal.h" |
7 | |
8 | #define (val, bit) (!!(val & BIT(bit))) |
9 | #define (val, s, e) ((val & GENMASK(e, s)) >> s) |
10 | |
11 | #define OCXL_DVSEC_AFU_IDX_MASK GENMASK(5, 0) |
12 | #define OCXL_DVSEC_ACTAG_MASK GENMASK(11, 0) |
13 | #define OCXL_DVSEC_PASID_MASK GENMASK(19, 0) |
14 | #define OCXL_DVSEC_PASID_LOG_MASK GENMASK(4, 0) |
15 | |
16 | #define OCXL_DVSEC_TEMPL_VERSION 0x0 |
17 | #define OCXL_DVSEC_TEMPL_NAME 0x4 |
18 | #define OCXL_DVSEC_TEMPL_AFU_VERSION 0x1C |
19 | #define OCXL_DVSEC_TEMPL_MMIO_GLOBAL 0x20 |
20 | #define OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ 0x28 |
21 | #define OCXL_DVSEC_TEMPL_MMIO_PP 0x30 |
22 | #define OCXL_DVSEC_TEMPL_MMIO_PP_SZ 0x38 |
23 | #define OCXL_DVSEC_TEMPL_ALL_MEM_SZ 0x3C |
24 | #define OCXL_DVSEC_TEMPL_LPC_MEM_START 0x40 |
25 | #define OCXL_DVSEC_TEMPL_WWID 0x48 |
26 | #define OCXL_DVSEC_TEMPL_LPC_MEM_SZ 0x58 |
27 | |
28 | #define OCXL_MAX_AFU_PER_FUNCTION 64 |
29 | #define OCXL_TEMPL_LEN_1_0 0x58 |
30 | #define OCXL_TEMPL_LEN_1_1 0x60 |
31 | #define OCXL_TEMPL_NAME_LEN 24 |
32 | #define OCXL_CFG_TIMEOUT 3 |
33 | |
34 | static int find_dvsec(struct pci_dev *dev, int dvsec_id) |
35 | { |
36 | return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM, dvsec: dvsec_id); |
37 | } |
38 | |
39 | static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) |
40 | { |
41 | int vsec = 0; |
42 | u16 vendor, id; |
43 | u8 idx; |
44 | |
45 | while ((vsec = pci_find_next_ext_capability(dev, pos: vsec, |
46 | OCXL_EXT_CAP_ID_DVSEC))) { |
47 | pci_read_config_word(dev, where: vsec + OCXL_DVSEC_VENDOR_OFFSET, |
48 | val: &vendor); |
49 | pci_read_config_word(dev, where: vsec + OCXL_DVSEC_ID_OFFSET, val: &id); |
50 | |
51 | if (vendor == PCI_VENDOR_ID_IBM && |
52 | id == OCXL_DVSEC_AFU_CTRL_ID) { |
53 | pci_read_config_byte(dev, |
54 | where: vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX, |
55 | val: &idx); |
56 | if (idx == afu_idx) |
57 | return vsec; |
58 | } |
59 | } |
60 | return 0; |
61 | } |
62 | |
63 | /** |
64 | * get_function_0() - Find a related PCI device (function 0) |
65 | * @dev: PCI device to match |
66 | * |
67 | * Returns a pointer to the related device, or null if not found |
68 | */ |
69 | static struct pci_dev *get_function_0(struct pci_dev *dev) |
70 | { |
71 | unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0); |
72 | |
73 | return pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: dev->bus), |
74 | bus: dev->bus->number, devfn); |
75 | } |
76 | |
77 | static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) |
78 | { |
79 | u16 val; |
80 | int pos; |
81 | |
82 | pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PASID); |
83 | if (!pos) { |
84 | /* |
85 | * PASID capability is not mandatory, but there |
86 | * shouldn't be any AFU |
87 | */ |
88 | dev_dbg(&dev->dev, "Function doesn't require any PASID\n" ); |
89 | fn->max_pasid_log = -1; |
90 | goto out; |
91 | } |
92 | pci_read_config_word(dev, where: pos + PCI_PASID_CAP, val: &val); |
93 | fn->max_pasid_log = EXTRACT_BITS(val, 8, 12); |
94 | |
95 | out: |
96 | dev_dbg(&dev->dev, "PASID capability:\n" ); |
97 | dev_dbg(&dev->dev, " Max PASID log = %d\n" , fn->max_pasid_log); |
98 | } |
99 | |
100 | static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn) |
101 | { |
102 | int pos; |
103 | |
104 | pos = find_dvsec(dev, OCXL_DVSEC_TL_ID); |
105 | if (!pos && PCI_FUNC(dev->devfn) == 0) { |
106 | dev_err(&dev->dev, "Can't find TL DVSEC\n" ); |
107 | return -ENODEV; |
108 | } |
109 | if (pos && PCI_FUNC(dev->devfn) != 0) { |
110 | dev_err(&dev->dev, "TL DVSEC is only allowed on function 0\n" ); |
111 | return -ENODEV; |
112 | } |
113 | fn->dvsec_tl_pos = pos; |
114 | return 0; |
115 | } |
116 | |
117 | static int read_dvsec_function(struct pci_dev *dev, struct ocxl_fn_config *fn) |
118 | { |
119 | int pos, afu_present; |
120 | u32 val; |
121 | |
122 | pos = find_dvsec(dev, OCXL_DVSEC_FUNC_ID); |
123 | if (!pos) { |
124 | dev_err(&dev->dev, "Can't find function DVSEC\n" ); |
125 | return -ENODEV; |
126 | } |
127 | fn->dvsec_function_pos = pos; |
128 | |
129 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_FUNC_OFF_INDEX, val: &val); |
130 | afu_present = EXTRACT_BIT(val, 31); |
131 | if (!afu_present) { |
132 | fn->max_afu_index = -1; |
133 | dev_dbg(&dev->dev, "Function doesn't define any AFU\n" ); |
134 | goto out; |
135 | } |
136 | fn->max_afu_index = EXTRACT_BITS(val, 24, 29); |
137 | |
138 | out: |
139 | dev_dbg(&dev->dev, "Function DVSEC:\n" ); |
140 | dev_dbg(&dev->dev, " Max AFU index = %d\n" , fn->max_afu_index); |
141 | return 0; |
142 | } |
143 | |
144 | static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn) |
145 | { |
146 | int pos; |
147 | |
148 | if (fn->max_afu_index < 0) { |
149 | fn->dvsec_afu_info_pos = -1; |
150 | return 0; |
151 | } |
152 | |
153 | pos = find_dvsec(dev, OCXL_DVSEC_AFU_INFO_ID); |
154 | if (!pos) { |
155 | dev_err(&dev->dev, "Can't find AFU information DVSEC\n" ); |
156 | return -ENODEV; |
157 | } |
158 | fn->dvsec_afu_info_pos = pos; |
159 | return 0; |
160 | } |
161 | |
162 | static int read_dvsec_vendor(struct pci_dev *dev) |
163 | { |
164 | int pos; |
165 | u32 cfg, tlx, dlx, reset_reload; |
166 | |
167 | /* |
168 | * vendor specific DVSEC, for IBM images only. Some older |
169 | * images may not have it |
170 | * |
171 | * It's only used on function 0 to specify the version of some |
172 | * logic blocks and to give access to special registers to |
173 | * enable host-based flashing. |
174 | */ |
175 | if (PCI_FUNC(dev->devfn) != 0) |
176 | return 0; |
177 | |
178 | pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID); |
179 | if (!pos) |
180 | return 0; |
181 | |
182 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_VENDOR_CFG_VERS, val: &cfg); |
183 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_VENDOR_TLX_VERS, val: &tlx); |
184 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_VENDOR_DLX_VERS, val: &dlx); |
185 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_VENDOR_RESET_RELOAD, |
186 | val: &reset_reload); |
187 | |
188 | dev_dbg(&dev->dev, "Vendor specific DVSEC:\n" ); |
189 | dev_dbg(&dev->dev, " CFG version = 0x%x\n" , cfg); |
190 | dev_dbg(&dev->dev, " TLX version = 0x%x\n" , tlx); |
191 | dev_dbg(&dev->dev, " DLX version = 0x%x\n" , dlx); |
192 | dev_dbg(&dev->dev, " ResetReload = 0x%x\n" , reset_reload); |
193 | return 0; |
194 | } |
195 | |
196 | /** |
197 | * get_dvsec_vendor0() - Find a related PCI device (function 0) |
198 | * @dev: PCI device to match |
199 | * @dev0: The PCI device (function 0) found |
200 | * @out_pos: The position of PCI device (function 0) |
201 | * |
202 | * Returns 0 on success, negative on failure. |
203 | * |
204 | * NOTE: If it's successful, the reference of dev0 is increased, |
205 | * so after using it, the callers must call pci_dev_put() to give |
206 | * up the reference. |
207 | */ |
208 | static int get_dvsec_vendor0(struct pci_dev *dev, struct pci_dev **dev0, |
209 | int *out_pos) |
210 | { |
211 | int pos; |
212 | |
213 | if (PCI_FUNC(dev->devfn) != 0) { |
214 | dev = get_function_0(dev); |
215 | if (!dev) |
216 | return -1; |
217 | } else { |
218 | dev = pci_dev_get(dev); |
219 | } |
220 | pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID); |
221 | if (!pos) { |
222 | pci_dev_put(dev); |
223 | return -1; |
224 | } |
225 | *dev0 = dev; |
226 | *out_pos = pos; |
227 | return 0; |
228 | } |
229 | |
230 | int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val) |
231 | { |
232 | struct pci_dev *dev0; |
233 | u32 reset_reload; |
234 | int pos; |
235 | |
236 | if (get_dvsec_vendor0(dev, dev0: &dev0, out_pos: &pos)) |
237 | return -1; |
238 | |
239 | pci_read_config_dword(dev: dev0, where: pos + OCXL_DVSEC_VENDOR_RESET_RELOAD, |
240 | val: &reset_reload); |
241 | pci_dev_put(dev: dev0); |
242 | *val = !!(reset_reload & BIT(0)); |
243 | return 0; |
244 | } |
245 | |
246 | int ocxl_config_set_reset_reload(struct pci_dev *dev, int val) |
247 | { |
248 | struct pci_dev *dev0; |
249 | u32 reset_reload; |
250 | int pos; |
251 | |
252 | if (get_dvsec_vendor0(dev, dev0: &dev0, out_pos: &pos)) |
253 | return -1; |
254 | |
255 | pci_read_config_dword(dev: dev0, where: pos + OCXL_DVSEC_VENDOR_RESET_RELOAD, |
256 | val: &reset_reload); |
257 | if (val) |
258 | reset_reload |= BIT(0); |
259 | else |
260 | reset_reload &= ~BIT(0); |
261 | pci_write_config_dword(dev: dev0, where: pos + OCXL_DVSEC_VENDOR_RESET_RELOAD, |
262 | val: reset_reload); |
263 | pci_dev_put(dev: dev0); |
264 | return 0; |
265 | } |
266 | |
267 | static int validate_function(struct pci_dev *dev, struct ocxl_fn_config *fn) |
268 | { |
269 | if (fn->max_pasid_log == -1 && fn->max_afu_index >= 0) { |
270 | dev_err(&dev->dev, |
271 | "AFUs are defined but no PASIDs are requested\n" ); |
272 | return -EINVAL; |
273 | } |
274 | |
275 | if (fn->max_afu_index > OCXL_MAX_AFU_PER_FUNCTION) { |
276 | dev_err(&dev->dev, |
277 | "Max AFU index out of architectural limit (%d vs %d)\n" , |
278 | fn->max_afu_index, OCXL_MAX_AFU_PER_FUNCTION); |
279 | return -EINVAL; |
280 | } |
281 | return 0; |
282 | } |
283 | |
284 | int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn) |
285 | { |
286 | int rc; |
287 | |
288 | read_pasid(dev, fn); |
289 | |
290 | rc = read_dvsec_tl(dev, fn); |
291 | if (rc) { |
292 | dev_err(&dev->dev, |
293 | "Invalid Transaction Layer DVSEC configuration: %d\n" , |
294 | rc); |
295 | return -ENODEV; |
296 | } |
297 | |
298 | rc = read_dvsec_function(dev, fn); |
299 | if (rc) { |
300 | dev_err(&dev->dev, |
301 | "Invalid Function DVSEC configuration: %d\n" , rc); |
302 | return -ENODEV; |
303 | } |
304 | |
305 | rc = read_dvsec_afu_info(dev, fn); |
306 | if (rc) { |
307 | dev_err(&dev->dev, "Invalid AFU configuration: %d\n" , rc); |
308 | return -ENODEV; |
309 | } |
310 | |
311 | rc = read_dvsec_vendor(dev); |
312 | if (rc) { |
313 | dev_err(&dev->dev, |
314 | "Invalid vendor specific DVSEC configuration: %d\n" , |
315 | rc); |
316 | return -ENODEV; |
317 | } |
318 | |
319 | rc = validate_function(dev, fn); |
320 | return rc; |
321 | } |
322 | EXPORT_SYMBOL_GPL(ocxl_config_read_function); |
323 | |
324 | static int read_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn, |
325 | int offset, u32 *data) |
326 | { |
327 | u32 val; |
328 | unsigned long timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT); |
329 | int pos = fn->dvsec_afu_info_pos; |
330 | |
331 | /* Protect 'data valid' bit */ |
332 | if (EXTRACT_BIT(offset, 31)) { |
333 | dev_err(&dev->dev, "Invalid offset in AFU info DVSEC\n" ); |
334 | return -EINVAL; |
335 | } |
336 | |
337 | pci_write_config_dword(dev, where: pos + OCXL_DVSEC_AFU_INFO_OFF, val: offset); |
338 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_AFU_INFO_OFF, val: &val); |
339 | while (!EXTRACT_BIT(val, 31)) { |
340 | if (time_after_eq(jiffies, timeout)) { |
341 | dev_err(&dev->dev, |
342 | "Timeout while reading AFU info DVSEC (offset=%d)\n" , |
343 | offset); |
344 | return -EBUSY; |
345 | } |
346 | cpu_relax(); |
347 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_AFU_INFO_OFF, val: &val); |
348 | } |
349 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_AFU_INFO_DATA, val: data); |
350 | return 0; |
351 | } |
352 | |
353 | /** |
354 | * read_template_version() - Read the template version from the AFU |
355 | * @dev: the device for the AFU |
356 | * @fn: the AFU offsets |
357 | * @len: outputs the template length |
358 | * @version: outputs the major<<8,minor version |
359 | * |
360 | * Returns 0 on success, negative on failure |
361 | */ |
362 | static int read_template_version(struct pci_dev *dev, struct ocxl_fn_config *fn, |
363 | u16 *len, u16 *version) |
364 | { |
365 | u32 val32; |
366 | u8 major, minor; |
367 | int rc; |
368 | |
369 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_VERSION, data: &val32); |
370 | if (rc) |
371 | return rc; |
372 | |
373 | *len = EXTRACT_BITS(val32, 16, 31); |
374 | major = EXTRACT_BITS(val32, 8, 15); |
375 | minor = EXTRACT_BITS(val32, 0, 7); |
376 | *version = (major << 8) + minor; |
377 | return 0; |
378 | } |
379 | |
380 | int ocxl_config_check_afu_index(struct pci_dev *dev, |
381 | struct ocxl_fn_config *fn, int afu_idx) |
382 | { |
383 | int rc; |
384 | u16 templ_version; |
385 | u16 len, expected_len; |
386 | |
387 | pci_write_config_byte(dev, |
388 | where: fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX, |
389 | val: afu_idx); |
390 | |
391 | rc = read_template_version(dev, fn, len: &len, version: &templ_version); |
392 | if (rc) |
393 | return rc; |
394 | |
395 | /* AFU index map can have holes, in which case we read all 0's */ |
396 | if (!templ_version && !len) |
397 | return 0; |
398 | |
399 | dev_dbg(&dev->dev, "AFU descriptor template version %d.%d\n" , |
400 | templ_version >> 8, templ_version & 0xFF); |
401 | |
402 | switch (templ_version) { |
403 | case 0x0005: // v0.5 was used prior to the spec approval |
404 | case 0x0100: |
405 | expected_len = OCXL_TEMPL_LEN_1_0; |
406 | break; |
407 | case 0x0101: |
408 | expected_len = OCXL_TEMPL_LEN_1_1; |
409 | break; |
410 | default: |
411 | dev_warn(&dev->dev, "Unknown AFU template version %#x\n" , |
412 | templ_version); |
413 | expected_len = len; |
414 | } |
415 | if (len != expected_len) |
416 | dev_warn(&dev->dev, |
417 | "Unexpected template length %#x in AFU information, expected %#x for version %#x\n" , |
418 | len, expected_len, templ_version); |
419 | return 1; |
420 | } |
421 | |
422 | static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn, |
423 | struct ocxl_afu_config *afu) |
424 | { |
425 | int i, rc; |
426 | u32 val, *ptr; |
427 | |
428 | BUILD_BUG_ON(OCXL_AFU_NAME_SZ < OCXL_TEMPL_NAME_LEN); |
429 | for (i = 0; i < OCXL_TEMPL_NAME_LEN; i += 4) { |
430 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_NAME + i, data: &val); |
431 | if (rc) |
432 | return rc; |
433 | ptr = (u32 *) &afu->name[i]; |
434 | *ptr = le32_to_cpu((__force __le32) val); |
435 | } |
436 | afu->name[OCXL_AFU_NAME_SZ - 1] = '\0'; /* play safe */ |
437 | return 0; |
438 | } |
439 | |
440 | static int read_afu_mmio(struct pci_dev *dev, struct ocxl_fn_config *fn, |
441 | struct ocxl_afu_config *afu) |
442 | { |
443 | int rc; |
444 | u32 val; |
445 | |
446 | /* |
447 | * Global MMIO |
448 | */ |
449 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL, data: &val); |
450 | if (rc) |
451 | return rc; |
452 | afu->global_mmio_bar = EXTRACT_BITS(val, 0, 2); |
453 | afu->global_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16; |
454 | |
455 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL + 4, data: &val); |
456 | if (rc) |
457 | return rc; |
458 | afu->global_mmio_offset += (u64) val << 32; |
459 | |
460 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ, data: &val); |
461 | if (rc) |
462 | return rc; |
463 | afu->global_mmio_size = val; |
464 | |
465 | /* |
466 | * Per-process MMIO |
467 | */ |
468 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP, data: &val); |
469 | if (rc) |
470 | return rc; |
471 | afu->pp_mmio_bar = EXTRACT_BITS(val, 0, 2); |
472 | afu->pp_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16; |
473 | |
474 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP + 4, data: &val); |
475 | if (rc) |
476 | return rc; |
477 | afu->pp_mmio_offset += (u64) val << 32; |
478 | |
479 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP_SZ, data: &val); |
480 | if (rc) |
481 | return rc; |
482 | afu->pp_mmio_stride = val; |
483 | |
484 | return 0; |
485 | } |
486 | |
487 | static int read_afu_control(struct pci_dev *dev, struct ocxl_afu_config *afu) |
488 | { |
489 | int pos; |
490 | u8 val8; |
491 | u16 val16; |
492 | |
493 | pos = find_dvsec_afu_ctrl(dev, afu_idx: afu->idx); |
494 | if (!pos) { |
495 | dev_err(&dev->dev, "Can't find AFU control DVSEC for AFU %d\n" , |
496 | afu->idx); |
497 | return -ENODEV; |
498 | } |
499 | afu->dvsec_afu_control_pos = pos; |
500 | |
501 | pci_read_config_byte(dev, where: pos + OCXL_DVSEC_AFU_CTRL_PASID_SUP, val: &val8); |
502 | afu->pasid_supported_log = EXTRACT_BITS(val8, 0, 4); |
503 | |
504 | pci_read_config_word(dev, where: pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, val: &val16); |
505 | afu->actag_supported = EXTRACT_BITS(val16, 0, 11); |
506 | return 0; |
507 | } |
508 | |
509 | static bool char_allowed(int c) |
510 | { |
511 | /* |
512 | * Permitted Characters : Alphanumeric, hyphen, underscore, comma |
513 | */ |
514 | if ((c >= 0x30 && c <= 0x39) /* digits */ || |
515 | (c >= 0x41 && c <= 0x5A) /* upper case */ || |
516 | (c >= 0x61 && c <= 0x7A) /* lower case */ || |
517 | c == 0 /* NULL */ || |
518 | c == 0x2D /* - */ || |
519 | c == 0x5F /* _ */ || |
520 | c == 0x2C /* , */) |
521 | return true; |
522 | return false; |
523 | } |
524 | |
525 | static int validate_afu(struct pci_dev *dev, struct ocxl_afu_config *afu) |
526 | { |
527 | int i; |
528 | |
529 | if (!afu->name[0]) { |
530 | dev_err(&dev->dev, "Empty AFU name\n" ); |
531 | return -EINVAL; |
532 | } |
533 | for (i = 0; i < OCXL_TEMPL_NAME_LEN; i++) { |
534 | if (!char_allowed(c: afu->name[i])) { |
535 | dev_err(&dev->dev, |
536 | "Invalid character in AFU name\n" ); |
537 | return -EINVAL; |
538 | } |
539 | } |
540 | |
541 | if (afu->global_mmio_bar != 0 && |
542 | afu->global_mmio_bar != 2 && |
543 | afu->global_mmio_bar != 4) { |
544 | dev_err(&dev->dev, "Invalid global MMIO bar number\n" ); |
545 | return -EINVAL; |
546 | } |
547 | if (afu->pp_mmio_bar != 0 && |
548 | afu->pp_mmio_bar != 2 && |
549 | afu->pp_mmio_bar != 4) { |
550 | dev_err(&dev->dev, "Invalid per-process MMIO bar number\n" ); |
551 | return -EINVAL; |
552 | } |
553 | return 0; |
554 | } |
555 | |
556 | /** |
557 | * read_afu_lpc_memory_info() - Populate AFU metadata regarding LPC memory |
558 | * @dev: the device for the AFU |
559 | * @fn: the AFU offsets |
560 | * @afu: the AFU struct to populate the LPC metadata into |
561 | * |
562 | * Returns 0 on success, negative on failure |
563 | */ |
564 | static int read_afu_lpc_memory_info(struct pci_dev *dev, |
565 | struct ocxl_fn_config *fn, |
566 | struct ocxl_afu_config *afu) |
567 | { |
568 | int rc; |
569 | u32 val32; |
570 | u16 templ_version; |
571 | u16 templ_len; |
572 | u64 total_mem_size = 0; |
573 | u64 lpc_mem_size = 0; |
574 | |
575 | afu->lpc_mem_offset = 0; |
576 | afu->lpc_mem_size = 0; |
577 | afu->special_purpose_mem_offset = 0; |
578 | afu->special_purpose_mem_size = 0; |
579 | /* |
580 | * For AFUs following template v1.0, the LPC memory covers the |
581 | * total memory. Its size is a power of 2. |
582 | * |
583 | * For AFUs with template >= v1.01, the total memory size is |
584 | * still a power of 2, but it is split in 2 parts: |
585 | * - the LPC memory, whose size can now be anything |
586 | * - the remainder memory is a special purpose memory, whose |
587 | * definition is AFU-dependent. It is not accessible through |
588 | * the usual commands for LPC memory |
589 | */ |
590 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_ALL_MEM_SZ, data: &val32); |
591 | if (rc) |
592 | return rc; |
593 | |
594 | val32 = EXTRACT_BITS(val32, 0, 7); |
595 | if (!val32) |
596 | return 0; /* No LPC memory */ |
597 | |
598 | /* |
599 | * The configuration space spec allows for a memory size of up |
600 | * to 2^255 bytes. |
601 | * |
602 | * Current generation hardware uses 56-bit physical addresses, |
603 | * but we won't be able to get near close to that, as we won't |
604 | * have a hole big enough in the memory map. Let it pass in |
605 | * the driver for now. We'll get an error from the firmware |
606 | * when trying to configure something too big. |
607 | */ |
608 | total_mem_size = 1ull << val32; |
609 | |
610 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START, data: &val32); |
611 | if (rc) |
612 | return rc; |
613 | |
614 | afu->lpc_mem_offset = val32; |
615 | |
616 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_LPC_MEM_START + 4, data: &val32); |
617 | if (rc) |
618 | return rc; |
619 | |
620 | afu->lpc_mem_offset |= (u64) val32 << 32; |
621 | |
622 | rc = read_template_version(dev, fn, len: &templ_len, version: &templ_version); |
623 | if (rc) |
624 | return rc; |
625 | |
626 | if (templ_version >= 0x0101) { |
627 | rc = read_afu_info(dev, fn, |
628 | OCXL_DVSEC_TEMPL_LPC_MEM_SZ, data: &val32); |
629 | if (rc) |
630 | return rc; |
631 | lpc_mem_size = val32; |
632 | |
633 | rc = read_afu_info(dev, fn, |
634 | OCXL_DVSEC_TEMPL_LPC_MEM_SZ + 4, data: &val32); |
635 | if (rc) |
636 | return rc; |
637 | lpc_mem_size |= (u64) val32 << 32; |
638 | } else { |
639 | lpc_mem_size = total_mem_size; |
640 | } |
641 | afu->lpc_mem_size = lpc_mem_size; |
642 | |
643 | if (lpc_mem_size < total_mem_size) { |
644 | afu->special_purpose_mem_offset = |
645 | afu->lpc_mem_offset + lpc_mem_size; |
646 | afu->special_purpose_mem_size = |
647 | total_mem_size - lpc_mem_size; |
648 | } |
649 | return 0; |
650 | } |
651 | |
652 | int ocxl_config_read_afu(struct pci_dev *dev, struct ocxl_fn_config *fn, |
653 | struct ocxl_afu_config *afu, u8 afu_idx) |
654 | { |
655 | int rc; |
656 | u32 val32; |
657 | |
658 | /* |
659 | * First, we need to write the AFU idx for the AFU we want to |
660 | * access. |
661 | */ |
662 | WARN_ON((afu_idx & OCXL_DVSEC_AFU_IDX_MASK) != afu_idx); |
663 | afu->idx = afu_idx; |
664 | pci_write_config_byte(dev, |
665 | where: fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX, |
666 | val: afu->idx); |
667 | |
668 | rc = read_afu_name(dev, fn, afu); |
669 | if (rc) |
670 | return rc; |
671 | |
672 | rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_AFU_VERSION, data: &val32); |
673 | if (rc) |
674 | return rc; |
675 | afu->version_major = EXTRACT_BITS(val32, 24, 31); |
676 | afu->version_minor = EXTRACT_BITS(val32, 16, 23); |
677 | afu->afuc_type = EXTRACT_BITS(val32, 14, 15); |
678 | afu->afum_type = EXTRACT_BITS(val32, 12, 13); |
679 | afu->profile = EXTRACT_BITS(val32, 0, 7); |
680 | |
681 | rc = read_afu_mmio(dev, fn, afu); |
682 | if (rc) |
683 | return rc; |
684 | |
685 | rc = read_afu_lpc_memory_info(dev, fn, afu); |
686 | if (rc) |
687 | return rc; |
688 | |
689 | rc = read_afu_control(dev, afu); |
690 | if (rc) |
691 | return rc; |
692 | |
693 | dev_dbg(&dev->dev, "AFU configuration:\n" ); |
694 | dev_dbg(&dev->dev, " name = %s\n" , afu->name); |
695 | dev_dbg(&dev->dev, " version = %d.%d\n" , afu->version_major, |
696 | afu->version_minor); |
697 | dev_dbg(&dev->dev, " global mmio bar = %hhu\n" , afu->global_mmio_bar); |
698 | dev_dbg(&dev->dev, " global mmio offset = %#llx\n" , |
699 | afu->global_mmio_offset); |
700 | dev_dbg(&dev->dev, " global mmio size = %#x\n" , afu->global_mmio_size); |
701 | dev_dbg(&dev->dev, " pp mmio bar = %hhu\n" , afu->pp_mmio_bar); |
702 | dev_dbg(&dev->dev, " pp mmio offset = %#llx\n" , afu->pp_mmio_offset); |
703 | dev_dbg(&dev->dev, " pp mmio stride = %#x\n" , afu->pp_mmio_stride); |
704 | dev_dbg(&dev->dev, " lpc_mem offset = %#llx\n" , afu->lpc_mem_offset); |
705 | dev_dbg(&dev->dev, " lpc_mem size = %#llx\n" , afu->lpc_mem_size); |
706 | dev_dbg(&dev->dev, " special purpose mem offset = %#llx\n" , |
707 | afu->special_purpose_mem_offset); |
708 | dev_dbg(&dev->dev, " special purpose mem size = %#llx\n" , |
709 | afu->special_purpose_mem_size); |
710 | dev_dbg(&dev->dev, " pasid supported (log) = %u\n" , |
711 | afu->pasid_supported_log); |
712 | dev_dbg(&dev->dev, " actag supported = %u\n" , |
713 | afu->actag_supported); |
714 | |
715 | rc = validate_afu(dev, afu); |
716 | return rc; |
717 | } |
718 | EXPORT_SYMBOL_GPL(ocxl_config_read_afu); |
719 | |
720 | int ocxl_config_get_actag_info(struct pci_dev *dev, u16 *base, u16 *enabled, |
721 | u16 *supported) |
722 | { |
723 | int rc; |
724 | |
725 | /* |
726 | * This is really a simple wrapper for the kernel API, to |
727 | * avoid an external driver using ocxl as a library to call |
728 | * platform-dependent code |
729 | */ |
730 | rc = pnv_ocxl_get_actag(dev, base, enabled, supported); |
731 | if (rc) { |
732 | dev_err(&dev->dev, "Can't get actag for device: %d\n" , rc); |
733 | return rc; |
734 | } |
735 | return 0; |
736 | } |
737 | EXPORT_SYMBOL_GPL(ocxl_config_get_actag_info); |
738 | |
739 | void ocxl_config_set_afu_actag(struct pci_dev *dev, int pos, int actag_base, |
740 | int actag_count) |
741 | { |
742 | u16 val; |
743 | |
744 | val = actag_count & OCXL_DVSEC_ACTAG_MASK; |
745 | pci_write_config_byte(dev, where: pos + OCXL_DVSEC_AFU_CTRL_ACTAG_EN, val); |
746 | |
747 | val = actag_base & OCXL_DVSEC_ACTAG_MASK; |
748 | pci_write_config_dword(dev, where: pos + OCXL_DVSEC_AFU_CTRL_ACTAG_BASE, val); |
749 | } |
750 | EXPORT_SYMBOL_GPL(ocxl_config_set_afu_actag); |
751 | |
752 | int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count) |
753 | { |
754 | return pnv_ocxl_get_pasid_count(dev, count); |
755 | } |
756 | |
757 | void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base, |
758 | u32 pasid_count_log) |
759 | { |
760 | u8 val8; |
761 | u32 val32; |
762 | |
763 | val8 = pasid_count_log & OCXL_DVSEC_PASID_LOG_MASK; |
764 | pci_write_config_byte(dev, where: pos + OCXL_DVSEC_AFU_CTRL_PASID_EN, val: val8); |
765 | |
766 | pci_read_config_dword(dev, where: pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE, |
767 | val: &val32); |
768 | val32 &= ~OCXL_DVSEC_PASID_MASK; |
769 | val32 |= pasid_base & OCXL_DVSEC_PASID_MASK; |
770 | pci_write_config_dword(dev, where: pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE, |
771 | val: val32); |
772 | } |
773 | EXPORT_SYMBOL_GPL(ocxl_config_set_afu_pasid); |
774 | |
775 | void ocxl_config_set_afu_state(struct pci_dev *dev, int pos, int enable) |
776 | { |
777 | u8 val; |
778 | |
779 | pci_read_config_byte(dev, where: pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val: &val); |
780 | if (enable) |
781 | val |= 1; |
782 | else |
783 | val &= 0xFE; |
784 | pci_write_config_byte(dev, where: pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val); |
785 | } |
786 | EXPORT_SYMBOL_GPL(ocxl_config_set_afu_state); |
787 | |
788 | int ocxl_config_set_TL(struct pci_dev *dev, int tl_dvsec) |
789 | { |
790 | u32 val; |
791 | __be32 *be32ptr; |
792 | u8 timers; |
793 | int i, rc; |
794 | long recv_cap; |
795 | char *recv_rate; |
796 | |
797 | /* |
798 | * Skip on function != 0, as the TL can only be defined on 0 |
799 | */ |
800 | if (PCI_FUNC(dev->devfn) != 0) |
801 | return 0; |
802 | |
803 | recv_rate = kzalloc(size: PNV_OCXL_TL_RATE_BUF_SIZE, GFP_KERNEL); |
804 | if (!recv_rate) |
805 | return -ENOMEM; |
806 | /* |
807 | * The spec defines 64 templates for messages in the |
808 | * Transaction Layer (TL). |
809 | * |
810 | * The host and device each support a subset, so we need to |
811 | * configure the transmitters on each side to send only |
812 | * templates the receiver understands, at a rate the receiver |
813 | * can process. Per the spec, template 0 must be supported by |
814 | * everybody. That's the template which has been used by the |
815 | * host and device so far. |
816 | * |
817 | * The sending rate limit must be set before the template is |
818 | * enabled. |
819 | */ |
820 | |
821 | /* |
822 | * Device -> host |
823 | */ |
824 | rc = pnv_ocxl_get_tl_cap(dev, &recv_cap, recv_rate, |
825 | PNV_OCXL_TL_RATE_BUF_SIZE); |
826 | if (rc) |
827 | goto out; |
828 | |
829 | for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) { |
830 | be32ptr = (__be32 *) &recv_rate[i]; |
831 | pci_write_config_dword(dev, |
832 | where: tl_dvsec + OCXL_DVSEC_TL_SEND_RATE + i, |
833 | be32_to_cpu(*be32ptr)); |
834 | } |
835 | val = recv_cap >> 32; |
836 | pci_write_config_dword(dev, where: tl_dvsec + OCXL_DVSEC_TL_SEND_CAP, val); |
837 | val = recv_cap & GENMASK(31, 0); |
838 | pci_write_config_dword(dev, where: tl_dvsec + OCXL_DVSEC_TL_SEND_CAP + 4, val); |
839 | |
840 | /* |
841 | * Host -> device |
842 | */ |
843 | for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) { |
844 | pci_read_config_dword(dev, |
845 | where: tl_dvsec + OCXL_DVSEC_TL_RECV_RATE + i, |
846 | val: &val); |
847 | be32ptr = (__be32 *) &recv_rate[i]; |
848 | *be32ptr = cpu_to_be32(val); |
849 | } |
850 | pci_read_config_dword(dev, where: tl_dvsec + OCXL_DVSEC_TL_RECV_CAP, val: &val); |
851 | recv_cap = (long) val << 32; |
852 | pci_read_config_dword(dev, where: tl_dvsec + OCXL_DVSEC_TL_RECV_CAP + 4, val: &val); |
853 | recv_cap |= val; |
854 | |
855 | rc = pnv_ocxl_set_tl_conf(dev, recv_cap, __pa(recv_rate), |
856 | PNV_OCXL_TL_RATE_BUF_SIZE); |
857 | if (rc) |
858 | goto out; |
859 | |
860 | /* |
861 | * Opencapi commands needing to be retried are classified per |
862 | * the TL in 2 groups: short and long commands. |
863 | * |
864 | * The short back off timer it not used for now. It will be |
865 | * for opencapi 4.0. |
866 | * |
867 | * The long back off timer is typically used when an AFU hits |
868 | * a page fault but the NPU is already processing one. So the |
869 | * AFU needs to wait before it can resubmit. Having a value |
870 | * too low doesn't break anything, but can generate extra |
871 | * traffic on the link. |
872 | * We set it to 1.6 us for now. It's shorter than, but in the |
873 | * same order of magnitude as the time spent to process a page |
874 | * fault. |
875 | */ |
876 | timers = 0x2 << 4; /* long timer = 1.6 us */ |
877 | pci_write_config_byte(dev, where: tl_dvsec + OCXL_DVSEC_TL_BACKOFF_TIMERS, |
878 | val: timers); |
879 | |
880 | rc = 0; |
881 | out: |
882 | kfree(objp: recv_rate); |
883 | return rc; |
884 | } |
885 | EXPORT_SYMBOL_GPL(ocxl_config_set_TL); |
886 | |
887 | int ocxl_config_terminate_pasid(struct pci_dev *dev, int afu_control, int pasid) |
888 | { |
889 | u32 val; |
890 | unsigned long timeout; |
891 | |
892 | pci_read_config_dword(dev, where: afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID, |
893 | val: &val); |
894 | if (EXTRACT_BIT(val, 20)) { |
895 | dev_err(&dev->dev, |
896 | "Can't terminate PASID %#x, previous termination didn't complete\n" , |
897 | pasid); |
898 | return -EBUSY; |
899 | } |
900 | |
901 | val &= ~OCXL_DVSEC_PASID_MASK; |
902 | val |= pasid & OCXL_DVSEC_PASID_MASK; |
903 | val |= BIT(20); |
904 | pci_write_config_dword(dev, |
905 | where: afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID, |
906 | val); |
907 | |
908 | timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT); |
909 | pci_read_config_dword(dev, where: afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID, |
910 | val: &val); |
911 | while (EXTRACT_BIT(val, 20)) { |
912 | if (time_after_eq(jiffies, timeout)) { |
913 | dev_err(&dev->dev, |
914 | "Timeout while waiting for AFU to terminate PASID %#x\n" , |
915 | pasid); |
916 | return -EBUSY; |
917 | } |
918 | cpu_relax(); |
919 | pci_read_config_dword(dev, |
920 | where: afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID, |
921 | val: &val); |
922 | } |
923 | return 0; |
924 | } |
925 | EXPORT_SYMBOL_GPL(ocxl_config_terminate_pasid); |
926 | |
927 | void ocxl_config_set_actag(struct pci_dev *dev, int func_dvsec, u32 tag_first, |
928 | u32 tag_count) |
929 | { |
930 | u32 val; |
931 | |
932 | val = (tag_first & OCXL_DVSEC_ACTAG_MASK) << 16; |
933 | val |= tag_count & OCXL_DVSEC_ACTAG_MASK; |
934 | pci_write_config_dword(dev, where: func_dvsec + OCXL_DVSEC_FUNC_OFF_ACTAG, |
935 | val); |
936 | } |
937 | EXPORT_SYMBOL_GPL(ocxl_config_set_actag); |
938 | |