1// SPDX-License-Identifier: GPL-2.0+
2// Copyright 2019 IBM Corp.
3#include <linux/idr.h>
4#include "ocxl_internal.h"
5
6static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
7{
8 return (get_device(dev: &fn->dev) == NULL) ? NULL : fn;
9}
10
11static void ocxl_fn_put(struct ocxl_fn *fn)
12{
13 put_device(dev: &fn->dev);
14}
15
16static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
17{
18 struct ocxl_afu *afu;
19
20 afu = kzalloc(size: sizeof(struct ocxl_afu), GFP_KERNEL);
21 if (!afu)
22 return NULL;
23
24 kref_init(kref: &afu->kref);
25 mutex_init(&afu->contexts_lock);
26 mutex_init(&afu->afu_control_lock);
27 idr_init(idr: &afu->contexts_idr);
28 afu->fn = fn;
29 ocxl_fn_get(fn);
30 return afu;
31}
32
33static void free_afu(struct kref *kref)
34{
35 struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref);
36
37 idr_destroy(&afu->contexts_idr);
38 ocxl_fn_put(fn: afu->fn);
39 kfree(objp: afu);
40}
41
42void ocxl_afu_get(struct ocxl_afu *afu)
43{
44 kref_get(kref: &afu->kref);
45}
46EXPORT_SYMBOL_GPL(ocxl_afu_get);
47
48void ocxl_afu_put(struct ocxl_afu *afu)
49{
50 kref_put(kref: &afu->kref, release: free_afu);
51}
52EXPORT_SYMBOL_GPL(ocxl_afu_put);
53
54static int assign_afu_actag(struct ocxl_afu *afu)
55{
56 struct ocxl_fn *fn = afu->fn;
57 int actag_count, actag_offset;
58 struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
59
60 /*
61 * if there were not enough actags for the function, each afu
62 * reduces its count as well
63 */
64 actag_count = afu->config.actag_supported *
65 fn->actag_enabled / fn->actag_supported;
66 actag_offset = ocxl_actag_afu_alloc(fn, size: actag_count);
67 if (actag_offset < 0) {
68 dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n",
69 actag_count, actag_offset);
70 return actag_offset;
71 }
72 afu->actag_base = fn->actag_base + actag_offset;
73 afu->actag_enabled = actag_count;
74
75 ocxl_config_set_afu_actag(dev: pci_dev, afu_control_offset: afu->config.dvsec_afu_control_pos,
76 actag_base: afu->actag_base, actag_count: afu->actag_enabled);
77 dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n",
78 afu->actag_base, afu->actag_enabled);
79 return 0;
80}
81
82static void reclaim_afu_actag(struct ocxl_afu *afu)
83{
84 struct ocxl_fn *fn = afu->fn;
85 int start_offset, size;
86
87 start_offset = afu->actag_base - fn->actag_base;
88 size = afu->actag_enabled;
89 ocxl_actag_afu_free(fn: afu->fn, start: start_offset, size);
90}
91
92static int assign_afu_pasid(struct ocxl_afu *afu)
93{
94 struct ocxl_fn *fn = afu->fn;
95 int pasid_count, pasid_offset;
96 struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
97
98 /*
99 * We only support the case where the function configuration
100 * requested enough PASIDs to cover all AFUs.
101 */
102 pasid_count = 1 << afu->config.pasid_supported_log;
103 pasid_offset = ocxl_pasid_afu_alloc(fn, size: pasid_count);
104 if (pasid_offset < 0) {
105 dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n",
106 pasid_count, pasid_offset);
107 return pasid_offset;
108 }
109 afu->pasid_base = fn->pasid_base + pasid_offset;
110 afu->pasid_count = 0;
111 afu->pasid_max = pasid_count;
112
113 ocxl_config_set_afu_pasid(dev: pci_dev, afu_control_offset: afu->config.dvsec_afu_control_pos,
114 pasid_base: afu->pasid_base,
115 pasid_count_log: afu->config.pasid_supported_log);
116 dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n",
117 afu->pasid_base, pasid_count);
118 return 0;
119}
120
121static void reclaim_afu_pasid(struct ocxl_afu *afu)
122{
123 struct ocxl_fn *fn = afu->fn;
124 int start_offset, size;
125
126 start_offset = afu->pasid_base - fn->pasid_base;
127 size = 1 << afu->config.pasid_supported_log;
128 ocxl_pasid_afu_free(fn: afu->fn, start: start_offset, size);
129}
130
131static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
132{
133 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
134 int rc, idx;
135
136 if (bar != 0 && bar != 2 && bar != 4)
137 return -EINVAL;
138
139 idx = bar >> 1;
140 if (fn->bar_used[idx]++ == 0) {
141 rc = pci_request_region(dev, bar, "ocxl");
142 if (rc)
143 return rc;
144 }
145 return 0;
146}
147
148static void release_fn_bar(struct ocxl_fn *fn, int bar)
149{
150 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
151 int idx;
152
153 if (bar != 0 && bar != 2 && bar != 4)
154 return;
155
156 idx = bar >> 1;
157 if (--fn->bar_used[idx] == 0)
158 pci_release_region(dev, bar);
159 WARN_ON(fn->bar_used[idx] < 0);
160}
161
162static int map_mmio_areas(struct ocxl_afu *afu)
163{
164 int rc;
165 struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent);
166
167 rc = reserve_fn_bar(fn: afu->fn, bar: afu->config.global_mmio_bar);
168 if (rc)
169 return rc;
170
171 rc = reserve_fn_bar(fn: afu->fn, bar: afu->config.pp_mmio_bar);
172 if (rc) {
173 release_fn_bar(fn: afu->fn, bar: afu->config.global_mmio_bar);
174 return rc;
175 }
176
177 afu->global_mmio_start =
178 pci_resource_start(pci_dev, afu->config.global_mmio_bar) +
179 afu->config.global_mmio_offset;
180 afu->pp_mmio_start =
181 pci_resource_start(pci_dev, afu->config.pp_mmio_bar) +
182 afu->config.pp_mmio_offset;
183
184 afu->global_mmio_ptr = ioremap(offset: afu->global_mmio_start,
185 size: afu->config.global_mmio_size);
186 if (!afu->global_mmio_ptr) {
187 release_fn_bar(fn: afu->fn, bar: afu->config.pp_mmio_bar);
188 release_fn_bar(fn: afu->fn, bar: afu->config.global_mmio_bar);
189 dev_err(&pci_dev->dev, "Error mapping global mmio area\n");
190 return -ENOMEM;
191 }
192
193 /*
194 * Leave an empty page between the per-process mmio area and
195 * the AFU interrupt mappings
196 */
197 afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
198 return 0;
199}
200
201static void unmap_mmio_areas(struct ocxl_afu *afu)
202{
203 if (afu->global_mmio_ptr) {
204 iounmap(addr: afu->global_mmio_ptr);
205 afu->global_mmio_ptr = NULL;
206 }
207 afu->global_mmio_start = 0;
208 afu->pp_mmio_start = 0;
209 release_fn_bar(fn: afu->fn, bar: afu->config.pp_mmio_bar);
210 release_fn_bar(fn: afu->fn, bar: afu->config.global_mmio_bar);
211}
212
213static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
214{
215 int rc;
216
217 rc = ocxl_config_read_afu(dev, fn: &afu->fn->config, afu: &afu->config, afu_idx);
218 if (rc)
219 return rc;
220
221 rc = assign_afu_actag(afu);
222 if (rc)
223 return rc;
224
225 rc = assign_afu_pasid(afu);
226 if (rc)
227 goto err_free_actag;
228
229 rc = map_mmio_areas(afu);
230 if (rc)
231 goto err_free_pasid;
232
233 return 0;
234
235err_free_pasid:
236 reclaim_afu_pasid(afu);
237err_free_actag:
238 reclaim_afu_actag(afu);
239 return rc;
240}
241
242static void deconfigure_afu(struct ocxl_afu *afu)
243{
244 unmap_mmio_areas(afu);
245 reclaim_afu_pasid(afu);
246 reclaim_afu_actag(afu);
247}
248
249static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
250{
251 ocxl_config_set_afu_state(dev, afu_control_offset: afu->config.dvsec_afu_control_pos, enable: 1);
252
253 return 0;
254}
255
256static void deactivate_afu(struct ocxl_afu *afu)
257{
258 struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
259
260 ocxl_config_set_afu_state(dev, afu_control_offset: afu->config.dvsec_afu_control_pos, enable: 0);
261}
262
263static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
264{
265 int rc;
266 struct ocxl_afu *afu;
267
268 afu = alloc_afu(fn);
269 if (!afu)
270 return -ENOMEM;
271
272 rc = configure_afu(afu, afu_idx, dev);
273 if (rc) {
274 ocxl_afu_put(afu);
275 return rc;
276 }
277
278 rc = activate_afu(dev, afu);
279 if (rc) {
280 deconfigure_afu(afu);
281 ocxl_afu_put(afu);
282 return rc;
283 }
284
285 list_add_tail(new: &afu->list, head: &fn->afu_list);
286
287 return 0;
288}
289
290static void remove_afu(struct ocxl_afu *afu)
291{
292 list_del(entry: &afu->list);
293 ocxl_context_detach_all(afu);
294 deactivate_afu(afu);
295 deconfigure_afu(afu);
296 ocxl_afu_put(afu); // matches the implicit get in alloc_afu
297}
298
299static struct ocxl_fn *alloc_function(void)
300{
301 struct ocxl_fn *fn;
302
303 fn = kzalloc(size: sizeof(struct ocxl_fn), GFP_KERNEL);
304 if (!fn)
305 return NULL;
306
307 INIT_LIST_HEAD(list: &fn->afu_list);
308 INIT_LIST_HEAD(list: &fn->pasid_list);
309 INIT_LIST_HEAD(list: &fn->actag_list);
310
311 return fn;
312}
313
314static void free_function(struct ocxl_fn *fn)
315{
316 WARN_ON(!list_empty(&fn->afu_list));
317 WARN_ON(!list_empty(&fn->pasid_list));
318 kfree(objp: fn);
319}
320
321static void free_function_dev(struct device *dev)
322{
323 struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev);
324
325 free_function(fn);
326}
327
328static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
329{
330 fn->dev.parent = &dev->dev;
331 fn->dev.release = free_function_dev;
332 return dev_set_name(dev: &fn->dev, name: "ocxlfn.%s", dev_name(dev: &dev->dev));
333}
334
335static int assign_function_actag(struct ocxl_fn *fn)
336{
337 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
338 u16 base, enabled, supported;
339 int rc;
340
341 rc = ocxl_config_get_actag_info(dev, base: &base, enabled: &enabled, supported: &supported);
342 if (rc)
343 return rc;
344
345 fn->actag_base = base;
346 fn->actag_enabled = enabled;
347 fn->actag_supported = supported;
348
349 ocxl_config_set_actag(dev, func_offset: fn->config.dvsec_function_pos,
350 actag_base: fn->actag_base, actag_count: fn->actag_enabled);
351 dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
352 fn->actag_base, fn->actag_enabled);
353 return 0;
354}
355
356static int set_function_pasid(struct ocxl_fn *fn)
357{
358 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
359 int rc, desired_count, max_count;
360
361 /* A function may not require any PASID */
362 if (fn->config.max_pasid_log < 0)
363 return 0;
364
365 rc = ocxl_config_get_pasid_info(dev, count: &max_count);
366 if (rc)
367 return rc;
368
369 desired_count = 1 << fn->config.max_pasid_log;
370
371 if (desired_count > max_count) {
372 dev_err(&fn->dev,
373 "Function requires more PASIDs than is available (%d vs. %d)\n",
374 desired_count, max_count);
375 return -ENOSPC;
376 }
377
378 fn->pasid_base = 0;
379 return 0;
380}
381
382static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
383{
384 int rc;
385
386 rc = pci_enable_device(dev);
387 if (rc) {
388 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
389 return rc;
390 }
391
392 /*
393 * Once it has been confirmed to work on our hardware, we
394 * should reset the function, to force the adapter to restart
395 * from scratch.
396 * A function reset would also reset all its AFUs.
397 *
398 * Some hints for implementation:
399 *
400 * - there's not status bit to know when the reset is done. We
401 * should try reading the config space to know when it's
402 * done.
403 * - probably something like:
404 * Reset
405 * wait 100ms
406 * issue config read
407 * allow device up to 1 sec to return success on config
408 * read before declaring it broken
409 *
410 * Some shared logic on the card (CFG, TLX) won't be reset, so
411 * there's no guarantee that it will be enough.
412 */
413 rc = ocxl_config_read_function(dev, fn: &fn->config);
414 if (rc)
415 return rc;
416
417 rc = set_function_device(fn, dev);
418 if (rc)
419 return rc;
420
421 rc = assign_function_actag(fn);
422 if (rc)
423 return rc;
424
425 rc = set_function_pasid(fn);
426 if (rc)
427 return rc;
428
429 rc = ocxl_link_setup(dev, PE_mask: 0, link_handle: &fn->link);
430 if (rc)
431 return rc;
432
433 rc = ocxl_config_set_TL(dev, tl_dvsec: fn->config.dvsec_tl_pos);
434 if (rc) {
435 ocxl_link_release(dev, link_handle: fn->link);
436 return rc;
437 }
438 return 0;
439}
440
441static void deconfigure_function(struct ocxl_fn *fn)
442{
443 struct pci_dev *dev = to_pci_dev(fn->dev.parent);
444
445 ocxl_link_release(dev, link_handle: fn->link);
446 pci_disable_device(dev);
447}
448
449static struct ocxl_fn *init_function(struct pci_dev *dev)
450{
451 struct ocxl_fn *fn;
452 int rc;
453
454 fn = alloc_function();
455 if (!fn)
456 return ERR_PTR(error: -ENOMEM);
457
458 rc = configure_function(fn, dev);
459 if (rc) {
460 free_function(fn);
461 return ERR_PTR(error: rc);
462 }
463
464 rc = device_register(dev: &fn->dev);
465 if (rc) {
466 deconfigure_function(fn);
467 put_device(dev: &fn->dev);
468 return ERR_PTR(error: rc);
469 }
470 return fn;
471}
472
473// Device detection & initialisation
474
475struct ocxl_fn *ocxl_function_open(struct pci_dev *dev)
476{
477 int rc, afu_count = 0;
478 u8 afu;
479 struct ocxl_fn *fn;
480
481 if (!radix_enabled()) {
482 dev_err(&dev->dev, "Unsupported memory model (hash)\n");
483 return ERR_PTR(error: -ENODEV);
484 }
485
486 fn = init_function(dev);
487 if (IS_ERR(ptr: fn)) {
488 dev_err(&dev->dev, "function init failed: %li\n",
489 PTR_ERR(fn));
490 return fn;
491 }
492
493 for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
494 rc = ocxl_config_check_afu_index(dev, fn: &fn->config, afu_idx: afu);
495 if (rc > 0) {
496 rc = init_afu(dev, fn, afu_idx: afu);
497 if (rc) {
498 dev_err(&dev->dev,
499 "Can't initialize AFU index %d\n", afu);
500 continue;
501 }
502 afu_count++;
503 }
504 }
505 dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
506 return fn;
507}
508EXPORT_SYMBOL_GPL(ocxl_function_open);
509
510struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn)
511{
512 return &fn->afu_list;
513}
514EXPORT_SYMBOL_GPL(ocxl_function_afu_list);
515
516struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx)
517{
518 struct ocxl_afu *afu;
519
520 list_for_each_entry(afu, &fn->afu_list, list) {
521 if (afu->config.idx == afu_idx)
522 return afu;
523 }
524
525 return NULL;
526}
527EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu);
528
529const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn)
530{
531 return &fn->config;
532}
533EXPORT_SYMBOL_GPL(ocxl_function_config);
534
535void ocxl_function_close(struct ocxl_fn *fn)
536{
537 struct ocxl_afu *afu, *tmp;
538
539 list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
540 remove_afu(afu);
541 }
542
543 deconfigure_function(fn);
544 device_unregister(dev: &fn->dev);
545}
546EXPORT_SYMBOL_GPL(ocxl_function_close);
547
548// AFU Metadata
549
550struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu)
551{
552 return &afu->config;
553}
554EXPORT_SYMBOL_GPL(ocxl_afu_config);
555
556void ocxl_afu_set_private(struct ocxl_afu *afu, void *private)
557{
558 afu->private = private;
559}
560EXPORT_SYMBOL_GPL(ocxl_afu_set_private);
561
562void *ocxl_afu_get_private(struct ocxl_afu *afu)
563{
564 if (afu)
565 return afu->private;
566
567 return NULL;
568}
569EXPORT_SYMBOL_GPL(ocxl_afu_get_private);
570

source code of linux/drivers/misc/ocxl/core.c