1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * PCI Endpoint *Function* (EPF) library |
4 | * |
5 | * Copyright (C) 2017 Texas Instruments |
6 | * Author: Kishon Vijay Abraham I <kishon@ti.com> |
7 | */ |
8 | |
9 | #include <linux/device.h> |
10 | #include <linux/dma-mapping.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/module.h> |
13 | |
14 | #include <linux/pci-epc.h> |
15 | #include <linux/pci-epf.h> |
16 | #include <linux/pci-ep-cfs.h> |
17 | |
18 | static DEFINE_MUTEX(pci_epf_mutex); |
19 | |
20 | static const struct bus_type pci_epf_bus_type; |
21 | static const struct device_type pci_epf_type; |
22 | |
23 | /** |
24 | * pci_epf_unbind() - Notify the function driver that the binding between the |
25 | * EPF device and EPC device has been lost |
26 | * @epf: the EPF device which has lost the binding with the EPC device |
27 | * |
28 | * Invoke to notify the function driver that the binding between the EPF device |
29 | * and EPC device has been lost. |
30 | */ |
31 | void pci_epf_unbind(struct pci_epf *epf) |
32 | { |
33 | struct pci_epf *epf_vf; |
34 | |
35 | if (!epf->driver) { |
36 | dev_WARN(&epf->dev, "epf device not bound to driver\n" ); |
37 | return; |
38 | } |
39 | |
40 | mutex_lock(&epf->lock); |
41 | list_for_each_entry(epf_vf, &epf->pci_vepf, list) { |
42 | if (epf_vf->is_bound) |
43 | epf_vf->driver->ops->unbind(epf_vf); |
44 | } |
45 | if (epf->is_bound) |
46 | epf->driver->ops->unbind(epf); |
47 | mutex_unlock(lock: &epf->lock); |
48 | module_put(module: epf->driver->owner); |
49 | } |
50 | EXPORT_SYMBOL_GPL(pci_epf_unbind); |
51 | |
52 | /** |
53 | * pci_epf_bind() - Notify the function driver that the EPF device has been |
54 | * bound to a EPC device |
55 | * @epf: the EPF device which has been bound to the EPC device |
56 | * |
57 | * Invoke to notify the function driver that it has been bound to a EPC device |
58 | */ |
59 | int pci_epf_bind(struct pci_epf *epf) |
60 | { |
61 | struct device *dev = &epf->dev; |
62 | struct pci_epf *epf_vf; |
63 | u8 func_no, vfunc_no; |
64 | struct pci_epc *epc; |
65 | int ret; |
66 | |
67 | if (!epf->driver) { |
68 | dev_WARN(dev, "epf device not bound to driver\n" ); |
69 | return -EINVAL; |
70 | } |
71 | |
72 | if (!try_module_get(module: epf->driver->owner)) |
73 | return -EAGAIN; |
74 | |
75 | mutex_lock(&epf->lock); |
76 | list_for_each_entry(epf_vf, &epf->pci_vepf, list) { |
77 | vfunc_no = epf_vf->vfunc_no; |
78 | |
79 | if (vfunc_no < 1) { |
80 | dev_err(dev, "Invalid virtual function number\n" ); |
81 | ret = -EINVAL; |
82 | goto ret; |
83 | } |
84 | |
85 | epc = epf->epc; |
86 | func_no = epf->func_no; |
87 | if (!IS_ERR_OR_NULL(ptr: epc)) { |
88 | if (!epc->max_vfs) { |
89 | dev_err(dev, "No support for virt function\n" ); |
90 | ret = -EINVAL; |
91 | goto ret; |
92 | } |
93 | |
94 | if (vfunc_no > epc->max_vfs[func_no]) { |
95 | dev_err(dev, "PF%d: Exceeds max vfunc number\n" , |
96 | func_no); |
97 | ret = -EINVAL; |
98 | goto ret; |
99 | } |
100 | } |
101 | |
102 | epc = epf->sec_epc; |
103 | func_no = epf->sec_epc_func_no; |
104 | if (!IS_ERR_OR_NULL(ptr: epc)) { |
105 | if (!epc->max_vfs) { |
106 | dev_err(dev, "No support for virt function\n" ); |
107 | ret = -EINVAL; |
108 | goto ret; |
109 | } |
110 | |
111 | if (vfunc_no > epc->max_vfs[func_no]) { |
112 | dev_err(dev, "PF%d: Exceeds max vfunc number\n" , |
113 | func_no); |
114 | ret = -EINVAL; |
115 | goto ret; |
116 | } |
117 | } |
118 | |
119 | epf_vf->func_no = epf->func_no; |
120 | epf_vf->sec_epc_func_no = epf->sec_epc_func_no; |
121 | epf_vf->epc = epf->epc; |
122 | epf_vf->sec_epc = epf->sec_epc; |
123 | ret = epf_vf->driver->ops->bind(epf_vf); |
124 | if (ret) |
125 | goto ret; |
126 | epf_vf->is_bound = true; |
127 | } |
128 | |
129 | ret = epf->driver->ops->bind(epf); |
130 | if (ret) |
131 | goto ret; |
132 | epf->is_bound = true; |
133 | |
134 | mutex_unlock(lock: &epf->lock); |
135 | return 0; |
136 | |
137 | ret: |
138 | mutex_unlock(lock: &epf->lock); |
139 | pci_epf_unbind(epf); |
140 | |
141 | return ret; |
142 | } |
143 | EXPORT_SYMBOL_GPL(pci_epf_bind); |
144 | |
145 | /** |
146 | * pci_epf_add_vepf() - associate virtual EP function to physical EP function |
147 | * @epf_pf: the physical EP function to which the virtual EP function should be |
148 | * associated |
149 | * @epf_vf: the virtual EP function to be added |
150 | * |
151 | * A physical endpoint function can be associated with multiple virtual |
152 | * endpoint functions. Invoke pci_epf_add_epf() to add a virtual PCI endpoint |
153 | * function to a physical PCI endpoint function. |
154 | */ |
155 | int pci_epf_add_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) |
156 | { |
157 | u32 vfunc_no; |
158 | |
159 | if (IS_ERR_OR_NULL(ptr: epf_pf) || IS_ERR_OR_NULL(ptr: epf_vf)) |
160 | return -EINVAL; |
161 | |
162 | if (epf_pf->epc || epf_vf->epc || epf_vf->epf_pf) |
163 | return -EBUSY; |
164 | |
165 | if (epf_pf->sec_epc || epf_vf->sec_epc) |
166 | return -EBUSY; |
167 | |
168 | mutex_lock(&epf_pf->lock); |
169 | vfunc_no = find_first_zero_bit(addr: &epf_pf->vfunction_num_map, |
170 | BITS_PER_LONG); |
171 | if (vfunc_no >= BITS_PER_LONG) { |
172 | mutex_unlock(lock: &epf_pf->lock); |
173 | return -EINVAL; |
174 | } |
175 | |
176 | set_bit(nr: vfunc_no, addr: &epf_pf->vfunction_num_map); |
177 | epf_vf->vfunc_no = vfunc_no; |
178 | |
179 | epf_vf->epf_pf = epf_pf; |
180 | epf_vf->is_vf = true; |
181 | |
182 | list_add_tail(new: &epf_vf->list, head: &epf_pf->pci_vepf); |
183 | mutex_unlock(lock: &epf_pf->lock); |
184 | |
185 | return 0; |
186 | } |
187 | EXPORT_SYMBOL_GPL(pci_epf_add_vepf); |
188 | |
189 | /** |
190 | * pci_epf_remove_vepf() - remove virtual EP function from physical EP function |
191 | * @epf_pf: the physical EP function from which the virtual EP function should |
192 | * be removed |
193 | * @epf_vf: the virtual EP function to be removed |
194 | * |
195 | * Invoke to remove a virtual endpoint function from the physical endpoint |
196 | * function. |
197 | */ |
198 | void pci_epf_remove_vepf(struct pci_epf *epf_pf, struct pci_epf *epf_vf) |
199 | { |
200 | if (IS_ERR_OR_NULL(ptr: epf_pf) || IS_ERR_OR_NULL(ptr: epf_vf)) |
201 | return; |
202 | |
203 | mutex_lock(&epf_pf->lock); |
204 | clear_bit(nr: epf_vf->vfunc_no, addr: &epf_pf->vfunction_num_map); |
205 | list_del(entry: &epf_vf->list); |
206 | mutex_unlock(lock: &epf_pf->lock); |
207 | } |
208 | EXPORT_SYMBOL_GPL(pci_epf_remove_vepf); |
209 | |
210 | /** |
211 | * pci_epf_free_space() - free the allocated PCI EPF register space |
212 | * @epf: the EPF device from whom to free the memory |
213 | * @addr: the virtual address of the PCI EPF register space |
214 | * @bar: the BAR number corresponding to the register space |
215 | * @type: Identifies if the allocated space is for primary EPC or secondary EPC |
216 | * |
217 | * Invoke to free the allocated PCI EPF register space. |
218 | */ |
219 | void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar, |
220 | enum pci_epc_interface_type type) |
221 | { |
222 | struct device *dev; |
223 | struct pci_epf_bar *epf_bar; |
224 | struct pci_epc *epc; |
225 | |
226 | if (!addr) |
227 | return; |
228 | |
229 | if (type == PRIMARY_INTERFACE) { |
230 | epc = epf->epc; |
231 | epf_bar = epf->bar; |
232 | } else { |
233 | epc = epf->sec_epc; |
234 | epf_bar = epf->sec_epc_bar; |
235 | } |
236 | |
237 | dev = epc->dev.parent; |
238 | dma_free_coherent(dev, size: epf_bar[bar].size, cpu_addr: addr, |
239 | dma_handle: epf_bar[bar].phys_addr); |
240 | |
241 | epf_bar[bar].phys_addr = 0; |
242 | epf_bar[bar].addr = NULL; |
243 | epf_bar[bar].size = 0; |
244 | epf_bar[bar].barno = 0; |
245 | epf_bar[bar].flags = 0; |
246 | } |
247 | EXPORT_SYMBOL_GPL(pci_epf_free_space); |
248 | |
249 | /** |
250 | * pci_epf_alloc_space() - allocate memory for the PCI EPF register space |
251 | * @epf: the EPF device to whom allocate the memory |
252 | * @size: the size of the memory that has to be allocated |
253 | * @bar: the BAR number corresponding to the allocated register space |
254 | * @epc_features: the features provided by the EPC specific to this EPF |
255 | * @type: Identifies if the allocation is for primary EPC or secondary EPC |
256 | * |
257 | * Invoke to allocate memory for the PCI EPF register space. |
258 | */ |
259 | void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar, |
260 | const struct pci_epc_features *epc_features, |
261 | enum pci_epc_interface_type type) |
262 | { |
263 | u64 bar_fixed_size = epc_features->bar[bar].fixed_size; |
264 | size_t align = epc_features->align; |
265 | struct pci_epf_bar *epf_bar; |
266 | dma_addr_t phys_addr; |
267 | struct pci_epc *epc; |
268 | struct device *dev; |
269 | void *space; |
270 | |
271 | if (size < 128) |
272 | size = 128; |
273 | |
274 | if (epc_features->bar[bar].type == BAR_FIXED && bar_fixed_size) { |
275 | if (size > bar_fixed_size) { |
276 | dev_err(&epf->dev, |
277 | "requested BAR size is larger than fixed size\n" ); |
278 | return NULL; |
279 | } |
280 | size = bar_fixed_size; |
281 | } |
282 | |
283 | if (align) |
284 | size = ALIGN(size, align); |
285 | else |
286 | size = roundup_pow_of_two(size); |
287 | |
288 | if (type == PRIMARY_INTERFACE) { |
289 | epc = epf->epc; |
290 | epf_bar = epf->bar; |
291 | } else { |
292 | epc = epf->sec_epc; |
293 | epf_bar = epf->sec_epc_bar; |
294 | } |
295 | |
296 | dev = epc->dev.parent; |
297 | space = dma_alloc_coherent(dev, size, dma_handle: &phys_addr, GFP_KERNEL); |
298 | if (!space) { |
299 | dev_err(dev, "failed to allocate mem space\n" ); |
300 | return NULL; |
301 | } |
302 | |
303 | epf_bar[bar].phys_addr = phys_addr; |
304 | epf_bar[bar].addr = space; |
305 | epf_bar[bar].size = size; |
306 | epf_bar[bar].barno = bar; |
307 | epf_bar[bar].flags |= upper_32_bits(size) ? |
308 | PCI_BASE_ADDRESS_MEM_TYPE_64 : |
309 | PCI_BASE_ADDRESS_MEM_TYPE_32; |
310 | |
311 | return space; |
312 | } |
313 | EXPORT_SYMBOL_GPL(pci_epf_alloc_space); |
314 | |
315 | static void pci_epf_remove_cfs(struct pci_epf_driver *driver) |
316 | { |
317 | struct config_group *group, *tmp; |
318 | |
319 | if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS)) |
320 | return; |
321 | |
322 | mutex_lock(&pci_epf_mutex); |
323 | list_for_each_entry_safe(group, tmp, &driver->epf_group, group_entry) |
324 | pci_ep_cfs_remove_epf_group(group); |
325 | list_del(entry: &driver->epf_group); |
326 | mutex_unlock(lock: &pci_epf_mutex); |
327 | } |
328 | |
329 | /** |
330 | * pci_epf_unregister_driver() - unregister the PCI EPF driver |
331 | * @driver: the PCI EPF driver that has to be unregistered |
332 | * |
333 | * Invoke to unregister the PCI EPF driver. |
334 | */ |
335 | void pci_epf_unregister_driver(struct pci_epf_driver *driver) |
336 | { |
337 | pci_epf_remove_cfs(driver); |
338 | driver_unregister(drv: &driver->driver); |
339 | } |
340 | EXPORT_SYMBOL_GPL(pci_epf_unregister_driver); |
341 | |
342 | static int pci_epf_add_cfs(struct pci_epf_driver *driver) |
343 | { |
344 | struct config_group *group; |
345 | const struct pci_epf_device_id *id; |
346 | |
347 | if (!IS_ENABLED(CONFIG_PCI_ENDPOINT_CONFIGFS)) |
348 | return 0; |
349 | |
350 | INIT_LIST_HEAD(list: &driver->epf_group); |
351 | |
352 | id = driver->id_table; |
353 | while (id->name[0]) { |
354 | group = pci_ep_cfs_add_epf_group(name: id->name); |
355 | if (IS_ERR(ptr: group)) { |
356 | pci_epf_remove_cfs(driver); |
357 | return PTR_ERR(ptr: group); |
358 | } |
359 | |
360 | mutex_lock(&pci_epf_mutex); |
361 | list_add_tail(new: &group->group_entry, head: &driver->epf_group); |
362 | mutex_unlock(lock: &pci_epf_mutex); |
363 | id++; |
364 | } |
365 | |
366 | return 0; |
367 | } |
368 | |
369 | /** |
370 | * __pci_epf_register_driver() - register a new PCI EPF driver |
371 | * @driver: structure representing PCI EPF driver |
372 | * @owner: the owner of the module that registers the PCI EPF driver |
373 | * |
374 | * Invoke to register a new PCI EPF driver. |
375 | */ |
376 | int __pci_epf_register_driver(struct pci_epf_driver *driver, |
377 | struct module *owner) |
378 | { |
379 | int ret; |
380 | |
381 | if (!driver->ops) |
382 | return -EINVAL; |
383 | |
384 | if (!driver->ops->bind || !driver->ops->unbind) |
385 | return -EINVAL; |
386 | |
387 | driver->driver.bus = &pci_epf_bus_type; |
388 | driver->driver.owner = owner; |
389 | |
390 | ret = driver_register(drv: &driver->driver); |
391 | if (ret) |
392 | return ret; |
393 | |
394 | pci_epf_add_cfs(driver); |
395 | |
396 | return 0; |
397 | } |
398 | EXPORT_SYMBOL_GPL(__pci_epf_register_driver); |
399 | |
400 | /** |
401 | * pci_epf_destroy() - destroy the created PCI EPF device |
402 | * @epf: the PCI EPF device that has to be destroyed. |
403 | * |
404 | * Invoke to destroy the PCI EPF device created by invoking pci_epf_create(). |
405 | */ |
406 | void pci_epf_destroy(struct pci_epf *epf) |
407 | { |
408 | device_unregister(dev: &epf->dev); |
409 | } |
410 | EXPORT_SYMBOL_GPL(pci_epf_destroy); |
411 | |
412 | /** |
413 | * pci_epf_create() - create a new PCI EPF device |
414 | * @name: the name of the PCI EPF device. This name will be used to bind the |
415 | * EPF device to a EPF driver |
416 | * |
417 | * Invoke to create a new PCI EPF device by providing the name of the function |
418 | * device. |
419 | */ |
420 | struct pci_epf *pci_epf_create(const char *name) |
421 | { |
422 | int ret; |
423 | struct pci_epf *epf; |
424 | struct device *dev; |
425 | int len; |
426 | |
427 | epf = kzalloc(size: sizeof(*epf), GFP_KERNEL); |
428 | if (!epf) |
429 | return ERR_PTR(error: -ENOMEM); |
430 | |
431 | len = strchrnul(name, '.') - name; |
432 | epf->name = kstrndup(s: name, len, GFP_KERNEL); |
433 | if (!epf->name) { |
434 | kfree(objp: epf); |
435 | return ERR_PTR(error: -ENOMEM); |
436 | } |
437 | |
438 | /* VFs are numbered starting with 1. So set BIT(0) by default */ |
439 | epf->vfunction_num_map = 1; |
440 | INIT_LIST_HEAD(list: &epf->pci_vepf); |
441 | |
442 | dev = &epf->dev; |
443 | device_initialize(dev); |
444 | dev->bus = &pci_epf_bus_type; |
445 | dev->type = &pci_epf_type; |
446 | mutex_init(&epf->lock); |
447 | |
448 | ret = dev_set_name(dev, name: "%s" , name); |
449 | if (ret) { |
450 | put_device(dev); |
451 | return ERR_PTR(error: ret); |
452 | } |
453 | |
454 | ret = device_add(dev); |
455 | if (ret) { |
456 | put_device(dev); |
457 | return ERR_PTR(error: ret); |
458 | } |
459 | |
460 | return epf; |
461 | } |
462 | EXPORT_SYMBOL_GPL(pci_epf_create); |
463 | |
464 | static void pci_epf_dev_release(struct device *dev) |
465 | { |
466 | struct pci_epf *epf = to_pci_epf(dev); |
467 | |
468 | kfree(objp: epf->name); |
469 | kfree(objp: epf); |
470 | } |
471 | |
472 | static const struct device_type pci_epf_type = { |
473 | .release = pci_epf_dev_release, |
474 | }; |
475 | |
476 | static const struct pci_epf_device_id * |
477 | pci_epf_match_id(const struct pci_epf_device_id *id, const struct pci_epf *epf) |
478 | { |
479 | while (id->name[0]) { |
480 | if (strcmp(epf->name, id->name) == 0) |
481 | return id; |
482 | id++; |
483 | } |
484 | |
485 | return NULL; |
486 | } |
487 | |
488 | static int pci_epf_device_match(struct device *dev, struct device_driver *drv) |
489 | { |
490 | struct pci_epf *epf = to_pci_epf(dev); |
491 | struct pci_epf_driver *driver = to_pci_epf_driver(drv); |
492 | |
493 | if (driver->id_table) |
494 | return !!pci_epf_match_id(id: driver->id_table, epf); |
495 | |
496 | return !strcmp(epf->name, drv->name); |
497 | } |
498 | |
499 | static int pci_epf_device_probe(struct device *dev) |
500 | { |
501 | struct pci_epf *epf = to_pci_epf(dev); |
502 | struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver); |
503 | |
504 | if (!driver->probe) |
505 | return -ENODEV; |
506 | |
507 | epf->driver = driver; |
508 | |
509 | return driver->probe(epf, pci_epf_match_id(id: driver->id_table, epf)); |
510 | } |
511 | |
512 | static void pci_epf_device_remove(struct device *dev) |
513 | { |
514 | struct pci_epf *epf = to_pci_epf(dev); |
515 | struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver); |
516 | |
517 | if (driver->remove) |
518 | driver->remove(epf); |
519 | epf->driver = NULL; |
520 | } |
521 | |
522 | static const struct bus_type pci_epf_bus_type = { |
523 | .name = "pci-epf" , |
524 | .match = pci_epf_device_match, |
525 | .probe = pci_epf_device_probe, |
526 | .remove = pci_epf_device_remove, |
527 | }; |
528 | |
529 | static int __init pci_epf_init(void) |
530 | { |
531 | int ret; |
532 | |
533 | ret = bus_register(bus: &pci_epf_bus_type); |
534 | if (ret) { |
535 | pr_err("failed to register pci epf bus --> %d\n" , ret); |
536 | return ret; |
537 | } |
538 | |
539 | return 0; |
540 | } |
541 | module_init(pci_epf_init); |
542 | |
543 | static void __exit pci_epf_exit(void) |
544 | { |
545 | bus_unregister(bus: &pci_epf_bus_type); |
546 | } |
547 | module_exit(pci_epf_exit); |
548 | |
549 | MODULE_DESCRIPTION("PCI EPF Library" ); |
550 | MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>" ); |
551 | |