1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (c) 2016 Mellanox Technologies. All rights reserved. |
4 | * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com> |
5 | */ |
6 | |
7 | #include <net/genetlink.h> |
8 | #define CREATE_TRACE_POINTS |
9 | #include <trace/events/devlink.h> |
10 | |
11 | #include "devl_internal.h" |
12 | |
13 | EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg); |
14 | EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr); |
15 | EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); |
16 | |
17 | DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); |
18 | |
19 | static struct devlink *devlinks_xa_get(unsigned long index) |
20 | { |
21 | struct devlink *devlink; |
22 | |
23 | rcu_read_lock(); |
24 | devlink = xa_find(xa: &devlinks, index: &index, max: index, DEVLINK_REGISTERED); |
25 | if (!devlink || !devlink_try_get(devlink)) |
26 | devlink = NULL; |
27 | rcu_read_unlock(); |
28 | return devlink; |
29 | } |
30 | |
31 | /* devlink_rels xarray contains 1:1 relationships between |
32 | * devlink object and related nested devlink instance. |
33 | * The xarray index is used to get the nested object from |
34 | * the nested-in object code. |
35 | */ |
36 | static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); |
37 | |
38 | #define DEVLINK_REL_IN_USE XA_MARK_0 |
39 | |
40 | struct devlink_rel { |
41 | u32 index; |
42 | refcount_t refcount; |
43 | u32 devlink_index; |
44 | struct { |
45 | u32 devlink_index; |
46 | u32 obj_index; |
47 | devlink_rel_notify_cb_t *notify_cb; |
48 | devlink_rel_cleanup_cb_t *cleanup_cb; |
49 | struct work_struct notify_work; |
50 | } nested_in; |
51 | }; |
52 | |
53 | static void devlink_rel_free(struct devlink_rel *rel) |
54 | { |
55 | xa_erase(&devlink_rels, index: rel->index); |
56 | kfree(objp: rel); |
57 | } |
58 | |
59 | static void __devlink_rel_get(struct devlink_rel *rel) |
60 | { |
61 | refcount_inc(r: &rel->refcount); |
62 | } |
63 | |
64 | static void __devlink_rel_put(struct devlink_rel *rel) |
65 | { |
66 | if (refcount_dec_and_test(r: &rel->refcount)) |
67 | devlink_rel_free(rel); |
68 | } |
69 | |
70 | static void devlink_rel_nested_in_notify_work(struct work_struct *work) |
71 | { |
72 | struct devlink_rel *rel = container_of(work, struct devlink_rel, |
73 | nested_in.notify_work); |
74 | struct devlink *devlink; |
75 | |
76 | devlink = devlinks_xa_get(index: rel->nested_in.devlink_index); |
77 | if (!devlink) |
78 | goto rel_put; |
79 | if (!devl_trylock(devlink)) { |
80 | devlink_put(devlink); |
81 | goto reschedule_work; |
82 | } |
83 | if (!devl_is_registered(devlink)) { |
84 | devl_unlock(devlink); |
85 | devlink_put(devlink); |
86 | goto rel_put; |
87 | } |
88 | if (!xa_get_mark(&devlink_rels, index: rel->index, DEVLINK_REL_IN_USE)) |
89 | rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); |
90 | rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); |
91 | devl_unlock(devlink); |
92 | devlink_put(devlink); |
93 | |
94 | rel_put: |
95 | __devlink_rel_put(rel); |
96 | return; |
97 | |
98 | reschedule_work: |
99 | schedule_work(work: &rel->nested_in.notify_work); |
100 | } |
101 | |
102 | static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) |
103 | { |
104 | __devlink_rel_get(rel); |
105 | schedule_work(work: &rel->nested_in.notify_work); |
106 | } |
107 | |
108 | static struct devlink_rel *devlink_rel_alloc(void) |
109 | { |
110 | struct devlink_rel *rel; |
111 | static u32 next; |
112 | int err; |
113 | |
114 | rel = kzalloc(size: sizeof(*rel), GFP_KERNEL); |
115 | if (!rel) |
116 | return ERR_PTR(error: -ENOMEM); |
117 | |
118 | err = xa_alloc_cyclic(xa: &devlink_rels, id: &rel->index, entry: rel, |
119 | xa_limit_32b, next: &next, GFP_KERNEL); |
120 | if (err) { |
121 | kfree(objp: rel); |
122 | return ERR_PTR(error: err); |
123 | } |
124 | |
125 | refcount_set(r: &rel->refcount, n: 1); |
126 | INIT_WORK(&rel->nested_in.notify_work, |
127 | &devlink_rel_nested_in_notify_work); |
128 | return rel; |
129 | } |
130 | |
131 | static void devlink_rel_put(struct devlink *devlink) |
132 | { |
133 | struct devlink_rel *rel = devlink->rel; |
134 | |
135 | if (!rel) |
136 | return; |
137 | xa_clear_mark(&devlink_rels, index: rel->index, DEVLINK_REL_IN_USE); |
138 | devlink_rel_nested_in_notify_work_schedule(rel); |
139 | __devlink_rel_put(rel); |
140 | devlink->rel = NULL; |
141 | } |
142 | |
143 | void devlink_rel_nested_in_clear(u32 rel_index) |
144 | { |
145 | xa_clear_mark(&devlink_rels, index: rel_index, DEVLINK_REL_IN_USE); |
146 | } |
147 | |
148 | int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, |
149 | u32 obj_index, devlink_rel_notify_cb_t *notify_cb, |
150 | devlink_rel_cleanup_cb_t *cleanup_cb, |
151 | struct devlink *devlink) |
152 | { |
153 | struct devlink_rel *rel = devlink_rel_alloc(); |
154 | |
155 | ASSERT_DEVLINK_NOT_REGISTERED(devlink); |
156 | |
157 | if (IS_ERR(ptr: rel)) |
158 | return PTR_ERR(ptr: rel); |
159 | |
160 | rel->devlink_index = devlink->index; |
161 | rel->nested_in.devlink_index = devlink_index; |
162 | rel->nested_in.obj_index = obj_index; |
163 | rel->nested_in.notify_cb = notify_cb; |
164 | rel->nested_in.cleanup_cb = cleanup_cb; |
165 | *rel_index = rel->index; |
166 | xa_set_mark(&devlink_rels, index: rel->index, DEVLINK_REL_IN_USE); |
167 | devlink->rel = rel; |
168 | return 0; |
169 | } |
170 | |
171 | /** |
172 | * devlink_rel_nested_in_notify - Notify the object this devlink |
173 | * instance is nested in. |
174 | * @devlink: devlink |
175 | * |
176 | * This is called upon network namespace change of devlink instance. |
177 | * In case this devlink instance is nested in another devlink object, |
178 | * a notification of a change of this object should be sent |
179 | * over netlink. The parent devlink instance lock needs to be |
180 | * taken during the notification preparation. |
181 | * However, since the devlink lock of nested instance is held here, |
182 | * we would end with wrong devlink instance lock ordering and |
183 | * deadlock. Therefore the work is utilized to avoid that. |
184 | */ |
185 | void devlink_rel_nested_in_notify(struct devlink *devlink) |
186 | { |
187 | struct devlink_rel *rel = devlink->rel; |
188 | |
189 | if (!rel) |
190 | return; |
191 | devlink_rel_nested_in_notify_work_schedule(rel); |
192 | } |
193 | |
194 | static struct devlink_rel *devlink_rel_find(unsigned long rel_index) |
195 | { |
196 | return xa_find(xa: &devlink_rels, index: &rel_index, max: rel_index, |
197 | DEVLINK_REL_IN_USE); |
198 | } |
199 | |
200 | static struct devlink *devlink_rel_devlink_get(u32 rel_index) |
201 | { |
202 | struct devlink_rel *rel; |
203 | u32 devlink_index; |
204 | |
205 | if (!rel_index) |
206 | return NULL; |
207 | xa_lock(&devlink_rels); |
208 | rel = devlink_rel_find(rel_index); |
209 | if (rel) |
210 | devlink_index = rel->devlink_index; |
211 | xa_unlock(&devlink_rels); |
212 | if (!rel) |
213 | return NULL; |
214 | return devlinks_xa_get(index: devlink_index); |
215 | } |
216 | |
217 | int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, |
218 | u32 rel_index, int attrtype, |
219 | bool *msg_updated) |
220 | { |
221 | struct net *net = devlink_net(devlink); |
222 | struct devlink *rel_devlink; |
223 | int err; |
224 | |
225 | rel_devlink = devlink_rel_devlink_get(rel_index); |
226 | if (!rel_devlink) |
227 | return 0; |
228 | err = devlink_nl_put_nested_handle(msg, net, devlink: rel_devlink, attrtype); |
229 | devlink_put(devlink: rel_devlink); |
230 | if (!err && msg_updated) |
231 | *msg_updated = true; |
232 | return err; |
233 | } |
234 | |
235 | void *devlink_priv(struct devlink *devlink) |
236 | { |
237 | return &devlink->priv; |
238 | } |
239 | EXPORT_SYMBOL_GPL(devlink_priv); |
240 | |
241 | struct devlink *priv_to_devlink(void *priv) |
242 | { |
243 | return container_of(priv, struct devlink, priv); |
244 | } |
245 | EXPORT_SYMBOL_GPL(priv_to_devlink); |
246 | |
247 | struct device *devlink_to_dev(const struct devlink *devlink) |
248 | { |
249 | return devlink->dev; |
250 | } |
251 | EXPORT_SYMBOL_GPL(devlink_to_dev); |
252 | |
253 | struct net *devlink_net(const struct devlink *devlink) |
254 | { |
255 | return read_pnet(pnet: &devlink->_net); |
256 | } |
257 | EXPORT_SYMBOL_GPL(devlink_net); |
258 | |
259 | void devl_assert_locked(struct devlink *devlink) |
260 | { |
261 | lockdep_assert_held(&devlink->lock); |
262 | } |
263 | EXPORT_SYMBOL_GPL(devl_assert_locked); |
264 | |
265 | #ifdef CONFIG_LOCKDEP |
266 | /* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */ |
267 | bool devl_lock_is_held(struct devlink *devlink) |
268 | { |
269 | return lockdep_is_held(&devlink->lock); |
270 | } |
271 | EXPORT_SYMBOL_GPL(devl_lock_is_held); |
272 | #endif |
273 | |
274 | void devl_lock(struct devlink *devlink) |
275 | { |
276 | mutex_lock(&devlink->lock); |
277 | } |
278 | EXPORT_SYMBOL_GPL(devl_lock); |
279 | |
280 | int devl_trylock(struct devlink *devlink) |
281 | { |
282 | return mutex_trylock(lock: &devlink->lock); |
283 | } |
284 | EXPORT_SYMBOL_GPL(devl_trylock); |
285 | |
286 | void devl_unlock(struct devlink *devlink) |
287 | { |
288 | mutex_unlock(lock: &devlink->lock); |
289 | } |
290 | EXPORT_SYMBOL_GPL(devl_unlock); |
291 | |
292 | /** |
293 | * devlink_try_get() - try to obtain a reference on a devlink instance |
294 | * @devlink: instance to reference |
295 | * |
296 | * Obtain a reference on a devlink instance. A reference on a devlink instance |
297 | * only implies that it's safe to take the instance lock. It does not imply |
298 | * that the instance is registered, use devl_is_registered() after taking |
299 | * the instance lock to check registration status. |
300 | */ |
301 | struct devlink *__must_check devlink_try_get(struct devlink *devlink) |
302 | { |
303 | if (refcount_inc_not_zero(r: &devlink->refcount)) |
304 | return devlink; |
305 | return NULL; |
306 | } |
307 | |
308 | static void devlink_release(struct work_struct *work) |
309 | { |
310 | struct devlink *devlink; |
311 | |
312 | devlink = container_of(to_rcu_work(work), struct devlink, rwork); |
313 | |
314 | mutex_destroy(lock: &devlink->lock); |
315 | lockdep_unregister_key(key: &devlink->lock_key); |
316 | put_device(dev: devlink->dev); |
317 | kfree(objp: devlink); |
318 | } |
319 | |
320 | void devlink_put(struct devlink *devlink) |
321 | { |
322 | if (refcount_dec_and_test(r: &devlink->refcount)) |
323 | queue_rcu_work(wq: system_wq, rwork: &devlink->rwork); |
324 | } |
325 | |
326 | struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp) |
327 | { |
328 | struct devlink *devlink = NULL; |
329 | |
330 | rcu_read_lock(); |
331 | retry: |
332 | devlink = xa_find(xa: &devlinks, index: indexp, ULONG_MAX, DEVLINK_REGISTERED); |
333 | if (!devlink) |
334 | goto unlock; |
335 | |
336 | if (!devlink_try_get(devlink)) |
337 | goto next; |
338 | if (!net_eq(net1: devlink_net(devlink), net2: net)) { |
339 | devlink_put(devlink); |
340 | goto next; |
341 | } |
342 | unlock: |
343 | rcu_read_unlock(); |
344 | return devlink; |
345 | |
346 | next: |
347 | (*indexp)++; |
348 | goto retry; |
349 | } |
350 | |
351 | /** |
352 | * devl_register - Register devlink instance |
353 | * @devlink: devlink |
354 | */ |
355 | int devl_register(struct devlink *devlink) |
356 | { |
357 | ASSERT_DEVLINK_NOT_REGISTERED(devlink); |
358 | devl_assert_locked(devlink); |
359 | |
360 | xa_set_mark(&devlinks, index: devlink->index, DEVLINK_REGISTERED); |
361 | devlink_notify_register(devlink); |
362 | devlink_rel_nested_in_notify(devlink); |
363 | |
364 | return 0; |
365 | } |
366 | EXPORT_SYMBOL_GPL(devl_register); |
367 | |
368 | void devlink_register(struct devlink *devlink) |
369 | { |
370 | devl_lock(devlink); |
371 | devl_register(devlink); |
372 | devl_unlock(devlink); |
373 | } |
374 | EXPORT_SYMBOL_GPL(devlink_register); |
375 | |
376 | /** |
377 | * devl_unregister - Unregister devlink instance |
378 | * @devlink: devlink |
379 | */ |
380 | void devl_unregister(struct devlink *devlink) |
381 | { |
382 | ASSERT_DEVLINK_REGISTERED(devlink); |
383 | devl_assert_locked(devlink); |
384 | |
385 | devlink_notify_unregister(devlink); |
386 | xa_clear_mark(&devlinks, index: devlink->index, DEVLINK_REGISTERED); |
387 | devlink_rel_put(devlink); |
388 | } |
389 | EXPORT_SYMBOL_GPL(devl_unregister); |
390 | |
391 | void devlink_unregister(struct devlink *devlink) |
392 | { |
393 | devl_lock(devlink); |
394 | devl_unregister(devlink); |
395 | devl_unlock(devlink); |
396 | } |
397 | EXPORT_SYMBOL_GPL(devlink_unregister); |
398 | |
399 | /** |
400 | * devlink_alloc_ns - Allocate new devlink instance resources |
401 | * in specific namespace |
402 | * |
403 | * @ops: ops |
404 | * @priv_size: size of user private data |
405 | * @net: net namespace |
406 | * @dev: parent device |
407 | * |
408 | * Allocate new devlink instance resources, including devlink index |
409 | * and name. |
410 | */ |
411 | struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, |
412 | size_t priv_size, struct net *net, |
413 | struct device *dev) |
414 | { |
415 | struct devlink *devlink; |
416 | static u32 last_id; |
417 | int ret; |
418 | |
419 | WARN_ON(!ops || !dev); |
420 | if (!devlink_reload_actions_valid(ops)) |
421 | return NULL; |
422 | |
423 | devlink = kzalloc(size: sizeof(*devlink) + priv_size, GFP_KERNEL); |
424 | if (!devlink) |
425 | return NULL; |
426 | |
427 | ret = xa_alloc_cyclic(xa: &devlinks, id: &devlink->index, entry: devlink, xa_limit_31b, |
428 | next: &last_id, GFP_KERNEL); |
429 | if (ret < 0) |
430 | goto err_xa_alloc; |
431 | |
432 | devlink->dev = get_device(dev); |
433 | devlink->ops = ops; |
434 | xa_init_flags(xa: &devlink->ports, XA_FLAGS_ALLOC); |
435 | xa_init_flags(xa: &devlink->params, XA_FLAGS_ALLOC); |
436 | xa_init_flags(xa: &devlink->snapshot_ids, XA_FLAGS_ALLOC); |
437 | xa_init_flags(xa: &devlink->nested_rels, XA_FLAGS_ALLOC); |
438 | write_pnet(pnet: &devlink->_net, net); |
439 | INIT_LIST_HEAD(list: &devlink->rate_list); |
440 | INIT_LIST_HEAD(list: &devlink->linecard_list); |
441 | INIT_LIST_HEAD(list: &devlink->sb_list); |
442 | INIT_LIST_HEAD_RCU(list: &devlink->dpipe_table_list); |
443 | INIT_LIST_HEAD(list: &devlink->resource_list); |
444 | INIT_LIST_HEAD(list: &devlink->region_list); |
445 | INIT_LIST_HEAD(list: &devlink->reporter_list); |
446 | INIT_LIST_HEAD(list: &devlink->trap_list); |
447 | INIT_LIST_HEAD(list: &devlink->trap_group_list); |
448 | INIT_LIST_HEAD(list: &devlink->trap_policer_list); |
449 | INIT_RCU_WORK(&devlink->rwork, devlink_release); |
450 | lockdep_register_key(key: &devlink->lock_key); |
451 | mutex_init(&devlink->lock); |
452 | lockdep_set_class(&devlink->lock, &devlink->lock_key); |
453 | refcount_set(r: &devlink->refcount, n: 1); |
454 | |
455 | return devlink; |
456 | |
457 | err_xa_alloc: |
458 | kfree(objp: devlink); |
459 | return NULL; |
460 | } |
461 | EXPORT_SYMBOL_GPL(devlink_alloc_ns); |
462 | |
463 | /** |
464 | * devlink_free - Free devlink instance resources |
465 | * |
466 | * @devlink: devlink |
467 | */ |
468 | void devlink_free(struct devlink *devlink) |
469 | { |
470 | ASSERT_DEVLINK_NOT_REGISTERED(devlink); |
471 | |
472 | WARN_ON(!list_empty(&devlink->trap_policer_list)); |
473 | WARN_ON(!list_empty(&devlink->trap_group_list)); |
474 | WARN_ON(!list_empty(&devlink->trap_list)); |
475 | WARN_ON(!list_empty(&devlink->reporter_list)); |
476 | WARN_ON(!list_empty(&devlink->region_list)); |
477 | WARN_ON(!list_empty(&devlink->resource_list)); |
478 | WARN_ON(!list_empty(&devlink->dpipe_table_list)); |
479 | WARN_ON(!list_empty(&devlink->sb_list)); |
480 | WARN_ON(!list_empty(&devlink->rate_list)); |
481 | WARN_ON(!list_empty(&devlink->linecard_list)); |
482 | WARN_ON(!xa_empty(&devlink->ports)); |
483 | |
484 | xa_destroy(&devlink->nested_rels); |
485 | xa_destroy(&devlink->snapshot_ids); |
486 | xa_destroy(&devlink->params); |
487 | xa_destroy(&devlink->ports); |
488 | |
489 | xa_erase(&devlinks, index: devlink->index); |
490 | |
491 | devlink_put(devlink); |
492 | } |
493 | EXPORT_SYMBOL_GPL(devlink_free); |
494 | |
495 | static void __net_exit devlink_pernet_pre_exit(struct net *net) |
496 | { |
497 | struct devlink *devlink; |
498 | u32 actions_performed; |
499 | unsigned long index; |
500 | int err; |
501 | |
502 | /* In case network namespace is getting destroyed, reload |
503 | * all devlink instances from this namespace into init_net. |
504 | */ |
505 | devlinks_xa_for_each_registered_get(net, index, devlink) { |
506 | devl_lock(devlink); |
507 | err = 0; |
508 | if (devl_is_registered(devlink)) |
509 | err = devlink_reload(devlink, dest_net: &init_net, |
510 | action: DEVLINK_RELOAD_ACTION_DRIVER_REINIT, |
511 | limit: DEVLINK_RELOAD_LIMIT_UNSPEC, |
512 | actions_performed: &actions_performed, NULL); |
513 | devl_unlock(devlink); |
514 | devlink_put(devlink); |
515 | if (err && err != -EOPNOTSUPP) |
516 | pr_warn("Failed to reload devlink instance into init_net\n" ); |
517 | } |
518 | } |
519 | |
520 | static struct pernet_operations devlink_pernet_ops __net_initdata = { |
521 | .pre_exit = devlink_pernet_pre_exit, |
522 | }; |
523 | |
524 | static struct notifier_block devlink_port_netdevice_nb = { |
525 | .notifier_call = devlink_port_netdevice_event, |
526 | }; |
527 | |
528 | static int __init devlink_init(void) |
529 | { |
530 | int err; |
531 | |
532 | err = genl_register_family(family: &devlink_nl_family); |
533 | if (err) |
534 | goto out; |
535 | err = register_pernet_subsys(&devlink_pernet_ops); |
536 | if (err) |
537 | goto out; |
538 | err = register_netdevice_notifier(nb: &devlink_port_netdevice_nb); |
539 | |
540 | out: |
541 | WARN_ON(err); |
542 | return err; |
543 | } |
544 | |
545 | subsys_initcall(devlink_init); |
546 | |