1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* |
3 | * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved. |
4 | */ |
5 | |
6 | #include <rdma/rdma_cm.h> |
7 | #include <rdma/ib_verbs.h> |
8 | #include <rdma/restrack.h> |
9 | #include <rdma/rdma_counter.h> |
10 | #include <linux/mutex.h> |
11 | #include <linux/sched/task.h> |
12 | #include <linux/pid_namespace.h> |
13 | |
14 | #include "cma_priv.h" |
15 | #include "restrack.h" |
16 | |
17 | /** |
18 | * rdma_restrack_init() - initialize and allocate resource tracking |
19 | * @dev: IB device |
20 | * |
21 | * Return: 0 on success |
22 | */ |
23 | int rdma_restrack_init(struct ib_device *dev) |
24 | { |
25 | struct rdma_restrack_root *rt; |
26 | int i; |
27 | |
28 | dev->res = kcalloc(n: RDMA_RESTRACK_MAX, size: sizeof(*rt), GFP_KERNEL); |
29 | if (!dev->res) |
30 | return -ENOMEM; |
31 | |
32 | rt = dev->res; |
33 | |
34 | for (i = 0; i < RDMA_RESTRACK_MAX; i++) |
35 | xa_init_flags(xa: &rt[i].xa, XA_FLAGS_ALLOC); |
36 | |
37 | return 0; |
38 | } |
39 | |
40 | static const char *type2str(enum rdma_restrack_type type) |
41 | { |
42 | static const char * const names[RDMA_RESTRACK_MAX] = { |
43 | [RDMA_RESTRACK_PD] = "PD" , |
44 | [RDMA_RESTRACK_CQ] = "CQ" , |
45 | [RDMA_RESTRACK_QP] = "QP" , |
46 | [RDMA_RESTRACK_CM_ID] = "CM_ID" , |
47 | [RDMA_RESTRACK_MR] = "MR" , |
48 | [RDMA_RESTRACK_CTX] = "CTX" , |
49 | [RDMA_RESTRACK_COUNTER] = "COUNTER" , |
50 | [RDMA_RESTRACK_SRQ] = "SRQ" , |
51 | }; |
52 | |
53 | return names[type]; |
54 | }; |
55 | |
56 | /** |
57 | * rdma_restrack_clean() - clean resource tracking |
58 | * @dev: IB device |
59 | */ |
60 | void rdma_restrack_clean(struct ib_device *dev) |
61 | { |
62 | struct rdma_restrack_root *rt = dev->res; |
63 | struct rdma_restrack_entry *e; |
64 | char buf[TASK_COMM_LEN]; |
65 | bool found = false; |
66 | const char *owner; |
67 | int i; |
68 | |
69 | for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) { |
70 | struct xarray *xa = &dev->res[i].xa; |
71 | |
72 | if (!xa_empty(xa)) { |
73 | unsigned long index; |
74 | |
75 | if (!found) { |
76 | pr_err("restrack: %s" , CUT_HERE); |
77 | dev_err(&dev->dev, "BUG: RESTRACK detected leak of resources\n" ); |
78 | } |
79 | xa_for_each(xa, index, e) { |
80 | if (rdma_is_kernel_res(res: e)) { |
81 | owner = e->kern_name; |
82 | } else { |
83 | /* |
84 | * There is no need to call get_task_struct here, |
85 | * because we can be here only if there are more |
86 | * get_task_struct() call than put_task_struct(). |
87 | */ |
88 | get_task_comm(buf, e->task); |
89 | owner = buf; |
90 | } |
91 | |
92 | pr_err("restrack: %s %s object allocated by %s is not freed\n" , |
93 | rdma_is_kernel_res(e) ? "Kernel" : |
94 | "User" , |
95 | type2str(e->type), owner); |
96 | } |
97 | found = true; |
98 | } |
99 | xa_destroy(xa); |
100 | } |
101 | if (found) |
102 | pr_err("restrack: %s" , CUT_HERE); |
103 | |
104 | kfree(objp: rt); |
105 | } |
106 | |
107 | /** |
108 | * rdma_restrack_count() - the current usage of specific object |
109 | * @dev: IB device |
110 | * @type: actual type of object to operate |
111 | */ |
112 | int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type) |
113 | { |
114 | struct rdma_restrack_root *rt = &dev->res[type]; |
115 | struct rdma_restrack_entry *e; |
116 | XA_STATE(xas, &rt->xa, 0); |
117 | u32 cnt = 0; |
118 | |
119 | xa_lock(&rt->xa); |
120 | xas_for_each(&xas, e, U32_MAX) |
121 | cnt++; |
122 | xa_unlock(&rt->xa); |
123 | return cnt; |
124 | } |
125 | EXPORT_SYMBOL(rdma_restrack_count); |
126 | |
127 | static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) |
128 | { |
129 | switch (res->type) { |
130 | case RDMA_RESTRACK_PD: |
131 | return container_of(res, struct ib_pd, res)->device; |
132 | case RDMA_RESTRACK_CQ: |
133 | return container_of(res, struct ib_cq, res)->device; |
134 | case RDMA_RESTRACK_QP: |
135 | return container_of(res, struct ib_qp, res)->device; |
136 | case RDMA_RESTRACK_CM_ID: |
137 | return container_of(res, struct rdma_id_private, |
138 | res)->id.device; |
139 | case RDMA_RESTRACK_MR: |
140 | return container_of(res, struct ib_mr, res)->device; |
141 | case RDMA_RESTRACK_CTX: |
142 | return container_of(res, struct ib_ucontext, res)->device; |
143 | case RDMA_RESTRACK_COUNTER: |
144 | return container_of(res, struct rdma_counter, res)->device; |
145 | case RDMA_RESTRACK_SRQ: |
146 | return container_of(res, struct ib_srq, res)->device; |
147 | default: |
148 | WARN_ONCE(true, "Wrong resource tracking type %u\n" , res->type); |
149 | return NULL; |
150 | } |
151 | } |
152 | |
153 | /** |
154 | * rdma_restrack_attach_task() - attach the task onto this resource, |
155 | * valid for user space restrack entries. |
156 | * @res: resource entry |
157 | * @task: the task to attach |
158 | */ |
159 | static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, |
160 | struct task_struct *task) |
161 | { |
162 | if (WARN_ON_ONCE(!task)) |
163 | return; |
164 | |
165 | if (res->task) |
166 | put_task_struct(t: res->task); |
167 | get_task_struct(t: task); |
168 | res->task = task; |
169 | res->user = true; |
170 | } |
171 | |
172 | /** |
173 | * rdma_restrack_set_name() - set the task for this resource |
174 | * @res: resource entry |
175 | * @caller: kernel name, the current task will be used if the caller is NULL. |
176 | */ |
177 | void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller) |
178 | { |
179 | if (caller) { |
180 | res->kern_name = caller; |
181 | return; |
182 | } |
183 | |
184 | rdma_restrack_attach_task(res, current); |
185 | } |
186 | EXPORT_SYMBOL(rdma_restrack_set_name); |
187 | |
188 | /** |
189 | * rdma_restrack_parent_name() - set the restrack name properties based |
190 | * on parent restrack |
191 | * @dst: destination resource entry |
192 | * @parent: parent resource entry |
193 | */ |
194 | void rdma_restrack_parent_name(struct rdma_restrack_entry *dst, |
195 | const struct rdma_restrack_entry *parent) |
196 | { |
197 | if (rdma_is_kernel_res(res: parent)) |
198 | dst->kern_name = parent->kern_name; |
199 | else |
200 | rdma_restrack_attach_task(res: dst, task: parent->task); |
201 | } |
202 | EXPORT_SYMBOL(rdma_restrack_parent_name); |
203 | |
204 | /** |
205 | * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface |
206 | * to release memory in fully automatic way. |
207 | * @res: Entry to initialize |
208 | * @type: REstrack type |
209 | */ |
210 | void rdma_restrack_new(struct rdma_restrack_entry *res, |
211 | enum rdma_restrack_type type) |
212 | { |
213 | kref_init(kref: &res->kref); |
214 | init_completion(x: &res->comp); |
215 | res->type = type; |
216 | } |
217 | EXPORT_SYMBOL(rdma_restrack_new); |
218 | |
219 | /** |
220 | * rdma_restrack_add() - add object to the reource tracking database |
221 | * @res: resource entry |
222 | */ |
223 | void rdma_restrack_add(struct rdma_restrack_entry *res) |
224 | { |
225 | struct ib_device *dev = res_to_dev(res); |
226 | struct rdma_restrack_root *rt; |
227 | int ret = 0; |
228 | |
229 | if (!dev) |
230 | return; |
231 | |
232 | if (res->no_track) |
233 | goto out; |
234 | |
235 | rt = &dev->res[res->type]; |
236 | |
237 | if (res->type == RDMA_RESTRACK_QP) { |
238 | /* Special case to ensure that LQPN points to right QP */ |
239 | struct ib_qp *qp = container_of(res, struct ib_qp, res); |
240 | |
241 | WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8, |
242 | "QP number 0x%0X and port 0x%0X" , qp->qp_num, |
243 | qp->port); |
244 | res->id = qp->qp_num; |
245 | if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI) |
246 | res->id |= qp->port << 24; |
247 | ret = xa_insert(xa: &rt->xa, index: res->id, entry: res, GFP_KERNEL); |
248 | if (ret) |
249 | res->id = 0; |
250 | } else if (res->type == RDMA_RESTRACK_COUNTER) { |
251 | /* Special case to ensure that cntn points to right counter */ |
252 | struct rdma_counter *counter; |
253 | |
254 | counter = container_of(res, struct rdma_counter, res); |
255 | ret = xa_insert(xa: &rt->xa, index: counter->id, entry: res, GFP_KERNEL); |
256 | res->id = ret ? 0 : counter->id; |
257 | } else { |
258 | ret = xa_alloc_cyclic(xa: &rt->xa, id: &res->id, entry: res, xa_limit_32b, |
259 | next: &rt->next_id, GFP_KERNEL); |
260 | ret = (ret < 0) ? ret : 0; |
261 | } |
262 | |
263 | out: |
264 | if (!ret) |
265 | res->valid = true; |
266 | } |
267 | EXPORT_SYMBOL(rdma_restrack_add); |
268 | |
269 | int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) |
270 | { |
271 | return kref_get_unless_zero(kref: &res->kref); |
272 | } |
273 | EXPORT_SYMBOL(rdma_restrack_get); |
274 | |
275 | /** |
276 | * rdma_restrack_get_byid() - translate from ID to restrack object |
277 | * @dev: IB device |
278 | * @type: resource track type |
279 | * @id: ID to take a look |
280 | * |
281 | * Return: Pointer to restrack entry or -ENOENT in case of error. |
282 | */ |
283 | struct rdma_restrack_entry * |
284 | rdma_restrack_get_byid(struct ib_device *dev, |
285 | enum rdma_restrack_type type, u32 id) |
286 | { |
287 | struct rdma_restrack_root *rt = &dev->res[type]; |
288 | struct rdma_restrack_entry *res; |
289 | |
290 | xa_lock(&rt->xa); |
291 | res = xa_load(&rt->xa, index: id); |
292 | if (!res || !rdma_restrack_get(res)) |
293 | res = ERR_PTR(error: -ENOENT); |
294 | xa_unlock(&rt->xa); |
295 | |
296 | return res; |
297 | } |
298 | EXPORT_SYMBOL(rdma_restrack_get_byid); |
299 | |
300 | static void restrack_release(struct kref *kref) |
301 | { |
302 | struct rdma_restrack_entry *res; |
303 | |
304 | res = container_of(kref, struct rdma_restrack_entry, kref); |
305 | if (res->task) { |
306 | put_task_struct(t: res->task); |
307 | res->task = NULL; |
308 | } |
309 | complete(&res->comp); |
310 | } |
311 | |
312 | int rdma_restrack_put(struct rdma_restrack_entry *res) |
313 | { |
314 | return kref_put(kref: &res->kref, release: restrack_release); |
315 | } |
316 | EXPORT_SYMBOL(rdma_restrack_put); |
317 | |
318 | /** |
319 | * rdma_restrack_del() - delete object from the reource tracking database |
320 | * @res: resource entry |
321 | */ |
322 | void rdma_restrack_del(struct rdma_restrack_entry *res) |
323 | { |
324 | struct rdma_restrack_entry *old; |
325 | struct rdma_restrack_root *rt; |
326 | struct ib_device *dev; |
327 | |
328 | if (!res->valid) { |
329 | if (res->task) { |
330 | put_task_struct(t: res->task); |
331 | res->task = NULL; |
332 | } |
333 | return; |
334 | } |
335 | |
336 | if (res->no_track) |
337 | goto out; |
338 | |
339 | dev = res_to_dev(res); |
340 | if (WARN_ON(!dev)) |
341 | return; |
342 | |
343 | rt = &dev->res[res->type]; |
344 | |
345 | old = xa_erase(&rt->xa, index: res->id); |
346 | WARN_ON(old != res); |
347 | |
348 | out: |
349 | res->valid = false; |
350 | rdma_restrack_put(res); |
351 | wait_for_completion(&res->comp); |
352 | } |
353 | EXPORT_SYMBOL(rdma_restrack_del); |
354 | |