1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/*
3	* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4	*/
5
6	#include <rdma/ib_user_verbs.h>
7	#include <rdma/ib_verbs.h>
8	#include <rdma/uverbs_types.h>
9	#include <rdma/uverbs_ioctl.h>
10	#include <rdma/mlx5_user_ioctl_cmds.h>
11	#include <rdma/mlx5_user_ioctl_verbs.h>
12	#include <rdma/ib_umem.h>
13	#include <rdma/uverbs_std_types.h>
14	#include <linux/mlx5/driver.h>
15	#include <linux/mlx5/fs.h>
16	#include <rdma/ib_ucaps.h>
17	#include "mlx5_ib.h"
18	#include "devx.h"
19	#include "qp.h"
20	#include <linux/xarray.h>
21
22	#define UVERBS_MODULE_NAME mlx5_ib
23	#include <rdma/uverbs_named_ioctl.h>
24
25	static void dispatch_event_fd(struct list_head *fd_list, const void *data);
26
27	enum devx_obj_flags {
28	DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
29	DEVX_OBJ_FLAGS_DCT = 1 << 1,
30	DEVX_OBJ_FLAGS_CQ = 1 << 2,
31	DEVX_OBJ_FLAGS_HW_FREED = 1 << 3,
32	};
33
34	#define MAX_ASYNC_CMDS 8
35
36	struct mlx5_async_cmd {
37	struct ib_uobject *uobject;
38	void *in;
39	int in_size;
40	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
41	int err;
42	struct mlx5_async_work cb_work;
43	struct completion comp;
44	};
45
46	struct devx_async_data {
47	struct mlx5_ib_dev *mdev;
48	struct list_head list;
49	struct devx_async_cmd_event_file *ev_file;
50	struct mlx5_async_work cb_work;
51	u16 cmd_out_len;
52	/* must be last field in this structure */
53	struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
54	};
55
56	struct devx_async_event_data {
57	struct list_head list; /* headed in ev_file->event_list */
58	struct mlx5_ib_uapi_devx_async_event_hdr hdr;
59	};
60
61	/* first level XA value data structure */
62	struct devx_event {
63	struct xarray object_ids; /* second XA level, Key = object id */
64	struct list_head unaffiliated_list;
65	};
66
67	/* second level XA value data structure */
68	struct devx_obj_event {
69	struct rcu_head rcu;
70	struct list_head obj_sub_list;
71	};
72
73	struct devx_event_subscription {
74	struct list_head file_list; /* headed in ev_file->
75	* subscribed_events_list
76	*/
77	struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
78	* devx_obj_event->obj_sub_list
79	*/
80	struct list_head obj_list; /* headed in devx_object */
81	struct list_head event_list; /* headed in ev_file->event_list or in
82	* temp list via subscription
83	*/
84
85	u8 is_cleaned:1;
86	u32 xa_key_level1;
87	u32 xa_key_level2;
88	struct rcu_head rcu;
89	u64 cookie;
90	struct devx_async_event_file *ev_file;
91	struct eventfd_ctx *eventfd;
92	};
93
94	struct devx_async_event_file {
95	struct ib_uobject uobj;
96	/* Head of events that are subscribed to this FD */
97	struct list_head subscribed_events_list;
98	spinlock_t lock;
99	wait_queue_head_t poll_wait;
100	struct list_head event_list;
101	struct mlx5_ib_dev *dev;
102	u8 omit_data:1;
103	u8 is_overflow_err:1;
104	u8 is_destroyed:1;
105	};
106
107	struct devx_umem {
108	struct mlx5_core_dev *mdev;
109	struct ib_umem *umem;
110	u32 dinlen;
111	u32 dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];
112	};
113
114	struct devx_umem_reg_cmd {
115	void *in;
116	u32 inlen;
117	u32 out[MLX5_ST_SZ_DW(create_umem_out)];
118	};
119
120	static struct mlx5_ib_ucontext *
121	devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
122	{
123	return to_mucontext(ibucontext: ib_uverbs_get_ucontext(attrs));
124	}
125
126	static int set_uctx_ucaps(struct mlx5_ib_dev *dev, u64 req_ucaps, u32 *cap)
127	{
128	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_LOCAL)) {
129	if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
130	*cap |= MLX5_UCTX_CAP_RDMA_CTRL;
131	else
132	return -EOPNOTSUPP;
133	}
134
135	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA)) {
136	if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
137	MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
138	*cap |= MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA;
139	else
140	return -EOPNOTSUPP;
141	}
142
143	return 0;
144	}
145
146	int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)
147	{
148	u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};
149	u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
150	void *uctx;
151	int err;
152	u16 uid;
153	u32 cap = 0;
154
155	/* 0 means not supported */
156	if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
157	return -EINVAL;
158
159	uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
160	if (is_user &&
161	(MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX) &&
162	rdma_dev_has_raw_cap(dev: &dev->ib_dev))
163	cap |= MLX5_UCTX_CAP_RAW_TX;
164	if (is_user &&
165	(MLX5_CAP_GEN(dev->mdev, uctx_cap) &
166	MLX5_UCTX_CAP_INTERNAL_DEV_RES) &&
167	capable(CAP_SYS_RAWIO))
168	cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
169
170	if (req_ucaps) {
171	err = set_uctx_ucaps(dev, req_ucaps, cap: &cap);
172	if (err)
173	return err;
174	}
175
176	MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
177	MLX5_SET(uctx, uctx, cap, cap);
178
179	err = mlx5_cmd_exec(dev: dev->mdev, in, in_size: sizeof(in), out, out_size: sizeof(out));
180	if (err)
181	return err;
182
183	uid = MLX5_GET(create_uctx_out, out, uid);
184	return uid;
185	}
186
187	void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
188	{
189	u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
190	u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
191
192	MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
193	MLX5_SET(destroy_uctx_in, in, uid, uid);
194
195	mlx5_cmd_exec(dev: dev->mdev, in, in_size: sizeof(in), out, out_size: sizeof(out));
196	}
197
198	static bool is_legacy_unaffiliated_event_num(u16 event_num)
199	{
200	switch (event_num) {
201	case MLX5_EVENT_TYPE_PORT_CHANGE:
202	return true;
203	default:
204	return false;
205	}
206	}
207
208	static bool is_legacy_obj_event_num(u16 event_num)
209	{
210	switch (event_num) {
211	case MLX5_EVENT_TYPE_PATH_MIG:
212	case MLX5_EVENT_TYPE_COMM_EST:
213	case MLX5_EVENT_TYPE_SQ_DRAINED:
214	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
215	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
216	case MLX5_EVENT_TYPE_CQ_ERROR:
217	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
218	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
219	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
220	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
221	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
222	case MLX5_EVENT_TYPE_DCT_DRAINED:
223	case MLX5_EVENT_TYPE_COMP:
224	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
225	case MLX5_EVENT_TYPE_XRQ_ERROR:
226	return true;
227	default:
228	return false;
229	}
230	}
231
232	static u16 get_legacy_obj_type(u16 opcode)
233	{
234	switch (opcode) {
235	case MLX5_CMD_OP_CREATE_RQ:
236	case MLX5_CMD_OP_CREATE_RMP:
237	return MLX5_EVENT_QUEUE_TYPE_RQ;
238	case MLX5_CMD_OP_CREATE_QP:
239	return MLX5_EVENT_QUEUE_TYPE_QP;
240	case MLX5_CMD_OP_CREATE_SQ:
241	return MLX5_EVENT_QUEUE_TYPE_SQ;
242	case MLX5_CMD_OP_CREATE_DCT:
243	return MLX5_EVENT_QUEUE_TYPE_DCT;
244	default:
245	return 0;
246	}
247	}
248
249	static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
250	{
251	u16 opcode;
252
253	opcode = (obj->obj_id >> 32) & 0xffff;
254
255	if (is_legacy_obj_event_num(event_num))
256	return get_legacy_obj_type(opcode);
257
258	switch (opcode) {
259	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
260	return (obj->obj_id >> 48);
261	case MLX5_CMD_OP_CREATE_RQ:
262	return MLX5_OBJ_TYPE_RQ;
263	case MLX5_CMD_OP_CREATE_QP:
264	return MLX5_OBJ_TYPE_QP;
265	case MLX5_CMD_OP_CREATE_SQ:
266	return MLX5_OBJ_TYPE_SQ;
267	case MLX5_CMD_OP_CREATE_DCT:
268	return MLX5_OBJ_TYPE_DCT;
269	case MLX5_CMD_OP_CREATE_TIR:
270	return MLX5_OBJ_TYPE_TIR;
271	case MLX5_CMD_OP_CREATE_TIS:
272	return MLX5_OBJ_TYPE_TIS;
273	case MLX5_CMD_OP_CREATE_PSV:
274	return MLX5_OBJ_TYPE_PSV;
275	case MLX5_OBJ_TYPE_MKEY:
276	return MLX5_OBJ_TYPE_MKEY;
277	case MLX5_CMD_OP_CREATE_RMP:
278	return MLX5_OBJ_TYPE_RMP;
279	case MLX5_CMD_OP_CREATE_XRC_SRQ:
280	return MLX5_OBJ_TYPE_XRC_SRQ;
281	case MLX5_CMD_OP_CREATE_XRQ:
282	return MLX5_OBJ_TYPE_XRQ;
283	case MLX5_CMD_OP_CREATE_RQT:
284	return MLX5_OBJ_TYPE_RQT;
285	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
286	return MLX5_OBJ_TYPE_FLOW_COUNTER;
287	case MLX5_CMD_OP_CREATE_CQ:
288	return MLX5_OBJ_TYPE_CQ;
289	default:
290	return 0;
291	}
292	}
293
294	static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
295	{
296	switch (event_type) {
297	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
298	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
299	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
300	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
301	case MLX5_EVENT_TYPE_PATH_MIG:
302	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
303	case MLX5_EVENT_TYPE_COMM_EST:
304	case MLX5_EVENT_TYPE_SQ_DRAINED:
305	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
306	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
307	return eqe->data.qp_srq.type;
308	case MLX5_EVENT_TYPE_CQ_ERROR:
309	case MLX5_EVENT_TYPE_XRQ_ERROR:
310	return 0;
311	case MLX5_EVENT_TYPE_DCT_DRAINED:
312	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
313	return MLX5_EVENT_QUEUE_TYPE_DCT;
314	default:
315	return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
316	}
317	}
318
319	static u32 get_dec_obj_id(u64 obj_id)
320	{
321	return (obj_id & 0xffffffff);
322	}
323
324	/*
325	* As the obj_id in the firmware is not globally unique the object type
326	* must be considered upon checking for a valid object id.
327	* For that the opcode of the creator command is encoded as part of the obj_id.
328	*/
329	static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
330	{
331	return ((u64)opcode << 32) | obj_id;
332	}
333
334	static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)
335	{
336	switch (opcode) {
337	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
338	return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
339	case MLX5_CMD_OP_CREATE_UMEM:
340	return MLX5_GET(create_umem_out, out, umem_id);
341	case MLX5_CMD_OP_CREATE_MKEY:
342	return MLX5_GET(create_mkey_out, out, mkey_index);
343	case MLX5_CMD_OP_CREATE_CQ:
344	return MLX5_GET(create_cq_out, out, cqn);
345	case MLX5_CMD_OP_ALLOC_PD:
346	return MLX5_GET(alloc_pd_out, out, pd);
347	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
348	return MLX5_GET(alloc_transport_domain_out, out,
349	transport_domain);
350	case MLX5_CMD_OP_CREATE_RMP:
351	return MLX5_GET(create_rmp_out, out, rmpn);
352	case MLX5_CMD_OP_CREATE_SQ:
353	return MLX5_GET(create_sq_out, out, sqn);
354	case MLX5_CMD_OP_CREATE_RQ:
355	return MLX5_GET(create_rq_out, out, rqn);
356	case MLX5_CMD_OP_CREATE_RQT:
357	return MLX5_GET(create_rqt_out, out, rqtn);
358	case MLX5_CMD_OP_CREATE_TIR:
359	return MLX5_GET(create_tir_out, out, tirn);
360	case MLX5_CMD_OP_CREATE_TIS:
361	return MLX5_GET(create_tis_out, out, tisn);
362	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
363	return MLX5_GET(alloc_q_counter_out, out, counter_set_id);
364	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
365	return MLX5_GET(create_flow_table_out, out, table_id);
366	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
367	return MLX5_GET(create_flow_group_out, out, group_id);
368	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
369	return MLX5_GET(set_fte_in, in, flow_index);
370	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
371	return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
372	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
373	return MLX5_GET(alloc_packet_reformat_context_out, out,
374	packet_reformat_id);
375	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
376	return MLX5_GET(alloc_modify_header_context_out, out,
377	modify_header_id);
378	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
379	return MLX5_GET(create_scheduling_element_out, out,
380	scheduling_element_id);
381	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
382	return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
383	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
384	return MLX5_GET(set_l2_table_entry_in, in, table_index);
385	case MLX5_CMD_OP_CREATE_QP:
386	return MLX5_GET(create_qp_out, out, qpn);
387	case MLX5_CMD_OP_CREATE_SRQ:
388	return MLX5_GET(create_srq_out, out, srqn);
389	case MLX5_CMD_OP_CREATE_XRC_SRQ:
390	return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
391	case MLX5_CMD_OP_CREATE_DCT:
392	return MLX5_GET(create_dct_out, out, dctn);
393	case MLX5_CMD_OP_CREATE_XRQ:
394	return MLX5_GET(create_xrq_out, out, xrqn);
395	case MLX5_CMD_OP_ATTACH_TO_MCG:
396	return MLX5_GET(attach_to_mcg_in, in, qpn);
397	case MLX5_CMD_OP_ALLOC_XRCD:
398	return MLX5_GET(alloc_xrcd_out, out, xrcd);
399	case MLX5_CMD_OP_CREATE_PSV:
400	return MLX5_GET(create_psv_out, out, psv0_index);
401	default:
402	/* The entry must match to one of the devx_is_obj_create_cmd */
403	WARN_ON(true);
404	return 0;
405	}
406	}
407
408	static u64 devx_get_obj_id(const void *in)
409	{
410	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
411	u64 obj_id;
412
413	switch (opcode) {
414	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
415	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
416	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
417	MLX5_GET(general_obj_in_cmd_hdr, in,
418	obj_type) << 16,
419	MLX5_GET(general_obj_in_cmd_hdr, in,
420	obj_id));
421	break;
422	case MLX5_CMD_OP_QUERY_MKEY:
423	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_MKEY,
424	MLX5_GET(query_mkey_in, in,
425	mkey_index));
426	break;
427	case MLX5_CMD_OP_QUERY_CQ:
428	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_CQ,
429	MLX5_GET(query_cq_in, in, cqn));
430	break;
431	case MLX5_CMD_OP_MODIFY_CQ:
432	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_CQ,
433	MLX5_GET(modify_cq_in, in, cqn));
434	break;
435	case MLX5_CMD_OP_QUERY_SQ:
436	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SQ,
437	MLX5_GET(query_sq_in, in, sqn));
438	break;
439	case MLX5_CMD_OP_MODIFY_SQ:
440	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SQ,
441	MLX5_GET(modify_sq_in, in, sqn));
442	break;
443	case MLX5_CMD_OP_QUERY_RQ:
444	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQ,
445	MLX5_GET(query_rq_in, in, rqn));
446	break;
447	case MLX5_CMD_OP_MODIFY_RQ:
448	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQ,
449	MLX5_GET(modify_rq_in, in, rqn));
450	break;
451	case MLX5_CMD_OP_QUERY_RMP:
452	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RMP,
453	MLX5_GET(query_rmp_in, in, rmpn));
454	break;
455	case MLX5_CMD_OP_MODIFY_RMP:
456	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RMP,
457	MLX5_GET(modify_rmp_in, in, rmpn));
458	break;
459	case MLX5_CMD_OP_QUERY_RQT:
460	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQT,
461	MLX5_GET(query_rqt_in, in, rqtn));
462	break;
463	case MLX5_CMD_OP_MODIFY_RQT:
464	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQT,
465	MLX5_GET(modify_rqt_in, in, rqtn));
466	break;
467	case MLX5_CMD_OP_QUERY_TIR:
468	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIR,
469	MLX5_GET(query_tir_in, in, tirn));
470	break;
471	case MLX5_CMD_OP_MODIFY_TIR:
472	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIR,
473	MLX5_GET(modify_tir_in, in, tirn));
474	break;
475	case MLX5_CMD_OP_QUERY_TIS:
476	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIS,
477	MLX5_GET(query_tis_in, in, tisn));
478	break;
479	case MLX5_CMD_OP_MODIFY_TIS:
480	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIS,
481	MLX5_GET(modify_tis_in, in, tisn));
482	break;
483	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
484	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_FLOW_TABLE,
485	MLX5_GET(query_flow_table_in, in,
486	table_id));
487	break;
488	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
489	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_FLOW_TABLE,
490	MLX5_GET(modify_flow_table_in, in,
491	table_id));
492	break;
493	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
494	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_FLOW_GROUP,
495	MLX5_GET(query_flow_group_in, in,
496	group_id));
497	break;
498	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
499	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
500	MLX5_GET(query_fte_in, in,
501	flow_index));
502	break;
503	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
504	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
505	MLX5_GET(set_fte_in, in, flow_index));
506	break;
507	case MLX5_CMD_OP_QUERY_Q_COUNTER:
508	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_ALLOC_Q_COUNTER,
509	MLX5_GET(query_q_counter_in, in,
510	counter_set_id));
511	break;
512	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
513	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
514	MLX5_GET(query_flow_counter_in, in,
515	flow_counter_id));
516	break;
517	case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
518	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
519	MLX5_GET(query_modify_header_context_in,
520	in, modify_header_id));
521	break;
522	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
523	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
524	MLX5_GET(query_scheduling_element_in,
525	in, scheduling_element_id));
526	break;
527	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
528	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
529	MLX5_GET(modify_scheduling_element_in,
530	in, scheduling_element_id));
531	break;
532	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
533	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
534	MLX5_GET(add_vxlan_udp_dport_in, in,
535	vxlan_udp_port));
536	break;
537	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
538	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
539	MLX5_GET(query_l2_table_entry_in, in,
540	table_index));
541	break;
542	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
543	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
544	MLX5_GET(set_l2_table_entry_in, in,
545	table_index));
546	break;
547	case MLX5_CMD_OP_QUERY_QP:
548	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
549	MLX5_GET(query_qp_in, in, qpn));
550	break;
551	case MLX5_CMD_OP_RST2INIT_QP:
552	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
553	MLX5_GET(rst2init_qp_in, in, qpn));
554	break;
555	case MLX5_CMD_OP_INIT2INIT_QP:
556	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
557	MLX5_GET(init2init_qp_in, in, qpn));
558	break;
559	case MLX5_CMD_OP_INIT2RTR_QP:
560	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
561	MLX5_GET(init2rtr_qp_in, in, qpn));
562	break;
563	case MLX5_CMD_OP_RTR2RTS_QP:
564	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
565	MLX5_GET(rtr2rts_qp_in, in, qpn));
566	break;
567	case MLX5_CMD_OP_RTS2RTS_QP:
568	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
569	MLX5_GET(rts2rts_qp_in, in, qpn));
570	break;
571	case MLX5_CMD_OP_SQERR2RTS_QP:
572	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
573	MLX5_GET(sqerr2rts_qp_in, in, qpn));
574	break;
575	case MLX5_CMD_OP_2ERR_QP:
576	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
577	MLX5_GET(qp_2err_in, in, qpn));
578	break;
579	case MLX5_CMD_OP_2RST_QP:
580	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
581	MLX5_GET(qp_2rst_in, in, qpn));
582	break;
583	case MLX5_CMD_OP_QUERY_DCT:
584	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_DCT,
585	MLX5_GET(query_dct_in, in, dctn));
586	break;
587	case MLX5_CMD_OP_QUERY_XRQ:
588	case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
589	case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
590	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_XRQ,
591	MLX5_GET(query_xrq_in, in, xrqn));
592	break;
593	case MLX5_CMD_OP_QUERY_XRC_SRQ:
594	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_XRC_SRQ,
595	MLX5_GET(query_xrc_srq_in, in,
596	xrc_srqn));
597	break;
598	case MLX5_CMD_OP_ARM_XRC_SRQ:
599	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_XRC_SRQ,
600	MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
601	break;
602	case MLX5_CMD_OP_QUERY_SRQ:
603	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SRQ,
604	MLX5_GET(query_srq_in, in, srqn));
605	break;
606	case MLX5_CMD_OP_ARM_RQ:
607	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQ,
608	MLX5_GET(arm_rq_in, in, srq_number));
609	break;
610	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
611	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_DCT,
612	MLX5_GET(drain_dct_in, in, dctn));
613	break;
614	case MLX5_CMD_OP_ARM_XRQ:
615	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
616	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
617	case MLX5_CMD_OP_MODIFY_XRQ:
618	obj_id = get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_XRQ,
619	MLX5_GET(arm_xrq_in, in, xrqn));
620	break;
621	case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
622	obj_id = get_enc_obj_id
623	(opcode: MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
624	MLX5_GET(query_packet_reformat_context_in,
625	in, packet_reformat_id));
626	break;
627	default:
628	obj_id = 0;
629	}
630
631	return obj_id;
632	}
633
634	static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
635	struct ib_uobject *uobj, const void *in)
636	{
637	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
638	u64 obj_id = devx_get_obj_id(in);
639
640	if (!obj_id)
641	return false;
642
643	switch (uobj_get_object_id(uobj)) {
644	case UVERBS_OBJECT_CQ:
645	return get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_CQ,
646	obj_id: to_mcq(ibcq: uobj->object)->mcq.cqn) ==
647	obj_id;
648
649	case UVERBS_OBJECT_SRQ:
650	{
651	struct mlx5_core_srq *srq = &(to_msrq(ibsrq: uobj->object)->msrq);
652	u16 opcode;
653
654	switch (srq->common.res) {
655	case MLX5_RES_XSRQ:
656	opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
657	break;
658	case MLX5_RES_XRQ:
659	opcode = MLX5_CMD_OP_CREATE_XRQ;
660	break;
661	default:
662	if (!dev->mdev->issi)
663	opcode = MLX5_CMD_OP_CREATE_SRQ;
664	else
665	opcode = MLX5_CMD_OP_CREATE_RMP;
666	}
667
668	return get_enc_obj_id(opcode,
669	obj_id: to_msrq(ibsrq: uobj->object)->msrq.srqn) ==
670	obj_id;
671	}
672
673	case UVERBS_OBJECT_QP:
674	{
675	struct mlx5_ib_qp *qp = to_mqp(ibqp: uobj->object);
676
677	if (qp->type == IB_QPT_RAW_PACKET ||
678	(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
679	struct mlx5_ib_raw_packet_qp *raw_packet_qp =
680	&qp->raw_packet_qp;
681	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
682	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
683
684	return (get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQ,
685	obj_id: rq->base.mqp.qpn) == obj_id ||
686	get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_SQ,
687	obj_id: sq->base.mqp.qpn) == obj_id ||
688	get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIR,
689	obj_id: rq->tirn) == obj_id ||
690	get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_TIS,
691	obj_id: sq->tisn) == obj_id);
692	}
693
694	if (qp->type == MLX5_IB_QPT_DCT)
695	return get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_DCT,
696	obj_id: qp->dct.mdct.mqp.qpn) == obj_id;
697	return get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_QP,
698	obj_id: qp->ibqp.qp_num) == obj_id;
699	}
700
701	case UVERBS_OBJECT_WQ:
702	return get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQ,
703	obj_id: to_mrwq(ibwq: uobj->object)->core_qp.qpn) ==
704	obj_id;
705
706	case UVERBS_OBJECT_RWQ_IND_TBL:
707	return get_enc_obj_id(opcode: MLX5_CMD_OP_CREATE_RQT,
708	obj_id: to_mrwq_ind_table(ib_rwq_ind_tbl: uobj->object)->rqtn) ==
709	obj_id;
710
711	case MLX5_IB_OBJECT_DEVX_OBJ:
712	{
713	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
714	struct devx_obj *devx_uobj = uobj->object;
715
716	if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &&
717	devx_uobj->flow_counter_bulk_size) {
718	u64 end;
719
720	end = devx_uobj->obj_id +
721	devx_uobj->flow_counter_bulk_size;
722	return devx_uobj->obj_id <= obj_id && end > obj_id;
723	}
724
725	return devx_uobj->obj_id == obj_id;
726	}
727
728	default:
729	return false;
730	}
731	}
732
733	static void devx_set_umem_valid(const void *in)
734	{
735	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
736
737	switch (opcode) {
738	case MLX5_CMD_OP_CREATE_MKEY:
739	MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
740	break;
741	case MLX5_CMD_OP_CREATE_CQ:
742	{
743	void *cqc;
744
745	MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
746	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
747	MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
748	break;
749	}
750	case MLX5_CMD_OP_CREATE_QP:
751	{
752	void *qpc;
753
754	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
755	MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
756	MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
757	break;
758	}
759
760	case MLX5_CMD_OP_CREATE_RQ:
761	{
762	void *rqc, *wq;
763
764	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
765	wq = MLX5_ADDR_OF(rqc, rqc, wq);
766	MLX5_SET(wq, wq, dbr_umem_valid, 1);
767	MLX5_SET(wq, wq, wq_umem_valid, 1);
768	break;
769	}
770
771	case MLX5_CMD_OP_CREATE_SQ:
772	{
773	void *sqc, *wq;
774
775	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
776	wq = MLX5_ADDR_OF(sqc, sqc, wq);
777	MLX5_SET(wq, wq, dbr_umem_valid, 1);
778	MLX5_SET(wq, wq, wq_umem_valid, 1);
779	break;
780	}
781
782	case MLX5_CMD_OP_MODIFY_CQ:
783	MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
784	break;
785
786	case MLX5_CMD_OP_CREATE_RMP:
787	{
788	void *rmpc, *wq;
789
790	rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
791	wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
792	MLX5_SET(wq, wq, dbr_umem_valid, 1);
793	MLX5_SET(wq, wq, wq_umem_valid, 1);
794	break;
795	}
796
797	case MLX5_CMD_OP_CREATE_XRQ:
798	{
799	void *xrqc, *wq;
800
801	xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
802	wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
803	MLX5_SET(wq, wq, dbr_umem_valid, 1);
804	MLX5_SET(wq, wq, wq_umem_valid, 1);
805	break;
806	}
807
808	case MLX5_CMD_OP_CREATE_XRC_SRQ:
809	{
810	void *xrc_srqc;
811
812	MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
813	xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
814	xrc_srq_context_entry);
815	MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
816	break;
817	}
818
819	default:
820	return;
821	}
822	}
823
824	static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
825	{
826	*opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
827
828	switch (*opcode) {
829	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
830	case MLX5_CMD_OP_CREATE_MKEY:
831	case MLX5_CMD_OP_CREATE_CQ:
832	case MLX5_CMD_OP_ALLOC_PD:
833	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
834	case MLX5_CMD_OP_CREATE_RMP:
835	case MLX5_CMD_OP_CREATE_SQ:
836	case MLX5_CMD_OP_CREATE_RQ:
837	case MLX5_CMD_OP_CREATE_RQT:
838	case MLX5_CMD_OP_CREATE_TIR:
839	case MLX5_CMD_OP_CREATE_TIS:
840	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
841	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
842	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
843	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
844	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
845	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
846	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
847	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
848	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
849	case MLX5_CMD_OP_CREATE_QP:
850	case MLX5_CMD_OP_CREATE_SRQ:
851	case MLX5_CMD_OP_CREATE_XRC_SRQ:
852	case MLX5_CMD_OP_CREATE_DCT:
853	case MLX5_CMD_OP_CREATE_XRQ:
854	case MLX5_CMD_OP_ATTACH_TO_MCG:
855	case MLX5_CMD_OP_ALLOC_XRCD:
856	return true;
857	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
858	{
859	u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
860	if (op_mod == 0)
861	return true;
862	return false;
863	}
864	case MLX5_CMD_OP_CREATE_PSV:
865	{
866	u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
867
868	if (num_psv == 1)
869	return true;
870	return false;
871	}
872	default:
873	return false;
874	}
875	}
876
877	static bool devx_is_obj_modify_cmd(const void *in)
878	{
879	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
880
881	switch (opcode) {
882	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
883	case MLX5_CMD_OP_MODIFY_CQ:
884	case MLX5_CMD_OP_MODIFY_RMP:
885	case MLX5_CMD_OP_MODIFY_SQ:
886	case MLX5_CMD_OP_MODIFY_RQ:
887	case MLX5_CMD_OP_MODIFY_RQT:
888	case MLX5_CMD_OP_MODIFY_TIR:
889	case MLX5_CMD_OP_MODIFY_TIS:
890	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
891	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
892	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
893	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
894	case MLX5_CMD_OP_RST2INIT_QP:
895	case MLX5_CMD_OP_INIT2RTR_QP:
896	case MLX5_CMD_OP_INIT2INIT_QP:
897	case MLX5_CMD_OP_RTR2RTS_QP:
898	case MLX5_CMD_OP_RTS2RTS_QP:
899	case MLX5_CMD_OP_SQERR2RTS_QP:
900	case MLX5_CMD_OP_2ERR_QP:
901	case MLX5_CMD_OP_2RST_QP:
902	case MLX5_CMD_OP_ARM_XRC_SRQ:
903	case MLX5_CMD_OP_ARM_RQ:
904	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
905	case MLX5_CMD_OP_ARM_XRQ:
906	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
907	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
908	case MLX5_CMD_OP_MODIFY_XRQ:
909	return true;
910	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
911	{
912	u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
913
914	if (op_mod == 1)
915	return true;
916	return false;
917	}
918	default:
919	return false;
920	}
921	}
922
923	static bool devx_is_obj_query_cmd(const void *in)
924	{
925	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
926
927	switch (opcode) {
928	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
929	case MLX5_CMD_OP_QUERY_MKEY:
930	case MLX5_CMD_OP_QUERY_CQ:
931	case MLX5_CMD_OP_QUERY_RMP:
932	case MLX5_CMD_OP_QUERY_SQ:
933	case MLX5_CMD_OP_QUERY_RQ:
934	case MLX5_CMD_OP_QUERY_RQT:
935	case MLX5_CMD_OP_QUERY_TIR:
936	case MLX5_CMD_OP_QUERY_TIS:
937	case MLX5_CMD_OP_QUERY_Q_COUNTER:
938	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
939	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
940	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
941	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
942	case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
943	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
944	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
945	case MLX5_CMD_OP_QUERY_QP:
946	case MLX5_CMD_OP_QUERY_SRQ:
947	case MLX5_CMD_OP_QUERY_XRC_SRQ:
948	case MLX5_CMD_OP_QUERY_DCT:
949	case MLX5_CMD_OP_QUERY_XRQ:
950	case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
951	case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
952	case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
953	return true;
954	default:
955	return false;
956	}
957	}
958
959	static bool devx_is_whitelist_cmd(void *in)
960	{
961	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
962
963	switch (opcode) {
964	case MLX5_CMD_OP_QUERY_HCA_CAP:
965	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
966	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
967	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
968	return true;
969	default:
970	return false;
971	}
972	}
973
974	static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
975	{
976	if (devx_is_whitelist_cmd(in: cmd_in)) {
977	struct mlx5_ib_dev *dev;
978
979	if (c->devx_uid)
980	return c->devx_uid;
981
982	dev = to_mdev(ibdev: c->ibucontext.device);
983	if (dev->devx_whitelist_uid)
984	return dev->devx_whitelist_uid;
985
986	return -EOPNOTSUPP;
987	}
988
989	if (!c->devx_uid)
990	return -EINVAL;
991
992	return c->devx_uid;
993	}
994
995	static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
996	{
997	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
998
999	/* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
1000	if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
1001	MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
1002	(opcode >= MLX5_CMD_OP_GENERAL_START &&
1003	opcode < MLX5_CMD_OP_GENERAL_END))
1004	return true;
1005
1006	switch (opcode) {
1007	case MLX5_CMD_OP_QUERY_HCA_CAP:
1008	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
1009	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
1010	case MLX5_CMD_OP_QUERY_VPORT_STATE:
1011	case MLX5_CMD_OP_QUERY_ADAPTER:
1012	case MLX5_CMD_OP_QUERY_ISSI:
1013	case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
1014	case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
1015	case MLX5_CMD_OP_QUERY_VNIC_ENV:
1016	case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
1017	case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
1018	case MLX5_CMD_OP_NOP:
1019	case MLX5_CMD_OP_QUERY_CONG_STATUS:
1020	case MLX5_CMD_OP_QUERY_CONG_PARAMS:
1021	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
1022	case MLX5_CMD_OP_QUERY_LAG:
1023	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
1024	return true;
1025	default:
1026	return false;
1027	}
1028	}
1029
1030	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
1031	struct uverbs_attr_bundle *attrs)
1032	{
1033	struct mlx5_ib_ucontext *c;
1034	struct mlx5_ib_dev *dev;
1035	int user_vector;
1036	int dev_eqn;
1037	int err;
1038
1039	if (uverbs_copy_from(&user_vector, attrs,
1040	MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
1041	return -EFAULT;
1042
1043	c = devx_ufile2uctx(attrs);
1044	if (IS_ERR(ptr: c))
1045	return PTR_ERR(ptr: c);
1046	dev = to_mdev(ibdev: c->ibucontext.device);
1047
1048	err = mlx5_comp_eqn_get(dev: dev->mdev, vecidx: user_vector, eqn: &dev_eqn);
1049	if (err < 0)
1050	return err;
1051
1052	if (uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1053	from: &dev_eqn, size: sizeof(dev_eqn)))
1054	return -EFAULT;
1055
1056	return 0;
1057	}
1058
1059	/*
1060	*Security note:
1061	* The hardware protection mechanism works like this: Each device object that
1062	* is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
1063	* the device specification manual) upon its creation. Then upon doorbell,
1064	* hardware fetches the object context for which the doorbell was rang, and
1065	* validates that the UAR through which the DB was rang matches the UAR ID
1066	* of the object.
1067	* If no match the doorbell is silently ignored by the hardware. Of course,
1068	* the user cannot ring a doorbell on a UAR that was not mapped to it.
1069	* Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1070	* mailboxes (except tagging them with UID), we expose to the user its UAR
1071	* ID, so it can embed it in these objects in the expected specification
1072	* format. So the only thing the user can do is hurt itself by creating a
1073	* QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1074	* may ring a doorbell on its objects.
1075	* The consequence of that will be that another user can schedule a QP/SQ
1076	* of the buggy user for execution (just insert it to the hardware schedule
1077	* queue or arm its CQ for event generation), no further harm is expected.
1078	*/
1079	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1080	struct uverbs_attr_bundle *attrs)
1081	{
1082	struct mlx5_ib_ucontext *c;
1083	struct mlx5_ib_dev *dev;
1084	u32 user_idx;
1085	s32 dev_idx;
1086
1087	c = devx_ufile2uctx(attrs);
1088	if (IS_ERR(ptr: c))
1089	return PTR_ERR(ptr: c);
1090	dev = to_mdev(ibdev: c->ibucontext.device);
1091
1092	if (uverbs_copy_from(&user_idx, attrs,
1093	MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1094	return -EFAULT;
1095
1096	dev_idx = bfregn_to_uar_index(dev, bfregi: &c->bfregi, bfregn: user_idx, dyn_bfreg: true);
1097	if (dev_idx < 0)
1098	return dev_idx;
1099
1100	if (uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1101	from: &dev_idx, size: sizeof(dev_idx)))
1102	return -EFAULT;
1103
1104	return 0;
1105	}
1106
1107	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1108	struct uverbs_attr_bundle *attrs)
1109	{
1110	struct mlx5_ib_ucontext *c;
1111	struct mlx5_ib_dev *dev;
1112	void *cmd_in = uverbs_attr_get_alloced_ptr(
1113	attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1114	int cmd_out_len = uverbs_attr_get_len(attrs_bundle: attrs,
1115	idx: MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1116	void *cmd_out;
1117	int err, err2;
1118	int uid;
1119
1120	c = devx_ufile2uctx(attrs);
1121	if (IS_ERR(ptr: c))
1122	return PTR_ERR(ptr: c);
1123	dev = to_mdev(ibdev: c->ibucontext.device);
1124
1125	uid = devx_get_uid(c, cmd_in);
1126	if (uid < 0)
1127	return uid;
1128
1129	/* Only white list of some general HCA commands are allowed for this method. */
1130	if (!devx_is_general_cmd(in: cmd_in, dev))
1131	return -EINVAL;
1132
1133	cmd_out = uverbs_zalloc(bundle: attrs, size: cmd_out_len);
1134	if (IS_ERR(ptr: cmd_out))
1135	return PTR_ERR(ptr: cmd_out);
1136
1137	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1138	err = mlx5_cmd_do(dev: dev->mdev, in: cmd_in,
1139	in_size: uverbs_attr_get_len(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1140	out: cmd_out, out_size: cmd_out_len);
1141	if (err && err != -EREMOTEIO)
1142	return err;
1143
1144	err2 = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, from: cmd_out,
1145	size: cmd_out_len);
1146
1147	return err2 ?: err;
1148	}
1149
1150	static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1151	u32 *dinlen,
1152	u32 *obj_id)
1153	{
1154	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
1155	u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1156
1157	*obj_id = devx_get_created_obj_id(in, out, opcode);
1158	*dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1159	MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1160
1161	switch (opcode) {
1162	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1163	MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1164	MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1165	MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,
1166	MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));
1167	break;
1168
1169	case MLX5_CMD_OP_CREATE_UMEM:
1170	MLX5_SET(destroy_umem_in, din, opcode,
1171	MLX5_CMD_OP_DESTROY_UMEM);
1172	MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);
1173	break;
1174	case MLX5_CMD_OP_CREATE_MKEY:
1175	MLX5_SET(destroy_mkey_in, din, opcode,
1176	MLX5_CMD_OP_DESTROY_MKEY);
1177	MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);
1178	break;
1179	case MLX5_CMD_OP_CREATE_CQ:
1180	MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1181	MLX5_SET(destroy_cq_in, din, cqn, *obj_id);
1182	break;
1183	case MLX5_CMD_OP_ALLOC_PD:
1184	MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1185	MLX5_SET(dealloc_pd_in, din, pd, *obj_id);
1186	break;
1187	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1188	MLX5_SET(dealloc_transport_domain_in, din, opcode,
1189	MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1190	MLX5_SET(dealloc_transport_domain_in, din, transport_domain,
1191	*obj_id);
1192	break;
1193	case MLX5_CMD_OP_CREATE_RMP:
1194	MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1195	MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);
1196	break;
1197	case MLX5_CMD_OP_CREATE_SQ:
1198	MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1199	MLX5_SET(destroy_sq_in, din, sqn, *obj_id);
1200	break;
1201	case MLX5_CMD_OP_CREATE_RQ:
1202	MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1203	MLX5_SET(destroy_rq_in, din, rqn, *obj_id);
1204	break;
1205	case MLX5_CMD_OP_CREATE_RQT:
1206	MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1207	MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);
1208	break;
1209	case MLX5_CMD_OP_CREATE_TIR:
1210	MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1211	MLX5_SET(destroy_tir_in, din, tirn, *obj_id);
1212	break;
1213	case MLX5_CMD_OP_CREATE_TIS:
1214	MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1215	MLX5_SET(destroy_tis_in, din, tisn, *obj_id);
1216	break;
1217	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1218	MLX5_SET(dealloc_q_counter_in, din, opcode,
1219	MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1220	MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);
1221	break;
1222	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1223	*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1224	MLX5_SET(destroy_flow_table_in, din, other_vport,
1225	MLX5_GET(create_flow_table_in, in, other_vport));
1226	MLX5_SET(destroy_flow_table_in, din, vport_number,
1227	MLX5_GET(create_flow_table_in, in, vport_number));
1228	MLX5_SET(destroy_flow_table_in, din, other_eswitch,
1229	MLX5_GET(create_flow_table_in, in, other_eswitch));
1230	MLX5_SET(destroy_flow_table_in, din, eswitch_owner_vhca_id,
1231	MLX5_GET(create_flow_table_in, in,
1232	eswitch_owner_vhca_id));
1233	MLX5_SET(destroy_flow_table_in, din, table_type,
1234	MLX5_GET(create_flow_table_in, in, table_type));
1235	MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1236	MLX5_SET(destroy_flow_table_in, din, opcode,
1237	MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1238	break;
1239	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1240	*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1241	MLX5_SET(destroy_flow_group_in, din, other_vport,
1242	MLX5_GET(create_flow_group_in, in, other_vport));
1243	MLX5_SET(destroy_flow_group_in, din, vport_number,
1244	MLX5_GET(create_flow_group_in, in, vport_number));
1245	MLX5_SET(destroy_flow_group_in, din, other_eswitch,
1246	MLX5_GET(create_flow_group_in, in, other_eswitch));
1247	MLX5_SET(destroy_flow_group_in, din, eswitch_owner_vhca_id,
1248	MLX5_GET(create_flow_group_in, in,
1249	eswitch_owner_vhca_id));
1250	MLX5_SET(destroy_flow_group_in, din, table_type,
1251	MLX5_GET(create_flow_group_in, in, table_type));
1252	MLX5_SET(destroy_flow_group_in, din, table_id,
1253	MLX5_GET(create_flow_group_in, in, table_id));
1254	MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1255	MLX5_SET(destroy_flow_group_in, din, opcode,
1256	MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1257	break;
1258	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1259	*dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1260	MLX5_SET(delete_fte_in, din, other_vport,
1261	MLX5_GET(set_fte_in, in, other_vport));
1262	MLX5_SET(delete_fte_in, din, vport_number,
1263	MLX5_GET(set_fte_in, in, vport_number));
1264	MLX5_SET(delete_fte_in, din, other_eswitch,
1265	MLX5_GET(set_fte_in, in, other_eswitch));
1266	MLX5_SET(delete_fte_in, din, eswitch_owner_vhca_id,
1267	MLX5_GET(set_fte_in, in, eswitch_owner_vhca_id));
1268	MLX5_SET(delete_fte_in, din, table_type,
1269	MLX5_GET(set_fte_in, in, table_type));
1270	MLX5_SET(delete_fte_in, din, table_id,
1271	MLX5_GET(set_fte_in, in, table_id));
1272	MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1273	MLX5_SET(delete_fte_in, din, opcode,
1274	MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1275	break;
1276	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1277	MLX5_SET(dealloc_flow_counter_in, din, opcode,
1278	MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1279	MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,
1280	*obj_id);
1281	break;
1282	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1283	MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,
1284	MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1285	MLX5_SET(dealloc_packet_reformat_context_in, din,
1286	packet_reformat_id, *obj_id);
1287	break;
1288	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1289	MLX5_SET(dealloc_modify_header_context_in, din, opcode,
1290	MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1291	MLX5_SET(dealloc_modify_header_context_in, din,
1292	modify_header_id, *obj_id);
1293	break;
1294	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1295	*dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1296	MLX5_SET(destroy_scheduling_element_in, din,
1297	scheduling_hierarchy,
1298	MLX5_GET(create_scheduling_element_in, in,
1299	scheduling_hierarchy));
1300	MLX5_SET(destroy_scheduling_element_in, din,
1301	scheduling_element_id, *obj_id);
1302	MLX5_SET(destroy_scheduling_element_in, din, opcode,
1303	MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1304	break;
1305	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1306	*dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1307	MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1308	MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,
1309	MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1310	break;
1311	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1312	*dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1313	MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1314	MLX5_SET(delete_l2_table_entry_in, din, opcode,
1315	MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1316	break;
1317	case MLX5_CMD_OP_CREATE_QP:
1318	MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1319	MLX5_SET(destroy_qp_in, din, qpn, *obj_id);
1320	break;
1321	case MLX5_CMD_OP_CREATE_SRQ:
1322	MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1323	MLX5_SET(destroy_srq_in, din, srqn, *obj_id);
1324	break;
1325	case MLX5_CMD_OP_CREATE_XRC_SRQ:
1326	MLX5_SET(destroy_xrc_srq_in, din, opcode,
1327	MLX5_CMD_OP_DESTROY_XRC_SRQ);
1328	MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);
1329	break;
1330	case MLX5_CMD_OP_CREATE_DCT:
1331	MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1332	MLX5_SET(destroy_dct_in, din, dctn, *obj_id);
1333	break;
1334	case MLX5_CMD_OP_CREATE_XRQ:
1335	MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1336	MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);
1337	break;
1338	case MLX5_CMD_OP_ATTACH_TO_MCG:
1339	*dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1340	MLX5_SET(detach_from_mcg_in, din, qpn,
1341	MLX5_GET(attach_to_mcg_in, in, qpn));
1342	memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1343	MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1344	MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1345	MLX5_SET(detach_from_mcg_in, din, opcode,
1346	MLX5_CMD_OP_DETACH_FROM_MCG);
1347	MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);
1348	break;
1349	case MLX5_CMD_OP_ALLOC_XRCD:
1350	MLX5_SET(dealloc_xrcd_in, din, opcode,
1351	MLX5_CMD_OP_DEALLOC_XRCD);
1352	MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);
1353	break;
1354	case MLX5_CMD_OP_CREATE_PSV:
1355	MLX5_SET(destroy_psv_in, din, opcode,
1356	MLX5_CMD_OP_DESTROY_PSV);
1357	MLX5_SET(destroy_psv_in, din, psvn, *obj_id);
1358	break;
1359	default:
1360	/* The entry must match to one of the devx_is_obj_create_cmd */
1361	WARN_ON(true);
1362	break;
1363	}
1364	}
1365
1366	static int devx_handle_mkey_indirect(struct devx_obj *obj,
1367	struct mlx5_ib_dev *dev,
1368	void *in, void *out)
1369	{
1370	struct mlx5_ib_mkey *mkey = &obj->mkey;
1371	void *mkc;
1372	u8 key;
1373
1374	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1375	key = MLX5_GET(mkc, mkc, mkey_7_0);
1376	mkey->key = mlx5_idx_to_mkey(
1377	MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1378	mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1379	mkey->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1380	init_waitqueue_head(&mkey->wait);
1381
1382	return mlx5r_store_odp_mkey(dev, mmkey: mkey);
1383	}
1384
1385	static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1386	struct devx_obj *obj,
1387	void *in, int in_len)
1388	{
1389	int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1390	MLX5_FLD_SZ_BYTES(create_mkey_in,
1391	memory_key_mkey_entry);
1392	void *mkc;
1393	u8 access_mode;
1394
1395	if (in_len < min_len)
1396	return -EINVAL;
1397
1398	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1399
1400	access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1401	access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1402
1403	if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1404	access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1405	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1406	obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1407	return 0;
1408	}
1409
1410	MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1411	/* TPH is not allowed to bypass the regular kernel's verbs flow */
1412	MLX5_SET(mkc, mkc, pcie_tph_en, 0);
1413	MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
1414	MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX);
1415	return 0;
1416	}
1417
1418	static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1419	struct devx_event_subscription *sub)
1420	{
1421	struct devx_event *event;
1422	struct devx_obj_event *xa_val_level2;
1423
1424	if (sub->is_cleaned)
1425	return;
1426
1427	sub->is_cleaned = 1;
1428	list_del_rcu(entry: &sub->xa_list);
1429
1430	if (list_empty(head: &sub->obj_list))
1431	return;
1432
1433	list_del_rcu(entry: &sub->obj_list);
1434	/* check whether key level 1 for this obj_sub_list is empty */
1435	event = xa_load(&dev->devx_event_table.event_xa,
1436	index: sub->xa_key_level1);
1437	WARN_ON(!event);
1438
1439	xa_val_level2 = xa_load(&event->object_ids, index: sub->xa_key_level2);
1440	if (list_empty(head: &xa_val_level2->obj_sub_list)) {
1441	xa_erase(&event->object_ids,
1442	index: sub->xa_key_level2);
1443	kfree_rcu(xa_val_level2, rcu);
1444	}
1445	}
1446
1447	static int devx_obj_cleanup(struct ib_uobject *uobject,
1448	enum rdma_remove_reason why,
1449	struct uverbs_attr_bundle *attrs)
1450	{
1451	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1452	struct mlx5_devx_event_table *devx_event_table;
1453	struct devx_obj *obj = uobject->object;
1454	struct devx_event_subscription *sub_entry, *tmp;
1455	struct mlx5_ib_dev *dev;
1456	int ret;
1457
1458	dev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
1459	if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
1460	xa_erase(&obj->ib_dev->odp_mkeys,
1461	index: mlx5_base_mkey(key: obj->mkey.key)))
1462	/*
1463	* The pagefault_single_data_segment() does commands against
1464	* the mmkey, we must wait for that to stop before freeing the
1465	* mkey, as another allocation could get the same mkey #.
1466	*/
1467	mlx5r_deref_wait_odp_mkey(mmkey: &obj->mkey);
1468
1469	if (obj->flags & DEVX_OBJ_FLAGS_HW_FREED)
1470	ret = 0;
1471	else if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1472	ret = mlx5_core_destroy_dct(dev: obj->ib_dev, dct: &obj->core_dct);
1473	else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1474	ret = mlx5_core_destroy_cq(dev: obj->ib_dev->mdev, cq: &obj->core_cq);
1475	else
1476	ret = mlx5_cmd_exec(dev: obj->ib_dev->mdev, in: obj->dinbox,
1477	in_size: obj->dinlen, out, out_size: sizeof(out));
1478	if (ret)
1479	return ret;
1480
1481	devx_event_table = &dev->devx_event_table;
1482
1483	mutex_lock(&devx_event_table->event_xa_lock);
1484	list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1485	devx_cleanup_subscription(dev, sub: sub_entry);
1486	mutex_unlock(lock: &devx_event_table->event_xa_lock);
1487
1488	kfree(objp: obj);
1489	return ret;
1490	}
1491
1492	static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1493	{
1494	struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1495	struct mlx5_devx_event_table *table;
1496	struct devx_event *event;
1497	struct devx_obj_event *obj_event;
1498	u32 obj_id = mcq->cqn;
1499
1500	table = &obj->ib_dev->devx_event_table;
1501	rcu_read_lock();
1502	event = xa_load(&table->event_xa, index: MLX5_EVENT_TYPE_COMP);
1503	if (!event)
1504	goto out;
1505
1506	obj_event = xa_load(&event->object_ids, index: obj_id);
1507	if (!obj_event)
1508	goto out;
1509
1510	dispatch_event_fd(fd_list: &obj_event->obj_sub_list, data: eqe);
1511	out:
1512	rcu_read_unlock();
1513	}
1514
1515	static bool is_apu_cq(struct mlx5_ib_dev *dev, const void *in)
1516	{
1517	if (!MLX5_CAP_GEN(dev->mdev, apu) ||
1518	!MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), apu_cq))
1519	return false;
1520
1521	return true;
1522	}
1523
1524	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1525	struct uverbs_attr_bundle *attrs)
1526	{
1527	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1528	int cmd_out_len = uverbs_attr_get_len(attrs_bundle: attrs,
1529	idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1530	int cmd_in_len = uverbs_attr_get_len(attrs_bundle: attrs,
1531	idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1532	void *cmd_out;
1533	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1534	attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1535	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1536	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1537	struct mlx5_ib_dev *dev = to_mdev(ibdev: c->ibucontext.device);
1538	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1539	struct devx_obj *obj;
1540	u16 obj_type = 0;
1541	int err, err2 = 0;
1542	int uid;
1543	u32 obj_id;
1544	u16 opcode;
1545
1546	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1547	return -EINVAL;
1548
1549	uid = devx_get_uid(c, cmd_in);
1550	if (uid < 0)
1551	return uid;
1552
1553	if (!devx_is_obj_create_cmd(in: cmd_in, opcode: &opcode))
1554	return -EINVAL;
1555
1556	cmd_out = uverbs_zalloc(bundle: attrs, size: cmd_out_len);
1557	if (IS_ERR(ptr: cmd_out))
1558	return PTR_ERR(ptr: cmd_out);
1559
1560	obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1561	if (!obj)
1562	return -ENOMEM;
1563
1564	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1565	if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1566	err = devx_handle_mkey_create(dev, obj, in: cmd_in, in_len: cmd_in_len);
1567	if (err)
1568	goto obj_free;
1569	} else {
1570	devx_set_umem_valid(in: cmd_in);
1571	}
1572
1573	if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1574	obj->flags |= DEVX_OBJ_FLAGS_DCT;
1575	err = mlx5_core_create_dct(dev, qp: &obj->core_dct, in: cmd_in,
1576	inlen: cmd_in_len, out: cmd_out, outlen: cmd_out_len);
1577	} else if (opcode == MLX5_CMD_OP_CREATE_CQ &&
1578	!is_apu_cq(dev, in: cmd_in)) {
1579	obj->flags |= DEVX_OBJ_FLAGS_CQ;
1580	obj->core_cq.comp = devx_cq_comp;
1581	err = mlx5_create_cq(dev: dev->mdev, cq: &obj->core_cq,
1582	in: cmd_in, inlen: cmd_in_len, out: cmd_out,
1583	outlen: cmd_out_len);
1584	} else {
1585	err = mlx5_cmd_do(dev: dev->mdev, in: cmd_in, in_size: cmd_in_len,
1586	out: cmd_out, out_size: cmd_out_len);
1587	}
1588
1589	if (err == -EREMOTEIO)
1590	err2 = uverbs_copy_to(attrs_bundle: attrs,
1591	idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1592	from: cmd_out, size: cmd_out_len);
1593	if (err)
1594	goto obj_free;
1595
1596	if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1597	u32 bulk = MLX5_GET(alloc_flow_counter_in,
1598	cmd_in,
1599	flow_counter_bulk_log_size);
1600
1601	if (bulk)
1602	bulk = 1 << bulk;
1603	else
1604	bulk = 128UL * MLX5_GET(alloc_flow_counter_in,
1605	cmd_in,
1606	flow_counter_bulk);
1607	obj->flow_counter_bulk_size = bulk;
1608	}
1609
1610	uobj->object = obj;
1611	INIT_LIST_HEAD(list: &obj->event_sub);
1612	obj->ib_dev = dev;
1613	devx_obj_build_destroy_cmd(in: cmd_in, out: cmd_out, din: obj->dinbox, dinlen: &obj->dinlen,
1614	obj_id: &obj_id);
1615	WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1616
1617	err = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, from: cmd_out, size: cmd_out_len);
1618	if (err)
1619	goto obj_destroy;
1620
1621	if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1622	obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1623	obj->obj_id = get_enc_obj_id(opcode: opcode | obj_type << 16, obj_id);
1624
1625	if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1626	err = devx_handle_mkey_indirect(obj, dev, in: cmd_in, out: cmd_out);
1627	if (err)
1628	goto obj_destroy;
1629	}
1630	return 0;
1631
1632	obj_destroy:
1633	if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1634	mlx5_core_destroy_dct(dev: obj->ib_dev, dct: &obj->core_dct);
1635	else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1636	mlx5_core_destroy_cq(dev: obj->ib_dev->mdev, cq: &obj->core_cq);
1637	else
1638	mlx5_cmd_exec(dev: obj->ib_dev->mdev, in: obj->dinbox, in_size: obj->dinlen, out,
1639	out_size: sizeof(out));
1640	obj_free:
1641	kfree(objp: obj);
1642	return err2 ?: err;
1643	}
1644
1645	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1646	struct uverbs_attr_bundle *attrs)
1647	{
1648	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1649	int cmd_out_len = uverbs_attr_get_len(attrs_bundle: attrs,
1650	idx: MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1651	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs_bundle: attrs,
1652	idx: MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1653	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1654	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1655	struct mlx5_ib_dev *mdev = to_mdev(ibdev: c->ibucontext.device);
1656	void *cmd_out;
1657	int err, err2;
1658	int uid;
1659
1660	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1661	return -EINVAL;
1662
1663	uid = devx_get_uid(c, cmd_in);
1664	if (uid < 0)
1665	return uid;
1666
1667	if (!devx_is_obj_modify_cmd(in: cmd_in))
1668	return -EINVAL;
1669
1670	if (!devx_is_valid_obj_id(attrs, uobj, in: cmd_in))
1671	return -EINVAL;
1672
1673	cmd_out = uverbs_zalloc(bundle: attrs, size: cmd_out_len);
1674	if (IS_ERR(ptr: cmd_out))
1675	return PTR_ERR(ptr: cmd_out);
1676
1677	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1678	devx_set_umem_valid(in: cmd_in);
1679
1680	err = mlx5_cmd_do(dev: mdev->mdev, in: cmd_in,
1681	in_size: uverbs_attr_get_len(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1682	out: cmd_out, out_size: cmd_out_len);
1683	if (err && err != -EREMOTEIO)
1684	return err;
1685
1686	err2 = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1687	from: cmd_out, size: cmd_out_len);
1688
1689	return err2 ?: err;
1690	}
1691
1692	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1693	struct uverbs_attr_bundle *attrs)
1694	{
1695	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1696	int cmd_out_len = uverbs_attr_get_len(attrs_bundle: attrs,
1697	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1698	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs_bundle: attrs,
1699	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1700	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1701	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1702	void *cmd_out;
1703	int err, err2;
1704	int uid;
1705	struct mlx5_ib_dev *mdev = to_mdev(ibdev: c->ibucontext.device);
1706
1707	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1708	return -EINVAL;
1709
1710	uid = devx_get_uid(c, cmd_in);
1711	if (uid < 0)
1712	return uid;
1713
1714	if (!devx_is_obj_query_cmd(in: cmd_in))
1715	return -EINVAL;
1716
1717	if (!devx_is_valid_obj_id(attrs, uobj, in: cmd_in))
1718	return -EINVAL;
1719
1720	cmd_out = uverbs_zalloc(bundle: attrs, size: cmd_out_len);
1721	if (IS_ERR(ptr: cmd_out))
1722	return PTR_ERR(ptr: cmd_out);
1723
1724	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1725	err = mlx5_cmd_do(dev: mdev->mdev, in: cmd_in,
1726	in_size: uverbs_attr_get_len(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1727	out: cmd_out, out_size: cmd_out_len);
1728	if (err && err != -EREMOTEIO)
1729	return err;
1730
1731	err2 = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1732	from: cmd_out, size: cmd_out_len);
1733
1734	return err2 ?: err;
1735	}
1736
1737	struct devx_async_event_queue {
1738	spinlock_t lock;
1739	wait_queue_head_t poll_wait;
1740	struct list_head event_list;
1741	atomic_t bytes_in_use;
1742	u8 is_destroyed:1;
1743	};
1744
1745	struct devx_async_cmd_event_file {
1746	struct ib_uobject uobj;
1747	struct devx_async_event_queue ev_queue;
1748	struct mlx5_async_ctx async_ctx;
1749	};
1750
1751	static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1752	{
1753	spin_lock_init(&ev_queue->lock);
1754	INIT_LIST_HEAD(list: &ev_queue->event_list);
1755	init_waitqueue_head(&ev_queue->poll_wait);
1756	atomic_set(v: &ev_queue->bytes_in_use, i: 0);
1757	ev_queue->is_destroyed = 0;
1758	}
1759
1760	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1761	struct uverbs_attr_bundle *attrs)
1762	{
1763	struct devx_async_cmd_event_file *ev_file;
1764
1765	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1766	attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1767	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(udata: &attrs->driver_udata);
1768
1769	ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1770	uobj);
1771	devx_init_event_queue(ev_queue: &ev_file->ev_queue);
1772	mlx5_cmd_init_async_ctx(dev: mdev->mdev, ctx: &ev_file->async_ctx);
1773	return 0;
1774	}
1775
1776	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1777	struct uverbs_attr_bundle *attrs)
1778	{
1779	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1780	attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1781	struct devx_async_event_file *ev_file;
1782	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1783	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1784	struct mlx5_ib_dev *dev = to_mdev(ibdev: c->ibucontext.device);
1785	u32 flags;
1786	int err;
1787
1788	err = uverbs_get_flags32(to: &flags, attrs_bundle: attrs,
1789	idx: MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1790	allowed_bits: MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1791
1792	if (err)
1793	return err;
1794
1795	ev_file = container_of(uobj, struct devx_async_event_file,
1796	uobj);
1797	spin_lock_init(&ev_file->lock);
1798	INIT_LIST_HEAD(list: &ev_file->event_list);
1799	init_waitqueue_head(&ev_file->poll_wait);
1800	if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1801	ev_file->omit_data = 1;
1802	INIT_LIST_HEAD(list: &ev_file->subscribed_events_list);
1803	ev_file->dev = dev;
1804	get_device(dev: &dev->ib_dev.dev);
1805	return 0;
1806	}
1807
1808	static void devx_query_callback(int status, struct mlx5_async_work *context)
1809	{
1810	struct devx_async_data *async_data =
1811	container_of(context, struct devx_async_data, cb_work);
1812	struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1813	struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1814	unsigned long flags;
1815
1816	/*
1817	* Note that if the struct devx_async_cmd_event_file uobj begins to be
1818	* destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1819	* routine returns, ensuring that it always remains valid here.
1820	*/
1821	spin_lock_irqsave(&ev_queue->lock, flags);
1822	list_add_tail(new: &async_data->list, head: &ev_queue->event_list);
1823	spin_unlock_irqrestore(lock: &ev_queue->lock, flags);
1824
1825	wake_up_interruptible(&ev_queue->poll_wait);
1826	}
1827
1828	#define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1829
1830	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1831	struct uverbs_attr_bundle *attrs)
1832	{
1833	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs,
1834	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1835	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1836	attrs_bundle: attrs,
1837	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1838	u16 cmd_out_len;
1839	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1840	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1841	struct ib_uobject *fd_uobj;
1842	int err;
1843	int uid;
1844	struct mlx5_ib_dev *mdev = to_mdev(ibdev: c->ibucontext.device);
1845	struct devx_async_cmd_event_file *ev_file;
1846	struct devx_async_data *async_data;
1847
1848	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1849	return -EINVAL;
1850
1851	uid = devx_get_uid(c, cmd_in);
1852	if (uid < 0)
1853	return uid;
1854
1855	if (!devx_is_obj_query_cmd(in: cmd_in))
1856	return -EINVAL;
1857
1858	err = uverbs_get_const(&cmd_out_len, attrs,
1859	MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1860	if (err)
1861	return err;
1862
1863	if (!devx_is_valid_obj_id(attrs, uobj, in: cmd_in))
1864	return -EINVAL;
1865
1866	fd_uobj = uverbs_attr_get_uobject(attrs_bundle: attrs,
1867	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1868	if (IS_ERR(ptr: fd_uobj))
1869	return PTR_ERR(ptr: fd_uobj);
1870
1871	ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1872	uobj);
1873
1874	if (atomic_add_return(i: cmd_out_len, v: &ev_file->ev_queue.bytes_in_use) >
1875	MAX_ASYNC_BYTES_IN_USE) {
1876	atomic_sub(i: cmd_out_len, v: &ev_file->ev_queue.bytes_in_use);
1877	return -EAGAIN;
1878	}
1879
1880	async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1881	cmd_out_len), GFP_KERNEL);
1882	if (!async_data) {
1883	err = -ENOMEM;
1884	goto sub_bytes;
1885	}
1886
1887	err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1888	MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1889	if (err)
1890	goto free_async;
1891
1892	async_data->cmd_out_len = cmd_out_len;
1893	async_data->mdev = mdev;
1894	async_data->ev_file = ev_file;
1895
1896	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1897	err = mlx5_cmd_exec_cb(ctx: &ev_file->async_ctx, in: cmd_in,
1898	in_size: uverbs_attr_get_len(attrs_bundle: attrs,
1899	idx: MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1900	out: async_data->hdr.out_data,
1901	out_size: async_data->cmd_out_len,
1902	callback: devx_query_callback, work: &async_data->cb_work);
1903
1904	if (err)
1905	goto free_async;
1906
1907	return 0;
1908
1909	free_async:
1910	kvfree(addr: async_data);
1911	sub_bytes:
1912	atomic_sub(i: cmd_out_len, v: &ev_file->ev_queue.bytes_in_use);
1913	return err;
1914	}
1915
1916	static void
1917	subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1918	u32 key_level1,
1919	bool is_level2,
1920	u32 key_level2)
1921	{
1922	struct devx_event *event;
1923	struct devx_obj_event *xa_val_level2;
1924
1925	/* Level 1 is valid for future use, no need to free */
1926	if (!is_level2)
1927	return;
1928
1929	event = xa_load(&devx_event_table->event_xa, index: key_level1);
1930	WARN_ON(!event);
1931
1932	xa_val_level2 = xa_load(&event->object_ids,
1933	index: key_level2);
1934	if (list_empty(head: &xa_val_level2->obj_sub_list)) {
1935	xa_erase(&event->object_ids,
1936	index: key_level2);
1937	kfree_rcu(xa_val_level2, rcu);
1938	}
1939	}
1940
1941	static int
1942	subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1943	u32 key_level1,
1944	bool is_level2,
1945	u32 key_level2)
1946	{
1947	struct devx_obj_event *obj_event;
1948	struct devx_event *event;
1949	int err;
1950
1951	event = xa_load(&devx_event_table->event_xa, index: key_level1);
1952	if (!event) {
1953	event = kzalloc(sizeof(*event), GFP_KERNEL);
1954	if (!event)
1955	return -ENOMEM;
1956
1957	INIT_LIST_HEAD(list: &event->unaffiliated_list);
1958	xa_init(xa: &event->object_ids);
1959
1960	err = xa_insert(xa: &devx_event_table->event_xa,
1961	index: key_level1,
1962	entry: event,
1963	GFP_KERNEL);
1964	if (err) {
1965	kfree(objp: event);
1966	return err;
1967	}
1968	}
1969
1970	if (!is_level2)
1971	return 0;
1972
1973	obj_event = xa_load(&event->object_ids, index: key_level2);
1974	if (!obj_event) {
1975	obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1976	if (!obj_event)
1977	/* Level1 is valid for future use, no need to free */
1978	return -ENOMEM;
1979
1980	INIT_LIST_HEAD(list: &obj_event->obj_sub_list);
1981	err = xa_insert(xa: &event->object_ids,
1982	index: key_level2,
1983	entry: obj_event,
1984	GFP_KERNEL);
1985	if (err) {
1986	kfree(objp: obj_event);
1987	return err;
1988	}
1989	}
1990
1991	return 0;
1992	}
1993
1994	static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1995	struct devx_obj *obj)
1996	{
1997	int i;
1998
1999	for (i = 0; i < num_events; i++) {
2000	if (obj) {
2001	if (!is_legacy_obj_event_num(event_num: event_type_num_list[i]))
2002	return false;
2003	} else if (!is_legacy_unaffiliated_event_num(
2004	event_num: event_type_num_list[i])) {
2005	return false;
2006	}
2007	}
2008
2009	return true;
2010	}
2011
2012	#define MAX_SUPP_EVENT_NUM 255
2013	static bool is_valid_events(struct mlx5_core_dev *dev,
2014	int num_events, u16 *event_type_num_list,
2015	struct devx_obj *obj)
2016	{
2017	__be64 *aff_events;
2018	__be64 *unaff_events;
2019	int mask_entry;
2020	int mask_bit;
2021	int i;
2022
2023	if (MLX5_CAP_GEN(dev, event_cap)) {
2024	aff_events = MLX5_CAP_DEV_EVENT(dev,
2025	user_affiliated_events);
2026	unaff_events = MLX5_CAP_DEV_EVENT(dev,
2027	user_unaffiliated_events);
2028	} else {
2029	return is_valid_events_legacy(num_events, event_type_num_list,
2030	obj);
2031	}
2032
2033	for (i = 0; i < num_events; i++) {
2034	if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
2035	return false;
2036
2037	mask_entry = event_type_num_list[i] / 64;
2038	mask_bit = event_type_num_list[i] % 64;
2039
2040	if (obj) {
2041	/* CQ completion */
2042	if (event_type_num_list[i] == 0)
2043	continue;
2044
2045	if (!(be64_to_cpu(aff_events[mask_entry]) &
2046	(1ull << mask_bit)))
2047	return false;
2048
2049	continue;
2050	}
2051
2052	if (!(be64_to_cpu(unaff_events[mask_entry]) &
2053	(1ull << mask_bit)))
2054	return false;
2055	}
2056
2057	return true;
2058	}
2059
2060	#define MAX_NUM_EVENTS 16
2061	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
2062	struct uverbs_attr_bundle *attrs)
2063	{
2064	struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
2065	attrs_bundle: attrs,
2066	idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
2067	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2068	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2069	struct mlx5_ib_dev *dev = to_mdev(ibdev: c->ibucontext.device);
2070	struct ib_uobject *fd_uobj;
2071	struct devx_obj *obj = NULL;
2072	struct devx_async_event_file *ev_file;
2073	struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
2074	u16 *event_type_num_list;
2075	struct devx_event_subscription *event_sub, *tmp_sub;
2076	struct list_head sub_list;
2077	int redirect_fd;
2078	bool use_eventfd = false;
2079	int num_events;
2080	u16 obj_type = 0;
2081	u64 cookie = 0;
2082	u32 obj_id = 0;
2083	int err;
2084	int i;
2085
2086	if (!c->devx_uid)
2087	return -EINVAL;
2088
2089	if (!IS_ERR(ptr: devx_uobj)) {
2090	obj = (struct devx_obj *)devx_uobj->object;
2091	if (obj)
2092	obj_id = get_dec_obj_id(obj_id: obj->obj_id);
2093	}
2094
2095	fd_uobj = uverbs_attr_get_uobject(attrs_bundle: attrs,
2096	idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
2097	if (IS_ERR(ptr: fd_uobj))
2098	return PTR_ERR(ptr: fd_uobj);
2099
2100	ev_file = container_of(fd_uobj, struct devx_async_event_file,
2101	uobj);
2102
2103	if (uverbs_attr_is_valid(attrs_bundle: attrs,
2104	idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
2105	err = uverbs_copy_from(&redirect_fd, attrs,
2106	MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
2107	if (err)
2108	return err;
2109
2110	use_eventfd = true;
2111	}
2112
2113	if (uverbs_attr_is_valid(attrs_bundle: attrs,
2114	idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
2115	if (use_eventfd)
2116	return -EINVAL;
2117
2118	err = uverbs_copy_from(&cookie, attrs,
2119	MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
2120	if (err)
2121	return err;
2122	}
2123
2124	num_events = uverbs_attr_ptr_get_array_size(
2125	attrs, idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2126	elem_size: sizeof(u16));
2127
2128	if (num_events < 0)
2129	return num_events;
2130
2131	if (num_events > MAX_NUM_EVENTS)
2132	return -EINVAL;
2133
2134	event_type_num_list = uverbs_attr_get_alloced_ptr(attrs_bundle: attrs,
2135	idx: MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2136
2137	if (!is_valid_events(dev: dev->mdev, num_events, event_type_num_list, obj))
2138	return -EINVAL;
2139
2140	INIT_LIST_HEAD(list: &sub_list);
2141
2142	/* Protect from concurrent subscriptions to same XA entries to allow
2143	* both to succeed
2144	*/
2145	mutex_lock(&devx_event_table->event_xa_lock);
2146	for (i = 0; i < num_events; i++) {
2147	u32 key_level1;
2148
2149	if (obj)
2150	obj_type = get_dec_obj_type(obj,
2151	event_num: event_type_num_list[i]);
2152	key_level1 = event_type_num_list[i] | obj_type << 16;
2153
2154	err = subscribe_event_xa_alloc(devx_event_table,
2155	key_level1,
2156	is_level2: obj,
2157	key_level2: obj_id);
2158	if (err)
2159	goto err;
2160
2161	event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2162	if (!event_sub) {
2163	err = -ENOMEM;
2164	goto err;
2165	}
2166
2167	list_add_tail(new: &event_sub->event_list, head: &sub_list);
2168	uverbs_uobject_get(uobject: &ev_file->uobj);
2169	if (use_eventfd) {
2170	event_sub->eventfd =
2171	eventfd_ctx_fdget(fd: redirect_fd);
2172
2173	if (IS_ERR(ptr: event_sub->eventfd)) {
2174	err = PTR_ERR(ptr: event_sub->eventfd);
2175	event_sub->eventfd = NULL;
2176	goto err;
2177	}
2178	}
2179
2180	event_sub->cookie = cookie;
2181	event_sub->ev_file = ev_file;
2182	/* May be needed upon cleanup the devx object/subscription */
2183	event_sub->xa_key_level1 = key_level1;
2184	event_sub->xa_key_level2 = obj_id;
2185	INIT_LIST_HEAD(list: &event_sub->obj_list);
2186	}
2187
2188	/* Once all the allocations and the XA data insertions were done we
2189	* can go ahead and add all the subscriptions to the relevant lists
2190	* without concern of a failure.
2191	*/
2192	list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2193	struct devx_event *event;
2194	struct devx_obj_event *obj_event;
2195
2196	list_del_init(entry: &event_sub->event_list);
2197
2198	spin_lock_irq(lock: &ev_file->lock);
2199	list_add_tail_rcu(new: &event_sub->file_list,
2200	head: &ev_file->subscribed_events_list);
2201	spin_unlock_irq(lock: &ev_file->lock);
2202
2203	event = xa_load(&devx_event_table->event_xa,
2204	index: event_sub->xa_key_level1);
2205	WARN_ON(!event);
2206
2207	if (!obj) {
2208	list_add_tail_rcu(new: &event_sub->xa_list,
2209	head: &event->unaffiliated_list);
2210	continue;
2211	}
2212
2213	obj_event = xa_load(&event->object_ids, index: obj_id);
2214	WARN_ON(!obj_event);
2215	list_add_tail_rcu(new: &event_sub->xa_list,
2216	head: &obj_event->obj_sub_list);
2217	list_add_tail_rcu(new: &event_sub->obj_list,
2218	head: &obj->event_sub);
2219	}
2220
2221	mutex_unlock(lock: &devx_event_table->event_xa_lock);
2222	return 0;
2223
2224	err:
2225	list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2226	list_del(entry: &event_sub->event_list);
2227
2228	subscribe_event_xa_dealloc(devx_event_table,
2229	key_level1: event_sub->xa_key_level1,
2230	is_level2: obj,
2231	key_level2: obj_id);
2232
2233	if (event_sub->eventfd)
2234	eventfd_ctx_put(ctx: event_sub->eventfd);
2235	uverbs_uobject_put(uobject: &event_sub->ev_file->uobj);
2236	kfree(objp: event_sub);
2237	}
2238
2239	mutex_unlock(lock: &devx_event_table->event_xa_lock);
2240	return err;
2241	}
2242
2243	static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2244	struct uverbs_attr_bundle *attrs,
2245	struct devx_umem *obj, u32 access_flags)
2246	{
2247	u64 addr;
2248	size_t size;
2249	int err;
2250
2251	if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2252	uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2253	return -EFAULT;
2254
2255	err = ib_check_mr_access(ib_dev: &dev->ib_dev, flags: access_flags);
2256	if (err)
2257	return err;
2258
2259	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) {
2260	struct ib_umem_dmabuf *umem_dmabuf;
2261	int dmabuf_fd;
2262
2263	err = uverbs_get_raw_fd(to: &dmabuf_fd, attrs_bundle: attrs,
2264	idx: MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD);
2265	if (err)
2266	return -EFAULT;
2267
2268	umem_dmabuf = ib_umem_dmabuf_get_pinned(
2269	device: &dev->ib_dev, offset: addr, size, fd: dmabuf_fd, access: access_flags);
2270	if (IS_ERR(ptr: umem_dmabuf))
2271	return PTR_ERR(ptr: umem_dmabuf);
2272	obj->umem = &umem_dmabuf->umem;
2273	} else {
2274	obj->umem = ib_umem_get(device: &dev->ib_dev, addr, size, access: access_flags);
2275	if (IS_ERR(ptr: obj->umem))
2276	return PTR_ERR(ptr: obj->umem);
2277	}
2278	return 0;
2279	}
2280
2281	static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,
2282	unsigned long pgsz_bitmap)
2283	{
2284	unsigned long page_size;
2285
2286	/* Don't bother checking larger page sizes as offset must be zero and
2287	* total DEVX umem length must be equal to total umem length.
2288	*/
2289	pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
2290	PAGE_SHIFT),
2291	MLX5_ADAPTER_PAGE_SHIFT);
2292	if (!pgsz_bitmap)
2293	return 0;
2294
2295	page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);
2296	if (!page_size)
2297	return 0;
2298
2299	/* If the page_size is less than the CPU page size then we can use the
2300	* offset and create a umem which is a subset of the page list.
2301	* For larger page sizes we can't be sure the DMA list reflects the
2302	* VA so we must ensure that the umem extent is exactly equal to the
2303	* page list. Reduce the page size until one of these cases is true.
2304	*/
2305	while ((ib_umem_dma_offset(umem, pgsz: page_size) != 0 ||
2306	(umem->length % page_size) != 0) &&
2307	page_size > PAGE_SIZE)
2308	page_size /= 2;
2309
2310	return page_size;
2311	}
2312
2313	static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2314	struct uverbs_attr_bundle *attrs,
2315	struct devx_umem *obj,
2316	struct devx_umem_reg_cmd *cmd,
2317	int access)
2318	{
2319	unsigned long pgsz_bitmap;
2320	unsigned int page_size;
2321	__be64 *mtt;
2322	void *umem;
2323	int ret;
2324
2325	/*
2326	* If the user does not pass in pgsz_bitmap then the user promises not
2327	* to use umem_offset!=0 in any commands that allocate on top of the
2328	* umem.
2329	*
2330	* If the user wants to use a umem_offset then it must pass in
2331	* pgsz_bitmap which guides the maximum page size and thus maximum
2332	* object alignment inside the umem. See the PRM.
2333	*
2334	* Users are not allowed to use IOVA here, mkeys are not supported on
2335	* umem.
2336	*/
2337	ret = uverbs_get_const_default(&pgsz_bitmap, attrs,
2338	MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2339	GENMASK_ULL(63,
2340	min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));
2341	if (ret)
2342	return ret;
2343
2344	page_size = devx_umem_find_best_pgsize(umem: obj->umem, pgsz_bitmap);
2345	if (!page_size)
2346	return -EINVAL;
2347
2348	cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2349	(MLX5_ST_SZ_BYTES(mtt) *
2350	ib_umem_num_dma_blocks(umem: obj->umem, pgsz: page_size));
2351	cmd->in = uverbs_zalloc(bundle: attrs, size: cmd->inlen);
2352	if (IS_ERR(ptr: cmd->in))
2353	return PTR_ERR(ptr: cmd->in);
2354
2355	umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2356	mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2357
2358	MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2359	MLX5_SET64(umem, umem, num_of_mtt,
2360	ib_umem_num_dma_blocks(obj->umem, page_size));
2361	MLX5_SET(umem, umem, log_page_size,
2362	order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2363	MLX5_SET(umem, umem, page_offset,
2364	ib_umem_dma_offset(obj->umem, page_size));
2365
2366	if (mlx5_umem_needs_ats(dev, umem: obj->umem, access_flags: access))
2367	MLX5_SET(umem, umem, ats, 1);
2368
2369	mlx5_ib_populate_pas(umem: obj->umem, page_size, pas: mtt,
2370	access_flags: (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2371	MLX5_IB_MTT_READ);
2372	return 0;
2373	}
2374
2375	static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2376	struct uverbs_attr_bundle *attrs)
2377	{
2378	struct devx_umem_reg_cmd cmd;
2379	struct devx_umem *obj;
2380	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2381	attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2382	u32 obj_id;
2383	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2384	&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2385	struct mlx5_ib_dev *dev = to_mdev(ibdev: c->ibucontext.device);
2386	int access_flags;
2387	int err;
2388
2389	if (!c->devx_uid)
2390	return -EINVAL;
2391
2392	err = uverbs_get_flags32(to: &access_flags, attrs_bundle: attrs,
2393	idx: MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2394	allowed_bits: IB_ACCESS_LOCAL_WRITE |
2395	IB_ACCESS_REMOTE_WRITE |
2396	IB_ACCESS_REMOTE_READ |
2397	IB_ACCESS_RELAXED_ORDERING);
2398	if (err)
2399	return err;
2400
2401	obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2402	if (!obj)
2403	return -ENOMEM;
2404
2405	err = devx_umem_get(dev, ucontext: &c->ibucontext, attrs, obj, access_flags);
2406	if (err)
2407	goto err_obj_free;
2408
2409	err = devx_umem_reg_cmd_alloc(dev, attrs, obj, cmd: &cmd, access: access_flags);
2410	if (err)
2411	goto err_umem_release;
2412
2413	MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2414	err = mlx5_cmd_exec(dev: dev->mdev, in: cmd.in, in_size: cmd.inlen, out: cmd.out,
2415	out_size: sizeof(cmd.out));
2416	if (err)
2417	goto err_umem_release;
2418
2419	obj->mdev = dev->mdev;
2420	uobj->object = obj;
2421	devx_obj_build_destroy_cmd(in: cmd.in, out: cmd.out, din: obj->dinbox, dinlen: &obj->dinlen, obj_id: &obj_id);
2422	uverbs_finalize_uobj_create(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2423
2424	err = uverbs_copy_to(attrs_bundle: attrs, idx: MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, from: &obj_id,
2425	size: sizeof(obj_id));
2426	return err;
2427
2428	err_umem_release:
2429	ib_umem_release(umem: obj->umem);
2430	err_obj_free:
2431	kfree(objp: obj);
2432	return err;
2433	}
2434
2435	static int devx_umem_cleanup(struct ib_uobject *uobject,
2436	enum rdma_remove_reason why,
2437	struct uverbs_attr_bundle *attrs)
2438	{
2439	struct devx_umem *obj = uobject->object;
2440	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2441	int err;
2442
2443	err = mlx5_cmd_exec(dev: obj->mdev, in: obj->dinbox, in_size: obj->dinlen, out, out_size: sizeof(out));
2444	if (err)
2445	return err;
2446
2447	ib_umem_release(umem: obj->umem);
2448	kfree(objp: obj);
2449	return 0;
2450	}
2451
2452	static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2453	unsigned long event_type)
2454	{
2455	__be64 *unaff_events;
2456	int mask_entry;
2457	int mask_bit;
2458
2459	if (!MLX5_CAP_GEN(dev, event_cap))
2460	return is_legacy_unaffiliated_event_num(event_num: event_type);
2461
2462	unaff_events = MLX5_CAP_DEV_EVENT(dev,
2463	user_unaffiliated_events);
2464	WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2465
2466	mask_entry = event_type / 64;
2467	mask_bit = event_type % 64;
2468
2469	if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2470	return false;
2471
2472	return true;
2473	}
2474
2475	static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2476	{
2477	struct mlx5_eqe *eqe = data;
2478	u32 obj_id = 0;
2479
2480	switch (event_type) {
2481	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2482	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2483	case MLX5_EVENT_TYPE_PATH_MIG:
2484	case MLX5_EVENT_TYPE_COMM_EST:
2485	case MLX5_EVENT_TYPE_SQ_DRAINED:
2486	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2487	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2488	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2489	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2490	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2491	obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2492	break;
2493	case MLX5_EVENT_TYPE_XRQ_ERROR:
2494	obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2495	break;
2496	case MLX5_EVENT_TYPE_DCT_DRAINED:
2497	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2498	obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2499	break;
2500	case MLX5_EVENT_TYPE_CQ_ERROR:
2501	obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2502	break;
2503	default:
2504	obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2505	break;
2506	}
2507
2508	return obj_id;
2509	}
2510
2511	static int deliver_event(struct devx_event_subscription *event_sub,
2512	const void *data)
2513	{
2514	struct devx_async_event_file *ev_file;
2515	struct devx_async_event_data *event_data;
2516	unsigned long flags;
2517
2518	ev_file = event_sub->ev_file;
2519
2520	if (ev_file->omit_data) {
2521	spin_lock_irqsave(&ev_file->lock, flags);
2522	if (!list_empty(head: &event_sub->event_list) ||
2523	ev_file->is_destroyed) {
2524	spin_unlock_irqrestore(lock: &ev_file->lock, flags);
2525	return 0;
2526	}
2527
2528	list_add_tail(new: &event_sub->event_list, head: &ev_file->event_list);
2529	spin_unlock_irqrestore(lock: &ev_file->lock, flags);
2530	wake_up_interruptible(&ev_file->poll_wait);
2531	return 0;
2532	}
2533
2534	event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2535	GFP_ATOMIC);
2536	if (!event_data) {
2537	spin_lock_irqsave(&ev_file->lock, flags);
2538	ev_file->is_overflow_err = 1;
2539	spin_unlock_irqrestore(lock: &ev_file->lock, flags);
2540	return -ENOMEM;
2541	}
2542
2543	event_data->hdr.cookie = event_sub->cookie;
2544	memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2545
2546	spin_lock_irqsave(&ev_file->lock, flags);
2547	if (!ev_file->is_destroyed)
2548	list_add_tail(new: &event_data->list, head: &ev_file->event_list);
2549	else
2550	kfree(objp: event_data);
2551	spin_unlock_irqrestore(lock: &ev_file->lock, flags);
2552	wake_up_interruptible(&ev_file->poll_wait);
2553
2554	return 0;
2555	}
2556
2557	static void dispatch_event_fd(struct list_head *fd_list,
2558	const void *data)
2559	{
2560	struct devx_event_subscription *item;
2561
2562	list_for_each_entry_rcu(item, fd_list, xa_list) {
2563	if (item->eventfd)
2564	eventfd_signal(ctx: item->eventfd);
2565	else
2566	deliver_event(event_sub: item, data);
2567	}
2568	}
2569
2570	static int devx_event_notifier(struct notifier_block *nb,
2571	unsigned long event_type, void *data)
2572	{
2573	struct mlx5_devx_event_table *table;
2574	struct mlx5_ib_dev *dev;
2575	struct devx_event *event;
2576	struct devx_obj_event *obj_event;
2577	u16 obj_type = 0;
2578	bool is_unaffiliated;
2579	u32 obj_id;
2580
2581	/* Explicit filtering to kernel events which may occur frequently */
2582	if (event_type == MLX5_EVENT_TYPE_CMD ||
2583	event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2584	return NOTIFY_OK;
2585
2586	table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2587	dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2588	is_unaffiliated = is_unaffiliated_event(dev: dev->mdev, event_type);
2589
2590	if (!is_unaffiliated)
2591	obj_type = get_event_obj_type(event_type, eqe: data);
2592
2593	rcu_read_lock();
2594	event = xa_load(&table->event_xa, index: event_type | (obj_type << 16));
2595	if (!event) {
2596	rcu_read_unlock();
2597	return NOTIFY_DONE;
2598	}
2599
2600	if (is_unaffiliated) {
2601	dispatch_event_fd(fd_list: &event->unaffiliated_list, data);
2602	rcu_read_unlock();
2603	return NOTIFY_OK;
2604	}
2605
2606	obj_id = devx_get_obj_id_from_event(event_type, data);
2607	obj_event = xa_load(&event->object_ids, index: obj_id);
2608	if (!obj_event) {
2609	rcu_read_unlock();
2610	return NOTIFY_DONE;
2611	}
2612
2613	dispatch_event_fd(fd_list: &obj_event->obj_sub_list, data);
2614
2615	rcu_read_unlock();
2616	return NOTIFY_OK;
2617	}
2618
2619	int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
2620	{
2621	struct mlx5_devx_event_table *table = &dev->devx_event_table;
2622	int uid;
2623
2624	uid = mlx5_ib_devx_create(dev, is_user: false, req_ucaps: 0);
2625	if (uid > 0) {
2626	dev->devx_whitelist_uid = uid;
2627	xa_init(xa: &table->event_xa);
2628	mutex_init(&table->event_xa_lock);
2629	MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2630	mlx5_eq_notifier_register(dev: dev->mdev, nb: &table->devx_nb);
2631	}
2632
2633	return 0;
2634	}
2635
2636	void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
2637	{
2638	struct mlx5_devx_event_table *table = &dev->devx_event_table;
2639	struct devx_event_subscription *sub, *tmp;
2640	struct devx_event *event;
2641	void *entry;
2642	unsigned long id;
2643
2644	if (dev->devx_whitelist_uid) {
2645	mlx5_eq_notifier_unregister(dev: dev->mdev, nb: &table->devx_nb);
2646	mutex_lock(&dev->devx_event_table.event_xa_lock);
2647	xa_for_each(&table->event_xa, id, entry) {
2648	event = entry;
2649	list_for_each_entry_safe(
2650	sub, tmp, &event->unaffiliated_list, xa_list)
2651	devx_cleanup_subscription(dev, sub);
2652	kfree(objp: entry);
2653	}
2654	mutex_unlock(lock: &dev->devx_event_table.event_xa_lock);
2655	xa_destroy(&table->event_xa);
2656
2657	mlx5_ib_devx_destroy(dev, uid: dev->devx_whitelist_uid);
2658	}
2659	}
2660
2661	static void devx_async_destroy_cb(int status, struct mlx5_async_work *context)
2662	{
2663	struct mlx5_async_cmd *devx_out = container_of(context,
2664	struct mlx5_async_cmd, cb_work);
2665	struct devx_obj *obj = devx_out->uobject->object;
2666
2667	if (!status)
2668	obj->flags |= DEVX_OBJ_FLAGS_HW_FREED;
2669
2670	complete(&devx_out->comp);
2671	}
2672
2673	static void devx_async_destroy(struct mlx5_ib_dev *dev,
2674	struct mlx5_async_cmd *cmd)
2675	{
2676	init_completion(x: &cmd->comp);
2677	cmd->err = mlx5_cmd_exec_cb(ctx: &dev->async_ctx, in: cmd->in, in_size: cmd->in_size,
2678	out: &cmd->out, out_size: sizeof(cmd->out),
2679	callback: devx_async_destroy_cb, work: &cmd->cb_work);
2680	}
2681
2682	static void devx_wait_async_destroy(struct mlx5_async_cmd *cmd)
2683	{
2684	if (!cmd->err)
2685	wait_for_completion(&cmd->comp);
2686	atomic_set(v: &cmd->uobject->usecnt, i: 0);
2687	}
2688
2689	void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)
2690	{
2691	struct mlx5_async_cmd *async_cmd;
2692	struct ib_ucontext *ucontext = ufile->ucontext;
2693	struct ib_device *device = ucontext->device;
2694	struct mlx5_ib_dev *dev = to_mdev(ibdev: device);
2695	struct ib_uobject *uobject;
2696	struct devx_obj *obj;
2697	int head = 0;
2698	int tail = 0;
2699
2700	async_cmd = kcalloc(MAX_ASYNC_CMDS, sizeof(*async_cmd), GFP_KERNEL);
2701	if (!async_cmd)
2702	return;
2703
2704	list_for_each_entry(uobject, &ufile->uobjects, list) {
2705	WARN_ON(uverbs_try_lock_object(uobject, UVERBS_LOOKUP_WRITE));
2706
2707	/*
2708	* Currently we only support QP destruction, if other objects
2709	* are to be destroyed need to add type synchronization to the
2710	* cleanup algorithm and handle pre/post FW cleanup for the
2711	* new types if needed.
2712	*/
2713	if (uobj_get_object_id(uobj: uobject) != MLX5_IB_OBJECT_DEVX_OBJ ||
2714	(get_dec_obj_type(obj: uobject->object, event_num: MLX5_EVENT_TYPE_MAX) !=
2715	MLX5_OBJ_TYPE_QP)) {
2716	atomic_set(v: &uobject->usecnt, i: 0);
2717	continue;
2718	}
2719
2720	obj = uobject->object;
2721
2722	async_cmd[tail % MAX_ASYNC_CMDS].in = obj->dinbox;
2723	async_cmd[tail % MAX_ASYNC_CMDS].in_size = obj->dinlen;
2724	async_cmd[tail % MAX_ASYNC_CMDS].uobject = uobject;
2725
2726	devx_async_destroy(dev, cmd: &async_cmd[tail % MAX_ASYNC_CMDS]);
2727	tail++;
2728
2729	if (tail - head == MAX_ASYNC_CMDS) {
2730	devx_wait_async_destroy(cmd: &async_cmd[head % MAX_ASYNC_CMDS]);
2731	head++;
2732	}
2733	}
2734
2735	while (head != tail) {
2736	devx_wait_async_destroy(cmd: &async_cmd[head % MAX_ASYNC_CMDS]);
2737	head++;
2738	}
2739
2740	kfree(objp: async_cmd);
2741	}
2742
2743	static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2744	size_t count, loff_t *pos)
2745	{
2746	struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2747	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2748	struct devx_async_data *event;
2749	int ret = 0;
2750	size_t eventsz;
2751
2752	spin_lock_irq(lock: &ev_queue->lock);
2753
2754	while (list_empty(head: &ev_queue->event_list)) {
2755	spin_unlock_irq(lock: &ev_queue->lock);
2756
2757	if (filp->f_flags & O_NONBLOCK)
2758	return -EAGAIN;
2759
2760	if (wait_event_interruptible(
2761	ev_queue->poll_wait,
2762	(!list_empty(&ev_queue->event_list) ||
2763	ev_queue->is_destroyed))) {
2764	return -ERESTARTSYS;
2765	}
2766
2767	spin_lock_irq(lock: &ev_queue->lock);
2768	if (ev_queue->is_destroyed) {
2769	spin_unlock_irq(lock: &ev_queue->lock);
2770	return -EIO;
2771	}
2772	}
2773
2774	event = list_entry(ev_queue->event_list.next,
2775	struct devx_async_data, list);
2776	eventsz = event->cmd_out_len +
2777	sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2778
2779	if (eventsz > count) {
2780	spin_unlock_irq(lock: &ev_queue->lock);
2781	return -ENOSPC;
2782	}
2783
2784	list_del(entry: ev_queue->event_list.next);
2785	spin_unlock_irq(lock: &ev_queue->lock);
2786
2787	if (copy_to_user(to: buf, from: &event->hdr, n: eventsz))
2788	ret = -EFAULT;
2789	else
2790	ret = eventsz;
2791
2792	atomic_sub(i: event->cmd_out_len, v: &ev_queue->bytes_in_use);
2793	kvfree(addr: event);
2794	return ret;
2795	}
2796
2797	static __poll_t devx_async_cmd_event_poll(struct file *filp,
2798	struct poll_table_struct *wait)
2799	{
2800	struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2801	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2802	__poll_t pollflags = 0;
2803
2804	poll_wait(filp, wait_address: &ev_queue->poll_wait, p: wait);
2805
2806	spin_lock_irq(lock: &ev_queue->lock);
2807	if (ev_queue->is_destroyed)
2808	pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2809	else if (!list_empty(head: &ev_queue->event_list))
2810	pollflags = EPOLLIN | EPOLLRDNORM;
2811	spin_unlock_irq(lock: &ev_queue->lock);
2812
2813	return pollflags;
2814	}
2815
2816	static const struct file_operations devx_async_cmd_event_fops = {
2817	.owner = THIS_MODULE,
2818	.read = devx_async_cmd_event_read,
2819	.poll = devx_async_cmd_event_poll,
2820	.release = uverbs_uobject_fd_release,
2821	};
2822
2823	static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2824	size_t count, loff_t *pos)
2825	{
2826	struct devx_async_event_file *ev_file = filp->private_data;
2827	struct devx_event_subscription *event_sub;
2828	struct devx_async_event_data *event;
2829	int ret = 0;
2830	size_t eventsz;
2831	bool omit_data;
2832	void *event_data;
2833
2834	omit_data = ev_file->omit_data;
2835
2836	spin_lock_irq(lock: &ev_file->lock);
2837
2838	if (ev_file->is_overflow_err) {
2839	ev_file->is_overflow_err = 0;
2840	spin_unlock_irq(lock: &ev_file->lock);
2841	return -EOVERFLOW;
2842	}
2843
2844
2845	while (list_empty(head: &ev_file->event_list)) {
2846	spin_unlock_irq(lock: &ev_file->lock);
2847
2848	if (filp->f_flags & O_NONBLOCK)
2849	return -EAGAIN;
2850
2851	if (wait_event_interruptible(ev_file->poll_wait,
2852	(!list_empty(&ev_file->event_list) ||
2853	ev_file->is_destroyed))) {
2854	return -ERESTARTSYS;
2855	}
2856
2857	spin_lock_irq(lock: &ev_file->lock);
2858	if (ev_file->is_destroyed) {
2859	spin_unlock_irq(lock: &ev_file->lock);
2860	return -EIO;
2861	}
2862	}
2863
2864	if (omit_data) {
2865	event_sub = list_first_entry(&ev_file->event_list,
2866	struct devx_event_subscription,
2867	event_list);
2868	eventsz = sizeof(event_sub->cookie);
2869	event_data = &event_sub->cookie;
2870	} else {
2871	event = list_first_entry(&ev_file->event_list,
2872	struct devx_async_event_data, list);
2873	eventsz = sizeof(struct mlx5_eqe) +
2874	sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2875	event_data = &event->hdr;
2876	}
2877
2878	if (eventsz > count) {
2879	spin_unlock_irq(lock: &ev_file->lock);
2880	return -EINVAL;
2881	}
2882
2883	if (omit_data)
2884	list_del_init(entry: &event_sub->event_list);
2885	else
2886	list_del(entry: &event->list);
2887
2888	spin_unlock_irq(lock: &ev_file->lock);
2889
2890	if (copy_to_user(to: buf, from: event_data, n: eventsz))
2891	/* This points to an application issue, not a kernel concern */
2892	ret = -EFAULT;
2893	else
2894	ret = eventsz;
2895
2896	if (!omit_data)
2897	kfree(objp: event);
2898	return ret;
2899	}
2900
2901	static __poll_t devx_async_event_poll(struct file *filp,
2902	struct poll_table_struct *wait)
2903	{
2904	struct devx_async_event_file *ev_file = filp->private_data;
2905	__poll_t pollflags = 0;
2906
2907	poll_wait(filp, wait_address: &ev_file->poll_wait, p: wait);
2908
2909	spin_lock_irq(lock: &ev_file->lock);
2910	if (ev_file->is_destroyed)
2911	pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2912	else if (!list_empty(head: &ev_file->event_list))
2913	pollflags = EPOLLIN | EPOLLRDNORM;
2914	spin_unlock_irq(lock: &ev_file->lock);
2915
2916	return pollflags;
2917	}
2918
2919	static void devx_free_subscription(struct rcu_head *rcu)
2920	{
2921	struct devx_event_subscription *event_sub =
2922	container_of(rcu, struct devx_event_subscription, rcu);
2923
2924	if (event_sub->eventfd)
2925	eventfd_ctx_put(ctx: event_sub->eventfd);
2926	uverbs_uobject_put(uobject: &event_sub->ev_file->uobj);
2927	kfree(objp: event_sub);
2928	}
2929
2930	static const struct file_operations devx_async_event_fops = {
2931	.owner = THIS_MODULE,
2932	.read = devx_async_event_read,
2933	.poll = devx_async_event_poll,
2934	.release = uverbs_uobject_fd_release,
2935	};
2936
2937	static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2938	enum rdma_remove_reason why)
2939	{
2940	struct devx_async_cmd_event_file *comp_ev_file =
2941	container_of(uobj, struct devx_async_cmd_event_file,
2942	uobj);
2943	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2944	struct devx_async_data *entry, *tmp;
2945
2946	spin_lock_irq(lock: &ev_queue->lock);
2947	ev_queue->is_destroyed = 1;
2948	spin_unlock_irq(lock: &ev_queue->lock);
2949	wake_up_interruptible(&ev_queue->poll_wait);
2950
2951	mlx5_cmd_cleanup_async_ctx(ctx: &comp_ev_file->async_ctx);
2952
2953	spin_lock_irq(lock: &comp_ev_file->ev_queue.lock);
2954	list_for_each_entry_safe(entry, tmp,
2955	&comp_ev_file->ev_queue.event_list, list) {
2956	list_del(entry: &entry->list);
2957	kvfree(addr: entry);
2958	}
2959	spin_unlock_irq(lock: &comp_ev_file->ev_queue.lock);
2960	};
2961
2962	static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2963	enum rdma_remove_reason why)
2964	{
2965	struct devx_async_event_file *ev_file =
2966	container_of(uobj, struct devx_async_event_file,
2967	uobj);
2968	struct devx_event_subscription *event_sub, *event_sub_tmp;
2969	struct mlx5_ib_dev *dev = ev_file->dev;
2970
2971	spin_lock_irq(lock: &ev_file->lock);
2972	ev_file->is_destroyed = 1;
2973
2974	/* free the pending events allocation */
2975	if (ev_file->omit_data) {
2976	struct devx_event_subscription *event_sub, *tmp;
2977
2978	list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2979	event_list)
2980	list_del_init(entry: &event_sub->event_list);
2981
2982	} else {
2983	struct devx_async_event_data *entry, *tmp;
2984
2985	list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2986	list) {
2987	list_del(entry: &entry->list);
2988	kfree(objp: entry);
2989	}
2990	}
2991
2992	spin_unlock_irq(lock: &ev_file->lock);
2993	wake_up_interruptible(&ev_file->poll_wait);
2994
2995	mutex_lock(&dev->devx_event_table.event_xa_lock);
2996	/* delete the subscriptions which are related to this FD */
2997	list_for_each_entry_safe(event_sub, event_sub_tmp,
2998	&ev_file->subscribed_events_list, file_list) {
2999	devx_cleanup_subscription(dev, sub: event_sub);
3000	list_del_rcu(entry: &event_sub->file_list);
3001	/* subscription may not be used by the read API any more */
3002	call_rcu(head: &event_sub->rcu, func: devx_free_subscription);
3003	}
3004	mutex_unlock(lock: &dev->devx_event_table.event_xa_lock);
3005
3006	put_device(dev: &dev->ib_dev.dev);
3007	};
3008
3009	DECLARE_UVERBS_NAMED_METHOD(
3010	MLX5_IB_METHOD_DEVX_UMEM_REG,
3011	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
3012	MLX5_IB_OBJECT_DEVX_UMEM,
3013	UVERBS_ACCESS_NEW,
3014	UA_MANDATORY),
3015	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
3016	UVERBS_ATTR_TYPE(u64),
3017	UA_MANDATORY),
3018	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
3019	UVERBS_ATTR_TYPE(u64),
3020	UA_MANDATORY),
3021	UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD,
3022	UA_OPTIONAL),
3023	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
3024	enum ib_access_flags),
3025	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
3026	u64),
3027	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
3028	UVERBS_ATTR_TYPE(u32),
3029	UA_MANDATORY));
3030
3031	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3032	MLX5_IB_METHOD_DEVX_UMEM_DEREG,
3033	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
3034	MLX5_IB_OBJECT_DEVX_UMEM,
3035	UVERBS_ACCESS_DESTROY,
3036	UA_MANDATORY));
3037
3038	DECLARE_UVERBS_NAMED_METHOD(
3039	MLX5_IB_METHOD_DEVX_QUERY_EQN,
3040	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
3041	UVERBS_ATTR_TYPE(u32),
3042	UA_MANDATORY),
3043	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
3044	UVERBS_ATTR_TYPE(u32),
3045	UA_MANDATORY));
3046
3047	DECLARE_UVERBS_NAMED_METHOD(
3048	MLX5_IB_METHOD_DEVX_QUERY_UAR,
3049	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
3050	UVERBS_ATTR_TYPE(u32),
3051	UA_MANDATORY),
3052	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
3053	UVERBS_ATTR_TYPE(u32),
3054	UA_MANDATORY));
3055
3056	DECLARE_UVERBS_NAMED_METHOD(
3057	MLX5_IB_METHOD_DEVX_OTHER,
3058	UVERBS_ATTR_PTR_IN(
3059	MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
3060	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3061	UA_MANDATORY,
3062	UA_ALLOC_AND_COPY),
3063	UVERBS_ATTR_PTR_OUT(
3064	MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
3065	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3066	UA_MANDATORY));
3067
3068	DECLARE_UVERBS_NAMED_METHOD(
3069	MLX5_IB_METHOD_DEVX_OBJ_CREATE,
3070	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
3071	MLX5_IB_OBJECT_DEVX_OBJ,
3072	UVERBS_ACCESS_NEW,
3073	UA_MANDATORY),
3074	UVERBS_ATTR_PTR_IN(
3075	MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
3076	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3077	UA_MANDATORY,
3078	UA_ALLOC_AND_COPY),
3079	UVERBS_ATTR_PTR_OUT(
3080	MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
3081	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3082	UA_MANDATORY));
3083
3084	DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3085	MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
3086	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
3087	MLX5_IB_OBJECT_DEVX_OBJ,
3088	UVERBS_ACCESS_DESTROY,
3089	UA_MANDATORY));
3090
3091	DECLARE_UVERBS_NAMED_METHOD(
3092	MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
3093	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
3094	UVERBS_IDR_ANY_OBJECT,
3095	UVERBS_ACCESS_READ,
3096	UA_MANDATORY),
3097	UVERBS_ATTR_PTR_IN(
3098	MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
3099	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3100	UA_MANDATORY,
3101	UA_ALLOC_AND_COPY),
3102	UVERBS_ATTR_PTR_OUT(
3103	MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
3104	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3105	UA_MANDATORY));
3106
3107	DECLARE_UVERBS_NAMED_METHOD(
3108	MLX5_IB_METHOD_DEVX_OBJ_QUERY,
3109	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3110	UVERBS_IDR_ANY_OBJECT,
3111	UVERBS_ACCESS_READ,
3112	UA_MANDATORY),
3113	UVERBS_ATTR_PTR_IN(
3114	MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3115	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3116	UA_MANDATORY,
3117	UA_ALLOC_AND_COPY),
3118	UVERBS_ATTR_PTR_OUT(
3119	MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
3120	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3121	UA_MANDATORY));
3122
3123	DECLARE_UVERBS_NAMED_METHOD(
3124	MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
3125	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3126	UVERBS_IDR_ANY_OBJECT,
3127	UVERBS_ACCESS_READ,
3128	UA_MANDATORY),
3129	UVERBS_ATTR_PTR_IN(
3130	MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3131	UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3132	UA_MANDATORY,
3133	UA_ALLOC_AND_COPY),
3134	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
3135	u16, UA_MANDATORY),
3136	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
3137	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3138	UVERBS_ACCESS_READ,
3139	UA_MANDATORY),
3140	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
3141	UVERBS_ATTR_TYPE(u64),
3142	UA_MANDATORY));
3143
3144	DECLARE_UVERBS_NAMED_METHOD(
3145	MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
3146	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
3147	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3148	UVERBS_ACCESS_READ,
3149	UA_MANDATORY),
3150	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
3151	MLX5_IB_OBJECT_DEVX_OBJ,
3152	UVERBS_ACCESS_READ,
3153	UA_OPTIONAL),
3154	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
3155	UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
3156	UA_MANDATORY,
3157	UA_ALLOC_AND_COPY),
3158	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
3159	UVERBS_ATTR_TYPE(u64),
3160	UA_OPTIONAL),
3161	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
3162	UVERBS_ATTR_TYPE(u32),
3163	UA_OPTIONAL));
3164
3165	DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
3166	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
3167	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
3168	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
3169	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
3170
3171	DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
3172	UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
3173	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
3174	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
3175	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
3176	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
3177	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
3178
3179	DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
3180	UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
3181	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
3182	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
3183
3184
3185	DECLARE_UVERBS_NAMED_METHOD(
3186	MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
3187	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
3188	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3189	UVERBS_ACCESS_NEW,
3190	UA_MANDATORY));
3191
3192	DECLARE_UVERBS_NAMED_OBJECT(
3193	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3194	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
3195	devx_async_cmd_event_destroy_uobj,
3196	&devx_async_cmd_event_fops, "[devx_async_cmd]",
3197	O_RDONLY),
3198	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
3199
3200	DECLARE_UVERBS_NAMED_METHOD(
3201	MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
3202	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
3203	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3204	UVERBS_ACCESS_NEW,
3205	UA_MANDATORY),
3206	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
3207	enum mlx5_ib_uapi_devx_create_event_channel_flags,
3208	UA_MANDATORY));
3209
3210	DECLARE_UVERBS_NAMED_OBJECT(
3211	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3212	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
3213	devx_async_event_destroy_uobj,
3214	&devx_async_event_fops, "[devx_async_event]",
3215	O_RDONLY),
3216	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
3217
3218	static bool devx_is_supported(struct ib_device *device)
3219	{
3220	struct mlx5_ib_dev *dev = to_mdev(ibdev: device);
3221
3222	return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
3223	}
3224
3225	const struct uapi_definition mlx5_ib_devx_defs[] = {
3226	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3227	MLX5_IB_OBJECT_DEVX,
3228	UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3229	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3230	MLX5_IB_OBJECT_DEVX_OBJ,
3231	UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3232	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3233	MLX5_IB_OBJECT_DEVX_UMEM,
3234	UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3235	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3236	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3237	UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3238	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3239	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3240	UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3241	{},
3242	};
3243