1	/*
2	* Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <linux/highmem.h>
34	#include <linux/errno.h>
35	#include <linux/pci.h>
36	#include <linux/dma-mapping.h>
37	#include <linux/slab.h>
38	#include <linux/delay.h>
39	#include <linux/random.h>
40	#include <linux/mlx5/driver.h>
41	#include <linux/mlx5/eq.h>
42	#include <linux/debugfs.h>
43
44	#include "mlx5_core.h"
45	#include "lib/eq.h"
46	#include "lib/tout.h"
47	#define CREATE_TRACE_POINTS
48	#include "diag/cmd_tracepoint.h"
49
50	struct mlx5_ifc_mbox_out_bits {
51	u8 status[0x8];
52	u8 reserved_at_8[0x18];
53
54	u8 syndrome[0x20];
55
56	u8 reserved_at_40[0x40];
57	};
58
59	struct mlx5_ifc_mbox_in_bits {
60	u8 opcode[0x10];
61	u8 uid[0x10];
62
63	u8 reserved_at_20[0x10];
64	u8 op_mod[0x10];
65
66	u8 reserved_at_40[0x40];
67	};
68
69	enum {
70	CMD_IF_REV = 5,
71	};
72
73	enum {
74	CMD_MODE_POLLING,
75	CMD_MODE_EVENTS
76	};
77
78	enum {
79	MLX5_CMD_DELIVERY_STAT_OK = 0x0,
80	MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
81	MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
82	MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
83	MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
84	MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
85	MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
86	MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
87	MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
88	MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
89	MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
90	};
91
92	static u16 in_to_opcode(void *in)
93	{
94	return MLX5_GET(mbox_in, in, opcode);
95	}
96
97	static u16 in_to_uid(void *in)
98	{
99	return MLX5_GET(mbox_in, in, uid);
100	}
101
102	/* Returns true for opcodes that might be triggered very frequently and throttle
103	* the command interface. Limit their command slots usage.
104	*/
105	static bool mlx5_cmd_is_throttle_opcode(u16 op)
106	{
107	switch (op) {
108	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
109	case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
110	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
111	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
112	case MLX5_CMD_OP_SYNC_CRYPTO:
113	return true;
114	}
115	return false;
116	}
117
118	static struct mlx5_cmd_work_ent *
119	cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
120	struct mlx5_cmd_msg *out, void *uout, int uout_size,
121	mlx5_cmd_cbk_t cbk, void *context, int page_queue)
122	{
123	gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
124	struct mlx5_cmd_work_ent *ent;
125
126	ent = kzalloc(sizeof(*ent), alloc_flags);
127	if (!ent)
128	return ERR_PTR(error: -ENOMEM);
129
130	ent->idx = -EINVAL;
131	ent->in = in;
132	ent->out = out;
133	ent->uout = uout;
134	ent->uout_size = uout_size;
135	ent->callback = cbk;
136	ent->context = context;
137	ent->cmd = cmd;
138	ent->page_queue = page_queue;
139	ent->op = in_to_opcode(in: in->first.data);
140	refcount_set(r: &ent->refcnt, n: 1);
141
142	return ent;
143	}
144
145	static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
146	{
147	kfree(objp: ent);
148	}
149
150	static u8 alloc_token(struct mlx5_cmd *cmd)
151	{
152	u8 token;
153
154	spin_lock(lock: &cmd->token_lock);
155	cmd->token++;
156	if (cmd->token == 0)
157	cmd->token++;
158	token = cmd->token;
159	spin_unlock(lock: &cmd->token_lock);
160
161	return token;
162	}
163
164	static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)
165	{
166	unsigned long flags;
167	int ret;
168
169	spin_lock_irqsave(&cmd->alloc_lock, flags);
170	ret = find_first_bit(addr: &cmd->vars.bitmask, size: cmd->vars.max_reg_cmds);
171	if (ret < cmd->vars.max_reg_cmds) {
172	clear_bit(nr: ret, addr: &cmd->vars.bitmask);
173	ent->idx = ret;
174	cmd->ent_arr[ent->idx] = ent;
175	}
176	spin_unlock_irqrestore(lock: &cmd->alloc_lock, flags);
177
178	return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;
179	}
180
181	static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
182	{
183	lockdep_assert_held(&cmd->alloc_lock);
184	cmd->ent_arr[idx] = NULL;
185	set_bit(nr: idx, addr: &cmd->vars.bitmask);
186	}
187
188	static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
189	{
190	refcount_inc(r: &ent->refcnt);
191	}
192
193	static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
194	{
195	struct mlx5_cmd *cmd = ent->cmd;
196	unsigned long flags;
197
198	spin_lock_irqsave(&cmd->alloc_lock, flags);
199	if (!refcount_dec_and_test(r: &ent->refcnt))
200	goto out;
201
202	if (ent->idx >= 0) {
203	cmd_free_index(cmd, idx: ent->idx);
204	up(sem: ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);
205	}
206
207	cmd_free_ent(ent);
208	out:
209	spin_unlock_irqrestore(lock: &cmd->alloc_lock, flags);
210	}
211
212	static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
213	{
214	return cmd->cmd_buf + (idx << cmd->vars.log_stride);
215	}
216
217	static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
218	{
219	int size = msg->len;
220	int blen = size - min_t(int, sizeof(msg->first.data), size);
221
222	return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
223	}
224
225	static u8 xor8_buf(void *buf, size_t offset, int len)
226	{
227	u8 *ptr = buf;
228	u8 sum = 0;
229	int i;
230	int end = len + offset;
231
232	for (i = offset; i < end; i++)
233	sum ^= ptr[i];
234
235	return sum;
236	}
237
238	static int verify_block_sig(struct mlx5_cmd_prot_block *block)
239	{
240	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
241	int xor_len = sizeof(*block) - sizeof(block->data) - 1;
242
243	if (xor8_buf(buf: block, offset: rsvd0_off, len: xor_len) != 0xff)
244	return -EHWPOISON;
245
246	if (xor8_buf(buf: block, offset: 0, len: sizeof(*block)) != 0xff)
247	return -EHWPOISON;
248
249	return 0;
250	}
251
252	static void calc_block_sig(struct mlx5_cmd_prot_block *block)
253	{
254	int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
255	size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
256
257	block->ctrl_sig = ~xor8_buf(buf: block, offset: rsvd0_off, len: ctrl_xor_len);
258	block->sig = ~xor8_buf(buf: block, offset: 0, len: sizeof(*block) - 1);
259	}
260
261	static void calc_chain_sig(struct mlx5_cmd_msg *msg)
262	{
263	struct mlx5_cmd_mailbox *next = msg->next;
264	int n = mlx5_calc_cmd_blocks(msg);
265	int i = 0;
266
267	for (i = 0; i < n && next; i++) {
268	calc_block_sig(block: next->buf);
269	next = next->next;
270	}
271	}
272
273	static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
274	{
275	ent->lay->sig = ~xor8_buf(buf: ent->lay, offset: 0, len: sizeof(*ent->lay));
276	if (csum) {
277	calc_chain_sig(msg: ent->in);
278	calc_chain_sig(msg: ent->out);
279	}
280	}
281
282	static void poll_timeout(struct mlx5_cmd_work_ent *ent)
283	{
284	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
285	u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
286	unsigned long poll_end;
287	u8 own;
288
289	poll_end = jiffies + msecs_to_jiffies(m: cmd_to_ms + 1000);
290
291	do {
292	own = READ_ONCE(ent->lay->status_own);
293	if (!(own & CMD_OWNER_HW)) {
294	ent->ret = 0;
295	return;
296	}
297	cond_resched();
298	if (mlx5_cmd_is_down(dev)) {
299	ent->ret = -ENXIO;
300	return;
301	}
302	} while (time_before(jiffies, poll_end));
303
304	ent->ret = -ETIMEDOUT;
305	}
306
307	static int verify_signature(struct mlx5_cmd_work_ent *ent)
308	{
309	struct mlx5_cmd_mailbox *next = ent->out->next;
310	int n = mlx5_calc_cmd_blocks(msg: ent->out);
311	int err;
312	u8 sig;
313	int i = 0;
314
315	sig = xor8_buf(buf: ent->lay, offset: 0, len: sizeof(*ent->lay));
316	if (sig != 0xff)
317	return -EHWPOISON;
318
319	for (i = 0; i < n && next; i++) {
320	err = verify_block_sig(block: next->buf);
321	if (err)
322	return -EHWPOISON;
323
324	next = next->next;
325	}
326
327	return 0;
328	}
329
330	static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
331	{
332	__be32 *p = buf;
333	int i;
334
335	for (i = 0; i < size; i += 16) {
336	pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
337	be32_to_cpu(p[0]), be32_to_cpu(p[1]),
338	be32_to_cpu(p[2]), be32_to_cpu(p[3]));
339	p += 4;
340	offset += 16;
341	}
342	if (!data_only)
343	pr_debug("\n");
344	}
345
346	static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
347	u32 *synd, u8 *status)
348	{
349	*synd = 0;
350	*status = 0;
351
352	switch (op) {
353	case MLX5_CMD_OP_TEARDOWN_HCA:
354	case MLX5_CMD_OP_DISABLE_HCA:
355	case MLX5_CMD_OP_MANAGE_PAGES:
356	case MLX5_CMD_OP_DESTROY_MKEY:
357	case MLX5_CMD_OP_DESTROY_EQ:
358	case MLX5_CMD_OP_DESTROY_CQ:
359	case MLX5_CMD_OP_DESTROY_QP:
360	case MLX5_CMD_OP_DESTROY_PSV:
361	case MLX5_CMD_OP_DESTROY_SRQ:
362	case MLX5_CMD_OP_DESTROY_XRC_SRQ:
363	case MLX5_CMD_OP_DESTROY_XRQ:
364	case MLX5_CMD_OP_DESTROY_DCT:
365	case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
366	case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
367	case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
368	case MLX5_CMD_OP_DEALLOC_PD:
369	case MLX5_CMD_OP_DEALLOC_UAR:
370	case MLX5_CMD_OP_DETACH_FROM_MCG:
371	case MLX5_CMD_OP_DEALLOC_XRCD:
372	case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
373	case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
374	case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
375	case MLX5_CMD_OP_DESTROY_LAG:
376	case MLX5_CMD_OP_DESTROY_VPORT_LAG:
377	case MLX5_CMD_OP_DESTROY_TIR:
378	case MLX5_CMD_OP_DESTROY_SQ:
379	case MLX5_CMD_OP_DESTROY_RQ:
380	case MLX5_CMD_OP_DESTROY_RMP:
381	case MLX5_CMD_OP_DESTROY_TIS:
382	case MLX5_CMD_OP_DESTROY_RQT:
383	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
384	case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
385	case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
386	case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
387	case MLX5_CMD_OP_2ERR_QP:
388	case MLX5_CMD_OP_2RST_QP:
389	case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
390	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
391	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
392	case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
393	case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
394	case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
395	case MLX5_CMD_OP_FPGA_DESTROY_QP:
396	case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
397	case MLX5_CMD_OP_DEALLOC_MEMIC:
398	case MLX5_CMD_OP_PAGE_FAULT_RESUME:
399	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
400	case MLX5_CMD_OP_DEALLOC_SF:
401	case MLX5_CMD_OP_DESTROY_UCTX:
402	case MLX5_CMD_OP_DESTROY_UMEM:
403	case MLX5_CMD_OP_MODIFY_RQT:
404	return MLX5_CMD_STAT_OK;
405
406	case MLX5_CMD_OP_QUERY_HCA_CAP:
407	case MLX5_CMD_OP_QUERY_ADAPTER:
408	case MLX5_CMD_OP_INIT_HCA:
409	case MLX5_CMD_OP_ENABLE_HCA:
410	case MLX5_CMD_OP_QUERY_PAGES:
411	case MLX5_CMD_OP_SET_HCA_CAP:
412	case MLX5_CMD_OP_QUERY_ISSI:
413	case MLX5_CMD_OP_SET_ISSI:
414	case MLX5_CMD_OP_CREATE_MKEY:
415	case MLX5_CMD_OP_QUERY_MKEY:
416	case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
417	case MLX5_CMD_OP_CREATE_EQ:
418	case MLX5_CMD_OP_QUERY_EQ:
419	case MLX5_CMD_OP_GEN_EQE:
420	case MLX5_CMD_OP_CREATE_CQ:
421	case MLX5_CMD_OP_QUERY_CQ:
422	case MLX5_CMD_OP_MODIFY_CQ:
423	case MLX5_CMD_OP_CREATE_QP:
424	case MLX5_CMD_OP_RST2INIT_QP:
425	case MLX5_CMD_OP_INIT2RTR_QP:
426	case MLX5_CMD_OP_RTR2RTS_QP:
427	case MLX5_CMD_OP_RTS2RTS_QP:
428	case MLX5_CMD_OP_SQERR2RTS_QP:
429	case MLX5_CMD_OP_QUERY_QP:
430	case MLX5_CMD_OP_SQD_RTS_QP:
431	case MLX5_CMD_OP_INIT2INIT_QP:
432	case MLX5_CMD_OP_CREATE_PSV:
433	case MLX5_CMD_OP_CREATE_SRQ:
434	case MLX5_CMD_OP_QUERY_SRQ:
435	case MLX5_CMD_OP_ARM_RQ:
436	case MLX5_CMD_OP_CREATE_XRC_SRQ:
437	case MLX5_CMD_OP_QUERY_XRC_SRQ:
438	case MLX5_CMD_OP_ARM_XRC_SRQ:
439	case MLX5_CMD_OP_CREATE_XRQ:
440	case MLX5_CMD_OP_QUERY_XRQ:
441	case MLX5_CMD_OP_ARM_XRQ:
442	case MLX5_CMD_OP_CREATE_DCT:
443	case MLX5_CMD_OP_DRAIN_DCT:
444	case MLX5_CMD_OP_QUERY_DCT:
445	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
446	case MLX5_CMD_OP_QUERY_VPORT_STATE:
447	case MLX5_CMD_OP_MODIFY_VPORT_STATE:
448	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
449	case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
450	case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
451	case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
452	case MLX5_CMD_OP_SET_ROCE_ADDRESS:
453	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
454	case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
455	case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
456	case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
457	case MLX5_CMD_OP_QUERY_VNIC_ENV:
458	case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
459	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
460	case MLX5_CMD_OP_QUERY_Q_COUNTER:
461	case MLX5_CMD_OP_SET_MONITOR_COUNTER:
462	case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
463	case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
464	case MLX5_CMD_OP_QUERY_RATE_LIMIT:
465	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
466	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
467	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
468	case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
469	case MLX5_CMD_OP_ALLOC_PD:
470	case MLX5_CMD_OP_ALLOC_UAR:
471	case MLX5_CMD_OP_CONFIG_INT_MODERATION:
472	case MLX5_CMD_OP_ACCESS_REG:
473	case MLX5_CMD_OP_ATTACH_TO_MCG:
474	case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
475	case MLX5_CMD_OP_MAD_IFC:
476	case MLX5_CMD_OP_QUERY_MAD_DEMUX:
477	case MLX5_CMD_OP_SET_MAD_DEMUX:
478	case MLX5_CMD_OP_NOP:
479	case MLX5_CMD_OP_ALLOC_XRCD:
480	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
481	case MLX5_CMD_OP_QUERY_CONG_STATUS:
482	case MLX5_CMD_OP_MODIFY_CONG_STATUS:
483	case MLX5_CMD_OP_QUERY_CONG_PARAMS:
484	case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
485	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
486	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
487	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
488	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
489	case MLX5_CMD_OP_CREATE_LAG:
490	case MLX5_CMD_OP_MODIFY_LAG:
491	case MLX5_CMD_OP_QUERY_LAG:
492	case MLX5_CMD_OP_CREATE_VPORT_LAG:
493	case MLX5_CMD_OP_CREATE_TIR:
494	case MLX5_CMD_OP_MODIFY_TIR:
495	case MLX5_CMD_OP_QUERY_TIR:
496	case MLX5_CMD_OP_CREATE_SQ:
497	case MLX5_CMD_OP_MODIFY_SQ:
498	case MLX5_CMD_OP_QUERY_SQ:
499	case MLX5_CMD_OP_CREATE_RQ:
500	case MLX5_CMD_OP_MODIFY_RQ:
501	case MLX5_CMD_OP_QUERY_RQ:
502	case MLX5_CMD_OP_CREATE_RMP:
503	case MLX5_CMD_OP_MODIFY_RMP:
504	case MLX5_CMD_OP_QUERY_RMP:
505	case MLX5_CMD_OP_CREATE_TIS:
506	case MLX5_CMD_OP_MODIFY_TIS:
507	case MLX5_CMD_OP_QUERY_TIS:
508	case MLX5_CMD_OP_CREATE_RQT:
509	case MLX5_CMD_OP_QUERY_RQT:
510
511	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
512	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
513	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
514	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
515	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
516	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
517	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
518	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
519	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
520	case MLX5_CMD_OP_FPGA_CREATE_QP:
521	case MLX5_CMD_OP_FPGA_MODIFY_QP:
522	case MLX5_CMD_OP_FPGA_QUERY_QP:
523	case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
524	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
525	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
526	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
527	case MLX5_CMD_OP_CREATE_UCTX:
528	case MLX5_CMD_OP_CREATE_UMEM:
529	case MLX5_CMD_OP_ALLOC_MEMIC:
530	case MLX5_CMD_OP_MODIFY_XRQ:
531	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
532	case MLX5_CMD_OP_QUERY_VHCA_STATE:
533	case MLX5_CMD_OP_MODIFY_VHCA_STATE:
534	case MLX5_CMD_OP_ALLOC_SF:
535	case MLX5_CMD_OP_SUSPEND_VHCA:
536	case MLX5_CMD_OP_RESUME_VHCA:
537	case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
538	case MLX5_CMD_OP_SAVE_VHCA_STATE:
539	case MLX5_CMD_OP_LOAD_VHCA_STATE:
540	case MLX5_CMD_OP_SYNC_CRYPTO:
541	case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:
542	*status = MLX5_DRIVER_STATUS_ABORTED;
543	*synd = MLX5_DRIVER_SYND;
544	return -ENOLINK;
545	default:
546	mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
547	return -EINVAL;
548	}
549	}
550
551	const char *mlx5_command_str(int command)
552	{
553	#define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
554
555	switch (command) {
556	MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
557	MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
558	MLX5_COMMAND_STR_CASE(INIT_HCA);
559	MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
560	MLX5_COMMAND_STR_CASE(ENABLE_HCA);
561	MLX5_COMMAND_STR_CASE(DISABLE_HCA);
562	MLX5_COMMAND_STR_CASE(QUERY_PAGES);
563	MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
564	MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
565	MLX5_COMMAND_STR_CASE(QUERY_ISSI);
566	MLX5_COMMAND_STR_CASE(SET_ISSI);
567	MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
568	MLX5_COMMAND_STR_CASE(CREATE_MKEY);
569	MLX5_COMMAND_STR_CASE(QUERY_MKEY);
570	MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
571	MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
572	MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
573	MLX5_COMMAND_STR_CASE(CREATE_EQ);
574	MLX5_COMMAND_STR_CASE(DESTROY_EQ);
575	MLX5_COMMAND_STR_CASE(QUERY_EQ);
576	MLX5_COMMAND_STR_CASE(GEN_EQE);
577	MLX5_COMMAND_STR_CASE(CREATE_CQ);
578	MLX5_COMMAND_STR_CASE(DESTROY_CQ);
579	MLX5_COMMAND_STR_CASE(QUERY_CQ);
580	MLX5_COMMAND_STR_CASE(MODIFY_CQ);
581	MLX5_COMMAND_STR_CASE(CREATE_QP);
582	MLX5_COMMAND_STR_CASE(DESTROY_QP);
583	MLX5_COMMAND_STR_CASE(RST2INIT_QP);
584	MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
585	MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
586	MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
587	MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
588	MLX5_COMMAND_STR_CASE(2ERR_QP);
589	MLX5_COMMAND_STR_CASE(2RST_QP);
590	MLX5_COMMAND_STR_CASE(QUERY_QP);
591	MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
592	MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
593	MLX5_COMMAND_STR_CASE(CREATE_PSV);
594	MLX5_COMMAND_STR_CASE(DESTROY_PSV);
595	MLX5_COMMAND_STR_CASE(CREATE_SRQ);
596	MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
597	MLX5_COMMAND_STR_CASE(QUERY_SRQ);
598	MLX5_COMMAND_STR_CASE(ARM_RQ);
599	MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
600	MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
601	MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
602	MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
603	MLX5_COMMAND_STR_CASE(CREATE_DCT);
604	MLX5_COMMAND_STR_CASE(DESTROY_DCT);
605	MLX5_COMMAND_STR_CASE(DRAIN_DCT);
606	MLX5_COMMAND_STR_CASE(QUERY_DCT);
607	MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
608	MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
609	MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
610	MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
611	MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
612	MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
613	MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
614	MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
615	MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
616	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
617	MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
618	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
619	MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
620	MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
621	MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
622	MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
623	MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
624	MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
625	MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
626	MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
627	MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
628	MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
629	MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
630	MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
631	MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
632	MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
633	MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
634	MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
635	MLX5_COMMAND_STR_CASE(ALLOC_PD);
636	MLX5_COMMAND_STR_CASE(DEALLOC_PD);
637	MLX5_COMMAND_STR_CASE(ALLOC_UAR);
638	MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
639	MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
640	MLX5_COMMAND_STR_CASE(ACCESS_REG);
641	MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
642	MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
643	MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
644	MLX5_COMMAND_STR_CASE(MAD_IFC);
645	MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
646	MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
647	MLX5_COMMAND_STR_CASE(NOP);
648	MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
649	MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
650	MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
651	MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
652	MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
653	MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
654	MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
655	MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
656	MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
657	MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
658	MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
659	MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
660	MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
661	MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
662	MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
663	MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
664	MLX5_COMMAND_STR_CASE(CREATE_LAG);
665	MLX5_COMMAND_STR_CASE(MODIFY_LAG);
666	MLX5_COMMAND_STR_CASE(QUERY_LAG);
667	MLX5_COMMAND_STR_CASE(DESTROY_LAG);
668	MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
669	MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
670	MLX5_COMMAND_STR_CASE(CREATE_TIR);
671	MLX5_COMMAND_STR_CASE(MODIFY_TIR);
672	MLX5_COMMAND_STR_CASE(DESTROY_TIR);
673	MLX5_COMMAND_STR_CASE(QUERY_TIR);
674	MLX5_COMMAND_STR_CASE(CREATE_SQ);
675	MLX5_COMMAND_STR_CASE(MODIFY_SQ);
676	MLX5_COMMAND_STR_CASE(DESTROY_SQ);
677	MLX5_COMMAND_STR_CASE(QUERY_SQ);
678	MLX5_COMMAND_STR_CASE(CREATE_RQ);
679	MLX5_COMMAND_STR_CASE(MODIFY_RQ);
680	MLX5_COMMAND_STR_CASE(DESTROY_RQ);
681	MLX5_COMMAND_STR_CASE(QUERY_RQ);
682	MLX5_COMMAND_STR_CASE(CREATE_RMP);
683	MLX5_COMMAND_STR_CASE(MODIFY_RMP);
684	MLX5_COMMAND_STR_CASE(DESTROY_RMP);
685	MLX5_COMMAND_STR_CASE(QUERY_RMP);
686	MLX5_COMMAND_STR_CASE(CREATE_TIS);
687	MLX5_COMMAND_STR_CASE(MODIFY_TIS);
688	MLX5_COMMAND_STR_CASE(DESTROY_TIS);
689	MLX5_COMMAND_STR_CASE(QUERY_TIS);
690	MLX5_COMMAND_STR_CASE(CREATE_RQT);
691	MLX5_COMMAND_STR_CASE(MODIFY_RQT);
692	MLX5_COMMAND_STR_CASE(DESTROY_RQT);
693	MLX5_COMMAND_STR_CASE(QUERY_RQT);
694	MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
695	MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
696	MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
697	MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
698	MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
699	MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
700	MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
701	MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
702	MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
703	MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
704	MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
705	MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
706	MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
707	MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
708	MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
709	MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
710	MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
711	MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
712	MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
713	MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
714	MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
715	MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
716	MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
717	MLX5_COMMAND_STR_CASE(CREATE_XRQ);
718	MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
719	MLX5_COMMAND_STR_CASE(QUERY_XRQ);
720	MLX5_COMMAND_STR_CASE(ARM_XRQ);
721	MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
722	MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
723	MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
724	MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
725	MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
726	MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
727	MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
728	MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
729	MLX5_COMMAND_STR_CASE(CREATE_UCTX);
730	MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
731	MLX5_COMMAND_STR_CASE(CREATE_UMEM);
732	MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
733	MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
734	MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
735	MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
736	MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
737	MLX5_COMMAND_STR_CASE(ALLOC_SF);
738	MLX5_COMMAND_STR_CASE(DEALLOC_SF);
739	MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
740	MLX5_COMMAND_STR_CASE(RESUME_VHCA);
741	MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
742	MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
743	MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
744	MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);
745	MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);
746	default: return "unknown command opcode";
747	}
748	}
749
750	static const char *cmd_status_str(u8 status)
751	{
752	switch (status) {
753	case MLX5_CMD_STAT_OK:
754	return "OK";
755	case MLX5_CMD_STAT_INT_ERR:
756	return "internal error";
757	case MLX5_CMD_STAT_BAD_OP_ERR:
758	return "bad operation";
759	case MLX5_CMD_STAT_BAD_PARAM_ERR:
760	return "bad parameter";
761	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
762	return "bad system state";
763	case MLX5_CMD_STAT_BAD_RES_ERR:
764	return "bad resource";
765	case MLX5_CMD_STAT_RES_BUSY:
766	return "resource busy";
767	case MLX5_CMD_STAT_NOT_READY:
768	return "FW not ready";
769	case MLX5_CMD_STAT_LIM_ERR:
770	return "limits exceeded";
771	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
772	return "bad resource state";
773	case MLX5_CMD_STAT_IX_ERR:
774	return "bad index";
775	case MLX5_CMD_STAT_NO_RES_ERR:
776	return "no resources";
777	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
778	return "bad input length";
779	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
780	return "bad output length";
781	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
782	return "bad QP state";
783	case MLX5_CMD_STAT_BAD_PKT_ERR:
784	return "bad packet (discarded)";
785	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
786	return "bad size too many outstanding CQEs";
787	default:
788	return "unknown status";
789	}
790	}
791
792	static int cmd_status_to_err(u8 status)
793	{
794	switch (status) {
795	case MLX5_CMD_STAT_OK: return 0;
796	case MLX5_CMD_STAT_INT_ERR: return -EIO;
797	case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
798	case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
799	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
800	case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
801	case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
802	case MLX5_CMD_STAT_NOT_READY: return -EAGAIN;
803	case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;
804	case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
805	case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
806	case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
807	case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
808	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
809	case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
810	case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
811	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
812	default: return -EIO;
813	}
814	}
815
816	void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
817	{
818	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
819	u8 status = MLX5_GET(mbox_out, out, status);
820
821	mlx5_core_err_rl(dev,
822	"%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
823	mlx5_command_str(opcode), opcode, op_mod,
824	cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
825	}
826	EXPORT_SYMBOL(mlx5_cmd_out_err);
827
828	static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
829	{
830	u16 opcode, op_mod;
831	u8 status;
832	u16 uid;
833
834	opcode = in_to_opcode(in);
835	op_mod = MLX5_GET(mbox_in, in, op_mod);
836	uid = in_to_uid(in);
837	status = MLX5_GET(mbox_out, out, status);
838
839	if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&
840	opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)
841	mlx5_cmd_out_err(dev, opcode, op_mod, out);
842	}
843
844	int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
845	{
846	/* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
847	if (err == -ENXIO) {
848	u16 opcode = in_to_opcode(in);
849	u32 syndrome;
850	u8 status;
851
852	/* PCI Error, emulate command return status, for smooth reset */
853	err = mlx5_internal_err_ret_value(dev, op: opcode, synd: &syndrome, status: &status);
854	MLX5_SET(mbox_out, out, status, status);
855	MLX5_SET(mbox_out, out, syndrome, syndrome);
856	if (!err)
857	return 0;
858	}
859
860	/* driver or FW delivery error */
861	if (err != -EREMOTEIO && err)
862	return err;
863
864	/* check outbox status */
865	err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
866	if (err)
867	cmd_status_print(dev, in, out);
868
869	return err;
870	}
871	EXPORT_SYMBOL(mlx5_cmd_check);
872
873	static void dump_command(struct mlx5_core_dev *dev,
874	struct mlx5_cmd_work_ent *ent, int input)
875	{
876	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
877	struct mlx5_cmd_mailbox *next = msg->next;
878	int n = mlx5_calc_cmd_blocks(msg);
879	u16 op = ent->op;
880	int data_only;
881	u32 offset = 0;
882	int dump_len;
883	int i;
884
885	mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
886	data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
887
888	if (data_only)
889	mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
890	"cmd[%d]: dump command data %s(0x%x) %s\n",
891	ent->idx, mlx5_command_str(op), op,
892	input ? "INPUT" : "OUTPUT");
893	else
894	mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
895	ent->idx, mlx5_command_str(op), op,
896	input ? "INPUT" : "OUTPUT");
897
898	if (data_only) {
899	if (input) {
900	dump_buf(buf: ent->lay->in, size: sizeof(ent->lay->in), data_only: 1, offset, idx: ent->idx);
901	offset += sizeof(ent->lay->in);
902	} else {
903	dump_buf(buf: ent->lay->out, size: sizeof(ent->lay->out), data_only: 1, offset, idx: ent->idx);
904	offset += sizeof(ent->lay->out);
905	}
906	} else {
907	dump_buf(buf: ent->lay, size: sizeof(*ent->lay), data_only: 0, offset, idx: ent->idx);
908	offset += sizeof(*ent->lay);
909	}
910
911	for (i = 0; i < n && next; i++) {
912	if (data_only) {
913	dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
914	dump_buf(buf: next->buf, size: dump_len, data_only: 1, offset, idx: ent->idx);
915	offset += MLX5_CMD_DATA_BLOCK_SIZE;
916	} else {
917	mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
918	dump_buf(buf: next->buf, size: sizeof(struct mlx5_cmd_prot_block), data_only: 0, offset,
919	idx: ent->idx);
920	offset += sizeof(struct mlx5_cmd_prot_block);
921	}
922	next = next->next;
923	}
924
925	if (data_only)
926	pr_debug("\n");
927
928	mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
929	}
930
931	static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
932
933	static void cb_timeout_handler(struct work_struct *work)
934	{
935	struct delayed_work *dwork = to_delayed_work(work);
936	struct mlx5_cmd_work_ent *ent = container_of(dwork,
937	struct mlx5_cmd_work_ent,
938	cb_timeout_work);
939	struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
940	cmd);
941
942	mlx5_cmd_eq_recover(dev);
943
944	/* Maybe got handled by eq recover ? */
945	if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
946	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
947	mlx5_command_str(ent->op), ent->op);
948	goto out; /* phew, already handled */
949	}
950
951	ent->ret = -ETIMEDOUT;
952	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
953	ent->idx, mlx5_command_str(ent->op), ent->op);
954	mlx5_cmd_comp_handler(dev, vec: 1ULL << ent->idx, forced: true);
955
956	out:
957	cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
958	}
959
960	static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
961	static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
962	struct mlx5_cmd_msg *msg);
963
964	static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
965	{
966	if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
967	return true;
968
969	return cmd->allowed_opcode == opcode;
970	}
971
972	bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
973	{
974	return pci_channel_offline(pdev: dev->pdev) ||
975	dev->cmd.state != MLX5_CMDIF_STATE_UP ||
976	dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
977	}
978
979	static void cmd_work_handler(struct work_struct *work)
980	{
981	struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
982	struct mlx5_cmd *cmd = ent->cmd;
983	bool poll_cmd = ent->polling;
984	struct mlx5_cmd_layout *lay;
985	struct mlx5_core_dev *dev;
986	unsigned long timeout;
987	unsigned long flags;
988	int alloc_ret;
989	int cmd_mode;
990
991	complete(&ent->handling);
992
993	dev = container_of(cmd, struct mlx5_core_dev, cmd);
994	timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
995
996	if (!ent->page_queue) {
997	if (down_timeout(sem: &cmd->vars.sem, jiffies: timeout)) {
998	mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",
999	mlx5_command_str(ent->op), ent->op);
1000	if (ent->callback) {
1001	ent->callback(-EBUSY, ent->context);
1002	mlx5_free_cmd_msg(dev, msg: ent->out);
1003	free_msg(dev, msg: ent->in);
1004	cmd_ent_put(ent);
1005	} else {
1006	ent->ret = -EBUSY;
1007	complete(&ent->done);
1008	}
1009	complete(&ent->slotted);
1010	return;
1011	}
1012	alloc_ret = cmd_alloc_index(cmd, ent);
1013	if (alloc_ret < 0) {
1014	mlx5_core_err_rl(dev, "failed to allocate command entry\n");
1015	if (ent->callback) {
1016	ent->callback(-EAGAIN, ent->context);
1017	mlx5_free_cmd_msg(dev, msg: ent->out);
1018	free_msg(dev, msg: ent->in);
1019	cmd_ent_put(ent);
1020	} else {
1021	ent->ret = -EAGAIN;
1022	complete(&ent->done);
1023	}
1024	up(sem: &cmd->vars.sem);
1025	complete(&ent->slotted);
1026	return;
1027	}
1028	} else {
1029	down(sem: &cmd->vars.pages_sem);
1030	ent->idx = cmd->vars.max_reg_cmds;
1031	spin_lock_irqsave(&cmd->alloc_lock, flags);
1032	clear_bit(nr: ent->idx, addr: &cmd->vars.bitmask);
1033	cmd->ent_arr[ent->idx] = ent;
1034	spin_unlock_irqrestore(lock: &cmd->alloc_lock, flags);
1035	}
1036
1037	complete(&ent->slotted);
1038
1039	lay = get_inst(cmd, idx: ent->idx);
1040	ent->lay = lay;
1041	memset(lay, 0, sizeof(*lay));
1042	memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
1043	if (ent->in->next)
1044	lay->in_ptr = cpu_to_be64(ent->in->next->dma);
1045	lay->inlen = cpu_to_be32(ent->in->len);
1046	if (ent->out->next)
1047	lay->out_ptr = cpu_to_be64(ent->out->next->dma);
1048	lay->outlen = cpu_to_be32(ent->out->len);
1049	lay->type = MLX5_PCI_CMD_XPORT;
1050	lay->token = ent->token;
1051	lay->status_own = CMD_OWNER_HW;
1052	set_signature(ent, csum: !cmd->checksum_disabled);
1053	dump_command(dev, ent, input: 1);
1054	ent->ts1 = ktime_get_ns();
1055	cmd_mode = cmd->mode;
1056
1057	if (ent->callback && schedule_delayed_work(dwork: &ent->cb_timeout_work, delay: timeout))
1058	cmd_ent_get(ent);
1059	set_bit(nr: MLX5_CMD_ENT_STATE_PENDING_COMP, addr: &ent->state);
1060
1061	cmd_ent_get(ent); /* for the _real_ FW event on completion */
1062	/* Skip sending command to fw if internal error */
1063	if (mlx5_cmd_is_down(dev) || !opcode_allowed(cmd: &dev->cmd, opcode: ent->op)) {
1064	ent->ret = -ENXIO;
1065	mlx5_cmd_comp_handler(dev, vec: 1ULL << ent->idx, forced: true);
1066	return;
1067	}
1068
1069	/* ring doorbell after the descriptor is valid */
1070	mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1071	wmb();
1072	iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1073	/* if not in polling don't use ent after this point */
1074	if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1075	poll_timeout(ent);
1076	/* make sure we read the descriptor after ownership is SW */
1077	rmb();
1078	mlx5_cmd_comp_handler(dev, vec: 1ULL << ent->idx, forced: !!ent->ret);
1079	}
1080	}
1081
1082	static int deliv_status_to_err(u8 status)
1083	{
1084	switch (status) {
1085	case MLX5_CMD_DELIVERY_STAT_OK:
1086	case MLX5_DRIVER_STATUS_ABORTED:
1087	return 0;
1088	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1089	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1090	return -EBADR;
1091	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1092	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1093	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1094	return -EFAULT; /* Bad address */
1095	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1096	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1097	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1098	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1099	return -ENOMSG;
1100	case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1101	return -EIO;
1102	default:
1103	return -EINVAL;
1104	}
1105	}
1106
1107	static const char *deliv_status_to_str(u8 status)
1108	{
1109	switch (status) {
1110	case MLX5_CMD_DELIVERY_STAT_OK:
1111	return "no errors";
1112	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1113	return "signature error";
1114	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1115	return "token error";
1116	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1117	return "bad block number";
1118	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1119	return "output pointer not aligned to block size";
1120	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1121	return "input pointer not aligned to block size";
1122	case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1123	return "firmware internal error";
1124	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1125	return "command input length error";
1126	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1127	return "command output length error";
1128	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1129	return "reserved fields not cleared";
1130	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1131	return "bad command descriptor type";
1132	default:
1133	return "unknown status code";
1134	}
1135	}
1136
1137	enum {
1138	MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
1139	};
1140
1141	static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1142	struct mlx5_cmd_work_ent *ent)
1143	{
1144	unsigned long timeout = msecs_to_jiffies(m: MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1145
1146	mlx5_cmd_eq_recover(dev);
1147
1148	/* Re-wait on the ent->done after executing the recovery flow. If the
1149	* recovery flow (or any other recovery flow running simultaneously)
1150	* has recovered an EQE, it should cause the entry to be completed by
1151	* the command interface.
1152	*/
1153	if (wait_for_completion_timeout(x: &ent->done, timeout)) {
1154	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1155	mlx5_command_str(ent->op), ent->op);
1156	return;
1157	}
1158
1159	mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1160	mlx5_command_str(ent->op), ent->op);
1161
1162	ent->ret = -ETIMEDOUT;
1163	mlx5_cmd_comp_handler(dev, vec: 1ULL << ent->idx, forced: true);
1164	}
1165
1166	static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1167	{
1168	unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1169	struct mlx5_cmd *cmd = &dev->cmd;
1170	int err;
1171
1172	if (!wait_for_completion_timeout(x: &ent->handling, timeout) &&
1173	cancel_work_sync(work: &ent->work)) {
1174	ent->ret = -ECANCELED;
1175	goto out_err;
1176	}
1177
1178	wait_for_completion(&ent->slotted);
1179
1180	if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1181	wait_for_completion(&ent->done);
1182	else if (!wait_for_completion_timeout(x: &ent->done, timeout))
1183	wait_func_handle_exec_timeout(dev, ent);
1184
1185	out_err:
1186	err = ent->ret;
1187
1188	if (err == -ETIMEDOUT) {
1189	mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1190	mlx5_command_str(ent->op), ent->op);
1191	} else if (err == -ECANCELED) {
1192	mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1193	mlx5_command_str(ent->op), ent->op);
1194	} else if (err == -EBUSY) {
1195	mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",
1196	mlx5_command_str(ent->op), ent->op);
1197	}
1198	mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1199	err, deliv_status_to_str(ent->status), ent->status);
1200
1201	return err;
1202	}
1203
1204	/* Check if all command slots are stalled (timed out and not recovered).
1205	* returns true if all slots timed out on a recent command and have not been
1206	* completed by FW yet. (stalled state)
1207	* false otherwise (at least one slot is not stalled).
1208	*
1209	* In such odd situation "all_stalled", this serves as a protection mechanism
1210	* to avoid blocking the kernel for long periods of time in case FW is not
1211	* responding to commands.
1212	*/
1213	static bool mlx5_cmd_all_stalled(struct mlx5_core_dev *dev)
1214	{
1215	struct mlx5_cmd *cmd = &dev->cmd;
1216	bool all_stalled = true;
1217	unsigned long flags;
1218	int i;
1219
1220	spin_lock_irqsave(&cmd->alloc_lock, flags);
1221
1222	/* at least one command slot is free */
1223	if (bitmap_weight(src: &cmd->vars.bitmask, nbits: cmd->vars.max_reg_cmds) > 0) {
1224	all_stalled = false;
1225	goto out;
1226	}
1227
1228	for_each_clear_bit(i, &cmd->vars.bitmask, cmd->vars.max_reg_cmds) {
1229	struct mlx5_cmd_work_ent *ent = dev->cmd.ent_arr[i];
1230
1231	if (!test_bit(MLX5_CMD_ENT_STATE_TIMEDOUT, &ent->state)) {
1232	all_stalled = false;
1233	break;
1234	}
1235	}
1236	out:
1237	spin_unlock_irqrestore(lock: &cmd->alloc_lock, flags);
1238
1239	return all_stalled;
1240	}
1241
1242	/* Notes:
1243	* 1. Callback functions may not sleep
1244	* 2. page queue commands do not support asynchrous completion
1245	*
1246	* return value in case (!callback):
1247	* ret < 0 : Command execution couldn't be submitted by driver
1248	* ret > 0 : Command execution couldn't be performed by firmware
1249	* ret == 0: Command was executed by FW, Caller must check FW outbox status.
1250	*
1251	* return value in case (callback):
1252	* ret < 0 : Command execution couldn't be submitted by driver
1253	* ret == 0: Command will be submitted to FW for execution
1254	* and the callback will be called for further status updates
1255	*/
1256	static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1257	struct mlx5_cmd_msg *out, void *uout, int uout_size,
1258	mlx5_cmd_cbk_t callback,
1259	void *context, int page_queue,
1260	u8 token, bool force_polling)
1261	{
1262	struct mlx5_cmd *cmd = &dev->cmd;
1263	struct mlx5_cmd_work_ent *ent;
1264	struct mlx5_cmd_stats *stats;
1265	u8 status = 0;
1266	int err = 0;
1267	s64 ds;
1268
1269	if (callback && page_queue)
1270	return -EINVAL;
1271
1272	if (!page_queue && mlx5_cmd_all_stalled(dev)) {
1273	mlx5_core_err_rl(dev,
1274	"All CMD slots are stalled, aborting command\n");
1275	/* there's no reason to wait and block the whole kernel if FW
1276	* isn't currently responding to all slots, fail immediately
1277	*/
1278	return -EAGAIN;
1279	}
1280
1281	ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1282	cbk: callback, context, page_queue);
1283	if (IS_ERR(ptr: ent))
1284	return PTR_ERR(ptr: ent);
1285
1286	/* put for this ent is when consumed, depending on the use case
1287	* 1) (!callback) blocking flow: by caller after wait_func completes
1288	* 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1289	*/
1290
1291	ent->token = token;
1292	ent->polling = force_polling;
1293
1294	init_completion(x: &ent->handling);
1295	init_completion(x: &ent->slotted);
1296	if (!callback)
1297	init_completion(x: &ent->done);
1298
1299	INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1300	INIT_WORK(&ent->work, cmd_work_handler);
1301	if (page_queue) {
1302	cmd_work_handler(work: &ent->work);
1303	} else if (!queue_work(wq: cmd->wq, work: &ent->work)) {
1304	mlx5_core_warn(dev, "failed to queue work\n");
1305	err = -EALREADY;
1306	goto out_free;
1307	}
1308
1309	if (callback)
1310	return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1311
1312	err = wait_func(dev, ent);
1313	if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)
1314	goto out_free;
1315
1316	ds = ent->ts2 - ent->ts1;
1317	stats = xa_load(&cmd->stats, index: ent->op);
1318	if (stats) {
1319	spin_lock_irq(lock: &stats->lock);
1320	stats->sum += ds;
1321	++stats->n;
1322	spin_unlock_irq(lock: &stats->lock);
1323	}
1324	mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1325	"fw exec time for %s is %lld nsec\n",
1326	mlx5_command_str(ent->op), ds);
1327
1328	out_free:
1329	status = ent->status;
1330	cmd_ent_put(ent);
1331	return err ? : status;
1332	}
1333
1334	static ssize_t dbg_write(struct file *filp, const char __user *buf,
1335	size_t count, loff_t *pos)
1336	{
1337	struct mlx5_core_dev *dev = filp->private_data;
1338	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1339	char lbuf[3];
1340	int err;
1341
1342	if (!dbg->in_msg || !dbg->out_msg)
1343	return -ENOMEM;
1344
1345	if (count < sizeof(lbuf) - 1)
1346	return -EINVAL;
1347
1348	if (copy_from_user(to: lbuf, from: buf, n: sizeof(lbuf) - 1))
1349	return -EFAULT;
1350
1351	lbuf[sizeof(lbuf) - 1] = 0;
1352
1353	if (strcmp(lbuf, "go"))
1354	return -EINVAL;
1355
1356	err = mlx5_cmd_exec(dev, in: dbg->in_msg, in_size: dbg->inlen, out: dbg->out_msg, out_size: dbg->outlen);
1357
1358	return err ? err : count;
1359	}
1360
1361	static const struct file_operations fops = {
1362	.owner = THIS_MODULE,
1363	.open = simple_open,
1364	.write = dbg_write,
1365	};
1366
1367	static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1368	u8 token)
1369	{
1370	struct mlx5_cmd_prot_block *block;
1371	struct mlx5_cmd_mailbox *next;
1372	int copy;
1373
1374	if (!to || !from)
1375	return -ENOMEM;
1376
1377	copy = min_t(int, size, sizeof(to->first.data));
1378	memcpy(to->first.data, from, copy);
1379	size -= copy;
1380	from += copy;
1381
1382	next = to->next;
1383	while (size) {
1384	if (!next) {
1385	/* this is a BUG */
1386	return -ENOMEM;
1387	}
1388
1389	copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1390	block = next->buf;
1391	memcpy(block->data, from, copy);
1392	from += copy;
1393	size -= copy;
1394	block->token = token;
1395	next = next->next;
1396	}
1397
1398	return 0;
1399	}
1400
1401	static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1402	{
1403	struct mlx5_cmd_prot_block *block;
1404	struct mlx5_cmd_mailbox *next;
1405	int copy;
1406
1407	if (!to || !from)
1408	return -ENOMEM;
1409
1410	copy = min_t(int, size, sizeof(from->first.data));
1411	memcpy(to, from->first.data, copy);
1412	size -= copy;
1413	to += copy;
1414
1415	next = from->next;
1416	while (size) {
1417	if (!next) {
1418	/* this is a BUG */
1419	return -ENOMEM;
1420	}
1421
1422	copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1423	block = next->buf;
1424
1425	memcpy(to, block->data, copy);
1426	to += copy;
1427	size -= copy;
1428	next = next->next;
1429	}
1430
1431	return 0;
1432	}
1433
1434	static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1435	gfp_t flags)
1436	{
1437	struct mlx5_cmd_mailbox *mailbox;
1438
1439	mailbox = kmalloc(sizeof(*mailbox), flags);
1440	if (!mailbox)
1441	return ERR_PTR(error: -ENOMEM);
1442
1443	mailbox->buf = dma_pool_zalloc(pool: dev->cmd.pool, mem_flags: flags,
1444	handle: &mailbox->dma);
1445	if (!mailbox->buf) {
1446	mlx5_core_dbg(dev, "failed allocation\n");
1447	kfree(objp: mailbox);
1448	return ERR_PTR(error: -ENOMEM);
1449	}
1450	mailbox->next = NULL;
1451
1452	return mailbox;
1453	}
1454
1455	static void free_cmd_box(struct mlx5_core_dev *dev,
1456	struct mlx5_cmd_mailbox *mailbox)
1457	{
1458	dma_pool_free(pool: dev->cmd.pool, vaddr: mailbox->buf, addr: mailbox->dma);
1459	kfree(objp: mailbox);
1460	}
1461
1462	static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1463	gfp_t flags, int size,
1464	u8 token)
1465	{
1466	struct mlx5_cmd_mailbox *tmp, *head = NULL;
1467	struct mlx5_cmd_prot_block *block;
1468	struct mlx5_cmd_msg *msg;
1469	int err;
1470	int n;
1471	int i;
1472
1473	msg = kzalloc(sizeof(*msg), flags);
1474	if (!msg)
1475	return ERR_PTR(error: -ENOMEM);
1476
1477	msg->len = size;
1478	n = mlx5_calc_cmd_blocks(msg);
1479
1480	for (i = 0; i < n; i++) {
1481	tmp = alloc_cmd_box(dev, flags);
1482	if (IS_ERR(ptr: tmp)) {
1483	mlx5_core_warn(dev, "failed allocating block\n");
1484	err = PTR_ERR(ptr: tmp);
1485	goto err_alloc;
1486	}
1487
1488	block = tmp->buf;
1489	tmp->next = head;
1490	block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1491	block->block_num = cpu_to_be32(n - i - 1);
1492	block->token = token;
1493	head = tmp;
1494	}
1495	msg->next = head;
1496	return msg;
1497
1498	err_alloc:
1499	while (head) {
1500	tmp = head->next;
1501	free_cmd_box(dev, mailbox: head);
1502	head = tmp;
1503	}
1504	kfree(objp: msg);
1505
1506	return ERR_PTR(error: err);
1507	}
1508
1509	static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1510	struct mlx5_cmd_msg *msg)
1511	{
1512	struct mlx5_cmd_mailbox *head = msg->next;
1513	struct mlx5_cmd_mailbox *next;
1514
1515	while (head) {
1516	next = head->next;
1517	free_cmd_box(dev, mailbox: head);
1518	head = next;
1519	}
1520	kfree(objp: msg);
1521	}
1522
1523	static ssize_t data_write(struct file *filp, const char __user *buf,
1524	size_t count, loff_t *pos)
1525	{
1526	struct mlx5_core_dev *dev = filp->private_data;
1527	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1528	void *ptr;
1529
1530	if (*pos != 0)
1531	return -EINVAL;
1532
1533	kfree(objp: dbg->in_msg);
1534	dbg->in_msg = NULL;
1535	dbg->inlen = 0;
1536	ptr = memdup_user(buf, count);
1537	if (IS_ERR(ptr))
1538	return PTR_ERR(ptr);
1539	dbg->in_msg = ptr;
1540	dbg->inlen = count;
1541
1542	*pos = count;
1543
1544	return count;
1545	}
1546
1547	static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1548	loff_t *pos)
1549	{
1550	struct mlx5_core_dev *dev = filp->private_data;
1551	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1552
1553	if (!dbg->out_msg)
1554	return -ENOMEM;
1555
1556	return simple_read_from_buffer(to: buf, count, ppos: pos, from: dbg->out_msg,
1557	available: dbg->outlen);
1558	}
1559
1560	static const struct file_operations dfops = {
1561	.owner = THIS_MODULE,
1562	.open = simple_open,
1563	.write = data_write,
1564	.read = data_read,
1565	};
1566
1567	static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1568	loff_t *pos)
1569	{
1570	struct mlx5_core_dev *dev = filp->private_data;
1571	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1572	char outlen[8];
1573	int err;
1574
1575	err = snprintf(buf: outlen, size: sizeof(outlen), fmt: "%d", dbg->outlen);
1576	if (err < 0)
1577	return err;
1578
1579	return simple_read_from_buffer(to: buf, count, ppos: pos, from: outlen, available: err);
1580	}
1581
1582	static ssize_t outlen_write(struct file *filp, const char __user *buf,
1583	size_t count, loff_t *pos)
1584	{
1585	struct mlx5_core_dev *dev = filp->private_data;
1586	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1587	char outlen_str[8] = {0};
1588	int outlen;
1589	void *ptr;
1590	int err;
1591
1592	if (*pos != 0 || count > 6)
1593	return -EINVAL;
1594
1595	kfree(objp: dbg->out_msg);
1596	dbg->out_msg = NULL;
1597	dbg->outlen = 0;
1598
1599	if (copy_from_user(to: outlen_str, from: buf, n: count))
1600	return -EFAULT;
1601
1602	err = sscanf(outlen_str, "%d", &outlen);
1603	if (err != 1)
1604	return -EINVAL;
1605
1606	ptr = kzalloc(outlen, GFP_KERNEL);
1607	if (!ptr)
1608	return -ENOMEM;
1609
1610	dbg->out_msg = ptr;
1611	dbg->outlen = outlen;
1612
1613	*pos = count;
1614
1615	return count;
1616	}
1617
1618	static const struct file_operations olfops = {
1619	.owner = THIS_MODULE,
1620	.open = simple_open,
1621	.write = outlen_write,
1622	.read = outlen_read,
1623	};
1624
1625	static void set_wqname(struct mlx5_core_dev *dev)
1626	{
1627	struct mlx5_cmd *cmd = &dev->cmd;
1628
1629	snprintf(buf: cmd->wq_name, size: sizeof(cmd->wq_name), fmt: "mlx5_cmd_%s",
1630	dev_name(dev: dev->device));
1631	}
1632
1633	static void clean_debug_files(struct mlx5_core_dev *dev)
1634	{
1635	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1636
1637	if (!mlx5_debugfs_root)
1638	return;
1639
1640	debugfs_remove_recursive(dentry: dbg->dbg_root);
1641	}
1642
1643	static void create_debugfs_files(struct mlx5_core_dev *dev)
1644	{
1645	struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1646
1647	dbg->dbg_root = debugfs_create_dir(name: "cmd", parent: mlx5_debugfs_get_dev_root(dev));
1648
1649	debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1650	debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1651	debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1652	debugfs_create_u8(name: "status", mode: 0600, parent: dbg->dbg_root, value: &dbg->status);
1653	debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1654	}
1655
1656	void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1657	{
1658	struct mlx5_cmd *cmd = &dev->cmd;
1659	int i;
1660
1661	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1662	down(sem: &cmd->vars.sem);
1663	down(sem: &cmd->vars.pages_sem);
1664
1665	cmd->allowed_opcode = opcode;
1666
1667	up(sem: &cmd->vars.pages_sem);
1668	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1669	up(sem: &cmd->vars.sem);
1670	}
1671
1672	static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1673	{
1674	struct mlx5_cmd *cmd = &dev->cmd;
1675	int i;
1676
1677	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1678	down(sem: &cmd->vars.sem);
1679	down(sem: &cmd->vars.pages_sem);
1680
1681	cmd->mode = mode;
1682
1683	up(sem: &cmd->vars.pages_sem);
1684	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1685	up(sem: &cmd->vars.sem);
1686	}
1687
1688	static int cmd_comp_notifier(struct notifier_block *nb,
1689	unsigned long type, void *data)
1690	{
1691	struct mlx5_core_dev *dev;
1692	struct mlx5_cmd *cmd;
1693	struct mlx5_eqe *eqe;
1694
1695	cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1696	dev = container_of(cmd, struct mlx5_core_dev, cmd);
1697	eqe = data;
1698
1699	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1700	return NOTIFY_DONE;
1701
1702	mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), forced: false);
1703
1704	return NOTIFY_OK;
1705	}
1706	void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1707	{
1708	MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1709	mlx5_eq_notifier_register(dev, nb: &dev->cmd.nb);
1710	mlx5_cmd_change_mod(dev, mode: CMD_MODE_EVENTS);
1711	}
1712
1713	void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1714	{
1715	mlx5_cmd_change_mod(dev, mode: CMD_MODE_POLLING);
1716	mlx5_eq_notifier_unregister(dev, nb: &dev->cmd.nb);
1717	}
1718
1719	static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1720	{
1721	unsigned long flags;
1722
1723	if (msg->parent) {
1724	spin_lock_irqsave(&msg->parent->lock, flags);
1725	list_add_tail(new: &msg->list, head: &msg->parent->head);
1726	spin_unlock_irqrestore(lock: &msg->parent->lock, flags);
1727	} else {
1728	mlx5_free_cmd_msg(dev, msg);
1729	}
1730	}
1731
1732	static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1733	{
1734	struct mlx5_cmd *cmd = &dev->cmd;
1735	struct mlx5_cmd_work_ent *ent;
1736	mlx5_cmd_cbk_t callback;
1737	void *context;
1738	int err;
1739	int i;
1740	s64 ds;
1741	struct mlx5_cmd_stats *stats;
1742	unsigned long flags;
1743	unsigned long vector;
1744
1745	/* there can be at most 32 command queues */
1746	vector = vec & 0xffffffff;
1747	for (i = 0; i < (1 << cmd->vars.log_sz); i++) {
1748	if (test_bit(i, &vector)) {
1749	ent = cmd->ent_arr[i];
1750
1751	if (forced && ent->ret == -ETIMEDOUT)
1752	set_bit(nr: MLX5_CMD_ENT_STATE_TIMEDOUT,
1753	addr: &ent->state);
1754	else if (!forced) /* real FW completion */
1755	clear_bit(nr: MLX5_CMD_ENT_STATE_TIMEDOUT,
1756	addr: &ent->state);
1757
1758	/* if we already completed the command, ignore it */
1759	if (!test_and_clear_bit(nr: MLX5_CMD_ENT_STATE_PENDING_COMP,
1760	addr: &ent->state)) {
1761	/* only real completion can free the cmd slot */
1762	if (!forced) {
1763	mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1764	ent->idx);
1765	cmd_ent_put(ent);
1766	}
1767	continue;
1768	}
1769
1770	if (ent->callback && cancel_delayed_work(dwork: &ent->cb_timeout_work))
1771	cmd_ent_put(ent); /* timeout work was canceled */
1772
1773	if (!forced || /* Real FW completion */
1774	mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1775	!opcode_allowed(cmd, opcode: ent->op))
1776	cmd_ent_put(ent);
1777
1778	ent->ts2 = ktime_get_ns();
1779	memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1780	dump_command(dev, ent, input: 0);
1781
1782	if (vec & MLX5_TRIGGERED_CMD_COMP)
1783	ent->ret = -ENXIO;
1784
1785	if (!ent->ret) { /* Command completed by FW */
1786	if (!cmd->checksum_disabled)
1787	ent->ret = verify_signature(ent);
1788
1789	ent->status = ent->lay->status_own >> 1;
1790
1791	mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1792	ent->ret, deliv_status_to_str(ent->status), ent->status);
1793	}
1794
1795	if (ent->callback) {
1796	ds = ent->ts2 - ent->ts1;
1797	stats = xa_load(&cmd->stats, index: ent->op);
1798	if (stats) {
1799	spin_lock_irqsave(&stats->lock, flags);
1800	stats->sum += ds;
1801	++stats->n;
1802	spin_unlock_irqrestore(lock: &stats->lock, flags);
1803	}
1804
1805	callback = ent->callback;
1806	context = ent->context;
1807	err = ent->ret ? : ent->status;
1808	if (err > 0) /* Failed in FW, command didn't execute */
1809	err = deliv_status_to_err(status: err);
1810
1811	if (!err)
1812	err = mlx5_copy_from_msg(to: ent->uout,
1813	from: ent->out,
1814	size: ent->uout_size);
1815
1816	mlx5_free_cmd_msg(dev, msg: ent->out);
1817	free_msg(dev, msg: ent->in);
1818
1819	/* final consumer is done, release ent */
1820	cmd_ent_put(ent);
1821	callback(err, context);
1822	} else {
1823	/* release wait_func() so mlx5_cmd_invoke()
1824	* can make the final ent_put()
1825	*/
1826	complete(&ent->done);
1827	}
1828	}
1829	}
1830	}
1831
1832	#define MLX5_MAX_MANAGE_PAGES_CMD_ENT 1
1833	#define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \
1834	MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1)
1835
1836	static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1837	{
1838	struct mlx5_cmd *cmd = &dev->cmd;
1839	unsigned long bitmask;
1840	unsigned long flags;
1841	u64 vector;
1842	int i;
1843
1844	/* wait for pending handlers to complete */
1845	mlx5_eq_synchronize_cmd_irq(dev);
1846	spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1847	vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK;
1848	if (!vector)
1849	goto no_trig;
1850
1851	bitmask = vector;
1852	/* we must increment the allocated entries refcount before triggering the completions
1853	* to guarantee pending commands will not get freed in the meanwhile.
1854	* For that reason, it also has to be done inside the alloc_lock.
1855	*/
1856	for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1857	cmd_ent_get(ent: cmd->ent_arr[i]);
1858	vector |= MLX5_TRIGGERED_CMD_COMP;
1859	spin_unlock_irqrestore(lock: &dev->cmd.alloc_lock, flags);
1860
1861	mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1862	mlx5_cmd_comp_handler(dev, vec: vector, forced: true);
1863	for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))
1864	cmd_ent_put(ent: cmd->ent_arr[i]);
1865	return;
1866
1867	no_trig:
1868	spin_unlock_irqrestore(lock: &dev->cmd.alloc_lock, flags);
1869	}
1870
1871	void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1872	{
1873	struct mlx5_cmd *cmd = &dev->cmd;
1874	int i;
1875
1876	for (i = 0; i < cmd->vars.max_reg_cmds; i++) {
1877	while (down_trylock(sem: &cmd->vars.sem)) {
1878	mlx5_cmd_trigger_completions(dev);
1879	cond_resched();
1880	}
1881	}
1882
1883	while (down_trylock(sem: &cmd->vars.pages_sem)) {
1884	mlx5_cmd_trigger_completions(dev);
1885	cond_resched();
1886	}
1887
1888	/* Unlock cmdif */
1889	up(sem: &cmd->vars.pages_sem);
1890	for (i = 0; i < cmd->vars.max_reg_cmds; i++)
1891	up(sem: &cmd->vars.sem);
1892	}
1893
1894	static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1895	gfp_t gfp)
1896	{
1897	struct mlx5_cmd_msg *msg = ERR_PTR(error: -ENOMEM);
1898	struct cmd_msg_cache *ch = NULL;
1899	struct mlx5_cmd *cmd = &dev->cmd;
1900	int i;
1901
1902	if (in_size <= 16)
1903	goto cache_miss;
1904
1905	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
1906	ch = &cmd->cache[i];
1907	if (in_size > ch->max_inbox_size)
1908	continue;
1909	spin_lock_irq(lock: &ch->lock);
1910	if (list_empty(head: &ch->head)) {
1911	spin_unlock_irq(lock: &ch->lock);
1912	continue;
1913	}
1914	msg = list_entry(ch->head.next, typeof(*msg), list);
1915	/* For cached lists, we must explicitly state what is
1916	* the real size
1917	*/
1918	msg->len = in_size;
1919	list_del(entry: &msg->list);
1920	spin_unlock_irq(lock: &ch->lock);
1921	break;
1922	}
1923
1924	if (!IS_ERR(ptr: msg))
1925	return msg;
1926
1927	cache_miss:
1928	msg = mlx5_alloc_cmd_msg(dev, flags: gfp, size: in_size, token: 0);
1929	return msg;
1930	}
1931
1932	static int is_manage_pages(void *in)
1933	{
1934	return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;
1935	}
1936
1937	static bool mlx5_has_privileged_uid(struct mlx5_core_dev *dev)
1938	{
1939	return !xa_empty(xa: &dev->cmd.vars.privileged_uids);
1940	}
1941
1942	static bool mlx5_cmd_is_privileged_uid(struct mlx5_core_dev *dev,
1943	u16 uid)
1944	{
1945	return !!xa_load(&dev->cmd.vars.privileged_uids, index: uid);
1946	}
1947
1948	/* Notes:
1949	* 1. Callback functions may not sleep
1950	* 2. Page queue commands do not support asynchrous completion
1951	*/
1952	static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1953	int out_size, mlx5_cmd_cbk_t callback, void *context,
1954	bool force_polling)
1955	{
1956	struct mlx5_cmd_msg *inb, *outb;
1957	u16 opcode = in_to_opcode(in);
1958	bool throttle_locked = false;
1959	bool unpriv_locked = false;
1960	u16 uid = in_to_uid(in);
1961	int pages_queue;
1962	gfp_t gfp;
1963	u8 token;
1964	int err;
1965
1966	if (mlx5_cmd_is_down(dev) || !opcode_allowed(cmd: &dev->cmd, opcode))
1967	return -ENXIO;
1968
1969	if (!callback) {
1970	/* The semaphore is already held for callback commands. It was
1971	* acquired in mlx5_cmd_exec_cb()
1972	*/
1973	if (uid && mlx5_has_privileged_uid(dev)) {
1974	if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
1975	unpriv_locked = true;
1976	down(sem: &dev->cmd.vars.unprivileged_sem);
1977	}
1978	} else if (mlx5_cmd_is_throttle_opcode(op: opcode)) {
1979	throttle_locked = true;
1980	down(sem: &dev->cmd.vars.throttle_sem);
1981	}
1982	}
1983
1984	pages_queue = is_manage_pages(in);
1985	gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1986
1987	inb = alloc_msg(dev, in_size, gfp);
1988	if (IS_ERR(inb)) {
1989	err = PTR_ERR(inb);
1990	goto out_up;
1991	}
1992
1993	token = alloc_token(cmd: &dev->cmd);
1994
1995	err = mlx5_copy_to_msg(inb, from: in, size: in_size, token);
1996	if (err) {
1997	mlx5_core_warn(dev, "err %d\n", err);
1998	goto out_in;
1999	}
2000
2001	outb = mlx5_alloc_cmd_msg(dev, flags: gfp, size: out_size, token);
2002	if (IS_ERR(outb)) {
2003	err = PTR_ERR(outb);
2004	goto out_in;
2005	}
2006
2007	err = mlx5_cmd_invoke(dev, inb, outb, uout: out, uout_size: out_size, callback, context,
2008	page_queue: pages_queue, token, force_polling);
2009	if (callback && !err)
2010	return 0;
2011
2012	if (err > 0) /* Failed in FW, command didn't execute */
2013	err = deliv_status_to_err(status: err);
2014
2015	if (err)
2016	goto out_out;
2017
2018	/* command completed by FW */
2019	err = mlx5_copy_from_msg(to: out, outb, size: out_size);
2020	out_out:
2021	mlx5_free_cmd_msg(dev, outb);
2022	out_in:
2023	free_msg(dev, inb);
2024	out_up:
2025	if (throttle_locked)
2026	up(sem: &dev->cmd.vars.throttle_sem);
2027	if (unpriv_locked)
2028	up(sem: &dev->cmd.vars.unprivileged_sem);
2029
2030	return err;
2031	}
2032
2033	static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
2034	{
2035	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2036	u8 status = MLX5_GET(mbox_out, out, status);
2037
2038	trace_mlx5_cmd(command_str: mlx5_command_str(command: opcode), opcode, op_mod,
2039	status_str: cmd_status_str(status), status, syndrome,
2040	err: cmd_status_to_err(status));
2041	}
2042
2043	static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
2044	u32 syndrome, int err)
2045	{
2046	const char *namep = mlx5_command_str(command: opcode);
2047	struct mlx5_cmd_stats *stats;
2048	unsigned long flags;
2049
2050	if (!err || !(strcmp(namep, "unknown command opcode")))
2051	return;
2052
2053	stats = xa_load(&dev->cmd.stats, index: opcode);
2054	if (!stats)
2055	return;
2056	spin_lock_irqsave(&stats->lock, flags);
2057	stats->failed++;
2058	if (err < 0)
2059	stats->last_failed_errno = -err;
2060	if (err == -EREMOTEIO) {
2061	stats->failed_mbox_status++;
2062	stats->last_failed_mbox_status = status;
2063	stats->last_failed_syndrome = syndrome;
2064	}
2065	spin_unlock_irqrestore(lock: &stats->lock, flags);
2066	}
2067
2068	/* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
2069	static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
2070	{
2071	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
2072	u8 status = MLX5_GET(mbox_out, out, status);
2073
2074	if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
2075	err = -EIO;
2076
2077	if (!err && status != MLX5_CMD_STAT_OK) {
2078	err = -EREMOTEIO;
2079	mlx5_cmd_err_trace(dev, opcode, op_mod, out);
2080	}
2081
2082	cmd_status_log(dev, opcode, status, syndrome, err);
2083	return err;
2084	}
2085
2086	/**
2087	* mlx5_cmd_do - Executes a fw command, wait for completion.
2088	* Unlike mlx5_cmd_exec, this function will not translate or intercept
2089	* outbox.status and will return -EREMOTEIO when
2090	* outbox.status != MLX5_CMD_STAT_OK
2091	*
2092	* @dev: mlx5 core device
2093	* @in: inbox mlx5_ifc command buffer
2094	* @in_size: inbox buffer size
2095	* @out: outbox mlx5_ifc buffer
2096	* @out_size: outbox size
2097	*
2098	* @return:
2099	* -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
2100	* Caller must check FW outbox status.
2101	* 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
2102	* < 0 : Command execution couldn't be performed by firmware or driver
2103	*/
2104	int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
2105	{
2106	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, force_polling: false);
2107	u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2108	u16 opcode = in_to_opcode(in);
2109
2110	return cmd_status_err(dev, err, opcode, op_mod, out);
2111	}
2112	EXPORT_SYMBOL(mlx5_cmd_do);
2113
2114	/**
2115	* mlx5_cmd_exec - Executes a fw command, wait for completion
2116	*
2117	* @dev: mlx5 core device
2118	* @in: inbox mlx5_ifc command buffer
2119	* @in_size: inbox buffer size
2120	* @out: outbox mlx5_ifc buffer
2121	* @out_size: outbox size
2122	*
2123	* @return: 0 if no error, FW command execution was successful
2124	* and outbox status is ok.
2125	*/
2126	int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
2127	int out_size)
2128	{
2129	int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
2130
2131	return mlx5_cmd_check(dev, err, in, out);
2132	}
2133	EXPORT_SYMBOL(mlx5_cmd_exec);
2134
2135	/**
2136	* mlx5_cmd_exec_polling - Executes a fw command, poll for completion
2137	* Needed for driver force teardown, when command completion EQ
2138	* will not be available to complete the command
2139	*
2140	* @dev: mlx5 core device
2141	* @in: inbox mlx5_ifc command buffer
2142	* @in_size: inbox buffer size
2143	* @out: outbox mlx5_ifc buffer
2144	* @out_size: outbox size
2145	*
2146	* @return: 0 if no error, FW command execution was successful
2147	* and outbox status is ok.
2148	*/
2149	int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
2150	void *out, int out_size)
2151	{
2152	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, force_polling: true);
2153	u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2154	u16 opcode = in_to_opcode(in);
2155
2156	err = cmd_status_err(dev, err, opcode, op_mod, out);
2157	return mlx5_cmd_check(dev, err, in, out);
2158	}
2159	EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2160
2161	void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2162	struct mlx5_async_ctx *ctx)
2163	{
2164	ctx->dev = dev;
2165	/* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2166	atomic_set(v: &ctx->num_inflight, i: 1);
2167	init_completion(x: &ctx->inflight_done);
2168	}
2169	EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2170
2171	/**
2172	* mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2173	* @ctx: The ctx to clean
2174	*
2175	* Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2176	* caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2177	* the call mlx5_cleanup_async_ctx().
2178	*/
2179	void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2180	{
2181	if (!atomic_dec_and_test(v: &ctx->num_inflight))
2182	wait_for_completion(&ctx->inflight_done);
2183	}
2184	EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2185
2186	static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2187	{
2188	struct mlx5_async_work *work = _work;
2189	struct mlx5_async_ctx *ctx;
2190	struct mlx5_core_dev *dev;
2191	bool throttle_locked;
2192	bool unpriv_locked;
2193
2194	ctx = work->ctx;
2195	dev = ctx->dev;
2196	throttle_locked = work->throttle_locked;
2197	unpriv_locked = work->unpriv_locked;
2198	status = cmd_status_err(dev, err: status, opcode: work->opcode, op_mod: work->op_mod, out: work->out);
2199	work->user_callback(status, work);
2200	/* Can't access "work" from this point on. It could have been freed in
2201	* the callback.
2202	*/
2203	if (throttle_locked)
2204	up(sem: &dev->cmd.vars.throttle_sem);
2205	if (unpriv_locked)
2206	up(sem: &dev->cmd.vars.unprivileged_sem);
2207	if (atomic_dec_and_test(v: &ctx->num_inflight))
2208	complete(&ctx->inflight_done);
2209	}
2210
2211	int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2212	void *out, int out_size, mlx5_async_cbk_t callback,
2213	struct mlx5_async_work *work)
2214	{
2215	struct mlx5_core_dev *dev = ctx->dev;
2216	u16 uid;
2217	int ret;
2218
2219	work->ctx = ctx;
2220	work->user_callback = callback;
2221	work->opcode = in_to_opcode(in);
2222	work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2223	work->out = out;
2224	work->throttle_locked = false;
2225	work->unpriv_locked = false;
2226	uid = in_to_uid(in);
2227
2228	if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2229	return -EIO;
2230
2231	if (uid && mlx5_has_privileged_uid(dev)) {
2232	if (!mlx5_cmd_is_privileged_uid(dev, uid)) {
2233	if (down_trylock(sem: &dev->cmd.vars.unprivileged_sem)) {
2234	ret = -EBUSY;
2235	goto dec_num_inflight;
2236	}
2237	work->unpriv_locked = true;
2238	}
2239	} else if (mlx5_cmd_is_throttle_opcode(op: in_to_opcode(in))) {
2240	if (down_trylock(sem: &dev->cmd.vars.throttle_sem)) {
2241	ret = -EBUSY;
2242	goto dec_num_inflight;
2243	}
2244	work->throttle_locked = true;
2245	}
2246
2247	ret = cmd_exec(dev, in, in_size, out, out_size,
2248	callback: mlx5_cmd_exec_cb_handler, context: work, force_polling: false);
2249	if (ret)
2250	goto sem_up;
2251
2252	return 0;
2253
2254	sem_up:
2255	if (work->throttle_locked)
2256	up(sem: &dev->cmd.vars.throttle_sem);
2257	if (work->unpriv_locked)
2258	up(sem: &dev->cmd.vars.unprivileged_sem);
2259	dec_num_inflight:
2260	if (atomic_dec_and_test(v: &ctx->num_inflight))
2261	complete(&ctx->inflight_done);
2262
2263	return ret;
2264	}
2265	EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2266
2267	int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,
2268	struct mlx5_cmd_allow_other_vhca_access_attr *attr)
2269	{
2270	u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};
2271	u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};
2272	void *key;
2273
2274	MLX5_SET(allow_other_vhca_access_in,
2275	in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);
2276	MLX5_SET(allow_other_vhca_access_in,
2277	in, object_type_to_be_accessed, attr->obj_type);
2278	MLX5_SET(allow_other_vhca_access_in,
2279	in, object_id_to_be_accessed, attr->obj_id);
2280
2281	key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);
2282	memcpy(key, attr->access_key, sizeof(attr->access_key));
2283
2284	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2285	}
2286
2287	int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
2288	struct mlx5_cmd_alias_obj_create_attr *alias_attr,
2289	u32 *obj_id)
2290	{
2291	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2292	u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};
2293	void *param;
2294	void *attr;
2295	void *key;
2296	int ret;
2297
2298	attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);
2299	MLX5_SET(general_obj_in_cmd_hdr,
2300	attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2301	MLX5_SET(general_obj_in_cmd_hdr,
2302	attr, obj_type, alias_attr->obj_type);
2303	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
2304	MLX5_SET(general_obj_create_param, param, alias_object, 1);
2305
2306	attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
2307	MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
2308	MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
2309
2310	key = MLX5_ADDR_OF(alias_context, attr, access_key);
2311	memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));
2312
2313	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2314	if (ret)
2315	return ret;
2316
2317	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2318
2319	return 0;
2320	}
2321
2322	int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,
2323	u16 obj_type)
2324	{
2325	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
2326	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
2327
2328	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
2329	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);
2330	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
2331
2332	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
2333	}
2334
2335	static void destroy_msg_cache(struct mlx5_core_dev *dev)
2336	{
2337	struct cmd_msg_cache *ch;
2338	struct mlx5_cmd_msg *msg;
2339	struct mlx5_cmd_msg *n;
2340	int i;
2341
2342	for (i = 0; i < dev->profile.num_cmd_caches; i++) {
2343	ch = &dev->cmd.cache[i];
2344	list_for_each_entry_safe(msg, n, &ch->head, list) {
2345	list_del(entry: &msg->list);
2346	mlx5_free_cmd_msg(dev, msg);
2347	}
2348	}
2349	}
2350
2351	static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2352	512, 32, 16, 8, 2
2353	};
2354
2355	static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2356	16 + MLX5_CMD_DATA_BLOCK_SIZE,
2357	16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2358	16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2359	16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2360	16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2361	};
2362
2363	static void create_msg_cache(struct mlx5_core_dev *dev)
2364	{
2365	struct mlx5_cmd *cmd = &dev->cmd;
2366	struct cmd_msg_cache *ch;
2367	struct mlx5_cmd_msg *msg;
2368	int i;
2369	int k;
2370
2371	/* Initialize and fill the caches with initial entries */
2372	for (k = 0; k < dev->profile.num_cmd_caches; k++) {
2373	ch = &cmd->cache[k];
2374	spin_lock_init(&ch->lock);
2375	INIT_LIST_HEAD(list: &ch->head);
2376	ch->num_ent = cmd_cache_num_ent[k];
2377	ch->max_inbox_size = cmd_cache_ent_size[k];
2378	for (i = 0; i < ch->num_ent; i++) {
2379	msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2380	size: ch->max_inbox_size, token: 0);
2381	if (IS_ERR(ptr: msg))
2382	break;
2383	msg->parent = ch;
2384	list_add_tail(new: &msg->list, head: &ch->head);
2385	}
2386	}
2387	}
2388
2389	static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2390	{
2391	cmd->cmd_alloc_buf = dma_alloc_coherent(dev: mlx5_core_dma_dev(dev), size: MLX5_ADAPTER_PAGE_SIZE,
2392	dma_handle: &cmd->alloc_dma, GFP_KERNEL);
2393	if (!cmd->cmd_alloc_buf)
2394	return -ENOMEM;
2395
2396	/* make sure it is aligned to 4K */
2397	if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2398	cmd->cmd_buf = cmd->cmd_alloc_buf;
2399	cmd->dma = cmd->alloc_dma;
2400	cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2401	return 0;
2402	}
2403
2404	dma_free_coherent(dev: mlx5_core_dma_dev(dev), size: MLX5_ADAPTER_PAGE_SIZE, cpu_addr: cmd->cmd_alloc_buf,
2405	dma_handle: cmd->alloc_dma);
2406	cmd->cmd_alloc_buf = dma_alloc_coherent(dev: mlx5_core_dma_dev(dev),
2407	size: 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2408	dma_handle: &cmd->alloc_dma, GFP_KERNEL);
2409	if (!cmd->cmd_alloc_buf)
2410	return -ENOMEM;
2411
2412	cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2413	cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2414	cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2415	return 0;
2416	}
2417
2418	static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2419	{
2420	dma_free_coherent(dev: mlx5_core_dma_dev(dev), size: cmd->alloc_size, cpu_addr: cmd->cmd_alloc_buf,
2421	dma_handle: cmd->alloc_dma);
2422	}
2423
2424	static u16 cmdif_rev(struct mlx5_core_dev *dev)
2425	{
2426	return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2427	}
2428
2429	int mlx5_cmd_init(struct mlx5_core_dev *dev)
2430	{
2431	struct mlx5_cmd *cmd = &dev->cmd;
2432
2433	cmd->checksum_disabled = 1;
2434
2435	spin_lock_init(&cmd->alloc_lock);
2436	spin_lock_init(&cmd->token_lock);
2437
2438	set_wqname(dev);
2439	cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2440	if (!cmd->wq) {
2441	mlx5_core_err(dev, "failed to create command workqueue\n");
2442	return -ENOMEM;
2443	}
2444
2445	mlx5_cmdif_debugfs_init(dev);
2446
2447	return 0;
2448	}
2449
2450	void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2451	{
2452	struct mlx5_cmd *cmd = &dev->cmd;
2453
2454	mlx5_cmdif_debugfs_cleanup(dev);
2455	destroy_workqueue(wq: cmd->wq);
2456	}
2457
2458	int mlx5_cmd_enable(struct mlx5_core_dev *dev)
2459	{
2460	int size = sizeof(struct mlx5_cmd_prot_block);
2461	int align = roundup_pow_of_two(size);
2462	struct mlx5_cmd *cmd = &dev->cmd;
2463	u32 cmd_h, cmd_l;
2464	int err;
2465
2466	memset(&cmd->vars, 0, sizeof(cmd->vars));
2467	cmd->vars.cmdif_rev = cmdif_rev(dev);
2468	if (cmd->vars.cmdif_rev != CMD_IF_REV) {
2469	mlx5_core_err(dev,
2470	"Driver cmdif rev(%d) differs from firmware's(%d)\n",
2471	CMD_IF_REV, cmd->vars.cmdif_rev);
2472	return -EINVAL;
2473	}
2474
2475	cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2476	cmd->vars.log_sz = cmd_l >> 4 & 0xf;
2477	cmd->vars.log_stride = cmd_l & 0xf;
2478	if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {
2479	mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2480	1 << cmd->vars.log_sz);
2481	return -EINVAL;
2482	}
2483
2484	if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2485	mlx5_core_err(dev, "command queue size overflow\n");
2486	return -EINVAL;
2487	}
2488
2489	cmd->state = MLX5_CMDIF_STATE_DOWN;
2490	cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;
2491	cmd->vars.bitmask = MLX5_CMD_MASK;
2492
2493	sema_init(sem: &cmd->vars.sem, val: cmd->vars.max_reg_cmds);
2494	sema_init(sem: &cmd->vars.pages_sem, val: 1);
2495	sema_init(sem: &cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2496	sema_init(sem: &cmd->vars.unprivileged_sem,
2497	DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));
2498
2499	xa_init(xa: &cmd->vars.privileged_uids);
2500
2501	cmd->pool = dma_pool_create(name: "mlx5_cmd", dev: mlx5_core_dma_dev(dev), size, align, boundary: 0);
2502	if (!cmd->pool) {
2503	err = -ENOMEM;
2504	goto err_destroy_xa;
2505	}
2506
2507	err = alloc_cmd_page(dev, cmd);
2508	if (err)
2509	goto err_free_pool;
2510
2511	cmd_h = (u32)((u64)(cmd->dma) >> 32);
2512	cmd_l = (u32)(cmd->dma);
2513	if (cmd_l & 0xfff) {
2514	mlx5_core_err(dev, "invalid command queue address\n");
2515	err = -ENOMEM;
2516	goto err_cmd_page;
2517	}
2518
2519	iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2520	iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2521
2522	/* Make sure firmware sees the complete address before we proceed */
2523	wmb();
2524
2525	mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2526
2527	cmd->mode = CMD_MODE_POLLING;
2528	cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2529
2530	create_msg_cache(dev);
2531	create_debugfs_files(dev);
2532
2533	return 0;
2534
2535	err_cmd_page:
2536	free_cmd_page(dev, cmd);
2537	err_free_pool:
2538	dma_pool_destroy(pool: cmd->pool);
2539	err_destroy_xa:
2540	xa_destroy(&dev->cmd.vars.privileged_uids);
2541	return err;
2542	}
2543
2544	void mlx5_cmd_disable(struct mlx5_core_dev *dev)
2545	{
2546	struct mlx5_cmd *cmd = &dev->cmd;
2547
2548	flush_workqueue(cmd->wq);
2549	clean_debug_files(dev);
2550	destroy_msg_cache(dev);
2551	free_cmd_page(dev, cmd);
2552	dma_pool_destroy(pool: cmd->pool);
2553	xa_destroy(&dev->cmd.vars.privileged_uids);
2554	}
2555
2556	void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2557	enum mlx5_cmdif_state cmdif_state)
2558	{
2559	dev->cmd.state = cmdif_state;
2560	}
2561
2562	int mlx5_cmd_add_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2563	{
2564	return xa_insert(xa: &dev->cmd.vars.privileged_uids, index: uid,
2565	entry: xa_mk_value(v: uid), GFP_KERNEL);
2566	}
2567	EXPORT_SYMBOL(mlx5_cmd_add_privileged_uid);
2568
2569	void mlx5_cmd_remove_privileged_uid(struct mlx5_core_dev *dev, u16 uid)
2570	{
2571	void *data = xa_erase(&dev->cmd.vars.privileged_uids, index: uid);
2572
2573	WARN(!data, "Privileged UID %u does not exist\n", uid);
2574	}
2575	EXPORT_SYMBOL(mlx5_cmd_remove_privileged_uid);
2576