1	/*
2	* Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3	* Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4	*
5	* This software is available to you under a choice of one of two
6	* licenses. You may choose to be licensed under the terms of the GNU
7	* General Public License (GPL) Version 2, available from the file
8	* COPYING in the main directory of this source tree, or the
9	* OpenIB.org BSD license below:
10	*
11	* Redistribution and use in source and binary forms, with or
12	* without modification, are permitted provided that the following
13	* conditions are met:
14	*
15	* - Redistributions of source code must retain the above
16	* copyright notice, this list of conditions and the following
17	* disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above
20	* copyright notice, this list of conditions and the following
21	* disclaimer in the documentation and/or other materials
22	* provided with the distribution.
23	*
24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31	* SOFTWARE.
32	*/
33
34	#include <linux/string.h>
35	#include <linux/etherdevice.h>
36
37	#include <linux/mlx4/cmd.h>
38	#include <linux/mlx4/qp.h>
39	#include <linux/export.h>
40
41	#include "mlx4.h"
42
43	int mlx4_get_mgm_entry_size(struct mlx4_dev *dev)
44	{
45	return 1 << dev->oper_log_mgm_entry_size;
46	}
47
48	int mlx4_get_qp_per_mgm(struct mlx4_dev *dev)
49	{
50	return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2);
51	}
52
53	static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev,
54	struct mlx4_cmd_mailbox *mailbox,
55	u32 size,
56	u64 *reg_id)
57	{
58	u64 imm;
59	int err = 0;
60
61	err = mlx4_cmd_imm(dev, in_param: mailbox->dma, out_param: &imm, in_modifier: size, op_modifier: 0,
62	op: MLX4_QP_FLOW_STEERING_ATTACH, timeout: MLX4_CMD_TIME_CLASS_A,
63	native: MLX4_CMD_NATIVE);
64	if (err)
65	return err;
66	*reg_id = imm;
67
68	return err;
69	}
70
71	static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid)
72	{
73	int err = 0;
74
75	err = mlx4_cmd(dev, in_param: regid, in_modifier: 0, op_modifier: 0,
76	op: MLX4_QP_FLOW_STEERING_DETACH, timeout: MLX4_CMD_TIME_CLASS_A,
77	native: MLX4_CMD_NATIVE);
78
79	return err;
80	}
81
82	static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index,
83	struct mlx4_cmd_mailbox *mailbox)
84	{
85	return mlx4_cmd_box(dev, in_param: 0, out_param: mailbox->dma, in_modifier: index, op_modifier: 0, op: MLX4_CMD_READ_MCG,
86	timeout: MLX4_CMD_TIME_CLASS_A, native: MLX4_CMD_NATIVE);
87	}
88
89	static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index,
90	struct mlx4_cmd_mailbox *mailbox)
91	{
92	return mlx4_cmd(dev, in_param: mailbox->dma, in_modifier: index, op_modifier: 0, op: MLX4_CMD_WRITE_MCG,
93	timeout: MLX4_CMD_TIME_CLASS_A, native: MLX4_CMD_NATIVE);
94	}
95
96	static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer,
97	struct mlx4_cmd_mailbox *mailbox)
98	{
99	u32 in_mod;
100
101	in_mod = (u32) port << 16 | steer << 1;
102	return mlx4_cmd(dev, in_param: mailbox->dma, in_modifier: in_mod, op_modifier: 0x1,
103	op: MLX4_CMD_WRITE_MCG, timeout: MLX4_CMD_TIME_CLASS_A,
104	native: MLX4_CMD_NATIVE);
105	}
106
107	static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
108	u16 *hash, u8 op_mod)
109	{
110	u64 imm;
111	int err;
112
113	err = mlx4_cmd_imm(dev, in_param: mailbox->dma, out_param: &imm, in_modifier: 0, op_modifier: op_mod,
114	op: MLX4_CMD_MGID_HASH, timeout: MLX4_CMD_TIME_CLASS_A,
115	native: MLX4_CMD_NATIVE);
116
117	if (!err)
118	*hash = imm;
119
120	return err;
121	}
122
123	static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port,
124	enum mlx4_steer_type steer,
125	u32 qpn)
126	{
127	struct mlx4_steer *s_steer;
128	struct mlx4_promisc_qp *pqp;
129
130	if (port < 1 || port > dev->caps.num_ports)
131	return NULL;
132
133	s_steer = &mlx4_priv(dev)->steer[port - 1];
134
135	list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) {
136	if (pqp->qpn == qpn)
137	return pqp;
138	}
139	/* not found */
140	return NULL;
141	}
142
143	/*
144	* Add new entry to steering data structure.
145	* All promisc QPs should be added as well
146	*/
147	static int new_steering_entry(struct mlx4_dev *dev, u8 port,
148	enum mlx4_steer_type steer,
149	unsigned int index, u32 qpn)
150	{
151	struct mlx4_steer *s_steer;
152	struct mlx4_cmd_mailbox *mailbox;
153	struct mlx4_mgm *mgm;
154	u32 members_count;
155	struct mlx4_steer_index *new_entry;
156	struct mlx4_promisc_qp *pqp;
157	struct mlx4_promisc_qp *dqp = NULL;
158	u32 prot;
159	int err;
160
161	if (port < 1 || port > dev->caps.num_ports)
162	return -EINVAL;
163
164	s_steer = &mlx4_priv(dev)->steer[port - 1];
165	new_entry = kzalloc(size: sizeof(*new_entry), GFP_KERNEL);
166	if (!new_entry)
167	return -ENOMEM;
168
169	INIT_LIST_HEAD(list: &new_entry->duplicates);
170	new_entry->index = index;
171	list_add_tail(new: &new_entry->list, head: &s_steer->steer_entries[steer]);
172
173	/* If the given qpn is also a promisc qp,
174	* it should be inserted to duplicates list
175	*/
176	pqp = get_promisc_qp(dev, port, steer, qpn);
177	if (pqp) {
178	dqp = kmalloc(size: sizeof(*dqp), GFP_KERNEL);
179	if (!dqp) {
180	err = -ENOMEM;
181	goto out_alloc;
182	}
183	dqp->qpn = qpn;
184	list_add_tail(new: &dqp->list, head: &new_entry->duplicates);
185	}
186
187	/* if no promisc qps for this vep, we are done */
188	if (list_empty(head: &s_steer->promisc_qps[steer]))
189	return 0;
190
191	/* now need to add all the promisc qps to the new
192	* steering entry, as they should also receive the packets
193	* destined to this address */
194	mailbox = mlx4_alloc_cmd_mailbox(dev);
195	if (IS_ERR(ptr: mailbox)) {
196	err = -ENOMEM;
197	goto out_alloc;
198	}
199	mgm = mailbox->buf;
200
201	err = mlx4_READ_ENTRY(dev, index, mailbox);
202	if (err)
203	goto out_mailbox;
204
205	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
206	prot = be32_to_cpu(mgm->members_count) >> 30;
207	list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) {
208	/* don't add already existing qpn */
209	if (pqp->qpn == qpn)
210	continue;
211	if (members_count == dev->caps.num_qp_per_mgm) {
212	/* out of space */
213	err = -ENOMEM;
214	goto out_mailbox;
215	}
216
217	/* add the qpn */
218	mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK);
219	}
220	/* update the qps count and update the entry with all the promisc qps*/
221	mgm->members_count = cpu_to_be32(members_count | (prot << 30));
222	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
223
224	out_mailbox:
225	mlx4_free_cmd_mailbox(dev, mailbox);
226	if (!err)
227	return 0;
228	out_alloc:
229	if (dqp) {
230	list_del(entry: &dqp->list);
231	kfree(objp: dqp);
232	}
233	list_del(entry: &new_entry->list);
234	kfree(objp: new_entry);
235	return err;
236	}
237
238	/* update the data structures with existing steering entry */
239	static int existing_steering_entry(struct mlx4_dev *dev, u8 port,
240	enum mlx4_steer_type steer,
241	unsigned int index, u32 qpn)
242	{
243	struct mlx4_steer *s_steer;
244	struct mlx4_steer_index *tmp_entry, *entry = NULL;
245	struct mlx4_promisc_qp *pqp;
246	struct mlx4_promisc_qp *dqp;
247
248	if (port < 1 || port > dev->caps.num_ports)
249	return -EINVAL;
250
251	s_steer = &mlx4_priv(dev)->steer[port - 1];
252
253	pqp = get_promisc_qp(dev, port, steer, qpn);
254	if (!pqp)
255	return 0; /* nothing to do */
256
257	list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) {
258	if (tmp_entry->index == index) {
259	entry = tmp_entry;
260	break;
261	}
262	}
263	if (unlikely(!entry)) {
264	mlx4_warn(dev, "Steering entry at index %x is not registered\n", index);
265	return -EINVAL;
266	}
267
268	/* the given qpn is listed as a promisc qpn
269	* we need to add it as a duplicate to this entry
270	* for future references */
271	list_for_each_entry(dqp, &entry->duplicates, list) {
272	if (qpn == dqp->qpn)
273	return 0; /* qp is already duplicated */
274	}
275
276	/* add the qp as a duplicate on this index */
277	dqp = kmalloc(size: sizeof(*dqp), GFP_KERNEL);
278	if (!dqp)
279	return -ENOMEM;
280	dqp->qpn = qpn;
281	list_add_tail(new: &dqp->list, head: &entry->duplicates);
282
283	return 0;
284	}
285
286	/* Check whether a qpn is a duplicate on steering entry
287	* If so, it should not be removed from mgm */
288	static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port,
289	enum mlx4_steer_type steer,
290	unsigned int index, u32 qpn)
291	{
292	struct mlx4_steer *s_steer;
293	struct mlx4_steer_index *tmp_entry, *entry = NULL;
294	struct mlx4_promisc_qp *dqp, *tmp_dqp;
295
296	if (port < 1 || port > dev->caps.num_ports)
297	return false;
298
299	s_steer = &mlx4_priv(dev)->steer[port - 1];
300
301	/* if qp is not promisc, it cannot be duplicated */
302	if (!get_promisc_qp(dev, port, steer, qpn))
303	return false;
304
305	/* The qp is promisc qp so it is a duplicate on this index
306	* Find the index entry, and remove the duplicate */
307	list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) {
308	if (tmp_entry->index == index) {
309	entry = tmp_entry;
310	break;
311	}
312	}
313	if (unlikely(!entry)) {
314	mlx4_warn(dev, "Steering entry for index %x is not registered\n", index);
315	return false;
316	}
317	list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) {
318	if (dqp->qpn == qpn) {
319	list_del(entry: &dqp->list);
320	kfree(objp: dqp);
321	}
322	}
323	return true;
324	}
325
326	/* Returns true if all the QPs != tqpn contained in this entry
327	* are Promisc QPs. Returns false otherwise.
328	*/
329	static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port,
330	enum mlx4_steer_type steer,
331	unsigned int index, u32 tqpn,
332	u32 *members_count)
333	{
334	struct mlx4_cmd_mailbox *mailbox;
335	struct mlx4_mgm *mgm;
336	u32 m_count;
337	bool ret = false;
338	int i;
339
340	if (port < 1 || port > dev->caps.num_ports)
341	return false;
342
343	mailbox = mlx4_alloc_cmd_mailbox(dev);
344	if (IS_ERR(ptr: mailbox))
345	return false;
346	mgm = mailbox->buf;
347
348	if (mlx4_READ_ENTRY(dev, index, mailbox))
349	goto out;
350	m_count = be32_to_cpu(mgm->members_count) & 0xffffff;
351	if (members_count)
352	*members_count = m_count;
353
354	for (i = 0; i < m_count; i++) {
355	u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK;
356	if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) {
357	/* the qp is not promisc, the entry can't be removed */
358	goto out;
359	}
360	}
361	ret = true;
362	out:
363	mlx4_free_cmd_mailbox(dev, mailbox);
364	return ret;
365	}
366
367	/* IF a steering entry contains only promisc QPs, it can be removed. */
368	static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port,
369	enum mlx4_steer_type steer,
370	unsigned int index, u32 tqpn)
371	{
372	struct mlx4_steer *s_steer;
373	struct mlx4_steer_index *entry = NULL, *tmp_entry;
374	u32 members_count;
375	bool ret = false;
376
377	if (port < 1 || port > dev->caps.num_ports)
378	return false;
379
380	s_steer = &mlx4_priv(dev)->steer[port - 1];
381
382	if (!promisc_steering_entry(dev, port, steer, index,
383	tqpn, members_count: &members_count))
384	goto out;
385
386	/* All the qps currently registered for this entry are promiscuous,
387	* Checking for duplicates */
388	ret = true;
389	list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) {
390	if (entry->index == index) {
391	if (list_empty(head: &entry->duplicates) ||
392	members_count == 1) {
393	struct mlx4_promisc_qp *pqp, *tmp_pqp;
394	/* If there is only 1 entry in duplicates then
395	* this is the QP we want to delete, going over
396	* the list and deleting the entry.
397	*/
398	list_del(entry: &entry->list);
399	list_for_each_entry_safe(pqp, tmp_pqp,
400	&entry->duplicates,
401	list) {
402	list_del(entry: &pqp->list);
403	kfree(objp: pqp);
404	}
405	kfree(objp: entry);
406	} else {
407	/* This entry contains duplicates so it shouldn't be removed */
408	ret = false;
409	goto out;
410	}
411	}
412	}
413
414	out:
415	return ret;
416	}
417
418	static int add_promisc_qp(struct mlx4_dev *dev, u8 port,
419	enum mlx4_steer_type steer, u32 qpn)
420	{
421	struct mlx4_steer *s_steer;
422	struct mlx4_cmd_mailbox *mailbox;
423	struct mlx4_mgm *mgm;
424	struct mlx4_steer_index *entry;
425	struct mlx4_promisc_qp *pqp;
426	struct mlx4_promisc_qp *dqp;
427	u32 members_count;
428	u32 prot;
429	int i;
430	bool found;
431	int err;
432	struct mlx4_priv *priv = mlx4_priv(dev);
433
434	if (port < 1 || port > dev->caps.num_ports)
435	return -EINVAL;
436
437	s_steer = &mlx4_priv(dev)->steer[port - 1];
438
439	mutex_lock(&priv->mcg_table.mutex);
440
441	if (get_promisc_qp(dev, port, steer, qpn)) {
442	err = 0; /* Noting to do, already exists */
443	goto out_mutex;
444	}
445
446	pqp = kmalloc(size: sizeof(*pqp), GFP_KERNEL);
447	if (!pqp) {
448	err = -ENOMEM;
449	goto out_mutex;
450	}
451	pqp->qpn = qpn;
452
453	mailbox = mlx4_alloc_cmd_mailbox(dev);
454	if (IS_ERR(ptr: mailbox)) {
455	err = -ENOMEM;
456	goto out_alloc;
457	}
458	mgm = mailbox->buf;
459
460	if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) {
461	/* The promisc QP needs to be added for each one of the steering
462	* entries. If it already exists, needs to be added as
463	* a duplicate for this entry.
464	*/
465	list_for_each_entry(entry,
466	&s_steer->steer_entries[steer],
467	list) {
468	err = mlx4_READ_ENTRY(dev, index: entry->index, mailbox);
469	if (err)
470	goto out_mailbox;
471
472	members_count = be32_to_cpu(mgm->members_count) &
473	0xffffff;
474	prot = be32_to_cpu(mgm->members_count) >> 30;
475	found = false;
476	for (i = 0; i < members_count; i++) {
477	if ((be32_to_cpu(mgm->qp[i]) &
478	MGM_QPN_MASK) == qpn) {
479	/* Entry already exists.
480	* Add to duplicates.
481	*/
482	dqp = kmalloc(size: sizeof(*dqp), GFP_KERNEL);
483	if (!dqp) {
484	err = -ENOMEM;
485	goto out_mailbox;
486	}
487	dqp->qpn = qpn;
488	list_add_tail(new: &dqp->list,
489	head: &entry->duplicates);
490	found = true;
491	}
492	}
493	if (!found) {
494	/* Need to add the qpn to mgm */
495	if (members_count ==
496	dev->caps.num_qp_per_mgm) {
497	/* entry is full */
498	err = -ENOMEM;
499	goto out_mailbox;
500	}
501	mgm->qp[members_count++] =
502	cpu_to_be32(qpn & MGM_QPN_MASK);
503	mgm->members_count =
504	cpu_to_be32(members_count |
505	(prot << 30));
506	err = mlx4_WRITE_ENTRY(dev, index: entry->index,
507	mailbox);
508	if (err)
509	goto out_mailbox;
510	}
511	}
512	}
513
514	/* add the new qpn to list of promisc qps */
515	list_add_tail(new: &pqp->list, head: &s_steer->promisc_qps[steer]);
516	/* now need to add all the promisc qps to default entry */
517	memset(mgm, 0, sizeof(*mgm));
518	members_count = 0;
519	list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) {
520	if (members_count == dev->caps.num_qp_per_mgm) {
521	/* entry is full */
522	err = -ENOMEM;
523	goto out_list;
524	}
525	mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK);
526	}
527	mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30);
528
529	err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox);
530	if (err)
531	goto out_list;
532
533	mlx4_free_cmd_mailbox(dev, mailbox);
534	mutex_unlock(lock: &priv->mcg_table.mutex);
535	return 0;
536
537	out_list:
538	list_del(entry: &pqp->list);
539	out_mailbox:
540	mlx4_free_cmd_mailbox(dev, mailbox);
541	out_alloc:
542	kfree(objp: pqp);
543	out_mutex:
544	mutex_unlock(lock: &priv->mcg_table.mutex);
545	return err;
546	}
547
548	static int remove_promisc_qp(struct mlx4_dev *dev, u8 port,
549	enum mlx4_steer_type steer, u32 qpn)
550	{
551	struct mlx4_priv *priv = mlx4_priv(dev);
552	struct mlx4_steer *s_steer;
553	struct mlx4_cmd_mailbox *mailbox;
554	struct mlx4_mgm *mgm;
555	struct mlx4_steer_index *entry, *tmp_entry;
556	struct mlx4_promisc_qp *pqp;
557	struct mlx4_promisc_qp *dqp;
558	u32 members_count;
559	bool found;
560	bool back_to_list = false;
561	int i;
562	int err;
563
564	if (port < 1 || port > dev->caps.num_ports)
565	return -EINVAL;
566
567	s_steer = &mlx4_priv(dev)->steer[port - 1];
568	mutex_lock(&priv->mcg_table.mutex);
569
570	pqp = get_promisc_qp(dev, port, steer, qpn);
571	if (unlikely(!pqp)) {
572	mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn);
573	/* nothing to do */
574	err = 0;
575	goto out_mutex;
576	}
577
578	/*remove from list of promisc qps */
579	list_del(entry: &pqp->list);
580
581	/* set the default entry not to include the removed one */
582	mailbox = mlx4_alloc_cmd_mailbox(dev);
583	if (IS_ERR(ptr: mailbox)) {
584	err = -ENOMEM;
585	back_to_list = true;
586	goto out_list;
587	}
588	mgm = mailbox->buf;
589	members_count = 0;
590	list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list)
591	mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK);
592	mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30);
593
594	err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox);
595	if (err)
596	goto out_mailbox;
597
598	if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) {
599	/* Remove the QP from all the steering entries */
600	list_for_each_entry_safe(entry, tmp_entry,
601	&s_steer->steer_entries[steer],
602	list) {
603	found = false;
604	list_for_each_entry(dqp, &entry->duplicates, list) {
605	if (dqp->qpn == qpn) {
606	found = true;
607	break;
608	}
609	}
610	if (found) {
611	/* A duplicate, no need to change the MGM,
612	* only update the duplicates list
613	*/
614	list_del(entry: &dqp->list);
615	kfree(objp: dqp);
616	} else {
617	int loc = -1;
618
619	err = mlx4_READ_ENTRY(dev,
620	index: entry->index,
621	mailbox);
622	if (err)
623	goto out_mailbox;
624	members_count =
625	be32_to_cpu(mgm->members_count) &
626	0xffffff;
627	if (!members_count) {
628	mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0. deleting entry...\n",
629	qpn, entry->index);
630	list_del(entry: &entry->list);
631	kfree(objp: entry);
632	continue;
633	}
634
635	for (i = 0; i < members_count; ++i)
636	if ((be32_to_cpu(mgm->qp[i]) &
637	MGM_QPN_MASK) == qpn) {
638	loc = i;
639	break;
640	}
641
642	if (loc < 0) {
643	mlx4_err(dev, "QP %06x wasn't found in entry %d\n",
644	qpn, entry->index);
645	err = -EINVAL;
646	goto out_mailbox;
647	}
648
649	/* Copy the last QP in this MGM
650	* over removed QP
651	*/
652	mgm->qp[loc] = mgm->qp[members_count - 1];
653	mgm->qp[members_count - 1] = 0;
654	mgm->members_count =
655	cpu_to_be32(--members_count |
656	(MLX4_PROT_ETH << 30));
657
658	err = mlx4_WRITE_ENTRY(dev,
659	index: entry->index,
660	mailbox);
661	if (err)
662	goto out_mailbox;
663	}
664	}
665	}
666
667	out_mailbox:
668	mlx4_free_cmd_mailbox(dev, mailbox);
669	out_list:
670	if (back_to_list)
671	list_add_tail(new: &pqp->list, head: &s_steer->promisc_qps[steer]);
672	else
673	kfree(objp: pqp);
674	out_mutex:
675	mutex_unlock(lock: &priv->mcg_table.mutex);
676	return err;
677	}
678
679	/*
680	* Caller must hold MCG table semaphore. gid and mgm parameters must
681	* be properly aligned for command interface.
682	*
683	* Returns 0 unless a firmware command error occurs.
684	*
685	* If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1
686	* and *mgm holds MGM entry.
687	*
688	* if GID is found in AMGM, *index = index in AMGM, *prev = index of
689	* previous entry in hash chain and *mgm holds AMGM entry.
690	*
691	* If no AMGM exists for given gid, *index = -1, *prev = index of last
692	* entry in hash chain and *mgm holds end of hash chain.
693	*/
694	static int find_entry(struct mlx4_dev *dev, u8 port,
695	u8 *gid, enum mlx4_protocol prot,
696	struct mlx4_cmd_mailbox *mgm_mailbox,
697	int *prev, int *index)
698	{
699	struct mlx4_cmd_mailbox *mailbox;
700	struct mlx4_mgm *mgm = mgm_mailbox->buf;
701	u8 *mgid;
702	int err;
703	u16 hash;
704	u8 op_mod = (prot == MLX4_PROT_ETH) ?
705	!!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0;
706
707	mailbox = mlx4_alloc_cmd_mailbox(dev);
708	if (IS_ERR(ptr: mailbox))
709	return -ENOMEM;
710	mgid = mailbox->buf;
711
712	memcpy(mgid, gid, 16);
713
714	err = mlx4_GID_HASH(dev, mailbox, hash: &hash, op_mod);
715	mlx4_free_cmd_mailbox(dev, mailbox);
716	if (err)
717	return err;
718
719	if (0)
720	mlx4_dbg(dev, "Hash for %pI6 is %04x\n", gid, hash);
721
722	*index = hash;
723	*prev = -1;
724
725	do {
726	err = mlx4_READ_ENTRY(dev, index: *index, mailbox: mgm_mailbox);
727	if (err)
728	return err;
729
730	if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) {
731	if (*index != hash) {
732	mlx4_err(dev, "Found zero MGID in AMGM\n");
733	err = -EINVAL;
734	}
735	return err;
736	}
737
738	if (!memcmp(p: mgm->gid, q: gid, size: 16) &&
739	be32_to_cpu(mgm->members_count) >> 30 == prot)
740	return err;
741
742	*prev = *index;
743	*index = be32_to_cpu(mgm->next_gid_index) >> 6;
744	} while (*index);
745
746	*index = -1;
747	return err;
748	}
749
750	static const u8 __promisc_mode[] = {
751	[MLX4_FS_REGULAR] = 0x0,
752	[MLX4_FS_ALL_DEFAULT] = 0x1,
753	[MLX4_FS_MC_DEFAULT] = 0x3,
754	[MLX4_FS_MIRROR_RX_PORT] = 0x4,
755	[MLX4_FS_MIRROR_SX_PORT] = 0x5,
756	[MLX4_FS_UC_SNIFFER] = 0x6,
757	[MLX4_FS_MC_SNIFFER] = 0x7,
758	};
759
760	int mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev *dev,
761	enum mlx4_net_trans_promisc_mode flow_type)
762	{
763	if (flow_type >= MLX4_FS_MODE_NUM) {
764	mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type);
765	return -EINVAL;
766	}
767	return __promisc_mode[flow_type];
768	}
769	EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_mode);
770
771	static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl,
772	struct mlx4_net_trans_rule_hw_ctrl *hw)
773	{
774	u8 flags = 0;
775
776	flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0;
777	flags |= ctrl->exclusive ? (1 << 2) : 0;
778	flags |= ctrl->allow_loopback ? (1 << 3) : 0;
779
780	hw->flags = flags;
781	hw->type = __promisc_mode[ctrl->promisc_mode];
782	hw->prio = cpu_to_be16(ctrl->priority);
783	hw->port = ctrl->port;
784	hw->qpn = cpu_to_be32(ctrl->qpn);
785	}
786
787	const u16 __sw_id_hw[] = {
788	[MLX4_NET_TRANS_RULE_ID_ETH] = 0xE001,
789	[MLX4_NET_TRANS_RULE_ID_IB] = 0xE005,
790	[MLX4_NET_TRANS_RULE_ID_IPV6] = 0xE003,
791	[MLX4_NET_TRANS_RULE_ID_IPV4] = 0xE002,
792	[MLX4_NET_TRANS_RULE_ID_TCP] = 0xE004,
793	[MLX4_NET_TRANS_RULE_ID_UDP] = 0xE006,
794	[MLX4_NET_TRANS_RULE_ID_VXLAN] = 0xE008
795	};
796
797	int mlx4_map_sw_to_hw_steering_id(struct mlx4_dev *dev,
798	enum mlx4_net_trans_rule_id id)
799	{
800	if (id >= MLX4_NET_TRANS_RULE_NUM) {
801	mlx4_err(dev, "Invalid network rule id. id = %d\n", id);
802	return -EINVAL;
803	}
804	return __sw_id_hw[id];
805	}
806	EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_id);
807
808	static const int __rule_hw_sz[] = {
809	[MLX4_NET_TRANS_RULE_ID_ETH] =
810	sizeof(struct mlx4_net_trans_rule_hw_eth),
811	[MLX4_NET_TRANS_RULE_ID_IB] =
812	sizeof(struct mlx4_net_trans_rule_hw_ib),
813	[MLX4_NET_TRANS_RULE_ID_IPV6] = 0,
814	[MLX4_NET_TRANS_RULE_ID_IPV4] =
815	sizeof(struct mlx4_net_trans_rule_hw_ipv4),
816	[MLX4_NET_TRANS_RULE_ID_TCP] =
817	sizeof(struct mlx4_net_trans_rule_hw_tcp_udp),
818	[MLX4_NET_TRANS_RULE_ID_UDP] =
819	sizeof(struct mlx4_net_trans_rule_hw_tcp_udp),
820	[MLX4_NET_TRANS_RULE_ID_VXLAN] =
821	sizeof(struct mlx4_net_trans_rule_hw_vxlan)
822	};
823
824	int mlx4_hw_rule_sz(struct mlx4_dev *dev,
825	enum mlx4_net_trans_rule_id id)
826	{
827	if (id >= MLX4_NET_TRANS_RULE_NUM) {
828	mlx4_err(dev, "Invalid network rule id. id = %d\n", id);
829	return -EINVAL;
830	}
831
832	return __rule_hw_sz[id];
833	}
834	EXPORT_SYMBOL_GPL(mlx4_hw_rule_sz);
835
836	static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec,
837	struct _rule_hw *rule_hw)
838	{
839	if (mlx4_hw_rule_sz(dev, spec->id) < 0)
840	return -EINVAL;
841	memset(rule_hw, 0, mlx4_hw_rule_sz(dev, spec->id));
842	rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]);
843	rule_hw->size = mlx4_hw_rule_sz(dev, spec->id) >> 2;
844
845	switch (spec->id) {
846	case MLX4_NET_TRANS_RULE_ID_ETH:
847	memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN);
848	memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk,
849	ETH_ALEN);
850	memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN);
851	memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk,
852	ETH_ALEN);
853	if (spec->eth.ether_type_enable) {
854	rule_hw->eth.ether_type_enable = 1;
855	rule_hw->eth.ether_type = spec->eth.ether_type;
856	}
857	rule_hw->eth.vlan_tag = spec->eth.vlan_id;
858	rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk;
859	break;
860
861	case MLX4_NET_TRANS_RULE_ID_IB:
862	rule_hw->ib.l3_qpn = spec->ib.l3_qpn;
863	rule_hw->ib.qpn_mask = spec->ib.qpn_msk;
864	memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16);
865	memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16);
866	break;
867
868	case MLX4_NET_TRANS_RULE_ID_IPV6:
869	return -EOPNOTSUPP;
870
871	case MLX4_NET_TRANS_RULE_ID_IPV4:
872	rule_hw->ipv4.src_ip = spec->ipv4.src_ip;
873	rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk;
874	rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip;
875	rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk;
876	break;
877
878	case MLX4_NET_TRANS_RULE_ID_TCP:
879	case MLX4_NET_TRANS_RULE_ID_UDP:
880	rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port;
881	rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk;
882	rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port;
883	rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk;
884	break;
885
886	case MLX4_NET_TRANS_RULE_ID_VXLAN:
887	rule_hw->vxlan.vni =
888	cpu_to_be32(be32_to_cpu(spec->vxlan.vni) << 8);
889	rule_hw->vxlan.vni_mask =
890	cpu_to_be32(be32_to_cpu(spec->vxlan.vni_mask) << 8);
891	break;
892
893	default:
894	return -EINVAL;
895	}
896
897	return __rule_hw_sz[spec->id];
898	}
899
900	static void mlx4_err_rule(struct mlx4_dev *dev, char *str,
901	struct mlx4_net_trans_rule *rule)
902	{
903	#define BUF_SIZE 256
904	struct mlx4_spec_list *cur;
905	char buf[BUF_SIZE];
906	int len = 0;
907
908	mlx4_err(dev, "%s", str);
909	len += scnprintf(buf: buf + len, BUF_SIZE - len,
910	fmt: "port = %d prio = 0x%x qp = 0x%x ",
911	rule->port, rule->priority, rule->qpn);
912
913	list_for_each_entry(cur, &rule->list, list) {
914	switch (cur->id) {
915	case MLX4_NET_TRANS_RULE_ID_ETH:
916	len += scnprintf(buf: buf + len, BUF_SIZE - len,
917	fmt: "dmac = %pM ", &cur->eth.dst_mac);
918	if (cur->eth.ether_type)
919	len += scnprintf(buf: buf + len, BUF_SIZE - len,
920	fmt: "ethertype = 0x%x ",
921	be16_to_cpu(cur->eth.ether_type));
922	if (cur->eth.vlan_id)
923	len += scnprintf(buf: buf + len, BUF_SIZE - len,
924	fmt: "vlan-id = %d ",
925	be16_to_cpu(cur->eth.vlan_id));
926	break;
927
928	case MLX4_NET_TRANS_RULE_ID_IPV4:
929	if (cur->ipv4.src_ip)
930	len += scnprintf(buf: buf + len, BUF_SIZE - len,
931	fmt: "src-ip = %pI4 ",
932	&cur->ipv4.src_ip);
933	if (cur->ipv4.dst_ip)
934	len += scnprintf(buf: buf + len, BUF_SIZE - len,
935	fmt: "dst-ip = %pI4 ",
936	&cur->ipv4.dst_ip);
937	break;
938
939	case MLX4_NET_TRANS_RULE_ID_TCP:
940	case MLX4_NET_TRANS_RULE_ID_UDP:
941	if (cur->tcp_udp.src_port)
942	len += scnprintf(buf: buf + len, BUF_SIZE - len,
943	fmt: "src-port = %d ",
944	be16_to_cpu(cur->tcp_udp.src_port));
945	if (cur->tcp_udp.dst_port)
946	len += scnprintf(buf: buf + len, BUF_SIZE - len,
947	fmt: "dst-port = %d ",
948	be16_to_cpu(cur->tcp_udp.dst_port));
949	break;
950
951	case MLX4_NET_TRANS_RULE_ID_IB:
952	len += scnprintf(buf: buf + len, BUF_SIZE - len,
953	fmt: "dst-gid = %pI6\n", cur->ib.dst_gid);
954	len += scnprintf(buf: buf + len, BUF_SIZE - len,
955	fmt: "dst-gid-mask = %pI6\n",
956	cur->ib.dst_gid_msk);
957	break;
958
959	case MLX4_NET_TRANS_RULE_ID_VXLAN:
960	len += scnprintf(buf: buf + len, BUF_SIZE - len,
961	fmt: "VNID = %d ", be32_to_cpu(cur->vxlan.vni));
962	break;
963	case MLX4_NET_TRANS_RULE_ID_IPV6:
964	break;
965
966	default:
967	break;
968	}
969	}
970	len += scnprintf(buf: buf + len, BUF_SIZE - len, fmt: "\n");
971	mlx4_err(dev, "%s", buf);
972
973	if (len >= BUF_SIZE)
974	mlx4_err(dev, "Network rule error message was truncated, print buffer is too small\n");
975	}
976
977	int mlx4_flow_attach(struct mlx4_dev *dev,
978	struct mlx4_net_trans_rule *rule, u64 *reg_id)
979	{
980	struct mlx4_cmd_mailbox *mailbox;
981	struct mlx4_spec_list *cur;
982	u32 size = 0;
983	int ret;
984
985	mailbox = mlx4_alloc_cmd_mailbox(dev);
986	if (IS_ERR(ptr: mailbox))
987	return PTR_ERR(ptr: mailbox);
988
989	if (!mlx4_qp_lookup(dev, qpn: rule->qpn)) {
990	mlx4_err_rule(dev, str: "QP doesn't exist\n", rule);
991	ret = -EINVAL;
992	goto out;
993	}
994
995	trans_rule_ctrl_to_hw(ctrl: rule, hw: mailbox->buf);
996
997	size += sizeof(struct mlx4_net_trans_rule_hw_ctrl);
998
999	list_for_each_entry(cur, &rule->list, list) {
1000	ret = parse_trans_rule(dev, spec: cur, rule_hw: mailbox->buf + size);
1001	if (ret < 0)
1002	goto out;
1003
1004	size += ret;
1005	}
1006
1007	ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size: size >> 2, reg_id);
1008	if (ret == -ENOMEM) {
1009	mlx4_err_rule(dev,
1010	str: "mcg table is full. Fail to register network rule\n",
1011	rule);
1012	} else if (ret) {
1013	if (ret == -ENXIO) {
1014	if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED)
1015	mlx4_err_rule(dev,
1016	str: "DMFS is not enabled, "
1017	"failed to register network rule.\n",
1018	rule);
1019	else
1020	mlx4_err_rule(dev,
1021	str: "Rule exceeds the dmfs_high_rate_mode limitations, "
1022	"failed to register network rule.\n",
1023	rule);
1024
1025	} else {
1026	mlx4_err_rule(dev, str: "Fail to register network rule.\n", rule);
1027	}
1028	}
1029
1030	out:
1031	mlx4_free_cmd_mailbox(dev, mailbox);
1032
1033	return ret;
1034	}
1035	EXPORT_SYMBOL_GPL(mlx4_flow_attach);
1036
1037	int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id)
1038	{
1039	int err;
1040
1041	err = mlx4_QP_FLOW_STEERING_DETACH(dev, regid: reg_id);
1042	if (err)
1043	mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n",
1044	reg_id);
1045	return err;
1046	}
1047	EXPORT_SYMBOL_GPL(mlx4_flow_detach);
1048
1049	int mlx4_tunnel_steer_add(struct mlx4_dev *dev, const unsigned char *addr,
1050	int port, int qpn, u16 prio, u64 *reg_id)
1051	{
1052	int err;
1053	struct mlx4_spec_list spec_eth_outer = { {NULL} };
1054	struct mlx4_spec_list spec_vxlan = { {NULL} };
1055	struct mlx4_spec_list spec_eth_inner = { {NULL} };
1056
1057	struct mlx4_net_trans_rule rule = {
1058	.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1059	.exclusive = 0,
1060	.allow_loopback = 1,
1061	.promisc_mode = MLX4_FS_REGULAR,
1062	};
1063
1064	__be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
1065
1066	rule.port = port;
1067	rule.qpn = qpn;
1068	rule.priority = prio;
1069	INIT_LIST_HEAD(list: &rule.list);
1070
1071	spec_eth_outer.id = MLX4_NET_TRANS_RULE_ID_ETH;
1072	memcpy(spec_eth_outer.eth.dst_mac, addr, ETH_ALEN);
1073	memcpy(spec_eth_outer.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
1074
1075	spec_vxlan.id = MLX4_NET_TRANS_RULE_ID_VXLAN; /* any vxlan header */
1076	spec_eth_inner.id = MLX4_NET_TRANS_RULE_ID_ETH; /* any inner eth header */
1077
1078	list_add_tail(new: &spec_eth_outer.list, head: &rule.list);
1079	list_add_tail(new: &spec_vxlan.list, head: &rule.list);
1080	list_add_tail(new: &spec_eth_inner.list, head: &rule.list);
1081
1082	err = mlx4_flow_attach(dev, &rule, reg_id);
1083	return err;
1084	}
1085	EXPORT_SYMBOL(mlx4_tunnel_steer_add);
1086
1087	int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn,
1088	u32 max_range_qpn)
1089	{
1090	int err;
1091	u64 in_param;
1092
1093	in_param = ((u64) min_range_qpn) << 32;
1094	in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF;
1095
1096	err = mlx4_cmd(dev, in_param, in_modifier: 0, op_modifier: 0,
1097	op: MLX4_FLOW_STEERING_IB_UC_QP_RANGE,
1098	timeout: MLX4_CMD_TIME_CLASS_A, native: MLX4_CMD_NATIVE);
1099
1100	return err;
1101	}
1102	EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE);
1103
1104	int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1105	int block_mcast_loopback, enum mlx4_protocol prot,
1106	enum mlx4_steer_type steer)
1107	{
1108	struct mlx4_priv *priv = mlx4_priv(dev);
1109	struct mlx4_cmd_mailbox *mailbox;
1110	struct mlx4_mgm *mgm;
1111	u32 members_count;
1112	int index = -1, prev;
1113	int link = 0;
1114	int i;
1115	int err;
1116	u8 port = gid[5];
1117	u8 new_entry = 0;
1118
1119	mailbox = mlx4_alloc_cmd_mailbox(dev);
1120	if (IS_ERR(ptr: mailbox))
1121	return PTR_ERR(ptr: mailbox);
1122	mgm = mailbox->buf;
1123
1124	mutex_lock(&priv->mcg_table.mutex);
1125	err = find_entry(dev, port, gid, prot,
1126	mgm_mailbox: mailbox, prev: &prev, index: &index);
1127	if (err)
1128	goto out;
1129
1130	if (index != -1) {
1131	if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) {
1132	new_entry = 1;
1133	memcpy(mgm->gid, gid, 16);
1134	}
1135	} else {
1136	link = 1;
1137
1138	index = mlx4_bitmap_alloc(bitmap: &priv->mcg_table.bitmap);
1139	if (index == -1) {
1140	mlx4_err(dev, "No AMGM entries left\n");
1141	err = -ENOMEM;
1142	goto out;
1143	}
1144	index += dev->caps.num_mgms;
1145
1146	new_entry = 1;
1147	memset(mgm, 0, sizeof(*mgm));
1148	memcpy(mgm->gid, gid, 16);
1149	}
1150
1151	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
1152	if (members_count == dev->caps.num_qp_per_mgm) {
1153	mlx4_err(dev, "MGM at index %x is full\n", index);
1154	err = -ENOMEM;
1155	goto out;
1156	}
1157
1158	for (i = 0; i < members_count; ++i)
1159	if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) {
1160	mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn);
1161	err = 0;
1162	goto out;
1163	}
1164
1165	if (block_mcast_loopback)
1166	mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) |
1167	(1U << MGM_BLCK_LB_BIT));
1168	else
1169	mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK);
1170
1171	mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30);
1172
1173	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1174	if (err)
1175	goto out;
1176
1177	if (!link)
1178	goto out;
1179
1180	err = mlx4_READ_ENTRY(dev, index: prev, mailbox);
1181	if (err)
1182	goto out;
1183
1184	mgm->next_gid_index = cpu_to_be32(index << 6);
1185
1186	err = mlx4_WRITE_ENTRY(dev, index: prev, mailbox);
1187	if (err)
1188	goto out;
1189
1190	out:
1191	if (prot == MLX4_PROT_ETH && index != -1) {
1192	/* manage the steering entry for promisc mode */
1193	if (new_entry)
1194	err = new_steering_entry(dev, port, steer,
1195	index, qpn: qp->qpn);
1196	else
1197	err = existing_steering_entry(dev, port, steer,
1198	index, qpn: qp->qpn);
1199	}
1200	if (err && link && index != -1) {
1201	if (index < dev->caps.num_mgms)
1202	mlx4_warn(dev, "Got AMGM index %d < %d\n",
1203	index, dev->caps.num_mgms);
1204	else
1205	mlx4_bitmap_free(bitmap: &priv->mcg_table.bitmap,
1206	obj: index - dev->caps.num_mgms, use_rr: MLX4_USE_RR);
1207	}
1208	mutex_unlock(lock: &priv->mcg_table.mutex);
1209
1210	mlx4_free_cmd_mailbox(dev, mailbox);
1211	return err;
1212	}
1213
1214	int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1215	enum mlx4_protocol prot, enum mlx4_steer_type steer)
1216	{
1217	struct mlx4_priv *priv = mlx4_priv(dev);
1218	struct mlx4_cmd_mailbox *mailbox;
1219	struct mlx4_mgm *mgm;
1220	u32 members_count;
1221	int prev, index;
1222	int i, loc = -1;
1223	int err;
1224	u8 port = gid[5];
1225	bool removed_entry = false;
1226
1227	mailbox = mlx4_alloc_cmd_mailbox(dev);
1228	if (IS_ERR(ptr: mailbox))
1229	return PTR_ERR(ptr: mailbox);
1230	mgm = mailbox->buf;
1231
1232	mutex_lock(&priv->mcg_table.mutex);
1233
1234	err = find_entry(dev, port, gid, prot,
1235	mgm_mailbox: mailbox, prev: &prev, index: &index);
1236	if (err)
1237	goto out;
1238
1239	if (index == -1) {
1240	mlx4_err(dev, "MGID %pI6 not found\n", gid);
1241	err = -EINVAL;
1242	goto out;
1243	}
1244
1245	/* If this QP is also a promisc QP, it shouldn't be removed only if
1246	* at least one none promisc QP is also attached to this MCG
1247	*/
1248	if (prot == MLX4_PROT_ETH &&
1249	check_duplicate_entry(dev, port, steer, index, qpn: qp->qpn) &&
1250	!promisc_steering_entry(dev, port, steer, index, tqpn: qp->qpn, NULL))
1251	goto out;
1252
1253	members_count = be32_to_cpu(mgm->members_count) & 0xffffff;
1254	for (i = 0; i < members_count; ++i)
1255	if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) {
1256	loc = i;
1257	break;
1258	}
1259
1260	if (loc == -1) {
1261	mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn);
1262	err = -EINVAL;
1263	goto out;
1264	}
1265
1266	/* copy the last QP in this MGM over removed QP */
1267	mgm->qp[loc] = mgm->qp[members_count - 1];
1268	mgm->qp[members_count - 1] = 0;
1269	mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30);
1270
1271	if (prot == MLX4_PROT_ETH)
1272	removed_entry = can_remove_steering_entry(dev, port, steer,
1273	index, tqpn: qp->qpn);
1274	if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) {
1275	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1276	goto out;
1277	}
1278
1279	/* We are going to delete the entry, members count should be 0 */
1280	mgm->members_count = cpu_to_be32((u32) prot << 30);
1281
1282	if (prev == -1) {
1283	/* Remove entry from MGM */
1284	int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6;
1285	if (amgm_index) {
1286	err = mlx4_READ_ENTRY(dev, index: amgm_index, mailbox);
1287	if (err)
1288	goto out;
1289	} else
1290	memset(mgm->gid, 0, 16);
1291
1292	err = mlx4_WRITE_ENTRY(dev, index, mailbox);
1293	if (err)
1294	goto out;
1295
1296	if (amgm_index) {
1297	if (amgm_index < dev->caps.num_mgms)
1298	mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d\n",
1299	index, amgm_index, dev->caps.num_mgms);
1300	else
1301	mlx4_bitmap_free(bitmap: &priv->mcg_table.bitmap,
1302	obj: amgm_index - dev->caps.num_mgms, use_rr: MLX4_USE_RR);
1303	}
1304	} else {
1305	/* Remove entry from AMGM */
1306	int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6;
1307	err = mlx4_READ_ENTRY(dev, index: prev, mailbox);
1308	if (err)
1309	goto out;
1310
1311	mgm->next_gid_index = cpu_to_be32(cur_next_index << 6);
1312
1313	err = mlx4_WRITE_ENTRY(dev, index: prev, mailbox);
1314	if (err)
1315	goto out;
1316
1317	if (index < dev->caps.num_mgms)
1318	mlx4_warn(dev, "entry %d had next AMGM index %d < %d\n",
1319	prev, index, dev->caps.num_mgms);
1320	else
1321	mlx4_bitmap_free(bitmap: &priv->mcg_table.bitmap,
1322	obj: index - dev->caps.num_mgms, use_rr: MLX4_USE_RR);
1323	}
1324
1325	out:
1326	mutex_unlock(lock: &priv->mcg_table.mutex);
1327
1328	mlx4_free_cmd_mailbox(dev, mailbox);
1329	if (err && dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
1330	/* In case device is under an error, return success as a closing command */
1331	err = 0;
1332	return err;
1333	}
1334
1335	static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp,
1336	u8 gid[16], u8 attach, u8 block_loopback,
1337	enum mlx4_protocol prot)
1338	{
1339	struct mlx4_cmd_mailbox *mailbox;
1340	int err = 0;
1341	int qpn;
1342
1343	if (!mlx4_is_mfunc(dev))
1344	return -EBADF;
1345
1346	mailbox = mlx4_alloc_cmd_mailbox(dev);
1347	if (IS_ERR(ptr: mailbox))
1348	return PTR_ERR(ptr: mailbox);
1349
1350	memcpy(mailbox->buf, gid, 16);
1351	qpn = qp->qpn;
1352	qpn |= (prot << 28);
1353	if (attach && block_loopback)
1354	qpn |= (1 << 31);
1355
1356	err = mlx4_cmd(dev, in_param: mailbox->dma, in_modifier: qpn, op_modifier: attach,
1357	op: MLX4_CMD_QP_ATTACH, timeout: MLX4_CMD_TIME_CLASS_A,
1358	native: MLX4_CMD_WRAPPED);
1359
1360	mlx4_free_cmd_mailbox(dev, mailbox);
1361	if (err && !attach &&
1362	dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
1363	err = 0;
1364	return err;
1365	}
1366
1367	int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp,
1368	u8 gid[16], u8 port,
1369	int block_mcast_loopback,
1370	enum mlx4_protocol prot, u64 *reg_id)
1371	{
1372	struct mlx4_spec_list spec = { {NULL} };
1373	__be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
1374
1375	struct mlx4_net_trans_rule rule = {
1376	.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1377	.exclusive = 0,
1378	.promisc_mode = MLX4_FS_REGULAR,
1379	.priority = MLX4_DOMAIN_NIC,
1380	};
1381
1382	rule.allow_loopback = !block_mcast_loopback;
1383	rule.port = port;
1384	rule.qpn = qp->qpn;
1385	INIT_LIST_HEAD(list: &rule.list);
1386
1387	switch (prot) {
1388	case MLX4_PROT_ETH:
1389	spec.id = MLX4_NET_TRANS_RULE_ID_ETH;
1390	memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN);
1391	memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
1392	break;
1393
1394	case MLX4_PROT_IB_IPV6:
1395	spec.id = MLX4_NET_TRANS_RULE_ID_IB;
1396	memcpy(spec.ib.dst_gid, gid, 16);
1397	memset(&spec.ib.dst_gid_msk, 0xff, 16);
1398	break;
1399	default:
1400	return -EINVAL;
1401	}
1402	list_add_tail(new: &spec.list, head: &rule.list);
1403
1404	return mlx4_flow_attach(dev, &rule, reg_id);
1405	}
1406
1407	int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1408	u8 port, int block_mcast_loopback,
1409	enum mlx4_protocol prot, u64 *reg_id)
1410	{
1411	switch (dev->caps.steering_mode) {
1412	case MLX4_STEERING_MODE_A0:
1413	if (prot == MLX4_PROT_ETH)
1414	return 0;
1415	fallthrough;
1416
1417	case MLX4_STEERING_MODE_B0:
1418	if (prot == MLX4_PROT_ETH)
1419	gid[7] |= (MLX4_MC_STEER << 1);
1420
1421	if (mlx4_is_mfunc(dev))
1422	return mlx4_QP_ATTACH(dev, qp, gid, attach: 1,
1423	block_loopback: block_mcast_loopback, prot);
1424	return mlx4_qp_attach_common(dev, qp, gid,
1425	block_mcast_loopback, prot,
1426	steer: MLX4_MC_STEER);
1427
1428	case MLX4_STEERING_MODE_DEVICE_MANAGED:
1429	return mlx4_trans_to_dmfs_attach(dev, qp, gid, port,
1430	block_mcast_loopback,
1431	prot, reg_id);
1432	default:
1433	return -EINVAL;
1434	}
1435	}
1436	EXPORT_SYMBOL_GPL(mlx4_multicast_attach);
1437
1438	int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
1439	enum mlx4_protocol prot, u64 reg_id)
1440	{
1441	switch (dev->caps.steering_mode) {
1442	case MLX4_STEERING_MODE_A0:
1443	if (prot == MLX4_PROT_ETH)
1444	return 0;
1445	fallthrough;
1446
1447	case MLX4_STEERING_MODE_B0:
1448	if (prot == MLX4_PROT_ETH)
1449	gid[7] |= (MLX4_MC_STEER << 1);
1450
1451	if (mlx4_is_mfunc(dev))
1452	return mlx4_QP_ATTACH(dev, qp, gid, attach: 0, block_loopback: 0, prot);
1453
1454	return mlx4_qp_detach_common(dev, qp, gid, prot,
1455	steer: MLX4_MC_STEER);
1456
1457	case MLX4_STEERING_MODE_DEVICE_MANAGED:
1458	return mlx4_flow_detach(dev, reg_id);
1459
1460	default:
1461	return -EINVAL;
1462	}
1463	}
1464	EXPORT_SYMBOL_GPL(mlx4_multicast_detach);
1465
1466	int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port,
1467	u32 qpn, enum mlx4_net_trans_promisc_mode mode)
1468	{
1469	struct mlx4_net_trans_rule rule = {
1470	.queue_mode = MLX4_NET_TRANS_Q_FIFO,
1471	.exclusive = 0,
1472	.allow_loopback = 1,
1473	};
1474
1475	u64 *regid_p;
1476
1477	switch (mode) {
1478	case MLX4_FS_ALL_DEFAULT:
1479	regid_p = &dev->regid_promisc_array[port];
1480	break;
1481	case MLX4_FS_MC_DEFAULT:
1482	regid_p = &dev->regid_allmulti_array[port];
1483	break;
1484	default:
1485	return -1;
1486	}
1487
1488	if (*regid_p != 0)
1489	return -1;
1490
1491	rule.promisc_mode = mode;
1492	rule.port = port;
1493	rule.qpn = qpn;
1494	INIT_LIST_HEAD(list: &rule.list);
1495	mlx4_info(dev, "going promisc on %x\n", port);
1496
1497	return mlx4_flow_attach(dev, &rule, regid_p);
1498	}
1499	EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add);
1500
1501	int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port,
1502	enum mlx4_net_trans_promisc_mode mode)
1503	{
1504	int ret;
1505	u64 *regid_p;
1506
1507	switch (mode) {
1508	case MLX4_FS_ALL_DEFAULT:
1509	regid_p = &dev->regid_promisc_array[port];
1510	break;
1511	case MLX4_FS_MC_DEFAULT:
1512	regid_p = &dev->regid_allmulti_array[port];
1513	break;
1514	default:
1515	return -1;
1516	}
1517
1518	if (*regid_p == 0)
1519	return -1;
1520
1521	ret = mlx4_flow_detach(dev, *regid_p);
1522	if (ret == 0)
1523	*regid_p = 0;
1524
1525	return ret;
1526	}
1527	EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove);
1528
1529	int mlx4_unicast_attach(struct mlx4_dev *dev,
1530	struct mlx4_qp *qp, u8 gid[16],
1531	int block_mcast_loopback, enum mlx4_protocol prot)
1532	{
1533	if (prot == MLX4_PROT_ETH)
1534	gid[7] |= (MLX4_UC_STEER << 1);
1535
1536	if (mlx4_is_mfunc(dev))
1537	return mlx4_QP_ATTACH(dev, qp, gid, attach: 1,
1538	block_loopback: block_mcast_loopback, prot);
1539
1540	return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback,
1541	prot, steer: MLX4_UC_STEER);
1542	}
1543	EXPORT_SYMBOL_GPL(mlx4_unicast_attach);
1544
1545	int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp,
1546	u8 gid[16], enum mlx4_protocol prot)
1547	{
1548	if (prot == MLX4_PROT_ETH)
1549	gid[7] |= (MLX4_UC_STEER << 1);
1550
1551	if (mlx4_is_mfunc(dev))
1552	return mlx4_QP_ATTACH(dev, qp, gid, attach: 0, block_loopback: 0, prot);
1553
1554	return mlx4_qp_detach_common(dev, qp, gid, prot, steer: MLX4_UC_STEER);
1555	}
1556	EXPORT_SYMBOL_GPL(mlx4_unicast_detach);
1557
1558	int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave,
1559	struct mlx4_vhcr *vhcr,
1560	struct mlx4_cmd_mailbox *inbox,
1561	struct mlx4_cmd_mailbox *outbox,
1562	struct mlx4_cmd_info *cmd)
1563	{
1564	u32 qpn = (u32) vhcr->in_param & 0xffffffff;
1565	int port = mlx4_slave_convert_port(dev, slave, port: vhcr->in_param >> 62);
1566	enum mlx4_steer_type steer = vhcr->in_modifier;
1567
1568	if (port < 0)
1569	return -EINVAL;
1570
1571	/* Promiscuous unicast is not allowed in mfunc */
1572	if (mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)
1573	return 0;
1574
1575	if (vhcr->op_modifier)
1576	return add_promisc_qp(dev, port, steer, qpn);
1577	else
1578	return remove_promisc_qp(dev, port, steer, qpn);
1579	}
1580
1581	static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn,
1582	enum mlx4_steer_type steer, u8 add, u8 port)
1583	{
1584	return mlx4_cmd(dev, in_param: (u64) qpn | (u64) port << 62, in_modifier: (u32) steer, op_modifier: add,
1585	op: MLX4_CMD_PROMISC, timeout: MLX4_CMD_TIME_CLASS_A,
1586	native: MLX4_CMD_WRAPPED);
1587	}
1588
1589	int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port)
1590	{
1591	if (mlx4_is_mfunc(dev))
1592	return mlx4_PROMISC(dev, qpn, steer: MLX4_MC_STEER, add: 1, port);
1593
1594	return add_promisc_qp(dev, port, steer: MLX4_MC_STEER, qpn);
1595	}
1596	EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add);
1597
1598	int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port)
1599	{
1600	if (mlx4_is_mfunc(dev))
1601	return mlx4_PROMISC(dev, qpn, steer: MLX4_MC_STEER, add: 0, port);
1602
1603	return remove_promisc_qp(dev, port, steer: MLX4_MC_STEER, qpn);
1604	}
1605	EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove);
1606
1607	int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port)
1608	{
1609	if (mlx4_is_mfunc(dev))
1610	return mlx4_PROMISC(dev, qpn, steer: MLX4_UC_STEER, add: 1, port);
1611
1612	return add_promisc_qp(dev, port, steer: MLX4_UC_STEER, qpn);
1613	}
1614	EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add);
1615
1616	int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port)
1617	{
1618	if (mlx4_is_mfunc(dev))
1619	return mlx4_PROMISC(dev, qpn, steer: MLX4_UC_STEER, add: 0, port);
1620
1621	return remove_promisc_qp(dev, port, steer: MLX4_UC_STEER, qpn);
1622	}
1623	EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove);
1624
1625	int mlx4_init_mcg_table(struct mlx4_dev *dev)
1626	{
1627	struct mlx4_priv *priv = mlx4_priv(dev);
1628	int err;
1629
1630	/* No need for mcg_table when fw managed the mcg table*/
1631	if (dev->caps.steering_mode ==
1632	MLX4_STEERING_MODE_DEVICE_MANAGED)
1633	return 0;
1634	err = mlx4_bitmap_init(bitmap: &priv->mcg_table.bitmap, num: dev->caps.num_amgms,
1635	mask: dev->caps.num_amgms - 1, reserved_bot: 0, resetrved_top: 0);
1636	if (err)
1637	return err;
1638
1639	mutex_init(&priv->mcg_table.mutex);
1640
1641	return 0;
1642	}
1643
1644	void mlx4_cleanup_mcg_table(struct mlx4_dev *dev)
1645	{
1646	if (dev->caps.steering_mode !=
1647	MLX4_STEERING_MODE_DEVICE_MANAGED)
1648	mlx4_bitmap_cleanup(bitmap: &mlx4_priv(dev)->mcg_table.bitmap);
1649	}
1650