1	/*
2	* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4	* Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
5	* Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6	*
7	* This software is available to you under a choice of one of two
8	* licenses. You may choose to be licensed under the terms of the GNU
9	* General Public License (GPL) Version 2, available from the file
10	* COPYING in the main directory of this source tree, or the
11	* OpenIB.org BSD license below:
12	*
13	* Redistribution and use in source and binary forms, with or
14	* without modification, are permitted provided that the following
15	* conditions are met:
16	*
17	* - Redistributions of source code must retain the above
18	* copyright notice, this list of conditions and the following
19	* disclaimer.
20	*
21	* - Redistributions in binary form must reproduce the above
22	* copyright notice, this list of conditions and the following
23	* disclaimer in the documentation and/or other materials
24	* provided with the distribution.
25	*
26	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33	* SOFTWARE.
34	*/
35
36	#include <linux/module.h>
37	#include <linux/kernel.h>
38	#include <linux/init.h>
39	#include <linux/errno.h>
40	#include <linux/pci.h>
41	#include <linux/dma-mapping.h>
42	#include <linux/slab.h>
43	#include <linux/io-mapping.h>
44	#include <linux/delay.h>
45	#include <linux/etherdevice.h>
46	#include <net/devlink.h>
47
48	#include <uapi/rdma/mlx4-abi.h>
49	#include <linux/mlx4/device.h>
50	#include <linux/mlx4/doorbell.h>
51
52	#include "mlx4.h"
53	#include "fw.h"
54	#include "icm.h"
55
56	MODULE_AUTHOR("Roland Dreier");
57	MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
58	MODULE_LICENSE("Dual BSD/GPL");
59	MODULE_VERSION(DRV_VERSION);
60
61	struct workqueue_struct *mlx4_wq;
62
63	#ifdef CONFIG_MLX4_DEBUG
64
65	int mlx4_debug_level; /* 0 by default */
66	module_param_named(debug_level, mlx4_debug_level, int, 0644);
67	MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
68
69	#endif /* CONFIG_MLX4_DEBUG */
70
71	#ifdef CONFIG_PCI_MSI
72
73	static int msi_x = 1;
74	module_param(msi_x, int, 0444);
75	MODULE_PARM_DESC(msi_x, "0 - don't use MSI-X, 1 - use MSI-X, >1 - limit number of MSI-X irqs to msi_x");
76
77	#else /* CONFIG_PCI_MSI */
78
79	#define msi_x (0)
80
81	#endif /* CONFIG_PCI_MSI */
82
83	static uint8_t num_vfs[3] = {0, 0, 0};
84	static int num_vfs_argc;
85	module_param_array(num_vfs, byte, &num_vfs_argc, 0444);
86	MODULE_PARM_DESC(num_vfs, "enable #num_vfs functions if num_vfs > 0\n"
87	"num_vfs=port1,port2,port1+2");
88
89	static uint8_t probe_vf[3] = {0, 0, 0};
90	static int probe_vfs_argc;
91	module_param_array(probe_vf, byte, &probe_vfs_argc, 0444);
92	MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (num_vfs > 0)\n"
93	"probe_vf=port1,port2,port1+2");
94
95	static int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
96	module_param_named(log_num_mgm_entry_size,
97	mlx4_log_num_mgm_entry_size, int, 0444);
98	MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num"
99	" of qp per mcg, for example:"
100	" 10 gives 248.range: 7 <="
101	" log_num_mgm_entry_size <= 12."
102	" To activate device managed"
103	" flow steering when available, set to -1");
104
105	static bool enable_64b_cqe_eqe = true;
106	module_param(enable_64b_cqe_eqe, bool, 0444);
107	MODULE_PARM_DESC(enable_64b_cqe_eqe,
108	"Enable 64 byte CQEs/EQEs when the FW supports this (default: True)");
109
110	static bool enable_4k_uar;
111	module_param(enable_4k_uar, bool, 0444);
112	MODULE_PARM_DESC(enable_4k_uar,
113	"Enable using 4K UAR. Should not be enabled if have VFs which do not support 4K UARs (default: false)");
114
115	#define PF_CONTEXT_BEHAVIOUR_MASK (MLX4_FUNC_CAP_64B_EQE_CQE | \
116	MLX4_FUNC_CAP_EQE_CQE_STRIDE | \
117	MLX4_FUNC_CAP_DMFS_A0_STATIC)
118
119	#define RESET_PERSIST_MASK_FLAGS (MLX4_FLAG_SRIOV)
120
121	static char mlx4_version[] =
122	DRV_NAME ": Mellanox ConnectX core driver v"
123	DRV_VERSION "\n";
124
125	static const struct mlx4_profile default_profile = {
126	.num_qp = 1 << 18,
127	.num_srq = 1 << 16,
128	.rdmarc_per_qp = 1 << 4,
129	.num_cq = 1 << 16,
130	.num_mcg = 1 << 13,
131	.num_mpt = 1 << 19,
132	.num_mtt = 1 << 20, /* It is really num mtt segements */
133	};
134
135	static const struct mlx4_profile low_mem_profile = {
136	.num_qp = 1 << 17,
137	.num_srq = 1 << 6,
138	.rdmarc_per_qp = 1 << 4,
139	.num_cq = 1 << 8,
140	.num_mcg = 1 << 8,
141	.num_mpt = 1 << 9,
142	.num_mtt = 1 << 7,
143	};
144
145	static int log_num_mac = 7;
146	module_param_named(log_num_mac, log_num_mac, int, 0444);
147	MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)");
148
149	static int log_num_vlan;
150	module_param_named(log_num_vlan, log_num_vlan, int, 0444);
151	MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)");
152	/* Log2 max number of VLANs per ETH port (0-7) */
153	#define MLX4_LOG_NUM_VLANS 7
154	#define MLX4_MIN_LOG_NUM_VLANS 0
155	#define MLX4_MIN_LOG_NUM_MAC 1
156
157	static bool use_prio;
158	module_param_named(use_prio, use_prio, bool, 0444);
159	MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)");
160
161	int log_mtts_per_seg = ilog2(1);
162	module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444);
163	MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment "
164	"(0-7) (default: 0)");
165
166	static int port_type_array[2] = {MLX4_PORT_TYPE_NONE, MLX4_PORT_TYPE_NONE};
167	static int arr_argc = 2;
168	module_param_array(port_type_array, int, &arr_argc, 0444);
169	MODULE_PARM_DESC(port_type_array, "Array of port types: HW_DEFAULT (0) is default "
170	"1 for IB, 2 for Ethernet");
171
172	struct mlx4_port_config {
173	struct list_head list;
174	enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
175	struct pci_dev *pdev;
176	};
177
178	static atomic_t pf_loading = ATOMIC_INIT(0);
179
180	static int mlx4_devlink_ierr_reset_get(struct devlink *devlink, u32 id,
181	struct devlink_param_gset_ctx *ctx)
182	{
183	ctx->val.vbool = !!mlx4_internal_err_reset;
184	return 0;
185	}
186
187	static int mlx4_devlink_ierr_reset_set(struct devlink *devlink, u32 id,
188	struct devlink_param_gset_ctx *ctx)
189	{
190	mlx4_internal_err_reset = ctx->val.vbool;
191	return 0;
192	}
193
194	static int mlx4_devlink_crdump_snapshot_get(struct devlink *devlink, u32 id,
195	struct devlink_param_gset_ctx *ctx)
196	{
197	struct mlx4_priv *priv = devlink_priv(devlink);
198	struct mlx4_dev *dev = &priv->dev;
199
200	ctx->val.vbool = dev->persist->crdump.snapshot_enable;
201	return 0;
202	}
203
204	static int mlx4_devlink_crdump_snapshot_set(struct devlink *devlink, u32 id,
205	struct devlink_param_gset_ctx *ctx)
206	{
207	struct mlx4_priv *priv = devlink_priv(devlink);
208	struct mlx4_dev *dev = &priv->dev;
209
210	dev->persist->crdump.snapshot_enable = ctx->val.vbool;
211	return 0;
212	}
213
214	static int
215	mlx4_devlink_max_macs_validate(struct devlink *devlink, u32 id,
216	union devlink_param_value val,
217	struct netlink_ext_ack *extack)
218	{
219	u32 value = val.vu32;
220
221	if (value < 1 || value > 128)
222	return -ERANGE;
223
224	if (!is_power_of_2(n: value)) {
225	NL_SET_ERR_MSG_MOD(extack, "max_macs supported must be power of 2");
226	return -EINVAL;
227	}
228
229	return 0;
230	}
231
232	enum mlx4_devlink_param_id {
233	MLX4_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
234	MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
235	MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
236	};
237
238	static const struct devlink_param mlx4_devlink_params[] = {
239	DEVLINK_PARAM_GENERIC(INT_ERR_RESET,
240	BIT(DEVLINK_PARAM_CMODE_RUNTIME) |
241	BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
242	mlx4_devlink_ierr_reset_get,
243	mlx4_devlink_ierr_reset_set, NULL),
244	DEVLINK_PARAM_GENERIC(MAX_MACS,
245	BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
246	NULL, NULL, mlx4_devlink_max_macs_validate),
247	DEVLINK_PARAM_GENERIC(REGION_SNAPSHOT,
248	BIT(DEVLINK_PARAM_CMODE_RUNTIME) |
249	BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
250	mlx4_devlink_crdump_snapshot_get,
251	mlx4_devlink_crdump_snapshot_set, NULL),
252	DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
253	"enable_64b_cqe_eqe", DEVLINK_PARAM_TYPE_BOOL,
254	BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
255	NULL, NULL, NULL),
256	DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
257	"enable_4k_uar", DEVLINK_PARAM_TYPE_BOOL,
258	BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
259	NULL, NULL, NULL),
260	};
261
262	static void mlx4_devlink_set_params_init_values(struct devlink *devlink)
263	{
264	union devlink_param_value value;
265
266	value.vbool = !!mlx4_internal_err_reset;
267	devl_param_driverinit_value_set(devlink,
268	param_id: DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
269	init_val: value);
270
271	value.vu32 = 1UL << log_num_mac;
272	devl_param_driverinit_value_set(devlink,
273	param_id: DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
274	init_val: value);
275
276	value.vbool = enable_64b_cqe_eqe;
277	devl_param_driverinit_value_set(devlink,
278	param_id: MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
279	init_val: value);
280
281	value.vbool = enable_4k_uar;
282	devl_param_driverinit_value_set(devlink,
283	param_id: MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
284	init_val: value);
285
286	value.vbool = false;
287	devl_param_driverinit_value_set(devlink,
288	param_id: DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
289	init_val: value);
290	}
291
292	static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev,
293	struct mlx4_dev_cap *dev_cap)
294	{
295	/* The reserved_uars is calculated by system page size unit.
296	* Therefore, adjustment is added when the uar page size is less
297	* than the system page size
298	*/
299	dev->caps.reserved_uars =
300	max_t(int,
301	mlx4_get_num_reserved_uar(dev),
302	dev_cap->reserved_uars /
303	(1 << (PAGE_SHIFT - dev->uar_page_shift)));
304	}
305
306	int mlx4_check_port_params(struct mlx4_dev *dev,
307	enum mlx4_port_type *port_type)
308	{
309	int i;
310
311	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
312	for (i = 0; i < dev->caps.num_ports - 1; i++) {
313	if (port_type[i] != port_type[i + 1]) {
314	mlx4_err(dev, "Only same port types supported on this HCA, aborting\n");
315	return -EOPNOTSUPP;
316	}
317	}
318	}
319
320	for (i = 0; i < dev->caps.num_ports; i++) {
321	if (!(port_type[i] & dev->caps.supported_type[i+1])) {
322	mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n",
323	i + 1);
324	return -EOPNOTSUPP;
325	}
326	}
327	return 0;
328	}
329
330	static void mlx4_set_port_mask(struct mlx4_dev *dev)
331	{
332	int i;
333
334	for (i = 1; i <= dev->caps.num_ports; ++i)
335	dev->caps.port_mask[i] = dev->caps.port_type[i];
336	}
337
338	enum {
339	MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0,
340	};
341
342	static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
343	{
344	int err = 0;
345	struct mlx4_func func;
346
347	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
348	err = mlx4_QUERY_FUNC(dev, func: &func, slave: 0);
349	if (err) {
350	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
351	return err;
352	}
353	dev_cap->max_eqs = func.max_eq;
354	dev_cap->reserved_eqs = func.rsvd_eqs;
355	dev_cap->reserved_uars = func.rsvd_uars;
356	err |= MLX4_QUERY_FUNC_NUM_SYS_EQS;
357	}
358	return err;
359	}
360
361	static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev)
362	{
363	struct mlx4_caps *dev_cap = &dev->caps;
364
365	/* FW not supporting or cancelled by user */
366	if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) ||
367	!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE))
368	return;
369
370	/* Must have 64B CQE_EQE enabled by FW to use bigger stride
371	* When FW has NCSI it may decide not to report 64B CQE/EQEs
372	*/
373	if (!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_EQE) ||
374	!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_CQE)) {
375	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
376	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
377	return;
378	}
379
380	if (cache_line_size() == 128 || cache_line_size() == 256) {
381	mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n");
382	/* Changing the real data inside CQE size to 32B */
383	dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
384	dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
385
386	if (mlx4_is_master(dev))
387	dev_cap->function_caps |= MLX4_FUNC_CAP_EQE_CQE_STRIDE;
388	} else {
389	if (cache_line_size() != 32 && cache_line_size() != 64)
390	mlx4_dbg(dev, "Disabling CQE stride, cacheLine size unsupported\n");
391	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
392	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
393	}
394	}
395
396	static int _mlx4_dev_port(struct mlx4_dev *dev, int port,
397	struct mlx4_port_cap *port_cap)
398	{
399	dev->caps.vl_cap[port] = port_cap->max_vl;
400	dev->caps.ib_mtu_cap[port] = port_cap->ib_mtu;
401	dev->phys_caps.gid_phys_table_len[port] = port_cap->max_gids;
402	dev->phys_caps.pkey_phys_table_len[port] = port_cap->max_pkeys;
403	/* set gid and pkey table operating lengths by default
404	* to non-sriov values
405	*/
406	dev->caps.gid_table_len[port] = port_cap->max_gids;
407	dev->caps.pkey_table_len[port] = port_cap->max_pkeys;
408	dev->caps.port_width_cap[port] = port_cap->max_port_width;
409	dev->caps.eth_mtu_cap[port] = port_cap->eth_mtu;
410	dev->caps.max_tc_eth = port_cap->max_tc_eth;
411	dev->caps.def_mac[port] = port_cap->def_mac;
412	dev->caps.supported_type[port] = port_cap->supported_port_types;
413	dev->caps.suggested_type[port] = port_cap->suggested_type;
414	dev->caps.default_sense[port] = port_cap->default_sense;
415	dev->caps.trans_type[port] = port_cap->trans_type;
416	dev->caps.vendor_oui[port] = port_cap->vendor_oui;
417	dev->caps.wavelength[port] = port_cap->wavelength;
418	dev->caps.trans_code[port] = port_cap->trans_code;
419
420	return 0;
421	}
422
423	static int mlx4_dev_port(struct mlx4_dev *dev, int port,
424	struct mlx4_port_cap *port_cap)
425	{
426	int err = 0;
427
428	err = mlx4_QUERY_PORT(dev, port, port_cap);
429
430	if (err)
431	mlx4_err(dev, "QUERY_PORT command failed.\n");
432
433	return err;
434	}
435
436	static inline void mlx4_enable_ignore_fcs(struct mlx4_dev *dev)
437	{
438	if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS))
439	return;
440
441	if (mlx4_is_mfunc(dev)) {
442	mlx4_dbg(dev, "SRIOV mode - Disabling Ignore FCS");
443	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;
444	return;
445	}
446
447	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
448	mlx4_dbg(dev,
449	"Keep FCS is not supported - Disabling Ignore FCS");
450	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;
451	return;
452	}
453	}
454
455	#define MLX4_A0_STEERING_TABLE_SIZE 256
456	static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
457	{
458	int err;
459	int i;
460
461	err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
462	if (err) {
463	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
464	return err;
465	}
466	mlx4_dev_cap_dump(dev, dev_cap);
467
468	if (dev_cap->min_page_sz > PAGE_SIZE) {
469	mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
470	dev_cap->min_page_sz, PAGE_SIZE);
471	return -ENODEV;
472	}
473	if (dev_cap->num_ports > MLX4_MAX_PORTS) {
474	mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
475	dev_cap->num_ports, MLX4_MAX_PORTS);
476	return -ENODEV;
477	}
478
479	if (dev_cap->uar_size > pci_resource_len(dev->persist->pdev, 2)) {
480	mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
481	dev_cap->uar_size,
482	(unsigned long long)
483	pci_resource_len(dev->persist->pdev, 2));
484	return -ENODEV;
485	}
486
487	dev->caps.num_ports = dev_cap->num_ports;
488	dev->caps.num_sys_eqs = dev_cap->num_sys_eqs;
489	dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ?
490	dev->caps.num_sys_eqs :
491	MLX4_MAX_EQ_NUM;
492	for (i = 1; i <= dev->caps.num_ports; ++i) {
493	err = _mlx4_dev_port(dev, port: i, port_cap: dev_cap->port_cap + i);
494	if (err) {
495	mlx4_err(dev, "QUERY_PORT command failed, aborting\n");
496	return err;
497	}
498	}
499
500	dev->caps.map_clock_to_user = dev_cap->map_clock_to_user;
501	dev->caps.uar_page_size = PAGE_SIZE;
502	dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE;
503	dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
504	dev->caps.bf_reg_size = dev_cap->bf_reg_size;
505	dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page;
506	dev->caps.max_sq_sg = dev_cap->max_sq_sg;
507	dev->caps.max_rq_sg = dev_cap->max_rq_sg;
508	dev->caps.max_wqes = dev_cap->max_qp_sz;
509	dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp;
510	dev->caps.max_srq_wqes = dev_cap->max_srq_sz;
511	dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1;
512	dev->caps.reserved_srqs = dev_cap->reserved_srqs;
513	dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz;
514	dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz;
515	/*
516	* Subtract 1 from the limit because we need to allocate a
517	* spare CQE to enable resizing the CQ.
518	*/
519	dev->caps.max_cqes = dev_cap->max_cq_sz - 1;
520	dev->caps.reserved_cqs = dev_cap->reserved_cqs;
521	dev->caps.reserved_eqs = dev_cap->reserved_eqs;
522	dev->caps.reserved_mtts = dev_cap->reserved_mtts;
523	dev->caps.reserved_mrws = dev_cap->reserved_mrws;
524
525	dev->caps.reserved_pds = dev_cap->reserved_pds;
526	dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
527	dev_cap->reserved_xrcds : 0;
528	dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
529	dev_cap->max_xrcds : 0;
530	dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz;
531
532	dev->caps.max_msg_sz = dev_cap->max_msg_sz;
533	dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1);
534	dev->caps.flags = dev_cap->flags;
535	dev->caps.flags2 = dev_cap->flags2;
536	dev->caps.bmme_flags = dev_cap->bmme_flags;
537	dev->caps.reserved_lkey = dev_cap->reserved_lkey;
538	dev->caps.stat_rate_support = dev_cap->stat_rate_support;
539	dev->caps.max_gso_sz = dev_cap->max_gso_sz;
540	dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz;
541	dev->caps.wol_port[1] = dev_cap->wol_port[1];
542	dev->caps.wol_port[2] = dev_cap->wol_port[2];
543	dev->caps.health_buffer_addrs = dev_cap->health_buffer_addrs;
544
545	/* Save uar page shift */
546	if (!mlx4_is_slave(dev)) {
547	/* Virtual PCI function needs to determine UAR page size from
548	* firmware. Only master PCI function can set the uar page size
549	*/
550	if (enable_4k_uar || !dev->persist->num_vfs)
551	dev->uar_page_shift = DEFAULT_UAR_PAGE_SHIFT;
552	else
553	dev->uar_page_shift = PAGE_SHIFT;
554
555	mlx4_set_num_reserved_uars(dev, dev_cap);
556	}
557
558	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN) {
559	struct mlx4_init_hca_param hca_param;
560
561	memset(&hca_param, 0, sizeof(hca_param));
562	err = mlx4_QUERY_HCA(dev, param: &hca_param);
563	/* Turn off PHV_EN flag in case phv_check_en is set.
564	* phv_check_en is a HW check that parse the packet and verify
565	* phv bit was reported correctly in the wqe. To allow QinQ
566	* PHV_EN flag should be set and phv_check_en must be cleared
567	* otherwise QinQ packets will be drop by the HW.
568	*/
569	if (err || hca_param.phv_check_en)
570	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_PHV_EN;
571	}
572
573	/* Sense port always allowed on supported devices for ConnectX-1 and -2 */
574	if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT)
575	dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
576	/* Don't do sense port on multifunction devices (for now at least) */
577	if (mlx4_is_mfunc(dev))
578	dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
579
580	if (mlx4_low_memory_profile()) {
581	dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC;
582	dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS;
583	} else {
584	dev->caps.log_num_macs = log_num_mac;
585	dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS;
586	}
587
588	for (i = 1; i <= dev->caps.num_ports; ++i) {
589	dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE;
590	if (dev->caps.supported_type[i]) {
591	/* if only ETH is supported - assign ETH */
592	if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH)
593	dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
594	/* if only IB is supported, assign IB */
595	else if (dev->caps.supported_type[i] ==
596	MLX4_PORT_TYPE_IB)
597	dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
598	else {
599	/* if IB and ETH are supported, we set the port
600	* type according to user selection of port type;
601	* if user selected none, take the FW hint */
602	if (port_type_array[i - 1] == MLX4_PORT_TYPE_NONE)
603	dev->caps.port_type[i] = dev->caps.suggested_type[i] ?
604	MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB;
605	else
606	dev->caps.port_type[i] = port_type_array[i - 1];
607	}
608	}
609	/*
610	* Link sensing is allowed on the port if 3 conditions are true:
611	* 1. Both protocols are supported on the port.
612	* 2. Different types are supported on the port
613	* 3. FW declared that it supports link sensing
614	*/
615	mlx4_priv(dev)->sense.sense_allowed[i] =
616	((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) &&
617	(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
618	(dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT));
619
620	/*
621	* If "default_sense" bit is set, we move the port to "AUTO" mode
622	* and perform sense_port FW command to try and set the correct
623	* port type from beginning
624	*/
625	if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) {
626	enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE;
627	dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO;
628	mlx4_SENSE_PORT(dev, port: i, type: &sensed_port);
629	if (sensed_port != MLX4_PORT_TYPE_NONE)
630	dev->caps.port_type[i] = sensed_port;
631	} else {
632	dev->caps.possible_type[i] = dev->caps.port_type[i];
633	}
634
635	if (dev->caps.log_num_macs > dev_cap->port_cap[i].log_max_macs) {
636	dev->caps.log_num_macs = dev_cap->port_cap[i].log_max_macs;
637	mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n",
638	i, 1 << dev->caps.log_num_macs);
639	}
640	if (dev->caps.log_num_vlans > dev_cap->port_cap[i].log_max_vlans) {
641	dev->caps.log_num_vlans = dev_cap->port_cap[i].log_max_vlans;
642	mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n",
643	i, 1 << dev->caps.log_num_vlans);
644	}
645	}
646
647	if (mlx4_is_master(dev) && (dev->caps.num_ports == 2) &&
648	(port_type_array[0] == MLX4_PORT_TYPE_IB) &&
649	(port_type_array[1] == MLX4_PORT_TYPE_ETH)) {
650	mlx4_warn(dev,
651	"Granular QoS per VF not supported with IB/Eth configuration\n");
652	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_QOS_VPP;
653	}
654
655	dev->caps.max_counters = dev_cap->max_counters;
656
657	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
658	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
659	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
660	(1 << dev->caps.log_num_macs) *
661	(1 << dev->caps.log_num_vlans) *
662	dev->caps.num_ports;
663	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
664
665	if (dev_cap->dmfs_high_rate_qpn_base > 0 &&
666	dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN)
667	dev->caps.dmfs_high_rate_qpn_base = dev_cap->dmfs_high_rate_qpn_base;
668	else
669	dev->caps.dmfs_high_rate_qpn_base =
670	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
671
672	if (dev_cap->dmfs_high_rate_qpn_range > 0 &&
673	dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) {
674	dev->caps.dmfs_high_rate_qpn_range = dev_cap->dmfs_high_rate_qpn_range;
675	dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DEFAULT;
676	dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_FS_A0;
677	} else {
678	dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_NOT_SUPPORTED;
679	dev->caps.dmfs_high_rate_qpn_base =
680	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
681	dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE;
682	}
683
684	dev->caps.rl_caps = dev_cap->rl_caps;
685
686	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] =
687	dev->caps.dmfs_high_rate_qpn_range;
688
689	dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
690	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
691	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +
692	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
693
694	dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0;
695
696	if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) {
697	if (dev_cap->flags &
698	(MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) {
699	mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n");
700	dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
701	dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
702	}
703
704	if (dev_cap->flags2 &
705	(MLX4_DEV_CAP_FLAG2_CQE_STRIDE |
706	MLX4_DEV_CAP_FLAG2_EQE_STRIDE)) {
707	mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n");
708	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
709	dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
710	}
711	}
712
713	if ((dev->caps.flags &
714	(MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) &&
715	mlx4_is_master(dev))
716	dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE;
717
718	if (!mlx4_is_slave(dev)) {
719	mlx4_enable_cqe_eqe_stride(dev);
720	dev->caps.alloc_res_qp_mask =
721	(dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) |
722	MLX4_RESERVE_A0_QP;
723
724	if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) &&
725	dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) {
726	mlx4_warn(dev, "Old device ETS support detected\n");
727	mlx4_warn(dev, "Consider upgrading device FW.\n");
728	dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG;
729	}
730
731	} else {
732	dev->caps.alloc_res_qp_mask = 0;
733	}
734
735	mlx4_enable_ignore_fcs(dev);
736
737	return 0;
738	}
739
740	/*The function checks if there are live vf, return the num of them*/
741	static int mlx4_how_many_lives_vf(struct mlx4_dev *dev)
742	{
743	struct mlx4_priv *priv = mlx4_priv(dev);
744	struct mlx4_slave_state *s_state;
745	int i;
746	int ret = 0;
747
748	for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) {
749	s_state = &priv->mfunc.master.slave_state[i];
750	if (s_state->active && s_state->last_cmd !=
751	MLX4_COMM_CMD_RESET) {
752	mlx4_warn(dev, "%s: slave: %d is still active\n",
753	__func__, i);
754	ret++;
755	}
756	}
757	return ret;
758	}
759
760	int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey)
761	{
762	u32 qk = MLX4_RESERVED_QKEY_BASE;
763
764	if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||
765	qpn < dev->phys_caps.base_proxy_sqpn)
766	return -EINVAL;
767
768	if (qpn >= dev->phys_caps.base_tunnel_sqpn)
769	/* tunnel qp */
770	qk += qpn - dev->phys_caps.base_tunnel_sqpn;
771	else
772	qk += qpn - dev->phys_caps.base_proxy_sqpn;
773	*qkey = qk;
774	return 0;
775	}
776	EXPORT_SYMBOL(mlx4_get_parav_qkey);
777
778	void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val)
779	{
780	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
781
782	if (!mlx4_is_master(dev))
783	return;
784
785	priv->virt2phys_pkey[slave][port - 1][i] = val;
786	}
787	EXPORT_SYMBOL(mlx4_sync_pkey_table);
788
789	void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid)
790	{
791	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
792
793	if (!mlx4_is_master(dev))
794	return;
795
796	priv->slave_node_guids[slave] = guid;
797	}
798	EXPORT_SYMBOL(mlx4_put_slave_node_guid);
799
800	__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave)
801	{
802	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
803
804	if (!mlx4_is_master(dev))
805	return 0;
806
807	return priv->slave_node_guids[slave];
808	}
809	EXPORT_SYMBOL(mlx4_get_slave_node_guid);
810
811	int mlx4_is_slave_active(struct mlx4_dev *dev, int slave)
812	{
813	struct mlx4_priv *priv = mlx4_priv(dev);
814	struct mlx4_slave_state *s_slave;
815
816	if (!mlx4_is_master(dev))
817	return 0;
818
819	s_slave = &priv->mfunc.master.slave_state[slave];
820	return !!s_slave->active;
821	}
822	EXPORT_SYMBOL(mlx4_is_slave_active);
823
824	void mlx4_handle_eth_header_mcast_prio(struct mlx4_net_trans_rule_hw_ctrl *ctrl,
825	struct _rule_hw *eth_header)
826	{
827	if (is_multicast_ether_addr(addr: eth_header->eth.dst_mac) ||
828	is_broadcast_ether_addr(addr: eth_header->eth.dst_mac)) {
829	struct mlx4_net_trans_rule_hw_eth *eth =
830	(struct mlx4_net_trans_rule_hw_eth *)eth_header;
831	struct _rule_hw *next_rule = (struct _rule_hw *)(eth + 1);
832	bool last_rule = next_rule->size == 0 && next_rule->id == 0 &&
833	next_rule->rsvd == 0;
834
835	if (last_rule)
836	ctrl->prio = cpu_to_be16(MLX4_DOMAIN_NIC);
837	}
838	}
839	EXPORT_SYMBOL(mlx4_handle_eth_header_mcast_prio);
840
841	static void slave_adjust_steering_mode(struct mlx4_dev *dev,
842	struct mlx4_dev_cap *dev_cap,
843	struct mlx4_init_hca_param *hca_param)
844	{
845	dev->caps.steering_mode = hca_param->steering_mode;
846	if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
847	dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
848	dev->caps.fs_log_max_ucast_qp_range_size =
849	dev_cap->fs_log_max_ucast_qp_range_size;
850	} else
851	dev->caps.num_qp_per_mgm =
852	4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2);
853
854	mlx4_dbg(dev, "Steering mode is: %s\n",
855	mlx4_steering_mode_str(dev->caps.steering_mode));
856	}
857
858	static void mlx4_slave_destroy_special_qp_cap(struct mlx4_dev *dev)
859	{
860	kfree(objp: dev->caps.spec_qps);
861	dev->caps.spec_qps = NULL;
862	}
863
864	static int mlx4_slave_special_qp_cap(struct mlx4_dev *dev)
865	{
866	struct mlx4_func_cap *func_cap;
867	struct mlx4_caps *caps = &dev->caps;
868	int i, err = 0;
869
870	func_cap = kzalloc(size: sizeof(*func_cap), GFP_KERNEL);
871	caps->spec_qps = kcalloc(n: caps->num_ports, size: sizeof(*caps->spec_qps), GFP_KERNEL);
872
873	if (!func_cap || !caps->spec_qps) {
874	mlx4_err(dev, "Failed to allocate memory for special qps cap\n");
875	err = -ENOMEM;
876	goto err_mem;
877	}
878
879	for (i = 1; i <= caps->num_ports; ++i) {
880	err = mlx4_QUERY_FUNC_CAP(dev, gen_or_port: i, func_cap);
881	if (err) {
882	mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",
883	i, err);
884	goto err_mem;
885	}
886	caps->spec_qps[i - 1] = func_cap->spec_qps;
887	caps->port_mask[i] = caps->port_type[i];
888	caps->phys_port_id[i] = func_cap->phys_port_id;
889	err = mlx4_get_slave_pkey_gid_tbl_len(dev, port: i,
890	gid_tbl_len: &caps->gid_table_len[i],
891	pkey_tbl_len: &caps->pkey_table_len[i]);
892	if (err) {
893	mlx4_err(dev, "QUERY_PORT command failed for port %d, aborting (%d)\n",
894	i, err);
895	goto err_mem;
896	}
897	}
898
899	err_mem:
900	if (err)
901	mlx4_slave_destroy_special_qp_cap(dev);
902	kfree(objp: func_cap);
903	return err;
904	}
905
906	static int mlx4_slave_cap(struct mlx4_dev *dev)
907	{
908	int err;
909	u32 page_size;
910	struct mlx4_dev_cap *dev_cap;
911	struct mlx4_func_cap *func_cap;
912	struct mlx4_init_hca_param *hca_param;
913
914	hca_param = kzalloc(size: sizeof(*hca_param), GFP_KERNEL);
915	func_cap = kzalloc(size: sizeof(*func_cap), GFP_KERNEL);
916	dev_cap = kzalloc(size: sizeof(*dev_cap), GFP_KERNEL);
917	if (!hca_param || !func_cap || !dev_cap) {
918	mlx4_err(dev, "Failed to allocate memory for slave_cap\n");
919	err = -ENOMEM;
920	goto free_mem;
921	}
922
923	err = mlx4_QUERY_HCA(dev, param: hca_param);
924	if (err) {
925	mlx4_err(dev, "QUERY_HCA command failed, aborting\n");
926	goto free_mem;
927	}
928
929	/* fail if the hca has an unknown global capability
930	* at this time global_caps should be always zeroed
931	*/
932	if (hca_param->global_caps) {
933	mlx4_err(dev, "Unknown hca global capabilities\n");
934	err = -EINVAL;
935	goto free_mem;
936	}
937
938	dev->caps.hca_core_clock = hca_param->hca_core_clock;
939
940	dev->caps.max_qp_dest_rdma = 1 << hca_param->log_rd_per_qp;
941	err = mlx4_dev_cap(dev, dev_cap);
942	if (err) {
943	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
944	goto free_mem;
945	}
946
947	err = mlx4_QUERY_FW(dev);
948	if (err)
949	mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n");
950
951	page_size = ~dev->caps.page_size_cap + 1;
952	mlx4_warn(dev, "HCA minimum page size:%d\n", page_size);
953	if (page_size > PAGE_SIZE) {
954	mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
955	page_size, PAGE_SIZE);
956	err = -ENODEV;
957	goto free_mem;
958	}
959
960	/* Set uar_page_shift for VF */
961	dev->uar_page_shift = hca_param->uar_page_sz + 12;
962
963	/* Make sure the master uar page size is valid */
964	if (dev->uar_page_shift > PAGE_SHIFT) {
965	mlx4_err(dev,
966	"Invalid configuration: uar page size is larger than system page size\n");
967	err = -ENODEV;
968	goto free_mem;
969	}
970
971	/* Set reserved_uars based on the uar_page_shift */
972	mlx4_set_num_reserved_uars(dev, dev_cap);
973
974	/* Although uar page size in FW differs from system page size,
975	* upper software layers (mlx4_ib, mlx4_en and part of mlx4_core)
976	* still works with assumption that uar page size == system page size
977	*/
978	dev->caps.uar_page_size = PAGE_SIZE;
979
980	err = mlx4_QUERY_FUNC_CAP(dev, gen_or_port: 0, func_cap);
981	if (err) {
982	mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n",
983	err);
984	goto free_mem;
985	}
986
987	if ((func_cap->pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) !=
988	PF_CONTEXT_BEHAVIOUR_MASK) {
989	mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n",
990	func_cap->pf_context_behaviour,
991	PF_CONTEXT_BEHAVIOUR_MASK);
992	err = -EINVAL;
993	goto free_mem;
994	}
995
996	dev->caps.num_ports = func_cap->num_ports;
997	dev->quotas.qp = func_cap->qp_quota;
998	dev->quotas.srq = func_cap->srq_quota;
999	dev->quotas.cq = func_cap->cq_quota;
1000	dev->quotas.mpt = func_cap->mpt_quota;
1001	dev->quotas.mtt = func_cap->mtt_quota;
1002	dev->caps.num_qps = 1 << hca_param->log_num_qps;
1003	dev->caps.num_srqs = 1 << hca_param->log_num_srqs;
1004	dev->caps.num_cqs = 1 << hca_param->log_num_cqs;
1005	dev->caps.num_mpts = 1 << hca_param->log_mpt_sz;
1006	dev->caps.num_eqs = func_cap->max_eq;
1007	dev->caps.reserved_eqs = func_cap->reserved_eq;
1008	dev->caps.reserved_lkey = func_cap->reserved_lkey;
1009	dev->caps.num_pds = MLX4_NUM_PDS;
1010	dev->caps.num_mgms = 0;
1011	dev->caps.num_amgms = 0;
1012
1013	if (dev->caps.num_ports > MLX4_MAX_PORTS) {
1014	mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
1015	dev->caps.num_ports, MLX4_MAX_PORTS);
1016	err = -ENODEV;
1017	goto free_mem;
1018	}
1019
1020	mlx4_replace_zero_macs(dev);
1021
1022	err = mlx4_slave_special_qp_cap(dev);
1023	if (err) {
1024	mlx4_err(dev, "Set special QP caps failed. aborting\n");
1025	goto free_mem;
1026	}
1027
1028	if (dev->caps.uar_page_size * (dev->caps.num_uars -
1029	dev->caps.reserved_uars) >
1030	pci_resource_len(dev->persist->pdev,
1031	2)) {
1032	mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
1033	dev->caps.uar_page_size * dev->caps.num_uars,
1034	(unsigned long long)
1035	pci_resource_len(dev->persist->pdev, 2));
1036	err = -ENOMEM;
1037	goto err_mem;
1038	}
1039
1040	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
1041	dev->caps.eqe_size = 64;
1042	dev->caps.eqe_factor = 1;
1043	} else {
1044	dev->caps.eqe_size = 32;
1045	dev->caps.eqe_factor = 0;
1046	}
1047
1048	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
1049	dev->caps.cqe_size = 64;
1050	dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
1051	} else {
1052	dev->caps.cqe_size = 32;
1053	}
1054
1055	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) {
1056	dev->caps.eqe_size = hca_param->eqe_size;
1057	dev->caps.eqe_factor = 0;
1058	}
1059
1060	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) {
1061	dev->caps.cqe_size = hca_param->cqe_size;
1062	/* User still need to know when CQE > 32B */
1063	dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
1064	}
1065
1066	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1067	mlx4_warn(dev, "Timestamping is not supported in slave mode\n");
1068
1069	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_USER_MAC_EN;
1070	mlx4_dbg(dev, "User MAC FW update is not supported in slave mode\n");
1071
1072	slave_adjust_steering_mode(dev, dev_cap, hca_param);
1073	mlx4_dbg(dev, "RSS support for IP fragments is %s\n",
1074	hca_param->rss_ip_frags ? "on" : "off");
1075
1076	if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP &&
1077	dev->caps.bf_reg_size)
1078	dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;
1079
1080	if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP)
1081	dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP;
1082
1083	err_mem:
1084	if (err)
1085	mlx4_slave_destroy_special_qp_cap(dev);
1086	free_mem:
1087	kfree(objp: hca_param);
1088	kfree(objp: func_cap);
1089	kfree(objp: dev_cap);
1090	return err;
1091	}
1092
1093	/*
1094	* Change the port configuration of the device.
1095	* Every user of this function must hold the port mutex.
1096	*/
1097	int mlx4_change_port_types(struct mlx4_dev *dev,
1098	enum mlx4_port_type *port_types)
1099	{
1100	int err = 0;
1101	int change = 0;
1102	int port;
1103
1104	for (port = 0; port < dev->caps.num_ports; port++) {
1105	/* Change the port type only if the new type is different
1106	* from the current, and not set to Auto */
1107	if (port_types[port] != dev->caps.port_type[port + 1])
1108	change = 1;
1109	}
1110	if (change) {
1111	mlx4_unregister_device(dev);
1112	for (port = 1; port <= dev->caps.num_ports; port++) {
1113	mlx4_CLOSE_PORT(dev, port);
1114	dev->caps.port_type[port] = port_types[port - 1];
1115	err = mlx4_SET_PORT(dev, port, pkey_tbl_sz: -1);
1116	if (err) {
1117	mlx4_err(dev, "Failed to set port %d, aborting\n",
1118	port);
1119	goto out;
1120	}
1121	}
1122	mlx4_set_port_mask(dev);
1123	err = mlx4_register_device(dev);
1124	if (err) {
1125	mlx4_err(dev, "Failed to register device\n");
1126	goto out;
1127	}
1128	}
1129
1130	out:
1131	return err;
1132	}
1133
1134	static ssize_t show_port_type(struct device *dev,
1135	struct device_attribute *attr,
1136	char *buf)
1137	{
1138	struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1139	port_attr);
1140	struct mlx4_dev *mdev = info->dev;
1141	char type[8];
1142
1143	sprintf(buf: type, fmt: "%s",
1144	(mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
1145	"ib" : "eth");
1146	if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
1147	sprintf(buf, fmt: "auto (%s)\n", type);
1148	else
1149	sprintf(buf, fmt: "%s\n", type);
1150
1151	return strlen(buf);
1152	}
1153
1154	static int __set_port_type(struct mlx4_port_info *info,
1155	enum mlx4_port_type port_type)
1156	{
1157	struct mlx4_dev *mdev = info->dev;
1158	struct mlx4_priv *priv = mlx4_priv(dev: mdev);
1159	enum mlx4_port_type types[MLX4_MAX_PORTS];
1160	enum mlx4_port_type new_types[MLX4_MAX_PORTS];
1161	int i;
1162	int err = 0;
1163
1164	if ((port_type & mdev->caps.supported_type[info->port]) != port_type) {
1165	mlx4_err(mdev,
1166	"Requested port type for port %d is not supported on this HCA\n",
1167	info->port);
1168	return -EOPNOTSUPP;
1169	}
1170
1171	mlx4_stop_sense(dev: mdev);
1172	mutex_lock(&priv->port_mutex);
1173	info->tmp_type = port_type;
1174
1175	/* Possible type is always the one that was delivered */
1176	mdev->caps.possible_type[info->port] = info->tmp_type;
1177
1178	for (i = 0; i < mdev->caps.num_ports; i++) {
1179	types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
1180	mdev->caps.possible_type[i+1];
1181	if (types[i] == MLX4_PORT_TYPE_AUTO)
1182	types[i] = mdev->caps.port_type[i+1];
1183	}
1184
1185	if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
1186	!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) {
1187	for (i = 1; i <= mdev->caps.num_ports; i++) {
1188	if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
1189	mdev->caps.possible_type[i] = mdev->caps.port_type[i];
1190	err = -EOPNOTSUPP;
1191	}
1192	}
1193	}
1194	if (err) {
1195	mlx4_err(mdev, "Auto sensing is not supported on this HCA. Set only 'eth' or 'ib' for both ports (should be the same)\n");
1196	goto out;
1197	}
1198
1199	mlx4_do_sense_ports(dev: mdev, stype: new_types, defaults: types);
1200
1201	err = mlx4_check_port_params(dev: mdev, port_type: new_types);
1202	if (err)
1203	goto out;
1204
1205	/* We are about to apply the changes after the configuration
1206	* was verified, no need to remember the temporary types
1207	* any more */
1208	for (i = 0; i < mdev->caps.num_ports; i++)
1209	priv->port[i + 1].tmp_type = 0;
1210
1211	err = mlx4_change_port_types(dev: mdev, port_types: new_types);
1212
1213	out:
1214	mlx4_start_sense(dev: mdev);
1215	mutex_unlock(lock: &priv->port_mutex);
1216
1217	return err;
1218	}
1219
1220	static ssize_t set_port_type(struct device *dev,
1221	struct device_attribute *attr,
1222	const char *buf, size_t count)
1223	{
1224	struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1225	port_attr);
1226	struct mlx4_dev *mdev = info->dev;
1227	enum mlx4_port_type port_type;
1228	static DEFINE_MUTEX(set_port_type_mutex);
1229	int err;
1230
1231	mutex_lock(&set_port_type_mutex);
1232
1233	if (!strcmp(buf, "ib\n")) {
1234	port_type = MLX4_PORT_TYPE_IB;
1235	} else if (!strcmp(buf, "eth\n")) {
1236	port_type = MLX4_PORT_TYPE_ETH;
1237	} else if (!strcmp(buf, "auto\n")) {
1238	port_type = MLX4_PORT_TYPE_AUTO;
1239	} else {
1240	mlx4_err(mdev, "%s is not supported port type\n", buf);
1241	err = -EINVAL;
1242	goto err_out;
1243	}
1244
1245	err = __set_port_type(info, port_type);
1246
1247	err_out:
1248	mutex_unlock(lock: &set_port_type_mutex);
1249
1250	return err ? err : count;
1251	}
1252
1253	enum ibta_mtu {
1254	IB_MTU_256 = 1,
1255	IB_MTU_512 = 2,
1256	IB_MTU_1024 = 3,
1257	IB_MTU_2048 = 4,
1258	IB_MTU_4096 = 5
1259	};
1260
1261	static inline int int_to_ibta_mtu(int mtu)
1262	{
1263	switch (mtu) {
1264	case 256: return IB_MTU_256;
1265	case 512: return IB_MTU_512;
1266	case 1024: return IB_MTU_1024;
1267	case 2048: return IB_MTU_2048;
1268	case 4096: return IB_MTU_4096;
1269	default: return -1;
1270	}
1271	}
1272
1273	static inline int ibta_mtu_to_int(enum ibta_mtu mtu)
1274	{
1275	switch (mtu) {
1276	case IB_MTU_256: return 256;
1277	case IB_MTU_512: return 512;
1278	case IB_MTU_1024: return 1024;
1279	case IB_MTU_2048: return 2048;
1280	case IB_MTU_4096: return 4096;
1281	default: return -1;
1282	}
1283	}
1284
1285	static ssize_t show_port_ib_mtu(struct device *dev,
1286	struct device_attribute *attr,
1287	char *buf)
1288	{
1289	struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1290	port_mtu_attr);
1291	struct mlx4_dev *mdev = info->dev;
1292
1293	if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH)
1294	mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1295
1296	sprintf(buf, fmt: "%d\n",
1297	ibta_mtu_to_int(mtu: mdev->caps.port_ib_mtu[info->port]));
1298	return strlen(buf);
1299	}
1300
1301	static ssize_t set_port_ib_mtu(struct device *dev,
1302	struct device_attribute *attr,
1303	const char *buf, size_t count)
1304	{
1305	struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1306	port_mtu_attr);
1307	struct mlx4_dev *mdev = info->dev;
1308	struct mlx4_priv *priv = mlx4_priv(dev: mdev);
1309	int err, port, mtu, ibta_mtu = -1;
1310
1311	if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) {
1312	mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1313	return -EINVAL;
1314	}
1315
1316	err = kstrtoint(s: buf, base: 0, res: &mtu);
1317	if (!err)
1318	ibta_mtu = int_to_ibta_mtu(mtu);
1319
1320	if (err || ibta_mtu < 0) {
1321	mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf);
1322	return -EINVAL;
1323	}
1324
1325	mdev->caps.port_ib_mtu[info->port] = ibta_mtu;
1326
1327	mlx4_stop_sense(dev: mdev);
1328	mutex_lock(&priv->port_mutex);
1329	mlx4_unregister_device(dev: mdev);
1330	for (port = 1; port <= mdev->caps.num_ports; port++) {
1331	mlx4_CLOSE_PORT(dev: mdev, port);
1332	err = mlx4_SET_PORT(dev: mdev, port, pkey_tbl_sz: -1);
1333	if (err) {
1334	mlx4_err(mdev, "Failed to set port %d, aborting\n",
1335	port);
1336	goto err_set_port;
1337	}
1338	}
1339	err = mlx4_register_device(dev: mdev);
1340	err_set_port:
1341	mutex_unlock(lock: &priv->port_mutex);
1342	mlx4_start_sense(dev: mdev);
1343	return err ? err : count;
1344	}
1345
1346	/* bond for multi-function device */
1347	#define MAX_MF_BOND_ALLOWED_SLAVES 63
1348	static int mlx4_mf_bond(struct mlx4_dev *dev)
1349	{
1350	int err = 0;
1351	int nvfs;
1352	struct mlx4_slaves_pport slaves_port1;
1353	struct mlx4_slaves_pport slaves_port2;
1354
1355	slaves_port1 = mlx4_phys_to_slaves_pport(dev, port: 1);
1356	slaves_port2 = mlx4_phys_to_slaves_pport(dev, port: 2);
1357
1358	/* only single port vfs are allowed */
1359	if (bitmap_weight_and(src1: slaves_port1.slaves, src2: slaves_port2.slaves,
1360	nbits: dev->persist->num_vfs + 1) > 1) {
1361	mlx4_warn(dev, "HA mode unsupported for dual ported VFs\n");
1362	return -EINVAL;
1363	}
1364
1365	/* number of virtual functions is number of total functions minus one
1366	* physical function for each port.
1367	*/
1368	nvfs = bitmap_weight(src: slaves_port1.slaves, nbits: dev->persist->num_vfs + 1) +
1369	bitmap_weight(src: slaves_port2.slaves, nbits: dev->persist->num_vfs + 1) - 2;
1370
1371	/* limit on maximum allowed VFs */
1372	if (nvfs > MAX_MF_BOND_ALLOWED_SLAVES) {
1373	mlx4_warn(dev, "HA mode is not supported for %d VFs (max %d are allowed)\n",
1374	nvfs, MAX_MF_BOND_ALLOWED_SLAVES);
1375	return -EINVAL;
1376	}
1377
1378	if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) {
1379	mlx4_warn(dev, "HA mode unsupported for NON DMFS steering\n");
1380	return -EINVAL;
1381	}
1382
1383	err = mlx4_bond_mac_table(dev);
1384	if (err)
1385	return err;
1386	err = mlx4_bond_vlan_table(dev);
1387	if (err)
1388	goto err1;
1389	err = mlx4_bond_fs_rules(dev);
1390	if (err)
1391	goto err2;
1392
1393	return 0;
1394	err2:
1395	(void)mlx4_unbond_vlan_table(dev);
1396	err1:
1397	(void)mlx4_unbond_mac_table(dev);
1398	return err;
1399	}
1400
1401	static int mlx4_mf_unbond(struct mlx4_dev *dev)
1402	{
1403	int ret, ret1;
1404
1405	ret = mlx4_unbond_fs_rules(dev);
1406	if (ret)
1407	mlx4_warn(dev, "multifunction unbond for flow rules failed (%d)\n", ret);
1408	ret1 = mlx4_unbond_mac_table(dev);
1409	if (ret1) {
1410	mlx4_warn(dev, "multifunction unbond for MAC table failed (%d)\n", ret1);
1411	ret = ret1;
1412	}
1413	ret1 = mlx4_unbond_vlan_table(dev);
1414	if (ret1) {
1415	mlx4_warn(dev, "multifunction unbond for VLAN table failed (%d)\n", ret1);
1416	ret = ret1;
1417	}
1418	return ret;
1419	}
1420
1421	static int mlx4_bond(struct mlx4_dev *dev)
1422	{
1423	int ret = 0;
1424	struct mlx4_priv *priv = mlx4_priv(dev);
1425
1426	mutex_lock(&priv->bond_mutex);
1427
1428	if (!mlx4_is_bonded(dev)) {
1429	ret = mlx4_do_bond(dev, enable: true);
1430	if (ret)
1431	mlx4_err(dev, "Failed to bond device: %d\n", ret);
1432	if (!ret && mlx4_is_master(dev)) {
1433	ret = mlx4_mf_bond(dev);
1434	if (ret) {
1435	mlx4_err(dev, "bond for multifunction failed\n");
1436	mlx4_do_bond(dev, enable: false);
1437	}
1438	}
1439	}
1440
1441	mutex_unlock(lock: &priv->bond_mutex);
1442	if (!ret)
1443	mlx4_dbg(dev, "Device is bonded\n");
1444
1445	return ret;
1446	}
1447
1448	static int mlx4_unbond(struct mlx4_dev *dev)
1449	{
1450	int ret = 0;
1451	struct mlx4_priv *priv = mlx4_priv(dev);
1452
1453	mutex_lock(&priv->bond_mutex);
1454
1455	if (mlx4_is_bonded(dev)) {
1456	int ret2 = 0;
1457
1458	ret = mlx4_do_bond(dev, enable: false);
1459	if (ret)
1460	mlx4_err(dev, "Failed to unbond device: %d\n", ret);
1461	if (mlx4_is_master(dev))
1462	ret2 = mlx4_mf_unbond(dev);
1463	if (ret2) {
1464	mlx4_warn(dev, "Failed to unbond device for multifunction (%d)\n", ret2);
1465	ret = ret2;
1466	}
1467	}
1468
1469	mutex_unlock(lock: &priv->bond_mutex);
1470	if (!ret)
1471	mlx4_dbg(dev, "Device is unbonded\n");
1472
1473	return ret;
1474	}
1475
1476	static int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p)
1477	{
1478	u8 port1 = v2p->port1;
1479	u8 port2 = v2p->port2;
1480	struct mlx4_priv *priv = mlx4_priv(dev);
1481	int err;
1482
1483	if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP))
1484	return -EOPNOTSUPP;
1485
1486	mutex_lock(&priv->bond_mutex);
1487
1488	/* zero means keep current mapping for this port */
1489	if (port1 == 0)
1490	port1 = priv->v2p.port1;
1491	if (port2 == 0)
1492	port2 = priv->v2p.port2;
1493
1494	if ((port1 < 1) || (port1 > MLX4_MAX_PORTS) ||
1495	(port2 < 1) || (port2 > MLX4_MAX_PORTS) ||
1496	(port1 == 2 && port2 == 1)) {
1497	/* besides boundary checks cross mapping makes
1498	* no sense and therefore not allowed */
1499	err = -EINVAL;
1500	} else if ((port1 == priv->v2p.port1) &&
1501	(port2 == priv->v2p.port2)) {
1502	err = 0;
1503	} else {
1504	err = mlx4_virt2phy_port_map(dev, port1, port2);
1505	if (!err) {
1506	mlx4_dbg(dev, "port map changed: [%d][%d]\n",
1507	port1, port2);
1508	priv->v2p.port1 = port1;
1509	priv->v2p.port2 = port2;
1510	} else {
1511	mlx4_err(dev, "Failed to change port mape: %d\n", err);
1512	}
1513	}
1514
1515	mutex_unlock(lock: &priv->bond_mutex);
1516	return err;
1517	}
1518
1519	struct mlx4_bond {
1520	struct work_struct work;
1521	struct mlx4_dev *dev;
1522	int is_bonded;
1523	struct mlx4_port_map port_map;
1524	};
1525
1526	static void mlx4_bond_work(struct work_struct *work)
1527	{
1528	struct mlx4_bond *bond = container_of(work, struct mlx4_bond, work);
1529	int err = 0;
1530
1531	if (bond->is_bonded) {
1532	if (!mlx4_is_bonded(dev: bond->dev)) {
1533	err = mlx4_bond(dev: bond->dev);
1534	if (err)
1535	mlx4_err(bond->dev, "Fail to bond device\n");
1536	}
1537	if (!err) {
1538	err = mlx4_port_map_set(dev: bond->dev, v2p: &bond->port_map);
1539	if (err)
1540	mlx4_err(bond->dev,
1541	"Fail to set port map [%d][%d]: %d\n",
1542	bond->port_map.port1,
1543	bond->port_map.port2, err);
1544	}
1545	} else if (mlx4_is_bonded(dev: bond->dev)) {
1546	err = mlx4_unbond(dev: bond->dev);
1547	if (err)
1548	mlx4_err(bond->dev, "Fail to unbond device\n");
1549	}
1550	put_device(dev: &bond->dev->persist->pdev->dev);
1551	kfree(objp: bond);
1552	}
1553
1554	int mlx4_queue_bond_work(struct mlx4_dev *dev, int is_bonded, u8 v2p_p1,
1555	u8 v2p_p2)
1556	{
1557	struct mlx4_bond *bond;
1558
1559	bond = kzalloc(size: sizeof(*bond), GFP_ATOMIC);
1560	if (!bond)
1561	return -ENOMEM;
1562
1563	INIT_WORK(&bond->work, mlx4_bond_work);
1564	get_device(dev: &dev->persist->pdev->dev);
1565	bond->dev = dev;
1566	bond->is_bonded = is_bonded;
1567	bond->port_map.port1 = v2p_p1;
1568	bond->port_map.port2 = v2p_p2;
1569	queue_work(wq: mlx4_wq, work: &bond->work);
1570	return 0;
1571	}
1572	EXPORT_SYMBOL(mlx4_queue_bond_work);
1573
1574	static int mlx4_load_fw(struct mlx4_dev *dev)
1575	{
1576	struct mlx4_priv *priv = mlx4_priv(dev);
1577	int err;
1578
1579	priv->fw.fw_icm = mlx4_alloc_icm(dev, npages: priv->fw.fw_pages,
1580	GFP_HIGHUSER | __GFP_NOWARN, coherent: 0);
1581	if (!priv->fw.fw_icm) {
1582	mlx4_err(dev, "Couldn't allocate FW area, aborting\n");
1583	return -ENOMEM;
1584	}
1585
1586	err = mlx4_MAP_FA(dev, icm: priv->fw.fw_icm);
1587	if (err) {
1588	mlx4_err(dev, "MAP_FA command failed, aborting\n");
1589	goto err_free;
1590	}
1591
1592	err = mlx4_RUN_FW(dev);
1593	if (err) {
1594	mlx4_err(dev, "RUN_FW command failed, aborting\n");
1595	goto err_unmap_fa;
1596	}
1597
1598	return 0;
1599
1600	err_unmap_fa:
1601	mlx4_UNMAP_FA(dev);
1602
1603	err_free:
1604	mlx4_free_icm(dev, icm: priv->fw.fw_icm, coherent: 0);
1605	return err;
1606	}
1607
1608	static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
1609	int cmpt_entry_sz)
1610	{
1611	struct mlx4_priv *priv = mlx4_priv(dev);
1612	int err;
1613	int num_eqs;
1614
1615	err = mlx4_init_icm_table(dev, table: &priv->qp_table.cmpt_table,
1616	virt: cmpt_base +
1617	((u64) (MLX4_CMPT_TYPE_QP *
1618	cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1619	obj_size: cmpt_entry_sz, nobj: dev->caps.num_qps,
1620	reserved: dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1621	use_lowmem: 0, use_coherent: 0);
1622	if (err)
1623	goto err;
1624
1625	err = mlx4_init_icm_table(dev, table: &priv->srq_table.cmpt_table,
1626	virt: cmpt_base +
1627	((u64) (MLX4_CMPT_TYPE_SRQ *
1628	cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1629	obj_size: cmpt_entry_sz, nobj: dev->caps.num_srqs,
1630	reserved: dev->caps.reserved_srqs, use_lowmem: 0, use_coherent: 0);
1631	if (err)
1632	goto err_qp;
1633
1634	err = mlx4_init_icm_table(dev, table: &priv->cq_table.cmpt_table,
1635	virt: cmpt_base +
1636	((u64) (MLX4_CMPT_TYPE_CQ *
1637	cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1638	obj_size: cmpt_entry_sz, nobj: dev->caps.num_cqs,
1639	reserved: dev->caps.reserved_cqs, use_lowmem: 0, use_coherent: 0);
1640	if (err)
1641	goto err_srq;
1642
1643	num_eqs = dev->phys_caps.num_phys_eqs;
1644	err = mlx4_init_icm_table(dev, table: &priv->eq_table.cmpt_table,
1645	virt: cmpt_base +
1646	((u64) (MLX4_CMPT_TYPE_EQ *
1647	cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1648	obj_size: cmpt_entry_sz, nobj: num_eqs, reserved: num_eqs, use_lowmem: 0, use_coherent: 0);
1649	if (err)
1650	goto err_cq;
1651
1652	return 0;
1653
1654	err_cq:
1655	mlx4_cleanup_icm_table(dev, table: &priv->cq_table.cmpt_table);
1656
1657	err_srq:
1658	mlx4_cleanup_icm_table(dev, table: &priv->srq_table.cmpt_table);
1659
1660	err_qp:
1661	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.cmpt_table);
1662
1663	err:
1664	return err;
1665	}
1666
1667	static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
1668	struct mlx4_init_hca_param *init_hca, u64 icm_size)
1669	{
1670	struct mlx4_priv *priv = mlx4_priv(dev);
1671	u64 aux_pages;
1672	int num_eqs;
1673	int err;
1674
1675	err = mlx4_SET_ICM_SIZE(dev, icm_size, aux_pages: &aux_pages);
1676	if (err) {
1677	mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n");
1678	return err;
1679	}
1680
1681	mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n",
1682	(unsigned long long) icm_size >> 10,
1683	(unsigned long long) aux_pages << 2);
1684
1685	priv->fw.aux_icm = mlx4_alloc_icm(dev, npages: aux_pages,
1686	GFP_HIGHUSER | __GFP_NOWARN, coherent: 0);
1687	if (!priv->fw.aux_icm) {
1688	mlx4_err(dev, "Couldn't allocate aux memory, aborting\n");
1689	return -ENOMEM;
1690	}
1691
1692	err = mlx4_MAP_ICM_AUX(dev, icm: priv->fw.aux_icm);
1693	if (err) {
1694	mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n");
1695	goto err_free_aux;
1696	}
1697
1698	err = mlx4_init_cmpt_table(dev, cmpt_base: init_hca->cmpt_base, cmpt_entry_sz: dev_cap->cmpt_entry_sz);
1699	if (err) {
1700	mlx4_err(dev, "Failed to map cMPT context memory, aborting\n");
1701	goto err_unmap_aux;
1702	}
1703
1704
1705	num_eqs = dev->phys_caps.num_phys_eqs;
1706	err = mlx4_init_icm_table(dev, table: &priv->eq_table.table,
1707	virt: init_hca->eqc_base, obj_size: dev_cap->eqc_entry_sz,
1708	nobj: num_eqs, reserved: num_eqs, use_lowmem: 0, use_coherent: 0);
1709	if (err) {
1710	mlx4_err(dev, "Failed to map EQ context memory, aborting\n");
1711	goto err_unmap_cmpt;
1712	}
1713
1714	/*
1715	* Reserved MTT entries must be aligned up to a cacheline
1716	* boundary, since the FW will write to them, while the driver
1717	* writes to all other MTT entries. (The variable
1718	* dev->caps.mtt_entry_sz below is really the MTT segment
1719	* size, not the raw entry size)
1720	*/
1721	dev->caps.reserved_mtts =
1722	ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
1723	dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
1724
1725	err = mlx4_init_icm_table(dev, table: &priv->mr_table.mtt_table,
1726	virt: init_hca->mtt_base,
1727	obj_size: dev->caps.mtt_entry_sz,
1728	nobj: dev->caps.num_mtts,
1729	reserved: dev->caps.reserved_mtts, use_lowmem: 1, use_coherent: 0);
1730	if (err) {
1731	mlx4_err(dev, "Failed to map MTT context memory, aborting\n");
1732	goto err_unmap_eq;
1733	}
1734
1735	err = mlx4_init_icm_table(dev, table: &priv->mr_table.dmpt_table,
1736	virt: init_hca->dmpt_base,
1737	obj_size: dev_cap->dmpt_entry_sz,
1738	nobj: dev->caps.num_mpts,
1739	reserved: dev->caps.reserved_mrws, use_lowmem: 1, use_coherent: 1);
1740	if (err) {
1741	mlx4_err(dev, "Failed to map dMPT context memory, aborting\n");
1742	goto err_unmap_mtt;
1743	}
1744
1745	err = mlx4_init_icm_table(dev, table: &priv->qp_table.qp_table,
1746	virt: init_hca->qpc_base,
1747	obj_size: dev_cap->qpc_entry_sz,
1748	nobj: dev->caps.num_qps,
1749	reserved: dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1750	use_lowmem: 0, use_coherent: 0);
1751	if (err) {
1752	mlx4_err(dev, "Failed to map QP context memory, aborting\n");
1753	goto err_unmap_dmpt;
1754	}
1755
1756	err = mlx4_init_icm_table(dev, table: &priv->qp_table.auxc_table,
1757	virt: init_hca->auxc_base,
1758	obj_size: dev_cap->aux_entry_sz,
1759	nobj: dev->caps.num_qps,
1760	reserved: dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1761	use_lowmem: 0, use_coherent: 0);
1762	if (err) {
1763	mlx4_err(dev, "Failed to map AUXC context memory, aborting\n");
1764	goto err_unmap_qp;
1765	}
1766
1767	err = mlx4_init_icm_table(dev, table: &priv->qp_table.altc_table,
1768	virt: init_hca->altc_base,
1769	obj_size: dev_cap->altc_entry_sz,
1770	nobj: dev->caps.num_qps,
1771	reserved: dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1772	use_lowmem: 0, use_coherent: 0);
1773	if (err) {
1774	mlx4_err(dev, "Failed to map ALTC context memory, aborting\n");
1775	goto err_unmap_auxc;
1776	}
1777
1778	err = mlx4_init_icm_table(dev, table: &priv->qp_table.rdmarc_table,
1779	virt: init_hca->rdmarc_base,
1780	obj_size: dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
1781	nobj: dev->caps.num_qps,
1782	reserved: dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1783	use_lowmem: 0, use_coherent: 0);
1784	if (err) {
1785	mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
1786	goto err_unmap_altc;
1787	}
1788
1789	err = mlx4_init_icm_table(dev, table: &priv->cq_table.table,
1790	virt: init_hca->cqc_base,
1791	obj_size: dev_cap->cqc_entry_sz,
1792	nobj: dev->caps.num_cqs,
1793	reserved: dev->caps.reserved_cqs, use_lowmem: 0, use_coherent: 0);
1794	if (err) {
1795	mlx4_err(dev, "Failed to map CQ context memory, aborting\n");
1796	goto err_unmap_rdmarc;
1797	}
1798
1799	err = mlx4_init_icm_table(dev, table: &priv->srq_table.table,
1800	virt: init_hca->srqc_base,
1801	obj_size: dev_cap->srq_entry_sz,
1802	nobj: dev->caps.num_srqs,
1803	reserved: dev->caps.reserved_srqs, use_lowmem: 0, use_coherent: 0);
1804	if (err) {
1805	mlx4_err(dev, "Failed to map SRQ context memory, aborting\n");
1806	goto err_unmap_cq;
1807	}
1808
1809	/*
1810	* For flow steering device managed mode it is required to use
1811	* mlx4_init_icm_table. For B0 steering mode it's not strictly
1812	* required, but for simplicity just map the whole multicast
1813	* group table now. The table isn't very big and it's a lot
1814	* easier than trying to track ref counts.
1815	*/
1816	err = mlx4_init_icm_table(dev, table: &priv->mcg_table.table,
1817	virt: init_hca->mc_base,
1818	obj_size: mlx4_get_mgm_entry_size(dev),
1819	nobj: dev->caps.num_mgms + dev->caps.num_amgms,
1820	reserved: dev->caps.num_mgms + dev->caps.num_amgms,
1821	use_lowmem: 0, use_coherent: 0);
1822	if (err) {
1823	mlx4_err(dev, "Failed to map MCG context memory, aborting\n");
1824	goto err_unmap_srq;
1825	}
1826
1827	return 0;
1828
1829	err_unmap_srq:
1830	mlx4_cleanup_icm_table(dev, table: &priv->srq_table.table);
1831
1832	err_unmap_cq:
1833	mlx4_cleanup_icm_table(dev, table: &priv->cq_table.table);
1834
1835	err_unmap_rdmarc:
1836	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.rdmarc_table);
1837
1838	err_unmap_altc:
1839	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.altc_table);
1840
1841	err_unmap_auxc:
1842	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.auxc_table);
1843
1844	err_unmap_qp:
1845	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.qp_table);
1846
1847	err_unmap_dmpt:
1848	mlx4_cleanup_icm_table(dev, table: &priv->mr_table.dmpt_table);
1849
1850	err_unmap_mtt:
1851	mlx4_cleanup_icm_table(dev, table: &priv->mr_table.mtt_table);
1852
1853	err_unmap_eq:
1854	mlx4_cleanup_icm_table(dev, table: &priv->eq_table.table);
1855
1856	err_unmap_cmpt:
1857	mlx4_cleanup_icm_table(dev, table: &priv->eq_table.cmpt_table);
1858	mlx4_cleanup_icm_table(dev, table: &priv->cq_table.cmpt_table);
1859	mlx4_cleanup_icm_table(dev, table: &priv->srq_table.cmpt_table);
1860	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.cmpt_table);
1861
1862	err_unmap_aux:
1863	mlx4_UNMAP_ICM_AUX(dev);
1864
1865	err_free_aux:
1866	mlx4_free_icm(dev, icm: priv->fw.aux_icm, coherent: 0);
1867
1868	return err;
1869	}
1870
1871	static void mlx4_free_icms(struct mlx4_dev *dev)
1872	{
1873	struct mlx4_priv *priv = mlx4_priv(dev);
1874
1875	mlx4_cleanup_icm_table(dev, table: &priv->mcg_table.table);
1876	mlx4_cleanup_icm_table(dev, table: &priv->srq_table.table);
1877	mlx4_cleanup_icm_table(dev, table: &priv->cq_table.table);
1878	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.rdmarc_table);
1879	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.altc_table);
1880	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.auxc_table);
1881	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.qp_table);
1882	mlx4_cleanup_icm_table(dev, table: &priv->mr_table.dmpt_table);
1883	mlx4_cleanup_icm_table(dev, table: &priv->mr_table.mtt_table);
1884	mlx4_cleanup_icm_table(dev, table: &priv->eq_table.table);
1885	mlx4_cleanup_icm_table(dev, table: &priv->eq_table.cmpt_table);
1886	mlx4_cleanup_icm_table(dev, table: &priv->cq_table.cmpt_table);
1887	mlx4_cleanup_icm_table(dev, table: &priv->srq_table.cmpt_table);
1888	mlx4_cleanup_icm_table(dev, table: &priv->qp_table.cmpt_table);
1889
1890	mlx4_UNMAP_ICM_AUX(dev);
1891	mlx4_free_icm(dev, icm: priv->fw.aux_icm, coherent: 0);
1892	}
1893
1894	static void mlx4_slave_exit(struct mlx4_dev *dev)
1895	{
1896	struct mlx4_priv *priv = mlx4_priv(dev);
1897
1898	mutex_lock(&priv->cmd.slave_cmd_mutex);
1899	if (mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_RESET, param: 0, MLX4_COMM_CMD_NA_OP,
1900	MLX4_COMM_TIME))
1901	mlx4_warn(dev, "Failed to close slave function\n");
1902	mutex_unlock(lock: &priv->cmd.slave_cmd_mutex);
1903	}
1904
1905	static int map_bf_area(struct mlx4_dev *dev)
1906	{
1907	struct mlx4_priv *priv = mlx4_priv(dev);
1908	resource_size_t bf_start;
1909	resource_size_t bf_len;
1910	int err = 0;
1911
1912	if (!dev->caps.bf_reg_size)
1913	return -ENXIO;
1914
1915	bf_start = pci_resource_start(dev->persist->pdev, 2) +
1916	(dev->caps.num_uars << PAGE_SHIFT);
1917	bf_len = pci_resource_len(dev->persist->pdev, 2) -
1918	(dev->caps.num_uars << PAGE_SHIFT);
1919	priv->bf_mapping = io_mapping_create_wc(base: bf_start, size: bf_len);
1920	if (!priv->bf_mapping)
1921	err = -ENOMEM;
1922
1923	return err;
1924	}
1925
1926	static void unmap_bf_area(struct mlx4_dev *dev)
1927	{
1928	if (mlx4_priv(dev)->bf_mapping)
1929	io_mapping_free(iomap: mlx4_priv(dev)->bf_mapping);
1930	}
1931
1932	u64 mlx4_read_clock(struct mlx4_dev *dev)
1933	{
1934	u32 clockhi, clocklo, clockhi1;
1935	u64 cycles;
1936	int i;
1937	struct mlx4_priv *priv = mlx4_priv(dev);
1938
1939	for (i = 0; i < 10; i++) {
1940	clockhi = swab32(readl(priv->clock_mapping));
1941	clocklo = swab32(readl(priv->clock_mapping + 4));
1942	clockhi1 = swab32(readl(priv->clock_mapping));
1943	if (clockhi == clockhi1)
1944	break;
1945	}
1946
1947	cycles = (u64) clockhi << 32 | (u64) clocklo;
1948
1949	return cycles;
1950	}
1951	EXPORT_SYMBOL_GPL(mlx4_read_clock);
1952
1953
1954	static int map_internal_clock(struct mlx4_dev *dev)
1955	{
1956	struct mlx4_priv *priv = mlx4_priv(dev);
1957
1958	priv->clock_mapping =
1959	ioremap(pci_resource_start(dev->persist->pdev,
1960	priv->fw.clock_bar) +
1961	priv->fw.clock_offset, size: MLX4_CLOCK_SIZE);
1962
1963	if (!priv->clock_mapping)
1964	return -ENOMEM;
1965
1966	return 0;
1967	}
1968
1969	int mlx4_get_internal_clock_params(struct mlx4_dev *dev,
1970	struct mlx4_clock_params *params)
1971	{
1972	struct mlx4_priv *priv = mlx4_priv(dev);
1973
1974	if (mlx4_is_slave(dev))
1975	return -EOPNOTSUPP;
1976
1977	if (!dev->caps.map_clock_to_user) {
1978	mlx4_dbg(dev, "Map clock to user is not supported.\n");
1979	return -EOPNOTSUPP;
1980	}
1981
1982	if (!params)
1983	return -EINVAL;
1984
1985	params->bar = priv->fw.clock_bar;
1986	params->offset = priv->fw.clock_offset;
1987	params->size = MLX4_CLOCK_SIZE;
1988
1989	return 0;
1990	}
1991	EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params);
1992
1993	static void unmap_internal_clock(struct mlx4_dev *dev)
1994	{
1995	struct mlx4_priv *priv = mlx4_priv(dev);
1996
1997	if (priv->clock_mapping)
1998	iounmap(addr: priv->clock_mapping);
1999	}
2000
2001	static void mlx4_close_hca(struct mlx4_dev *dev)
2002	{
2003	unmap_internal_clock(dev);
2004	unmap_bf_area(dev);
2005	if (mlx4_is_slave(dev))
2006	mlx4_slave_exit(dev);
2007	else {
2008	mlx4_CLOSE_HCA(dev, panic: 0);
2009	mlx4_free_icms(dev);
2010	}
2011	}
2012
2013	static void mlx4_close_fw(struct mlx4_dev *dev)
2014	{
2015	if (!mlx4_is_slave(dev)) {
2016	mlx4_UNMAP_FA(dev);
2017	mlx4_free_icm(dev, icm: mlx4_priv(dev)->fw.fw_icm, coherent: 0);
2018	}
2019	}
2020
2021	static int mlx4_comm_check_offline(struct mlx4_dev *dev)
2022	{
2023	#define COMM_CHAN_OFFLINE_OFFSET 0x09
2024
2025	u32 comm_flags;
2026	u32 offline_bit;
2027	unsigned long end;
2028	struct mlx4_priv *priv = mlx4_priv(dev);
2029
2030	end = msecs_to_jiffies(MLX4_COMM_OFFLINE_TIME_OUT) + jiffies;
2031	while (time_before(jiffies, end)) {
2032	comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm +
2033	MLX4_COMM_CHAN_FLAGS));
2034	offline_bit = (comm_flags &
2035	(u32)(1 << COMM_CHAN_OFFLINE_OFFSET));
2036	if (!offline_bit)
2037	return 0;
2038
2039	/* If device removal has been requested,
2040	* do not continue retrying.
2041	*/
2042	if (dev->persist->interface_state &
2043	MLX4_INTERFACE_STATE_NOWAIT)
2044	break;
2045
2046	/* There are cases as part of AER/Reset flow that PF needs
2047	* around 100 msec to load. We therefore sleep for 100 msec
2048	* to allow other tasks to make use of that CPU during this
2049	* time interval.
2050	*/
2051	msleep(msecs: 100);
2052	}
2053	mlx4_err(dev, "Communication channel is offline.\n");
2054	return -EIO;
2055	}
2056
2057	static void mlx4_reset_vf_support(struct mlx4_dev *dev)
2058	{
2059	#define COMM_CHAN_RST_OFFSET 0x1e
2060
2061	struct mlx4_priv *priv = mlx4_priv(dev);
2062	u32 comm_rst;
2063	u32 comm_caps;
2064
2065	comm_caps = swab32(readl((__iomem char *)priv->mfunc.comm +
2066	MLX4_COMM_CHAN_CAPS));
2067	comm_rst = (comm_caps & (u32)(1 << COMM_CHAN_RST_OFFSET));
2068
2069	if (comm_rst)
2070	dev->caps.vf_caps |= MLX4_VF_CAP_FLAG_RESET;
2071	}
2072
2073	static int mlx4_init_slave(struct mlx4_dev *dev)
2074	{
2075	struct mlx4_priv *priv = mlx4_priv(dev);
2076	u64 dma = (u64) priv->mfunc.vhcr_dma;
2077	int ret_from_reset = 0;
2078	u32 slave_read;
2079	u32 cmd_channel_ver;
2080
2081	if (atomic_read(v: &pf_loading)) {
2082	mlx4_warn(dev, "PF is not ready - Deferring probe\n");
2083	return -EPROBE_DEFER;
2084	}
2085
2086	mutex_lock(&priv->cmd.slave_cmd_mutex);
2087	priv->cmd.max_cmds = 1;
2088	if (mlx4_comm_check_offline(dev)) {
2089	mlx4_err(dev, "PF is not responsive, skipping initialization\n");
2090	goto err_offline;
2091	}
2092
2093	mlx4_reset_vf_support(dev);
2094	mlx4_warn(dev, "Sending reset\n");
2095	ret_from_reset = mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_RESET, param: 0,
2096	MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME);
2097	/* if we are in the middle of flr the slave will try
2098	* NUM_OF_RESET_RETRIES times before leaving.*/
2099	if (ret_from_reset) {
2100	if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) {
2101	mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n");
2102	mutex_unlock(lock: &priv->cmd.slave_cmd_mutex);
2103	return -EPROBE_DEFER;
2104	} else
2105	goto err;
2106	}
2107
2108	/* check the driver version - the slave I/F revision
2109	* must match the master's */
2110	slave_read = swab32(readl(&priv->mfunc.comm->slave_read));
2111	cmd_channel_ver = mlx4_comm_get_version();
2112
2113	if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) !=
2114	MLX4_COMM_GET_IF_REV(slave_read)) {
2115	mlx4_err(dev, "slave driver version is not supported by the master\n");
2116	goto err;
2117	}
2118
2119	mlx4_warn(dev, "Sending vhcr0\n");
2120	if (mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_VHCR0, param: dma >> 48,
2121	MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
2122	goto err;
2123	if (mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_VHCR1, param: dma >> 32,
2124	MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
2125	goto err;
2126	if (mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_VHCR2, param: dma >> 16,
2127	MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
2128	goto err;
2129	if (mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_VHCR_EN, param: dma,
2130	MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
2131	goto err;
2132
2133	mutex_unlock(lock: &priv->cmd.slave_cmd_mutex);
2134	return 0;
2135
2136	err:
2137	mlx4_comm_cmd(dev, cmd: MLX4_COMM_CMD_RESET, param: 0, MLX4_COMM_CMD_NA_OP, timeout: 0);
2138	err_offline:
2139	mutex_unlock(lock: &priv->cmd.slave_cmd_mutex);
2140	return -EIO;
2141	}
2142
2143	static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev)
2144	{
2145	int i;
2146
2147	for (i = 1; i <= dev->caps.num_ports; i++) {
2148	if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)
2149	dev->caps.gid_table_len[i] =
2150	mlx4_get_slave_num_gids(dev, slave: 0, port: i);
2151	else
2152	dev->caps.gid_table_len[i] = 1;
2153	dev->caps.pkey_table_len[i] =
2154	dev->phys_caps.pkey_phys_table_len[i] - 1;
2155	}
2156	}
2157
2158	static int choose_log_fs_mgm_entry_size(int qp_per_entry)
2159	{
2160	int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE;
2161
2162	for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE;
2163	i++) {
2164	if (qp_per_entry <= 4 * ((1 << i) / 16 - 2))
2165	break;
2166	}
2167
2168	return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1;
2169	}
2170
2171	static const char *dmfs_high_rate_steering_mode_str(int dmfs_high_steer_mode)
2172	{
2173	switch (dmfs_high_steer_mode) {
2174	case MLX4_STEERING_DMFS_A0_DEFAULT:
2175	return "default performance";
2176
2177	case MLX4_STEERING_DMFS_A0_DYNAMIC:
2178	return "dynamic hybrid mode";
2179
2180	case MLX4_STEERING_DMFS_A0_STATIC:
2181	return "performance optimized for limited rule configuration (static)";
2182
2183	case MLX4_STEERING_DMFS_A0_DISABLE:
2184	return "disabled performance optimized steering";
2185
2186	case MLX4_STEERING_DMFS_A0_NOT_SUPPORTED:
2187	return "performance optimized steering not supported";
2188
2189	default:
2190	return "Unrecognized mode";
2191	}
2192	}
2193
2194	#define MLX4_DMFS_A0_STEERING (1UL << 2)
2195
2196	static void choose_steering_mode(struct mlx4_dev *dev,
2197	struct mlx4_dev_cap *dev_cap)
2198	{
2199	if (mlx4_log_num_mgm_entry_size <= 0) {
2200	if ((-mlx4_log_num_mgm_entry_size) & MLX4_DMFS_A0_STEERING) {
2201	if (dev->caps.dmfs_high_steer_mode ==
2202	MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2203	mlx4_err(dev, "DMFS high rate mode not supported\n");
2204	else
2205	dev->caps.dmfs_high_steer_mode =
2206	MLX4_STEERING_DMFS_A0_STATIC;
2207	}
2208	}
2209
2210	if (mlx4_log_num_mgm_entry_size <= 0 &&
2211	dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN &&
2212	(!mlx4_is_mfunc(dev) ||
2213	(dev_cap->fs_max_num_qp_per_entry >=
2214	(dev->persist->num_vfs + 1))) &&
2215	choose_log_fs_mgm_entry_size(qp_per_entry: dev_cap->fs_max_num_qp_per_entry) >=
2216	MLX4_MIN_MGM_LOG_ENTRY_SIZE) {
2217	dev->oper_log_mgm_entry_size =
2218	choose_log_fs_mgm_entry_size(qp_per_entry: dev_cap->fs_max_num_qp_per_entry);
2219	dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED;
2220	dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
2221	dev->caps.fs_log_max_ucast_qp_range_size =
2222	dev_cap->fs_log_max_ucast_qp_range_size;
2223	} else {
2224	if (dev->caps.dmfs_high_steer_mode !=
2225	MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2226	dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DISABLE;
2227	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER &&
2228	dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
2229	dev->caps.steering_mode = MLX4_STEERING_MODE_B0;
2230	else {
2231	dev->caps.steering_mode = MLX4_STEERING_MODE_A0;
2232
2233	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER ||
2234	dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
2235	mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back to A0 steering mode\n");
2236	}
2237	dev->oper_log_mgm_entry_size =
2238	mlx4_log_num_mgm_entry_size > 0 ?
2239	mlx4_log_num_mgm_entry_size :
2240	MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
2241	dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev);
2242	}
2243	mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size = %d\n",
2244	mlx4_steering_mode_str(dev->caps.steering_mode),
2245	dev->oper_log_mgm_entry_size,
2246	mlx4_log_num_mgm_entry_size);
2247	}
2248
2249	static void choose_tunnel_offload_mode(struct mlx4_dev *dev,
2250	struct mlx4_dev_cap *dev_cap)
2251	{
2252	if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED &&
2253	dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS)
2254	dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN;
2255	else
2256	dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE;
2257
2258	mlx4_dbg(dev, "Tunneling offload mode is: %s\n", (dev->caps.tunnel_offload_mode
2259	== MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) ? "vxlan" : "none");
2260	}
2261
2262	static int mlx4_validate_optimized_steering(struct mlx4_dev *dev)
2263	{
2264	int i;
2265	struct mlx4_port_cap port_cap;
2266
2267	if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2268	return -EINVAL;
2269
2270	for (i = 1; i <= dev->caps.num_ports; i++) {
2271	if (mlx4_dev_port(dev, port: i, port_cap: &port_cap)) {
2272	mlx4_err(dev,
2273	"QUERY_DEV_CAP command failed, can't verify DMFS high rate steering.\n");
2274	} else if ((dev->caps.dmfs_high_steer_mode !=
2275	MLX4_STEERING_DMFS_A0_DEFAULT) &&
2276	(port_cap.dmfs_optimized_state ==
2277	!!(dev->caps.dmfs_high_steer_mode ==
2278	MLX4_STEERING_DMFS_A0_DISABLE))) {
2279	mlx4_err(dev,
2280	"DMFS high rate steer mode differ, driver requested %s but %s in FW.\n",
2281	dmfs_high_rate_steering_mode_str(
2282	dev->caps.dmfs_high_steer_mode),
2283	(port_cap.dmfs_optimized_state ?
2284	"enabled" : "disabled"));
2285	}
2286	}
2287
2288	return 0;
2289	}
2290
2291	static int mlx4_init_fw(struct mlx4_dev *dev)
2292	{
2293	struct mlx4_mod_stat_cfg mlx4_cfg;
2294	int err = 0;
2295
2296	if (!mlx4_is_slave(dev)) {
2297	err = mlx4_QUERY_FW(dev);
2298	if (err) {
2299	if (err == -EACCES)
2300	mlx4_info(dev, "non-primary physical function, skipping\n");
2301	else
2302	mlx4_err(dev, "QUERY_FW command failed, aborting\n");
2303	return err;
2304	}
2305
2306	err = mlx4_load_fw(dev);
2307	if (err) {
2308	mlx4_err(dev, "Failed to start FW, aborting\n");
2309	return err;
2310	}
2311
2312	mlx4_cfg.log_pg_sz_m = 1;
2313	mlx4_cfg.log_pg_sz = 0;
2314	err = mlx4_MOD_STAT_CFG(dev, cfg: &mlx4_cfg);
2315	if (err)
2316	mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
2317	}
2318
2319	return err;
2320	}
2321
2322	static int mlx4_init_hca(struct mlx4_dev *dev)
2323	{
2324	struct mlx4_priv *priv = mlx4_priv(dev);
2325	struct mlx4_init_hca_param *init_hca = NULL;
2326	struct mlx4_dev_cap *dev_cap = NULL;
2327	struct mlx4_adapter adapter;
2328	struct mlx4_profile profile;
2329	u64 icm_size;
2330	struct mlx4_config_dev_params params;
2331	int err;
2332
2333	if (!mlx4_is_slave(dev)) {
2334	dev_cap = kzalloc(size: sizeof(*dev_cap), GFP_KERNEL);
2335	init_hca = kzalloc(size: sizeof(*init_hca), GFP_KERNEL);
2336
2337	if (!dev_cap || !init_hca) {
2338	err = -ENOMEM;
2339	goto out_free;
2340	}
2341
2342	err = mlx4_dev_cap(dev, dev_cap);
2343	if (err) {
2344	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
2345	goto out_free;
2346	}
2347
2348	choose_steering_mode(dev, dev_cap);
2349	choose_tunnel_offload_mode(dev, dev_cap);
2350
2351	if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC &&
2352	mlx4_is_master(dev))
2353	dev->caps.function_caps |= MLX4_FUNC_CAP_DMFS_A0_STATIC;
2354
2355	err = mlx4_get_phys_port_id(dev);
2356	if (err)
2357	mlx4_err(dev, "Fail to get physical port id\n");
2358
2359	if (mlx4_is_master(dev))
2360	mlx4_parav_master_pf_caps(dev);
2361
2362	if (mlx4_low_memory_profile()) {
2363	mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");
2364	profile = low_mem_profile;
2365	} else {
2366	profile = default_profile;
2367	}
2368	if (dev->caps.steering_mode ==
2369	MLX4_STEERING_MODE_DEVICE_MANAGED)
2370	profile.num_mcg = MLX4_FS_NUM_MCG;
2371
2372	icm_size = mlx4_make_profile(dev, request: &profile, dev_cap,
2373	init_hca);
2374	if ((long long) icm_size < 0) {
2375	err = icm_size;
2376	goto out_free;
2377	}
2378
2379	if (enable_4k_uar || !dev->persist->num_vfs) {
2380	init_hca->log_uar_sz = ilog2(dev->caps.num_uars) +
2381	PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT;
2382	init_hca->uar_page_sz = DEFAULT_UAR_PAGE_SHIFT - 12;
2383	} else {
2384	init_hca->log_uar_sz = ilog2(dev->caps.num_uars);
2385	init_hca->uar_page_sz = PAGE_SHIFT - 12;
2386	}
2387
2388	init_hca->mw_enabled = 0;
2389	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
2390	dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN)
2391	init_hca->mw_enabled = INIT_HCA_TPT_MW_ENABLE;
2392
2393	err = mlx4_init_icm(dev, dev_cap, init_hca, icm_size);
2394	if (err)
2395	goto out_free;
2396
2397	err = mlx4_INIT_HCA(dev, param: init_hca);
2398	if (err) {
2399	mlx4_err(dev, "INIT_HCA command failed, aborting\n");
2400	goto err_free_icm;
2401	}
2402
2403	if (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
2404	err = mlx4_query_func(dev, dev_cap);
2405	if (err < 0) {
2406	mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n");
2407	goto err_close;
2408	} else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) {
2409	dev->caps.num_eqs = dev_cap->max_eqs;
2410	dev->caps.reserved_eqs = dev_cap->reserved_eqs;
2411	dev->caps.reserved_uars = dev_cap->reserved_uars;
2412	}
2413	}
2414
2415	/*
2416	* If TS is supported by FW
2417	* read HCA frequency by QUERY_HCA command
2418	*/
2419	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) {
2420	err = mlx4_QUERY_HCA(dev, param: init_hca);
2421	if (err) {
2422	mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n");
2423	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
2424	} else {
2425	dev->caps.hca_core_clock =
2426	init_hca->hca_core_clock;
2427	}
2428
2429	/* In case we got HCA frequency 0 - disable timestamping
2430	* to avoid dividing by zero
2431	*/
2432	if (!dev->caps.hca_core_clock) {
2433	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
2434	mlx4_err(dev,
2435	"HCA frequency is 0 - timestamping is not supported\n");
2436	} else if (map_internal_clock(dev)) {
2437	/*
2438	* Map internal clock,
2439	* in case of failure disable timestamping
2440	*/
2441	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
2442	mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n");
2443	}
2444	}
2445
2446	if (dev->caps.dmfs_high_steer_mode !=
2447	MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) {
2448	if (mlx4_validate_optimized_steering(dev))
2449	mlx4_warn(dev, "Optimized steering validation failed\n");
2450
2451	if (dev->caps.dmfs_high_steer_mode ==
2452	MLX4_STEERING_DMFS_A0_DISABLE) {
2453	dev->caps.dmfs_high_rate_qpn_base =
2454	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
2455	dev->caps.dmfs_high_rate_qpn_range =
2456	MLX4_A0_STEERING_TABLE_SIZE;
2457	}
2458
2459	mlx4_info(dev, "DMFS high rate steer mode is: %s\n",
2460	dmfs_high_rate_steering_mode_str(
2461	dev->caps.dmfs_high_steer_mode));
2462	}
2463	} else {
2464	err = mlx4_init_slave(dev);
2465	if (err) {
2466	if (err != -EPROBE_DEFER)
2467	mlx4_err(dev, "Failed to initialize slave\n");
2468	return err;
2469	}
2470
2471	err = mlx4_slave_cap(dev);
2472	if (err) {
2473	mlx4_err(dev, "Failed to obtain slave caps\n");
2474	goto err_close;
2475	}
2476	}
2477
2478	if (map_bf_area(dev))
2479	mlx4_dbg(dev, "Failed to map blue flame area\n");
2480
2481	/*Only the master set the ports, all the rest got it from it.*/
2482	if (!mlx4_is_slave(dev))
2483	mlx4_set_port_mask(dev);
2484
2485	err = mlx4_QUERY_ADAPTER(dev, adapter: &adapter);
2486	if (err) {
2487	mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n");
2488	goto unmap_bf;
2489	}
2490
2491	/* Query CONFIG_DEV parameters */
2492	err = mlx4_config_dev_retrieval(dev, params: &params);
2493	if (err && err != -EOPNOTSUPP) {
2494	mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n");
2495	} else if (!err) {
2496	dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1;
2497	dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2;
2498	}
2499	priv->eq_table.inta_pin = adapter.inta_pin;
2500	memcpy(dev->board_id, adapter.board_id, sizeof(dev->board_id));
2501
2502	err = 0;
2503	goto out_free;
2504
2505	unmap_bf:
2506	unmap_internal_clock(dev);
2507	unmap_bf_area(dev);
2508
2509	if (mlx4_is_slave(dev))
2510	mlx4_slave_destroy_special_qp_cap(dev);
2511
2512	err_close:
2513	if (mlx4_is_slave(dev))
2514	mlx4_slave_exit(dev);
2515	else
2516	mlx4_CLOSE_HCA(dev, panic: 0);
2517
2518	err_free_icm:
2519	if (!mlx4_is_slave(dev))
2520	mlx4_free_icms(dev);
2521
2522	out_free:
2523	kfree(objp: dev_cap);
2524	kfree(objp: init_hca);
2525
2526	return err;
2527	}
2528
2529	static int mlx4_init_counters_table(struct mlx4_dev *dev)
2530	{
2531	struct mlx4_priv *priv = mlx4_priv(dev);
2532	int nent_pow2;
2533
2534	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2535	return -ENOENT;
2536
2537	if (!dev->caps.max_counters)
2538	return -ENOSPC;
2539
2540	nent_pow2 = roundup_pow_of_two(dev->caps.max_counters);
2541	/* reserve last counter index for sink counter */
2542	return mlx4_bitmap_init(bitmap: &priv->counters_bitmap, num: nent_pow2,
2543	mask: nent_pow2 - 1, reserved_bot: 0,
2544	resetrved_top: nent_pow2 - dev->caps.max_counters + 1);
2545	}
2546
2547	static void mlx4_cleanup_counters_table(struct mlx4_dev *dev)
2548	{
2549	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2550	return;
2551
2552	if (!dev->caps.max_counters)
2553	return;
2554
2555	mlx4_bitmap_cleanup(bitmap: &mlx4_priv(dev)->counters_bitmap);
2556	}
2557
2558	static void mlx4_cleanup_default_counters(struct mlx4_dev *dev)
2559	{
2560	struct mlx4_priv *priv = mlx4_priv(dev);
2561	int port;
2562
2563	for (port = 0; port < dev->caps.num_ports; port++)
2564	if (priv->def_counter[port] != -1)
2565	mlx4_counter_free(dev, idx: priv->def_counter[port]);
2566	}
2567
2568	static int mlx4_allocate_default_counters(struct mlx4_dev *dev)
2569	{
2570	struct mlx4_priv *priv = mlx4_priv(dev);
2571	int port, err = 0;
2572	u32 idx;
2573
2574	for (port = 0; port < dev->caps.num_ports; port++)
2575	priv->def_counter[port] = -1;
2576
2577	for (port = 0; port < dev->caps.num_ports; port++) {
2578	err = mlx4_counter_alloc(dev, idx: &idx, usage: MLX4_RES_USAGE_DRIVER);
2579
2580	if (!err || err == -ENOSPC) {
2581	priv->def_counter[port] = idx;
2582	err = 0;
2583	} else if (err == -ENOENT) {
2584	err = 0;
2585	continue;
2586	} else if (mlx4_is_slave(dev) && err == -EINVAL) {
2587	priv->def_counter[port] = MLX4_SINK_COUNTER_INDEX(dev);
2588	mlx4_warn(dev, "can't allocate counter from old PF driver, using index %d\n",
2589	MLX4_SINK_COUNTER_INDEX(dev));
2590	err = 0;
2591	} else {
2592	mlx4_err(dev, "%s: failed to allocate default counter port %d err %d\n",
2593	__func__, port + 1, err);
2594	mlx4_cleanup_default_counters(dev);
2595	return err;
2596	}
2597
2598	mlx4_dbg(dev, "%s: default counter index %d for port %d\n",
2599	__func__, priv->def_counter[port], port + 1);
2600	}
2601
2602	return err;
2603	}
2604
2605	int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
2606	{
2607	struct mlx4_priv *priv = mlx4_priv(dev);
2608
2609	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2610	return -ENOENT;
2611
2612	*idx = mlx4_bitmap_alloc(bitmap: &priv->counters_bitmap);
2613	if (*idx == -1) {
2614	*idx = MLX4_SINK_COUNTER_INDEX(dev);
2615	return -ENOSPC;
2616	}
2617
2618	return 0;
2619	}
2620
2621	int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage)
2622	{
2623	u32 in_modifier = RES_COUNTER | (((u32)usage & 3) << 30);
2624	u64 out_param;
2625	int err;
2626
2627	if (mlx4_is_mfunc(dev)) {
2628	err = mlx4_cmd_imm(dev, in_param: 0, out_param: &out_param, in_modifier,
2629	op_modifier: RES_OP_RESERVE, op: MLX4_CMD_ALLOC_RES,
2630	timeout: MLX4_CMD_TIME_CLASS_A, native: MLX4_CMD_WRAPPED);
2631	if (!err)
2632	*idx = get_param_l(arg: &out_param);
2633	if (WARN_ON(err == -ENOSPC))
2634	err = -EINVAL;
2635	return err;
2636	}
2637	return __mlx4_counter_alloc(dev, idx);
2638	}
2639	EXPORT_SYMBOL_GPL(mlx4_counter_alloc);
2640
2641	static int __mlx4_clear_if_stat(struct mlx4_dev *dev,
2642	u8 counter_index)
2643	{
2644	struct mlx4_cmd_mailbox *if_stat_mailbox;
2645	int err;
2646	u32 if_stat_in_mod = (counter_index & 0xff) | MLX4_QUERY_IF_STAT_RESET;
2647
2648	if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev);
2649	if (IS_ERR(ptr: if_stat_mailbox))
2650	return PTR_ERR(ptr: if_stat_mailbox);
2651
2652	err = mlx4_cmd_box(dev, in_param: 0, out_param: if_stat_mailbox->dma, in_modifier: if_stat_in_mod, op_modifier: 0,
2653	op: MLX4_CMD_QUERY_IF_STAT, timeout: MLX4_CMD_TIME_CLASS_C,
2654	native: MLX4_CMD_NATIVE);
2655
2656	mlx4_free_cmd_mailbox(dev, mailbox: if_stat_mailbox);
2657	return err;
2658	}
2659
2660	void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
2661	{
2662	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2663	return;
2664
2665	if (idx == MLX4_SINK_COUNTER_INDEX(dev))
2666	return;
2667
2668	__mlx4_clear_if_stat(dev, counter_index: idx);
2669
2670	mlx4_bitmap_free(bitmap: &mlx4_priv(dev)->counters_bitmap, obj: idx, use_rr: MLX4_USE_RR);
2671	return;
2672	}
2673
2674	void mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
2675	{
2676	u64 in_param = 0;
2677
2678	if (mlx4_is_mfunc(dev)) {
2679	set_param_l(arg: &in_param, val: idx);
2680	mlx4_cmd(dev, in_param, in_modifier: RES_COUNTER, op_modifier: RES_OP_RESERVE,
2681	op: MLX4_CMD_FREE_RES, timeout: MLX4_CMD_TIME_CLASS_A,
2682	native: MLX4_CMD_WRAPPED);
2683	return;
2684	}
2685	__mlx4_counter_free(dev, idx);
2686	}
2687	EXPORT_SYMBOL_GPL(mlx4_counter_free);
2688
2689	int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port)
2690	{
2691	struct mlx4_priv *priv = mlx4_priv(dev);
2692
2693	return priv->def_counter[port - 1];
2694	}
2695	EXPORT_SYMBOL_GPL(mlx4_get_default_counter_index);
2696
2697	void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, int port)
2698	{
2699	struct mlx4_priv *priv = mlx4_priv(dev);
2700
2701	priv->mfunc.master.vf_admin[entry].vport[port].guid = guid;
2702	}
2703	EXPORT_SYMBOL_GPL(mlx4_set_admin_guid);
2704
2705	__be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port)
2706	{
2707	struct mlx4_priv *priv = mlx4_priv(dev);
2708
2709	return priv->mfunc.master.vf_admin[entry].vport[port].guid;
2710	}
2711	EXPORT_SYMBOL_GPL(mlx4_get_admin_guid);
2712
2713	void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port)
2714	{
2715	struct mlx4_priv *priv = mlx4_priv(dev);
2716	__be64 guid;
2717
2718	/* hw GUID */
2719	if (entry == 0)
2720	return;
2721
2722	get_random_bytes(buf: (char *)&guid, len: sizeof(guid));
2723	guid &= ~(cpu_to_be64(1ULL << 56));
2724	guid |= cpu_to_be64(1ULL << 57);
2725	priv->mfunc.master.vf_admin[entry].vport[port].guid = guid;
2726	}
2727
2728	static int mlx4_setup_hca(struct mlx4_dev *dev)
2729	{
2730	struct mlx4_priv *priv = mlx4_priv(dev);
2731	int err;
2732	int port;
2733	__be32 ib_port_default_caps;
2734
2735	err = mlx4_init_uar_table(dev);
2736	if (err) {
2737	mlx4_err(dev, "Failed to initialize user access region table, aborting\n");
2738	return err;
2739	}
2740
2741	err = mlx4_uar_alloc(dev, uar: &priv->driver_uar);
2742	if (err) {
2743	mlx4_err(dev, "Failed to allocate driver access region, aborting\n");
2744	goto err_uar_table_free;
2745	}
2746
2747	priv->kar = ioremap(offset: (phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
2748	if (!priv->kar) {
2749	mlx4_err(dev, "Couldn't map kernel access region, aborting\n");
2750	err = -ENOMEM;
2751	goto err_uar_free;
2752	}
2753
2754	err = mlx4_init_pd_table(dev);
2755	if (err) {
2756	mlx4_err(dev, "Failed to initialize protection domain table, aborting\n");
2757	goto err_kar_unmap;
2758	}
2759
2760	err = mlx4_init_xrcd_table(dev);
2761	if (err) {
2762	mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n");
2763	goto err_pd_table_free;
2764	}
2765
2766	err = mlx4_init_mr_table(dev);
2767	if (err) {
2768	mlx4_err(dev, "Failed to initialize memory region table, aborting\n");
2769	goto err_xrcd_table_free;
2770	}
2771
2772	if (!mlx4_is_slave(dev)) {
2773	err = mlx4_init_mcg_table(dev);
2774	if (err) {
2775	mlx4_err(dev, "Failed to initialize multicast group table, aborting\n");
2776	goto err_mr_table_free;
2777	}
2778	err = mlx4_config_mad_demux(dev);
2779	if (err) {
2780	mlx4_err(dev, "Failed in config_mad_demux, aborting\n");
2781	goto err_mcg_table_free;
2782	}
2783	}
2784
2785	err = mlx4_init_eq_table(dev);
2786	if (err) {
2787	mlx4_err(dev, "Failed to initialize event queue table, aborting\n");
2788	goto err_mcg_table_free;
2789	}
2790
2791	err = mlx4_cmd_use_events(dev);
2792	if (err) {
2793	mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n");
2794	goto err_eq_table_free;
2795	}
2796
2797	err = mlx4_NOP(dev);
2798	if (err) {
2799	if (dev->flags & MLX4_FLAG_MSI_X) {
2800	mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n",
2801	priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
2802	mlx4_warn(dev, "Trying again without MSI-X\n");
2803	} else {
2804	mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n",
2805	priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
2806	mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
2807	}
2808
2809	goto err_cmd_poll;
2810	}
2811
2812	mlx4_dbg(dev, "NOP command IRQ test passed\n");
2813
2814	err = mlx4_init_cq_table(dev);
2815	if (err) {
2816	mlx4_err(dev, "Failed to initialize completion queue table, aborting\n");
2817	goto err_cmd_poll;
2818	}
2819
2820	err = mlx4_init_srq_table(dev);
2821	if (err) {
2822	mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n");
2823	goto err_cq_table_free;
2824	}
2825
2826	err = mlx4_init_qp_table(dev);
2827	if (err) {
2828	mlx4_err(dev, "Failed to initialize queue pair table, aborting\n");
2829	goto err_srq_table_free;
2830	}
2831
2832	if (!mlx4_is_slave(dev)) {
2833	err = mlx4_init_counters_table(dev);
2834	if (err && err != -ENOENT) {
2835	mlx4_err(dev, "Failed to initialize counters table, aborting\n");
2836	goto err_qp_table_free;
2837	}
2838	}
2839
2840	err = mlx4_allocate_default_counters(dev);
2841	if (err) {
2842	mlx4_err(dev, "Failed to allocate default counters, aborting\n");
2843	goto err_counters_table_free;
2844	}
2845
2846	if (!mlx4_is_slave(dev)) {
2847	for (port = 1; port <= dev->caps.num_ports; port++) {
2848	ib_port_default_caps = 0;
2849	err = mlx4_get_port_ib_caps(dev, port,
2850	caps: &ib_port_default_caps);
2851	if (err)
2852	mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n",
2853	port, err);
2854	dev->caps.ib_port_def_cap[port] = ib_port_default_caps;
2855
2856	/* initialize per-slave default ib port capabilities */
2857	if (mlx4_is_master(dev)) {
2858	int i;
2859	for (i = 0; i < dev->num_slaves; i++) {
2860	if (i == mlx4_master_func_num(dev))
2861	continue;
2862	priv->mfunc.master.slave_state[i].ib_cap_mask[port] =
2863	ib_port_default_caps;
2864	}
2865	}
2866
2867	if (mlx4_is_mfunc(dev))
2868	dev->caps.port_ib_mtu[port] = IB_MTU_2048;
2869	else
2870	dev->caps.port_ib_mtu[port] = IB_MTU_4096;
2871
2872	err = mlx4_SET_PORT(dev, port, pkey_tbl_sz: mlx4_is_master(dev) ?
2873	dev->caps.pkey_table_len[port] : -1);
2874	if (err) {
2875	mlx4_err(dev, "Failed to set port %d, aborting\n",
2876	port);
2877	goto err_default_countes_free;
2878	}
2879	}
2880	}
2881
2882	return 0;
2883
2884	err_default_countes_free:
2885	mlx4_cleanup_default_counters(dev);
2886
2887	err_counters_table_free:
2888	if (!mlx4_is_slave(dev))
2889	mlx4_cleanup_counters_table(dev);
2890
2891	err_qp_table_free:
2892	mlx4_cleanup_qp_table(dev);
2893
2894	err_srq_table_free:
2895	mlx4_cleanup_srq_table(dev);
2896
2897	err_cq_table_free:
2898	mlx4_cleanup_cq_table(dev);
2899
2900	err_cmd_poll:
2901	mlx4_cmd_use_polling(dev);
2902
2903	err_eq_table_free:
2904	mlx4_cleanup_eq_table(dev);
2905
2906	err_mcg_table_free:
2907	if (!mlx4_is_slave(dev))
2908	mlx4_cleanup_mcg_table(dev);
2909
2910	err_mr_table_free:
2911	mlx4_cleanup_mr_table(dev);
2912
2913	err_xrcd_table_free:
2914	mlx4_cleanup_xrcd_table(dev);
2915
2916	err_pd_table_free:
2917	mlx4_cleanup_pd_table(dev);
2918
2919	err_kar_unmap:
2920	iounmap(addr: priv->kar);
2921
2922	err_uar_free:
2923	mlx4_uar_free(dev, uar: &priv->driver_uar);
2924
2925	err_uar_table_free:
2926	mlx4_cleanup_uar_table(dev);
2927	return err;
2928	}
2929
2930	static int mlx4_init_affinity_hint(struct mlx4_dev *dev, int port, int eqn)
2931	{
2932	int requested_cpu = 0;
2933	struct mlx4_priv *priv = mlx4_priv(dev);
2934	struct mlx4_eq *eq;
2935	int off = 0;
2936	int i;
2937
2938	if (eqn > dev->caps.num_comp_vectors)
2939	return -EINVAL;
2940
2941	for (i = 1; i < port; i++)
2942	off += mlx4_get_eqs_per_port(dev, port: i);
2943
2944	requested_cpu = eqn - off - !!(eqn > MLX4_EQ_ASYNC);
2945
2946	/* Meaning EQs are shared, and this call comes from the second port */
2947	if (requested_cpu < 0)
2948	return 0;
2949
2950	eq = &priv->eq_table.eq[eqn];
2951
2952	if (!zalloc_cpumask_var(mask: &eq->affinity_mask, GFP_KERNEL))
2953	return -ENOMEM;
2954
2955	cpumask_set_cpu(cpu: requested_cpu, dstp: eq->affinity_mask);
2956
2957	return 0;
2958	}
2959
2960	static void mlx4_enable_msi_x(struct mlx4_dev *dev)
2961	{
2962	struct mlx4_priv *priv = mlx4_priv(dev);
2963	struct msix_entry *entries;
2964	int i;
2965	int port = 0;
2966
2967	if (msi_x) {
2968	int nreq = min3(dev->caps.num_ports *
2969	(int)num_online_cpus() + 1,
2970	dev->caps.num_eqs - dev->caps.reserved_eqs,
2971	MAX_MSIX);
2972
2973	if (msi_x > 1)
2974	nreq = min_t(int, nreq, msi_x);
2975
2976	entries = kcalloc(n: nreq, size: sizeof(*entries), GFP_KERNEL);
2977	if (!entries)
2978	goto no_msi;
2979
2980	for (i = 0; i < nreq; ++i)
2981	entries[i].entry = i;
2982
2983	nreq = pci_enable_msix_range(dev: dev->persist->pdev, entries, minvec: 2,
2984	maxvec: nreq);
2985
2986	if (nreq < 0 || nreq < MLX4_EQ_ASYNC) {
2987	kfree(objp: entries);
2988	goto no_msi;
2989	}
2990	/* 1 is reserved for events (asyncrounous EQ) */
2991	dev->caps.num_comp_vectors = nreq - 1;
2992
2993	priv->eq_table.eq[MLX4_EQ_ASYNC].irq = entries[0].vector;
2994	bitmap_zero(dst: priv->eq_table.eq[MLX4_EQ_ASYNC].actv_ports.ports,
2995	nbits: dev->caps.num_ports);
2996
2997	for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) {
2998	if (i == MLX4_EQ_ASYNC)
2999	continue;
3000
3001	priv->eq_table.eq[i].irq =
3002	entries[i + 1 - !!(i > MLX4_EQ_ASYNC)].vector;
3003
3004	if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) {
3005	bitmap_fill(dst: priv->eq_table.eq[i].actv_ports.ports,
3006	nbits: dev->caps.num_ports);
3007	/* We don't set affinity hint when there
3008	* aren't enough EQs
3009	*/
3010	} else {
3011	set_bit(nr: port,
3012	addr: priv->eq_table.eq[i].actv_ports.ports);
3013	if (mlx4_init_affinity_hint(dev, port: port + 1, eqn: i))
3014	mlx4_warn(dev, "Couldn't init hint cpumask for EQ %d\n",
3015	i);
3016	}
3017	/* We divide the Eqs evenly between the two ports.
3018	* (dev->caps.num_comp_vectors / dev->caps.num_ports)
3019	* refers to the number of Eqs per port
3020	* (i.e eqs_per_port). Theoretically, we would like to
3021	* write something like (i + 1) % eqs_per_port == 0.
3022	* However, since there's an asynchronous Eq, we have
3023	* to skip over it by comparing this condition to
3024	* !!((i + 1) > MLX4_EQ_ASYNC).
3025	*/
3026	if ((dev->caps.num_comp_vectors > dev->caps.num_ports) &&
3027	((i + 1) %
3028	(dev->caps.num_comp_vectors / dev->caps.num_ports)) ==
3029	!!((i + 1) > MLX4_EQ_ASYNC))
3030	/* If dev->caps.num_comp_vectors < dev->caps.num_ports,
3031	* everything is shared anyway.
3032	*/
3033	port++;
3034	}
3035
3036	dev->flags |= MLX4_FLAG_MSI_X;
3037
3038	kfree(objp: entries);
3039	return;
3040	}
3041
3042	no_msi:
3043	dev->caps.num_comp_vectors = 1;
3044
3045	BUG_ON(MLX4_EQ_ASYNC >= 2);
3046	for (i = 0; i < 2; ++i) {
3047	priv->eq_table.eq[i].irq = dev->persist->pdev->irq;
3048	if (i != MLX4_EQ_ASYNC) {
3049	bitmap_fill(dst: priv->eq_table.eq[i].actv_ports.ports,
3050	nbits: dev->caps.num_ports);
3051	}
3052	}
3053	}
3054
3055	static int mlx4_devlink_port_type_set(struct devlink_port *devlink_port,
3056	enum devlink_port_type port_type)
3057	{
3058	struct mlx4_port_info *info = container_of(devlink_port,
3059	struct mlx4_port_info,
3060	devlink_port);
3061	enum mlx4_port_type mlx4_port_type;
3062
3063	switch (port_type) {
3064	case DEVLINK_PORT_TYPE_AUTO:
3065	mlx4_port_type = MLX4_PORT_TYPE_AUTO;
3066	break;
3067	case DEVLINK_PORT_TYPE_ETH:
3068	mlx4_port_type = MLX4_PORT_TYPE_ETH;
3069	break;
3070	case DEVLINK_PORT_TYPE_IB:
3071	mlx4_port_type = MLX4_PORT_TYPE_IB;
3072	break;
3073	default:
3074	return -EOPNOTSUPP;
3075	}
3076
3077	return __set_port_type(info, port_type: mlx4_port_type);
3078	}
3079
3080	static const struct devlink_port_ops mlx4_devlink_port_ops = {
3081	.port_type_set = mlx4_devlink_port_type_set,
3082	};
3083
3084	static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
3085	{
3086	struct devlink *devlink = priv_to_devlink(priv: mlx4_priv(dev));
3087	struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
3088	int err;
3089
3090	err = devl_port_register_with_ops(devlink, devlink_port: &info->devlink_port, port_index: port,
3091	ops: &mlx4_devlink_port_ops);
3092	if (err)
3093	return err;
3094
3095	/* Ethernet and IB drivers will normally set the port type,
3096	* but if they are not built set the type now to prevent
3097	* devlink_port_type_warn() from firing.
3098	*/
3099	if (!IS_ENABLED(CONFIG_MLX4_EN) &&
3100	dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
3101	devlink_port_type_eth_set(devlink_port: &info->devlink_port);
3102	else if (!IS_ENABLED(CONFIG_MLX4_INFINIBAND) &&
3103	dev->caps.port_type[port] == MLX4_PORT_TYPE_IB)
3104	devlink_port_type_ib_set(devlink_port: &info->devlink_port, NULL);
3105
3106	info->dev = dev;
3107	info->port = port;
3108	if (!mlx4_is_slave(dev)) {
3109	mlx4_init_mac_table(dev, table: &info->mac_table);
3110	mlx4_init_vlan_table(dev, table: &info->vlan_table);
3111	mlx4_init_roce_gid_table(dev, table: &info->gid_table);
3112	info->base_qpn = mlx4_get_base_qpn(dev, port);
3113	}
3114
3115	sprintf(buf: info->dev_name, fmt: "mlx4_port%d", port);
3116	info->port_attr.attr.name = info->dev_name;
3117	if (mlx4_is_mfunc(dev)) {
3118	info->port_attr.attr.mode = 0444;
3119	} else {
3120	info->port_attr.attr.mode = 0644;
3121	info->port_attr.store = set_port_type;
3122	}
3123	info->port_attr.show = show_port_type;
3124	sysfs_attr_init(&info->port_attr.attr);
3125
3126	err = device_create_file(device: &dev->persist->pdev->dev, entry: &info->port_attr);
3127	if (err) {
3128	mlx4_err(dev, "Failed to create file for port %d\n", port);
3129	devlink_port_type_clear(devlink_port: &info->devlink_port);
3130	devl_port_unregister(devlink_port: &info->devlink_port);
3131	info->port = -1;
3132	return err;
3133	}
3134
3135	sprintf(buf: info->dev_mtu_name, fmt: "mlx4_port%d_mtu", port);
3136	info->port_mtu_attr.attr.name = info->dev_mtu_name;
3137	if (mlx4_is_mfunc(dev)) {
3138	info->port_mtu_attr.attr.mode = 0444;
3139	} else {
3140	info->port_mtu_attr.attr.mode = 0644;
3141	info->port_mtu_attr.store = set_port_ib_mtu;
3142	}
3143	info->port_mtu_attr.show = show_port_ib_mtu;
3144	sysfs_attr_init(&info->port_mtu_attr.attr);
3145
3146	err = device_create_file(device: &dev->persist->pdev->dev,
3147	entry: &info->port_mtu_attr);
3148	if (err) {
3149	mlx4_err(dev, "Failed to create mtu file for port %d\n", port);
3150	device_remove_file(dev: &info->dev->persist->pdev->dev,
3151	attr: &info->port_attr);
3152	devlink_port_type_clear(devlink_port: &info->devlink_port);
3153	devl_port_unregister(devlink_port: &info->devlink_port);
3154	info->port = -1;
3155	return err;
3156	}
3157
3158	return 0;
3159	}
3160
3161	static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
3162	{
3163	if (info->port < 0)
3164	return;
3165
3166	device_remove_file(dev: &info->dev->persist->pdev->dev, attr: &info->port_attr);
3167	device_remove_file(dev: &info->dev->persist->pdev->dev,
3168	attr: &info->port_mtu_attr);
3169	devlink_port_type_clear(devlink_port: &info->devlink_port);
3170	devl_port_unregister(devlink_port: &info->devlink_port);
3171
3172	#ifdef CONFIG_RFS_ACCEL
3173	free_irq_cpu_rmap(rmap: info->rmap);
3174	info->rmap = NULL;
3175	#endif
3176	}
3177
3178	static int mlx4_init_steering(struct mlx4_dev *dev)
3179	{
3180	struct mlx4_priv *priv = mlx4_priv(dev);
3181	int num_entries = dev->caps.num_ports;
3182	int i, j;
3183
3184	priv->steer = kcalloc(n: num_entries, size: sizeof(struct mlx4_steer),
3185	GFP_KERNEL);
3186	if (!priv->steer)
3187	return -ENOMEM;
3188
3189	for (i = 0; i < num_entries; i++)
3190	for (j = 0; j < MLX4_NUM_STEERS; j++) {
3191	INIT_LIST_HEAD(list: &priv->steer[i].promisc_qps[j]);
3192	INIT_LIST_HEAD(list: &priv->steer[i].steer_entries[j]);
3193	}
3194	return 0;
3195	}
3196
3197	static void mlx4_clear_steering(struct mlx4_dev *dev)
3198	{
3199	struct mlx4_priv *priv = mlx4_priv(dev);
3200	struct mlx4_steer_index *entry, *tmp_entry;
3201	struct mlx4_promisc_qp *pqp, *tmp_pqp;
3202	int num_entries = dev->caps.num_ports;
3203	int i, j;
3204
3205	for (i = 0; i < num_entries; i++) {
3206	for (j = 0; j < MLX4_NUM_STEERS; j++) {
3207	list_for_each_entry_safe(pqp, tmp_pqp,
3208	&priv->steer[i].promisc_qps[j],
3209	list) {
3210	list_del(entry: &pqp->list);
3211	kfree(objp: pqp);
3212	}
3213	list_for_each_entry_safe(entry, tmp_entry,
3214	&priv->steer[i].steer_entries[j],
3215	list) {
3216	list_del(entry: &entry->list);
3217	list_for_each_entry_safe(pqp, tmp_pqp,
3218	&entry->duplicates,
3219	list) {
3220	list_del(entry: &pqp->list);
3221	kfree(objp: pqp);
3222	}
3223	kfree(objp: entry);
3224	}
3225	}
3226	}
3227	kfree(objp: priv->steer);
3228	}
3229
3230	static int extended_func_num(struct pci_dev *pdev)
3231	{
3232	return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn);
3233	}
3234
3235	#define MLX4_OWNER_BASE 0x8069c
3236	#define MLX4_OWNER_SIZE 4
3237
3238	static int mlx4_get_ownership(struct mlx4_dev *dev)
3239	{
3240	void __iomem *owner;
3241	u32 ret;
3242
3243	if (pci_channel_offline(pdev: dev->persist->pdev))
3244	return -EIO;
3245
3246	owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +
3247	MLX4_OWNER_BASE,
3248	MLX4_OWNER_SIZE);
3249	if (!owner) {
3250	mlx4_err(dev, "Failed to obtain ownership bit\n");
3251	return -ENOMEM;
3252	}
3253
3254	ret = readl(addr: owner);
3255	iounmap(addr: owner);
3256	return (int) !!ret;
3257	}
3258
3259	static void mlx4_free_ownership(struct mlx4_dev *dev)
3260	{
3261	void __iomem *owner;
3262
3263	if (pci_channel_offline(pdev: dev->persist->pdev))
3264	return;
3265
3266	owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +
3267	MLX4_OWNER_BASE,
3268	MLX4_OWNER_SIZE);
3269	if (!owner) {
3270	mlx4_err(dev, "Failed to obtain ownership bit\n");
3271	return;
3272	}
3273	writel(val: 0, addr: owner);
3274	msleep(msecs: 1000);
3275	iounmap(addr: owner);
3276	}
3277
3278	#define SRIOV_VALID_STATE(flags) (!!((flags) & MLX4_FLAG_SRIOV) ==\
3279	!!((flags) & MLX4_FLAG_MASTER))
3280
3281	static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev,
3282	u8 total_vfs, int existing_vfs, int reset_flow)
3283	{
3284	u64 dev_flags = dev->flags;
3285	int err = 0;
3286	int fw_enabled_sriov_vfs = min(pci_sriov_get_totalvfs(pdev),
3287	MLX4_MAX_NUM_VF);
3288
3289	if (reset_flow) {
3290	dev->dev_vfs = kcalloc(n: total_vfs, size: sizeof(*dev->dev_vfs),
3291	GFP_KERNEL);
3292	if (!dev->dev_vfs)
3293	goto free_mem;
3294	return dev_flags;
3295	}
3296
3297	atomic_inc(v: &pf_loading);
3298	if (dev->flags & MLX4_FLAG_SRIOV) {
3299	if (existing_vfs != total_vfs) {
3300	mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n",
3301	existing_vfs, total_vfs);
3302	total_vfs = existing_vfs;
3303	}
3304	}
3305
3306	dev->dev_vfs = kcalloc(n: total_vfs, size: sizeof(*dev->dev_vfs), GFP_KERNEL);
3307	if (NULL == dev->dev_vfs) {
3308	mlx4_err(dev, "Failed to allocate memory for VFs\n");
3309	goto disable_sriov;
3310	}
3311
3312	if (!(dev->flags & MLX4_FLAG_SRIOV)) {
3313	if (total_vfs > fw_enabled_sriov_vfs) {
3314	mlx4_err(dev, "requested vfs (%d) > available vfs (%d). Continuing without SR_IOV\n",
3315	total_vfs, fw_enabled_sriov_vfs);
3316	err = -ENOMEM;
3317	goto disable_sriov;
3318	}
3319	mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs);
3320	err = pci_enable_sriov(dev: pdev, nr_virtfn: total_vfs);
3321	}
3322	if (err) {
3323	mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n",
3324	err);
3325	goto disable_sriov;
3326	} else {
3327	mlx4_warn(dev, "Running in master mode\n");
3328	dev_flags |= MLX4_FLAG_SRIOV |
3329	MLX4_FLAG_MASTER;
3330	dev_flags &= ~MLX4_FLAG_SLAVE;
3331	dev->persist->num_vfs = total_vfs;
3332	}
3333	return dev_flags;
3334
3335	disable_sriov:
3336	atomic_dec(v: &pf_loading);
3337	free_mem:
3338	dev->persist->num_vfs = 0;
3339	kfree(objp: dev->dev_vfs);
3340	dev->dev_vfs = NULL;
3341	return dev_flags & ~MLX4_FLAG_MASTER;
3342	}
3343
3344	enum {
3345	MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64 = -1,
3346	};
3347
3348	static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
3349	int *nvfs)
3350	{
3351	int requested_vfs = nvfs[0] + nvfs[1] + nvfs[2];
3352	/* Checking for 64 VFs as a limitation of CX2 */
3353	if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_80_VFS) &&
3354	requested_vfs >= 64) {
3355	mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n",
3356	requested_vfs);
3357	return MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64;
3358	}
3359	return 0;
3360	}
3361
3362	static int mlx4_pci_enable_device(struct mlx4_dev *dev)
3363	{
3364	struct pci_dev *pdev = dev->persist->pdev;
3365	int err = 0;
3366
3367	mutex_lock(&dev->persist->pci_status_mutex);
3368	if (dev->persist->pci_status == MLX4_PCI_STATUS_DISABLED) {
3369	err = pci_enable_device(dev: pdev);
3370	if (!err)
3371	dev->persist->pci_status = MLX4_PCI_STATUS_ENABLED;
3372	}
3373	mutex_unlock(lock: &dev->persist->pci_status_mutex);
3374
3375	return err;
3376	}
3377
3378	static void mlx4_pci_disable_device(struct mlx4_dev *dev)
3379	{
3380	struct pci_dev *pdev = dev->persist->pdev;
3381
3382	mutex_lock(&dev->persist->pci_status_mutex);
3383	if (dev->persist->pci_status == MLX4_PCI_STATUS_ENABLED) {
3384	pci_disable_device(dev: pdev);
3385	dev->persist->pci_status = MLX4_PCI_STATUS_DISABLED;
3386	}
3387	mutex_unlock(lock: &dev->persist->pci_status_mutex);
3388	}
3389
3390	static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data,
3391	int total_vfs, int *nvfs, struct mlx4_priv *priv,
3392	int reset_flow)
3393	{
3394	struct devlink *devlink = priv_to_devlink(priv);
3395	struct mlx4_dev *dev;
3396	unsigned sum = 0;
3397	int err;
3398	int port;
3399	int i;
3400	struct mlx4_dev_cap *dev_cap = NULL;
3401	int existing_vfs = 0;
3402
3403	devl_assert_locked(devlink);
3404	dev = &priv->dev;
3405
3406	err = mlx4_adev_init(dev);
3407	if (err)
3408	return err;
3409
3410	ATOMIC_INIT_NOTIFIER_HEAD(&priv->event_nh);
3411
3412	mutex_init(&priv->port_mutex);
3413	mutex_init(&priv->bond_mutex);
3414
3415	INIT_LIST_HEAD(list: &priv->pgdir_list);
3416	mutex_init(&priv->pgdir_mutex);
3417	spin_lock_init(&priv->cmd.context_lock);
3418
3419	INIT_LIST_HEAD(list: &priv->bf_list);
3420	mutex_init(&priv->bf_mutex);
3421
3422	dev->rev_id = pdev->revision;
3423	dev->numa_node = dev_to_node(dev: &pdev->dev);
3424
3425	/* Detect if this device is a virtual function */
3426	if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
3427	mlx4_warn(dev, "Detected virtual function - running in slave mode\n");
3428	dev->flags |= MLX4_FLAG_SLAVE;
3429	} else {
3430	/* We reset the device and enable SRIOV only for physical
3431	* devices. Try to claim ownership on the device;
3432	* if already taken, skip -- do not allow multiple PFs */
3433	err = mlx4_get_ownership(dev);
3434	if (err) {
3435	if (err < 0)
3436	goto err_adev;
3437	else {
3438	mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n");
3439	err = -EINVAL;
3440	goto err_adev;
3441	}
3442	}
3443
3444	atomic_set(v: &priv->opreq_count, i: 0);
3445	INIT_WORK(&priv->opreq_task, mlx4_opreq_action);
3446
3447	/*
3448	* Now reset the HCA before we touch the PCI capabilities or
3449	* attempt a firmware command, since a boot ROM may have left
3450	* the HCA in an undefined state.
3451	*/
3452	err = mlx4_reset(dev);
3453	if (err) {
3454	mlx4_err(dev, "Failed to reset HCA, aborting\n");
3455	goto err_sriov;
3456	}
3457
3458	if (total_vfs) {
3459	dev->flags = MLX4_FLAG_MASTER;
3460	existing_vfs = pci_num_vf(dev: pdev);
3461	if (existing_vfs)
3462	dev->flags |= MLX4_FLAG_SRIOV;
3463	dev->persist->num_vfs = total_vfs;
3464	}
3465	}
3466
3467	/* on load remove any previous indication of internal error,
3468	* device is up.
3469	*/
3470	dev->persist->state = MLX4_DEVICE_STATE_UP;
3471
3472	slave_start:
3473	err = mlx4_cmd_init(dev);
3474	if (err) {
3475	mlx4_err(dev, "Failed to init command interface, aborting\n");
3476	goto err_sriov;
3477	}
3478
3479	/* In slave functions, the communication channel must be initialized
3480	* before posting commands. Also, init num_slaves before calling
3481	* mlx4_init_hca */
3482	if (mlx4_is_mfunc(dev)) {
3483	if (mlx4_is_master(dev)) {
3484	dev->num_slaves = MLX4_MAX_NUM_SLAVES;
3485
3486	} else {
3487	dev->num_slaves = 0;
3488	err = mlx4_multi_func_init(dev);
3489	if (err) {
3490	mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n");
3491	goto err_cmd;
3492	}
3493	}
3494	}
3495
3496	err = mlx4_init_fw(dev);
3497	if (err) {
3498	mlx4_err(dev, "Failed to init fw, aborting.\n");
3499	goto err_mfunc;
3500	}
3501
3502	if (mlx4_is_master(dev)) {
3503	/* when we hit the goto slave_start below, dev_cap already initialized */
3504	if (!dev_cap) {
3505	dev_cap = kzalloc(size: sizeof(*dev_cap), GFP_KERNEL);
3506
3507	if (!dev_cap) {
3508	err = -ENOMEM;
3509	goto err_fw;
3510	}
3511
3512	err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
3513	if (err) {
3514	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
3515	goto err_fw;
3516	}
3517
3518	if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
3519	goto err_fw;
3520
3521	if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
3522	u64 dev_flags = mlx4_enable_sriov(dev, pdev,
3523	total_vfs,
3524	existing_vfs,
3525	reset_flow);
3526
3527	mlx4_close_fw(dev);
3528	mlx4_cmd_cleanup(dev, cleanup_mask: MLX4_CMD_CLEANUP_ALL);
3529	dev->flags = dev_flags;
3530	if (!SRIOV_VALID_STATE(dev->flags)) {
3531	mlx4_err(dev, "Invalid SRIOV state\n");
3532	goto err_sriov;
3533	}
3534	err = mlx4_reset(dev);
3535	if (err) {
3536	mlx4_err(dev, "Failed to reset HCA, aborting.\n");
3537	goto err_sriov;
3538	}
3539	goto slave_start;
3540	}
3541	} else {
3542	/* Legacy mode FW requires SRIOV to be enabled before
3543	* doing QUERY_DEV_CAP, since max_eq's value is different if
3544	* SRIOV is enabled.
3545	*/
3546	memset(dev_cap, 0, sizeof(*dev_cap));
3547	err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
3548	if (err) {
3549	mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
3550	goto err_fw;
3551	}
3552
3553	if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
3554	goto err_fw;
3555	}
3556	}
3557
3558	err = mlx4_init_hca(dev);
3559	if (err) {
3560	if (err == -EACCES) {
3561	/* Not primary Physical function
3562	* Running in slave mode */
3563	mlx4_cmd_cleanup(dev, cleanup_mask: MLX4_CMD_CLEANUP_ALL);
3564	/* We're not a PF */
3565	if (dev->flags & MLX4_FLAG_SRIOV) {
3566	if (!existing_vfs)
3567	pci_disable_sriov(dev: pdev);
3568	if (mlx4_is_master(dev) && !reset_flow)
3569	atomic_dec(v: &pf_loading);
3570	dev->flags &= ~MLX4_FLAG_SRIOV;
3571	}
3572	if (!mlx4_is_slave(dev))
3573	mlx4_free_ownership(dev);
3574	dev->flags |= MLX4_FLAG_SLAVE;
3575	dev->flags &= ~MLX4_FLAG_MASTER;
3576	goto slave_start;
3577	} else
3578	goto err_fw;
3579	}
3580
3581	if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
3582	u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs,
3583	existing_vfs, reset_flow);
3584
3585	if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) {
3586	mlx4_cmd_cleanup(dev, cleanup_mask: MLX4_CMD_CLEANUP_VHCR);
3587	dev->flags = dev_flags;
3588	err = mlx4_cmd_init(dev);
3589	if (err) {
3590	/* Only VHCR is cleaned up, so could still
3591	* send FW commands
3592	*/
3593	mlx4_err(dev, "Failed to init VHCR command interface, aborting\n");
3594	goto err_close;
3595	}
3596	} else {
3597	dev->flags = dev_flags;
3598	}
3599
3600	if (!SRIOV_VALID_STATE(dev->flags)) {
3601	mlx4_err(dev, "Invalid SRIOV state\n");
3602	err = -EINVAL;
3603	goto err_close;
3604	}
3605	}
3606
3607	/* check if the device is functioning at its maximum possible speed.
3608	* No return code for this call, just warn the user in case of PCI
3609	* express device capabilities are under-satisfied by the bus.
3610	*/
3611	if (!mlx4_is_slave(dev))
3612	pcie_print_link_status(dev: dev->persist->pdev);
3613
3614	/* In master functions, the communication channel must be initialized
3615	* after obtaining its address from fw */
3616	if (mlx4_is_master(dev)) {
3617	if (dev->caps.num_ports < 2 &&
3618	num_vfs_argc > 1) {
3619	err = -EINVAL;
3620	mlx4_err(dev,
3621	"Error: Trying to configure VFs on port 2, but HCA has only %d physical ports\n",
3622	dev->caps.num_ports);
3623	goto err_close;
3624	}
3625	memcpy(dev->persist->nvfs, nvfs, sizeof(dev->persist->nvfs));
3626
3627	for (i = 0;
3628	i < sizeof(dev->persist->nvfs)/
3629	sizeof(dev->persist->nvfs[0]); i++) {
3630	unsigned j;
3631
3632	for (j = 0; j < dev->persist->nvfs[i]; ++sum, ++j) {
3633	dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1;
3634	dev->dev_vfs[sum].n_ports = i < 2 ? 1 :
3635	dev->caps.num_ports;
3636	}
3637	}
3638
3639	/* In master functions, the communication channel
3640	* must be initialized after obtaining its address from fw
3641	*/
3642	err = mlx4_multi_func_init(dev);
3643	if (err) {
3644	mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n");
3645	goto err_close;
3646	}
3647	}
3648
3649	err = mlx4_alloc_eq_table(dev);
3650	if (err)
3651	goto err_master_mfunc;
3652
3653	bitmap_zero(dst: priv->msix_ctl.pool_bm, MAX_MSIX);
3654	mutex_init(&priv->msix_ctl.pool_lock);
3655
3656	mlx4_enable_msi_x(dev);
3657	if ((mlx4_is_mfunc(dev)) &&
3658	!(dev->flags & MLX4_FLAG_MSI_X)) {
3659	err = -EOPNOTSUPP;
3660	mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n");
3661	goto err_free_eq;
3662	}
3663
3664	if (!mlx4_is_slave(dev)) {
3665	err = mlx4_init_steering(dev);
3666	if (err)
3667	goto err_disable_msix;
3668	}
3669
3670	mlx4_init_quotas(dev);
3671
3672	err = mlx4_setup_hca(dev);
3673	if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) &&
3674	!mlx4_is_mfunc(dev)) {
3675	dev->flags &= ~MLX4_FLAG_MSI_X;
3676	dev->caps.num_comp_vectors = 1;
3677	pci_disable_msix(dev: pdev);
3678	err = mlx4_setup_hca(dev);
3679	}
3680
3681	if (err)
3682	goto err_steer;
3683
3684	/* When PF resources are ready arm its comm channel to enable
3685	* getting commands
3686	*/
3687	if (mlx4_is_master(dev)) {
3688	err = mlx4_ARM_COMM_CHANNEL(dev);
3689	if (err) {
3690	mlx4_err(dev, " Failed to arm comm channel eq: %x\n",
3691	err);
3692	goto err_steer;
3693	}
3694	}
3695
3696	for (port = 1; port <= dev->caps.num_ports; port++) {
3697	err = mlx4_init_port_info(dev, port);
3698	if (err)
3699	goto err_port;
3700	}
3701
3702	priv->v2p.port1 = 1;
3703	priv->v2p.port2 = 2;
3704
3705	err = mlx4_register_device(dev);
3706	if (err)
3707	goto err_port;
3708
3709	mlx4_sense_init(dev);
3710	mlx4_start_sense(dev);
3711
3712	priv->removed = 0;
3713
3714	if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)
3715	atomic_dec(v: &pf_loading);
3716
3717	kfree(objp: dev_cap);
3718	return 0;
3719
3720	err_port:
3721	for (--port; port >= 1; --port)
3722	mlx4_cleanup_port_info(info: &priv->port[port]);
3723
3724	mlx4_cleanup_default_counters(dev);
3725	if (!mlx4_is_slave(dev))
3726	mlx4_cleanup_counters_table(dev);
3727	mlx4_cleanup_qp_table(dev);
3728	mlx4_cleanup_srq_table(dev);
3729	mlx4_cleanup_cq_table(dev);
3730	mlx4_cmd_use_polling(dev);
3731	mlx4_cleanup_eq_table(dev);
3732	mlx4_cleanup_mcg_table(dev);
3733	mlx4_cleanup_mr_table(dev);
3734	mlx4_cleanup_xrcd_table(dev);
3735	mlx4_cleanup_pd_table(dev);
3736	mlx4_cleanup_uar_table(dev);
3737
3738	err_steer:
3739	if (!mlx4_is_slave(dev))
3740	mlx4_clear_steering(dev);
3741
3742	err_disable_msix:
3743	if (dev->flags & MLX4_FLAG_MSI_X)
3744	pci_disable_msix(dev: pdev);
3745
3746	err_free_eq:
3747	mlx4_free_eq_table(dev);
3748
3749	err_master_mfunc:
3750	if (mlx4_is_master(dev)) {
3751	mlx4_free_resource_tracker(dev, type: RES_TR_FREE_STRUCTS_ONLY);
3752	mlx4_multi_func_cleanup(dev);
3753	}
3754
3755	if (mlx4_is_slave(dev))
3756	mlx4_slave_destroy_special_qp_cap(dev);
3757
3758	err_close:
3759	mlx4_close_hca(dev);
3760
3761	err_fw:
3762	mlx4_close_fw(dev);
3763
3764	err_mfunc:
3765	if (mlx4_is_slave(dev))
3766	mlx4_multi_func_cleanup(dev);
3767
3768	err_cmd:
3769	mlx4_cmd_cleanup(dev, cleanup_mask: MLX4_CMD_CLEANUP_ALL);
3770
3771	err_sriov:
3772	if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs) {
3773	pci_disable_sriov(dev: pdev);
3774	dev->flags &= ~MLX4_FLAG_SRIOV;
3775	}
3776
3777	if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)
3778	atomic_dec(v: &pf_loading);
3779
3780	kfree(objp: priv->dev.dev_vfs);
3781
3782	if (!mlx4_is_slave(dev))
3783	mlx4_free_ownership(dev);
3784
3785	kfree(objp: dev_cap);
3786
3787	err_adev:
3788	mlx4_adev_cleanup(dev);
3789	return err;
3790	}
3791
3792	static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
3793	struct mlx4_priv *priv)
3794	{
3795	int err;
3796	int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
3797	int prb_vf[MLX4_MAX_PORTS + 1] = {0, 0, 0};
3798	const int param_map[MLX4_MAX_PORTS + 1][MLX4_MAX_PORTS + 1] = {
3799	{2, 0, 0}, {0, 1, 2}, {0, 1, 2} };
3800	unsigned total_vfs = 0;
3801	unsigned int i;
3802
3803	pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev));
3804
3805	err = mlx4_pci_enable_device(dev: &priv->dev);
3806	if (err) {
3807	dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
3808	return err;
3809	}
3810
3811	/* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS
3812	* per port, we must limit the number of VFs to 63 (since their are
3813	* 128 MACs)
3814	*/
3815	for (i = 0; i < ARRAY_SIZE(nvfs) && i < num_vfs_argc;
3816	total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) {
3817	nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i];
3818	if (nvfs[i] < 0) {
3819	dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n");
3820	err = -EINVAL;
3821	goto err_disable_pdev;
3822	}
3823	}
3824	for (i = 0; i < ARRAY_SIZE(prb_vf) && i < probe_vfs_argc;
3825	i++) {
3826	prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i];
3827	if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) {
3828	dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n");
3829	err = -EINVAL;
3830	goto err_disable_pdev;
3831	}
3832	}
3833	if (total_vfs > MLX4_MAX_NUM_VF) {
3834	dev_err(&pdev->dev,
3835	"Requested more VF's (%d) than allowed by hw (%d)\n",
3836	total_vfs, MLX4_MAX_NUM_VF);
3837	err = -EINVAL;
3838	goto err_disable_pdev;
3839	}
3840
3841	for (i = 0; i < MLX4_MAX_PORTS; i++) {
3842	if (nvfs[i] + nvfs[2] > MLX4_MAX_NUM_VF_P_PORT) {
3843	dev_err(&pdev->dev,
3844	"Requested more VF's (%d) for port (%d) than allowed by driver (%d)\n",
3845	nvfs[i] + nvfs[2], i + 1,
3846	MLX4_MAX_NUM_VF_P_PORT);
3847	err = -EINVAL;
3848	goto err_disable_pdev;
3849	}
3850	}
3851
3852	/* Check for BARs. */
3853	if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) &&
3854	!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
3855	dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\n",
3856	pci_dev_data, pci_resource_flags(pdev, 0));
3857	err = -ENODEV;
3858	goto err_disable_pdev;
3859	}
3860	if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
3861	dev_err(&pdev->dev, "Missing UAR, aborting\n");
3862	err = -ENODEV;
3863	goto err_disable_pdev;
3864	}
3865
3866	err = pci_request_regions(pdev, DRV_NAME);
3867	if (err) {
3868	dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
3869	goto err_disable_pdev;
3870	}
3871
3872	pci_set_master(dev: pdev);
3873
3874	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
3875	if (err) {
3876	dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
3877	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32));
3878	if (err) {
3879	dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
3880	goto err_release_regions;
3881	}
3882	}
3883
3884	/* Allow large DMA segments, up to the firmware limit of 1 GB */
3885	dma_set_max_seg_size(dev: &pdev->dev, size: 1024 * 1024 * 1024);
3886	/* Detect if this device is a virtual function */
3887	if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
3888	/* When acting as pf, we normally skip vfs unless explicitly
3889	* requested to probe them.
3890	*/
3891	if (total_vfs) {
3892	unsigned vfs_offset = 0;
3893
3894	for (i = 0; i < ARRAY_SIZE(nvfs) &&
3895	vfs_offset + nvfs[i] < extended_func_num(pdev);
3896	vfs_offset += nvfs[i], i++)
3897	;
3898	if (i == ARRAY_SIZE(nvfs)) {
3899	err = -ENODEV;
3900	goto err_release_regions;
3901	}
3902	if ((extended_func_num(pdev) - vfs_offset)
3903	> prb_vf[i]) {
3904	dev_warn(&pdev->dev, "Skipping virtual function:%d\n",
3905	extended_func_num(pdev));
3906	err = -ENODEV;
3907	goto err_release_regions;
3908	}
3909	}
3910	}
3911
3912	err = mlx4_crdump_init(dev: &priv->dev);
3913	if (err)
3914	goto err_release_regions;
3915
3916	err = mlx4_catas_init(dev: &priv->dev);
3917	if (err)
3918	goto err_crdump;
3919
3920	err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, reset_flow: 0);
3921	if (err)
3922	goto err_catas;
3923
3924	return 0;
3925
3926	err_catas:
3927	mlx4_catas_end(dev: &priv->dev);
3928
3929	err_crdump:
3930	mlx4_crdump_end(dev: &priv->dev);
3931
3932	err_release_regions:
3933	pci_release_regions(pdev);
3934
3935	err_disable_pdev:
3936	mlx4_pci_disable_device(dev: &priv->dev);
3937	return err;
3938	}
3939
3940	static void mlx4_devlink_param_load_driverinit_values(struct devlink *devlink)
3941	{
3942	struct mlx4_priv *priv = devlink_priv(devlink);
3943	struct mlx4_dev *dev = &priv->dev;
3944	struct mlx4_fw_crdump *crdump = &dev->persist->crdump;
3945	union devlink_param_value saved_value;
3946	int err;
3947
3948	err = devl_param_driverinit_value_get(devlink,
3949	param_id: DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
3950	val: &saved_value);
3951	if (!err && mlx4_internal_err_reset != saved_value.vbool) {
3952	mlx4_internal_err_reset = saved_value.vbool;
3953	/* Notify on value changed on runtime configuration mode */
3954	devl_param_value_changed(devlink,
3955	param_id: DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET);
3956	}
3957	err = devl_param_driverinit_value_get(devlink,
3958	param_id: DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
3959	val: &saved_value);
3960	if (!err)
3961	log_num_mac = order_base_2(saved_value.vu32);
3962	err = devl_param_driverinit_value_get(devlink,
3963	param_id: MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
3964	val: &saved_value);
3965	if (!err)
3966	enable_64b_cqe_eqe = saved_value.vbool;
3967	err = devl_param_driverinit_value_get(devlink,
3968	param_id: MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
3969	val: &saved_value);
3970	if (!err)
3971	enable_4k_uar = saved_value.vbool;
3972	err = devl_param_driverinit_value_get(devlink,
3973	param_id: DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
3974	val: &saved_value);
3975	if (!err && crdump->snapshot_enable != saved_value.vbool) {
3976	crdump->snapshot_enable = saved_value.vbool;
3977	devl_param_value_changed(devlink,
3978	param_id: DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT);
3979	}
3980	}
3981
3982	static void mlx4_restart_one_down(struct pci_dev *pdev);
3983	static int mlx4_restart_one_up(struct pci_dev *pdev, bool reload,
3984	struct devlink *devlink);
3985
3986	static int mlx4_devlink_reload_down(struct devlink *devlink, bool netns_change,
3987	enum devlink_reload_action action,
3988	enum devlink_reload_limit limit,
3989	struct netlink_ext_ack *extack)
3990	{
3991	struct mlx4_priv *priv = devlink_priv(devlink);
3992	struct mlx4_dev *dev = &priv->dev;
3993	struct mlx4_dev_persistent *persist = dev->persist;
3994
3995	if (netns_change) {
3996	NL_SET_ERR_MSG_MOD(extack, "Namespace change is not supported");
3997	return -EOPNOTSUPP;
3998	}
3999	if (persist->num_vfs)
4000	mlx4_warn(persist->dev, "Reload performed on PF, will cause reset on operating Virtual Functions\n");
4001	mlx4_restart_one_down(pdev: persist->pdev);
4002	return 0;
4003	}
4004
4005	static int mlx4_devlink_reload_up(struct devlink *devlink, enum devlink_reload_action action,
4006	enum devlink_reload_limit limit, u32 *actions_performed,
4007	struct netlink_ext_ack *extack)
4008	{
4009	struct mlx4_priv *priv = devlink_priv(devlink);
4010	struct mlx4_dev *dev = &priv->dev;
4011	struct mlx4_dev_persistent *persist = dev->persist;
4012	int err;
4013
4014	*actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT);
4015	err = mlx4_restart_one_up(pdev: persist->pdev, reload: true, devlink);
4016	if (err)
4017	mlx4_err(persist->dev, "mlx4_restart_one_up failed, ret=%d\n",
4018	err);
4019
4020	return err;
4021	}
4022
4023	static const struct devlink_ops mlx4_devlink_ops = {
4024	.reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT),
4025	.reload_down = mlx4_devlink_reload_down,
4026	.reload_up = mlx4_devlink_reload_up,
4027	};
4028
4029	static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
4030	{
4031	struct devlink *devlink;
4032	struct mlx4_priv *priv;
4033	struct mlx4_dev *dev;
4034	int ret;
4035
4036	printk_once(KERN_INFO "%s", mlx4_version);
4037
4038	devlink = devlink_alloc(ops: &mlx4_devlink_ops, priv_size: sizeof(*priv), dev: &pdev->dev);
4039	if (!devlink)
4040	return -ENOMEM;
4041	devl_lock(devlink);
4042	priv = devlink_priv(devlink);
4043
4044	dev = &priv->dev;
4045	dev->persist = kzalloc(size: sizeof(*dev->persist), GFP_KERNEL);
4046	if (!dev->persist) {
4047	ret = -ENOMEM;
4048	goto err_devlink_free;
4049	}
4050	dev->persist->pdev = pdev;
4051	dev->persist->dev = dev;
4052	pci_set_drvdata(pdev, data: dev->persist);
4053	priv->pci_dev_data = id->driver_data;
4054	mutex_init(&dev->persist->device_state_mutex);
4055	mutex_init(&dev->persist->interface_state_mutex);
4056	mutex_init(&dev->persist->pci_status_mutex);
4057
4058	ret = devl_params_register(devlink, params: mlx4_devlink_params,
4059	ARRAY_SIZE(mlx4_devlink_params));
4060	if (ret)
4061	goto err_devlink_unregister;
4062	mlx4_devlink_set_params_init_values(devlink);
4063	ret = __mlx4_init_one(pdev, pci_dev_data: id->driver_data, priv);
4064	if (ret)
4065	goto err_params_unregister;
4066
4067	pci_save_state(dev: pdev);
4068	devl_unlock(devlink);
4069	devlink_register(devlink);
4070	return 0;
4071
4072	err_params_unregister:
4073	devl_params_unregister(devlink, params: mlx4_devlink_params,
4074	ARRAY_SIZE(mlx4_devlink_params));
4075	err_devlink_unregister:
4076	kfree(objp: dev->persist);
4077	err_devlink_free:
4078	devl_unlock(devlink);
4079	devlink_free(devlink);
4080	return ret;
4081	}
4082
4083	static void mlx4_clean_dev(struct mlx4_dev *dev)
4084	{
4085	struct mlx4_dev_persistent *persist = dev->persist;
4086	struct mlx4_priv *priv = mlx4_priv(dev);
4087	unsigned long flags = (dev->flags & RESET_PERSIST_MASK_FLAGS);
4088
4089	memset(priv, 0, sizeof(*priv));
4090	priv->dev.persist = persist;
4091	priv->dev.flags = flags;
4092	}
4093
4094	static void mlx4_unload_one(struct pci_dev *pdev)
4095	{
4096	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4097	struct mlx4_dev *dev = persist->dev;
4098	struct mlx4_priv *priv = mlx4_priv(dev);
4099	int pci_dev_data;
4100	struct devlink *devlink;
4101	int p, i;
4102
4103	devlink = priv_to_devlink(priv);
4104	devl_assert_locked(devlink);
4105	if (priv->removed)
4106	return;
4107
4108	/* saving current ports type for further use */
4109	for (i = 0; i < dev->caps.num_ports; i++) {
4110	dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1];
4111	dev->persist->curr_port_poss_type[i] = dev->caps.
4112	possible_type[i + 1];
4113	}
4114
4115	pci_dev_data = priv->pci_dev_data;
4116
4117	mlx4_stop_sense(dev);
4118	mlx4_unregister_device(dev);
4119
4120	for (p = 1; p <= dev->caps.num_ports; p++) {
4121	mlx4_cleanup_port_info(info: &priv->port[p]);
4122	mlx4_CLOSE_PORT(dev, port: p);
4123	}
4124
4125	if (mlx4_is_master(dev))
4126	mlx4_free_resource_tracker(dev,
4127	type: RES_TR_FREE_SLAVES_ONLY);
4128
4129	mlx4_cleanup_default_counters(dev);
4130	if (!mlx4_is_slave(dev))
4131	mlx4_cleanup_counters_table(dev);
4132	mlx4_cleanup_qp_table(dev);
4133	mlx4_cleanup_srq_table(dev);
4134	mlx4_cleanup_cq_table(dev);
4135	mlx4_cmd_use_polling(dev);
4136	mlx4_cleanup_eq_table(dev);
4137	mlx4_cleanup_mcg_table(dev);
4138	mlx4_cleanup_mr_table(dev);
4139	mlx4_cleanup_xrcd_table(dev);
4140	mlx4_cleanup_pd_table(dev);
4141
4142	if (mlx4_is_master(dev))
4143	mlx4_free_resource_tracker(dev,
4144	type: RES_TR_FREE_STRUCTS_ONLY);
4145
4146	iounmap(addr: priv->kar);
4147	mlx4_uar_free(dev, uar: &priv->driver_uar);
4148	mlx4_cleanup_uar_table(dev);
4149	if (!mlx4_is_slave(dev))
4150	mlx4_clear_steering(dev);
4151	mlx4_free_eq_table(dev);
4152	if (mlx4_is_master(dev))
4153	mlx4_multi_func_cleanup(dev);
4154	mlx4_close_hca(dev);
4155	mlx4_close_fw(dev);
4156	if (mlx4_is_slave(dev))
4157	mlx4_multi_func_cleanup(dev);
4158	mlx4_cmd_cleanup(dev, cleanup_mask: MLX4_CMD_CLEANUP_ALL);
4159
4160	if (dev->flags & MLX4_FLAG_MSI_X)
4161	pci_disable_msix(dev: pdev);
4162
4163	if (!mlx4_is_slave(dev))
4164	mlx4_free_ownership(dev);
4165
4166	mlx4_slave_destroy_special_qp_cap(dev);
4167	kfree(objp: dev->dev_vfs);
4168
4169	mlx4_adev_cleanup(dev);
4170
4171	mlx4_clean_dev(dev);
4172	priv->pci_dev_data = pci_dev_data;
4173	priv->removed = 1;
4174	}
4175
4176	static void mlx4_remove_one(struct pci_dev *pdev)
4177	{
4178	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4179	struct mlx4_dev *dev = persist->dev;
4180	struct mlx4_priv *priv = mlx4_priv(dev);
4181	struct devlink *devlink = priv_to_devlink(priv);
4182	int active_vfs = 0;
4183
4184	devlink_unregister(devlink);
4185
4186	devl_lock(devlink);
4187	if (mlx4_is_slave(dev))
4188	persist->interface_state |= MLX4_INTERFACE_STATE_NOWAIT;
4189
4190	mutex_lock(&persist->interface_state_mutex);
4191	persist->interface_state |= MLX4_INTERFACE_STATE_DELETION;
4192	mutex_unlock(lock: &persist->interface_state_mutex);
4193
4194	/* Disabling SR-IOV is not allowed while there are active vf's */
4195	if (mlx4_is_master(dev) && dev->flags & MLX4_FLAG_SRIOV) {
4196	active_vfs = mlx4_how_many_lives_vf(dev);
4197	if (active_vfs) {
4198	pr_warn("Removing PF when there are active VF's !!\n");
4199	pr_warn("Will not disable SR-IOV.\n");
4200	}
4201	}
4202
4203	/* device marked to be under deletion running now without the lock
4204	* letting other tasks to be terminated
4205	*/
4206	if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4207	mlx4_unload_one(pdev);
4208	else
4209	mlx4_info(dev, "%s: interface is down\n", __func__);
4210	mlx4_catas_end(dev);
4211	mlx4_crdump_end(dev);
4212	if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) {
4213	mlx4_warn(dev, "Disabling SR-IOV\n");
4214	pci_disable_sriov(dev: pdev);
4215	}
4216
4217	pci_release_regions(pdev);
4218	mlx4_pci_disable_device(dev);
4219	devl_params_unregister(devlink, params: mlx4_devlink_params,
4220	ARRAY_SIZE(mlx4_devlink_params));
4221	kfree(objp: dev->persist);
4222	devl_unlock(devlink);
4223	devlink_free(devlink);
4224	}
4225
4226	static int restore_current_port_types(struct mlx4_dev *dev,
4227	enum mlx4_port_type *types,
4228	enum mlx4_port_type *poss_types)
4229	{
4230	struct mlx4_priv *priv = mlx4_priv(dev);
4231	int err, i;
4232
4233	mlx4_stop_sense(dev);
4234
4235	mutex_lock(&priv->port_mutex);
4236	for (i = 0; i < dev->caps.num_ports; i++)
4237	dev->caps.possible_type[i + 1] = poss_types[i];
4238	err = mlx4_change_port_types(dev, port_types: types);
4239	mlx4_start_sense(dev);
4240	mutex_unlock(lock: &priv->port_mutex);
4241
4242	return err;
4243	}
4244
4245	static void mlx4_restart_one_down(struct pci_dev *pdev)
4246	{
4247	mlx4_unload_one(pdev);
4248	}
4249
4250	static int mlx4_restart_one_up(struct pci_dev *pdev, bool reload,
4251	struct devlink *devlink)
4252	{
4253	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4254	struct mlx4_dev *dev = persist->dev;
4255	struct mlx4_priv *priv = mlx4_priv(dev);
4256	int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4257	int pci_dev_data, err, total_vfs;
4258
4259	pci_dev_data = priv->pci_dev_data;
4260	total_vfs = dev->persist->num_vfs;
4261	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
4262
4263	if (reload)
4264	mlx4_devlink_param_load_driverinit_values(devlink);
4265	err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, reset_flow: 1);
4266	if (err) {
4267	mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n",
4268	__func__, pci_name(pdev), err);
4269	return err;
4270	}
4271
4272	err = restore_current_port_types(dev, types: dev->persist->curr_port_type,
4273	poss_types: dev->persist->curr_port_poss_type);
4274	if (err)
4275	mlx4_err(dev, "could not restore original port types (%d)\n",
4276	err);
4277
4278	return err;
4279	}
4280
4281	int mlx4_restart_one(struct pci_dev *pdev)
4282	{
4283	mlx4_restart_one_down(pdev);
4284	return mlx4_restart_one_up(pdev, reload: false, NULL);
4285	}
4286
4287	#define MLX_SP(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_FORCE_SENSE_PORT }
4288	#define MLX_VF(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_IS_VF }
4289	#define MLX_GN(id) { PCI_VDEVICE(MELLANOX, id), 0 }
4290
4291	static const struct pci_device_id mlx4_pci_table[] = {
4292	#ifdef CONFIG_MLX4_CORE_GEN2
4293	/* MT25408 "Hermon" */
4294	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_SDR), /* SDR */
4295	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR), /* DDR */
4296	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR), /* QDR */
4297	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2), /* DDR Gen2 */
4298	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2), /* QDR Gen2 */
4299	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN), /* EN 10GigE */
4300	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2), /* EN 10GigE Gen2 */
4301	/* MT25458 ConnectX EN 10GBASE-T */
4302	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN),
4303	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2), /* Gen2 */
4304	/* MT26468 ConnectX EN 10GigE PCIe Gen2*/
4305	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2),
4306	/* MT26438 ConnectX EN 40GigE PCIe Gen2 5GT/s */
4307	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2),
4308	/* MT26478 ConnectX2 40GigE PCIe Gen2 */
4309	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX2),
4310	/* MT25400 Family [ConnectX-2] */
4311	MLX_VF(0x1002), /* Virtual Function */
4312	#endif /* CONFIG_MLX4_CORE_GEN2 */
4313	/* MT27500 Family [ConnectX-3] */
4314	MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3),
4315	MLX_VF(0x1004), /* Virtual Function */
4316	MLX_GN(0x1005), /* MT27510 Family */
4317	MLX_GN(0x1006), /* MT27511 Family */
4318	MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO), /* MT27520 Family */
4319	MLX_GN(0x1008), /* MT27521 Family */
4320	MLX_GN(0x1009), /* MT27530 Family */
4321	MLX_GN(0x100a), /* MT27531 Family */
4322	MLX_GN(0x100b), /* MT27540 Family */
4323	MLX_GN(0x100c), /* MT27541 Family */
4324	MLX_GN(0x100d), /* MT27550 Family */
4325	MLX_GN(0x100e), /* MT27551 Family */
4326	MLX_GN(0x100f), /* MT27560 Family */
4327	MLX_GN(0x1010), /* MT27561 Family */
4328
4329	/*
4330	* See the mellanox_check_broken_intx_masking() quirk when
4331	* adding devices
4332	*/
4333
4334	{ 0, }
4335	};
4336
4337	MODULE_DEVICE_TABLE(pci, mlx4_pci_table);
4338
4339	static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev,
4340	pci_channel_state_t state)
4341	{
4342	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4343	struct mlx4_dev *dev = persist->dev;
4344	struct devlink *devlink;
4345
4346	mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n");
4347	mlx4_enter_error_state(persist);
4348
4349	devlink = priv_to_devlink(priv: mlx4_priv(dev));
4350	devl_lock(devlink);
4351	mutex_lock(&persist->interface_state_mutex);
4352	if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4353	mlx4_unload_one(pdev);
4354
4355	mutex_unlock(lock: &persist->interface_state_mutex);
4356	devl_unlock(devlink);
4357	if (state == pci_channel_io_perm_failure)
4358	return PCI_ERS_RESULT_DISCONNECT;
4359
4360	mlx4_pci_disable_device(dev: persist->dev);
4361	return PCI_ERS_RESULT_NEED_RESET;
4362	}
4363
4364	static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev)
4365	{
4366	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4367	struct mlx4_dev *dev = persist->dev;
4368	int err;
4369
4370	mlx4_err(dev, "mlx4_pci_slot_reset was called\n");
4371	err = mlx4_pci_enable_device(dev);
4372	if (err) {
4373	mlx4_err(dev, "Can not re-enable device, err=%d\n", err);
4374	return PCI_ERS_RESULT_DISCONNECT;
4375	}
4376
4377	pci_set_master(dev: pdev);
4378	pci_restore_state(dev: pdev);
4379	pci_save_state(dev: pdev);
4380	return PCI_ERS_RESULT_RECOVERED;
4381	}
4382
4383	static void mlx4_pci_resume(struct pci_dev *pdev)
4384	{
4385	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4386	struct mlx4_dev *dev = persist->dev;
4387	struct mlx4_priv *priv = mlx4_priv(dev);
4388	int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4389	struct devlink *devlink;
4390	int total_vfs;
4391	int err;
4392
4393	mlx4_err(dev, "%s was called\n", __func__);
4394	total_vfs = dev->persist->num_vfs;
4395	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
4396
4397	devlink = priv_to_devlink(priv);
4398	devl_lock(devlink);
4399	mutex_lock(&persist->interface_state_mutex);
4400	if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) {
4401	err = mlx4_load_one(pdev, pci_dev_data: priv->pci_dev_data, total_vfs, nvfs,
4402	priv, reset_flow: 1);
4403	if (err) {
4404	mlx4_err(dev, "%s: mlx4_load_one failed, err=%d\n",
4405	__func__, err);
4406	goto end;
4407	}
4408
4409	err = restore_current_port_types(dev, types: dev->persist->
4410	curr_port_type, poss_types: dev->persist->
4411	curr_port_poss_type);
4412	if (err)
4413	mlx4_err(dev, "could not restore original port types (%d)\n", err);
4414	}
4415	end:
4416	mutex_unlock(lock: &persist->interface_state_mutex);
4417	devl_unlock(devlink);
4418	}
4419
4420	static void mlx4_shutdown(struct pci_dev *pdev)
4421	{
4422	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4423	struct mlx4_dev *dev = persist->dev;
4424	struct devlink *devlink;
4425
4426	mlx4_info(persist->dev, "mlx4_shutdown was called\n");
4427	devlink = priv_to_devlink(priv: mlx4_priv(dev));
4428	devl_lock(devlink);
4429	mutex_lock(&persist->interface_state_mutex);
4430	if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4431	mlx4_unload_one(pdev);
4432	mutex_unlock(lock: &persist->interface_state_mutex);
4433	devl_unlock(devlink);
4434	mlx4_pci_disable_device(dev);
4435	}
4436
4437	static const struct pci_error_handlers mlx4_err_handler = {
4438	.error_detected = mlx4_pci_err_detected,
4439	.slot_reset = mlx4_pci_slot_reset,
4440	.resume = mlx4_pci_resume,
4441	};
4442
4443	static int __maybe_unused mlx4_suspend(struct device *dev_d)
4444	{
4445	struct pci_dev *pdev = to_pci_dev(dev_d);
4446	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4447	struct mlx4_dev *dev = persist->dev;
4448	struct devlink *devlink;
4449
4450	mlx4_err(dev, "suspend was called\n");
4451	devlink = priv_to_devlink(priv: mlx4_priv(dev));
4452	devl_lock(devlink);
4453	mutex_lock(&persist->interface_state_mutex);
4454	if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4455	mlx4_unload_one(pdev);
4456	mutex_unlock(lock: &persist->interface_state_mutex);
4457	devl_unlock(devlink);
4458
4459	return 0;
4460	}
4461
4462	static int __maybe_unused mlx4_resume(struct device *dev_d)
4463	{
4464	struct pci_dev *pdev = to_pci_dev(dev_d);
4465	struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4466	struct mlx4_dev *dev = persist->dev;
4467	struct mlx4_priv *priv = mlx4_priv(dev);
4468	int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4469	struct devlink *devlink;
4470	int total_vfs;
4471	int ret = 0;
4472
4473	mlx4_err(dev, "resume was called\n");
4474	total_vfs = dev->persist->num_vfs;
4475	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
4476
4477	devlink = priv_to_devlink(priv);
4478	devl_lock(devlink);
4479	mutex_lock(&persist->interface_state_mutex);
4480	if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) {
4481	ret = mlx4_load_one(pdev, pci_dev_data: priv->pci_dev_data, total_vfs,
4482	nvfs, priv, reset_flow: 1);
4483	if (!ret) {
4484	ret = restore_current_port_types(dev,
4485	types: dev->persist->curr_port_type,
4486	poss_types: dev->persist->curr_port_poss_type);
4487	if (ret)
4488	mlx4_err(dev, "resume: could not restore original port types (%d)\n", ret);
4489	}
4490	}
4491	mutex_unlock(lock: &persist->interface_state_mutex);
4492	devl_unlock(devlink);
4493
4494	return ret;
4495	}
4496
4497	static SIMPLE_DEV_PM_OPS(mlx4_pm_ops, mlx4_suspend, mlx4_resume);
4498
4499	static struct pci_driver mlx4_driver = {
4500	.name = DRV_NAME,
4501	.id_table = mlx4_pci_table,
4502	.probe = mlx4_init_one,
4503	.shutdown = mlx4_shutdown,
4504	.remove = mlx4_remove_one,
4505	.driver.pm = &mlx4_pm_ops,
4506	.err_handler = &mlx4_err_handler,
4507	};
4508
4509	static int __init mlx4_verify_params(void)
4510	{
4511	if (msi_x < 0) {
4512	pr_warn("mlx4_core: bad msi_x: %d\n", msi_x);
4513	return -1;
4514	}
4515
4516	if ((log_num_mac < 0) || (log_num_mac > 7)) {
4517	pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac);
4518	return -1;
4519	}
4520
4521	if (log_num_vlan != 0)
4522	pr_warn("mlx4_core: log_num_vlan - obsolete module param, using %d\n",
4523	MLX4_LOG_NUM_VLANS);
4524
4525	if (use_prio != 0)
4526	pr_warn("mlx4_core: use_prio - obsolete module param, ignored\n");
4527
4528	if ((log_mtts_per_seg < 0) || (log_mtts_per_seg > 7)) {
4529	pr_warn("mlx4_core: bad log_mtts_per_seg: %d\n",
4530	log_mtts_per_seg);
4531	return -1;
4532	}
4533
4534	/* Check if module param for ports type has legal combination */
4535	if (port_type_array[0] == false && port_type_array[1] == true) {
4536	pr_warn("Module parameter configuration ETH/IB is not supported. Switching to default configuration IB/IB\n");
4537	port_type_array[0] = true;
4538	}
4539
4540	if (mlx4_log_num_mgm_entry_size < -7 ||
4541	(mlx4_log_num_mgm_entry_size > 0 &&
4542	(mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE ||
4543	mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE))) {
4544	pr_warn("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not in legal range (-7..0 or %d..%d)\n",
4545	mlx4_log_num_mgm_entry_size,
4546	MLX4_MIN_MGM_LOG_ENTRY_SIZE,
4547	MLX4_MAX_MGM_LOG_ENTRY_SIZE);
4548	return -1;
4549	}
4550
4551	return 0;
4552	}
4553
4554	static int __init mlx4_init(void)
4555	{
4556	int ret;
4557
4558	WARN_ONCE(strcmp(MLX4_ADEV_NAME, KBUILD_MODNAME),
4559	"mlx4_core name not in sync with kernel module name");
4560
4561	if (mlx4_verify_params())
4562	return -EINVAL;
4563
4564
4565	mlx4_wq = create_singlethread_workqueue("mlx4");
4566	if (!mlx4_wq)
4567	return -ENOMEM;
4568
4569	ret = pci_register_driver(&mlx4_driver);
4570	if (ret < 0)
4571	destroy_workqueue(wq: mlx4_wq);
4572	return ret < 0 ? ret : 0;
4573	}
4574
4575	static void __exit mlx4_cleanup(void)
4576	{
4577	pci_unregister_driver(dev: &mlx4_driver);
4578	destroy_workqueue(wq: mlx4_wq);
4579	}
4580
4581	module_init(mlx4_init);
4582	module_exit(mlx4_cleanup);
4583