1 | /* |
2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/highmem.h> |
34 | #include <linux/module.h> |
35 | #include <linux/init.h> |
36 | #include <linux/errno.h> |
37 | #include <linux/pci.h> |
38 | #include <linux/dma-mapping.h> |
39 | #include <linux/slab.h> |
40 | #include <linux/interrupt.h> |
41 | #include <linux/delay.h> |
42 | #include <linux/mlx5/driver.h> |
43 | #include <linux/mlx5/cq.h> |
44 | #include <linux/mlx5/qp.h> |
45 | #include <linux/debugfs.h> |
46 | #include <linux/kmod.h> |
47 | #include <linux/mlx5/mlx5_ifc.h> |
48 | #include <linux/mlx5/vport.h> |
49 | #include <linux/version.h> |
50 | #include <net/devlink.h> |
51 | #include "mlx5_core.h" |
52 | #include "lib/eq.h" |
53 | #include "fs_core.h" |
54 | #include "lib/mpfs.h" |
55 | #include "eswitch.h" |
56 | #include "devlink.h" |
57 | #include "fw_reset.h" |
58 | #include "lib/mlx5.h" |
59 | #include "lib/tout.h" |
60 | #include "fpga/core.h" |
61 | #include "en_accel/ipsec.h" |
62 | #include "lib/clock.h" |
63 | #include "lib/vxlan.h" |
64 | #include "lib/geneve.h" |
65 | #include "lib/devcom.h" |
66 | #include "lib/pci_vsc.h" |
67 | #include "diag/fw_tracer.h" |
68 | #include "ecpf.h" |
69 | #include "lib/hv_vhca.h" |
70 | #include "diag/rsc_dump.h" |
71 | #include "sf/vhca_event.h" |
72 | #include "sf/dev/dev.h" |
73 | #include "sf/sf.h" |
74 | #include "mlx5_irq.h" |
75 | #include "hwmon.h" |
76 | #include "lag/lag.h" |
77 | |
78 | MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>" ); |
79 | MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver" ); |
80 | MODULE_LICENSE("Dual BSD/GPL" ); |
81 | |
82 | unsigned int mlx5_core_debug_mask; |
83 | module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644); |
84 | MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0" ); |
85 | |
86 | static unsigned int prof_sel = MLX5_DEFAULT_PROF; |
87 | module_param_named(prof_sel, prof_sel, uint, 0444); |
88 | MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2" ); |
89 | |
90 | static u32 sw_owner_id[4]; |
91 | #define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1) |
92 | static DEFINE_IDA(sw_vhca_ida); |
93 | |
94 | enum { |
95 | MLX5_ATOMIC_REQ_MODE_BE = 0x0, |
96 | MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1, |
97 | }; |
98 | |
99 | #define LOG_MAX_SUPPORTED_QPS 0xff |
100 | |
101 | static struct mlx5_profile profile[] = { |
102 | [0] = { |
103 | .mask = 0, |
104 | .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, |
105 | }, |
106 | [1] = { |
107 | .mask = MLX5_PROF_MASK_QP_SIZE, |
108 | .log_max_qp = 12, |
109 | .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, |
110 | |
111 | }, |
112 | [2] = { |
113 | .mask = MLX5_PROF_MASK_QP_SIZE | |
114 | MLX5_PROF_MASK_MR_CACHE, |
115 | .log_max_qp = LOG_MAX_SUPPORTED_QPS, |
116 | .num_cmd_caches = MLX5_NUM_COMMAND_CACHES, |
117 | .mr_cache[0] = { |
118 | .size = 500, |
119 | .limit = 250 |
120 | }, |
121 | .mr_cache[1] = { |
122 | .size = 500, |
123 | .limit = 250 |
124 | }, |
125 | .mr_cache[2] = { |
126 | .size = 500, |
127 | .limit = 250 |
128 | }, |
129 | .mr_cache[3] = { |
130 | .size = 500, |
131 | .limit = 250 |
132 | }, |
133 | .mr_cache[4] = { |
134 | .size = 500, |
135 | .limit = 250 |
136 | }, |
137 | .mr_cache[5] = { |
138 | .size = 500, |
139 | .limit = 250 |
140 | }, |
141 | .mr_cache[6] = { |
142 | .size = 500, |
143 | .limit = 250 |
144 | }, |
145 | .mr_cache[7] = { |
146 | .size = 500, |
147 | .limit = 250 |
148 | }, |
149 | .mr_cache[8] = { |
150 | .size = 500, |
151 | .limit = 250 |
152 | }, |
153 | .mr_cache[9] = { |
154 | .size = 500, |
155 | .limit = 250 |
156 | }, |
157 | .mr_cache[10] = { |
158 | .size = 500, |
159 | .limit = 250 |
160 | }, |
161 | .mr_cache[11] = { |
162 | .size = 500, |
163 | .limit = 250 |
164 | }, |
165 | .mr_cache[12] = { |
166 | .size = 64, |
167 | .limit = 32 |
168 | }, |
169 | .mr_cache[13] = { |
170 | .size = 32, |
171 | .limit = 16 |
172 | }, |
173 | .mr_cache[14] = { |
174 | .size = 16, |
175 | .limit = 8 |
176 | }, |
177 | .mr_cache[15] = { |
178 | .size = 8, |
179 | .limit = 4 |
180 | }, |
181 | }, |
182 | [3] = { |
183 | .mask = MLX5_PROF_MASK_QP_SIZE, |
184 | .log_max_qp = LOG_MAX_SUPPORTED_QPS, |
185 | .num_cmd_caches = 0, |
186 | }, |
187 | }; |
188 | |
189 | static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili, |
190 | u32 warn_time_mili, const char *init_state) |
191 | { |
192 | unsigned long warn = jiffies + msecs_to_jiffies(m: warn_time_mili); |
193 | unsigned long end = jiffies + msecs_to_jiffies(m: max_wait_mili); |
194 | u32 fw_initializing; |
195 | |
196 | do { |
197 | fw_initializing = ioread32be(&dev->iseg->initializing); |
198 | if (!(fw_initializing >> 31)) |
199 | break; |
200 | if (time_after(jiffies, end)) { |
201 | mlx5_core_err(dev, "Firmware over %u MS in %s state, aborting\n" , |
202 | max_wait_mili, init_state); |
203 | return -ETIMEDOUT; |
204 | } |
205 | if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) { |
206 | mlx5_core_warn(dev, "device is being removed, stop waiting for FW %s\n" , |
207 | init_state); |
208 | return -ENODEV; |
209 | } |
210 | if (warn_time_mili && time_after(jiffies, warn)) { |
211 | mlx5_core_warn(dev, "Waiting for FW %s, timeout abort in %ds (0x%x)\n" , |
212 | init_state, jiffies_to_msecs(end - warn) / 1000, |
213 | fw_initializing); |
214 | warn = jiffies + msecs_to_jiffies(m: warn_time_mili); |
215 | } |
216 | msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT)); |
217 | } while (true); |
218 | |
219 | return 0; |
220 | } |
221 | |
222 | static void mlx5_set_driver_version(struct mlx5_core_dev *dev) |
223 | { |
224 | int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in, |
225 | driver_version); |
226 | u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {}; |
227 | char *string; |
228 | |
229 | if (!MLX5_CAP_GEN(dev, driver_version)) |
230 | return; |
231 | |
232 | string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version); |
233 | |
234 | snprintf(buf: string, size: driver_ver_sz, fmt: "Linux,%s,%u.%u.%u" , |
235 | KBUILD_MODNAME, LINUX_VERSION_MAJOR, |
236 | LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL); |
237 | |
238 | /*Send the command*/ |
239 | MLX5_SET(set_driver_version_in, in, opcode, |
240 | MLX5_CMD_OP_SET_DRIVER_VERSION); |
241 | |
242 | mlx5_cmd_exec_in(dev, set_driver_version, in); |
243 | } |
244 | |
245 | static int set_dma_caps(struct pci_dev *pdev) |
246 | { |
247 | int err; |
248 | |
249 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
250 | if (err) { |
251 | dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n" ); |
252 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32)); |
253 | if (err) { |
254 | dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n" ); |
255 | return err; |
256 | } |
257 | } |
258 | |
259 | dma_set_max_seg_size(dev: &pdev->dev, size: 2u * 1024 * 1024 * 1024); |
260 | return err; |
261 | } |
262 | |
263 | static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) |
264 | { |
265 | struct pci_dev *pdev = dev->pdev; |
266 | int err = 0; |
267 | |
268 | mutex_lock(&dev->pci_status_mutex); |
269 | if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { |
270 | err = pci_enable_device(dev: pdev); |
271 | if (!err) |
272 | dev->pci_status = MLX5_PCI_STATUS_ENABLED; |
273 | } |
274 | mutex_unlock(lock: &dev->pci_status_mutex); |
275 | |
276 | return err; |
277 | } |
278 | |
279 | static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) |
280 | { |
281 | struct pci_dev *pdev = dev->pdev; |
282 | |
283 | mutex_lock(&dev->pci_status_mutex); |
284 | if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { |
285 | pci_disable_device(dev: pdev); |
286 | dev->pci_status = MLX5_PCI_STATUS_DISABLED; |
287 | } |
288 | mutex_unlock(lock: &dev->pci_status_mutex); |
289 | } |
290 | |
291 | static int request_bar(struct pci_dev *pdev) |
292 | { |
293 | int err = 0; |
294 | |
295 | if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { |
296 | dev_err(&pdev->dev, "Missing registers BAR, aborting\n" ); |
297 | return -ENODEV; |
298 | } |
299 | |
300 | err = pci_request_regions(pdev, KBUILD_MODNAME); |
301 | if (err) |
302 | dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n" ); |
303 | |
304 | return err; |
305 | } |
306 | |
307 | static void release_bar(struct pci_dev *pdev) |
308 | { |
309 | pci_release_regions(pdev); |
310 | } |
311 | |
312 | struct mlx5_reg_host_endianness { |
313 | u8 he; |
314 | u8 rsvd[15]; |
315 | }; |
316 | |
317 | static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) |
318 | { |
319 | switch (size) { |
320 | case 128: |
321 | return 0; |
322 | case 256: |
323 | return 1; |
324 | case 512: |
325 | return 2; |
326 | case 1024: |
327 | return 3; |
328 | case 2048: |
329 | return 4; |
330 | case 4096: |
331 | return 5; |
332 | default: |
333 | mlx5_core_warn(dev, "invalid pkey table size %d\n" , size); |
334 | return 0; |
335 | } |
336 | } |
337 | |
338 | void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev) |
339 | { |
340 | mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); |
341 | dev->mlx5e_res.uplink_netdev = netdev; |
342 | mlx5_blocking_notifier_call_chain(dev, event: MLX5_DRIVER_EVENT_UPLINK_NETDEV, |
343 | data: netdev); |
344 | mutex_unlock(lock: &dev->mlx5e_res.uplink_netdev_lock); |
345 | } |
346 | |
347 | void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev) |
348 | { |
349 | mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); |
350 | mlx5_blocking_notifier_call_chain(dev, event: MLX5_DRIVER_EVENT_UPLINK_NETDEV, |
351 | data: dev->mlx5e_res.uplink_netdev); |
352 | mutex_unlock(lock: &dev->mlx5e_res.uplink_netdev_lock); |
353 | } |
354 | EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay); |
355 | |
356 | void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data) |
357 | { |
358 | mlx5_blocking_notifier_call_chain(dev, event, data); |
359 | } |
360 | EXPORT_SYMBOL(mlx5_core_mp_event_replay); |
361 | |
362 | int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, |
363 | enum mlx5_cap_mode cap_mode) |
364 | { |
365 | u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; |
366 | int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); |
367 | void *out, *hca_caps; |
368 | u16 opmod = (cap_type << 1) | (cap_mode & 0x01); |
369 | int err; |
370 | |
371 | memset(in, 0, sizeof(in)); |
372 | out = kzalloc(size: out_sz, GFP_KERNEL); |
373 | if (!out) |
374 | return -ENOMEM; |
375 | |
376 | MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); |
377 | MLX5_SET(query_hca_cap_in, in, op_mod, opmod); |
378 | err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out); |
379 | if (err) { |
380 | mlx5_core_warn(dev, |
381 | "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n" , |
382 | cap_type, cap_mode, err); |
383 | goto query_ex; |
384 | } |
385 | |
386 | hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); |
387 | |
388 | switch (cap_mode) { |
389 | case HCA_CAP_OPMOD_GET_MAX: |
390 | memcpy(dev->caps.hca[cap_type]->max, hca_caps, |
391 | MLX5_UN_SZ_BYTES(hca_cap_union)); |
392 | break; |
393 | case HCA_CAP_OPMOD_GET_CUR: |
394 | memcpy(dev->caps.hca[cap_type]->cur, hca_caps, |
395 | MLX5_UN_SZ_BYTES(hca_cap_union)); |
396 | break; |
397 | default: |
398 | mlx5_core_warn(dev, |
399 | "Tried to query dev cap type(%x) with wrong opmode(%x)\n" , |
400 | cap_type, cap_mode); |
401 | err = -EINVAL; |
402 | break; |
403 | } |
404 | query_ex: |
405 | kfree(objp: out); |
406 | return err; |
407 | } |
408 | |
409 | int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type) |
410 | { |
411 | int ret; |
412 | |
413 | ret = mlx5_core_get_caps_mode(dev, cap_type, cap_mode: HCA_CAP_OPMOD_GET_CUR); |
414 | if (ret) |
415 | return ret; |
416 | return mlx5_core_get_caps_mode(dev, cap_type, cap_mode: HCA_CAP_OPMOD_GET_MAX); |
417 | } |
418 | |
419 | static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod) |
420 | { |
421 | MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); |
422 | MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1); |
423 | return mlx5_cmd_exec_in(dev, set_hca_cap, in); |
424 | } |
425 | |
426 | static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx) |
427 | { |
428 | void *set_hca_cap; |
429 | int req_endianness; |
430 | int err; |
431 | |
432 | if (!MLX5_CAP_GEN(dev, atomic)) |
433 | return 0; |
434 | |
435 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_ATOMIC); |
436 | if (err) |
437 | return err; |
438 | |
439 | req_endianness = |
440 | MLX5_CAP_ATOMIC(dev, |
441 | supported_atomic_req_8B_endianness_mode_1); |
442 | |
443 | if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS) |
444 | return 0; |
445 | |
446 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); |
447 | |
448 | /* Set requestor to host endianness */ |
449 | MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode, |
450 | MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS); |
451 | |
452 | return set_caps(dev, in: set_ctx, opmod: MLX5_SET_HCA_CAP_OP_MOD_ATOMIC); |
453 | } |
454 | |
455 | static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx) |
456 | { |
457 | void *set_hca_cap; |
458 | bool do_set = false; |
459 | int err; |
460 | |
461 | if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) || |
462 | !MLX5_CAP_GEN(dev, pg)) |
463 | return 0; |
464 | |
465 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_ODP); |
466 | if (err) |
467 | return err; |
468 | |
469 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); |
470 | memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur, |
471 | MLX5_ST_SZ_BYTES(odp_cap)); |
472 | |
473 | #define ODP_CAP_SET_MAX(dev, field) \ |
474 | do { \ |
475 | u32 _res = MLX5_CAP_ODP_MAX(dev, field); \ |
476 | if (_res) { \ |
477 | do_set = true; \ |
478 | MLX5_SET(odp_cap, set_hca_cap, field, _res); \ |
479 | } \ |
480 | } while (0) |
481 | |
482 | ODP_CAP_SET_MAX(dev, ud_odp_caps.srq_receive); |
483 | ODP_CAP_SET_MAX(dev, rc_odp_caps.srq_receive); |
484 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.srq_receive); |
485 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.send); |
486 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.receive); |
487 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.write); |
488 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.read); |
489 | ODP_CAP_SET_MAX(dev, xrc_odp_caps.atomic); |
490 | ODP_CAP_SET_MAX(dev, dc_odp_caps.srq_receive); |
491 | ODP_CAP_SET_MAX(dev, dc_odp_caps.send); |
492 | ODP_CAP_SET_MAX(dev, dc_odp_caps.receive); |
493 | ODP_CAP_SET_MAX(dev, dc_odp_caps.write); |
494 | ODP_CAP_SET_MAX(dev, dc_odp_caps.read); |
495 | ODP_CAP_SET_MAX(dev, dc_odp_caps.atomic); |
496 | |
497 | if (!do_set) |
498 | return 0; |
499 | |
500 | return set_caps(dev, in: set_ctx, opmod: MLX5_SET_HCA_CAP_OP_MOD_ODP); |
501 | } |
502 | |
503 | static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) |
504 | { |
505 | struct devlink *devlink = priv_to_devlink(priv: dev); |
506 | union devlink_param_value val; |
507 | int err; |
508 | |
509 | err = devl_param_driverinit_value_get(devlink, |
510 | param_id: DEVLINK_PARAM_GENERIC_ID_MAX_MACS, |
511 | val: &val); |
512 | if (!err) |
513 | return val.vu32; |
514 | mlx5_core_dbg(dev, "Failed to get param. err = %d\n" , err); |
515 | return err; |
516 | } |
517 | |
518 | bool mlx5_is_roce_on(struct mlx5_core_dev *dev) |
519 | { |
520 | struct devlink *devlink = priv_to_devlink(priv: dev); |
521 | union devlink_param_value val; |
522 | int err; |
523 | |
524 | err = devl_param_driverinit_value_get(devlink, |
525 | param_id: DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, |
526 | val: &val); |
527 | |
528 | if (!err) |
529 | return val.vbool; |
530 | |
531 | mlx5_core_dbg(dev, "Failed to get param. err = %d\n" , err); |
532 | return MLX5_CAP_GEN(dev, roce); |
533 | } |
534 | EXPORT_SYMBOL(mlx5_is_roce_on); |
535 | |
536 | static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) |
537 | { |
538 | void *set_hca_cap; |
539 | int err; |
540 | |
541 | if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) |
542 | return 0; |
543 | |
544 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_GENERAL_2); |
545 | if (err) |
546 | return err; |
547 | |
548 | if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) || |
549 | !(dev->priv.sw_vhca_id > 0)) |
550 | return 0; |
551 | |
552 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, |
553 | capability); |
554 | memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, |
555 | MLX5_ST_SZ_BYTES(cmd_hca_cap_2)); |
556 | MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1); |
557 | |
558 | return set_caps(dev, in: set_ctx, opmod: MLX5_CAP_GENERAL_2); |
559 | } |
560 | |
561 | static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) |
562 | { |
563 | struct mlx5_profile *prof = &dev->profile; |
564 | void *set_hca_cap; |
565 | int max_uc_list; |
566 | int err; |
567 | |
568 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_GENERAL); |
569 | if (err) |
570 | return err; |
571 | |
572 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, |
573 | capability); |
574 | memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur, |
575 | MLX5_ST_SZ_BYTES(cmd_hca_cap)); |
576 | |
577 | mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n" , |
578 | mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), |
579 | 128); |
580 | /* we limit the size of the pkey table to 128 entries for now */ |
581 | MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, |
582 | to_fw_pkey_sz(dev, 128)); |
583 | |
584 | /* Check log_max_qp from HCA caps to set in current profile */ |
585 | if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) { |
586 | prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp)); |
587 | } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) { |
588 | mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n" , |
589 | prof->log_max_qp, |
590 | MLX5_CAP_GEN_MAX(dev, log_max_qp)); |
591 | prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); |
592 | } |
593 | if (prof->mask & MLX5_PROF_MASK_QP_SIZE) |
594 | MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, |
595 | prof->log_max_qp); |
596 | |
597 | /* disable cmdif checksum */ |
598 | MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); |
599 | |
600 | /* Enable 4K UAR only when HCA supports it and page size is bigger |
601 | * than 4K. |
602 | */ |
603 | if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096) |
604 | MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1); |
605 | |
606 | MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); |
607 | |
608 | if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte)) |
609 | MLX5_SET(cmd_hca_cap, |
610 | set_hca_cap, |
611 | cache_line_128byte, |
612 | cache_line_size() >= 128 ? 1 : 0); |
613 | |
614 | if (MLX5_CAP_GEN_MAX(dev, dct)) |
615 | MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1); |
616 | |
617 | if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event)) |
618 | MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_event, 1); |
619 | if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload)) |
620 | MLX5_SET(cmd_hca_cap, set_hca_cap, |
621 | pci_sync_for_fw_update_with_driver_unload, 1); |
622 | |
623 | if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports)) |
624 | MLX5_SET(cmd_hca_cap, |
625 | set_hca_cap, |
626 | num_vhca_ports, |
627 | MLX5_CAP_GEN_MAX(dev, num_vhca_ports)); |
628 | |
629 | if (MLX5_CAP_GEN_MAX(dev, release_all_pages)) |
630 | MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1); |
631 | |
632 | if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) |
633 | MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1); |
634 | |
635 | mlx5_vhca_state_cap_handle(dev, set_hca_cap); |
636 | |
637 | if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)) |
638 | MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix, |
639 | MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)); |
640 | |
641 | if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce)) |
642 | MLX5_SET(cmd_hca_cap, set_hca_cap, roce, |
643 | mlx5_is_roce_on(dev)); |
644 | |
645 | max_uc_list = max_uc_list_get_devlink_param(dev); |
646 | if (max_uc_list > 0) |
647 | MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list, |
648 | ilog2(max_uc_list)); |
649 | |
650 | return set_caps(dev, in: set_ctx, opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); |
651 | } |
652 | |
653 | /* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the |
654 | * boot process. |
655 | * In case RoCE cap is writable in FW and user/devlink requested to change the |
656 | * cap, we are yet to query the final state of the above cap. |
657 | * Hence, the need for this function. |
658 | * |
659 | * Returns |
660 | * True: |
661 | * 1) RoCE cap is read only in FW and already disabled |
662 | * OR: |
663 | * 2) RoCE cap is writable in FW and user/devlink requested it off. |
664 | * |
665 | * In any other case, return False. |
666 | */ |
667 | static bool is_roce_fw_disabled(struct mlx5_core_dev *dev) |
668 | { |
669 | return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) || |
670 | (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce)); |
671 | } |
672 | |
673 | static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx) |
674 | { |
675 | void *set_hca_cap; |
676 | int err; |
677 | |
678 | if (is_roce_fw_disabled(dev)) |
679 | return 0; |
680 | |
681 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_ROCE); |
682 | if (err) |
683 | return err; |
684 | |
685 | if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) || |
686 | !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port)) |
687 | return 0; |
688 | |
689 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); |
690 | memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur, |
691 | MLX5_ST_SZ_BYTES(roce_cap)); |
692 | MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1); |
693 | |
694 | if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default)) |
695 | MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1); |
696 | |
697 | err = set_caps(dev, in: set_ctx, opmod: MLX5_SET_HCA_CAP_OP_MOD_ROCE); |
698 | return err; |
699 | } |
700 | |
701 | static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev, |
702 | void *set_ctx) |
703 | { |
704 | void *set_hca_cap; |
705 | int err; |
706 | |
707 | if (!MLX5_CAP_GEN(dev, port_selection_cap)) |
708 | return 0; |
709 | |
710 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_PORT_SELECTION); |
711 | if (err) |
712 | return err; |
713 | |
714 | if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) || |
715 | !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass)) |
716 | return 0; |
717 | |
718 | set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); |
719 | memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur, |
720 | MLX5_ST_SZ_BYTES(port_selection_cap)); |
721 | MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1); |
722 | |
723 | err = set_caps(dev, in: set_ctx, opmod: MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION); |
724 | |
725 | return err; |
726 | } |
727 | |
728 | static int set_hca_cap(struct mlx5_core_dev *dev) |
729 | { |
730 | int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); |
731 | void *set_ctx; |
732 | int err; |
733 | |
734 | set_ctx = kzalloc(size: set_sz, GFP_KERNEL); |
735 | if (!set_ctx) |
736 | return -ENOMEM; |
737 | |
738 | err = handle_hca_cap(dev, set_ctx); |
739 | if (err) { |
740 | mlx5_core_err(dev, "handle_hca_cap failed\n" ); |
741 | goto out; |
742 | } |
743 | |
744 | memset(set_ctx, 0, set_sz); |
745 | err = handle_hca_cap_atomic(dev, set_ctx); |
746 | if (err) { |
747 | mlx5_core_err(dev, "handle_hca_cap_atomic failed\n" ); |
748 | goto out; |
749 | } |
750 | |
751 | memset(set_ctx, 0, set_sz); |
752 | err = handle_hca_cap_odp(dev, set_ctx); |
753 | if (err) { |
754 | mlx5_core_err(dev, "handle_hca_cap_odp failed\n" ); |
755 | goto out; |
756 | } |
757 | |
758 | memset(set_ctx, 0, set_sz); |
759 | err = handle_hca_cap_roce(dev, set_ctx); |
760 | if (err) { |
761 | mlx5_core_err(dev, "handle_hca_cap_roce failed\n" ); |
762 | goto out; |
763 | } |
764 | |
765 | memset(set_ctx, 0, set_sz); |
766 | err = handle_hca_cap_2(dev, set_ctx); |
767 | if (err) { |
768 | mlx5_core_err(dev, "handle_hca_cap_2 failed\n" ); |
769 | goto out; |
770 | } |
771 | |
772 | memset(set_ctx, 0, set_sz); |
773 | err = handle_hca_cap_port_selection(dev, set_ctx); |
774 | if (err) { |
775 | mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n" ); |
776 | goto out; |
777 | } |
778 | |
779 | out: |
780 | kfree(objp: set_ctx); |
781 | return err; |
782 | } |
783 | |
784 | static int set_hca_ctrl(struct mlx5_core_dev *dev) |
785 | { |
786 | struct mlx5_reg_host_endianness he_in; |
787 | struct mlx5_reg_host_endianness he_out; |
788 | int err; |
789 | |
790 | if (!mlx5_core_is_pf(dev)) |
791 | return 0; |
792 | |
793 | memset(&he_in, 0, sizeof(he_in)); |
794 | he_in.he = MLX5_SET_HOST_ENDIANNESS; |
795 | err = mlx5_core_access_reg(dev, data_in: &he_in, size_in: sizeof(he_in), |
796 | data_out: &he_out, size_out: sizeof(he_out), |
797 | reg_num: MLX5_REG_HOST_ENDIANNESS, arg: 0, write: 1); |
798 | return err; |
799 | } |
800 | |
801 | static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev) |
802 | { |
803 | int ret = 0; |
804 | |
805 | /* Disable local_lb by default */ |
806 | if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) |
807 | ret = mlx5_nic_vport_update_local_lb(mdev: dev, enable: false); |
808 | |
809 | return ret; |
810 | } |
811 | |
812 | int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id) |
813 | { |
814 | u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {}; |
815 | |
816 | MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA); |
817 | MLX5_SET(enable_hca_in, in, function_id, func_id); |
818 | MLX5_SET(enable_hca_in, in, embedded_cpu_function, |
819 | dev->caps.embedded_cpu); |
820 | return mlx5_cmd_exec_in(dev, enable_hca, in); |
821 | } |
822 | |
823 | int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id) |
824 | { |
825 | u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {}; |
826 | |
827 | MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA); |
828 | MLX5_SET(disable_hca_in, in, function_id, func_id); |
829 | MLX5_SET(enable_hca_in, in, embedded_cpu_function, |
830 | dev->caps.embedded_cpu); |
831 | return mlx5_cmd_exec_in(dev, disable_hca, in); |
832 | } |
833 | |
834 | static int mlx5_core_set_issi(struct mlx5_core_dev *dev) |
835 | { |
836 | u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {}; |
837 | u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {}; |
838 | u32 sup_issi; |
839 | int err; |
840 | |
841 | MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); |
842 | err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out); |
843 | if (err) { |
844 | u32 syndrome = MLX5_GET(query_issi_out, query_out, syndrome); |
845 | u8 status = MLX5_GET(query_issi_out, query_out, status); |
846 | |
847 | if (!status || syndrome == MLX5_DRIVER_SYND) { |
848 | mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n" , |
849 | err, status, syndrome); |
850 | return err; |
851 | } |
852 | |
853 | mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n" ); |
854 | dev->issi = 0; |
855 | return 0; |
856 | } |
857 | |
858 | sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); |
859 | |
860 | if (sup_issi & (1 << 1)) { |
861 | u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {}; |
862 | |
863 | MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); |
864 | MLX5_SET(set_issi_in, set_in, current_issi, 1); |
865 | err = mlx5_cmd_exec_in(dev, set_issi, set_in); |
866 | if (err) { |
867 | mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n" , |
868 | err); |
869 | return err; |
870 | } |
871 | |
872 | dev->issi = 1; |
873 | |
874 | return 0; |
875 | } else if (sup_issi & (1 << 0) || !sup_issi) { |
876 | return 0; |
877 | } |
878 | |
879 | return -EOPNOTSUPP; |
880 | } |
881 | |
882 | static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, |
883 | const struct pci_device_id *id) |
884 | { |
885 | int err = 0; |
886 | |
887 | mutex_init(&dev->pci_status_mutex); |
888 | pci_set_drvdata(pdev: dev->pdev, data: dev); |
889 | |
890 | dev->bar_addr = pci_resource_start(pdev, 0); |
891 | |
892 | err = mlx5_pci_enable_device(dev); |
893 | if (err) { |
894 | mlx5_core_err(dev, "Cannot enable PCI device, aborting\n" ); |
895 | return err; |
896 | } |
897 | |
898 | err = request_bar(pdev); |
899 | if (err) { |
900 | mlx5_core_err(dev, "error requesting BARs, aborting\n" ); |
901 | goto err_disable; |
902 | } |
903 | |
904 | pci_set_master(dev: pdev); |
905 | |
906 | err = set_dma_caps(pdev); |
907 | if (err) { |
908 | mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n" ); |
909 | goto err_clr_master; |
910 | } |
911 | |
912 | if (pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && |
913 | pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && |
914 | pci_enable_atomic_ops_to_root(dev: pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) |
915 | mlx5_core_dbg(dev, "Enabling pci atomics failed\n" ); |
916 | |
917 | dev->iseg_base = dev->bar_addr; |
918 | dev->iseg = ioremap(offset: dev->iseg_base, size: sizeof(*dev->iseg)); |
919 | if (!dev->iseg) { |
920 | err = -ENOMEM; |
921 | mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n" ); |
922 | goto err_clr_master; |
923 | } |
924 | |
925 | mlx5_pci_vsc_init(dev); |
926 | return 0; |
927 | |
928 | err_clr_master: |
929 | release_bar(pdev: dev->pdev); |
930 | err_disable: |
931 | mlx5_pci_disable_device(dev); |
932 | return err; |
933 | } |
934 | |
935 | static void mlx5_pci_close(struct mlx5_core_dev *dev) |
936 | { |
937 | /* health work might still be active, and it needs pci bar in |
938 | * order to know the NIC state. Therefore, drain the health WQ |
939 | * before removing the pci bars |
940 | */ |
941 | mlx5_drain_health_wq(dev); |
942 | iounmap(addr: dev->iseg); |
943 | release_bar(pdev: dev->pdev); |
944 | mlx5_pci_disable_device(dev); |
945 | } |
946 | |
947 | static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev) |
948 | { |
949 | /* This component is use to sync adding core_dev to lag_dev and to sync |
950 | * changes of mlx5_adev_devices between LAG layer and other layers. |
951 | */ |
952 | if (!mlx5_lag_is_supported(dev)) |
953 | return; |
954 | |
955 | dev->priv.hca_devcom_comp = |
956 | mlx5_devcom_register_component(devc: dev->priv.devc, id: MLX5_DEVCOM_HCA_PORTS, |
957 | key: mlx5_query_nic_system_image_guid(mdev: dev), |
958 | NULL, data: dev); |
959 | if (IS_ERR(ptr: dev->priv.hca_devcom_comp)) |
960 | mlx5_core_err(dev, "Failed to register devcom HCA component\n" ); |
961 | } |
962 | |
963 | static void mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev *dev) |
964 | { |
965 | mlx5_devcom_unregister_component(devcom: dev->priv.hca_devcom_comp); |
966 | } |
967 | |
968 | static int mlx5_init_once(struct mlx5_core_dev *dev) |
969 | { |
970 | int err; |
971 | |
972 | dev->priv.devc = mlx5_devcom_register_device(dev); |
973 | if (IS_ERR(ptr: dev->priv.devc)) |
974 | mlx5_core_warn(dev, "failed to register devcom device %ld\n" , |
975 | PTR_ERR(dev->priv.devc)); |
976 | mlx5_register_hca_devcom_comp(dev); |
977 | |
978 | err = mlx5_query_board_id(dev); |
979 | if (err) { |
980 | mlx5_core_err(dev, "query board id failed\n" ); |
981 | goto err_devcom; |
982 | } |
983 | |
984 | err = mlx5_irq_table_init(dev); |
985 | if (err) { |
986 | mlx5_core_err(dev, "failed to initialize irq table\n" ); |
987 | goto err_devcom; |
988 | } |
989 | |
990 | err = mlx5_eq_table_init(dev); |
991 | if (err) { |
992 | mlx5_core_err(dev, "failed to initialize eq\n" ); |
993 | goto err_irq_cleanup; |
994 | } |
995 | |
996 | err = mlx5_events_init(dev); |
997 | if (err) { |
998 | mlx5_core_err(dev, "failed to initialize events\n" ); |
999 | goto err_eq_cleanup; |
1000 | } |
1001 | |
1002 | err = mlx5_fw_reset_init(dev); |
1003 | if (err) { |
1004 | mlx5_core_err(dev, "failed to initialize fw reset events\n" ); |
1005 | goto err_events_cleanup; |
1006 | } |
1007 | |
1008 | mlx5_cq_debugfs_init(dev); |
1009 | |
1010 | mlx5_init_reserved_gids(dev); |
1011 | |
1012 | mlx5_init_clock(mdev: dev); |
1013 | |
1014 | dev->vxlan = mlx5_vxlan_create(mdev: dev); |
1015 | dev->geneve = mlx5_geneve_create(mdev: dev); |
1016 | |
1017 | err = mlx5_init_rl_table(dev); |
1018 | if (err) { |
1019 | mlx5_core_err(dev, "Failed to init rate limiting\n" ); |
1020 | goto err_tables_cleanup; |
1021 | } |
1022 | |
1023 | err = mlx5_mpfs_init(dev); |
1024 | if (err) { |
1025 | mlx5_core_err(dev, "Failed to init l2 table %d\n" , err); |
1026 | goto err_rl_cleanup; |
1027 | } |
1028 | |
1029 | err = mlx5_sriov_init(dev); |
1030 | if (err) { |
1031 | mlx5_core_err(dev, "Failed to init sriov %d\n" , err); |
1032 | goto err_mpfs_cleanup; |
1033 | } |
1034 | |
1035 | err = mlx5_eswitch_init(dev); |
1036 | if (err) { |
1037 | mlx5_core_err(dev, "Failed to init eswitch %d\n" , err); |
1038 | goto err_sriov_cleanup; |
1039 | } |
1040 | |
1041 | err = mlx5_fpga_init(mdev: dev); |
1042 | if (err) { |
1043 | mlx5_core_err(dev, "Failed to init fpga device %d\n" , err); |
1044 | goto err_eswitch_cleanup; |
1045 | } |
1046 | |
1047 | err = mlx5_vhca_event_init(dev); |
1048 | if (err) { |
1049 | mlx5_core_err(dev, "Failed to init vhca event notifier %d\n" , err); |
1050 | goto err_fpga_cleanup; |
1051 | } |
1052 | |
1053 | err = mlx5_sf_hw_table_init(dev); |
1054 | if (err) { |
1055 | mlx5_core_err(dev, "Failed to init SF HW table %d\n" , err); |
1056 | goto err_sf_hw_table_cleanup; |
1057 | } |
1058 | |
1059 | err = mlx5_sf_table_init(dev); |
1060 | if (err) { |
1061 | mlx5_core_err(dev, "Failed to init SF table %d\n" , err); |
1062 | goto err_sf_table_cleanup; |
1063 | } |
1064 | |
1065 | err = mlx5_fs_core_alloc(dev); |
1066 | if (err) { |
1067 | mlx5_core_err(dev, "Failed to alloc flow steering\n" ); |
1068 | goto err_fs; |
1069 | } |
1070 | |
1071 | dev->dm = mlx5_dm_create(dev); |
1072 | if (IS_ERR(ptr: dev->dm)) |
1073 | mlx5_core_warn(dev, "Failed to init device memory %ld\n" , PTR_ERR(dev->dm)); |
1074 | |
1075 | dev->tracer = mlx5_fw_tracer_create(dev); |
1076 | dev->hv_vhca = mlx5_hv_vhca_create(dev); |
1077 | dev->rsc_dump = mlx5_rsc_dump_create(dev); |
1078 | |
1079 | return 0; |
1080 | |
1081 | err_fs: |
1082 | mlx5_sf_table_cleanup(dev); |
1083 | err_sf_table_cleanup: |
1084 | mlx5_sf_hw_table_cleanup(dev); |
1085 | err_sf_hw_table_cleanup: |
1086 | mlx5_vhca_event_cleanup(dev); |
1087 | err_fpga_cleanup: |
1088 | mlx5_fpga_cleanup(mdev: dev); |
1089 | err_eswitch_cleanup: |
1090 | mlx5_eswitch_cleanup(esw: dev->priv.eswitch); |
1091 | err_sriov_cleanup: |
1092 | mlx5_sriov_cleanup(dev); |
1093 | err_mpfs_cleanup: |
1094 | mlx5_mpfs_cleanup(dev); |
1095 | err_rl_cleanup: |
1096 | mlx5_cleanup_rl_table(dev); |
1097 | err_tables_cleanup: |
1098 | mlx5_geneve_destroy(geneve: dev->geneve); |
1099 | mlx5_vxlan_destroy(vxlan: dev->vxlan); |
1100 | mlx5_cleanup_clock(mdev: dev); |
1101 | mlx5_cleanup_reserved_gids(dev); |
1102 | mlx5_cq_debugfs_cleanup(dev); |
1103 | mlx5_fw_reset_cleanup(dev); |
1104 | err_events_cleanup: |
1105 | mlx5_events_cleanup(dev); |
1106 | err_eq_cleanup: |
1107 | mlx5_eq_table_cleanup(dev); |
1108 | err_irq_cleanup: |
1109 | mlx5_irq_table_cleanup(dev); |
1110 | err_devcom: |
1111 | mlx5_unregister_hca_devcom_comp(dev); |
1112 | mlx5_devcom_unregister_device(devc: dev->priv.devc); |
1113 | |
1114 | return err; |
1115 | } |
1116 | |
1117 | static void mlx5_cleanup_once(struct mlx5_core_dev *dev) |
1118 | { |
1119 | mlx5_rsc_dump_destroy(dev); |
1120 | mlx5_hv_vhca_destroy(hv_vhca: dev->hv_vhca); |
1121 | mlx5_fw_tracer_destroy(tracer: dev->tracer); |
1122 | mlx5_dm_cleanup(dev); |
1123 | mlx5_fs_core_free(dev); |
1124 | mlx5_sf_table_cleanup(dev); |
1125 | mlx5_sf_hw_table_cleanup(dev); |
1126 | mlx5_vhca_event_cleanup(dev); |
1127 | mlx5_fpga_cleanup(mdev: dev); |
1128 | mlx5_eswitch_cleanup(esw: dev->priv.eswitch); |
1129 | mlx5_sriov_cleanup(dev); |
1130 | mlx5_mpfs_cleanup(dev); |
1131 | mlx5_cleanup_rl_table(dev); |
1132 | mlx5_geneve_destroy(geneve: dev->geneve); |
1133 | mlx5_vxlan_destroy(vxlan: dev->vxlan); |
1134 | mlx5_cleanup_clock(mdev: dev); |
1135 | mlx5_cleanup_reserved_gids(dev); |
1136 | mlx5_cq_debugfs_cleanup(dev); |
1137 | mlx5_fw_reset_cleanup(dev); |
1138 | mlx5_events_cleanup(dev); |
1139 | mlx5_eq_table_cleanup(dev); |
1140 | mlx5_irq_table_cleanup(dev); |
1141 | mlx5_unregister_hca_devcom_comp(dev); |
1142 | mlx5_devcom_unregister_device(devc: dev->priv.devc); |
1143 | } |
1144 | |
1145 | static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout) |
1146 | { |
1147 | int err; |
1148 | |
1149 | mlx5_core_info(dev, "firmware version: %d.%d.%d\n" , fw_rev_maj(dev), |
1150 | fw_rev_min(dev), fw_rev_sub(dev)); |
1151 | |
1152 | /* Only PFs hold the relevant PCIe information for this query */ |
1153 | if (mlx5_core_is_pf(dev)) |
1154 | pcie_print_link_status(dev: dev->pdev); |
1155 | |
1156 | /* wait for firmware to accept initialization segments configurations |
1157 | */ |
1158 | err = wait_fw_init(dev, max_wait_mili: timeout, |
1159 | mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL), |
1160 | init_state: "pre-initializing" ); |
1161 | if (err) |
1162 | return err; |
1163 | |
1164 | err = mlx5_cmd_enable(dev); |
1165 | if (err) { |
1166 | mlx5_core_err(dev, "Failed initializing command interface, aborting\n" ); |
1167 | return err; |
1168 | } |
1169 | |
1170 | mlx5_tout_query_iseg(dev); |
1171 | |
1172 | err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), warn_time_mili: 0, init_state: "initializing" ); |
1173 | if (err) |
1174 | goto err_cmd_cleanup; |
1175 | |
1176 | dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev); |
1177 | mlx5_cmd_set_state(dev, cmdif_state: MLX5_CMDIF_STATE_UP); |
1178 | |
1179 | mlx5_start_health_poll(dev); |
1180 | |
1181 | err = mlx5_core_enable_hca(dev, func_id: 0); |
1182 | if (err) { |
1183 | mlx5_core_err(dev, "enable hca failed\n" ); |
1184 | goto stop_health_poll; |
1185 | } |
1186 | |
1187 | err = mlx5_core_set_issi(dev); |
1188 | if (err) { |
1189 | mlx5_core_err(dev, "failed to set issi\n" ); |
1190 | goto err_disable_hca; |
1191 | } |
1192 | |
1193 | err = mlx5_satisfy_startup_pages(dev, boot: 1); |
1194 | if (err) { |
1195 | mlx5_core_err(dev, "failed to allocate boot pages\n" ); |
1196 | goto err_disable_hca; |
1197 | } |
1198 | |
1199 | err = mlx5_tout_query_dtor(dev); |
1200 | if (err) { |
1201 | mlx5_core_err(dev, "failed to read dtor\n" ); |
1202 | goto reclaim_boot_pages; |
1203 | } |
1204 | |
1205 | return 0; |
1206 | |
1207 | reclaim_boot_pages: |
1208 | mlx5_reclaim_startup_pages(dev); |
1209 | err_disable_hca: |
1210 | mlx5_core_disable_hca(dev, func_id: 0); |
1211 | stop_health_poll: |
1212 | mlx5_stop_health_poll(dev, disable_health: boot); |
1213 | err_cmd_cleanup: |
1214 | mlx5_cmd_set_state(dev, cmdif_state: MLX5_CMDIF_STATE_DOWN); |
1215 | mlx5_cmd_disable(dev); |
1216 | |
1217 | return err; |
1218 | } |
1219 | |
1220 | static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot) |
1221 | { |
1222 | mlx5_reclaim_startup_pages(dev); |
1223 | mlx5_core_disable_hca(dev, func_id: 0); |
1224 | mlx5_stop_health_poll(dev, disable_health: boot); |
1225 | mlx5_cmd_set_state(dev, cmdif_state: MLX5_CMDIF_STATE_DOWN); |
1226 | mlx5_cmd_disable(dev); |
1227 | } |
1228 | |
1229 | static int mlx5_function_open(struct mlx5_core_dev *dev) |
1230 | { |
1231 | int err; |
1232 | |
1233 | err = set_hca_ctrl(dev); |
1234 | if (err) { |
1235 | mlx5_core_err(dev, "set_hca_ctrl failed\n" ); |
1236 | return err; |
1237 | } |
1238 | |
1239 | err = set_hca_cap(dev); |
1240 | if (err) { |
1241 | mlx5_core_err(dev, "set_hca_cap failed\n" ); |
1242 | return err; |
1243 | } |
1244 | |
1245 | err = mlx5_satisfy_startup_pages(dev, boot: 0); |
1246 | if (err) { |
1247 | mlx5_core_err(dev, "failed to allocate init pages\n" ); |
1248 | return err; |
1249 | } |
1250 | |
1251 | err = mlx5_cmd_init_hca(dev, sw_owner_id); |
1252 | if (err) { |
1253 | mlx5_core_err(dev, "init hca failed\n" ); |
1254 | return err; |
1255 | } |
1256 | |
1257 | mlx5_set_driver_version(dev); |
1258 | |
1259 | err = mlx5_query_hca_caps(dev); |
1260 | if (err) { |
1261 | mlx5_core_err(dev, "query hca failed\n" ); |
1262 | return err; |
1263 | } |
1264 | mlx5_start_health_fw_log_up(dev); |
1265 | return 0; |
1266 | } |
1267 | |
1268 | static int mlx5_function_close(struct mlx5_core_dev *dev) |
1269 | { |
1270 | int err; |
1271 | |
1272 | err = mlx5_cmd_teardown_hca(dev); |
1273 | if (err) { |
1274 | mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n" ); |
1275 | return err; |
1276 | } |
1277 | |
1278 | return 0; |
1279 | } |
1280 | |
1281 | static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout) |
1282 | { |
1283 | int err; |
1284 | |
1285 | err = mlx5_function_enable(dev, boot, timeout); |
1286 | if (err) |
1287 | return err; |
1288 | |
1289 | err = mlx5_function_open(dev); |
1290 | if (err) |
1291 | mlx5_function_disable(dev, boot); |
1292 | return err; |
1293 | } |
1294 | |
1295 | static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) |
1296 | { |
1297 | int err = mlx5_function_close(dev); |
1298 | |
1299 | if (!err) |
1300 | mlx5_function_disable(dev, boot); |
1301 | return err; |
1302 | } |
1303 | |
1304 | static int mlx5_load(struct mlx5_core_dev *dev) |
1305 | { |
1306 | int err; |
1307 | |
1308 | dev->priv.uar = mlx5_get_uars_page(mdev: dev); |
1309 | if (IS_ERR(ptr: dev->priv.uar)) { |
1310 | mlx5_core_err(dev, "Failed allocating uar, aborting\n" ); |
1311 | err = PTR_ERR(ptr: dev->priv.uar); |
1312 | return err; |
1313 | } |
1314 | |
1315 | mlx5_events_start(dev); |
1316 | mlx5_pagealloc_start(dev); |
1317 | |
1318 | err = mlx5_irq_table_create(dev); |
1319 | if (err) { |
1320 | mlx5_core_err(dev, "Failed to alloc IRQs\n" ); |
1321 | goto err_irq_table; |
1322 | } |
1323 | |
1324 | err = mlx5_eq_table_create(dev); |
1325 | if (err) { |
1326 | mlx5_core_err(dev, "Failed to create EQs\n" ); |
1327 | goto err_eq_table; |
1328 | } |
1329 | |
1330 | err = mlx5_fw_tracer_init(tracer: dev->tracer); |
1331 | if (err) { |
1332 | mlx5_core_err(dev, "Failed to init FW tracer %d\n" , err); |
1333 | mlx5_fw_tracer_destroy(tracer: dev->tracer); |
1334 | dev->tracer = NULL; |
1335 | } |
1336 | |
1337 | mlx5_fw_reset_events_start(dev); |
1338 | mlx5_hv_vhca_init(hv_vhca: dev->hv_vhca); |
1339 | |
1340 | err = mlx5_rsc_dump_init(dev); |
1341 | if (err) { |
1342 | mlx5_core_err(dev, "Failed to init Resource dump %d\n" , err); |
1343 | mlx5_rsc_dump_destroy(dev); |
1344 | dev->rsc_dump = NULL; |
1345 | } |
1346 | |
1347 | err = mlx5_fpga_device_start(mdev: dev); |
1348 | if (err) { |
1349 | mlx5_core_err(dev, "fpga device start failed %d\n" , err); |
1350 | goto err_fpga_start; |
1351 | } |
1352 | |
1353 | err = mlx5_fs_core_init(dev); |
1354 | if (err) { |
1355 | mlx5_core_err(dev, "Failed to init flow steering\n" ); |
1356 | goto err_fs; |
1357 | } |
1358 | |
1359 | err = mlx5_core_set_hca_defaults(dev); |
1360 | if (err) { |
1361 | mlx5_core_err(dev, "Failed to set hca defaults\n" ); |
1362 | goto err_set_hca; |
1363 | } |
1364 | |
1365 | mlx5_vhca_event_start(dev); |
1366 | |
1367 | err = mlx5_sf_hw_table_create(dev); |
1368 | if (err) { |
1369 | mlx5_core_err(dev, "sf table create failed %d\n" , err); |
1370 | goto err_vhca; |
1371 | } |
1372 | |
1373 | err = mlx5_ec_init(dev); |
1374 | if (err) { |
1375 | mlx5_core_err(dev, "Failed to init embedded CPU\n" ); |
1376 | goto err_ec; |
1377 | } |
1378 | |
1379 | mlx5_lag_add_mdev(dev); |
1380 | err = mlx5_sriov_attach(dev); |
1381 | if (err) { |
1382 | mlx5_core_err(dev, "sriov init failed %d\n" , err); |
1383 | goto err_sriov; |
1384 | } |
1385 | |
1386 | mlx5_sf_dev_table_create(dev); |
1387 | |
1388 | err = mlx5_devlink_traps_register(devlink: priv_to_devlink(priv: dev)); |
1389 | if (err) |
1390 | goto err_traps_reg; |
1391 | |
1392 | return 0; |
1393 | |
1394 | err_traps_reg: |
1395 | mlx5_sf_dev_table_destroy(dev); |
1396 | mlx5_sriov_detach(dev); |
1397 | err_sriov: |
1398 | mlx5_lag_remove_mdev(dev); |
1399 | mlx5_ec_cleanup(dev); |
1400 | err_ec: |
1401 | mlx5_sf_hw_table_destroy(dev); |
1402 | err_vhca: |
1403 | mlx5_vhca_event_stop(dev); |
1404 | err_set_hca: |
1405 | mlx5_fs_core_cleanup(dev); |
1406 | err_fs: |
1407 | mlx5_fpga_device_stop(mdev: dev); |
1408 | err_fpga_start: |
1409 | mlx5_rsc_dump_cleanup(dev); |
1410 | mlx5_hv_vhca_cleanup(hv_vhca: dev->hv_vhca); |
1411 | mlx5_fw_reset_events_stop(dev); |
1412 | mlx5_fw_tracer_cleanup(tracer: dev->tracer); |
1413 | mlx5_eq_table_destroy(dev); |
1414 | err_eq_table: |
1415 | mlx5_irq_table_destroy(dev); |
1416 | err_irq_table: |
1417 | mlx5_pagealloc_stop(dev); |
1418 | mlx5_events_stop(dev); |
1419 | mlx5_put_uars_page(mdev: dev, up: dev->priv.uar); |
1420 | return err; |
1421 | } |
1422 | |
1423 | static void mlx5_unload(struct mlx5_core_dev *dev) |
1424 | { |
1425 | mlx5_eswitch_disable(esw: dev->priv.eswitch); |
1426 | mlx5_devlink_traps_unregister(devlink: priv_to_devlink(priv: dev)); |
1427 | mlx5_sf_dev_table_destroy(dev); |
1428 | mlx5_sriov_detach(dev); |
1429 | mlx5_lag_remove_mdev(dev); |
1430 | mlx5_ec_cleanup(dev); |
1431 | mlx5_sf_hw_table_destroy(dev); |
1432 | mlx5_vhca_event_stop(dev); |
1433 | mlx5_fs_core_cleanup(dev); |
1434 | mlx5_fpga_device_stop(mdev: dev); |
1435 | mlx5_rsc_dump_cleanup(dev); |
1436 | mlx5_hv_vhca_cleanup(hv_vhca: dev->hv_vhca); |
1437 | mlx5_fw_reset_events_stop(dev); |
1438 | mlx5_fw_tracer_cleanup(tracer: dev->tracer); |
1439 | mlx5_eq_table_destroy(dev); |
1440 | mlx5_irq_table_destroy(dev); |
1441 | mlx5_pagealloc_stop(dev); |
1442 | mlx5_events_stop(dev); |
1443 | mlx5_put_uars_page(mdev: dev, up: dev->priv.uar); |
1444 | } |
1445 | |
1446 | int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev) |
1447 | { |
1448 | bool light_probe = mlx5_dev_is_lightweight(dev); |
1449 | int err = 0; |
1450 | |
1451 | mutex_lock(&dev->intf_state_mutex); |
1452 | dev->state = MLX5_DEVICE_STATE_UP; |
1453 | |
1454 | err = mlx5_function_setup(dev, boot: true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); |
1455 | if (err) |
1456 | goto err_function; |
1457 | |
1458 | err = mlx5_init_once(dev); |
1459 | if (err) { |
1460 | mlx5_core_err(dev, "sw objs init failed\n" ); |
1461 | goto function_teardown; |
1462 | } |
1463 | |
1464 | /* In case of light_probe, mlx5_devlink is already registered. |
1465 | * Hence, don't register devlink again. |
1466 | */ |
1467 | if (!light_probe) { |
1468 | err = mlx5_devlink_params_register(devlink: priv_to_devlink(priv: dev)); |
1469 | if (err) |
1470 | goto err_devlink_params_reg; |
1471 | } |
1472 | |
1473 | err = mlx5_load(dev); |
1474 | if (err) |
1475 | goto err_load; |
1476 | |
1477 | set_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1478 | |
1479 | err = mlx5_register_device(dev); |
1480 | if (err) |
1481 | goto err_register; |
1482 | |
1483 | err = mlx5_crdump_enable(dev); |
1484 | if (err) |
1485 | mlx5_core_err(dev, "mlx5_crdump_enable failed with error code %d\n" , err); |
1486 | |
1487 | err = mlx5_hwmon_dev_register(mdev: dev); |
1488 | if (err) |
1489 | mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n" , err); |
1490 | |
1491 | mutex_unlock(lock: &dev->intf_state_mutex); |
1492 | return 0; |
1493 | |
1494 | err_register: |
1495 | clear_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1496 | mlx5_unload(dev); |
1497 | err_load: |
1498 | if (!light_probe) |
1499 | mlx5_devlink_params_unregister(devlink: priv_to_devlink(priv: dev)); |
1500 | err_devlink_params_reg: |
1501 | mlx5_cleanup_once(dev); |
1502 | function_teardown: |
1503 | mlx5_function_teardown(dev, boot: true); |
1504 | err_function: |
1505 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
1506 | mutex_unlock(lock: &dev->intf_state_mutex); |
1507 | return err; |
1508 | } |
1509 | |
1510 | int mlx5_init_one(struct mlx5_core_dev *dev) |
1511 | { |
1512 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1513 | int err; |
1514 | |
1515 | devl_lock(devlink); |
1516 | devl_register(devlink); |
1517 | err = mlx5_init_one_devl_locked(dev); |
1518 | if (err) |
1519 | devl_unregister(devlink); |
1520 | devl_unlock(devlink); |
1521 | return err; |
1522 | } |
1523 | |
1524 | void mlx5_uninit_one(struct mlx5_core_dev *dev) |
1525 | { |
1526 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1527 | |
1528 | devl_lock(devlink); |
1529 | mutex_lock(&dev->intf_state_mutex); |
1530 | |
1531 | mlx5_hwmon_dev_unregister(mdev: dev); |
1532 | mlx5_crdump_disable(dev); |
1533 | mlx5_unregister_device(dev); |
1534 | |
1535 | if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { |
1536 | mlx5_core_warn(dev, "%s: interface is down, NOP\n" , |
1537 | __func__); |
1538 | mlx5_devlink_params_unregister(devlink: priv_to_devlink(priv: dev)); |
1539 | mlx5_cleanup_once(dev); |
1540 | goto out; |
1541 | } |
1542 | |
1543 | clear_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1544 | mlx5_unload(dev); |
1545 | mlx5_devlink_params_unregister(devlink: priv_to_devlink(priv: dev)); |
1546 | mlx5_cleanup_once(dev); |
1547 | mlx5_function_teardown(dev, boot: true); |
1548 | out: |
1549 | mutex_unlock(lock: &dev->intf_state_mutex); |
1550 | devl_unregister(devlink); |
1551 | devl_unlock(devlink); |
1552 | } |
1553 | |
1554 | int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) |
1555 | { |
1556 | int err = 0; |
1557 | u64 timeout; |
1558 | |
1559 | devl_assert_locked(devlink: priv_to_devlink(priv: dev)); |
1560 | mutex_lock(&dev->intf_state_mutex); |
1561 | if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { |
1562 | mlx5_core_warn(dev, "interface is up, NOP\n" ); |
1563 | goto out; |
1564 | } |
1565 | /* remove any previous indication of internal error */ |
1566 | dev->state = MLX5_DEVICE_STATE_UP; |
1567 | |
1568 | if (recovery) |
1569 | timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT); |
1570 | else |
1571 | timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT); |
1572 | err = mlx5_function_setup(dev, boot: false, timeout); |
1573 | if (err) |
1574 | goto err_function; |
1575 | |
1576 | err = mlx5_load(dev); |
1577 | if (err) |
1578 | goto err_load; |
1579 | |
1580 | set_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1581 | |
1582 | err = mlx5_attach_device(dev); |
1583 | if (err) |
1584 | goto err_attach; |
1585 | |
1586 | mutex_unlock(lock: &dev->intf_state_mutex); |
1587 | return 0; |
1588 | |
1589 | err_attach: |
1590 | clear_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1591 | mlx5_unload(dev); |
1592 | err_load: |
1593 | mlx5_function_teardown(dev, boot: false); |
1594 | err_function: |
1595 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
1596 | out: |
1597 | mutex_unlock(lock: &dev->intf_state_mutex); |
1598 | return err; |
1599 | } |
1600 | |
1601 | int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) |
1602 | { |
1603 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1604 | int ret; |
1605 | |
1606 | devl_lock(devlink); |
1607 | ret = mlx5_load_one_devl_locked(dev, recovery); |
1608 | devl_unlock(devlink); |
1609 | return ret; |
1610 | } |
1611 | |
1612 | void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend) |
1613 | { |
1614 | devl_assert_locked(devlink: priv_to_devlink(priv: dev)); |
1615 | mutex_lock(&dev->intf_state_mutex); |
1616 | |
1617 | mlx5_detach_device(dev, suspend); |
1618 | |
1619 | if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { |
1620 | mlx5_core_warn(dev, "%s: interface is down, NOP\n" , |
1621 | __func__); |
1622 | goto out; |
1623 | } |
1624 | |
1625 | clear_bit(nr: MLX5_INTERFACE_STATE_UP, addr: &dev->intf_state); |
1626 | mlx5_unload(dev); |
1627 | mlx5_function_teardown(dev, boot: false); |
1628 | out: |
1629 | mutex_unlock(lock: &dev->intf_state_mutex); |
1630 | } |
1631 | |
1632 | void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend) |
1633 | { |
1634 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1635 | |
1636 | devl_lock(devlink); |
1637 | mlx5_unload_one_devl_locked(dev, suspend); |
1638 | devl_unlock(devlink); |
1639 | } |
1640 | |
1641 | /* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps. |
1642 | * A full query of hca_caps will be done when the device will reload. |
1643 | */ |
1644 | static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev) |
1645 | { |
1646 | int err; |
1647 | |
1648 | err = mlx5_core_get_caps(dev, cap_type: MLX5_CAP_GENERAL); |
1649 | if (err) |
1650 | return err; |
1651 | |
1652 | if (MLX5_CAP_GEN(dev, eth_net_offloads)) { |
1653 | err = mlx5_core_get_caps_mode(dev, cap_type: MLX5_CAP_ETHERNET_OFFLOADS, |
1654 | cap_mode: HCA_CAP_OPMOD_GET_CUR); |
1655 | if (err) |
1656 | return err; |
1657 | } |
1658 | |
1659 | if (MLX5_CAP_GEN(dev, nic_flow_table) || |
1660 | MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) { |
1661 | err = mlx5_core_get_caps_mode(dev, cap_type: MLX5_CAP_FLOW_TABLE, |
1662 | cap_mode: HCA_CAP_OPMOD_GET_CUR); |
1663 | if (err) |
1664 | return err; |
1665 | } |
1666 | |
1667 | if (MLX5_CAP_GEN_64(dev, general_obj_types) & |
1668 | MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) { |
1669 | err = mlx5_core_get_caps_mode(dev, cap_type: MLX5_CAP_VDPA_EMULATION, |
1670 | cap_mode: HCA_CAP_OPMOD_GET_CUR); |
1671 | if (err) |
1672 | return err; |
1673 | } |
1674 | |
1675 | return 0; |
1676 | } |
1677 | |
1678 | int mlx5_init_one_light(struct mlx5_core_dev *dev) |
1679 | { |
1680 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1681 | int err; |
1682 | |
1683 | dev->state = MLX5_DEVICE_STATE_UP; |
1684 | err = mlx5_function_enable(dev, boot: true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); |
1685 | if (err) { |
1686 | mlx5_core_warn(dev, "mlx5_function_enable err=%d\n" , err); |
1687 | goto out; |
1688 | } |
1689 | |
1690 | err = mlx5_query_hca_caps_light(dev); |
1691 | if (err) { |
1692 | mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n" , err); |
1693 | goto query_hca_caps_err; |
1694 | } |
1695 | |
1696 | devl_lock(devlink); |
1697 | devl_register(devlink); |
1698 | |
1699 | err = mlx5_devlink_params_register(devlink: priv_to_devlink(priv: dev)); |
1700 | if (err) { |
1701 | mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n" , err); |
1702 | goto params_reg_err; |
1703 | } |
1704 | |
1705 | devl_unlock(devlink); |
1706 | return 0; |
1707 | |
1708 | params_reg_err: |
1709 | devl_unregister(devlink); |
1710 | devl_unlock(devlink); |
1711 | query_hca_caps_err: |
1712 | devl_unregister(devlink); |
1713 | devl_unlock(devlink); |
1714 | mlx5_function_disable(dev, boot: true); |
1715 | out: |
1716 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
1717 | return err; |
1718 | } |
1719 | |
1720 | void mlx5_uninit_one_light(struct mlx5_core_dev *dev) |
1721 | { |
1722 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1723 | |
1724 | devl_lock(devlink); |
1725 | mlx5_devlink_params_unregister(devlink: priv_to_devlink(priv: dev)); |
1726 | devl_unregister(devlink); |
1727 | devl_unlock(devlink); |
1728 | if (dev->state != MLX5_DEVICE_STATE_UP) |
1729 | return; |
1730 | mlx5_function_disable(dev, boot: true); |
1731 | } |
1732 | |
1733 | /* xxx_light() function are used in order to configure the device without full |
1734 | * init (light init). e.g.: There isn't a point in reload a device to light state. |
1735 | * Hence, mlx5_load_one_light() isn't needed. |
1736 | */ |
1737 | |
1738 | void mlx5_unload_one_light(struct mlx5_core_dev *dev) |
1739 | { |
1740 | if (dev->state != MLX5_DEVICE_STATE_UP) |
1741 | return; |
1742 | mlx5_function_disable(dev, boot: false); |
1743 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
1744 | } |
1745 | |
1746 | static const int types[] = { |
1747 | MLX5_CAP_GENERAL, |
1748 | MLX5_CAP_GENERAL_2, |
1749 | MLX5_CAP_ETHERNET_OFFLOADS, |
1750 | MLX5_CAP_IPOIB_ENHANCED_OFFLOADS, |
1751 | MLX5_CAP_ODP, |
1752 | MLX5_CAP_ATOMIC, |
1753 | MLX5_CAP_ROCE, |
1754 | MLX5_CAP_IPOIB_OFFLOADS, |
1755 | MLX5_CAP_FLOW_TABLE, |
1756 | MLX5_CAP_ESWITCH_FLOW_TABLE, |
1757 | MLX5_CAP_ESWITCH, |
1758 | MLX5_CAP_QOS, |
1759 | MLX5_CAP_DEBUG, |
1760 | MLX5_CAP_DEV_MEM, |
1761 | MLX5_CAP_DEV_EVENT, |
1762 | MLX5_CAP_TLS, |
1763 | MLX5_CAP_VDPA_EMULATION, |
1764 | MLX5_CAP_IPSEC, |
1765 | MLX5_CAP_PORT_SELECTION, |
1766 | MLX5_CAP_MACSEC, |
1767 | MLX5_CAP_ADV_VIRTUALIZATION, |
1768 | MLX5_CAP_CRYPTO, |
1769 | }; |
1770 | |
1771 | static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) |
1772 | { |
1773 | int type; |
1774 | int i; |
1775 | |
1776 | for (i = 0; i < ARRAY_SIZE(types); i++) { |
1777 | type = types[i]; |
1778 | kfree(objp: dev->caps.hca[type]); |
1779 | } |
1780 | } |
1781 | |
1782 | static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev) |
1783 | { |
1784 | struct mlx5_hca_cap *cap; |
1785 | int type; |
1786 | int i; |
1787 | |
1788 | for (i = 0; i < ARRAY_SIZE(types); i++) { |
1789 | cap = kzalloc(size: sizeof(*cap), GFP_KERNEL); |
1790 | if (!cap) |
1791 | goto err; |
1792 | type = types[i]; |
1793 | dev->caps.hca[type] = cap; |
1794 | } |
1795 | |
1796 | return 0; |
1797 | |
1798 | err: |
1799 | mlx5_hca_caps_free(dev); |
1800 | return -ENOMEM; |
1801 | } |
1802 | |
1803 | static int vhca_id_show(struct seq_file *file, void *priv) |
1804 | { |
1805 | struct mlx5_core_dev *dev = file->private; |
1806 | |
1807 | seq_printf(m: file, fmt: "0x%x\n" , MLX5_CAP_GEN(dev, vhca_id)); |
1808 | return 0; |
1809 | } |
1810 | |
1811 | DEFINE_SHOW_ATTRIBUTE(vhca_id); |
1812 | |
1813 | int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) |
1814 | { |
1815 | struct mlx5_priv *priv = &dev->priv; |
1816 | int err; |
1817 | |
1818 | memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); |
1819 | lockdep_register_key(key: &dev->lock_key); |
1820 | mutex_init(&dev->intf_state_mutex); |
1821 | lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); |
1822 | mutex_init(&dev->mlx5e_res.uplink_netdev_lock); |
1823 | |
1824 | mutex_init(&priv->bfregs.reg_head.lock); |
1825 | mutex_init(&priv->bfregs.wc_head.lock); |
1826 | INIT_LIST_HEAD(list: &priv->bfregs.reg_head.list); |
1827 | INIT_LIST_HEAD(list: &priv->bfregs.wc_head.list); |
1828 | |
1829 | mutex_init(&priv->alloc_mutex); |
1830 | mutex_init(&priv->pgdir_mutex); |
1831 | INIT_LIST_HEAD(list: &priv->pgdir_list); |
1832 | |
1833 | priv->numa_node = dev_to_node(dev: mlx5_core_dma_dev(dev)); |
1834 | priv->dbg.dbg_root = debugfs_create_dir(name: dev_name(dev: dev->device), |
1835 | parent: mlx5_debugfs_root); |
1836 | debugfs_create_file(name: "vhca_id" , mode: 0400, parent: priv->dbg.dbg_root, data: dev, fops: &vhca_id_fops); |
1837 | INIT_LIST_HEAD(list: &priv->traps); |
1838 | |
1839 | err = mlx5_cmd_init(dev); |
1840 | if (err) { |
1841 | mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n" ); |
1842 | goto err_cmd_init; |
1843 | } |
1844 | |
1845 | err = mlx5_tout_init(dev); |
1846 | if (err) { |
1847 | mlx5_core_err(dev, "Failed initializing timeouts, aborting\n" ); |
1848 | goto err_timeout_init; |
1849 | } |
1850 | |
1851 | err = mlx5_health_init(dev); |
1852 | if (err) |
1853 | goto err_health_init; |
1854 | |
1855 | err = mlx5_pagealloc_init(dev); |
1856 | if (err) |
1857 | goto err_pagealloc_init; |
1858 | |
1859 | err = mlx5_adev_init(dev); |
1860 | if (err) |
1861 | goto err_adev_init; |
1862 | |
1863 | err = mlx5_hca_caps_alloc(dev); |
1864 | if (err) |
1865 | goto err_hca_caps; |
1866 | |
1867 | /* The conjunction of sw_vhca_id with sw_owner_id will be a global |
1868 | * unique id per function which uses mlx5_core. |
1869 | * Those values are supplied to FW as part of the init HCA command to |
1870 | * be used by both driver and FW when it's applicable. |
1871 | */ |
1872 | dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, min: 1, |
1873 | MAX_SW_VHCA_ID, |
1874 | GFP_KERNEL); |
1875 | if (dev->priv.sw_vhca_id < 0) |
1876 | mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n" , |
1877 | dev->priv.sw_vhca_id); |
1878 | |
1879 | return 0; |
1880 | |
1881 | err_hca_caps: |
1882 | mlx5_adev_cleanup(dev); |
1883 | err_adev_init: |
1884 | mlx5_pagealloc_cleanup(dev); |
1885 | err_pagealloc_init: |
1886 | mlx5_health_cleanup(dev); |
1887 | err_health_init: |
1888 | mlx5_tout_cleanup(dev); |
1889 | err_timeout_init: |
1890 | mlx5_cmd_cleanup(dev); |
1891 | err_cmd_init: |
1892 | debugfs_remove(dentry: dev->priv.dbg.dbg_root); |
1893 | mutex_destroy(lock: &priv->pgdir_mutex); |
1894 | mutex_destroy(lock: &priv->alloc_mutex); |
1895 | mutex_destroy(lock: &priv->bfregs.wc_head.lock); |
1896 | mutex_destroy(lock: &priv->bfregs.reg_head.lock); |
1897 | mutex_destroy(lock: &dev->intf_state_mutex); |
1898 | lockdep_unregister_key(key: &dev->lock_key); |
1899 | return err; |
1900 | } |
1901 | |
1902 | void mlx5_mdev_uninit(struct mlx5_core_dev *dev) |
1903 | { |
1904 | struct mlx5_priv *priv = &dev->priv; |
1905 | |
1906 | if (priv->sw_vhca_id > 0) |
1907 | ida_free(&sw_vhca_ida, id: dev->priv.sw_vhca_id); |
1908 | |
1909 | mlx5_hca_caps_free(dev); |
1910 | mlx5_adev_cleanup(dev); |
1911 | mlx5_pagealloc_cleanup(dev); |
1912 | mlx5_health_cleanup(dev); |
1913 | mlx5_tout_cleanup(dev); |
1914 | mlx5_cmd_cleanup(dev); |
1915 | debugfs_remove_recursive(dentry: dev->priv.dbg.dbg_root); |
1916 | mutex_destroy(lock: &priv->pgdir_mutex); |
1917 | mutex_destroy(lock: &priv->alloc_mutex); |
1918 | mutex_destroy(lock: &priv->bfregs.wc_head.lock); |
1919 | mutex_destroy(lock: &priv->bfregs.reg_head.lock); |
1920 | mutex_destroy(lock: &dev->mlx5e_res.uplink_netdev_lock); |
1921 | mutex_destroy(lock: &dev->intf_state_mutex); |
1922 | lockdep_unregister_key(key: &dev->lock_key); |
1923 | } |
1924 | |
1925 | static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) |
1926 | { |
1927 | struct mlx5_core_dev *dev; |
1928 | struct devlink *devlink; |
1929 | int err; |
1930 | |
1931 | devlink = mlx5_devlink_alloc(dev: &pdev->dev); |
1932 | if (!devlink) { |
1933 | dev_err(&pdev->dev, "devlink alloc failed\n" ); |
1934 | return -ENOMEM; |
1935 | } |
1936 | |
1937 | dev = devlink_priv(devlink); |
1938 | dev->device = &pdev->dev; |
1939 | dev->pdev = pdev; |
1940 | |
1941 | dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ? |
1942 | MLX5_COREDEV_VF : MLX5_COREDEV_PF; |
1943 | |
1944 | dev->priv.adev_idx = mlx5_adev_idx_alloc(); |
1945 | if (dev->priv.adev_idx < 0) { |
1946 | err = dev->priv.adev_idx; |
1947 | goto adev_init_err; |
1948 | } |
1949 | |
1950 | err = mlx5_mdev_init(dev, profile_idx: prof_sel); |
1951 | if (err) |
1952 | goto mdev_init_err; |
1953 | |
1954 | err = mlx5_pci_init(dev, pdev, id); |
1955 | if (err) { |
1956 | mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n" , |
1957 | err); |
1958 | goto pci_init_err; |
1959 | } |
1960 | |
1961 | err = mlx5_init_one(dev); |
1962 | if (err) { |
1963 | mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n" , |
1964 | err); |
1965 | goto err_init_one; |
1966 | } |
1967 | |
1968 | pci_save_state(dev: pdev); |
1969 | return 0; |
1970 | |
1971 | err_init_one: |
1972 | mlx5_pci_close(dev); |
1973 | pci_init_err: |
1974 | mlx5_mdev_uninit(dev); |
1975 | mdev_init_err: |
1976 | mlx5_adev_idx_free(idx: dev->priv.adev_idx); |
1977 | adev_init_err: |
1978 | mlx5_devlink_free(devlink); |
1979 | |
1980 | return err; |
1981 | } |
1982 | |
1983 | static void remove_one(struct pci_dev *pdev) |
1984 | { |
1985 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
1986 | struct devlink *devlink = priv_to_devlink(priv: dev); |
1987 | |
1988 | set_bit(nr: MLX5_BREAK_FW_WAIT, addr: &dev->intf_state); |
1989 | mlx5_drain_fw_reset(dev); |
1990 | mlx5_drain_health_wq(dev); |
1991 | mlx5_sriov_disable(pdev, num_vf_change: false); |
1992 | mlx5_uninit_one(dev); |
1993 | mlx5_pci_close(dev); |
1994 | mlx5_mdev_uninit(dev); |
1995 | mlx5_adev_idx_free(idx: dev->priv.adev_idx); |
1996 | mlx5_devlink_free(devlink); |
1997 | } |
1998 | |
1999 | #define mlx5_pci_trace(dev, fmt, ...) ({ \ |
2000 | struct mlx5_core_dev *__dev = (dev); \ |
2001 | mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \ |
2002 | __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \ |
2003 | __dev->pci_status, ##__VA_ARGS__); \ |
2004 | }) |
2005 | |
2006 | static const char *result2str(enum pci_ers_result result) |
2007 | { |
2008 | return result == PCI_ERS_RESULT_NEED_RESET ? "need reset" : |
2009 | result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" : |
2010 | result == PCI_ERS_RESULT_RECOVERED ? "recovered" : |
2011 | "unknown" ; |
2012 | } |
2013 | |
2014 | static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, |
2015 | pci_channel_state_t state) |
2016 | { |
2017 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2018 | enum pci_ers_result res; |
2019 | |
2020 | mlx5_pci_trace(dev, "Enter, pci channel state = %d\n" , state); |
2021 | |
2022 | mlx5_enter_error_state(dev, force: false); |
2023 | mlx5_error_sw_reset(dev); |
2024 | mlx5_unload_one(dev, suspend: false); |
2025 | mlx5_drain_health_wq(dev); |
2026 | mlx5_pci_disable_device(dev); |
2027 | |
2028 | res = state == pci_channel_io_perm_failure ? |
2029 | PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; |
2030 | |
2031 | mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n" , |
2032 | __func__, dev->state, dev->pci_status, res, result2str(res)); |
2033 | return res; |
2034 | } |
2035 | |
2036 | /* wait for the device to show vital signs by waiting |
2037 | * for the health counter to start counting. |
2038 | */ |
2039 | static int wait_vital(struct pci_dev *pdev) |
2040 | { |
2041 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2042 | struct mlx5_core_health *health = &dev->priv.health; |
2043 | const int niter = 100; |
2044 | u32 last_count = 0; |
2045 | u32 count; |
2046 | int i; |
2047 | |
2048 | for (i = 0; i < niter; i++) { |
2049 | count = ioread32be(health->health_counter); |
2050 | if (count && count != 0xffffffff) { |
2051 | if (last_count && last_count != count) { |
2052 | mlx5_core_info(dev, |
2053 | "wait vital counter value 0x%x after %d iterations\n" , |
2054 | count, i); |
2055 | return 0; |
2056 | } |
2057 | last_count = count; |
2058 | } |
2059 | msleep(msecs: 50); |
2060 | } |
2061 | |
2062 | return -ETIMEDOUT; |
2063 | } |
2064 | |
2065 | static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) |
2066 | { |
2067 | enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT; |
2068 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2069 | int err; |
2070 | |
2071 | mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n" , |
2072 | __func__, dev->state, dev->pci_status); |
2073 | |
2074 | err = mlx5_pci_enable_device(dev); |
2075 | if (err) { |
2076 | mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" , |
2077 | __func__, err); |
2078 | goto out; |
2079 | } |
2080 | |
2081 | pci_set_master(dev: pdev); |
2082 | pci_restore_state(dev: pdev); |
2083 | pci_save_state(dev: pdev); |
2084 | |
2085 | err = wait_vital(pdev); |
2086 | if (err) { |
2087 | mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n" , |
2088 | __func__, err); |
2089 | goto out; |
2090 | } |
2091 | |
2092 | res = PCI_ERS_RESULT_RECOVERED; |
2093 | out: |
2094 | mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n" , |
2095 | __func__, dev->state, dev->pci_status, err, res, result2str(res)); |
2096 | return res; |
2097 | } |
2098 | |
2099 | static void mlx5_pci_resume(struct pci_dev *pdev) |
2100 | { |
2101 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2102 | int err; |
2103 | |
2104 | mlx5_pci_trace(dev, "Enter, loading driver..\n" ); |
2105 | |
2106 | err = mlx5_load_one(dev, recovery: false); |
2107 | |
2108 | if (!err) |
2109 | devlink_health_reporter_state_update(reporter: dev->priv.health.fw_fatal_reporter, |
2110 | state: DEVLINK_HEALTH_REPORTER_STATE_HEALTHY); |
2111 | |
2112 | mlx5_pci_trace(dev, "Done, err = %d, device %s\n" , err, |
2113 | !err ? "recovered" : "Failed" ); |
2114 | } |
2115 | |
2116 | static const struct pci_error_handlers mlx5_err_handler = { |
2117 | .error_detected = mlx5_pci_err_detected, |
2118 | .slot_reset = mlx5_pci_slot_reset, |
2119 | .resume = mlx5_pci_resume |
2120 | }; |
2121 | |
2122 | static int mlx5_try_fast_unload(struct mlx5_core_dev *dev) |
2123 | { |
2124 | bool fast_teardown = false, force_teardown = false; |
2125 | int ret = 1; |
2126 | |
2127 | fast_teardown = MLX5_CAP_GEN(dev, fast_teardown); |
2128 | force_teardown = MLX5_CAP_GEN(dev, force_teardown); |
2129 | |
2130 | mlx5_core_dbg(dev, "force teardown firmware support=%d\n" , force_teardown); |
2131 | mlx5_core_dbg(dev, "fast teardown firmware support=%d\n" , fast_teardown); |
2132 | |
2133 | if (!fast_teardown && !force_teardown) |
2134 | return -EOPNOTSUPP; |
2135 | |
2136 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
2137 | mlx5_core_dbg(dev, "Device in internal error state, giving up\n" ); |
2138 | return -EAGAIN; |
2139 | } |
2140 | |
2141 | /* Panic tear down fw command will stop the PCI bus communication |
2142 | * with the HCA, so the health poll is no longer needed. |
2143 | */ |
2144 | mlx5_drain_health_wq(dev); |
2145 | mlx5_stop_health_poll(dev, disable_health: false); |
2146 | |
2147 | ret = mlx5_cmd_fast_teardown_hca(dev); |
2148 | if (!ret) |
2149 | goto succeed; |
2150 | |
2151 | ret = mlx5_cmd_force_teardown_hca(dev); |
2152 | if (!ret) |
2153 | goto succeed; |
2154 | |
2155 | mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n" , ret); |
2156 | mlx5_start_health_poll(dev); |
2157 | return ret; |
2158 | |
2159 | succeed: |
2160 | mlx5_enter_error_state(dev, force: true); |
2161 | |
2162 | /* Some platforms requiring freeing the IRQ's in the shutdown |
2163 | * flow. If they aren't freed they can't be allocated after |
2164 | * kexec. There is no need to cleanup the mlx5_core software |
2165 | * contexts. |
2166 | */ |
2167 | mlx5_core_eq_free_irqs(dev); |
2168 | |
2169 | return 0; |
2170 | } |
2171 | |
2172 | static void shutdown(struct pci_dev *pdev) |
2173 | { |
2174 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2175 | int err; |
2176 | |
2177 | mlx5_core_info(dev, "Shutdown was called\n" ); |
2178 | set_bit(nr: MLX5_BREAK_FW_WAIT, addr: &dev->intf_state); |
2179 | err = mlx5_try_fast_unload(dev); |
2180 | if (err) |
2181 | mlx5_unload_one(dev, suspend: false); |
2182 | mlx5_pci_disable_device(dev); |
2183 | } |
2184 | |
2185 | static int mlx5_suspend(struct pci_dev *pdev, pm_message_t state) |
2186 | { |
2187 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2188 | |
2189 | mlx5_unload_one(dev, suspend: true); |
2190 | |
2191 | return 0; |
2192 | } |
2193 | |
2194 | static int mlx5_resume(struct pci_dev *pdev) |
2195 | { |
2196 | struct mlx5_core_dev *dev = pci_get_drvdata(pdev); |
2197 | |
2198 | return mlx5_load_one(dev, recovery: false); |
2199 | } |
2200 | |
2201 | static const struct pci_device_id mlx5_core_pci_table[] = { |
2202 | { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) }, |
2203 | { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */ |
2204 | { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) }, |
2205 | { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */ |
2206 | { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) }, |
2207 | { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */ |
2208 | { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */ |
2209 | { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */ |
2210 | { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */ |
2211 | { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */ |
2212 | { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */ |
2213 | { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */ |
2214 | { PCI_VDEVICE(MELLANOX, 0x101d) }, /* ConnectX-6 Dx */ |
2215 | { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */ |
2216 | { PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */ |
2217 | { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */ |
2218 | { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */ |
2219 | { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */ |
2220 | { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */ |
2221 | { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */ |
2222 | { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */ |
2223 | { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */ |
2224 | { 0, } |
2225 | }; |
2226 | |
2227 | MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); |
2228 | |
2229 | void mlx5_disable_device(struct mlx5_core_dev *dev) |
2230 | { |
2231 | mlx5_error_sw_reset(dev); |
2232 | mlx5_unload_one_devl_locked(dev, suspend: false); |
2233 | } |
2234 | |
2235 | int mlx5_recover_device(struct mlx5_core_dev *dev) |
2236 | { |
2237 | if (!mlx5_core_is_sf(dev)) { |
2238 | mlx5_pci_disable_device(dev); |
2239 | if (mlx5_pci_slot_reset(pdev: dev->pdev) != PCI_ERS_RESULT_RECOVERED) |
2240 | return -EIO; |
2241 | } |
2242 | |
2243 | return mlx5_load_one_devl_locked(dev, recovery: true); |
2244 | } |
2245 | |
2246 | static struct pci_driver mlx5_core_driver = { |
2247 | .name = KBUILD_MODNAME, |
2248 | .id_table = mlx5_core_pci_table, |
2249 | .probe = probe_one, |
2250 | .remove = remove_one, |
2251 | .suspend = mlx5_suspend, |
2252 | .resume = mlx5_resume, |
2253 | .shutdown = shutdown, |
2254 | .err_handler = &mlx5_err_handler, |
2255 | .sriov_configure = mlx5_core_sriov_configure, |
2256 | .sriov_get_vf_total_msix = mlx5_sriov_get_vf_total_msix, |
2257 | .sriov_set_msix_vec_count = mlx5_core_sriov_set_msix_vec_count, |
2258 | }; |
2259 | |
2260 | /** |
2261 | * mlx5_vf_get_core_dev - Get the mlx5 core device from a given VF PCI device if |
2262 | * mlx5_core is its driver. |
2263 | * @pdev: The associated PCI device. |
2264 | * |
2265 | * Upon return the interface state lock stay held to let caller uses it safely. |
2266 | * Caller must ensure to use the returned mlx5 device for a narrow window |
2267 | * and put it back with mlx5_vf_put_core_dev() immediately once usage was over. |
2268 | * |
2269 | * Return: Pointer to the associated mlx5_core_dev or NULL. |
2270 | */ |
2271 | struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev) |
2272 | { |
2273 | struct mlx5_core_dev *mdev; |
2274 | |
2275 | mdev = pci_iov_get_pf_drvdata(dev: pdev, pf_driver: &mlx5_core_driver); |
2276 | if (IS_ERR(ptr: mdev)) |
2277 | return NULL; |
2278 | |
2279 | mutex_lock(&mdev->intf_state_mutex); |
2280 | if (!test_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state)) { |
2281 | mutex_unlock(lock: &mdev->intf_state_mutex); |
2282 | return NULL; |
2283 | } |
2284 | |
2285 | return mdev; |
2286 | } |
2287 | EXPORT_SYMBOL(mlx5_vf_get_core_dev); |
2288 | |
2289 | /** |
2290 | * mlx5_vf_put_core_dev - Put the mlx5 core device back. |
2291 | * @mdev: The mlx5 core device. |
2292 | * |
2293 | * Upon return the interface state lock is unlocked and caller should not |
2294 | * access the mdev any more. |
2295 | */ |
2296 | void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev) |
2297 | { |
2298 | mutex_unlock(lock: &mdev->intf_state_mutex); |
2299 | } |
2300 | EXPORT_SYMBOL(mlx5_vf_put_core_dev); |
2301 | |
2302 | static void mlx5_core_verify_params(void) |
2303 | { |
2304 | if (prof_sel >= ARRAY_SIZE(profile)) { |
2305 | pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n" , |
2306 | prof_sel, |
2307 | ARRAY_SIZE(profile) - 1, |
2308 | MLX5_DEFAULT_PROF); |
2309 | prof_sel = MLX5_DEFAULT_PROF; |
2310 | } |
2311 | } |
2312 | |
2313 | static int __init mlx5_init(void) |
2314 | { |
2315 | int err; |
2316 | |
2317 | WARN_ONCE(strcmp(MLX5_ADEV_NAME, KBUILD_MODNAME), |
2318 | "mlx5_core name not in sync with kernel module name" ); |
2319 | |
2320 | get_random_bytes(buf: &sw_owner_id, len: sizeof(sw_owner_id)); |
2321 | |
2322 | mlx5_core_verify_params(); |
2323 | mlx5_register_debugfs(); |
2324 | |
2325 | err = mlx5e_init(); |
2326 | if (err) |
2327 | goto err_debug; |
2328 | |
2329 | err = mlx5_sf_driver_register(); |
2330 | if (err) |
2331 | goto err_sf; |
2332 | |
2333 | err = pci_register_driver(&mlx5_core_driver); |
2334 | if (err) |
2335 | goto err_pci; |
2336 | |
2337 | return 0; |
2338 | |
2339 | err_pci: |
2340 | mlx5_sf_driver_unregister(); |
2341 | err_sf: |
2342 | mlx5e_cleanup(); |
2343 | err_debug: |
2344 | mlx5_unregister_debugfs(); |
2345 | return err; |
2346 | } |
2347 | |
2348 | static void __exit mlx5_cleanup(void) |
2349 | { |
2350 | pci_unregister_driver(dev: &mlx5_core_driver); |
2351 | mlx5_sf_driver_unregister(); |
2352 | mlx5e_cleanup(); |
2353 | mlx5_unregister_debugfs(); |
2354 | } |
2355 | |
2356 | module_init(mlx5_init); |
2357 | module_exit(mlx5_cleanup); |
2358 | |