1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* MHI PCI driver - MHI over PCI controller driver
4	*
5	* This module is a generic driver for registering MHI-over-PCI devices,
6	* such as PCIe QCOM modems.
7	*
8	* Copyright (C) 2020 Linaro Ltd <loic.poulain@linaro.org>
9	*/
10
11	#include <linux/delay.h>
12	#include <linux/device.h>
13	#include <linux/mhi.h>
14	#include <linux/module.h>
15	#include <linux/pci.h>
16	#include <linux/pm_runtime.h>
17	#include <linux/timer.h>
18	#include <linux/workqueue.h>
19
20	#define MHI_PCI_DEFAULT_BAR_NUM 0
21
22	#define MHI_POST_RESET_DELAY_MS 2000
23
24	#define HEALTH_CHECK_PERIOD (HZ * 2)
25
26	/* PCI VID definitions */
27	#define PCI_VENDOR_ID_THALES 0x1269
28	#define PCI_VENDOR_ID_QUECTEL 0x1eac
29	#define PCI_VENDOR_ID_NETPRISMA 0x203e
30
31	#define MHI_EDL_DB 91
32	#define MHI_EDL_COOKIE 0xEDEDEDED
33
34	/**
35	* struct mhi_pci_dev_info - MHI PCI device specific information
36	* @config: MHI controller configuration
37	* @name: name of the PCI module
38	* @fw: firmware path (if any)
39	* @edl: emergency download mode firmware path (if any)
40	* @edl_trigger: capable of triggering EDL mode in the device (if supported)
41	* @bar_num: PCI base address register to use for MHI MMIO register space
42	* @dma_data_width: DMA transfer word size (32 or 64 bits)
43	* @mru_default: default MRU size for MBIM network packets
44	* @sideband_wake: Devices using dedicated sideband GPIO for wakeup instead
45	* of inband wake support (such as sdx24)
46	*/
47	struct mhi_pci_dev_info {
48	const struct mhi_controller_config *config;
49	const char *name;
50	const char *fw;
51	const char *edl;
52	bool edl_trigger;
53	unsigned int bar_num;
54	unsigned int dma_data_width;
55	unsigned int mru_default;
56	bool sideband_wake;
57	};
58
59	#define MHI_CHANNEL_CONFIG_UL(ch_num, ch_name, el_count, ev_ring) \
60	{ \
61	.num = ch_num, \
62	.name = ch_name, \
63	.num_elements = el_count, \
64	.event_ring = ev_ring, \
65	.dir = DMA_TO_DEVICE, \
66	.ee_mask = BIT(MHI_EE_AMSS), \
67	.pollcfg = 0, \
68	.doorbell = MHI_DB_BRST_DISABLE, \
69	.lpm_notify = false, \
70	.offload_channel = false, \
71	.doorbell_mode_switch = false, \
72	} \
73
74	#define MHI_CHANNEL_CONFIG_DL(ch_num, ch_name, el_count, ev_ring) \
75	{ \
76	.num = ch_num, \
77	.name = ch_name, \
78	.num_elements = el_count, \
79	.event_ring = ev_ring, \
80	.dir = DMA_FROM_DEVICE, \
81	.ee_mask = BIT(MHI_EE_AMSS), \
82	.pollcfg = 0, \
83	.doorbell = MHI_DB_BRST_DISABLE, \
84	.lpm_notify = false, \
85	.offload_channel = false, \
86	.doorbell_mode_switch = false, \
87	}
88
89	#define MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(ch_num, ch_name, el_count, ev_ring) \
90	{ \
91	.num = ch_num, \
92	.name = ch_name, \
93	.num_elements = el_count, \
94	.event_ring = ev_ring, \
95	.dir = DMA_FROM_DEVICE, \
96	.ee_mask = BIT(MHI_EE_AMSS), \
97	.pollcfg = 0, \
98	.doorbell = MHI_DB_BRST_DISABLE, \
99	.lpm_notify = false, \
100	.offload_channel = false, \
101	.doorbell_mode_switch = false, \
102	.auto_queue = true, \
103	}
104
105	#define MHI_EVENT_CONFIG_CTRL(ev_ring, el_count) \
106	{ \
107	.num_elements = el_count, \
108	.irq_moderation_ms = 0, \
109	.irq = (ev_ring) + 1, \
110	.priority = 1, \
111	.mode = MHI_DB_BRST_DISABLE, \
112	.data_type = MHI_ER_CTRL, \
113	.hardware_event = false, \
114	.client_managed = false, \
115	.offload_channel = false, \
116	}
117
118	#define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \
119	{ \
120	.num = ch_num, \
121	.name = ch_name, \
122	.num_elements = el_count, \
123	.event_ring = ev_ring, \
124	.dir = DMA_TO_DEVICE, \
125	.ee_mask = BIT(MHI_EE_AMSS), \
126	.pollcfg = 0, \
127	.doorbell = MHI_DB_BRST_ENABLE, \
128	.lpm_notify = false, \
129	.offload_channel = false, \
130	.doorbell_mode_switch = true, \
131	} \
132
133	#define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \
134	{ \
135	.num = ch_num, \
136	.name = ch_name, \
137	.num_elements = el_count, \
138	.event_ring = ev_ring, \
139	.dir = DMA_FROM_DEVICE, \
140	.ee_mask = BIT(MHI_EE_AMSS), \
141	.pollcfg = 0, \
142	.doorbell = MHI_DB_BRST_ENABLE, \
143	.lpm_notify = false, \
144	.offload_channel = false, \
145	.doorbell_mode_switch = true, \
146	}
147
148	#define MHI_CHANNEL_CONFIG_UL_SBL(ch_num, ch_name, el_count, ev_ring) \
149	{ \
150	.num = ch_num, \
151	.name = ch_name, \
152	.num_elements = el_count, \
153	.event_ring = ev_ring, \
154	.dir = DMA_TO_DEVICE, \
155	.ee_mask = BIT(MHI_EE_SBL), \
156	.pollcfg = 0, \
157	.doorbell = MHI_DB_BRST_DISABLE, \
158	.lpm_notify = false, \
159	.offload_channel = false, \
160	.doorbell_mode_switch = false, \
161	} \
162
163	#define MHI_CHANNEL_CONFIG_DL_SBL(ch_num, ch_name, el_count, ev_ring) \
164	{ \
165	.num = ch_num, \
166	.name = ch_name, \
167	.num_elements = el_count, \
168	.event_ring = ev_ring, \
169	.dir = DMA_FROM_DEVICE, \
170	.ee_mask = BIT(MHI_EE_SBL), \
171	.pollcfg = 0, \
172	.doorbell = MHI_DB_BRST_DISABLE, \
173	.lpm_notify = false, \
174	.offload_channel = false, \
175	.doorbell_mode_switch = false, \
176	}
177
178	#define MHI_CHANNEL_CONFIG_UL_FP(ch_num, ch_name, el_count, ev_ring) \
179	{ \
180	.num = ch_num, \
181	.name = ch_name, \
182	.num_elements = el_count, \
183	.event_ring = ev_ring, \
184	.dir = DMA_TO_DEVICE, \
185	.ee_mask = BIT(MHI_EE_FP), \
186	.pollcfg = 0, \
187	.doorbell = MHI_DB_BRST_DISABLE, \
188	.lpm_notify = false, \
189	.offload_channel = false, \
190	.doorbell_mode_switch = false, \
191	} \
192
193	#define MHI_CHANNEL_CONFIG_DL_FP(ch_num, ch_name, el_count, ev_ring) \
194	{ \
195	.num = ch_num, \
196	.name = ch_name, \
197	.num_elements = el_count, \
198	.event_ring = ev_ring, \
199	.dir = DMA_FROM_DEVICE, \
200	.ee_mask = BIT(MHI_EE_FP), \
201	.pollcfg = 0, \
202	.doorbell = MHI_DB_BRST_DISABLE, \
203	.lpm_notify = false, \
204	.offload_channel = false, \
205	.doorbell_mode_switch = false, \
206	}
207
208	#define MHI_EVENT_CONFIG_DATA(ev_ring, el_count) \
209	{ \
210	.num_elements = el_count, \
211	.irq_moderation_ms = 5, \
212	.irq = (ev_ring) + 1, \
213	.priority = 1, \
214	.mode = MHI_DB_BRST_DISABLE, \
215	.data_type = MHI_ER_DATA, \
216	.hardware_event = false, \
217	.client_managed = false, \
218	.offload_channel = false, \
219	}
220
221	#define MHI_EVENT_CONFIG_SW_DATA(ev_ring, el_count) \
222	{ \
223	.num_elements = el_count, \
224	.irq_moderation_ms = 0, \
225	.irq = (ev_ring) + 1, \
226	.priority = 1, \
227	.mode = MHI_DB_BRST_DISABLE, \
228	.data_type = MHI_ER_DATA, \
229	.hardware_event = false, \
230	.client_managed = false, \
231	.offload_channel = false, \
232	}
233
234	#define MHI_EVENT_CONFIG_HW_DATA(ev_ring, el_count, ch_num) \
235	{ \
236	.num_elements = el_count, \
237	.irq_moderation_ms = 1, \
238	.irq = (ev_ring) + 1, \
239	.priority = 1, \
240	.mode = MHI_DB_BRST_DISABLE, \
241	.data_type = MHI_ER_DATA, \
242	.hardware_event = true, \
243	.client_managed = false, \
244	.offload_channel = false, \
245	.channel = ch_num, \
246	}
247
248	static const struct mhi_channel_config mhi_qcom_qdu100_channels[] = {
249	MHI_CHANNEL_CONFIG_UL(0, "LOOPBACK", 32, 2),
250	MHI_CHANNEL_CONFIG_DL(1, "LOOPBACK", 32, 2),
251	MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 128, 1),
252	MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 128, 1),
253	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 64, 3),
254	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 64, 3),
255	MHI_CHANNEL_CONFIG_UL(9, "QDSS", 64, 3),
256	MHI_CHANNEL_CONFIG_UL(14, "NMEA", 32, 4),
257	MHI_CHANNEL_CONFIG_DL(15, "NMEA", 32, 4),
258	MHI_CHANNEL_CONFIG_UL(16, "CSM_CTRL", 32, 4),
259	MHI_CHANNEL_CONFIG_DL(17, "CSM_CTRL", 32, 4),
260	MHI_CHANNEL_CONFIG_UL(40, "MHI_PHC", 32, 4),
261	MHI_CHANNEL_CONFIG_DL(41, "MHI_PHC", 32, 4),
262	MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 256, 5),
263	MHI_CHANNEL_CONFIG_DL(47, "IP_SW0", 256, 5),
264	};
265
266	static struct mhi_event_config mhi_qcom_qdu100_events[] = {
267	/* first ring is control+data ring */
268	MHI_EVENT_CONFIG_CTRL(0, 64),
269	/* SAHARA dedicated event ring */
270	MHI_EVENT_CONFIG_SW_DATA(1, 256),
271	/* Software channels dedicated event ring */
272	MHI_EVENT_CONFIG_SW_DATA(2, 64),
273	MHI_EVENT_CONFIG_SW_DATA(3, 256),
274	MHI_EVENT_CONFIG_SW_DATA(4, 256),
275	/* Software IP channels dedicated event ring */
276	MHI_EVENT_CONFIG_SW_DATA(5, 512),
277	MHI_EVENT_CONFIG_SW_DATA(6, 512),
278	MHI_EVENT_CONFIG_SW_DATA(7, 512),
279	};
280
281	static const struct mhi_controller_config mhi_qcom_qdu100_config = {
282	.max_channels = 128,
283	.timeout_ms = 120000,
284	.num_channels = ARRAY_SIZE(mhi_qcom_qdu100_channels),
285	.ch_cfg = mhi_qcom_qdu100_channels,
286	.num_events = ARRAY_SIZE(mhi_qcom_qdu100_events),
287	.event_cfg = mhi_qcom_qdu100_events,
288	};
289
290	static const struct mhi_pci_dev_info mhi_qcom_qdu100_info = {
291	.name = "qcom-qdu100",
292	.fw = "qcom/qdu100/xbl_s.melf",
293	.edl_trigger = true,
294	.config = &mhi_qcom_qdu100_config,
295	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
296	.dma_data_width = 32,
297	.sideband_wake = false,
298	};
299
300	static const struct mhi_channel_config mhi_qcom_sa8775p_channels[] = {
301	MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 2048, 1),
302	MHI_CHANNEL_CONFIG_DL(47, "IP_SW0", 2048, 2),
303	};
304
305	static struct mhi_event_config mhi_qcom_sa8775p_events[] = {
306	/* first ring is control+data ring */
307	MHI_EVENT_CONFIG_CTRL(0, 64),
308	/* Software channels dedicated event ring */
309	MHI_EVENT_CONFIG_SW_DATA(1, 64),
310	MHI_EVENT_CONFIG_SW_DATA(2, 64),
311	};
312
313	static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = {
314	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1),
315	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1),
316	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0),
317	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0),
318	MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0),
319	MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0),
320	MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0),
321	MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 8, 0),
322	MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
323	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
324	MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 64, 2),
325	MHI_CHANNEL_CONFIG_DL(47, "IP_SW0", 64, 3),
326	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 4),
327	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 5),
328	};
329
330	static struct mhi_event_config modem_qcom_v1_mhi_events[] = {
331	/* first ring is control+data ring */
332	MHI_EVENT_CONFIG_CTRL(0, 64),
333	/* DIAG dedicated event ring */
334	MHI_EVENT_CONFIG_DATA(1, 128),
335	/* Software channels dedicated event ring */
336	MHI_EVENT_CONFIG_SW_DATA(2, 64),
337	MHI_EVENT_CONFIG_SW_DATA(3, 64),
338	/* Hardware channels request dedicated hardware event rings */
339	MHI_EVENT_CONFIG_HW_DATA(4, 1024, 100),
340	MHI_EVENT_CONFIG_HW_DATA(5, 2048, 101)
341	};
342
343	static const struct mhi_controller_config mhi_qcom_sa8775p_config = {
344	.max_channels = 128,
345	.timeout_ms = 8000,
346	.num_channels = ARRAY_SIZE(mhi_qcom_sa8775p_channels),
347	.ch_cfg = mhi_qcom_sa8775p_channels,
348	.num_events = ARRAY_SIZE(mhi_qcom_sa8775p_events),
349	.event_cfg = mhi_qcom_sa8775p_events,
350	};
351
352	static const struct mhi_controller_config modem_qcom_v2_mhiv_config = {
353	.max_channels = 128,
354	.timeout_ms = 8000,
355	.ready_timeout_ms = 50000,
356	.num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels),
357	.ch_cfg = modem_qcom_v1_mhi_channels,
358	.num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events),
359	.event_cfg = modem_qcom_v1_mhi_events,
360	};
361
362	static const struct mhi_controller_config modem_qcom_v1_mhiv_config = {
363	.max_channels = 128,
364	.timeout_ms = 8000,
365	.num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels),
366	.ch_cfg = modem_qcom_v1_mhi_channels,
367	.num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events),
368	.event_cfg = modem_qcom_v1_mhi_events,
369	};
370
371	static const struct mhi_pci_dev_info mhi_qcom_sa8775p_info = {
372	.name = "qcom-sa8775p",
373	.edl_trigger = false,
374	.config = &mhi_qcom_sa8775p_config,
375	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
376	.dma_data_width = 32,
377	.mru_default = 32768,
378	.sideband_wake = false,
379	};
380
381	static const struct mhi_pci_dev_info mhi_qcom_sdx75_info = {
382	.name = "qcom-sdx75m",
383	.fw = "qcom/sdx75m/xbl.elf",
384	.edl = "qcom/sdx75m/edl.mbn",
385	.edl_trigger = true,
386	.config = &modem_qcom_v2_mhiv_config,
387	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
388	.dma_data_width = 32,
389	.sideband_wake = false,
390	};
391
392	static const struct mhi_pci_dev_info mhi_qcom_sdx65_info = {
393	.name = "qcom-sdx65m",
394	.fw = "qcom/sdx65m/xbl.elf",
395	.edl = "qcom/sdx65m/edl.mbn",
396	.edl_trigger = true,
397	.config = &modem_qcom_v1_mhiv_config,
398	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
399	.dma_data_width = 32,
400	.sideband_wake = false,
401	};
402
403	static const struct mhi_pci_dev_info mhi_qcom_sdx55_info = {
404	.name = "qcom-sdx55m",
405	.fw = "qcom/sdx55m/sbl1.mbn",
406	.edl = "qcom/sdx55m/edl.mbn",
407	.edl_trigger = true,
408	.config = &modem_qcom_v1_mhiv_config,
409	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
410	.dma_data_width = 32,
411	.mru_default = 32768,
412	.sideband_wake = false,
413	};
414
415	static const struct mhi_pci_dev_info mhi_qcom_sdx24_info = {
416	.name = "qcom-sdx24",
417	.edl = "qcom/prog_firehose_sdx24.mbn",
418	.config = &modem_qcom_v1_mhiv_config,
419	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
420	.dma_data_width = 32,
421	.sideband_wake = true,
422	};
423
424	static const struct mhi_channel_config mhi_quectel_em1xx_channels[] = {
425	MHI_CHANNEL_CONFIG_UL(0, "NMEA", 32, 0),
426	MHI_CHANNEL_CONFIG_DL(1, "NMEA", 32, 0),
427	MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 32, 0),
428	MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 32, 0),
429	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 32, 1),
430	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 32, 1),
431	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 32, 0),
432	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
433	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
434	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
435	/* The EDL firmware is a flash-programmer exposing firehose protocol */
436	MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
437	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
438	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
439	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
440	};
441
442	static struct mhi_event_config mhi_quectel_em1xx_events[] = {
443	MHI_EVENT_CONFIG_CTRL(0, 128),
444	MHI_EVENT_CONFIG_DATA(1, 128),
445	MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
446	MHI_EVENT_CONFIG_HW_DATA(3, 1024, 101)
447	};
448
449	static const struct mhi_controller_config modem_quectel_em1xx_config = {
450	.max_channels = 128,
451	.timeout_ms = 20000,
452	.num_channels = ARRAY_SIZE(mhi_quectel_em1xx_channels),
453	.ch_cfg = mhi_quectel_em1xx_channels,
454	.num_events = ARRAY_SIZE(mhi_quectel_em1xx_events),
455	.event_cfg = mhi_quectel_em1xx_events,
456	};
457
458	static const struct mhi_pci_dev_info mhi_quectel_em1xx_info = {
459	.name = "quectel-em1xx",
460	.edl = "qcom/prog_firehose_sdx24.mbn",
461	.config = &modem_quectel_em1xx_config,
462	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
463	.dma_data_width = 32,
464	.mru_default = 32768,
465	.sideband_wake = true,
466	};
467
468	static const struct mhi_pci_dev_info mhi_quectel_rm5xx_info = {
469	.name = "quectel-rm5xx",
470	.edl = "qcom/prog_firehose_sdx6x.elf",
471	.config = &modem_quectel_em1xx_config,
472	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
473	.dma_data_width = 32,
474	.mru_default = 32768,
475	.sideband_wake = true,
476	};
477
478	static const struct mhi_channel_config mhi_foxconn_sdx55_channels[] = {
479	MHI_CHANNEL_CONFIG_UL(0, "LOOPBACK", 32, 0),
480	MHI_CHANNEL_CONFIG_DL(1, "LOOPBACK", 32, 0),
481	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 32, 1),
482	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 32, 1),
483	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 32, 0),
484	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
485	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
486	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
487	MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
488	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
489	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
490	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
491	};
492
493	static struct mhi_event_config mhi_foxconn_sdx55_events[] = {
494	MHI_EVENT_CONFIG_CTRL(0, 128),
495	MHI_EVENT_CONFIG_DATA(1, 128),
496	MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
497	MHI_EVENT_CONFIG_HW_DATA(3, 1024, 101)
498	};
499
500	static const struct mhi_controller_config modem_foxconn_sdx55_config = {
501	.max_channels = 128,
502	.timeout_ms = 20000,
503	.num_channels = ARRAY_SIZE(mhi_foxconn_sdx55_channels),
504	.ch_cfg = mhi_foxconn_sdx55_channels,
505	.num_events = ARRAY_SIZE(mhi_foxconn_sdx55_events),
506	.event_cfg = mhi_foxconn_sdx55_events,
507	};
508
509	static const struct mhi_controller_config modem_foxconn_sdx72_config = {
510	.max_channels = 128,
511	.timeout_ms = 20000,
512	.ready_timeout_ms = 50000,
513	.num_channels = ARRAY_SIZE(mhi_foxconn_sdx55_channels),
514	.ch_cfg = mhi_foxconn_sdx55_channels,
515	.num_events = ARRAY_SIZE(mhi_foxconn_sdx55_events),
516	.event_cfg = mhi_foxconn_sdx55_events,
517	};
518
519	static const struct mhi_pci_dev_info mhi_foxconn_sdx55_info = {
520	.name = "foxconn-sdx55",
521	.edl = "qcom/sdx55m/foxconn/prog_firehose_sdx55.mbn",
522	.edl_trigger = true,
523	.config = &modem_foxconn_sdx55_config,
524	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
525	.dma_data_width = 32,
526	.mru_default = 32768,
527	.sideband_wake = false,
528	};
529
530	static const struct mhi_pci_dev_info mhi_foxconn_t99w175_info = {
531	.name = "foxconn-t99w175",
532	.edl = "qcom/sdx55m/foxconn/prog_firehose_sdx55.mbn",
533	.edl_trigger = true,
534	.config = &modem_foxconn_sdx55_config,
535	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
536	.dma_data_width = 32,
537	.mru_default = 32768,
538	.sideband_wake = false,
539	};
540
541	static const struct mhi_pci_dev_info mhi_foxconn_dw5930e_info = {
542	.name = "foxconn-dw5930e",
543	.edl = "qcom/sdx55m/foxconn/prog_firehose_sdx55.mbn",
544	.edl_trigger = true,
545	.config = &modem_foxconn_sdx55_config,
546	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
547	.dma_data_width = 32,
548	.mru_default = 32768,
549	.sideband_wake = false,
550	};
551
552	static const struct mhi_pci_dev_info mhi_foxconn_t99w368_info = {
553	.name = "foxconn-t99w368",
554	.edl = "qcom/sdx65m/foxconn/prog_firehose_lite.elf",
555	.edl_trigger = true,
556	.config = &modem_foxconn_sdx55_config,
557	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
558	.dma_data_width = 32,
559	.mru_default = 32768,
560	.sideband_wake = false,
561	};
562
563	static const struct mhi_pci_dev_info mhi_foxconn_t99w373_info = {
564	.name = "foxconn-t99w373",
565	.edl = "qcom/sdx65m/foxconn/prog_firehose_lite.elf",
566	.edl_trigger = true,
567	.config = &modem_foxconn_sdx55_config,
568	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
569	.dma_data_width = 32,
570	.mru_default = 32768,
571	.sideband_wake = false,
572	};
573
574	static const struct mhi_pci_dev_info mhi_foxconn_t99w510_info = {
575	.name = "foxconn-t99w510",
576	.edl = "qcom/sdx24m/foxconn/prog_firehose_sdx24.mbn",
577	.edl_trigger = true,
578	.config = &modem_foxconn_sdx55_config,
579	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
580	.dma_data_width = 32,
581	.mru_default = 32768,
582	.sideband_wake = false,
583	};
584
585	static const struct mhi_pci_dev_info mhi_foxconn_dw5932e_info = {
586	.name = "foxconn-dw5932e",
587	.edl = "qcom/sdx65m/foxconn/prog_firehose_lite.elf",
588	.edl_trigger = true,
589	.config = &modem_foxconn_sdx55_config,
590	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
591	.dma_data_width = 32,
592	.mru_default = 32768,
593	.sideband_wake = false,
594	};
595
596	static const struct mhi_pci_dev_info mhi_foxconn_t99w515_info = {
597	.name = "foxconn-t99w515",
598	.edl = "qcom/sdx72m/foxconn/edl.mbn",
599	.edl_trigger = true,
600	.config = &modem_foxconn_sdx72_config,
601	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
602	.dma_data_width = 32,
603	.mru_default = 32768,
604	.sideband_wake = false,
605	};
606
607	static const struct mhi_pci_dev_info mhi_foxconn_dw5934e_info = {
608	.name = "foxconn-dw5934e",
609	.edl = "qcom/sdx72m/foxconn/edl.mbn",
610	.edl_trigger = true,
611	.config = &modem_foxconn_sdx72_config,
612	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
613	.dma_data_width = 32,
614	.mru_default = 32768,
615	.sideband_wake = false,
616	};
617
618	static const struct mhi_channel_config mhi_mv3x_channels[] = {
619	MHI_CHANNEL_CONFIG_UL(0, "LOOPBACK", 64, 0),
620	MHI_CHANNEL_CONFIG_DL(1, "LOOPBACK", 64, 0),
621	/* MBIM Control Channel */
622	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 64, 0),
623	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 64, 0),
624	/* MBIM Data Channel */
625	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 512, 2),
626	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 512, 3),
627	};
628
629	static struct mhi_event_config mhi_mv3x_events[] = {
630	MHI_EVENT_CONFIG_CTRL(0, 256),
631	MHI_EVENT_CONFIG_DATA(1, 256),
632	MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
633	MHI_EVENT_CONFIG_HW_DATA(3, 1024, 101),
634	};
635
636	static const struct mhi_controller_config modem_mv3x_config = {
637	.max_channels = 128,
638	.timeout_ms = 20000,
639	.num_channels = ARRAY_SIZE(mhi_mv3x_channels),
640	.ch_cfg = mhi_mv3x_channels,
641	.num_events = ARRAY_SIZE(mhi_mv3x_events),
642	.event_cfg = mhi_mv3x_events,
643	};
644
645	static const struct mhi_pci_dev_info mhi_mv31_info = {
646	.name = "cinterion-mv31",
647	.config = &modem_mv3x_config,
648	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
649	.dma_data_width = 32,
650	.mru_default = 32768,
651	};
652
653	static const struct mhi_pci_dev_info mhi_mv32_info = {
654	.name = "cinterion-mv32",
655	.config = &modem_mv3x_config,
656	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
657	.dma_data_width = 32,
658	.mru_default = 32768,
659	};
660
661	static const struct mhi_channel_config mhi_sierra_em919x_channels[] = {
662	MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 32, 0),
663	MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 256, 0),
664	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 32, 0),
665	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 32, 0),
666	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 128, 0),
667	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 128, 0),
668	MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0),
669	MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0),
670	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
671	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
672	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 512, 1),
673	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 512, 2),
674	};
675
676	static struct mhi_event_config modem_sierra_em919x_mhi_events[] = {
677	/* first ring is control+data and DIAG ring */
678	MHI_EVENT_CONFIG_CTRL(0, 2048),
679	/* Hardware channels request dedicated hardware event rings */
680	MHI_EVENT_CONFIG_HW_DATA(1, 2048, 100),
681	MHI_EVENT_CONFIG_HW_DATA(2, 2048, 101)
682	};
683
684	static const struct mhi_controller_config modem_sierra_em919x_config = {
685	.max_channels = 128,
686	.timeout_ms = 24000,
687	.num_channels = ARRAY_SIZE(mhi_sierra_em919x_channels),
688	.ch_cfg = mhi_sierra_em919x_channels,
689	.num_events = ARRAY_SIZE(modem_sierra_em919x_mhi_events),
690	.event_cfg = modem_sierra_em919x_mhi_events,
691	};
692
693	static const struct mhi_pci_dev_info mhi_sierra_em919x_info = {
694	.name = "sierra-em919x",
695	.config = &modem_sierra_em919x_config,
696	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
697	.dma_data_width = 32,
698	.sideband_wake = false,
699	};
700
701	static const struct mhi_channel_config mhi_telit_fn980_hw_v1_channels[] = {
702	MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0),
703	MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0),
704	MHI_CHANNEL_CONFIG_UL(20, "IPCR", 16, 0),
705	MHI_CHANNEL_CONFIG_DL_AUTOQUEUE(21, "IPCR", 16, 0),
706	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 1),
707	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 2),
708	};
709
710	static struct mhi_event_config mhi_telit_fn980_hw_v1_events[] = {
711	MHI_EVENT_CONFIG_CTRL(0, 128),
712	MHI_EVENT_CONFIG_HW_DATA(1, 1024, 100),
713	MHI_EVENT_CONFIG_HW_DATA(2, 2048, 101)
714	};
715
716	static const struct mhi_controller_config modem_telit_fn980_hw_v1_config = {
717	.max_channels = 128,
718	.timeout_ms = 20000,
719	.num_channels = ARRAY_SIZE(mhi_telit_fn980_hw_v1_channels),
720	.ch_cfg = mhi_telit_fn980_hw_v1_channels,
721	.num_events = ARRAY_SIZE(mhi_telit_fn980_hw_v1_events),
722	.event_cfg = mhi_telit_fn980_hw_v1_events,
723	};
724
725	static const struct mhi_pci_dev_info mhi_telit_fn980_hw_v1_info = {
726	.name = "telit-fn980-hwv1",
727	.fw = "qcom/sdx55m/sbl1.mbn",
728	.edl = "qcom/sdx55m/edl.mbn",
729	.config = &modem_telit_fn980_hw_v1_config,
730	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
731	.dma_data_width = 32,
732	.mru_default = 32768,
733	.sideband_wake = false,
734	};
735
736	static const struct mhi_channel_config mhi_telit_fn990_channels[] = {
737	MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 32, 0),
738	MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 32, 0),
739	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 64, 1),
740	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 64, 1),
741	MHI_CHANNEL_CONFIG_UL(12, "MBIM", 32, 0),
742	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
743	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
744	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
745	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
746	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
747	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
748	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
749	};
750
751	static struct mhi_event_config mhi_telit_fn990_events[] = {
752	MHI_EVENT_CONFIG_CTRL(0, 128),
753	MHI_EVENT_CONFIG_DATA(1, 128),
754	MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
755	MHI_EVENT_CONFIG_HW_DATA(3, 2048, 101)
756	};
757
758	static const struct mhi_controller_config modem_telit_fn990_config = {
759	.max_channels = 128,
760	.timeout_ms = 20000,
761	.num_channels = ARRAY_SIZE(mhi_telit_fn990_channels),
762	.ch_cfg = mhi_telit_fn990_channels,
763	.num_events = ARRAY_SIZE(mhi_telit_fn990_events),
764	.event_cfg = mhi_telit_fn990_events,
765	};
766
767	static const struct mhi_pci_dev_info mhi_telit_fn990_info = {
768	.name = "telit-fn990",
769	.config = &modem_telit_fn990_config,
770	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
771	.dma_data_width = 32,
772	.sideband_wake = false,
773	.mru_default = 32768,
774	};
775
776	static const struct mhi_pci_dev_info mhi_telit_fe990a_info = {
777	.name = "telit-fe990a",
778	.config = &modem_telit_fn990_config,
779	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
780	.dma_data_width = 32,
781	.sideband_wake = false,
782	.mru_default = 32768,
783	};
784
785	static const struct mhi_channel_config mhi_telit_fn920c04_channels[] = {
786	MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 32, 0),
787	MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 32, 0),
788	MHI_CHANNEL_CONFIG_UL(4, "DIAG", 64, 1),
789	MHI_CHANNEL_CONFIG_DL(5, "DIAG", 64, 1),
790	MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0),
791	MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0),
792	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
793	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
794	MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
795	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
796	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
797	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
798	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2),
799	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3),
800	};
801
802	static const struct mhi_controller_config modem_telit_fn920c04_config = {
803	.max_channels = 128,
804	.timeout_ms = 50000,
805	.num_channels = ARRAY_SIZE(mhi_telit_fn920c04_channels),
806	.ch_cfg = mhi_telit_fn920c04_channels,
807	.num_events = ARRAY_SIZE(mhi_telit_fn990_events),
808	.event_cfg = mhi_telit_fn990_events,
809	};
810
811	static const struct mhi_pci_dev_info mhi_telit_fn920c04_info = {
812	.name = "telit-fn920c04",
813	.config = &modem_telit_fn920c04_config,
814	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
815	.dma_data_width = 32,
816	.sideband_wake = false,
817	.mru_default = 32768,
818	.edl_trigger = true,
819	};
820
821	static const struct mhi_pci_dev_info mhi_netprisma_lcur57_info = {
822	.name = "netprisma-lcur57",
823	.edl = "qcom/prog_firehose_sdx24.mbn",
824	.config = &modem_quectel_em1xx_config,
825	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
826	.dma_data_width = 32,
827	.mru_default = 32768,
828	.sideband_wake = true,
829	};
830
831	static const struct mhi_pci_dev_info mhi_netprisma_fcun69_info = {
832	.name = "netprisma-fcun69",
833	.edl = "qcom/prog_firehose_sdx6x.elf",
834	.config = &modem_quectel_em1xx_config,
835	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
836	.dma_data_width = 32,
837	.mru_default = 32768,
838	.sideband_wake = true,
839	};
840
841	/* Keep the list sorted based on the PID. New VID should be added as the last entry */
842	static const struct pci_device_id mhi_pci_id_table[] = {
843	{PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0116),
844	.driver_data = (kernel_ulong_t) &mhi_qcom_sa8775p_info },
845	/* Telit FN920C04 (sdx35) */
846	{PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x2020),
847	.driver_data = (kernel_ulong_t) &mhi_telit_fn920c04_info },
848	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0304),
849	.driver_data = (kernel_ulong_t) &mhi_qcom_sdx24_info },
850	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0306, PCI_VENDOR_ID_QCOM, 0x010c),
851	.driver_data = (kernel_ulong_t) &mhi_foxconn_sdx55_info },
852	/* EM919x (sdx55), use the same vid:pid as qcom-sdx55m */
853	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0306, 0x18d7, 0x0200),
854	.driver_data = (kernel_ulong_t) &mhi_sierra_em919x_info },
855	/* Telit FN980 hardware revision v1 */
856	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0306, 0x1C5D, 0x2000),
857	.driver_data = (kernel_ulong_t) &mhi_telit_fn980_hw_v1_info },
858	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0306),
859	.driver_data = (kernel_ulong_t) &mhi_qcom_sdx55_info },
860	/* Telit FN990 */
861	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0308, 0x1c5d, 0x2010),
862	.driver_data = (kernel_ulong_t) &mhi_telit_fn990_info },
863	/* Telit FE990A */
864	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x0308, 0x1c5d, 0x2015),
865	.driver_data = (kernel_ulong_t) &mhi_telit_fe990a_info },
866	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0308),
867	.driver_data = (kernel_ulong_t) &mhi_qcom_sdx65_info },
868	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0309),
869	.driver_data = (kernel_ulong_t) &mhi_qcom_sdx75_info },
870	/* QDU100, x100-DU */
871	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0601),
872	.driver_data = (kernel_ulong_t) &mhi_qcom_qdu100_info },
873	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1001), /* EM120R-GL (sdx24) */
874	.driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
875	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1002), /* EM160R-GL (sdx24) */
876	.driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
877	/* RM520N-GL (sdx6x), eSIM */
878	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1004),
879	.driver_data = (kernel_ulong_t) &mhi_quectel_rm5xx_info },
880	/* RM520N-GL (sdx6x), Lenovo variant */
881	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1007),
882	.driver_data = (kernel_ulong_t) &mhi_quectel_rm5xx_info },
883	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x100d), /* EM160R-GL (sdx24) */
884	.driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
885	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x2001), /* EM120R-GL for FCCL (sdx24) */
886	.driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
887	/* T99W175 (sdx55), Both for eSIM and Non-eSIM */
888	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0ab),
889	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w175_info },
890	/* DW5930e (sdx55), With eSIM, It's also T99W175 */
891	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0b0),
892	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5930e_info },
893	/* DW5930e (sdx55), Non-eSIM, It's also T99W175 */
894	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0b1),
895	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5930e_info },
896	/* T99W175 (sdx55), Based on Qualcomm new baseline */
897	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0bf),
898	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w175_info },
899	/* T99W175 (sdx55) */
900	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0c3),
901	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w175_info },
902	/* T99W368 (sdx65) */
903	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0d8),
904	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w368_info },
905	/* T99W373 (sdx62) */
906	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0d9),
907	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w373_info },
908	/* T99W510 (sdx24), variant 1 */
909	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f0),
910	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w510_info },
911	/* T99W510 (sdx24), variant 2 */
912	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f1),
913	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w510_info },
914	/* T99W510 (sdx24), variant 3 */
915	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f2),
916	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w510_info },
917	/* DW5932e-eSIM (sdx62), With eSIM */
918	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f5),
919	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5932e_info },
920	/* DW5932e (sdx62), Non-eSIM */
921	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0f9),
922	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5932e_info },
923	/* T99W515 (sdx72) */
924	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe118),
925	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w515_info },
926	/* DW5934e(sdx72), With eSIM */
927	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe11d),
928	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5934e_info },
929	/* DW5934e(sdx72), Non-eSIM */
930	{ PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe11e),
931	.driver_data = (kernel_ulong_t) &mhi_foxconn_dw5934e_info },
932	/* MV31-W (Cinterion) */
933	{ PCI_DEVICE(PCI_VENDOR_ID_THALES, 0x00b3),
934	.driver_data = (kernel_ulong_t) &mhi_mv31_info },
935	/* MV31-W (Cinterion), based on new baseline */
936	{ PCI_DEVICE(PCI_VENDOR_ID_THALES, 0x00b4),
937	.driver_data = (kernel_ulong_t) &mhi_mv31_info },
938	/* MV32-WA (Cinterion) */
939	{ PCI_DEVICE(PCI_VENDOR_ID_THALES, 0x00ba),
940	.driver_data = (kernel_ulong_t) &mhi_mv32_info },
941	/* MV32-WB (Cinterion) */
942	{ PCI_DEVICE(PCI_VENDOR_ID_THALES, 0x00bb),
943	.driver_data = (kernel_ulong_t) &mhi_mv32_info },
944	/* T99W175 (sdx55), HP variant */
945	{ PCI_DEVICE(0x03f0, 0x0a6c),
946	.driver_data = (kernel_ulong_t) &mhi_foxconn_t99w175_info },
947	/* NETPRISMA LCUR57 (SDX24) */
948	{ PCI_DEVICE(PCI_VENDOR_ID_NETPRISMA, 0x1000),
949	.driver_data = (kernel_ulong_t) &mhi_netprisma_lcur57_info },
950	/* NETPRISMA FCUN69 (SDX6X) */
951	{ PCI_DEVICE(PCI_VENDOR_ID_NETPRISMA, 0x1001),
952	.driver_data = (kernel_ulong_t) &mhi_netprisma_fcun69_info },
953	{ }
954	};
955	MODULE_DEVICE_TABLE(pci, mhi_pci_id_table);
956
957	enum mhi_pci_device_status {
958	MHI_PCI_DEV_STARTED,
959	MHI_PCI_DEV_SUSPENDED,
960	};
961
962	struct mhi_pci_device {
963	struct mhi_controller mhi_cntrl;
964	struct pci_saved_state *pci_state;
965	struct work_struct recovery_work;
966	struct timer_list health_check_timer;
967	unsigned long status;
968	};
969
970	static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl,
971	void __iomem *addr, u32 *out)
972	{
973	*out = readl(addr);
974	return 0;
975	}
976
977	static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl,
978	void __iomem *addr, u32 val)
979	{
980	writel(val, addr);
981	}
982
983	static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl,
984	enum mhi_callback cb)
985	{
986	struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
987
988	/* Nothing to do for now */
989	switch (cb) {
990	case MHI_CB_FATAL_ERROR:
991	case MHI_CB_SYS_ERROR:
992	dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb);
993	pm_runtime_forbid(dev: &pdev->dev);
994	break;
995	case MHI_CB_EE_MISSION_MODE:
996	pm_runtime_allow(dev: &pdev->dev);
997	break;
998	default:
999	break;
1000	}
1001	}
1002
1003	static void mhi_pci_wake_get_nop(struct mhi_controller *mhi_cntrl, bool force)
1004	{
1005	/* no-op */
1006	}
1007
1008	static void mhi_pci_wake_put_nop(struct mhi_controller *mhi_cntrl, bool override)
1009	{
1010	/* no-op */
1011	}
1012
1013	static void mhi_pci_wake_toggle_nop(struct mhi_controller *mhi_cntrl)
1014	{
1015	/* no-op */
1016	}
1017
1018	static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl)
1019	{
1020	struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
1021	u16 vendor = 0;
1022
1023	if (pci_read_config_word(dev: pdev, PCI_VENDOR_ID, val: &vendor))
1024	return false;
1025
1026	if (vendor == (u16) ~0 || vendor == 0)
1027	return false;
1028
1029	return true;
1030	}
1031
1032	static int mhi_pci_claim(struct mhi_controller *mhi_cntrl,
1033	unsigned int bar_num, u64 dma_mask)
1034	{
1035	struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
1036	int err;
1037
1038	err = pcim_enable_device(pdev);
1039	if (err) {
1040	dev_err(&pdev->dev, "failed to enable pci device: %d\n", err);
1041	return err;
1042	}
1043
1044	mhi_cntrl->regs = pcim_iomap_region(pdev, bar: bar_num, name: pci_name(pdev));
1045	if (IS_ERR(ptr: mhi_cntrl->regs)) {
1046	err = PTR_ERR(ptr: mhi_cntrl->regs);
1047	dev_err(&pdev->dev, "failed to map pci region: %d\n", err);
1048	return err;
1049	}
1050	mhi_cntrl->reg_len = pci_resource_len(pdev, bar_num);
1051
1052	err = dma_set_mask_and_coherent(dev: &pdev->dev, mask: dma_mask);
1053	if (err) {
1054	dev_err(&pdev->dev, "Cannot set proper DMA mask\n");
1055	return err;
1056	}
1057
1058	pci_set_master(dev: pdev);
1059
1060	return 0;
1061	}
1062
1063	static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl,
1064	const struct mhi_controller_config *mhi_cntrl_config)
1065	{
1066	struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
1067	int nr_vectors, i;
1068	int *irq;
1069
1070	/*
1071	* Alloc one MSI vector for BHI + one vector per event ring, ideally...
1072	* No explicit pci_free_irq_vectors required, done by pcim_release.
1073	*/
1074	mhi_cntrl->nr_irqs = 1 + mhi_cntrl_config->num_events;
1075
1076	nr_vectors = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: mhi_cntrl->nr_irqs, PCI_IRQ_MSIX | PCI_IRQ_MSI);
1077	if (nr_vectors < 0) {
1078	dev_err(&pdev->dev, "Error allocating MSI vectors %d\n",
1079	nr_vectors);
1080	return nr_vectors;
1081	}
1082
1083	if (nr_vectors < mhi_cntrl->nr_irqs) {
1084	dev_warn(&pdev->dev, "using shared MSI\n");
1085
1086	/* Patch msi vectors, use only one (shared) */
1087	for (i = 0; i < mhi_cntrl_config->num_events; i++)
1088	mhi_cntrl_config->event_cfg[i].irq = 0;
1089	mhi_cntrl->nr_irqs = 1;
1090	}
1091
1092	irq = devm_kcalloc(dev: &pdev->dev, n: mhi_cntrl->nr_irqs, size: sizeof(int), GFP_KERNEL);
1093	if (!irq)
1094	return -ENOMEM;
1095
1096	for (i = 0; i < mhi_cntrl->nr_irqs; i++) {
1097	int vector = i >= nr_vectors ? (nr_vectors - 1) : i;
1098
1099	irq[i] = pci_irq_vector(dev: pdev, nr: vector);
1100	}
1101
1102	mhi_cntrl->irq = irq;
1103
1104	return 0;
1105	}
1106
1107	static int mhi_pci_runtime_get(struct mhi_controller *mhi_cntrl)
1108	{
1109	/* The runtime_get() MHI callback means:
1110	* Do whatever is requested to leave M3.
1111	*/
1112	return pm_runtime_get(dev: mhi_cntrl->cntrl_dev);
1113	}
1114
1115	static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl)
1116	{
1117	/* The runtime_put() MHI callback means:
1118	* Device can be moved in M3 state.
1119	*/
1120	pm_runtime_mark_last_busy(dev: mhi_cntrl->cntrl_dev);
1121	pm_runtime_put(dev: mhi_cntrl->cntrl_dev);
1122	}
1123
1124	static void mhi_pci_recovery_work(struct work_struct *work)
1125	{
1126	struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device,
1127	recovery_work);
1128	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1129	struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
1130	int err;
1131
1132	dev_warn(&pdev->dev, "device recovery started\n");
1133
1134	timer_delete(timer: &mhi_pdev->health_check_timer);
1135	pm_runtime_forbid(dev: &pdev->dev);
1136
1137	/* Clean up MHI state */
1138	if (test_and_clear_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status)) {
1139	mhi_power_down(mhi_cntrl, graceful: false);
1140	mhi_unprepare_after_power_down(mhi_cntrl);
1141	}
1142
1143	pci_set_power_state(dev: pdev, PCI_D0);
1144	pci_load_saved_state(dev: pdev, state: mhi_pdev->pci_state);
1145	pci_restore_state(dev: pdev);
1146
1147	if (!mhi_pci_is_alive(mhi_cntrl))
1148	goto err_try_reset;
1149
1150	err = mhi_prepare_for_power_up(mhi_cntrl);
1151	if (err)
1152	goto err_try_reset;
1153
1154	err = mhi_sync_power_up(mhi_cntrl);
1155	if (err)
1156	goto err_unprepare;
1157
1158	dev_dbg(&pdev->dev, "Recovery completed\n");
1159
1160	set_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status);
1161	mod_timer(timer: &mhi_pdev->health_check_timer, expires: jiffies + HEALTH_CHECK_PERIOD);
1162	return;
1163
1164	err_unprepare:
1165	mhi_unprepare_after_power_down(mhi_cntrl);
1166	err_try_reset:
1167	err = pci_try_reset_function(dev: pdev);
1168	if (err)
1169	dev_err(&pdev->dev, "Recovery failed: %d\n", err);
1170	}
1171
1172	static void health_check(struct timer_list *t)
1173	{
1174	struct mhi_pci_device *mhi_pdev = timer_container_of(mhi_pdev, t,
1175	health_check_timer);
1176	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1177
1178	if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
1179	test_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status))
1180	return;
1181
1182	if (!mhi_pci_is_alive(mhi_cntrl)) {
1183	dev_err(mhi_cntrl->cntrl_dev, "Device died\n");
1184	queue_work(wq: system_long_wq, work: &mhi_pdev->recovery_work);
1185	return;
1186	}
1187
1188	/* reschedule in two seconds */
1189	mod_timer(timer: &mhi_pdev->health_check_timer, expires: jiffies + HEALTH_CHECK_PERIOD);
1190	}
1191
1192	static int mhi_pci_generic_edl_trigger(struct mhi_controller *mhi_cntrl)
1193	{
1194	void __iomem *base = mhi_cntrl->regs;
1195	void __iomem *edl_db;
1196	int ret;
1197	u32 val;
1198
1199	ret = mhi_device_get_sync(mhi_dev: mhi_cntrl->mhi_dev);
1200	if (ret) {
1201	dev_err(mhi_cntrl->cntrl_dev, "Failed to wakeup the device\n");
1202	return ret;
1203	}
1204
1205	pm_wakeup_event(dev: &mhi_cntrl->mhi_dev->dev, msec: 0);
1206	mhi_cntrl->runtime_get(mhi_cntrl);
1207
1208	ret = mhi_get_channel_doorbell_offset(mhi_cntrl, chdb_offset: &val);
1209	if (ret)
1210	goto err_get_chdb;
1211
1212	edl_db = base + val + (8 * MHI_EDL_DB);
1213
1214	mhi_cntrl->write_reg(mhi_cntrl, edl_db + 4, upper_32_bits(MHI_EDL_COOKIE));
1215	mhi_cntrl->write_reg(mhi_cntrl, edl_db, lower_32_bits(MHI_EDL_COOKIE));
1216
1217	mhi_soc_reset(mhi_cntrl);
1218
1219	err_get_chdb:
1220	mhi_cntrl->runtime_put(mhi_cntrl);
1221	mhi_device_put(mhi_dev: mhi_cntrl->mhi_dev);
1222
1223	return ret;
1224	}
1225
1226	static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1227	{
1228	const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data;
1229	const struct mhi_controller_config *mhi_cntrl_config;
1230	struct mhi_pci_device *mhi_pdev;
1231	struct mhi_controller *mhi_cntrl;
1232	int err;
1233
1234	dev_info(&pdev->dev, "MHI PCI device found: %s\n", info->name);
1235
1236	/* mhi_pdev.mhi_cntrl must be zero-initialized */
1237	mhi_pdev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*mhi_pdev), GFP_KERNEL);
1238	if (!mhi_pdev)
1239	return -ENOMEM;
1240
1241	INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work);
1242	timer_setup(&mhi_pdev->health_check_timer, health_check, 0);
1243
1244	mhi_cntrl_config = info->config;
1245	mhi_cntrl = &mhi_pdev->mhi_cntrl;
1246
1247	mhi_cntrl->cntrl_dev = &pdev->dev;
1248	mhi_cntrl->iova_start = 0;
1249	mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width);
1250	mhi_cntrl->fw_image = info->fw;
1251	mhi_cntrl->edl_image = info->edl;
1252
1253	mhi_cntrl->read_reg = mhi_pci_read_reg;
1254	mhi_cntrl->write_reg = mhi_pci_write_reg;
1255	mhi_cntrl->status_cb = mhi_pci_status_cb;
1256	mhi_cntrl->runtime_get = mhi_pci_runtime_get;
1257	mhi_cntrl->runtime_put = mhi_pci_runtime_put;
1258	mhi_cntrl->mru = info->mru_default;
1259	mhi_cntrl->name = info->name;
1260
1261	if (info->edl_trigger)
1262	mhi_cntrl->edl_trigger = mhi_pci_generic_edl_trigger;
1263
1264	if (info->sideband_wake) {
1265	mhi_cntrl->wake_get = mhi_pci_wake_get_nop;
1266	mhi_cntrl->wake_put = mhi_pci_wake_put_nop;
1267	mhi_cntrl->wake_toggle = mhi_pci_wake_toggle_nop;
1268	}
1269
1270	err = mhi_pci_claim(mhi_cntrl, bar_num: info->bar_num, DMA_BIT_MASK(info->dma_data_width));
1271	if (err)
1272	return err;
1273
1274	err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config);
1275	if (err)
1276	return err;
1277
1278	pci_set_drvdata(pdev, data: mhi_pdev);
1279
1280	/* Have stored pci confspace at hand for restore in sudden PCI error.
1281	* cache the state locally and discard the PCI core one.
1282	*/
1283	pci_save_state(dev: pdev);
1284	mhi_pdev->pci_state = pci_store_saved_state(dev: pdev);
1285	pci_load_saved_state(dev: pdev, NULL);
1286
1287	err = mhi_register_controller(mhi_cntrl, config: mhi_cntrl_config);
1288	if (err)
1289	return err;
1290
1291	/* MHI bus does not power up the controller by default */
1292	err = mhi_prepare_for_power_up(mhi_cntrl);
1293	if (err) {
1294	dev_err(&pdev->dev, "failed to prepare MHI controller\n");
1295	goto err_unregister;
1296	}
1297
1298	err = mhi_sync_power_up(mhi_cntrl);
1299	if (err) {
1300	dev_err(&pdev->dev, "failed to power up MHI controller\n");
1301	goto err_unprepare;
1302	}
1303
1304	set_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status);
1305
1306	/* start health check */
1307	mod_timer(timer: &mhi_pdev->health_check_timer, expires: jiffies + HEALTH_CHECK_PERIOD);
1308
1309	/* Only allow runtime-suspend if PME capable (for wakeup) */
1310	if (pci_pme_capable(dev: pdev, PCI_D3hot)) {
1311	pm_runtime_set_autosuspend_delay(dev: &pdev->dev, delay: 2000);
1312	pm_runtime_use_autosuspend(dev: &pdev->dev);
1313	pm_runtime_mark_last_busy(dev: &pdev->dev);
1314	pm_runtime_put_noidle(dev: &pdev->dev);
1315	}
1316
1317	return 0;
1318
1319	err_unprepare:
1320	mhi_unprepare_after_power_down(mhi_cntrl);
1321	err_unregister:
1322	mhi_unregister_controller(mhi_cntrl);
1323
1324	return err;
1325	}
1326
1327	static void mhi_pci_remove(struct pci_dev *pdev)
1328	{
1329	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
1330	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1331
1332	timer_delete_sync(timer: &mhi_pdev->health_check_timer);
1333	cancel_work_sync(work: &mhi_pdev->recovery_work);
1334
1335	if (test_and_clear_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status)) {
1336	mhi_power_down(mhi_cntrl, graceful: true);
1337	mhi_unprepare_after_power_down(mhi_cntrl);
1338	}
1339
1340	/* balancing probe put_noidle */
1341	if (pci_pme_capable(dev: pdev, PCI_D3hot))
1342	pm_runtime_get_noresume(dev: &pdev->dev);
1343
1344	mhi_unregister_controller(mhi_cntrl);
1345	}
1346
1347	static void mhi_pci_shutdown(struct pci_dev *pdev)
1348	{
1349	mhi_pci_remove(pdev);
1350	pci_set_power_state(dev: pdev, PCI_D3hot);
1351	}
1352
1353	static void mhi_pci_reset_prepare(struct pci_dev *pdev)
1354	{
1355	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
1356	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1357
1358	dev_info(&pdev->dev, "reset\n");
1359
1360	timer_delete(timer: &mhi_pdev->health_check_timer);
1361
1362	/* Clean up MHI state */
1363	if (test_and_clear_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status)) {
1364	mhi_power_down(mhi_cntrl, graceful: false);
1365	mhi_unprepare_after_power_down(mhi_cntrl);
1366	}
1367
1368	/* cause internal device reset */
1369	mhi_soc_reset(mhi_cntrl);
1370
1371	/* Be sure device reset has been executed */
1372	msleep(MHI_POST_RESET_DELAY_MS);
1373	}
1374
1375	static void mhi_pci_reset_done(struct pci_dev *pdev)
1376	{
1377	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
1378	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1379	int err;
1380
1381	/* Restore initial known working PCI state */
1382	pci_load_saved_state(dev: pdev, state: mhi_pdev->pci_state);
1383	pci_restore_state(dev: pdev);
1384
1385	/* Is device status available ? */
1386	if (!mhi_pci_is_alive(mhi_cntrl)) {
1387	dev_err(&pdev->dev, "reset failed\n");
1388	return;
1389	}
1390
1391	err = mhi_prepare_for_power_up(mhi_cntrl);
1392	if (err) {
1393	dev_err(&pdev->dev, "failed to prepare MHI controller\n");
1394	return;
1395	}
1396
1397	err = mhi_sync_power_up(mhi_cntrl);
1398	if (err) {
1399	dev_err(&pdev->dev, "failed to power up MHI controller\n");
1400	mhi_unprepare_after_power_down(mhi_cntrl);
1401	return;
1402	}
1403
1404	set_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status);
1405	mod_timer(timer: &mhi_pdev->health_check_timer, expires: jiffies + HEALTH_CHECK_PERIOD);
1406	}
1407
1408	static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev,
1409	pci_channel_state_t state)
1410	{
1411	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
1412	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1413
1414	dev_err(&pdev->dev, "PCI error detected, state = %u\n", state);
1415
1416	if (state == pci_channel_io_perm_failure)
1417	return PCI_ERS_RESULT_DISCONNECT;
1418
1419	/* Clean up MHI state */
1420	if (test_and_clear_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status)) {
1421	mhi_power_down(mhi_cntrl, graceful: false);
1422	mhi_unprepare_after_power_down(mhi_cntrl);
1423	} else {
1424	/* Nothing to do */
1425	return PCI_ERS_RESULT_RECOVERED;
1426	}
1427
1428	pci_disable_device(dev: pdev);
1429
1430	return PCI_ERS_RESULT_NEED_RESET;
1431	}
1432
1433	static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev)
1434	{
1435	if (pci_enable_device(dev: pdev)) {
1436	dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n");
1437	return PCI_ERS_RESULT_DISCONNECT;
1438	}
1439
1440	return PCI_ERS_RESULT_RECOVERED;
1441	}
1442
1443	static void mhi_pci_io_resume(struct pci_dev *pdev)
1444	{
1445	struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
1446
1447	dev_err(&pdev->dev, "PCI slot reset done\n");
1448
1449	queue_work(wq: system_long_wq, work: &mhi_pdev->recovery_work);
1450	}
1451
1452	static const struct pci_error_handlers mhi_pci_err_handler = {
1453	.error_detected = mhi_pci_error_detected,
1454	.slot_reset = mhi_pci_slot_reset,
1455	.resume = mhi_pci_io_resume,
1456	.reset_prepare = mhi_pci_reset_prepare,
1457	.reset_done = mhi_pci_reset_done,
1458	};
1459
1460	static int __maybe_unused mhi_pci_runtime_suspend(struct device *dev)
1461	{
1462	struct pci_dev *pdev = to_pci_dev(dev);
1463	struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
1464	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1465	int err;
1466
1467	if (test_and_set_bit(nr: MHI_PCI_DEV_SUSPENDED, addr: &mhi_pdev->status))
1468	return 0;
1469
1470	timer_delete(timer: &mhi_pdev->health_check_timer);
1471	cancel_work_sync(work: &mhi_pdev->recovery_work);
1472
1473	if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
1474	mhi_cntrl->ee != MHI_EE_AMSS)
1475	goto pci_suspend; /* Nothing to do at MHI level */
1476
1477	/* Transition to M3 state */
1478	err = mhi_pm_suspend(mhi_cntrl);
1479	if (err) {
1480	dev_err(&pdev->dev, "failed to suspend device: %d\n", err);
1481	clear_bit(nr: MHI_PCI_DEV_SUSPENDED, addr: &mhi_pdev->status);
1482	return -EBUSY;
1483	}
1484
1485	pci_suspend:
1486	pci_disable_device(dev: pdev);
1487	pci_wake_from_d3(dev: pdev, enable: true);
1488
1489	return 0;
1490	}
1491
1492	static int __maybe_unused mhi_pci_runtime_resume(struct device *dev)
1493	{
1494	struct pci_dev *pdev = to_pci_dev(dev);
1495	struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
1496	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1497	int err;
1498
1499	if (!test_and_clear_bit(nr: MHI_PCI_DEV_SUSPENDED, addr: &mhi_pdev->status))
1500	return 0;
1501
1502	err = pci_enable_device(dev: pdev);
1503	if (err)
1504	goto err_recovery;
1505
1506	pci_set_master(dev: pdev);
1507	pci_wake_from_d3(dev: pdev, enable: false);
1508
1509	if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
1510	mhi_cntrl->ee != MHI_EE_AMSS)
1511	return 0; /* Nothing to do at MHI level */
1512
1513	/* Exit M3, transition to M0 state */
1514	err = mhi_pm_resume(mhi_cntrl);
1515	if (err) {
1516	dev_err(&pdev->dev, "failed to resume device: %d\n", err);
1517	goto err_recovery;
1518	}
1519
1520	/* Resume health check */
1521	mod_timer(timer: &mhi_pdev->health_check_timer, expires: jiffies + HEALTH_CHECK_PERIOD);
1522
1523	/* It can be a remote wakeup (no mhi runtime_get), update access time */
1524	pm_runtime_mark_last_busy(dev);
1525
1526	return 0;
1527
1528	err_recovery:
1529	/* Do not fail to not mess up our PCI device state, the device likely
1530	* lost power (d3cold) and we simply need to reset it from the recovery
1531	* procedure, trigger the recovery asynchronously to prevent system
1532	* suspend exit delaying.
1533	*/
1534	queue_work(wq: system_long_wq, work: &mhi_pdev->recovery_work);
1535	pm_runtime_mark_last_busy(dev);
1536
1537	return 0;
1538	}
1539
1540	static int __maybe_unused mhi_pci_suspend(struct device *dev)
1541	{
1542	pm_runtime_disable(dev);
1543	return mhi_pci_runtime_suspend(dev);
1544	}
1545
1546	static int __maybe_unused mhi_pci_resume(struct device *dev)
1547	{
1548	int ret;
1549
1550	/* Depending the platform, device may have lost power (d3cold), we need
1551	* to resume it now to check its state and recover when necessary.
1552	*/
1553	ret = mhi_pci_runtime_resume(dev);
1554	pm_runtime_enable(dev);
1555
1556	return ret;
1557	}
1558
1559	static int __maybe_unused mhi_pci_freeze(struct device *dev)
1560	{
1561	struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
1562	struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
1563
1564	/* We want to stop all operations, hibernation does not guarantee that
1565	* device will be in the same state as before freezing, especially if
1566	* the intermediate restore kernel reinitializes MHI device with new
1567	* context.
1568	*/
1569	flush_work(work: &mhi_pdev->recovery_work);
1570	if (test_and_clear_bit(nr: MHI_PCI_DEV_STARTED, addr: &mhi_pdev->status)) {
1571	mhi_power_down(mhi_cntrl, graceful: true);
1572	mhi_unprepare_after_power_down(mhi_cntrl);
1573	}
1574
1575	return 0;
1576	}
1577
1578	static int __maybe_unused mhi_pci_restore(struct device *dev)
1579	{
1580	struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
1581
1582	/* Reinitialize the device */
1583	queue_work(wq: system_long_wq, work: &mhi_pdev->recovery_work);
1584
1585	return 0;
1586	}
1587
1588	static const struct dev_pm_ops mhi_pci_pm_ops = {
1589	SET_RUNTIME_PM_OPS(mhi_pci_runtime_suspend, mhi_pci_runtime_resume, NULL)
1590	#ifdef CONFIG_PM_SLEEP
1591	.suspend = mhi_pci_suspend,
1592	.resume = mhi_pci_resume,
1593	.freeze = mhi_pci_freeze,
1594	.thaw = mhi_pci_restore,
1595	.poweroff = mhi_pci_freeze,
1596	.restore = mhi_pci_restore,
1597	#endif
1598	};
1599
1600	static struct pci_driver mhi_pci_driver = {
1601	.name = "mhi-pci-generic",
1602	.id_table = mhi_pci_id_table,
1603	.probe = mhi_pci_probe,
1604	.remove = mhi_pci_remove,
1605	.shutdown = mhi_pci_shutdown,
1606	.err_handler = &mhi_pci_err_handler,
1607	.driver.pm = &mhi_pci_pm_ops
1608	};
1609	module_pci_driver(mhi_pci_driver);
1610
1611	MODULE_AUTHOR("Loic Poulain <loic.poulain@linaro.org>");
1612	MODULE_DESCRIPTION("Modem Host Interface (MHI) PCI controller driver");
1613	MODULE_LICENSE("GPL");
1614