1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, MediaTek Inc. |
4 | * Copyright (c) 2021-2022, Intel Corporation. |
5 | * |
6 | * Authors: |
7 | * Haijun Liu <haijun.liu@mediatek.com> |
8 | * Ricardo Martinez <ricardo.martinez@linux.intel.com> |
9 | * Sreehari Kancharla <sreehari.kancharla@intel.com> |
10 | * |
11 | * Contributors: |
12 | * Amir Hanania <amir.hanania@intel.com> |
13 | * Andy Shevchenko <andriy.shevchenko@linux.intel.com> |
14 | * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> |
15 | * Eliot Lee <eliot.lee@intel.com> |
16 | * Moises Veleta <moises.veleta@intel.com> |
17 | */ |
18 | |
19 | #include <linux/atomic.h> |
20 | #include <linux/bits.h> |
21 | #include <linux/completion.h> |
22 | #include <linux/device.h> |
23 | #include <linux/dma-mapping.h> |
24 | #include <linux/gfp.h> |
25 | #include <linux/interrupt.h> |
26 | #include <linux/io.h> |
27 | #include <linux/iopoll.h> |
28 | #include <linux/jiffies.h> |
29 | #include <linux/list.h> |
30 | #include <linux/module.h> |
31 | #include <linux/mutex.h> |
32 | #include <linux/pci.h> |
33 | #include <linux/pm.h> |
34 | #include <linux/pm_runtime.h> |
35 | #include <linux/pm_wakeup.h> |
36 | #include <linux/spinlock.h> |
37 | |
38 | #include "t7xx_mhccif.h" |
39 | #include "t7xx_modem_ops.h" |
40 | #include "t7xx_pci.h" |
41 | #include "t7xx_pcie_mac.h" |
42 | #include "t7xx_reg.h" |
43 | #include "t7xx_state_monitor.h" |
44 | |
45 | #define T7XX_PCI_IREG_BASE 0 |
46 | #define T7XX_PCI_EREG_BASE 2 |
47 | |
48 | #define T7XX_INIT_TIMEOUT 20 |
49 | #define PM_SLEEP_DIS_TIMEOUT_MS 20 |
50 | #define PM_ACK_TIMEOUT_MS 1500 |
51 | #define PM_AUTOSUSPEND_MS 20000 |
52 | #define PM_RESOURCE_POLL_TIMEOUT_US 10000 |
53 | #define PM_RESOURCE_POLL_STEP_US 100 |
54 | |
55 | static const char * const t7xx_mode_names[] = { |
56 | [T7XX_UNKNOWN] = "unknown" , |
57 | [T7XX_READY] = "ready" , |
58 | [T7XX_RESET] = "reset" , |
59 | [T7XX_FASTBOOT_SWITCHING] = "fastboot_switching" , |
60 | [T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download" , |
61 | [T7XX_FASTBOOT_DUMP] = "fastboot_dump" , |
62 | }; |
63 | |
64 | static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST); |
65 | |
66 | static ssize_t t7xx_mode_store(struct device *dev, |
67 | struct device_attribute *attr, |
68 | const char *buf, size_t count) |
69 | { |
70 | struct t7xx_pci_dev *t7xx_dev; |
71 | struct pci_dev *pdev; |
72 | int index = 0; |
73 | |
74 | pdev = to_pci_dev(dev); |
75 | t7xx_dev = pci_get_drvdata(pdev); |
76 | if (!t7xx_dev) |
77 | return -ENODEV; |
78 | |
79 | index = sysfs_match_string(t7xx_mode_names, buf); |
80 | if (index == T7XX_FASTBOOT_SWITCHING) { |
81 | WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING); |
82 | } else if (index == T7XX_RESET) { |
83 | WRITE_ONCE(t7xx_dev->mode, T7XX_RESET); |
84 | t7xx_acpi_pldr_func(t7xx_dev); |
85 | } |
86 | |
87 | return count; |
88 | }; |
89 | |
90 | static ssize_t t7xx_mode_show(struct device *dev, |
91 | struct device_attribute *attr, |
92 | char *buf) |
93 | { |
94 | enum t7xx_mode mode = T7XX_UNKNOWN; |
95 | struct t7xx_pci_dev *t7xx_dev; |
96 | struct pci_dev *pdev; |
97 | |
98 | pdev = to_pci_dev(dev); |
99 | t7xx_dev = pci_get_drvdata(pdev); |
100 | if (!t7xx_dev) |
101 | return -ENODEV; |
102 | |
103 | mode = READ_ONCE(t7xx_dev->mode); |
104 | if (mode < T7XX_MODE_LAST) |
105 | return sysfs_emit(buf, fmt: "%s\n" , t7xx_mode_names[mode]); |
106 | |
107 | return sysfs_emit(buf, fmt: "%s\n" , t7xx_mode_names[T7XX_UNKNOWN]); |
108 | } |
109 | |
110 | static DEVICE_ATTR_RW(t7xx_mode); |
111 | |
112 | static struct attribute *t7xx_mode_attr[] = { |
113 | &dev_attr_t7xx_mode.attr, |
114 | NULL |
115 | }; |
116 | |
117 | static const struct attribute_group t7xx_mode_attribute_group = { |
118 | .attrs = t7xx_mode_attr, |
119 | }; |
120 | |
121 | void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode) |
122 | { |
123 | if (!t7xx_dev) |
124 | return; |
125 | |
126 | WRITE_ONCE(t7xx_dev->mode, mode); |
127 | sysfs_notify(kobj: &t7xx_dev->pdev->dev.kobj, NULL, attr: "t7xx_mode" ); |
128 | } |
129 | |
130 | enum t7xx_pm_state { |
131 | MTK_PM_EXCEPTION, |
132 | MTK_PM_INIT, /* Device initialized, but handshake not completed */ |
133 | MTK_PM_SUSPENDED, |
134 | MTK_PM_RESUMED, |
135 | }; |
136 | |
137 | static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable) |
138 | { |
139 | void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL; |
140 | u32 value; |
141 | |
142 | value = ioread32(ctrl_reg); |
143 | |
144 | if (enable) |
145 | value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS; |
146 | else |
147 | value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS; |
148 | |
149 | iowrite32(value, ctrl_reg); |
150 | } |
151 | |
152 | static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev) |
153 | { |
154 | int ret, val; |
155 | |
156 | ret = read_poll_timeout(ioread32, val, |
157 | (val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK, |
158 | PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true, |
159 | IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); |
160 | if (ret == -ETIMEDOUT) |
161 | dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n" ); |
162 | |
163 | return ret; |
164 | } |
165 | |
166 | static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev) |
167 | { |
168 | struct pci_dev *pdev = t7xx_dev->pdev; |
169 | |
170 | INIT_LIST_HEAD(list: &t7xx_dev->md_pm_entities); |
171 | mutex_init(&t7xx_dev->md_pm_entity_mtx); |
172 | spin_lock_init(&t7xx_dev->md_pm_lock); |
173 | init_completion(x: &t7xx_dev->sleep_lock_acquire); |
174 | init_completion(x: &t7xx_dev->pm_sr_ack); |
175 | init_completion(x: &t7xx_dev->init_done); |
176 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_INIT); |
177 | |
178 | device_init_wakeup(dev: &pdev->dev, enable: true); |
179 | dev_pm_set_driver_flags(dev: &pdev->dev, flags: pdev->dev.power.driver_flags | |
180 | DPM_FLAG_NO_DIRECT_COMPLETE); |
181 | |
182 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); |
183 | pm_runtime_set_autosuspend_delay(dev: &pdev->dev, PM_AUTOSUSPEND_MS); |
184 | pm_runtime_use_autosuspend(dev: &pdev->dev); |
185 | |
186 | return 0; |
187 | } |
188 | |
189 | void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev) |
190 | { |
191 | /* Enable the PCIe resource lock only after MD deep sleep is done */ |
192 | t7xx_mhccif_mask_clr(t7xx_dev, |
193 | D2H_INT_DS_LOCK_ACK | |
194 | D2H_INT_SUSPEND_ACK | |
195 | D2H_INT_RESUME_ACK | |
196 | D2H_INT_SUSPEND_ACK_AP | |
197 | D2H_INT_RESUME_ACK_AP); |
198 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
199 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_RESUMED); |
200 | |
201 | pm_runtime_mark_last_busy(dev: &t7xx_dev->pdev->dev); |
202 | pm_runtime_allow(dev: &t7xx_dev->pdev->dev); |
203 | pm_runtime_put_noidle(dev: &t7xx_dev->pdev->dev); |
204 | complete_all(&t7xx_dev->init_done); |
205 | } |
206 | |
207 | static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev) |
208 | { |
209 | /* The device is kept in FSM re-init flow |
210 | * so just roll back PM setting to the init setting. |
211 | */ |
212 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_INIT); |
213 | |
214 | pm_runtime_get_noresume(dev: &t7xx_dev->pdev->dev); |
215 | |
216 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); |
217 | return t7xx_wait_pm_config(t7xx_dev); |
218 | } |
219 | |
220 | void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev) |
221 | { |
222 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); |
223 | t7xx_wait_pm_config(t7xx_dev); |
224 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_EXCEPTION); |
225 | } |
226 | |
227 | int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) |
228 | { |
229 | struct md_pm_entity *entity; |
230 | |
231 | mutex_lock(&t7xx_dev->md_pm_entity_mtx); |
232 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
233 | if (entity->id == pm_entity->id) { |
234 | mutex_unlock(lock: &t7xx_dev->md_pm_entity_mtx); |
235 | return -EEXIST; |
236 | } |
237 | } |
238 | |
239 | list_add_tail(new: &pm_entity->entity, head: &t7xx_dev->md_pm_entities); |
240 | mutex_unlock(lock: &t7xx_dev->md_pm_entity_mtx); |
241 | return 0; |
242 | } |
243 | |
244 | int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity) |
245 | { |
246 | struct md_pm_entity *entity, *tmp_entity; |
247 | |
248 | mutex_lock(&t7xx_dev->md_pm_entity_mtx); |
249 | list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) { |
250 | if (entity->id == pm_entity->id) { |
251 | list_del(entry: &pm_entity->entity); |
252 | mutex_unlock(lock: &t7xx_dev->md_pm_entity_mtx); |
253 | return 0; |
254 | } |
255 | } |
256 | |
257 | mutex_unlock(lock: &t7xx_dev->md_pm_entity_mtx); |
258 | |
259 | return -ENXIO; |
260 | } |
261 | |
262 | int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev) |
263 | { |
264 | struct device *dev = &t7xx_dev->pdev->dev; |
265 | int ret; |
266 | |
267 | ret = wait_for_completion_timeout(x: &t7xx_dev->sleep_lock_acquire, |
268 | timeout: msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS)); |
269 | if (!ret) |
270 | dev_err_ratelimited(dev, "Resource wait complete timed out\n" ); |
271 | |
272 | return ret; |
273 | } |
274 | |
275 | /** |
276 | * t7xx_pci_disable_sleep() - Disable deep sleep capability. |
277 | * @t7xx_dev: MTK device. |
278 | * |
279 | * Lock the deep sleep capability, note that the device can still go into deep sleep |
280 | * state while device is in D0 state, from the host's point-of-view. |
281 | * |
282 | * If device is in deep sleep state, wake up the device and disable deep sleep capability. |
283 | */ |
284 | void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev) |
285 | { |
286 | unsigned long flags; |
287 | |
288 | spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); |
289 | t7xx_dev->sleep_disable_count++; |
290 | if (atomic_read(v: &t7xx_dev->md_pm_state) < MTK_PM_RESUMED) |
291 | goto unlock_and_complete; |
292 | |
293 | if (t7xx_dev->sleep_disable_count == 1) { |
294 | u32 status; |
295 | |
296 | reinit_completion(x: &t7xx_dev->sleep_lock_acquire); |
297 | t7xx_dev_set_sleep_capability(t7xx_dev, enable: false); |
298 | |
299 | status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS); |
300 | if (status & T7XX_PCIE_RESOURCE_STS_MSK) |
301 | goto unlock_and_complete; |
302 | |
303 | t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK); |
304 | } |
305 | spin_unlock_irqrestore(lock: &t7xx_dev->md_pm_lock, flags); |
306 | return; |
307 | |
308 | unlock_and_complete: |
309 | spin_unlock_irqrestore(lock: &t7xx_dev->md_pm_lock, flags); |
310 | complete_all(&t7xx_dev->sleep_lock_acquire); |
311 | } |
312 | |
313 | /** |
314 | * t7xx_pci_enable_sleep() - Enable deep sleep capability. |
315 | * @t7xx_dev: MTK device. |
316 | * |
317 | * After enabling deep sleep, device can enter into deep sleep state. |
318 | */ |
319 | void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev) |
320 | { |
321 | unsigned long flags; |
322 | |
323 | spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags); |
324 | t7xx_dev->sleep_disable_count--; |
325 | if (atomic_read(v: &t7xx_dev->md_pm_state) < MTK_PM_RESUMED) |
326 | goto unlock; |
327 | |
328 | if (t7xx_dev->sleep_disable_count == 0) |
329 | t7xx_dev_set_sleep_capability(t7xx_dev, enable: true); |
330 | |
331 | unlock: |
332 | spin_unlock_irqrestore(lock: &t7xx_dev->md_pm_lock, flags); |
333 | } |
334 | |
335 | static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request) |
336 | { |
337 | unsigned long wait_ret; |
338 | |
339 | reinit_completion(x: &t7xx_dev->pm_sr_ack); |
340 | t7xx_mhccif_h2d_swint_trigger(t7xx_dev, channel: request); |
341 | wait_ret = wait_for_completion_timeout(x: &t7xx_dev->pm_sr_ack, |
342 | timeout: msecs_to_jiffies(PM_ACK_TIMEOUT_MS)); |
343 | if (!wait_ret) |
344 | return -ETIMEDOUT; |
345 | |
346 | return 0; |
347 | } |
348 | |
349 | static int __t7xx_pci_pm_suspend(struct pci_dev *pdev) |
350 | { |
351 | enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID; |
352 | struct t7xx_pci_dev *t7xx_dev; |
353 | struct md_pm_entity *entity; |
354 | int ret; |
355 | |
356 | t7xx_dev = pci_get_drvdata(pdev); |
357 | if (atomic_read(v: &t7xx_dev->md_pm_state) <= MTK_PM_INIT || |
358 | READ_ONCE(t7xx_dev->mode) != T7XX_READY) { |
359 | dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n" ); |
360 | return -EFAULT; |
361 | } |
362 | |
363 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); |
364 | ret = t7xx_wait_pm_config(t7xx_dev); |
365 | if (ret) { |
366 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
367 | return ret; |
368 | } |
369 | |
370 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_SUSPENDED); |
371 | t7xx_pcie_mac_clear_int(t7xx_dev, int_type: SAP_RGU_INT); |
372 | t7xx_dev->rgu_pci_irq_en = false; |
373 | |
374 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
375 | if (!entity->suspend) |
376 | continue; |
377 | |
378 | ret = entity->suspend(t7xx_dev, entity->entity_param); |
379 | if (ret) { |
380 | entity_id = entity->id; |
381 | dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n" , ret, entity_id); |
382 | goto abort_suspend; |
383 | } |
384 | } |
385 | |
386 | ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ); |
387 | if (ret) { |
388 | dev_err(&pdev->dev, "[PM] MD suspend error: %d\n" , ret); |
389 | goto abort_suspend; |
390 | } |
391 | |
392 | ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP); |
393 | if (ret) { |
394 | t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); |
395 | dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n" , ret); |
396 | goto abort_suspend; |
397 | } |
398 | |
399 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
400 | if (entity->suspend_late) |
401 | entity->suspend_late(t7xx_dev, entity->entity_param); |
402 | } |
403 | |
404 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
405 | return 0; |
406 | |
407 | abort_suspend: |
408 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
409 | if (entity_id == entity->id) |
410 | break; |
411 | |
412 | if (entity->resume) |
413 | entity->resume(t7xx_dev, entity->entity_param); |
414 | } |
415 | |
416 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
417 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_RESUMED); |
418 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: SAP_RGU_INT); |
419 | return ret; |
420 | } |
421 | |
422 | static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev) |
423 | { |
424 | t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); |
425 | |
426 | /* Disable interrupt first and let the IPs enable them */ |
427 | iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0); |
428 | |
429 | /* Device disables PCIe interrupts during resume and |
430 | * following function will re-enable PCIe interrupts. |
431 | */ |
432 | t7xx_pcie_mac_interrupts_en(t7xx_dev); |
433 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: MHCCIF_INT); |
434 | } |
435 | |
436 | static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3) |
437 | { |
438 | int ret; |
439 | |
440 | ret = pcim_enable_device(pdev: t7xx_dev->pdev); |
441 | if (ret) |
442 | return ret; |
443 | |
444 | t7xx_pcie_mac_atr_init(t7xx_dev); |
445 | t7xx_pcie_interrupt_reinit(t7xx_dev); |
446 | |
447 | if (is_d3) { |
448 | t7xx_mhccif_init(t7xx_dev); |
449 | return t7xx_pci_pm_reinit(t7xx_dev); |
450 | } |
451 | |
452 | return 0; |
453 | } |
454 | |
455 | static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event) |
456 | { |
457 | struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl; |
458 | struct device *dev = &t7xx_dev->pdev->dev; |
459 | int ret = -EINVAL; |
460 | |
461 | switch (event) { |
462 | case FSM_CMD_STOP: |
463 | ret = t7xx_fsm_append_cmd(ctl: fsm_ctl, cmd_id: FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION); |
464 | break; |
465 | |
466 | case FSM_CMD_START: |
467 | t7xx_pcie_mac_clear_int(t7xx_dev, int_type: SAP_RGU_INT); |
468 | t7xx_pcie_mac_clear_int_status(t7xx_dev, int_type: SAP_RGU_INT); |
469 | t7xx_dev->rgu_pci_irq_en = true; |
470 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: SAP_RGU_INT); |
471 | ret = t7xx_fsm_append_cmd(ctl: fsm_ctl, cmd_id: FSM_CMD_START, flag: 0); |
472 | break; |
473 | |
474 | default: |
475 | break; |
476 | } |
477 | |
478 | if (ret) |
479 | dev_err(dev, "Failure handling FSM command %u, %d\n" , event, ret); |
480 | |
481 | return ret; |
482 | } |
483 | |
484 | static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check) |
485 | { |
486 | struct t7xx_pci_dev *t7xx_dev; |
487 | struct md_pm_entity *entity; |
488 | u32 prev_state; |
489 | int ret = 0; |
490 | |
491 | t7xx_dev = pci_get_drvdata(pdev); |
492 | if (atomic_read(v: &t7xx_dev->md_pm_state) <= MTK_PM_INIT) { |
493 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
494 | return 0; |
495 | } |
496 | |
497 | t7xx_pcie_mac_interrupts_en(t7xx_dev); |
498 | prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE); |
499 | |
500 | if (state_check) { |
501 | /* For D3/L3 resume, the device could boot so quickly that the |
502 | * initial value of the dummy register might be overwritten. |
503 | * Identify new boots if the ATR source address register is not initialized. |
504 | */ |
505 | u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) + |
506 | ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR); |
507 | if (prev_state == PM_RESUME_REG_STATE_L3 || |
508 | (prev_state == PM_RESUME_REG_STATE_INIT && |
509 | atr_reg_val == ATR_SRC_ADDR_INVALID)) { |
510 | ret = t7xx_send_fsm_command(t7xx_dev, event: FSM_CMD_STOP); |
511 | if (ret) |
512 | return ret; |
513 | |
514 | ret = t7xx_pcie_reinit(t7xx_dev, is_d3: true); |
515 | if (ret) |
516 | return ret; |
517 | |
518 | t7xx_clear_rgu_irq(t7xx_dev); |
519 | return t7xx_send_fsm_command(t7xx_dev, event: FSM_CMD_START); |
520 | } |
521 | |
522 | if (prev_state == PM_RESUME_REG_STATE_EXP || |
523 | prev_state == PM_RESUME_REG_STATE_L2_EXP) { |
524 | if (prev_state == PM_RESUME_REG_STATE_L2_EXP) { |
525 | ret = t7xx_pcie_reinit(t7xx_dev, is_d3: false); |
526 | if (ret) |
527 | return ret; |
528 | } |
529 | |
530 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_SUSPENDED); |
531 | t7xx_dev->rgu_pci_irq_en = true; |
532 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: SAP_RGU_INT); |
533 | |
534 | t7xx_mhccif_mask_clr(t7xx_dev, |
535 | D2H_INT_EXCEPTION_INIT | |
536 | D2H_INT_EXCEPTION_INIT_DONE | |
537 | D2H_INT_EXCEPTION_CLEARQ_DONE | |
538 | D2H_INT_EXCEPTION_ALLQ_RESET | |
539 | D2H_INT_PORT_ENUM); |
540 | |
541 | return ret; |
542 | } |
543 | |
544 | if (prev_state == PM_RESUME_REG_STATE_L2) { |
545 | ret = t7xx_pcie_reinit(t7xx_dev, is_d3: false); |
546 | if (ret) |
547 | return ret; |
548 | |
549 | } else if (prev_state != PM_RESUME_REG_STATE_L1 && |
550 | prev_state != PM_RESUME_REG_STATE_INIT) { |
551 | ret = t7xx_send_fsm_command(t7xx_dev, event: FSM_CMD_STOP); |
552 | if (ret) |
553 | return ret; |
554 | |
555 | t7xx_clear_rgu_irq(t7xx_dev); |
556 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_SUSPENDED); |
557 | return 0; |
558 | } |
559 | } |
560 | |
561 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR); |
562 | t7xx_wait_pm_config(t7xx_dev); |
563 | |
564 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
565 | if (entity->resume_early) |
566 | entity->resume_early(t7xx_dev, entity->entity_param); |
567 | } |
568 | |
569 | ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); |
570 | if (ret) |
571 | dev_err(&pdev->dev, "[PM] MD resume error: %d\n" , ret); |
572 | |
573 | ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP); |
574 | if (ret) |
575 | dev_err(&pdev->dev, "[PM] SAP resume error: %d\n" , ret); |
576 | |
577 | list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) { |
578 | if (entity->resume) { |
579 | ret = entity->resume(t7xx_dev, entity->entity_param); |
580 | if (ret) |
581 | dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n" , |
582 | entity->id, ret); |
583 | } |
584 | } |
585 | |
586 | t7xx_dev->rgu_pci_irq_en = true; |
587 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: SAP_RGU_INT); |
588 | iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR); |
589 | pm_runtime_mark_last_busy(dev: &pdev->dev); |
590 | atomic_set(v: &t7xx_dev->md_pm_state, i: MTK_PM_RESUMED); |
591 | |
592 | return ret; |
593 | } |
594 | |
595 | static int t7xx_pci_pm_resume_noirq(struct device *dev) |
596 | { |
597 | struct pci_dev *pdev = to_pci_dev(dev); |
598 | struct t7xx_pci_dev *t7xx_dev; |
599 | |
600 | t7xx_dev = pci_get_drvdata(pdev); |
601 | t7xx_pcie_mac_interrupts_dis(t7xx_dev); |
602 | |
603 | return 0; |
604 | } |
605 | |
606 | static void t7xx_pci_shutdown(struct pci_dev *pdev) |
607 | { |
608 | __t7xx_pci_pm_suspend(pdev); |
609 | } |
610 | |
611 | static int t7xx_pci_pm_prepare(struct device *dev) |
612 | { |
613 | struct pci_dev *pdev = to_pci_dev(dev); |
614 | struct t7xx_pci_dev *t7xx_dev; |
615 | |
616 | t7xx_dev = pci_get_drvdata(pdev); |
617 | if (!wait_for_completion_timeout(x: &t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) { |
618 | dev_warn(dev, "Not ready for system sleep.\n" ); |
619 | return -ETIMEDOUT; |
620 | } |
621 | |
622 | return 0; |
623 | } |
624 | |
625 | static int t7xx_pci_pm_suspend(struct device *dev) |
626 | { |
627 | return __t7xx_pci_pm_suspend(to_pci_dev(dev)); |
628 | } |
629 | |
630 | static int t7xx_pci_pm_resume(struct device *dev) |
631 | { |
632 | return __t7xx_pci_pm_resume(to_pci_dev(dev), state_check: true); |
633 | } |
634 | |
635 | static int t7xx_pci_pm_thaw(struct device *dev) |
636 | { |
637 | return __t7xx_pci_pm_resume(to_pci_dev(dev), state_check: false); |
638 | } |
639 | |
640 | static int t7xx_pci_pm_runtime_suspend(struct device *dev) |
641 | { |
642 | return __t7xx_pci_pm_suspend(to_pci_dev(dev)); |
643 | } |
644 | |
645 | static int t7xx_pci_pm_runtime_resume(struct device *dev) |
646 | { |
647 | return __t7xx_pci_pm_resume(to_pci_dev(dev), state_check: true); |
648 | } |
649 | |
650 | static const struct dev_pm_ops t7xx_pci_pm_ops = { |
651 | .prepare = t7xx_pci_pm_prepare, |
652 | .suspend = t7xx_pci_pm_suspend, |
653 | .resume = t7xx_pci_pm_resume, |
654 | .resume_noirq = t7xx_pci_pm_resume_noirq, |
655 | .freeze = t7xx_pci_pm_suspend, |
656 | .thaw = t7xx_pci_pm_thaw, |
657 | .poweroff = t7xx_pci_pm_suspend, |
658 | .restore = t7xx_pci_pm_resume, |
659 | .restore_noirq = t7xx_pci_pm_resume_noirq, |
660 | .runtime_suspend = t7xx_pci_pm_runtime_suspend, |
661 | .runtime_resume = t7xx_pci_pm_runtime_resume |
662 | }; |
663 | |
664 | static int t7xx_request_irq(struct pci_dev *pdev) |
665 | { |
666 | struct t7xx_pci_dev *t7xx_dev; |
667 | int ret = 0, i; |
668 | |
669 | t7xx_dev = pci_get_drvdata(pdev); |
670 | |
671 | for (i = 0; i < EXT_INT_NUM; i++) { |
672 | const char *irq_descr; |
673 | int irq_vec; |
674 | |
675 | if (!t7xx_dev->intr_handler[i]) |
676 | continue; |
677 | |
678 | irq_descr = devm_kasprintf(dev: &pdev->dev, GFP_KERNEL, fmt: "%s_%d" , |
679 | dev_driver_string(dev: &pdev->dev), i); |
680 | if (!irq_descr) { |
681 | ret = -ENOMEM; |
682 | break; |
683 | } |
684 | |
685 | irq_vec = pci_irq_vector(dev: pdev, nr: i); |
686 | ret = request_threaded_irq(irq: irq_vec, handler: t7xx_dev->intr_handler[i], |
687 | thread_fn: t7xx_dev->intr_thread[i], flags: 0, name: irq_descr, |
688 | dev: t7xx_dev->callback_param[i]); |
689 | if (ret) { |
690 | dev_err(&pdev->dev, "Failed to request IRQ: %d\n" , ret); |
691 | break; |
692 | } |
693 | } |
694 | |
695 | if (ret) { |
696 | while (i--) { |
697 | if (!t7xx_dev->intr_handler[i]) |
698 | continue; |
699 | |
700 | free_irq(pci_irq_vector(dev: pdev, nr: i), t7xx_dev->callback_param[i]); |
701 | } |
702 | } |
703 | |
704 | return ret; |
705 | } |
706 | |
707 | static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev) |
708 | { |
709 | struct pci_dev *pdev = t7xx_dev->pdev; |
710 | int ret; |
711 | |
712 | /* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */ |
713 | ret = pci_alloc_irq_vectors(dev: pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX); |
714 | if (ret < 0) { |
715 | dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n" , ret); |
716 | return ret; |
717 | } |
718 | |
719 | ret = t7xx_request_irq(pdev); |
720 | if (ret) { |
721 | pci_free_irq_vectors(dev: pdev); |
722 | return ret; |
723 | } |
724 | |
725 | t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM); |
726 | return 0; |
727 | } |
728 | |
729 | static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev) |
730 | { |
731 | int ret, i; |
732 | |
733 | if (!t7xx_dev->pdev->msix_cap) |
734 | return -EINVAL; |
735 | |
736 | ret = t7xx_setup_msix(t7xx_dev); |
737 | if (ret) |
738 | return ret; |
739 | |
740 | /* IPs enable interrupts when ready */ |
741 | for (i = 0; i < EXT_INT_NUM; i++) |
742 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: i); |
743 | |
744 | return 0; |
745 | } |
746 | |
747 | static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev) |
748 | { |
749 | t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base + |
750 | INFRACFG_AO_DEV_CHIP - |
751 | t7xx_dev->base_addr.pcie_dev_reg_trsl_addr; |
752 | } |
753 | |
754 | static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
755 | { |
756 | struct t7xx_pci_dev *t7xx_dev; |
757 | int ret; |
758 | |
759 | t7xx_dev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*t7xx_dev), GFP_KERNEL); |
760 | if (!t7xx_dev) |
761 | return -ENOMEM; |
762 | |
763 | pci_set_drvdata(pdev, data: t7xx_dev); |
764 | t7xx_dev->pdev = pdev; |
765 | |
766 | ret = pcim_enable_device(pdev); |
767 | if (ret) |
768 | return ret; |
769 | |
770 | pci_set_master(dev: pdev); |
771 | |
772 | ret = pcim_iomap_regions(pdev, BIT(T7XX_PCI_IREG_BASE) | BIT(T7XX_PCI_EREG_BASE), |
773 | name: pci_name(pdev)); |
774 | if (ret) { |
775 | dev_err(&pdev->dev, "Could not request BARs: %d\n" , ret); |
776 | return -ENOMEM; |
777 | } |
778 | |
779 | ret = dma_set_mask(dev: &pdev->dev, DMA_BIT_MASK(64)); |
780 | if (ret) { |
781 | dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n" , ret); |
782 | return ret; |
783 | } |
784 | |
785 | ret = dma_set_coherent_mask(dev: &pdev->dev, DMA_BIT_MASK(64)); |
786 | if (ret) { |
787 | dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n" , ret); |
788 | return ret; |
789 | } |
790 | |
791 | IREG_BASE(t7xx_dev) = pcim_iomap_table(pdev)[T7XX_PCI_IREG_BASE]; |
792 | t7xx_dev->base_addr.pcie_ext_reg_base = pcim_iomap_table(pdev)[T7XX_PCI_EREG_BASE]; |
793 | |
794 | ret = t7xx_pci_pm_init(t7xx_dev); |
795 | if (ret) |
796 | return ret; |
797 | |
798 | t7xx_pcie_mac_atr_init(t7xx_dev); |
799 | t7xx_pci_infracfg_ao_calc(t7xx_dev); |
800 | t7xx_mhccif_init(t7xx_dev); |
801 | |
802 | ret = t7xx_md_init(t7xx_dev); |
803 | if (ret) |
804 | return ret; |
805 | |
806 | t7xx_pcie_mac_interrupts_dis(t7xx_dev); |
807 | |
808 | ret = sysfs_create_group(kobj: &t7xx_dev->pdev->dev.kobj, |
809 | grp: &t7xx_mode_attribute_group); |
810 | if (ret) |
811 | goto err_md_exit; |
812 | |
813 | ret = t7xx_interrupt_init(t7xx_dev); |
814 | if (ret) |
815 | goto err_remove_group; |
816 | |
817 | |
818 | t7xx_pcie_mac_set_int(t7xx_dev, int_type: MHCCIF_INT); |
819 | t7xx_pcie_mac_interrupts_en(t7xx_dev); |
820 | |
821 | return 0; |
822 | |
823 | err_remove_group: |
824 | sysfs_remove_group(kobj: &t7xx_dev->pdev->dev.kobj, |
825 | grp: &t7xx_mode_attribute_group); |
826 | |
827 | err_md_exit: |
828 | t7xx_md_exit(t7xx_dev); |
829 | return ret; |
830 | } |
831 | |
832 | static void t7xx_pci_remove(struct pci_dev *pdev) |
833 | { |
834 | struct t7xx_pci_dev *t7xx_dev; |
835 | int i; |
836 | |
837 | t7xx_dev = pci_get_drvdata(pdev); |
838 | |
839 | sysfs_remove_group(kobj: &t7xx_dev->pdev->dev.kobj, |
840 | grp: &t7xx_mode_attribute_group); |
841 | t7xx_md_exit(t7xx_dev); |
842 | |
843 | for (i = 0; i < EXT_INT_NUM; i++) { |
844 | if (!t7xx_dev->intr_handler[i]) |
845 | continue; |
846 | |
847 | free_irq(pci_irq_vector(dev: pdev, nr: i), t7xx_dev->callback_param[i]); |
848 | } |
849 | |
850 | pci_free_irq_vectors(dev: t7xx_dev->pdev); |
851 | } |
852 | |
853 | static const struct pci_device_id t7xx_pci_table[] = { |
854 | { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) }, |
855 | { } |
856 | }; |
857 | MODULE_DEVICE_TABLE(pci, t7xx_pci_table); |
858 | |
859 | static struct pci_driver t7xx_pci_driver = { |
860 | .name = "mtk_t7xx" , |
861 | .id_table = t7xx_pci_table, |
862 | .probe = t7xx_pci_probe, |
863 | .remove = t7xx_pci_remove, |
864 | .driver.pm = &t7xx_pci_pm_ops, |
865 | .shutdown = t7xx_pci_shutdown, |
866 | }; |
867 | |
868 | module_pci_driver(t7xx_pci_driver); |
869 | |
870 | MODULE_AUTHOR("MediaTek Inc" ); |
871 | MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver" ); |
872 | MODULE_LICENSE("GPL" ); |
873 | |