1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2019 MediaTek Inc.
4 * Authors:
5 * Stanley Chu <stanley.chu@mediatek.com>
6 * Peter Wang <peter.wang@mediatek.com>
7 */
8
9#include <linux/arm-smccc.h>
10#include <linux/bitfield.h>
11#include <linux/clk.h>
12#include <linux/delay.h>
13#include <linux/module.h>
14#include <linux/of.h>
15#include <linux/of_address.h>
16#include <linux/of_device.h>
17#include <linux/of_platform.h>
18#include <linux/phy/phy.h>
19#include <linux/platform_device.h>
20#include <linux/regulator/consumer.h>
21#include <linux/reset.h>
22#include <linux/soc/mediatek/mtk_sip_svc.h>
23
24#include <ufs/ufshcd.h>
25#include "ufshcd-pltfrm.h"
26#include <ufs/ufs_quirks.h>
27#include <ufs/unipro.h>
28#include "ufs-mediatek.h"
29
30static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq);
31
32#define CREATE_TRACE_POINTS
33#include "ufs-mediatek-trace.h"
34#undef CREATE_TRACE_POINTS
35
36#define MAX_SUPP_MAC 64
37#define MCQ_QUEUE_OFFSET(c) ((((c) >> 16) & 0xFF) * 0x200)
38
39static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = {
40 { .wmanufacturerid = UFS_ANY_VENDOR,
41 .model = UFS_ANY_MODEL,
42 .quirk = UFS_DEVICE_QUIRK_DELAY_AFTER_LPM |
43 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
44 { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
45 .model = "H9HQ21AFAMZDAR",
46 .quirk = UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES },
47 {}
48};
49
50static const struct of_device_id ufs_mtk_of_match[] = {
51 { .compatible = "mediatek,mt8183-ufshci" },
52 {},
53};
54
55/*
56 * Details of UIC Errors
57 */
58static const char *const ufs_uic_err_str[] = {
59 "PHY Adapter Layer",
60 "Data Link Layer",
61 "Network Link Layer",
62 "Transport Link Layer",
63 "DME"
64};
65
66static const char *const ufs_uic_pa_err_str[] = {
67 "PHY error on Lane 0",
68 "PHY error on Lane 1",
69 "PHY error on Lane 2",
70 "PHY error on Lane 3",
71 "Generic PHY Adapter Error. This should be the LINERESET indication"
72};
73
74static const char *const ufs_uic_dl_err_str[] = {
75 "NAC_RECEIVED",
76 "TCx_REPLAY_TIMER_EXPIRED",
77 "AFCx_REQUEST_TIMER_EXPIRED",
78 "FCx_PROTECTION_TIMER_EXPIRED",
79 "CRC_ERROR",
80 "RX_BUFFER_OVERFLOW",
81 "MAX_FRAME_LENGTH_EXCEEDED",
82 "WRONG_SEQUENCE_NUMBER",
83 "AFC_FRAME_SYNTAX_ERROR",
84 "NAC_FRAME_SYNTAX_ERROR",
85 "EOF_SYNTAX_ERROR",
86 "FRAME_SYNTAX_ERROR",
87 "BAD_CTRL_SYMBOL_TYPE",
88 "PA_INIT_ERROR",
89 "PA_ERROR_IND_RECEIVED",
90 "PA_INIT"
91};
92
93static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba)
94{
95 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
96
97 return !!(host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE);
98}
99
100static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba)
101{
102 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
103
104 return !!(host->caps & UFS_MTK_CAP_VA09_PWR_CTRL);
105}
106
107static bool ufs_mtk_is_broken_vcc(struct ufs_hba *hba)
108{
109 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
110
111 return !!(host->caps & UFS_MTK_CAP_BROKEN_VCC);
112}
113
114static bool ufs_mtk_is_pmc_via_fastauto(struct ufs_hba *hba)
115{
116 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
117
118 return !!(host->caps & UFS_MTK_CAP_PMC_VIA_FASTAUTO);
119}
120
121static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
122{
123 u32 tmp;
124
125 if (enable) {
126 ufshcd_dme_get(hba,
127 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: &tmp);
128 tmp = tmp |
129 (1 << RX_SYMBOL_CLK_GATE_EN) |
130 (1 << SYS_CLK_GATE_EN) |
131 (1 << TX_CLK_GATE_EN);
132 ufshcd_dme_set(hba,
133 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: tmp);
134
135 ufshcd_dme_get(hba,
136 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), mib_val: &tmp);
137 tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE);
138 ufshcd_dme_set(hba,
139 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), mib_val: tmp);
140 } else {
141 ufshcd_dme_get(hba,
142 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: &tmp);
143 tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) |
144 (1 << SYS_CLK_GATE_EN) |
145 (1 << TX_CLK_GATE_EN));
146 ufshcd_dme_set(hba,
147 UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: tmp);
148
149 ufshcd_dme_get(hba,
150 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), mib_val: &tmp);
151 tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE);
152 ufshcd_dme_set(hba,
153 UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), mib_val: tmp);
154 }
155}
156
157static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
158{
159 struct arm_smccc_res res;
160
161 ufs_mtk_crypto_ctrl(res, 1);
162 if (res.a0) {
163 dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n",
164 __func__, res.a0);
165 hba->caps &= ~UFSHCD_CAP_CRYPTO;
166 }
167}
168
169static void ufs_mtk_host_reset(struct ufs_hba *hba)
170{
171 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
172
173 reset_control_assert(rstc: host->hci_reset);
174 reset_control_assert(rstc: host->crypto_reset);
175 reset_control_assert(rstc: host->unipro_reset);
176
177 usleep_range(min: 100, max: 110);
178
179 reset_control_deassert(rstc: host->unipro_reset);
180 reset_control_deassert(rstc: host->crypto_reset);
181 reset_control_deassert(rstc: host->hci_reset);
182}
183
184static void ufs_mtk_init_reset_control(struct ufs_hba *hba,
185 struct reset_control **rc,
186 char *str)
187{
188 *rc = devm_reset_control_get(dev: hba->dev, id: str);
189 if (IS_ERR(ptr: *rc)) {
190 dev_info(hba->dev, "Failed to get reset control %s: %ld\n",
191 str, PTR_ERR(*rc));
192 *rc = NULL;
193 }
194}
195
196static void ufs_mtk_init_reset(struct ufs_hba *hba)
197{
198 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
199
200 ufs_mtk_init_reset_control(hba, rc: &host->hci_reset,
201 str: "hci_rst");
202 ufs_mtk_init_reset_control(hba, rc: &host->unipro_reset,
203 str: "unipro_rst");
204 ufs_mtk_init_reset_control(hba, rc: &host->crypto_reset,
205 str: "crypto_rst");
206}
207
208static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
209 enum ufs_notify_change_status status)
210{
211 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
212
213 if (status == PRE_CHANGE) {
214 if (host->unipro_lpm) {
215 hba->vps->hba_enable_delay_us = 0;
216 } else {
217 hba->vps->hba_enable_delay_us = 600;
218 ufs_mtk_host_reset(hba);
219 }
220
221 if (hba->caps & UFSHCD_CAP_CRYPTO)
222 ufs_mtk_crypto_enable(hba);
223
224 if (host->caps & UFS_MTK_CAP_DISABLE_AH8) {
225 ufshcd_writel(hba, 0,
226 REG_AUTO_HIBERNATE_IDLE_TIMER);
227 hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT;
228 hba->ahit = 0;
229 }
230
231 /*
232 * Turn on CLK_CG early to bypass abnormal ERR_CHK signal
233 * to prevent host hang issue
234 */
235 ufshcd_writel(hba,
236 ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) | 0x80,
237 REG_UFS_XOUFS_CTRL);
238 }
239
240 return 0;
241}
242
243static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
244{
245 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
246 struct device *dev = hba->dev;
247 struct device_node *np = dev->of_node;
248 int err = 0;
249
250 host->mphy = devm_of_phy_get_by_index(dev, np, index: 0);
251
252 if (host->mphy == ERR_PTR(error: -EPROBE_DEFER)) {
253 /*
254 * UFS driver might be probed before the phy driver does.
255 * In that case we would like to return EPROBE_DEFER code.
256 */
257 err = -EPROBE_DEFER;
258 dev_info(dev,
259 "%s: required phy hasn't probed yet. err = %d\n",
260 __func__, err);
261 } else if (IS_ERR(ptr: host->mphy)) {
262 err = PTR_ERR(ptr: host->mphy);
263 if (err != -ENODEV) {
264 dev_info(dev, "%s: PHY get failed %d\n", __func__,
265 err);
266 }
267 }
268
269 if (err)
270 host->mphy = NULL;
271 /*
272 * Allow unbound mphy because not every platform needs specific
273 * mphy control.
274 */
275 if (err == -ENODEV)
276 err = 0;
277
278 return err;
279}
280
281static int ufs_mtk_setup_ref_clk(struct ufs_hba *hba, bool on)
282{
283 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
284 struct arm_smccc_res res;
285 ktime_t timeout, time_checked;
286 u32 value;
287
288 if (host->ref_clk_enabled == on)
289 return 0;
290
291 ufs_mtk_ref_clk_notify(on, PRE_CHANGE, res);
292
293 if (on) {
294 ufshcd_writel(hba, REFCLK_REQUEST, REG_UFS_REFCLK_CTRL);
295 } else {
296 ufshcd_delay_us(us: host->ref_clk_gating_wait_us, tolerance: 10);
297 ufshcd_writel(hba, REFCLK_RELEASE, REG_UFS_REFCLK_CTRL);
298 }
299
300 /* Wait for ack */
301 timeout = ktime_add_us(kt: ktime_get(), REFCLK_REQ_TIMEOUT_US);
302 do {
303 time_checked = ktime_get();
304 value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL);
305
306 /* Wait until ack bit equals to req bit */
307 if (((value & REFCLK_ACK) >> 1) == (value & REFCLK_REQUEST))
308 goto out;
309
310 usleep_range(min: 100, max: 200);
311 } while (ktime_before(cmp1: time_checked, cmp2: timeout));
312
313 dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value);
314
315 ufs_mtk_ref_clk_notify(host->ref_clk_enabled, POST_CHANGE, res);
316
317 return -ETIMEDOUT;
318
319out:
320 host->ref_clk_enabled = on;
321 if (on)
322 ufshcd_delay_us(us: host->ref_clk_ungating_wait_us, tolerance: 10);
323
324 ufs_mtk_ref_clk_notify(on, POST_CHANGE, res);
325
326 return 0;
327}
328
329static void ufs_mtk_setup_ref_clk_wait_us(struct ufs_hba *hba,
330 u16 gating_us)
331{
332 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
333
334 if (hba->dev_info.clk_gating_wait_us) {
335 host->ref_clk_gating_wait_us =
336 hba->dev_info.clk_gating_wait_us;
337 } else {
338 host->ref_clk_gating_wait_us = gating_us;
339 }
340
341 host->ref_clk_ungating_wait_us = REFCLK_DEFAULT_WAIT_US;
342}
343
344static void ufs_mtk_dbg_sel(struct ufs_hba *hba)
345{
346 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
347
348 if (((host->ip_ver >> 16) & 0xFF) >= 0x36) {
349 ufshcd_writel(hba, 0x820820, REG_UFS_DEBUG_SEL);
350 ufshcd_writel(hba, 0x0, REG_UFS_DEBUG_SEL_B0);
351 ufshcd_writel(hba, 0x55555555, REG_UFS_DEBUG_SEL_B1);
352 ufshcd_writel(hba, 0xaaaaaaaa, REG_UFS_DEBUG_SEL_B2);
353 ufshcd_writel(hba, 0xffffffff, REG_UFS_DEBUG_SEL_B3);
354 } else {
355 ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
356 }
357}
358
359static void ufs_mtk_wait_idle_state(struct ufs_hba *hba,
360 unsigned long retry_ms)
361{
362 u64 timeout, time_checked;
363 u32 val, sm;
364 bool wait_idle;
365
366 /* cannot use plain ktime_get() in suspend */
367 timeout = ktime_get_mono_fast_ns() + retry_ms * 1000000UL;
368
369 /* wait a specific time after check base */
370 udelay(10);
371 wait_idle = false;
372
373 do {
374 time_checked = ktime_get_mono_fast_ns();
375 ufs_mtk_dbg_sel(hba);
376 val = ufshcd_readl(hba, REG_UFS_PROBE);
377
378 sm = val & 0x1f;
379
380 /*
381 * if state is in H8 enter and H8 enter confirm
382 * wait until return to idle state.
383 */
384 if ((sm >= VS_HIB_ENTER) && (sm <= VS_HIB_EXIT)) {
385 wait_idle = true;
386 udelay(50);
387 continue;
388 } else if (!wait_idle)
389 break;
390
391 if (wait_idle && (sm == VS_HCE_BASE))
392 break;
393 } while (time_checked < timeout);
394
395 if (wait_idle && sm != VS_HCE_BASE)
396 dev_info(hba->dev, "wait idle tmo: 0x%x\n", val);
397}
398
399static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
400 unsigned long max_wait_ms)
401{
402 ktime_t timeout, time_checked;
403 u32 val;
404
405 timeout = ktime_add_ms(kt: ktime_get(), msec: max_wait_ms);
406 do {
407 time_checked = ktime_get();
408 ufs_mtk_dbg_sel(hba);
409 val = ufshcd_readl(hba, REG_UFS_PROBE);
410 val = val >> 28;
411
412 if (val == state)
413 return 0;
414
415 /* Sleep for max. 200us */
416 usleep_range(min: 100, max: 200);
417 } while (ktime_before(cmp1: time_checked, cmp2: timeout));
418
419 return -ETIMEDOUT;
420}
421
422static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
423{
424 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
425 struct phy *mphy = host->mphy;
426 struct arm_smccc_res res;
427 int ret = 0;
428
429 if (!mphy || !(on ^ host->mphy_powered_on))
430 return 0;
431
432 if (on) {
433 if (ufs_mtk_is_va09_supported(hba)) {
434 ret = regulator_enable(regulator: host->reg_va09);
435 if (ret < 0)
436 goto out;
437 /* wait 200 us to stablize VA09 */
438 usleep_range(min: 200, max: 210);
439 ufs_mtk_va09_pwr_ctrl(res, 1);
440 }
441 phy_power_on(phy: mphy);
442 } else {
443 phy_power_off(phy: mphy);
444 if (ufs_mtk_is_va09_supported(hba)) {
445 ufs_mtk_va09_pwr_ctrl(res, 0);
446 ret = regulator_disable(regulator: host->reg_va09);
447 }
448 }
449out:
450 if (ret) {
451 dev_info(hba->dev,
452 "failed to %s va09: %d\n",
453 on ? "enable" : "disable",
454 ret);
455 } else {
456 host->mphy_powered_on = on;
457 }
458
459 return ret;
460}
461
462static int ufs_mtk_get_host_clk(struct device *dev, const char *name,
463 struct clk **clk_out)
464{
465 struct clk *clk;
466 int err = 0;
467
468 clk = devm_clk_get(dev, id: name);
469 if (IS_ERR(ptr: clk))
470 err = PTR_ERR(ptr: clk);
471 else
472 *clk_out = clk;
473
474 return err;
475}
476
477static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost)
478{
479 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
480 struct ufs_mtk_crypt_cfg *cfg;
481 struct regulator *reg;
482 int volt, ret;
483
484 if (!ufs_mtk_is_boost_crypt_enabled(hba))
485 return;
486
487 cfg = host->crypt;
488 volt = cfg->vcore_volt;
489 reg = cfg->reg_vcore;
490
491 ret = clk_prepare_enable(clk: cfg->clk_crypt_mux);
492 if (ret) {
493 dev_info(hba->dev, "clk_prepare_enable(): %d\n",
494 ret);
495 return;
496 }
497
498 if (boost) {
499 ret = regulator_set_voltage(regulator: reg, min_uV: volt, INT_MAX);
500 if (ret) {
501 dev_info(hba->dev,
502 "failed to set vcore to %d\n", volt);
503 goto out;
504 }
505
506 ret = clk_set_parent(clk: cfg->clk_crypt_mux,
507 parent: cfg->clk_crypt_perf);
508 if (ret) {
509 dev_info(hba->dev,
510 "failed to set clk_crypt_perf\n");
511 regulator_set_voltage(regulator: reg, min_uV: 0, INT_MAX);
512 goto out;
513 }
514 } else {
515 ret = clk_set_parent(clk: cfg->clk_crypt_mux,
516 parent: cfg->clk_crypt_lp);
517 if (ret) {
518 dev_info(hba->dev,
519 "failed to set clk_crypt_lp\n");
520 goto out;
521 }
522
523 ret = regulator_set_voltage(regulator: reg, min_uV: 0, INT_MAX);
524 if (ret) {
525 dev_info(hba->dev,
526 "failed to set vcore to MIN\n");
527 }
528 }
529out:
530 clk_disable_unprepare(clk: cfg->clk_crypt_mux);
531}
532
533static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name,
534 struct clk **clk)
535{
536 int ret;
537
538 ret = ufs_mtk_get_host_clk(dev: hba->dev, name, clk_out: clk);
539 if (ret) {
540 dev_info(hba->dev, "%s: failed to get %s: %d", __func__,
541 name, ret);
542 }
543
544 return ret;
545}
546
547static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba)
548{
549 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
550 struct ufs_mtk_crypt_cfg *cfg;
551 struct device *dev = hba->dev;
552 struct regulator *reg;
553 u32 volt;
554
555 host->crypt = devm_kzalloc(dev, size: sizeof(*(host->crypt)),
556 GFP_KERNEL);
557 if (!host->crypt)
558 goto disable_caps;
559
560 reg = devm_regulator_get_optional(dev, id: "dvfsrc-vcore");
561 if (IS_ERR(ptr: reg)) {
562 dev_info(dev, "failed to get dvfsrc-vcore: %ld",
563 PTR_ERR(reg));
564 goto disable_caps;
565 }
566
567 if (of_property_read_u32(np: dev->of_node, propname: "boost-crypt-vcore-min",
568 out_value: &volt)) {
569 dev_info(dev, "failed to get boost-crypt-vcore-min");
570 goto disable_caps;
571 }
572
573 cfg = host->crypt;
574 if (ufs_mtk_init_host_clk(hba, name: "crypt_mux",
575 clk: &cfg->clk_crypt_mux))
576 goto disable_caps;
577
578 if (ufs_mtk_init_host_clk(hba, name: "crypt_lp",
579 clk: &cfg->clk_crypt_lp))
580 goto disable_caps;
581
582 if (ufs_mtk_init_host_clk(hba, name: "crypt_perf",
583 clk: &cfg->clk_crypt_perf))
584 goto disable_caps;
585
586 cfg->reg_vcore = reg;
587 cfg->vcore_volt = volt;
588 host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE;
589
590disable_caps:
591 return;
592}
593
594static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba)
595{
596 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
597
598 host->reg_va09 = regulator_get(dev: hba->dev, id: "va09");
599 if (IS_ERR(ptr: host->reg_va09))
600 dev_info(hba->dev, "failed to get va09");
601 else
602 host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL;
603}
604
605static void ufs_mtk_init_host_caps(struct ufs_hba *hba)
606{
607 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
608 struct device_node *np = hba->dev->of_node;
609
610 if (of_property_read_bool(np, propname: "mediatek,ufs-boost-crypt"))
611 ufs_mtk_init_boost_crypt(hba);
612
613 if (of_property_read_bool(np, propname: "mediatek,ufs-support-va09"))
614 ufs_mtk_init_va09_pwr_ctrl(hba);
615
616 if (of_property_read_bool(np, propname: "mediatek,ufs-disable-ah8"))
617 host->caps |= UFS_MTK_CAP_DISABLE_AH8;
618
619 if (of_property_read_bool(np, propname: "mediatek,ufs-broken-vcc"))
620 host->caps |= UFS_MTK_CAP_BROKEN_VCC;
621
622 if (of_property_read_bool(np, propname: "mediatek,ufs-pmc-via-fastauto"))
623 host->caps |= UFS_MTK_CAP_PMC_VIA_FASTAUTO;
624
625 dev_info(hba->dev, "caps: 0x%x", host->caps);
626}
627
628static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool scale_up)
629{
630 ufs_mtk_boost_crypt(hba, boost: scale_up);
631}
632
633static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on)
634{
635 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
636
637 if (on) {
638 phy_power_on(phy: host->mphy);
639 ufs_mtk_setup_ref_clk(hba, on);
640 if (!ufshcd_is_clkscaling_supported(hba))
641 ufs_mtk_scale_perf(hba, scale_up: on);
642 } else {
643 if (!ufshcd_is_clkscaling_supported(hba))
644 ufs_mtk_scale_perf(hba, scale_up: on);
645 ufs_mtk_setup_ref_clk(hba, on);
646 phy_power_off(phy: host->mphy);
647 }
648}
649
650static void ufs_mtk_mcq_disable_irq(struct ufs_hba *hba)
651{
652 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
653 u32 irq, i;
654
655 if (!is_mcq_enabled(hba))
656 return;
657
658 if (host->mcq_nr_intr == 0)
659 return;
660
661 for (i = 0; i < host->mcq_nr_intr; i++) {
662 irq = host->mcq_intr_info[i].irq;
663 disable_irq(irq);
664 }
665 host->is_mcq_intr_enabled = false;
666}
667
668static void ufs_mtk_mcq_enable_irq(struct ufs_hba *hba)
669{
670 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
671 u32 irq, i;
672
673 if (!is_mcq_enabled(hba))
674 return;
675
676 if (host->mcq_nr_intr == 0)
677 return;
678
679 if (host->is_mcq_intr_enabled == true)
680 return;
681
682 for (i = 0; i < host->mcq_nr_intr; i++) {
683 irq = host->mcq_intr_info[i].irq;
684 enable_irq(irq);
685 }
686 host->is_mcq_intr_enabled = true;
687}
688
689/**
690 * ufs_mtk_setup_clocks - enables/disable clocks
691 * @hba: host controller instance
692 * @on: If true, enable clocks else disable them.
693 * @status: PRE_CHANGE or POST_CHANGE notify
694 *
695 * Return: 0 on success, non-zero on failure.
696 */
697static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
698 enum ufs_notify_change_status status)
699{
700 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
701 bool clk_pwr_off = false;
702 int ret = 0;
703
704 /*
705 * In case ufs_mtk_init() is not yet done, simply ignore.
706 * This ufs_mtk_setup_clocks() shall be called from
707 * ufs_mtk_init() after init is done.
708 */
709 if (!host)
710 return 0;
711
712 if (!on && status == PRE_CHANGE) {
713 if (ufshcd_is_link_off(hba)) {
714 clk_pwr_off = true;
715 } else if (ufshcd_is_link_hibern8(hba) ||
716 (!ufshcd_can_hibern8_during_gating(hba) &&
717 ufshcd_is_auto_hibern8_enabled(hba))) {
718 /*
719 * Gate ref-clk and poweroff mphy if link state is in
720 * OFF or Hibern8 by either Auto-Hibern8 or
721 * ufshcd_link_state_transition().
722 */
723 ret = ufs_mtk_wait_link_state(hba,
724 state: VS_LINK_HIBERN8,
725 max_wait_ms: 15);
726 if (!ret)
727 clk_pwr_off = true;
728 }
729
730 if (clk_pwr_off)
731 ufs_mtk_pwr_ctrl(hba, on: false);
732 ufs_mtk_mcq_disable_irq(hba);
733 } else if (on && status == POST_CHANGE) {
734 ufs_mtk_pwr_ctrl(hba, on: true);
735 ufs_mtk_mcq_enable_irq(hba);
736 }
737
738 return ret;
739}
740
741static void ufs_mtk_get_controller_version(struct ufs_hba *hba)
742{
743 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
744 int ret, ver = 0;
745
746 if (host->hw_ver.major)
747 return;
748
749 /* Set default (minimum) version anyway */
750 host->hw_ver.major = 2;
751
752 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), mib_val: &ver);
753 if (!ret) {
754 if (ver >= UFS_UNIPRO_VER_1_8) {
755 host->hw_ver.major = 3;
756 /*
757 * Fix HCI version for some platforms with
758 * incorrect version
759 */
760 if (hba->ufs_version < ufshci_version(major: 3, minor: 0))
761 hba->ufs_version = ufshci_version(major: 3, minor: 0);
762 }
763 }
764}
765
766static u32 ufs_mtk_get_ufs_hci_version(struct ufs_hba *hba)
767{
768 return hba->ufs_version;
769}
770
771/**
772 * ufs_mtk_init_clocks - Init mtk driver private clocks
773 *
774 * @hba: per adapter instance
775 */
776static void ufs_mtk_init_clocks(struct ufs_hba *hba)
777{
778 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
779 struct list_head *head = &hba->clk_list_head;
780 struct ufs_mtk_clk *mclk = &host->mclk;
781 struct ufs_clk_info *clki, *clki_tmp;
782
783 /*
784 * Find private clocks and store them in struct ufs_mtk_clk.
785 * Remove "ufs_sel_min_src" and "ufs_sel_min_src" from list to avoid
786 * being switched on/off in clock gating.
787 */
788 list_for_each_entry_safe(clki, clki_tmp, head, list) {
789 if (!strcmp(clki->name, "ufs_sel")) {
790 host->mclk.ufs_sel_clki = clki;
791 } else if (!strcmp(clki->name, "ufs_sel_max_src")) {
792 host->mclk.ufs_sel_max_clki = clki;
793 clk_disable_unprepare(clk: clki->clk);
794 list_del(entry: &clki->list);
795 } else if (!strcmp(clki->name, "ufs_sel_min_src")) {
796 host->mclk.ufs_sel_min_clki = clki;
797 clk_disable_unprepare(clk: clki->clk);
798 list_del(entry: &clki->list);
799 }
800 }
801
802 if (!mclk->ufs_sel_clki || !mclk->ufs_sel_max_clki ||
803 !mclk->ufs_sel_min_clki) {
804 hba->caps &= ~UFSHCD_CAP_CLK_SCALING;
805 dev_info(hba->dev,
806 "%s: Clk-scaling not ready. Feature disabled.",
807 __func__);
808 }
809}
810
811#define MAX_VCC_NAME 30
812static int ufs_mtk_vreg_fix_vcc(struct ufs_hba *hba)
813{
814 struct ufs_vreg_info *info = &hba->vreg_info;
815 struct device_node *np = hba->dev->of_node;
816 struct device *dev = hba->dev;
817 char vcc_name[MAX_VCC_NAME];
818 struct arm_smccc_res res;
819 int err, ver;
820
821 if (hba->vreg_info.vcc)
822 return 0;
823
824 if (of_property_read_bool(np, propname: "mediatek,ufs-vcc-by-num")) {
825 ufs_mtk_get_vcc_num(res);
826 if (res.a1 > UFS_VCC_NONE && res.a1 < UFS_VCC_MAX)
827 snprintf(buf: vcc_name, MAX_VCC_NAME, fmt: "vcc-opt%lu", res.a1);
828 else
829 return -ENODEV;
830 } else if (of_property_read_bool(np, propname: "mediatek,ufs-vcc-by-ver")) {
831 ver = (hba->dev_info.wspecversion & 0xF00) >> 8;
832 snprintf(buf: vcc_name, MAX_VCC_NAME, fmt: "vcc-ufs%u", ver);
833 } else {
834 return 0;
835 }
836
837 err = ufshcd_populate_vreg(dev, name: vcc_name, out_vreg: &info->vcc, skip_current: false);
838 if (err)
839 return err;
840
841 err = ufshcd_get_vreg(dev, vreg: info->vcc);
842 if (err)
843 return err;
844
845 err = regulator_enable(regulator: info->vcc->reg);
846 if (!err) {
847 info->vcc->enabled = true;
848 dev_info(dev, "%s: %s enabled\n", __func__, vcc_name);
849 }
850
851 return err;
852}
853
854static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba)
855{
856 struct ufs_vreg_info *info = &hba->vreg_info;
857 struct ufs_vreg **vreg_on, **vreg_off;
858
859 if (hba->dev_info.wspecversion >= 0x0300) {
860 vreg_on = &info->vccq;
861 vreg_off = &info->vccq2;
862 } else {
863 vreg_on = &info->vccq2;
864 vreg_off = &info->vccq;
865 }
866
867 if (*vreg_on)
868 (*vreg_on)->always_on = true;
869
870 if (*vreg_off) {
871 regulator_disable(regulator: (*vreg_off)->reg);
872 devm_kfree(dev: hba->dev, p: (*vreg_off)->name);
873 devm_kfree(dev: hba->dev, p: *vreg_off);
874 *vreg_off = NULL;
875 }
876}
877
878static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba)
879{
880 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
881 struct platform_device *pdev;
882 int i;
883 int irq;
884
885 host->mcq_nr_intr = UFSHCD_MAX_Q_NR;
886 pdev = container_of(hba->dev, struct platform_device, dev);
887
888 for (i = 0; i < host->mcq_nr_intr; i++) {
889 /* irq index 0 is legacy irq, sq/cq irq start from index 1 */
890 irq = platform_get_irq(pdev, i + 1);
891 if (irq < 0) {
892 host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
893 goto failed;
894 }
895 host->mcq_intr_info[i].hba = hba;
896 host->mcq_intr_info[i].irq = irq;
897 dev_info(hba->dev, "get platform mcq irq: %d, %d\n", i, irq);
898 }
899
900 return;
901failed:
902 /* invalidate irq info */
903 for (i = 0; i < host->mcq_nr_intr; i++)
904 host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ;
905
906 host->mcq_nr_intr = 0;
907}
908
909/**
910 * ufs_mtk_init - find other essential mmio bases
911 * @hba: host controller instance
912 *
913 * Binds PHY with controller and powers up PHY enabling clocks
914 * and regulators.
915 *
916 * Return: -EPROBE_DEFER if binding fails, returns negative error
917 * on phy power up failure and returns zero on success.
918 */
919static int ufs_mtk_init(struct ufs_hba *hba)
920{
921 const struct of_device_id *id;
922 struct device *dev = hba->dev;
923 struct ufs_mtk_host *host;
924 struct Scsi_Host *shost = hba->host;
925 int err = 0;
926
927 host = devm_kzalloc(dev, size: sizeof(*host), GFP_KERNEL);
928 if (!host) {
929 err = -ENOMEM;
930 dev_info(dev, "%s: no memory for mtk ufs host\n", __func__);
931 goto out;
932 }
933
934 host->hba = hba;
935 ufshcd_set_variant(hba, variant: host);
936
937 id = of_match_device(matches: ufs_mtk_of_match, dev);
938 if (!id) {
939 err = -EINVAL;
940 goto out;
941 }
942
943 /* Initialize host capability */
944 ufs_mtk_init_host_caps(hba);
945
946 ufs_mtk_init_mcq_irq(hba);
947
948 err = ufs_mtk_bind_mphy(hba);
949 if (err)
950 goto out_variant_clear;
951
952 ufs_mtk_init_reset(hba);
953
954 /* Enable runtime autosuspend */
955 hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
956
957 /* Enable clock-gating */
958 hba->caps |= UFSHCD_CAP_CLK_GATING;
959
960 /* Enable inline encryption */
961 hba->caps |= UFSHCD_CAP_CRYPTO;
962
963 /* Enable WriteBooster */
964 hba->caps |= UFSHCD_CAP_WB_EN;
965
966 /* Enable clk scaling*/
967 hba->caps |= UFSHCD_CAP_CLK_SCALING;
968
969 /* Set runtime pm delay to replace default */
970 shost->rpm_autosuspend_delay = MTK_RPM_AUTOSUSPEND_DELAY_MS;
971
972 hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL;
973 hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_INTR;
974 hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_RTC;
975 hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
976
977 if (host->caps & UFS_MTK_CAP_DISABLE_AH8)
978 hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
979
980 ufs_mtk_init_clocks(hba);
981
982 /*
983 * ufshcd_vops_init() is invoked after
984 * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
985 * phy clock setup is skipped.
986 *
987 * Enable phy clocks specifically here.
988 */
989 ufs_mtk_mphy_power_on(hba, on: true);
990 ufs_mtk_setup_clocks(hba, on: true, status: POST_CHANGE);
991
992 host->ip_ver = ufshcd_readl(hba, REG_UFS_MTK_IP_VER);
993
994 goto out;
995
996out_variant_clear:
997 ufshcd_set_variant(hba, NULL);
998out:
999 return err;
1000}
1001
1002static bool ufs_mtk_pmc_via_fastauto(struct ufs_hba *hba,
1003 struct ufs_pa_layer_attr *dev_req_params)
1004{
1005 if (!ufs_mtk_is_pmc_via_fastauto(hba))
1006 return false;
1007
1008 if (dev_req_params->hs_rate == hba->pwr_info.hs_rate)
1009 return false;
1010
1011 if (dev_req_params->pwr_tx != FAST_MODE &&
1012 dev_req_params->gear_tx < UFS_HS_G4)
1013 return false;
1014
1015 if (dev_req_params->pwr_rx != FAST_MODE &&
1016 dev_req_params->gear_rx < UFS_HS_G4)
1017 return false;
1018
1019 return true;
1020}
1021
1022static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
1023 struct ufs_pa_layer_attr *dev_max_params,
1024 struct ufs_pa_layer_attr *dev_req_params)
1025{
1026 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1027 struct ufs_host_params host_params;
1028 int ret;
1029
1030 ufshcd_init_host_params(host_params: &host_params);
1031 host_params.hs_rx_gear = UFS_HS_G5;
1032 host_params.hs_tx_gear = UFS_HS_G5;
1033
1034 ret = ufshcd_negotiate_pwr_params(host_params: &host_params, dev_max: dev_max_params, agreed_pwr: dev_req_params);
1035 if (ret) {
1036 pr_info("%s: failed to determine capabilities\n",
1037 __func__);
1038 }
1039
1040 if (ufs_mtk_pmc_via_fastauto(hba, dev_req_params)) {
1041 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), mib_val: true);
1042 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), mib_val: UFS_HS_G1);
1043
1044 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), mib_val: true);
1045 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), mib_val: UFS_HS_G1);
1046
1047 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
1048 mib_val: dev_req_params->lane_tx);
1049 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
1050 mib_val: dev_req_params->lane_rx);
1051 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
1052 mib_val: dev_req_params->hs_rate);
1053
1054 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE),
1055 PA_NO_ADAPT);
1056
1057 ret = ufshcd_uic_change_pwr_mode(hba,
1058 mode: FASTAUTO_MODE << 4 | FASTAUTO_MODE);
1059
1060 if (ret) {
1061 dev_err(hba->dev, "%s: HSG1B FASTAUTO failed ret=%d\n",
1062 __func__, ret);
1063 }
1064 }
1065
1066 if (host->hw_ver.major >= 3) {
1067 ret = ufshcd_dme_configure_adapt(hba,
1068 agreed_gear: dev_req_params->gear_tx,
1069 PA_INITIAL_ADAPT);
1070 }
1071
1072 return ret;
1073}
1074
1075static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
1076 enum ufs_notify_change_status stage,
1077 struct ufs_pa_layer_attr *dev_max_params,
1078 struct ufs_pa_layer_attr *dev_req_params)
1079{
1080 int ret = 0;
1081
1082 switch (stage) {
1083 case PRE_CHANGE:
1084 ret = ufs_mtk_pre_pwr_change(hba, dev_max_params,
1085 dev_req_params);
1086 break;
1087 case POST_CHANGE:
1088 break;
1089 default:
1090 ret = -EINVAL;
1091 break;
1092 }
1093
1094 return ret;
1095}
1096
1097static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm)
1098{
1099 int ret;
1100 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1101
1102 ret = ufshcd_dme_set(hba,
1103 UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0),
1104 mib_val: lpm ? 1 : 0);
1105 if (!ret || !lpm) {
1106 /*
1107 * Forcibly set as non-LPM mode if UIC commands is failed
1108 * to use default hba_enable_delay_us value for re-enabling
1109 * the host.
1110 */
1111 host->unipro_lpm = lpm;
1112 }
1113
1114 return ret;
1115}
1116
1117static int ufs_mtk_pre_link(struct ufs_hba *hba)
1118{
1119 int ret;
1120 u32 tmp;
1121
1122 ufs_mtk_get_controller_version(hba);
1123
1124 ret = ufs_mtk_unipro_set_lpm(hba, lpm: false);
1125 if (ret)
1126 return ret;
1127
1128 /*
1129 * Setting PA_Local_TX_LCC_Enable to 0 before link startup
1130 * to make sure that both host and device TX LCC are disabled
1131 * once link startup is completed.
1132 */
1133 ret = ufshcd_disable_host_tx_lcc(hba);
1134 if (ret)
1135 return ret;
1136
1137 /* disable deep stall */
1138 ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: &tmp);
1139 if (ret)
1140 return ret;
1141
1142 tmp &= ~(1 << 6);
1143
1144 ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), mib_val: tmp);
1145
1146 return ret;
1147}
1148
1149static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
1150{
1151 u32 ah_ms;
1152
1153 if (ufshcd_is_clkgating_allowed(hba)) {
1154 if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit)
1155 ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK,
1156 hba->ahit);
1157 else
1158 ah_ms = 10;
1159 ufshcd_clkgate_delay_set(dev: hba->dev, value: ah_ms + 5);
1160 }
1161}
1162
1163static void ufs_mtk_post_link(struct ufs_hba *hba)
1164{
1165 /* enable unipro clock gating feature */
1166 ufs_mtk_cfg_unipro_cg(hba, enable: true);
1167
1168 /* will be configured during probe hba */
1169 if (ufshcd_is_auto_hibern8_supported(hba))
1170 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 10) |
1171 FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
1172
1173 ufs_mtk_setup_clk_gating(hba);
1174}
1175
1176static int ufs_mtk_link_startup_notify(struct ufs_hba *hba,
1177 enum ufs_notify_change_status stage)
1178{
1179 int ret = 0;
1180
1181 switch (stage) {
1182 case PRE_CHANGE:
1183 ret = ufs_mtk_pre_link(hba);
1184 break;
1185 case POST_CHANGE:
1186 ufs_mtk_post_link(hba);
1187 break;
1188 default:
1189 ret = -EINVAL;
1190 break;
1191 }
1192
1193 return ret;
1194}
1195
1196static int ufs_mtk_device_reset(struct ufs_hba *hba)
1197{
1198 struct arm_smccc_res res;
1199
1200 /* disable hba before device reset */
1201 ufshcd_hba_stop(hba);
1202
1203 ufs_mtk_device_reset_ctrl(0, res);
1204
1205 /*
1206 * The reset signal is active low. UFS devices shall detect
1207 * more than or equal to 1us of positive or negative RST_n
1208 * pulse width.
1209 *
1210 * To be on safe side, keep the reset low for at least 10us.
1211 */
1212 usleep_range(min: 10, max: 15);
1213
1214 ufs_mtk_device_reset_ctrl(1, res);
1215
1216 /* Some devices may need time to respond to rst_n */
1217 usleep_range(min: 10000, max: 15000);
1218
1219 dev_info(hba->dev, "device reset done\n");
1220
1221 return 0;
1222}
1223
1224static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
1225{
1226 int err;
1227
1228 err = ufshcd_hba_enable(hba);
1229 if (err)
1230 return err;
1231
1232 err = ufs_mtk_unipro_set_lpm(hba, lpm: false);
1233 if (err)
1234 return err;
1235
1236 err = ufshcd_uic_hibern8_exit(hba);
1237 if (err)
1238 return err;
1239
1240 /* Check link state to make sure exit h8 success */
1241 ufs_mtk_wait_idle_state(hba, retry_ms: 5);
1242 err = ufs_mtk_wait_link_state(hba, state: VS_LINK_UP, max_wait_ms: 100);
1243 if (err) {
1244 dev_warn(hba->dev, "exit h8 state fail, err=%d\n", err);
1245 return err;
1246 }
1247 ufshcd_set_link_active(hba);
1248
1249 err = ufshcd_make_hba_operational(hba);
1250 if (err)
1251 return err;
1252
1253 if (is_mcq_enabled(hba)) {
1254 ufs_mtk_config_mcq(hba, irq: false);
1255 ufshcd_mcq_make_queues_operational(hba);
1256 ufshcd_mcq_config_mac(hba, max_active_cmds: hba->nutrs);
1257 ufshcd_mcq_enable(hba);
1258 }
1259
1260 return 0;
1261}
1262
1263static int ufs_mtk_link_set_lpm(struct ufs_hba *hba)
1264{
1265 int err;
1266
1267 /* Disable reset confirm feature by UniPro */
1268 ufshcd_writel(hba,
1269 (ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) & ~0x100),
1270 REG_UFS_XOUFS_CTRL);
1271
1272 err = ufs_mtk_unipro_set_lpm(hba, lpm: true);
1273 if (err) {
1274 /* Resume UniPro state for following error recovery */
1275 ufs_mtk_unipro_set_lpm(hba, lpm: false);
1276 return err;
1277 }
1278
1279 return 0;
1280}
1281
1282static void ufs_mtk_vccqx_set_lpm(struct ufs_hba *hba, bool lpm)
1283{
1284 struct ufs_vreg *vccqx = NULL;
1285
1286 if (hba->vreg_info.vccq)
1287 vccqx = hba->vreg_info.vccq;
1288 else
1289 vccqx = hba->vreg_info.vccq2;
1290
1291 regulator_set_mode(regulator: vccqx->reg,
1292 mode: lpm ? REGULATOR_MODE_IDLE : REGULATOR_MODE_NORMAL);
1293}
1294
1295static void ufs_mtk_vsx_set_lpm(struct ufs_hba *hba, bool lpm)
1296{
1297 struct arm_smccc_res res;
1298
1299 ufs_mtk_device_pwr_ctrl(!lpm,
1300 (unsigned long)hba->dev_info.wspecversion,
1301 res);
1302}
1303
1304static void ufs_mtk_dev_vreg_set_lpm(struct ufs_hba *hba, bool lpm)
1305{
1306 if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2)
1307 return;
1308
1309 /* Skip if VCC is assumed always-on */
1310 if (!hba->vreg_info.vcc)
1311 return;
1312
1313 /* Bypass LPM when device is still active */
1314 if (lpm && ufshcd_is_ufs_dev_active(hba))
1315 return;
1316
1317 /* Bypass LPM if VCC is enabled */
1318 if (lpm && hba->vreg_info.vcc->enabled)
1319 return;
1320
1321 if (lpm) {
1322 ufs_mtk_vccqx_set_lpm(hba, lpm);
1323 ufs_mtk_vsx_set_lpm(hba, lpm);
1324 } else {
1325 ufs_mtk_vsx_set_lpm(hba, lpm);
1326 ufs_mtk_vccqx_set_lpm(hba, lpm);
1327 }
1328}
1329
1330static void ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba)
1331{
1332 int ret;
1333
1334 /* disable auto-hibern8 */
1335 ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
1336
1337 /* wait host return to idle state when auto-hibern8 off */
1338 ufs_mtk_wait_idle_state(hba, retry_ms: 5);
1339
1340 ret = ufs_mtk_wait_link_state(hba, state: VS_LINK_UP, max_wait_ms: 100);
1341 if (ret)
1342 dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret);
1343}
1344
1345static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
1346 enum ufs_notify_change_status status)
1347{
1348 int err;
1349 struct arm_smccc_res res;
1350
1351 if (status == PRE_CHANGE) {
1352 if (ufshcd_is_auto_hibern8_supported(hba))
1353 ufs_mtk_auto_hibern8_disable(hba);
1354 return 0;
1355 }
1356
1357 if (ufshcd_is_link_hibern8(hba)) {
1358 err = ufs_mtk_link_set_lpm(hba);
1359 if (err)
1360 goto fail;
1361 }
1362
1363 if (!ufshcd_is_link_active(hba)) {
1364 /*
1365 * Make sure no error will be returned to prevent
1366 * ufshcd_suspend() re-enabling regulators while vreg is still
1367 * in low-power mode.
1368 */
1369 err = ufs_mtk_mphy_power_on(hba, on: false);
1370 if (err)
1371 goto fail;
1372 }
1373
1374 if (ufshcd_is_link_off(hba))
1375 ufs_mtk_device_reset_ctrl(0, res);
1376
1377 ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, false, res);
1378
1379 return 0;
1380fail:
1381 /*
1382 * Set link as off state enforcedly to trigger
1383 * ufshcd_host_reset_and_restore() in ufshcd_suspend()
1384 * for completed host reset.
1385 */
1386 ufshcd_set_link_off(hba);
1387 return -EAGAIN;
1388}
1389
1390static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
1391{
1392 int err;
1393 struct arm_smccc_res res;
1394
1395 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1396 ufs_mtk_dev_vreg_set_lpm(hba, lpm: false);
1397
1398 ufs_mtk_host_pwr_ctrl(HOST_PWR_HCI, true, res);
1399
1400 err = ufs_mtk_mphy_power_on(hba, on: true);
1401 if (err)
1402 goto fail;
1403
1404 if (ufshcd_is_link_hibern8(hba)) {
1405 err = ufs_mtk_link_set_hpm(hba);
1406 if (err)
1407 goto fail;
1408 }
1409
1410 return 0;
1411fail:
1412 return ufshcd_link_recovery(hba);
1413}
1414
1415static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba)
1416{
1417 /* Dump ufshci register 0x140 ~ 0x14C */
1418 ufshcd_dump_regs(hba, REG_UFS_XOUFS_CTRL, len: 0x10,
1419 prefix: "XOUFS Ctrl (0x140): ");
1420
1421 ufshcd_dump_regs(hba, REG_UFS_EXTREG, len: 0x4, prefix: "Ext Reg ");
1422
1423 /* Dump ufshci register 0x2200 ~ 0x22AC */
1424 ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL,
1425 REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4,
1426 prefix: "MPHY Ctrl (0x2200): ");
1427
1428 /* Direct debugging information to REG_MTK_PROBE */
1429 ufs_mtk_dbg_sel(hba);
1430 ufshcd_dump_regs(hba, REG_UFS_PROBE, len: 0x4, prefix: "Debug Probe ");
1431}
1432
1433static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
1434{
1435 struct ufs_dev_info *dev_info = &hba->dev_info;
1436 u16 mid = dev_info->wmanufacturerid;
1437
1438 if (mid == UFS_VENDOR_SAMSUNG) {
1439 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), mib_val: 6);
1440 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), mib_val: 10);
1441 }
1442
1443 /*
1444 * Decide waiting time before gating reference clock and
1445 * after ungating reference clock according to vendors'
1446 * requirements.
1447 */
1448 if (mid == UFS_VENDOR_SAMSUNG)
1449 ufs_mtk_setup_ref_clk_wait_us(hba, gating_us: 1);
1450 else if (mid == UFS_VENDOR_SKHYNIX)
1451 ufs_mtk_setup_ref_clk_wait_us(hba, gating_us: 30);
1452 else if (mid == UFS_VENDOR_TOSHIBA)
1453 ufs_mtk_setup_ref_clk_wait_us(hba, gating_us: 100);
1454 else
1455 ufs_mtk_setup_ref_clk_wait_us(hba,
1456 REFCLK_DEFAULT_WAIT_US);
1457 return 0;
1458}
1459
1460static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
1461{
1462 ufshcd_fixup_dev_quirks(hba, fixups: ufs_mtk_dev_fixups);
1463
1464 if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc &&
1465 (hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)) {
1466 hba->vreg_info.vcc->always_on = true;
1467 /*
1468 * VCC will be kept always-on thus we don't
1469 * need any delay during regulator operations
1470 */
1471 hba->dev_quirks &= ~(UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
1472 UFS_DEVICE_QUIRK_DELAY_AFTER_LPM);
1473 }
1474
1475 ufs_mtk_vreg_fix_vcc(hba);
1476 ufs_mtk_vreg_fix_vccqx(hba);
1477}
1478
1479static void ufs_mtk_event_notify(struct ufs_hba *hba,
1480 enum ufs_event_type evt, void *data)
1481{
1482 unsigned int val = *(u32 *)data;
1483 unsigned long reg;
1484 u8 bit;
1485
1486 trace_ufs_mtk_event(type: evt, data: val);
1487
1488 /* Print details of UIC Errors */
1489 if (evt <= UFS_EVT_DME_ERR) {
1490 dev_info(hba->dev,
1491 "Host UIC Error Code (%s): %08x\n",
1492 ufs_uic_err_str[evt], val);
1493 reg = val;
1494 }
1495
1496 if (evt == UFS_EVT_PA_ERR) {
1497 for_each_set_bit(bit, &reg, ARRAY_SIZE(ufs_uic_pa_err_str))
1498 dev_info(hba->dev, "%s\n", ufs_uic_pa_err_str[bit]);
1499 }
1500
1501 if (evt == UFS_EVT_DL_ERR) {
1502 for_each_set_bit(bit, &reg, ARRAY_SIZE(ufs_uic_dl_err_str))
1503 dev_info(hba->dev, "%s\n", ufs_uic_dl_err_str[bit]);
1504 }
1505}
1506
1507static void ufs_mtk_config_scaling_param(struct ufs_hba *hba,
1508 struct devfreq_dev_profile *profile,
1509 struct devfreq_simple_ondemand_data *data)
1510{
1511 /* Customize min gear in clk scaling */
1512 hba->clk_scaling.min_gear = UFS_HS_G4;
1513
1514 hba->vps->devfreq_profile.polling_ms = 200;
1515 hba->vps->ondemand_data.upthreshold = 50;
1516 hba->vps->ondemand_data.downdifferential = 20;
1517}
1518
1519/**
1520 * ufs_mtk_clk_scale - Internal clk scaling operation
1521 *
1522 * MTK platform supports clk scaling by switching parent of ufs_sel(mux).
1523 * The ufs_sel downstream to ufs_ck which feeds directly to UFS hardware.
1524 * Max and min clocks rate of ufs_sel defined in dts should match rate of
1525 * "ufs_sel_max_src" and "ufs_sel_min_src" respectively.
1526 * This prevent changing rate of pll clock that is shared between modules.
1527 *
1528 * @hba: per adapter instance
1529 * @scale_up: True for scaling up and false for scaling down
1530 */
1531static void ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up)
1532{
1533 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1534 struct ufs_mtk_clk *mclk = &host->mclk;
1535 struct ufs_clk_info *clki = mclk->ufs_sel_clki;
1536 int ret = 0;
1537
1538 ret = clk_prepare_enable(clk: clki->clk);
1539 if (ret) {
1540 dev_info(hba->dev,
1541 "clk_prepare_enable() fail, ret: %d\n", ret);
1542 return;
1543 }
1544
1545 if (scale_up) {
1546 ret = clk_set_parent(clk: clki->clk, parent: mclk->ufs_sel_max_clki->clk);
1547 clki->curr_freq = clki->max_freq;
1548 } else {
1549 ret = clk_set_parent(clk: clki->clk, parent: mclk->ufs_sel_min_clki->clk);
1550 clki->curr_freq = clki->min_freq;
1551 }
1552
1553 if (ret) {
1554 dev_info(hba->dev,
1555 "Failed to set ufs_sel_clki, ret: %d\n", ret);
1556 }
1557
1558 clk_disable_unprepare(clk: clki->clk);
1559
1560 trace_ufs_mtk_clk_scale(name: clki->name, scale_up, clk_rate: clk_get_rate(clk: clki->clk));
1561}
1562
1563static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
1564 enum ufs_notify_change_status status)
1565{
1566 if (!ufshcd_is_clkscaling_supported(hba))
1567 return 0;
1568
1569 if (status == PRE_CHANGE) {
1570 /* Switch parent before clk_set_rate() */
1571 ufs_mtk_clk_scale(hba, scale_up);
1572 } else {
1573 /* Request interrupt latency QoS accordingly */
1574 ufs_mtk_scale_perf(hba, scale_up);
1575 }
1576
1577 return 0;
1578}
1579
1580static int ufs_mtk_get_hba_mac(struct ufs_hba *hba)
1581{
1582 return MAX_SUPP_MAC;
1583}
1584
1585static int ufs_mtk_op_runtime_config(struct ufs_hba *hba)
1586{
1587 struct ufshcd_mcq_opr_info_t *opr;
1588 int i;
1589
1590 hba->mcq_opr[OPR_SQD].offset = REG_UFS_MTK_SQD;
1591 hba->mcq_opr[OPR_SQIS].offset = REG_UFS_MTK_SQIS;
1592 hba->mcq_opr[OPR_CQD].offset = REG_UFS_MTK_CQD;
1593 hba->mcq_opr[OPR_CQIS].offset = REG_UFS_MTK_CQIS;
1594
1595 for (i = 0; i < OPR_MAX; i++) {
1596 opr = &hba->mcq_opr[i];
1597 opr->stride = REG_UFS_MCQ_STRIDE;
1598 opr->base = hba->mmio_base + opr->offset;
1599 }
1600
1601 return 0;
1602}
1603
1604static int ufs_mtk_mcq_config_resource(struct ufs_hba *hba)
1605{
1606 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1607
1608 /* fail mcq initialization if interrupt is not filled properly */
1609 if (!host->mcq_nr_intr) {
1610 dev_info(hba->dev, "IRQs not ready. MCQ disabled.");
1611 return -EINVAL;
1612 }
1613
1614 hba->mcq_base = hba->mmio_base + MCQ_QUEUE_OFFSET(hba->mcq_capabilities);
1615 return 0;
1616}
1617
1618static irqreturn_t ufs_mtk_mcq_intr(int irq, void *__intr_info)
1619{
1620 struct ufs_mtk_mcq_intr_info *mcq_intr_info = __intr_info;
1621 struct ufs_hba *hba = mcq_intr_info->hba;
1622 struct ufs_hw_queue *hwq;
1623 u32 events;
1624 int qid = mcq_intr_info->qid;
1625
1626 hwq = &hba->uhq[qid];
1627
1628 events = ufshcd_mcq_read_cqis(hba, i: qid);
1629 if (events)
1630 ufshcd_mcq_write_cqis(hba, val: events, i: qid);
1631
1632 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
1633 ufshcd_mcq_poll_cqe_lock(hba, hwq);
1634
1635 return IRQ_HANDLED;
1636}
1637
1638static int ufs_mtk_config_mcq_irq(struct ufs_hba *hba)
1639{
1640 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1641 u32 irq, i;
1642 int ret;
1643
1644 for (i = 0; i < host->mcq_nr_intr; i++) {
1645 irq = host->mcq_intr_info[i].irq;
1646 if (irq == MTK_MCQ_INVALID_IRQ) {
1647 dev_err(hba->dev, "invalid irq. %d\n", i);
1648 return -ENOPARAM;
1649 }
1650
1651 host->mcq_intr_info[i].qid = i;
1652 ret = devm_request_irq(dev: hba->dev, irq, handler: ufs_mtk_mcq_intr, irqflags: 0, UFSHCD,
1653 dev_id: &host->mcq_intr_info[i]);
1654
1655 dev_dbg(hba->dev, "request irq %d intr %s\n", irq, ret ? "failed" : "");
1656
1657 if (ret) {
1658 dev_err(hba->dev, "Cannot request irq %d\n", ret);
1659 return ret;
1660 }
1661 }
1662
1663 return 0;
1664}
1665
1666static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq)
1667{
1668 struct ufs_mtk_host *host = ufshcd_get_variant(hba);
1669 int ret = 0;
1670
1671 if (!host->mcq_set_intr) {
1672 /* Disable irq option register */
1673 ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, val: 0, REG_UFS_MMIO_OPT_CTRL_0);
1674
1675 if (irq) {
1676 ret = ufs_mtk_config_mcq_irq(hba);
1677 if (ret)
1678 return ret;
1679 }
1680
1681 host->mcq_set_intr = true;
1682 }
1683
1684 ufshcd_rmwl(hba, MCQ_AH8, MCQ_AH8, REG_UFS_MMIO_OPT_CTRL_0);
1685 ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, MCQ_MULTI_INTR_EN, REG_UFS_MMIO_OPT_CTRL_0);
1686
1687 return 0;
1688}
1689
1690static int ufs_mtk_config_esi(struct ufs_hba *hba)
1691{
1692 return ufs_mtk_config_mcq(hba, irq: true);
1693}
1694
1695/*
1696 * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
1697 *
1698 * The variant operations configure the necessary controller and PHY
1699 * handshake during initialization.
1700 */
1701static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
1702 .name = "mediatek.ufshci",
1703 .init = ufs_mtk_init,
1704 .get_ufs_hci_version = ufs_mtk_get_ufs_hci_version,
1705 .setup_clocks = ufs_mtk_setup_clocks,
1706 .hce_enable_notify = ufs_mtk_hce_enable_notify,
1707 .link_startup_notify = ufs_mtk_link_startup_notify,
1708 .pwr_change_notify = ufs_mtk_pwr_change_notify,
1709 .apply_dev_quirks = ufs_mtk_apply_dev_quirks,
1710 .fixup_dev_quirks = ufs_mtk_fixup_dev_quirks,
1711 .suspend = ufs_mtk_suspend,
1712 .resume = ufs_mtk_resume,
1713 .dbg_register_dump = ufs_mtk_dbg_register_dump,
1714 .device_reset = ufs_mtk_device_reset,
1715 .event_notify = ufs_mtk_event_notify,
1716 .config_scaling_param = ufs_mtk_config_scaling_param,
1717 .clk_scale_notify = ufs_mtk_clk_scale_notify,
1718 /* mcq vops */
1719 .get_hba_mac = ufs_mtk_get_hba_mac,
1720 .op_runtime_config = ufs_mtk_op_runtime_config,
1721 .mcq_config_resource = ufs_mtk_mcq_config_resource,
1722 .config_esi = ufs_mtk_config_esi,
1723};
1724
1725/**
1726 * ufs_mtk_probe - probe routine of the driver
1727 * @pdev: pointer to Platform device handle
1728 *
1729 * Return: zero for success and non-zero for failure.
1730 */
1731static int ufs_mtk_probe(struct platform_device *pdev)
1732{
1733 int err;
1734 struct device *dev = &pdev->dev;
1735 struct device_node *reset_node;
1736 struct platform_device *reset_pdev;
1737 struct device_link *link;
1738
1739 reset_node = of_find_compatible_node(NULL, NULL,
1740 compat: "ti,syscon-reset");
1741 if (!reset_node) {
1742 dev_notice(dev, "find ti,syscon-reset fail\n");
1743 goto skip_reset;
1744 }
1745 reset_pdev = of_find_device_by_node(np: reset_node);
1746 if (!reset_pdev) {
1747 dev_notice(dev, "find reset_pdev fail\n");
1748 goto skip_reset;
1749 }
1750 link = device_link_add(consumer: dev, supplier: &reset_pdev->dev,
1751 DL_FLAG_AUTOPROBE_CONSUMER);
1752 put_device(dev: &reset_pdev->dev);
1753 if (!link) {
1754 dev_notice(dev, "add reset device_link fail\n");
1755 goto skip_reset;
1756 }
1757 /* supplier is not probed */
1758 if (link->status == DL_STATE_DORMANT) {
1759 err = -EPROBE_DEFER;
1760 goto out;
1761 }
1762
1763skip_reset:
1764 /* perform generic probe */
1765 err = ufshcd_pltfrm_init(pdev, vops: &ufs_hba_mtk_vops);
1766
1767out:
1768 if (err)
1769 dev_err(dev, "probe failed %d\n", err);
1770
1771 of_node_put(node: reset_node);
1772 return err;
1773}
1774
1775/**
1776 * ufs_mtk_remove - set driver_data of the device to NULL
1777 * @pdev: pointer to platform device handle
1778 *
1779 * Always return 0
1780 */
1781static void ufs_mtk_remove(struct platform_device *pdev)
1782{
1783 struct ufs_hba *hba = platform_get_drvdata(pdev);
1784
1785 pm_runtime_get_sync(dev: &(pdev)->dev);
1786 ufshcd_remove(hba);
1787}
1788
1789#ifdef CONFIG_PM_SLEEP
1790static int ufs_mtk_system_suspend(struct device *dev)
1791{
1792 struct ufs_hba *hba = dev_get_drvdata(dev);
1793 int ret;
1794
1795 ret = ufshcd_system_suspend(dev);
1796 if (ret)
1797 return ret;
1798
1799 ufs_mtk_dev_vreg_set_lpm(hba, lpm: true);
1800
1801 return 0;
1802}
1803
1804static int ufs_mtk_system_resume(struct device *dev)
1805{
1806 struct ufs_hba *hba = dev_get_drvdata(dev);
1807
1808 ufs_mtk_dev_vreg_set_lpm(hba, lpm: false);
1809
1810 return ufshcd_system_resume(dev);
1811}
1812#endif
1813
1814#ifdef CONFIG_PM
1815static int ufs_mtk_runtime_suspend(struct device *dev)
1816{
1817 struct ufs_hba *hba = dev_get_drvdata(dev);
1818 int ret = 0;
1819
1820 ret = ufshcd_runtime_suspend(dev);
1821 if (ret)
1822 return ret;
1823
1824 ufs_mtk_dev_vreg_set_lpm(hba, lpm: true);
1825
1826 return 0;
1827}
1828
1829static int ufs_mtk_runtime_resume(struct device *dev)
1830{
1831 struct ufs_hba *hba = dev_get_drvdata(dev);
1832
1833 ufs_mtk_dev_vreg_set_lpm(hba, lpm: false);
1834
1835 return ufshcd_runtime_resume(dev);
1836}
1837#endif
1838
1839static const struct dev_pm_ops ufs_mtk_pm_ops = {
1840 SET_SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend,
1841 ufs_mtk_system_resume)
1842 SET_RUNTIME_PM_OPS(ufs_mtk_runtime_suspend,
1843 ufs_mtk_runtime_resume, NULL)
1844 .prepare = ufshcd_suspend_prepare,
1845 .complete = ufshcd_resume_complete,
1846};
1847
1848static struct platform_driver ufs_mtk_pltform = {
1849 .probe = ufs_mtk_probe,
1850 .remove_new = ufs_mtk_remove,
1851 .driver = {
1852 .name = "ufshcd-mtk",
1853 .pm = &ufs_mtk_pm_ops,
1854 .of_match_table = ufs_mtk_of_match,
1855 },
1856};
1857
1858MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
1859MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>");
1860MODULE_DESCRIPTION("MediaTek UFS Host Driver");
1861MODULE_LICENSE("GPL v2");
1862
1863module_platform_driver(ufs_mtk_pltform);
1864

source code of linux/drivers/ufs/host/ufs-mediatek.c