drivers/scsi/ufs/ufs-mediatek.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2019 MediaTek Inc.
   4  * Authors:
   5  *      Stanley Chu <stanley.chu@mediatek.com>
   6  *      Peter Wang <peter.wang@mediatek.com>
   7  */
   8
   9 #include <linux/arm-smccc.h>
  10 #include <linux/bitfield.h>
  11 #include <linux/of.h>
  12 #include <linux/of_address.h>
  13 #include <linux/of_device.h>
  14 #include <linux/phy/phy.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/regulator/consumer.h>
  17 #include <linux/reset.h>
  18 #include <linux/soc/mediatek/mtk_sip_svc.h>
  19
  20 #include "ufshcd.h"
  21 #include "ufshcd-crypto.h"
  22 #include "ufshcd-pltfrm.h"
  23 #include "ufs_quirks.h"
  24 #include "unipro.h"
  25 #include "ufs-mediatek.h"
  26
  27 #define CREATE_TRACE_POINTS
  28 #include "ufs-mediatek-trace.h"
  29
  30 #define ufs_mtk_smc(cmd, val, res) \
  31         arm_smccc_smc(MTK_SIP_UFS_CONTROL, \
  32                       cmd, val, 0, 0, 0, 0, 0, &(res))
  33
  34 #define ufs_mtk_va09_pwr_ctrl(res, on) \
  35         ufs_mtk_smc(UFS_MTK_SIP_VA09_PWR_CTRL, on, res)
  36
  37 #define ufs_mtk_crypto_ctrl(res, enable) \
  38         ufs_mtk_smc(UFS_MTK_SIP_CRYPTO_CTRL, enable, res)
  39
  40 #define ufs_mtk_ref_clk_notify(on, res) \
  41         ufs_mtk_smc(UFS_MTK_SIP_REF_CLK_NOTIFICATION, on, res)
  42
  43 #define ufs_mtk_device_reset_ctrl(high, res) \
  44         ufs_mtk_smc(UFS_MTK_SIP_DEVICE_RESET, high, res)
  45
  46 static struct ufs_dev_fix ufs_mtk_dev_fixups[] = {
  47         UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
  48                 UFS_DEVICE_QUIRK_DELAY_AFTER_LPM),
  49         UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AFAMZDAR",
  50                 UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES),
  51         END_FIX
  52 };
  53
  54 static const struct of_device_id ufs_mtk_of_match[] = {
  55         { .compatible = "mediatek,mt8183-ufshci" },
  56         {},
  57 };
  58
  59 static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba)
  60 {
  61         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
  62
  63         return !!(host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE);
  64 }
  65
  66 static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba)
  67 {
  68         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
  69
  70         return !!(host->caps & UFS_MTK_CAP_VA09_PWR_CTRL);
  71 }
  72
  73 static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
  74 {
  75         u32 tmp;
  76
  77         if (enable) {
  78                 ufshcd_dme_get(hba,
  79                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
  80                 tmp = tmp |
  81                       (1 << RX_SYMBOL_CLK_GATE_EN) |
  82                       (1 << SYS_CLK_GATE_EN) |
  83                       (1 << TX_CLK_GATE_EN);
  84                 ufshcd_dme_set(hba,
  85                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
  86
  87                 ufshcd_dme_get(hba,
  88                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
  89                 tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE);
  90                 ufshcd_dme_set(hba,
  91                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
  92         } else {
  93                 ufshcd_dme_get(hba,
  94                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
  95                 tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) |
  96                               (1 << SYS_CLK_GATE_EN) |
  97                               (1 << TX_CLK_GATE_EN));
  98                 ufshcd_dme_set(hba,
  99                                UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
 100
 101                 ufshcd_dme_get(hba,
 102                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
 103                 tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE);
 104                 ufshcd_dme_set(hba,
 105                                UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
 106         }
 107 }
 108
 109 static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
 110 {
 111         struct arm_smccc_res res;
 112
 113         ufs_mtk_crypto_ctrl(res, 1);
 114         if (res.a0) {
 115                 dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n",
 116                          __func__, res.a0);
 117                 hba->caps &= ~UFSHCD_CAP_CRYPTO;
 118         }
 119 }
 120
 121 static void ufs_mtk_host_reset(struct ufs_hba *hba)
 122 {
 123         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 124
 125         reset_control_assert(host->hci_reset);
 126         reset_control_assert(host->crypto_reset);
 127         reset_control_assert(host->unipro_reset);
 128
 129         usleep_range(100, 110);
 130
 131         reset_control_deassert(host->unipro_reset);
 132         reset_control_deassert(host->crypto_reset);
 133         reset_control_deassert(host->hci_reset);
 134 }
 135
 136 static void ufs_mtk_init_reset_control(struct ufs_hba *hba,
 137                                        struct reset_control **rc,
 138                                        char *str)
 139 {
 140         *rc = devm_reset_control_get(hba->dev, str);
 141         if (IS_ERR(*rc)) {
 142                 dev_info(hba->dev, "Failed to get reset control %s: %ld\n",
 143                          str, PTR_ERR(*rc));
 144                 *rc = NULL;
 145         }
 146 }
 147
 148 static void ufs_mtk_init_reset(struct ufs_hba *hba)
 149 {
 150         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 151
 152         ufs_mtk_init_reset_control(hba, &host->hci_reset,
 153                                    "hci_rst");
 154         ufs_mtk_init_reset_control(hba, &host->unipro_reset,
 155                                    "unipro_rst");
 156         ufs_mtk_init_reset_control(hba, &host->crypto_reset,
 157                                    "crypto_rst");
 158 }
 159
 160 static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
 161                                      enum ufs_notify_change_status status)
 162 {
 163         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 164         unsigned long flags;
 165
 166         if (status == PRE_CHANGE) {
 167                 if (host->unipro_lpm) {
 168                         hba->vps->hba_enable_delay_us = 0;
 169                 } else {
 170                         hba->vps->hba_enable_delay_us = 600;
 171                         ufs_mtk_host_reset(hba);
 172                 }
 173
 174                 if (hba->caps & UFSHCD_CAP_CRYPTO)
 175                         ufs_mtk_crypto_enable(hba);
 176
 177                 if (host->caps & UFS_MTK_CAP_DISABLE_AH8) {
 178                         spin_lock_irqsave(hba->host->host_lock, flags);
 179                         ufshcd_writel(hba, 0,
 180                                       REG_AUTO_HIBERNATE_IDLE_TIMER);
 181                         spin_unlock_irqrestore(hba->host->host_lock,
 182                                                flags);
 183
 184                         hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT;
 185                         hba->ahit = 0;
 186                 }
 187         }
 188
 189         return 0;
 190 }
 191
 192 static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
 193 {
 194         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 195         struct device *dev = hba->dev;
 196         struct device_node *np = dev->of_node;
 197         int err = 0;
 198
 199         host->mphy = devm_of_phy_get_by_index(dev, np, 0);
 200
 201         if (host->mphy == ERR_PTR(-EPROBE_DEFER)) {
 202                 /*
 203                  * UFS driver might be probed before the phy driver does.
 204                  * In that case we would like to return EPROBE_DEFER code.
 205                  */
 206                 err = -EPROBE_DEFER;
 207                 dev_info(dev,
 208                          "%s: required phy hasn't probed yet. err = %d\n",
 209                         __func__, err);
 210         } else if (IS_ERR(host->mphy)) {
 211                 err = PTR_ERR(host->mphy);
 212                 if (err != -ENODEV) {
 213                         dev_info(dev, "%s: PHY get failed %d\n", __func__,
 214                                  err);
 215                 }
 216         }
 217
 218         if (err)
 219                 host->mphy = NULL;
 220         /*
 221          * Allow unbound mphy because not every platform needs specific
 222          * mphy control.
 223          */
 224         if (err == -ENODEV)
 225                 err = 0;
 226
 227         return err;
 228 }
 229
 230 static int ufs_mtk_setup_ref_clk(struct ufs_hba *hba, bool on)
 231 {
 232         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 233         struct arm_smccc_res res;
 234         ktime_t timeout, time_checked;
 235         u32 value;
 236
 237         if (host->ref_clk_enabled == on)
 238                 return 0;
 239
 240         if (on) {
 241                 ufs_mtk_ref_clk_notify(on, res);
 242                 ufshcd_delay_us(host->ref_clk_ungating_wait_us, 10);
 243                 ufshcd_writel(hba, REFCLK_REQUEST, REG_UFS_REFCLK_CTRL);
 244         } else {
 245                 ufshcd_writel(hba, REFCLK_RELEASE, REG_UFS_REFCLK_CTRL);
 246         }
 247
 248         /* Wait for ack */
 249         timeout = ktime_add_us(ktime_get(), REFCLK_REQ_TIMEOUT_US);
 250         do {
 251                 time_checked = ktime_get();
 252                 value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL);
 253
 254                 /* Wait until ack bit equals to req bit */
 255                 if (((value & REFCLK_ACK) >> 1) == (value & REFCLK_REQUEST))
 256                         goto out;
 257
 258                 usleep_range(100, 200);
 259         } while (ktime_before(time_checked, timeout));
 260
 261         dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value);
 262
 263         ufs_mtk_ref_clk_notify(host->ref_clk_enabled, res);
 264
 265         return -ETIMEDOUT;
 266
 267 out:
 268         host->ref_clk_enabled = on;
 269         if (!on) {
 270                 ufshcd_delay_us(host->ref_clk_gating_wait_us, 10);
 271                 ufs_mtk_ref_clk_notify(on, res);
 272         }
 273
 274         return 0;
 275 }
 276
 277 static void ufs_mtk_setup_ref_clk_wait_us(struct ufs_hba *hba,
 278                                           u16 gating_us, u16 ungating_us)
 279 {
 280         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 281
 282         if (hba->dev_info.clk_gating_wait_us) {
 283                 host->ref_clk_gating_wait_us =
 284                         hba->dev_info.clk_gating_wait_us;
 285         } else {
 286                 host->ref_clk_gating_wait_us = gating_us;
 287         }
 288
 289         host->ref_clk_ungating_wait_us = ungating_us;
 290 }
 291
 292 static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state,
 293                                    unsigned long max_wait_ms)
 294 {
 295         ktime_t timeout, time_checked;
 296         u32 val;
 297
 298         timeout = ktime_add_ms(ktime_get(), max_wait_ms);
 299         do {
 300                 time_checked = ktime_get();
 301                 ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
 302                 val = ufshcd_readl(hba, REG_UFS_PROBE);
 303                 val = val >> 28;
 304
 305                 if (val == state)
 306                         return 0;
 307
 308                 /* Sleep for max. 200us */
 309                 usleep_range(100, 200);
 310         } while (ktime_before(time_checked, timeout));
 311
 312         if (val == state)
 313                 return 0;
 314
 315         return -ETIMEDOUT;
 316 }
 317
 318 static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
 319 {
 320         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 321         struct phy *mphy = host->mphy;
 322         struct arm_smccc_res res;
 323         int ret = 0;
 324
 325         if (!mphy || !(on ^ host->mphy_powered_on))
 326                 return 0;
 327
 328         if (on) {
 329                 if (ufs_mtk_is_va09_supported(hba)) {
 330                         ret = regulator_enable(host->reg_va09);
 331                         if (ret < 0)
 332                                 goto out;
 333                         /* wait 200 us to stablize VA09 */
 334                         usleep_range(200, 210);
 335                         ufs_mtk_va09_pwr_ctrl(res, 1);
 336                 }
 337                 phy_power_on(mphy);
 338         } else {
 339                 phy_power_off(mphy);
 340                 if (ufs_mtk_is_va09_supported(hba)) {
 341                         ufs_mtk_va09_pwr_ctrl(res, 0);
 342                         ret = regulator_disable(host->reg_va09);
 343                         if (ret < 0)
 344                                 goto out;
 345                 }
 346         }
 347 out:
 348         if (ret) {
 349                 dev_info(hba->dev,
 350                          "failed to %s va09: %d\n",
 351                          on ? "enable" : "disable",
 352                          ret);
 353         } else {
 354                 host->mphy_powered_on = on;
 355         }
 356
 357         return ret;
 358 }
 359
 360 static int ufs_mtk_get_host_clk(struct device *dev, const char *name,
 361                                 struct clk **clk_out)
 362 {
 363         struct clk *clk;
 364         int err = 0;
 365
 366         clk = devm_clk_get(dev, name);
 367         if (IS_ERR(clk))
 368                 err = PTR_ERR(clk);
 369         else
 370                 *clk_out = clk;
 371
 372         return err;
 373 }
 374
 375 static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost)
 376 {
 377         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 378         struct ufs_mtk_crypt_cfg *cfg;
 379         struct regulator *reg;
 380         int volt, ret;
 381
 382         if (!ufs_mtk_is_boost_crypt_enabled(hba))
 383                 return;
 384
 385         cfg = host->crypt;
 386         volt = cfg->vcore_volt;
 387         reg = cfg->reg_vcore;
 388
 389         ret = clk_prepare_enable(cfg->clk_crypt_mux);
 390         if (ret) {
 391                 dev_info(hba->dev, "clk_prepare_enable(): %d\n",
 392                          ret);
 393                 return;
 394         }
 395
 396         if (boost) {
 397                 ret = regulator_set_voltage(reg, volt, INT_MAX);
 398                 if (ret) {
 399                         dev_info(hba->dev,
 400                                  "failed to set vcore to %d\n", volt);
 401                         goto out;
 402                 }
 403
 404                 ret = clk_set_parent(cfg->clk_crypt_mux,
 405                                      cfg->clk_crypt_perf);
 406                 if (ret) {
 407                         dev_info(hba->dev,
 408                                  "failed to set clk_crypt_perf\n");
 409                         regulator_set_voltage(reg, 0, INT_MAX);
 410                         goto out;
 411                 }
 412         } else {
 413                 ret = clk_set_parent(cfg->clk_crypt_mux,
 414                                      cfg->clk_crypt_lp);
 415                 if (ret) {
 416                         dev_info(hba->dev,
 417                                  "failed to set clk_crypt_lp\n");
 418                         goto out;
 419                 }
 420
 421                 ret = regulator_set_voltage(reg, 0, INT_MAX);
 422                 if (ret) {
 423                         dev_info(hba->dev,
 424                                  "failed to set vcore to MIN\n");
 425                 }
 426         }
 427 out:
 428         clk_disable_unprepare(cfg->clk_crypt_mux);
 429 }
 430
 431 static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name,
 432                                  struct clk **clk)
 433 {
 434         int ret;
 435
 436         ret = ufs_mtk_get_host_clk(hba->dev, name, clk);
 437         if (ret) {
 438                 dev_info(hba->dev, "%s: failed to get %s: %d", __func__,
 439                          name, ret);
 440         }
 441
 442         return ret;
 443 }
 444
 445 static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba)
 446 {
 447         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 448         struct ufs_mtk_crypt_cfg *cfg;
 449         struct device *dev = hba->dev;
 450         struct regulator *reg;
 451         u32 volt;
 452
 453         host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)),
 454                                    GFP_KERNEL);
 455         if (!host->crypt)
 456                 goto disable_caps;
 457
 458         reg = devm_regulator_get_optional(dev, "dvfsrc-vcore");
 459         if (IS_ERR(reg)) {
 460                 dev_info(dev, "failed to get dvfsrc-vcore: %ld",
 461                          PTR_ERR(reg));
 462                 goto disable_caps;
 463         }
 464
 465         if (of_property_read_u32(dev->of_node, "boost-crypt-vcore-min",
 466                                  &volt)) {
 467                 dev_info(dev, "failed to get boost-crypt-vcore-min");
 468                 goto disable_caps;
 469         }
 470
 471         cfg = host->crypt;
 472         if (ufs_mtk_init_host_clk(hba, "crypt_mux",
 473                                   &cfg->clk_crypt_mux))
 474                 goto disable_caps;
 475
 476         if (ufs_mtk_init_host_clk(hba, "crypt_lp",
 477                                   &cfg->clk_crypt_lp))
 478                 goto disable_caps;
 479
 480         if (ufs_mtk_init_host_clk(hba, "crypt_perf",
 481                                   &cfg->clk_crypt_perf))
 482                 goto disable_caps;
 483
 484         cfg->reg_vcore = reg;
 485         cfg->vcore_volt = volt;
 486         host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE;
 487
 488 disable_caps:
 489         return;
 490 }
 491
 492 static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba)
 493 {
 494         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 495
 496         host->reg_va09 = regulator_get(hba->dev, "va09");
 497         if (!host->reg_va09)
 498                 dev_info(hba->dev, "failed to get va09");
 499         else
 500                 host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL;
 501 }
 502
 503 static void ufs_mtk_init_host_caps(struct ufs_hba *hba)
 504 {
 505         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 506         struct device_node *np = hba->dev->of_node;
 507
 508         if (of_property_read_bool(np, "mediatek,ufs-boost-crypt"))
 509                 ufs_mtk_init_boost_crypt(hba);
 510
 511         if (of_property_read_bool(np, "mediatek,ufs-support-va09"))
 512                 ufs_mtk_init_va09_pwr_ctrl(hba);
 513
 514         if (of_property_read_bool(np, "mediatek,ufs-disable-ah8"))
 515                 host->caps |= UFS_MTK_CAP_DISABLE_AH8;
 516
 517         dev_info(hba->dev, "caps: 0x%x", host->caps);
 518 }
 519
 520 static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool up)
 521 {
 522         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 523
 524         ufs_mtk_boost_crypt(hba, up);
 525         ufs_mtk_setup_ref_clk(hba, up);
 526
 527         if (up)
 528                 phy_power_on(host->mphy);
 529         else
 530                 phy_power_off(host->mphy);
 531 }
 532
 533 /**
 534  * ufs_mtk_setup_clocks - enables/disable clocks
 535  * @hba: host controller instance
 536  * @on: If true, enable clocks else disable them.
 537  * @status: PRE_CHANGE or POST_CHANGE notify
 538  *
 539  * Returns 0 on success, non-zero on failure.
 540  */
 541 static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
 542                                 enum ufs_notify_change_status status)
 543 {
 544         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 545         bool clk_pwr_off = false;
 546         int ret = 0;
 547
 548         /*
 549          * In case ufs_mtk_init() is not yet done, simply ignore.
 550          * This ufs_mtk_setup_clocks() shall be called from
 551          * ufs_mtk_init() after init is done.
 552          */
 553         if (!host)
 554                 return 0;
 555
 556         if (!on && status == PRE_CHANGE) {
 557                 if (ufshcd_is_link_off(hba)) {
 558                         clk_pwr_off = true;
 559                 } else if (ufshcd_is_link_hibern8(hba) ||
 560                          (!ufshcd_can_hibern8_during_gating(hba) &&
 561                          ufshcd_is_auto_hibern8_enabled(hba))) {
 562                         /*
 563                          * Gate ref-clk and poweroff mphy if link state is in
 564                          * OFF or Hibern8 by either Auto-Hibern8 or
 565                          * ufshcd_link_state_transition().
 566                          */
 567                         ret = ufs_mtk_wait_link_state(hba,
 568                                                       VS_LINK_HIBERN8,
 569                                                       15);
 570                         if (!ret)
 571                                 clk_pwr_off = true;
 572                 }
 573
 574                 if (clk_pwr_off)
 575                         ufs_mtk_scale_perf(hba, false);
 576         } else if (on && status == POST_CHANGE) {
 577                 ufs_mtk_scale_perf(hba, true);
 578         }
 579
 580         return ret;
 581 }
 582
 583 static void ufs_mtk_get_controller_version(struct ufs_hba *hba)
 584 {
 585         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 586         int ret, ver = 0;
 587
 588         if (host->hw_ver.major)
 589                 return;
 590
 591         /* Set default (minimum) version anyway */
 592         host->hw_ver.major = 2;
 593
 594         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &ver);
 595         if (!ret) {
 596                 if (ver >= UFS_UNIPRO_VER_1_8)
 597                         host->hw_ver.major = 3;
 598         }
 599 }
 600
 601 /**
 602  * ufs_mtk_init - find other essential mmio bases
 603  * @hba: host controller instance
 604  *
 605  * Binds PHY with controller and powers up PHY enabling clocks
 606  * and regulators.
 607  *
 608  * Returns -EPROBE_DEFER if binding fails, returns negative error
 609  * on phy power up failure and returns zero on success.
 610  */
 611 static int ufs_mtk_init(struct ufs_hba *hba)
 612 {
 613         const struct of_device_id *id;
 614         struct device *dev = hba->dev;
 615         struct ufs_mtk_host *host;
 616         int err = 0;
 617
 618         host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
 619         if (!host) {
 620                 err = -ENOMEM;
 621                 dev_info(dev, "%s: no memory for mtk ufs host\n", __func__);
 622                 goto out;
 623         }
 624
 625         host->hba = hba;
 626         ufshcd_set_variant(hba, host);
 627
 628         id = of_match_device(ufs_mtk_of_match, dev);
 629         if (!id) {
 630                 err = -EINVAL;
 631                 goto out;
 632         }
 633
 634         /* Initialize host capability */
 635         ufs_mtk_init_host_caps(hba);
 636
 637         err = ufs_mtk_bind_mphy(hba);
 638         if (err)
 639                 goto out_variant_clear;
 640
 641         ufs_mtk_init_reset(hba);
 642
 643         /* Enable runtime autosuspend */
 644         hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
 645
 646         /* Enable clock-gating */
 647         hba->caps |= UFSHCD_CAP_CLK_GATING;
 648
 649         /* Enable inline encryption */
 650         hba->caps |= UFSHCD_CAP_CRYPTO;
 651
 652         /* Enable WriteBooster */
 653         hba->caps |= UFSHCD_CAP_WB_EN;
 654         hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
 655
 656         if (host->caps & UFS_MTK_CAP_DISABLE_AH8)
 657                 hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
 658
 659         /*
 660          * ufshcd_vops_init() is invoked after
 661          * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
 662          * phy clock setup is skipped.
 663          *
 664          * Enable phy clocks specifically here.
 665          */
 666         ufs_mtk_mphy_power_on(hba, true);
 667         ufs_mtk_setup_clocks(hba, true, POST_CHANGE);
 668
 669         goto out;
 670
 671 out_variant_clear:
 672         ufshcd_set_variant(hba, NULL);
 673 out:
 674         return err;
 675 }
 676
 677 static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
 678                                   struct ufs_pa_layer_attr *dev_max_params,
 679                                   struct ufs_pa_layer_attr *dev_req_params)
 680 {
 681         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 682         struct ufs_dev_params host_cap;
 683         int ret;
 684
 685         ufshcd_init_pwr_dev_param(&host_cap);
 686         host_cap.hs_rx_gear = UFS_HS_G4;
 687         host_cap.hs_tx_gear = UFS_HS_G4;
 688
 689         ret = ufshcd_get_pwr_dev_param(&host_cap,
 690                                        dev_max_params,
 691                                        dev_req_params);
 692         if (ret) {
 693                 pr_info("%s: failed to determine capabilities\n",
 694                         __func__);
 695         }
 696
 697         if (host->hw_ver.major >= 3) {
 698                 ret = ufshcd_dme_configure_adapt(hba,
 699                                            dev_req_params->gear_tx,
 700                                            PA_INITIAL_ADAPT);
 701         }
 702
 703         return ret;
 704 }
 705
 706 static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
 707                                      enum ufs_notify_change_status stage,
 708                                      struct ufs_pa_layer_attr *dev_max_params,
 709                                      struct ufs_pa_layer_attr *dev_req_params)
 710 {
 711         int ret = 0;
 712
 713         switch (stage) {
 714         case PRE_CHANGE:
 715                 ret = ufs_mtk_pre_pwr_change(hba, dev_max_params,
 716                                              dev_req_params);
 717                 break;
 718         case POST_CHANGE:
 719                 break;
 720         default:
 721                 ret = -EINVAL;
 722                 break;
 723         }
 724
 725         return ret;
 726 }
 727
 728 static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm)
 729 {
 730         int ret;
 731         struct ufs_mtk_host *host = ufshcd_get_variant(hba);
 732
 733         ret = ufshcd_dme_set(hba,
 734                              UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0),
 735                              lpm ? 1 : 0);
 736         if (!ret || !lpm) {
 737                 /*
 738                  * Forcibly set as non-LPM mode if UIC commands is failed
 739                  * to use default hba_enable_delay_us value for re-enabling
 740                  * the host.
 741                  */
 742                 host->unipro_lpm = lpm;
 743         }
 744
 745         return ret;
 746 }
 747
 748 static int ufs_mtk_pre_link(struct ufs_hba *hba)
 749 {
 750         int ret;
 751         u32 tmp;
 752
 753         ufs_mtk_get_controller_version(hba);
 754
 755         ret = ufs_mtk_unipro_set_lpm(hba, false);
 756         if (ret)
 757                 return ret;
 758
 759         /*
 760          * Setting PA_Local_TX_LCC_Enable to 0 before link startup
 761          * to make sure that both host and device TX LCC are disabled
 762          * once link startup is completed.
 763          */
 764         ret = ufshcd_disable_host_tx_lcc(hba);
 765         if (ret)
 766                 return ret;
 767
 768         /* disable deep stall */
 769         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
 770         if (ret)
 771                 return ret;
 772
 773         tmp &= ~(1 << 6);
 774
 775         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
 776
 777         return ret;
 778 }
 779
 780 static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
 781 {
 782         unsigned long flags;
 783         u32 ah_ms;
 784
 785         if (ufshcd_is_clkgating_allowed(hba)) {
 786                 if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit)
 787                         ah_ms = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK,
 788                                           hba->ahit);
 789                 else
 790                         ah_ms = 10;
 791                 spin_lock_irqsave(hba->host->host_lock, flags);
 792                 hba->clk_gating.delay_ms = ah_ms + 5;
 793                 spin_unlock_irqrestore(hba->host->host_lock, flags);
 794         }
 795 }
 796
 797 static int ufs_mtk_post_link(struct ufs_hba *hba)
 798 {
 799         /* enable unipro clock gating feature */
 800         ufs_mtk_cfg_unipro_cg(hba, true);
 801
 802         /* configure auto-hibern8 timer to 10ms */
 803         if (ufshcd_is_auto_hibern8_supported(hba)) {
 804                 ufshcd_auto_hibern8_update(hba,
 805                         FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 10) |
 806                         FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3));
 807         }
 808
 809         ufs_mtk_setup_clk_gating(hba);
 810
 811         return 0;
 812 }
 813
 814 static int ufs_mtk_link_startup_notify(struct ufs_hba *hba,
 815                                        enum ufs_notify_change_status stage)
 816 {
 817         int ret = 0;
 818
 819         switch (stage) {
 820         case PRE_CHANGE:
 821                 ret = ufs_mtk_pre_link(hba);
 822                 break;
 823         case POST_CHANGE:
 824                 ret = ufs_mtk_post_link(hba);
 825                 break;
 826         default:
 827                 ret = -EINVAL;
 828                 break;
 829         }
 830
 831         return ret;
 832 }
 833
 834 static int ufs_mtk_device_reset(struct ufs_hba *hba)
 835 {
 836         struct arm_smccc_res res;
 837
 838         ufs_mtk_device_reset_ctrl(0, res);
 839
 840         /*
 841          * The reset signal is active low. UFS devices shall detect
 842          * more than or equal to 1us of positive or negative RST_n
 843          * pulse width.
 844          *
 845          * To be on safe side, keep the reset low for at least 10us.
 846          */
 847         usleep_range(10, 15);
 848
 849         ufs_mtk_device_reset_ctrl(1, res);
 850
 851         /* Some devices may need time to respond to rst_n */
 852         usleep_range(10000, 15000);
 853
 854         dev_info(hba->dev, "device reset done\n");
 855
 856         return 0;
 857 }
 858
 859 static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
 860 {
 861         int err;
 862
 863         err = ufshcd_hba_enable(hba);
 864         if (err)
 865                 return err;
 866
 867         err = ufs_mtk_unipro_set_lpm(hba, false);
 868         if (err)
 869                 return err;
 870
 871         err = ufshcd_uic_hibern8_exit(hba);
 872         if (!err)
 873                 ufshcd_set_link_active(hba);
 874         else
 875                 return err;
 876
 877         err = ufshcd_make_hba_operational(hba);
 878         if (err)
 879                 return err;
 880
 881         return 0;
 882 }
 883
 884 static int ufs_mtk_link_set_lpm(struct ufs_hba *hba)
 885 {
 886         int err;
 887
 888         err = ufs_mtk_unipro_set_lpm(hba, true);
 889         if (err) {
 890                 /* Resume UniPro state for following error recovery */
 891                 ufs_mtk_unipro_set_lpm(hba, false);
 892                 return err;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static void ufs_mtk_vreg_set_lpm(struct ufs_hba *hba, bool lpm)
 899 {
 900         if (!hba->vreg_info.vccq2 || !hba->vreg_info.vcc)
 901                 return;
 902
 903         if (lpm & !hba->vreg_info.vcc->enabled)
 904                 regulator_set_mode(hba->vreg_info.vccq2->reg,
 905                                    REGULATOR_MODE_IDLE);
 906         else if (!lpm)
 907                 regulator_set_mode(hba->vreg_info.vccq2->reg,
 908                                    REGULATOR_MODE_NORMAL);
 909 }
 910
 911 static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 912 {
 913         int err;
 914
 915         if (ufshcd_is_link_hibern8(hba)) {
 916                 err = ufs_mtk_link_set_lpm(hba);
 917                 if (err)
 918                         goto fail;
 919         }
 920
 921         if (!ufshcd_is_link_active(hba)) {
 922                 /*
 923                  * Make sure no error will be returned to prevent
 924                  * ufshcd_suspend() re-enabling regulators while vreg is still
 925                  * in low-power mode.
 926                  */
 927                 ufs_mtk_vreg_set_lpm(hba, true);
 928                 err = ufs_mtk_mphy_power_on(hba, false);
 929                 if (err)
 930                         goto fail;
 931         }
 932
 933         return 0;
 934 fail:
 935         /*
 936          * Set link as off state enforcedly to trigger
 937          * ufshcd_host_reset_and_restore() in ufshcd_suspend()
 938          * for completed host reset.
 939          */
 940         ufshcd_set_link_off(hba);
 941         return -EAGAIN;
 942 }
 943
 944 static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 945 {
 946         int err;
 947
 948         err = ufs_mtk_mphy_power_on(hba, true);
 949         if (err)
 950                 goto fail;
 951
 952         ufs_mtk_vreg_set_lpm(hba, false);
 953
 954         if (ufshcd_is_link_hibern8(hba)) {
 955                 err = ufs_mtk_link_set_hpm(hba);
 956                 if (err)
 957                         goto fail;
 958         }
 959
 960         return 0;
 961 fail:
 962         return ufshcd_link_recovery(hba);
 963 }
 964
 965 static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba)
 966 {
 967         ufshcd_dump_regs(hba, REG_UFS_REFCLK_CTRL, 0x4, "Ref-Clk Ctrl ");
 968
 969         ufshcd_dump_regs(hba, REG_UFS_EXTREG, 0x4, "Ext Reg ");
 970
 971         ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL,
 972                          REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4,
 973                          "MPHY Ctrl ");
 974
 975         /* Direct debugging information to REG_MTK_PROBE */
 976         ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL);
 977         ufshcd_dump_regs(hba, REG_UFS_PROBE, 0x4, "Debug Probe ");
 978 }
 979
 980 static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
 981 {
 982         struct ufs_dev_info *dev_info = &hba->dev_info;
 983         u16 mid = dev_info->wmanufacturerid;
 984
 985         if (mid == UFS_VENDOR_SAMSUNG)
 986                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 6);
 987
 988         /*
 989          * Decide waiting time before gating reference clock and
 990          * after ungating reference clock according to vendors'
 991          * requirements.
 992          */
 993         if (mid == UFS_VENDOR_SAMSUNG)
 994                 ufs_mtk_setup_ref_clk_wait_us(hba, 1, 1);
 995         else if (mid == UFS_VENDOR_SKHYNIX)
 996                 ufs_mtk_setup_ref_clk_wait_us(hba, 30, 30);
 997         else if (mid == UFS_VENDOR_TOSHIBA)
 998                 ufs_mtk_setup_ref_clk_wait_us(hba, 100, 32);
 999
1000         return 0;
1001 }
1002
1003 static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
1004 {
1005         ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
1006 }
1007
1008 static void ufs_mtk_event_notify(struct ufs_hba *hba,
1009                                  enum ufs_event_type evt, void *data)
1010 {
1011         unsigned int val = *(u32 *)data;
1012
1013         trace_ufs_mtk_event(evt, val);
1014 }
1015
1016 /*
1017  * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
1018  *
1019  * The variant operations configure the necessary controller and PHY
1020  * handshake during initialization.
1021  */
1022 static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
1023         .name                = "mediatek.ufshci",
1024         .init                = ufs_mtk_init,
1025         .setup_clocks        = ufs_mtk_setup_clocks,
1026         .hce_enable_notify   = ufs_mtk_hce_enable_notify,
1027         .link_startup_notify = ufs_mtk_link_startup_notify,
1028         .pwr_change_notify   = ufs_mtk_pwr_change_notify,
1029         .apply_dev_quirks    = ufs_mtk_apply_dev_quirks,
1030         .fixup_dev_quirks    = ufs_mtk_fixup_dev_quirks,
1031         .suspend             = ufs_mtk_suspend,
1032         .resume              = ufs_mtk_resume,
1033         .dbg_register_dump   = ufs_mtk_dbg_register_dump,
1034         .device_reset        = ufs_mtk_device_reset,
1035         .event_notify        = ufs_mtk_event_notify,
1036 };
1037
1038 /**
1039  * ufs_mtk_probe - probe routine of the driver
1040  * @pdev: pointer to Platform device handle
1041  *
1042  * Return zero for success and non-zero for failure
1043  */
1044 static int ufs_mtk_probe(struct platform_device *pdev)
1045 {
1046         int err;
1047         struct device *dev = &pdev->dev;
1048
1049         /* perform generic probe */
1050         err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops);
1051         if (err)
1052                 dev_info(dev, "probe failed %d\n", err);
1053
1054         return err;
1055 }
1056
1057 /**
1058  * ufs_mtk_remove - set driver_data of the device to NULL
1059  * @pdev: pointer to platform device handle
1060  *
1061  * Always return 0
1062  */
1063 static int ufs_mtk_remove(struct platform_device *pdev)
1064 {
1065         struct ufs_hba *hba =  platform_get_drvdata(pdev);
1066
1067         pm_runtime_get_sync(&(pdev)->dev);
1068         ufshcd_remove(hba);
1069         return 0;
1070 }
1071
1072 static const struct dev_pm_ops ufs_mtk_pm_ops = {
1073         .suspend         = ufshcd_pltfrm_suspend,
1074         .resume          = ufshcd_pltfrm_resume,
1075         .runtime_suspend = ufshcd_pltfrm_runtime_suspend,
1076         .runtime_resume  = ufshcd_pltfrm_runtime_resume,
1077         .runtime_idle    = ufshcd_pltfrm_runtime_idle,
1078 };
1079
1080 static struct platform_driver ufs_mtk_pltform = {
1081         .probe      = ufs_mtk_probe,
1082         .remove     = ufs_mtk_remove,
1083         .shutdown   = ufshcd_pltfrm_shutdown,
1084         .driver = {
1085                 .name   = "ufshcd-mtk",
1086                 .pm     = &ufs_mtk_pm_ops,
1087                 .of_match_table = ufs_mtk_of_match,
1088         },
1089 };
1090
1091 MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
1092 MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>");
1093 MODULE_DESCRIPTION("MediaTek UFS Host Driver");
1094 MODULE_LICENSE("GPL v2");
1095
1096 module_platform_driver(ufs_mtk_pltform);