drivers/memory/tegra/tegra210-emc-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/clk.h>
   8 #include <linux/clk/tegra.h>
   9 #include <linux/debugfs.h>
  10 #include <linux/delay.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/of_address.h>
  14 #include <linux/of_platform.h>
  15 #include <linux/of_reserved_mem.h>
  16 #include <linux/slab.h>
  17 #include <linux/thermal.h>
  18 #include <soc/tegra/fuse.h>
  19 #include <soc/tegra/mc.h>
  20
  21 #include "tegra210-emc.h"
  22 #include "tegra210-mc.h"
  23
  24 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC */
  25 #define EMC_CLK_EMC_2X_CLK_SRC_SHIFT                    29
  26 #define EMC_CLK_EMC_2X_CLK_SRC_MASK                     \
  27         (0x7 << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)
  28 #define EMC_CLK_SOURCE_PLLM_LJ                          0x4
  29 #define EMC_CLK_SOURCE_PLLMB_LJ                         0x5
  30 #define EMC_CLK_FORCE_CC_TRIGGER                        BIT(27)
  31 #define EMC_CLK_MC_EMC_SAME_FREQ                        BIT(16)
  32 #define EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT                0
  33 #define EMC_CLK_EMC_2X_CLK_DIVISOR_MASK                 \
  34         (0xff << EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT)
  35
  36 /* CLK_RST_CONTROLLER_CLK_SOURCE_EMC_DLL */
  37 #define DLL_CLK_EMC_DLL_CLK_SRC_SHIFT                   29
  38 #define DLL_CLK_EMC_DLL_CLK_SRC_MASK                    \
  39         (0x7 << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT)
  40 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT              10
  41 #define DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK               \
  42         (0x3 << DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT)
  43 #define PLLM_VCOA                                       0
  44 #define PLLM_VCOB                                       1
  45 #define EMC_DLL_SWITCH_OUT                              2
  46 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT               0
  47 #define DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK                \
  48         (0xff << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT)
  49
  50 /* MC_EMEM_ARB_MISC0 */
  51 #define MC_EMEM_ARB_MISC0_EMC_SAME_FREQ                 BIT(27)
  52
  53 /* EMC_DATA_BRLSHFT_X */
  54 #define EMC0_EMC_DATA_BRLSHFT_0_INDEX   2
  55 #define EMC1_EMC_DATA_BRLSHFT_0_INDEX   3
  56 #define EMC0_EMC_DATA_BRLSHFT_1_INDEX   4
  57 #define EMC1_EMC_DATA_BRLSHFT_1_INDEX   5
  58
  59 #define TRIM_REG(chan, rank, reg, byte)                                 \
  60         (((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##       \
  61            _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK &    \
  62            next->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ##           \
  63                                  rank ## _ ## reg ## _INDEX]) >>        \
  64           EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##        \
  65           _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT)     \
  66          +                                                              \
  67          (((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##      \
  68             byte ## _DATA_BRLSHFT_MASK &                                \
  69             next->trim_perch_regs[EMC ## chan ##                        \
  70                               _EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
  71            EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ##       \
  72            byte ## _DATA_BRLSHFT_SHIFT) * 64))
  73
  74 #define CALC_TEMP(rank, reg, byte1, byte2, n)                           \
  75         (((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##    \
  76            reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
  77           EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##        \
  78           _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK)     \
  79          |                                                              \
  80          ((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ##\
  81            reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
  82           EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ##        \
  83           _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK))
  84
  85 #define REFRESH_SPEEDUP(value, speedup) \
  86                 (((value) & 0xffff0000) | ((value) & 0xffff) * (speedup))
  87
  88 #define LPDDR2_MR4_SRR GENMASK(2, 0)
  89
  90 static const struct tegra210_emc_sequence *tegra210_emc_sequences[] = {
  91         &tegra210_emc_r21021,
  92 };
  93
  94 static const struct tegra210_emc_table_register_offsets
  95 tegra210_emc_table_register_offsets = {
  96         .burst = {
  97                 EMC_RC,
  98                 EMC_RFC,
  99                 EMC_RFCPB,
 100                 EMC_REFCTRL2,
 101                 EMC_RFC_SLR,
 102                 EMC_RAS,
 103                 EMC_RP,
 104                 EMC_R2W,
 105                 EMC_W2R,
 106                 EMC_R2P,
 107                 EMC_W2P,
 108                 EMC_R2R,
 109                 EMC_TPPD,
 110                 EMC_CCDMW,
 111                 EMC_RD_RCD,
 112                 EMC_WR_RCD,
 113                 EMC_RRD,
 114                 EMC_REXT,
 115                 EMC_WEXT,
 116                 EMC_WDV_CHK,
 117                 EMC_WDV,
 118                 EMC_WSV,
 119                 EMC_WEV,
 120                 EMC_WDV_MASK,
 121                 EMC_WS_DURATION,
 122                 EMC_WE_DURATION,
 123                 EMC_QUSE,
 124                 EMC_QUSE_WIDTH,
 125                 EMC_IBDLY,
 126                 EMC_OBDLY,
 127                 EMC_EINPUT,
 128                 EMC_MRW6,
 129                 EMC_EINPUT_DURATION,
 130                 EMC_PUTERM_EXTRA,
 131                 EMC_PUTERM_WIDTH,
 132                 EMC_QRST,
 133                 EMC_QSAFE,
 134                 EMC_RDV,
 135                 EMC_RDV_MASK,
 136                 EMC_RDV_EARLY,
 137                 EMC_RDV_EARLY_MASK,
 138                 EMC_REFRESH,
 139                 EMC_BURST_REFRESH_NUM,
 140                 EMC_PRE_REFRESH_REQ_CNT,
 141                 EMC_PDEX2WR,
 142                 EMC_PDEX2RD,
 143                 EMC_PCHG2PDEN,
 144                 EMC_ACT2PDEN,
 145                 EMC_AR2PDEN,
 146                 EMC_RW2PDEN,
 147                 EMC_CKE2PDEN,
 148                 EMC_PDEX2CKE,
 149                 EMC_PDEX2MRR,
 150                 EMC_TXSR,
 151                 EMC_TXSRDLL,
 152                 EMC_TCKE,
 153                 EMC_TCKESR,
 154                 EMC_TPD,
 155                 EMC_TFAW,
 156                 EMC_TRPAB,
 157                 EMC_TCLKSTABLE,
 158                 EMC_TCLKSTOP,
 159                 EMC_MRW7,
 160                 EMC_TREFBW,
 161                 EMC_ODT_WRITE,
 162                 EMC_FBIO_CFG5,
 163                 EMC_FBIO_CFG7,
 164                 EMC_CFG_DIG_DLL,
 165                 EMC_CFG_DIG_DLL_PERIOD,
 166                 EMC_PMACRO_IB_RXRT,
 167                 EMC_CFG_PIPE_1,
 168                 EMC_CFG_PIPE_2,
 169                 EMC_PMACRO_QUSE_DDLL_RANK0_4,
 170                 EMC_PMACRO_QUSE_DDLL_RANK0_5,
 171                 EMC_PMACRO_QUSE_DDLL_RANK1_4,
 172                 EMC_PMACRO_QUSE_DDLL_RANK1_5,
 173                 EMC_MRW8,
 174                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_4,
 175                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_5,
 176                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_0,
 177                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_1,
 178                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_2,
 179                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_3,
 180                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_4,
 181                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK0_5,
 182                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_0,
 183                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_1,
 184                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_2,
 185                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_3,
 186                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_4,
 187                 EMC_PMACRO_OB_DDLL_LONG_DQS_RANK1_5,
 188                 EMC_PMACRO_DDLL_LONG_CMD_0,
 189                 EMC_PMACRO_DDLL_LONG_CMD_1,
 190                 EMC_PMACRO_DDLL_LONG_CMD_2,
 191                 EMC_PMACRO_DDLL_LONG_CMD_3,
 192                 EMC_PMACRO_DDLL_LONG_CMD_4,
 193                 EMC_PMACRO_DDLL_SHORT_CMD_0,
 194                 EMC_PMACRO_DDLL_SHORT_CMD_1,
 195                 EMC_PMACRO_DDLL_SHORT_CMD_2,
 196                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_3,
 197                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_3,
 198                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_3,
 199                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_3,
 200                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_3,
 201                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_3,
 202                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_3,
 203                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_3,
 204                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_3,
 205                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_3,
 206                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_3,
 207                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_3,
 208                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_3,
 209                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_3,
 210                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_3,
 211                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_3,
 212                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_3,
 213                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_3,
 214                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_3,
 215                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_3,
 216                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_0,
 217                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_1,
 218                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_2,
 219                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD0_3,
 220                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_0,
 221                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_1,
 222                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_2,
 223                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD1_3,
 224                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_0,
 225                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_1,
 226                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_2,
 227                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD2_3,
 228                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_0,
 229                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_1,
 230                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_2,
 231                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_CMD3_3,
 232                 EMC_TXDSRVTTGEN,
 233                 EMC_FDPD_CTRL_DQ,
 234                 EMC_FDPD_CTRL_CMD,
 235                 EMC_FBIO_SPARE,
 236                 EMC_ZCAL_INTERVAL,
 237                 EMC_ZCAL_WAIT_CNT,
 238                 EMC_MRS_WAIT_CNT,
 239                 EMC_MRS_WAIT_CNT2,
 240                 EMC_AUTO_CAL_CHANNEL,
 241                 EMC_DLL_CFG_0,
 242                 EMC_DLL_CFG_1,
 243                 EMC_PMACRO_AUTOCAL_CFG_COMMON,
 244                 EMC_PMACRO_ZCTRL,
 245                 EMC_CFG,
 246                 EMC_CFG_PIPE,
 247                 EMC_DYN_SELF_REF_CONTROL,
 248                 EMC_QPOP,
 249                 EMC_DQS_BRLSHFT_0,
 250                 EMC_DQS_BRLSHFT_1,
 251                 EMC_CMD_BRLSHFT_2,
 252                 EMC_CMD_BRLSHFT_3,
 253                 EMC_PMACRO_PAD_CFG_CTRL,
 254                 EMC_PMACRO_DATA_PAD_RX_CTRL,
 255                 EMC_PMACRO_CMD_PAD_RX_CTRL,
 256                 EMC_PMACRO_DATA_RX_TERM_MODE,
 257                 EMC_PMACRO_CMD_RX_TERM_MODE,
 258                 EMC_PMACRO_CMD_PAD_TX_CTRL,
 259                 EMC_PMACRO_DATA_PAD_TX_CTRL,
 260                 EMC_PMACRO_COMMON_PAD_TX_CTRL,
 261                 EMC_PMACRO_VTTGEN_CTRL_0,
 262                 EMC_PMACRO_VTTGEN_CTRL_1,
 263                 EMC_PMACRO_VTTGEN_CTRL_2,
 264                 EMC_PMACRO_BRICK_CTRL_RFU1,
 265                 EMC_PMACRO_CMD_BRICK_CTRL_FDPD,
 266                 EMC_PMACRO_BRICK_CTRL_RFU2,
 267                 EMC_PMACRO_DATA_BRICK_CTRL_FDPD,
 268                 EMC_PMACRO_BG_BIAS_CTRL_0,
 269                 EMC_CFG_3,
 270                 EMC_PMACRO_TX_PWRD_0,
 271                 EMC_PMACRO_TX_PWRD_1,
 272                 EMC_PMACRO_TX_PWRD_2,
 273                 EMC_PMACRO_TX_PWRD_3,
 274                 EMC_PMACRO_TX_PWRD_4,
 275                 EMC_PMACRO_TX_PWRD_5,
 276                 EMC_CONFIG_SAMPLE_DELAY,
 277                 EMC_PMACRO_TX_SEL_CLK_SRC_0,
 278                 EMC_PMACRO_TX_SEL_CLK_SRC_1,
 279                 EMC_PMACRO_TX_SEL_CLK_SRC_2,
 280                 EMC_PMACRO_TX_SEL_CLK_SRC_3,
 281                 EMC_PMACRO_TX_SEL_CLK_SRC_4,
 282                 EMC_PMACRO_TX_SEL_CLK_SRC_5,
 283                 EMC_PMACRO_DDLL_BYPASS,
 284                 EMC_PMACRO_DDLL_PWRD_0,
 285                 EMC_PMACRO_DDLL_PWRD_1,
 286                 EMC_PMACRO_DDLL_PWRD_2,
 287                 EMC_PMACRO_CMD_CTRL_0,
 288                 EMC_PMACRO_CMD_CTRL_1,
 289                 EMC_PMACRO_CMD_CTRL_2,
 290                 EMC_TR_TIMING_0,
 291                 EMC_TR_DVFS,
 292                 EMC_TR_CTRL_1,
 293                 EMC_TR_RDV,
 294                 EMC_TR_QPOP,
 295                 EMC_TR_RDV_MASK,
 296                 EMC_MRW14,
 297                 EMC_TR_QSAFE,
 298                 EMC_TR_QRST,
 299                 EMC_TRAINING_CTRL,
 300                 EMC_TRAINING_SETTLE,
 301                 EMC_TRAINING_VREF_SETTLE,
 302                 EMC_TRAINING_CA_FINE_CTRL,
 303                 EMC_TRAINING_CA_CTRL_MISC,
 304                 EMC_TRAINING_CA_CTRL_MISC1,
 305                 EMC_TRAINING_CA_VREF_CTRL,
 306                 EMC_TRAINING_QUSE_CORS_CTRL,
 307                 EMC_TRAINING_QUSE_FINE_CTRL,
 308                 EMC_TRAINING_QUSE_CTRL_MISC,
 309                 EMC_TRAINING_QUSE_VREF_CTRL,
 310                 EMC_TRAINING_READ_FINE_CTRL,
 311                 EMC_TRAINING_READ_CTRL_MISC,
 312                 EMC_TRAINING_READ_VREF_CTRL,
 313                 EMC_TRAINING_WRITE_FINE_CTRL,
 314                 EMC_TRAINING_WRITE_CTRL_MISC,
 315                 EMC_TRAINING_WRITE_VREF_CTRL,
 316                 EMC_TRAINING_MPC,
 317                 EMC_MRW15,
 318         },
 319         .trim = {
 320                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0,
 321                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1,
 322                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2,
 323                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3,
 324                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0,
 325                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1,
 326                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2,
 327                 EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3,
 328                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
 329                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
 330                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
 331                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
 332                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
 333                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
 334                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
 335                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
 336                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
 337                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
 338                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
 339                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
 340                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
 341                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
 342                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
 343                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
 344                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
 345                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
 346                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
 347                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
 348                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
 349                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
 350                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
 351                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
 352                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
 353                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
 354                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
 355                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
 356                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
 357                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
 358                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
 359                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
 360                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
 361                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
 362                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
 363                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
 364                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
 365                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
 366                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
 367                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
 368                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
 369                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
 370                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
 371                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
 372                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
 373                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
 374                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
 375                 EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
 376                 EMC_PMACRO_IB_VREF_DQS_0,
 377                 EMC_PMACRO_IB_VREF_DQS_1,
 378                 EMC_PMACRO_IB_VREF_DQ_0,
 379                 EMC_PMACRO_IB_VREF_DQ_1,
 380                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
 381                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
 382                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
 383                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
 384                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4,
 385                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5,
 386                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
 387                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
 388                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
 389                 EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
 390                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0,
 391                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1,
 392                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2,
 393                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0,
 394                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1,
 395                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2,
 396                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0,
 397                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1,
 398                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2,
 399                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0,
 400                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1,
 401                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2,
 402                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0,
 403                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1,
 404                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2,
 405                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0,
 406                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1,
 407                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2,
 408                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0,
 409                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1,
 410                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2,
 411                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0,
 412                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1,
 413                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2,
 414                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0,
 415                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1,
 416                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2,
 417                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0,
 418                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1,
 419                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2,
 420                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0,
 421                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1,
 422                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2,
 423                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0,
 424                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1,
 425                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2,
 426                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0,
 427                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1,
 428                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2,
 429                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0,
 430                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1,
 431                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2,
 432                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0,
 433                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1,
 434                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2,
 435                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0,
 436                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1,
 437                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2,
 438                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0,
 439                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1,
 440                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2,
 441                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0,
 442                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1,
 443                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2,
 444                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0,
 445                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1,
 446                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2,
 447                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0,
 448                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1,
 449                 EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2,
 450                 EMC_PMACRO_QUSE_DDLL_RANK0_0,
 451                 EMC_PMACRO_QUSE_DDLL_RANK0_1,
 452                 EMC_PMACRO_QUSE_DDLL_RANK0_2,
 453                 EMC_PMACRO_QUSE_DDLL_RANK0_3,
 454                 EMC_PMACRO_QUSE_DDLL_RANK1_0,
 455                 EMC_PMACRO_QUSE_DDLL_RANK1_1,
 456                 EMC_PMACRO_QUSE_DDLL_RANK1_2,
 457                 EMC_PMACRO_QUSE_DDLL_RANK1_3
 458         },
 459         .burst_mc = {
 460                 MC_EMEM_ARB_CFG,
 461                 MC_EMEM_ARB_OUTSTANDING_REQ,
 462                 MC_EMEM_ARB_REFPB_HP_CTRL,
 463                 MC_EMEM_ARB_REFPB_BANK_CTRL,
 464                 MC_EMEM_ARB_TIMING_RCD,
 465                 MC_EMEM_ARB_TIMING_RP,
 466                 MC_EMEM_ARB_TIMING_RC,
 467                 MC_EMEM_ARB_TIMING_RAS,
 468                 MC_EMEM_ARB_TIMING_FAW,
 469                 MC_EMEM_ARB_TIMING_RRD,
 470                 MC_EMEM_ARB_TIMING_RAP2PRE,
 471                 MC_EMEM_ARB_TIMING_WAP2PRE,
 472                 MC_EMEM_ARB_TIMING_R2R,
 473                 MC_EMEM_ARB_TIMING_W2W,
 474                 MC_EMEM_ARB_TIMING_R2W,
 475                 MC_EMEM_ARB_TIMING_CCDMW,
 476                 MC_EMEM_ARB_TIMING_W2R,
 477                 MC_EMEM_ARB_TIMING_RFCPB,
 478                 MC_EMEM_ARB_DA_TURNS,
 479                 MC_EMEM_ARB_DA_COVERS,
 480                 MC_EMEM_ARB_MISC0,
 481                 MC_EMEM_ARB_MISC1,
 482                 MC_EMEM_ARB_MISC2,
 483                 MC_EMEM_ARB_RING1_THROTTLE,
 484                 MC_EMEM_ARB_DHYST_CTRL,
 485                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_0,
 486                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_1,
 487                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_2,
 488                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_3,
 489                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_4,
 490                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_5,
 491                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_6,
 492                 MC_EMEM_ARB_DHYST_TIMEOUT_UTIL_7,
 493         },
 494         .la_scale = {
 495                 MC_MLL_MPCORER_PTSA_RATE,
 496                 MC_FTOP_PTSA_RATE,
 497                 MC_PTSA_GRANT_DECREMENT,
 498                 MC_LATENCY_ALLOWANCE_XUSB_0,
 499                 MC_LATENCY_ALLOWANCE_XUSB_1,
 500                 MC_LATENCY_ALLOWANCE_TSEC_0,
 501                 MC_LATENCY_ALLOWANCE_SDMMCA_0,
 502                 MC_LATENCY_ALLOWANCE_SDMMCAA_0,
 503                 MC_LATENCY_ALLOWANCE_SDMMC_0,
 504                 MC_LATENCY_ALLOWANCE_SDMMCAB_0,
 505                 MC_LATENCY_ALLOWANCE_PPCS_0,
 506                 MC_LATENCY_ALLOWANCE_PPCS_1,
 507                 MC_LATENCY_ALLOWANCE_MPCORE_0,
 508                 MC_LATENCY_ALLOWANCE_HC_0,
 509                 MC_LATENCY_ALLOWANCE_HC_1,
 510                 MC_LATENCY_ALLOWANCE_AVPC_0,
 511                 MC_LATENCY_ALLOWANCE_GPU_0,
 512                 MC_LATENCY_ALLOWANCE_GPU2_0,
 513                 MC_LATENCY_ALLOWANCE_NVENC_0,
 514                 MC_LATENCY_ALLOWANCE_NVDEC_0,
 515                 MC_LATENCY_ALLOWANCE_VIC_0,
 516                 MC_LATENCY_ALLOWANCE_VI2_0,
 517                 MC_LATENCY_ALLOWANCE_ISP2_0,
 518                 MC_LATENCY_ALLOWANCE_ISP2_1,
 519         },
 520         .burst_per_channel = {
 521                 { .bank = 0, .offset = EMC_MRW10, },
 522                 { .bank = 1, .offset = EMC_MRW10, },
 523                 { .bank = 0, .offset = EMC_MRW11, },
 524                 { .bank = 1, .offset = EMC_MRW11, },
 525                 { .bank = 0, .offset = EMC_MRW12, },
 526                 { .bank = 1, .offset = EMC_MRW12, },
 527                 { .bank = 0, .offset = EMC_MRW13, },
 528                 { .bank = 1, .offset = EMC_MRW13, },
 529         },
 530         .trim_per_channel = {
 531                 { .bank = 0, .offset = EMC_CMD_BRLSHFT_0, },
 532                 { .bank = 1, .offset = EMC_CMD_BRLSHFT_1, },
 533                 { .bank = 0, .offset = EMC_DATA_BRLSHFT_0, },
 534                 { .bank = 1, .offset = EMC_DATA_BRLSHFT_0, },
 535                 { .bank = 0, .offset = EMC_DATA_BRLSHFT_1, },
 536                 { .bank = 1, .offset = EMC_DATA_BRLSHFT_1, },
 537                 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_0, },
 538                 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_1, },
 539                 { .bank = 0, .offset = EMC_QUSE_BRLSHFT_2, },
 540                 { .bank = 1, .offset = EMC_QUSE_BRLSHFT_3, },
 541         },
 542         .vref_per_channel = {
 543                 {
 544                         .bank = 0,
 545                         .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
 546                 }, {
 547                         .bank = 1,
 548                         .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK0,
 549                 }, {
 550                         .bank = 0,
 551                         .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
 552                 }, {
 553                         .bank = 1,
 554                         .offset = EMC_TRAINING_OPT_DQS_IB_VREF_RANK1,
 555                 },
 556         },
 557 };
 558
 559 static void tegra210_emc_train(struct timer_list *timer)
 560 {
 561         struct tegra210_emc *emc = from_timer(emc, timer, training);
 562         unsigned long flags;
 563
 564         if (!emc->last)
 565                 return;
 566
 567         spin_lock_irqsave(&emc->lock, flags);
 568
 569         if (emc->sequence->periodic_compensation)
 570                 emc->sequence->periodic_compensation(emc);
 571
 572         spin_unlock_irqrestore(&emc->lock, flags);
 573
 574         mod_timer(&emc->training,
 575                   jiffies + msecs_to_jiffies(emc->training_interval));
 576 }
 577
 578 static void tegra210_emc_training_start(struct tegra210_emc *emc)
 579 {
 580         mod_timer(&emc->training,
 581                   jiffies + msecs_to_jiffies(emc->training_interval));
 582 }
 583
 584 static void tegra210_emc_training_stop(struct tegra210_emc *emc)
 585 {
 586         del_timer(&emc->training);
 587 }
 588
 589 static unsigned int tegra210_emc_get_temperature(struct tegra210_emc *emc)
 590 {
 591         unsigned long flags;
 592         u32 value, max = 0;
 593         unsigned int i;
 594
 595         spin_lock_irqsave(&emc->lock, flags);
 596
 597         for (i = 0; i < emc->num_devices; i++) {
 598                 value = tegra210_emc_mrr_read(emc, i, 4);
 599
 600                 if (value & BIT(7))
 601                         dev_dbg(emc->dev,
 602                                 "sensor reading changed for device %u: %08x\n",
 603                                 i, value);
 604
 605                 value = FIELD_GET(LPDDR2_MR4_SRR, value);
 606                 if (value > max)
 607                         max = value;
 608         }
 609
 610         spin_unlock_irqrestore(&emc->lock, flags);
 611
 612         return max;
 613 }
 614
 615 static void tegra210_emc_poll_refresh(struct timer_list *timer)
 616 {
 617         struct tegra210_emc *emc = from_timer(emc, timer, refresh_timer);
 618         unsigned int temperature;
 619
 620         if (!emc->debugfs.temperature)
 621                 temperature = tegra210_emc_get_temperature(emc);
 622         else
 623                 temperature = emc->debugfs.temperature;
 624
 625         if (temperature == emc->temperature)
 626                 goto reset;
 627
 628         switch (temperature) {
 629         case 0 ... 3:
 630                 /* temperature is fine, using regular refresh */
 631                 dev_dbg(emc->dev, "switching to nominal refresh...\n");
 632                 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_NOMINAL);
 633                 break;
 634
 635         case 4:
 636                 dev_dbg(emc->dev, "switching to 2x refresh...\n");
 637                 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_2X);
 638                 break;
 639
 640         case 5:
 641                 dev_dbg(emc->dev, "switching to 4x refresh...\n");
 642                 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_4X);
 643                 break;
 644
 645         case 6 ... 7:
 646                 dev_dbg(emc->dev, "switching to throttle refresh...\n");
 647                 tegra210_emc_set_refresh(emc, TEGRA210_EMC_REFRESH_THROTTLE);
 648                 break;
 649
 650         default:
 651                 WARN(1, "invalid DRAM temperature state %u\n", temperature);
 652                 return;
 653         }
 654
 655         emc->temperature = temperature;
 656
 657 reset:
 658         if (atomic_read(&emc->refresh_poll) > 0) {
 659                 unsigned int interval = emc->refresh_poll_interval;
 660                 unsigned int timeout = msecs_to_jiffies(interval);
 661
 662                 mod_timer(&emc->refresh_timer, jiffies + timeout);
 663         }
 664 }
 665
 666 static void tegra210_emc_poll_refresh_stop(struct tegra210_emc *emc)
 667 {
 668         atomic_set(&emc->refresh_poll, 0);
 669         del_timer_sync(&emc->refresh_timer);
 670 }
 671
 672 static void tegra210_emc_poll_refresh_start(struct tegra210_emc *emc)
 673 {
 674         atomic_set(&emc->refresh_poll, 1);
 675
 676         mod_timer(&emc->refresh_timer,
 677                   jiffies + msecs_to_jiffies(emc->refresh_poll_interval));
 678 }
 679
 680 static int tegra210_emc_cd_max_state(struct thermal_cooling_device *cd,
 681                                      unsigned long *state)
 682 {
 683         *state = 1;
 684
 685         return 0;
 686 }
 687
 688 static int tegra210_emc_cd_get_state(struct thermal_cooling_device *cd,
 689                                      unsigned long *state)
 690 {
 691         struct tegra210_emc *emc = cd->devdata;
 692
 693         *state = atomic_read(&emc->refresh_poll);
 694
 695         return 0;
 696 }
 697
 698 static int tegra210_emc_cd_set_state(struct thermal_cooling_device *cd,
 699                                      unsigned long state)
 700 {
 701         struct tegra210_emc *emc = cd->devdata;
 702
 703         if (state == atomic_read(&emc->refresh_poll))
 704                 return 0;
 705
 706         if (state)
 707                 tegra210_emc_poll_refresh_start(emc);
 708         else
 709                 tegra210_emc_poll_refresh_stop(emc);
 710
 711         return 0;
 712 }
 713
 714 static struct thermal_cooling_device_ops tegra210_emc_cd_ops = {
 715         .get_max_state = tegra210_emc_cd_max_state,
 716         .get_cur_state = tegra210_emc_cd_get_state,
 717         .set_cur_state = tegra210_emc_cd_set_state,
 718 };
 719
 720 static void tegra210_emc_set_clock(struct tegra210_emc *emc, u32 clksrc)
 721 {
 722         emc->sequence->set_clock(emc, clksrc);
 723
 724         if (emc->next->periodic_training)
 725                 tegra210_emc_training_start(emc);
 726         else
 727                 tegra210_emc_training_stop(emc);
 728 }
 729
 730 static void tegra210_change_dll_src(struct tegra210_emc *emc,
 731                                     u32 clksrc)
 732 {
 733         u32 dll_setting = emc->next->dll_clk_src;
 734         u32 emc_clk_src;
 735         u32 emc_clk_div;
 736
 737         emc_clk_src = (clksrc & EMC_CLK_EMC_2X_CLK_SRC_MASK) >>
 738                        EMC_CLK_EMC_2X_CLK_SRC_SHIFT;
 739         emc_clk_div = (clksrc & EMC_CLK_EMC_2X_CLK_DIVISOR_MASK) >>
 740                        EMC_CLK_EMC_2X_CLK_DIVISOR_SHIFT;
 741
 742         dll_setting &= ~(DLL_CLK_EMC_DLL_CLK_SRC_MASK |
 743                          DLL_CLK_EMC_DLL_CLK_DIVISOR_MASK);
 744         dll_setting |= emc_clk_src << DLL_CLK_EMC_DLL_CLK_SRC_SHIFT;
 745         dll_setting |= emc_clk_div << DLL_CLK_EMC_DLL_CLK_DIVISOR_SHIFT;
 746
 747         dll_setting &= ~DLL_CLK_EMC_DLL_DDLL_CLK_SEL_MASK;
 748         if (emc_clk_src == EMC_CLK_SOURCE_PLLMB_LJ)
 749                 dll_setting |= (PLLM_VCOB <<
 750                                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
 751         else if (emc_clk_src == EMC_CLK_SOURCE_PLLM_LJ)
 752                 dll_setting |= (PLLM_VCOA <<
 753                                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
 754         else
 755                 dll_setting |= (EMC_DLL_SWITCH_OUT <<
 756                                 DLL_CLK_EMC_DLL_DDLL_CLK_SEL_SHIFT);
 757
 758         tegra210_clk_emc_dll_update_setting(dll_setting);
 759
 760         if (emc->next->clk_out_enb_x_0_clk_enb_emc_dll)
 761                 tegra210_clk_emc_dll_enable(true);
 762         else
 763                 tegra210_clk_emc_dll_enable(false);
 764 }
 765
 766 int tegra210_emc_set_refresh(struct tegra210_emc *emc,
 767                              enum tegra210_emc_refresh refresh)
 768 {
 769         struct tegra210_emc_timing *timings;
 770         unsigned long flags;
 771
 772         if ((emc->dram_type != DRAM_TYPE_LPDDR2 &&
 773              emc->dram_type != DRAM_TYPE_LPDDR4) ||
 774             !emc->last)
 775                 return -ENODEV;
 776
 777         if (refresh > TEGRA210_EMC_REFRESH_THROTTLE)
 778                 return -EINVAL;
 779
 780         if (refresh == emc->refresh)
 781                 return 0;
 782
 783         spin_lock_irqsave(&emc->lock, flags);
 784
 785         if (refresh == TEGRA210_EMC_REFRESH_THROTTLE && emc->derated)
 786                 timings = emc->derated;
 787         else
 788                 timings = emc->nominal;
 789
 790         if (timings != emc->timings) {
 791                 unsigned int index = emc->last - emc->timings;
 792                 u32 clksrc;
 793
 794                 clksrc = emc->provider.configs[index].value |
 795                          EMC_CLK_FORCE_CC_TRIGGER;
 796
 797                 emc->next = &timings[index];
 798                 emc->timings = timings;
 799
 800                 tegra210_emc_set_clock(emc, clksrc);
 801         } else {
 802                 tegra210_emc_adjust_timing(emc, emc->last);
 803                 tegra210_emc_timing_update(emc);
 804
 805                 if (refresh != TEGRA210_EMC_REFRESH_NOMINAL)
 806                         emc_writel(emc, EMC_REF_REF_CMD, EMC_REF);
 807         }
 808
 809         spin_unlock_irqrestore(&emc->lock, flags);
 810
 811         return 0;
 812 }
 813
 814 u32 tegra210_emc_mrr_read(struct tegra210_emc *emc, unsigned int chip,
 815                           unsigned int address)
 816 {
 817         u32 value, ret = 0;
 818         unsigned int i;
 819
 820         value = (chip & EMC_MRR_DEV_SEL_MASK) << EMC_MRR_DEV_SEL_SHIFT |
 821                 (address & EMC_MRR_MA_MASK) << EMC_MRR_MA_SHIFT;
 822         emc_writel(emc, value, EMC_MRR);
 823
 824         for (i = 0; i < emc->num_channels; i++)
 825                 WARN(tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
 826                                                   EMC_EMC_STATUS_MRR_DIVLD, 1),
 827                      "Timed out waiting for MRR %u (ch=%u)\n", address, i);
 828
 829         for (i = 0; i < emc->num_channels; i++) {
 830                 value = emc_channel_readl(emc, i, EMC_MRR);
 831                 value &= EMC_MRR_DATA_MASK;
 832
 833                 ret = (ret << 16) | value;
 834         }
 835
 836         return ret;
 837 }
 838
 839 void tegra210_emc_do_clock_change(struct tegra210_emc *emc, u32 clksrc)
 840 {
 841         int err;
 842
 843         mc_readl(emc->mc, MC_EMEM_ADR_CFG);
 844         emc_readl(emc, EMC_INTSTATUS);
 845
 846         tegra210_clk_emc_update_setting(clksrc);
 847
 848         err = tegra210_emc_wait_for_update(emc, 0, EMC_INTSTATUS,
 849                                            EMC_INTSTATUS_CLKCHANGE_COMPLETE,
 850                                            true);
 851         if (err)
 852                 dev_warn(emc->dev, "clock change completion error: %d\n", err);
 853 }
 854
 855 struct tegra210_emc_timing *tegra210_emc_find_timing(struct tegra210_emc *emc,
 856                                                      unsigned long rate)
 857 {
 858         unsigned int i;
 859
 860         for (i = 0; i < emc->num_timings; i++)
 861                 if (emc->timings[i].rate * 1000UL == rate)
 862                         return &emc->timings[i];
 863
 864         return NULL;
 865 }
 866
 867 int tegra210_emc_wait_for_update(struct tegra210_emc *emc, unsigned int channel,
 868                                  unsigned int offset, u32 bit_mask, bool state)
 869 {
 870         unsigned int i;
 871         u32 value;
 872
 873         for (i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
 874                 value = emc_channel_readl(emc, channel, offset);
 875                 if (!!(value & bit_mask) == state)
 876                         return 0;
 877
 878                 udelay(1);
 879         }
 880
 881         return -ETIMEDOUT;
 882 }
 883
 884 void tegra210_emc_set_shadow_bypass(struct tegra210_emc *emc, int set)
 885 {
 886         u32 emc_dbg = emc_readl(emc, EMC_DBG);
 887
 888         if (set)
 889                 emc_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
 890         else
 891                 emc_writel(emc, emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
 892 }
 893
 894 u32 tegra210_emc_get_dll_state(struct tegra210_emc_timing *next)
 895 {
 896         if (next->emc_emrs & 0x1)
 897                 return 0;
 898
 899         return 1;
 900 }
 901
 902 void tegra210_emc_timing_update(struct tegra210_emc *emc)
 903 {
 904         unsigned int i;
 905         int err = 0;
 906
 907         emc_writel(emc, 0x1, EMC_TIMING_CONTROL);
 908
 909         for (i = 0; i < emc->num_channels; i++) {
 910                 err |= tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
 911                                                     EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
 912                                                     false);
 913         }
 914
 915         if (err)
 916                 dev_warn(emc->dev, "timing update error: %d\n", err);
 917 }
 918
 919 unsigned long tegra210_emc_actual_osc_clocks(u32 in)
 920 {
 921         if (in < 0x40)
 922                 return in * 16;
 923         else if (in < 0x80)
 924                 return 2048;
 925         else if (in < 0xc0)
 926                 return 4096;
 927         else
 928                 return 8192;
 929 }
 930
 931 void tegra210_emc_start_periodic_compensation(struct tegra210_emc *emc)
 932 {
 933         u32 mpc_req = 0x4b;
 934
 935         emc_writel(emc, mpc_req, EMC_MPC);
 936         mpc_req = emc_readl(emc, EMC_MPC);
 937 }
 938
 939 u32 tegra210_emc_compensate(struct tegra210_emc_timing *next, u32 offset)
 940 {
 941         u32 temp = 0, rate = next->rate / 1000;
 942         s32 delta[4], delta_taps[4];
 943         s32 new[] = {
 944                 TRIM_REG(0, 0, 0, 0),
 945                 TRIM_REG(0, 0, 0, 1),
 946                 TRIM_REG(0, 0, 1, 2),
 947                 TRIM_REG(0, 0, 1, 3),
 948
 949                 TRIM_REG(1, 0, 2, 4),
 950                 TRIM_REG(1, 0, 2, 5),
 951                 TRIM_REG(1, 0, 3, 6),
 952                 TRIM_REG(1, 0, 3, 7),
 953
 954                 TRIM_REG(0, 1, 0, 0),
 955                 TRIM_REG(0, 1, 0, 1),
 956                 TRIM_REG(0, 1, 1, 2),
 957                 TRIM_REG(0, 1, 1, 3),
 958
 959                 TRIM_REG(1, 1, 2, 4),
 960                 TRIM_REG(1, 1, 2, 5),
 961                 TRIM_REG(1, 1, 3, 6),
 962                 TRIM_REG(1, 1, 3, 7)
 963         };
 964         unsigned i;
 965
 966         switch (offset) {
 967         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
 968         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
 969         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
 970         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
 971         case EMC_DATA_BRLSHFT_0:
 972                 delta[0] = 128 * (next->current_dram_clktree[C0D0U0] -
 973                                   next->trained_dram_clktree[C0D0U0]);
 974                 delta[1] = 128 * (next->current_dram_clktree[C0D0U1] -
 975                                   next->trained_dram_clktree[C0D0U1]);
 976                 delta[2] = 128 * (next->current_dram_clktree[C1D0U0] -
 977                                   next->trained_dram_clktree[C1D0U0]);
 978                 delta[3] = 128 * (next->current_dram_clktree[C1D0U1] -
 979                                   next->trained_dram_clktree[C1D0U1]);
 980
 981                 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
 982                 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
 983                 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
 984                 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
 985
 986                 for (i = 0; i < 4; i++) {
 987                         if ((delta_taps[i] > next->tree_margin) ||
 988                             (delta_taps[i] < (-1 * next->tree_margin))) {
 989                                 new[i * 2] = new[i * 2] + delta_taps[i];
 990                                 new[i * 2 + 1] = new[i * 2 + 1] +
 991                                                         delta_taps[i];
 992                         }
 993                 }
 994
 995                 if (offset == EMC_DATA_BRLSHFT_0) {
 996                         for (i = 0; i < 8; i++)
 997                                 new[i] = new[i] / 64;
 998                 } else {
 999                         for (i = 0; i < 8; i++)
1000                                 new[i] = new[i] % 64;
1001                 }
1002
1003                 break;
1004
1005         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1006         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1007         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1008         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1009         case EMC_DATA_BRLSHFT_1:
1010                 delta[0] = 128 * (next->current_dram_clktree[C0D1U0] -
1011                                   next->trained_dram_clktree[C0D1U0]);
1012                 delta[1] = 128 * (next->current_dram_clktree[C0D1U1] -
1013                                   next->trained_dram_clktree[C0D1U1]);
1014                 delta[2] = 128 * (next->current_dram_clktree[C1D1U0] -
1015                                   next->trained_dram_clktree[C1D1U0]);
1016                 delta[3] = 128 * (next->current_dram_clktree[C1D1U1] -
1017                                   next->trained_dram_clktree[C1D1U1]);
1018
1019                 delta_taps[0] = (delta[0] * (s32)rate) / 1000000;
1020                 delta_taps[1] = (delta[1] * (s32)rate) / 1000000;
1021                 delta_taps[2] = (delta[2] * (s32)rate) / 1000000;
1022                 delta_taps[3] = (delta[3] * (s32)rate) / 1000000;
1023
1024                 for (i = 0; i < 4; i++) {
1025                         if ((delta_taps[i] > next->tree_margin) ||
1026                             (delta_taps[i] < (-1 * next->tree_margin))) {
1027                                 new[8 + i * 2] = new[8 + i * 2] +
1028                                                         delta_taps[i];
1029                                 new[8 + i * 2 + 1] = new[8 + i * 2 + 1] +
1030                                                         delta_taps[i];
1031                         }
1032                 }
1033
1034                 if (offset == EMC_DATA_BRLSHFT_1) {
1035                         for (i = 0; i < 8; i++)
1036                                 new[i + 8] = new[i + 8] / 64;
1037                 } else {
1038                         for (i = 0; i < 8; i++)
1039                                 new[i + 8] = new[i + 8] % 64;
1040                 }
1041
1042                 break;
1043         }
1044
1045         switch (offset) {
1046         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
1047                 temp = CALC_TEMP(0, 0, 0, 1, 0);
1048                 break;
1049
1050         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
1051                 temp = CALC_TEMP(0, 1, 2, 3, 2);
1052                 break;
1053
1054         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
1055                 temp = CALC_TEMP(0, 2, 4, 5, 4);
1056                 break;
1057
1058         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
1059                 temp = CALC_TEMP(0, 3, 6, 7, 6);
1060                 break;
1061
1062         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
1063                 temp = CALC_TEMP(1, 0, 0, 1, 8);
1064                 break;
1065
1066         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
1067                 temp = CALC_TEMP(1, 1, 2, 3, 10);
1068                 break;
1069
1070         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
1071                 temp = CALC_TEMP(1, 2, 4, 5, 12);
1072                 break;
1073
1074         case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
1075                 temp = CALC_TEMP(1, 3, 6, 7, 14);
1076                 break;
1077
1078         case EMC_DATA_BRLSHFT_0:
1079                 temp = ((new[0] <<
1080                          EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
1081                          EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
1082                        ((new[1] <<
1083                          EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
1084                          EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
1085                        ((new[2] <<
1086                          EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
1087                          EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
1088                        ((new[3] <<
1089                          EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
1090                          EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
1091                        ((new[4] <<
1092                          EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
1093                          EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
1094                        ((new[5] <<
1095                          EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
1096                          EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
1097                        ((new[6] <<
1098                          EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
1099                          EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
1100                        ((new[7] <<
1101                          EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
1102                          EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
1103                 break;
1104
1105         case EMC_DATA_BRLSHFT_1:
1106                 temp = ((new[8] <<
1107                          EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
1108                          EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
1109                        ((new[9] <<
1110                          EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
1111                          EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
1112                        ((new[10] <<
1113                          EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
1114                          EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
1115                        ((new[11] <<
1116                          EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
1117                          EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
1118                        ((new[12] <<
1119                          EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
1120                          EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
1121                        ((new[13] <<
1122                          EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
1123                          EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
1124                        ((new[14] <<
1125                          EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
1126                          EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
1127                        ((new[15] <<
1128                          EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
1129                          EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
1130                 break;
1131
1132         default:
1133                 break;
1134         }
1135
1136         return temp;
1137 }
1138
1139 u32 tegra210_emc_dll_prelock(struct tegra210_emc *emc, u32 clksrc)
1140 {
1141         unsigned int i;
1142         u32 value;
1143
1144         value = emc_readl(emc, EMC_CFG_DIG_DLL);
1145         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK;
1146         value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
1147         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1148         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
1149         value |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
1150         value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
1151         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
1152         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
1153         emc_writel(emc, value, EMC_CFG_DIG_DLL);
1154         emc_writel(emc, 1, EMC_TIMING_CONTROL);
1155
1156         for (i = 0; i < emc->num_channels; i++)
1157                 tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
1158                                              EMC_EMC_STATUS_TIMING_UPDATE_STALLED,
1159                                              0);
1160
1161         for (i = 0; i < emc->num_channels; i++) {
1162                 while (true) {
1163                         value = emc_channel_readl(emc, i, EMC_CFG_DIG_DLL);
1164                         if ((value & EMC_CFG_DIG_DLL_CFG_DLL_EN) == 0)
1165                                 break;
1166                 }
1167         }
1168
1169         value = emc->next->burst_regs[EMC_DLL_CFG_0_INDEX];
1170         emc_writel(emc, value, EMC_DLL_CFG_0);
1171
1172         value = emc_readl(emc, EMC_DLL_CFG_1);
1173         value &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
1174
1175         if (emc->next->rate >= 400000 && emc->next->rate < 600000)
1176                 value |= 150;
1177         else if (emc->next->rate >= 600000 && emc->next->rate < 800000)
1178                 value |= 100;
1179         else if (emc->next->rate >= 800000 && emc->next->rate < 1000000)
1180                 value |= 70;
1181         else if (emc->next->rate >= 1000000 && emc->next->rate < 1200000)
1182                 value |= 30;
1183         else
1184                 value |= 20;
1185
1186         emc_writel(emc, value, EMC_DLL_CFG_1);
1187
1188         tegra210_change_dll_src(emc, clksrc);
1189
1190         value = emc_readl(emc, EMC_CFG_DIG_DLL);
1191         value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1192         emc_writel(emc, value, EMC_CFG_DIG_DLL);
1193
1194         tegra210_emc_timing_update(emc);
1195
1196         for (i = 0; i < emc->num_channels; i++) {
1197                 while (true) {
1198                         value = emc_channel_readl(emc, 0, EMC_CFG_DIG_DLL);
1199                         if (value & EMC_CFG_DIG_DLL_CFG_DLL_EN)
1200                                 break;
1201                 }
1202         }
1203
1204         while (true) {
1205                 value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1206
1207                 if ((value & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED) == 0)
1208                         continue;
1209
1210                 if ((value & EMC_DIG_DLL_STATUS_DLL_LOCK) == 0)
1211                         continue;
1212
1213                 break;
1214         }
1215
1216         value = emc_readl(emc, EMC_DIG_DLL_STATUS);
1217
1218         return value & EMC_DIG_DLL_STATUS_DLL_OUT_MASK;
1219 }
1220
1221 u32 tegra210_emc_dvfs_power_ramp_up(struct tegra210_emc *emc, u32 clk,
1222                                     bool flip_backward)
1223 {
1224         u32 cmd_pad, dq_pad, rfu1, cfg5, common_tx, ramp_up_wait = 0;
1225         const struct tegra210_emc_timing *timing;
1226
1227         if (flip_backward)
1228                 timing = emc->last;
1229         else
1230                 timing = emc->next;
1231
1232         cmd_pad = timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1233         dq_pad = timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1234         rfu1 = timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1235         cfg5 = timing->burst_regs[EMC_FBIO_CFG5_INDEX];
1236         common_tx = timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1237
1238         cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1239
1240         if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1241                 ccfifo_writel(emc, common_tx & 0xa,
1242                               EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1243                 ccfifo_writel(emc, common_tx & 0xf,
1244                               EMC_PMACRO_COMMON_PAD_TX_CTRL,
1245                               (100000 / clk) + 1);
1246                 ramp_up_wait += 100000;
1247         } else {
1248                 ccfifo_writel(emc, common_tx | 0x8,
1249                               EMC_PMACRO_COMMON_PAD_TX_CTRL, 0);
1250         }
1251
1252         if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
1253                 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1254                         cmd_pad |=
1255                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1256                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1257                         cmd_pad &=
1258                                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1259                                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1260                         ccfifo_writel(emc, cmd_pad,
1261                                       EMC_PMACRO_CMD_PAD_TX_CTRL,
1262                                       (100000 / clk) + 1);
1263                         ramp_up_wait += 100000;
1264
1265                         dq_pad |=
1266                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1267                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1268                         dq_pad &=
1269                                ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1270                                  EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1271                         ccfifo_writel(emc, dq_pad,
1272                                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1273                         ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1274                                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1275                 } else {
1276                         ccfifo_writel(emc, rfu1 & 0xfe40fe40,
1277                                       EMC_PMACRO_BRICK_CTRL_RFU1,
1278                                       (100000 / clk) + 1);
1279                         ramp_up_wait += 100000;
1280                 }
1281
1282                 ccfifo_writel(emc, rfu1 & 0xfeedfeed,
1283                               EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1284                 ramp_up_wait += 100000;
1285
1286                 if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
1287                         cmd_pad |=
1288                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1289                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
1290                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1291                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC;
1292                         ccfifo_writel(emc, cmd_pad,
1293                                       EMC_PMACRO_CMD_PAD_TX_CTRL,
1294                                       (100000 / clk) + 1);
1295                         ramp_up_wait += 100000;
1296
1297                         dq_pad |=
1298                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1299                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
1300                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1301                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC;
1302                         ccfifo_writel(emc, dq_pad,
1303                                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1304                         ccfifo_writel(emc, rfu1,
1305                                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1306                 } else {
1307                         ccfifo_writel(emc, rfu1,
1308                                       EMC_PMACRO_BRICK_CTRL_RFU1,
1309                                       (100000 / clk) + 1);
1310                         ramp_up_wait += 100000;
1311                 }
1312
1313                 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1314                               EMC_FBIO_CFG5, (100000 / clk) + 10);
1315                 ramp_up_wait += 100000 + (10 * clk);
1316         } else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
1317                 ccfifo_writel(emc, rfu1 | 0x06000600,
1318                               EMC_PMACRO_BRICK_CTRL_RFU1, (100000 / clk) + 1);
1319                 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1320                               EMC_FBIO_CFG5, (100000 / clk) + 10);
1321                 ramp_up_wait += 100000 + 10 * clk;
1322         } else {
1323                 ccfifo_writel(emc, rfu1 | 0x00000600,
1324                               EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1325                 ccfifo_writel(emc, cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS,
1326                               EMC_FBIO_CFG5, 12);
1327                 ramp_up_wait += 12 * clk;
1328         }
1329
1330         cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1331         ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 5);
1332
1333         return ramp_up_wait;
1334 }
1335
1336 u32 tegra210_emc_dvfs_power_ramp_down(struct tegra210_emc *emc, u32 clk,
1337                                       bool flip_backward)
1338 {
1339         u32 ramp_down_wait = 0, cmd_pad, dq_pad, rfu1, cfg5, common_tx;
1340         const struct tegra210_emc_timing *entry;
1341         u32 seq_wait;
1342
1343         if (flip_backward)
1344                 entry = emc->next;
1345         else
1346                 entry = emc->last;
1347
1348         cmd_pad = entry->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
1349         dq_pad = entry->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
1350         rfu1 = entry->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
1351         cfg5 = entry->burst_regs[EMC_FBIO_CFG5_INDEX];
1352         common_tx = entry->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
1353
1354         cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
1355
1356         ccfifo_writel(emc, cmd_pad, EMC_PMACRO_CMD_PAD_TX_CTRL, 0);
1357         ccfifo_writel(emc, cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS,
1358                       EMC_FBIO_CFG5, 12);
1359         ramp_down_wait = 12 * clk;
1360
1361         seq_wait = (100000 / clk) + 1;
1362
1363         if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
1364                 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1365                         cmd_pad &=
1366                                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1367                                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
1368                         cmd_pad |=
1369                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1370                                 EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
1371                         ccfifo_writel(emc, cmd_pad,
1372                                       EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1373                         ramp_down_wait += 100000;
1374
1375                         dq_pad &=
1376                               ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1377                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
1378                         dq_pad |=
1379                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1380                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
1381                         ccfifo_writel(emc, dq_pad,
1382                                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1383                         ccfifo_writel(emc, rfu1 & ~0x01120112,
1384                                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1385                 } else {
1386                         ccfifo_writel(emc, rfu1 & ~0x01120112,
1387                                       EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1388                         ramp_down_wait += 100000;
1389                 }
1390
1391                 ccfifo_writel(emc, rfu1 & ~0x01bf01bf,
1392                               EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1393                 ramp_down_wait += 100000;
1394
1395                 if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
1396                         cmd_pad &=
1397                                 ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
1398                                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC |
1399                                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
1400                                   EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
1401                         ccfifo_writel(emc, cmd_pad,
1402                                       EMC_PMACRO_CMD_PAD_TX_CTRL, seq_wait);
1403                         ramp_down_wait += 100000;
1404
1405                         dq_pad &=
1406                               ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
1407                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC |
1408                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
1409                                 EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
1410                         ccfifo_writel(emc, dq_pad,
1411                                       EMC_PMACRO_DATA_PAD_TX_CTRL, 0);
1412                         ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1413                                       EMC_PMACRO_BRICK_CTRL_RFU1, 0);
1414                 } else {
1415                         ccfifo_writel(emc, rfu1 & ~0x07ff07ff,
1416                                       EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait);
1417                         ramp_down_wait += 100000;
1418                 }
1419         } else {
1420                 ccfifo_writel(emc, rfu1 & ~0xffff07ff,
1421                               EMC_PMACRO_BRICK_CTRL_RFU1, seq_wait + 19);
1422                 ramp_down_wait += 100000 + (20 * clk);
1423         }
1424
1425         if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
1426                 ramp_down_wait += 100000;
1427                 ccfifo_writel(emc, common_tx & ~0x5,
1428                               EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1429                 ramp_down_wait += 100000;
1430                 ccfifo_writel(emc, common_tx & ~0xf,
1431                               EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1432                 ramp_down_wait += 100000;
1433                 ccfifo_writel(emc, 0, 0, seq_wait);
1434                 ramp_down_wait += 100000;
1435         } else {
1436                 ccfifo_writel(emc, common_tx & ~0xf,
1437                               EMC_PMACRO_COMMON_PAD_TX_CTRL, seq_wait);
1438         }
1439
1440         return ramp_down_wait;
1441 }
1442
1443 void tegra210_emc_reset_dram_clktree_values(struct tegra210_emc_timing *timing)
1444 {
1445         timing->current_dram_clktree[C0D0U0] =
1446                 timing->trained_dram_clktree[C0D0U0];
1447         timing->current_dram_clktree[C0D0U1] =
1448                 timing->trained_dram_clktree[C0D0U1];
1449         timing->current_dram_clktree[C1D0U0] =
1450                 timing->trained_dram_clktree[C1D0U0];
1451         timing->current_dram_clktree[C1D0U1] =
1452                 timing->trained_dram_clktree[C1D0U1];
1453         timing->current_dram_clktree[C1D1U0] =
1454                 timing->trained_dram_clktree[C1D1U0];
1455         timing->current_dram_clktree[C1D1U1] =
1456                 timing->trained_dram_clktree[C1D1U1];
1457 }
1458
1459 static void update_dll_control(struct tegra210_emc *emc, u32 value, bool state)
1460 {
1461         unsigned int i;
1462
1463         emc_writel(emc, value, EMC_CFG_DIG_DLL);
1464         tegra210_emc_timing_update(emc);
1465
1466         for (i = 0; i < emc->num_channels; i++)
1467                 tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
1468                                              EMC_CFG_DIG_DLL_CFG_DLL_EN,
1469                                              state);
1470 }
1471
1472 void tegra210_emc_dll_disable(struct tegra210_emc *emc)
1473 {
1474         u32 value;
1475
1476         value = emc_readl(emc, EMC_CFG_DIG_DLL);
1477         value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
1478
1479         update_dll_control(emc, value, false);
1480 }
1481
1482 void tegra210_emc_dll_enable(struct tegra210_emc *emc)
1483 {
1484         u32 value;
1485
1486         value = emc_readl(emc, EMC_CFG_DIG_DLL);
1487         value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
1488
1489         update_dll_control(emc, value, true);
1490 }
1491
1492 void tegra210_emc_adjust_timing(struct tegra210_emc *emc,
1493                                 struct tegra210_emc_timing *timing)
1494 {
1495         u32 dsr_cntrl = timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
1496         u32 pre_ref = timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
1497         u32 ref = timing->burst_regs[EMC_REFRESH_INDEX];
1498
1499         switch (emc->refresh) {
1500         case TEGRA210_EMC_REFRESH_NOMINAL:
1501         case TEGRA210_EMC_REFRESH_THROTTLE:
1502                 break;
1503
1504         case TEGRA210_EMC_REFRESH_2X:
1505                 ref = REFRESH_SPEEDUP(ref, 2);
1506                 pre_ref = REFRESH_SPEEDUP(pre_ref, 2);
1507                 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 2);
1508                 break;
1509
1510         case TEGRA210_EMC_REFRESH_4X:
1511                 ref = REFRESH_SPEEDUP(ref, 4);
1512                 pre_ref = REFRESH_SPEEDUP(pre_ref, 4);
1513                 dsr_cntrl = REFRESH_SPEEDUP(dsr_cntrl, 4);
1514                 break;
1515
1516         default:
1517                 dev_warn(emc->dev, "failed to set refresh: %d\n", emc->refresh);
1518                 return;
1519         }
1520
1521         emc_writel(emc, ref, emc->offsets->burst[EMC_REFRESH_INDEX]);
1522         emc_writel(emc, pre_ref,
1523                    emc->offsets->burst[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
1524         emc_writel(emc, dsr_cntrl,
1525                    emc->offsets->burst[EMC_DYN_SELF_REF_CONTROL_INDEX]);
1526 }
1527
1528 static int tegra210_emc_set_rate(struct device *dev,
1529                                  const struct tegra210_clk_emc_config *config)
1530 {
1531         struct tegra210_emc *emc = dev_get_drvdata(dev);
1532         struct tegra210_emc_timing *timing = NULL;
1533         unsigned long rate = config->rate;
1534         s64 last_change_delay;
1535         unsigned long flags;
1536         unsigned int i;
1537
1538         if (rate == emc->last->rate * 1000UL)
1539                 return 0;
1540
1541         for (i = 0; i < emc->num_timings; i++) {
1542                 if (emc->timings[i].rate * 1000UL == rate) {
1543                         timing = &emc->timings[i];
1544                         break;
1545                 }
1546         }
1547
1548         if (!timing)
1549                 return -EINVAL;
1550
1551         if (rate > 204000000 && !timing->trained)
1552                 return -EINVAL;
1553
1554         emc->next = timing;
1555         last_change_delay = ktime_us_delta(ktime_get(), emc->clkchange_time);
1556
1557         /* XXX use non-busy-looping sleep? */
1558         if ((last_change_delay >= 0) &&
1559             (last_change_delay < emc->clkchange_delay))
1560                 udelay(emc->clkchange_delay - (int)last_change_delay);
1561
1562         spin_lock_irqsave(&emc->lock, flags);
1563         tegra210_emc_set_clock(emc, config->value);
1564         emc->clkchange_time = ktime_get();
1565         emc->last = timing;
1566         spin_unlock_irqrestore(&emc->lock, flags);
1567
1568         return 0;
1569 }
1570
1571 /*
1572  * debugfs interface
1573  *
1574  * The memory controller driver exposes some files in debugfs that can be used
1575  * to control the EMC frequency. The top-level directory can be found here:
1576  *
1577  *   /sys/kernel/debug/emc
1578  *
1579  * It contains the following files:
1580  *
1581  *   - available_rates: This file contains a list of valid, space-separated
1582  *     EMC frequencies.
1583  *
1584  *   - min_rate: Writing a value to this file sets the given frequency as the
1585  *       floor of the permitted range. If this is higher than the currently
1586  *       configured EMC frequency, this will cause the frequency to be
1587  *       increased so that it stays within the valid range.
1588  *
1589  *   - max_rate: Similarily to the min_rate file, writing a value to this file
1590  *       sets the given frequency as the ceiling of the permitted range. If
1591  *       the value is lower than the currently configured EMC frequency, this
1592  *       will cause the frequency to be decreased so that it stays within the
1593  *       valid range.
1594  */
1595
1596 static bool tegra210_emc_validate_rate(struct tegra210_emc *emc,
1597                                        unsigned long rate)
1598 {
1599         unsigned int i;
1600
1601         for (i = 0; i < emc->num_timings; i++)
1602                 if (rate == emc->timings[i].rate * 1000UL)
1603                         return true;
1604
1605         return false;
1606 }
1607
1608 static int tegra210_emc_debug_available_rates_show(struct seq_file *s,
1609                                                    void *data)
1610 {
1611         struct tegra210_emc *emc = s->private;
1612         const char *prefix = "";
1613         unsigned int i;
1614
1615         for (i = 0; i < emc->num_timings; i++) {
1616                 seq_printf(s, "%s%u", prefix, emc->timings[i].rate * 1000);
1617                 prefix = " ";
1618         }
1619
1620         seq_puts(s, "\n");
1621
1622         return 0;
1623 }
1624
1625 static int tegra210_emc_debug_available_rates_open(struct inode *inode,
1626                                                    struct file *file)
1627 {
1628         return single_open(file, tegra210_emc_debug_available_rates_show,
1629                            inode->i_private);
1630 }
1631
1632 static const struct file_operations tegra210_emc_debug_available_rates_fops = {
1633         .open = tegra210_emc_debug_available_rates_open,
1634         .read = seq_read,
1635         .llseek = seq_lseek,
1636         .release = single_release,
1637 };
1638
1639 static int tegra210_emc_debug_min_rate_get(void *data, u64 *rate)
1640 {
1641         struct tegra210_emc *emc = data;
1642
1643         *rate = emc->debugfs.min_rate;
1644
1645         return 0;
1646 }
1647
1648 static int tegra210_emc_debug_min_rate_set(void *data, u64 rate)
1649 {
1650         struct tegra210_emc *emc = data;
1651         int err;
1652
1653         if (!tegra210_emc_validate_rate(emc, rate))
1654                 return -EINVAL;
1655
1656         err = clk_set_min_rate(emc->clk, rate);
1657         if (err < 0)
1658                 return err;
1659
1660         emc->debugfs.min_rate = rate;
1661
1662         return 0;
1663 }
1664
1665 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_min_rate_fops,
1666                         tegra210_emc_debug_min_rate_get,
1667                         tegra210_emc_debug_min_rate_set, "%llu\n");
1668
1669 static int tegra210_emc_debug_max_rate_get(void *data, u64 *rate)
1670 {
1671         struct tegra210_emc *emc = data;
1672
1673         *rate = emc->debugfs.max_rate;
1674
1675         return 0;
1676 }
1677
1678 static int tegra210_emc_debug_max_rate_set(void *data, u64 rate)
1679 {
1680         struct tegra210_emc *emc = data;
1681         int err;
1682
1683         if (!tegra210_emc_validate_rate(emc, rate))
1684                 return -EINVAL;
1685
1686         err = clk_set_max_rate(emc->clk, rate);
1687         if (err < 0)
1688                 return err;
1689
1690         emc->debugfs.max_rate = rate;
1691
1692         return 0;
1693 }
1694
1695 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_max_rate_fops,
1696                         tegra210_emc_debug_max_rate_get,
1697                         tegra210_emc_debug_max_rate_set, "%llu\n");
1698
1699 static int tegra210_emc_debug_temperature_get(void *data, u64 *temperature)
1700 {
1701         struct tegra210_emc *emc = data;
1702         unsigned int value;
1703
1704         if (!emc->debugfs.temperature)
1705                 value = tegra210_emc_get_temperature(emc);
1706         else
1707                 value = emc->debugfs.temperature;
1708
1709         *temperature = value;
1710
1711         return 0;
1712 }
1713
1714 static int tegra210_emc_debug_temperature_set(void *data, u64 temperature)
1715 {
1716         struct tegra210_emc *emc = data;
1717
1718         if (temperature > 7)
1719                 return -EINVAL;
1720
1721         emc->debugfs.temperature = temperature;
1722
1723         return 0;
1724 }
1725
1726 DEFINE_SIMPLE_ATTRIBUTE(tegra210_emc_debug_temperature_fops,
1727                         tegra210_emc_debug_temperature_get,
1728                         tegra210_emc_debug_temperature_set, "%llu\n");
1729
1730 static void tegra210_emc_debugfs_init(struct tegra210_emc *emc)
1731 {
1732         struct device *dev = emc->dev;
1733         unsigned int i;
1734         int err;
1735
1736         emc->debugfs.min_rate = ULONG_MAX;
1737         emc->debugfs.max_rate = 0;
1738
1739         for (i = 0; i < emc->num_timings; i++) {
1740                 if (emc->timings[i].rate * 1000UL < emc->debugfs.min_rate)
1741                         emc->debugfs.min_rate = emc->timings[i].rate * 1000UL;
1742
1743                 if (emc->timings[i].rate * 1000UL > emc->debugfs.max_rate)
1744                         emc->debugfs.max_rate = emc->timings[i].rate * 1000UL;
1745         }
1746
1747         if (!emc->num_timings) {
1748                 emc->debugfs.min_rate = clk_get_rate(emc->clk);
1749                 emc->debugfs.max_rate = emc->debugfs.min_rate;
1750         }
1751
1752         err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
1753                                  emc->debugfs.max_rate);
1754         if (err < 0) {
1755                 dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
1756                         emc->debugfs.min_rate, emc->debugfs.max_rate,
1757                         emc->clk);
1758                 return;
1759         }
1760
1761         emc->debugfs.root = debugfs_create_dir("emc", NULL);
1762
1763         debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
1764                             &tegra210_emc_debug_available_rates_fops);
1765         debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
1766                             &tegra210_emc_debug_min_rate_fops);
1767         debugfs_create_file("max_rate", 0644, emc->debugfs.root, emc,
1768                             &tegra210_emc_debug_max_rate_fops);
1769         debugfs_create_file("temperature", 0644, emc->debugfs.root, emc,
1770                             &tegra210_emc_debug_temperature_fops);
1771 }
1772
1773 static void tegra210_emc_detect(struct tegra210_emc *emc)
1774 {
1775         u32 value;
1776
1777         /* probe the number of connected DRAM devices */
1778         value = mc_readl(emc->mc, MC_EMEM_ADR_CFG);
1779
1780         if (value & MC_EMEM_ADR_CFG_EMEM_NUMDEV)
1781                 emc->num_devices = 2;
1782         else
1783                 emc->num_devices = 1;
1784
1785         /* probe the type of DRAM */
1786         value = emc_readl(emc, EMC_FBIO_CFG5);
1787         emc->dram_type = value & 0x3;
1788
1789         /* probe the number of channels */
1790         value = emc_readl(emc, EMC_FBIO_CFG7);
1791
1792         if ((value & EMC_FBIO_CFG7_CH1_ENABLE) &&
1793             (value & EMC_FBIO_CFG7_CH0_ENABLE))
1794                 emc->num_channels = 2;
1795         else
1796                 emc->num_channels = 1;
1797 }
1798
1799 static int tegra210_emc_validate_timings(struct tegra210_emc *emc,
1800                                          struct tegra210_emc_timing *timings,
1801                                          unsigned int num_timings)
1802 {
1803         unsigned int i;
1804
1805         for (i = 0; i < num_timings; i++) {
1806                 u32 min_volt = timings[i].min_volt;
1807                 u32 rate = timings[i].rate;
1808
1809                 if (!rate)
1810                         return -EINVAL;
1811
1812                 if ((i > 0) && ((rate <= timings[i - 1].rate) ||
1813                     (min_volt < timings[i - 1].min_volt)))
1814                         return -EINVAL;
1815
1816                 if (timings[i].revision != timings[0].revision)
1817                         continue;
1818         }
1819
1820         return 0;
1821 }
1822
1823 static int tegra210_emc_probe(struct platform_device *pdev)
1824 {
1825         struct thermal_cooling_device *cd;
1826         unsigned long current_rate;
1827         struct tegra210_emc *emc;
1828         struct device_node *np;
1829         unsigned int i;
1830         int err;
1831
1832         emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);
1833         if (!emc)
1834                 return -ENOMEM;
1835
1836         emc->clk = devm_clk_get(&pdev->dev, "emc");
1837         if (IS_ERR(emc->clk))
1838                 return PTR_ERR(emc->clk);
1839
1840         platform_set_drvdata(pdev, emc);
1841         spin_lock_init(&emc->lock);
1842         emc->dev = &pdev->dev;
1843
1844         emc->mc = devm_tegra_memory_controller_get(&pdev->dev);
1845         if (IS_ERR(emc->mc))
1846                 return PTR_ERR(emc->mc);
1847
1848         emc->regs = devm_platform_ioremap_resource(pdev, 0);
1849         if (IS_ERR(emc->regs))
1850                 return PTR_ERR(emc->regs);
1851
1852         for (i = 0; i < 2; i++) {
1853                 emc->channel[i] = devm_platform_ioremap_resource(pdev, 1 + i);
1854                 if (IS_ERR(emc->channel[i]))
1855                         return PTR_ERR(emc->channel[i]);
1856
1857         }
1858
1859         tegra210_emc_detect(emc);
1860         np = pdev->dev.of_node;
1861
1862         /* attach to the nominal and (optional) derated tables */
1863         err = of_reserved_mem_device_init_by_name(emc->dev, np, "nominal");
1864         if (err < 0) {
1865                 dev_err(emc->dev, "failed to get nominal EMC table: %d\n", err);
1866                 return err;
1867         }
1868
1869         err = of_reserved_mem_device_init_by_name(emc->dev, np, "derated");
1870         if (err < 0 && err != -ENODEV) {
1871                 dev_err(emc->dev, "failed to get derated EMC table: %d\n", err);
1872                 goto release;
1873         }
1874
1875         /* validate the tables */
1876         if (emc->nominal) {
1877                 err = tegra210_emc_validate_timings(emc, emc->nominal,
1878                                                     emc->num_timings);
1879                 if (err < 0)
1880                         goto release;
1881         }
1882
1883         if (emc->derated) {
1884                 err = tegra210_emc_validate_timings(emc, emc->derated,
1885                                                     emc->num_timings);
1886                 if (err < 0)
1887                         goto release;
1888         }
1889
1890         /* default to the nominal table */
1891         emc->timings = emc->nominal;
1892
1893         /* pick the current timing based on the current EMC clock rate */
1894         current_rate = clk_get_rate(emc->clk) / 1000;
1895
1896         for (i = 0; i < emc->num_timings; i++) {
1897                 if (emc->timings[i].rate == current_rate) {
1898                         emc->last = &emc->timings[i];
1899                         break;
1900                 }
1901         }
1902
1903         if (i == emc->num_timings) {
1904                 dev_err(emc->dev, "no EMC table entry found for %lu kHz\n",
1905                         current_rate);
1906                 err = -ENOENT;
1907                 goto release;
1908         }
1909
1910         /* pick a compatible clock change sequence for the EMC table */
1911         for (i = 0; i < ARRAY_SIZE(tegra210_emc_sequences); i++) {
1912                 const struct tegra210_emc_sequence *sequence =
1913                                 tegra210_emc_sequences[i];
1914
1915                 if (emc->timings[0].revision == sequence->revision) {
1916                         emc->sequence = sequence;
1917                         break;
1918                 }
1919         }
1920
1921         if (!emc->sequence) {
1922                 dev_err(&pdev->dev, "sequence %u not supported\n",
1923                         emc->timings[0].revision);
1924                 err = -ENOTSUPP;
1925                 goto release;
1926         }
1927
1928         emc->offsets = &tegra210_emc_table_register_offsets;
1929         emc->refresh = TEGRA210_EMC_REFRESH_NOMINAL;
1930
1931         emc->provider.owner = THIS_MODULE;
1932         emc->provider.dev = &pdev->dev;
1933         emc->provider.set_rate = tegra210_emc_set_rate;
1934
1935         emc->provider.configs = devm_kcalloc(&pdev->dev, emc->num_timings,
1936                                              sizeof(*emc->provider.configs),
1937                                              GFP_KERNEL);
1938         if (!emc->provider.configs) {
1939                 err = -ENOMEM;
1940                 goto release;
1941         }
1942
1943         emc->provider.num_configs = emc->num_timings;
1944
1945         for (i = 0; i < emc->provider.num_configs; i++) {
1946                 struct tegra210_emc_timing *timing = &emc->timings[i];
1947                 struct tegra210_clk_emc_config *config =
1948                                 &emc->provider.configs[i];
1949                 u32 value;
1950
1951                 config->rate = timing->rate * 1000UL;
1952                 config->value = timing->clk_src_emc;
1953
1954                 value = timing->burst_mc_regs[MC_EMEM_ARB_MISC0_INDEX];
1955
1956                 if ((value & MC_EMEM_ARB_MISC0_EMC_SAME_FREQ) == 0)
1957                         config->same_freq = false;
1958                 else
1959                         config->same_freq = true;
1960         }
1961
1962         err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
1963         if (err < 0) {
1964                 dev_err(&pdev->dev, "failed to attach to EMC clock: %d\n", err);
1965                 goto release;
1966         }
1967
1968         emc->clkchange_delay = 100;
1969         emc->training_interval = 100;
1970         dev_set_drvdata(emc->dev, emc);
1971
1972         timer_setup(&emc->refresh_timer, tegra210_emc_poll_refresh,
1973                     TIMER_DEFERRABLE);
1974         atomic_set(&emc->refresh_poll, 0);
1975         emc->refresh_poll_interval = 1000;
1976
1977         timer_setup(&emc->training, tegra210_emc_train, 0);
1978
1979         tegra210_emc_debugfs_init(emc);
1980
1981         cd = devm_thermal_of_cooling_device_register(emc->dev, np, "emc", emc,
1982                                                      &tegra210_emc_cd_ops);
1983         if (IS_ERR(cd)) {
1984                 err = PTR_ERR(cd);
1985                 dev_err(emc->dev, "failed to register cooling device: %d\n",
1986                         err);
1987                 goto detach;
1988         }
1989
1990         return 0;
1991
1992 detach:
1993         debugfs_remove_recursive(emc->debugfs.root);
1994         tegra210_clk_emc_detach(emc->clk);
1995 release:
1996         of_reserved_mem_device_release(emc->dev);
1997
1998         return err;
1999 }
2000
2001 static int tegra210_emc_remove(struct platform_device *pdev)
2002 {
2003         struct tegra210_emc *emc = platform_get_drvdata(pdev);
2004
2005         debugfs_remove_recursive(emc->debugfs.root);
2006         tegra210_clk_emc_detach(emc->clk);
2007         of_reserved_mem_device_release(emc->dev);
2008
2009         return 0;
2010 }
2011
2012 static int __maybe_unused tegra210_emc_suspend(struct device *dev)
2013 {
2014         struct tegra210_emc *emc = dev_get_drvdata(dev);
2015         int err;
2016
2017         err = clk_rate_exclusive_get(emc->clk);
2018         if (err < 0) {
2019                 dev_err(emc->dev, "failed to acquire clock: %d\n", err);
2020                 return err;
2021         }
2022
2023         emc->resume_rate = clk_get_rate(emc->clk);
2024
2025         clk_set_rate(emc->clk, 204000000);
2026         tegra210_clk_emc_detach(emc->clk);
2027
2028         dev_dbg(dev, "suspending at %lu Hz\n", clk_get_rate(emc->clk));
2029
2030         return 0;
2031 }
2032
2033 static int __maybe_unused tegra210_emc_resume(struct device *dev)
2034 {
2035         struct tegra210_emc *emc = dev_get_drvdata(dev);
2036         int err;
2037
2038         err = tegra210_clk_emc_attach(emc->clk, &emc->provider);
2039         if (err < 0) {
2040                 dev_err(dev, "failed to attach to EMC clock: %d\n", err);
2041                 return err;
2042         }
2043
2044         clk_set_rate(emc->clk, emc->resume_rate);
2045         clk_rate_exclusive_put(emc->clk);
2046
2047         dev_dbg(dev, "resuming at %lu Hz\n", clk_get_rate(emc->clk));
2048
2049         return 0;
2050 }
2051
2052 static const struct dev_pm_ops tegra210_emc_pm_ops = {
2053         SET_SYSTEM_SLEEP_PM_OPS(tegra210_emc_suspend, tegra210_emc_resume)
2054 };
2055
2056 static const struct of_device_id tegra210_emc_of_match[] = {
2057         { .compatible = "nvidia,tegra210-emc", },
2058         { },
2059 };
2060 MODULE_DEVICE_TABLE(of, tegra210_emc_of_match);
2061
2062 static struct platform_driver tegra210_emc_driver = {
2063         .driver = {
2064                 .name = "tegra210-emc",
2065                 .of_match_table = tegra210_emc_of_match,
2066                 .pm = &tegra210_emc_pm_ops,
2067         },
2068         .probe = tegra210_emc_probe,
2069         .remove = tegra210_emc_remove,
2070 };
2071
2072 module_platform_driver(tegra210_emc_driver);
2073
2074 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
2075 MODULE_AUTHOR("Joseph Lo <josephl@nvidia.com>");
2076 MODULE_DESCRIPTION("NVIDIA Tegra210 EMC driver");
2077 MODULE_LICENSE("GPL v2");