drivers/memory/samsung/exynos5422-dmc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Samsung Electronics Co., Ltd.
   4  * Author: Lukasz Luba <l.luba@partner.samsung.com>
   5  */
   6
   7 #include <linux/clk.h>
   8 #include <linux/devfreq.h>
   9 #include <linux/devfreq-event.h>
  10 #include <linux/device.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/io.h>
  13 #include <linux/mfd/syscon.h>
  14 #include <linux/module.h>
  15 #include <linux/of_device.h>
  16 #include <linux/pm_opp.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/regmap.h>
  19 #include <linux/regulator/consumer.h>
  20 #include <linux/slab.h>
  21 #include "../jedec_ddr.h"
  22 #include "../of_memory.h"
  23
  24 #define EXYNOS5_DREXI_TIMINGAREF                (0x0030)
  25 #define EXYNOS5_DREXI_TIMINGROW0                (0x0034)
  26 #define EXYNOS5_DREXI_TIMINGDATA0               (0x0038)
  27 #define EXYNOS5_DREXI_TIMINGPOWER0              (0x003C)
  28 #define EXYNOS5_DREXI_TIMINGROW1                (0x00E4)
  29 #define EXYNOS5_DREXI_TIMINGDATA1               (0x00E8)
  30 #define EXYNOS5_DREXI_TIMINGPOWER1              (0x00EC)
  31 #define CDREX_PAUSE                             (0x2091c)
  32 #define CDREX_LPDDR3PHY_CON3                    (0x20a20)
  33 #define CDREX_LPDDR3PHY_CLKM_SRC                (0x20700)
  34 #define EXYNOS5_TIMING_SET_SWI                  BIT(28)
  35 #define USE_MX_MSPLL_TIMINGS                    (1)
  36 #define USE_BPLL_TIMINGS                        (0)
  37 #define EXYNOS5_AREF_NORMAL                     (0x2e)
  38
  39 #define DREX_PPCCLKCON          (0x0130)
  40 #define DREX_PEREV2CONFIG       (0x013c)
  41 #define DREX_PMNC_PPC           (0xE000)
  42 #define DREX_CNTENS_PPC         (0xE010)
  43 #define DREX_CNTENC_PPC         (0xE020)
  44 #define DREX_INTENS_PPC         (0xE030)
  45 #define DREX_INTENC_PPC         (0xE040)
  46 #define DREX_FLAG_PPC           (0xE050)
  47 #define DREX_PMCNT2_PPC         (0xE130)
  48
  49 /*
  50  * A value for register DREX_PMNC_PPC which should be written to reset
  51  * the cycle counter CCNT (a reference wall clock). It sets zero to the
  52  * CCNT counter.
  53  */
  54 #define CC_RESET                BIT(2)
  55
  56 /*
  57  * A value for register DREX_PMNC_PPC which does the reset of all performance
  58  * counters to zero.
  59  */
  60 #define PPC_COUNTER_RESET       BIT(1)
  61
  62 /*
  63  * Enables all configured counters (including cycle counter). The value should
  64  * be written to the register DREX_PMNC_PPC.
  65  */
  66 #define PPC_ENABLE              BIT(0)
  67
  68 /* A value for register DREX_PPCCLKCON which enables performance events clock.
  69  * Must be written before first access to the performance counters register
  70  * set, otherwise it could crash.
  71  */
  72 #define PEREV_CLK_EN            BIT(0)
  73
  74 /*
  75  * Values which are used to enable counters, interrupts or configure flags of
  76  * the performance counters. They configure counter 2 and cycle counter.
  77  */
  78 #define PERF_CNT2               BIT(2)
  79 #define PERF_CCNT               BIT(31)
  80
  81 /*
  82  * Performance event types which are used for setting the preferred event
  83  * to track in the counters.
  84  * There is a set of different types, the values are from range 0 to 0x6f.
  85  * These settings should be written to the configuration register which manages
  86  * the type of the event (register DREX_PEREV2CONFIG).
  87  */
  88 #define READ_TRANSFER_CH0       (0x6d)
  89 #define READ_TRANSFER_CH1       (0x6f)
  90
  91 #define PERF_COUNTER_START_VALUE 0xff000000
  92 #define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
  93
  94 /**
  95  * struct dmc_opp_table - Operating level desciption
  96  *
  97  * Covers frequency and voltage settings of the DMC operating mode.
  98  */
  99 struct dmc_opp_table {
 100         u32 freq_hz;
 101         u32 volt_uv;
 102 };
 103
 104 /**
 105  * struct exynos5_dmc - main structure describing DMC device
 106  *
 107  * The main structure for the Dynamic Memory Controller which covers clocks,
 108  * memory regions, HW information, parameters and current operating mode.
 109  */
 110 struct exynos5_dmc {
 111         struct device *dev;
 112         struct devfreq *df;
 113         struct devfreq_simple_ondemand_data gov_data;
 114         void __iomem *base_drexi0;
 115         void __iomem *base_drexi1;
 116         struct regmap *clk_regmap;
 117         struct mutex lock;
 118         unsigned long curr_rate;
 119         unsigned long curr_volt;
 120         unsigned long bypass_rate;
 121         struct dmc_opp_table *opp;
 122         struct dmc_opp_table opp_bypass;
 123         int opp_count;
 124         u32 timings_arr_size;
 125         u32 *timing_row;
 126         u32 *timing_data;
 127         u32 *timing_power;
 128         const struct lpddr3_timings *timings;
 129         const struct lpddr3_min_tck *min_tck;
 130         u32 bypass_timing_row;
 131         u32 bypass_timing_data;
 132         u32 bypass_timing_power;
 133         struct regulator *vdd_mif;
 134         struct clk *fout_spll;
 135         struct clk *fout_bpll;
 136         struct clk *mout_spll;
 137         struct clk *mout_bpll;
 138         struct clk *mout_mclk_cdrex;
 139         struct clk *mout_mx_mspll_ccore;
 140         struct clk *mx_mspll_ccore_phy;
 141         struct clk *mout_mx_mspll_ccore_phy;
 142         struct devfreq_event_dev **counter;
 143         int num_counters;
 144         u64 last_overflow_ts[2];
 145         unsigned long load;
 146         unsigned long total;
 147         bool in_irq_mode;
 148 };
 149
 150 #define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
 151         { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
 152
 153 #define TIMING_VAL2REG(timing, t_val)                   \
 154 ({                                                      \
 155                 u32 __val;                              \
 156                 __val = (t_val) << (timing)->bit_beg;   \
 157                 __val;                                  \
 158 })
 159
 160 struct timing_reg {
 161         char *name;
 162         int bit_beg;
 163         int bit_end;
 164         unsigned int val;
 165 };
 166
 167 static const struct timing_reg timing_row[] = {
 168         TIMING_FIELD("tRFC", 24, 31),
 169         TIMING_FIELD("tRRD", 20, 23),
 170         TIMING_FIELD("tRP", 16, 19),
 171         TIMING_FIELD("tRCD", 12, 15),
 172         TIMING_FIELD("tRC", 6, 11),
 173         TIMING_FIELD("tRAS", 0, 5),
 174 };
 175
 176 static const struct timing_reg timing_data[] = {
 177         TIMING_FIELD("tWTR", 28, 31),
 178         TIMING_FIELD("tWR", 24, 27),
 179         TIMING_FIELD("tRTP", 20, 23),
 180         TIMING_FIELD("tW2W-C2C", 14, 14),
 181         TIMING_FIELD("tR2R-C2C", 12, 12),
 182         TIMING_FIELD("WL", 8, 11),
 183         TIMING_FIELD("tDQSCK", 4, 7),
 184         TIMING_FIELD("RL", 0, 3),
 185 };
 186
 187 static const struct timing_reg timing_power[] = {
 188         TIMING_FIELD("tFAW", 26, 31),
 189         TIMING_FIELD("tXSR", 16, 25),
 190         TIMING_FIELD("tXP", 8, 15),
 191         TIMING_FIELD("tCKE", 4, 7),
 192         TIMING_FIELD("tMRD", 0, 3),
 193 };
 194
 195 #define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \
 196                       ARRAY_SIZE(timing_power))
 197
 198 static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
 199 {
 200         int i, ret;
 201
 202         for (i = 0; i < dmc->num_counters; i++) {
 203                 if (!dmc->counter[i])
 204                         continue;
 205                 ret = devfreq_event_set_event(dmc->counter[i]);
 206                 if (ret < 0)
 207                         return ret;
 208         }
 209         return 0;
 210 }
 211
 212 static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
 213 {
 214         int i, ret;
 215
 216         for (i = 0; i < dmc->num_counters; i++) {
 217                 if (!dmc->counter[i])
 218                         continue;
 219                 ret = devfreq_event_enable_edev(dmc->counter[i]);
 220                 if (ret < 0)
 221                         return ret;
 222         }
 223         return 0;
 224 }
 225
 226 static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
 227 {
 228         int i, ret;
 229
 230         for (i = 0; i < dmc->num_counters; i++) {
 231                 if (!dmc->counter[i])
 232                         continue;
 233                 ret = devfreq_event_disable_edev(dmc->counter[i]);
 234                 if (ret < 0)
 235                         return ret;
 236         }
 237         return 0;
 238 }
 239
 240 /**
 241  * find_target_freq_id() - Finds requested frequency in local DMC configuration
 242  * @dmc:        device for which the information is checked
 243  * @target_rate:        requested frequency in KHz
 244  *
 245  * Seeks in the local DMC driver structure for the requested frequency value
 246  * and returns index or error value.
 247  */
 248 static int find_target_freq_idx(struct exynos5_dmc *dmc,
 249                                 unsigned long target_rate)
 250 {
 251         int i;
 252
 253         for (i = dmc->opp_count - 1; i >= 0; i--)
 254                 if (dmc->opp[i].freq_hz <= target_rate)
 255                         return i;
 256
 257         return -EINVAL;
 258 }
 259
 260 /**
 261  * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
 262  * @dmc:        device for which the new settings is going to be applied
 263  * @set:        boolean variable passing set value
 264  *
 265  * Changes the register set, which holds timing parameters.
 266  * There is two register sets: 0 and 1. The register set 0
 267  * is used in normal operation when the clock is provided from main PLL.
 268  * The bank register set 1 is used when the main PLL frequency is going to be
 269  * changed and the clock is taken from alternative, stable source.
 270  * This function switches between these banks according to the
 271  * currently used clock source.
 272  */
 273 static void exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
 274 {
 275         unsigned int reg;
 276         int ret;
 277
 278         ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, &reg);
 279
 280         if (set)
 281                 reg |= EXYNOS5_TIMING_SET_SWI;
 282         else
 283                 reg &= ~EXYNOS5_TIMING_SET_SWI;
 284
 285         regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
 286 }
 287
 288 /**
 289  * exynos5_init_freq_table() - Initialized PM OPP framework
 290  * @dmc:        DMC device for which the frequencies are used for OPP init
 291  * @profile:    devfreq device's profile
 292  *
 293  * Populate the devfreq device's OPP table based on current frequency, voltage.
 294  */
 295 static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
 296                                    struct devfreq_dev_profile *profile)
 297 {
 298         int i, ret;
 299         int idx;
 300         unsigned long freq;
 301
 302         ret = dev_pm_opp_of_add_table(dmc->dev);
 303         if (ret < 0) {
 304                 dev_err(dmc->dev, "Failed to get OPP table\n");
 305                 return ret;
 306         }
 307
 308         dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
 309
 310         dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
 311                                       sizeof(struct dmc_opp_table), GFP_KERNEL);
 312         if (!dmc->opp)
 313                 goto err_opp;
 314
 315         idx = dmc->opp_count - 1;
 316         for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
 317                 struct dev_pm_opp *opp;
 318
 319                 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
 320                 if (IS_ERR(opp))
 321                         goto err_opp;
 322
 323                 dmc->opp[idx - i].freq_hz = freq;
 324                 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
 325
 326                 dev_pm_opp_put(opp);
 327         }
 328
 329         return 0;
 330
 331 err_opp:
 332         dev_pm_opp_of_remove_table(dmc->dev);
 333
 334         return -EINVAL;
 335 }
 336
 337 /**
 338  * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
 339  * @dmc:        device for which the new settings is going to be applied
 340  * @param:      DRAM parameters which passes timing data
 341  *
 342  * Low-level function for changing timings for DRAM memory clocking from
 343  * 'bypass' clock source (fixed frequency @400MHz).
 344  * It uses timing bank registers set 1.
 345  */
 346 static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
 347 {
 348         writel(EXYNOS5_AREF_NORMAL,
 349                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
 350
 351         writel(dmc->bypass_timing_row,
 352                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
 353         writel(dmc->bypass_timing_row,
 354                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
 355         writel(dmc->bypass_timing_data,
 356                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
 357         writel(dmc->bypass_timing_data,
 358                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
 359         writel(dmc->bypass_timing_power,
 360                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
 361         writel(dmc->bypass_timing_power,
 362                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
 363 }
 364
 365 /**
 366  * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
 367  * @dmc:        device for which the new settings is going to be applied
 368  * @target_rate:        target frequency of the DMC
 369  *
 370  * Low-level function for changing timings for DRAM memory operating from main
 371  * clock source (BPLL), which can have different frequencies. Thus, each
 372  * frequency must have corresponding timings register values in order to keep
 373  * the needed delays.
 374  * It uses timing bank registers set 0.
 375  */
 376 static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
 377                                        unsigned long target_rate)
 378 {
 379         int idx;
 380
 381         for (idx = dmc->opp_count - 1; idx >= 0; idx--)
 382                 if (dmc->opp[idx].freq_hz <= target_rate)
 383                         break;
 384
 385         if (idx < 0)
 386                 return -EINVAL;
 387
 388         writel(EXYNOS5_AREF_NORMAL,
 389                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
 390
 391         writel(dmc->timing_row[idx],
 392                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
 393         writel(dmc->timing_row[idx],
 394                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
 395         writel(dmc->timing_data[idx],
 396                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
 397         writel(dmc->timing_data[idx],
 398                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
 399         writel(dmc->timing_power[idx],
 400                dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
 401         writel(dmc->timing_power[idx],
 402                dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
 403
 404         return 0;
 405 }
 406
 407 /**
 408  * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
 409  * @dmc:        device for which it is going to be set
 410  * @target_volt:        new voltage which is chosen to be final
 411  *
 412  * Function tries to align voltage to the safe level for 'normal' mode.
 413  * It checks the need of higher voltage and changes the value. The target
 414  * voltage might be lower that currently set and still the system will be
 415  * stable.
 416  */
 417 static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
 418                                             unsigned long target_volt)
 419 {
 420         int ret = 0;
 421
 422         if (dmc->curr_volt <= target_volt)
 423                 return 0;
 424
 425         ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
 426                                     target_volt);
 427         if (!ret)
 428                 dmc->curr_volt = target_volt;
 429
 430         return ret;
 431 }
 432
 433 /**
 434  * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
 435  * @dmc:        device for which it is going to be set
 436  * @target_volt:        new voltage which is chosen to be final
 437  *
 438  * Function tries to align voltage to the safe level for the 'bypass' mode.
 439  * It checks the need of higher voltage and changes the value.
 440  * The target voltage must not be less than currently needed, because
 441  * for current frequency the device might become unstable.
 442  */
 443 static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
 444                                             unsigned long target_volt)
 445 {
 446         int ret = 0;
 447         unsigned long bypass_volt = dmc->opp_bypass.volt_uv;
 448
 449         target_volt = max(bypass_volt, target_volt);
 450
 451         if (dmc->curr_volt >= target_volt)
 452                 return 0;
 453
 454         ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
 455                                     target_volt);
 456         if (!ret)
 457                 dmc->curr_volt = target_volt;
 458
 459         return ret;
 460 }
 461
 462 /**
 463  * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
 464  * @dmc:        device for which it is going to be set
 465  * @target_rate:        new frequency which is chosen to be final
 466  *
 467  * Function changes the DRAM timings for the temporary 'bypass' mode.
 468  */
 469 static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
 470                                                  unsigned long target_rate)
 471 {
 472         int idx = find_target_freq_idx(dmc, target_rate);
 473
 474         if (idx < 0)
 475                 return -EINVAL;
 476
 477         exynos5_set_bypass_dram_timings(dmc);
 478
 479         return 0;
 480 }
 481
 482 /**
 483  * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
 484  * @dmc:        DMC device for which the switching is going to happen
 485  * @target_rate:        new frequency which is going to be set as a final
 486  * @target_volt:        new voltage which is going to be set as a final
 487  *
 488  * Function configures DMC and clocks for operating in temporary 'bypass' mode.
 489  * This mode is used only temporary but if required, changes voltage and timings
 490  * for DRAM chips. It switches the main clock to stable clock source for the
 491  * period of the main PLL reconfiguration.
 492  */
 493 static int
 494 exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
 495                                            unsigned long target_rate,
 496                                            unsigned long target_volt)
 497 {
 498         int ret;
 499
 500         /*
 501          * Having higher voltage for a particular frequency does not harm
 502          * the chip. Use it for the temporary frequency change when one
 503          * voltage manipulation might be avoided.
 504          */
 505         ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
 506         if (ret)
 507                 return ret;
 508
 509         /*
 510          * Longer delays for DRAM does not cause crash, the opposite does.
 511          */
 512         ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
 513         if (ret)
 514                 return ret;
 515
 516         /*
 517          * Delays are long enough, so use them for the new coming clock.
 518          */
 519         exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
 520
 521         return ret;
 522 }
 523
 524 /**
 525  * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
 526  * using safe procedure
 527  * @dmc:        device for which the frequency is going to be changed
 528  * @target_rate:        requested new frequency
 529  * @target_volt:        requested voltage which corresponds to the new frequency
 530  *
 531  * The DMC frequency change procedure requires a few steps.
 532  * The main requirement is to change the clock source in the clk mux
 533  * for the time of main clock PLL locking. The assumption is that the
 534  * alternative clock source set as parent is stable.
 535  * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
 536  * clock. This requires alignment in DRAM timing parameters for the new
 537  * T-period. There is two bank sets for keeping DRAM
 538  * timings: set 0 and set 1. The set 0 is used when main clock source is
 539  * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
 540  * the two bank sets is part of the process.
 541  * The voltage must also be aligned to the minimum required level. There is
 542  * this intermediate step with switching to 'bypass' parent clock source.
 543  * if the old voltage is lower, it requires an increase of the voltage level.
 544  * The complexity of the voltage manipulation is hidden in low level function.
 545  * In this function there is last alignment of the voltage level at the end.
 546  */
 547 static int
 548 exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
 549                                  unsigned long target_rate,
 550                                  unsigned long target_volt)
 551 {
 552         int ret;
 553
 554         ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
 555                                                          target_volt);
 556         if (ret)
 557                 return ret;
 558
 559         /*
 560          * Voltage is set at least to a level needed for this frequency,
 561          * so switching clock source is safe now.
 562          */
 563         clk_prepare_enable(dmc->fout_spll);
 564         clk_prepare_enable(dmc->mout_spll);
 565         clk_prepare_enable(dmc->mout_mx_mspll_ccore);
 566
 567         ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
 568         if (ret)
 569                 goto disable_clocks;
 570
 571         /*
 572          * We are safe to increase the timings for current bypass frequency.
 573          * Thanks to this the settings will be ready for the upcoming clock
 574          * source change.
 575          */
 576         exynos5_dram_change_timings(dmc, target_rate);
 577
 578         clk_set_rate(dmc->fout_bpll, target_rate);
 579
 580         exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
 581
 582         ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
 583         if (ret)
 584                 goto disable_clocks;
 585
 586         /*
 587          * Make sure if the voltage is not from 'bypass' settings and align to
 588          * the right level for power efficiency.
 589          */
 590         ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
 591
 592 disable_clocks:
 593         clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
 594         clk_disable_unprepare(dmc->mout_spll);
 595         clk_disable_unprepare(dmc->fout_spll);
 596
 597         return ret;
 598 }
 599
 600 /**
 601  * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
 602  * table.
 603  * @dmc:        device for which the frequency is going to be changed
 604  * @freq:       requested frequency in KHz
 605  * @target_rate:        returned frequency which is the same or lower than
 606  *                      requested
 607  * @target_volt:        returned voltage which corresponds to the returned
 608  *                      frequency
 609  *
 610  * Function gets requested frequency and checks OPP framework for needed
 611  * frequency and voltage. It populates the values 'target_rate' and
 612  * 'target_volt' or returns error value when OPP framework fails.
 613  */
 614 static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
 615                                      unsigned long *freq,
 616                                      unsigned long *target_rate,
 617                                      unsigned long *target_volt, u32 flags)
 618 {
 619         struct dev_pm_opp *opp;
 620
 621         opp = devfreq_recommended_opp(dmc->dev, freq, flags);
 622         if (IS_ERR(opp))
 623                 return PTR_ERR(opp);
 624
 625         *target_rate = dev_pm_opp_get_freq(opp);
 626         *target_volt = dev_pm_opp_get_voltage(opp);
 627         dev_pm_opp_put(opp);
 628
 629         return 0;
 630 }
 631
 632 /**
 633  * exynos5_dmc_target() - Function responsible for changing frequency of DMC
 634  * @dev:        device for which the frequency is going to be changed
 635  * @freq:       requested frequency in KHz
 636  * @flags:      flags provided for this frequency change request
 637  *
 638  * An entry function provided to the devfreq framework which provides frequency
 639  * change of the DMC. The function gets the possible rate from OPP table based
 640  * on requested frequency. It calls the next function responsible for the
 641  * frequency and voltage change. In case of failure, does not set 'curr_rate'
 642  * and returns error value to the framework.
 643  */
 644 static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
 645                               u32 flags)
 646 {
 647         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 648         unsigned long target_rate = 0;
 649         unsigned long target_volt = 0;
 650         int ret;
 651
 652         ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
 653                                         flags);
 654
 655         if (ret)
 656                 return ret;
 657
 658         if (target_rate == dmc->curr_rate)
 659                 return 0;
 660
 661         mutex_lock(&dmc->lock);
 662
 663         ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
 664
 665         if (ret) {
 666                 mutex_unlock(&dmc->lock);
 667                 return ret;
 668         }
 669
 670         dmc->curr_rate = target_rate;
 671
 672         mutex_unlock(&dmc->lock);
 673         return 0;
 674 }
 675
 676 /**
 677  * exynos5_counters_get() - Gets the performance counters values.
 678  * @dmc:        device for which the counters are going to be checked
 679  * @load_count: variable which is populated with counter value
 680  * @total_count:        variable which is used as 'wall clock' reference
 681  *
 682  * Function which provides performance counters values. It sums up counters for
 683  * two DMC channels. The 'total_count' is used as a reference and max value.
 684  * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
 685  */
 686 static int exynos5_counters_get(struct exynos5_dmc *dmc,
 687                                 unsigned long *load_count,
 688                                 unsigned long *total_count)
 689 {
 690         unsigned long total = 0;
 691         struct devfreq_event_data event;
 692         int ret, i;
 693
 694         *load_count = 0;
 695
 696         /* Take into account only read+write counters, but stop all */
 697         for (i = 0; i < dmc->num_counters; i++) {
 698                 if (!dmc->counter[i])
 699                         continue;
 700
 701                 ret = devfreq_event_get_event(dmc->counter[i], &event);
 702                 if (ret < 0)
 703                         return ret;
 704
 705                 *load_count += event.load_count;
 706
 707                 if (total < event.total_count)
 708                         total = event.total_count;
 709         }
 710
 711         *total_count = total;
 712
 713         return 0;
 714 }
 715
 716 /**
 717  * exynos5_dmc_start_perf_events() - Setup and start performance event counters
 718  * @dmc:        device for which the counters are going to be checked
 719  * @beg_value:  initial value for the counter
 720  *
 721  * Function which enables needed counters, interrupts and sets initial values
 722  * then starts the counters.
 723  */
 724 static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
 725                                           u32 beg_value)
 726 {
 727         /* Enable interrupts for counter 2 */
 728         writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
 729         writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
 730
 731         /* Enable counter 2 and CCNT  */
 732         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
 733         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
 734
 735         /* Clear overflow flag for all counters */
 736         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
 737         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
 738
 739         /* Reset all counters */
 740         writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
 741         writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
 742
 743         /*
 744          * Set start value for the counters, the number of samples that
 745          * will be gathered is calculated as: 0xffffffff - beg_value
 746          */
 747         writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
 748         writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
 749
 750         /* Start all counters */
 751         writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
 752         writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
 753 }
 754
 755 /**
 756  * exynos5_dmc_perf_events_calc() - Calculate utilization
 757  * @dmc:        device for which the counters are going to be checked
 758  * @diff_ts:    time between last interrupt and current one
 759  *
 760  * Function which calculates needed utilization for the devfreq governor.
 761  * It prepares values for 'busy_time' and 'total_time' based on elapsed time
 762  * between interrupts, which approximates utilization.
 763  */
 764 static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
 765 {
 766         /*
 767          * This is a simple algorithm for managing traffic on DMC.
 768          * When there is almost no load the counters overflow every 4s,
 769          * no mater the DMC frequency.
 770          * The high load might be approximated using linear function.
 771          * Knowing that, simple calculation can provide 'busy_time' and
 772          * 'total_time' to the devfreq governor which picks up target
 773          * frequency.
 774          * We want a fast ramp up and slow decay in frequency change function.
 775          */
 776         if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
 777                 /*
 778                  * Set higher utilization for the simple_ondemand governor.
 779                  * The governor should increase the frequency of the DMC.
 780                  */
 781                 dmc->load = 70;
 782                 dmc->total = 100;
 783         } else {
 784                 /*
 785                  * Set low utilization for the simple_ondemand governor.
 786                  * The governor should decrease the frequency of the DMC.
 787                  */
 788                 dmc->load = 35;
 789                 dmc->total = 100;
 790         }
 791
 792         dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
 793 }
 794
 795 /**
 796  * exynos5_dmc_perf_events_check() - Checks the status of the counters
 797  * @dmc:        device for which the counters are going to be checked
 798  *
 799  * Function which is called from threaded IRQ to check the counters state
 800  * and to call approximation for the needed utilization.
 801  */
 802 static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
 803 {
 804         u32 val;
 805         u64 diff_ts, ts;
 806
 807         ts = ktime_get_ns();
 808
 809         /* Stop all counters */
 810         writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
 811         writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
 812
 813         /* Check the source in interrupt flag registers (which channel) */
 814         val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
 815         if (val) {
 816                 diff_ts = ts - dmc->last_overflow_ts[0];
 817                 dmc->last_overflow_ts[0] = ts;
 818                 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n",  val);
 819         } else {
 820                 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
 821                 diff_ts = ts - dmc->last_overflow_ts[1];
 822                 dmc->last_overflow_ts[1] = ts;
 823                 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n",  val);
 824         }
 825
 826         exynos5_dmc_perf_events_calc(dmc, diff_ts);
 827
 828         exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
 829 }
 830
 831 /**
 832  * exynos5_dmc_enable_perf_events() - Enable performance events
 833  * @dmc:        device for which the counters are going to be checked
 834  *
 835  * Function which is setup needed environment and enables counters.
 836  */
 837 static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
 838 {
 839         u64 ts;
 840
 841         /* Enable Performance Event Clock */
 842         writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
 843         writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
 844
 845         /* Select read transfers as performance event2 */
 846         writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
 847         writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
 848
 849         ts = ktime_get_ns();
 850         dmc->last_overflow_ts[0] = ts;
 851         dmc->last_overflow_ts[1] = ts;
 852
 853         /* Devfreq shouldn't be faster than initialization, play safe though. */
 854         dmc->load = 99;
 855         dmc->total = 100;
 856 }
 857
 858 /**
 859  * exynos5_dmc_disable_perf_events() - Disable performance events
 860  * @dmc:        device for which the counters are going to be checked
 861  *
 862  * Function which stops, disables performance event counters and interrupts.
 863  */
 864 static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
 865 {
 866         /* Stop all counters */
 867         writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
 868         writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
 869
 870         /* Disable interrupts for counter 2 */
 871         writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
 872         writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
 873
 874         /* Disable counter 2 and CCNT  */
 875         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
 876         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
 877
 878         /* Clear overflow flag for all counters */
 879         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
 880         writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
 881 }
 882
 883 /**
 884  * exynos5_dmc_get_status() - Read current DMC performance statistics.
 885  * @dev:        device for which the statistics are requested
 886  * @stat:       structure which has statistic fields
 887  *
 888  * Function reads the DMC performance counters and calculates 'busy_time'
 889  * and 'total_time'. To protect from overflow, the values are shifted right
 890  * by 10. After read out the counters are setup to count again.
 891  */
 892 static int exynos5_dmc_get_status(struct device *dev,
 893                                   struct devfreq_dev_status *stat)
 894 {
 895         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 896         unsigned long load, total;
 897         int ret;
 898
 899         if (dmc->in_irq_mode) {
 900                 stat->current_frequency = dmc->curr_rate;
 901                 stat->busy_time = dmc->load;
 902                 stat->total_time = dmc->total;
 903         } else {
 904                 ret = exynos5_counters_get(dmc, &load, &total);
 905                 if (ret < 0)
 906                         return -EINVAL;
 907
 908                 /* To protect from overflow, divide by 1024 */
 909                 stat->busy_time = load >> 10;
 910                 stat->total_time = total >> 10;
 911
 912                 ret = exynos5_counters_set_event(dmc);
 913                 if (ret < 0) {
 914                         dev_err(dev, "could not set event counter\n");
 915                         return ret;
 916                 }
 917         }
 918
 919         return 0;
 920 }
 921
 922 /**
 923  * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
 924  * @dev:        device for which the framework checks operating frequency
 925  * @freq:       returned frequency value
 926  *
 927  * It returns the currently used frequency of the DMC. The real operating
 928  * frequency might be lower when the clock source value could not be divided
 929  * to the requested value.
 930  */
 931 static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
 932 {
 933         struct exynos5_dmc *dmc = dev_get_drvdata(dev);
 934
 935         mutex_lock(&dmc->lock);
 936         *freq = dmc->curr_rate;
 937         mutex_unlock(&dmc->lock);
 938
 939         return 0;
 940 }
 941
 942 /**
 943  * exynos5_dmc_df_profile - Devfreq governor's profile structure
 944  *
 945  * It provides to the devfreq framework needed functions and polling period.
 946  */
 947 static struct devfreq_dev_profile exynos5_dmc_df_profile = {
 948         .target = exynos5_dmc_target,
 949         .get_dev_status = exynos5_dmc_get_status,
 950         .get_cur_freq = exynos5_dmc_get_cur_freq,
 951 };
 952
 953 /**
 954  * exynos5_dmc_align_initial_frequency() - Align initial frequency value
 955  * @dmc:        device for which the frequency is going to be set
 956  * @bootloader_init_freq:       initial frequency set by the bootloader in KHz
 957  *
 958  * The initial bootloader frequency, which is present during boot, might be
 959  * different that supported frequency values in the driver. It is possible
 960  * due to different PLL settings or used PLL as a source.
 961  * This function provides the 'initial_freq' for the devfreq framework
 962  * statistics engine which supports only registered values. Thus, some alignment
 963  * must be made.
 964  */
 965 static unsigned long
 966 exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
 967                             unsigned long bootloader_init_freq)
 968 {
 969         unsigned long aligned_freq;
 970         int idx;
 971
 972         idx = find_target_freq_idx(dmc, bootloader_init_freq);
 973         if (idx >= 0)
 974                 aligned_freq = dmc->opp[idx].freq_hz;
 975         else
 976                 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
 977
 978         return aligned_freq;
 979 }
 980
 981 /**
 982  * create_timings_aligned() - Create register values and align with standard
 983  * @dmc:        device for which the frequency is going to be set
 984  * @idx:        speed bin in the OPP table
 985  * @clk_period_ps:      the period of the clock, known as tCK
 986  *
 987  * The function calculates timings and creates a register value ready for
 988  * a frequency transition. The register contains a few timings. They are
 989  * shifted by a known offset. The timing value is calculated based on memory
 990  * specyfication: minimal time required and minimal cycles required.
 991  */
 992 static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
 993                                   u32 *reg_timing_data, u32 *reg_timing_power,
 994                                   u32 clk_period_ps)
 995 {
 996         u32 val;
 997         const struct timing_reg *reg;
 998
 999         if (clk_period_ps == 0)
1000                 return -EINVAL;
1001
1002         *reg_timing_row = 0;
1003         *reg_timing_data = 0;
1004         *reg_timing_power = 0;
1005
1006         val = dmc->timings->tRFC / clk_period_ps;
1007         val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1008         val = max(val, dmc->min_tck->tRFC);
1009         reg = &timing_row[0];
1010         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1011
1012         val = dmc->timings->tRRD / clk_period_ps;
1013         val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1014         val = max(val, dmc->min_tck->tRRD);
1015         reg = &timing_row[1];
1016         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1017
1018         val = dmc->timings->tRPab / clk_period_ps;
1019         val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1020         val = max(val, dmc->min_tck->tRPab);
1021         reg = &timing_row[2];
1022         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1023
1024         val = dmc->timings->tRCD / clk_period_ps;
1025         val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1026         val = max(val, dmc->min_tck->tRCD);
1027         reg = &timing_row[3];
1028         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1029
1030         val = dmc->timings->tRC / clk_period_ps;
1031         val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1032         val = max(val, dmc->min_tck->tRC);
1033         reg = &timing_row[4];
1034         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1035
1036         val = dmc->timings->tRAS / clk_period_ps;
1037         val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1038         val = max(val, dmc->min_tck->tRAS);
1039         reg = &timing_row[5];
1040         *reg_timing_row |= TIMING_VAL2REG(reg, val);
1041
1042         /* data related timings */
1043         val = dmc->timings->tWTR / clk_period_ps;
1044         val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1045         val = max(val, dmc->min_tck->tWTR);
1046         reg = &timing_data[0];
1047         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1048
1049         val = dmc->timings->tWR / clk_period_ps;
1050         val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1051         val = max(val, dmc->min_tck->tWR);
1052         reg = &timing_data[1];
1053         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1054
1055         val = dmc->timings->tRTP / clk_period_ps;
1056         val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1057         val = max(val, dmc->min_tck->tRTP);
1058         reg = &timing_data[2];
1059         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1060
1061         val = dmc->timings->tW2W_C2C / clk_period_ps;
1062         val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1063         val = max(val, dmc->min_tck->tW2W_C2C);
1064         reg = &timing_data[3];
1065         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1066
1067         val = dmc->timings->tR2R_C2C / clk_period_ps;
1068         val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1069         val = max(val, dmc->min_tck->tR2R_C2C);
1070         reg = &timing_data[4];
1071         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1072
1073         val = dmc->timings->tWL / clk_period_ps;
1074         val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1075         val = max(val, dmc->min_tck->tWL);
1076         reg = &timing_data[5];
1077         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1078
1079         val = dmc->timings->tDQSCK / clk_period_ps;
1080         val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1081         val = max(val, dmc->min_tck->tDQSCK);
1082         reg = &timing_data[6];
1083         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1084
1085         val = dmc->timings->tRL / clk_period_ps;
1086         val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1087         val = max(val, dmc->min_tck->tRL);
1088         reg = &timing_data[7];
1089         *reg_timing_data |= TIMING_VAL2REG(reg, val);
1090
1091         /* power related timings */
1092         val = dmc->timings->tFAW / clk_period_ps;
1093         val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1094         val = max(val, dmc->min_tck->tFAW);
1095         reg = &timing_power[0];
1096         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1097
1098         val = dmc->timings->tXSR / clk_period_ps;
1099         val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1100         val = max(val, dmc->min_tck->tXSR);
1101         reg = &timing_power[1];
1102         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1103
1104         val = dmc->timings->tXP / clk_period_ps;
1105         val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1106         val = max(val, dmc->min_tck->tXP);
1107         reg = &timing_power[2];
1108         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1109
1110         val = dmc->timings->tCKE / clk_period_ps;
1111         val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1112         val = max(val, dmc->min_tck->tCKE);
1113         reg = &timing_power[3];
1114         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1115
1116         val = dmc->timings->tMRD / clk_period_ps;
1117         val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1118         val = max(val, dmc->min_tck->tMRD);
1119         reg = &timing_power[4];
1120         *reg_timing_power |= TIMING_VAL2REG(reg, val);
1121
1122         return 0;
1123 }
1124
1125 /**
1126  * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1127  * @dmc:        device for which the frequency is going to be set
1128  *
1129  * The function parses DT entries with DRAM information.
1130  */
1131 static int of_get_dram_timings(struct exynos5_dmc *dmc)
1132 {
1133         int ret = 0;
1134         int idx;
1135         struct device_node *np_ddr;
1136         u32 freq_mhz, clk_period_ps;
1137
1138         np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1139         if (!np_ddr) {
1140                 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1141                 return -EINVAL;
1142         }
1143
1144         dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1145                                              sizeof(u32), GFP_KERNEL);
1146         if (!dmc->timing_row)
1147                 return -ENOMEM;
1148
1149         dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1150                                               sizeof(u32), GFP_KERNEL);
1151         if (!dmc->timing_data)
1152                 return -ENOMEM;
1153
1154         dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1155                                                sizeof(u32), GFP_KERNEL);
1156         if (!dmc->timing_power)
1157                 return -ENOMEM;
1158
1159         dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1160                                                  DDR_TYPE_LPDDR3,
1161                                                  &dmc->timings_arr_size);
1162         if (!dmc->timings) {
1163                 of_node_put(np_ddr);
1164                 dev_warn(dmc->dev, "could not get timings from DT\n");
1165                 return -EINVAL;
1166         }
1167
1168         dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1169         if (!dmc->min_tck) {
1170                 of_node_put(np_ddr);
1171                 dev_warn(dmc->dev, "could not get tck from DT\n");
1172                 return -EINVAL;
1173         }
1174
1175         /* Sorted array of OPPs with frequency ascending */
1176         for (idx = 0; idx < dmc->opp_count; idx++) {
1177                 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1178                 clk_period_ps = 1000000 / freq_mhz;
1179
1180                 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1181                                              &dmc->timing_data[idx],
1182                                              &dmc->timing_power[idx],
1183                                              clk_period_ps);
1184         }
1185
1186         of_node_put(np_ddr);
1187
1188         /* Take the highest frequency's timings as 'bypass' */
1189         dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1190         dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1191         dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1192
1193         return ret;
1194 }
1195
1196 /**
1197  * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1198  * @dmc:        DMC structure containing needed fields
1199  *
1200  * Get the needed clocks defined in DT device, enable and set the right parents.
1201  * Read current frequency and initialize the initial rate for governor.
1202  */
1203 static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1204 {
1205         int ret;
1206         unsigned long target_volt = 0;
1207         unsigned long target_rate = 0;
1208         unsigned int tmp;
1209
1210         dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1211         if (IS_ERR(dmc->fout_spll))
1212                 return PTR_ERR(dmc->fout_spll);
1213
1214         dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1215         if (IS_ERR(dmc->fout_bpll))
1216                 return PTR_ERR(dmc->fout_bpll);
1217
1218         dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1219         if (IS_ERR(dmc->mout_mclk_cdrex))
1220                 return PTR_ERR(dmc->mout_mclk_cdrex);
1221
1222         dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1223         if (IS_ERR(dmc->mout_bpll))
1224                 return PTR_ERR(dmc->mout_bpll);
1225
1226         dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1227                                                 "mout_mx_mspll_ccore");
1228         if (IS_ERR(dmc->mout_mx_mspll_ccore))
1229                 return PTR_ERR(dmc->mout_mx_mspll_ccore);
1230
1231         dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1232         if (IS_ERR(dmc->mout_spll)) {
1233                 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1234                 if (IS_ERR(dmc->mout_spll))
1235                         return PTR_ERR(dmc->mout_spll);
1236         }
1237
1238         /*
1239          * Convert frequency to KHz values and set it for the governor.
1240          */
1241         dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1242         dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1243         exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1244
1245         ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1246                                         &target_volt, 0);
1247         if (ret)
1248                 return ret;
1249
1250         dmc->curr_volt = target_volt;
1251
1252         clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1253
1254         dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore);
1255
1256         clk_prepare_enable(dmc->fout_bpll);
1257         clk_prepare_enable(dmc->mout_bpll);
1258
1259         /*
1260          * Some bootloaders do not set clock routes correctly.
1261          * Stop one path in clocks to PHY.
1262          */
1263         regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1264         tmp &= ~(BIT(1) | BIT(0));
1265         regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1266
1267         return 0;
1268 }
1269
1270 /**
1271  * exynos5_performance_counters_init() - Initializes performance DMC's counters
1272  * @dmc:        DMC for which it does the setup
1273  *
1274  * Initialization of performance counters in DMC for estimating usage.
1275  * The counter's values are used for calculation of a memory bandwidth and based
1276  * on that the governor changes the frequency.
1277  * The counters are not used when the governor is GOVERNOR_USERSPACE.
1278  */
1279 static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1280 {
1281         int counters_size;
1282         int ret, i;
1283
1284         dmc->num_counters = devfreq_event_get_edev_count(dmc->dev);
1285         if (dmc->num_counters < 0) {
1286                 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1287                 return dmc->num_counters;
1288         }
1289
1290         counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters;
1291         dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL);
1292         if (!dmc->counter)
1293                 return -ENOMEM;
1294
1295         for (i = 0; i < dmc->num_counters; i++) {
1296                 dmc->counter[i] =
1297                         devfreq_event_get_edev_by_phandle(dmc->dev, i);
1298                 if (IS_ERR_OR_NULL(dmc->counter[i]))
1299                         return -EPROBE_DEFER;
1300         }
1301
1302         ret = exynos5_counters_enable_edev(dmc);
1303         if (ret < 0) {
1304                 dev_err(dmc->dev, "could not enable event counter\n");
1305                 return ret;
1306         }
1307
1308         ret = exynos5_counters_set_event(dmc);
1309         if (ret < 0) {
1310                 exynos5_counters_disable_edev(dmc);
1311                 dev_err(dmc->dev, "could not set event counter\n");
1312                 return ret;
1313         }
1314
1315         return 0;
1316 }
1317
1318 /**
1319  * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1320  * @dmc:        device which is used for changing this feature
1321  * @set:        a boolean state passing enable/disable request
1322  *
1323  * There is a need of pausing DREX DMC when divider or MUX in clock tree
1324  * changes its configuration. In such situation access to the memory is blocked
1325  * in DMC automatically. This feature is used when clock frequency change
1326  * request appears and touches clock tree.
1327  */
1328 static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1329 {
1330         unsigned int val;
1331         int ret;
1332
1333         ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1334         if (ret)
1335                 return ret;
1336
1337         val |= 1UL;
1338         regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1339
1340         return 0;
1341 }
1342
1343 static irqreturn_t dmc_irq_thread(int irq, void *priv)
1344 {
1345         int res;
1346         struct exynos5_dmc *dmc = priv;
1347
1348         mutex_lock(&dmc->df->lock);
1349         exynos5_dmc_perf_events_check(dmc);
1350         res = update_devfreq(dmc->df);
1351         mutex_unlock(&dmc->df->lock);
1352
1353         if (res)
1354                 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1355
1356         return IRQ_HANDLED;
1357 }
1358
1359 /**
1360  * exynos5_dmc_probe() - Probe function for the DMC driver
1361  * @pdev:       platform device for which the driver is going to be initialized
1362  *
1363  * Initialize basic components: clocks, regulators, performance counters, etc.
1364  * Read out product version and based on the information setup
1365  * internal structures for the controller (frequency and voltage) and for DRAM
1366  * memory parameters: timings for each operating frequency.
1367  * Register new devfreq device for controlling DVFS of the DMC.
1368  */
1369 static int exynos5_dmc_probe(struct platform_device *pdev)
1370 {
1371         int ret = 0;
1372         struct device *dev = &pdev->dev;
1373         struct device_node *np = dev->of_node;
1374         struct exynos5_dmc *dmc;
1375         int irq[2];
1376
1377         dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1378         if (!dmc)
1379                 return -ENOMEM;
1380
1381         mutex_init(&dmc->lock);
1382
1383         dmc->dev = dev;
1384         platform_set_drvdata(pdev, dmc);
1385
1386         dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1387         if (IS_ERR(dmc->base_drexi0))
1388                 return PTR_ERR(dmc->base_drexi0);
1389
1390         dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1391         if (IS_ERR(dmc->base_drexi1))
1392                 return PTR_ERR(dmc->base_drexi1);
1393
1394         dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1395                                 "samsung,syscon-clk");
1396         if (IS_ERR(dmc->clk_regmap))
1397                 return PTR_ERR(dmc->clk_regmap);
1398
1399         ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1400         if (ret) {
1401                 dev_warn(dev, "couldn't initialize frequency settings\n");
1402                 return ret;
1403         }
1404
1405         dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1406         if (IS_ERR(dmc->vdd_mif)) {
1407                 ret = PTR_ERR(dmc->vdd_mif);
1408                 return ret;
1409         }
1410
1411         ret = exynos5_dmc_init_clks(dmc);
1412         if (ret)
1413                 return ret;
1414
1415         ret = of_get_dram_timings(dmc);
1416         if (ret) {
1417                 dev_warn(dev, "couldn't initialize timings settings\n");
1418                 goto remove_clocks;
1419         }
1420
1421         ret = exynos5_dmc_set_pause_on_switching(dmc);
1422         if (ret) {
1423                 dev_warn(dev, "couldn't get access to PAUSE register\n");
1424                 goto remove_clocks;
1425         }
1426
1427         /* There is two modes in which the driver works: polling or IRQ */
1428         irq[0] = platform_get_irq_byname(pdev, "drex_0");
1429         irq[1] = platform_get_irq_byname(pdev, "drex_1");
1430         if (irq[0] > 0 && irq[1] > 0) {
1431                 ret = devm_request_threaded_irq(dev, irq[0], NULL,
1432                                                 dmc_irq_thread, IRQF_ONESHOT,
1433                                                 dev_name(dev), dmc);
1434                 if (ret) {
1435                         dev_err(dev, "couldn't grab IRQ\n");
1436                         goto remove_clocks;
1437                 }
1438
1439                 ret = devm_request_threaded_irq(dev, irq[1], NULL,
1440                                                 dmc_irq_thread, IRQF_ONESHOT,
1441                                                 dev_name(dev), dmc);
1442                 if (ret) {
1443                         dev_err(dev, "couldn't grab IRQ\n");
1444                         goto remove_clocks;
1445                 }
1446
1447                 /*
1448                  * Setup default thresholds for the devfreq governor.
1449                  * The values are chosen based on experiments.
1450                  */
1451                 dmc->gov_data.upthreshold = 55;
1452                 dmc->gov_data.downdifferential = 5;
1453
1454                 exynos5_dmc_enable_perf_events(dmc);
1455
1456                 dmc->in_irq_mode = 1;
1457         } else {
1458                 ret = exynos5_performance_counters_init(dmc);
1459                 if (ret) {
1460                         dev_warn(dev, "couldn't probe performance counters\n");
1461                         goto remove_clocks;
1462                 }
1463
1464                 /*
1465                  * Setup default thresholds for the devfreq governor.
1466                  * The values are chosen based on experiments.
1467                  */
1468                 dmc->gov_data.upthreshold = 30;
1469                 dmc->gov_data.downdifferential = 5;
1470
1471                 exynos5_dmc_df_profile.polling_ms = 500;
1472         }
1473
1474
1475         dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1476                                           DEVFREQ_GOV_SIMPLE_ONDEMAND,
1477                                           &dmc->gov_data);
1478
1479         if (IS_ERR(dmc->df)) {
1480                 ret = PTR_ERR(dmc->df);
1481                 goto err_devfreq_add;
1482         }
1483
1484         if (dmc->in_irq_mode)
1485                 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1486
1487         dev_info(dev, "DMC initialized\n");
1488
1489         return 0;
1490
1491 err_devfreq_add:
1492         if (dmc->in_irq_mode)
1493                 exynos5_dmc_disable_perf_events(dmc);
1494         else
1495                 exynos5_counters_disable_edev(dmc);
1496 remove_clocks:
1497         clk_disable_unprepare(dmc->mout_bpll);
1498         clk_disable_unprepare(dmc->fout_bpll);
1499
1500         return ret;
1501 }
1502
1503 /**
1504  * exynos5_dmc_remove() - Remove function for the platform device
1505  * @pdev:       platform device which is going to be removed
1506  *
1507  * The function relies on 'devm' framework function which automatically
1508  * clean the device's resources. It just calls explicitly disable function for
1509  * the performance counters.
1510  */
1511 static int exynos5_dmc_remove(struct platform_device *pdev)
1512 {
1513         struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1514
1515         if (dmc->in_irq_mode)
1516                 exynos5_dmc_disable_perf_events(dmc);
1517         else
1518                 exynos5_counters_disable_edev(dmc);
1519
1520         clk_disable_unprepare(dmc->mout_bpll);
1521         clk_disable_unprepare(dmc->fout_bpll);
1522
1523         dev_pm_opp_remove_table(dmc->dev);
1524
1525         return 0;
1526 }
1527
1528 static const struct of_device_id exynos5_dmc_of_match[] = {
1529         { .compatible = "samsung,exynos5422-dmc", },
1530         { },
1531 };
1532 MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1533
1534 static struct platform_driver exynos5_dmc_platdrv = {
1535         .probe  = exynos5_dmc_probe,
1536         .remove = exynos5_dmc_remove,
1537         .driver = {
1538                 .name   = "exynos5-dmc",
1539                 .of_match_table = exynos5_dmc_of_match,
1540         },
1541 };
1542 module_platform_driver(exynos5_dmc_platdrv);
1543 MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1544 MODULE_LICENSE("GPL v2");
1545 MODULE_AUTHOR("Lukasz Luba");