drivers/power/avs/qcom-cpr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
   4  * Copyright (c) 2019, Linaro Limited
   5  */
   6
   7 #include <linux/module.h>
   8 #include <linux/err.h>
   9 #include <linux/debugfs.h>
  10 #include <linux/string.h>
  11 #include <linux/kernel.h>
  12 #include <linux/list.h>
  13 #include <linux/init.h>
  14 #include <linux/io.h>
  15 #include <linux/bitops.h>
  16 #include <linux/slab.h>
  17 #include <linux/of.h>
  18 #include <linux/of_device.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/pm_domain.h>
  21 #include <linux/pm_opp.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/regmap.h>
  24 #include <linux/mfd/syscon.h>
  25 #include <linux/regulator/consumer.h>
  26 #include <linux/clk.h>
  27 #include <linux/nvmem-consumer.h>
  28
  29 /* Register Offsets for RB-CPR and Bit Definitions */
  30
  31 /* RBCPR Version Register */
  32 #define REG_RBCPR_VERSION               0
  33 #define RBCPR_VER_2                     0x02
  34 #define FLAGS_IGNORE_1ST_IRQ_STATUS     BIT(0)
  35
  36 /* RBCPR Gate Count and Target Registers */
  37 #define REG_RBCPR_GCNT_TARGET(n)        (0x60 + 4 * (n))
  38
  39 #define RBCPR_GCNT_TARGET_TARGET_SHIFT  0
  40 #define RBCPR_GCNT_TARGET_TARGET_MASK   GENMASK(11, 0)
  41 #define RBCPR_GCNT_TARGET_GCNT_SHIFT    12
  42 #define RBCPR_GCNT_TARGET_GCNT_MASK     GENMASK(9, 0)
  43
  44 /* RBCPR Timer Control */
  45 #define REG_RBCPR_TIMER_INTERVAL        0x44
  46 #define REG_RBIF_TIMER_ADJUST           0x4c
  47
  48 #define RBIF_TIMER_ADJ_CONS_UP_MASK     GENMASK(3, 0)
  49 #define RBIF_TIMER_ADJ_CONS_UP_SHIFT    0
  50 #define RBIF_TIMER_ADJ_CONS_DOWN_MASK   GENMASK(3, 0)
  51 #define RBIF_TIMER_ADJ_CONS_DOWN_SHIFT  4
  52 #define RBIF_TIMER_ADJ_CLAMP_INT_MASK   GENMASK(7, 0)
  53 #define RBIF_TIMER_ADJ_CLAMP_INT_SHIFT  8
  54
  55 /* RBCPR Config Register */
  56 #define REG_RBIF_LIMIT                  0x48
  57 #define RBIF_LIMIT_CEILING_MASK         GENMASK(5, 0)
  58 #define RBIF_LIMIT_CEILING_SHIFT        6
  59 #define RBIF_LIMIT_FLOOR_BITS           6
  60 #define RBIF_LIMIT_FLOOR_MASK           GENMASK(5, 0)
  61
  62 #define RBIF_LIMIT_CEILING_DEFAULT      RBIF_LIMIT_CEILING_MASK
  63 #define RBIF_LIMIT_FLOOR_DEFAULT        0
  64
  65 #define REG_RBIF_SW_VLEVEL              0x94
  66 #define RBIF_SW_VLEVEL_DEFAULT          0x20
  67
  68 #define REG_RBCPR_STEP_QUOT             0x80
  69 #define RBCPR_STEP_QUOT_STEPQUOT_MASK   GENMASK(7, 0)
  70 #define RBCPR_STEP_QUOT_IDLE_CLK_MASK   GENMASK(3, 0)
  71 #define RBCPR_STEP_QUOT_IDLE_CLK_SHIFT  8
  72
  73 /* RBCPR Control Register */
  74 #define REG_RBCPR_CTL                   0x90
  75
  76 #define RBCPR_CTL_LOOP_EN                       BIT(0)
  77 #define RBCPR_CTL_TIMER_EN                      BIT(3)
  78 #define RBCPR_CTL_SW_AUTO_CONT_ACK_EN           BIT(5)
  79 #define RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN       BIT(6)
  80 #define RBCPR_CTL_COUNT_MODE                    BIT(10)
  81 #define RBCPR_CTL_UP_THRESHOLD_MASK     GENMASK(3, 0)
  82 #define RBCPR_CTL_UP_THRESHOLD_SHIFT    24
  83 #define RBCPR_CTL_DN_THRESHOLD_MASK     GENMASK(3, 0)
  84 #define RBCPR_CTL_DN_THRESHOLD_SHIFT    28
  85
  86 /* RBCPR Ack/Nack Response */
  87 #define REG_RBIF_CONT_ACK_CMD           0x98
  88 #define REG_RBIF_CONT_NACK_CMD          0x9c
  89
  90 /* RBCPR Result status Register */
  91 #define REG_RBCPR_RESULT_0              0xa0
  92
  93 #define RBCPR_RESULT0_BUSY_SHIFT        19
  94 #define RBCPR_RESULT0_BUSY_MASK         BIT(RBCPR_RESULT0_BUSY_SHIFT)
  95 #define RBCPR_RESULT0_ERROR_LT0_SHIFT   18
  96 #define RBCPR_RESULT0_ERROR_SHIFT       6
  97 #define RBCPR_RESULT0_ERROR_MASK        GENMASK(11, 0)
  98 #define RBCPR_RESULT0_ERROR_STEPS_SHIFT 2
  99 #define RBCPR_RESULT0_ERROR_STEPS_MASK  GENMASK(3, 0)
 100 #define RBCPR_RESULT0_STEP_UP_SHIFT     1
 101
 102 /* RBCPR Interrupt Control Register */
 103 #define REG_RBIF_IRQ_EN(n)              (0x100 + 4 * (n))
 104 #define REG_RBIF_IRQ_CLEAR              0x110
 105 #define REG_RBIF_IRQ_STATUS             0x114
 106
 107 #define CPR_INT_DONE            BIT(0)
 108 #define CPR_INT_MIN             BIT(1)
 109 #define CPR_INT_DOWN            BIT(2)
 110 #define CPR_INT_MID             BIT(3)
 111 #define CPR_INT_UP              BIT(4)
 112 #define CPR_INT_MAX             BIT(5)
 113 #define CPR_INT_CLAMP           BIT(6)
 114 #define CPR_INT_ALL     (CPR_INT_DONE | CPR_INT_MIN | CPR_INT_DOWN | \
 115                         CPR_INT_MID | CPR_INT_UP | CPR_INT_MAX | CPR_INT_CLAMP)
 116 #define CPR_INT_DEFAULT (CPR_INT_UP | CPR_INT_DOWN)
 117
 118 #define CPR_NUM_RING_OSC        8
 119
 120 /* CPR eFuse parameters */
 121 #define CPR_FUSE_TARGET_QUOT_BITS_MASK  GENMASK(11, 0)
 122
 123 #define CPR_FUSE_MIN_QUOT_DIFF          50
 124
 125 #define FUSE_REVISION_UNKNOWN           (-1)
 126
 127 enum voltage_change_dir {
 128         NO_CHANGE,
 129         DOWN,
 130         UP,
 131 };
 132
 133 struct cpr_fuse {
 134         char *ring_osc;
 135         char *init_voltage;
 136         char *quotient;
 137         char *quotient_offset;
 138 };
 139
 140 struct fuse_corner_data {
 141         int ref_uV;
 142         int max_uV;
 143         int min_uV;
 144         int max_volt_scale;
 145         int max_quot_scale;
 146         /* fuse quot */
 147         int quot_offset;
 148         int quot_scale;
 149         int quot_adjust;
 150         /* fuse quot_offset */
 151         int quot_offset_scale;
 152         int quot_offset_adjust;
 153 };
 154
 155 struct cpr_fuses {
 156         int init_voltage_step;
 157         int init_voltage_width;
 158         struct fuse_corner_data *fuse_corner_data;
 159 };
 160
 161 struct corner_data {
 162         unsigned int fuse_corner;
 163         unsigned long freq;
 164 };
 165
 166 struct cpr_desc {
 167         unsigned int num_fuse_corners;
 168         int min_diff_quot;
 169         int *step_quot;
 170
 171         unsigned int            timer_delay_us;
 172         unsigned int            timer_cons_up;
 173         unsigned int            timer_cons_down;
 174         unsigned int            up_threshold;
 175         unsigned int            down_threshold;
 176         unsigned int            idle_clocks;
 177         unsigned int            gcnt_us;
 178         unsigned int            vdd_apc_step_up_limit;
 179         unsigned int            vdd_apc_step_down_limit;
 180         unsigned int            clamp_timer_interval;
 181
 182         struct cpr_fuses cpr_fuses;
 183         bool reduce_to_fuse_uV;
 184         bool reduce_to_corner_uV;
 185 };
 186
 187 struct acc_desc {
 188         unsigned int    enable_reg;
 189         u32             enable_mask;
 190
 191         struct reg_sequence     *config;
 192         struct reg_sequence     *settings;
 193         int                     num_regs_per_fuse;
 194 };
 195
 196 struct cpr_acc_desc {
 197         const struct cpr_desc *cpr_desc;
 198         const struct acc_desc *acc_desc;
 199 };
 200
 201 struct fuse_corner {
 202         int min_uV;
 203         int max_uV;
 204         int uV;
 205         int quot;
 206         int step_quot;
 207         const struct reg_sequence *accs;
 208         int num_accs;
 209         unsigned long max_freq;
 210         u8 ring_osc_idx;
 211 };
 212
 213 struct corner {
 214         int min_uV;
 215         int max_uV;
 216         int uV;
 217         int last_uV;
 218         int quot_adjust;
 219         u32 save_ctl;
 220         u32 save_irq;
 221         unsigned long freq;
 222         struct fuse_corner *fuse_corner;
 223 };
 224
 225 struct cpr_drv {
 226         unsigned int            num_corners;
 227         unsigned int            ref_clk_khz;
 228
 229         struct generic_pm_domain pd;
 230         struct device           *dev;
 231         struct device           *attached_cpu_dev;
 232         struct mutex            lock;
 233         void __iomem            *base;
 234         struct corner           *corner;
 235         struct regulator        *vdd_apc;
 236         struct clk              *cpu_clk;
 237         struct regmap           *tcsr;
 238         bool                    loop_disabled;
 239         u32                     gcnt;
 240         unsigned long           flags;
 241
 242         struct fuse_corner      *fuse_corners;
 243         struct corner           *corners;
 244
 245         const struct cpr_desc *desc;
 246         const struct acc_desc *acc_desc;
 247         const struct cpr_fuse *cpr_fuses;
 248
 249         struct dentry *debugfs;
 250 };
 251
 252 static bool cpr_is_allowed(struct cpr_drv *drv)
 253 {
 254         return !drv->loop_disabled;
 255 }
 256
 257 static void cpr_write(struct cpr_drv *drv, u32 offset, u32 value)
 258 {
 259         writel_relaxed(value, drv->base + offset);
 260 }
 261
 262 static u32 cpr_read(struct cpr_drv *drv, u32 offset)
 263 {
 264         return readl_relaxed(drv->base + offset);
 265 }
 266
 267 static void
 268 cpr_masked_write(struct cpr_drv *drv, u32 offset, u32 mask, u32 value)
 269 {
 270         u32 val;
 271
 272         val = readl_relaxed(drv->base + offset);
 273         val &= ~mask;
 274         val |= value & mask;
 275         writel_relaxed(val, drv->base + offset);
 276 }
 277
 278 static void cpr_irq_clr(struct cpr_drv *drv)
 279 {
 280         cpr_write(drv, REG_RBIF_IRQ_CLEAR, CPR_INT_ALL);
 281 }
 282
 283 static void cpr_irq_clr_nack(struct cpr_drv *drv)
 284 {
 285         cpr_irq_clr(drv);
 286         cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 287 }
 288
 289 static void cpr_irq_clr_ack(struct cpr_drv *drv)
 290 {
 291         cpr_irq_clr(drv);
 292         cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 293 }
 294
 295 static void cpr_irq_set(struct cpr_drv *drv, u32 int_bits)
 296 {
 297         cpr_write(drv, REG_RBIF_IRQ_EN(0), int_bits);
 298 }
 299
 300 static void cpr_ctl_modify(struct cpr_drv *drv, u32 mask, u32 value)
 301 {
 302         cpr_masked_write(drv, REG_RBCPR_CTL, mask, value);
 303 }
 304
 305 static void cpr_ctl_enable(struct cpr_drv *drv, struct corner *corner)
 306 {
 307         u32 val, mask;
 308         const struct cpr_desc *desc = drv->desc;
 309
 310         /* Program Consecutive Up & Down */
 311         val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 312         val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 313         mask = RBIF_TIMER_ADJ_CONS_UP_MASK | RBIF_TIMER_ADJ_CONS_DOWN_MASK;
 314         cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST, mask, val);
 315         cpr_masked_write(drv, REG_RBCPR_CTL,
 316                          RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 317                          RBCPR_CTL_SW_AUTO_CONT_ACK_EN,
 318                          corner->save_ctl);
 319         cpr_irq_set(drv, corner->save_irq);
 320
 321         if (cpr_is_allowed(drv) && corner->max_uV > corner->min_uV)
 322                 val = RBCPR_CTL_LOOP_EN;
 323         else
 324                 val = 0;
 325         cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, val);
 326 }
 327
 328 static void cpr_ctl_disable(struct cpr_drv *drv)
 329 {
 330         cpr_irq_set(drv, 0);
 331         cpr_ctl_modify(drv, RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN |
 332                        RBCPR_CTL_SW_AUTO_CONT_ACK_EN, 0);
 333         cpr_masked_write(drv, REG_RBIF_TIMER_ADJUST,
 334                          RBIF_TIMER_ADJ_CONS_UP_MASK |
 335                          RBIF_TIMER_ADJ_CONS_DOWN_MASK, 0);
 336         cpr_irq_clr(drv);
 337         cpr_write(drv, REG_RBIF_CONT_ACK_CMD, 1);
 338         cpr_write(drv, REG_RBIF_CONT_NACK_CMD, 1);
 339         cpr_ctl_modify(drv, RBCPR_CTL_LOOP_EN, 0);
 340 }
 341
 342 static bool cpr_ctl_is_enabled(struct cpr_drv *drv)
 343 {
 344         u32 reg_val;
 345
 346         reg_val = cpr_read(drv, REG_RBCPR_CTL);
 347         return reg_val & RBCPR_CTL_LOOP_EN;
 348 }
 349
 350 static bool cpr_ctl_is_busy(struct cpr_drv *drv)
 351 {
 352         u32 reg_val;
 353
 354         reg_val = cpr_read(drv, REG_RBCPR_RESULT_0);
 355         return reg_val & RBCPR_RESULT0_BUSY_MASK;
 356 }
 357
 358 static void cpr_corner_save(struct cpr_drv *drv, struct corner *corner)
 359 {
 360         corner->save_ctl = cpr_read(drv, REG_RBCPR_CTL);
 361         corner->save_irq = cpr_read(drv, REG_RBIF_IRQ_EN(0));
 362 }
 363
 364 static void cpr_corner_restore(struct cpr_drv *drv, struct corner *corner)
 365 {
 366         u32 gcnt, ctl, irq, ro_sel, step_quot;
 367         struct fuse_corner *fuse = corner->fuse_corner;
 368         const struct cpr_desc *desc = drv->desc;
 369         int i;
 370
 371         ro_sel = fuse->ring_osc_idx;
 372         gcnt = drv->gcnt;
 373         gcnt |= fuse->quot - corner->quot_adjust;
 374
 375         /* Program the step quotient and idle clocks */
 376         step_quot = desc->idle_clocks << RBCPR_STEP_QUOT_IDLE_CLK_SHIFT;
 377         step_quot |= fuse->step_quot & RBCPR_STEP_QUOT_STEPQUOT_MASK;
 378         cpr_write(drv, REG_RBCPR_STEP_QUOT, step_quot);
 379
 380         /* Clear the target quotient value and gate count of all ROs */
 381         for (i = 0; i < CPR_NUM_RING_OSC; i++)
 382                 cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 383
 384         cpr_write(drv, REG_RBCPR_GCNT_TARGET(ro_sel), gcnt);
 385         ctl = corner->save_ctl;
 386         cpr_write(drv, REG_RBCPR_CTL, ctl);
 387         irq = corner->save_irq;
 388         cpr_irq_set(drv, irq);
 389         dev_dbg(drv->dev, "gcnt = %#08x, ctl = %#08x, irq = %#08x\n", gcnt,
 390                 ctl, irq);
 391 }
 392
 393 static void cpr_set_acc(struct regmap *tcsr, struct fuse_corner *f,
 394                         struct fuse_corner *end)
 395 {
 396         if (f == end)
 397                 return;
 398
 399         if (f < end) {
 400                 for (f += 1; f <= end; f++)
 401                         regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 402         } else {
 403                 for (f -= 1; f >= end; f--)
 404                         regmap_multi_reg_write(tcsr, f->accs, f->num_accs);
 405         }
 406 }
 407
 408 static int cpr_pre_voltage(struct cpr_drv *drv,
 409                            struct fuse_corner *fuse_corner,
 410                            enum voltage_change_dir dir)
 411 {
 412         struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 413
 414         if (drv->tcsr && dir == DOWN)
 415                 cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 416
 417         return 0;
 418 }
 419
 420 static int cpr_post_voltage(struct cpr_drv *drv,
 421                             struct fuse_corner *fuse_corner,
 422                             enum voltage_change_dir dir)
 423 {
 424         struct fuse_corner *prev_fuse_corner = drv->corner->fuse_corner;
 425
 426         if (drv->tcsr && dir == UP)
 427                 cpr_set_acc(drv->tcsr, prev_fuse_corner, fuse_corner);
 428
 429         return 0;
 430 }
 431
 432 static int cpr_scale_voltage(struct cpr_drv *drv, struct corner *corner,
 433                              int new_uV, enum voltage_change_dir dir)
 434 {
 435         int ret;
 436         struct fuse_corner *fuse_corner = corner->fuse_corner;
 437
 438         ret = cpr_pre_voltage(drv, fuse_corner, dir);
 439         if (ret)
 440                 return ret;
 441
 442         ret = regulator_set_voltage(drv->vdd_apc, new_uV, new_uV);
 443         if (ret) {
 444                 dev_err_ratelimited(drv->dev, "failed to set apc voltage %d\n",
 445                                     new_uV);
 446                 return ret;
 447         }
 448
 449         ret = cpr_post_voltage(drv, fuse_corner, dir);
 450         if (ret)
 451                 return ret;
 452
 453         return 0;
 454 }
 455
 456 static unsigned int cpr_get_cur_perf_state(struct cpr_drv *drv)
 457 {
 458         return drv->corner ? drv->corner - drv->corners + 1 : 0;
 459 }
 460
 461 static int cpr_scale(struct cpr_drv *drv, enum voltage_change_dir dir)
 462 {
 463         u32 val, error_steps, reg_mask;
 464         int last_uV, new_uV, step_uV, ret;
 465         struct corner *corner;
 466         const struct cpr_desc *desc = drv->desc;
 467
 468         if (dir != UP && dir != DOWN)
 469                 return 0;
 470
 471         step_uV = regulator_get_linear_step(drv->vdd_apc);
 472         if (!step_uV)
 473                 return -EINVAL;
 474
 475         corner = drv->corner;
 476
 477         val = cpr_read(drv, REG_RBCPR_RESULT_0);
 478
 479         error_steps = val >> RBCPR_RESULT0_ERROR_STEPS_SHIFT;
 480         error_steps &= RBCPR_RESULT0_ERROR_STEPS_MASK;
 481         last_uV = corner->last_uV;
 482
 483         if (dir == UP) {
 484                 if (desc->clamp_timer_interval &&
 485                     error_steps < desc->up_threshold) {
 486                         /*
 487                          * Handle the case where another measurement started
 488                          * after the interrupt was triggered due to a core
 489                          * exiting from power collapse.
 490                          */
 491                         error_steps = max(desc->up_threshold,
 492                                           desc->vdd_apc_step_up_limit);
 493                 }
 494
 495                 if (last_uV >= corner->max_uV) {
 496                         cpr_irq_clr_nack(drv);
 497
 498                         /* Maximize the UP threshold */
 499                         reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 500                         reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 501                         val = reg_mask;
 502                         cpr_ctl_modify(drv, reg_mask, val);
 503
 504                         /* Disable UP interrupt */
 505                         cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_UP);
 506
 507                         return 0;
 508                 }
 509
 510                 if (error_steps > desc->vdd_apc_step_up_limit)
 511                         error_steps = desc->vdd_apc_step_up_limit;
 512
 513                 /* Calculate new voltage */
 514                 new_uV = last_uV + error_steps * step_uV;
 515                 new_uV = min(new_uV, corner->max_uV);
 516
 517                 dev_dbg(drv->dev,
 518                         "UP: -> new_uV: %d last_uV: %d perf state: %u\n",
 519                         new_uV, last_uV, cpr_get_cur_perf_state(drv));
 520         } else {
 521                 if (desc->clamp_timer_interval &&
 522                     error_steps < desc->down_threshold) {
 523                         /*
 524                          * Handle the case where another measurement started
 525                          * after the interrupt was triggered due to a core
 526                          * exiting from power collapse.
 527                          */
 528                         error_steps = max(desc->down_threshold,
 529                                           desc->vdd_apc_step_down_limit);
 530                 }
 531
 532                 if (last_uV <= corner->min_uV) {
 533                         cpr_irq_clr_nack(drv);
 534
 535                         /* Enable auto nack down */
 536                         reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 537                         val = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 538
 539                         cpr_ctl_modify(drv, reg_mask, val);
 540
 541                         /* Disable DOWN interrupt */
 542                         cpr_irq_set(drv, CPR_INT_DEFAULT & ~CPR_INT_DOWN);
 543
 544                         return 0;
 545                 }
 546
 547                 if (error_steps > desc->vdd_apc_step_down_limit)
 548                         error_steps = desc->vdd_apc_step_down_limit;
 549
 550                 /* Calculate new voltage */
 551                 new_uV = last_uV - error_steps * step_uV;
 552                 new_uV = max(new_uV, corner->min_uV);
 553
 554                 dev_dbg(drv->dev,
 555                         "DOWN: -> new_uV: %d last_uV: %d perf state: %u\n",
 556                         new_uV, last_uV, cpr_get_cur_perf_state(drv));
 557         }
 558
 559         ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 560         if (ret) {
 561                 cpr_irq_clr_nack(drv);
 562                 return ret;
 563         }
 564         drv->corner->last_uV = new_uV;
 565
 566         if (dir == UP) {
 567                 /* Disable auto nack down */
 568                 reg_mask = RBCPR_CTL_SW_AUTO_CONT_NACK_DN_EN;
 569                 val = 0;
 570         } else {
 571                 /* Restore default threshold for UP */
 572                 reg_mask = RBCPR_CTL_UP_THRESHOLD_MASK;
 573                 reg_mask <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 574                 val = desc->up_threshold;
 575                 val <<= RBCPR_CTL_UP_THRESHOLD_SHIFT;
 576         }
 577
 578         cpr_ctl_modify(drv, reg_mask, val);
 579
 580         /* Re-enable default interrupts */
 581         cpr_irq_set(drv, CPR_INT_DEFAULT);
 582
 583         /* Ack */
 584         cpr_irq_clr_ack(drv);
 585
 586         return 0;
 587 }
 588
 589 static irqreturn_t cpr_irq_handler(int irq, void *dev)
 590 {
 591         struct cpr_drv *drv = dev;
 592         const struct cpr_desc *desc = drv->desc;
 593         irqreturn_t ret = IRQ_HANDLED;
 594         u32 val;
 595
 596         mutex_lock(&drv->lock);
 597
 598         val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 599         if (drv->flags & FLAGS_IGNORE_1ST_IRQ_STATUS)
 600                 val = cpr_read(drv, REG_RBIF_IRQ_STATUS);
 601
 602         dev_dbg(drv->dev, "IRQ_STATUS = %#02x\n", val);
 603
 604         if (!cpr_ctl_is_enabled(drv)) {
 605                 dev_dbg(drv->dev, "CPR is disabled\n");
 606                 ret = IRQ_NONE;
 607         } else if (cpr_ctl_is_busy(drv) && !desc->clamp_timer_interval) {
 608                 dev_dbg(drv->dev, "CPR measurement is not ready\n");
 609         } else if (!cpr_is_allowed(drv)) {
 610                 val = cpr_read(drv, REG_RBCPR_CTL);
 611                 dev_err_ratelimited(drv->dev,
 612                                     "Interrupt broken? RBCPR_CTL = %#02x\n",
 613                                     val);
 614                 ret = IRQ_NONE;
 615         } else {
 616                 /*
 617                  * Following sequence of handling is as per each IRQ's
 618                  * priority
 619                  */
 620                 if (val & CPR_INT_UP) {
 621                         cpr_scale(drv, UP);
 622                 } else if (val & CPR_INT_DOWN) {
 623                         cpr_scale(drv, DOWN);
 624                 } else if (val & CPR_INT_MIN) {
 625                         cpr_irq_clr_nack(drv);
 626                 } else if (val & CPR_INT_MAX) {
 627                         cpr_irq_clr_nack(drv);
 628                 } else if (val & CPR_INT_MID) {
 629                         /* RBCPR_CTL_SW_AUTO_CONT_ACK_EN is enabled */
 630                         dev_dbg(drv->dev, "IRQ occurred for Mid Flag\n");
 631                 } else {
 632                         dev_dbg(drv->dev,
 633                                 "IRQ occurred for unknown flag (%#08x)\n", val);
 634                 }
 635
 636                 /* Save register values for the corner */
 637                 cpr_corner_save(drv, drv->corner);
 638         }
 639
 640         mutex_unlock(&drv->lock);
 641
 642         return ret;
 643 }
 644
 645 static int cpr_enable(struct cpr_drv *drv)
 646 {
 647         int ret;
 648
 649         ret = regulator_enable(drv->vdd_apc);
 650         if (ret)
 651                 return ret;
 652
 653         mutex_lock(&drv->lock);
 654
 655         if (cpr_is_allowed(drv) && drv->corner) {
 656                 cpr_irq_clr(drv);
 657                 cpr_corner_restore(drv, drv->corner);
 658                 cpr_ctl_enable(drv, drv->corner);
 659         }
 660
 661         mutex_unlock(&drv->lock);
 662
 663         return 0;
 664 }
 665
 666 static int cpr_disable(struct cpr_drv *drv)
 667 {
 668         int ret;
 669
 670         mutex_lock(&drv->lock);
 671
 672         if (cpr_is_allowed(drv)) {
 673                 cpr_ctl_disable(drv);
 674                 cpr_irq_clr(drv);
 675         }
 676
 677         mutex_unlock(&drv->lock);
 678
 679         ret = regulator_disable(drv->vdd_apc);
 680         if (ret)
 681                 return ret;
 682
 683         return 0;
 684 }
 685
 686 static int cpr_config(struct cpr_drv *drv)
 687 {
 688         int i;
 689         u32 val, gcnt;
 690         struct corner *corner;
 691         const struct cpr_desc *desc = drv->desc;
 692
 693         /* Disable interrupt and CPR */
 694         cpr_write(drv, REG_RBIF_IRQ_EN(0), 0);
 695         cpr_write(drv, REG_RBCPR_CTL, 0);
 696
 697         /* Program the default HW ceiling, floor and vlevel */
 698         val = (RBIF_LIMIT_CEILING_DEFAULT & RBIF_LIMIT_CEILING_MASK)
 699                 << RBIF_LIMIT_CEILING_SHIFT;
 700         val |= RBIF_LIMIT_FLOOR_DEFAULT & RBIF_LIMIT_FLOOR_MASK;
 701         cpr_write(drv, REG_RBIF_LIMIT, val);
 702         cpr_write(drv, REG_RBIF_SW_VLEVEL, RBIF_SW_VLEVEL_DEFAULT);
 703
 704         /*
 705          * Clear the target quotient value and gate count of all
 706          * ring oscillators
 707          */
 708         for (i = 0; i < CPR_NUM_RING_OSC; i++)
 709                 cpr_write(drv, REG_RBCPR_GCNT_TARGET(i), 0);
 710
 711         /* Init and save gcnt */
 712         gcnt = (drv->ref_clk_khz * desc->gcnt_us) / 1000;
 713         gcnt = gcnt & RBCPR_GCNT_TARGET_GCNT_MASK;
 714         gcnt <<= RBCPR_GCNT_TARGET_GCNT_SHIFT;
 715         drv->gcnt = gcnt;
 716
 717         /* Program the delay count for the timer */
 718         val = (drv->ref_clk_khz * desc->timer_delay_us) / 1000;
 719         cpr_write(drv, REG_RBCPR_TIMER_INTERVAL, val);
 720         dev_dbg(drv->dev, "Timer count: %#0x (for %d us)\n", val,
 721                 desc->timer_delay_us);
 722
 723         /* Program Consecutive Up & Down */
 724         val = desc->timer_cons_down << RBIF_TIMER_ADJ_CONS_DOWN_SHIFT;
 725         val |= desc->timer_cons_up << RBIF_TIMER_ADJ_CONS_UP_SHIFT;
 726         val |= desc->clamp_timer_interval << RBIF_TIMER_ADJ_CLAMP_INT_SHIFT;
 727         cpr_write(drv, REG_RBIF_TIMER_ADJUST, val);
 728
 729         /* Program the control register */
 730         val = desc->up_threshold << RBCPR_CTL_UP_THRESHOLD_SHIFT;
 731         val |= desc->down_threshold << RBCPR_CTL_DN_THRESHOLD_SHIFT;
 732         val |= RBCPR_CTL_TIMER_EN | RBCPR_CTL_COUNT_MODE;
 733         val |= RBCPR_CTL_SW_AUTO_CONT_ACK_EN;
 734         cpr_write(drv, REG_RBCPR_CTL, val);
 735
 736         for (i = 0; i < drv->num_corners; i++) {
 737                 corner = &drv->corners[i];
 738                 corner->save_ctl = val;
 739                 corner->save_irq = CPR_INT_DEFAULT;
 740         }
 741
 742         cpr_irq_set(drv, CPR_INT_DEFAULT);
 743
 744         val = cpr_read(drv, REG_RBCPR_VERSION);
 745         if (val <= RBCPR_VER_2)
 746                 drv->flags |= FLAGS_IGNORE_1ST_IRQ_STATUS;
 747
 748         return 0;
 749 }
 750
 751 static int cpr_set_performance_state(struct generic_pm_domain *domain,
 752                                      unsigned int state)
 753 {
 754         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
 755         struct corner *corner, *end;
 756         enum voltage_change_dir dir;
 757         int ret = 0, new_uV;
 758
 759         mutex_lock(&drv->lock);
 760
 761         dev_dbg(drv->dev, "%s: setting perf state: %u (prev state: %u)\n",
 762                 __func__, state, cpr_get_cur_perf_state(drv));
 763
 764         /*
 765          * Determine new corner we're going to.
 766          * Remove one since lowest performance state is 1.
 767          */
 768         corner = drv->corners + state - 1;
 769         end = &drv->corners[drv->num_corners - 1];
 770         if (corner > end || corner < drv->corners) {
 771                 ret = -EINVAL;
 772                 goto unlock;
 773         }
 774
 775         /* Determine direction */
 776         if (drv->corner > corner)
 777                 dir = DOWN;
 778         else if (drv->corner < corner)
 779                 dir = UP;
 780         else
 781                 dir = NO_CHANGE;
 782
 783         if (cpr_is_allowed(drv))
 784                 new_uV = corner->last_uV;
 785         else
 786                 new_uV = corner->uV;
 787
 788         if (cpr_is_allowed(drv))
 789                 cpr_ctl_disable(drv);
 790
 791         ret = cpr_scale_voltage(drv, corner, new_uV, dir);
 792         if (ret)
 793                 goto unlock;
 794
 795         if (cpr_is_allowed(drv)) {
 796                 cpr_irq_clr(drv);
 797                 if (drv->corner != corner)
 798                         cpr_corner_restore(drv, corner);
 799                 cpr_ctl_enable(drv, corner);
 800         }
 801
 802         drv->corner = corner;
 803
 804 unlock:
 805         mutex_unlock(&drv->lock);
 806
 807         return ret;
 808 }
 809
 810 static int cpr_read_efuse(struct device *dev, const char *cname, u32 *data)
 811 {
 812         struct nvmem_cell *cell;
 813         ssize_t len;
 814         char *ret;
 815         int i;
 816
 817         *data = 0;
 818
 819         cell = nvmem_cell_get(dev, cname);
 820         if (IS_ERR(cell)) {
 821                 if (PTR_ERR(cell) != -EPROBE_DEFER)
 822                         dev_err(dev, "undefined cell %s\n", cname);
 823                 return PTR_ERR(cell);
 824         }
 825
 826         ret = nvmem_cell_read(cell, &len);
 827         nvmem_cell_put(cell);
 828         if (IS_ERR(ret)) {
 829                 dev_err(dev, "can't read cell %s\n", cname);
 830                 return PTR_ERR(ret);
 831         }
 832
 833         for (i = 0; i < len; i++)
 834                 *data |= ret[i] << (8 * i);
 835
 836         kfree(ret);
 837         dev_dbg(dev, "efuse read(%s) = %x, bytes %zd\n", cname, *data, len);
 838
 839         return 0;
 840 }
 841
 842 static int
 843 cpr_populate_ring_osc_idx(struct cpr_drv *drv)
 844 {
 845         struct fuse_corner *fuse = drv->fuse_corners;
 846         struct fuse_corner *end = fuse + drv->desc->num_fuse_corners;
 847         const struct cpr_fuse *fuses = drv->cpr_fuses;
 848         u32 data;
 849         int ret;
 850
 851         for (; fuse < end; fuse++, fuses++) {
 852                 ret = cpr_read_efuse(drv->dev, fuses->ring_osc,
 853                                      &data);
 854                 if (ret)
 855                         return ret;
 856                 fuse->ring_osc_idx = data;
 857         }
 858
 859         return 0;
 860 }
 861
 862 static int cpr_read_fuse_uV(const struct cpr_desc *desc,
 863                             const struct fuse_corner_data *fdata,
 864                             const char *init_v_efuse,
 865                             int step_volt,
 866                             struct cpr_drv *drv)
 867 {
 868         int step_size_uV, steps, uV;
 869         u32 bits = 0;
 870         int ret;
 871
 872         ret = cpr_read_efuse(drv->dev, init_v_efuse, &bits);
 873         if (ret)
 874                 return ret;
 875
 876         steps = bits & ~BIT(desc->cpr_fuses.init_voltage_width - 1);
 877         /* Not two's complement.. instead highest bit is sign bit */
 878         if (bits & BIT(desc->cpr_fuses.init_voltage_width - 1))
 879                 steps = -steps;
 880
 881         step_size_uV = desc->cpr_fuses.init_voltage_step;
 882
 883         uV = fdata->ref_uV + steps * step_size_uV;
 884         return DIV_ROUND_UP(uV, step_volt) * step_volt;
 885 }
 886
 887 static int cpr_fuse_corner_init(struct cpr_drv *drv)
 888 {
 889         const struct cpr_desc *desc = drv->desc;
 890         const struct cpr_fuse *fuses = drv->cpr_fuses;
 891         const struct acc_desc *acc_desc = drv->acc_desc;
 892         int i;
 893         unsigned int step_volt;
 894         struct fuse_corner_data *fdata;
 895         struct fuse_corner *fuse, *end;
 896         int uV;
 897         const struct reg_sequence *accs;
 898         int ret;
 899
 900         accs = acc_desc->settings;
 901
 902         step_volt = regulator_get_linear_step(drv->vdd_apc);
 903         if (!step_volt)
 904                 return -EINVAL;
 905
 906         /* Populate fuse_corner members */
 907         fuse = drv->fuse_corners;
 908         end = &fuse[desc->num_fuse_corners - 1];
 909         fdata = desc->cpr_fuses.fuse_corner_data;
 910
 911         for (i = 0; fuse <= end; fuse++, fuses++, i++, fdata++) {
 912                 /*
 913                  * Update SoC voltages: platforms might choose a different
 914                  * regulators than the one used to characterize the algorithms
 915                  * (ie, init_voltage_step).
 916                  */
 917                 fdata->min_uV = roundup(fdata->min_uV, step_volt);
 918                 fdata->max_uV = roundup(fdata->max_uV, step_volt);
 919
 920                 /* Populate uV */
 921                 uV = cpr_read_fuse_uV(desc, fdata, fuses->init_voltage,
 922                                       step_volt, drv);
 923                 if (uV < 0)
 924                         return uV;
 925
 926                 fuse->min_uV = fdata->min_uV;
 927                 fuse->max_uV = fdata->max_uV;
 928                 fuse->uV = clamp(uV, fuse->min_uV, fuse->max_uV);
 929
 930                 if (fuse == end) {
 931                         /*
 932                          * Allow the highest fuse corner's PVS voltage to
 933                          * define the ceiling voltage for that corner in order
 934                          * to support SoC's in which variable ceiling values
 935                          * are required.
 936                          */
 937                         end->max_uV = max(end->max_uV, end->uV);
 938                 }
 939
 940                 /* Populate target quotient by scaling */
 941                 ret = cpr_read_efuse(drv->dev, fuses->quotient, &fuse->quot);
 942                 if (ret)
 943                         return ret;
 944
 945                 fuse->quot *= fdata->quot_scale;
 946                 fuse->quot += fdata->quot_offset;
 947                 fuse->quot += fdata->quot_adjust;
 948                 fuse->step_quot = desc->step_quot[fuse->ring_osc_idx];
 949
 950                 /* Populate acc settings */
 951                 fuse->accs = accs;
 952                 fuse->num_accs = acc_desc->num_regs_per_fuse;
 953                 accs += acc_desc->num_regs_per_fuse;
 954         }
 955
 956         /*
 957          * Restrict all fuse corner PVS voltages based upon per corner
 958          * ceiling and floor voltages.
 959          */
 960         for (fuse = drv->fuse_corners, i = 0; fuse <= end; fuse++, i++) {
 961                 if (fuse->uV > fuse->max_uV)
 962                         fuse->uV = fuse->max_uV;
 963                 else if (fuse->uV < fuse->min_uV)
 964                         fuse->uV = fuse->min_uV;
 965
 966                 ret = regulator_is_supported_voltage(drv->vdd_apc,
 967                                                      fuse->min_uV,
 968                                                      fuse->min_uV);
 969                 if (!ret) {
 970                         dev_err(drv->dev,
 971                                 "min uV: %d (fuse corner: %d) not supported by regulator\n",
 972                                 fuse->min_uV, i);
 973                         return -EINVAL;
 974                 }
 975
 976                 ret = regulator_is_supported_voltage(drv->vdd_apc,
 977                                                      fuse->max_uV,
 978                                                      fuse->max_uV);
 979                 if (!ret) {
 980                         dev_err(drv->dev,
 981                                 "max uV: %d (fuse corner: %d) not supported by regulator\n",
 982                                 fuse->max_uV, i);
 983                         return -EINVAL;
 984                 }
 985
 986                 dev_dbg(drv->dev,
 987                         "fuse corner %d: [%d %d %d] RO%hhu quot %d squot %d\n",
 988                         i, fuse->min_uV, fuse->uV, fuse->max_uV,
 989                         fuse->ring_osc_idx, fuse->quot, fuse->step_quot);
 990         }
 991
 992         return 0;
 993 }
 994
 995 static int cpr_calculate_scaling(const char *quot_offset,
 996                                  struct cpr_drv *drv,
 997                                  const struct fuse_corner_data *fdata,
 998                                  const struct corner *corner)
 999 {
1000         u32 quot_diff = 0;
1001         unsigned long freq_diff;
1002         int scaling;
1003         const struct fuse_corner *fuse, *prev_fuse;
1004         int ret;
1005
1006         fuse = corner->fuse_corner;
1007         prev_fuse = fuse - 1;
1008
1009         if (quot_offset) {
1010                 ret = cpr_read_efuse(drv->dev, quot_offset, &quot_diff);
1011                 if (ret)
1012                         return ret;
1013
1014                 quot_diff *= fdata->quot_offset_scale;
1015                 quot_diff += fdata->quot_offset_adjust;
1016         } else {
1017                 quot_diff = fuse->quot - prev_fuse->quot;
1018         }
1019
1020         freq_diff = fuse->max_freq - prev_fuse->max_freq;
1021         freq_diff /= 1000000; /* Convert to MHz */
1022         scaling = 1000 * quot_diff / freq_diff;
1023         return min(scaling, fdata->max_quot_scale);
1024 }
1025
1026 static int cpr_interpolate(const struct corner *corner, int step_volt,
1027                            const struct fuse_corner_data *fdata)
1028 {
1029         unsigned long f_high, f_low, f_diff;
1030         int uV_high, uV_low, uV;
1031         u64 temp, temp_limit;
1032         const struct fuse_corner *fuse, *prev_fuse;
1033
1034         fuse = corner->fuse_corner;
1035         prev_fuse = fuse - 1;
1036
1037         f_high = fuse->max_freq;
1038         f_low = prev_fuse->max_freq;
1039         uV_high = fuse->uV;
1040         uV_low = prev_fuse->uV;
1041         f_diff = fuse->max_freq - corner->freq;
1042
1043         /*
1044          * Don't interpolate in the wrong direction. This could happen
1045          * if the adjusted fuse voltage overlaps with the previous fuse's
1046          * adjusted voltage.
1047          */
1048         if (f_high <= f_low || uV_high <= uV_low || f_high <= corner->freq)
1049                 return corner->uV;
1050
1051         temp = f_diff * (uV_high - uV_low);
1052         do_div(temp, f_high - f_low);
1053
1054         /*
1055          * max_volt_scale has units of uV/MHz while freq values
1056          * have units of Hz.  Divide by 1000000 to convert to.
1057          */
1058         temp_limit = f_diff * fdata->max_volt_scale;
1059         do_div(temp_limit, 1000000);
1060
1061         uV = uV_high - min(temp, temp_limit);
1062         return roundup(uV, step_volt);
1063 }
1064
1065 static unsigned int cpr_get_fuse_corner(struct dev_pm_opp *opp)
1066 {
1067         struct device_node *np;
1068         unsigned int fuse_corner = 0;
1069
1070         np = dev_pm_opp_get_of_node(opp);
1071         if (of_property_read_u32(np, "qcom,opp-fuse-level", &fuse_corner))
1072                 pr_err("%s: missing 'qcom,opp-fuse-level' property\n",
1073                        __func__);
1074
1075         of_node_put(np);
1076
1077         return fuse_corner;
1078 }
1079
1080 static unsigned long cpr_get_opp_hz_for_req(struct dev_pm_opp *ref,
1081                                             struct device *cpu_dev)
1082 {
1083         u64 rate = 0;
1084         struct device_node *ref_np;
1085         struct device_node *desc_np;
1086         struct device_node *child_np = NULL;
1087         struct device_node *child_req_np = NULL;
1088
1089         desc_np = dev_pm_opp_of_get_opp_desc_node(cpu_dev);
1090         if (!desc_np)
1091                 return 0;
1092
1093         ref_np = dev_pm_opp_get_of_node(ref);
1094         if (!ref_np)
1095                 goto out_ref;
1096
1097         do {
1098                 of_node_put(child_req_np);
1099                 child_np = of_get_next_available_child(desc_np, child_np);
1100                 child_req_np = of_parse_phandle(child_np, "required-opps", 0);
1101         } while (child_np && child_req_np != ref_np);
1102
1103         if (child_np && child_req_np == ref_np)
1104                 of_property_read_u64(child_np, "opp-hz", &rate);
1105
1106         of_node_put(child_req_np);
1107         of_node_put(child_np);
1108         of_node_put(ref_np);
1109 out_ref:
1110         of_node_put(desc_np);
1111
1112         return (unsigned long) rate;
1113 }
1114
1115 static int cpr_corner_init(struct cpr_drv *drv)
1116 {
1117         const struct cpr_desc *desc = drv->desc;
1118         const struct cpr_fuse *fuses = drv->cpr_fuses;
1119         int i, level, scaling = 0;
1120         unsigned int fnum, fc;
1121         const char *quot_offset;
1122         struct fuse_corner *fuse, *prev_fuse;
1123         struct corner *corner, *end;
1124         struct corner_data *cdata;
1125         const struct fuse_corner_data *fdata;
1126         bool apply_scaling;
1127         unsigned long freq_diff, freq_diff_mhz;
1128         unsigned long freq;
1129         int step_volt = regulator_get_linear_step(drv->vdd_apc);
1130         struct dev_pm_opp *opp;
1131
1132         if (!step_volt)
1133                 return -EINVAL;
1134
1135         corner = drv->corners;
1136         end = &corner[drv->num_corners - 1];
1137
1138         cdata = devm_kcalloc(drv->dev, drv->num_corners,
1139                              sizeof(struct corner_data),
1140                              GFP_KERNEL);
1141         if (!cdata)
1142                 return -ENOMEM;
1143
1144         /*
1145          * Store maximum frequency for each fuse corner based on the frequency
1146          * plan
1147          */
1148         for (level = 1; level <= drv->num_corners; level++) {
1149                 opp = dev_pm_opp_find_level_exact(&drv->pd.dev, level);
1150                 if (IS_ERR(opp))
1151                         return -EINVAL;
1152                 fc = cpr_get_fuse_corner(opp);
1153                 if (!fc) {
1154                         dev_pm_opp_put(opp);
1155                         return -EINVAL;
1156                 }
1157                 fnum = fc - 1;
1158                 freq = cpr_get_opp_hz_for_req(opp, drv->attached_cpu_dev);
1159                 if (!freq) {
1160                         dev_pm_opp_put(opp);
1161                         return -EINVAL;
1162                 }
1163                 cdata[level - 1].fuse_corner = fnum;
1164                 cdata[level - 1].freq = freq;
1165
1166                 fuse = &drv->fuse_corners[fnum];
1167                 dev_dbg(drv->dev, "freq: %lu level: %u fuse level: %u\n",
1168                         freq, dev_pm_opp_get_level(opp) - 1, fnum);
1169                 if (freq > fuse->max_freq)
1170                         fuse->max_freq = freq;
1171                 dev_pm_opp_put(opp);
1172         }
1173
1174         /*
1175          * Get the quotient adjustment scaling factor, according to:
1176          *
1177          * scaling = min(1000 * (QUOT(corner_N) - QUOT(corner_N-1))
1178          *              / (freq(corner_N) - freq(corner_N-1)), max_factor)
1179          *
1180          * QUOT(corner_N):      quotient read from fuse for fuse corner N
1181          * QUOT(corner_N-1):    quotient read from fuse for fuse corner (N - 1)
1182          * freq(corner_N):      max frequency in MHz supported by fuse corner N
1183          * freq(corner_N-1):    max frequency in MHz supported by fuse corner
1184          *                       (N - 1)
1185          *
1186          * Then walk through the corners mapped to each fuse corner
1187          * and calculate the quotient adjustment for each one using the
1188          * following formula:
1189          *
1190          * quot_adjust = (freq_max - freq_corner) * scaling / 1000
1191          *
1192          * freq_max: max frequency in MHz supported by the fuse corner
1193          * freq_corner: frequency in MHz corresponding to the corner
1194          * scaling: calculated from above equation
1195          *
1196          *
1197          *     +                           +
1198          *     |                         v |
1199          *   q |           f c           o |           f c
1200          *   u |         c               l |         c
1201          *   o |       f                 t |       f
1202          *   t |     c                   a |     c
1203          *     | c f                     g | c f
1204          *     |                         e |
1205          *     +---------------            +----------------
1206          *       0 1 2 3 4 5 6               0 1 2 3 4 5 6
1207          *          corner                      corner
1208          *
1209          *    c = corner
1210          *    f = fuse corner
1211          *
1212          */
1213         for (apply_scaling = false, i = 0; corner <= end; corner++, i++) {
1214                 fnum = cdata[i].fuse_corner;
1215                 fdata = &desc->cpr_fuses.fuse_corner_data[fnum];
1216                 quot_offset = fuses[fnum].quotient_offset;
1217                 fuse = &drv->fuse_corners[fnum];
1218                 if (fnum)
1219                         prev_fuse = &drv->fuse_corners[fnum - 1];
1220                 else
1221                         prev_fuse = NULL;
1222
1223                 corner->fuse_corner = fuse;
1224                 corner->freq = cdata[i].freq;
1225                 corner->uV = fuse->uV;
1226
1227                 if (prev_fuse && cdata[i - 1].freq == prev_fuse->max_freq) {
1228                         scaling = cpr_calculate_scaling(quot_offset, drv,
1229                                                         fdata, corner);
1230                         if (scaling < 0)
1231                                 return scaling;
1232
1233                         apply_scaling = true;
1234                 } else if (corner->freq == fuse->max_freq) {
1235                         /* This is a fuse corner; don't scale anything */
1236                         apply_scaling = false;
1237                 }
1238
1239                 if (apply_scaling) {
1240                         freq_diff = fuse->max_freq - corner->freq;
1241                         freq_diff_mhz = freq_diff / 1000000;
1242                         corner->quot_adjust = scaling * freq_diff_mhz / 1000;
1243
1244                         corner->uV = cpr_interpolate(corner, step_volt, fdata);
1245                 }
1246
1247                 corner->max_uV = fuse->max_uV;
1248                 corner->min_uV = fuse->min_uV;
1249                 corner->uV = clamp(corner->uV, corner->min_uV, corner->max_uV);
1250                 corner->last_uV = corner->uV;
1251
1252                 /* Reduce the ceiling voltage if needed */
1253                 if (desc->reduce_to_corner_uV && corner->uV < corner->max_uV)
1254                         corner->max_uV = corner->uV;
1255                 else if (desc->reduce_to_fuse_uV && fuse->uV < corner->max_uV)
1256                         corner->max_uV = max(corner->min_uV, fuse->uV);
1257
1258                 dev_dbg(drv->dev, "corner %d: [%d %d %d] quot %d\n", i,
1259                         corner->min_uV, corner->uV, corner->max_uV,
1260                         fuse->quot - corner->quot_adjust);
1261         }
1262
1263         return 0;
1264 }
1265
1266 static const struct cpr_fuse *cpr_get_fuses(struct cpr_drv *drv)
1267 {
1268         const struct cpr_desc *desc = drv->desc;
1269         struct cpr_fuse *fuses;
1270         int i;
1271
1272         fuses = devm_kcalloc(drv->dev, desc->num_fuse_corners,
1273                              sizeof(struct cpr_fuse),
1274                              GFP_KERNEL);
1275         if (!fuses)
1276                 return ERR_PTR(-ENOMEM);
1277
1278         for (i = 0; i < desc->num_fuse_corners; i++) {
1279                 char tbuf[32];
1280
1281                 snprintf(tbuf, 32, "cpr_ring_osc%d", i + 1);
1282                 fuses[i].ring_osc = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1283                 if (!fuses[i].ring_osc)
1284                         return ERR_PTR(-ENOMEM);
1285
1286                 snprintf(tbuf, 32, "cpr_init_voltage%d", i + 1);
1287                 fuses[i].init_voltage = devm_kstrdup(drv->dev, tbuf,
1288                                                      GFP_KERNEL);
1289                 if (!fuses[i].init_voltage)
1290                         return ERR_PTR(-ENOMEM);
1291
1292                 snprintf(tbuf, 32, "cpr_quotient%d", i + 1);
1293                 fuses[i].quotient = devm_kstrdup(drv->dev, tbuf, GFP_KERNEL);
1294                 if (!fuses[i].quotient)
1295                         return ERR_PTR(-ENOMEM);
1296
1297                 snprintf(tbuf, 32, "cpr_quotient_offset%d", i + 1);
1298                 fuses[i].quotient_offset = devm_kstrdup(drv->dev, tbuf,
1299                                                         GFP_KERNEL);
1300                 if (!fuses[i].quotient_offset)
1301                         return ERR_PTR(-ENOMEM);
1302         }
1303
1304         return fuses;
1305 }
1306
1307 static void cpr_set_loop_allowed(struct cpr_drv *drv)
1308 {
1309         drv->loop_disabled = false;
1310 }
1311
1312 static int cpr_init_parameters(struct cpr_drv *drv)
1313 {
1314         const struct cpr_desc *desc = drv->desc;
1315         struct clk *clk;
1316
1317         clk = clk_get(drv->dev, "ref");
1318         if (IS_ERR(clk))
1319                 return PTR_ERR(clk);
1320
1321         drv->ref_clk_khz = clk_get_rate(clk) / 1000;
1322         clk_put(clk);
1323
1324         if (desc->timer_cons_up > RBIF_TIMER_ADJ_CONS_UP_MASK ||
1325             desc->timer_cons_down > RBIF_TIMER_ADJ_CONS_DOWN_MASK ||
1326             desc->up_threshold > RBCPR_CTL_UP_THRESHOLD_MASK ||
1327             desc->down_threshold > RBCPR_CTL_DN_THRESHOLD_MASK ||
1328             desc->idle_clocks > RBCPR_STEP_QUOT_IDLE_CLK_MASK ||
1329             desc->clamp_timer_interval > RBIF_TIMER_ADJ_CLAMP_INT_MASK)
1330                 return -EINVAL;
1331
1332         dev_dbg(drv->dev, "up threshold = %u, down threshold = %u\n",
1333                 desc->up_threshold, desc->down_threshold);
1334
1335         return 0;
1336 }
1337
1338 static int cpr_find_initial_corner(struct cpr_drv *drv)
1339 {
1340         unsigned long rate;
1341         const struct corner *end;
1342         struct corner *iter;
1343         unsigned int i = 0;
1344
1345         if (!drv->cpu_clk) {
1346                 dev_err(drv->dev, "cannot get rate from NULL clk\n");
1347                 return -EINVAL;
1348         }
1349
1350         end = &drv->corners[drv->num_corners - 1];
1351         rate = clk_get_rate(drv->cpu_clk);
1352
1353         /*
1354          * Some bootloaders set a CPU clock frequency that is not defined
1355          * in the OPP table. When running at an unlisted frequency,
1356          * cpufreq_online() will change to the OPP which has the lowest
1357          * frequency, at or above the unlisted frequency.
1358          * Since cpufreq_online() always "rounds up" in the case of an
1359          * unlisted frequency, this function always "rounds down" in case
1360          * of an unlisted frequency. That way, when cpufreq_online()
1361          * triggers the first ever call to cpr_set_performance_state(),
1362          * it will correctly determine the direction as UP.
1363          */
1364         for (iter = drv->corners; iter <= end; iter++) {
1365                 if (iter->freq > rate)
1366                         break;
1367                 i++;
1368                 if (iter->freq == rate) {
1369                         drv->corner = iter;
1370                         break;
1371                 }
1372                 if (iter->freq < rate)
1373                         drv->corner = iter;
1374         }
1375
1376         if (!drv->corner) {
1377                 dev_err(drv->dev, "boot up corner not found\n");
1378                 return -EINVAL;
1379         }
1380
1381         dev_dbg(drv->dev, "boot up perf state: %u\n", i);
1382
1383         return 0;
1384 }
1385
1386 static const struct cpr_desc qcs404_cpr_desc = {
1387         .num_fuse_corners = 3,
1388         .min_diff_quot = CPR_FUSE_MIN_QUOT_DIFF,
1389         .step_quot = (int []){ 25, 25, 25, },
1390         .timer_delay_us = 5000,
1391         .timer_cons_up = 0,
1392         .timer_cons_down = 2,
1393         .up_threshold = 1,
1394         .down_threshold = 3,
1395         .idle_clocks = 15,
1396         .gcnt_us = 1,
1397         .vdd_apc_step_up_limit = 1,
1398         .vdd_apc_step_down_limit = 1,
1399         .cpr_fuses = {
1400                 .init_voltage_step = 8000,
1401                 .init_voltage_width = 6,
1402                 .fuse_corner_data = (struct fuse_corner_data[]){
1403                         /* fuse corner 0 */
1404                         {
1405                                 .ref_uV = 1224000,
1406                                 .max_uV = 1224000,
1407                                 .min_uV = 1048000,
1408                                 .max_volt_scale = 0,
1409                                 .max_quot_scale = 0,
1410                                 .quot_offset = 0,
1411                                 .quot_scale = 1,
1412                                 .quot_adjust = 0,
1413                                 .quot_offset_scale = 5,
1414                                 .quot_offset_adjust = 0,
1415                         },
1416                         /* fuse corner 1 */
1417                         {
1418                                 .ref_uV = 1288000,
1419                                 .max_uV = 1288000,
1420                                 .min_uV = 1048000,
1421                                 .max_volt_scale = 2000,
1422                                 .max_quot_scale = 1400,
1423                                 .quot_offset = 0,
1424                                 .quot_scale = 1,
1425                                 .quot_adjust = -20,
1426                                 .quot_offset_scale = 5,
1427                                 .quot_offset_adjust = 0,
1428                         },
1429                         /* fuse corner 2 */
1430                         {
1431                                 .ref_uV = 1352000,
1432                                 .max_uV = 1384000,
1433                                 .min_uV = 1088000,
1434                                 .max_volt_scale = 2000,
1435                                 .max_quot_scale = 1400,
1436                                 .quot_offset = 0,
1437                                 .quot_scale = 1,
1438                                 .quot_adjust = 0,
1439                                 .quot_offset_scale = 5,
1440                                 .quot_offset_adjust = 0,
1441                         },
1442                 },
1443         },
1444 };
1445
1446 static const struct acc_desc qcs404_acc_desc = {
1447         .settings = (struct reg_sequence[]){
1448                 { 0xb120, 0x1041040 },
1449                 { 0xb124, 0x41 },
1450                 { 0xb120, 0x0 },
1451                 { 0xb124, 0x0 },
1452                 { 0xb120, 0x0 },
1453                 { 0xb124, 0x0 },
1454         },
1455         .config = (struct reg_sequence[]){
1456                 { 0xb138, 0xff },
1457                 { 0xb130, 0x5555 },
1458         },
1459         .num_regs_per_fuse = 2,
1460 };
1461
1462 static const struct cpr_acc_desc qcs404_cpr_acc_desc = {
1463         .cpr_desc = &qcs404_cpr_desc,
1464         .acc_desc = &qcs404_acc_desc,
1465 };
1466
1467 static unsigned int cpr_get_performance_state(struct generic_pm_domain *genpd,
1468                                               struct dev_pm_opp *opp)
1469 {
1470         return dev_pm_opp_get_level(opp);
1471 }
1472
1473 static int cpr_power_off(struct generic_pm_domain *domain)
1474 {
1475         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1476
1477         return cpr_disable(drv);
1478 }
1479
1480 static int cpr_power_on(struct generic_pm_domain *domain)
1481 {
1482         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1483
1484         return cpr_enable(drv);
1485 }
1486
1487 static int cpr_pd_attach_dev(struct generic_pm_domain *domain,
1488                              struct device *dev)
1489 {
1490         struct cpr_drv *drv = container_of(domain, struct cpr_drv, pd);
1491         const struct acc_desc *acc_desc = drv->acc_desc;
1492         int ret = 0;
1493
1494         mutex_lock(&drv->lock);
1495
1496         dev_dbg(drv->dev, "attach callback for: %s\n", dev_name(dev));
1497
1498         /*
1499          * This driver only supports scaling voltage for a CPU cluster
1500          * where all CPUs in the cluster share a single regulator.
1501          * Therefore, save the struct device pointer only for the first
1502          * CPU device that gets attached. There is no need to do any
1503          * additional initialization when further CPUs get attached.
1504          */
1505         if (drv->attached_cpu_dev)
1506                 goto unlock;
1507
1508         /*
1509          * cpr_scale_voltage() requires the direction (if we are changing
1510          * to a higher or lower OPP). The first time
1511          * cpr_set_performance_state() is called, there is no previous
1512          * performance state defined. Therefore, we call
1513          * cpr_find_initial_corner() that gets the CPU clock frequency
1514          * set by the bootloader, so that we can determine the direction
1515          * the first time cpr_set_performance_state() is called.
1516          */
1517         drv->cpu_clk = devm_clk_get(dev, NULL);
1518         if (IS_ERR(drv->cpu_clk)) {
1519                 ret = PTR_ERR(drv->cpu_clk);
1520                 if (ret != -EPROBE_DEFER)
1521                         dev_err(drv->dev, "could not get cpu clk: %d\n", ret);
1522                 goto unlock;
1523         }
1524         drv->attached_cpu_dev = dev;
1525
1526         dev_dbg(drv->dev, "using cpu clk from: %s\n",
1527                 dev_name(drv->attached_cpu_dev));
1528
1529         /*
1530          * Everything related to (virtual) corners has to be initialized
1531          * here, when attaching to the power domain, since we need to know
1532          * the maximum frequency for each fuse corner, and this is only
1533          * available after the cpufreq driver has attached to us.
1534          * The reason for this is that we need to know the highest
1535          * frequency associated with each fuse corner.
1536          */
1537         ret = dev_pm_opp_get_opp_count(&drv->pd.dev);
1538         if (ret < 0) {
1539                 dev_err(drv->dev, "could not get OPP count\n");
1540                 goto unlock;
1541         }
1542         drv->num_corners = ret;
1543
1544         if (drv->num_corners < 2) {
1545                 dev_err(drv->dev, "need at least 2 OPPs to use CPR\n");
1546                 ret = -EINVAL;
1547                 goto unlock;
1548         }
1549
1550         drv->corners = devm_kcalloc(drv->dev, drv->num_corners,
1551                                     sizeof(*drv->corners),
1552                                     GFP_KERNEL);
1553         if (!drv->corners) {
1554                 ret = -ENOMEM;
1555                 goto unlock;
1556         }
1557
1558         ret = cpr_corner_init(drv);
1559         if (ret)
1560                 goto unlock;
1561
1562         cpr_set_loop_allowed(drv);
1563
1564         ret = cpr_init_parameters(drv);
1565         if (ret)
1566                 goto unlock;
1567
1568         /* Configure CPR HW but keep it disabled */
1569         ret = cpr_config(drv);
1570         if (ret)
1571                 goto unlock;
1572
1573         ret = cpr_find_initial_corner(drv);
1574         if (ret)
1575                 goto unlock;
1576
1577         if (acc_desc->config)
1578                 regmap_multi_reg_write(drv->tcsr, acc_desc->config,
1579                                        acc_desc->num_regs_per_fuse);
1580
1581         /* Enable ACC if required */
1582         if (acc_desc->enable_mask)
1583                 regmap_update_bits(drv->tcsr, acc_desc->enable_reg,
1584                                    acc_desc->enable_mask,
1585                                    acc_desc->enable_mask);
1586
1587         dev_info(drv->dev, "driver initialized with %u OPPs\n",
1588                  drv->num_corners);
1589
1590 unlock:
1591         mutex_unlock(&drv->lock);
1592
1593         return ret;
1594 }
1595
1596 static int cpr_debug_info_show(struct seq_file *s, void *unused)
1597 {
1598         u32 gcnt, ro_sel, ctl, irq_status, reg, error_steps;
1599         u32 step_dn, step_up, error, error_lt0, busy;
1600         struct cpr_drv *drv = s->private;
1601         struct fuse_corner *fuse_corner;
1602         struct corner *corner;
1603
1604         corner = drv->corner;
1605         fuse_corner = corner->fuse_corner;
1606
1607         seq_printf(s, "corner, current_volt = %d uV\n",
1608                        corner->last_uV);
1609
1610         ro_sel = fuse_corner->ring_osc_idx;
1611         gcnt = cpr_read(drv, REG_RBCPR_GCNT_TARGET(ro_sel));
1612         seq_printf(s, "rbcpr_gcnt_target (%u) = %#02X\n", ro_sel, gcnt);
1613
1614         ctl = cpr_read(drv, REG_RBCPR_CTL);
1615         seq_printf(s, "rbcpr_ctl = %#02X\n", ctl);
1616
1617         irq_status = cpr_read(drv, REG_RBIF_IRQ_STATUS);
1618         seq_printf(s, "rbcpr_irq_status = %#02X\n", irq_status);
1619
1620         reg = cpr_read(drv, REG_RBCPR_RESULT_0);
1621         seq_printf(s, "rbcpr_result_0 = %#02X\n", reg);
1622
1623         step_dn = reg & 0x01;
1624         step_up = (reg >> RBCPR_RESULT0_STEP_UP_SHIFT) & 0x01;
1625         seq_printf(s, "  [step_dn = %u", step_dn);
1626
1627         seq_printf(s, ", step_up = %u", step_up);
1628
1629         error_steps = (reg >> RBCPR_RESULT0_ERROR_STEPS_SHIFT)
1630                                 & RBCPR_RESULT0_ERROR_STEPS_MASK;
1631         seq_printf(s, ", error_steps = %u", error_steps);
1632
1633         error = (reg >> RBCPR_RESULT0_ERROR_SHIFT) & RBCPR_RESULT0_ERROR_MASK;
1634         seq_printf(s, ", error = %u", error);
1635
1636         error_lt0 = (reg >> RBCPR_RESULT0_ERROR_LT0_SHIFT) & 0x01;
1637         seq_printf(s, ", error_lt_0 = %u", error_lt0);
1638
1639         busy = (reg >> RBCPR_RESULT0_BUSY_SHIFT) & 0x01;
1640         seq_printf(s, ", busy = %u]\n", busy);
1641
1642         return 0;
1643 }
1644 DEFINE_SHOW_ATTRIBUTE(cpr_debug_info);
1645
1646 static void cpr_debugfs_init(struct cpr_drv *drv)
1647 {
1648         drv->debugfs = debugfs_create_dir("qcom_cpr", NULL);
1649
1650         debugfs_create_file("debug_info", 0444, drv->debugfs,
1651                             drv, &cpr_debug_info_fops);
1652 }
1653
1654 static int cpr_probe(struct platform_device *pdev)
1655 {
1656         struct resource *res;
1657         struct device *dev = &pdev->dev;
1658         struct cpr_drv *drv;
1659         int irq, ret;
1660         const struct cpr_acc_desc *data;
1661         struct device_node *np;
1662         u32 cpr_rev = FUSE_REVISION_UNKNOWN;
1663
1664         data = of_device_get_match_data(dev);
1665         if (!data || !data->cpr_desc || !data->acc_desc)
1666                 return -EINVAL;
1667
1668         drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
1669         if (!drv)
1670                 return -ENOMEM;
1671         drv->dev = dev;
1672         drv->desc = data->cpr_desc;
1673         drv->acc_desc = data->acc_desc;
1674
1675         drv->fuse_corners = devm_kcalloc(dev, drv->desc->num_fuse_corners,
1676                                          sizeof(*drv->fuse_corners),
1677                                          GFP_KERNEL);
1678         if (!drv->fuse_corners)
1679                 return -ENOMEM;
1680
1681         np = of_parse_phandle(dev->of_node, "acc-syscon", 0);
1682         if (!np)
1683                 return -ENODEV;
1684
1685         drv->tcsr = syscon_node_to_regmap(np);
1686         of_node_put(np);
1687         if (IS_ERR(drv->tcsr))
1688                 return PTR_ERR(drv->tcsr);
1689
1690         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1691         drv->base = devm_ioremap_resource(dev, res);
1692         if (IS_ERR(drv->base))
1693                 return PTR_ERR(drv->base);
1694
1695         irq = platform_get_irq(pdev, 0);
1696         if (irq < 0)
1697                 return -EINVAL;
1698
1699         drv->vdd_apc = devm_regulator_get(dev, "vdd-apc");
1700         if (IS_ERR(drv->vdd_apc))
1701                 return PTR_ERR(drv->vdd_apc);
1702
1703         /*
1704          * Initialize fuse corners, since it simply depends
1705          * on data in efuses.
1706          * Everything related to (virtual) corners has to be
1707          * initialized after attaching to the power domain,
1708          * since it depends on the CPU's OPP table.
1709          */
1710         ret = cpr_read_efuse(dev, "cpr_fuse_revision", &cpr_rev);
1711         if (ret)
1712                 return ret;
1713
1714         drv->cpr_fuses = cpr_get_fuses(drv);
1715         if (IS_ERR(drv->cpr_fuses))
1716                 return PTR_ERR(drv->cpr_fuses);
1717
1718         ret = cpr_populate_ring_osc_idx(drv);
1719         if (ret)
1720                 return ret;
1721
1722         ret = cpr_fuse_corner_init(drv);
1723         if (ret)
1724                 return ret;
1725
1726         mutex_init(&drv->lock);
1727
1728         ret = devm_request_threaded_irq(dev, irq, NULL,
1729                                         cpr_irq_handler,
1730                                         IRQF_ONESHOT | IRQF_TRIGGER_RISING,
1731                                         "cpr", drv);
1732         if (ret)
1733                 return ret;
1734
1735         drv->pd.name = devm_kstrdup_const(dev, dev->of_node->full_name,
1736                                           GFP_KERNEL);
1737         if (!drv->pd.name)
1738                 return -EINVAL;
1739
1740         drv->pd.power_off = cpr_power_off;
1741         drv->pd.power_on = cpr_power_on;
1742         drv->pd.set_performance_state = cpr_set_performance_state;
1743         drv->pd.opp_to_performance_state = cpr_get_performance_state;
1744         drv->pd.attach_dev = cpr_pd_attach_dev;
1745
1746         ret = pm_genpd_init(&drv->pd, NULL, true);
1747         if (ret)
1748                 return ret;
1749
1750         ret = of_genpd_add_provider_simple(dev->of_node, &drv->pd);
1751         if (ret)
1752                 return ret;
1753
1754         platform_set_drvdata(pdev, drv);
1755         cpr_debugfs_init(drv);
1756
1757         return 0;
1758 }
1759
1760 static int cpr_remove(struct platform_device *pdev)
1761 {
1762         struct cpr_drv *drv = platform_get_drvdata(pdev);
1763
1764         if (cpr_is_allowed(drv)) {
1765                 cpr_ctl_disable(drv);
1766                 cpr_irq_set(drv, 0);
1767         }
1768
1769         of_genpd_del_provider(pdev->dev.of_node);
1770         pm_genpd_remove(&drv->pd);
1771
1772         debugfs_remove_recursive(drv->debugfs);
1773
1774         return 0;
1775 }
1776
1777 static const struct of_device_id cpr_match_table[] = {
1778         { .compatible = "qcom,qcs404-cpr", .data = &qcs404_cpr_acc_desc },
1779         { }
1780 };
1781 MODULE_DEVICE_TABLE(of, cpr_match_table);
1782
1783 static struct platform_driver cpr_driver = {
1784         .probe          = cpr_probe,
1785         .remove         = cpr_remove,
1786         .driver         = {
1787                 .name   = "qcom-cpr",
1788                 .of_match_table = cpr_match_table,
1789         },
1790 };
1791 module_platform_driver(cpr_driver);
1792
1793 MODULE_DESCRIPTION("Core Power Reduction (CPR) driver");
1794 MODULE_LICENSE("GPL v2");