drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
   4  * Copyright (C) 2013 Red Hat
   5  * Author: Rob Clark <robdclark@gmail.com>
   6  */
   7
   8 #define pr_fmt(fmt)     "[drm:%s:%d] " fmt, __func__, __LINE__
   9
  10 #include <linux/debugfs.h>
  11 #include <linux/dma-buf.h>
  12 #include <linux/of_irq.h>
  13 #include <linux/pm_opp.h>
  14
  15 #include <drm/drm_crtc.h>
  16 #include <drm/drm_file.h>
  17 #include <drm/drm_vblank.h>
  18
  19 #include "msm_drv.h"
  20 #include "msm_mmu.h"
  21 #include "msm_gem.h"
  22 #include "disp/msm_disp_snapshot.h"
  23
  24 #include "dpu_kms.h"
  25 #include "dpu_core_irq.h"
  26 #include "dpu_formats.h"
  27 #include "dpu_hw_vbif.h"
  28 #include "dpu_vbif.h"
  29 #include "dpu_encoder.h"
  30 #include "dpu_plane.h"
  31 #include "dpu_crtc.h"
  32
  33 #define CREATE_TRACE_POINTS
  34 #include "dpu_trace.h"
  35
  36 /*
  37  * To enable overall DRM driver logging
  38  * # echo 0x2 > /sys/module/drm/parameters/debug
  39  *
  40  * To enable DRM driver h/w logging
  41  * # echo <mask> > /sys/kernel/debug/dri/0/debug/hw_log_mask
  42  *
  43  * See dpu_hw_mdss.h for h/w logging mask definitions (search for DPU_DBG_MASK_)
  44  */
  45 #define DPU_DEBUGFS_DIR "msm_dpu"
  46 #define DPU_DEBUGFS_HWMASKNAME "hw_log_mask"
  47
  48 #define MIN_IB_BW       400000000ULL /* Min ib vote 400MB */
  49
  50 static int dpu_kms_hw_init(struct msm_kms *kms);
  51 static void _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms);
  52
  53 #ifdef CONFIG_DEBUG_FS
  54 static int _dpu_danger_signal_status(struct seq_file *s,
  55                 bool danger_status)
  56 {
  57         struct dpu_kms *kms = (struct dpu_kms *)s->private;
  58         struct dpu_danger_safe_status status;
  59         int i;
  60
  61         if (!kms->hw_mdp) {
  62                 DPU_ERROR("invalid arg(s)\n");
  63                 return 0;
  64         }
  65
  66         memset(&status, 0, sizeof(struct dpu_danger_safe_status));
  67
  68         pm_runtime_get_sync(&kms->pdev->dev);
  69         if (danger_status) {
  70                 seq_puts(s, "\nDanger signal status:\n");
  71                 if (kms->hw_mdp->ops.get_danger_status)
  72                         kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
  73                                         &status);
  74         } else {
  75                 seq_puts(s, "\nSafe signal status:\n");
  76                 if (kms->hw_mdp->ops.get_danger_status)
  77                         kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
  78                                         &status);
  79         }
  80         pm_runtime_put_sync(&kms->pdev->dev);
  81
  82         seq_printf(s, "MDP     :  0x%x\n", status.mdp);
  83
  84         for (i = SSPP_VIG0; i < SSPP_MAX; i++)
  85                 seq_printf(s, "SSPP%d   :  0x%x  \t", i - SSPP_VIG0,
  86                                 status.sspp[i]);
  87         seq_puts(s, "\n");
  88
  89         return 0;
  90 }
  91
  92 static int dpu_debugfs_danger_stats_show(struct seq_file *s, void *v)
  93 {
  94         return _dpu_danger_signal_status(s, true);
  95 }
  96 DEFINE_SHOW_ATTRIBUTE(dpu_debugfs_danger_stats);
  97
  98 static int dpu_debugfs_safe_stats_show(struct seq_file *s, void *v)
  99 {
 100         return _dpu_danger_signal_status(s, false);
 101 }
 102 DEFINE_SHOW_ATTRIBUTE(dpu_debugfs_safe_stats);
 103
 104 static void dpu_debugfs_danger_init(struct dpu_kms *dpu_kms,
 105                 struct dentry *parent)
 106 {
 107         struct dentry *entry = debugfs_create_dir("danger", parent);
 108
 109         debugfs_create_file("danger_status", 0600, entry,
 110                         dpu_kms, &dpu_debugfs_danger_stats_fops);
 111         debugfs_create_file("safe_status", 0600, entry,
 112                         dpu_kms, &dpu_debugfs_safe_stats_fops);
 113 }
 114
 115 static int _dpu_debugfs_show_regset32(struct seq_file *s, void *data)
 116 {
 117         struct dpu_debugfs_regset32 *regset = s->private;
 118         struct dpu_kms *dpu_kms = regset->dpu_kms;
 119         void __iomem *base;
 120         uint32_t i, addr;
 121
 122         if (!dpu_kms->mmio)
 123                 return 0;
 124
 125         base = dpu_kms->mmio + regset->offset;
 126
 127         /* insert padding spaces, if needed */
 128         if (regset->offset & 0xF) {
 129                 seq_printf(s, "[%x]", regset->offset & ~0xF);
 130                 for (i = 0; i < (regset->offset & 0xF); i += 4)
 131                         seq_puts(s, "         ");
 132         }
 133
 134         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 135
 136         /* main register output */
 137         for (i = 0; i < regset->blk_len; i += 4) {
 138                 addr = regset->offset + i;
 139                 if ((addr & 0xF) == 0x0)
 140                         seq_printf(s, i ? "\n[%x]" : "[%x]", addr);
 141                 seq_printf(s, " %08x", readl_relaxed(base + i));
 142         }
 143         seq_puts(s, "\n");
 144         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 145
 146         return 0;
 147 }
 148
 149 static int dpu_debugfs_open_regset32(struct inode *inode,
 150                 struct file *file)
 151 {
 152         return single_open(file, _dpu_debugfs_show_regset32, inode->i_private);
 153 }
 154
 155 static const struct file_operations dpu_fops_regset32 = {
 156         .open =         dpu_debugfs_open_regset32,
 157         .read =         seq_read,
 158         .llseek =       seq_lseek,
 159         .release =      single_release,
 160 };
 161
 162 void dpu_debugfs_setup_regset32(struct dpu_debugfs_regset32 *regset,
 163                 uint32_t offset, uint32_t length, struct dpu_kms *dpu_kms)
 164 {
 165         if (regset) {
 166                 regset->offset = offset;
 167                 regset->blk_len = length;
 168                 regset->dpu_kms = dpu_kms;
 169         }
 170 }
 171
 172 void dpu_debugfs_create_regset32(const char *name, umode_t mode,
 173                 void *parent, struct dpu_debugfs_regset32 *regset)
 174 {
 175         if (!name || !regset || !regset->dpu_kms || !regset->blk_len)
 176                 return;
 177
 178         /* make sure offset is a multiple of 4 */
 179         regset->offset = round_down(regset->offset, 4);
 180
 181         debugfs_create_file(name, mode, parent, regset, &dpu_fops_regset32);
 182 }
 183
 184 static int dpu_kms_debugfs_init(struct msm_kms *kms, struct drm_minor *minor)
 185 {
 186         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 187         void *p = dpu_hw_util_get_log_mask_ptr();
 188         struct dentry *entry;
 189         struct drm_device *dev;
 190         struct msm_drm_private *priv;
 191
 192         if (!p)
 193                 return -EINVAL;
 194
 195         dev = dpu_kms->dev;
 196         priv = dev->dev_private;
 197
 198         entry = debugfs_create_dir("debug", minor->debugfs_root);
 199
 200         debugfs_create_x32(DPU_DEBUGFS_HWMASKNAME, 0600, entry, p);
 201
 202         dpu_debugfs_danger_init(dpu_kms, entry);
 203         dpu_debugfs_vbif_init(dpu_kms, entry);
 204         dpu_debugfs_core_irq_init(dpu_kms, entry);
 205
 206         if (priv->dp)
 207                 msm_dp_debugfs_init(priv->dp, minor);
 208
 209         return dpu_core_perf_debugfs_init(dpu_kms, entry);
 210 }
 211 #endif
 212
 213 /* Global/shared object state funcs */
 214
 215 /*
 216  * This is a helper that returns the private state currently in operation.
 217  * Note that this would return the "old_state" if called in the atomic check
 218  * path, and the "new_state" after the atomic swap has been done.
 219  */
 220 struct dpu_global_state *
 221 dpu_kms_get_existing_global_state(struct dpu_kms *dpu_kms)
 222 {
 223         return to_dpu_global_state(dpu_kms->global_state.state);
 224 }
 225
 226 /*
 227  * This acquires the modeset lock set aside for global state, creates
 228  * a new duplicated private object state.
 229  */
 230 struct dpu_global_state *dpu_kms_get_global_state(struct drm_atomic_state *s)
 231 {
 232         struct msm_drm_private *priv = s->dev->dev_private;
 233         struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
 234         struct drm_private_state *priv_state;
 235         int ret;
 236
 237         ret = drm_modeset_lock(&dpu_kms->global_state_lock, s->acquire_ctx);
 238         if (ret)
 239                 return ERR_PTR(ret);
 240
 241         priv_state = drm_atomic_get_private_obj_state(s,
 242                                                 &dpu_kms->global_state);
 243         if (IS_ERR(priv_state))
 244                 return ERR_CAST(priv_state);
 245
 246         return to_dpu_global_state(priv_state);
 247 }
 248
 249 static struct drm_private_state *
 250 dpu_kms_global_duplicate_state(struct drm_private_obj *obj)
 251 {
 252         struct dpu_global_state *state;
 253
 254         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 255         if (!state)
 256                 return NULL;
 257
 258         __drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 259
 260         return &state->base;
 261 }
 262
 263 static void dpu_kms_global_destroy_state(struct drm_private_obj *obj,
 264                                       struct drm_private_state *state)
 265 {
 266         struct dpu_global_state *dpu_state = to_dpu_global_state(state);
 267
 268         kfree(dpu_state);
 269 }
 270
 271 static const struct drm_private_state_funcs dpu_kms_global_state_funcs = {
 272         .atomic_duplicate_state = dpu_kms_global_duplicate_state,
 273         .atomic_destroy_state = dpu_kms_global_destroy_state,
 274 };
 275
 276 static int dpu_kms_global_obj_init(struct dpu_kms *dpu_kms)
 277 {
 278         struct dpu_global_state *state;
 279
 280         drm_modeset_lock_init(&dpu_kms->global_state_lock);
 281
 282         state = kzalloc(sizeof(*state), GFP_KERNEL);
 283         if (!state)
 284                 return -ENOMEM;
 285
 286         drm_atomic_private_obj_init(dpu_kms->dev, &dpu_kms->global_state,
 287                                     &state->base,
 288                                     &dpu_kms_global_state_funcs);
 289         return 0;
 290 }
 291
 292 static int dpu_kms_parse_data_bus_icc_path(struct dpu_kms *dpu_kms)
 293 {
 294         struct icc_path *path0;
 295         struct icc_path *path1;
 296         struct drm_device *dev = dpu_kms->dev;
 297
 298         path0 = of_icc_get(dev->dev, "mdp0-mem");
 299         path1 = of_icc_get(dev->dev, "mdp1-mem");
 300
 301         if (IS_ERR_OR_NULL(path0))
 302                 return PTR_ERR_OR_ZERO(path0);
 303
 304         dpu_kms->path[0] = path0;
 305         dpu_kms->num_paths = 1;
 306
 307         if (!IS_ERR_OR_NULL(path1)) {
 308                 dpu_kms->path[1] = path1;
 309                 dpu_kms->num_paths++;
 310         }
 311         return 0;
 312 }
 313
 314 static int dpu_kms_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 315 {
 316         return dpu_crtc_vblank(crtc, true);
 317 }
 318
 319 static void dpu_kms_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 320 {
 321         dpu_crtc_vblank(crtc, false);
 322 }
 323
 324 static void dpu_kms_enable_commit(struct msm_kms *kms)
 325 {
 326         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 327         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 328 }
 329
 330 static void dpu_kms_disable_commit(struct msm_kms *kms)
 331 {
 332         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 333         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 334 }
 335
 336 static ktime_t dpu_kms_vsync_time(struct msm_kms *kms, struct drm_crtc *crtc)
 337 {
 338         struct drm_encoder *encoder;
 339
 340         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
 341                 ktime_t vsync_time;
 342
 343                 if (dpu_encoder_vsync_time(encoder, &vsync_time) == 0)
 344                         return vsync_time;
 345         }
 346
 347         return ktime_get();
 348 }
 349
 350 static void dpu_kms_prepare_commit(struct msm_kms *kms,
 351                 struct drm_atomic_state *state)
 352 {
 353         struct drm_crtc *crtc;
 354         struct drm_crtc_state *crtc_state;
 355         struct drm_encoder *encoder;
 356         int i;
 357
 358         if (!kms)
 359                 return;
 360
 361         /* Call prepare_commit for all affected encoders */
 362         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 363                 drm_for_each_encoder_mask(encoder, crtc->dev,
 364                                           crtc_state->encoder_mask) {
 365                         dpu_encoder_prepare_commit(encoder);
 366                 }
 367         }
 368 }
 369
 370 static void dpu_kms_flush_commit(struct msm_kms *kms, unsigned crtc_mask)
 371 {
 372         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 373         struct drm_crtc *crtc;
 374
 375         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask) {
 376                 if (!crtc->state->active)
 377                         continue;
 378
 379                 trace_dpu_kms_commit(DRMID(crtc));
 380                 dpu_crtc_commit_kickoff(crtc);
 381         }
 382 }
 383
 384 /*
 385  * Override the encoder enable since we need to setup the inline rotator and do
 386  * some crtc magic before enabling any bridge that might be present.
 387  */
 388 void dpu_kms_encoder_enable(struct drm_encoder *encoder)
 389 {
 390         const struct drm_encoder_helper_funcs *funcs = encoder->helper_private;
 391         struct drm_device *dev = encoder->dev;
 392         struct drm_crtc *crtc;
 393
 394         /* Forward this enable call to the commit hook */
 395         if (funcs && funcs->commit)
 396                 funcs->commit(encoder);
 397
 398         drm_for_each_crtc(crtc, dev) {
 399                 if (!(crtc->state->encoder_mask & drm_encoder_mask(encoder)))
 400                         continue;
 401
 402                 trace_dpu_kms_enc_enable(DRMID(crtc));
 403         }
 404 }
 405
 406 static void dpu_kms_complete_commit(struct msm_kms *kms, unsigned crtc_mask)
 407 {
 408         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 409         struct drm_crtc *crtc;
 410
 411         DPU_ATRACE_BEGIN("kms_complete_commit");
 412
 413         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask)
 414                 dpu_crtc_complete_commit(crtc);
 415
 416         DPU_ATRACE_END("kms_complete_commit");
 417 }
 418
 419 static void dpu_kms_wait_for_commit_done(struct msm_kms *kms,
 420                 struct drm_crtc *crtc)
 421 {
 422         struct drm_encoder *encoder;
 423         struct drm_device *dev;
 424         int ret;
 425
 426         if (!kms || !crtc || !crtc->state) {
 427                 DPU_ERROR("invalid params\n");
 428                 return;
 429         }
 430
 431         dev = crtc->dev;
 432
 433         if (!crtc->state->enable) {
 434                 DPU_DEBUG("[crtc:%d] not enable\n", crtc->base.id);
 435                 return;
 436         }
 437
 438         if (!crtc->state->active) {
 439                 DPU_DEBUG("[crtc:%d] not active\n", crtc->base.id);
 440                 return;
 441         }
 442
 443         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 444                 if (encoder->crtc != crtc)
 445                         continue;
 446                 /*
 447                  * Wait for post-flush if necessary to delay before
 448                  * plane_cleanup. For example, wait for vsync in case of video
 449                  * mode panels. This may be a no-op for command mode panels.
 450                  */
 451                 trace_dpu_kms_wait_for_commit_done(DRMID(crtc));
 452                 ret = dpu_encoder_wait_for_event(encoder, MSM_ENC_COMMIT_DONE);
 453                 if (ret && ret != -EWOULDBLOCK) {
 454                         DPU_ERROR("wait for commit done returned %d\n", ret);
 455                         break;
 456                 }
 457         }
 458 }
 459
 460 static void dpu_kms_wait_flush(struct msm_kms *kms, unsigned crtc_mask)
 461 {
 462         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 463         struct drm_crtc *crtc;
 464
 465         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask)
 466                 dpu_kms_wait_for_commit_done(kms, crtc);
 467 }
 468
 469 static int _dpu_kms_initialize_dsi(struct drm_device *dev,
 470                                     struct msm_drm_private *priv,
 471                                     struct dpu_kms *dpu_kms)
 472 {
 473         struct drm_encoder *encoder = NULL;
 474         struct msm_display_info info;
 475         int i, rc = 0;
 476
 477         if (!(priv->dsi[0] || priv->dsi[1]))
 478                 return rc;
 479
 480         /*
 481          * We support following confiurations:
 482          * - Single DSI host (dsi0 or dsi1)
 483          * - Two independent DSI hosts
 484          * - Bonded DSI0 and DSI1 hosts
 485          *
 486          * TODO: Support swapping DSI0 and DSI1 in the bonded setup.
 487          */
 488         for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) {
 489                 int other = (i + 1) % 2;
 490
 491                 if (!priv->dsi[i])
 492                         continue;
 493
 494                 if (msm_dsi_is_bonded_dsi(priv->dsi[i]) &&
 495                     !msm_dsi_is_master_dsi(priv->dsi[i]))
 496                         continue;
 497
 498                 encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DSI);
 499                 if (IS_ERR(encoder)) {
 500                         DPU_ERROR("encoder init failed for dsi display\n");
 501                         return PTR_ERR(encoder);
 502                 }
 503
 504                 priv->encoders[priv->num_encoders++] = encoder;
 505
 506                 memset(&info, 0, sizeof(info));
 507                 info.intf_type = encoder->encoder_type;
 508
 509                 rc = msm_dsi_modeset_init(priv->dsi[i], dev, encoder);
 510                 if (rc) {
 511                         DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n",
 512                                 i, rc);
 513                         break;
 514                 }
 515
 516                 info.h_tile_instance[info.num_of_h_tiles++] = i;
 517                 info.capabilities = msm_dsi_is_cmd_mode(priv->dsi[i]) ?
 518                         MSM_DISPLAY_CAP_CMD_MODE :
 519                         MSM_DISPLAY_CAP_VID_MODE;
 520
 521                 if (msm_dsi_is_bonded_dsi(priv->dsi[i]) && priv->dsi[other]) {
 522                         rc = msm_dsi_modeset_init(priv->dsi[other], dev, encoder);
 523                         if (rc) {
 524                                 DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n",
 525                                         other, rc);
 526                                 break;
 527                         }
 528
 529                         info.h_tile_instance[info.num_of_h_tiles++] = other;
 530                 }
 531
 532                 rc = dpu_encoder_setup(dev, encoder, &info);
 533                 if (rc)
 534                         DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n",
 535                                   encoder->base.id, rc);
 536         }
 537
 538         return rc;
 539 }
 540
 541 static int _dpu_kms_initialize_displayport(struct drm_device *dev,
 542                                             struct msm_drm_private *priv,
 543                                             struct dpu_kms *dpu_kms)
 544 {
 545         struct drm_encoder *encoder = NULL;
 546         struct msm_display_info info;
 547         int rc = 0;
 548
 549         if (!priv->dp)
 550                 return rc;
 551
 552         encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_TMDS);
 553         if (IS_ERR(encoder)) {
 554                 DPU_ERROR("encoder init failed for dsi display\n");
 555                 return PTR_ERR(encoder);
 556         }
 557
 558         memset(&info, 0, sizeof(info));
 559         rc = msm_dp_modeset_init(priv->dp, dev, encoder);
 560         if (rc) {
 561                 DPU_ERROR("modeset_init failed for DP, rc = %d\n", rc);
 562                 drm_encoder_cleanup(encoder);
 563                 return rc;
 564         }
 565
 566         priv->encoders[priv->num_encoders++] = encoder;
 567
 568         info.num_of_h_tiles = 1;
 569         info.capabilities = MSM_DISPLAY_CAP_VID_MODE;
 570         info.intf_type = encoder->encoder_type;
 571         rc = dpu_encoder_setup(dev, encoder, &info);
 572         if (rc)
 573                 DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n",
 574                           encoder->base.id, rc);
 575         return rc;
 576 }
 577
 578 /**
 579  * _dpu_kms_setup_displays - create encoders, bridges and connectors
 580  *                           for underlying displays
 581  * @dev:        Pointer to drm device structure
 582  * @priv:       Pointer to private drm device data
 583  * @dpu_kms:    Pointer to dpu kms structure
 584  * Returns:     Zero on success
 585  */
 586 static int _dpu_kms_setup_displays(struct drm_device *dev,
 587                                     struct msm_drm_private *priv,
 588                                     struct dpu_kms *dpu_kms)
 589 {
 590         int rc = 0;
 591
 592         rc = _dpu_kms_initialize_dsi(dev, priv, dpu_kms);
 593         if (rc) {
 594                 DPU_ERROR("initialize_dsi failed, rc = %d\n", rc);
 595                 return rc;
 596         }
 597
 598         rc = _dpu_kms_initialize_displayport(dev, priv, dpu_kms);
 599         if (rc) {
 600                 DPU_ERROR("initialize_DP failed, rc = %d\n", rc);
 601                 return rc;
 602         }
 603
 604         return rc;
 605 }
 606
 607 static void _dpu_kms_drm_obj_destroy(struct dpu_kms *dpu_kms)
 608 {
 609         struct msm_drm_private *priv;
 610         int i;
 611
 612         priv = dpu_kms->dev->dev_private;
 613
 614         for (i = 0; i < priv->num_crtcs; i++)
 615                 priv->crtcs[i]->funcs->destroy(priv->crtcs[i]);
 616         priv->num_crtcs = 0;
 617
 618         for (i = 0; i < priv->num_planes; i++)
 619                 priv->planes[i]->funcs->destroy(priv->planes[i]);
 620         priv->num_planes = 0;
 621
 622         for (i = 0; i < priv->num_connectors; i++)
 623                 priv->connectors[i]->funcs->destroy(priv->connectors[i]);
 624         priv->num_connectors = 0;
 625
 626         for (i = 0; i < priv->num_encoders; i++)
 627                 priv->encoders[i]->funcs->destroy(priv->encoders[i]);
 628         priv->num_encoders = 0;
 629 }
 630
 631 static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms)
 632 {
 633         struct drm_device *dev;
 634         struct drm_plane *primary_planes[MAX_PLANES], *plane;
 635         struct drm_plane *cursor_planes[MAX_PLANES] = { NULL };
 636         struct drm_crtc *crtc;
 637
 638         struct msm_drm_private *priv;
 639         struct dpu_mdss_cfg *catalog;
 640
 641         int primary_planes_idx = 0, cursor_planes_idx = 0, i, ret;
 642         int max_crtc_count;
 643         dev = dpu_kms->dev;
 644         priv = dev->dev_private;
 645         catalog = dpu_kms->catalog;
 646
 647         /*
 648          * Create encoder and query display drivers to create
 649          * bridges and connectors
 650          */
 651         ret = _dpu_kms_setup_displays(dev, priv, dpu_kms);
 652         if (ret)
 653                 goto fail;
 654
 655         max_crtc_count = min(catalog->mixer_count, priv->num_encoders);
 656
 657         /* Create the planes, keeping track of one primary/cursor per crtc */
 658         for (i = 0; i < catalog->sspp_count; i++) {
 659                 enum drm_plane_type type;
 660
 661                 if ((catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR))
 662                         && cursor_planes_idx < max_crtc_count)
 663                         type = DRM_PLANE_TYPE_CURSOR;
 664                 else if (primary_planes_idx < max_crtc_count)
 665                         type = DRM_PLANE_TYPE_PRIMARY;
 666                 else
 667                         type = DRM_PLANE_TYPE_OVERLAY;
 668
 669                 DPU_DEBUG("Create plane type %d with features %lx (cur %lx)\n",
 670                           type, catalog->sspp[i].features,
 671                           catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR));
 672
 673                 plane = dpu_plane_init(dev, catalog->sspp[i].id, type,
 674                                        (1UL << max_crtc_count) - 1, 0);
 675                 if (IS_ERR(plane)) {
 676                         DPU_ERROR("dpu_plane_init failed\n");
 677                         ret = PTR_ERR(plane);
 678                         goto fail;
 679                 }
 680                 priv->planes[priv->num_planes++] = plane;
 681
 682                 if (type == DRM_PLANE_TYPE_CURSOR)
 683                         cursor_planes[cursor_planes_idx++] = plane;
 684                 else if (type == DRM_PLANE_TYPE_PRIMARY)
 685                         primary_planes[primary_planes_idx++] = plane;
 686         }
 687
 688         max_crtc_count = min(max_crtc_count, primary_planes_idx);
 689
 690         /* Create one CRTC per encoder */
 691         for (i = 0; i < max_crtc_count; i++) {
 692                 crtc = dpu_crtc_init(dev, primary_planes[i], cursor_planes[i]);
 693                 if (IS_ERR(crtc)) {
 694                         ret = PTR_ERR(crtc);
 695                         goto fail;
 696                 }
 697                 priv->crtcs[priv->num_crtcs++] = crtc;
 698         }
 699
 700         /* All CRTCs are compatible with all encoders */
 701         for (i = 0; i < priv->num_encoders; i++)
 702                 priv->encoders[i]->possible_crtcs = (1 << priv->num_crtcs) - 1;
 703
 704         return 0;
 705 fail:
 706         _dpu_kms_drm_obj_destroy(dpu_kms);
 707         return ret;
 708 }
 709
 710 static long dpu_kms_round_pixclk(struct msm_kms *kms, unsigned long rate,
 711                 struct drm_encoder *encoder)
 712 {
 713         return rate;
 714 }
 715
 716 static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms)
 717 {
 718         int i;
 719
 720         if (dpu_kms->hw_intr)
 721                 dpu_hw_intr_destroy(dpu_kms->hw_intr);
 722         dpu_kms->hw_intr = NULL;
 723
 724         /* safe to call these more than once during shutdown */
 725         _dpu_kms_mmu_destroy(dpu_kms);
 726
 727         if (dpu_kms->catalog) {
 728                 for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
 729                         u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
 730
 731                         if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx])
 732                                 dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]);
 733                 }
 734         }
 735
 736         if (dpu_kms->rm_init)
 737                 dpu_rm_destroy(&dpu_kms->rm);
 738         dpu_kms->rm_init = false;
 739
 740         if (dpu_kms->catalog)
 741                 dpu_hw_catalog_deinit(dpu_kms->catalog);
 742         dpu_kms->catalog = NULL;
 743
 744         if (dpu_kms->vbif[VBIF_NRT])
 745                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_NRT]);
 746         dpu_kms->vbif[VBIF_NRT] = NULL;
 747
 748         if (dpu_kms->vbif[VBIF_RT])
 749                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_RT]);
 750         dpu_kms->vbif[VBIF_RT] = NULL;
 751
 752         if (dpu_kms->hw_mdp)
 753                 dpu_hw_mdp_destroy(dpu_kms->hw_mdp);
 754         dpu_kms->hw_mdp = NULL;
 755
 756         if (dpu_kms->mmio)
 757                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->mmio);
 758         dpu_kms->mmio = NULL;
 759 }
 760
 761 static void dpu_kms_destroy(struct msm_kms *kms)
 762 {
 763         struct dpu_kms *dpu_kms;
 764
 765         if (!kms) {
 766                 DPU_ERROR("invalid kms\n");
 767                 return;
 768         }
 769
 770         dpu_kms = to_dpu_kms(kms);
 771
 772         _dpu_kms_hw_destroy(dpu_kms);
 773
 774         msm_kms_destroy(&dpu_kms->base);
 775 }
 776
 777 static irqreturn_t dpu_irq(struct msm_kms *kms)
 778 {
 779         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 780
 781         return dpu_core_irq(dpu_kms);
 782 }
 783
 784 static void dpu_irq_preinstall(struct msm_kms *kms)
 785 {
 786         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 787
 788         dpu_core_irq_preinstall(dpu_kms);
 789 }
 790
 791 static int dpu_irq_postinstall(struct msm_kms *kms)
 792 {
 793         struct msm_drm_private *priv;
 794         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 795
 796         if (!dpu_kms || !dpu_kms->dev)
 797                 return -EINVAL;
 798
 799         priv = dpu_kms->dev->dev_private;
 800         if (!priv)
 801                 return -EINVAL;
 802
 803         msm_dp_irq_postinstall(priv->dp);
 804
 805         return 0;
 806 }
 807
 808 static void dpu_irq_uninstall(struct msm_kms *kms)
 809 {
 810         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 811
 812         dpu_core_irq_uninstall(dpu_kms);
 813 }
 814
 815 static void dpu_kms_mdp_snapshot(struct msm_disp_state *disp_state, struct msm_kms *kms)
 816 {
 817         int i;
 818         struct dpu_kms *dpu_kms;
 819         struct dpu_mdss_cfg *cat;
 820         struct dpu_hw_mdp *top;
 821
 822         dpu_kms = to_dpu_kms(kms);
 823
 824         cat = dpu_kms->catalog;
 825         top = dpu_kms->hw_mdp;
 826
 827         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 828
 829         /* dump CTL sub-blocks HW regs info */
 830         for (i = 0; i < cat->ctl_count; i++)
 831                 msm_disp_snapshot_add_block(disp_state, cat->ctl[i].len,
 832                                 dpu_kms->mmio + cat->ctl[i].base, "ctl_%d", i);
 833
 834         /* dump DSPP sub-blocks HW regs info */
 835         for (i = 0; i < cat->dspp_count; i++)
 836                 msm_disp_snapshot_add_block(disp_state, cat->dspp[i].len,
 837                                 dpu_kms->mmio + cat->dspp[i].base, "dspp_%d", i);
 838
 839         /* dump INTF sub-blocks HW regs info */
 840         for (i = 0; i < cat->intf_count; i++)
 841                 msm_disp_snapshot_add_block(disp_state, cat->intf[i].len,
 842                                 dpu_kms->mmio + cat->intf[i].base, "intf_%d", i);
 843
 844         /* dump PP sub-blocks HW regs info */
 845         for (i = 0; i < cat->pingpong_count; i++)
 846                 msm_disp_snapshot_add_block(disp_state, cat->pingpong[i].len,
 847                                 dpu_kms->mmio + cat->pingpong[i].base, "pingpong_%d", i);
 848
 849         /* dump SSPP sub-blocks HW regs info */
 850         for (i = 0; i < cat->sspp_count; i++)
 851                 msm_disp_snapshot_add_block(disp_state, cat->sspp[i].len,
 852                                 dpu_kms->mmio + cat->sspp[i].base, "sspp_%d", i);
 853
 854         msm_disp_snapshot_add_block(disp_state, top->hw.length,
 855                         dpu_kms->mmio + top->hw.blk_off, "top");
 856
 857         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 858 }
 859
 860 static const struct msm_kms_funcs kms_funcs = {
 861         .hw_init         = dpu_kms_hw_init,
 862         .irq_preinstall  = dpu_irq_preinstall,
 863         .irq_postinstall = dpu_irq_postinstall,
 864         .irq_uninstall   = dpu_irq_uninstall,
 865         .irq             = dpu_irq,
 866         .enable_commit   = dpu_kms_enable_commit,
 867         .disable_commit  = dpu_kms_disable_commit,
 868         .vsync_time      = dpu_kms_vsync_time,
 869         .prepare_commit  = dpu_kms_prepare_commit,
 870         .flush_commit    = dpu_kms_flush_commit,
 871         .wait_flush      = dpu_kms_wait_flush,
 872         .complete_commit = dpu_kms_complete_commit,
 873         .enable_vblank   = dpu_kms_enable_vblank,
 874         .disable_vblank  = dpu_kms_disable_vblank,
 875         .check_modified_format = dpu_format_check_modified_format,
 876         .get_format      = dpu_get_msm_format,
 877         .round_pixclk    = dpu_kms_round_pixclk,
 878         .destroy         = dpu_kms_destroy,
 879         .snapshot        = dpu_kms_mdp_snapshot,
 880 #ifdef CONFIG_DEBUG_FS
 881         .debugfs_init    = dpu_kms_debugfs_init,
 882 #endif
 883 };
 884
 885 static void _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms)
 886 {
 887         struct msm_mmu *mmu;
 888
 889         if (!dpu_kms->base.aspace)
 890                 return;
 891
 892         mmu = dpu_kms->base.aspace->mmu;
 893
 894         mmu->funcs->detach(mmu);
 895         msm_gem_address_space_put(dpu_kms->base.aspace);
 896
 897         dpu_kms->base.aspace = NULL;
 898 }
 899
 900 static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms)
 901 {
 902         struct iommu_domain *domain;
 903         struct msm_gem_address_space *aspace;
 904         struct msm_mmu *mmu;
 905
 906         domain = iommu_domain_alloc(&platform_bus_type);
 907         if (!domain)
 908                 return 0;
 909
 910         mmu = msm_iommu_new(dpu_kms->dev->dev, domain);
 911         aspace = msm_gem_address_space_create(mmu, "dpu1",
 912                 0x1000, 0x100000000 - 0x1000);
 913
 914         if (IS_ERR(aspace)) {
 915                 mmu->funcs->destroy(mmu);
 916                 return PTR_ERR(aspace);
 917         }
 918
 919         dpu_kms->base.aspace = aspace;
 920         return 0;
 921 }
 922
 923 static struct dss_clk *_dpu_kms_get_clk(struct dpu_kms *dpu_kms,
 924                 char *clock_name)
 925 {
 926         struct dss_module_power *mp = &dpu_kms->mp;
 927         int i;
 928
 929         for (i = 0; i < mp->num_clk; i++) {
 930                 if (!strcmp(mp->clk_config[i].clk_name, clock_name))
 931                         return &mp->clk_config[i];
 932         }
 933
 934         return NULL;
 935 }
 936
 937 u64 dpu_kms_get_clk_rate(struct dpu_kms *dpu_kms, char *clock_name)
 938 {
 939         struct dss_clk *clk;
 940
 941         clk = _dpu_kms_get_clk(dpu_kms, clock_name);
 942         if (!clk)
 943                 return -EINVAL;
 944
 945         return clk_get_rate(clk->clk);
 946 }
 947
 948 static int dpu_kms_hw_init(struct msm_kms *kms)
 949 {
 950         struct dpu_kms *dpu_kms;
 951         struct drm_device *dev;
 952         int i, rc = -EINVAL;
 953
 954         if (!kms) {
 955                 DPU_ERROR("invalid kms\n");
 956                 return rc;
 957         }
 958
 959         dpu_kms = to_dpu_kms(kms);
 960         dev = dpu_kms->dev;
 961
 962         rc = dpu_kms_global_obj_init(dpu_kms);
 963         if (rc)
 964                 return rc;
 965
 966         atomic_set(&dpu_kms->bandwidth_ref, 0);
 967
 968         dpu_kms->mmio = msm_ioremap(dpu_kms->pdev, "mdp", "mdp");
 969         if (IS_ERR(dpu_kms->mmio)) {
 970                 rc = PTR_ERR(dpu_kms->mmio);
 971                 DPU_ERROR("mdp register memory map failed: %d\n", rc);
 972                 dpu_kms->mmio = NULL;
 973                 goto error;
 974         }
 975         DRM_DEBUG("mapped dpu address space @%pK\n", dpu_kms->mmio);
 976
 977         dpu_kms->vbif[VBIF_RT] = msm_ioremap(dpu_kms->pdev, "vbif", "vbif");
 978         if (IS_ERR(dpu_kms->vbif[VBIF_RT])) {
 979                 rc = PTR_ERR(dpu_kms->vbif[VBIF_RT]);
 980                 DPU_ERROR("vbif register memory map failed: %d\n", rc);
 981                 dpu_kms->vbif[VBIF_RT] = NULL;
 982                 goto error;
 983         }
 984         dpu_kms->vbif[VBIF_NRT] = msm_ioremap_quiet(dpu_kms->pdev, "vbif_nrt", "vbif_nrt");
 985         if (IS_ERR(dpu_kms->vbif[VBIF_NRT])) {
 986                 dpu_kms->vbif[VBIF_NRT] = NULL;
 987                 DPU_DEBUG("VBIF NRT is not defined");
 988         }
 989
 990         dpu_kms->reg_dma = msm_ioremap_quiet(dpu_kms->pdev, "regdma", "regdma");
 991         if (IS_ERR(dpu_kms->reg_dma)) {
 992                 dpu_kms->reg_dma = NULL;
 993                 DPU_DEBUG("REG_DMA is not defined");
 994         }
 995
 996         dpu_kms_parse_data_bus_icc_path(dpu_kms);
 997
 998         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 999
1000         dpu_kms->core_rev = readl_relaxed(dpu_kms->mmio + 0x0);
1001
1002         pr_info("dpu hardware revision:0x%x\n", dpu_kms->core_rev);
1003
1004         dpu_kms->catalog = dpu_hw_catalog_init(dpu_kms->core_rev);
1005         if (IS_ERR_OR_NULL(dpu_kms->catalog)) {
1006                 rc = PTR_ERR(dpu_kms->catalog);
1007                 if (!dpu_kms->catalog)
1008                         rc = -EINVAL;
1009                 DPU_ERROR("catalog init failed: %d\n", rc);
1010                 dpu_kms->catalog = NULL;
1011                 goto power_error;
1012         }
1013
1014         /*
1015          * Now we need to read the HW catalog and initialize resources such as
1016          * clocks, regulators, GDSC/MMAGIC, ioremap the register ranges etc
1017          */
1018         rc = _dpu_kms_mmu_init(dpu_kms);
1019         if (rc) {
1020                 DPU_ERROR("dpu_kms_mmu_init failed: %d\n", rc);
1021                 goto power_error;
1022         }
1023
1024         rc = dpu_rm_init(&dpu_kms->rm, dpu_kms->catalog, dpu_kms->mmio);
1025         if (rc) {
1026                 DPU_ERROR("rm init failed: %d\n", rc);
1027                 goto power_error;
1028         }
1029
1030         dpu_kms->rm_init = true;
1031
1032         dpu_kms->hw_mdp = dpu_hw_mdptop_init(MDP_TOP, dpu_kms->mmio,
1033                                              dpu_kms->catalog);
1034         if (IS_ERR(dpu_kms->hw_mdp)) {
1035                 rc = PTR_ERR(dpu_kms->hw_mdp);
1036                 DPU_ERROR("failed to get hw_mdp: %d\n", rc);
1037                 dpu_kms->hw_mdp = NULL;
1038                 goto power_error;
1039         }
1040
1041         for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
1042                 u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
1043
1044                 dpu_kms->hw_vbif[i] = dpu_hw_vbif_init(vbif_idx,
1045                                 dpu_kms->vbif[vbif_idx], dpu_kms->catalog);
1046                 if (IS_ERR_OR_NULL(dpu_kms->hw_vbif[vbif_idx])) {
1047                         rc = PTR_ERR(dpu_kms->hw_vbif[vbif_idx]);
1048                         if (!dpu_kms->hw_vbif[vbif_idx])
1049                                 rc = -EINVAL;
1050                         DPU_ERROR("failed to init vbif %d: %d\n", vbif_idx, rc);
1051                         dpu_kms->hw_vbif[vbif_idx] = NULL;
1052                         goto power_error;
1053                 }
1054         }
1055
1056         rc = dpu_core_perf_init(&dpu_kms->perf, dev, dpu_kms->catalog,
1057                         _dpu_kms_get_clk(dpu_kms, "core"));
1058         if (rc) {
1059                 DPU_ERROR("failed to init perf %d\n", rc);
1060                 goto perf_err;
1061         }
1062
1063         dpu_kms->hw_intr = dpu_hw_intr_init(dpu_kms->mmio, dpu_kms->catalog);
1064         if (IS_ERR_OR_NULL(dpu_kms->hw_intr)) {
1065                 rc = PTR_ERR(dpu_kms->hw_intr);
1066                 DPU_ERROR("hw_intr init failed: %d\n", rc);
1067                 dpu_kms->hw_intr = NULL;
1068                 goto hw_intr_init_err;
1069         }
1070
1071         dev->mode_config.min_width = 0;
1072         dev->mode_config.min_height = 0;
1073
1074         /*
1075          * max crtc width is equal to the max mixer width * 2 and max height is
1076          * is 4K
1077          */
1078         dev->mode_config.max_width =
1079                         dpu_kms->catalog->caps->max_mixer_width * 2;
1080         dev->mode_config.max_height = 4096;
1081
1082         dev->max_vblank_count = 0xffffffff;
1083         /* Disable vblank irqs aggressively for power-saving */
1084         dev->vblank_disable_immediate = true;
1085
1086         /*
1087          * _dpu_kms_drm_obj_init should create the DRM related objects
1088          * i.e. CRTCs, planes, encoders, connectors and so forth
1089          */
1090         rc = _dpu_kms_drm_obj_init(dpu_kms);
1091         if (rc) {
1092                 DPU_ERROR("modeset init failed: %d\n", rc);
1093                 goto drm_obj_init_err;
1094         }
1095
1096         dpu_vbif_init_memtypes(dpu_kms);
1097
1098         pm_runtime_put_sync(&dpu_kms->pdev->dev);
1099
1100         return 0;
1101
1102 drm_obj_init_err:
1103         dpu_core_perf_destroy(&dpu_kms->perf);
1104 hw_intr_init_err:
1105 perf_err:
1106 power_error:
1107         pm_runtime_put_sync(&dpu_kms->pdev->dev);
1108 error:
1109         _dpu_kms_hw_destroy(dpu_kms);
1110
1111         return rc;
1112 }
1113
1114 struct msm_kms *dpu_kms_init(struct drm_device *dev)
1115 {
1116         struct msm_drm_private *priv;
1117         struct dpu_kms *dpu_kms;
1118         int irq;
1119
1120         if (!dev) {
1121                 DPU_ERROR("drm device node invalid\n");
1122                 return ERR_PTR(-EINVAL);
1123         }
1124
1125         priv = dev->dev_private;
1126         dpu_kms = to_dpu_kms(priv->kms);
1127
1128         irq = irq_of_parse_and_map(dpu_kms->pdev->dev.of_node, 0);
1129         if (irq < 0) {
1130                 DPU_ERROR("failed to get irq: %d\n", irq);
1131                 return ERR_PTR(irq);
1132         }
1133         dpu_kms->base.irq = irq;
1134
1135         return &dpu_kms->base;
1136 }
1137
1138 static int dpu_bind(struct device *dev, struct device *master, void *data)
1139 {
1140         struct drm_device *ddev = dev_get_drvdata(master);
1141         struct platform_device *pdev = to_platform_device(dev);
1142         struct msm_drm_private *priv = ddev->dev_private;
1143         struct dpu_kms *dpu_kms;
1144         struct dss_module_power *mp;
1145         int ret = 0;
1146
1147         dpu_kms = devm_kzalloc(&pdev->dev, sizeof(*dpu_kms), GFP_KERNEL);
1148         if (!dpu_kms)
1149                 return -ENOMEM;
1150
1151         ret = devm_pm_opp_set_clkname(dev, "core");
1152         if (ret)
1153                 return ret;
1154         /* OPP table is optional */
1155         ret = devm_pm_opp_of_add_table(dev);
1156         if (ret && ret != -ENODEV) {
1157                 dev_err(dev, "invalid OPP table in device tree\n");
1158                 return ret;
1159         }
1160
1161         mp = &dpu_kms->mp;
1162         ret = msm_dss_parse_clock(pdev, mp);
1163         if (ret) {
1164                 DPU_ERROR("failed to parse clocks, ret=%d\n", ret);
1165                 return ret;
1166         }
1167
1168         platform_set_drvdata(pdev, dpu_kms);
1169
1170         ret = msm_kms_init(&dpu_kms->base, &kms_funcs);
1171         if (ret) {
1172                 DPU_ERROR("failed to init kms, ret=%d\n", ret);
1173                 return ret;
1174         }
1175         dpu_kms->dev = ddev;
1176         dpu_kms->pdev = pdev;
1177
1178         pm_runtime_enable(&pdev->dev);
1179         dpu_kms->rpm_enabled = true;
1180
1181         priv->kms = &dpu_kms->base;
1182
1183         return ret;
1184 }
1185
1186 static void dpu_unbind(struct device *dev, struct device *master, void *data)
1187 {
1188         struct platform_device *pdev = to_platform_device(dev);
1189         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1190         struct dss_module_power *mp = &dpu_kms->mp;
1191
1192         msm_dss_put_clk(mp->clk_config, mp->num_clk);
1193         devm_kfree(&pdev->dev, mp->clk_config);
1194         mp->num_clk = 0;
1195
1196         if (dpu_kms->rpm_enabled)
1197                 pm_runtime_disable(&pdev->dev);
1198 }
1199
1200 static const struct component_ops dpu_ops = {
1201         .bind   = dpu_bind,
1202         .unbind = dpu_unbind,
1203 };
1204
1205 static int dpu_dev_probe(struct platform_device *pdev)
1206 {
1207         return component_add(&pdev->dev, &dpu_ops);
1208 }
1209
1210 static int dpu_dev_remove(struct platform_device *pdev)
1211 {
1212         component_del(&pdev->dev, &dpu_ops);
1213         return 0;
1214 }
1215
1216 static int __maybe_unused dpu_runtime_suspend(struct device *dev)
1217 {
1218         int i, rc = -1;
1219         struct platform_device *pdev = to_platform_device(dev);
1220         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1221         struct dss_module_power *mp = &dpu_kms->mp;
1222
1223         /* Drop the performance state vote */
1224         dev_pm_opp_set_rate(dev, 0);
1225         rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, false);
1226         if (rc)
1227                 DPU_ERROR("clock disable failed rc:%d\n", rc);
1228
1229         for (i = 0; i < dpu_kms->num_paths; i++)
1230                 icc_set_bw(dpu_kms->path[i], 0, 0);
1231
1232         return rc;
1233 }
1234
1235 static int __maybe_unused dpu_runtime_resume(struct device *dev)
1236 {
1237         int rc = -1;
1238         struct platform_device *pdev = to_platform_device(dev);
1239         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1240         struct drm_encoder *encoder;
1241         struct drm_device *ddev;
1242         struct dss_module_power *mp = &dpu_kms->mp;
1243         int i;
1244
1245         ddev = dpu_kms->dev;
1246
1247         WARN_ON(!(dpu_kms->num_paths));
1248         /* Min vote of BW is required before turning on AXI clk */
1249         for (i = 0; i < dpu_kms->num_paths; i++)
1250                 icc_set_bw(dpu_kms->path[i], 0, Bps_to_icc(MIN_IB_BW));
1251
1252         rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, true);
1253         if (rc) {
1254                 DPU_ERROR("clock enable failed rc:%d\n", rc);
1255                 return rc;
1256         }
1257
1258         dpu_vbif_init_memtypes(dpu_kms);
1259
1260         drm_for_each_encoder(encoder, ddev)
1261                 dpu_encoder_virt_runtime_resume(encoder);
1262
1263         return rc;
1264 }
1265
1266 static const struct dev_pm_ops dpu_pm_ops = {
1267         SET_RUNTIME_PM_OPS(dpu_runtime_suspend, dpu_runtime_resume, NULL)
1268         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1269                                 pm_runtime_force_resume)
1270 };
1271
1272 static const struct of_device_id dpu_dt_match[] = {
1273         { .compatible = "qcom,sdm845-dpu", },
1274         { .compatible = "qcom,sc7180-dpu", },
1275         { .compatible = "qcom,sc7280-dpu", },
1276         { .compatible = "qcom,sm8150-dpu", },
1277         { .compatible = "qcom,sm8250-dpu", },
1278         {}
1279 };
1280 MODULE_DEVICE_TABLE(of, dpu_dt_match);
1281
1282 static struct platform_driver dpu_driver = {
1283         .probe = dpu_dev_probe,
1284         .remove = dpu_dev_remove,
1285         .driver = {
1286                 .name = "msm_dpu",
1287                 .of_match_table = dpu_dt_match,
1288                 .pm = &dpu_pm_ops,
1289         },
1290 };
1291
1292 void __init msm_dpu_register(void)
1293 {
1294         platform_driver_register(&dpu_driver);
1295 }
1296
1297 void __exit msm_dpu_unregister(void)
1298 {
1299         platform_driver_unregister(&dpu_driver);
1300 }