drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
   4  * Copyright (C) 2013 Red Hat
   5  * Author: Rob Clark <robdclark@gmail.com>
   6  */
   7
   8 #define pr_fmt(fmt)     "[drm:%s:%d] " fmt, __func__, __LINE__
   9
  10 #include <linux/debugfs.h>
  11 #include <linux/dma-buf.h>
  12 #include <linux/of_irq.h>
  13 #include <linux/pm_opp.h>
  14
  15 #include <drm/drm_crtc.h>
  16 #include <drm/drm_file.h>
  17 #include <drm/drm_vblank.h>
  18
  19 #include "msm_drv.h"
  20 #include "msm_mmu.h"
  21 #include "msm_gem.h"
  22 #include "disp/msm_disp_snapshot.h"
  23
  24 #include "dpu_kms.h"
  25 #include "dpu_core_irq.h"
  26 #include "dpu_formats.h"
  27 #include "dpu_hw_vbif.h"
  28 #include "dpu_vbif.h"
  29 #include "dpu_encoder.h"
  30 #include "dpu_plane.h"
  31 #include "dpu_crtc.h"
  32
  33 #define CREATE_TRACE_POINTS
  34 #include "dpu_trace.h"
  35
  36 /*
  37  * To enable overall DRM driver logging
  38  * # echo 0x2 > /sys/module/drm/parameters/debug
  39  *
  40  * To enable DRM driver h/w logging
  41  * # echo <mask> > /sys/kernel/debug/dri/0/debug/hw_log_mask
  42  *
  43  * See dpu_hw_mdss.h for h/w logging mask definitions (search for DPU_DBG_MASK_)
  44  */
  45 #define DPU_DEBUGFS_DIR "msm_dpu"
  46 #define DPU_DEBUGFS_HWMASKNAME "hw_log_mask"
  47
  48 #define MIN_IB_BW       400000000ULL /* Min ib vote 400MB */
  49
  50 static int dpu_kms_hw_init(struct msm_kms *kms);
  51 static void _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms);
  52
  53 #ifdef CONFIG_DEBUG_FS
  54 static int _dpu_danger_signal_status(struct seq_file *s,
  55                 bool danger_status)
  56 {
  57         struct dpu_kms *kms = (struct dpu_kms *)s->private;
  58         struct dpu_danger_safe_status status;
  59         int i;
  60
  61         if (!kms->hw_mdp) {
  62                 DPU_ERROR("invalid arg(s)\n");
  63                 return 0;
  64         }
  65
  66         memset(&status, 0, sizeof(struct dpu_danger_safe_status));
  67
  68         pm_runtime_get_sync(&kms->pdev->dev);
  69         if (danger_status) {
  70                 seq_puts(s, "\nDanger signal status:\n");
  71                 if (kms->hw_mdp->ops.get_danger_status)
  72                         kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
  73                                         &status);
  74         } else {
  75                 seq_puts(s, "\nSafe signal status:\n");
  76                 if (kms->hw_mdp->ops.get_danger_status)
  77                         kms->hw_mdp->ops.get_danger_status(kms->hw_mdp,
  78                                         &status);
  79         }
  80         pm_runtime_put_sync(&kms->pdev->dev);
  81
  82         seq_printf(s, "MDP     :  0x%x\n", status.mdp);
  83
  84         for (i = SSPP_VIG0; i < SSPP_MAX; i++)
  85                 seq_printf(s, "SSPP%d   :  0x%x  \t", i - SSPP_VIG0,
  86                                 status.sspp[i]);
  87         seq_puts(s, "\n");
  88
  89         return 0;
  90 }
  91
  92 static int dpu_debugfs_danger_stats_show(struct seq_file *s, void *v)
  93 {
  94         return _dpu_danger_signal_status(s, true);
  95 }
  96 DEFINE_SHOW_ATTRIBUTE(dpu_debugfs_danger_stats);
  97
  98 static int dpu_debugfs_safe_stats_show(struct seq_file *s, void *v)
  99 {
 100         return _dpu_danger_signal_status(s, false);
 101 }
 102 DEFINE_SHOW_ATTRIBUTE(dpu_debugfs_safe_stats);
 103
 104 static void dpu_debugfs_danger_init(struct dpu_kms *dpu_kms,
 105                 struct dentry *parent)
 106 {
 107         struct dentry *entry = debugfs_create_dir("danger", parent);
 108
 109         debugfs_create_file("danger_status", 0600, entry,
 110                         dpu_kms, &dpu_debugfs_danger_stats_fops);
 111         debugfs_create_file("safe_status", 0600, entry,
 112                         dpu_kms, &dpu_debugfs_safe_stats_fops);
 113 }
 114
 115 static int _dpu_debugfs_show_regset32(struct seq_file *s, void *data)
 116 {
 117         struct dpu_debugfs_regset32 *regset = s->private;
 118         struct dpu_kms *dpu_kms = regset->dpu_kms;
 119         void __iomem *base;
 120         uint32_t i, addr;
 121
 122         if (!dpu_kms->mmio)
 123                 return 0;
 124
 125         base = dpu_kms->mmio + regset->offset;
 126
 127         /* insert padding spaces, if needed */
 128         if (regset->offset & 0xF) {
 129                 seq_printf(s, "[%x]", regset->offset & ~0xF);
 130                 for (i = 0; i < (regset->offset & 0xF); i += 4)
 131                         seq_puts(s, "         ");
 132         }
 133
 134         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 135
 136         /* main register output */
 137         for (i = 0; i < regset->blk_len; i += 4) {
 138                 addr = regset->offset + i;
 139                 if ((addr & 0xF) == 0x0)
 140                         seq_printf(s, i ? "\n[%x]" : "[%x]", addr);
 141                 seq_printf(s, " %08x", readl_relaxed(base + i));
 142         }
 143         seq_puts(s, "\n");
 144         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 145
 146         return 0;
 147 }
 148
 149 static int dpu_debugfs_open_regset32(struct inode *inode,
 150                 struct file *file)
 151 {
 152         return single_open(file, _dpu_debugfs_show_regset32, inode->i_private);
 153 }
 154
 155 static const struct file_operations dpu_fops_regset32 = {
 156         .open =         dpu_debugfs_open_regset32,
 157         .read =         seq_read,
 158         .llseek =       seq_lseek,
 159         .release =      single_release,
 160 };
 161
 162 void dpu_debugfs_setup_regset32(struct dpu_debugfs_regset32 *regset,
 163                 uint32_t offset, uint32_t length, struct dpu_kms *dpu_kms)
 164 {
 165         if (regset) {
 166                 regset->offset = offset;
 167                 regset->blk_len = length;
 168                 regset->dpu_kms = dpu_kms;
 169         }
 170 }
 171
 172 void dpu_debugfs_create_regset32(const char *name, umode_t mode,
 173                 void *parent, struct dpu_debugfs_regset32 *regset)
 174 {
 175         if (!name || !regset || !regset->dpu_kms || !regset->blk_len)
 176                 return;
 177
 178         /* make sure offset is a multiple of 4 */
 179         regset->offset = round_down(regset->offset, 4);
 180
 181         debugfs_create_file(name, mode, parent, regset, &dpu_fops_regset32);
 182 }
 183
 184 static int dpu_kms_debugfs_init(struct msm_kms *kms, struct drm_minor *minor)
 185 {
 186         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 187         void *p = dpu_hw_util_get_log_mask_ptr();
 188         struct dentry *entry;
 189         struct drm_device *dev;
 190         struct msm_drm_private *priv;
 191
 192         if (!p)
 193                 return -EINVAL;
 194
 195         dev = dpu_kms->dev;
 196         priv = dev->dev_private;
 197
 198         entry = debugfs_create_dir("debug", minor->debugfs_root);
 199
 200         debugfs_create_x32(DPU_DEBUGFS_HWMASKNAME, 0600, entry, p);
 201
 202         dpu_debugfs_danger_init(dpu_kms, entry);
 203         dpu_debugfs_vbif_init(dpu_kms, entry);
 204         dpu_debugfs_core_irq_init(dpu_kms, entry);
 205
 206         if (priv->dp)
 207                 msm_dp_debugfs_init(priv->dp, minor);
 208
 209         return dpu_core_perf_debugfs_init(dpu_kms, entry);
 210 }
 211 #endif
 212
 213 /* Global/shared object state funcs */
 214
 215 /*
 216  * This is a helper that returns the private state currently in operation.
 217  * Note that this would return the "old_state" if called in the atomic check
 218  * path, and the "new_state" after the atomic swap has been done.
 219  */
 220 struct dpu_global_state *
 221 dpu_kms_get_existing_global_state(struct dpu_kms *dpu_kms)
 222 {
 223         return to_dpu_global_state(dpu_kms->global_state.state);
 224 }
 225
 226 /*
 227  * This acquires the modeset lock set aside for global state, creates
 228  * a new duplicated private object state.
 229  */
 230 struct dpu_global_state *dpu_kms_get_global_state(struct drm_atomic_state *s)
 231 {
 232         struct msm_drm_private *priv = s->dev->dev_private;
 233         struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
 234         struct drm_private_state *priv_state;
 235         int ret;
 236
 237         ret = drm_modeset_lock(&dpu_kms->global_state_lock, s->acquire_ctx);
 238         if (ret)
 239                 return ERR_PTR(ret);
 240
 241         priv_state = drm_atomic_get_private_obj_state(s,
 242                                                 &dpu_kms->global_state);
 243         if (IS_ERR(priv_state))
 244                 return ERR_CAST(priv_state);
 245
 246         return to_dpu_global_state(priv_state);
 247 }
 248
 249 static struct drm_private_state *
 250 dpu_kms_global_duplicate_state(struct drm_private_obj *obj)
 251 {
 252         struct dpu_global_state *state;
 253
 254         state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
 255         if (!state)
 256                 return NULL;
 257
 258         __drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 259
 260         return &state->base;
 261 }
 262
 263 static void dpu_kms_global_destroy_state(struct drm_private_obj *obj,
 264                                       struct drm_private_state *state)
 265 {
 266         struct dpu_global_state *dpu_state = to_dpu_global_state(state);
 267
 268         kfree(dpu_state);
 269 }
 270
 271 static const struct drm_private_state_funcs dpu_kms_global_state_funcs = {
 272         .atomic_duplicate_state = dpu_kms_global_duplicate_state,
 273         .atomic_destroy_state = dpu_kms_global_destroy_state,
 274 };
 275
 276 static int dpu_kms_global_obj_init(struct dpu_kms *dpu_kms)
 277 {
 278         struct dpu_global_state *state;
 279
 280         drm_modeset_lock_init(&dpu_kms->global_state_lock);
 281
 282         state = kzalloc(sizeof(*state), GFP_KERNEL);
 283         if (!state)
 284                 return -ENOMEM;
 285
 286         drm_atomic_private_obj_init(dpu_kms->dev, &dpu_kms->global_state,
 287                                     &state->base,
 288                                     &dpu_kms_global_state_funcs);
 289         return 0;
 290 }
 291
 292 static int dpu_kms_parse_data_bus_icc_path(struct dpu_kms *dpu_kms)
 293 {
 294         struct icc_path *path0;
 295         struct icc_path *path1;
 296         struct drm_device *dev = dpu_kms->dev;
 297
 298         path0 = of_icc_get(dev->dev, "mdp0-mem");
 299         path1 = of_icc_get(dev->dev, "mdp1-mem");
 300
 301         if (IS_ERR_OR_NULL(path0))
 302                 return PTR_ERR_OR_ZERO(path0);
 303
 304         dpu_kms->path[0] = path0;
 305         dpu_kms->num_paths = 1;
 306
 307         if (!IS_ERR_OR_NULL(path1)) {
 308                 dpu_kms->path[1] = path1;
 309                 dpu_kms->num_paths++;
 310         }
 311         return 0;
 312 }
 313
 314 static int dpu_kms_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 315 {
 316         return dpu_crtc_vblank(crtc, true);
 317 }
 318
 319 static void dpu_kms_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 320 {
 321         dpu_crtc_vblank(crtc, false);
 322 }
 323
 324 static void dpu_kms_enable_commit(struct msm_kms *kms)
 325 {
 326         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 327         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 328 }
 329
 330 static void dpu_kms_disable_commit(struct msm_kms *kms)
 331 {
 332         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 333         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 334 }
 335
 336 static ktime_t dpu_kms_vsync_time(struct msm_kms *kms, struct drm_crtc *crtc)
 337 {
 338         struct drm_encoder *encoder;
 339
 340         drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
 341                 ktime_t vsync_time;
 342
 343                 if (dpu_encoder_vsync_time(encoder, &vsync_time) == 0)
 344                         return vsync_time;
 345         }
 346
 347         return ktime_get();
 348 }
 349
 350 static void dpu_kms_prepare_commit(struct msm_kms *kms,
 351                 struct drm_atomic_state *state)
 352 {
 353         struct drm_crtc *crtc;
 354         struct drm_crtc_state *crtc_state;
 355         struct drm_encoder *encoder;
 356         int i;
 357
 358         if (!kms)
 359                 return;
 360
 361         /* Call prepare_commit for all affected encoders */
 362         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 363                 drm_for_each_encoder_mask(encoder, crtc->dev,
 364                                           crtc_state->encoder_mask) {
 365                         dpu_encoder_prepare_commit(encoder);
 366                 }
 367         }
 368 }
 369
 370 static void dpu_kms_flush_commit(struct msm_kms *kms, unsigned crtc_mask)
 371 {
 372         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 373         struct drm_crtc *crtc;
 374
 375         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask) {
 376                 if (!crtc->state->active)
 377                         continue;
 378
 379                 trace_dpu_kms_commit(DRMID(crtc));
 380                 dpu_crtc_commit_kickoff(crtc);
 381         }
 382 }
 383
 384 /*
 385  * Override the encoder enable since we need to setup the inline rotator and do
 386  * some crtc magic before enabling any bridge that might be present.
 387  */
 388 void dpu_kms_encoder_enable(struct drm_encoder *encoder)
 389 {
 390         const struct drm_encoder_helper_funcs *funcs = encoder->helper_private;
 391         struct drm_device *dev = encoder->dev;
 392         struct drm_crtc *crtc;
 393
 394         /* Forward this enable call to the commit hook */
 395         if (funcs && funcs->commit)
 396                 funcs->commit(encoder);
 397
 398         drm_for_each_crtc(crtc, dev) {
 399                 if (!(crtc->state->encoder_mask & drm_encoder_mask(encoder)))
 400                         continue;
 401
 402                 trace_dpu_kms_enc_enable(DRMID(crtc));
 403         }
 404 }
 405
 406 static void dpu_kms_complete_commit(struct msm_kms *kms, unsigned crtc_mask)
 407 {
 408         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 409         struct drm_crtc *crtc;
 410
 411         DPU_ATRACE_BEGIN("kms_complete_commit");
 412
 413         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask)
 414                 dpu_crtc_complete_commit(crtc);
 415
 416         DPU_ATRACE_END("kms_complete_commit");
 417 }
 418
 419 static void dpu_kms_wait_for_commit_done(struct msm_kms *kms,
 420                 struct drm_crtc *crtc)
 421 {
 422         struct drm_encoder *encoder;
 423         struct drm_device *dev;
 424         int ret;
 425
 426         if (!kms || !crtc || !crtc->state) {
 427                 DPU_ERROR("invalid params\n");
 428                 return;
 429         }
 430
 431         dev = crtc->dev;
 432
 433         if (!crtc->state->enable) {
 434                 DPU_DEBUG("[crtc:%d] not enable\n", crtc->base.id);
 435                 return;
 436         }
 437
 438         if (!crtc->state->active) {
 439                 DPU_DEBUG("[crtc:%d] not active\n", crtc->base.id);
 440                 return;
 441         }
 442
 443         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 444                 if (encoder->crtc != crtc)
 445                         continue;
 446                 /*
 447                  * Wait for post-flush if necessary to delay before
 448                  * plane_cleanup. For example, wait for vsync in case of video
 449                  * mode panels. This may be a no-op for command mode panels.
 450                  */
 451                 trace_dpu_kms_wait_for_commit_done(DRMID(crtc));
 452                 ret = dpu_encoder_wait_for_event(encoder, MSM_ENC_COMMIT_DONE);
 453                 if (ret && ret != -EWOULDBLOCK) {
 454                         DPU_ERROR("wait for commit done returned %d\n", ret);
 455                         break;
 456                 }
 457         }
 458 }
 459
 460 static void dpu_kms_wait_flush(struct msm_kms *kms, unsigned crtc_mask)
 461 {
 462         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 463         struct drm_crtc *crtc;
 464
 465         for_each_crtc_mask(dpu_kms->dev, crtc, crtc_mask)
 466                 dpu_kms_wait_for_commit_done(kms, crtc);
 467 }
 468
 469 static int _dpu_kms_initialize_dsi(struct drm_device *dev,
 470                                     struct msm_drm_private *priv,
 471                                     struct dpu_kms *dpu_kms)
 472 {
 473         struct drm_encoder *encoder = NULL;
 474         int i, rc = 0;
 475
 476         if (!(priv->dsi[0] || priv->dsi[1]))
 477                 return rc;
 478
 479         /*TODO: Support two independent DSI connectors */
 480         encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_DSI);
 481         if (IS_ERR(encoder)) {
 482                 DPU_ERROR("encoder init failed for dsi display\n");
 483                 return PTR_ERR(encoder);
 484         }
 485
 486         priv->encoders[priv->num_encoders++] = encoder;
 487
 488         for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) {
 489                 if (!priv->dsi[i])
 490                         continue;
 491
 492                 rc = msm_dsi_modeset_init(priv->dsi[i], dev, encoder);
 493                 if (rc) {
 494                         DPU_ERROR("modeset_init failed for dsi[%d], rc = %d\n",
 495                                 i, rc);
 496                         break;
 497                 }
 498         }
 499
 500         return rc;
 501 }
 502
 503 static int _dpu_kms_initialize_displayport(struct drm_device *dev,
 504                                             struct msm_drm_private *priv,
 505                                             struct dpu_kms *dpu_kms)
 506 {
 507         struct drm_encoder *encoder = NULL;
 508         int rc = 0;
 509
 510         if (!priv->dp)
 511                 return rc;
 512
 513         encoder = dpu_encoder_init(dev, DRM_MODE_ENCODER_TMDS);
 514         if (IS_ERR(encoder)) {
 515                 DPU_ERROR("encoder init failed for dsi display\n");
 516                 return PTR_ERR(encoder);
 517         }
 518
 519         rc = msm_dp_modeset_init(priv->dp, dev, encoder);
 520         if (rc) {
 521                 DPU_ERROR("modeset_init failed for DP, rc = %d\n", rc);
 522                 drm_encoder_cleanup(encoder);
 523                 return rc;
 524         }
 525
 526         priv->encoders[priv->num_encoders++] = encoder;
 527         return rc;
 528 }
 529
 530 /**
 531  * _dpu_kms_setup_displays - create encoders, bridges and connectors
 532  *                           for underlying displays
 533  * @dev:        Pointer to drm device structure
 534  * @priv:       Pointer to private drm device data
 535  * @dpu_kms:    Pointer to dpu kms structure
 536  * Returns:     Zero on success
 537  */
 538 static int _dpu_kms_setup_displays(struct drm_device *dev,
 539                                     struct msm_drm_private *priv,
 540                                     struct dpu_kms *dpu_kms)
 541 {
 542         int rc = 0;
 543
 544         rc = _dpu_kms_initialize_dsi(dev, priv, dpu_kms);
 545         if (rc) {
 546                 DPU_ERROR("initialize_dsi failed, rc = %d\n", rc);
 547                 return rc;
 548         }
 549
 550         rc = _dpu_kms_initialize_displayport(dev, priv, dpu_kms);
 551         if (rc) {
 552                 DPU_ERROR("initialize_DP failed, rc = %d\n", rc);
 553                 return rc;
 554         }
 555
 556         return rc;
 557 }
 558
 559 static void _dpu_kms_drm_obj_destroy(struct dpu_kms *dpu_kms)
 560 {
 561         struct msm_drm_private *priv;
 562         int i;
 563
 564         priv = dpu_kms->dev->dev_private;
 565
 566         for (i = 0; i < priv->num_crtcs; i++)
 567                 priv->crtcs[i]->funcs->destroy(priv->crtcs[i]);
 568         priv->num_crtcs = 0;
 569
 570         for (i = 0; i < priv->num_planes; i++)
 571                 priv->planes[i]->funcs->destroy(priv->planes[i]);
 572         priv->num_planes = 0;
 573
 574         for (i = 0; i < priv->num_connectors; i++)
 575                 priv->connectors[i]->funcs->destroy(priv->connectors[i]);
 576         priv->num_connectors = 0;
 577
 578         for (i = 0; i < priv->num_encoders; i++)
 579                 priv->encoders[i]->funcs->destroy(priv->encoders[i]);
 580         priv->num_encoders = 0;
 581 }
 582
 583 static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms)
 584 {
 585         struct drm_device *dev;
 586         struct drm_plane *primary_planes[MAX_PLANES], *plane;
 587         struct drm_plane *cursor_planes[MAX_PLANES] = { NULL };
 588         struct drm_crtc *crtc;
 589
 590         struct msm_drm_private *priv;
 591         struct dpu_mdss_cfg *catalog;
 592
 593         int primary_planes_idx = 0, cursor_planes_idx = 0, i, ret;
 594         int max_crtc_count;
 595         dev = dpu_kms->dev;
 596         priv = dev->dev_private;
 597         catalog = dpu_kms->catalog;
 598
 599         /*
 600          * Create encoder and query display drivers to create
 601          * bridges and connectors
 602          */
 603         ret = _dpu_kms_setup_displays(dev, priv, dpu_kms);
 604         if (ret)
 605                 goto fail;
 606
 607         max_crtc_count = min(catalog->mixer_count, priv->num_encoders);
 608
 609         /* Create the planes, keeping track of one primary/cursor per crtc */
 610         for (i = 0; i < catalog->sspp_count; i++) {
 611                 enum drm_plane_type type;
 612
 613                 if ((catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR))
 614                         && cursor_planes_idx < max_crtc_count)
 615                         type = DRM_PLANE_TYPE_CURSOR;
 616                 else if (primary_planes_idx < max_crtc_count)
 617                         type = DRM_PLANE_TYPE_PRIMARY;
 618                 else
 619                         type = DRM_PLANE_TYPE_OVERLAY;
 620
 621                 DPU_DEBUG("Create plane type %d with features %lx (cur %lx)\n",
 622                           type, catalog->sspp[i].features,
 623                           catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR));
 624
 625                 plane = dpu_plane_init(dev, catalog->sspp[i].id, type,
 626                                        (1UL << max_crtc_count) - 1, 0);
 627                 if (IS_ERR(plane)) {
 628                         DPU_ERROR("dpu_plane_init failed\n");
 629                         ret = PTR_ERR(plane);
 630                         goto fail;
 631                 }
 632                 priv->planes[priv->num_planes++] = plane;
 633
 634                 if (type == DRM_PLANE_TYPE_CURSOR)
 635                         cursor_planes[cursor_planes_idx++] = plane;
 636                 else if (type == DRM_PLANE_TYPE_PRIMARY)
 637                         primary_planes[primary_planes_idx++] = plane;
 638         }
 639
 640         max_crtc_count = min(max_crtc_count, primary_planes_idx);
 641
 642         /* Create one CRTC per encoder */
 643         for (i = 0; i < max_crtc_count; i++) {
 644                 crtc = dpu_crtc_init(dev, primary_planes[i], cursor_planes[i]);
 645                 if (IS_ERR(crtc)) {
 646                         ret = PTR_ERR(crtc);
 647                         goto fail;
 648                 }
 649                 priv->crtcs[priv->num_crtcs++] = crtc;
 650         }
 651
 652         /* All CRTCs are compatible with all encoders */
 653         for (i = 0; i < priv->num_encoders; i++)
 654                 priv->encoders[i]->possible_crtcs = (1 << priv->num_crtcs) - 1;
 655
 656         return 0;
 657 fail:
 658         _dpu_kms_drm_obj_destroy(dpu_kms);
 659         return ret;
 660 }
 661
 662 static long dpu_kms_round_pixclk(struct msm_kms *kms, unsigned long rate,
 663                 struct drm_encoder *encoder)
 664 {
 665         return rate;
 666 }
 667
 668 static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms)
 669 {
 670         int i;
 671
 672         if (dpu_kms->hw_intr)
 673                 dpu_hw_intr_destroy(dpu_kms->hw_intr);
 674         dpu_kms->hw_intr = NULL;
 675
 676         /* safe to call these more than once during shutdown */
 677         _dpu_kms_mmu_destroy(dpu_kms);
 678
 679         if (dpu_kms->catalog) {
 680                 for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
 681                         u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
 682
 683                         if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx])
 684                                 dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]);
 685                 }
 686         }
 687
 688         if (dpu_kms->rm_init)
 689                 dpu_rm_destroy(&dpu_kms->rm);
 690         dpu_kms->rm_init = false;
 691
 692         if (dpu_kms->catalog)
 693                 dpu_hw_catalog_deinit(dpu_kms->catalog);
 694         dpu_kms->catalog = NULL;
 695
 696         if (dpu_kms->vbif[VBIF_NRT])
 697                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_NRT]);
 698         dpu_kms->vbif[VBIF_NRT] = NULL;
 699
 700         if (dpu_kms->vbif[VBIF_RT])
 701                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->vbif[VBIF_RT]);
 702         dpu_kms->vbif[VBIF_RT] = NULL;
 703
 704         if (dpu_kms->hw_mdp)
 705                 dpu_hw_mdp_destroy(dpu_kms->hw_mdp);
 706         dpu_kms->hw_mdp = NULL;
 707
 708         if (dpu_kms->mmio)
 709                 devm_iounmap(&dpu_kms->pdev->dev, dpu_kms->mmio);
 710         dpu_kms->mmio = NULL;
 711 }
 712
 713 static void dpu_kms_destroy(struct msm_kms *kms)
 714 {
 715         struct dpu_kms *dpu_kms;
 716
 717         if (!kms) {
 718                 DPU_ERROR("invalid kms\n");
 719                 return;
 720         }
 721
 722         dpu_kms = to_dpu_kms(kms);
 723
 724         _dpu_kms_hw_destroy(dpu_kms);
 725
 726         msm_kms_destroy(&dpu_kms->base);
 727 }
 728
 729 static void _dpu_kms_set_encoder_mode(struct msm_kms *kms,
 730                                  struct drm_encoder *encoder,
 731                                  bool cmd_mode)
 732 {
 733         struct msm_display_info info;
 734         struct msm_drm_private *priv = encoder->dev->dev_private;
 735         int i, rc = 0;
 736
 737         memset(&info, 0, sizeof(info));
 738
 739         info.intf_type = encoder->encoder_type;
 740         info.capabilities = cmd_mode ? MSM_DISPLAY_CAP_CMD_MODE :
 741                         MSM_DISPLAY_CAP_VID_MODE;
 742
 743         switch (info.intf_type) {
 744         case DRM_MODE_ENCODER_DSI:
 745                 /* TODO: No support for DSI swap */
 746                 for (i = 0; i < ARRAY_SIZE(priv->dsi); i++) {
 747                         if (priv->dsi[i]) {
 748                                 info.h_tile_instance[info.num_of_h_tiles] = i;
 749                                 info.num_of_h_tiles++;
 750                         }
 751                 }
 752                 break;
 753         case DRM_MODE_ENCODER_TMDS:
 754                 info.num_of_h_tiles = 1;
 755                 break;
 756         }
 757
 758         rc = dpu_encoder_setup(encoder->dev, encoder, &info);
 759         if (rc)
 760                 DPU_ERROR("failed to setup DPU encoder %d: rc:%d\n",
 761                         encoder->base.id, rc);
 762 }
 763
 764 static irqreturn_t dpu_irq(struct msm_kms *kms)
 765 {
 766         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 767
 768         return dpu_core_irq(dpu_kms);
 769 }
 770
 771 static void dpu_irq_preinstall(struct msm_kms *kms)
 772 {
 773         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 774
 775         dpu_core_irq_preinstall(dpu_kms);
 776 }
 777
 778 static int dpu_irq_postinstall(struct msm_kms *kms)
 779 {
 780         struct msm_drm_private *priv;
 781         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 782
 783         if (!dpu_kms || !dpu_kms->dev)
 784                 return -EINVAL;
 785
 786         priv = dpu_kms->dev->dev_private;
 787         if (!priv)
 788                 return -EINVAL;
 789
 790         msm_dp_irq_postinstall(priv->dp);
 791
 792         return 0;
 793 }
 794
 795 static void dpu_irq_uninstall(struct msm_kms *kms)
 796 {
 797         struct dpu_kms *dpu_kms = to_dpu_kms(kms);
 798
 799         dpu_core_irq_uninstall(dpu_kms);
 800 }
 801
 802 static void dpu_kms_mdp_snapshot(struct msm_disp_state *disp_state, struct msm_kms *kms)
 803 {
 804         int i;
 805         struct dpu_kms *dpu_kms;
 806         struct dpu_mdss_cfg *cat;
 807         struct dpu_hw_mdp *top;
 808
 809         dpu_kms = to_dpu_kms(kms);
 810
 811         cat = dpu_kms->catalog;
 812         top = dpu_kms->hw_mdp;
 813
 814         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 815
 816         /* dump CTL sub-blocks HW regs info */
 817         for (i = 0; i < cat->ctl_count; i++)
 818                 msm_disp_snapshot_add_block(disp_state, cat->ctl[i].len,
 819                                 dpu_kms->mmio + cat->ctl[i].base, "ctl_%d", i);
 820
 821         /* dump DSPP sub-blocks HW regs info */
 822         for (i = 0; i < cat->dspp_count; i++)
 823                 msm_disp_snapshot_add_block(disp_state, cat->dspp[i].len,
 824                                 dpu_kms->mmio + cat->dspp[i].base, "dspp_%d", i);
 825
 826         /* dump INTF sub-blocks HW regs info */
 827         for (i = 0; i < cat->intf_count; i++)
 828                 msm_disp_snapshot_add_block(disp_state, cat->intf[i].len,
 829                                 dpu_kms->mmio + cat->intf[i].base, "intf_%d", i);
 830
 831         /* dump PP sub-blocks HW regs info */
 832         for (i = 0; i < cat->pingpong_count; i++)
 833                 msm_disp_snapshot_add_block(disp_state, cat->pingpong[i].len,
 834                                 dpu_kms->mmio + cat->pingpong[i].base, "pingpong_%d", i);
 835
 836         /* dump SSPP sub-blocks HW regs info */
 837         for (i = 0; i < cat->sspp_count; i++)
 838                 msm_disp_snapshot_add_block(disp_state, cat->sspp[i].len,
 839                                 dpu_kms->mmio + cat->sspp[i].base, "sspp_%d", i);
 840
 841         msm_disp_snapshot_add_block(disp_state, top->hw.length,
 842                         dpu_kms->mmio + top->hw.blk_off, "top");
 843
 844         pm_runtime_put_sync(&dpu_kms->pdev->dev);
 845 }
 846
 847 static const struct msm_kms_funcs kms_funcs = {
 848         .hw_init         = dpu_kms_hw_init,
 849         .irq_preinstall  = dpu_irq_preinstall,
 850         .irq_postinstall = dpu_irq_postinstall,
 851         .irq_uninstall   = dpu_irq_uninstall,
 852         .irq             = dpu_irq,
 853         .enable_commit   = dpu_kms_enable_commit,
 854         .disable_commit  = dpu_kms_disable_commit,
 855         .vsync_time      = dpu_kms_vsync_time,
 856         .prepare_commit  = dpu_kms_prepare_commit,
 857         .flush_commit    = dpu_kms_flush_commit,
 858         .wait_flush      = dpu_kms_wait_flush,
 859         .complete_commit = dpu_kms_complete_commit,
 860         .enable_vblank   = dpu_kms_enable_vblank,
 861         .disable_vblank  = dpu_kms_disable_vblank,
 862         .check_modified_format = dpu_format_check_modified_format,
 863         .get_format      = dpu_get_msm_format,
 864         .round_pixclk    = dpu_kms_round_pixclk,
 865         .destroy         = dpu_kms_destroy,
 866         .set_encoder_mode = _dpu_kms_set_encoder_mode,
 867         .snapshot        = dpu_kms_mdp_snapshot,
 868 #ifdef CONFIG_DEBUG_FS
 869         .debugfs_init    = dpu_kms_debugfs_init,
 870 #endif
 871 };
 872
 873 static void _dpu_kms_mmu_destroy(struct dpu_kms *dpu_kms)
 874 {
 875         struct msm_mmu *mmu;
 876
 877         if (!dpu_kms->base.aspace)
 878                 return;
 879
 880         mmu = dpu_kms->base.aspace->mmu;
 881
 882         mmu->funcs->detach(mmu);
 883         msm_gem_address_space_put(dpu_kms->base.aspace);
 884
 885         dpu_kms->base.aspace = NULL;
 886 }
 887
 888 static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms)
 889 {
 890         struct iommu_domain *domain;
 891         struct msm_gem_address_space *aspace;
 892         struct msm_mmu *mmu;
 893
 894         domain = iommu_domain_alloc(&platform_bus_type);
 895         if (!domain)
 896                 return 0;
 897
 898         mmu = msm_iommu_new(dpu_kms->dev->dev, domain);
 899         aspace = msm_gem_address_space_create(mmu, "dpu1",
 900                 0x1000, 0x100000000 - 0x1000);
 901
 902         if (IS_ERR(aspace)) {
 903                 mmu->funcs->destroy(mmu);
 904                 return PTR_ERR(aspace);
 905         }
 906
 907         dpu_kms->base.aspace = aspace;
 908         return 0;
 909 }
 910
 911 static struct dss_clk *_dpu_kms_get_clk(struct dpu_kms *dpu_kms,
 912                 char *clock_name)
 913 {
 914         struct dss_module_power *mp = &dpu_kms->mp;
 915         int i;
 916
 917         for (i = 0; i < mp->num_clk; i++) {
 918                 if (!strcmp(mp->clk_config[i].clk_name, clock_name))
 919                         return &mp->clk_config[i];
 920         }
 921
 922         return NULL;
 923 }
 924
 925 u64 dpu_kms_get_clk_rate(struct dpu_kms *dpu_kms, char *clock_name)
 926 {
 927         struct dss_clk *clk;
 928
 929         clk = _dpu_kms_get_clk(dpu_kms, clock_name);
 930         if (!clk)
 931                 return -EINVAL;
 932
 933         return clk_get_rate(clk->clk);
 934 }
 935
 936 static int dpu_kms_hw_init(struct msm_kms *kms)
 937 {
 938         struct dpu_kms *dpu_kms;
 939         struct drm_device *dev;
 940         int i, rc = -EINVAL;
 941
 942         if (!kms) {
 943                 DPU_ERROR("invalid kms\n");
 944                 return rc;
 945         }
 946
 947         dpu_kms = to_dpu_kms(kms);
 948         dev = dpu_kms->dev;
 949
 950         rc = dpu_kms_global_obj_init(dpu_kms);
 951         if (rc)
 952                 return rc;
 953
 954         atomic_set(&dpu_kms->bandwidth_ref, 0);
 955
 956         dpu_kms->mmio = msm_ioremap(dpu_kms->pdev, "mdp", "mdp");
 957         if (IS_ERR(dpu_kms->mmio)) {
 958                 rc = PTR_ERR(dpu_kms->mmio);
 959                 DPU_ERROR("mdp register memory map failed: %d\n", rc);
 960                 dpu_kms->mmio = NULL;
 961                 goto error;
 962         }
 963         DRM_DEBUG("mapped dpu address space @%pK\n", dpu_kms->mmio);
 964
 965         dpu_kms->vbif[VBIF_RT] = msm_ioremap(dpu_kms->pdev, "vbif", "vbif");
 966         if (IS_ERR(dpu_kms->vbif[VBIF_RT])) {
 967                 rc = PTR_ERR(dpu_kms->vbif[VBIF_RT]);
 968                 DPU_ERROR("vbif register memory map failed: %d\n", rc);
 969                 dpu_kms->vbif[VBIF_RT] = NULL;
 970                 goto error;
 971         }
 972         dpu_kms->vbif[VBIF_NRT] = msm_ioremap_quiet(dpu_kms->pdev, "vbif_nrt", "vbif_nrt");
 973         if (IS_ERR(dpu_kms->vbif[VBIF_NRT])) {
 974                 dpu_kms->vbif[VBIF_NRT] = NULL;
 975                 DPU_DEBUG("VBIF NRT is not defined");
 976         }
 977
 978         dpu_kms->reg_dma = msm_ioremap_quiet(dpu_kms->pdev, "regdma", "regdma");
 979         if (IS_ERR(dpu_kms->reg_dma)) {
 980                 dpu_kms->reg_dma = NULL;
 981                 DPU_DEBUG("REG_DMA is not defined");
 982         }
 983
 984         dpu_kms_parse_data_bus_icc_path(dpu_kms);
 985
 986         pm_runtime_get_sync(&dpu_kms->pdev->dev);
 987
 988         dpu_kms->core_rev = readl_relaxed(dpu_kms->mmio + 0x0);
 989
 990         pr_info("dpu hardware revision:0x%x\n", dpu_kms->core_rev);
 991
 992         dpu_kms->catalog = dpu_hw_catalog_init(dpu_kms->core_rev);
 993         if (IS_ERR_OR_NULL(dpu_kms->catalog)) {
 994                 rc = PTR_ERR(dpu_kms->catalog);
 995                 if (!dpu_kms->catalog)
 996                         rc = -EINVAL;
 997                 DPU_ERROR("catalog init failed: %d\n", rc);
 998                 dpu_kms->catalog = NULL;
 999                 goto power_error;
1000         }
1001
1002         /*
1003          * Now we need to read the HW catalog and initialize resources such as
1004          * clocks, regulators, GDSC/MMAGIC, ioremap the register ranges etc
1005          */
1006         rc = _dpu_kms_mmu_init(dpu_kms);
1007         if (rc) {
1008                 DPU_ERROR("dpu_kms_mmu_init failed: %d\n", rc);
1009                 goto power_error;
1010         }
1011
1012         rc = dpu_rm_init(&dpu_kms->rm, dpu_kms->catalog, dpu_kms->mmio);
1013         if (rc) {
1014                 DPU_ERROR("rm init failed: %d\n", rc);
1015                 goto power_error;
1016         }
1017
1018         dpu_kms->rm_init = true;
1019
1020         dpu_kms->hw_mdp = dpu_hw_mdptop_init(MDP_TOP, dpu_kms->mmio,
1021                                              dpu_kms->catalog);
1022         if (IS_ERR(dpu_kms->hw_mdp)) {
1023                 rc = PTR_ERR(dpu_kms->hw_mdp);
1024                 DPU_ERROR("failed to get hw_mdp: %d\n", rc);
1025                 dpu_kms->hw_mdp = NULL;
1026                 goto power_error;
1027         }
1028
1029         for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
1030                 u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
1031
1032                 dpu_kms->hw_vbif[i] = dpu_hw_vbif_init(vbif_idx,
1033                                 dpu_kms->vbif[vbif_idx], dpu_kms->catalog);
1034                 if (IS_ERR_OR_NULL(dpu_kms->hw_vbif[vbif_idx])) {
1035                         rc = PTR_ERR(dpu_kms->hw_vbif[vbif_idx]);
1036                         if (!dpu_kms->hw_vbif[vbif_idx])
1037                                 rc = -EINVAL;
1038                         DPU_ERROR("failed to init vbif %d: %d\n", vbif_idx, rc);
1039                         dpu_kms->hw_vbif[vbif_idx] = NULL;
1040                         goto power_error;
1041                 }
1042         }
1043
1044         rc = dpu_core_perf_init(&dpu_kms->perf, dev, dpu_kms->catalog,
1045                         _dpu_kms_get_clk(dpu_kms, "core"));
1046         if (rc) {
1047                 DPU_ERROR("failed to init perf %d\n", rc);
1048                 goto perf_err;
1049         }
1050
1051         dpu_kms->hw_intr = dpu_hw_intr_init(dpu_kms->mmio, dpu_kms->catalog);
1052         if (IS_ERR_OR_NULL(dpu_kms->hw_intr)) {
1053                 rc = PTR_ERR(dpu_kms->hw_intr);
1054                 DPU_ERROR("hw_intr init failed: %d\n", rc);
1055                 dpu_kms->hw_intr = NULL;
1056                 goto hw_intr_init_err;
1057         }
1058
1059         dev->mode_config.min_width = 0;
1060         dev->mode_config.min_height = 0;
1061
1062         /*
1063          * max crtc width is equal to the max mixer width * 2 and max height is
1064          * is 4K
1065          */
1066         dev->mode_config.max_width =
1067                         dpu_kms->catalog->caps->max_mixer_width * 2;
1068         dev->mode_config.max_height = 4096;
1069
1070         /*
1071          * Support format modifiers for compression etc.
1072          */
1073         dev->mode_config.allow_fb_modifiers = true;
1074
1075         dev->max_vblank_count = 0xffffffff;
1076         /* Disable vblank irqs aggressively for power-saving */
1077         dev->vblank_disable_immediate = true;
1078
1079         /*
1080          * _dpu_kms_drm_obj_init should create the DRM related objects
1081          * i.e. CRTCs, planes, encoders, connectors and so forth
1082          */
1083         rc = _dpu_kms_drm_obj_init(dpu_kms);
1084         if (rc) {
1085                 DPU_ERROR("modeset init failed: %d\n", rc);
1086                 goto drm_obj_init_err;
1087         }
1088
1089         dpu_vbif_init_memtypes(dpu_kms);
1090
1091         pm_runtime_put_sync(&dpu_kms->pdev->dev);
1092
1093         return 0;
1094
1095 drm_obj_init_err:
1096         dpu_core_perf_destroy(&dpu_kms->perf);
1097 hw_intr_init_err:
1098 perf_err:
1099 power_error:
1100         pm_runtime_put_sync(&dpu_kms->pdev->dev);
1101 error:
1102         _dpu_kms_hw_destroy(dpu_kms);
1103
1104         return rc;
1105 }
1106
1107 struct msm_kms *dpu_kms_init(struct drm_device *dev)
1108 {
1109         struct msm_drm_private *priv;
1110         struct dpu_kms *dpu_kms;
1111         int irq;
1112
1113         if (!dev) {
1114                 DPU_ERROR("drm device node invalid\n");
1115                 return ERR_PTR(-EINVAL);
1116         }
1117
1118         priv = dev->dev_private;
1119         dpu_kms = to_dpu_kms(priv->kms);
1120
1121         irq = irq_of_parse_and_map(dpu_kms->pdev->dev.of_node, 0);
1122         if (irq < 0) {
1123                 DPU_ERROR("failed to get irq: %d\n", irq);
1124                 return ERR_PTR(irq);
1125         }
1126         dpu_kms->base.irq = irq;
1127
1128         return &dpu_kms->base;
1129 }
1130
1131 static int dpu_bind(struct device *dev, struct device *master, void *data)
1132 {
1133         struct drm_device *ddev = dev_get_drvdata(master);
1134         struct platform_device *pdev = to_platform_device(dev);
1135         struct msm_drm_private *priv = ddev->dev_private;
1136         struct dpu_kms *dpu_kms;
1137         struct dss_module_power *mp;
1138         int ret = 0;
1139
1140         dpu_kms = devm_kzalloc(&pdev->dev, sizeof(*dpu_kms), GFP_KERNEL);
1141         if (!dpu_kms)
1142                 return -ENOMEM;
1143
1144         dpu_kms->opp_table = dev_pm_opp_set_clkname(dev, "core");
1145         if (IS_ERR(dpu_kms->opp_table))
1146                 return PTR_ERR(dpu_kms->opp_table);
1147         /* OPP table is optional */
1148         ret = dev_pm_opp_of_add_table(dev);
1149         if (ret && ret != -ENODEV) {
1150                 dev_err(dev, "invalid OPP table in device tree\n");
1151                 goto put_clkname;
1152         }
1153
1154         mp = &dpu_kms->mp;
1155         ret = msm_dss_parse_clock(pdev, mp);
1156         if (ret) {
1157                 DPU_ERROR("failed to parse clocks, ret=%d\n", ret);
1158                 goto err;
1159         }
1160
1161         platform_set_drvdata(pdev, dpu_kms);
1162
1163         ret = msm_kms_init(&dpu_kms->base, &kms_funcs);
1164         if (ret) {
1165                 DPU_ERROR("failed to init kms, ret=%d\n", ret);
1166                 goto err;
1167         }
1168         dpu_kms->dev = ddev;
1169         dpu_kms->pdev = pdev;
1170
1171         pm_runtime_enable(&pdev->dev);
1172         dpu_kms->rpm_enabled = true;
1173
1174         priv->kms = &dpu_kms->base;
1175         return ret;
1176 err:
1177         dev_pm_opp_of_remove_table(dev);
1178 put_clkname:
1179         dev_pm_opp_put_clkname(dpu_kms->opp_table);
1180         return ret;
1181 }
1182
1183 static void dpu_unbind(struct device *dev, struct device *master, void *data)
1184 {
1185         struct platform_device *pdev = to_platform_device(dev);
1186         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1187         struct dss_module_power *mp = &dpu_kms->mp;
1188
1189         msm_dss_put_clk(mp->clk_config, mp->num_clk);
1190         devm_kfree(&pdev->dev, mp->clk_config);
1191         mp->num_clk = 0;
1192
1193         if (dpu_kms->rpm_enabled)
1194                 pm_runtime_disable(&pdev->dev);
1195
1196         dev_pm_opp_of_remove_table(dev);
1197         dev_pm_opp_put_clkname(dpu_kms->opp_table);
1198 }
1199
1200 static const struct component_ops dpu_ops = {
1201         .bind   = dpu_bind,
1202         .unbind = dpu_unbind,
1203 };
1204
1205 static int dpu_dev_probe(struct platform_device *pdev)
1206 {
1207         return component_add(&pdev->dev, &dpu_ops);
1208 }
1209
1210 static int dpu_dev_remove(struct platform_device *pdev)
1211 {
1212         component_del(&pdev->dev, &dpu_ops);
1213         return 0;
1214 }
1215
1216 static int __maybe_unused dpu_runtime_suspend(struct device *dev)
1217 {
1218         int i, rc = -1;
1219         struct platform_device *pdev = to_platform_device(dev);
1220         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1221         struct dss_module_power *mp = &dpu_kms->mp;
1222
1223         /* Drop the performance state vote */
1224         dev_pm_opp_set_rate(dev, 0);
1225         rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, false);
1226         if (rc)
1227                 DPU_ERROR("clock disable failed rc:%d\n", rc);
1228
1229         for (i = 0; i < dpu_kms->num_paths; i++)
1230                 icc_set_bw(dpu_kms->path[i], 0, 0);
1231
1232         return rc;
1233 }
1234
1235 static int __maybe_unused dpu_runtime_resume(struct device *dev)
1236 {
1237         int rc = -1;
1238         struct platform_device *pdev = to_platform_device(dev);
1239         struct dpu_kms *dpu_kms = platform_get_drvdata(pdev);
1240         struct drm_encoder *encoder;
1241         struct drm_device *ddev;
1242         struct dss_module_power *mp = &dpu_kms->mp;
1243         int i;
1244
1245         ddev = dpu_kms->dev;
1246
1247         WARN_ON(!(dpu_kms->num_paths));
1248         /* Min vote of BW is required before turning on AXI clk */
1249         for (i = 0; i < dpu_kms->num_paths; i++)
1250                 icc_set_bw(dpu_kms->path[i], 0, Bps_to_icc(MIN_IB_BW));
1251
1252         rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, true);
1253         if (rc) {
1254                 DPU_ERROR("clock enable failed rc:%d\n", rc);
1255                 return rc;
1256         }
1257
1258         dpu_vbif_init_memtypes(dpu_kms);
1259
1260         drm_for_each_encoder(encoder, ddev)
1261                 dpu_encoder_virt_runtime_resume(encoder);
1262
1263         return rc;
1264 }
1265
1266 static const struct dev_pm_ops dpu_pm_ops = {
1267         SET_RUNTIME_PM_OPS(dpu_runtime_suspend, dpu_runtime_resume, NULL)
1268         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1269                                 pm_runtime_force_resume)
1270 };
1271
1272 static const struct of_device_id dpu_dt_match[] = {
1273         { .compatible = "qcom,sdm845-dpu", },
1274         { .compatible = "qcom,sc7180-dpu", },
1275         { .compatible = "qcom,sc7280-dpu", },
1276         { .compatible = "qcom,sm8150-dpu", },
1277         { .compatible = "qcom,sm8250-dpu", },
1278         {}
1279 };
1280 MODULE_DEVICE_TABLE(of, dpu_dt_match);
1281
1282 static struct platform_driver dpu_driver = {
1283         .probe = dpu_dev_probe,
1284         .remove = dpu_dev_remove,
1285         .driver = {
1286                 .name = "msm_dpu",
1287                 .of_match_table = dpu_dt_match,
1288                 .pm = &dpu_pm_ops,
1289         },
1290 };
1291
1292 void __init msm_dpu_register(void)
1293 {
1294         platform_driver_register(&dpu_driver);
1295 }
1296
1297 void __exit msm_dpu_unregister(void)
1298 {
1299         platform_driver_unregister(&dpu_driver);
1300 }